decimalai 0.3.0__py3-none-any.whl

decimalai/__init__.py

@@ -0,0 +1,298 @@
+"""DecimalAI SDK — Agent dataset lifecycle platform.
+
+Quick start::
+
+    import decimalai
+    decimalai.init()  # reads DECIMAL_API_KEY from env
+
+    # LangChain — one-liner
+    decimalai.init(langchain=True)
+
+    # Or manual install
+    from decimalai.langchain import install
+    install()
+
+    # Generic / any framework
+    @decimalai.trace(agent_name="my-agent")
+    def run_agent(query):
+        decimalai.log_llm_call(model="gpt-4o", input=msgs, output=resp)
+        return resp.choices[0].message.content
+"""
+
+__version__ = "0.3.0"
+
+import logging
+import os
+from typing import Optional
+
+logger = logging.getLogger("decimalai")
+
+
+def init(
+    api_key: Optional[str] = None,
+    base_url: Optional[str] = None,
+    project: Optional[str] = None,
+    enabled: bool = True,
+    langchain: bool = False,
+    agent_name: Optional[str] = None,
+) -> None:
+    """Initialize the DecimalAI SDK.
+
+    Must be called once before using any integration. Configuration is
+    resolved in order: explicit parameter → environment variable → default.
+
+    Args:
+        api_key: API key. Falls back to ``DECIMAL_API_KEY`` env var.
+        base_url: Backend URL. Falls back to ``DECIMAL_BASE_URL``, then
+            ``https://api.decimal.ai``.
+        project: Optional project grouping.
+        enabled: Set ``False`` to disable all tracing (integrations become no-ops).
+        langchain: If ``True``, auto-calls ``decimalai.langchain.install()``.
+        agent_name: Default agent name for langchain auto-install.
+
+    Raises:
+        DecimalConfigError: If ``api_key`` is not provided and not in env.
+    """
+    from ._config import DecimalConfig, DecimalConfigError
+    from ._client import DecimalAIClient
+
+    import decimalai._config as _cfg
+
+    # Resolve API key
+    resolved_key = api_key or os.environ.get("DECIMAL_API_KEY", "")
+    if not resolved_key and enabled:
+        raise DecimalConfigError(
+            "No API key provided. Pass api_key= to decimalai.init() "
+            "or set the DECIMAL_API_KEY environment variable."
+        )
+
+    # Resolve base URL
+    resolved_url = (
+        base_url
+        or os.environ.get("DECIMAL_BASE_URL", "")
+        or "https://api.decimal.ai"
+    )
+
+    config = DecimalConfig(
+        api_key=resolved_key,
+        base_url=resolved_url.rstrip("/"),
+        project=project,
+        enabled=enabled,
+    )
+    _cfg._config = config
+
+    if enabled:
+        _cfg._client = DecimalAIClient(
+            api_key=config.api_key,
+            base_url=config.base_url,
+            project=config.project,
+        )
+        logger.info(
+            "DecimalAI SDK initialized: base_url=%s project=%s",
+            config.base_url,
+            config.project,
+        )
+    else:
+        _cfg._client = None
+        logger.info("DecimalAI SDK initialized in disabled mode (no-op)")
+
+    # Auto-install langchain tracing if requested
+    if langchain:
+        from .langchain import install as _lc_install
+        _lc_install(agent_name=agent_name)
+
+
+def send(trace) -> None:
+    """Manually send a trace to the backend.
+
+    For advanced usage when ``auto_send=False``.
+    """
+    from ._config import _get_client
+
+    client = _get_client()
+    client.ingest_trace(trace)
+
+
+# ── Eval convenience functions ─────────────────────────────────
+
+
+def eval(
+    trace_id: str,
+    name: str,
+    score: float,
+    *,
+    source: str = "custom",
+    source_label: Optional[str] = None,
+    passed: Optional[bool] = None,
+    reason: Optional[str] = None,
+    category: str = "quality",
+) -> dict:
+    """Push a single eval score to a trace.
+
+    This is the simplest way to attach evaluation results to a trace.
+    All scores are visible in the dashboard's Evaluation Breakdown card
+    grouped by source.
+
+    Args:
+        trace_id: The trace to attach the score to.
+        name: Metric name (e.g., "factual_accuracy", "coherence").
+        score: Score value between 0.0 and 1.0.
+        source: Eval source identifier. Defaults to "custom".
+        source_label: Human-readable display name (e.g., "My RAG Eval").
+        passed: Binary pass/fail. Defaults to score >= 0.5.
+        reason: Human-readable explanation of the score.
+        category: "quality" (default) or "compatibility".
+
+    Returns:
+        API response with stored score details and recomputed verdict.
+
+    Example::
+
+        import decimalai
+        decimalai.init()
+
+        decimalai.eval(
+            trace_id="abc123",
+            name="factual_accuracy",
+            score=0.75,
+            reason="3/4 facts verified against source docs",
+        )
+    """
+    from ._config import _get_client
+
+    client = _get_client()
+    score_entry = {
+        "name": name,
+        "score": score,
+        "passed": passed if passed is not None else score >= 0.5,
+    }
+    if reason:
+        score_entry["reason"] = reason
+
+    metadata = {}
+    if source_label:
+        metadata["source_label"] = source_label
+
+    return client.push_eval_scores(
+        trace_id=trace_id,
+        source=source,
+        scores=[score_entry],
+    )
+
+
+def score(
+    trace_id: str,
+    name: str,
+    value: float,
+    reason: Optional[str] = None,
+) -> dict:
+    """Shorthand for pushing a single eval score.
+
+    Args:
+        trace_id: The trace to attach the score to.
+        name: Metric name.
+        value: Score between 0.0 and 1.0.
+        reason: Optional explanation.
+
+    Returns:
+        API response.
+
+    Example::
+
+        decimalai.score("abc123", "factual_accuracy", 0.75)
+    """
+    return eval(
+        trace_id=trace_id,
+        name=name,
+        score=value,
+        reason=reason,
+    )
+
+
+def get_eval_breakdown(trace_id: str) -> dict:
+    """Get the full eval breakdown for a trace with provenance info.
+
+    Returns scores grouped by source (Manifest Diff, DeepEval, LangSmith,
+    Custom, etc.) with icons, labels, and decision reasons.
+
+    Returns:
+        Dict with eval_verdict, quality_avg, compat_avg, source_groups,
+        and decision_reasons.
+
+    Example::
+
+        import decimalai
+        decimalai.init()
+
+        bd = decimalai.get_eval_breakdown("abc123")
+        print(f"Verdict: {bd['eval_verdict']}")
+    """
+    from ._config import _get_client
+
+    client = _get_client()
+    return client.get_eval_breakdown(trace_id)
+
+
+# ── Re-export generic tracing API ──────────────────────────────
+
+from .generic import (  # noqa: E402, F401
+    log_llm_call,
+    log_tool_call,
+    start_trace,
+    trace,
+)
+
+# Re-export eval adapters from new location
+from .evals.adapters import (  # noqa: E402, F401
+    push_deepeval_results,
+    push_langsmith_scores,
+    push_custom_scores,
+)
+
+from .evals import batch_eval  # noqa: E402, F401
+
+__all__ = [
+    "__version__",
+    "init",
+    "send",
+    "eval",
+    "score",
+    "get_eval_breakdown",
+    "trace",
+    "start_trace",
+    "log_llm_call",
+    "log_tool_call",
+    "push_deepeval_results",
+    "push_langsmith_scores",
+    "push_custom_scores",
+    "batch_eval",
+]
+
+
+# ── Auto-init from environment variable ────────────────────────
+# Setting DECIMAL_AUTO_TRACE=langchain will auto-init and install tracing.
+
+def _auto_init_from_env() -> None:
+    """Auto-initialize from environment variables if configured."""
+    auto_trace = os.environ.get("DECIMAL_AUTO_TRACE", "").strip().lower()
+    if not auto_trace:
+        return
+
+    api_key = os.environ.get("DECIMAL_API_KEY", "")
+    if not api_key:
+        logger.debug(
+            "DECIMAL_AUTO_TRACE=%s set but no DECIMAL_API_KEY found, skipping",
+            auto_trace,
+        )
+        return
+
+    try:
+        init(
+            langchain=(auto_trace == "langchain"),
+        )
+        logger.info("DecimalAI auto-initialized via DECIMAL_AUTO_TRACE=%s", auto_trace)
+    except Exception:
+        logger.debug("Auto-init failed", exc_info=True)
+
+
+_auto_init_from_env()

decimalai/_client.py

@@ -0,0 +1,334 @@
+"""HTTP client for communicating with the Decimal platform."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, Dict, List, Optional
+from uuid import UUID
+
+import httpx
+
+from .schema.trace import RunTrace
+
+logger = logging.getLogger("decimalai")
+
+_MAX_RETRIES = 3
+_DEFAULT_RETRY_DELAY = 1.0  # seconds
+
+
+class DecimalRateLimitError(Exception):
+    """Raised when the Decimal platform returns 429 after all retries are exhausted."""
+
+    def __init__(self, retry_after: float = 0, message: str = ""):
+        self.retry_after = retry_after
+        super().__init__(message or f"Rate limit exceeded. Retry after {retry_after}s")
+
+
+class DecimalAIClient:
+    """Client for the Decimal platform API.
+
+    Handles authentication, trace ingestion, and manifest registration.
+    Can be used standalone or created automatically via ``decimalai.init()``.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = "https://api.decimal.ai",
+        project: Optional[str] = None,
+        timeout: float = 30.0,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self.project = project
+
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+        if project:
+            headers["X-Decimal-Project"] = project
+
+        self._http = httpx.Client(
+            base_url=self.base_url,
+            headers=headers,
+            timeout=timeout,
+        )
+        self._trace_buffer: List[RunTrace] = []
+
+    # ── Auth ────────────────────────────────────────────────────
+
+    def verify_auth(self) -> Dict[str, Any]:
+        """Verify the API key and return project configuration."""
+        resp = self._http.get("/api/v1/auth/verify")
+        resp.raise_for_status()
+        return resp.json()
+
+    # ── Retry logic ────────────────────────────────────────────
+
+    def _request_with_retry(
+        self, method: str, url: str, **kwargs: Any
+    ) -> httpx.Response:
+        """Make an HTTP request with retry-on-429.
+
+        Retries up to ``_MAX_RETRIES`` times when the server responds with
+        HTTP 429. Uses the ``Retry-After`` header if present, otherwise
+        falls back to exponential backoff (1s, 2s, 4s).
+        """
+        last_exc: Optional[httpx.HTTPStatusError] = None
+
+        for attempt in range(_MAX_RETRIES + 1):  # 0, 1, 2, 3
+            resp = self._http.request(method, url, **kwargs)
+
+            if resp.status_code != 429:
+                resp.raise_for_status()
+                return resp
+
+            # 429 — parse Retry-After and maybe retry
+            retry_after = float(resp.headers.get("Retry-After", 0))
+            delay = max(retry_after, _DEFAULT_RETRY_DELAY * (2 ** attempt))
+
+            last_exc = httpx.HTTPStatusError(
+                f"429 Too Many Requests",
+                request=resp.request,
+                response=resp,
+            )
+
+            if attempt < _MAX_RETRIES:
+                logger.warning(
+                    "Rate limited (429). Retrying in %.1fs (attempt %d/%d)",
+                    delay, attempt + 1, _MAX_RETRIES,
+                )
+                time.sleep(delay)
+            else:
+                raise DecimalRateLimitError(
+                    retry_after=retry_after,
+                    message=(
+                        f"Rate limit exceeded after {_MAX_RETRIES} retries. "
+                        f"Server says retry after {retry_after}s."
+                    ),
+                )
+
+        # Should never reach here, but satisfy type checker
+        raise last_exc  # type: ignore[misc]
+
+    # ── Trace ingestion ────────────────────────────────────────
+
+    def ingest_trace(self, trace: RunTrace) -> Dict[str, Any]:
+        """Send a single trace to the platform."""
+        payload = trace.model_dump(mode="json")
+        resp = self._request_with_retry("POST", "/api/v1/traces", json=payload)
+        logger.debug("Ingested trace %s", trace.id)
+        return resp.json()
+
+    def ingest_traces_batch(self, traces: List[RunTrace]) -> Dict[str, Any]:
+        """Send a batch of traces to the platform."""
+        payload = [t.model_dump(mode="json") for t in traces]
+        resp = self._request_with_retry("POST", "/api/v1/traces/batch", json=payload)
+        logger.debug("Ingested %d traces", len(traces))
+        return resp.json()
+
+    def buffer_trace(self, trace: RunTrace) -> None:
+        """Buffer a trace for batched sending."""
+        self._trace_buffer.append(trace)
+        if len(self._trace_buffer) >= 50:
+            self.flush()
+
+    def flush(self) -> None:
+        """Flush all buffered traces to the platform.
+
+        On rate limit errors (429), the buffer is **preserved** so that
+        traces are not lost — the next call to ``flush()`` will retry.
+        For all other errors the buffer is cleared.
+        """
+        if not self._trace_buffer:
+            return
+        try:
+            self.ingest_traces_batch(self._trace_buffer)
+            self._trace_buffer.clear()
+        except DecimalRateLimitError:
+            logger.warning(
+                "Rate limited — preserving %d buffered traces for later flush",
+                len(self._trace_buffer),
+            )
+        except Exception:
+            logger.exception("Failed to flush %d traces", len(self._trace_buffer))
+            self._trace_buffer.clear()
+
+    # ── Trace queries ──────────────────────────────────────────
+
+    def list_traces(
+        self,
+        limit: int = 20,
+        offset: int = 0,
+        status: Optional[str] = None,
+        agent_name: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """List traces for the current project."""
+        params: Dict[str, Any] = {"limit": limit, "offset": offset}
+        if status:
+            params["status"] = status
+        if agent_name:
+            params["agent_name"] = agent_name
+        resp = self._http.get("/api/v1/traces", params=params)
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_trace(self, trace_id: str | UUID) -> Dict[str, Any]:
+        """Get a single trace with its full span tree."""
+        resp = self._http.get(f"/api/v1/traces/{trace_id}")
+        resp.raise_for_status()
+        return resp.json()
+
+    # ── Manifest registration ─────────────────────────────────
+
+    def register_manifest(self, manifest: Any) -> Dict[str, Any]:
+        """Register a manifest snapshot with the platform.
+
+        Args:
+            manifest: A ManifestSnapshot (from decimalai.schema.manifest).
+
+        Returns:
+            Registration response with manifest_id and compatibility info.
+        """
+        payload = manifest.model_dump(mode="json")
+        resp = self._http.post("/api/v1/manifests", json=payload)
+        resp.raise_for_status()
+        logger.debug("Registered manifest %s (hash=%s)", manifest.id, manifest.manifest_hash)
+        return resp.json()
+
+    def list_manifests(
+        self,
+        limit: int = 20,
+        offset: int = 0,
+        agent_name: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """List manifests from the platform."""
+        params: Dict[str, Any] = {"limit": limit, "offset": offset}
+        if agent_name:
+            params["agent_name"] = agent_name
+        resp = self._http.get("/api/v1/manifests", params=params)
+        resp.raise_for_status()
+        return resp.json()
+
+    # ── Eval Scores ──────────────────────────────────────────────
+
+    def push_eval_scores(
+        self,
+        trace_id: str | UUID,
+        source: str,
+        scores: List[Dict[str, Any]],
+    ) -> Dict[str, Any]:
+        """Push external evaluation scores to a trace.
+
+        Args:
+            trace_id: The trace to attach scores to.
+            source: Origin of the scores (e.g., "deepeval", "langsmith", "custom").
+            scores: List of score dicts, each with at least "name" and "score".
+                    Optional fields: "passed", "reason", "category".
+
+        Returns:
+            Ingestion response with stored score count.
+
+        Example::
+
+            client.push_eval_scores(
+                trace_id="abc123",
+                source="deepeval",
+                scores=[
+                    {"name": "correctness", "score": 0.92, "reason": "Accurate"},
+                    {"name": "faithfulness", "score": 0.85},
+                ],
+            )
+        """
+        payload = {"source": source, "scores": scores}
+        resp = self._request_with_retry(
+            "POST", f"/api/v1/traces/{trace_id}/eval-scores", json=payload,
+        )
+        logger.debug("Pushed %d eval scores to trace %s", len(scores), str(trace_id)[:8])
+        return resp.json()
+
+    def get_eval_scores(self, trace_id: str | UUID) -> Dict[str, Any]:
+        """Get all evaluation scores (quality + compatibility) for a trace.
+
+        Returns:
+            Dict with quality_scores, compatibility_scores, and aggregates.
+        """
+        resp = self._http.get(f"/api/v1/traces/{trace_id}/eval-scores")
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_eval_breakdown(self, trace_id: str | UUID) -> Dict[str, Any]:
+        """Get the full eval breakdown with provenance for a trace.
+
+        Returns scores grouped by source (Manifest Diff, DeepEval, LangSmith,
+        Custom, etc.) with icons, labels, badge colors, and decision reasons
+        explaining how the final verdict was computed.
+
+        Returns:
+            Dict with eval_verdict, quality_avg, compat_avg, source_groups,
+            and decision_reasons.
+
+        Example::
+
+            breakdown = client.get_eval_breakdown("trace-123")
+            print(breakdown["eval_verdict"])  # "keep" / "drop" / ...
+            for group in breakdown["source_groups"]:
+                print(f"{group['source_label']}: {group['source_avg']}")
+                for score in group["scores"]:
+                    print(f"  {score['name']}: {score['score']}")
+        """
+        resp = self._http.get(f"/api/v1/traces/{trace_id}/eval-breakdown")
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_decision(self, trace_id: str | UUID) -> Dict[str, Any]:
+        """Compute and get the unified verdict for a trace.
+
+        Returns:
+            Dict with verdict (keep/repair/replay/drop), quality_avg,
+            compat_avg, and per-score breakdowns.
+        """
+        resp = self._request_with_retry(
+            "POST", f"/api/v1/traces/{trace_id}/decision",
+        )
+        return resp.json()
+
+    def batch_decision(
+        self,
+        trace_ids: Optional[List[str]] = None,
+        manifest_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Batch compute unified verdicts for multiple traces.
+
+        Args:
+            trace_ids: Specific trace IDs to score.
+            manifest_id: Score all traces from this manifest (alternative to trace_ids).
+
+        Returns:
+            Dict with decisions list, total count, and verdict_counts breakdown.
+        """
+        payload: Dict[str, Any] = {}
+        if trace_ids:
+            payload["trace_ids"] = trace_ids
+        if manifest_id:
+            payload["manifest_id"] = manifest_id
+
+        resp = self._request_with_retry(
+            "POST", "/api/v1/traces/batch-decision", json=payload,
+        )
+        return resp.json()
+
+    # ── Lifecycle ──────────────────────────────────────────────
+
+    def close(self) -> None:
+        """Flush remaining traces and close the HTTP client."""
+        self.flush()
+        self._http.close()
+
+    def __enter__(self) -> "DecimalAIClient":
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        self.close()