variably-sdk 2.5.0__tar.gz → 2.6.0__tar.gz

{variably_sdk-2.5.0 → variably_sdk-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.5.0
+Version: 2.6.0
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
 Author: Variably
 Author-email: Variably <support@variably.com>

{variably_sdk-2.5.0 → variably_sdk-2.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "variably-sdk"
-version = "2.5.0"
+version = "2.6.0"
 description = "Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform"
 readme = "README.md"
 requires-python = ">=3.7"

{variably_sdk-2.5.0 → variably_sdk-2.6.0}/src/variably/__init__.py

@@ -16,6 +16,7 @@ from .errors import (
     TimeoutError,
     ConfigurationError,
 )
+from .observe import observe
 from .version import __version__
 
 __all__ = [
@@ -35,5 +36,6 @@ __all__ = [
     "RateLimitError",
     "TimeoutError",
     "ConfigurationError",
+    "observe",
     "__version__",
 ]

variably_sdk-2.6.0/src/variably/observe.py

@@ -0,0 +1,257 @@
+"""
+Variably Observe Mode — drop-in integration for multi-dimension LLM evaluation.
+
+Usage:
+    from variably import observe
+
+    observe(prompt=user_query, response=llm_response, provider_response=completion)
+
+Never raises. Returns {"observation_id": "...", "status": "queued"} or None.
+"""
+
+import os
+import sys
+import threading
+from typing import Any, Dict, List, Optional
+
+from .client import VariablyClient
+
+# ---------------------------------------------------------------------------
+# Lazy singleton client
+# ---------------------------------------------------------------------------
+
+_client: Optional[VariablyClient] = None
+_client_lock = threading.Lock()
+
+
+def _get_client() -> Optional[VariablyClient]:
+    """Return a shared VariablyClient, creating it on first call. Thread-safe."""
+    global _client
+    if _client is not None:
+        return _client
+
+    with _client_lock:
+        # Double-check after acquiring lock
+        if _client is not None:
+            return _client
+
+        # Try loading .env if python-dotenv is available
+        try:
+            from dotenv import load_dotenv  # type: ignore[import-untyped]
+
+            load_dotenv()
+        except ImportError:
+            pass
+
+        api_key = os.environ.get("VARIABLY_API_KEY")
+        if not api_key:
+            return None
+
+        base_url = os.environ.get(
+            "VARIABLY_BASE_URL", "https://api.variably.tech"
+        )
+
+        try:
+            _client = VariablyClient(
+                {
+                    "api_key": api_key,
+                    "base_url": base_url,
+                    "log_level": os.environ.get("VARIABLY_LOG_LEVEL", "WARNING"),
+                }
+            )
+            return _client
+        except Exception as exc:
+            print(
+                f"[variably.observe] Failed to initialize client: {exc}",
+                file=sys.stderr,
+            )
+            return None
+
+
+# ---------------------------------------------------------------------------
+# Provider response auto-extraction (duck-typed, no SDK imports needed)
+# ---------------------------------------------------------------------------
+
+
+def _extract_from_provider(provider_response: Any) -> Dict[str, Any]:
+    """Extract model, tokens, and provider from a raw LLM response object.
+
+    Supports OpenAI, Anthropic, and Google/Gemini response shapes via duck typing.
+    No provider SDK imports required.
+    """
+    extracted: Dict[str, Any] = {}
+
+    if provider_response is None:
+        return extracted
+
+    # --- Model name ---
+    model = getattr(provider_response, "model", None)
+    if model:
+        extracted["model"] = str(model)
+
+    # --- OpenAI-style: resp.usage.prompt_tokens / completion_tokens ---
+    usage = getattr(provider_response, "usage", None)
+    if usage is not None:
+        pt = getattr(usage, "prompt_tokens", None)
+        ct = getattr(usage, "completion_tokens", None)
+        if pt is not None:
+            extracted["prompt_tokens"] = int(pt)
+        if ct is not None:
+            extracted["completion_tokens"] = int(ct)
+
+        # Anthropic-style: resp.usage.input_tokens / output_tokens
+        if pt is None:
+            it = getattr(usage, "input_tokens", None)
+            if it is not None:
+                extracted["prompt_tokens"] = int(it)
+        if ct is None:
+            ot = getattr(usage, "output_tokens", None)
+            if ot is not None:
+                extracted["completion_tokens"] = int(ot)
+
+    # --- Google/Gemini-style: resp.usage_metadata ---
+    usage_meta = getattr(provider_response, "usage_metadata", None)
+    if usage_meta is not None and usage is None:
+        ptc = getattr(usage_meta, "prompt_token_count", None)
+        ctc = getattr(usage_meta, "candidates_token_count", None)
+        if ptc is not None:
+            extracted["prompt_tokens"] = int(ptc)
+        if ctc is not None:
+            extracted["completion_tokens"] = int(ctc)
+
+    # --- Auto-detect provider from model name ---
+    m = extracted.get("model", "")
+    if m:
+        m_lower = m.lower()
+        if any(k in m_lower for k in ("gpt-", "o1-", "o3-", "chatgpt")):
+            extracted["provider"] = "openai"
+        elif "claude" in m_lower:
+            extracted["provider"] = "anthropic"
+        elif any(k in m_lower for k in ("gemini", "palm")):
+            extracted["provider"] = "google"
+
+    return extracted
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def observe(
+    prompt: str,
+    response: str,
+    *,
+    # Auto-extraction from provider response object
+    provider_response: Any = None,
+    # Manual overrides (take precedence over auto-extracted)
+    model: Optional[str] = None,
+    provider: Optional[str] = None,
+    latency_ms: Optional[int] = None,
+    prompt_tokens: Optional[int] = None,
+    completion_tokens: Optional[int] = None,
+    cost: Optional[float] = None,
+    # RAG context
+    reference_materials: Optional[List[Dict[str, Any]]] = None,
+    retrieval_query: Optional[str] = None,
+    # Multi-turn context
+    conversation_history: Optional[List[Dict[str, str]]] = None,
+    # Metadata
+    tags: Optional[List[str]] = None,
+    user_id: Optional[str] = None,
+    session_id: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Optional[Dict[str, Any]]:
+    """Log a prompt/response pair to Variably for multi-dimension evaluation.
+
+    Never raises. Returns ``{"observation_id": "...", "status": "queued"}``
+    on success, or ``None`` on any error (missing API key, network failure, etc.).
+
+    Args:
+        prompt: The user's input / question sent to the LLM.
+        response: The LLM's generated response text.
+        provider_response: Raw OpenAI / Anthropic / Google response object.
+            Model name, token counts, and provider are auto-extracted via
+            duck typing — no provider SDK imports needed.
+        model: Model name override (e.g. ``"gpt-4o"``).
+            Takes precedence over auto-extracted value.
+        provider: Provider override (e.g. ``"openai"``).
+            Takes precedence over auto-detected value.
+        latency_ms: Response generation time in milliseconds.
+        prompt_tokens: Input token count override.
+            Takes precedence over auto-extracted value.
+        completion_tokens: Output token count override.
+            Takes precedence over auto-extracted value.
+        cost: Cost in USD for this LLM call.
+        reference_materials: RAG chunks for grounding / hallucination scoring.
+            Each dict should have ``"id"``, ``"content"``, and optionally
+            ``"source"``.
+        retrieval_query: The query sent to your retriever (enables retrieval
+            quality scoring).
+        conversation_history: Prior turns for coherence scoring.
+            Each dict should have ``"role"`` and ``"content"``.
+        tags: Grouping labels, e.g. ``["production", "rag"]``.
+        user_id: Your end-user's ID.
+        session_id: Conversation session ID (groups multi-turn observations).
+        metadata: Arbitrary key-value data.
+
+    Returns:
+        Dict with ``observation_id`` and ``status`` on success, or ``None``.
+    """
+    try:
+        client = _get_client()
+        if client is None:
+            return None
+
+        # Auto-extract from provider response; manual params override
+        auto = _extract_from_provider(provider_response)
+
+        resolved_model = model or auto.get("model")
+        resolved_provider = provider or auto.get("provider")
+        resolved_prompt_tokens = (
+            prompt_tokens if prompt_tokens is not None else auto.get("prompt_tokens")
+        )
+        resolved_completion_tokens = (
+            completion_tokens
+            if completion_tokens is not None
+            else auto.get("completion_tokens")
+        )
+
+        result = client.log(
+            prompt=prompt,
+            response=response,
+            model=resolved_model,
+            provider=resolved_provider,
+            latency_ms=latency_ms,
+            prompt_tokens=resolved_prompt_tokens,
+            completion_tokens=resolved_completion_tokens,
+            cost=cost,
+            reference_materials=reference_materials,
+            retrieval_query=retrieval_query,
+            conversation_history=conversation_history,
+            tags=tags,
+            user_id=user_id,
+            session_id=session_id,
+            metadata=metadata,
+        )
+
+        return {
+            "observation_id": result.observation_id,
+            "status": result.status,
+        }
+
+    except Exception as exc:
+        print(f"[variably.observe] {exc}", file=sys.stderr)
+        return None
+
+
+def reset() -> None:
+    """Reset the singleton client. Useful for testing or reconfiguration."""
+    global _client
+    with _client_lock:
+        if _client is not None:
+            try:
+                _client.close()
+            except Exception:
+                pass
+        _client = None

{variably_sdk-2.5.0 → variably_sdk-2.6.0}/src/variably/version.py

@@ -1,3 +1,3 @@
 """Version information for Variably SDK."""
 
-__version__ = "2.3.0"
+__version__ = "2.6.0"

{variably_sdk-2.5.0 → variably_sdk-2.6.0}/src/variably_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: variably-sdk
-Version: 2.5.0
+Version: 2.6.0
 Summary: Official Python SDK for Variably feature flags, LLM experimentation, and prompt optimization platform
 Author: Variably
 Author-email: Variably <support@variably.com>

{variably_sdk-2.5.0 → variably_sdk-2.6.0}/src/variably_sdk.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ src/variably/config.py
 src/variably/errors.py
 src/variably/http_client.py
 src/variably/logger.py
+src/variably/observe.py
 src/variably/types.py
 src/variably/version.py
 src/variably_sdk.egg-info/PKG-INFO