kalibr 1.0.28__py3-none-any.whl → 1.1.0__py3-none-any.whl

kalibr/cost_adapter.py

@@ -0,0 +1,222 @@
+"""Vendor-agnostic cost adapters for LLM pricing.
+
+Each adapter computes cost in USD based on:
+- Model name
+- Input tokens
+- Output tokens
+- Pricing table (versioned)
+
+Supports:
+- OpenAI (GPT-4, GPT-3.5, etc.)
+- Anthropic (Claude models)
+- Extensible for other vendors
+"""
+
+import json
+import os
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+
+class BaseCostAdapter(ABC):
+    """Base class for vendor cost adapters."""
+
+    @abstractmethod
+    def compute_cost(self, model_name: str, tokens_in: int, tokens_out: int) -> float:
+        """Compute cost in USD for given model and token counts.
+
+        Args:
+            model_name: Model identifier
+            tokens_in: Input token count
+            tokens_out: Output token count
+
+        Returns:
+            Cost in USD (e.g., 0.0123)
+        """
+        pass
+
+    @abstractmethod
+    def get_vendor_name(self) -> str:
+        """Return vendor name (e.g., 'openai', 'anthropic')."""
+        pass
+
+
+class OpenAICostAdapter(BaseCostAdapter):
+    """Cost adapter for OpenAI models."""
+
+    # OpenAI pricing as of 2025 (per 1M tokens)
+    # Source: https://openai.com/pricing
+    PRICING = {
+        "gpt-4": {
+            "input": 30.00,  # $30/1M input tokens
+            "output": 60.00,  # $60/1M output tokens
+        },
+        "gpt-4-turbo": {
+            "input": 10.00,
+            "output": 30.00,
+        },
+        "gpt-4o": {
+            "input": 2.50,
+            "output": 10.00,
+        },
+        "gpt-3.5-turbo": {
+            "input": 0.50,
+            "output": 1.50,
+        },
+        "gpt-4o-mini": {
+            "input": 0.15,
+            "output": 0.60,
+        },
+    }
+
+    def get_vendor_name(self) -> str:
+        return "openai"
+
+    def compute_cost(self, model_name: str, tokens_in: int, tokens_out: int) -> float:
+        """Compute cost for OpenAI models."""
+        # Normalize model name
+        model_key = self._normalize_model_name(model_name)
+
+        # Get pricing (default to gpt-4 if unknown)
+        pricing = self.PRICING.get(model_key, self.PRICING["gpt-4"])
+
+        # Calculate cost (pricing is per 1M tokens)
+        input_cost = (tokens_in / 1_000_000) * pricing["input"]
+        output_cost = (tokens_out / 1_000_000) * pricing["output"]
+
+        return round(input_cost + output_cost, 6)
+
+    def _normalize_model_name(self, model_name: str) -> str:
+        """Normalize model name to match pricing table."""
+        model_lower = model_name.lower()
+
+        # Direct matches
+        if model_lower in self.PRICING:
+            return model_lower
+
+        # Fuzzy matches
+        if "gpt-4o-mini" in model_lower:
+            return "gpt-4o-mini"
+        elif "gpt-4o" in model_lower:
+            return "gpt-4o"
+        elif "gpt-4-turbo" in model_lower:
+            return "gpt-4-turbo"
+        elif "gpt-4" in model_lower:
+            return "gpt-4"
+        elif "gpt-3.5" in model_lower:
+            return "gpt-3.5-turbo"
+
+        # Default to gpt-4 for unknown models
+        return "gpt-4"
+
+
+class AnthropicCostAdapter(BaseCostAdapter):
+    """Cost adapter for Anthropic Claude models."""
+
+    # Anthropic pricing as of 2025 (per 1M tokens)
+    # Source: https://www.anthropic.com/pricing
+    PRICING = {
+        "claude-3-opus": {
+            "input": 15.00,
+            "output": 75.00,
+        },
+        "claude-3-sonnet": {
+            "input": 3.00,
+            "output": 15.00,
+        },
+        "claude-3-haiku": {
+            "input": 0.25,
+            "output": 1.25,
+        },
+        "claude-3.5-sonnet": {
+            "input": 3.00,
+            "output": 15.00,
+        },
+    }
+
+    def get_vendor_name(self) -> str:
+        return "anthropic"
+
+    def compute_cost(self, model_name: str, tokens_in: int, tokens_out: int) -> float:
+        """Compute cost for Anthropic models."""
+        # Normalize model name
+        model_key = self._normalize_model_name(model_name)
+
+        # Get pricing (default to opus if unknown)
+        pricing = self.PRICING.get(model_key, self.PRICING["claude-3-opus"])
+
+        # Calculate cost (pricing is per 1M tokens)
+        input_cost = (tokens_in / 1_000_000) * pricing["input"]
+        output_cost = (tokens_out / 1_000_000) * pricing["output"]
+
+        return round(input_cost + output_cost, 6)
+
+    def _normalize_model_name(self, model_name: str) -> str:
+        """Normalize model name to match pricing table."""
+        model_lower = model_name.lower()
+
+        # Direct matches
+        if model_lower in self.PRICING:
+            return model_lower
+
+        # Fuzzy matches
+        if "claude-3.5-sonnet" in model_lower or "claude-3-5-sonnet" in model_lower:
+            return "claude-3.5-sonnet"
+        elif "claude-3-opus" in model_lower:
+            return "claude-3-opus"
+        elif "claude-3-sonnet" in model_lower:
+            return "claude-3-sonnet"
+        elif "claude-3-haiku" in model_lower:
+            return "claude-3-haiku"
+
+        # Default to opus for unknown models
+        return "claude-3-opus"
+
+
+class CostAdapterFactory:
+    """Factory to get appropriate cost adapter for a vendor."""
+
+    _adapters: Dict[str, BaseCostAdapter] = {
+        "openai": OpenAICostAdapter(),
+        "anthropic": AnthropicCostAdapter(),
+    }
+
+    @classmethod
+    def get_adapter(cls, vendor: str) -> Optional[BaseCostAdapter]:
+        """Get cost adapter for vendor.
+
+        Args:
+            vendor: Vendor name (openai, anthropic, etc.)
+
+        Returns:
+            Cost adapter instance or None if not supported
+        """
+        return cls._adapters.get(vendor.lower())
+
+    @classmethod
+    def register_adapter(cls, vendor: str, adapter: BaseCostAdapter):
+        """Register a custom cost adapter.
+
+        Args:
+            vendor: Vendor name
+            adapter: Cost adapter instance
+        """
+        cls._adapters[vendor.lower()] = adapter
+
+    @classmethod
+    def compute_cost(cls, vendor: str, model_name: str, tokens_in: int, tokens_out: int) -> float:
+        """Convenience method to compute cost.
+
+        Args:
+            vendor: Vendor name
+            model_name: Model identifier
+            tokens_in: Input token count
+            tokens_out: Output token count
+
+        Returns:
+            Cost in USD, or 0.0 if vendor not supported
+        """
+        adapter = cls.get_adapter(vendor)
+        if adapter:
+            return adapter.compute_cost(model_name, tokens_in, tokens_out)
+        return 0.0

kalibr/decorators.py

@@ -0,0 +1,140 @@
+"""Decorator functions for automatic tracing.
+
+Provides clean decorator-based API for tracing LLM calls:
+
+@trace(operation="chat_completion", vendor="openai", model="gpt-4")
+def my_llm_call(prompt):
+    return client.chat.completions.create(...)
+"""
+
+from functools import wraps
+from typing import Any, Callable, Optional
+
+from .tokens import count_tokens
+from .tracer import Tracer
+
+
+def create_trace_decorator(tracer: Tracer):
+    """Create a trace decorator bound to a tracer instance.
+
+    Args:
+        tracer: Tracer instance
+
+    Returns:
+        Trace decorator function
+    """
+
+    def trace(
+        operation: str = "model_call",
+        vendor: str = "unknown",
+        model: str = "unknown",
+        endpoint: Optional[str] = None,
+        extract_tokens: bool = True,
+    ):
+        """Decorator to trace function calls.
+
+        Args:
+            operation: Operation type (chat_completion, embedding, etc.)
+            vendor: Vendor name (openai, anthropic, etc.)
+            model: Model identifier
+            endpoint: API endpoint or function name
+            extract_tokens: Whether to extract token counts from args/result
+
+        Example:
+            @trace(operation="chat_completion", vendor="openai", model="gpt-4")
+            def call_openai(prompt):
+                return openai.chat.completions.create(
+                    model="gpt-4",
+                    messages=[{"role": "user", "content": prompt}]
+                )
+        """
+
+        def decorator(func: Callable) -> Callable:
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                # Create span context
+                with tracer.create_span(
+                    operation=operation,
+                    vendor=vendor,
+                    model_name=model,
+                    endpoint=endpoint or func.__name__,
+                ) as span:
+                    try:
+                        # Execute function
+                        result = func(*args, **kwargs)
+
+                        # Extract tokens if enabled
+                        if extract_tokens:
+                            tokens_in, tokens_out = _extract_tokens(args, kwargs, result, model)
+                            span.set_tokens(tokens_in, tokens_out)
+
+                        return result
+
+                    except Exception as e:
+                        # Capture error
+                        span.set_error(e)
+                        raise
+
+            return wrapper
+
+        return decorator
+
+    return trace
+
+
+def _extract_tokens(args, kwargs, result, model: str) -> tuple[int, int]:
+    """Extract token counts from function args and result.
+
+    Args:
+        args: Function positional arguments
+        kwargs: Function keyword arguments
+        result: Function return value
+        model: Model identifier
+
+    Returns:
+        Tuple of (tokens_in, tokens_out)
+    """
+    tokens_in = 0
+    tokens_out = 0
+
+    # Try to extract prompt from common arg patterns
+    prompt = None
+    response_text = None
+
+    # Extract from OpenAI-style calls
+    if "messages" in kwargs:
+        messages = kwargs["messages"]
+        if isinstance(messages, list):
+            prompt = "\n".join([str(m.get("content", "")) for m in messages])
+    elif "prompt" in kwargs:
+        prompt = kwargs["prompt"]
+    elif args and isinstance(args[0], str):
+        prompt = args[0]
+
+    # Extract response
+    if hasattr(result, "choices") and result.choices:  # OpenAI response
+        choice = result.choices[0]
+        if hasattr(choice, "message") and hasattr(choice.message, "content"):
+            response_text = choice.message.content
+    elif hasattr(result, "content"):  # Anthropic response
+        if isinstance(result.content, list):
+            response_text = "\n".join(
+                [block.text for block in result.content if hasattr(block, "text")]
+            )
+        else:
+            response_text = str(result.content)
+    elif isinstance(result, dict):
+        if "content" in result:
+            response_text = result["content"]
+        elif "text" in result:
+            response_text = result["text"]
+    elif isinstance(result, str):
+        response_text = result
+
+    # Count tokens
+    if prompt:
+        tokens_in = count_tokens(prompt, model)
+    if response_text:
+        tokens_out = count_tokens(response_text, model)
+
+    return tokens_in, tokens_out

kalibr/instrumentation/__init__.py

@@ -0,0 +1,13 @@
+"""
+Kalibr SDK Instrumentation Module
+
+Provides automatic instrumentation for LLM SDKs (OpenAI, Anthropic, Google)
+using monkey-patching to emit OpenTelemetry-compatible spans.
+"""
+
+import os
+from typing import List, Optional
+
+from .registry import auto_instrument, get_instrumented_providers
+
+__all__ = ["auto_instrument", "get_instrumented_providers"]

kalibr/instrumentation/anthropic_instr.py

@@ -0,0 +1,282 @@
+"""
+Anthropic SDK Instrumentation
+
+Monkey-patches the Anthropic SDK to automatically emit OpenTelemetry spans
+for all message API calls.
+"""
+
+import time
+from functools import wraps
+from typing import Any, Dict, Optional
+
+from opentelemetry.trace import SpanKind
+
+from .base import BaseCostAdapter, BaseInstrumentation
+
+
+class AnthropicCostAdapter(BaseCostAdapter):
+    """Cost calculation adapter for Anthropic models"""
+
+    # Pricing per 1K tokens (USD) - Updated November 2025
+    PRICING = {
+        # Claude 4 models
+        "claude-4-opus": {"input": 0.015, "output": 0.075},
+        "claude-4-sonnet": {"input": 0.003, "output": 0.015},
+        # Claude 3 models (Sonnet 4 is actually Claude 3.7)
+        "claude-sonnet-4": {"input": 0.003, "output": 0.015},
+        "claude-3-7-sonnet": {"input": 0.003, "output": 0.015},
+        "claude-3-5-sonnet": {"input": 0.003, "output": 0.015},
+        "claude-3-opus": {"input": 0.015, "output": 0.075},
+        "claude-3-sonnet": {"input": 0.003, "output": 0.015},
+        "claude-3-haiku": {"input": 0.00025, "output": 0.00125},
+        # Claude 2 models
+        "claude-2.1": {"input": 0.008, "output": 0.024},
+        "claude-2.0": {"input": 0.008, "output": 0.024},
+        "claude-instant-1.2": {"input": 0.0008, "output": 0.0024},
+    }
+
+    def calculate_cost(self, model: str, usage: Dict[str, int]) -> float:
+        """Calculate cost in USD for an Anthropic API call"""
+        # Normalize model name
+        base_model = model.lower()
+
+        # Try exact match first
+        pricing = self.get_pricing(base_model)
+
+        # Try fuzzy matching for versioned models
+        if not pricing:
+            for known_model in self.PRICING.keys():
+                if known_model in base_model or base_model in known_model:
+                    pricing = self.PRICING[known_model]
+                    break
+
+        if not pricing:
+            # Default to Claude 3 Sonnet pricing if unknown
+            pricing = {"input": 0.003, "output": 0.015}
+
+        input_tokens = usage.get("input_tokens", 0)
+        output_tokens = usage.get("output_tokens", 0)
+
+        input_cost = (input_tokens / 1000) * pricing["input"]
+        output_cost = (output_tokens / 1000) * pricing["output"]
+
+        return round(input_cost + output_cost, 6)
+
+
+class AnthropicInstrumentation(BaseInstrumentation):
+    """Instrumentation for Anthropic SDK"""
+
+    def __init__(self):
+        super().__init__("kalibr.anthropic")
+        self._original_create = None
+        self._original_async_create = None
+        self.cost_adapter = AnthropicCostAdapter()
+
+    def instrument(self) -> bool:
+        """Apply monkey-patching to Anthropic SDK"""
+        if self._is_instrumented:
+            return True
+
+        try:
+            import anthropic
+            from anthropic.resources import messages
+
+            # Patch sync method
+            if hasattr(messages.Messages, "create"):
+                self._original_create = messages.Messages.create
+                messages.Messages.create = self._traced_create_wrapper(messages.Messages.create)
+
+            # Patch async method
+            if hasattr(messages.AsyncMessages, "create"):
+                self._original_async_create = messages.AsyncMessages.create
+                messages.AsyncMessages.create = self._traced_async_create_wrapper(
+                    messages.AsyncMessages.create
+                )
+
+            self._is_instrumented = True
+            return True
+
+        except ImportError:
+            print("⚠️  Anthropic SDK not installed, skipping instrumentation")
+            return False
+        except Exception as e:
+            print(f"❌ Failed to instrument Anthropic SDK: {e}")
+            return False
+
+    def uninstrument(self) -> bool:
+        """Remove monkey-patching from Anthropic SDK"""
+        if not self._is_instrumented:
+            return True
+
+        try:
+            import anthropic
+            from anthropic.resources import messages
+
+            # Restore sync method
+            if self._original_create:
+                messages.Messages.create = self._original_create
+
+            # Restore async method
+            if self._original_async_create:
+                messages.AsyncMessages.create = self._original_async_create
+
+            self._is_instrumented = False
+            return True
+
+        except Exception as e:
+            print(f"❌ Failed to uninstrument Anthropic SDK: {e}")
+            return False
+
+    def _traced_create_wrapper(self, original_func):
+        """Wrapper for sync create method"""
+
+        @wraps(original_func)
+        def wrapper(self_instance, *args, **kwargs):
+            # Extract model from kwargs
+            model = kwargs.get("model", "unknown")
+
+            # Create span with initial attributes
+            with self.tracer.start_as_current_span(
+                "anthropic.messages.create",
+                kind=SpanKind.CLIENT,
+                attributes={
+                    "llm.vendor": "anthropic",
+                    "llm.request.model": model,
+                    "llm.system": "anthropic",
+                },
+            ) as span:
+                start_time = time.time()
+
+                # Phase 3: Inject Kalibr context for HTTP→SDK linking
+                try:
+                    from kalibr.context import inject_kalibr_context_into_span
+
+                    inject_kalibr_context_into_span(span)
+                except Exception:
+                    pass  # Fail silently if context not available
+
+                try:
+                    # Call original method
+                    result = original_func(self_instance, *args, **kwargs)
+
+                    # Extract and set response metadata
+                    self._set_response_attributes(span, result, start_time)
+
+                    return result
+
+                except Exception as e:
+                    self.set_error(span, e)
+                    raise
+
+        return wrapper
+
+    def _traced_async_create_wrapper(self, original_func):
+        """Wrapper for async create method"""
+
+        @wraps(original_func)
+        async def wrapper(self_instance, *args, **kwargs):
+            # Extract model from kwargs
+            model = kwargs.get("model", "unknown")
+
+            # Create span with initial attributes
+            with self.tracer.start_as_current_span(
+                "anthropic.messages.create",
+                kind=SpanKind.CLIENT,
+                attributes={
+                    "llm.vendor": "anthropic",
+                    "llm.request.model": model,
+                    "llm.system": "anthropic",
+                },
+            ) as span:
+                start_time = time.time()
+
+                # Phase 3: Inject Kalibr context for HTTP→SDK linking
+                try:
+                    from kalibr.context import inject_kalibr_context_into_span
+
+                    inject_kalibr_context_into_span(span)
+                except Exception:
+                    pass  # Fail silently if context not available
+
+                try:
+                    # Call original async method
+                    result = await original_func(self_instance, *args, **kwargs)
+
+                    # Extract and set response metadata
+                    self._set_response_attributes(span, result, start_time)
+
+                    return result
+
+                except Exception as e:
+                    self.set_error(span, e)
+                    raise
+
+        return wrapper
+
+    def _set_response_attributes(self, span, result, start_time: float) -> None:
+        """Extract metadata from response and set span attributes"""
+        try:
+            # Model
+            if hasattr(result, "model"):
+                span.set_attribute("llm.response.model", result.model)
+
+            # Token usage
+            if hasattr(result, "usage") and result.usage:
+                usage = result.usage
+                if hasattr(usage, "input_tokens"):
+                    span.set_attribute("llm.usage.input_tokens", usage.input_tokens)
+                    span.set_attribute("llm.usage.prompt_tokens", usage.input_tokens)  # Alias
+                if hasattr(usage, "output_tokens"):
+                    span.set_attribute("llm.usage.output_tokens", usage.output_tokens)
+                    span.set_attribute("llm.usage.completion_tokens", usage.output_tokens)  # Alias
+
+                total_tokens = usage.input_tokens + usage.output_tokens
+                span.set_attribute("llm.usage.total_tokens", total_tokens)
+
+                # Calculate cost
+                cost = self.cost_adapter.calculate_cost(
+                    result.model,
+                    {
+                        "input_tokens": usage.input_tokens,
+                        "output_tokens": usage.output_tokens,
+                    },
+                )
+                span.set_attribute("llm.cost_usd", cost)
+
+            # Latency
+            latency_ms = (time.time() - start_time) * 1000
+            span.set_attribute("llm.latency_ms", round(latency_ms, 2))
+
+            # Response ID
+            if hasattr(result, "id"):
+                span.set_attribute("llm.response.id", result.id)
+
+            # Stop reason
+            if hasattr(result, "stop_reason"):
+                span.set_attribute("llm.response.stop_reason", result.stop_reason)
+
+        except Exception as e:
+            # Don't fail the call if metadata extraction fails
+            span.set_attribute("llm.metadata_extraction_error", str(e))
+
+
+# Singleton instance
+_anthropic_instrumentation = None
+
+
+def get_instrumentation() -> AnthropicInstrumentation:
+    """Get or create the Anthropic instrumentation singleton"""
+    global _anthropic_instrumentation
+    if _anthropic_instrumentation is None:
+        _anthropic_instrumentation = AnthropicInstrumentation()
+    return _anthropic_instrumentation
+
+
+def instrument() -> bool:
+    """Instrument Anthropic SDK"""
+    return get_instrumentation().instrument()
+
+
+def uninstrument() -> bool:
+    """Uninstrument Anthropic SDK"""
+    return get_instrumentation().uninstrument()