eval-protocol 0.3.15.dev1__tar.gz → 0.3.16__tar.gz

{eval_protocol-0.3.15.dev1/eval_protocol.egg-info → eval_protocol-0.3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.15.dev1
+Version: 0.3.16
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -116,6 +116,9 @@ Provides-Extra: proxy
 Requires-Dist: redis>=5.0.0; extra == "proxy"
 Requires-Dist: langfuse>=2.0.0; extra == "proxy"
 Requires-Dist: uuid6>=2025.0.0; extra == "proxy"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "proxy"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "proxy"
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == "proxy"
 Dynamic: license-file
 
 # Eval Protocol

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/__init__.py

@@ -13,6 +13,10 @@ import sys
 import warnings
 from typing import TYPE_CHECKING
 
+import litellm
+
+litellm.disable_add_transform_inline_image_block = True
+
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 # Eager imports for symbols that conflict with module names - ONLY when pytest is running.

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-01-23T23:15:42-0800",
+ "date": "2026-01-26T17:57:51-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "154394f68fbe20a91f663a4fc87f9d0b3d98ec68",
- "version": "0.3.15.dev.1"
+ "full-revisionid": "d6acf244b2072a305511451cc98f2dd784e98bf5",
+ "version": "0.3.16"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/adapters/fireworks_tracing.py

@@ -8,8 +8,10 @@ from __future__ import annotations
 import logging
 import requests
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Protocol
+import ast
+import json
 import os
+from typing import Any, Dict, List, Optional, Protocol
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -44,6 +46,43 @@ class TraceDictConverter(Protocol):
         ...
 
 
+def extract_openai_response(observations: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Attempt to extract and parse attributes from raw_gen_ai_request observation. This only works when stored in OTEL format.
+
+    Args:
+        observations: List of observation dictionaries from the trace
+
+    Returns:
+        Dict with all attributes parsed. Or None if not found.
+    """
+    for obs in observations:
+        if obs.get("name") == "raw_gen_ai_request" and obs.get("type") == "SPAN":
+            metadata = obs.get("metadata") or {}
+            attributes = metadata.get("attributes") or {}
+
+            result: Dict[str, Any] = {}
+
+            for key, value in attributes.items():
+                # Try to parse stringified objects (could be Python repr or JSON)
+                if isinstance(value, str) and value.startswith(("[", "{")):
+                    try:
+                        result[key] = ast.literal_eval(value)
+                    except Exception as e:
+                        logger.debug("Failed to parse %s with ast.literal_eval: %s", key, e)
+                        try:
+                            result[key] = json.loads(value)
+                        except Exception as e:
+                            logger.debug("Failed to parse %s with json.loads: %s", key, e)
+                            result[key] = value
+                else:
+                    result[key] = value
+
+            if result:
+                return result
+
+    return None
+
+
 def convert_trace_dict_to_evaluation_row(
     trace: Dict[str, Any], include_tool_calls: bool = True, span_name: Optional[str] = None
 ) -> Optional[EvaluationRow]:
@@ -96,6 +135,14 @@ def convert_trace_dict_to_evaluation_row(
                 ):
                     break  # Break early if we've found all the metadata we need
 
+        observations = trace.get("observations") or []
+        # We can only extract when stored in OTEL format.
+        openai_response = extract_openai_response(observations)
+        if openai_response:
+            choices = openai_response.get("llm.openai.choices")
+            if choices and len(choices) > 0:
+                execution_metadata.finish_reason = choices[0].get("finish_reason")
+
         return EvaluationRow(
             messages=messages,
             tools=tools,
@@ -160,7 +207,7 @@ def extract_messages_from_trace_dict(
         # Fallback: use the last GENERATION observation which typically contains full chat history
         if not messages:
             try:
-                all_observations = trace.get("observations", [])
+                all_observations = trace.get("observations") or []
                 gens = [obs for obs in all_observations if obs.get("type") == "GENERATION"]
                 if gens:
                     gens.sort(key=lambda x: x.get("start_time", ""))
@@ -186,7 +233,7 @@ def get_final_generation_in_span_dict(trace: Dict[str, Any], span_name: str) ->
         The final generation dictionary, or None if not found
     """
     # Get all observations from the trace
-    all_observations = trace.get("observations", [])
+    all_observations = trace.get("observations") or []
 
     # Find a span with the given name that has generation children
     parent_span = None

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/mcp/execution/policy.py

@@ -22,9 +22,6 @@ from .base_policy import LLMBasePolicy
 
 logger = logging.getLogger(__name__)
 
-# Disable LiteLLM's document inlining feature that appends #transform=inline
-# to base64 image URLs, which breaks base64 decoding
-litellm.disable_add_transform_inline_image_block = True
 
 class LiteLLMPolicy(LLMBasePolicy):
     """

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/proxy/proxy_core/app.py

@@ -15,7 +15,7 @@ from contextlib import asynccontextmanager
 
 from .models import ProxyConfig, LangfuseTracesResponse, TracesParams, ChatParams, ChatRequestHook, TracesRequestHook
 from .auth import AuthProvider, NoAuthProvider
-from .litellm import handle_chat_completion, proxy_to_litellm
+from .litellm import handle_chat_completion
 from .langfuse import fetch_langfuse_traces, pointwise_fetch_langfuse_trace
 
 # Configure logging before any other imports (so all modules inherit this config)
@@ -35,10 +35,6 @@ def build_proxy_config(
     preprocess_traces_request: Optional[TracesRequestHook] = None,
 ) -> ProxyConfig:
     """Load environment and secrets, and build ProxyConfig"""
-    # Env
-    litellm_url = os.getenv("LITELLM_URL")
-    if not litellm_url:
-        raise ValueError("LITELLM_URL environment variable must be set")
     request_timeout = float(os.getenv("REQUEST_TIMEOUT", "300.0"))
     langfuse_host = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
 
@@ -66,7 +62,6 @@ def build_proxy_config(
         raise ValueError(f"Invalid format in secrets file {secrets_path.name}: {e}")
 
     return ProxyConfig(
-        litellm_url=litellm_url,
         request_timeout=request_timeout,
         langfuse_host=langfuse_host,
         langfuse_keys=langfuse_keys,
@@ -113,6 +108,16 @@ def create_app(
         app.state.config = build_proxy_config(preprocess_chat_request, preprocess_traces_request)
         app.state.redis = init_redis()
 
+        config = app.state.config
+        default_keys = config.langfuse_keys[config.default_project_id]
+        os.environ["LANGFUSE_PUBLIC_KEY"] = default_keys["public_key"]
+        os.environ["LANGFUSE_SECRET_KEY"] = default_keys["secret_key"]
+        os.environ.setdefault("LANGFUSE_HOST", config.langfuse_host)
+
+        import litellm
+
+        litellm.callbacks = ["langfuse_otel"]
+
         try:
             yield
         finally:
@@ -297,13 +302,4 @@ def create_app(
     async def health():
         return {"status": "healthy", "service": "metadata-proxy"}
 
-    # Catch-all
-    @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
-    async def catch_all_proxy(
-        path: str,
-        request: Request,
-        config: ProxyConfig = Depends(get_config),
-    ):
-        return await proxy_to_litellm(config, path, request)
-
     return app

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/proxy/proxy_core/langfuse.py

@@ -50,6 +50,7 @@ def _serialize_trace_to_dict(trace_full: Any) -> Dict[str, Any]:
                 "input": getattr(obs, "input", None),
                 "output": getattr(obs, "output", None),
                 "parent_observation_id": getattr(obs, "parent_observation_id", None),
+                "metadata": getattr(obs, "metadata", None),
             }
             for obs in getattr(trace_full, "observations", [])
         ]

eval_protocol-0.3.16/eval_protocol/proxy/proxy_core/litellm.py

@@ -0,0 +1,150 @@
+"""
+LiteLLM client - handles all LLM calls directly via LiteLLM SDK with Langfuse OTEL integration.
+"""
+
+import json
+import base64
+import logging
+from uuid6 import uuid7
+from fastapi import Request, Response, HTTPException
+from fastapi.responses import StreamingResponse
+import redis
+import openai
+from litellm import acompletion
+
+from .redis_utils import register_insertion_id
+from .models import ProxyConfig, ChatParams
+
+logger = logging.getLogger(__name__)
+
+
+async def handle_chat_completion(
+    config: ProxyConfig,
+    redis_client: redis.Redis,
+    request: Request,
+    params: ChatParams,
+) -> Response:
+    """
+    Handle chat completion requests using LiteLLM SDK directly with Langfuse OTEL.
+
+    If metadata IDs (rollout_id, etc.) are provided, they'll be added as tags
+    and the assistant message count will be tracked in Redis.
+
+    If encoded_base_url is provided, it will be decoded and used as api_base.
+    """
+    body = await request.body()
+    data = json.loads(body) if body else {}
+
+    if config.preprocess_chat_request:
+        data, params = config.preprocess_chat_request(data, request, params)
+
+    project_id = params.project_id
+    rollout_id = params.rollout_id
+    invocation_id = params.invocation_id
+    experiment_id = params.experiment_id
+    run_id = params.run_id
+    row_id = params.row_id
+    encoded_base_url = params.encoded_base_url
+
+    # Use default project if not specified
+    if project_id is None:
+        project_id = config.default_project_id
+
+    # Decode and add base_url if provided
+    if encoded_base_url:
+        try:
+            decoded_bytes = base64.urlsafe_b64decode(encoded_base_url)
+            data["base_url"] = decoded_bytes.decode("utf-8")
+            logger.debug(f"Decoded base_url: {data['base_url']}")
+        except Exception as e:
+            logger.error(f"Failed to decode base_url: {e}")
+            raise HTTPException(status_code=400, detail=f"Invalid encoded_base_url: {str(e)}")
+
+    # Extract API key from Authorization header and add to data
+    auth_header = request.headers.get("authorization", "")
+    if auth_header.startswith("Bearer "):
+        data["api_key"] = auth_header.replace("Bearer ", "").strip()
+
+    # Build metadata with tags for Langfuse
+    insertion_id = None
+    metadata = data.pop("metadata", {}) or {}
+    tags = list(metadata.pop("tags", []) or [])
+
+    if rollout_id is not None:
+        insertion_id = str(uuid7())
+        tags.extend(
+            [
+                f"rollout_id:{rollout_id}",
+                f"insertion_id:{insertion_id}",
+                f"invocation_id:{invocation_id}",
+                f"experiment_id:{experiment_id}",
+                f"run_id:{run_id}",
+                f"row_id:{row_id}",
+            ]
+        )
+
+    # Build Langfuse metadata (tags)
+    litellm_metadata = {"tags": tags, **metadata}
+
+    langfuse_keys = config.langfuse_keys[project_id]
+
+    # Check if streaming is requested
+    is_streaming = data.get("stream", False)
+
+    # Pop fields that we pass explicitly to avoid duplicate kwarg errors
+    request_timeout = data.pop("timeout", None) or config.request_timeout
+    data.pop("langfuse_public_key", None)
+    data.pop("langfuse_secret_key", None)
+
+    try:
+        # Make the completion call - pass all params through
+        # Note: langfuse_host is set via LANGFUSE_HOST env var at startup; OTEL doesn't support per-request host override
+        response = await acompletion(
+            **data,
+            metadata=litellm_metadata,
+            timeout=request_timeout,
+            langfuse_public_key=langfuse_keys["public_key"],
+            langfuse_secret_key=langfuse_keys["secret_key"],
+        )
+
+        if is_streaming:
+            # For streaming, return a StreamingResponse with SSE format
+            # Register insertion_id only after stream completes successfully
+            async def stream_generator():
+                async for chunk in response:  # type: ignore[union-attr]
+                    yield f"data: {chunk.model_dump_json()}\n\n"
+                yield "data: [DONE]\n\n"
+                # Stream completed successfully - now register
+                if insertion_id is not None and rollout_id is not None:
+                    register_insertion_id(redis_client, rollout_id, insertion_id)
+
+            return StreamingResponse(
+                stream_generator(),
+                media_type="text/event-stream",
+                headers={
+                    "Cache-Control": "no-cache",
+                    "Connection": "keep-alive",
+                },
+            )
+        else:
+            # Non-streaming: register insertion_id on success
+            if insertion_id is not None and rollout_id is not None:
+                register_insertion_id(redis_client, rollout_id, insertion_id)
+
+            return Response(
+                content=response.model_dump_json(),
+                status_code=200,
+                media_type="application/json",
+            )
+
+    except HTTPException:
+        raise
+    except openai.APIError as e:
+        # Convert to HTTPException and let FastAPI handle it
+        raise HTTPException(
+            status_code=getattr(e, "status_code", 500),
+            detail=str(e),
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/proxy/proxy_core/models.py

@@ -53,7 +53,6 @@ class TracesParams(BaseModel):
 class ProxyConfig(BaseModel):
     """Configuration model for the LiteLLM Metadata Proxy"""
 
-    litellm_url: str
     request_timeout: float = 300.0
     langfuse_host: str
     langfuse_keys: Dict[str, Dict[str, str]]

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/pytest/handle_persist_flow.py

@@ -209,7 +209,7 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
 
                         store_experiment_link(
                             experiment_id,
-                            f"https://app.fireworks.ai/dashboard/evaluation-jobs/{job_id}",
+                            f"https://app.fireworks.ai/dashboard/evaluation-jobs/{job_id}?account={fireworks_account_id}",
                             "success",
                         )
                     else:

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol/reward_function.py

@@ -12,7 +12,6 @@ import requests
 from .models import EvaluateResult, MetricResult
 from .typed_interface import reward_function
 
-logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 T = TypeVar("T", bound=Callable[..., EvaluateResult])

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.15.dev1
+Version: 0.3.16
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -116,6 +116,9 @@ Provides-Extra: proxy
 Requires-Dist: redis>=5.0.0; extra == "proxy"
 Requires-Dist: langfuse>=2.0.0; extra == "proxy"
 Requires-Dist: uuid6>=2025.0.0; extra == "proxy"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "proxy"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "proxy"
+Requires-Dist: opentelemetry-exporter-otlp>=1.20.0; extra == "proxy"
 Dynamic: license-file
 
 # Eval Protocol

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/eval_protocol.egg-info/requires.txt

@@ -108,6 +108,9 @@ openevals>=0.1.0
 redis>=5.0.0
 langfuse>=2.0.0
 uuid6>=2025.0.0
+opentelemetry-api>=1.20.0
+opentelemetry-sdk>=1.20.0
+opentelemetry-exporter-otlp>=1.20.0
 
 [pydantic]
 pydantic-ai>=1.0.2

{eval_protocol-0.3.15.dev1 → eval_protocol-0.3.16}/pyproject.toml

@@ -153,6 +153,9 @@ proxy = [
     "redis>=5.0.0",
     "langfuse>=2.0.0",
     "uuid6>=2025.0.0",
+    "opentelemetry-api>=1.20.0",
+    "opentelemetry-sdk>=1.20.0",
+    "opentelemetry-exporter-otlp>=1.20.0",
 ]
 
 [project.scripts]

eval_protocol-0.3.15.dev1/eval_protocol/proxy/proxy_core/litellm.py

@@ -1,197 +0,0 @@
-"""
-LiteLLM client - handles all communication with LiteLLM service.
-"""
-
-import json
-import base64
-import asyncio
-import httpx
-import logging
-from uuid6 import uuid7
-from fastapi import Request, Response, HTTPException
-import redis
-from .redis_utils import register_insertion_id
-from .models import ProxyConfig, ChatParams
-
-logger = logging.getLogger(__name__)
-
-# Retry configuration for 404 errors
-# 8 retries with exponential backoff (1, 2, 4, 8, 16, 32, 64, 128 seconds)
-# Total wait time: ~255 seconds (~4.25 minutes)
-MAX_RETRIES_ON_404 = 8
-RETRY_BASE_DELAY_SECONDS = 1
-
-
-async def handle_chat_completion(
-    config: ProxyConfig,
-    redis_client: redis.Redis,
-    request: Request,
-    params: ChatParams,
-) -> Response:
-    """
-    Handle chat completion requests and forward to LiteLLM.
-
-    If metadata IDs (rollout_id, etc.) are provided, they'll be added as tags
-    and the assistant message count will be tracked in Redis.
-
-    If encoded_base_url is provided, it will be decoded and added to the request.
-    """
-    body = await request.body()
-    data = json.loads(body) if body else {}
-
-    if config.preprocess_chat_request:
-        data, params = config.preprocess_chat_request(data, request, params)
-
-    project_id = params.project_id
-    rollout_id = params.rollout_id
-    invocation_id = params.invocation_id
-    experiment_id = params.experiment_id
-    run_id = params.run_id
-    row_id = params.row_id
-    encoded_base_url = params.encoded_base_url
-
-    # Use default project if not specified
-    if project_id is None:
-        project_id = config.default_project_id
-
-    # Decode and add base_url if provided
-    if encoded_base_url:
-        try:
-            # Decode from URL-safe base64
-            decoded_bytes = base64.urlsafe_b64decode(encoded_base_url)
-            base_url = decoded_bytes.decode("utf-8")
-            data["base_url"] = base_url
-            logger.debug(f"Decoded base_url: {base_url}")
-        except Exception as e:
-            logger.error(f"Failed to decode base_url: {e}")
-            raise HTTPException(status_code=400, detail=f"Invalid encoded_base_url: {str(e)}")
-
-    # Extract API key from Authorization header and inject into request body
-    auth_header = request.headers.get("authorization", "")
-    if auth_header.startswith("Bearer "):
-        api_key = auth_header.replace("Bearer ", "").strip()
-        # Only inject API key if model is a Fireworks model
-        model = data.get("model")
-        if model and isinstance(model, str) and model.startswith("fireworks_ai"):
-            data["api_key"] = api_key
-
-    # If metadata IDs are provided, add them as tags
-    insertion_id = None
-    if rollout_id is not None:
-        insertion_id = str(uuid7())
-
-        if "metadata" not in data:
-            data["metadata"] = {}
-        if "tags" not in data["metadata"]:
-            data["metadata"]["tags"] = []
-
-        # Add extracted IDs as tags
-        data["metadata"]["tags"].extend(
-            [
-                f"rollout_id:{rollout_id}",
-                f"insertion_id:{insertion_id}",
-                f"invocation_id:{invocation_id}",
-                f"experiment_id:{experiment_id}",
-                f"run_id:{run_id}",
-                f"row_id:{row_id}",
-            ]
-        )
-
-    # Add Langfuse configuration
-    data["langfuse_public_key"] = config.langfuse_keys[project_id]["public_key"]
-    data["langfuse_secret_key"] = config.langfuse_keys[project_id]["secret_key"]
-    data["langfuse_host"] = config.langfuse_host
-
-    # Forward to LiteLLM's standard /chat/completions endpoint
-    # Set longer timeout for LLM API calls (LLMs can be slow)
-    timeout = httpx.Timeout(config.request_timeout)
-    async with httpx.AsyncClient(timeout=timeout) as client:
-        # Copy headers from original request but exclude content-length (httpx will set it correctly)
-        headers = dict(request.headers)
-        headers.pop("host", None)
-        headers.pop("content-length", None)  # Let httpx calculate the correct length
-        headers["content-type"] = "application/json"
-
-        # Forward to LiteLLM
-        litellm_url = f"{config.litellm_url}/chat/completions"
-
-        # Retry loop with exponential backoff for 404 errors
-        # Initial request
-        response = await client.post(
-            litellm_url,
-            json=data,  # httpx will serialize and set correct Content-Length
-            headers=headers,
-        )
-
-        for attempt in range(MAX_RETRIES_ON_404):
-            if response.status_code != 404:
-                break
-
-            # Wait with exponential backoff before retry
-            delay = RETRY_BASE_DELAY_SECONDS * (2**attempt)
-            logger.warning(f"Got 404 from LiteLLM, retrying in {delay}s (attempt {attempt + 1}/{MAX_RETRIES_ON_404})")
-            await asyncio.sleep(delay)
-
-            response = await client.post(
-                litellm_url,
-                json=data,
-                headers=headers,
-            )
-
-        # Register insertion_id in Redis only on successful response
-        if response.status_code == 200 and insertion_id is not None and rollout_id is not None:
-            register_insertion_id(redis_client, rollout_id, insertion_id)
-
-        # Return the response
-        return Response(
-            content=response.content,
-            status_code=response.status_code,
-            headers=dict(response.headers),
-        )
-
-
-async def proxy_to_litellm(config: ProxyConfig, path: str, request: Request) -> Response:
-    """
-    Catch-all proxy: Forward any request to LiteLLM, extracting API key from Authorization header.
-    """
-    # Set longer timeout for LLM API calls (LLMs can be slow)
-    timeout = httpx.Timeout(config.request_timeout)
-    async with httpx.AsyncClient(timeout=timeout) as client:
-        # Copy headers
-        headers = dict(request.headers)
-        headers.pop("host", None)
-        headers.pop("content-length", None)
-
-        # Get body
-        body = await request.body()
-
-        # Pass through API key from Authorization header
-        if request.method in ["POST", "PUT", "PATCH"] and body:
-            try:
-                data = json.loads(body)
-
-                auth_header = request.headers.get("authorization", "")
-                if auth_header.startswith("Bearer "):
-                    api_key = auth_header.replace("Bearer ", "").strip()
-                    data["api_key"] = api_key
-
-                # Re-serialize
-                body = json.dumps(data).encode()
-            except json.JSONDecodeError:
-                pass
-
-        # Forward to LiteLLM
-        litellm_url = f"{config.litellm_url}/{path}"
-
-        response = await client.request(
-            method=request.method,
-            url=litellm_url,
-            headers=headers,
-            content=body,
-        )
-
-        return Response(
-            content=response.content,
-            status_code=response.status_code,
-            headers=dict(response.headers),
-        )