eval-protocol 0.2.45__tar.gz → 0.2.45.dev0__tar.gz

{eval_protocol-0.2.45/eval_protocol.egg-info → eval_protocol-0.2.45.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.45
+Version: 0.2.45.dev0
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.45 → eval_protocol-0.2.45.dev0}/eval_protocol/__init__.py

@@ -64,6 +64,12 @@ try:
 except ImportError:
     LangSmithAdapter = None
 
+
+try:
+    from .adapters import WeaveAdapter
+except ImportError:
+    WeaveAdapter = None
+
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [

{eval_protocol-0.2.45 → eval_protocol-0.2.45.dev0}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-08T14:59:37-0700",
+ "date": "2025-10-09T01:23:30-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "b120611112b84df8476cefcc02660c542e61b2a9",
- "version": "0.2.45"
+ "full-revisionid": "c2ec0c8bb3f927b3c7f77c8a0e4fb955c7685ea6",
+ "version": "0.2.45-dev"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.45 → eval_protocol-0.2.45.dev0}/eval_protocol/adapters/__init__.py

@@ -92,3 +92,10 @@ try:
     __all__.extend(["LangSmithAdapter"])
 except ImportError:
     pass
+
+try:
+    from .weave import WeaveAdapter
+
+    __all__.extend(["WeaveAdapter"])
+except ImportError:
+    pass

{eval_protocol-0.2.45 → eval_protocol-0.2.45.dev0}/eval_protocol/adapters/fireworks_tracing.py

@@ -7,9 +7,9 @@ to pull data from Langfuse deployments with simplified retry logic handling.
 from __future__ import annotations
 import logging
 import requests
-import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
+import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -349,9 +349,11 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces"
 
+        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout)
+            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
             response.raise_for_status()
             result = response.json()
         except requests.exceptions.HTTPError as e:

eval_protocol-0.2.45.dev0/eval_protocol/adapters/weave.py

@@ -0,0 +1,130 @@
+"""Weave (Weights & Biases) adapter for Eval Protocol.
+
+This adapter fetches recent root traces from Weave Trace API and converts them
+to `EvaluationRow` format for use in evaluation pipelines. It is intentionally
+minimal and depends only on requests.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+import os
+import requests
+
+from eval_protocol.models import EvaluationRow, InputMetadata, Message, ExecutionMetadata
+from .base import BaseAdapter
+
+
+def _extract_messages_from_trace(trace: Dict[str, Any], include_tool_calls: bool = True) -> List[Message]:
+    messages: List[Message] = []
+
+    # Prefer explicit output messages if provided
+    output = trace.get("output") or {}
+    out_msgs = output.get("messages")
+    if isinstance(out_msgs, list):
+        for m in out_msgs:
+            messages.append(
+                Message(
+                    role=m.get("role"),
+                    content=m.get("content"),
+                    tool_calls=m.get("tool_calls") if include_tool_calls else None,
+                    tool_call_id=m.get("tool_call_id"),
+                    name=m.get("name"),
+                )
+            )
+
+    # If no explicit output messages, fall back to final bubble from choices
+    if not messages:
+        choices = output.get("choices")
+        if isinstance(choices, list) and choices:
+            msg = (choices[0] or {}).get("message", {})
+            if msg:
+                messages.append(Message(role=msg.get("role"), content=msg.get("content")))
+
+    # Prepend input messages if present and not already contained
+    inputs = trace.get("inputs") or {}
+    in_msgs = inputs.get("messages")
+    if isinstance(in_msgs, list):
+        prefixed = [Message(role=m.get("role"), content=m.get("content")) for m in in_msgs]
+        messages = prefixed + messages
+
+    return messages
+
+
+def _convert_trace_to_evaluation_row(
+    trace: Dict[str, Any], include_tool_calls: bool = True
+) -> Optional[EvaluationRow]:
+    messages = _extract_messages_from_trace(trace, include_tool_calls=include_tool_calls)
+    if not messages:
+        return None
+
+    # Provider-native IDs for UI joinability
+    session_data = {
+        "weave_trace_id": trace.get("id"),
+        "weave_project_id": trace.get("project_id"),
+    }
+
+    # Optional EP identifiers (if present in provider payload)
+    meta_in = (trace.get("inputs") or {}).get("metadata") or {}
+    meta_out = (trace.get("output") or {}).get("metadata") or {}
+    metadata = {**meta_in, **meta_out}
+
+    input_metadata = InputMetadata(row_id=metadata.get("row_id"), session_data=session_data)
+
+    # Preserve default factory behavior by only setting provided fields
+    exec_kwargs: Dict[str, Any] = {}
+    for k in ("invocation_id", "experiment_id", "rollout_id", "run_id"):
+        if metadata.get(k) is not None:
+            exec_kwargs[k] = metadata[k]
+    execution_metadata = ExecutionMetadata(**exec_kwargs)
+
+    # Capture tools if provider exposes them (prefer inputs)
+    tools = None
+    inputs = trace.get("inputs") or {}
+    if include_tool_calls and isinstance(inputs, dict) and "tools" in inputs:
+        tools = inputs.get("tools")
+
+    return EvaluationRow(
+        messages=messages, tools=tools, input_metadata=input_metadata, execution_metadata=execution_metadata
+    )
+
+
+class WeaveAdapter(BaseAdapter):
+    """Adapter to pull data from Weave Trace API and convert to EvaluationRow format."""
+
+    def __init__(
+        self, base_url: Optional[str] = None, api_token: Optional[str] = None, project_id: Optional[str] = None
+    ):
+        self.base_url = base_url or os.getenv("WEAVE_TRACE_BASE_URL", "https://trace.wandb.ai")
+        self.api_token = api_token or os.getenv("WANDB_API_KEY")
+        # project_id is in form "<entity>/<project>"
+        self.project_id = project_id or (f"{os.getenv('WANDB_ENTITY')}/{os.getenv('WANDB_PROJECT')}")
+        if not self.api_token or not self.project_id or "/" not in self.project_id:
+            raise ValueError("Missing Weave credentials or project (WANDB_API_KEY and WANDB_ENTITY/WANDB_PROJECT)")
+
+    def _fetch_traces(self, limit: int = 100) -> List[Dict[str, Any]]:
+        url = f"{self.base_url}/calls/stream_query"
+        payload = {
+            "project_id": self.project_id,
+            "filter": {"trace_roots_only": True},
+            "limit": limit,
+            "offset": 0,
+            "sort_by": [{"field": "started_at", "direction": "desc"}],
+            "include_feedback": False,
+        }
+        headers = {"Authorization": f"Bearer {self.api_token}", "Content-Type": "application/json"}
+        resp = requests.post(url, json=payload, headers=headers, timeout=30)
+        resp.raise_for_status()
+        body = resp.json() or {}
+        return body.get("data", [])
+
+    def get_evaluation_rows(self, *args, **kwargs) -> List[EvaluationRow]:
+        limit = kwargs.get("limit", 100)
+        include_tool_calls = kwargs.get("include_tool_calls", True)
+        traces = self._fetch_traces(limit=limit)
+        rows: List[EvaluationRow] = []
+        for tr in traces:
+            row = _convert_trace_to_evaluation_row(tr, include_tool_calls=include_tool_calls)
+            if row:
+                rows.append(row)
+        return rows

eval_protocol-0.2.45.dev0/eval_protocol/proxy/proxy_core/__init__.py

@@ -0,0 +1,10 @@
+from .models import ProxyConfig
+from .app import create_app
+from .auth import AuthProvider, NoAuthProvider
+
+__all__ = [
+    "ProxyConfig",
+    "create_app",
+    "AuthProvider",
+    "NoAuthProvider",
+]

eval_protocol-0.2.45.dev0/eval_protocol/proxy/proxy_core/app.py

@@ -0,0 +1,259 @@
+"""
+Metadata Extraction Gateway
+A FastAPI service that sits in front of LiteLLM and extracts metadata from URL paths.
+"""
+
+from fastapi import FastAPI, Depends, HTTPException, Request, Query
+from typing import Optional, List
+import os
+import redis
+import logging
+import json
+from pathlib import Path
+import sys
+from contextlib import asynccontextmanager
+
+from .models import ProxyConfig, LangfuseTracesResponse
+from .auth import AuthProvider, NoAuthProvider
+from .litellm import handle_chat_completion, proxy_to_litellm
+from .langfuse import fetch_langfuse_traces
+
+# Configure logging before any other imports (so all modules inherit this config)
+log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+logging.basicConfig(
+    level=getattr(logging, log_level),
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler(sys.stdout)],
+)
+
+logger = logging.getLogger(__name__)
+
+
+def build_proxy_config() -> ProxyConfig:
+    """Load environment and secrets, and build ProxyConfig (no Redis)."""
+    # Env
+    litellm_url = os.getenv("LITELLM_URL")
+    if not litellm_url:
+        raise ValueError("LITELLM_URL environment variable must be set")
+    request_timeout = float(os.getenv("REQUEST_TIMEOUT", "300.0"))
+
+    # Secrets - use SECRETS_PATH env var if set, otherwise default to proxy/secrets.json
+    secrets_path_str = os.getenv("SECRETS_PATH")
+    if secrets_path_str:
+        secrets_path = Path(secrets_path_str)
+    else:
+        secrets_path = Path(__file__).parent / "secrets.json"
+    if not secrets_path.exists():
+        raise ValueError(
+            "secrets.json not found! Please create it from secrets.json.example:\n"
+            "  cp litellm_proxy_config/proxy/secrets.json.example litellm_proxy_config/proxy/secrets.json\n"
+            "Then add your Langfuse API keys to secrets.json"
+        )
+    try:
+        with open(secrets_path, "r") as f:
+            secrets_config = json.load(f)
+        langfuse_keys = secrets_config["langfuse_keys"]
+        default_project_id = secrets_config["default_project_id"]
+        logger.info(f"Loaded {len(langfuse_keys)} Langfuse project(s) from secrets.json")
+    except KeyError as e:
+        raise ValueError(f"Missing required key in secrets.json: {e}")
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON in secrets.json: {e}")
+
+    return ProxyConfig(
+        litellm_url=litellm_url,
+        request_timeout=request_timeout,
+        langfuse_keys=langfuse_keys,
+        default_project_id=default_project_id,
+    )
+
+
+def init_redis() -> redis.Redis:
+    """Initialize and return a Redis client from environment variables."""
+    redis_host = os.getenv("REDIS_HOST")
+    if not redis_host:
+        raise ValueError("REDIS_HOST environment variable must be set")
+    redis_port = int(os.getenv("REDIS_PORT", "6379"))
+    redis_password = os.getenv("REDIS_PASSWORD")
+
+    try:
+        client = redis.Redis(
+            host=redis_host,
+            port=redis_port,
+            password=redis_password if redis_password else None,
+            decode_responses=True,
+            socket_connect_timeout=5,
+            socket_timeout=5,
+            retry_on_timeout=True,
+        )
+        client.ping()
+        logger.info(f"Connected to Redis at {redis_host}:{redis_port}")
+        return client
+    except Exception as e:
+        raise ConnectionError(f"Failed to connect to Redis at {redis_host}:{redis_port}: {e}")
+
+
+def create_app(
+    auth_provider: AuthProvider = NoAuthProvider(),
+) -> FastAPI:
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        # Build runtime on startup
+        app.state.config = build_proxy_config()
+        app.state.redis = init_redis()
+        try:
+            yield
+        finally:
+            try:
+                app.state.redis.close()
+            except Exception:
+                pass
+
+    app = FastAPI(title="LiteLLM Metadata Proxy", lifespan=lifespan)
+
+    def get_config(request: Request) -> ProxyConfig:
+        return request.app.state.config
+
+    def get_redis(request: Request) -> redis.Redis:
+        return request.app.state.redis
+
+    async def require_auth(request: Request) -> None:
+        auth_header = request.headers.get("authorization", "")
+        api_key = None
+        if auth_header.startswith("Bearer "):
+            api_key = auth_header.replace("Bearer ", "").strip()
+
+        auth_provider.validate(api_key)
+        return None
+
+    # =====================
+    # Chat completion routes
+    # =====================
+    @app.post(
+        "/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/v1/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/v1/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    @app.post(
+        "/v1/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    @app.post(
+        "/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    @app.post(
+        "/v1/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    async def chat_completion_with_full_metadata(
+        rollout_id: str,
+        invocation_id: str,
+        experiment_id: str,
+        run_id: str,
+        row_id: str,
+        request: Request,
+        project_id: Optional[str] = None,
+        encoded_base_url: Optional[str] = None,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+    ):
+        return await handle_chat_completion(
+            config=config,
+            redis_client=redis_client,
+            request=request,
+            project_id=project_id,
+            rollout_id=rollout_id,
+            invocation_id=invocation_id,
+            experiment_id=experiment_id,
+            run_id=run_id,
+            row_id=row_id,
+            encoded_base_url=encoded_base_url,
+        )
+
+    @app.post("/project_id/{project_id}/chat/completions")
+    @app.post("/v1/project_id/{project_id}/chat/completions")
+    async def chat_completion_with_project_only(
+        project_id: str,
+        request: Request,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+    ):
+        return await handle_chat_completion(
+            config=config,
+            redis_client=redis_client,
+            request=request,
+            project_id=project_id,
+        )
+
+    # ===============
+    # Traces routes
+    # ===============
+    @app.get("/traces", response_model=LangfuseTracesResponse)
+    @app.get("/v1/traces", response_model=LangfuseTracesResponse)
+    @app.get("/project_id/{project_id}/traces", response_model=LangfuseTracesResponse)
+    @app.get("/v1/project_id/{project_id}/traces", response_model=LangfuseTracesResponse)
+    async def get_langfuse_traces(
+        tags: List[str] = Query(...),  # REQUIRED query param
+        project_id: Optional[str] = None,
+        limit: int = 100,
+        sample_size: Optional[int] = None,
+        user_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        name: Optional[str] = None,
+        environment: Optional[str] = None,
+        version: Optional[str] = None,
+        release: Optional[str] = None,
+        fields: Optional[str] = None,
+        hours_back: Optional[int] = None,
+        from_timestamp: Optional[str] = None,
+        to_timestamp: Optional[str] = None,
+        sleep_between_gets: float = 2.5,
+        max_retries: int = 3,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+        _: None = Depends(require_auth),
+    ) -> LangfuseTracesResponse:
+        return await fetch_langfuse_traces(
+            config=config,
+            redis_client=redis_client,
+            tags=tags,
+            project_id=project_id,
+            limit=limit,
+            sample_size=sample_size,
+            user_id=user_id,
+            session_id=session_id,
+            name=name,
+            environment=environment,
+            version=version,
+            release=release,
+            fields=fields,
+            hours_back=hours_back,
+            from_timestamp=from_timestamp,
+            to_timestamp=to_timestamp,
+            sleep_between_gets=sleep_between_gets,
+            max_retries=max_retries,
+        )
+
+    # Health
+    @app.get("/health")
+    async def health():
+        return {"status": "healthy", "service": "metadata-proxy"}
+
+    # Catch-all
+    @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
+    async def catch_all_proxy(
+        path: str,
+        request: Request,
+        config: ProxyConfig = Depends(get_config),
+    ):
+        return await proxy_to_litellm(config, path, request)
+
+    return app

eval_protocol-0.2.45.dev0/eval_protocol/proxy/proxy_core/auth.py

@@ -0,0 +1,12 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class AuthProvider(ABC):
+    @abstractmethod
+    def validate(self, api_key: Optional[str]) -> Optional[str]: ...
+
+
+class NoAuthProvider(AuthProvider):
+    def validate(self, api_key: Optional[str]) -> Optional[str]:
+        return None