eval-protocol 0.2.46__tar.gz → 0.2.46.dev0__tar.gz

{eval_protocol-0.2.46/eval_protocol.egg-info → eval_protocol-0.2.46.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.46
+Version: 0.2.46.dev0
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.46 → eval_protocol-0.2.46.dev0}/eval_protocol/__init__.py

@@ -70,6 +70,13 @@ try:
 except ImportError:
     WeaveAdapter = None
 
+try:
+    from .proxy import create_app, AuthProvider
+except ImportError:
+    create_app = None
+    AuthProvider = None
+
+
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
@@ -130,6 +137,9 @@ __all__ = [
     "RolloutMetadata",
     "StatusResponse",
     "create_langfuse_config_tags",
+    # Proxy
+    "create_app",
+    "AuthProvider",
 ]
 
 from . import _version

{eval_protocol-0.2.46 → eval_protocol-0.2.46.dev0}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-09T13:58:14-0700",
+ "date": "2025-10-10T00:24:47-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "e066febd15f9056f74c40c8f4c34d3c68768fd59",
- "version": "0.2.46"
+ "full-revisionid": "98d81a75608e700f91e697f88c149641328d9507",
+ "version": "0.2.46-dev"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.46 → eval_protocol-0.2.46.dev0}/eval_protocol/adapters/fireworks_tracing.py

@@ -7,9 +7,9 @@ to pull data from Langfuse deployments with simplified retry logic handling.
 from __future__ import annotations
 import logging
 import requests
-import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
+import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -349,9 +349,11 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces"
 
+        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout)
+            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
             response.raise_for_status()
             result = response.json()
         except requests.exceptions.HTTPError as e:

eval_protocol-0.2.46.dev0/eval_protocol/proxy/__init__.py

@@ -0,0 +1,17 @@
+"""
+LiteLLM Metadata Extraction Gateway
+
+A proxy service for extracting evaluation metadata from URL paths and managing
+Langfuse tracing for distributed evaluation workflows.
+"""
+
+from .proxy_core import create_app, AuthProvider, NoAuthProvider, ProxyConfig, ChatParams, TracesParams
+
+__all__ = [
+    "create_app",
+    "AuthProvider",
+    "NoAuthProvider",
+    "ProxyConfig",
+    "ChatParams",
+    "TracesParams",
+]

eval_protocol-0.2.46.dev0/eval_protocol/proxy/proxy_core/__init__.py

@@ -0,0 +1,12 @@
+from .models import ProxyConfig, ChatParams, TracesParams
+from .app import create_app
+from .auth import AuthProvider, NoAuthProvider
+
+__all__ = [
+    "ProxyConfig",
+    "ChatParams",
+    "TracesParams",
+    "create_app",
+    "AuthProvider",
+    "NoAuthProvider",
+]

eval_protocol-0.2.46.dev0/eval_protocol/proxy/proxy_core/app.py

@@ -0,0 +1,284 @@
+"""
+Metadata Extraction Gateway
+A FastAPI service that sits in front of LiteLLM and extracts metadata from URL paths.
+"""
+
+from fastapi import FastAPI, Depends, HTTPException, Request, Query
+from typing import Optional, Callable, Dict, Any, List
+import os
+import redis
+import logging
+import yaml
+from pathlib import Path
+import sys
+from contextlib import asynccontextmanager
+
+from .models import ProxyConfig, LangfuseTracesResponse, TracesParams, ChatParams, ChatRequestHook, TracesRequestHook
+from .auth import AuthProvider, NoAuthProvider
+from .litellm import handle_chat_completion, proxy_to_litellm
+from .langfuse import fetch_langfuse_traces
+
+# Configure logging before any other imports (so all modules inherit this config)
+log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+if not logging.getLogger().hasHandlers():
+    logging.basicConfig(
+        level=getattr(logging, log_level),
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+logger = logging.getLogger(__name__)
+
+
+def build_proxy_config(
+    preprocess_chat_request: Optional[ChatRequestHook] = None,
+    preprocess_traces_request: Optional[TracesRequestHook] = None,
+) -> ProxyConfig:
+    """Load environment and secrets, and build ProxyConfig"""
+    # Env
+    litellm_url = os.getenv("LITELLM_URL")
+    if not litellm_url:
+        raise ValueError("LITELLM_URL environment variable must be set")
+    request_timeout = float(os.getenv("REQUEST_TIMEOUT", "300.0"))
+    langfuse_host = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
+
+    # Secrets - use SECRETS_PATH env var if set, otherwise default to proxy/secrets.yaml
+    secrets_path_str = os.getenv("SECRETS_PATH")
+    if secrets_path_str:
+        secrets_path = Path(secrets_path_str)
+    else:
+        secrets_path = Path(__file__).parent / "secrets.yaml"
+    if not secrets_path.exists():
+        raise ValueError(
+            "Secrets file not found! Please create it from secrets.yaml.example:\n"
+            "  cp eval_protocol/proxy/proxy_core/secrets.yaml.example eval_protocol/proxy/proxy_core/secrets.yaml\n"
+            "Then add your Langfuse API keys to the secrets file"
+        )
+    try:
+        with open(secrets_path, "r") as f:
+            secrets_config = yaml.safe_load(f)
+        langfuse_keys = secrets_config["langfuse_keys"]
+        default_project_id = secrets_config["default_project_id"]
+        logger.info(f"Loaded {len(langfuse_keys)} Langfuse project(s) from {secrets_path.name}")
+    except KeyError as e:
+        raise ValueError(f"Missing required key in secrets file: {e}")
+    except yaml.YAMLError as e:
+        raise ValueError(f"Invalid format in secrets file {secrets_path.name}: {e}")
+
+    return ProxyConfig(
+        litellm_url=litellm_url,
+        request_timeout=request_timeout,
+        langfuse_host=langfuse_host,
+        langfuse_keys=langfuse_keys,
+        default_project_id=default_project_id,
+        preprocess_chat_request=preprocess_chat_request,
+        preprocess_traces_request=preprocess_traces_request,
+    )
+
+
+def init_redis() -> redis.Redis:
+    """Initialize and return a Redis client from environment variables."""
+    redis_host = os.getenv("REDIS_HOST")
+    if not redis_host:
+        raise ValueError("REDIS_HOST environment variable must be set")
+    redis_port = int(os.getenv("REDIS_PORT", "6379"))
+    redis_password = os.getenv("REDIS_PASSWORD")
+
+    try:
+        client = redis.Redis(
+            host=redis_host,
+            port=redis_port,
+            password=redis_password if redis_password else None,
+            decode_responses=True,
+            socket_connect_timeout=5,
+            socket_timeout=5,
+            retry_on_timeout=True,
+        )
+        client.ping()
+        logger.info(f"Connected to Redis at {redis_host}:{redis_port}")
+        return client
+    except Exception as e:
+        raise ConnectionError(f"Failed to connect to Redis at {redis_host}:{redis_port}: {e}")
+
+
+def create_app(
+    auth_provider: AuthProvider = NoAuthProvider(),
+    preprocess_chat_request: Optional[ChatRequestHook] = None,
+    preprocess_traces_request: Optional[TracesRequestHook] = None,
+) -> FastAPI:
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        # Build runtime on startup
+        app.state.config = build_proxy_config(preprocess_chat_request, preprocess_traces_request)
+        app.state.redis = init_redis()
+
+        try:
+            yield
+        finally:
+            try:
+                app.state.redis.close()
+            except Exception:
+                pass
+
+    app = FastAPI(title="LiteLLM Metadata Proxy", lifespan=lifespan)
+
+    def get_config(request: Request) -> ProxyConfig:
+        return request.app.state.config
+
+    def get_redis(request: Request) -> redis.Redis:
+        return request.app.state.redis
+
+    def get_traces_params(
+        tags: Optional[List[str]] = Query(default=None),
+        project_id: Optional[str] = None,
+        limit: int = 100,
+        sample_size: Optional[int] = None,
+        user_id: Optional[str] = None,
+        session_id: Optional[str] = None,
+        name: Optional[str] = None,
+        environment: Optional[str] = None,
+        version: Optional[str] = None,
+        release: Optional[str] = None,
+        fields: Optional[str] = None,
+        hours_back: Optional[int] = None,
+        from_timestamp: Optional[str] = None,
+        to_timestamp: Optional[str] = None,
+        sleep_between_gets: float = 2.5,
+        max_retries: int = 3,
+    ) -> TracesParams:
+        return TracesParams(
+            tags=tags,
+            project_id=project_id,
+            limit=limit,
+            sample_size=sample_size,
+            user_id=user_id,
+            session_id=session_id,
+            name=name,
+            environment=environment,
+            version=version,
+            release=release,
+            fields=fields,
+            hours_back=hours_back,
+            from_timestamp=from_timestamp,
+            to_timestamp=to_timestamp,
+            sleep_between_gets=sleep_between_gets,
+            max_retries=max_retries,
+        )
+
+    async def require_auth(request: Request) -> None:
+        account_id = auth_provider.validate_and_return_account_id(request)
+        request.state.account_id = account_id
+        return None
+
+    # =====================
+    # Chat completion routes
+    # =====================
+    @app.post(
+        "/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/v1/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/v1/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/chat/completions"
+    )
+    @app.post(
+        "/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    @app.post(
+        "/v1/project_id/{project_id}/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    @app.post(
+        "/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    @app.post(
+        "/v1/rollout_id/{rollout_id}/invocation_id/{invocation_id}/experiment_id/{experiment_id}/run_id/{run_id}/row_id/{row_id}/encoded_base_url/{encoded_base_url}/chat/completions"
+    )
+    async def chat_completion_with_full_metadata(
+        rollout_id: str,
+        invocation_id: str,
+        experiment_id: str,
+        run_id: str,
+        row_id: str,
+        request: Request,
+        project_id: Optional[str] = None,
+        encoded_base_url: Optional[str] = None,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+        _: None = Depends(require_auth),
+    ):
+        params = ChatParams(
+            project_id=project_id,
+            rollout_id=rollout_id,
+            invocation_id=invocation_id,
+            experiment_id=experiment_id,
+            run_id=run_id,
+            row_id=row_id,
+            encoded_base_url=encoded_base_url,
+        )
+        return await handle_chat_completion(
+            config=config,
+            redis_client=redis_client,
+            request=request,
+            params=params,
+        )
+
+    @app.post("/project_id/{project_id}/chat/completions")
+    @app.post("/v1/project_id/{project_id}/chat/completions")
+    async def chat_completion_with_project_only(
+        project_id: str,
+        request: Request,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+        _: None = Depends(require_auth),
+    ):
+        params = ChatParams(project_id=project_id)
+        return await handle_chat_completion(
+            config=config,
+            redis_client=redis_client,
+            request=request,
+            params=params,
+        )
+
+    # ===============
+    # Traces routes
+    # ===============
+    @app.get("/traces", response_model=LangfuseTracesResponse)
+    @app.get("/v1/traces", response_model=LangfuseTracesResponse)
+    @app.get("/project_id/{project_id}/traces", response_model=LangfuseTracesResponse)
+    @app.get("/v1/project_id/{project_id}/traces", response_model=LangfuseTracesResponse)
+    async def get_langfuse_traces(
+        request: Request,
+        params: TracesParams = Depends(get_traces_params),
+        project_id: Optional[str] = None,
+        config: ProxyConfig = Depends(get_config),
+        redis_client: redis.Redis = Depends(get_redis),
+        _: None = Depends(require_auth),
+    ) -> LangfuseTracesResponse:
+        if project_id is not None:
+            params.project_id = project_id
+        return await fetch_langfuse_traces(
+            config=config,
+            redis_client=redis_client,
+            request=request,
+            params=params,
+        )
+
+    # Health
+    @app.get("/health")
+    async def health():
+        return {"status": "healthy", "service": "metadata-proxy"}
+
+    # Catch-all
+    @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
+    async def catch_all_proxy(
+        path: str,
+        request: Request,
+        config: ProxyConfig = Depends(get_config),
+    ):
+        return await proxy_to_litellm(config, path, request)
+
+    return app

eval_protocol-0.2.46.dev0/eval_protocol/proxy/proxy_core/auth.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+import logging
+from fastapi import Request
+from fastapi import HTTPException
+import httpx
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class AuthProvider(ABC):
+    @abstractmethod
+    def validate_and_return_account_id(self, request: Request) -> Optional[str]: ...
+
+
+class NoAuthProvider(AuthProvider):
+    def validate_and_return_account_id(self, request: Request) -> Optional[str]:
+        return None