eval-protocol 0.2.53__tar.gz → 0.2.54.dev0__tar.gz

{eval_protocol-0.2.53/eval_protocol.egg-info → eval_protocol-0.2.54.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.53
+Version: 0.2.54.dev0
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/__init__.py

@@ -31,14 +31,10 @@ from .reward_function import RewardFunction
 from .typed_interface import reward_function
 from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
 from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
-from .pytest.remote_rollout_processor import create_elasticsearch_config_from_env
 from .pytest.parameterize import DefaultParameterIdGenerator
 from .log_utils.elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
 from .log_utils.rollout_id_filter import RolloutIdFilter
 from .log_utils.util import setup_rollout_logging_for_elasticsearch_handler
-from .log_utils.fireworks_tracing_http_handler import FireworksTracingHttpHandler
-from .log_utils.elasticsearch_client import ElasticsearchConfig
-
 
 from .types.remote_rollout_processor import (
     InitRequest,
@@ -74,11 +70,16 @@ try:
 except ImportError:
     WeaveAdapter = None
 
+try:
+    from .proxy import create_app, AuthProvider
+except ImportError:
+    create_app = None
+    AuthProvider = None
+
+
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
-    "create_elasticsearch_config_from_env",
-    "ElasticsearchConfig",
     "ElasticsearchDirectHttpHandler",
     "RolloutIdFilter",
     "setup_rollout_logging_for_elasticsearch_handler",
@@ -101,7 +102,6 @@ __all__ = [
     "BraintrustAdapter",
     "create_braintrust_adapter",
     "LangSmithAdapter",
-    "FireworksTracingHttpHandler",
     # Core interfaces
     "Message",
     "MetricResult",
@@ -137,6 +137,9 @@ __all__ = [
     "RolloutMetadata",
     "StatusResponse",
     "create_langfuse_config_tags",
+    # Proxy
+    "create_app",
+    "AuthProvider",
 ]
 
 from . import _version

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-13T11:21:58-0700",
+ "date": "2025-10-13T17:28:46-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "3b326f2fcc52573154c9952ad882880de8096c6e",
- "version": "0.2.53"
+ "full-revisionid": "bfe8e3146c3971cadf5c7e43d259b40e7e26163a",
+ "version": "0.2.54-dev"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/adapters/fireworks_tracing.py

@@ -7,9 +7,9 @@ to pull data from Langfuse deployments with simplified retry logic handling.
 from __future__ import annotations
 import logging
 import requests
-import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
+import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -343,15 +343,17 @@ class FireworksTracingAdapter(BaseAdapter):
         # Remove None values
         params = {k: v for k, v in params.items() if v is not None}
 
-        # Make request to proxy
+        # Make request to proxy (using pointwise for efficiency)
         if self.project_id:
-            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces"
+            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces/pointwise"
         else:
-            url = f"{self.base_url}/v1/traces"
+            url = f"{self.base_url}/v1/traces/pointwise"
+
+        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
 
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout)
+            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
             response.raise_for_status()
             result = response.json()
         except requests.exceptions.HTTPError as e:
@@ -365,7 +367,7 @@ class FireworksTracingAdapter(BaseAdapter):
                 except Exception:  # In case e.response.json() fails
                     error_msg = f"Proxy error: {e.response.text}"
 
-            logger.error("Failed to fetch traces from proxy: %s", error_msg)
+            logger.error("Failed to fetch traces from proxy (HTTP %s): %s", e.response.status_code, error_msg)
             return eval_rows
         except requests.exceptions.RequestException as e:
             # Non-HTTP errors (network issues, timeouts, etc.)

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/adapters/openai_responses.py

@@ -169,9 +169,7 @@ class OpenAIResponsesAdapter(BaseAdapter):
                         raise NotImplementedError(f"Unsupported content type: {content_item.type}")
             elif item.type == "function_call_output":
                 # Collect tool call outputs to add before assistant message
-                tool_call_outputs.append(
-                    Message(role="tool", content=self._coerce_tool_output(item.output), tool_call_id=item.call_id)
-                )
+                tool_call_outputs.append(Message(role="tool", content=item.output, tool_call_id=item.call_id))
             elif item.type == "function_call":
                 tool_call = ChatCompletionMessageToolCall(
                     id=item.call_id, type="function", function=Function(name=item.name, arguments=item.arguments)
@@ -188,29 +186,3 @@ class OpenAIResponsesAdapter(BaseAdapter):
             messages.append(Message(role="assistant", tool_calls=current_tool_calls))
 
         return reversed(messages)
-
-    def _coerce_tool_output(self, output: Any) -> str:
-        """Coerce OpenAI Responses tool output into a string for Message.content.
-
-        The Responses API may return structured content lists. For our purposes,
-        we stringify non-string outputs to satisfy the Message.content type.
-        """
-        if isinstance(output, str):
-            return output
-        try:
-            # Attempt to join list of objects with any 'text' fields
-            if isinstance(output, list):
-                parts: list[str] = []
-                for part in output:
-                    text = None
-                    if isinstance(part, dict):
-                        text = part.get("text")
-                    if text:
-                        parts.append(str(text))
-                    else:
-                        parts.append(str(part))
-                return "\n".join(parts)
-            # Fallback to string conversion
-            return str(output)
-        except Exception:
-            return str(output)

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/auth.py

@@ -4,8 +4,6 @@ import os
 from pathlib import Path
 from typing import Dict, Optional  # Added Dict
 
-import requests
-
 logger = logging.getLogger(__name__)
 
 # Default locations (used for tests and as fallback). Actual resolution is dynamic via _get_auth_ini_file().
@@ -220,40 +218,3 @@ def get_fireworks_api_base() -> str:
     else:
         logger.debug("FIREWORKS_API_BASE not set in environment, defaulting to %s.", api_base)
     return api_base
-
-
-def verify_api_key_and_get_account_id(
-    api_key: Optional[str] = None,
-    api_base: Optional[str] = None,
-) -> Optional[str]:
-    """
-    Calls the Fireworks API verify endpoint to validate the API key and returns the
-    account id from response headers when available.
-
-    Args:
-        api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
-        api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
-
-    Returns:
-        The resolved account id if verification succeeds and the header is present; otherwise None.
-    """
-    try:
-        resolved_key = api_key or get_fireworks_api_key()
-        if not resolved_key:
-            return None
-        resolved_base = api_base or get_fireworks_api_base()
-        url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
-        resp = requests.get(url, headers=headers, timeout=10)
-        if resp.status_code != 200:
-            logger.debug("verifyApiKey returned status %s", resp.status_code)
-            return None
-        # Header keys could vary in case; requests provides case-insensitive dict
-        account_id = resp.headers.get("x-fireworks-account-id") or resp.headers.get("X-Fireworks-Account-Id")
-        if account_id and account_id.strip():
-            logger.debug("Resolved FIREWORKS_ACCOUNT_ID via verifyApiKey: %s", account_id)
-            return account_id.strip()
-        return None
-    except Exception as e:
-        logger.debug("Failed to verify API key for account id resolution: %s", e)
-        return None

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/cli.py

@@ -301,12 +301,6 @@ def parse_args(args=None):
     logs_parser = subparsers.add_parser("logs", help="Serve logs with file watching and real-time updates")
     logs_parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)")
     logs_parser.add_argument("--debug", action="store_true", help="Enable debug mode")
-    logs_parser.add_argument("--disable-elasticsearch-setup", action="store_true", help="Disable Elasticsearch setup")
-    logs_parser.add_argument(
-        "--use-env-elasticsearch-config",
-        action="store_true",
-        help="Use env vars for Elasticsearch config (requires ELASTICSEARCH_URL, ELASTICSEARCH_API_KEY, ELASTICSEARCH_INDEX_NAME)",
-    )
 
     # Upload command
     upload_parser = subparsers.add_parser(

eval_protocol-0.2.54.dev0/eval_protocol/cli_commands/logs.py

@@ -0,0 +1,36 @@
+"""
+CLI command for serving logs with file watching and real-time updates.
+"""
+
+import sys
+from pathlib import Path
+
+from ..utils.logs_server import serve_logs
+
+
+def logs_command(args):
+    """Serve logs with file watching and real-time updates"""
+
+    port = args.port
+    print("🚀 Starting Eval Protocol Logs Server")
+    print(f"🌐 URL: http://localhost:{port}")
+    print(f"🔌 WebSocket: ws://localhost:{port}/ws")
+    print(f"👀 Watching paths: {['current directory']}")
+    print(f"🔍 Debug mode: {args.debug}")
+    print("Press Ctrl+C to stop the server")
+    print("-" * 50)
+
+    # setup Elasticsearch
+    from eval_protocol.pytest.elasticsearch_setup import ElasticsearchSetup
+
+    elasticsearch_config = ElasticsearchSetup().setup_elasticsearch()
+
+    try:
+        serve_logs(port=args.port, elasticsearch_config=elasticsearch_config, debug=args.debug)
+        return 0
+    except KeyboardInterrupt:
+        print("\n🛑 Server stopped by user")
+        return 0
+    except Exception as e:
+        print(f"❌ Error starting server: {e}")
+        return 1

{eval_protocol-0.2.53 → eval_protocol-0.2.54.dev0}/eval_protocol/cli_commands/upload.py

@@ -12,12 +12,7 @@ from pathlib import Path
 from typing import Any, Callable, Iterable, Optional
 
 import pytest
-from eval_protocol.auth import (
-    get_fireworks_account_id,
-    get_fireworks_api_key,
-    get_fireworks_api_base,
-    verify_api_key_and_get_account_id,
-)
+from eval_protocol.auth import get_fireworks_account_id, get_fireworks_api_key
 from eval_protocol.platform_api import create_or_update_fireworks_secret
 
 from eval_protocol.evaluation import create_evaluation
@@ -264,7 +259,7 @@ def _parse_entry(entry: str, cwd: str) -> tuple[str, str]:
         raise ValueError("--entry must be in 'module::function', 'path::function', or 'module:function' format")
 
 
-def _resolve_entry_to_qual_and_source(entry: str, cwd: str) -> tuple[str, str]:
+def _generate_ts_mode_code_from_entry(entry: str, cwd: str) -> tuple[str, str, str, str]:
     target, func = _parse_entry(entry, cwd)
 
     # Check if target looks like a file path
@@ -298,12 +293,47 @@ def _resolve_entry_to_qual_and_source(entry: str, cwd: str) -> tuple[str, str]:
         raise ValueError(f"Function '{func}' not found in module '{module_name}'")
 
     qualname = f"{module_name}.{func}"
-    return qualname, os.path.abspath(source_file_path) if source_file_path else ""
+    code, file_name = _generate_ts_mode_code(
+        DiscoveredTest(
+            module_path=module_name,
+            module_name=module_name,
+            qualname=qualname,
+            file_path=getattr(module, "__file__", module_name),
+            lineno=None,
+            has_parametrize=False,
+            param_count=0,
+            nodeids=[],
+        )
+    )
+    return code, file_name, qualname, os.path.abspath(source_file_path) if source_file_path else ""
 
 
 def _generate_ts_mode_code(test: DiscoveredTest) -> tuple[str, str]:
-    # Deprecated: we no longer generate a shim; keep stub for import compatibility
-    return ("", "main.py")
+    # Generate a minimal main.py that imports the test module and calls the function
+    module = test.module_name
+    func = test.qualname.split(".")[-1]
+    code = f"""
+from typing import Any, Dict, List, Optional, Union
+
+from eval_protocol.models import EvaluationRow, Message
+from {module} import {func} as _ep_test
+
+def evaluate(messages: List[Dict[str, Any]], ground_truth: Optional[Union[str, List[Dict[str, Any]]]] = None, tools=None, **kwargs):
+    row = EvaluationRow(messages=[Message(**m) for m in messages], ground_truth=ground_truth)
+    result = _ep_test(row)  # Supports sync/async via decorator's dual-mode
+    if hasattr(result, "__await__"):
+        import asyncio
+        result = asyncio.get_event_loop().run_until_complete(result)
+    if result.evaluation_result is None:
+        return {{"score": 0.0, "reason": "No evaluation_result set"}}
+    out = {{
+        "score": float(result.evaluation_result.score or 0.0),
+        "reason": result.evaluation_result.reason,
+        "metrics": {{k: (v.model_dump() if hasattr(v, "model_dump") else v) for k, v in (result.evaluation_result.metrics or {{}}).items()}},
+    }}
+    return out
+"""
+    return (code, "main.py")
 
 
 def _normalize_evaluator_id(evaluator_id: str) -> str:
@@ -492,10 +522,10 @@ def upload_command(args: argparse.Namespace) -> int:
     entries_arg = getattr(args, "entry", None)
     if entries_arg:
         entries = [e.strip() for e in re.split(r"[,\s]+", entries_arg) if e.strip()]
-        selected_specs: list[tuple[str, str]] = []
+        selected_specs: list[tuple[str, str, str, str]] = []
         for e in entries:
-            qualname, resolved_path = _resolve_entry_to_qual_and_source(e, root)
-            selected_specs.append((qualname, resolved_path))
+            code, file_name, qualname, resolved_path = _generate_ts_mode_code_from_entry(e, root)
+            selected_specs.append((code, file_name, qualname, resolved_path))
     else:
         print("Scanning for evaluation tests...")
         tests = _discover_tests(root)
@@ -515,7 +545,11 @@ def upload_command(args: argparse.Namespace) -> int:
             print("      handles all parameter combinations. The evaluator will work with")
             print("      the same logic regardless of which model/parameters are used.")
 
-        selected_specs = [(t.qualname, t.file_path) for t in selected_tests]
+        selected_specs = []
+        for t in selected_tests:
+            code, file_name = _generate_ts_mode_code(t)
+            # Store test info for better ID generation
+            selected_specs.append((code, file_name, t.qualname, t.file_path))
 
     base_id = getattr(args, "id", None)
     display_name = getattr(args, "display_name", None)
@@ -526,14 +560,6 @@ def upload_command(args: argparse.Namespace) -> int:
     try:
         fw_account_id = get_fireworks_account_id()
         fw_api_key_value = get_fireworks_api_key()
-        if not fw_account_id and fw_api_key_value:
-            # Attempt to verify and resolve account id from server headers
-            resolved = verify_api_key_and_get_account_id(api_key=fw_api_key_value, api_base=get_fireworks_api_base())
-            if resolved:
-                fw_account_id = resolved
-                # Propagate to environment so downstream calls use it if needed
-                os.environ["FIREWORKS_ACCOUNT_ID"] = fw_account_id
-                print(f"Resolved FIREWORKS_ACCOUNT_ID via API verification: {fw_account_id}")
         if fw_account_id and fw_api_key_value:
             print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
             if create_or_update_fireworks_secret(
@@ -553,7 +579,8 @@ def upload_command(args: argparse.Namespace) -> int:
         print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
 
     exit_code = 0
-    for i, (qualname, source_file_path) in enumerate(selected_specs):
+    for i, (code, file_name, qualname, source_file_path) in enumerate(selected_specs):
+        # Use ts_mode to upload evaluator
         # Generate a short default ID from just the test function name
         if base_id:
             evaluator_id = base_id
@@ -591,12 +618,12 @@ def upload_command(args: argparse.Namespace) -> int:
 
         print(f"\nUploading evaluator '{evaluator_id}' for {qualname.split('.')[-1]}...")
         try:
-            # Always treat as a single evaluator (single-metric) even if folder has helper modules
-            test_dir = os.path.dirname(source_file_path) if source_file_path else root
-            metric_name = os.path.basename(test_dir) or "metric"
             result = create_evaluation(
                 evaluator_id=evaluator_id,
-                metric_folders=[f"{metric_name}={test_dir}"],
+                python_code_to_evaluate=code,
+                python_file_name_for_code=file_name,
+                criterion_name_for_code=qualname,
+                criterion_description_for_code=description or f"Evaluator for {qualname}",
                 display_name=display_name or evaluator_id,
                 description=description or f"Evaluator for {qualname}",
                 force=force,