eval-protocol 0.2.51.dev0__tar.gz → 0.2.52__tar.gz

{eval_protocol-0.2.51.dev0/eval_protocol.egg-info → eval_protocol-0.2.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.51.dev0
+Version: 0.2.52
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/__init__.py

@@ -31,10 +31,14 @@ from .reward_function import RewardFunction
 from .typed_interface import reward_function
 from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
 from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
+from .pytest.remote_rollout_processor import create_elasticsearch_config_from_env
 from .pytest.parameterize import DefaultParameterIdGenerator
 from .log_utils.elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
 from .log_utils.rollout_id_filter import RolloutIdFilter
 from .log_utils.util import setup_rollout_logging_for_elasticsearch_handler
+from .log_utils.fireworks_tracing_http_handler import FireworksTracingHttpHandler
+from .log_utils.elasticsearch_client import ElasticsearchConfig
+
 
 from .types.remote_rollout_processor import (
     InitRequest,
@@ -70,16 +74,11 @@ try:
 except ImportError:
     WeaveAdapter = None
 
-try:
-    from .proxy import create_app, AuthProvider
-except ImportError:
-    create_app = None
-    AuthProvider = None
-
-
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "create_elasticsearch_config_from_env",
+    "ElasticsearchConfig",
     "ElasticsearchDirectHttpHandler",
     "RolloutIdFilter",
     "setup_rollout_logging_for_elasticsearch_handler",
@@ -102,6 +101,7 @@ __all__ = [
     "BraintrustAdapter",
     "create_braintrust_adapter",
     "LangSmithAdapter",
+    "FireworksTracingHttpHandler",
     # Core interfaces
     "Message",
     "MetricResult",
@@ -137,9 +137,6 @@ __all__ = [
     "RolloutMetadata",
     "StatusResponse",
     "create_langfuse_config_tags",
-    # Proxy
-    "create_app",
-    "AuthProvider",
 ]
 
 from . import _version

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-10T21:44:48-0700",
+ "date": "2025-10-13T00:31:45-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "e753a0be05950541cdf51cd66ad22190ec1b1571",
- "version": "0.2.51-dev"
+ "full-revisionid": "270a91e21f730169bd3ff7f94c44f8c0502ace33",
+ "version": "0.2.52"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/adapters/fireworks_tracing.py

@@ -7,9 +7,9 @@ to pull data from Langfuse deployments with simplified retry logic handling.
 from __future__ import annotations
 import logging
 import requests
+import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
-import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -343,17 +343,15 @@ class FireworksTracingAdapter(BaseAdapter):
         # Remove None values
         params = {k: v for k, v in params.items() if v is not None}
 
-        # Make request to proxy (using pointwise for efficiency)
+        # Make request to proxy
         if self.project_id:
-            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces/pointwise"
+            url = f"{self.base_url}/v1/project_id/{self.project_id}/traces"
         else:
-            url = f"{self.base_url}/v1/traces/pointwise"
-
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+            url = f"{self.base_url}/v1/traces"
 
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
+            response = requests.get(url, params=params, timeout=self.timeout)
             response.raise_for_status()
             result = response.json()
         except requests.exceptions.HTTPError as e:
@@ -367,7 +365,7 @@ class FireworksTracingAdapter(BaseAdapter):
                 except Exception:  # In case e.response.json() fails
                     error_msg = f"Proxy error: {e.response.text}"
 
-            logger.error("Failed to fetch traces from proxy (HTTP %s): %s", e.response.status_code, error_msg)
+            logger.error("Failed to fetch traces from proxy: %s", error_msg)
             return eval_rows
         except requests.exceptions.RequestException as e:
             # Non-HTTP errors (network issues, timeouts, etc.)

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/adapters/openai_responses.py

@@ -169,7 +169,9 @@ class OpenAIResponsesAdapter(BaseAdapter):
                         raise NotImplementedError(f"Unsupported content type: {content_item.type}")
             elif item.type == "function_call_output":
                 # Collect tool call outputs to add before assistant message
-                tool_call_outputs.append(Message(role="tool", content=item.output, tool_call_id=item.call_id))
+                tool_call_outputs.append(
+                    Message(role="tool", content=self._coerce_tool_output(item.output), tool_call_id=item.call_id)
+                )
             elif item.type == "function_call":
                 tool_call = ChatCompletionMessageToolCall(
                     id=item.call_id, type="function", function=Function(name=item.name, arguments=item.arguments)
@@ -186,3 +188,29 @@ class OpenAIResponsesAdapter(BaseAdapter):
             messages.append(Message(role="assistant", tool_calls=current_tool_calls))
 
         return reversed(messages)
+
+    def _coerce_tool_output(self, output: Any) -> str:
+        """Coerce OpenAI Responses tool output into a string for Message.content.
+
+        The Responses API may return structured content lists. For our purposes,
+        we stringify non-string outputs to satisfy the Message.content type.
+        """
+        if isinstance(output, str):
+            return output
+        try:
+            # Attempt to join list of objects with any 'text' fields
+            if isinstance(output, list):
+                parts: list[str] = []
+                for part in output:
+                    text = None
+                    if isinstance(part, dict):
+                        text = part.get("text")
+                    if text:
+                        parts.append(str(text))
+                    else:
+                        parts.append(str(part))
+                return "\n".join(parts)
+            # Fallback to string conversion
+            return str(output)
+        except Exception:
+            return str(output)

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/auth.py

@@ -4,6 +4,8 @@ import os
 from pathlib import Path
 from typing import Dict, Optional  # Added Dict
 
+import requests
+
 logger = logging.getLogger(__name__)
 
 # Default locations (used for tests and as fallback). Actual resolution is dynamic via _get_auth_ini_file().
@@ -218,3 +220,40 @@ def get_fireworks_api_base() -> str:
     else:
         logger.debug("FIREWORKS_API_BASE not set in environment, defaulting to %s.", api_base)
     return api_base
+
+
+def verify_api_key_and_get_account_id(
+    api_key: Optional[str] = None,
+    api_base: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Calls the Fireworks API verify endpoint to validate the API key and returns the
+    account id from response headers when available.
+
+    Args:
+        api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
+        api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
+
+    Returns:
+        The resolved account id if verification succeeds and the header is present; otherwise None.
+    """
+    try:
+        resolved_key = api_key or get_fireworks_api_key()
+        if not resolved_key:
+            return None
+        resolved_base = api_base or get_fireworks_api_base()
+        url = f"{resolved_base.rstrip('/')}/verifyApiKey"
+        headers = {"Authorization": f"Bearer {resolved_key}"}
+        resp = requests.get(url, headers=headers, timeout=10)
+        if resp.status_code != 200:
+            logger.debug("verifyApiKey returned status %s", resp.status_code)
+            return None
+        # Header keys could vary in case; requests provides case-insensitive dict
+        account_id = resp.headers.get("x-fireworks-account-id") or resp.headers.get("X-Fireworks-Account-Id")
+        if account_id and account_id.strip():
+            logger.debug("Resolved FIREWORKS_ACCOUNT_ID via verifyApiKey: %s", account_id)
+            return account_id.strip()
+        return None
+    except Exception as e:
+        logger.debug("Failed to verify API key for account id resolution: %s", e)
+        return None

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/cli.py

@@ -301,6 +301,12 @@ def parse_args(args=None):
     logs_parser = subparsers.add_parser("logs", help="Serve logs with file watching and real-time updates")
     logs_parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)")
     logs_parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+    logs_parser.add_argument("--disable-elasticsearch-setup", action="store_true", help="Disable Elasticsearch setup")
+    logs_parser.add_argument(
+        "--use-env-elasticsearch-config",
+        action="store_true",
+        help="Use env vars for Elasticsearch config (requires ELASTICSEARCH_URL, ELASTICSEARCH_API_KEY, ELASTICSEARCH_INDEX_NAME)",
+    )
 
     # Upload command
     upload_parser = subparsers.add_parser(

eval_protocol-0.2.52/eval_protocol/cli_commands/logs.py

@@ -0,0 +1,76 @@
+"""
+CLI command for serving logs with file watching and real-time updates.
+"""
+
+import sys
+from pathlib import Path
+
+from ..utils.logs_server import serve_logs
+
+
+def logs_command(args):
+    """Serve logs with file watching and real-time updates"""
+
+    port = args.port
+    print("🚀 Starting Eval Protocol Logs Server")
+    print(f"🌐 URL: http://localhost:{port}")
+    print(f"🔌 WebSocket: ws://localhost:{port}/ws")
+    print(f"👀 Watching paths: {['current directory']}")
+    print(f"🔍 Debug mode: {args.debug}")
+    print("Press Ctrl+C to stop the server")
+    print("-" * 50)
+
+    # Setup Elasticsearch based on flags
+    elasticsearch_config = None
+    try:
+        if getattr(args, "use_env_elasticsearch_config", False):
+            # Use environment variables for configuration
+            print("⚙️ Using environment variables for Elasticsearch config")
+            from eval_protocol.pytest.remote_rollout_processor import (
+                create_elasticsearch_config_from_env,
+            )
+
+            elasticsearch_config = create_elasticsearch_config_from_env()
+            # Ensure index exists with correct mapping, mirroring Docker setup path
+            try:
+                from eval_protocol.log_utils.elasticsearch_index_manager import (
+                    ElasticsearchIndexManager,
+                )
+
+                index_manager = ElasticsearchIndexManager(
+                    elasticsearch_config.url,
+                    elasticsearch_config.index_name,
+                    elasticsearch_config.api_key,
+                )
+                created = index_manager.create_logging_index_mapping()
+                if created:
+                    print(
+                        f"🧭 Verified Elasticsearch index '{elasticsearch_config.index_name}' mapping (created or already correct)"
+                    )
+                else:
+                    print(
+                        f"⚠️ Could not verify/create mapping for index '{elasticsearch_config.index_name}'. Searches may behave unexpectedly."
+                    )
+            except Exception as e:
+                print(f"⚠️ Failed to ensure index mapping via IndexManager: {e}")
+        elif not getattr(args, "disable_elasticsearch_setup", False):
+            # Default behavior: start or connect to local Elasticsearch via Docker helper
+            from eval_protocol.pytest.elasticsearch_setup import ElasticsearchSetup
+
+            print("🧰 Auto-configuring local Elasticsearch (Docker)")
+            elasticsearch_config = ElasticsearchSetup().setup_elasticsearch()
+        else:
+            print("🚫 Elasticsearch setup disabled; running without Elasticsearch integration")
+    except Exception as e:
+        print(f"❌ Failed to configure Elasticsearch: {e}")
+        return 1
+
+    try:
+        serve_logs(port=args.port, elasticsearch_config=elasticsearch_config, debug=args.debug)
+        return 0
+    except KeyboardInterrupt:
+        print("\n🛑 Server stopped by user")
+        return 0
+    except Exception as e:
+        print(f"❌ Error starting server: {e}")
+        return 1

{eval_protocol-0.2.51.dev0 → eval_protocol-0.2.52}/eval_protocol/cli_commands/upload.py

@@ -12,7 +12,12 @@ from pathlib import Path
 from typing import Any, Callable, Iterable, Optional
 
 import pytest
-from eval_protocol.auth import get_fireworks_account_id, get_fireworks_api_key
+from eval_protocol.auth import (
+    get_fireworks_account_id,
+    get_fireworks_api_key,
+    get_fireworks_api_base,
+    verify_api_key_and_get_account_id,
+)
 from eval_protocol.platform_api import create_or_update_fireworks_secret
 
 from eval_protocol.evaluation import create_evaluation
@@ -259,7 +264,7 @@ def _parse_entry(entry: str, cwd: str) -> tuple[str, str]:
         raise ValueError("--entry must be in 'module::function', 'path::function', or 'module:function' format")
 
 
-def _generate_ts_mode_code_from_entry(entry: str, cwd: str) -> tuple[str, str, str, str]:
+def _resolve_entry_to_qual_and_source(entry: str, cwd: str) -> tuple[str, str]:
     target, func = _parse_entry(entry, cwd)
 
     # Check if target looks like a file path
@@ -293,47 +298,12 @@ def _generate_ts_mode_code_from_entry(entry: str, cwd: str) -> tuple[str, str, s
         raise ValueError(f"Function '{func}' not found in module '{module_name}'")
 
     qualname = f"{module_name}.{func}"
-    code, file_name = _generate_ts_mode_code(
-        DiscoveredTest(
-            module_path=module_name,
-            module_name=module_name,
-            qualname=qualname,
-            file_path=getattr(module, "__file__", module_name),
-            lineno=None,
-            has_parametrize=False,
-            param_count=0,
-            nodeids=[],
-        )
-    )
-    return code, file_name, qualname, os.path.abspath(source_file_path) if source_file_path else ""
+    return qualname, os.path.abspath(source_file_path) if source_file_path else ""
 
 
 def _generate_ts_mode_code(test: DiscoveredTest) -> tuple[str, str]:
-    # Generate a minimal main.py that imports the test module and calls the function
-    module = test.module_name
-    func = test.qualname.split(".")[-1]
-    code = f"""
-from typing import Any, Dict, List, Optional, Union
-
-from eval_protocol.models import EvaluationRow, Message
-from {module} import {func} as _ep_test
-
-def evaluate(messages: List[Dict[str, Any]], ground_truth: Optional[Union[str, List[Dict[str, Any]]]] = None, tools=None, **kwargs):
-    row = EvaluationRow(messages=[Message(**m) for m in messages], ground_truth=ground_truth)
-    result = _ep_test(row)  # Supports sync/async via decorator's dual-mode
-    if hasattr(result, "__await__"):
-        import asyncio
-        result = asyncio.get_event_loop().run_until_complete(result)
-    if result.evaluation_result is None:
-        return {{"score": 0.0, "reason": "No evaluation_result set"}}
-    out = {{
-        "score": float(result.evaluation_result.score or 0.0),
-        "reason": result.evaluation_result.reason,
-        "metrics": {{k: (v.model_dump() if hasattr(v, "model_dump") else v) for k, v in (result.evaluation_result.metrics or {{}}).items()}},
-    }}
-    return out
-"""
-    return (code, "main.py")
+    # Deprecated: we no longer generate a shim; keep stub for import compatibility
+    return ("", "main.py")
 
 
 def _normalize_evaluator_id(evaluator_id: str) -> str:
@@ -522,10 +492,10 @@ def upload_command(args: argparse.Namespace) -> int:
     entries_arg = getattr(args, "entry", None)
     if entries_arg:
         entries = [e.strip() for e in re.split(r"[,\s]+", entries_arg) if e.strip()]
-        selected_specs: list[tuple[str, str, str, str]] = []
+        selected_specs: list[tuple[str, str]] = []
         for e in entries:
-            code, file_name, qualname, resolved_path = _generate_ts_mode_code_from_entry(e, root)
-            selected_specs.append((code, file_name, qualname, resolved_path))
+            qualname, resolved_path = _resolve_entry_to_qual_and_source(e, root)
+            selected_specs.append((qualname, resolved_path))
     else:
         print("Scanning for evaluation tests...")
         tests = _discover_tests(root)
@@ -545,11 +515,7 @@ def upload_command(args: argparse.Namespace) -> int:
             print("      handles all parameter combinations. The evaluator will work with")
             print("      the same logic regardless of which model/parameters are used.")
 
-        selected_specs = []
-        for t in selected_tests:
-            code, file_name = _generate_ts_mode_code(t)
-            # Store test info for better ID generation
-            selected_specs.append((code, file_name, t.qualname, t.file_path))
+        selected_specs = [(t.qualname, t.file_path) for t in selected_tests]
 
     base_id = getattr(args, "id", None)
     display_name = getattr(args, "display_name", None)
@@ -560,6 +526,14 @@ def upload_command(args: argparse.Namespace) -> int:
     try:
         fw_account_id = get_fireworks_account_id()
         fw_api_key_value = get_fireworks_api_key()
+        if not fw_account_id and fw_api_key_value:
+            # Attempt to verify and resolve account id from server headers
+            resolved = verify_api_key_and_get_account_id(api_key=fw_api_key_value, api_base=get_fireworks_api_base())
+            if resolved:
+                fw_account_id = resolved
+                # Propagate to environment so downstream calls use it if needed
+                os.environ["FIREWORKS_ACCOUNT_ID"] = fw_account_id
+                print(f"Resolved FIREWORKS_ACCOUNT_ID via API verification: {fw_account_id}")
         if fw_account_id and fw_api_key_value:
             print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
             if create_or_update_fireworks_secret(
@@ -579,8 +553,7 @@ def upload_command(args: argparse.Namespace) -> int:
         print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
 
     exit_code = 0
-    for i, (code, file_name, qualname, source_file_path) in enumerate(selected_specs):
-        # Use ts_mode to upload evaluator
+    for i, (qualname, source_file_path) in enumerate(selected_specs):
         # Generate a short default ID from just the test function name
         if base_id:
             evaluator_id = base_id
@@ -618,12 +591,12 @@ def upload_command(args: argparse.Namespace) -> int:
 
         print(f"\nUploading evaluator '{evaluator_id}' for {qualname.split('.')[-1]}...")
         try:
+            # Always treat as a single evaluator (single-metric) even if folder has helper modules
+            test_dir = os.path.dirname(source_file_path) if source_file_path else root
+            metric_name = os.path.basename(test_dir) or "metric"
             result = create_evaluation(
                 evaluator_id=evaluator_id,
-                python_code_to_evaluate=code,
-                python_file_name_for_code=file_name,
-                criterion_name_for_code=qualname,
-                criterion_description_for_code=description or f"Evaluator for {qualname}",
+                metric_folders=[f"{metric_name}={test_dir}"],
                 display_name=display_name or evaluator_id,
                 description=description or f"Evaluator for {qualname}",
                 force=force,