eval-protocol 0.2.45.dev0__tar.gz → 0.2.46__tar.gz

{eval_protocol-0.2.45.dev0/eval_protocol.egg-info → eval_protocol-0.2.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.45.dev0
+Version: 0.2.46
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-09T01:23:30-0700",
+ "date": "2025-10-09T13:58:14-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "c2ec0c8bb3f927b3c7f77c8a0e4fb955c7685ea6",
- "version": "0.2.45-dev"
+ "full-revisionid": "e066febd15f9056f74c40c8f4c34d3c68768fd59",
+ "version": "0.2.46"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46}/eval_protocol/adapters/fireworks_tracing.py

@@ -7,9 +7,9 @@ to pull data from Langfuse deployments with simplified retry logic handling.
 from __future__ import annotations
 import logging
 import requests
+import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
-import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -349,11 +349,9 @@ class FireworksTracingAdapter(BaseAdapter):
         else:
             url = f"{self.base_url}/v1/traces"
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
-
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
+            response = requests.get(url, params=params, timeout=self.timeout)
             response.raise_for_status()
             result = response.json()
         except requests.exceptions.HTTPError as e:

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46}/eval_protocol/pytest/evaluation_test.py

@@ -62,7 +62,8 @@ from eval_protocol.pytest.utils import (
     run_tasks_with_eval_progress,
     run_tasks_with_run_progress,
 )
-from eval_protocol.utils.show_results_url import store_local_ui_results_url
+from eval_protocol.utils.show_results_url import store_local_ui_results_url, generate_invocation_filter_url
+from eval_protocol.utils.browser_utils import is_logs_server_running, open_browser_tab
 
 from ..common_utils import load_jsonl
 
@@ -80,6 +81,7 @@ def evaluation_test(
     rollout_processor_kwargs: RolloutProcessorInputParam | None = None,
     aggregation_method: AggregationMethod = "mean",
     passed_threshold: EvaluationThreshold | float | EvaluationThresholdDict | None = None,
+    disable_browser_open: bool = False,
     num_runs: int = 1,
     filtered_row_ids: Sequence[str] | None = None,
     max_dataset_rows: int | None = None,
@@ -246,10 +248,29 @@ def evaluation_test(
             else:
                 invocation_id = generate_id()
 
+            # Track whether we've opened browser for this invocation
+            browser_opened_for_invocation = False
+
             async def wrapper_body(**kwargs: Unpack[ParameterizedTestKwargs]) -> None:
+                nonlocal browser_opened_for_invocation
+
                 # Store URL for viewing results (after all postprocessing is complete)
                 store_local_ui_results_url(invocation_id)
 
+                # Auto-open browser if server is running and not disabled (only once per invocation)
+                if (
+                    not browser_opened_for_invocation
+                    and not disable_browser_open
+                    and os.environ.get("EP_DISABLE_AUTO_BROWSER") is None
+                ):
+                    is_running, port = is_logs_server_running()
+                    if is_running:
+                        # Generate URL for table view with invocation filter
+                        base_url = f"http://localhost:{port}" if port else "http://localhost:8000"
+                        table_url = generate_invocation_filter_url(invocation_id, f"{base_url}/table")
+                        open_browser_tab(table_url)
+                        browser_opened_for_invocation = True
+
                 eval_metadata = None
 
                 all_results: list[list[EvaluationRow]] = [[] for _ in range(num_runs)]

eval_protocol-0.2.46/eval_protocol/utils/browser_utils.py

@@ -0,0 +1,114 @@
+"""
+Browser utilities for auto-opening evaluation results in the local UI.
+"""
+
+import json
+import os
+import threading
+import time
+import webbrowser
+from pathlib import Path
+from typing import Tuple, Optional
+
+try:
+    import psutil
+
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+
+
+def _get_pid_file_path() -> Path:
+    """Get the path to the logs server PID file."""
+    from eval_protocol.directory_utils import find_eval_protocol_dir
+
+    return Path(find_eval_protocol_dir()) / "logs_server.pid"
+
+
+def write_pid_file(pid: int, port: int) -> None:
+    """
+    Write the server PID and port to a file for external processes to check.
+
+    Args:
+        pid: The process ID of the logs server
+        port: The port the server is running on
+    """
+    try:
+        pid_file = _get_pid_file_path()
+
+        data = {"pid": pid, "port": port}
+
+        with open(pid_file, "w") as f:
+            json.dump(data, f)
+
+        # Use print instead of logger to avoid circular imports
+        print(f"Wrote PID file: {pid_file} with PID {pid} and port {port}")
+    except Exception as e:
+        print(f"Warning: Failed to write PID file: {e}")
+
+
+def is_logs_server_running() -> Tuple[bool, Optional[int]]:
+    """
+    Check if the logs server is running by reading the PID file and verifying the process.
+
+    Returns:
+        Tuple of (is_running, port) where:
+        - is_running: True if server is running, False otherwise
+        - port: The port the server is running on, or None if not running
+    """
+    if not PSUTIL_AVAILABLE:
+        return False, None
+
+    pid_file = _get_pid_file_path()
+    if not pid_file.exists():
+        return False, None
+
+    try:
+        with open(pid_file, "r") as f:
+            data = json.load(f)
+            pid = data.get("pid")
+            port = data.get("port")
+    except (json.JSONDecodeError, KeyError, FileNotFoundError):
+        return False, None
+
+    if pid is None:
+        return False, None
+
+    try:
+        # Check if the process is still running
+        process = psutil.Process(pid)
+        if not process.is_running():
+            return False, None
+
+        # Optionally verify it's listening on the expected port
+        if port is not None:
+            try:
+                connections = process.net_connections()
+                for conn in connections:
+                    if conn.laddr.port == port and conn.status == "LISTEN":
+                        return True, port
+            except (psutil.AccessDenied, psutil.NoSuchProcess):
+                # If we can't check connections, assume it's running if process exists
+                pass
+
+        return True, port
+    except (psutil.NoSuchProcess, psutil.AccessDenied):
+        return False, None
+
+
+def open_browser_tab(url: str, delay: float = 0.5) -> None:
+    """
+    Open a URL in a new browser tab with an optional delay.
+
+    Args:
+        url: The URL to open
+        delay: Delay in seconds before opening browser (default: 0.5)
+    """
+
+    def _open():
+        time.sleep(delay)  # Give the server time to start
+        webbrowser.open_new_tab(url)
+
+    thread = threading.Thread(target=_open)
+    thread.daemon = True
+    thread.start()

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46}/eval_protocol/utils/logs_server.py

@@ -6,6 +6,7 @@ import threading
 import time
 from datetime import datetime
 from contextlib import asynccontextmanager
+from pathlib import Path
 from queue import Queue
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
@@ -23,6 +24,7 @@ from eval_protocol.utils.vite_server import ViteServer
 from eval_protocol.log_utils.elasticsearch_client import ElasticsearchClient
 from eval_protocol.types.remote_rollout_processor import ElasticsearchConfig
 from eval_protocol.utils.logs_models import LogEntry, LogsResponse
+from eval_protocol.utils.browser_utils import write_pid_file
 
 if TYPE_CHECKING:
     from eval_protocol.models import EvaluationRow
@@ -378,7 +380,7 @@ class LogsServer(ViteServer):
         event_bus.subscribe(self._handle_event)
         logger.debug("[LOGS_SERVER_INIT] Successfully subscribed to event bus")
 
-        logger.info(f"[LOGS_SERVER_INIT] LogsServer initialized on {host}:{port}")
+        logger.info(f"[LOGS_SERVER_INIT] LogsServer initialized on {self.host}:{self.port}")
 
     def _setup_websocket_routes(self):
         """Set up WebSocket routes for real-time communication."""
@@ -541,6 +543,12 @@ class LogsServer(ViteServer):
             )
 
             server = uvicorn.Server(config)
+
+            # Write PID file after server is configured but before serving
+            logger.debug(f"[LOGS_SERVER_RUN_ASYNC] Writing PID file for port {self.port}")
+            write_pid_file(os.getpid(), self.port)
+            logger.debug(f"[LOGS_SERVER_RUN_ASYNC] Successfully wrote PID file for port {self.port}")
+
             await server.serve()
 
         except KeyboardInterrupt:

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.45.dev0
+Version: 0.2.46
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46}/eval_protocol.egg-info/SOURCES.txt

@@ -165,14 +165,6 @@ eval_protocol/mcp_servers/tau2/tests/test_tau2_e2e.py
 eval_protocol/mcp_servers/tau2/tests/system_prompts/airline_agent_system_prompt.md
 eval_protocol/mcp_servers/tau2/tests/system_prompts/mock_agent_system_prompt.md
 eval_protocol/mcp_servers/tau2/tests/system_prompts/retail_agent_system_prompt.md
-eval_protocol/proxy/proxy_core/__init__.py
-eval_protocol/proxy/proxy_core/app.py
-eval_protocol/proxy/proxy_core/auth.py
-eval_protocol/proxy/proxy_core/langfuse.py
-eval_protocol/proxy/proxy_core/litellm.py
-eval_protocol/proxy/proxy_core/main.py
-eval_protocol/proxy/proxy_core/models.py
-eval_protocol/proxy/proxy_core/redis_utils.py
 eval_protocol/pytest/__init__.py
 eval_protocol/pytest/default_agent_rollout_processor.py
 eval_protocol/pytest/default_dataset_adapter.py
@@ -237,6 +229,7 @@ eval_protocol/types/types.py
 eval_protocol/utils/__init__.py
 eval_protocol/utils/batch_evaluation.py
 eval_protocol/utils/batch_transformation.py
+eval_protocol/utils/browser_utils.py
 eval_protocol/utils/check_server_status.py
 eval_protocol/utils/dataset_helpers.py
 eval_protocol/utils/logs_models.py

{eval_protocol-0.2.45.dev0 → eval_protocol-0.2.46}/tests/test_show_results_url.py

@@ -6,6 +6,13 @@ import socket
 from unittest.mock import patch, MagicMock
 import pytest
 
+try:
+    import psutil
+
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+
 from eval_protocol.utils.show_results_url import (
     is_server_running,
     generate_invocation_filter_url,
@@ -193,3 +200,137 @@ class TestIntegration:
         assert "table" in call_args[2]
         assert "integration-test" in call_args[1]
         assert "integration-test" in call_args[2]
+
+
+class TestBrowserUtilities:
+    """Test browser utility functions."""
+
+    def test_get_pid_file_path(self):
+        """Test PID file path generation."""
+        from eval_protocol.utils.browser_utils import _get_pid_file_path
+        from eval_protocol.directory_utils import find_eval_protocol_dir
+        from pathlib import Path
+
+        pid_file = _get_pid_file_path()
+        expected = Path(find_eval_protocol_dir()) / "logs_server.pid"
+        assert pid_file == expected
+
+    def test_is_logs_server_running_no_pid_file(self, tmp_path, monkeypatch):
+        """Test server detection when PID file doesn't exist."""
+        from eval_protocol.utils.browser_utils import is_logs_server_running
+
+        # Mock the PID file path to a non-existent file
+        monkeypatch.setattr(
+            "eval_protocol.utils.browser_utils._get_pid_file_path", lambda: tmp_path / "nonexistent.pid"
+        )
+
+        is_running, port = is_logs_server_running()
+        assert not is_running
+        assert port is None
+
+    def test_is_logs_server_running_invalid_pid_file(self, tmp_path, monkeypatch):
+        """Test server detection with invalid PID file content."""
+        from eval_protocol.utils.browser_utils import is_logs_server_running
+
+        # Create invalid PID file
+        pid_file = tmp_path / "invalid.pid"
+        pid_file.write_text("invalid json")
+        monkeypatch.setattr("eval_protocol.utils.browser_utils._get_pid_file_path", lambda: pid_file)
+
+        is_running, port = is_logs_server_running()
+        assert not is_running
+        assert port is None
+
+    def test_is_logs_server_running_missing_pid_key(self, tmp_path, monkeypatch):
+        """Test server detection with PID file missing required keys."""
+        from eval_protocol.utils.browser_utils import is_logs_server_running
+        import json
+
+        # Create PID file with missing pid key
+        pid_file = tmp_path / "missing_pid.pid"
+        pid_file.write_text(json.dumps({"port": 8000}))
+        monkeypatch.setattr("eval_protocol.utils.browser_utils._get_pid_file_path", lambda: pid_file)
+
+        is_running, port = is_logs_server_running()
+        assert not is_running
+        assert port is None
+
+    @pytest.mark.skipif(not PSUTIL_AVAILABLE, reason="psutil not available")
+    def test_is_logs_server_running_nonexistent_process(self, tmp_path, monkeypatch):
+        """Test server detection with PID file pointing to non-existent process."""
+        from eval_protocol.utils.browser_utils import is_logs_server_running
+        import json
+
+        # Create PID file with non-existent PID
+        pid_file = tmp_path / "nonexistent_process.pid"
+        pid_file.write_text(json.dumps({"pid": 999999, "port": 8000}))
+        monkeypatch.setattr("eval_protocol.utils.browser_utils._get_pid_file_path", lambda: pid_file)
+
+        is_running, port = is_logs_server_running()
+        assert not is_running
+        assert port is None
+
+    @pytest.mark.skipif(not PSUTIL_AVAILABLE, reason="psutil not available")
+    def test_is_logs_server_running_current_process(self, tmp_path, monkeypatch):
+        """Test server detection with PID file pointing to current process."""
+        from eval_protocol.utils.browser_utils import is_logs_server_running
+        import json
+        import os
+
+        # Create PID file with current process PID
+        pid_file = tmp_path / "current_process.pid"
+        pid_file.write_text(json.dumps({"pid": os.getpid(), "port": 8000}))
+        monkeypatch.setattr("eval_protocol.utils.browser_utils._get_pid_file_path", lambda: pid_file)
+
+        is_running, port = is_logs_server_running()
+        assert is_running
+        assert port == 8000
+
+    def test_open_browser_tab(self, monkeypatch):
+        """Test browser tab opening."""
+        from eval_protocol.utils.browser_utils import open_browser_tab
+
+        opened_urls = []
+
+        def mock_open_new_tab(url):
+            opened_urls.append(url)
+
+        monkeypatch.setattr("webbrowser.open_new_tab", mock_open_new_tab)
+
+        # Test with delay
+        open_browser_tab("http://example.com", delay=0.01)
+
+        # Wait a bit for the thread to execute
+        import time
+
+        time.sleep(0.02)
+
+        assert len(opened_urls) == 1
+        assert opened_urls[0] == "http://example.com"
+
+
+class TestLogsServerPidFile:
+    """Test logs server PID file functionality."""
+
+    def test_write_pid_file(self, tmp_path, monkeypatch):
+        """Test PID file writing."""
+        from eval_protocol.utils.browser_utils import write_pid_file
+        import json
+
+        # Mock the find_eval_protocol_dir function
+        monkeypatch.setattr("eval_protocol.directory_utils.find_eval_protocol_dir", lambda: str(tmp_path))
+
+        # Test writing PID file
+        write_pid_file(12345, 8000)
+
+        # Check that PID file was created
+        pid_file = tmp_path / "logs_server.pid"
+        assert pid_file.exists()
+
+        # Check content
+        with open(pid_file, "r") as f:
+            data = json.load(f)
+            assert "pid" in data
+            assert "port" in data
+            assert data["port"] == 8000
+            assert data["pid"] == 12345

eval_protocol-0.2.45.dev0/eval_protocol/proxy/proxy_core/__init__.py

@@ -1,10 +0,0 @@
-from .models import ProxyConfig
-from .app import create_app
-from .auth import AuthProvider, NoAuthProvider
-
-__all__ = [
-    "ProxyConfig",
-    "create_app",
-    "AuthProvider",
-    "NoAuthProvider",
-]