eval-protocol 0.2.62__tar.gz → 0.2.63__tar.gz

{eval_protocol-0.2.62/eval_protocol.egg-info → eval_protocol-0.2.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.62
+Version: 0.2.63
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.62 → eval_protocol-0.2.63}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-24T00:58:57-0700",
+ "date": "2025-10-25T15:18:19-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "e68248b5edf8e6f977c94ecd01f9972ef14b4b40",
- "version": "0.2.62"
+ "full-revisionid": "8ad7d26db059270502fbee55189421d884f0b13e",
+ "version": "0.2.63"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.62 → eval_protocol-0.2.63}/eval_protocol/cli_commands/upload.py

@@ -81,6 +81,12 @@ def _is_eval_protocol_test(obj: Any) -> bool:
         return False
     # Must have pytest marks from evaluation_test
     marks = getattr(obj, "pytestmark", [])
+    # Handle pytest proxy objects (APIRemovedInV1Proxy)
+    if not isinstance(marks, (list, tuple)):
+        try:
+            marks = list(marks) if marks else []
+        except (TypeError, AttributeError):
+            return False
     return len(marks) > 0
 
 
@@ -91,6 +97,14 @@ def _extract_param_info_from_marks(obj: Any) -> tuple[bool, int, list[str]]:
         (has_parametrize, param_count, param_ids)
     """
     marks = getattr(obj, "pytestmark", [])
+
+    # Handle pytest proxy objects (APIRemovedInV1Proxy) - same as _is_eval_protocol_test
+    if not isinstance(marks, (list, tuple)):
+        try:
+            marks = list(marks) if marks else []
+        except (TypeError, AttributeError):
+            marks = []
+
     has_parametrize = False
     total_combinations = 0
     all_param_ids: list[str] = []
@@ -131,51 +145,103 @@ def _discover_tests(root: str) -> list[DiscoveredTest]:
 
     discovered: list[DiscoveredTest] = []
 
-    # Collect all test functions from Python files
-    for file_path in _iter_python_files(root):
+    class CollectionPlugin:
+        """Plugin to capture collected items without running code."""
+
+        def __init__(self):
+            self.items = []
+
+        def pytest_ignore_collect(self, collection_path, config):
+            """Ignore problematic files before pytest tries to import them."""
+            # Ignore specific files
+            ignored_files = ["setup.py", "versioneer.py", "conf.py", "__main__.py"]
+            if collection_path.name in ignored_files:
+                return True
+
+            # Ignore hidden files (starting with .)
+            if collection_path.name.startswith("."):
+                return True
+
+            # Ignore test_discovery files
+            if collection_path.name.startswith("test_discovery"):
+                return True
+
+            return None
+
+        def pytest_collection_modifyitems(self, items):
+            """Hook called after collection is done."""
+            self.items = items
+
+    plugin = CollectionPlugin()
+
+    # Run pytest collection only (--collect-only prevents code execution)
+    # Override python_files to collect from ANY .py file
+    args = [
+        abs_root,
+        "--collect-only",
+        "-q",
+        "--pythonwarnings=ignore",
+        "-o",
+        "python_files=*.py",  # Override to collect all .py files
+    ]
+
+    try:
+        # Suppress pytest output
+        import io
+        import contextlib
+
+        with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
+            pytest.main(args, plugins=[plugin])
+    except Exception:
+        # If pytest collection fails, fall back to empty list
+        return []
+
+    # Process collected items
+    for item in plugin.items:
+        if not hasattr(item, "obj"):
+            continue
+
+        obj = item.obj
+        if not _is_eval_protocol_test(obj):
+            continue
+
+        origin = getattr(obj, "_origin_func", obj)
         try:
-            unique_name = "ep_upload_" + re.sub(r"[^a-zA-Z0-9_]", "_", os.path.abspath(file_path))
-            spec = importlib.util.spec_from_file_location(unique_name, file_path)
-            if spec and spec.loader:
-                module = importlib.util.module_from_spec(spec)
-                sys.modules[spec.name] = module
-                spec.loader.exec_module(module)  # type: ignore[attr-defined]
-            else:
-                continue
+            src_file = inspect.getsourcefile(origin) or str(item.path)
+            _, lineno = inspect.getsourcelines(origin)
         except Exception:
-            continue
+            src_file, lineno = str(item.path), None
 
-        for name, obj in inspect.getmembers(module):
-            if _is_eval_protocol_test(obj):
-                origin = getattr(obj, "_origin_func", obj)
-                try:
-                    src_file = inspect.getsourcefile(origin) or file_path
-                    _, lineno = inspect.getsourcelines(origin)
-                except Exception:
-                    src_file, lineno = file_path, None
-
-                # Extract parametrization info from marks
-                has_parametrize, param_count, param_ids = _extract_param_info_from_marks(obj)
-
-                # Generate synthetic nodeids for display
-                base_nodeid = f"{os.path.basename(file_path)}::{name}"
-                if has_parametrize and param_ids:
-                    nodeids = [f"{base_nodeid}[{pid}]" for pid in param_ids]
-                else:
-                    nodeids = [base_nodeid]
-
-                discovered.append(
-                    DiscoveredTest(
-                        module_path=module.__name__,
-                        module_name=module.__name__,
-                        qualname=f"{module.__name__}.{name}",
-                        file_path=os.path.abspath(src_file),
-                        lineno=lineno,
-                        has_parametrize=has_parametrize,
-                        param_count=param_count,
-                        nodeids=nodeids,
-                    )
-                )
+        # Extract parametrization info from marks
+        has_parametrize, param_count, param_ids = _extract_param_info_from_marks(obj)
+
+        # Get module name and function name
+        module_name = (
+            item.module.__name__
+            if hasattr(item, "module")
+            else item.nodeid.split("::")[0].replace("/", ".").replace(".py", "")
+        )
+        func_name = item.name.split("[")[0] if "[" in item.name else item.name
+
+        # Generate nodeids
+        base_nodeid = f"{os.path.basename(src_file)}::{func_name}"
+        if param_ids:
+            nodeids = [f"{base_nodeid}[{pid}]" for pid in param_ids]
+        else:
+            nodeids = [base_nodeid]
+
+        discovered.append(
+            DiscoveredTest(
+                module_path=module_name,
+                module_name=module_name,
+                qualname=f"{module_name}.{func_name}",
+                file_path=os.path.abspath(src_file),
+                lineno=lineno,
+                has_parametrize=has_parametrize,
+                param_count=param_count,
+                nodeids=nodeids,
+            )
+        )
 
     # Deduplicate by qualname (in case same test appears multiple times)
     by_qual: dict[str, DiscoveredTest] = {}

{eval_protocol-0.2.62 → eval_protocol-0.2.63}/eval_protocol/evaluation.py

@@ -22,6 +22,8 @@ from eval_protocol.auth import (
 )
 from eval_protocol.typed_interface import EvaluationMode
 
+from eval_protocol.get_pep440_version import get_pep440_version
+
 logger = logging.getLogger(__name__)
 
 # Flag to track if the preview API was successfully used
@@ -395,7 +397,7 @@ class Evaluator:
         }
 
         api_base = os.environ.get("FIREWORKS_API_BASE", "https://api.fireworks.ai")
-        print("show payload", payload)
+
         if "dev.api.fireworks.ai" in api_base and account_id == "fireworks":
             account_id = "pyroworks-dev"
 
@@ -518,6 +520,135 @@ class Evaluator:
         preview_result.total_runtime_ms = max(1, int((end_time - start_time) * 1000))
         return preview_result
 
+    def _build_minimal_criteria(self) -> List[Dict[str, str]]:
+        """Build minimal criteria (name, type, description) without code snippets."""
+
+        # Remote URL mode
+        if self.remote_url:
+            return [
+                {
+                    "name": "remote_eval_proxy",
+                    "type": "CODE_SNIPPETS",
+                    "description": f"Proxies evaluation to remote URL: {self.remote_url}",
+                }
+            ]
+
+        # TS mode (direct code snippet)
+        elif self.ts_mode_config:
+            criterion_name = self.ts_mode_config.get("criterion_name", "default_code_criterion")
+            description = self.ts_mode_config.get("description", "Python code execution")
+            return [
+                {
+                    "name": criterion_name,
+                    "type": "CODE_SNIPPETS",
+                    "description": description,
+                }
+            ]
+
+        # Multi-metrics mode
+        elif self.multi_metrics:
+            return [
+                {
+                    "name": "eval",
+                    "type": "CODE_SNIPPETS",
+                    "description": self.description or "Multi-metric evaluation",
+                }
+            ]
+
+        # Single metric folders
+        else:
+            criteria = []
+            for metric_name in self.metric_folders:
+                criteria.append(
+                    {
+                        "name": metric_name,
+                        "type": "CODE_SNIPPETS",
+                        "description": self.description or f"Evaluation metric: {metric_name}",
+                    }
+                )
+            return criteria
+
+    @staticmethod
+    def _parse_ignore_file(ignore_path: str) -> List[str]:
+        """Parse .gitignore or .dockerignore and return patterns."""
+        patterns = []
+        if not os.path.exists(ignore_path):
+            return patterns
+
+        try:
+            with open(ignore_path, "r") as f:
+                for line in f:
+                    line = line.strip()
+                    if line and not line.startswith("#"):
+                        patterns.append(line)
+        except Exception:
+            pass
+
+        return patterns
+
+    @staticmethod
+    def _ensure_requirements_present(source_dir: str) -> None:
+        req_path = os.path.join(source_dir, "requirements.txt")
+        if not os.path.isfile(req_path):
+            logger.error("Missing requirements.txt in upload directory: %s", source_dir)
+            raise ValueError(
+                "Upload requires requirements.txt in the project root. "
+                "Please add requirements.txt and re-run ep upload."
+            )
+
+    @staticmethod
+    def _should_ignore(path: str, ignore_patterns: List[str]) -> bool:
+        """Check if path matches any ignore pattern."""
+        from pathlib import Path
+        import fnmatch
+
+        default_ignores = [".git", "__pycache__", "*.pyc", ".venv", "venv", "node_modules", "*.egg-info"]
+        all_patterns = default_ignores + ignore_patterns
+
+        path_obj = Path(path)
+        for pattern in all_patterns:
+            if pattern.endswith("/"):
+                if path_obj.is_dir() and fnmatch.fnmatch(path_obj.name, pattern.rstrip("/")):
+                    return True
+            elif fnmatch.fnmatch(path_obj.name, pattern) or fnmatch.fnmatch(str(path_obj), pattern):
+                return True
+
+        return False
+
+    @staticmethod
+    def _create_tar_gz_with_ignores(output_path: str, source_dir: str) -> int:
+        """Create tar.gz of source_dir with parent directory included."""
+        import tarfile
+        from pathlib import Path
+
+        source_path = Path(source_dir)
+        gitignore_patterns = Evaluator._parse_ignore_file(str(source_path / ".gitignore"))
+        dockerignore_patterns = Evaluator._parse_ignore_file(str(source_path / ".dockerignore"))
+        all_ignore_patterns = gitignore_patterns + dockerignore_patterns
+
+        logger.info(f"Creating tar.gz with {len(all_ignore_patterns)} ignore patterns")
+
+        # Get directory name for the archive root
+        dir_name = os.path.basename(source_dir)
+        parent_dir = os.path.dirname(source_dir)
+
+        with tarfile.open(output_path, "w:gz") as tar:
+            for root, dirs, files in os.walk(source_dir):
+                dirs[:] = [d for d in dirs if not Evaluator._should_ignore(os.path.join(root, d), all_ignore_patterns)]
+
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if Evaluator._should_ignore(file_path, all_ignore_patterns):
+                        continue
+
+                    # Include parent directory in archive path
+                    rel_path = os.path.relpath(file_path, parent_dir)  # Relative to parent
+                    tar.add(file_path, arcname=rel_path)  # Keeps "python-sdk/..." structure
+
+        size_bytes = os.path.getsize(output_path)
+        logger.info(f"Created {output_path} ({size_bytes:,} bytes)")
+        return size_bytes
+
     def create(self, evaluator_id, display_name=None, description=None, force=False):
         if not self.remote_url and not self.ts_mode_config and not self.code_files:
             raise ValueError("No code files loaded. Load metric folder(s) or provide ts_mode_config/remote_url first.")
@@ -537,14 +668,21 @@ class Evaluator:
         # Keep multiMetrics/rollupSettings for backward compatibility with tests
         payload_multi_metrics = True
         payload_rollup_settings = {"skipRollup": True}
+        parent = f"accounts/{account_id}"
+
+        try:
+            version_str = get_pep440_version()
+        except Exception:
+            version_str = None
 
         payload_data = {
+            "parent": parent,
             "evaluator": {
                 "displayName": self.display_name,
                 "description": self.description,
                 "multiMetrics": payload_multi_metrics,
-                # "rewardFunctionMode": self.reward_function_mode,  # How input is processed by user func
-                "criteria": self._construct_criteria(criteria_data={}),
+                "commitHash": version_str,
+                "criteria": self._build_minimal_criteria(),
                 "requirements": "",
                 "rollupSettings": payload_rollup_settings,
             },
@@ -566,21 +704,26 @@ class Evaluator:
         if "dev.api.fireworks.ai" in self.api_base and account_id == "fireworks":
             account_id = "pyroworks-dev"
 
-        base_url = f"{self.api_base}/v1/accounts/{account_id}/evaluators"
+        base_url = f"{self.api_base}/v1/{parent}/evaluatorsV2"
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
         }
+
+        self._ensure_requirements_present(os.getcwd())
+
         logger.info(f"Creating evaluator '{evaluator_id}' for account '{account_id}'...")
 
         try:
             if force:
-                check_url = f"{base_url}/{evaluator_id}"
+                check_url = f"{self.api_base}/v1/{parent}/evaluators/{evaluator_id}"
                 try:
+                    logger.info(f"Checking if evaluator exists: {check_url}")
                     check_response = requests.get(check_url, headers=headers)
+
                     if check_response.status_code == 200:
                         logger.info(f"Evaluator '{evaluator_id}' already exists, deleting and recreating...")
-                        delete_url = f"{base_url}/{evaluator_id}"
+                        delete_url = f"{self.api_base}/v1/{parent}/evaluators/{evaluator_id}"
                         try:
                             delete_response = requests.delete(delete_url, headers=headers)
                             if delete_response.status_code < 400:
@@ -597,12 +740,134 @@ class Evaluator:
                 except requests.exceptions.RequestException:
                     response = requests.post(base_url, json=payload_data, headers=headers)
             else:
+                logger.info(f"Creating evaluator at: {base_url}")
                 response = requests.post(base_url, json=payload_data, headers=headers)
 
             response.raise_for_status()
             result = response.json()
             logger.info(f"Successfully created evaluator '{evaluator_id}'")
-            return result
+
+            # Upload code as tar.gz to GCS
+            evaluator_name = result.get("name")  # e.g., "accounts/pyroworks/evaluators/test-123"
+
+            if not evaluator_name:
+                raise ValueError(
+                    "Create evaluator response missing 'name' field. "
+                    f"Cannot proceed with code upload. Response: {result}"
+                )
+
+            try:
+                # Create tar.gz of current directory
+                cwd = os.getcwd()
+                dir_name = os.path.basename(cwd)
+                tar_filename = f"{dir_name}.tar.gz"
+                tar_path = os.path.join(cwd, tar_filename)
+
+                tar_size = self._create_tar_gz_with_ignores(tar_path, cwd)
+
+                # Call GetEvaluatorUploadEndpoint
+                upload_endpoint_url = f"{self.api_base}/v1/{evaluator_name}:getUploadEndpoint"
+                upload_payload = {"name": evaluator_name, "filename_to_size": {tar_filename: tar_size}}
+
+                logger.info(f"Requesting upload endpoint for {tar_filename}")
+                upload_response = requests.post(upload_endpoint_url, json=upload_payload, headers=headers)
+                upload_response.raise_for_status()
+
+                # Check for signed URLs
+                upload_response_data = upload_response.json()
+                signed_urls = upload_response_data.get("filenameToSignedUrls", {})
+
+                if not signed_urls:
+                    raise ValueError(f"GetUploadEndpoint returned no signed URLs. Response: {upload_response_data}")
+
+                signed_url = signed_urls.get(tar_filename)
+
+                if not signed_url:
+                    raise ValueError(
+                        f"No signed URL received for {tar_filename}. Available files: {list(signed_urls.keys())}"
+                    )
+
+                # Upload to GCS
+                logger.info(f"Uploading {tar_filename} to GCS...")
+
+                file_size = os.path.getsize(tar_path)
+
+                # Retry configuration
+                max_retries = 3
+                retry_delay = 2  # seconds
+
+                for attempt in range(max_retries):
+                    try:
+                        with open(tar_path, "rb") as f:
+                            # Create request exactly like Golang
+                            req = requests.Request(
+                                "PUT",
+                                signed_url,
+                                data=f,
+                                headers={
+                                    "Content-Type": "application/octet-stream",
+                                    "X-Goog-Content-Length-Range": f"{file_size},{file_size}",
+                                },
+                            )
+                            prepared = req.prepare()
+
+                            # Don't let requests add extra headers
+                            session = requests.Session()
+                            gcs_response = session.send(prepared, timeout=600)
+                            gcs_response.raise_for_status()
+
+                        logger.info(f"Successfully uploaded {tar_filename}")
+                        break  # Success, exit retry loop
+
+                    except (requests.exceptions.RequestException, IOError) as e:
+                        if attempt < max_retries - 1:
+                            # Check if it's a retryable error
+                            is_retryable = False
+                            if isinstance(e, requests.exceptions.RequestException):
+                                if hasattr(e, "response") and e.response is not None:
+                                    # Retry on 5xx errors or 408 (timeout)
+                                    is_retryable = e.response.status_code >= 500 or e.response.status_code == 408
+                                else:
+                                    # Network errors (no response) are retryable
+                                    is_retryable = True
+                            else:
+                                # IOError is retryable
+                                is_retryable = True
+
+                            if is_retryable:
+                                wait_time = retry_delay * (2**attempt)  # Exponential backoff
+                                logger.warning(
+                                    f"Upload attempt {attempt + 1}/{max_retries} failed: {e}. "
+                                    f"Retrying in {wait_time}s..."
+                                )
+                                time.sleep(wait_time)
+                            else:
+                                # Non-retryable error, raise immediately
+                                raise
+                        else:
+                            # Last attempt failed
+                            logger.error(f"Upload failed after {max_retries} attempts")
+                            raise
+
+                # Step 3: Validate upload
+                validate_url = f"{self.api_base}/v1/{evaluator_name}:validateUpload"
+                validate_payload = {"name": evaluator_name}
+                validate_response = requests.post(validate_url, json=validate_payload, headers=headers)
+                validate_response.raise_for_status()
+
+                validate_data = validate_response.json()
+
+                logger.info("Upload validated successfully")
+
+                # Clean up tar file
+                if os.path.exists(tar_path):
+                    os.remove(tar_path)
+
+            except Exception as upload_error:
+                logger.warning(f"Code upload failed (evaluator created but code not uploaded): {upload_error}")
+                # Don't fail - evaluator is created, just code upload failed
+
+            return result  # Return after attempting upload
         except Exception as e:
             logger.error(f"Error creating evaluator: {str(e)}")
             if isinstance(e, requests.exceptions.HTTPError) and hasattr(e, "response"):

{eval_protocol-0.2.62 → eval_protocol-0.2.63}/eval_protocol/mcp/execution/manager.py

@@ -504,7 +504,7 @@ class ExecutionManager:
                             )
 
                     logger.info(
-                        f"🏁 Environmnet indicates rollout {rollout_idx} terminated at step {step} (reward: {trajectory.total_reward}) in thread {threading.current_thread().name}"
+                        f"🏁 Environment indicates rollout {rollout_idx} terminated at step {step} (reward: {trajectory.total_reward}) in thread {threading.current_thread().name}"
                     )
                     break
 

{eval_protocol-0.2.62 → eval_protocol-0.2.63}/eval_protocol/pytest/exception_config.py

@@ -8,7 +8,7 @@ from typing import Callable, Set, Type, Union
 
 import backoff
 
-
+import litellm
 import requests
 import httpx
 
@@ -28,6 +28,12 @@ DEFAULT_RETRYABLE_EXCEPTIONS: Set[Type[Exception]] = {
     httpx.TimeoutException,
     httpx.NetworkError,
     httpx.RemoteProtocolError,
+    litellm.exceptions.RateLimitError,
+    litellm.exceptions.InternalServerError,
+    litellm.exceptions.Timeout,
+    litellm.exceptions.NotFoundError,
+    litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
+    litellm.exceptions.ServiceUnavailableError,
 }
 
 

{eval_protocol-0.2.62 → eval_protocol-0.2.63}/eval_protocol/quickstart/svg_agent/vercel_svg_server/api/init.py

@@ -8,6 +8,7 @@ The SVG evaluation logic remains in the test client.
 import json
 import os
 import logging
+import sys
 from http.server import BaseHTTPRequestHandler
 from openai import OpenAI
 from dotenv import load_dotenv
@@ -16,9 +17,32 @@ from eval_protocol import Status, InitRequest, FireworksTracingHttpHandler, Roll
 
 load_dotenv()
 
-# Attach Fireworks tracing handler to root logger
-fireworks_handler = FireworksTracingHttpHandler()
-logging.getLogger().addHandler(fireworks_handler)
+# Configure logging so INFO and below go to stdout, WARNING+ to stderr.
+# This avoids Vercel marking INFO logs as [error] (stderr).
+root_logger = logging.getLogger()
+root_logger.handlers.clear()
+root_logger.setLevel(logging.INFO)
+
+
+class _InfoOnly(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.levelno <= logging.INFO
+
+
+formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s")
+
+stdout_handler = logging.StreamHandler(sys.stdout)
+stdout_handler.addFilter(_InfoOnly())
+stdout_handler.setFormatter(formatter)
+root_logger.addHandler(stdout_handler)
+
+stderr_handler = logging.StreamHandler(sys.stderr)
+stderr_handler.setLevel(logging.WARNING)
+stderr_handler.setFormatter(formatter)
+root_logger.addHandler(stderr_handler)
+
+# Attach Fireworks tracing handler to root logger (non-stream HTTP sink)
+root_logger.addHandler(FireworksTracingHttpHandler())
 
 
 class handler(BaseHTTPRequestHandler):
@@ -43,22 +67,22 @@ class handler(BaseHTTPRequestHandler):
                 self._send_error(400, error_msg)
                 return
 
+            model = req.completion_params.get("model")
+            if model and isinstance(model, str) and model.startswith("fireworks_ai/"):
+                model = model[len("fireworks_ai/") :]
+
             # Prepare completion arguments
             completion_kwargs = {
                 "messages": req.messages,
-                **req.completion_params,
+                "model": model,
+                "temperature": req.completion_params.get("temperature"),
+                "max_tokens": req.completion_params.get("max_tokens"),
             }
 
             # Add tools if present
             if req.tools:
                 completion_kwargs["tools"] = req.tools
 
-            # Add completion parameters if they exist
-            # if hasattr(req, 'completion_params') and req.completion_params:
-            #     # Filter out any model override
-            #     params = {k: v for k, v in req.completion_params.items() if k != 'model'}
-            #     completion_kwargs.update(params)
-
             # Get API key (prefer request api_key, fallback to environment)
             api_key = req.api_key or os.environ.get("FIREWORKS_API_KEY")
             if not api_key: