eval-protocol 0.2.35__tar.gz → 0.2.35.dev2__tar.gz

{eval_protocol-0.2.35/eval_protocol.egg-info → eval_protocol-0.2.35.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.35
+Version: 0.2.35.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -20,30 +20,25 @@ Requires-Dist: aiosqlite
 Requires-Dist: aiohttp
 Requires-Dist: mcp>=1.9.2
 Requires-Dist: PyYAML>=5.0
-Requires-Dist: datasets>=3.0.0
-Requires-Dist: fsspec
 Requires-Dist: hydra-core>=1.3.2
 Requires-Dist: omegaconf>=2.3.0
-Requires-Dist: gymnasium>=0.29.0
 Requires-Dist: httpx>=0.24.0
 Requires-Dist: anthropic>=0.59.0
-Requires-Dist: ipykernel>=6.30.0
-Requires-Dist: jupyter>=1.1.1
+Requires-Dist: litellm<1.75.0
+Requires-Dist: pytest>=6.0.0
+Requires-Dist: pytest-asyncio>=0.21.0
+Requires-Dist: peewee>=3.18.2
+Requires-Dist: backoff>=2.2.0
+Requires-Dist: questionary>=2.0.0
 Requires-Dist: toml>=0.10.0
 Requires-Dist: loguru>=0.6.0
 Requires-Dist: docstring-parser>=0.15
 Requires-Dist: rich>=12.0.0
 Requires-Dist: psutil>=5.8.0
-Requires-Dist: litellm>=1.0.0
 Requires-Dist: addict>=2.4.0
 Requires-Dist: deepdiff>=6.0.0
-Requires-Dist: pandas>=1.5.0
 Requires-Dist: websockets>=15.0.1
 Requires-Dist: fastapi>=0.116.1
-Requires-Dist: pytest>=6.0.0
-Requires-Dist: pytest-asyncio>=0.21.0
-Requires-Dist: peewee>=3.18.2
-Requires-Dist: backoff>=2.2.0
 Provides-Extra: dev
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"
@@ -51,6 +46,7 @@ Requires-Dist: pytest-httpserver; extra == "dev"
 Requires-Dist: werkzeug>=2.0.0; extra == "dev"
 Requires-Dist: ruff>=0.5.0; extra == "dev"
 Requires-Dist: transformers>=4.0.0; extra == "dev"
+Requires-Dist: pandas>=1.5.0; extra == "dev"
 Requires-Dist: types-setuptools; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
@@ -86,10 +82,6 @@ Requires-Dist: langfuse>=2.0.0; extra == "langfuse"
 Provides-Extra: huggingface
 Requires-Dist: datasets>=3.0.0; extra == "huggingface"
 Requires-Dist: transformers>=4.0.0; extra == "huggingface"
-Provides-Extra: adapters
-Requires-Dist: langfuse>=2.0.0; extra == "adapters"
-Requires-Dist: datasets>=3.0.0; extra == "adapters"
-Requires-Dist: transformers>=4.0.0; extra == "adapters"
 Provides-Extra: langsmith
 Requires-Dist: langsmith>=0.1.86; extra == "langsmith"
 Provides-Extra: bigquery

{eval_protocol-0.2.35 → eval_protocol-0.2.35.dev2}/eval_protocol/__init__.py

@@ -23,17 +23,8 @@ from .mcp_env import (
     test_mcp,
 )
 from .data_loader import DynamicDataLoader, InlineDataLoader
-
-# Try to import FireworksPolicy if available
-try:
-    from .mcp_env import FireworksPolicy
-
-    _FIREWORKS_AVAILABLE = True
-except (ImportError, AttributeError):
-    _FIREWORKS_AVAILABLE = False
-# Import submodules to make them available via eval_protocol.rewards, etc.
 from . import mcp, rewards
-from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata
+from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata, Status
 from .playback_policy import PlaybackPolicyBase
 from .resources import create_llm_resource
 from .reward_function import RewardFunction
@@ -42,6 +33,13 @@ from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assista
 from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
 from .pytest.parameterize import DefaultParameterIdGenerator
 
+from .types.remote_rollout_processor import (
+    InitRequest,
+    RolloutMetadata,
+    StatusResponse,
+    create_langfuse_config_tags,
+)
+
 try:
     from .adapters import OpenAIResponsesAdapter
 except ImportError:
@@ -62,17 +60,10 @@ try:
 except ImportError:
     LangSmithAdapter = None
 
-# Remote server types
-from .types.remote_rollout_processor import (
-    InitRequest,
-    RolloutMetadata,
-    StatusResponse,
-    create_langfuse_config_tags,
-)
-
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "Status",
     "RemoteRolloutProcessor",
     "InputMetadata",
     "EvaluationRow",

{eval_protocol-0.2.35 → eval_protocol-0.2.35.dev2}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-10-01T13:28:59-0700",
+ "date": "2025-10-02T12:04:07-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "43ea8eaa8329931e6a9e61aa23a7aeca359f1d1c",
- "version": "0.2.35"
+ "full-revisionid": "52178b3b90bb27a7f53fcbbba0bfbb50e7ebb416",
+ "version": "0.2.35-dev2"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.35 → eval_protocol-0.2.35.dev2}/eval_protocol/adapters/huggingface.py

@@ -13,12 +13,9 @@ from .base import BaseAdapter
 logger = logging.getLogger(__name__)
 
 try:
-    from datasets import Dataset, DatasetDict, load_dataset
-
-    DATASETS_AVAILABLE = True
+    from datasets import Dataset, DatasetDict, load_dataset  # pyright: ignore[reportAttributeAccessIssue]
 except ImportError:
-    DATASETS_AVAILABLE = False
-    logger.warning("HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'")
+    raise ImportError("HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'")
 
 # Type alias for transformation function
 TransformFunction = Callable[[Dict[str, Any]], Dict[str, Any]]
@@ -80,11 +77,6 @@ class HuggingFaceAdapter(BaseAdapter):
             revision: Optional dataset revision/commit hash
             **load_dataset_kwargs: Additional arguments to pass to load_dataset
         """
-        if not DATASETS_AVAILABLE:
-            raise ImportError(
-                "HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'"
-            )
-
         self.dataset_id = dataset_id
         self.transform_fn = transform_fn
         self.config_name = config_name

{eval_protocol-0.2.35 → eval_protocol-0.2.35.dev2}/eval_protocol/cli.py

@@ -15,19 +15,14 @@ from pathlib import Path
 logger = logging.getLogger(__name__)
 
 
-from eval_protocol.evaluation import create_evaluation, preview_evaluation
-
 from .cli_commands.agent_eval_cmd import agent_eval_command
-from .cli_commands.common import (
-    check_agent_environment,
-    check_environment,
-    setup_logging,
-)
+from .cli_commands.common import setup_logging
 from .cli_commands.deploy import deploy_command
 from .cli_commands.deploy_mcp import deploy_mcp_command
 from .cli_commands.logs import logs_command
 from .cli_commands.preview import preview_command
 from .cli_commands.run_eval_cmd import hydra_cli_entry_point
+from .cli_commands.upload import upload_command
 
 
 def parse_args(args=None):
@@ -291,6 +286,44 @@ def parse_args(args=None):
     logs_parser = subparsers.add_parser("logs", help="Serve logs with file watching and real-time updates")
     logs_parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)")
 
+    # Upload command
+    upload_parser = subparsers.add_parser(
+        "upload",
+        help="Scan for evaluation tests, select, and upload as Fireworks evaluators",
+    )
+    upload_parser.add_argument(
+        "--path",
+        default=".",
+        help="Path to search for evaluation tests (default: current directory)",
+    )
+    upload_parser.add_argument(
+        "--entry",
+        help="Entrypoint of evaluation test to upload (module:function or path::function). For multiple, separate by commas.",
+    )
+    upload_parser.add_argument(
+        "--id",
+        help="Evaluator ID to use (if multiple selections, a numeric suffix is appended)",
+    )
+    upload_parser.add_argument(
+        "--display-name",
+        help="Display name for evaluator (defaults to ID)",
+    )
+    upload_parser.add_argument(
+        "--description",
+        help="Description for evaluator",
+    )
+    upload_parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Overwrite existing evaluator with the same ID",
+    )
+    upload_parser.add_argument(
+        "--yes",
+        "-y",
+        action="store_true",
+        help="Non-interactive: upload all discovered evaluation tests",
+    )
+
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.
     # All arguments after 'run' will be passed to Hydra by parse_known_args.
@@ -346,6 +379,8 @@ def main():
         return agent_eval_command(args)
     elif args.command == "logs":
         return logs_command(args)
+    elif args.command == "upload":
+        return upload_command(args)
     elif args.command == "run":
         # For the 'run' command, Hydra takes over argument parsing.
 

{eval_protocol-0.2.35 → eval_protocol-0.2.35.dev2}/eval_protocol/cli_commands/agent_eval_cmd.py

@@ -27,11 +27,7 @@ import logging  # For logger instance
 import os  # For environment variables
 from pathlib import Path
 
-from pydantic import ValidationError
-
-from eval_protocol.agent import Orchestrator
 from eval_protocol.agent.task_manager import TaskManager
-from eval_protocol.models import TaskDefinitionModel  # Import the new Pydantic model
 
 # setup_logging is already called in cli.py's main, but good for standalone use if any
 # from .common import setup_logging

{eval_protocol-0.2.35 → eval_protocol-0.2.35.dev2}/eval_protocol/cli_commands/run_eval_cmd.py

@@ -17,7 +17,6 @@ from omegaconf import (  # Ensure MISSING is imported if used in configs
     OmegaConf,
 )
 
-from eval_protocol.execution.pipeline import EvaluationPipeline
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +25,8 @@ def run_evaluation_command_logic(cfg: DictConfig) -> None:
     """
     Main logic for the 'run-evaluation' command.
     """
+    from eval_protocol.execution.pipeline import EvaluationPipeline
+
     logger.info("Starting 'run-evaluation' command with resolved Hydra config.")
 
     # Make Hydra's runtime output directory available to the pipeline if needed