eval-protocol 0.2.32__tar.gz → 0.2.34__tar.gz

{eval_protocol-0.2.32/eval_protocol.egg-info → eval_protocol-0.2.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.32
+Version: 0.2.34
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.32 → eval_protocol-0.2.34}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-29T16:40:31-0700",
+ "date": "2025-09-30T15:39:15-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "1744b558ceb39f3a910a898ed6dd3df6a4576691",
- "version": "0.2.32"
+ "full-revisionid": "c09755b30386c03c95bd79d7b142ed614419c7c4",
+ "version": "0.2.34"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.32 → eval_protocol-0.2.34}/eval_protocol/pytest/evaluation_test.py

@@ -52,10 +52,12 @@ from eval_protocol.pytest.utils import (
     add_cost_metrics,
     log_eval_status_and_rows,
     parse_ep_completion_params,
+    parse_ep_completion_params_overwrite,
     parse_ep_max_concurrent_rollouts,
     parse_ep_max_rows,
     parse_ep_num_runs,
     parse_ep_passed_threshold,
+    parse_ep_dataloaders,
     rollout_processor_with_retry,
     run_tasks_with_eval_progress,
     run_tasks_with_run_progress,
@@ -189,10 +191,18 @@ def evaluation_test(
     max_concurrent_rollouts = parse_ep_max_concurrent_rollouts(max_concurrent_rollouts)
     max_dataset_rows = parse_ep_max_rows(max_dataset_rows)
     completion_params = parse_ep_completion_params(completion_params)
+    completion_params = parse_ep_completion_params_overwrite(completion_params)
     original_completion_params = completion_params
     passed_threshold = parse_ep_passed_threshold(passed_threshold)
+    data_loaders = parse_ep_dataloaders(data_loaders)
     custom_invocation_id = os.environ.get("EP_INVOCATION_ID", None)
 
+    # ignore other data input params when dataloader is provided
+    if data_loaders:
+        input_dataset = None
+        input_messages = None
+        input_rows = None
+
     def decorator(
         test_func: TestFunction,
     ) -> TestFunction:

{eval_protocol-0.2.32 → eval_protocol-0.2.34}/eval_protocol/pytest/handle_persist_flow.py

@@ -16,9 +16,10 @@ import requests
 def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name: str):
     try:
         # Default is to save and upload experiment JSONL files, unless explicitly disabled
-        should_save_and_upload = os.getenv("EP_NO_UPLOAD") != "1"
+        custom_output_dir = os.getenv("EP_OUTPUT_DIR")
+        should_save = os.getenv("EP_NO_UPLOAD") != "1" or custom_output_dir is not None
 
-        if should_save_and_upload:
+        if should_save:
             current_run_rows = [item for sublist in all_results for item in sublist]
             if current_run_rows:
                 experiments: dict[str, list[EvaluationRow]] = defaultdict(list)
@@ -27,6 +28,8 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                         experiments[row.execution_metadata.experiment_id].append(row)
 
                 eval_protocol_dir = find_eval_protocol_dir()
+                if custom_output_dir:
+                    eval_protocol_dir = custom_output_dir
                 exp_dir = pathlib.Path(eval_protocol_dir) / "experiment_results"
                 exp_dir.mkdir(parents=True, exist_ok=True)
 
@@ -81,6 +84,10 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
                             json.dump(row_data, f, ensure_ascii=False)
                             f.write("\n")
 
+                    should_upload = os.getenv("EP_NO_UPLOAD") != "1"
+                    if not should_upload:
+                        continue
+
                     def get_auth_value(key: str) -> str | None:
                         """Get auth value from config file or environment."""
                         try:

{eval_protocol-0.2.32 → eval_protocol-0.2.34}/eval_protocol/pytest/plugin.py

@@ -19,6 +19,7 @@ import json
 import pathlib
 import sys
 from pytest import StashKey
+import pytest
 
 
 def pytest_addoption(parser) -> None:
@@ -56,6 +57,7 @@ def pytest_addoption(parser) -> None:
         default=None,
         help=("Write a JSON summary artifact at the given path (e.g., ./outputs/aime_low.json)."),
     )
+    # deprecate this later
     group.addoption(
         "--ep-input-param",
         action="append",
@@ -115,6 +117,27 @@ def pytest_addoption(parser) -> None:
             "Default: false (experiment JSONs are saved and uploaded by default)."
         ),
     )
+    group.addoption(
+        "--ep-jsonl-path",
+        default=None,
+        help=("Load input from a jsonl file that is already in EvaluationRow or openai CHAT format"),
+    )
+    group.addoption(
+        "--ep-completion-params",
+        default=[],
+        action="append",
+        help=("Overwrite completion params with json. Can be used multiple times. "),
+    )
+    group.addoption(
+        "--ep-remote-rollout-processor-base-url",
+        default=None,
+        help=("If set, use this base URL for remote rollout processing. Example: http://localhost:8000"),
+    )
+    group.addoption(
+        "--ep-output-dir",
+        default=None,
+        help=("If set, save evaluation results to this directory in jsonl format."),
+    )
 
 
 def _normalize_max_rows(val: Optional[str]) -> Optional[str]:
@@ -240,9 +263,22 @@ def pytest_configure(config) -> None:
     if threshold_env is not None:
         os.environ["EP_PASSED_THRESHOLD"] = threshold_env
 
+    if config.getoption("--ep-output-dir"):
+        # set this to save eval results to the target dir in jsonl format
+        os.environ["EP_OUTPUT_DIR"] = config.getoption("--ep-output-dir")
+
     if config.getoption("--ep-no-upload"):
         os.environ["EP_NO_UPLOAD"] = "1"
 
+    if config.getoption("--ep-jsonl-path"):
+        os.environ["EP_JSONL_PATH"] = config.getoption("--ep-jsonl-path")
+
+    if config.getoption("--ep-completion-params"):
+        # redump to json to make sure they are legit
+        os.environ["EP_COMPLETION_PARAMS"] = json.dumps(
+            [json.loads(s) for s in config.getoption("--ep-completion-params") or []]
+        )
+
     # Allow ad-hoc overrides of input params via CLI flags
     try:
         merged: dict = {}

{eval_protocol-0.2.32 → eval_protocol-0.2.34}/eval_protocol/pytest/remote_rollout_processor.py

@@ -9,6 +9,7 @@ from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader
 from eval_protocol.types.remote_rollout_processor import InitRequest, RolloutMetadata
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
+import os
 
 
 class RemoteRolloutProcessor(RolloutProcessor):
@@ -31,6 +32,8 @@ class RemoteRolloutProcessor(RolloutProcessor):
         # config.kwargs at call time for backward compatibility.
         self._remote_base_url = remote_base_url
         self._model_base_url = model_base_url
+        if os.getenv("EP_REMOTE_ROLLOUT_PROCESSOR_BASE_URL"):
+            self._remote_base_url = os.getenv("EP_REMOTE_ROLLOUT_PROCESSOR_BASE_URL")
         self._poll_interval = poll_interval
         self._timeout_seconds = timeout_seconds
         self._output_data_loader = output_data_loader

{eval_protocol-0.2.32 → eval_protocol-0.2.34}/eval_protocol/pytest/utils.py

@@ -19,6 +19,8 @@ from eval_protocol.models import (
     EvaluationThresholdDict,
     Status,
 )
+from eval_protocol.data_loader import DynamicDataLoader
+from eval_protocol.data_loader.models import EvaluationDataLoader
 from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import (
     RolloutProcessorConfig,
@@ -239,6 +241,45 @@ def parse_ep_completion_params(
     return completion_params
 
 
+def parse_ep_completion_params_overwrite(
+    completion_params: Sequence[CompletionParams | None] | None,
+) -> Sequence[CompletionParams | None]:
+    new_completion_params = os.getenv("EP_COMPLETION_PARAMS")
+    if new_completion_params:
+        try:
+            new_completion_params_list = json.loads(new_completion_params)
+            if isinstance(new_completion_params_list, list):
+                return new_completion_params_list
+        except Exception:
+            pass
+    return completion_params or []
+
+
+def _rows_from_jsonl(path: str) -> list[EvaluationRow]:
+    rows = []
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                rows.append(EvaluationRow(**json.loads(line)))
+    except Exception as e:
+        print(f"❌ Failed to load rows from JSONL at {path}: {e}")
+        return []
+
+    return rows
+
+
+def parse_ep_dataloaders(
+    dataloaders: Sequence[EvaluationDataLoader] | EvaluationDataLoader | None,
+) -> Sequence[EvaluationDataLoader] | EvaluationDataLoader | None:
+    try:
+        load_from_jsonl_path = os.getenv("EP_JSONL_PATH")
+        if load_from_jsonl_path:
+            return DynamicDataLoader(generators=[lambda path=load_from_jsonl_path: _rows_from_jsonl(path)])
+    except Exception:
+        pass
+    return dataloaders or None
+
+
 def parse_ep_passed_threshold(
     default_value: float | EvaluationThresholdDict | EvaluationThreshold | None,
 ) -> EvaluationThreshold | None:

{eval_protocol-0.2.32 → eval_protocol-0.2.34/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.32
+Version: 0.2.34
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT