eval-protocol 0.2.84.dev10__tar.gz → 0.2.86__tar.gz

{eval_protocol-0.2.84.dev10/eval_protocol.egg-info → eval_protocol-0.2.86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.84.dev10
+Version: 0.2.86
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -118,9 +118,7 @@ Dynamic: license-file
 [![PyPI - Version](https://img.shields.io/pypi/v/eval-protocol)](https://pypi.org/project/eval-protocol/)
 [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/eval-protocol/python-sdk)
 
-**Stop guessing which AI model to use. Build a data-driven model leaderboard.**
-
-With hundreds of models and configs, you need objective data to choose the right one for your use case. EP helps you evaluate real traces, compare models, and visualize results locally.
+**The open-source framework to help you write evals for RL.**
 
 ## 🚀 Features
 

{eval_protocol-0.2.84.dev10 → eval_protocol-0.2.86}/README.md

@@ -3,9 +3,7 @@
 [![PyPI - Version](https://img.shields.io/pypi/v/eval-protocol)](https://pypi.org/project/eval-protocol/)
 [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/eval-protocol/python-sdk)
 
-**Stop guessing which AI model to use. Build a data-driven model leaderboard.**
-
-With hundreds of models and configs, you need objective data to choose the right one for your use case. EP helps you evaluate real traces, compare models, and visualize results locally.
+**The open-source framework to help you write evals for RL.**
 
 ## 🚀 Features
 

{eval_protocol-0.2.84.dev10 → eval_protocol-0.2.86}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-11-11T15:13:03-0800",
+ "date": "2025-11-11T17:37:10-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "41b79daeafe6bcb53a8a3183738314596874696d",
- "version": "0.2.84.dev.10"
+ "full-revisionid": "3b2340644bd8894822a9e34445bc60552e3843cf",
+ "version": "0.2.86"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.84.dev10 → eval_protocol-0.2.86}/eval_protocol/cli.py

@@ -407,7 +407,7 @@ def parse_args(args=None):
     rft_parser.add_argument("--eval-auto-carveout", dest="eval_auto_carveout", action="store_true", default=True)
     rft_parser.add_argument("--no-eval-auto-carveout", dest="eval_auto_carveout", action="store_false")
     # Rollout chunking
-    rft_parser.add_argument("--chunk-size", type=int, default=10, help="Data chunk size for rollout batching")
+    rft_parser.add_argument("--chunk-size", type=int, default=100, help="Data chunk size for rollout batching")
     # Inference params
     rft_parser.add_argument("--temperature", type=float)
     rft_parser.add_argument("--top-p", type=float)
@@ -447,6 +447,16 @@ def parse_args(args=None):
         action="store_true",
         help="Non-interactive: if multiple tests exist and no --entry, fails with guidance",
     )
+    local_test_parser.add_argument(
+        "--docker-build-extra",
+        default="",
+        help="Extra flags to pass to 'docker build' (quoted string, e.g. \"--no-cache --pull --progress=plain\")",
+    )
+    local_test_parser.add_argument(
+        "--docker-run-extra",
+        default="",
+        help="Extra flags to pass to 'docker run' (quoted string, e.g. \"--env-file .env --memory=8g\")",
+    )
 
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.

{eval_protocol-0.2.84.dev10 → eval_protocol-0.2.86}/eval_protocol/cli_commands/local_test.py

@@ -2,6 +2,7 @@ import argparse
 import os
 import subprocess
 import sys
+import shlex
 from typing import List
 
 from .upload import _discover_tests, _prompt_select
@@ -24,16 +25,15 @@ def _run_pytest_host(pytest_target: str) -> int:
     return proc.returncode
 
 
-def _build_docker_image(dockerfile_path: str, image_tag: str) -> bool:
+def _build_docker_image(dockerfile_path: str, image_tag: str, build_extras: List[str] | None = None) -> bool:
     context_dir = os.path.dirname(dockerfile_path)
     print(f"Building Docker image '{image_tag}' from {dockerfile_path} ...")
     try:
-        proc = subprocess.run(
-            ["docker", "build", "-t", image_tag, "-f", dockerfile_path, context_dir],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-        )
+        base_cmd = ["docker", "build"]
+        if build_extras:
+            base_cmd += build_extras
+        base_cmd += ["-t", image_tag, "-f", dockerfile_path, context_dir]
+        proc = subprocess.run(base_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
         print(proc.stdout)
         return proc.returncode == 0
     except FileNotFoundError:
@@ -41,7 +41,9 @@ def _build_docker_image(dockerfile_path: str, image_tag: str) -> bool:
         return False
 
 
-def _run_pytest_in_docker(project_root: str, image_tag: str, pytest_target: str) -> int:
+def _run_pytest_in_docker(
+    project_root: str, image_tag: str, pytest_target: str, run_extras: List[str] | None = None
+) -> int:
     workdir = "/workspace"
     # Host HOME logs directory to map into container
     host_home = os.path.expanduser("~")
@@ -73,6 +75,8 @@ def _run_pytest_in_docker(project_root: str, image_tag: str, pytest_target: str)
         cmd += ["--user", f"{uid}:{gid}"]
     except Exception:
         pass
+    if run_extras:
+        cmd += run_extras
     cmd += [image_tag, "pytest", pytest_target, "-vs"]
     print("Running in Docker:", " ".join(cmd))
     try:
@@ -91,11 +95,16 @@ def local_test_command(args: argparse.Namespace) -> int:
     entry = getattr(args, "entry", None)
     if entry:
         if "::" in entry:
-            file_part = entry.split("::", 1)[0]
+            file_part, func_part = entry.split("::", 1)
             file_path = (
                 file_part if os.path.isabs(file_part) else os.path.abspath(os.path.join(project_root, file_part))
             )
-            pytest_target = entry
+            # Convert to project-relative like the non-:: path
+            try:
+                rel = os.path.relpath(file_path, project_root)
+            except Exception:
+                rel = file_path
+            pytest_target = f"{rel}::{func_part}"
         else:
             file_path = entry if os.path.isabs(entry) else os.path.abspath(os.path.join(project_root, entry))
             # Use path relative to project_root when possible
@@ -126,6 +135,10 @@ def local_test_command(args: argparse.Namespace) -> int:
         pytest_target = rel
 
     ignore_docker = bool(getattr(args, "ignore_docker", False))
+    build_extras_str = getattr(args, "docker_build_extra", "") or ""
+    run_extras_str = getattr(args, "docker_run_extra", "") or ""
+    build_extras = shlex.split(build_extras_str) if build_extras_str else []
+    run_extras = shlex.split(run_extras_str) if run_extras_str else []
     if ignore_docker:
         if not pytest_target:
             print("Error: Failed to resolve a pytest target to run.")
@@ -146,14 +159,14 @@ def local_test_command(args: argparse.Namespace) -> int:
         except Exception:
             pass
         image_tag = "ep-evaluator:local"
-        ok = _build_docker_image(dockerfiles[0], image_tag)
+        ok = _build_docker_image(dockerfiles[0], image_tag, build_extras=build_extras)
         if not ok:
             print("Docker build failed. See logs above.")
             return 1
         if not pytest_target:
             print("Error: Failed to resolve a pytest target to run.")
             return 1
-        return _run_pytest_in_docker(project_root, image_tag, pytest_target)
+        return _run_pytest_in_docker(project_root, image_tag, pytest_target, run_extras=run_extras)
 
     # No Dockerfile: run on host
     if not pytest_target:

{eval_protocol-0.2.84.dev10 → eval_protocol-0.2.86/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.84.dev10
+Version: 0.2.86
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -118,9 +118,7 @@ Dynamic: license-file
 [![PyPI - Version](https://img.shields.io/pypi/v/eval-protocol)](https://pypi.org/project/eval-protocol/)
 [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/eval-protocol/python-sdk)
 
-**Stop guessing which AI model to use. Build a data-driven model leaderboard.**
-
-With hundreds of models and configs, you need objective data to choose the right one for your use case. EP helps you evaluate real traces, compare models, and visualize results locally.
+**The open-source framework to help you write evals for RL.**
 
 ## 🚀 Features
 

{eval_protocol-0.2.84.dev10 → eval_protocol-0.2.86}/tests/test_cli_local_test.py

@@ -95,11 +95,11 @@ def test_local_test_builds_and_runs_in_docker(tmp_path, monkeypatch):
     from eval_protocol.cli_commands import local_test as lt
 
     monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [str(project / "Dockerfile")])
-    monkeypatch.setattr(lt, "_build_docker_image", lambda dockerfile, tag, platform=None: True)
+    monkeypatch.setattr(lt, "_build_docker_image", lambda dockerfile, tag, build_extras=None: True)
 
     captured = {"target": "", "image": ""}
 
-    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, platform=None) -> int:
+    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, run_extras=None) -> int:
         captured["target"] = pytest_target
         captured["image"] = image_tag
         return 0
@@ -123,12 +123,11 @@ def test_local_test_selector_single_test(tmp_path, monkeypatch):
     test_file.write_text("def test_dummy():\n    assert True\n", encoding="utf-8")
 
     from eval_protocol.cli_commands import local_test as lt
-    from eval_protocol.cli_commands import upload as up
 
     # No entry; force discover + selector
     disc = SimpleNamespace(qualname="metric.test_sel", file_path=str(test_file))
     monkeypatch.setattr(lt, "_discover_tests", lambda root: [disc])
-    monkeypatch.setattr(up, "_prompt_select", lambda tests, non_interactive=False: tests[:1])
+    monkeypatch.setattr(lt, "_prompt_select", lambda tests, non_interactive=False: tests[:1])
     monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [])
 
     called = {"host": False}
@@ -143,3 +142,115 @@ def test_local_test_selector_single_test(tmp_path, monkeypatch):
     rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
     assert rc == 0
     assert called["host"] is True
+
+
+def test_local_test_passes_docker_build_extra(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    test_file = project / "metric" / "test_build_extra.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("def test_dummy():\n    assert True\n", encoding="utf-8")
+
+    from eval_protocol.cli_commands import local_test as lt
+
+    monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [str(project / "Dockerfile")])
+
+    captured = {"extras": None}
+
+    def _fake_build(dockerfile, tag, build_extras=None):
+        captured["extras"] = build_extras
+        return True
+
+    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, run_extras=None) -> int:
+        return 0
+
+    monkeypatch.setattr(lt, "_build_docker_image", _fake_build)
+    monkeypatch.setattr(lt, "_run_pytest_in_docker", _fake_run_docker)
+
+    # Extras string with multiple flags and equals-arg
+    args = SimpleNamespace(
+        entry=str(test_file),
+        ignore_docker=False,
+        yes=True,
+        docker_build_extra="--no-cache --pull --progress=plain --build-arg KEY=VAL",
+        docker_run_extra="",
+    )
+    rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
+    assert rc == 0
+    # Expect split list preserving tokens order
+    assert captured["extras"] == ["--no-cache", "--pull", "--progress=plain", "--build-arg", "KEY=VAL"]
+
+
+def test_local_test_passes_docker_run_extra(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    test_file = project / "metric" / "test_run_extra.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("def test_dummy():\n    assert True\n", encoding="utf-8")
+
+    from eval_protocol.cli_commands import local_test as lt
+
+    monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [str(project / "Dockerfile")])
+    monkeypatch.setattr(lt, "_build_docker_image", lambda dockerfile, tag, build_extras=None: True)
+
+    captured = {"extras": None}
+
+    def _fake_run_docker(root: str, image_tag: str, pytest_target: str, run_extras=None) -> int:
+        captured["extras"] = run_extras
+        return 0
+
+    monkeypatch.setattr(lt, "_run_pytest_in_docker", _fake_run_docker)
+
+    args = SimpleNamespace(
+        entry=str(test_file),
+        ignore_docker=False,
+        yes=True,
+        docker_build_extra="",
+        docker_run_extra="--env-file .env --memory=8g --cpus=2 --add-host=host.docker.internal:host-gateway",
+    )
+    rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
+    assert rc == 0
+    assert captured["extras"] == [
+        "--env-file",
+        ".env",
+        "--memory=8g",
+        "--cpus=2",
+        "--add-host=host.docker.internal:host-gateway",
+    ]
+
+
+def test_local_test_normalizes_entry_with_selector(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Create a dummy test file
+    test_file = project / "metric" / "test_sel_abs.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("def test_dummy():\n    assert True\n", encoding="utf-8")
+
+    abs_entry = f"{str(test_file)}::test_dummy"
+
+    from eval_protocol.cli_commands import local_test as lt
+
+    # Avoid Docker path
+    monkeypatch.setattr(lt, "_find_dockerfiles", lambda root: [])
+
+    captured = {"target": ""}
+
+    def _fake_host(target: str) -> int:
+        captured["target"] = target
+        return 0
+
+    monkeypatch.setattr(lt, "_run_pytest_host", _fake_host)
+
+    args = SimpleNamespace(entry=abs_entry, ignore_docker=False, yes=True)
+    rc = lt.local_test_command(args)  # pyright: ignore[reportArgumentType]
+    assert rc == 0
+    # Expect project-relative path plus selector
+    rel = os.path.relpath(str(test_file), str(project))
+    assert captured["target"] == f"{rel}::test_dummy"