eval-protocol 0.2.80.dev4__tar.gz → 0.2.81__tar.gz

{eval_protocol-0.2.80.dev4/eval_protocol.egg-info → eval_protocol-0.2.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.80.dev4
+Version: 0.2.81
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.80.dev4 → eval_protocol-0.2.81}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-11-05T01:57:48-0800",
+ "date": "2025-11-07T14:46:57-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "6b9f1331a8ab25d0f057d3426a77849cb1b0a2f0",
- "version": "0.2.80-dev4"
+ "full-revisionid": "61ca3d755eb7ca874ad04f27c843e153ad0c0345",
+ "version": "0.2.81"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.80.dev4 → eval_protocol-0.2.81}/eval_protocol/cli_commands/create_rft.py

@@ -487,16 +487,23 @@ def create_rft_command(args) -> int:
                 return 1
 
     # Build training config/body
-    # Ensure base model is explicitly provided for clarity
-    if not getattr(args, "base_model", None):
-        print(
-            "Error: --base-model is required. Please specify the base model resource id (e.g., accounts/{account}/models/<model_id>)."
-        )
+    # Exactly one of base-model or warm-start-from must be provided
+    base_model_raw = getattr(args, "base_model", None)
+    warm_start_from_raw = getattr(args, "warm_start_from", None)
+    # Treat empty/whitespace strings as not provided
+    base_model = base_model_raw.strip() if isinstance(base_model_raw, str) else base_model_raw
+    warm_start_from = warm_start_from_raw.strip() if isinstance(warm_start_from_raw, str) else warm_start_from_raw
+    has_base_model = bool(base_model)
+    has_warm_start = bool(warm_start_from)
+    if (not has_base_model and not has_warm_start) or (has_base_model and has_warm_start):
+        print("Error: exactly one of --base-model or --warm-start-from must be specified.")
         return 1
 
-    training_config: Dict[str, Any] = {"baseModel": args.base_model}
-    if getattr(args, "warm_start_from", None):
-        training_config["warmStartFrom"] = args.warm_start_from
+    training_config: Dict[str, Any] = {}
+    if has_base_model:
+        training_config["baseModel"] = base_model
+    if has_warm_start:
+        training_config["warmStartFrom"] = warm_start_from
 
     # Optional hyperparameters
     for key, arg_name in [

{eval_protocol-0.2.80.dev4 → eval_protocol-0.2.81}/eval_protocol/pytest/evaluation_test.py

@@ -179,7 +179,11 @@ def evaluation_test(
         completion_params = [None]
     else:
         completion_params_provided = True
-    if rollout_processor is None:
+
+    # Override rollout processor if flag is set
+    if os.environ.get("EP_USE_NO_OP_ROLLOUT_PROCESSOR") == "1":
+        rollout_processor = NoOpRolloutProcessor()
+    elif rollout_processor is None:
         rollout_processor = NoOpRolloutProcessor()
 
     active_logger: DatasetLogger = logger if logger else default_logger
@@ -704,22 +708,16 @@ def evaluation_test(
         )
         pytest_wrapper = pytest.mark.asyncio(pytest_wrapper)
 
-        ep_params: dict[str, Any] = {
-            "rollout_processor": rollout_processor,
-            "server_script_path": server_script_path,
-            "mcp_config_path": mcp_config_path,
-            "rollout_processor_kwargs": rollout_processor_kwargs,
-            "mode": mode,
-        }
-
-        print(f"ep_params: {ep_params}")
-
         # Create the dual mode wrapper
         dual_mode_wrapper = create_dual_mode_wrapper(
             test_func, mode, max_concurrent_rollouts, max_concurrent_evaluations, pytest_wrapper
         )
 
-        setattr(dual_mode_wrapper, "__ep_params__", ep_params)
+        # Make this pytest discoverable regardless of pytest configuration. So
+        # you can name your eval whatever you want, as long as it's decorated
+        # with @evaluation_test.
+        dual_mode_wrapper.__test__ = True
+
         return dual_mode_wrapper  # pyright: ignore[reportReturnType, reportUnknownVariableType]
 
     return decorator

{eval_protocol-0.2.80.dev4 → eval_protocol-0.2.81}/eval_protocol/pytest/plugin.py

@@ -133,6 +133,14 @@ def pytest_addoption(parser) -> None:
         default=None,
         help=("If set, use this base URL for remote rollout processing. Example: http://localhost:8000"),
     )
+    group.addoption(
+        "--ep-no-op-rollout-processor",
+        action="store_true",
+        default=False,
+        help=(
+            "Override the rollout processor to use NoOpRolloutProcessor, which passes input dataset through unchanged."
+        ),
+    )
     group.addoption(
         "--ep-output-dir",
         default=None,
@@ -267,6 +275,9 @@ def pytest_configure(config) -> None:
         # set this to save eval results to the target dir in jsonl format
         os.environ["EP_OUTPUT_DIR"] = config.getoption("--ep-output-dir")
 
+    if config.getoption("--ep-no-op-rollout-processor"):
+        os.environ["EP_USE_NO_OP_ROLLOUT_PROCESSOR"] = "1"
+
     if config.getoption("--ep-no-upload"):
         os.environ["EP_NO_UPLOAD"] = "1"
 

{eval_protocol-0.2.80.dev4 → eval_protocol-0.2.81/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.80.dev4
+Version: 0.2.81
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT