eval-protocol 0.3.9.dev1__tar.gz → 0.3.9.dev2__tar.gz

{eval_protocol-0.3.9.dev1/eval_protocol.egg-info → eval_protocol-0.3.9.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.9.dev1
+Version: 0.3.9.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.9.dev1 → eval_protocol-0.3.9.dev2}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-01-08T13:29:17-0800",
+ "date": "2026-01-08T11:04:23-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "764ac4f132c35fe01c354b4150cbc19c7eedea12",
- "version": "0.3.9.dev.1"
+ "full-revisionid": "e504b404fd0b77190d42a807cea241e95785a441",
+ "version": "0.3.9.dev.2"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.9.dev1 → eval_protocol-0.3.9.dev2}/eval_protocol/pytest/evaluation_test.py

@@ -20,12 +20,13 @@ from eval_protocol.models import (
     EvaluationRow,
     EvaluationThreshold,
     EvaluationThresholdDict,
+    EvaluateResult,
     Status,
     EPParameters,
 )
 from eval_protocol.pytest.dual_mode_wrapper import create_dual_mode_wrapper
 from eval_protocol.pytest.evaluation_test_postprocess import postprocess
-from eval_protocol.pytest.execution import execute_pytest_with_exception_handling
+from eval_protocol.pytest.execution import execute_pytest, execute_pytest_with_exception_handling
 from eval_protocol.pytest.priority_scheduler import execute_priority_rollouts
 from eval_protocol.pytest.generate_parameter_combinations import (
     ParameterizedTestKwargs,
@@ -55,7 +56,6 @@ from eval_protocol.pytest.evaluation_test_utils import (
     AggregationMethod,
     add_cost_metrics,
     log_eval_status_and_rows,
-    normalize_fireworks_model,
     parse_ep_completion_params,
     parse_ep_completion_params_overwrite,
     parse_ep_max_concurrent_rollouts,
@@ -205,7 +205,6 @@ def evaluation_test(
     max_dataset_rows = parse_ep_max_rows(max_dataset_rows)
     completion_params = parse_ep_completion_params(completion_params)
     completion_params = parse_ep_completion_params_overwrite(completion_params)
-    completion_params = [normalize_fireworks_model(cp) for cp in completion_params]
     original_completion_params = completion_params
     passed_threshold = parse_ep_passed_threshold(passed_threshold)
     data_loaders = parse_ep_dataloaders(data_loaders)
@@ -366,7 +365,6 @@ def evaluation_test(
                             row.input_metadata.row_id = generate_id(seed=0, index=index)
 
                     completion_params = kwargs["completion_params"] if "completion_params" in kwargs else None
-                    completion_params = normalize_fireworks_model(completion_params)
                     # Create eval metadata with test function info and current commit hash
                     eval_metadata = EvalMetadata(
                         name=test_func.__name__,
@@ -411,22 +409,21 @@ def evaluation_test(
 
                     rollout_processor.setup()
 
-                    use_priority_scheduler = os.environ.get(
-                        "EP_USE_PRIORITY_SCHEDULER", "0"
-                    ) == "1" and not isinstance(rollout_processor, MCPGymRolloutProcessor)
+                    use_priority_scheduler = (
+                        (
+                            os.environ.get("EP_USE_PRIORITY_SCHEDULER", "0") == "1"
+                            and not isinstance(rollout_processor, MCPGymRolloutProcessor)
+                        )
+                    )
 
                     if use_priority_scheduler:
                         microbatch_output_size = os.environ.get("EP_MICRO_BATCH_OUTPUT_SIZE", None)
                         output_dir = os.environ.get("EP_OUTPUT_DIR", None)
                         if microbatch_output_size and output_dir:
-                            output_buffer = MicroBatchDataBuffer(
-                                num_runs=num_runs,
-                                batch_size=int(microbatch_output_size),
-                                output_path_template=os.path.join(output_dir, "buffer_{index}.jsonl"),
-                            )
+                            output_buffer = MicroBatchDataBuffer(num_runs=num_runs, batch_size=int(microbatch_output_size), output_path_template=os.path.join(output_dir, "buffer_{index}.jsonl"))
                         else:
                             output_buffer = None
-
+                        
                         try:
                             priority_results = await execute_priority_rollouts(
                                 dataset=data,
@@ -444,12 +441,12 @@ def evaluation_test(
                         finally:
                             if output_buffer:
                                 await output_buffer.close()
-
+                        
                         for res in priority_results:
                             run_idx = (res.execution_metadata.extra or {}).get("run_index", 0)
                             if run_idx < len(all_results):
                                 all_results[run_idx].append(res)
-
+                            
                             processed_rows_in_run.append(res)
 
                         postprocess(
@@ -465,7 +462,6 @@ def evaluation_test(
                         )
 
                     else:
-
                         async def execute_run(run_idx: int, config: RolloutProcessorConfig):
                             nonlocal all_results
 
@@ -510,7 +506,9 @@ def evaluation_test(
                                         raise ValueError(
                                             f"Test function {test_func.__name__} did not return an EvaluationRow instance. You must return an EvaluationRow instance from your test function decorated with @evaluation_test."
                                         )
-                                    result.execution_metadata.eval_duration_seconds = time.perf_counter() - start_time
+                                    result.execution_metadata.eval_duration_seconds = (
+                                        time.perf_counter() - start_time
+                                    )
                                     return result
 
                             async def _execute_groupwise_eval_with_semaphore(
@@ -521,9 +519,7 @@ def evaluation_test(
                                     evaluation_test_kwargs = kwargs.get("evaluation_test_kwargs") or {}
                                     primary_rollout_id = rows[0].execution_metadata.rollout_id if rows else None
                                     group_rollout_ids = [
-                                        r.execution_metadata.rollout_id
-                                        for r in rows
-                                        if r.execution_metadata.rollout_id
+                                        r.execution_metadata.rollout_id for r in rows if r.execution_metadata.rollout_id
                                     ]
                                     async with rollout_logging_context(
                                         primary_rollout_id or "",
@@ -600,9 +596,7 @@ def evaluation_test(
                                         row_groups[row.input_metadata.row_id].append(row)
                                 tasks = []
                                 for _, rows in row_groups.items():
-                                    tasks.append(
-                                        asyncio.create_task(_execute_groupwise_eval_with_semaphore(rows=rows))
-                                    )
+                                    tasks.append(asyncio.create_task(_execute_groupwise_eval_with_semaphore(rows=rows)))
                                 results = []
                                 for task in tasks:
                                     res = await task
@@ -698,9 +692,9 @@ def evaluation_test(
                             # For other processors, create all tasks at once and run in parallel
                             # Concurrency is now controlled by the shared semaphore in each rollout processor
                             await run_tasks_with_run_progress(execute_run, num_runs, config)
-
+                        
                         experiment_duration_seconds = time.perf_counter() - experiment_start_time
-
+                        
                         # for groupwise mode, the result contains eval output from multiple completion_params, we need to differentiate them
                         # rollout_id is used to differentiate the result from different completion_params
                         if mode == "groupwise":
@@ -736,12 +730,15 @@ def evaluation_test(
                                 experiment_duration_seconds,
                             )
 
+
+                
                     if not all(r.evaluation_result is not None for run_results in all_results for r in run_results):
                         raise AssertionError(
                             "Some EvaluationRow instances are missing evaluation_result. "
                             "Your @evaluation_test function must set `row.evaluation_result`"
                         )
 
+                    
                 except AssertionError:
                     _log_eval_error(
                         Status.eval_finished(),

{eval_protocol-0.3.9.dev1 → eval_protocol-0.3.9.dev2}/eval_protocol/pytest/evaluation_test_utils.py

@@ -619,22 +619,3 @@ def build_rollout_processor_config(
         server_script_path=None,
         kwargs=rollout_processor_kwargs,
     )
-
-
-def normalize_fireworks_model(completion_params: CompletionParams | None) -> CompletionParams | None:
-    """Fireworks model names like 'accounts/<org>/models/<model>' need the fireworks_ai/
-    prefix when routing through LiteLLM. This function adds the prefix if missing.
-    """
-    if completion_params is None:
-        return None
-
-    model = completion_params.get("model")
-    if (
-        model
-        and isinstance(model, str)
-        and not model.startswith("fireworks_ai/")
-        and re.match(r"^accounts/[^/]+/models/.+", model)
-    ):
-        completion_params = completion_params.copy()
-        completion_params["model"] = f"fireworks_ai/{model}"
-    return completion_params

{eval_protocol-0.3.9.dev1 → eval_protocol-0.3.9.dev2}/eval_protocol/pytest/remote_rollout_processor.py

@@ -122,9 +122,20 @@ class RemoteRolloutProcessor(RolloutProcessor):
                         status_logs.append(log)
 
                 if status_logs:
+                    if len(status_logs) > 1:
+                        logger.warning(
+                            "Found %s status logs for rollout %s; expected at most 1. Using the first one: %s",
+                            len(status_logs),
+                            row.execution_metadata.rollout_id,
+                            status_logs[0],
+                        )
                     # Use the first log with status information
                     status_log = status_logs[0]
                     status_dict = status_log.get("status")
+                    raw_extras = status_log.get("extras") or {}
+                    status_extras = {
+                        k: v for k, v in raw_extras.items() if k not in ("logger_name", "level", "timestamp")
+                    }
 
                     logger.info(
                         f"Found status log for rollout {row.execution_metadata.rollout_id}: {status_log.get('message', '')}"
@@ -149,6 +160,11 @@ class RemoteRolloutProcessor(RolloutProcessor):
                         details=status_details,
                     )
 
+                    if row.execution_metadata.extra:
+                        row.execution_metadata.extra.update(status_extras)
+                    else:
+                        row.execution_metadata.extra = status_extras
+
                     logger.info("Stopping polling for rollout %s", row.execution_metadata.rollout_id)
                     break
 

{eval_protocol-0.3.9.dev1 → eval_protocol-0.3.9.dev2}/eval_protocol/pytest/tracing_utils.py

@@ -179,7 +179,18 @@ def update_row_with_remote_trace(
                 if k not in row.input_metadata.dataset_info:
                     row.input_metadata.dataset_info[k] = v
 
-        row.execution_metadata = remote_row.execution_metadata
+        preserved_extra = row.execution_metadata.extra
+
+        row.execution_metadata = remote_row.execution_metadata.model_copy(deep=True)
+
+        if preserved_extra:
+            if row.execution_metadata.extra:
+                # Merge remote and local extras; local takes precedence on conflicts
+                merged = row.execution_metadata.extra or {}
+                merged.update(preserved_extra)
+                row.execution_metadata.extra = merged
+            else:
+                row.execution_metadata.extra = preserved_extra
         return None
     else:
         raise ValueError("Output data loader should return exactly one row.")

{eval_protocol-0.3.9.dev1 → eval_protocol-0.3.9.dev2/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.9.dev1
+Version: 0.3.9.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT