eval-protocol 0.3.25__tar.gz → 0.3.27__tar.gz

{eval_protocol-0.3.25/eval_protocol.egg-info → eval_protocol-0.3.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.25
+Version: 0.3.27
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.25 → eval_protocol-0.3.27}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-03-13T16:09:44-0700",
+ "date": "2026-04-06T16:29:39-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "3c8d8f23f7b301697f246c64e57d08fa1c7af50b",
- "version": "0.3.25"
+ "full-revisionid": "f77b26fd89b097ac0b09b56ccdd1b11d1feeeb72",
+ "version": "0.3.27"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.3.25 → eval_protocol-0.3.27}/eval_protocol/cli_commands/utils.py

@@ -738,7 +738,13 @@ def add_args_from_callable_signature(
                 prefix = name.replace("_", "-")
                 field_kebab = field_name.replace("_", "-")
                 flag_name = f"--{prefix}-{field_kebab}"
-                flags = [flag_name] + aliases.get(f"{name}.{field_name}", []) + [f"--{field_kebab}"]
+                # Skip short alias if already claimed by another TypedDict with the same field name
+                # (e.g. AwsS3Config.credentials_secret and AzureBlobStorageConfig.credentials_secret).
+                short_flag = f"--{field_kebab}"
+                existing = {s for a in parser._actions for s in a.option_strings}
+                flags = [flag_name] + aliases.get(f"{name}.{field_name}", [])
+                if short_flag not in existing and short_flag != flag_name:
+                    flags.append(short_flag)
                 help_text = help_overrides.get(f"{name}.{field_name}", field_help.get(field_name))
 
                 _add_flag(parser, flags, field_hints.get(field_name, field_type), help_text)

{eval_protocol-0.3.25 → eval_protocol-0.3.27}/eval_protocol/pytest/evaluation_test.py

@@ -211,7 +211,7 @@ def evaluation_test(
     completion_params = parse_ep_completion_params_overwrite(completion_params)
     original_completion_params = completion_params
     passed_threshold = parse_ep_passed_threshold(passed_threshold)
-    data_loaders = parse_ep_dataloaders(data_loaders)
+    data_loaders = parse_ep_dataloaders(data_loaders, dataset_adapter=dataset_adapter)
     custom_invocation_id = os.environ.get("EP_INVOCATION_ID", None)
 
     # ignore other data input params when dataloader is provided

{eval_protocol-0.3.25 → eval_protocol-0.3.27}/eval_protocol/pytest/evaluation_test_utils.py

@@ -21,6 +21,7 @@ from eval_protocol.models import (
     EvaluationThresholdDict,
     Status,
 )
+from eval_protocol.common_utils import load_jsonl
 from eval_protocol.data_loader import DynamicDataLoader
 from eval_protocol.data_loader.models import EvaluationDataLoader
 from eval_protocol.pytest.rollout_processor import RolloutProcessor
@@ -288,10 +289,21 @@ def _rows_from_jsonl(path: str) -> list[EvaluationRow]:
 
 def parse_ep_dataloaders(
     dataloaders: Sequence[EvaluationDataLoader] | EvaluationDataLoader | None,
+    *,
+    dataset_adapter: Callable[[list[dict[str, Any]]], list[EvaluationRow]] | None = None,
 ) -> Sequence[EvaluationDataLoader] | EvaluationDataLoader | None:
+    """When ``EP_JSONL_PATH`` is set, load JSONL as raw dicts and run ``dataset_adapter`` if provided.
+
+    Without ``dataset_adapter``, rows are built with ``EvaluationRow(**dict)`` (legacy behavior),
+    which skips custom label fields that adapters normally attach.
+    """
     try:
         load_from_jsonl_path = os.getenv("EP_JSONL_PATH")
         if load_from_jsonl_path:
+            if dataset_adapter is not None:
+                return DynamicDataLoader(
+                    generators=[lambda path=load_from_jsonl_path, da=dataset_adapter: da(load_jsonl(path))]
+                )
             return DynamicDataLoader(generators=[lambda path=load_from_jsonl_path: _rows_from_jsonl(path)])
     except Exception:
         pass

{eval_protocol-0.3.25 → eval_protocol-0.3.27/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.3.25
+Version: 0.3.27
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.3.25 → eval_protocol-0.3.27}/pyproject.toml

@@ -184,7 +184,7 @@ parentdir_prefix = "eval-protocol-"
 ### Black and isort removed; Ruff provides formatting and imports
 
 [tool.uv.sources]
-tau2 = { git = "https://github.com/sierra-research/tau2-bench.git" }
+tau2 = { git = "https://github.com/sierra-research/tau2-bench.git", rev = "v0.2.0" }
 
 [dependency-groups]
 dev = [