eval-protocol 0.2.83__tar.gz → 0.2.84__tar.gz

{eval_protocol-0.2.83/eval_protocol.egg-info → eval_protocol-0.2.84}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.83
+Version: 0.2.84
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.83 → eval_protocol-0.2.84}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-11-09T23:23:12-0800",
+ "date": "2025-11-10T00:30:58-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "a533dcb232528e3910d94adb922c6ab7df27bc4e",
- "version": "0.2.83"
+ "full-revisionid": "2d75acf5944468856d9f1bea787fce63dcabc16f",
+ "version": "0.2.84"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.83 → eval_protocol-0.2.84}/eval_protocol/cli_commands/create_rft.py

@@ -20,6 +20,7 @@ from ..fireworks_rft import (
     create_dataset_from_jsonl,
     create_reinforcement_fine_tuning_job,
 )
+from ..fireworks_rft import detect_dataset_builder, materialize_dataset_via_builder
 from .upload import _discover_tests, _normalize_evaluator_id, _prompt_select
 
 
@@ -250,6 +251,37 @@ def _build_trimmed_dataset_id(evaluator_id: str) -> str:
     return f"{base}{suffix}"
 
 
+def _resolve_selected_test(
+    project_root: str,
+    evaluator_id: Optional[str],
+    selected_tests: Optional[list] = None,
+) -> tuple[Optional[str], Optional[str]]:
+    """
+    Resolve a single test's source file path and function name to use downstream.
+    Priority:
+      1) If selected_tests provided and length == 1, use it.
+      2) Else discover tests; if exactly one test, use it.
+      3) Else, if evaluator_id provided, match by normalized '<file-stem>-<func-name>'.
+    Returns: (file_path, func_name) or (None, None) if unresolved.
+    """
+    try:
+        tests = selected_tests if selected_tests is not None else _discover_tests(project_root)
+        if not tests:
+            return None, None
+        if len(tests) == 1:
+            return tests[0].file_path, tests[0].qualname.split(".")[-1]
+        if evaluator_id:
+            for t in tests:
+                func_name = t.qualname.split(".")[-1]
+                source_file_name = os.path.splitext(os.path.basename(t.file_path))[0]
+                candidate = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
+                if candidate == evaluator_id:
+                    return t.file_path, func_name
+        return None, None
+    except Exception:
+        return None, None
+
+
 def _poll_evaluator_status(
     evaluator_resource_name: str, api_key: str, api_base: str, timeout_minutes: int = 10
 ) -> bool:
@@ -316,6 +348,9 @@ def create_rft_command(args) -> int:
     non_interactive: bool = bool(getattr(args, "yes", False))
     dry_run: bool = bool(getattr(args, "dry_run", False))
     force: bool = bool(getattr(args, "force", False))
+    # Track the specifically chosen test (if any) to aid dataset inference later
+    selected_test_file_path: Optional[str] = None
+    selected_test_func_name: Optional[str] = None
 
     api_key = get_fireworks_api_key()
     if not api_key:
@@ -348,12 +383,33 @@ def create_rft_command(args) -> int:
             print("No tests selected.")
             return 1
         if len(selected_tests) != 1:
-            print("Error: Please select exactly one evaluation test for 'create rft'.")
+            if non_interactive and len(selected_tests) > 1:
+                print("Error: Multiple evaluation tests found in --yes (non-interactive) mode.")
+                print("       Please pass --evaluator-id or --entry to disambiguate.")
+                try:
+                    # Offer candidate evaluator ids for convenience
+                    tests = _discover_tests(project_root)
+                    if tests:
+                        print("       Candidate evaluator ids:")
+                        for t in tests:
+                            func = t.qualname.split(".")[-1]
+                            stem = os.path.splitext(os.path.basename(t.file_path))[0]
+                            cand = _normalize_evaluator_id(f"{stem}-{func}")
+                            print(f"         - {cand}")
+                except Exception:
+                    pass
+            else:
+                print("Error: Please select exactly one evaluation test for 'create rft'.")
             return 1
+        # Derive evaluator_id from user's single selection
         chosen = selected_tests[0]
         func_name = chosen.qualname.split(".")[-1]
         source_file_name = os.path.splitext(os.path.basename(chosen.file_path))[0]
         evaluator_id = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
+        # Resolve selected test once for downstream
+        selected_test_file_path, selected_test_func_name = _resolve_selected_test(
+            project_root, evaluator_id, selected_tests=selected_tests
+        )
     # Resolve evaluator resource name to fully-qualified format required by API
     evaluator_resource_name = f"accounts/{account_id}/evaluators/{evaluator_id}"
 
@@ -386,6 +442,11 @@ def create_rft_command(args) -> int:
                     print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
                     return 1
                 skip_upload = True
+                # Populate selected test info for dataset inference later
+                st_path, st_func = _resolve_selected_test(project_root, evaluator_id)
+                if st_path and st_func:
+                    selected_test_file_path = st_path
+                    selected_test_func_name = st_func
         except requests.exceptions.RequestException:
             pass
 
@@ -396,28 +457,16 @@ def create_rft_command(args) -> int:
 
             tests = _discover_tests(project_root)
             selected_entry: Optional[str] = None
-            if len(tests) == 1:
-                func_name = tests[0].qualname.split(".")[-1]
-                abs_path = os.path.abspath(tests[0].file_path)
+            st_path, st_func = _resolve_selected_test(project_root, evaluator_id, selected_tests=tests)
+            if st_path and st_func:
+                abs_path = os.path.abspath(st_path)
                 try:
                     rel = os.path.relpath(abs_path, project_root)
                 except Exception:
                     rel = abs_path
-                selected_entry = f"{rel}::{func_name}"
-            else:
-                # Try to match evaluator_id to a discovered test's normalized ID
-                for t in tests:
-                    func_name = t.qualname.split(".")[-1]
-                    source_file_name = os.path.splitext(os.path.basename(t.file_path))[0]
-                    candidate = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
-                    if candidate == evaluator_id:
-                        abs_path = os.path.abspath(t.file_path)
-                        try:
-                            rel = os.path.relpath(abs_path, project_root)
-                        except Exception:
-                            rel = abs_path
-                        selected_entry = f"{rel}::{func_name}"
-                        break
+                selected_entry = f"{rel}::{st_func}"
+                selected_test_file_path = st_path
+                selected_test_func_name = st_func
             # If still unresolved and multiple tests exist, fail fast to avoid uploading unintended evaluators
             if selected_entry is None and len(tests) > 1:
                 print(
@@ -480,30 +529,48 @@ def create_rft_command(args) -> int:
     dataset_builder = getattr(args, "dataset_builder", None)  # accepted but unused in simplified flow
 
     if not dataset_id:
-        # Prefer explicit --dataset-jsonl, else attempt to extract from data loader or input_dataset of the single discovered test
+        # Prefer explicit --dataset-jsonl, else attempt to extract from the selected test's data loader or input_dataset.
         if not dataset_jsonl:
-            tests = _discover_tests(project_root)
-            if len(tests) == 1:
-                func_name = tests[0].qualname.split(".")[-1]
-                # Try data_loaders first (existing behavior)
-                dataset_jsonl = _extract_jsonl_from_dataloader(tests[0].file_path, func_name)
+            # Use specifically selected test if available; else only infer when exactly one test exists
+            test_file_for_infer = None
+            func_for_infer = None
+            if selected_test_file_path and selected_test_func_name:
+                test_file_for_infer = selected_test_file_path
+                func_for_infer = selected_test_func_name
+            else:
+                tests = _discover_tests(project_root)
+                if len(tests) == 1:
+                    test_file_for_infer = tests[0].file_path
+                    func_for_infer = tests[0].qualname.split(".")[-1]
+            if test_file_for_infer and func_for_infer:
+                # Try data_loaders first
+                dataset_jsonl = _extract_jsonl_from_dataloader(test_file_for_infer, func_for_infer)
                 if dataset_jsonl:
-                    # Display relative path for readability
                     try:
                         rel = os.path.relpath(dataset_jsonl, project_root)
                     except Exception:
                         rel = dataset_jsonl
                     print(f"✓ Using JSONL from data loader: {rel}")
-                else:
+                if not dataset_jsonl:
                     # Fall back to input_dataset (dataset_path)
-                    dataset_jsonl = _extract_jsonl_from_input_dataset(tests[0].file_path, func_name)
+                    dataset_jsonl = _extract_jsonl_from_input_dataset(test_file_for_infer, func_for_infer)
                     if dataset_jsonl:
-                        # Display relative path for readability
                         try:
                             rel = os.path.relpath(dataset_jsonl, project_root)
                         except Exception:
                             rel = dataset_jsonl
                         print(f"✓ Using JSONL from input_dataset: {rel}")
+                if not dataset_jsonl:
+                    # Last resort: attempt to detect and run a dataset builder in the test's directory
+                    metric_dir = os.path.dirname(test_file_for_infer)
+                    builder_spec = detect_dataset_builder(metric_dir)
+                    if builder_spec:
+                        try:
+                            tmp_jsonl, count = materialize_dataset_via_builder(builder_spec)
+                            dataset_jsonl = tmp_jsonl
+                            print(f"✓ Materialized {count} rows via dataset builder: {builder_spec}")
+                        except Exception as e:
+                            print(f"Warning: dataset builder failed: {e}")
         if not dataset_jsonl:
             print(
                 "Error: Could not determine dataset. Provide --dataset-id or --dataset-jsonl, or ensure a JSONL-based data loader or input_dataset is used in your single discovered test."

{eval_protocol-0.2.83 → eval_protocol-0.2.84}/eval_protocol/cli_commands/upload.py

@@ -21,7 +21,6 @@ from eval_protocol.auth import (
 from eval_protocol.platform_api import create_or_update_fireworks_secret
 
 from eval_protocol.evaluation import create_evaluation
-from eval_protocol.fireworks_rft import save_evaluator_trace, detect_dataset_builder
 
 
 @dataclass
@@ -444,49 +443,25 @@ def _prompt_select_interactive(tests: list[DiscoveredTest]) -> list[DiscoveredTe
             else:
                 return []
 
-        # Enter-only selection UX with optional multi-select via repeat
-        remaining_indices = list(range(len(tests)))
-        selected_indices: list[int] = []
-
+        # Single-select UX
         print("\n")
-        print("Tip: Use ↑/↓ arrows to navigate and press ENTER to select.")
-        print("     After selecting one, you can choose to add more.\n")
-
-        while remaining_indices:
-            # Build choices from remaining
-            choices = []
-            for idx, test_idx in enumerate(remaining_indices, 1):
-                t = tests[test_idx]
-                choice_text = _format_test_choice(t, idx)
-                choices.append({"name": choice_text, "value": test_idx})
-
-            selected = questionary.select(
-                "Select an evaluation test to upload:", choices=choices, style=custom_style
-            ).ask()
-
-            if selected is None:  # Ctrl+C
-                print("\nUpload cancelled.")
-                return []
+        print("Tip: Use ↑/↓ arrows to navigate and press ENTER to select.\n")
 
-            if isinstance(selected, int):
-                selected_indices.append(selected)
-                # Remove from remaining
-                if selected in remaining_indices:
-                    remaining_indices.remove(selected)
+        choices = []
+        for idx, t in enumerate(tests, 1):
+            choice_text = _format_test_choice(t, idx)
+            choices.append({"name": choice_text, "value": idx - 1})
 
-                # Ask whether to add another (ENTER to finish)
-                add_more = questionary.confirm("Add another?", default=False, style=custom_style).ask()
-                if not add_more:
-                    break
-            else:
-                break
+        selected = questionary.select(
+            "Select an evaluation test to upload:", choices=choices, style=custom_style
+        ).ask()
 
-        if not selected_indices:
-            print("\n⚠️  No tests were selected.")
+        if selected is None:  # Ctrl+C
+            print("\nUpload cancelled.")
             return []
 
-        print(f"\n✓ Selected {len(selected_indices)} test(s)")
-        return [tests[i] for i in selected_indices]
+        print("\n✓ Selected 1 test")
+        return [tests[selected]]
 
     except ImportError:
         # Fallback to simpler implementation
@@ -525,22 +500,19 @@ def _prompt_select_fallback(tests: list[DiscoveredTest]) -> list[DiscoveredTest]
 
     print("=" * 80)
     try:
-        choice = input("Enter numbers to upload (comma or space-separated), or 'all': ").strip()
+        choice = input("Enter the number to upload: ").strip()
     except KeyboardInterrupt:
         print("\n\nUpload cancelled.")
         return []
 
-    if choice.lower() in ("all", "a", "*"):
-        return tests
-
-    indices: list[int] = []
-    for token in re.split(r"[\s,]+", choice):
-        if token.isdigit():
-            n = int(token)
-            if 1 <= n <= len(tests):
-                indices.append(n - 1)
-    indices = sorted(set(indices))
-    return [tests[i] for i in indices]
+    if not choice.isdigit():
+        print("\n⚠️  Invalid selection.")
+        return []
+    n = int(choice)
+    if not (1 <= n <= len(tests)):
+        print("\n⚠️  Selection out of range.")
+        return []
+    return [tests[n - 1]]
 
 
 def _prompt_select(tests: list[DiscoveredTest], non_interactive: bool) -> list[DiscoveredTest]:
@@ -718,23 +690,6 @@ def upload_command(args: argparse.Namespace) -> int:
             )
             name = result.get("name", evaluator_id) if isinstance(result, dict) else evaluator_id
 
-            # Persist local evaluator trace for later `create rft`
-            try:
-                metric_dir = os.path.dirname(source_file_path) if source_file_path else root
-                builder_spec = detect_dataset_builder(metric_dir) or None
-                trace_payload = {
-                    "evaluator_id": evaluator_id,
-                    "evaluator_resource_name": name,
-                    "entry_point": entry_point,
-                    "metric_dir": metric_dir,
-                    "project_root": root,
-                    "dataset_builder": builder_spec,
-                }
-                save_evaluator_trace(project_root=root, evaluator_id=evaluator_id, trace=trace_payload)
-            except Exception:
-                # Non-fatal; continue
-                pass
-
             # Print success message with Fireworks dashboard link
             print(f"\n✅ Successfully uploaded evaluator: {evaluator_id}")
             print("📊 View in Fireworks Dashboard:")

{eval_protocol-0.2.83 → eval_protocol-0.2.84}/eval_protocol/fireworks_rft.py

@@ -37,25 +37,6 @@ def _map_api_host_to_app_host(api_base: str) -> str:
         return "https://app.fireworks.ai"
 
 
-def load_evaluator_trace(project_root: str, evaluator_id: str) -> Optional[Dict[str, Any]]:
-    trace_path = Path(project_root) / ".eval_protocol" / "evaluators" / f"{evaluator_id}.json"
-    if not trace_path.exists():
-        return None
-    try:
-        with open(trace_path, "r", encoding="utf-8") as f:
-            return json.load(f)
-    except Exception:
-        return None
-
-
-def save_evaluator_trace(project_root: str, evaluator_id: str, trace: Dict[str, Any]) -> None:
-    base_dir = Path(project_root) / ".eval_protocol" / "evaluators"
-    base_dir.mkdir(parents=True, exist_ok=True)
-    trace_path = base_dir / f"{evaluator_id}.json"
-    with open(trace_path, "w", encoding="utf-8") as f:
-        json.dump(trace, f, indent=2, ensure_ascii=False)
-
-
 def detect_dataset_builder(metric_dir: str) -> Optional[str]:
     """
     Best-effort scan for a dataset builder callable inside the metric directory.
@@ -228,8 +209,6 @@ def build_default_output_model(evaluator_id: str) -> str:
 
 
 __all__ = [
-    "load_evaluator_trace",
-    "save_evaluator_trace",
     "detect_dataset_builder",
     "materialize_dataset_via_builder",
     "create_dataset_from_jsonl",

{eval_protocol-0.2.83 → eval_protocol-0.2.84/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.83
+Version: 0.2.84
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT