eval-protocol 0.2.82__tar.gz → 0.2.83__tar.gz

{eval_protocol-0.2.82/eval_protocol.egg-info → eval_protocol-0.2.83}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.82
+Version: 0.2.83
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

{eval_protocol-0.2.82 → eval_protocol-0.2.83}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-11-08T19:39:06-0800",
+ "date": "2025-11-09T23:23:12-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "69e53a7d7a70440177cb5545eb23f1b953994da9",
- "version": "0.2.82"
+ "full-revisionid": "a533dcb232528e3910d94adb922c6ab7df27bc4e",
+ "version": "0.2.83"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.82 → eval_protocol-0.2.83}/eval_protocol/cli_commands/create_rft.py

@@ -20,88 +20,7 @@ from ..fireworks_rft import (
     create_dataset_from_jsonl,
     create_reinforcement_fine_tuning_job,
 )
-from .upload import _discover_tests, _normalize_evaluator_id, _resolve_entry_to_qual_and_source
-
-
-def _last_evaluator_paths(cwd: str) -> list[str]:
-    return [
-        os.path.join(cwd, ".eval_protocol", "last_evaluator.json"),
-        os.path.expanduser(os.path.join("~", ".eval_protocol", "last_evaluator.json")),
-    ]
-
-
-def _load_last_evaluator(cwd: str) -> Optional[str]:
-    import json
-
-    for p in _last_evaluator_paths(cwd):
-        try:
-            if os.path.isfile(p):
-                with open(p, "r", encoding="utf-8") as f:
-                    data = json.load(f)
-                if isinstance(data, dict) and data.get("evaluator_id"):
-                    return str(data["evaluator_id"])
-        except Exception:
-            # ignore and continue
-            pass
-    return None
-
-
-def _save_last_evaluator(cwd: str, evaluator_id: str) -> None:
-    import json
-
-    base = os.path.join(cwd, ".eval_protocol")
-    try:
-        os.makedirs(base, exist_ok=True)
-        with open(os.path.join(base, "last_evaluator.json"), "w", encoding="utf-8") as f:
-            json.dump({"evaluator_id": evaluator_id, "ts": time.time()}, f)
-    except Exception:
-        # best-effort only
-        pass
-
-
-def _gather_evaluator_traces(cwd: str) -> list[dict]:
-    roots = [
-        os.path.join(cwd, ".eval_protocol", "evaluators"),
-        os.path.expanduser(os.path.join("~", ".eval_protocol", "evaluators")),
-    ]
-    records: list[dict] = []
-    for root in roots:
-        if os.path.isdir(root):
-            for name in os.listdir(root):
-                if name.endswith(".json"):
-                    full = os.path.join(root, name)
-                    try:
-                        mtime = os.path.getmtime(full)
-                    except Exception:
-                        mtime = 0.0
-                    records.append({"id": name[:-5], "path": full, "mtime": mtime})
-    # dedupe by id keeping most recent mtime
-    dedup: dict[str, dict] = {}
-    for rec in records:
-        cur = dedup.get(rec["id"])
-        if not cur or rec["mtime"] > cur["mtime"]:
-            dedup[rec["id"]] = rec
-    return list(dedup.values())
-
-
-def _prompt_select_evaluator(candidates: list[dict]) -> Optional[str]:
-    print("\nMultiple evaluators detected. Select one:")
-    ordered = sorted(candidates, key=lambda x: -x["mtime"])
-    for i, c in enumerate(ordered, start=1):
-        print(f"  {i}) {c['id']}  (from {c['path']})")
-    try:
-        choice = input("Enter a number (or press Enter to cancel): ").strip()
-    except KeyboardInterrupt:
-        print("\nCancelled.")
-        return None
-    if not choice or not choice.isdigit():
-        return None
-    n = int(choice)
-    if 1 <= n <= len(ordered):
-        sel = ordered[n - 1]["id"]
-        print(f"✓ Using evaluator: {sel}")
-        return sel
-    return None
+from .upload import _discover_tests, _normalize_evaluator_id, _prompt_select
 
 
 def _ensure_account_id() -> Optional[str]:
@@ -331,37 +250,6 @@ def _build_trimmed_dataset_id(evaluator_id: str) -> str:
     return f"{base}{suffix}"
 
 
-def _auto_select_evaluator_id(cwd: str, *, non_interactive: bool = False) -> Optional[str]:
-    # 1) Use last used pointer if available
-    last = _load_last_evaluator(cwd)
-    if last:
-        return last
-
-    # 2) Look for evaluator traces in project and home
-    traces = _gather_evaluator_traces(cwd)
-    if len(traces) == 1:
-        return traces[0]["id"]
-    if len(traces) > 1:
-        if non_interactive:
-            sel = sorted(traces, key=lambda x: -x["mtime"])[0]["id"]
-            print(f"⚠️  Multiple evaluators found; using most recent: {sel}. Override with --evaluator-id.")
-            return sel
-        chosen = _prompt_select_evaluator(traces)
-        if chosen:
-            return chosen
-        return None
-
-    # 3) Fall back to discovering a single evaluation_test
-    tests = _discover_tests(cwd)
-    if len(tests) == 1:
-        qualname, source_file_path = tests[0].qualname, tests[0].file_path
-        test_func_name = qualname.split(".")[-1]
-        source_file_name = os.path.splitext(os.path.basename(source_file_path))[0]
-        evaluator_id = _normalize_evaluator_id(f"{source_file_name}-{test_func_name}")
-        return evaluator_id
-    return None
-
-
 def _poll_evaluator_status(
     evaluator_resource_name: str, api_key: str, api_base: str, timeout_minutes: int = 10
 ) -> bool:
@@ -441,13 +329,31 @@ def create_rft_command(args) -> int:
 
     api_base = get_fireworks_api_base()
 
-    # Resolve evaluator id if omitted
+    # Resolve evaluator id/entry if omitted (reuse upload's selector flow)
     project_root = os.getcwd()
     if not evaluator_id:
-        evaluator_id = _auto_select_evaluator_id(project_root, non_interactive=non_interactive)
-        if not evaluator_id:
-            print("Error: Could not infer evaluator id. Provide --evaluator-id or run 'eval-protocol upload' first.")
+        print("Scanning for evaluation tests...")
+        tests = _discover_tests(project_root)
+        if not tests:
+            print("No evaluation tests found.")
+            print("\nHint: Make sure your tests use the @evaluation_test decorator.")
+            return 1
+        # Always interactive selection here (no implicit quiet unless --evaluator-id was provided)
+        try:
+            selected_tests = _prompt_select(tests, non_interactive=non_interactive)
+        except Exception:
+            print("Error: Failed to open selector UI. Please pass --evaluator-id or --entry explicitly.")
+            return 1
+        if not selected_tests:
+            print("No tests selected.")
+            return 1
+        if len(selected_tests) != 1:
+            print("Error: Please select exactly one evaluation test for 'create rft'.")
             return 1
+        chosen = selected_tests[0]
+        func_name = chosen.qualname.split(".")[-1]
+        source_file_name = os.path.splitext(os.path.basename(chosen.file_path))[0]
+        evaluator_id = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
     # Resolve evaluator resource name to fully-qualified format required by API
     evaluator_resource_name = f"accounts/{account_id}/evaluators/{evaluator_id}"
 
@@ -479,7 +385,6 @@ def create_rft_command(args) -> int:
                     print(f"📊 Please check the evaluator status at: {dashboard_url}")
                     print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
                     return 1
-                _save_last_evaluator(project_root, evaluator_id)
                 skip_upload = True
         except requests.exceptions.RequestException:
             pass
@@ -561,8 +466,8 @@ def create_rft_command(args) -> int:
                     print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
                     return 1
                 else:
-                    # Only persist last-used evaluator after successful ensure + ACTIVE
-                    _save_last_evaluator(project_root, evaluator_id)
+                    # Evaluator ACTIVE; proceed
+                    pass
             else:
                 print("Warning: Evaluator upload did not complete successfully; proceeding to RFT creation.")
         except Exception as e:

{eval_protocol-0.2.82 → eval_protocol-0.2.83/eval_protocol.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.82
+Version: 0.2.83
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT

eval_protocol-0.2.83/tests/test_cli_create_rft_infer.py

@@ -0,0 +1,388 @@
+import json
+import os
+import time
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+
+from eval_protocol.cli_commands import create_rft as cr
+
+
+def _write_json(path: str, data: dict) -> None:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(data, f)
+
+
+def test_create_rft_picks_most_recent_evaluator_and_dataset_id_follows(tmp_path, monkeypatch):
+    # Isolate HOME so expanduser paths remain inside tmp
+    monkeypatch.setenv("HOME", str(tmp_path / "home"))
+
+    # Create a fake project and chdir into it (create_rft uses os.getcwd())
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Create a dummy dataset jsonl file
+    ds_path = project / "evaluator" / "dummy_dataset.jsonl"
+    ds_path.parent.mkdir(parents=True, exist_ok=True)
+    ds_path.write_text('{"input":"x"}\n', encoding="utf-8")
+
+    # Env required by create_rft_command
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Stub out networked/subcommands used by create_rft
+    # Patch selector and upload
+    import eval_protocol.cli_commands.upload as upload_mod
+
+    # Simulate exactly one discovered test and selector returning it
+    one_file = project / "metric" / "test_single.py"
+    one_file.parent.mkdir(parents=True, exist_ok=True)
+    one_file.write_text("# single", encoding="utf-8")
+    single_disc = SimpleNamespace(qualname="metric.test_single", file_path=str(one_file))
+    monkeypatch.setattr(cr, "_discover_tests", lambda cwd: [single_disc])
+    monkeypatch.setattr(upload_mod, "_prompt_select", lambda tests, non_interactive=False: tests[:1])
+    monkeypatch.setattr(upload_mod, "upload_command", lambda args: 0)
+    monkeypatch.setattr(cr, "_poll_evaluator_status", lambda **kwargs: True)
+
+    captured = {"dataset_id": None}
+
+    def _fake_create_dataset_from_jsonl(account_id, api_key, api_base, dataset_id, display_name, jsonl_path):
+        captured["dataset_id"] = dataset_id
+        return dataset_id, {"name": f"accounts/{account_id}/datasets/{dataset_id}", "state": "UPLOADING"}
+
+    monkeypatch.setattr(cr, "create_dataset_from_jsonl", _fake_create_dataset_from_jsonl)
+    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", lambda *a, **k: {"name": "jobs/123"})
+
+    # Build args: non_interactive (yes=True), no explicit evaluator_id, valid warm_start_from
+    args = type("Args", (), {})()
+    setattr(args, "evaluator_id", None)
+    setattr(args, "yes", True)
+    setattr(args, "dry_run", False)
+    setattr(args, "force", False)
+    setattr(args, "env_file", None)
+    setattr(args, "dataset_id", None)
+    setattr(args, "dataset_jsonl", str(ds_path))
+    setattr(args, "dataset_display_name", None)
+    setattr(args, "dataset_builder", None)
+    setattr(args, "base_model", None)
+    setattr(args, "warm_start_from", "accounts/acct123/models/ft-abc123")
+    setattr(args, "output_model", None)
+    setattr(args, "n", None)
+    setattr(args, "max_tokens", None)
+    setattr(args, "learning_rate", None)
+    setattr(args, "batch_size", None)
+    setattr(args, "epochs", None)
+    setattr(args, "lora_rank", None)
+    setattr(args, "max_context_length", None)
+    setattr(args, "chunk_size", None)
+    setattr(args, "eval_auto_carveout", None)
+
+    rc = cr.create_rft_command(args)
+    assert rc == 0
+
+    # Assert dataset id derived from selected test: metric-test_single
+    assert captured["dataset_id"] is not None
+    assert captured["dataset_id"].startswith("test-single-test-single-dataset-")
+
+
+def test_create_rft_passes_matching_evaluator_id_and_entry_with_multiple_tests(tmp_path, monkeypatch):
+    # Ensure expanduser paths stay under tmp
+    monkeypatch.setenv("HOME", str(tmp_path / "home"))
+
+    # Project structure and CWD
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Create dummy test files for discovery
+    eval_dir = project / "evaluator"
+    eval_dir.mkdir(parents=True, exist_ok=True)
+    cal_file = eval_dir / "foo_eval.py"
+    svg_file = eval_dir / "bar_eval.py"
+    cal_file.write_text("# foo", encoding="utf-8")
+    svg_file.write_text("# bar", encoding="utf-8")
+
+    # Fake discovered tests: foo and bar
+    cal_disc = SimpleNamespace(qualname="foo_eval.test_bar_evaluation", file_path=str(cal_file))
+    svg_disc = SimpleNamespace(qualname="bar_eval.test_baz_evaluation", file_path=str(svg_file))
+    monkeypatch.setattr(cr, "_discover_tests", lambda cwd: [cal_disc, svg_disc])
+
+    # Env for CLI
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Capture what upload receives (id and entry)
+    captured = {"id": None, "entry": None, "dataset_id": None}
+
+    # Monkeypatch the upload command from the upload module (the function imports it inside)
+    import eval_protocol.cli_commands.upload as upload_mod
+
+    def _fake_upload(ns):
+        captured["id"] = getattr(ns, "id", None)
+        captured["entry"] = getattr(ns, "entry", None)
+        return 0
+
+    monkeypatch.setattr(upload_mod, "upload_command", _fake_upload)
+
+    # Avoid network and capture dataset id
+    monkeypatch.setattr(cr, "_poll_evaluator_status", lambda **kwargs: True)
+
+    def _fake_create_dataset_from_jsonl(account_id, api_key, api_base, dataset_id, display_name, jsonl_path):
+        captured["dataset_id"] = dataset_id
+        return dataset_id, {"name": f"accounts/{account_id}/datasets/{dataset_id}", "state": "UPLOADING"}
+
+    monkeypatch.setattr(cr, "create_dataset_from_jsonl", _fake_create_dataset_from_jsonl)
+    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", lambda *a, **k: {"name": "jobs/123"})
+
+    # Provide a dataset jsonl so flow proceeds
+    ds_path = eval_dir / "dummy_dataset.jsonl"
+    ds_path.write_text('{"input":"x"}\n', encoding="utf-8")
+
+    # Build args: no explicit evaluator id, selector will not be used here; mapping by id
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id=cr._normalize_evaluator_id("foo_eval-test_bar_evaluation"),
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(ds_path),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+
+    rc = cr.create_rft_command(args)
+    assert rc == 0
+
+    # Assert evaluator_id passed to upload matches the provided id
+    assert captured["id"] == cr._normalize_evaluator_id("foo_eval-test_bar_evaluation")
+    # Assert entry points to the foo test (should map when id matches normalization)
+    assert captured["entry"] is not None and captured["entry"].endswith("foo_eval.py::test_bar_evaluation")
+    # Assert dataset id is derived from the same evaluator id (trimmed base + '-dataset-<timestamp>')
+    assert captured["dataset_id"] is not None
+    expected_prefix = (
+        cr._build_trimmed_dataset_id(cr._normalize_evaluator_id("foo_eval-test_bar_evaluation")).split("-dataset-")[0]
+        + "-dataset-"
+    )
+    assert captured["dataset_id"].startswith(expected_prefix)
+
+
+def test_create_rft_interactive_selector_single_test(tmp_path, monkeypatch):
+    # Setup project
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Single discovered test
+    test_file = project / "metric" / "test_one.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("# one", encoding="utf-8")
+    single_disc = SimpleNamespace(qualname="metric.test_one", file_path=str(test_file))
+    monkeypatch.setattr(cr, "_discover_tests", lambda cwd: [single_disc])
+
+    # Environment
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Stub selector to return the single test; stub upload and polling
+    import eval_protocol.cli_commands.upload as upload_mod
+
+    monkeypatch.setattr(upload_mod, "_prompt_select", lambda tests, non_interactive=False: tests[:1])
+    captured = {"id": None, "entry": None, "dataset_id": None}
+
+    def _fake_upload(ns):
+        captured["id"] = getattr(ns, "id", None)
+        captured["entry"] = getattr(ns, "entry", None)
+        return 0
+
+    monkeypatch.setattr(upload_mod, "upload_command", _fake_upload)
+    monkeypatch.setattr(cr, "_poll_evaluator_status", lambda **kwargs: True)
+
+    # Provide dataset jsonl
+    ds_path = project / "metric" / "dataset.jsonl"
+    ds_path.write_text('{"input":"x"}\n', encoding="utf-8")
+    monkeypatch.setattr(
+        cr,
+        "create_dataset_from_jsonl",
+        lambda account_id, api_key, api_base, dataset_id, display_name, jsonl_path: (
+            dataset_id,
+            {"name": f"accounts/{account_id}/datasets/{dataset_id}"},
+        ),
+    )
+    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", lambda *a, **k: {"name": "jobs/123"})
+
+    # Run without evaluator_id; use --yes so selector returns tests directly (no UI)
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id=None,
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(ds_path),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+
+    rc = cr.create_rft_command(args)
+    assert rc == 0
+    assert captured["id"] is not None
+    assert captured["entry"] is not None and captured["entry"].endswith("test_one.py::test_one")
+
+
+def test_create_rft_quiet_existing_evaluator_skips_upload(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Env
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Mock evaluator exists and is ACTIVE
+    class _Resp:
+        ok = True
+
+        def json(self):
+            return {"state": "ACTIVE"}
+
+        def raise_for_status(self):
+            return None
+
+    monkeypatch.setattr(cr.requests, "get", lambda *a, **k: _Resp())
+
+    # Provide dataset via --dataset-jsonl so no test discovery needed
+    ds_path = project / "dataset.jsonl"
+    ds_path.write_text('{"input":"x"}\n', encoding="utf-8")
+    monkeypatch.setattr(
+        cr,
+        "create_dataset_from_jsonl",
+        lambda account_id, api_key, api_base, dataset_id, display_name, jsonl_path: (
+            dataset_id,
+            {"name": f"accounts/{account_id}/datasets/{dataset_id}"},
+        ),
+    )
+    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", lambda *a, **k: {"name": "jobs/123"})
+
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id="some-eval",
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(ds_path),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+
+    rc = cr.create_rft_command(args)
+    assert rc == 0
+
+
+def test_create_rft_quiet_new_evaluator_ambiguous_without_entry_errors(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Env
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Evaluator does not exist (force path into upload section)
+    def _raise(*a, **k):
+        raise requests.exceptions.RequestException("nope")
+
+    import requests
+
+    monkeypatch.setattr(cr.requests, "get", _raise)
+
+    # Two discovered tests (ambiguous)
+    f1 = project / "a.py"
+    f2 = project / "b.py"
+    f1.write_text("# a", encoding="utf-8")
+    f2.write_text("# b", encoding="utf-8")
+    d1 = SimpleNamespace(qualname="a.test_one", file_path=str(f1))
+    d2 = SimpleNamespace(qualname="b.test_two", file_path=str(f2))
+    monkeypatch.setattr(cr, "_discover_tests", lambda cwd: [d1, d2])
+
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id="some-eval",
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(project / "dataset.jsonl"),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+    # create the dataset file so we don't fail earlier
+    (project / "dataset.jsonl").write_text('{"input":"x"}\n', encoding="utf-8")
+
+    rc = cr.create_rft_command(args)
+    assert rc == 1