runapprentice 0.4.2__tar.gz → 0.4.3__tar.gz

{runapprentice-0.4.2 → runapprentice-0.4.3}/.gitignore

@@ -52,3 +52,4 @@ docs/personal_learning.md
 .uv-cache/
 spike/Capture-*.png
 apprentice-ui/ClaudeDesign/screenshots/localhost-*.png
+supabase-magic-link-template.html

{runapprentice-0.4.2 → runapprentice-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runapprentice
-Version: 0.4.2
+Version: 0.4.3
 Summary: Cut LLM cost without losing quality: capture traffic, build golden datasets, and optimize prompts.
 Project-URL: Homepage, https://runapprentice.com
 Project-URL: Documentation, https://docs.runapprentice.com

{runapprentice-0.4.2 → runapprentice-0.4.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runapprentice"
-version = "0.4.2"
+version = "0.4.3"
 description = "Cut LLM cost without losing quality: capture traffic, build golden datasets, and optimize prompts."
 readme = "README.md"
 requires-python = ">=3.11"

{runapprentice-0.4.2 → runapprentice-0.4.3}/src/runapprentice/cli.py

@@ -112,7 +112,10 @@ def build_parser() -> argparse.ArgumentParser:
     optimize.add_argument(
         "--local",
         action="store_true",
-        help="run GEPA on this machine with your own OPENAI_API_KEY, no Apprentice account needed",
+        help=(
+            "run GEPA on this machine with your own OPENAI_API_KEY, no Apprentice account "
+            "needed; scores JSON extraction only, so every output must be a JSON object or array"
+        ),
     )
     optimize.add_argument(
         "--data",

{runapprentice-0.4.2 → runapprentice-0.4.3}/src/runapprentice/client.py

@@ -156,7 +156,9 @@ class Apprentice:
             ) from exc
         logger.debug("%s %s -> %s (%.0fms)", method, shown_path, resp.status_code, _ms(t0))
         if resp.status_code >= 400:
-            logger.error("%s %s -> %s: %s", method, shown_path, resp.status_code, resp.text[:500])
+            logger.error(
+                "%s %s -> %s error=http_response", method, shown_path, resp.status_code
+            )
             # The API sends a written reason in `detail`; lead with it so the first
             # thing a reader sees is why the call failed. The route cannot tell us
             # that: a 404 on /v1/prompts/{task} means the task has no optimized
@@ -430,8 +432,8 @@ class Apprentice:
             self._capture_warned = False
             return record.trace_id
         if resp.status_code >= 500:
-            raise ApprenticeError(f"backend {resp.status_code}: {resp.text[:200]}")
-        self._warn_capture_once(f"backend rejected trace: {resp.status_code} {resp.text[:200]}")
+            raise ApprenticeError("backend 5xx for /v1/traces")
+        self._warn_capture_once(f"backend rejected trace: {resp.status_code} for /v1/traces")
         return None
 
     def _warn_capture_once(self, message: str) -> None:

{runapprentice-0.4.2 → runapprentice-0.4.3}/src/runapprentice/local.py

@@ -27,8 +27,10 @@ from runapprentice.client import ApprenticeError
 from runapprentice.models import DatasetRow
 
 LOCAL_MODEL_ID = "mlx-community/Qwen3.5-4B-OptiQ-4bit"
-DEFAULT_STUDENT_MODEL = "openai/gpt-5-mini"
-DEFAULT_REFLECTION_MODEL = "openai/gpt-5.4"
+# Mirrors the API defaults (owner decision 2026-07-31). The exact id matters: the bare
+# `gpt-5.6` alias routes to Sol, not Luna.
+DEFAULT_STUDENT_MODEL = "openai/gpt-5.6-luna"
+DEFAULT_REFLECTION_MODEL = "openai/gpt-5.6-luna"
 
 # Every tier trains the SAME model quality: 16 LoRA layers (mlx-lm's default;
 # fewer layers reduces final quality per LORA.md) and the same number of passes
@@ -117,18 +119,64 @@ def score_json_field_f1(row: DatasetRow, actual: str) -> float:
         actual_obj = json.loads(_extract_json_text(actual))
     except json.JSONDecodeError:
         return 0.0
-    if not isinstance(expected_obj, dict) or not isinstance(actual_obj, dict):
+    # Objects AND arrays, matching the API scorer: a task whose gold answers are top-level
+    # arrays scored 0 here for a correct answer.
+    if not isinstance(expected_obj, (dict, list)) or not isinstance(actual_obj, (dict, list)):
         return 0.0
     expected = _json_leaf_pairs(expected_obj)
     actual_pairs = _json_leaf_pairs(actual_obj)
+    # Mirrors the hosted ceiling. Without it the same answer scores 1.0 here and 0.0 hosted,
+    # so a local run and a hosted run disagree about quality on identical data.
+    if len(expected) > _MAX_JSON_METRIC_LEAVES or len(actual_pairs) > _MAX_JSON_METRIC_LEAVES:
+        return 0.0
     f1 = _leaf_f1(expected, actual_pairs)
-    if len(expected_obj) == 1:
+    if isinstance(expected_obj, dict) and isinstance(actual_obj, dict) and len(expected_obj) == 1:
         root_key, root_value = next(iter(expected_obj.items()))
         if isinstance(root_value, dict) and root_key not in actual_obj:
             f1 = max(f1, _leaf_f1(_json_leaf_pairs(root_value), actual_pairs))
     return f1
 
 
+def _reject_unscoreable_rows(rows: Sequence[DatasetRow]) -> None:
+    """Stop before spending if the scorer cannot read this data.
+
+    Local optimize scores with `score_json_field_f1` and nothing else -- the CLI refuses
+    `--metric` in local mode. That scorer returns 0.0 for an answer it cannot parse as a JSON
+    object or array, so a dataset of plain labels ("yes"/"no") used to run to completion on the
+    user's own OpenAI key and report 0.0 to 0.0: money and minutes spent measuring nothing.
+    The same shape of bug reached production hosted-side on 2026-08-02.
+
+    The test mirrors the scorer's own accept rule rather than restating it, so the two cannot
+    drift apart.
+    """
+    bad = [
+        index
+        for index, row in enumerate(rows, start=1)
+        if not _is_json_container(row.output)
+    ]
+    if not bad:
+        return
+    shown = ", ".join(str(index) for index in bad[:5])
+    more = f" (and {len(bad) - 5} more)" if len(bad) > 5 else ""
+    # One action, and it is true on every path that reaches this message. An earlier draft
+    # also offered "run without --local", which is false advice here: the hosted branch of
+    # `_optimize` never reads --data, so dropping the flag silently ignores the CSV and
+    # optimizes whatever rows already exist server-side. The CLI has no upload command, so
+    # there is no hosted path for THIS file to name.
+    raise ApprenticeError(
+        f"{len(bad)} of {len(rows)} rows have an output that is not a JSON object or array, "
+        f"starting at row {shown}{more}. Local optimize scores JSON extraction only, so these "
+        "rows would all score zero. Give each row's output column a JSON object or array."
+    )
+
+
+def _is_json_container(output: str) -> bool:
+    try:
+        return isinstance(json.loads(output), (dict, list))
+    except json.JSONDecodeError:
+        return False
+
+
 def optimize_prompt_local(
     rows: Sequence[DatasetRow],
     *,
@@ -141,6 +189,7 @@ def optimize_prompt_local(
         raise ApprenticeError("OPENAI_API_KEY is required for local prompt optimization")
     if len(rows) < 2:
         raise ApprenticeError("local optimize needs at least two rows for train and held-out scoring")
+    _reject_unscoreable_rows(rows)
     dspy = _load_dspy()
     student_model = student_model or os.environ.get("APPRENTICE_STUDENT_MODEL", DEFAULT_STUDENT_MODEL)
     reflection_model = reflection_model or os.environ.get(
@@ -525,51 +574,75 @@ def _generate(row: DatasetRow, adapter_dir: Path, model_id: str) -> str:
 
 
 def _extract_json_text(text: str) -> str:
+    # Objects AND arrays, mirroring the API. Scanning for "{" alone returned the first inner
+    # OBJECT of a wrapped array, so an array answer was compared as an object and scored 0.
     candidate = text.strip()
-    start = candidate.find("{")
-    while start != -1:
-        in_string = False
-        escaped = False
-        depth = 0
-        for i, char in enumerate(candidate[start:], start=start):
-            if in_string:
-                if escaped:
-                    escaped = False
-                elif char == "\\":
-                    escaped = True
-                elif char == '"':
-                    in_string = False
-                continue
-            if char == '"':
-                in_string = True
-            elif char == "{":
-                depth += 1
-            elif char == "}":
-                depth -= 1
-                if depth == 0:
-                    return candidate[start : i + 1]
-        start = candidate.find("{", start + 1)
+    openers = [(candidate.find(o), o, c) for o, c in (("{", "}"), ("[", "]")) if candidate.find(o) != -1]
+    for _, opener, closer in sorted(openers):
+        start = candidate.find(opener)
+        while start != -1:
+            in_string = False
+            escaped = False
+            depth = 0
+            for i, char in enumerate(candidate[start:], start=start):
+                if in_string:
+                    if escaped:
+                        escaped = False
+                    elif char == "\\":
+                        escaped = True
+                    elif char == '"':
+                        in_string = False
+                    continue
+                if char == '"':
+                    in_string = True
+                elif char == opener:
+                    depth += 1
+                elif char == closer:
+                    depth -= 1
+                    if depth == 0:
+                        span = candidate[start : i + 1]
+                        try:
+                            json.loads(span)
+                            return span
+                        except json.JSONDecodeError:
+                            break
+            start = candidate.find(opener, start + 1)
     return candidate
 
 
-def _json_leaf_pairs(value: Any, path: str = "") -> set[tuple[str, str]]:
-    leaves: set[tuple[str, str]] = set()
+# Leaf kinds, mirroring the API. The kind is a separate field so an empty container can
+# never be confused with a scalar that spells one.
+# Mirrors apprentice-api optimizer._MAX_JSON_METRIC_LEAVES.
+_MAX_JSON_METRIC_LEAVES = 2000
+_LEAF_SCALAR = "scalar"
+_LEAF_EMPTY_OBJECT = "empty-object"
+_LEAF_EMPTY_ARRAY = "empty-array"
+
+
+def _json_leaf_pairs(value: Any, path: str = "") -> set[tuple[str, str, str]]:
+    leaves: set[tuple[str, str, str]] = set()
 
     def walk(node: Any, current_path: str) -> None:
         if isinstance(node, dict):
+            if not node:
+                leaves.add((current_path or "$", _LEAF_EMPTY_OBJECT, ""))
+                return
             for key in sorted(node):
                 walk(node[key], f"{current_path}.{key}" if current_path else str(key))
         elif isinstance(node, list):
+            if not node:
+                leaves.add((current_path or "$", _LEAF_EMPTY_ARRAY, ""))
+                return
             for i, item in enumerate(node):
                 walk(item, f"{current_path}[{i}]")
         else:
-            leaves.add((current_path or "$", _normalize_scalar(node)))
+            leaves.add((current_path or "$", _LEAF_SCALAR, _normalize_scalar(node)))
 
     walk(value, path)
     return leaves
 
 
-def _leaf_f1(expected: set[tuple[str, str]], actual: set[tuple[str, str]]) -> float:
+def _leaf_f1(expected: set[tuple[str, str, str]], actual: set[tuple[str, str, str]]) -> float:
     tp = len(expected & actual)
     precision = tp / len(actual) if actual else 0.0
     recall = tp / len(expected) if expected else 0.0

{runapprentice-0.4.2 → runapprentice-0.4.3}/src/runapprentice/models.py

@@ -64,6 +64,9 @@ class HoldoutEvidenceRow(BaseModel):
 class OptimizationReport(BaseModel):
     task: str | None = None
     baseline_score: float | None = None
+    # Set only when the baseline scored 0 on every held-out row, derived from the outputs it
+    # produced: output_not_structured | empty_output | evaluation_error | values_differ.
+    baseline_zero_reason: str | None = None
     optimized_score: float | None = None
     examples_used: int | None = None
     baseline_prompt: str | None = None
@@ -85,7 +88,16 @@ class OptimizationReport(BaseModel):
     judge_cache: dict[str, int] | None = None
     score_explanation: str | None = None
     worst_examples: list[dict[str, Any]] | None = None
+    # Deprecated, None on runs from 2026-08 onward: it counted rows scoring under 1.0, which
+    # on a continuous metric makes 0.97 a "failure". Older reports keep whatever they stored.
+    # Remove at 1.0.
     failure_count: int | None = None
+    label_distribution: dict[str, int] | None = None
+    constant_baseline: dict[str, str | float] | None = None
+    constant_baseline_reason: str | None = None
+    per_class_result: list[dict[str, str | int]] | None = None
+    holdout_tier_counts: dict[str, int] | None = None
+    evaluation_errors: int | None = None
     holdout_evidence: list[HoldoutEvidenceRow] | None = None
     improved_count: int | None = None
     regressed_count: int | None = None
@@ -149,6 +161,10 @@ class JobInfo(BaseModel):
     error: str | None = None
     report: SkipValidation[JobReport] | None = None
     progress: dict[str, Any] | None = None
+    # Deprecated, always None: the API stopped reporting a queue rank because every rank it
+    # could compute was false (it ignored run_after and the worker's own --queues subset).
+    # Kept so `job.queue_position` still reads as "no rank" for anyone on 0.4.2 or earlier
+    # instead of raising AttributeError. Remove at 1.0.
     queue_position: int | None = None
     baseline_score: float | None = None
     optimized_score: float | None = None

{runapprentice-0.4.2 → runapprentice-0.4.3}/tests/test_cli.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import json
 import os
 import stat
+import sys
 from io import StringIO
 from pathlib import Path
 from typing import Any, Callable
@@ -333,7 +334,7 @@ def test_login_writes_config_0600(
     config_path = config_dir / "config.json"
     monkeypatch.setattr(cli, "CONFIG_DIR", config_dir)
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys.stdin, "isatty", lambda: True)
+    monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
     monkeypatch.setattr("runapprentice.cli.getpass.getpass", lambda _: "secret-key")
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(200, json=[]))
 
@@ -351,7 +352,7 @@ def test_login_rejects_invalid_key_without_writing(
 ) -> None:
     config_path = tmp_path / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("invalid-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("invalid-key\n"))
     _mock_login_api(
         monkeypatch,
         lambda _request: httpx.Response(401, json={"detail": "invalid API key"}),
@@ -369,7 +370,7 @@ def test_failed_login_does_not_clobber_existing_config(
     original = '{"api_key": "working-key"}\n'
     config_path.write_text(original, encoding="utf-8")
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("invalid-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("invalid-key\n"))
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(403, json={"detail": "forbidden"}))
 
     assert cli.main(["login"]) == 1
@@ -380,7 +381,7 @@ def test_login_reports_unreachable_api_differently_from_rejected_key(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
 ) -> None:
     monkeypatch.setattr(cli, "CONFIG_PATH", tmp_path / "config.json")
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("candidate-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("candidate-key\n"))
 
     def unreachable(request: httpx.Request) -> httpx.Response:
         raise httpx.ConnectError("connection refused", request=request)
@@ -398,7 +399,7 @@ def test_login_reports_server_error_as_unavailable(
 ) -> None:
     config_path = tmp_path / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("candidate-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("candidate-key\n"))
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(503))
 
     assert cli.main(["login"]) == 1
@@ -425,7 +426,7 @@ def test_login_reads_piped_stdin_and_validates_it(
 ) -> None:
     config_path = tmp_path / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("  piped-key  \n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("  piped-key  \n"))
 
     def validate(request: httpx.Request) -> httpx.Response:
         assert request.url.path == "/v1/tasks"
@@ -547,7 +548,7 @@ def test_login_rejects_a_key_containing_whitespace(
     """
     config_path = tmp_path / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("good-key\nJUNK-SECOND-LINE\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("good-key\nJUNK-SECOND-LINE\n"))
 
     def explode(_request: httpx.Request) -> httpx.Response:
         raise AssertionError("a malformed key must never reach the network")
@@ -569,7 +570,7 @@ def test_login_directory_is_not_world_writable_under_a_permissive_umask(
     """
     config_path = tmp_path / "nested" / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("good-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("good-key\n"))
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(200, json=[]))
 
     previous = os.umask(0)
@@ -609,7 +610,7 @@ def test_login_rejects_a_key_that_is_not_printable_ascii(
     """
     config_path = tmp_path / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO(key + "\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO(key + "\n"))
 
     def explode(_request: httpx.Request) -> httpx.Response:
         raise AssertionError("a malformed key must never reach the network")
@@ -632,7 +633,7 @@ def test_login_never_leaves_the_key_world_readable(
     """
     config_path = tmp_path / "nested" / "config.json"
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("good-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("good-key\n"))
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(200, json=[]))
 
     seen_modes: list[int] = []
@@ -643,7 +644,7 @@ def test_login_never_leaves_the_key_world_readable(
         seen_modes.append(stat.S_IMODE(os.stat(src).st_mode))
         real_replace(src, dst)
 
-    monkeypatch.setattr(cli.os, "replace", spy)
+    monkeypatch.setattr(os, "replace", spy)
 
     previous = os.umask(0)
     try:
@@ -663,7 +664,7 @@ def test_login_replaces_a_symlinked_config_instead_of_its_target(
     config_path = tmp_path / "config.json"
     config_path.symlink_to(victim)
     monkeypatch.setattr(cli, "CONFIG_PATH", config_path)
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("good-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("good-key\n"))
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(200, json=[]))
 
     assert cli.main(["login"]) == 0
@@ -683,7 +684,7 @@ def test_login_reports_a_filesystem_failure_instead_of_a_traceback(
     """
     # A path whose parent cannot exist: /dev/null is a file, not a directory.
     monkeypatch.setattr(cli, "CONFIG_PATH", Path("/dev/null/nope/config.json"))
-    monkeypatch.setattr(cli.sys, "stdin", StringIO("good-key\n"))
+    monkeypatch.setattr(sys, "stdin", StringIO("good-key\n"))
     _mock_login_api(monkeypatch, lambda _request: httpx.Response(200, json=[]))
 
     assert cli.main(["--json", "login"]) == 1

{runapprentice-0.4.2 → runapprentice-0.4.3}/tests/test_client.py

@@ -252,10 +252,20 @@ def test_job_error_is_reachable() -> None:
     assert job.error == "training failed"
 
 
+def test_queue_position_still_reads_as_absent_after_the_api_stopped_sending_it() -> None:
+    """0.4.2 published this attribute; dropping it would raise AttributeError on upgrade."""
+    info = JobInfo.model_validate({"job_id": "j1", "task": "t1", "kind": "optimize", "status": "queued"})
+
+    assert info.queue_position is None
+
+
 def test_optimization_report_round_trips_every_api_field() -> None:
     payload = {
         "task": "extract",
         "baseline_score": 41.0,
+        # Set only on a run whose baseline scored zero everywhere; carried here so this
+        # round-trip keeps covering every field the API can send.
+        "baseline_zero_reason": None,
         "optimized_score": 88.0,
         "examples_used": 20,
         "baseline_prompt": "Extract fields.",
@@ -290,6 +300,15 @@ def test_optimization_report_round_trips_every_api_field() -> None:
             {"input": "invoice", "expected": "{}", "actual": "{}", "score": 1.0}
         ],
         "failure_count": 1,
+        "label_distribution": {"false": 12, "true": 4},
+        "constant_baseline": {"value": "false", "score": 75.0},
+        "constant_baseline_reason": None,
+        "per_class_result": [
+            {"label": "false", "correct": 12, "total": 12},
+            {"label": "true", "correct": 0, "total": 4},
+        ],
+        "holdout_tier_counts": {"gold": 3, "silver": 13},
+        "evaluation_errors": 0,
         "holdout_evidence": [
             {
                 "row_id": "row-1",
@@ -632,6 +651,42 @@ def test_error_leads_with_the_reason_the_api_gave() -> None:
     )
 
 
+def test_http_error_log_omits_response_body(caplog: pytest.LogCaptureFixture) -> None:
+    secret = "customer SSN 123-45-6789"
+    client = Apprentice(api_key="k", base_url="http://mock", capture_async=False)
+    client._http = httpx.Client(
+        base_url="http://mock",
+        transport=httpx.MockTransport(
+            lambda _request: httpx.Response(422, json={"detail": secret})
+        ),
+    )
+
+    with caplog.at_level(logging.ERROR, logger="runapprentice"):
+        with pytest.raises(ApprenticeError):
+            client._request("POST", "/v1/datasets/t1/rows", json={"input": secret})
+
+    assert "/v1/datasets/t1/rows" in caplog.text
+    assert "422" in caplog.text
+    assert secret not in caplog.text
+
+
+def test_capture_rejection_log_omits_response_body(caplog: pytest.LogCaptureFixture) -> None:
+    secret = "customer prompt in rejected body"
+    client = Apprentice(api_key="k", base_url="http://mock", capture_async=False)
+    client._http = httpx.Client(
+        base_url="http://mock",
+        transport=httpx.MockTransport(
+            lambda _request: httpx.Response(422, json={"detail": secret})
+        ),
+    )
+
+    with caplog.at_level(logging.WARNING, logger="runapprentice"):
+        assert client.capture("t1", input="i", output="o") is None
+
+    assert "backend rejected trace: 422 for /v1/traces" in caplog.text
+    assert secret not in caplog.text
+
+
 @pytest.mark.parametrize(
     "body",
     [["upstream failed"], "plain string", 42, None],
@@ -965,3 +1020,71 @@ def test_a_hostile_server_message_cannot_repaint_the_terminal() -> None:
     message = str(exc_info.value)
     assert "\x1b" not in message and "\x07" not in message and "\r" not in message
     assert "FAKE: login succeeded" in message  # kept, but inert
+
+
+def test_local_scorer_matches_the_hosted_one_on_arrays_and_empty_containers() -> None:
+    """local.py names the API scorer as its source of truth, so it has to keep up.
+
+    It rejected top-level arrays and ignored empty containers after the API stopped doing
+    both, so the same answer scored differently depending on where it ran.
+    """
+    from runapprentice.local import score_json_field_f1
+    from runapprentice.models import DatasetRow
+
+    def score(expected: str, actual: str) -> float:
+        return round(score_json_field_f1(DatasetRow(input="x", output=expected), actual), 3)
+
+    assert score("[1]", "[1]") == 1.0
+    assert score('[{"a": 1}]', '[{"a": 1}]') == 1.0
+    assert score('{"a": {}}', '{"a": {}}') == 1.0
+    # A missing empty field is counted, not ignored.
+    assert score('{"a": 1, "i": []}', '{"a": 1}') < 1.0
+    # The leaf kind is not forgeable by a string that spells it.
+    assert score('{"a": {}}', '{"a": "{}"}') == 0.0
+    # Unchanged behaviour.
+    assert score('{"a": 1, "b": 2}', '{"a": 1, "b": 2}') == 1.0
+    assert score('{"a": 1, "b": 2}', '{"a": 1, "b": 9}') == 0.5
+    assert score('{"root": {"a": 1}}', '{"a": 1}') == 1.0
+
+
+def test_local_optimize_refuses_data_its_scorer_cannot_read(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Non-JSON rows must fail BEFORE any model call, not score 0.0 after paying.
+
+    Local optimize scores with score_json_field_f1 and the CLI refuses --metric in local
+    mode, so a dataset of plain labels used to run to completion on the user's own OpenAI
+    key and report 0.0 to 0.0. The guard has to fire before the key check does any work.
+    """
+    from runapprentice import local
+    from runapprentice.client import ApprenticeError
+    from runapprentice.models import DatasetRow
+
+    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+    # Would blow up loudly if the guard let execution through to the optimizer.
+    monkeypatch.setattr(local, "_load_dspy", lambda: pytest.fail("spent before validating"))
+
+    labels = [DatasetRow(input=f"email {i}", output="yes" if i % 2 else "no") for i in range(6)]
+    with pytest.raises(ApprenticeError) as caught:
+        local.optimize_prompt_local(labels)
+
+    message = str(caught.value)
+    assert "6 of 6 rows" in message, "the count tells the user how much of their data is wrong"
+    assert "not a JSON object or array" in message
+    assert "output column a JSON object or array" in message, "one action, and a true one"
+    # No hosted suggestion: the hosted branch never reads --data, so "run without --local"
+    # would silently ignore this CSV.
+    assert "--local" not in message
+
+    # A scalar is still not a container, and neither is a bare JSON string.
+    with pytest.raises(ApprenticeError):
+        local.optimize_prompt_local([DatasetRow(input="a", output="42")] * 2)
+    with pytest.raises(ApprenticeError):
+        local.optimize_prompt_local([DatasetRow(input="a", output='"yes"')] * 2)
+
+    # Objects and arrays both pass the guard: it must not narrow what the scorer accepts.
+    mixed = [
+        DatasetRow(input="a", output='{"label": "yes"}'),
+        DatasetRow(input="b", output='[{"label": "no"}]'),
+    ]
+    local._reject_unscoreable_rows(mixed)