runapprentice 0.4.1__tar.gz → 0.4.3__tar.gz

{runapprentice-0.4.1 → runapprentice-0.4.3}/.gitignore

@@ -52,3 +52,4 @@ docs/personal_learning.md
 .uv-cache/
 spike/Capture-*.png
 apprentice-ui/ClaudeDesign/screenshots/localhost-*.png
+supabase-magic-link-template.html

{runapprentice-0.4.1 → runapprentice-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runapprentice
-Version: 0.4.1
+Version: 0.4.3
 Summary: Cut LLM cost without losing quality: capture traffic, build golden datasets, and optimize prompts.
 Project-URL: Homepage, https://runapprentice.com
 Project-URL: Documentation, https://docs.runapprentice.com
@@ -50,10 +50,10 @@ apprentice optimize <task> --local --data golden.csv              # free, your O
 apprentice train <task> --local --data golden.csv --effort high   # free, fine-tunes on your Mac
 ```
 
-Both commands work on a CSV alone, no Apprentice account. No golden dataset yet? `apprentice
-login` also lets Apprentice generate a starter set and route rows to a subject-matter expert
-for review, and tracks every run in the console. `apprentice --help` (or any
-subcommand `--help`) documents every flag; `--json` gives clean machine-readable output.
+Both `--local --data` commands work end to end from a CSV. Hosted setup requires the console
+or Python SDK to create a task and upload its dataset; the CLI has no create or upload command.
+`apprentice --help` (or any subcommand `--help`) documents every flag; `--json` gives clean
+machine-readable output.
 
 ## Install
 

{runapprentice-0.4.1 → runapprentice-0.4.3}/README.md

@@ -21,10 +21,10 @@ apprentice optimize <task> --local --data golden.csv              # free, your O
 apprentice train <task> --local --data golden.csv --effort high   # free, fine-tunes on your Mac
 ```
 
-Both commands work on a CSV alone, no Apprentice account. No golden dataset yet? `apprentice
-login` also lets Apprentice generate a starter set and route rows to a subject-matter expert
-for review, and tracks every run in the console. `apprentice --help` (or any
-subcommand `--help`) documents every flag; `--json` gives clean machine-readable output.
+Both `--local --data` commands work end to end from a CSV. Hosted setup requires the console
+or Python SDK to create a task and upload its dataset; the CLI has no create or upload command.
+`apprentice --help` (or any subcommand `--help`) documents every flag; `--json` gives clean
+machine-readable output.
 
 ## Install
 

{runapprentice-0.4.1 → runapprentice-0.4.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runapprentice"
-version = "0.4.1"
+version = "0.4.3"
 description = "Cut LLM cost without losing quality: capture traffic, build golden datasets, and optimize prompts."
 readme = "README.md"
 requires-python = ">=3.11"

{runapprentice-0.4.1 → runapprentice-0.4.3}/src/runapprentice/cli.py

@@ -8,9 +8,12 @@ import json
 import os
 import stat
 import sys
+import tempfile
 from pathlib import Path
 from typing import Any
 
+import httpx
+
 from runapprentice.client import Apprentice, ApprenticeError
 from runapprentice import local as _local
 from runapprentice.models import (
@@ -51,7 +54,10 @@ def main(argv: list[str] | None = None) -> int:
     args = parser.parse_args(argv)
     if args.command == "login":
         return _login(args)
-    api_key = _api_key()
+    try:
+        api_key = _api_key()
+    except ApprenticeError as exc:
+        return _fail(str(exc), bool(args.json))
     client = Apprentice(api_key=api_key)
     setattr(args, "_api_key_present", api_key is not None)
     keyless_local_optimize = args.command == "optimize" and args.local and api_key is None
@@ -106,7 +112,10 @@ def build_parser() -> argparse.ArgumentParser:
     optimize.add_argument(
         "--local",
         action="store_true",
-        help="run GEPA on this machine with your own OPENAI_API_KEY, no Apprentice account needed",
+        help=(
+            "run GEPA on this machine with your own OPENAI_API_KEY, no Apprentice account "
+            "needed; scores JSON extraction only, so every output must be a JSON object or array"
+        ),
     )
     optimize.add_argument(
         "--data",
@@ -168,7 +177,7 @@ def build_parser() -> argparse.ArgumentParser:
     )
     train.set_defaults(func=_train)
 
-    jobs = sub.add_parser("jobs", help="list hosted optimize/train jobs for a task, newest first")
+    jobs = sub.add_parser("jobs", help="list hosted jobs for a task, newest first")
     jobs.add_argument("task", help="task name")
     jobs.set_defaults(func=_jobs)
 
@@ -183,13 +192,70 @@ def build_parser() -> argparse.ArgumentParser:
 
 
 def _login(args: argparse.Namespace) -> int:
-    key = getpass.getpass("Apprentice API key: ").strip()
+    key = (
+        getpass.getpass("Apprentice API key: ") if sys.stdin.isatty() else sys.stdin.read()
+    ).strip()
     if not key:
         return _fail("API key required", bool(args.json))
-    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
-    CONFIG_PATH.write_text(json.dumps({"api_key": key}) + "\n", encoding="utf-8")
-    CONFIG_PATH.chmod(stat.S_IRUSR | stat.S_IWUSR)
-    return _emit({"config": str(CONFIG_PATH), "mode": "600"}, "Saved API key.", bool(args.json))
+    # The client rejects a key that cannot become an HTTP header (whitespace from a
+    # two-line paste, a smart quote). Letting it raise keeps one definition of what
+    # a usable key is, shared with the constructor and APPRENTICE_API_KEY.
+    # No captures happen during login, so skip the background dispatcher thread.
+    try:
+        client = Apprentice(api_key=key, capture_async=False)
+    except ApprenticeError as exc:
+        return _fail(f"{exc}, so it was not saved.", bool(args.json))
+    try:
+        response = client._http.get("/v1/tasks")
+    except httpx.HTTPError:
+        return _fail(
+            "Could not verify API key because Apprentice API is unavailable. "
+            "API key was not saved.",
+            bool(args.json),
+        )
+    if response.status_code in {401, 403}:
+        return _fail("API key was rejected and was not saved.", bool(args.json))
+    if not response.is_success:
+        return _fail(
+            "Could not verify API key because Apprentice API is unavailable. "
+            "API key was not saved.",
+            bool(args.json),
+        )
+    # The key is valid past this point, so a filesystem failure is the only thing
+    # left that can go wrong. Report it the same way as every other failure rather
+    # than as a traceback that emits no JSON in --json mode.
+    try:
+        return _save_api_key(key, bool(args.json))
+    except OSError as exc:
+        return _fail(f"Could not save the API key to {CONFIG_PATH}: {exc}", bool(args.json))
+
+
+def _save_api_key(key: str, as_json: bool) -> int:
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    # mkdir's mode is masked by the process umask, so set it explicitly. A
+    # world-writable directory lets another local account replace the key file
+    # whatever the file's own mode is, because directory write governs entries.
+    CONFIG_PATH.parent.chmod(stat.S_IRWXU)
+    # Write private-then-rename rather than write-then-chmod. Writing first would
+    # leave the key world-readable (0644 under a normal umask, 0666 under a
+    # permissive one) until the chmod lands, and a process killed in that window
+    # leaves it that way for good. mkstemp creates at 0600, and os.replace is
+    # atomic, so a concurrent reader sees either the old file or the new one and
+    # never a half-written one. The rename also lands on the path itself, so a
+    # config.json symlink is replaced instead of its target being overwritten.
+    handle_fd, temporary_path = tempfile.mkstemp(dir=CONFIG_PATH.parent, prefix=".config-")
+    try:
+        with os.fdopen(handle_fd, "w", encoding="utf-8") as handle:
+            handle.write(json.dumps({"api_key": key}) + "\n")
+        os.replace(temporary_path, CONFIG_PATH)
+    except BaseException:
+        os.unlink(temporary_path)
+        raise
+    return _emit(
+        {"config": str(CONFIG_PATH), "mode": "600", "validated": True},
+        f"API key verified and saved to {CONFIG_PATH}.",
+        as_json,
+    )
 
 
 def _status(args: argparse.Namespace, client: Apprentice) -> int:
@@ -474,9 +540,34 @@ def _api_key() -> str | None:
         return os.environ["APPRENTICE_API_KEY"]
     if not CONFIG_PATH.exists():
         return None
-    data = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
+    try:
+        data = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ApprenticeError(
+            f"Could not read {CONFIG_PATH}: invalid JSON. "
+            "Run `apprentice login` to replace it."
+        ) from exc
+    except (OSError, UnicodeError) as exc:
+        raise ApprenticeError(
+            f"Could not read {CONFIG_PATH}: {exc}. "
+            "Fix its permissions or set APPRENTICE_API_KEY."
+        ) from exc
+    if not isinstance(data, dict):
+        raise ApprenticeError(
+            f"Could not read {CONFIG_PATH}: expected a JSON object. "
+            "Run `apprentice login` to replace it."
+        )
     key = data.get("api_key")
-    return key if isinstance(key, str) else None
+    if key is None:
+        return None
+    # Present but not a string: the file is malformed, which is not the same as
+    # having no key. Saying so beats silently falling through to an auth error.
+    if not isinstance(key, str):
+        raise ApprenticeError(
+            f"Could not read {CONFIG_PATH}: api_key must be a string. "
+            "Run `apprentice login` to replace it."
+        )
+    return key
 
 
 def _job_summary(job: JobSummary) -> dict[str, Any]:

{runapprentice-0.4.1 → runapprentice-0.4.3}/src/runapprentice/client.py

@@ -35,6 +35,7 @@ from runapprentice.models import (
 )
 
 logger = logging.getLogger("runapprentice")
+logger.addHandler(logging.NullHandler())
 
 _DEFAULT_BASE_URL = "https://api.runapprentice.com"
 _DEFAULT_TIMEOUT = 10.0
@@ -64,6 +65,45 @@ class ApprenticeError(Exception):
     """Raised for control-plane API failures."""
 
 
+def _reject_unusable_api_key(api_key: str) -> None:
+    """Reject a key that cannot become an HTTP header.
+
+    httpx encodes header values as ASCII, so a key carrying a smart quote or an
+    accented character raises UnicodeEncodeError deep inside the transport on the
+    first request. Checked here so every path that supplies a key (constructor,
+    APPRENTICE_API_KEY, the CLI config file) fails with the same clear error.
+    """
+    if any(character.isspace() for character in api_key):
+        raise ApprenticeError("API key contains whitespace")
+    if not api_key.isascii() or not api_key.isprintable():
+        raise ApprenticeError("API key has characters that are not printable ASCII")
+
+
+def _printable(text: str) -> str:
+    """Drop control characters from server-controlled text.
+
+    The message goes to a terminal and to logs. Escape sequences in it can clear
+    the screen, move the cursor, or repaint earlier lines, so a hostile or
+    compromised backend could make its error read as something else entirely.
+    """
+    return "".join(character if character.isprintable() else " " for character in text)
+
+
+def _reason(resp: httpx.Response) -> str:
+    """The API's own explanation for a failed response, falling back to its body."""
+    try:
+        payload = resp.json()
+    except ValueError:
+        payload = None
+    # A body is not always a JSON object: a gateway can return a bare list or
+    # string, and FastAPI sends a list of dicts for 422. Only a string `detail`
+    # on an object is a written reason; everything else falls back to the body.
+    detail = payload.get("detail") if isinstance(payload, dict) else None
+    if isinstance(detail, str) and detail.strip():
+        return _printable(detail.strip())[:500]
+    return _printable(resp.text[:500]) or f"HTTP {resp.status_code}"
+
+
 class Apprentice:
     def __init__(
         self,
@@ -78,6 +118,8 @@ class Apprentice:
             base_url or os.environ.get("APPRENTICE_BASE_URL") or _DEFAULT_BASE_URL
         ).rstrip("/")
         api_key = api_key or os.environ.get("APPRENTICE_API_KEY")
+        if api_key is not None:
+            _reject_unusable_api_key(api_key)
         self._api_key_present = api_key is not None
         headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
         self._http = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout)
@@ -97,24 +139,50 @@ class Apprentice:
 
     def _request(self, method: str, path: str, json: Any | None = None) -> Any:
         t0 = time.monotonic()
+        # The path carries caller-supplied names, so an absurd one would otherwise
+        # put its whole length into every message and log line, defeating the cap
+        # on the response body. httpx.InvalidURL is not an HTTPError, so an
+        # over-long path would escape the handler below.
+        shown_path = path if len(path) <= 200 else f"{path[:200]}... ({len(path)} chars)"
         try:
             resp = self._http.request(method, path, json=json)
+        except httpx.InvalidURL as exc:
+            raise ApprenticeError(f"{method} {shown_path} is not a valid URL: {exc}") from exc
         except httpx.HTTPError as exc:
-            logger.error("%s %s failed after %.0fms: %s", method, path, _ms(t0), exc)
+            logger.error("%s %s failed after %.0fms: %s", method, shown_path, _ms(t0), exc)
             raise ApprenticeError(
-                f"{method} {path} failed: {exc}. "
+                f"{method} {shown_path} failed: {exc}. "
                 f"Is the Apprentice backend reachable at {self.base_url}?"
             ) from exc
-        logger.debug("%s %s -> %s (%.0fms)", method, path, resp.status_code, _ms(t0))
+        logger.debug("%s %s -> %s (%.0fms)", method, shown_path, resp.status_code, _ms(t0))
         if resp.status_code >= 400:
-            logger.error("%s %s -> %s: %s", method, path, resp.status_code, resp.text[:500])
+            logger.error(
+                "%s %s -> %s error=http_response", method, shown_path, resp.status_code
+            )
+            # The API sends a written reason in `detail`; lead with it so the first
+            # thing a reader sees is why the call failed. The route cannot tell us
+            # that: a 404 on /v1/prompts/{task} means the task has no optimized
+            # prompt yet, not that the task is missing.
             suffix = (
-                ". Get a free Apprentice API key at https://runapprentice.com"
+                " Get a free Apprentice API key at https://runapprentice.com"
                 if resp.status_code in {401, 403}
                 else ""
             )
-            raise ApprenticeError(f"{method} {path} -> {resp.status_code}: {resp.text[:500]}{suffix}")
-        return resp.json()
+            raise ApprenticeError(
+                f"{_reason(resp)} ({method} {shown_path} -> {resp.status_code}){suffix}"
+            )
+        try:
+            return resp.json()
+        except ValueError as exc:
+            # A success with no body (204) or a non-JSON body from a proxy would
+            # otherwise raise JSONDecodeError, escaping every caller that catches
+            # ApprenticeError.
+            if not resp.content:
+                return {}
+            raise ApprenticeError(
+                f"{method} {shown_path} -> {resp.status_code} returned a body that is not JSON: "
+                f"{resp.text[:200]}"
+            ) from exc
 
     def optimize(
         self,
@@ -123,6 +191,7 @@ class Apprentice:
         budget: Literal["quick", "thorough"] | None = None,
         metric: str | None = None,
     ) -> Job:
+        """Queue hosted optimization over gold and silver rows, returning its Job immediately."""
         payload = {"task": task}
         payload.update(
             {
@@ -145,6 +214,7 @@ class Apprentice:
         budget: Literal["quick", "thorough"] | None = None,
         metric: str | None = None,
     ) -> OptimizationPreview:
+        """Return metric guidance and cost estimates without queueing optimization."""
         payload = {"task": task}
         payload.update(
             {
@@ -161,21 +231,26 @@ class Apprentice:
         return OptimizationPreview.model_validate(data)
 
     def job(self, job_id: str) -> Job:
+        """Fetch current job information and return its Job proxy."""
         data = self._request("GET", f"/v1/jobs/{job_id}")
         return Job(self, JobInfo.model_validate(data))
 
     def jobs(self, task: str) -> list[JobSummary]:
+        """Return every hosted job for a task, newest first, whatever its kind."""
         data = self._request("GET", f"/v1/tasks/{quote(task, safe='')}/jobs")
         return [JobSummary.model_validate(item) for item in data]
 
     def cancel_job(self, job_id: str) -> dict[str, Any]:
+        """Request job cancellation and return the API response."""
         return dict(self._request("POST", f"/v1/jobs/{job_id}/cancel"))
 
     def local_runs(self, task: str) -> list[LocalRun]:
+        """Return tracked local training runs for a task."""
         data = self._request("GET", f"/v1/tasks/{quote(task, safe='')}/local-runs")
         return [LocalRun.model_validate(item) for item in data]
 
     def train(self, task: str) -> Job:
+        """Queue hosted training over gold rows only, returning its Job immediately."""
         data = self._request("POST", f"/v1/tasks/{quote(task, safe='')}/train", json={})
         return Job(self, JobInfo.model_validate(data))
 
@@ -185,6 +260,7 @@ class Apprentice:
         data: str | Path | None = None,
         **options: Any,
     ) -> dict[str, Any]:
+        """Optimize locally using a CSV or the task's gold and silver rows, returning results."""
         from runapprentice.local import load_csv_rows, optimize_prompt_local
 
         if data is None:
@@ -203,6 +279,7 @@ class Apprentice:
         effort: str | None = None,
         **options: Any,
     ) -> dict[str, Any]:
+        """Train locally using a CSV or the task's gold rows, returning run results."""
         from runapprentice.local import load_csv_rows, train_mlx_local
 
         if data is None:
@@ -222,6 +299,7 @@ class Apprentice:
         seed: int,
         config: dict[str, Any] | None = None,
     ) -> LocalRun:
+        """Register a local training run and return its tracking record."""
         data = self._request(
             "POST",
             "/v1/local-runs",
@@ -236,6 +314,7 @@ class Apprentice:
         return LocalRun.model_validate(data)
 
     def complete_local_run(self, run_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+        """Submit a local run result and return the API response."""
         return dict(self._request("POST", f"/v1/local-runs/{run_id}/complete", json=payload))
 
     def feedback(
@@ -245,6 +324,7 @@ class Apprentice:
         score: float | None = None,
         note: str | None = None,
     ) -> None:
+        """Flush pending captures, then submit feedback for a trace."""
         if self._dispatcher is not None and not self._dispatcher.flush():
             raise ApprenticeError(
                 "feedback not sent because pending captures did not finish before the flush timeout"
@@ -352,8 +432,8 @@ class Apprentice:
             self._capture_warned = False
             return record.trace_id
         if resp.status_code >= 500:
-            raise ApprenticeError(f"backend {resp.status_code}: {resp.text[:200]}")
-        self._warn_capture_once(f"backend rejected trace: {resp.status_code} {resp.text[:200]}")
+            raise ApprenticeError("backend 5xx for /v1/traces")
+        self._warn_capture_once(f"backend rejected trace: {resp.status_code} for /v1/traces")
         return None
 
     def _warn_capture_once(self, message: str) -> None:
@@ -368,6 +448,7 @@ class Apprentice:
             logger.debug("runapprentice: %s", message)
 
     def close(self) -> None:
+        """Flush pending captures and close transport resources."""
         if self._dispatcher is not None:
             self._dispatcher.flush()
             self._dispatcher.close()
@@ -387,11 +468,18 @@ class Job:
     def status(self) -> str:
         return self.info.status
 
+    @property
+    def error(self) -> str | None:
+        """Return failure detail reported for the job, if any."""
+        return self.info.error
+
     def refresh(self) -> Job:
+        """Refresh this proxy in place and return it."""
         self.info = self._client.job(self.info.job_id).info
         return self
 
     def wait(self, poll_seconds: float = 5.0, timeout_seconds: float = 3600.0) -> Job:
+        """Poll until the job finishes; raise ApprenticeError on timeout and return this Job."""
         deadline = time.monotonic() + timeout_seconds
         while self.refresh().info.status in ("queued", "running"):
             if time.monotonic() > deadline:
@@ -400,9 +488,11 @@ class Job:
         return self
 
     def cancel(self) -> dict[str, Any]:
+        """Request cancellation and return the API response."""
         return self._client.cancel_job(self.info.job_id)
 
     def report(self) -> JobReport:
+        """Return the finished job report, or raise ApprenticeError when unavailable."""
         if self.info.report is None:
             raise ApprenticeError(
                 f"job {self.info.job_id} has no report (status={self.info.status})"

{runapprentice-0.4.1 → runapprentice-0.4.3}/src/runapprentice/local.py

@@ -27,8 +27,10 @@ from runapprentice.client import ApprenticeError
 from runapprentice.models import DatasetRow
 
 LOCAL_MODEL_ID = "mlx-community/Qwen3.5-4B-OptiQ-4bit"
-DEFAULT_STUDENT_MODEL = "openai/gpt-5-mini"
-DEFAULT_REFLECTION_MODEL = "openai/gpt-5.4"
+# Mirrors the API defaults (owner decision 2026-07-31). The exact id matters: the bare
+# `gpt-5.6` alias routes to Sol, not Luna.
+DEFAULT_STUDENT_MODEL = "openai/gpt-5.6-luna"
+DEFAULT_REFLECTION_MODEL = "openai/gpt-5.6-luna"
 
 # Every tier trains the SAME model quality: 16 LoRA layers (mlx-lm's default;
 # fewer layers reduces final quality per LORA.md) and the same number of passes
@@ -117,18 +119,64 @@ def score_json_field_f1(row: DatasetRow, actual: str) -> float:
         actual_obj = json.loads(_extract_json_text(actual))
     except json.JSONDecodeError:
         return 0.0
-    if not isinstance(expected_obj, dict) or not isinstance(actual_obj, dict):
+    # Objects AND arrays, matching the API scorer: a task whose gold answers are top-level
+    # arrays scored 0 here for a correct answer.
+    if not isinstance(expected_obj, (dict, list)) or not isinstance(actual_obj, (dict, list)):
         return 0.0
     expected = _json_leaf_pairs(expected_obj)
     actual_pairs = _json_leaf_pairs(actual_obj)
+    # Mirrors the hosted ceiling. Without it the same answer scores 1.0 here and 0.0 hosted,
+    # so a local run and a hosted run disagree about quality on identical data.
+    if len(expected) > _MAX_JSON_METRIC_LEAVES or len(actual_pairs) > _MAX_JSON_METRIC_LEAVES:
+        return 0.0
     f1 = _leaf_f1(expected, actual_pairs)
-    if len(expected_obj) == 1:
+    if isinstance(expected_obj, dict) and isinstance(actual_obj, dict) and len(expected_obj) == 1:
         root_key, root_value = next(iter(expected_obj.items()))
         if isinstance(root_value, dict) and root_key not in actual_obj:
             f1 = max(f1, _leaf_f1(_json_leaf_pairs(root_value), actual_pairs))
     return f1
 
 
+def _reject_unscoreable_rows(rows: Sequence[DatasetRow]) -> None:
+    """Stop before spending if the scorer cannot read this data.
+
+    Local optimize scores with `score_json_field_f1` and nothing else -- the CLI refuses
+    `--metric` in local mode. That scorer returns 0.0 for an answer it cannot parse as a JSON
+    object or array, so a dataset of plain labels ("yes"/"no") used to run to completion on the
+    user's own OpenAI key and report 0.0 to 0.0: money and minutes spent measuring nothing.
+    The same shape of bug reached production hosted-side on 2026-08-02.
+
+    The test mirrors the scorer's own accept rule rather than restating it, so the two cannot
+    drift apart.
+    """
+    bad = [
+        index
+        for index, row in enumerate(rows, start=1)
+        if not _is_json_container(row.output)
+    ]
+    if not bad:
+        return
+    shown = ", ".join(str(index) for index in bad[:5])
+    more = f" (and {len(bad) - 5} more)" if len(bad) > 5 else ""
+    # One action, and it is true on every path that reaches this message. An earlier draft
+    # also offered "run without --local", which is false advice here: the hosted branch of
+    # `_optimize` never reads --data, so dropping the flag silently ignores the CSV and
+    # optimizes whatever rows already exist server-side. The CLI has no upload command, so
+    # there is no hosted path for THIS file to name.
+    raise ApprenticeError(
+        f"{len(bad)} of {len(rows)} rows have an output that is not a JSON object or array, "
+        f"starting at row {shown}{more}. Local optimize scores JSON extraction only, so these "
+        "rows would all score zero. Give each row's output column a JSON object or array."
+    )
+
+
+def _is_json_container(output: str) -> bool:
+    try:
+        return isinstance(json.loads(output), (dict, list))
+    except json.JSONDecodeError:
+        return False
+
+
 def optimize_prompt_local(
     rows: Sequence[DatasetRow],
     *,
@@ -141,6 +189,7 @@ def optimize_prompt_local(
         raise ApprenticeError("OPENAI_API_KEY is required for local prompt optimization")
     if len(rows) < 2:
         raise ApprenticeError("local optimize needs at least two rows for train and held-out scoring")
+    _reject_unscoreable_rows(rows)
     dspy = _load_dspy()
     student_model = student_model or os.environ.get("APPRENTICE_STUDENT_MODEL", DEFAULT_STUDENT_MODEL)
     reflection_model = reflection_model or os.environ.get(
@@ -525,51 +574,75 @@ def _generate(row: DatasetRow, adapter_dir: Path, model_id: str) -> str:
 
 
 def _extract_json_text(text: str) -> str:
+    # Objects AND arrays, mirroring the API. Scanning for "{" alone returned the first inner
+    # OBJECT of a wrapped array, so an array answer was compared as an object and scored 0.
     candidate = text.strip()
-    start = candidate.find("{")
-    while start != -1:
-        in_string = False
-        escaped = False
-        depth = 0
-        for i, char in enumerate(candidate[start:], start=start):
-            if in_string:
-                if escaped:
-                    escaped = False
-                elif char == "\\":
-                    escaped = True
-                elif char == '"':
-                    in_string = False
-                continue
-            if char == '"':
-                in_string = True
-            elif char == "{":
-                depth += 1
-            elif char == "}":
-                depth -= 1
-                if depth == 0:
-                    return candidate[start : i + 1]
-        start = candidate.find("{", start + 1)
+    openers = [(candidate.find(o), o, c) for o, c in (("{", "}"), ("[", "]")) if candidate.find(o) != -1]
+    for _, opener, closer in sorted(openers):
+        start = candidate.find(opener)
+        while start != -1:
+            in_string = False
+            escaped = False
+            depth = 0
+            for i, char in enumerate(candidate[start:], start=start):
+                if in_string:
+                    if escaped:
+                        escaped = False
+                    elif char == "\\":
+                        escaped = True
+                    elif char == '"':
+                        in_string = False
+                    continue
+                if char == '"':
+                    in_string = True
+                elif char == opener:
+                    depth += 1
+                elif char == closer:
+                    depth -= 1
+                    if depth == 0:
+                        span = candidate[start : i + 1]
+                        try:
+                            json.loads(span)
+                            return span
+                        except json.JSONDecodeError:
+                            break
+            start = candidate.find(opener, start + 1)
     return candidate
 
 
-def _json_leaf_pairs(value: Any, path: str = "") -> set[tuple[str, str]]:
-    leaves: set[tuple[str, str]] = set()
+# Leaf kinds, mirroring the API. The kind is a separate field so an empty container can
+# never be confused with a scalar that spells one.
+# Mirrors apprentice-api optimizer._MAX_JSON_METRIC_LEAVES.
+_MAX_JSON_METRIC_LEAVES = 2000
+_LEAF_SCALAR = "scalar"
+_LEAF_EMPTY_OBJECT = "empty-object"
+_LEAF_EMPTY_ARRAY = "empty-array"
+
+
+def _json_leaf_pairs(value: Any, path: str = "") -> set[tuple[str, str, str]]:
+    leaves: set[tuple[str, str, str]] = set()
 
     def walk(node: Any, current_path: str) -> None:
         if isinstance(node, dict):
+            if not node:
+                leaves.add((current_path or "$", _LEAF_EMPTY_OBJECT, ""))
+                return
             for key in sorted(node):
                 walk(node[key], f"{current_path}.{key}" if current_path else str(key))
         elif isinstance(node, list):
+            if not node:
+                leaves.add((current_path or "$", _LEAF_EMPTY_ARRAY, ""))
+                return
             for i, item in enumerate(node):
                 walk(item, f"{current_path}[{i}]")
         else:
-            leaves.add((current_path or "$", _normalize_scalar(node)))
+            leaves.add((current_path or "$", _LEAF_SCALAR, _normalize_scalar(node)))
 
     walk(value, path)
     return leaves
 
 
-def _leaf_f1(expected: set[tuple[str, str]], actual: set[tuple[str, str]]) -> float:
+def _leaf_f1(expected: set[tuple[str, str, str]], actual: set[tuple[str, str, str]]) -> float:
     tp = len(expected & actual)
     precision = tp / len(actual) if actual else 0.0
     recall = tp / len(expected) if expected else 0.0