synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/cli/task_apps.py

@@ -9,24 +9,21 @@ import hashlib
 import importlib
 import importlib.util
 import inspect
-import json
 import os
 import shlex
 import shutil
 import signal
-import sqlite3
 import subprocess
 import sys
 import tempfile
 import textwrap
 import time
 import types
-import uuid
 from collections.abc import Callable, Iterable, Iterator, Sequence
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from pathlib import Path
-from typing import Any, Optional, cast
+from typing import Any, cast
 
 try:  # Python 3.11+
     import tomllib as _toml
@@ -35,6 +32,9 @@ except Exception:  # pragma: no cover - fallback
 
 import click
 from click.exceptions import Abort
+from synth_ai.cli.commands import deploy as _deploy_commands
+from synth_ai.cli.commands.eval import core as eval_core
+from synth_ai.cli.commands.filter import core as filter_core
 
 # Tracing imports - make conditional for optional dependencies
 try:
@@ -92,14 +92,14 @@ except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load task app server utilities") from exc
 
 
-def _load_demo_directory() -> Optional[Path]:
+def _load_demo_directory() -> Path | None:
     """Return the demo task apps directory if available."""
 
     try:
         module = cast(
             Any, importlib.import_module("synth_ai.demos.demo_task_apps.core")
         )
-        loader = cast(Callable[[], Optional[str | Path]], module.load_demo_dir)
+        loader = cast(Callable[[], str | Path | None], module.load_demo_dir)
         demo_dir = loader()
         if isinstance(demo_dir, str | Path):
             demo_path = Path(demo_dir)
@@ -139,7 +139,7 @@ DEFAULT_SEARCH_RELATIVE = (
 )
 
 
-def _pearson(xs: Sequence[float], ys: Sequence[float]) -> Optional[float]:
+def _pearson(xs: Sequence[float], ys: Sequence[float]) -> float | None:
     if len(xs) != len(ys) or len(xs) < 2:
         return None
     mean_x = sum(xs) / len(xs)
@@ -164,7 +164,7 @@ class AppChoice:
     label: str
     path: Path
     source: str
-    description: Optional[str] = None
+    description: str | None = None
     aliases: tuple[str, ...] = ()
     entry: TaskAppEntryType | None = None
     entry_loader: Callable[[], TaskAppEntryType] | None = None
@@ -188,21 +188,21 @@ class JudgeSpec:
     kwargs: dict[str, Any]
 
 
-def _parse_datetime_for_trace(value: Any) -> Optional[datetime]:
+def _parse_datetime_for_trace(value: Any) -> datetime | None:
     if isinstance(value, datetime):
-        return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
+        return value if value.tzinfo else value.replace(tzinfo=UTC)
     if isinstance(value, str):
         value = value.replace("Z", "+00:00")
         try:
             dt = datetime.fromisoformat(value)
         except ValueError:
             try:
-                dt = datetime.fromtimestamp(float(value), tz=timezone.utc)
+                dt = datetime.fromtimestamp(float(value), tz=UTC)
             except Exception:
                 return None
-        return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
+        return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
     if isinstance(value, int | float):
-        return datetime.fromtimestamp(float(value), tz=timezone.utc)
+        return datetime.fromtimestamp(float(value), tz=UTC)
     return None
 
 
@@ -241,6 +241,24 @@ def _event_from_dict(payload: dict[str, Any]) -> BaseEvent:
             system_state_after=payload.get("system_state_after"),
             **base_kwargs,
         )
+    # Check for LM CAIS event fields
+    if any(key in payload for key in ("model_name", "provider", "call_records")):
+        from synth_ai.tracing_v3.abstractions import LMCAISEvent
+        # Note: call_records are left as dicts - the storage layer will handle serialization
+        call_records = payload.get("call_records") or []
+        return LMCAISEvent(
+            model_name=payload.get("model_name", ""),
+            provider=payload.get("provider", ""),
+            input_tokens=payload.get("input_tokens"),
+            output_tokens=payload.get("output_tokens"),
+            total_tokens=payload.get("total_tokens"),
+            cost_usd=payload.get("cost_usd"),
+            latency_ms=payload.get("latency_ms"),
+            span_id=payload.get("span_id"),
+            trace_id=payload.get("trace_id"),
+            call_records=call_records,
+            **base_kwargs,
+        )
     return BaseEvent(**base_kwargs)
 
 
@@ -279,7 +297,7 @@ def _step_from_dict(payload: dict[str, Any]) -> SessionTimeStep:
         for msg in payload.get("markov_blanket_messages", [])
         if isinstance(msg, dict)
     ]
-    timestamp = _parse_datetime_for_trace(payload.get("timestamp")) or datetime.now(timezone.utc)
+    timestamp = _parse_datetime_for_trace(payload.get("timestamp")) or datetime.now(UTC)
     completed_at = _parse_datetime_for_trace(payload.get("completed_at"))
     return SessionTimeStep(
         step_id=payload.get("step_id", ""),
@@ -293,7 +311,7 @@ def _step_from_dict(payload: dict[str, Any]) -> SessionTimeStep:
     )
 
 
-def _session_trace_from_dict(payload: dict[str, Any]) -> Optional[V3SessionTrace]:
+def _session_trace_from_dict(payload: dict[str, Any]) -> V3SessionTrace | None:
     if not isinstance(payload, dict):
         return None
     steps = [
@@ -311,7 +329,7 @@ def _session_trace_from_dict(payload: dict[str, Any]) -> Optional[V3SessionTrace
         for msg in payload.get("markov_blanket_message_history", [])
         if isinstance(msg, dict)
     ]
-    created_at = _parse_datetime_for_trace(payload.get("created_at")) or datetime.now(timezone.utc)
+    created_at = _parse_datetime_for_trace(payload.get("created_at")) or datetime.now(UTC)
     metadata = payload.get("metadata") or {}
     session_metadata = payload.get("session_metadata")
     return V3SessionTrace(
@@ -336,15 +354,32 @@ async def _store_trace(
     _logger.info(f"[STORE_TRACE_DEBUG] Called with tracer={tracer is not None}, trace_namespace={trace_namespace is not None}")
     
     if tracer is None or not isinstance(trace_namespace, dict):
-        _logger.warning(f"[STORE_TRACE_DEBUG] Early return: tracer={tracer is not None}, trace_namespace type={type(trace_namespace)}")
-        return
+        message = (
+            f"Trace storage requires a tracer instance and dict payload. "
+            f"Got tracer_present={tracer is not None}, payload_type={type(trace_namespace)}"
+        )
+        _logger.error("[STORE_TRACE_DEBUG] %s", message)
+        raise ValueError(message)
     
     _logger.info(f"[STORE_TRACE_DEBUG] trace_namespace keys: {list(trace_namespace.keys())}")
     
+    # Handle both formats:
+    # - With session_trace key: {"session_trace": {...}}
+    # - Without session_trace key (trace itself is the session): {"session_id": ..., "markov_blanket_message_history": ...}
     session_payload = trace_namespace.get("session_trace")
     if not isinstance(session_payload, dict):
-        _logger.warning(f"[STORE_TRACE_DEBUG] No session_trace found or wrong type: {type(session_payload)}")
-        return
+        # If no session_trace key, assume "full" format where trace itself is the session_trace
+        if "session_id" in trace_namespace:
+            session_payload = trace_namespace
+            _logger.info("[STORE_TRACE_DEBUG] Using trace_namespace directly as session_payload (no session_trace key)")
+        else:
+            message = (
+                "Trace payload did not contain a 'session_trace' dict and lacked top-level "
+                "session fields (session_id, markov_blanket_message_history). "
+                f"Payload keys: {list(trace_namespace.keys())}"
+            )
+            _logger.error("[STORE_TRACE_DEBUG] %s", message)
+            raise ValueError(message)
     
     _logger.info(f"[STORE_TRACE_DEBUG] session_payload keys: {list(session_payload.keys())}")
     msg_count = len(session_payload.get("markov_blanket_message_history", []))
@@ -352,8 +387,26 @@ async def _store_trace(
     
     trace_obj = _session_trace_from_dict(session_payload)
     if trace_obj is None:
-        _logger.warning(f"[STORE_TRACE_DEBUG] _session_trace_from_dict returned None")
-        return
+        message = "Session trace payload could not be parsed into a SessionTrace object."
+        _logger.error("[STORE_TRACE_DEBUG] %s", message)
+        raise ValueError(message)
+
+    if not trace_obj.markov_blanket_message_history:
+        message = (
+            "Session trace is missing markov_blanket_message_history; "
+            "eval output must include all prompts/tool calls. "
+            f"session_id={trace_obj.session_id}"
+        )
+        _logger.error("[STORE_TRACE_DEBUG] %s", message)
+        raise ValueError(message)
+
+    if not trace_obj.event_history:
+        message = (
+            "Session trace is missing event_history; rollout should emit environment/LLM events. "
+            f"session_id={trace_obj.session_id}"
+        )
+        _logger.error("[STORE_TRACE_DEBUG] %s", message)
+        raise ValueError(message)
     
     _logger.info(f"[STORE_TRACE_DEBUG] Created SessionTrace object with {len(trace_obj.markov_blanket_message_history)} messages")
     
@@ -366,7 +419,7 @@ async def _store_trace(
     
     _logger.info(f"[STORE_TRACE_DEBUG] Calling insert_session_trace for session_id={trace_obj.session_id}")
     await tracer.db.insert_session_trace(trace_obj)
-    _logger.info(f"[STORE_TRACE_DEBUG] Successfully inserted trace")
+    _logger.info("[STORE_TRACE_DEBUG] Successfully inserted trace")
 
 def _temporary_sys_path(paths: Sequence[Path]):
     """Context manager to prepend entries to sys.path temporarily."""
@@ -480,49 +533,6 @@ def _candidate_search_roots() -> list[Path]:
     return ordered
 
 
-def _eval_config_sort_key(path: Path) -> tuple[int, int, int, str]:
-    name = path.name.lower()
-    parent_names = {p.name.lower() for p in path.parents}
-    in_configs = 0 if "configs" in parent_names else 1
-    in_examples = 0 if "examples" in parent_names else 1
-    starts_eval = 0 if name.startswith("eval") else 1
-    return (in_configs, in_examples, starts_eval, str(path))
-
-
-def _discover_eval_config_paths() -> list[Path]:
-    """Find candidate eval TOML files near the current working directory."""
-
-    candidates: list[Path] = []
-    seen: set[Path] = set()
-    search_roots = _candidate_search_roots()
-    for root in search_roots:
-        if not root.exists() or not root.is_dir():
-            continue
-        try:
-            root = root.resolve()
-        except Exception:
-            continue
-        for path in root.rglob("*.toml"):
-            if not path.is_file():
-                continue
-            if _should_ignore_path(path):
-                continue
-            name_lower = path.name.lower()
-            if "eval" not in name_lower and "evaluation" not in name_lower:
-                continue
-            try:
-                resolved = path.resolve()
-            except Exception:
-                continue
-            if resolved in seen:
-                continue
-            seen.add(resolved)
-            candidates.append(resolved)
-
-    candidates.sort(key=_eval_config_sort_key)
-    return candidates
-
-
 class _TaskAppConfigVisitor(ast.NodeVisitor):
     def __init__(self) -> None:
         self.matches: list[tuple[str, int]] = []
@@ -913,43 +923,43 @@ def _build_modal_config_from_ast(modal_call: ast.Call) -> ModalDeploymentConfigT
         for kw in modal_call.keywords:
             if kw.arg and isinstance(kw.value, ast.Constant):
                 kwargs[kw.arg] = kw.value.value
-            elif kw.arg == "pip_packages" and isinstance(kw.value, (ast.List, ast.Tuple)):
+            elif kw.arg == "pip_packages" and isinstance(kw.value, ast.List | ast.Tuple):
                 # Handle pip_packages list/tuple
                 packages: list[str] = []
                 value_node = kw.value
-                if isinstance(value_node, (ast.List, ast.Tuple)):
+                if isinstance(value_node, ast.List | ast.Tuple):
                     for elt in value_node.elts:
                         if isinstance(elt, ast.Constant):
                             packages.append(elt.value)
                 kwargs[kw.arg] = tuple(packages)
-            elif kw.arg == "extra_local_dirs" and isinstance(kw.value, (ast.List, ast.Tuple)):
+            elif kw.arg == "extra_local_dirs" and isinstance(kw.value, ast.List | ast.Tuple):
                 # Handle extra_local_dirs list/tuple of tuples
                 dirs = []
                 value_node = kw.value
-                if isinstance(value_node, (ast.List, ast.Tuple)):
+                if isinstance(value_node, ast.List | ast.Tuple):
                     for elt in value_node.elts:
-                        if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
+                        if isinstance(elt, ast.List | ast.Tuple) and len(elt.elts) == 2:
                             src = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
                             dst = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
                             if src and dst:
                                 dirs.append((src, dst))
                 kwargs[kw.arg] = tuple(dirs)
-            elif kw.arg == "secret_names" and isinstance(kw.value, (ast.List, ast.Tuple)):
+            elif kw.arg == "secret_names" and isinstance(kw.value, ast.List | ast.Tuple):
                 # Handle secret_names list/tuple
                 secrets = []
                 value_node = kw.value
-                if isinstance(value_node, (ast.List, ast.Tuple)):
+                if isinstance(value_node, ast.List | ast.Tuple):
                     for elt in value_node.elts:
                         if isinstance(elt, ast.Constant):
                             secrets.append(elt.value)
                 kwargs[kw.arg] = tuple(secrets)
-            elif kw.arg == "volume_mounts" and isinstance(kw.value, (ast.List, ast.Tuple)):
+            elif kw.arg == "volume_mounts" and isinstance(kw.value, ast.List | ast.Tuple):
                 # Handle volume_mounts list/tuple of tuples
                 mounts = []
                 value_node = kw.value
-                if isinstance(value_node, (ast.List, ast.Tuple)):
+                if isinstance(value_node, ast.List | ast.Tuple):
                     for elt in value_node.elts:
-                        if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
+                        if isinstance(elt, ast.List | ast.Tuple) and len(elt.elts) == 2:
                             name = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
                             mount = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
                             if name and mount:
@@ -2238,14 +2248,13 @@ def validate_task_app_cmd(
     • Debug failing deployments: Use --verbose to see detailed endpoint responses
     • Test API key configuration: Verify authentication is set up correctly
     """
-    import asyncio
     import socket
     import subprocess
     import tempfile
     import time
     
     # Import the validate_task_app function defined in this module
-    from synth_ai.cli._validate_task_app import validate_task_app  # type: ignore[attr-defined]
+    from ._validate_task_app import validate_task_app  # type: ignore[attr-defined]
     
     proc = None
     task_app_url = url
@@ -2445,48 +2454,15 @@ def serve_command(
     trace_dir: str | None,
     trace_db: str | None,
 ) -> None:
-    demo_dir_path = _load_demo_directory()
-    if demo_dir_path:
-        if not demo_dir_path.is_dir():
-            raise click.ClickException(
-                f"Demo directory not found: {demo_dir_path}\nRun 'synth-ai setup' to create a demo."
-            )
-        os.chdir(demo_dir_path)
-        click.echo(f"Using demo directory: {demo_dir_path}\n")
-        os.environ["SYNTH_DEMO_DIR"] = str(demo_dir_path.resolve())
-
-    # Prompt for port if not provided
-    if port is None:
-        port = click.prompt("Port to serve on", type=int, default=8001)
-
-    # Prompt for trace directory if not provided
-    if trace_dir is None:
-        click.echo(
-            "\nTracing captures rollout data (actions, rewards, model outputs) to a local SQLite DB."
-        )
-        click.echo("This data can be exported to JSONL for supervised fine-tuning (SFT).")
-        enable_tracing = click.confirm("Enable tracing?", default=True)
-        if enable_tracing:
-            demo_base = Path(os.environ.get("SYNTH_DEMO_DIR") or Path.cwd())
-            default_trace_dir = str((demo_base / "traces/v3").resolve())
-            trace_dir = click.prompt(
-                "Trace directory", type=str, default=default_trace_dir, show_default=True
-            )
-        else:
-            trace_dir = None
-
-    # Prompt for trace DB if not provided and tracing is enabled
-    if trace_dir and trace_db is None:
-        demo_base = Path(os.environ.get("SYNTH_DEMO_DIR") or Path.cwd())
-        default_trace_db = str((demo_base / "traces/v3/synth_ai.db").resolve())
-        trace_db = click.prompt(
-            "Trace DB path", type=str, default=default_trace_db, show_default=True
-        )
-
-    choice = _select_app_choice(app_id, purpose="serve")
-    entry = choice.ensure_entry()
-    _serve_entry(
-        entry, host, port, env_file, reload_flag, force, trace_dir=trace_dir, trace_db=trace_db
+    _deploy_commands.run_uvicorn_runtime(
+        app_id,
+        host,
+        port,
+        env_file,
+        reload_flag,
+        force,
+        trace_dir,
+        trace_db,
     )
 
 
@@ -2599,49 +2575,19 @@ def serve_task_group(
     trace_dir: str | None,
     trace_db: str | None,
 ) -> None:
-    demo_dir_path = _load_demo_directory()
-    if demo_dir_path:
-        if not demo_dir_path.is_dir():
-            raise click.ClickException(
-                f"Demo directory not found: {demo_dir_path}\nRun 'synth-ai setup' to create a demo."
-            )
-        os.chdir(demo_dir_path)
-        click.echo(f"Using demo directory: {demo_dir_path}\n")
-        os.environ["SYNTH_DEMO_DIR"] = str(demo_dir_path.resolve())
-
-    # Prompt for port if not provided
-    if port is None:
-        port = click.prompt("Port to serve on", type=int, default=8001)
-
-    # Prompt for trace directory if not provided
-    if trace_dir is None:
-        click.echo(
-            "\nTracing captures rollout data (actions, rewards, model outputs) to a local SQLite DB."
-        )
-        click.echo("This data can be exported to JSONL for supervised fine-tuning (SFT).")
-        enable_tracing = click.confirm("Enable tracing?", default=True)
-        if enable_tracing:
-            demo_base = Path(os.environ.get("SYNTH_DEMO_DIR") or Path.cwd())
-            default_trace_dir = str((demo_base / "traces/v3").resolve())
-            trace_dir = click.prompt(
-                "Trace directory", type=str, default=default_trace_dir, show_default=True
-            )
-        else:
-            trace_dir = None
+    _deploy_commands.run_uvicorn_runtime(
+        app_id,
+        host,
+        port,
+        env_file,
+        reload_flag,
+        force,
+        trace_dir,
+        trace_db,
+    )
 
-    # Prompt for trace DB if not provided and tracing is enabled
-    if trace_dir and trace_db is None:
-        demo_base = Path(os.environ.get("SYNTH_DEMO_DIR") or Path.cwd())
-        default_trace_db = str((demo_base / "traces/v3/synth_ai.db").resolve())
-        trace_db = click.prompt(
-            "Trace DB path", type=str, default=default_trace_db, show_default=True
-        )
 
-    choice = _select_app_choice(app_id, purpose="serve")
-    entry = choice.ensure_entry()
-    _serve_entry(
-        entry, host, port, env_file, reload_flag, force, trace_dir=trace_dir, trace_db=trace_db
-    )
+_deploy_commands.register_task_app_commands(task_app_group)
 
 
 def _determine_env_files(
@@ -2936,87 +2882,6 @@ def _serve_entry(
     )
 
 
-@task_app_group.command("deploy")
-@click.argument("app_id", type=str, required=False)
-@click.option("--name", "modal_name", default=None, help="Override Modal app name")
-@click.option("--dry-run", is_flag=True, help="Print modal deploy command without executing")
-@click.option("--modal-cli", default="modal", help="Path to modal CLI executable")
-@click.option(
-    "--env-file",
-    multiple=True,
-    type=click.Path(),
-    help="Env file to load into the container (can be repeated)",
-)
-def deploy_app(
-    app_id: str | None,
-    modal_name: str | None,
-    dry_run: bool,
-    modal_cli: str,
-    env_file: Sequence[str],
-) -> None:
-    """Deploy a task app to Modal."""
-
-    demo_dir_path = _load_demo_directory()
-    if demo_dir_path:
-        if not demo_dir_path.is_dir():
-            raise click.ClickException(
-                f"Demo directory not found: {demo_dir_path}\nRun 'synth-ai demo' to create a demo."
-            )
-        os.chdir(demo_dir_path)
-        click.echo(f"Using demo directory: {demo_dir_path}\n")
-
-    choice = _select_app_choice(app_id, purpose="deploy")
-
-    if choice.modal_script:
-        env_paths = _resolve_env_paths_for_script(choice.modal_script, env_file)
-        click.echo("Using env file(s): " + ", ".join(str(p.resolve()) for p in env_paths))
-        _run_modal_script(
-            choice.modal_script,
-            modal_cli,
-            "deploy",
-            env_paths,
-            modal_name=modal_name,
-            dry_run=dry_run,
-        )
-        return
-
-    entry = choice.ensure_entry()
-    _deploy_entry(entry, modal_name, dry_run, modal_cli, env_file, original_path=choice.path)
-
-
-@task_app_group.command("modal-serve")
-@click.argument("app_id", type=str, required=False)
-@click.option("--modal-cli", default="modal", help="Path to modal CLI executable")
-@click.option("--name", "modal_name", default=None, help="Override Modal app name (optional)")
-@click.option(
-    "--env-file",
-    multiple=True,
-    type=click.Path(),
-    help="Env file to load into the container (can be repeated)",
-)
-def modal_serve_app(
-    app_id: str | None, modal_cli: str, modal_name: str | None, env_file: Sequence[str]
-) -> None:
-    click.echo(f"[modal-serve] requested app_id={app_id or '(auto)'} modal_cli={modal_cli}")
-    try:
-        choice = _select_app_choice(app_id, purpose="modal-serve")
-    except SystemExit as exc:  # bubble up with context (legacy argparse would trigger this)
-        raise click.ClickException(
-            f"Legacy CLI intercepted modal-serve (exit {exc.code}). "
-            "Make sure you're running the Click CLI (synth_ai.cli:cli)."
-        ) from exc
-
-    if choice.modal_script:
-        env_paths = _resolve_env_paths_for_script(choice.modal_script, env_file)
-        click.echo("Using env file(s): " + ", ".join(str(p.resolve()) for p in env_paths))
-        _run_modal_script(choice.modal_script, modal_cli, "serve", env_paths, modal_name=modal_name)
-        return
-
-    entry = choice.ensure_entry()
-    click.echo(f"[modal-serve] serving entry {entry.app_id} from {choice.path}")
-    _modal_serve_entry(entry, modal_name, modal_cli, env_file, original_path=choice.path)
-
-
 def _write_modal_entrypoint(
     entry: TaskAppEntryType,
     modal_cfg: ModalDeploymentConfigType,
@@ -3260,1263 +3125,9 @@ def register(cli: click.Group) -> None:
     cli.add_command(filter_command)
 
 
-@click.command(
-    "eval",
-    help="Run one-off rollouts against a task app and print judge/eval summaries.",
-)
-@click.argument("app_id", type=str, required=False)
-@click.option(
-    "--config",
-    type=click.Path(),
-    default=None,
-    help="Path to eval TOML (short schema). Auto-discovers the first matching file when omitted.",
-)
-@click.option(
-    "--url",
-    "task_app_url",
-    type=str,
-    default=None,
-    help="Base URL of a running task app instead of spawning locally (requires --env-file for secrets).",
-)
-@click.option(
-    "--seeds",
-    default="0,1,2,3,4",
-    help="Comma-separated seeds/indices to evaluate. Use negative numbers to wrap around the dataset.",
-)
-@click.option("--split", default="train", show_default=True, help="Dataset split to use")
-@click.option(
-    "--model",
-    default=None,
-    help="Model identifier. When omitted the CLI will prompt based on task metadata.",
-)
-@click.option(
-    "--env-file",
-    multiple=True,
-    type=click.Path(),
-    help="Env file(s) to load (API keys, etc.). Required when using --url or remote judges.",
-)
-@click.option(
-    "--trace-db",
-    default="traces/v3/synth_ai.db",
-    show_default=True,
-    help="SQLite/Turso URL for storing rollout traces set to 'none' to disable persistence.",
-)
-@click.option(
-    "--metadata",
-    multiple=True,
-    help="Filter tasks by key=value metadata (e.g., --metadata difficulty=easy)",
-)
-@click.option(
-    "--metadata-sql",
-    default=None,
-    help="SQLite query that returns seeds to evaluate (e.g., SELECT seed FROM tasks WHERE difficulty='easy' LIMIT 5)",
-)
-def eval_command(
-    app_id: str | None,
-    config: str | None,
-    task_app_url: str | None,
-    seeds: str,
-    split: str,
-    model: str | None,
-    env_file: Sequence[str],
-    trace_db: str,
-    metadata: Sequence[str],
-    metadata_sql: str | None,
-) -> None:
-    """Run rollouts against a task app and report judge statistics.
-
-    By default the command spins up the selected task app in-process, executes the
-    requested seeds, and prints aggregate scores (official and custom judges). When
-    pointing at a remote `--url`, supply matching `--env-file` values so the CLI can
-    forward authentication headers to the running service.
-    """
-    # Parse and validate TOML config
-    from synth_ai.task.config import EvalConfig
-    
-    cfg: dict[str, Any] = {}
-    eval_cfg: EvalConfig | None = None
-    config_path: Path | None = None
-    
-    if config:
-        config_path = Path(config)
-    else:
-        auto_configs = _discover_eval_config_paths()
-        if auto_configs:
-            config_path = auto_configs[0]
-            click.echo(f"Using eval config: {config_path}")
-
-    if config_path:
-        if _toml is None:
-            raise click.ClickException(
-                "TOML parser not available; use Python 3.11+ or install tomli"
-            )
-        if not config_path.exists():
-            raise click.ClickException(f"Eval config not found: {config_path}")
-        try:
-            data = config_path.read_bytes()
-            parsed = _toml.loads(data.decode("utf-8"))
-            if isinstance(parsed, dict):
-                section = parsed.get("eval")
-                cfg = dict(section) if isinstance(section, dict) else dict(parsed)
-            
-            # Validate config with dataclass
-            try:
-                eval_cfg = EvalConfig.from_dict(cfg)
-                click.echo(f"✓ Config validated: {len(eval_cfg.seeds)} seeds, model={eval_cfg.model}")
-            except (ValueError, TypeError) as validation_error:
-                raise click.ClickException(f"Invalid eval config: {validation_error}") from validation_error
-        except click.ClickException:
-            raise
-        except Exception as exc:
-            raise click.ClickException(f"Failed to parse TOML '{config_path}': {exc}") from exc
-
-    # CLI args override config
-    if eval_cfg:
-        app_id = app_id or eval_cfg.app_id
-    else:
-        app_id = app_id or (cfg.get("app_id") if isinstance(cfg.get("app_id"), str) else None)  # type: ignore
-
-    metadata_filters: dict[str, str] = {}
-    if eval_cfg:
-        metadata_filters.update(eval_cfg.metadata)
-    else:
-        cfg_metadata = cfg.get("metadata")
-        if isinstance(cfg_metadata, dict):
-            for key, value in cfg_metadata.items():
-                metadata_filters[str(key)] = str(value)
-        elif isinstance(cfg_metadata, list):
-            for item in cfg_metadata:
-                if isinstance(item, str) and "=" in item:
-                    key, value = item.split("=", 1)
-                    metadata_filters[key.strip()] = value.strip()
-
-    for item in metadata or ():
-        if "=" not in item:
-            raise click.ClickException(f"Metadata filters must be key=value (got: {item})")
-        key, value = item.split("=", 1)
-        key = key.strip()
-        value = value.strip()
-        if not key or not value:
-            raise click.ClickException(f"Invalid metadata filter: {item}")
-        metadata_filters[key] = value
-
-    metadata_sql_query: str | None = None
-    if eval_cfg and eval_cfg.metadata_sql:
-        metadata_sql_query = eval_cfg.metadata_sql
-    else:
-        cfg_metadata_sql = cfg.get("metadata_sql")
-        if isinstance(cfg_metadata_sql, dict):
-            metadata_sql_query = cfg_metadata_sql.get("query") or cfg_metadata_sql.get("sql")
-        elif isinstance(cfg_metadata_sql, str):
-            metadata_sql_query = cfg_metadata_sql
-
-    if metadata_sql:
-        metadata_sql_query = metadata_sql
-    if metadata_sql_query is not None:
-        metadata_sql_query = str(metadata_sql_query)
-
-    trace_db_url: str | None = None
-    trace_db = (trace_db or "").strip()
-    if trace_db and trace_db.lower() not in {"none", "off", "disable"}:
-        if "://" in trace_db:
-            trace_db_url = trace_db
-        else:
-            trace_path = Path(trace_db).expanduser()
-            trace_path.parent.mkdir(parents=True, exist_ok=True)
-            trace_db_url = f"sqlite+aiosqlite:///{trace_path}"
-    trace_tracer: SessionTracer | None = SessionTracer(db_url=trace_db_url, auto_save=True) if trace_db_url else None
-
-    # Determine selection params (CLI takes precedence; TOML only fills unset model/seeds/env)
-    if cfg.get("model") and not model:
-        model = str(cfg["model"])  # type: ignore[index]
-    if cfg.get("seeds") and seeds == "0,1,2,3,4":
-        val = cfg["seeds"]
-        if isinstance(val, list):
-            with contextlib.suppress(Exception):
-                seeds = ",".join(str(int(x)) for x in val)
-        elif isinstance(val, str):
-            seeds = val
-        elif isinstance(val, int):
-            seeds = str(val)
-    if cfg.get("env_file") and not env_file:
-        ef = cfg["env_file"]
-        if isinstance(ef, str):
-            env_file = (ef,)  # type: ignore[assignment]
-        elif isinstance(ef, list):
-            env_file = tuple(str(x) for x in ef)  # type: ignore[assignment]
-
-    choice_for_env: AppChoice | None = None
-    entry: TaskAppEntryType | None = None
-    if task_app_url is None:
-        choice_for_env = _select_app_choice(app_id, purpose="eval")
-        entry = choice_for_env.ensure_entry()
-
-    env_paths: list[Path] = []
-    if entry is not None:
-        original_env_path = choice_for_env.path if choice_for_env is not None else None
-        env_paths = _determine_env_files(entry, env_file, original_path=original_env_path)
-    else:
-        if not env_file:
-            raise click.ClickException("--env-file is required when using --url")
-        for candidate in env_file:
-            p = Path(candidate).expanduser()
-            if not p.exists():
-                raise click.ClickException(f"Env file not found: {p}")
-            env_paths.append(p)
-
-    click.echo("Using env file(s): " + ", ".join(str(p) for p in env_paths))
-    _load_env_files_into_process([str(Path(p)) for p in env_paths])
-
-    if task_app_url is None:
-        config = entry.config_factory()  # type: ignore[union-attr]
-        # Help the type checker; runtime check also enforced in server.run_task_app
-        if not isinstance(config, TaskAppConfig):
-            raise click.ClickException(
-                "Invalid task app: config_factory did not return TaskAppConfig"
-            )
-        app = create_task_app(config)
-
-    # Determine supported models
-    inference_meta: dict[str, Any] = {}
-    supported: list[str] = []
-    seen_models: set[str] = set()
-
-    def _add_supported_model(candidate: Any) -> None:
-        if not candidate:
-            return
-        text = str(candidate).strip()
-        if not text or text in seen_models:
-            return
-        supported.append(text)
-        seen_models.add(text)
-
-    if task_app_url is None:
-        try:
-            if hasattr(config, "base_task_info") and config.base_task_info:
-                inf_obj = getattr(config.base_task_info, "inference", None)
-                if inf_obj is not None:
-                    if hasattr(inf_obj, "model_dump"):
-                        inference_meta = dict(inf_obj.model_dump(exclude_none=True))  # type: ignore[attr-defined]
-                    elif isinstance(inf_obj, dict):
-                        inference_meta = dict(inf_obj)
-        except Exception:
-            inference_meta = {}
-    else:
-        try:
-            import httpx as _hx
-
-            headers = {}
-            api_key = (os.environ.get("ENVIRONMENT_API_KEY") or "").strip()
-            if api_key:
-                headers["X-API-Key"] = api_key
-            with _hx.Client(base_url=task_app_url, headers=headers, timeout=15.0) as c:
-                info = c.get("/info").json()
-            inf = info.get("inference") if isinstance(info, dict) else None
-            if isinstance(inf, dict):
-                inference_meta = dict(inf)
-        except Exception:
-            inference_meta = {}
-
-    default_model = inference_meta.get("model")
-    if isinstance(default_model, str):
-        _add_supported_model(default_model)
-
-    models_field = inference_meta.get("models")
-    if isinstance(models_field, list):
-        for candidate in models_field:
-            _add_supported_model(candidate)
-
-    supported_models = inference_meta.get("supported_models")
-    if isinstance(supported_models, list):
-        for candidate in supported_models:
-            _add_supported_model(candidate)
-
-    providers = inference_meta.get("providers")
-    if isinstance(providers, list):
-        if "openai" in providers:
-            _add_supported_model("gpt-5")
-        if "groq" in providers:
-            _add_supported_model("groq:llama-3.1-70b-versatile")
-
-    _add_supported_model("synth:qwen-0.6b")
-
-    selected_model = model
-    if not selected_model:
-        if not supported:
-            raise click.ClickException(
-                "No supported models; supply --model or add base_task_info.inference.model"
-            )
-        click.echo("Select model to evaluate:")
-        for idx, m in enumerate(supported, start=1):
-            click.echo(f"  {idx}) {m}")
-        choice_idx = click.prompt("Enter choice", type=click.IntRange(1, len(supported)))
-        selected_model = supported[choice_idx - 1]
-
-    try:
-        seed_values = [int(s.strip()) for s in seeds.split(",") if s.strip()]
-    except Exception as exc:
-        raise click.ClickException("Invalid --seeds; expected comma-separated integers") from exc
-
-    import httpx
-
-    headers = {}
-    api_key = (os.environ.get("ENVIRONMENT_API_KEY") or "").strip()
-    if api_key:
-        headers["X-API-Key"] = api_key
-
-    # Precompute optional policy overrides from TOML
-    policy_overrides: dict[str, Any] = {}
-    try:
-        # Accept [eval.policy] table or top-level keys for convenience
-        if isinstance(cfg.get("policy"), dict):
-            policy_overrides.update(dict(cfg["policy"]))
-        # Back-compat: allow temperature/max_tokens at top level
-        for k in (
-            "temperature",
-            "max_tokens",
-            "reasoning_effort",
-            "system_hint",
-            "tool_choice",
-            "inference_url",
-        ):
-            if k in cfg and k not in policy_overrides:
-                policy_overrides[k] = cfg.get(k)
-    except Exception:
-        policy_overrides = {}
-
-    raw_concurrency = cfg.get("concurrency")
-    try:
-        concurrency_limit = int(raw_concurrency) if raw_concurrency is not None else 1
-    except Exception:
-        concurrency_limit = 1
-    if concurrency_limit <= 0:
-        concurrency_limit = 1
-    concurrency_limit = min(concurrency_limit, max(1, len(seed_values)))
-
-    judge_specs: list[JudgeSpec] = []
-
-    def _register_judge(name_hint: str | None, judge_cfg: dict[str, Any]) -> None:
-        if not judge_cfg:
-            return
-        judge_module = judge_cfg.get("module")
-        judge_path = judge_cfg.get("path")
-        judge_callable_name = judge_cfg.get("callable") or judge_cfg.get("function")
-        if judge_module and judge_path:
-            raise click.ClickException("Judge config cannot set both 'module' and 'path'")
-        if not judge_module and not judge_path:
-            raise click.ClickException("Judge config requires 'module' or 'path'")
-        try:
-            if judge_module:
-                module = importlib.import_module(str(judge_module))
-            else:
-                path = Path(str(judge_path)).expanduser()
-                if not path.exists():
-                    raise click.ClickException(f"Judge module path not found: {path}")
-                spec = importlib.util.spec_from_file_location(
-                    f"_eval_judge_{path.stem}", path
-                )
-                if not spec or not spec.loader:
-                    raise click.ClickException(f"Failed to load judge module from {path}")
-                module = importlib.util.module_from_spec(spec)
-                sys.modules[spec.name] = module
-                spec.loader.exec_module(module)
-        except click.ClickException:
-            raise
-        except Exception as exc:
-            raise click.ClickException(f"Unable to load judge module: {exc}") from exc
-
-        if judge_callable_name:
-            try:
-                judge_fn = getattr(module, str(judge_callable_name))
-            except AttributeError as exc:
-                raise click.ClickException(
-                    f"Judge callable '{judge_callable_name}' not found in module"
-                ) from exc
-        else:
-            if hasattr(module, "judge"):
-                judge_fn = module.judge
-            else:
-                raise click.ClickException("Judge module must expose 'judge' callable")
-
-        if not callable(judge_fn):
-            raise click.ClickException("Judge callable is not callable")
-
-        judge_kwargs = {
-            k: v
-            for k, v in judge_cfg.items()
-            if k not in {"module", "path", "callable", "function", "name"}
-        }
-        display_name = str(
-            judge_cfg.get("name")
-            or name_hint
-            or f"judge{len(judge_specs) + 1}"
-        )
-        judge_specs.append(JudgeSpec(display_name, judge_fn, judge_kwargs))
-
-    raw_judge_cfg = cfg.get("judge")
-    if isinstance(raw_judge_cfg, dict) and raw_judge_cfg:
-        direct_keys = {"module", "path", "callable", "function", "name"}
-        has_direct_keys = any(key in raw_judge_cfg for key in direct_keys)
-        nested_candidates = [
-            (key, value)
-            for key, value in raw_judge_cfg.items()
-            if isinstance(value, dict)
-        ]
-        if has_direct_keys and not nested_candidates:
-            _register_judge(None, raw_judge_cfg)
-        else:
-            for sub_name, sub_cfg in nested_candidates:
-                _register_judge(sub_name, sub_cfg)
-
-    raw_judges_list = cfg.get("judges")
-    if isinstance(raw_judges_list, list):
-        for _index, entry in enumerate(raw_judges_list, start=1):
-            if isinstance(entry, dict):
-                _register_judge(entry.get("name") or f"judge{len(judge_specs) + 1}", entry)
-
-    records: list[dict[str, Any]] = []
-
-    successes = 0
-    failures = 0
-    # Aggregate outcome stats across successful seeds
-    outcome_sum: float = 0.0
-    outcome_count: int = 0
-    outcome_correct: int = 0
-
-    def _build_task_rows(taskset: Any) -> dict[int, dict[str, Any]]:
-        rows: dict[int, dict[str, Any]] = {}
-        if not isinstance(taskset, dict):
-            return rows
-
-        scenario_ids = taskset.get("scenario_ids") or []
-        loop_ids = taskset.get("loop_ids") or []
-        thread_ids = taskset.get("thread_ids") or []
-        difficulty_map = taskset.get("difficulty_map") or {}
-
-        max_len = max(len(scenario_ids), len(loop_ids), len(thread_ids))
-        for seed in range(max_len):
-            scenario_id = scenario_ids[seed] if seed < len(scenario_ids) else None
-            loop_id = loop_ids[seed] if seed < len(loop_ids) else None
-            thread_id = thread_ids[seed] if seed < len(thread_ids) else None
-            difficulty = None
-            if isinstance(difficulty_map, dict):
-                if scenario_id and scenario_id in difficulty_map:
-                    difficulty = difficulty_map.get(scenario_id)
-                elif str(seed) in difficulty_map:
-                    difficulty = difficulty_map.get(str(seed))
-
-            rows[seed] = {
-                "seed": seed,
-                "scenario_id": scenario_id,
-                "loop_id": loop_id,
-                "thread_id": thread_id,
-                "difficulty": difficulty,
-            }
-        return rows
-
-    def _apply_metadata_filters(
-        rows: dict[int, dict[str, Any]], seeds_list: list[int], filters: dict[str, str]
-    ) -> list[int]:
-        if not filters:
-            return seeds_list
-        filtered: list[int] = []
-        for seed in seeds_list:
-            row = rows.get(seed)
-            if not row:
-                continue
-            include = True
-            for key, expected in filters.items():
-                actual = row.get(key)
-                if actual is None:
-                    include = False
-                    break
-                if str(actual).lower() != expected.lower():
-                    include = False
-                    break
-            if include:
-                filtered.append(seed)
-        return filtered
-
-    def _apply_metadata_sql(
-        rows: dict[int, dict[str, Any]], seeds_list: list[int], query: str
-    ) -> list[int]:
-        """Return seeds that satisfy an arbitrary SQL query.
-
-        The query is executed against an in-memory SQLite table named `tasks`
-        with columns (seed INTEGER, scenario_id TEXT, loop_id TEXT, thread_id TEXT, difficulty TEXT).
-        Any rows whose `seed` value (or first column if `seed` is absent) appear in the result set are retained.
-        """
-        if not query:
-            return seeds_list
-        conn = sqlite3.connect(":memory:")
-        try:
-            cur = conn.cursor()
-            cur.execute(
-                "CREATE TABLE tasks (seed INTEGER, scenario_id TEXT, loop_id TEXT, thread_id TEXT, difficulty TEXT)"
-            )
-            insert_stmt = (
-                "INSERT INTO tasks (seed, scenario_id, loop_id, thread_id, difficulty) VALUES (?,?,?,?,?)"
-            )
-            for seed in seeds_list:
-                row = rows.get(seed, {})
-                cur.execute(
-                    insert_stmt,
-                    [
-                        seed,
-                        row.get("scenario_id"),
-                        row.get("loop_id"),
-                        row.get("thread_id"),
-                        row.get("difficulty"),
-                    ],
-                )
-
-            result = cur.execute(query)
-            fetched = result.fetchall()
-            if not fetched:
-                return []
-            description = result.description or []
-            col_names = [col[0] for col in description]
-            seeds_out: list[int] = []
-            for entry in fetched:
-                value = entry[col_names.index("seed")] if "seed" in col_names else entry[0]
-                try:
-                    seeds_out.append(int(value))
-                except Exception as exc:
-                    raise click.ClickException(
-                        "metadata SQL query must return seed integers"
-                    ) from exc
-            seeds_set = set(seeds_out)
-            return [seed for seed in seeds_list if seed in seeds_set]
-        except sqlite3.Error as exc:
-            raise click.ClickException(f"Failed to execute metadata SQL query: {exc}") from exc
-        finally:
-            conn.close()
-
-    async def _run_eval() -> None:
-        nonlocal successes, failures, outcome_sum, outcome_count, outcome_correct, records, seed_values
-
-        if trace_tracer is not None and trace_tracer.db is None:
-            await trace_tracer.initialize()
-
-        if task_app_url is None:
-            transport = httpx.ASGITransport(app=app)  # type: ignore[name-defined]
-            async_client = httpx.AsyncClient(
-                transport=cast(Any, transport),
-                base_url="http://eval.local",
-                timeout=300.0,
-                follow_redirects=True,
-                headers=headers,
-            )
-        else:
-            async_client = httpx.AsyncClient(
-                base_url=task_app_url,
-                timeout=300.0,
-                follow_redirects=True,
-                headers=headers,
-            )
-
-        try:
-            taskset_payload: dict[str, Any] | None = None
-            try:
-                task_info_response = await async_client.get("/task_info")
-            except Exception:
-                task_info_response = None
-            if task_info_response is not None and task_info_response.status_code == 200:
-                with contextlib.suppress(Exception):
-                    payload_json = task_info_response.json()
-                if isinstance(payload_json, dict) and "taskset" in payload_json:
-                    taskset_payload = payload_json.get("taskset")
-                    if not isinstance(taskset_payload, dict):
-                        taskset_payload = None
-                elif isinstance(payload_json, dict):
-                    taskset_payload = payload_json
-
-            available_seeds = list(seed_values)
-            if metadata_sql_query or metadata_filters:
-                if not taskset_payload:
-                    raise click.ClickException(
-                        "Task metadata filters require the task app to expose /task_info metadata"
-                    )
-                rows = _build_task_rows(taskset_payload)
-                if metadata_sql_query:
-                    available_seeds = _apply_metadata_sql(rows, available_seeds, metadata_sql_query)
-                if metadata_filters:
-                    available_seeds = _apply_metadata_filters(rows, available_seeds, metadata_filters)
-                if not available_seeds:
-                    raise click.ClickException("No seeds match the provided metadata filters")
-                seed_values = available_seeds
-
-            semaphore = asyncio.Semaphore(concurrency_limit)
-
-            async def _run_seed(seed_val: int) -> None:
-                nonlocal successes, failures, outcome_sum, outcome_count, outcome_correct, records
-                # Read env_name and policy_name from config if available
-                env_name = cfg.get("env_name") or (cfg.get("env", {}).get("env_name") if isinstance(cfg.get("env"), dict) else None)
-                policy_name = cfg.get("policy_name") or (cfg.get("policy", {}).get("policy_name") if isinstance(cfg.get("policy"), dict) else None)
-                env_config_overrides = cfg.get("env_config", {}) if isinstance(cfg.get("env_config"), dict) else {}
-                policy_config_overrides = cfg.get("policy_config", {}) if isinstance(cfg.get("policy_config"), dict) else {}
-                
-                # Debug: print config parsing
-                if seed_val == 0:
-                    click.echo(f"[DEBUG] env_name from config: {env_name}")
-                    click.echo(f"[DEBUG] policy_name from config: {policy_name}")
-                
-                # Generate default ops sequence if not provided
-                max_llm_calls = policy_config_overrides.get("max_llm_calls", 10)
-                ops_list = cfg.get("ops", [])
-                if not ops_list:
-                    # Generate default "agent, env" pairs for max_llm_calls
-                    ops_list = ["agent", "env"] * int(max_llm_calls)
-                
-                body = {
-                    "run_id": str(uuid.uuid4()),
-                    "env": {"config": {"split": split, "index": seed_val, **env_config_overrides}, "seed": seed_val},
-                    "policy": {
-                        "policy_name": policy_name or selected_model,
-                        "config": {"model": selected_model, **policy_overrides, **policy_config_overrides},
-                    },
-                    "ops": ops_list,
-                    "record": {
-                        "return_trace": cfg.get("return_trace", True),
-                        "trace_format": cfg.get("trace_format", "structured"),
-                    },
-                    "mode": "eval",  # RolloutMode.EVAL: use inference URLs as-is, no transformations
-                }
-                if env_name:
-                    body["env"]["env_name"] = env_name
-                
-                # Debug: print the body being sent
-                if seed_val == 0:
-                    click.echo(f"[DEBUG] rollout body env: {body['env']}")
-                    click.echo(f"[DEBUG] rollout body policy: {body['policy']}")
-                    click.echo(f"[DEBUG] rollout body mode: {body.get('mode', 'NOT SET')}")
-                rollout_elapsed: float | None = None
-                rollout_start = time.perf_counter()
-                try:
-                    import logging
-                    _log = logging.getLogger(__name__)
-                    _log.info(f"[EVAL_BODY_DEBUG] Sending body with mode={body.get('mode')}")
-                    async with semaphore:
-                        response = await async_client.post("/rollout", json=body)
-                    rollout_elapsed = time.perf_counter() - rollout_start
-                except Exception as exc:
-                    failures += 1
-                    click.echo(f"seed={seed_val} error={exc}")
-                    return
-
-                ok = 200 <= response.status_code < 300
-                if ok:
-                    successes += 1
-                else:
-                    failures += 1
-
-                summary = [f"seed={seed_val}", f"status={response.status_code}"]
-                data: Any
-                try:
-                    data = response.json()
-                except Exception:
-                    data = None
-                
-                # Debug: print validation errors
-                if response.status_code == 422 and data:
-                    click.echo(f"[DEBUG] 422 Validation Error: {data}")
-
-                metrics: dict[str, Any] | None = None
-                completion: str | None = None
-                prompt_index: int | None = None
-                prompt_text: str | None = None
-                task_id: str | None = None
-                task_split: str | None = None
-                task_rubric_id: str | None = None
-
-                trace_namespace: dict[str, Any] | None = None
-                session_trace_dict: dict[str, Any] | None = None
-
-                if isinstance(data, dict):
-                    import logging
-                    _logger = logging.getLogger(__name__)
-                    _logger.info(f"[EVAL_DEBUG] Response data keys: {list(data.keys())}")
-                    if "detail" in data:
-                        _logger.error(f"[EVAL_DEBUG] Task app returned error: {data['detail']}")
-                    trace_namespace = data.get("trace")
-                    _logger.info(f"[EVAL_DEBUG] trace_namespace type: {type(trace_namespace)}, value: {trace_namespace if not isinstance(trace_namespace, dict) else 'dict with keys: ' + str(list(trace_namespace.keys()) if trace_namespace else 'None')}")
-                    if not isinstance(trace_namespace, dict):
-                        raise RuntimeError(
-                            "The 'synth-ai eval' command requires trace payloads in rollout responses. "
-                            "Ensure the rollout request includes 'trace_format': 'structured' and 'return_trace': true, "
-                            "and that task app tracing is enabled (TASKAPP_TRACING_ENABLED=1). "
-                            "Note: This is specific to the eval command - general rollout endpoints don't require traces."
-                        )
-                    # Handle both "compact" and "full" trace formats:
-                    # - compact: trace_namespace contains {session_id, metadata, ...}
-                    # - full: trace_namespace IS the full session_trace dict
-                    session_trace_dict = trace_namespace.get("session_trace")
-                    if not isinstance(session_trace_dict, dict):
-                        # If no session_trace key, assume "full" format where trace itself is the session_trace
-                        if "session_id" in trace_namespace:
-                            session_trace_dict = trace_namespace
-                        else:
-                            raise RuntimeError(
-                                "The 'synth-ai eval' command requires 'session_trace' in the trace payload or a valid full trace format. "
-                                "Ensure the task app is using tracing_v3 and returning structured trace data."
-                            )
-                    metrics = data.get("metrics") if isinstance(data.get("metrics"), dict) else None
-                    if metrics:
-                        mean_return = metrics.get("mean_return") or metrics.get("total_reward")
-                        outcome = metrics.get("outcome_score")
-                        if mean_return is not None:
-                            summary.append(f"mean_return={mean_return}")
-                        if outcome is not None:
-                            summary.append(f"outcome={outcome}")
-                            try:
-                                val = float(outcome)
-                                outcome_sum += val
-                                outcome_count += 1
-                                if val >= 0.5:
-                                    outcome_correct += 1
-                            except Exception:
-                                pass
-                    trajs = (
-                        data.get("trajectories")
-                        if isinstance(data.get("trajectories"), list)
-                        else None
-                    )
-                    if trajs:
-                        first = trajs[0] if trajs else None
-                        steps = first.get("steps") if isinstance(first, dict) else None
-                        if isinstance(steps, list) and steps:
-                            step0 = steps[0]
-                            tool_calls = step0.get("tool_calls") or step0.get("tools") or []
-                            if isinstance(tool_calls, list):
-                                summary.append(f"tool_calls={len(tool_calls)}")
-                            obs = step0.get("obs") if isinstance(step0, dict) else None
-                            if isinstance(obs, dict):
-                                idx_val = obs.get("prompt_index")
-                                if isinstance(idx_val, int):
-                                    prompt_index = idx_val
-                                prompt_raw = obs.get("prompt")
-                                if isinstance(prompt_raw, str):
-                                    prompt_text = prompt_raw
-                                if task_id is None:
-                                    candidate_id = obs.get("task_id")
-                                    if isinstance(candidate_id, str) and candidate_id:
-                                        task_id = candidate_id
-                                if task_split is None:
-                                    candidate_split = obs.get("task_split")
-                                    if isinstance(candidate_split, str) and candidate_split:
-                                        task_split = candidate_split
-                                if task_rubric_id is None:
-                                    candidate_rid = obs.get("task_rubric_id")
-                                    if isinstance(candidate_rid, str) and candidate_rid:
-                                        task_rubric_id = candidate_rid
-                        final = first.get("final") if isinstance(first, dict) else None
-                        if isinstance(final, dict):
-                            final_obs = final.get("observation")
-                            if isinstance(final_obs, dict):
-                                comp_val = final_obs.get("completion")
-                                if isinstance(comp_val, str):
-                                    completion = comp_val
-                                if task_id is None:
-                                    candidate_id = final_obs.get("task_id")
-                                    if isinstance(candidate_id, str) and candidate_id:
-                                        task_id = candidate_id
-                                if task_split is None:
-                                    candidate_split = final_obs.get("task_split")
-                                    if isinstance(candidate_split, str) and candidate_split:
-                                        task_split = candidate_split
-                                if task_rubric_id is None:
-                                    candidate_rid = final_obs.get("task_rubric_id")
-                                    if isinstance(candidate_rid, str) and candidate_rid:
-                                        task_rubric_id = candidate_rid
-                            final_info = final.get("info")
-                            if isinstance(final_info, dict):
-                                if task_id is None:
-                                    candidate_id = final_info.get("task_id")
-                                    if isinstance(candidate_id, str) and candidate_id:
-                                        task_id = candidate_id
-                                if task_split is None:
-                                    candidate_split = final_info.get("task_split")
-                                    if isinstance(candidate_split, str) and candidate_split:
-                                        task_split = candidate_split
-                                if task_rubric_id is None:
-                                    candidate_rid = final_info.get("task_rubric_id")
-                                    if isinstance(candidate_rid, str) and candidate_rid:
-                                        task_rubric_id = candidate_rid
-                    if task_id:
-                        summary.append(f"task_id={task_id}")
-                    click.echo(" ".join(summary))
-                    with contextlib.suppress(Exception):
-                        click.echo(json.dumps(data, indent=2))
-                else:
-                    click.echo(" ".join(summary))
-
-                official_score = None
-                if isinstance(metrics, dict):
-                    for key in ("mean_return", "total_reward", "outcome_score"):
-                        val = metrics.get(key)
-                        if isinstance(val, int | float):
-                            official_score = float(val)
-                            break
-                if official_score is None and isinstance(data, dict):
-                    try:
-                        reward_val = data["trajectories"][0]["steps"][0].get("reward")
-                        if isinstance(reward_val, int | float):
-                            official_score = float(reward_val)
-                    except Exception:
-                        pass
-
-                if official_score is not None:
-                    if official_score < 0.0:
-                        official_score = 0.0
-                    elif official_score > 1.0:
-                        official_score = min(1.0, official_score)
-
-                judge_scores: dict[str, float | None] = {}
-                judges_timings: dict[str, float | None] = {}
-                timings: dict[str, Any] = {
-                    "rollout_s": rollout_elapsed,
-                    "judges": judges_timings,
-                }
-                if judge_specs:
-                    for spec in judge_specs:
-                        score_value: float | None = None
-                        judge_elapsed: float | None = None
-                        # Run judges for all tasks (text-based and trajectory-based)
-                        # Text-based tasks have completion, trajectory-based tasks use response
-                        judge_payload = {
-                            "seed": seed_val,
-                            "prompt_index": prompt_index,
-                            "prompt": prompt_text,
-                            "completion": completion,
-                            "metrics": metrics,
-                            "response": data,
-                            "trace": trace_namespace,
-                        }
-                        try:
-                            judge_start = time.perf_counter()
-                            result = spec.fn(judge_payload, **spec.kwargs)
-                            judge_elapsed = time.perf_counter() - judge_start
-                            if isinstance(result, int | float):
-                                score_value = float(result)
-                        except Exception as exc:
-                            if judge_elapsed is None:
-                                judge_elapsed = time.perf_counter() - judge_start
-                            click.echo(f"seed={seed_val} judge[{spec.name}]_error={exc}")
-                        judges_timings[spec.name] = judge_elapsed
-                        judge_scores[spec.name] = score_value
-
-                if trace_tracer is not None and trace_namespace:
-                    storage_metadata = {
-                        "eval_seed": seed_val,
-                        "prompt_index": prompt_index,
-                        "task_id": task_id,
-                        "task_split": task_split,
-                        "task_rubric_id": task_rubric_id,
-                        "official_score": official_score,
-                        "judge_scores": judge_scores,
-                        "model": selected_model,
-                        "prompt": prompt_text,
-                        "completion": completion,
-                    }
-                    await _store_trace(trace_tracer, trace_namespace, storage_metadata)
-
-                records.append(
-                    {
-                        "seed": seed_val,
-                        "prompt_index": prompt_index,
-                        "task_id": task_id,
-                        "task_split": task_split,
-                        "task_rubric_id": task_rubric_id,
-                        "official_score": official_score,
-                        "judge_scores": judge_scores,
-                        "timings": timings,
-                    }
-                )
-
-            await asyncio.gather(*[_run_seed(seed_val) for seed_val in seed_values])
-        finally:
-            await async_client.aclose()
-
-    try:
-        asyncio.run(_run_eval())
-    finally:
-        if trace_tracer is not None and trace_tracer.db is not None:
-            asyncio.run(trace_tracer.db.close())
-
-    click.echo(
-        f"Eval complete: {successes} ok, {failures} failed; model={selected_model}, split={split}"
-    )
-
-    if outcome_count > 0:
-        mean_outcome = outcome_sum / float(outcome_count)
-        frac_right = outcome_correct / float(outcome_count)
-        click.echo(
-            f"Outcome summary: correct={outcome_correct}/{outcome_count} ({frac_right:.2%}), mean_outcome={mean_outcome:.3f}"
-        )
-
-    if records:
-        judge_specs = judge_specs or []  # ensure iterable
-        official_scores = [
-            r["official_score"] for r in records if r["official_score"] is not None
-        ]
-        if official_scores:
-            click.echo(f"  Official mean: {sum(official_scores) / len(official_scores):.3f}")
-        else:
-            click.echo("  Official mean: n/a")
-
-        for spec in judge_specs:
-            spec_scores = [
-                record["judge_scores"].get(spec.name)
-                for record in records
-                if record["judge_scores"].get(spec.name) is not None
-            ]
-            if spec_scores:
-                mean_spec = sum(spec_scores) / len(spec_scores)
-                click.echo(f"  [{spec.name}] mean: {mean_spec:.3f}")
-            else:
-                click.echo(f"  [{spec.name}] mean: n/a")
-
-            paired = [
-                (
-                    record["official_score"],
-                    record["judge_scores"].get(spec.name),
-                )
-                for record in records
-                if record["official_score"] is not None
-                and record["judge_scores"].get(spec.name) is not None
-            ]
-            if len(paired) >= 2:
-                corr = _pearson(
-                    [p[0] for p in paired if p[0] is not None],
-                    [p[1] for p in paired if p[1] is not None],
-                )
-                if corr is not None:
-                    click.echo(f"    Pearson r: {corr:.3f}")
-                else:
-                    click.echo("    Pearson r: undefined (zero variance)")
-            else:
-                click.echo("    Pearson r: n/a (need ≥2 paired scores)")
-
-        header = ["Seed", "Prompt", "Official"]
-        header.extend(spec.name for spec in judge_specs)
-        rows: list[list[str]] = []
-        for record in sorted(records, key=lambda r: (r["seed"], r.get("prompt_index") or -1)):
-            seed_val = str(record["seed"])
-            prompt_idx = (
-                str(record["prompt_index"])
-                if record["prompt_index"] is not None
-                else "-"
-            )
-            official_val = (
-                f"{record['official_score']:.3f}"
-                if record["official_score"] is not None
-                else "-"
-            )
-            row = [seed_val, prompt_idx, official_val]
-            for spec in judge_specs:
-                score_val = record["judge_scores"].get(spec.name)
-                row.append(f"{score_val:.3f}" if isinstance(score_val, int | float) else "-")
-            rows.append(row)
-
-        widths = [len(col) for col in header]
-        for row in rows:
-            for idx, cell in enumerate(row):
-                widths[idx] = max(widths[idx], len(cell))
-
-        click.echo("")
-        click.echo("  ".join(h.ljust(widths[idx]) for idx, h in enumerate(header)))
-        click.echo("  ".join("-" * widths[idx] for idx in range(len(header))))
-        for row in rows:
-            click.echo("  ".join(cell.ljust(widths[idx]) for idx, cell in enumerate(row)))
-
-
-
-@click.command(
-    "filter",
-    help="Export filtered tracing sessions to SFT-ready JSONL based on a TOML config.",
-)
-@click.option(
-    "--config",
-    "config_path",
-    type=click.Path(),
-    required=True,
-    help="Path to TOML config describing the input trace DB, score thresholds, and output JSONL.",
-)
-def filter_command(config_path: str) -> None:
-    """Render tracing sessions that match filter rules into SFT JSONL.
-
-    The TOML file should contain a `[filter]` table with at least:
-
-        db = \"path/to/traces.db\"      # sqlite path or URL (sqlite+aiosqlite://...)
-        output = \"ft_data/out.jsonl\"  # destination JSONL
-
-    Optional keys such as `splits`, `task_ids`, `models`, `min_official_score`, or
-    `min_judge_scores.my_judge = 0.7` allow you to narrow the dataset down to
-    high-quality traces. See `customers/agora_single_file/configs/filter_local.toml`
-    for a working example.
-    """
-    # Parse and validate TOML config
-    from synth_ai.task.config import FilterConfig
-    
-    if _toml is None:
-        raise click.ClickException("TOML parser not available; install tomli or use Python 3.11+")
-
-    cfg_path = Path(config_path)
-    if not cfg_path.exists():
-        raise click.ClickException(f"Filter config not found: {cfg_path}")
-
-    try:
-        config_data = _toml.loads(cfg_path.read_text(encoding="utf-8"))
-    except Exception as exc:
-        raise click.ClickException(f"Failed to parse TOML '{cfg_path}': {exc}") from exc
-
-    filter_cfg_dict = config_data.get("filter") if isinstance(config_data, dict) else None
-    if not isinstance(filter_cfg_dict, dict):
-        raise click.ClickException("Config must contain a [filter] table")
-
-    # Validate config with dataclass
-    try:
-        filter_cfg = FilterConfig.from_dict(filter_cfg_dict)
-        click.echo(f"✓ Config validated: db={filter_cfg.db}, output={filter_cfg.output}")
-        if filter_cfg.min_official_score is not None:
-            click.echo(f"  → Filtering for official score >= {filter_cfg.min_official_score}")
-        if filter_cfg.limit:
-            click.echo(f"  → Limiting to {filter_cfg.limit} examples")
-    except (ValueError, TypeError) as validation_error:
-        raise click.ClickException(f"Invalid filter config: {validation_error}") from validation_error
-
-    # Use validated config
-    db_url = filter_cfg.get_db_url()
-    output_path = filter_cfg.get_output_path()
-
-    # Extract validated fields from dataclass
-    splits = set(filter_cfg.splits)
-    task_ids = set(filter_cfg.task_ids)
-    models = set(filter_cfg.models)
-    min_official = filter_cfg.min_official_score
-    max_official = filter_cfg.max_official_score
-    min_judge_scores = filter_cfg.min_judge_scores
-    max_judge_scores = filter_cfg.max_judge_scores
-    # Note: min_created_at and max_created_at not yet in FilterConfig dataclass
-    min_created = _parse_datetime_for_trace(filter_cfg_dict.get("min_created_at"))
-    max_created = _parse_datetime_for_trace(filter_cfg_dict.get("max_created_at"))
-    limit = filter_cfg.limit
-
-    def _score_ok(value: Any, min_val: Any, max_val: Any) -> bool:
-        try:
-            if value is None:
-                return min_val is None
-            value = float(value)
-        except Exception:
-            return False
-        if min_val is not None and value < float(min_val):
-            return False
-        return not (max_val is not None and value > float(max_val))
-
-    async def _run_filter() -> None:
-        tracer = SessionTracer(db_url=db_url, auto_save=False)
-        await tracer.initialize()
-
-        df = await tracer.db.query_traces(
-            "SELECT session_id, created_at, metadata FROM session_traces ORDER BY created_at"
-        )
-        if getattr(df, "empty", True):
-            raise click.ClickException("No traces found in database")
-
-        sessions = df.to_dict("records")
-        accepted: list[dict[str, Any]] = []
-
-        for row in sessions:
-            metadata_raw = row.get("metadata")
-            if isinstance(metadata_raw, str):
-                try:
-                    metadata = json.loads(metadata_raw)
-                except Exception:
-                    metadata = {}
-            elif isinstance(metadata_raw, dict):
-                metadata = dict(metadata_raw)
-            else:
-                metadata = {}
-
-            created_at_raw = row.get("created_at")
-            created_at_dt = _parse_datetime_for_trace(created_at_raw)
-
-            session_id = row.get("session_id")
-
-            if splits and metadata.get("task_split") not in splits:
-                continue
-            if task_ids and metadata.get("task_id") not in task_ids:
-                continue
-            if models and metadata.get("model") not in models:
-                continue
-
-            if min_created and (created_at_dt is None or created_at_dt < min_created):
-                continue
-            if max_created and (created_at_dt is None or created_at_dt > max_created):
-                continue
-
-            # Check against outcome_rewards if score filter is set
-            total_reward = None
-            achievements_count = None
-            if min_official is not None or max_official is not None:
-                reward_query = "SELECT total_reward, achievements_count FROM outcome_rewards WHERE session_id = :session_id"
-                reward_rows = await tracer.db.query_traces(reward_query, {"session_id": session_id})
-                reward_records = reward_rows.to_dict("records") if hasattr(reward_rows, "to_dict") else []
-                if reward_records:
-                    total_reward = reward_records[0].get("total_reward")
-                    achievements_count = reward_records[0].get("achievements_count")
-                    if not _score_ok(total_reward, min_official, max_official):
-                        continue
-                elif min_official is not None:
-                    # No reward found, but score filter requires it
-                    continue
-
-            judge_scores = metadata.get("judge_scores") or {}
-            include = True
-            for judge_name, threshold in (min_judge_scores or {}).items():
-                if not _score_ok(judge_scores.get(judge_name), threshold, None):
-                    include = False
-                    break
-            if not include:
-                continue
-            for judge_name, threshold in (max_judge_scores or {}).items():
-                if not _score_ok(judge_scores.get(judge_name), None, threshold):
-                    include = False
-                    break
-            if not include:
-                continue
-
-            # Query messages for this session
-            messages_query = """
-                SELECT message_type, content, timestamp 
-                FROM messages 
-                WHERE session_id = :session_id
-                ORDER BY timestamp ASC, id ASC
-            """
-            msg_df = await tracer.db.query_traces(messages_query, {"session_id": session_id})
-            message_rows = msg_df.to_dict("records") if hasattr(msg_df, "to_dict") else []
-            
-            if not message_rows:
-                # Fallback: check if prompt/completion in metadata (old format)
-                prompt = metadata.get("prompt") or ""
-                completion = metadata.get("completion") or ""
-                if prompt and completion:
-                    record = {
-                        "messages": [
-                            {"role": "user", "content": str(prompt)},
-                            {"role": "assistant", "content": str(completion)},
-                        ],
-                        "metadata": {
-                            "session_id": session_id,
-                            "env_name": metadata.get("env_name"),
-                            "policy_name": metadata.get("policy_name"),
-                            "seed": metadata.get("seed"),
-                            "total_reward": total_reward,
-                            "achievements_count": achievements_count,
-                            "model": metadata.get("model"),
-                            "created_at": created_at_dt.isoformat() if created_at_dt else created_at_raw,
-                        },
-                    }
-                    accepted.append(record)
-                continue
-
-            # Extract user/assistant pairs from messages
-            for i, msg_row in enumerate(message_rows):
-                msg_type = msg_row.get("message_type")
-                content_raw = msg_row.get("content")
-                
-                # Look for user message
-                if msg_type in ("user", "policy_user_prompt"):
-                    # Find next policy_system_prompt or assistant
-                    assistant_msg = None
-                    for j in range(i + 1, len(message_rows)):
-                        next_type = message_rows[j].get("message_type")
-                        if next_type in ("assistant", "policy_system_prompt"):
-                            if next_type == "assistant":
-                                assistant_msg = message_rows[j]
-                            break
-                    
-                    # Parse content
-                    try:
-                        user_content = json.loads(content_raw) if isinstance(content_raw, str) else content_raw
-                    except Exception:
-                        user_content = content_raw
-                    
-                    # Extract text from structured content
-                    def extract_text(content: Any) -> str:
-                        if isinstance(content, str):
-                            return content
-                        if isinstance(content, dict):
-                            # Try payload.content for user prompts
-                            if "payload" in content and isinstance(content["payload"], dict):
-                                payload = content["payload"]
-                                if "content" in payload:
-                                    return extract_text(payload["content"])
-                            # Try common keys
-                            for key in ["text", "content", "content_text"]:
-                                if key in content:
-                                    val = content[key]
-                                    if isinstance(val, str):
-                                        return val
-                            return json.dumps(content)
-                        if isinstance(content, list):
-                            # Multimodal content - concatenate text parts
-                            parts = []
-                            for item in content:
-                                if isinstance(item, dict) and item.get("type") == "text":
-                                    parts.append(item.get("text", ""))
-                            return " ".join(parts) if parts else str(content)
-                        return str(content)
-                    
-                    user_text = extract_text(user_content)
-                    
-                    # For assistant, we might not have it recorded, so use tool calls as completion
-                    assistant_text = ""
-                    if assistant_msg:
-                        assistant_content_raw = assistant_msg.get("content")
-                        try:
-                            assistant_content = json.loads(assistant_content_raw) if isinstance(assistant_content_raw, str) else assistant_content_raw
-                        except Exception:
-                            assistant_content = assistant_content_raw
-                        assistant_text = extract_text(assistant_content)
-                    
-                    if not user_text:
-                        continue
-
-                    record = {
-                        "messages": [
-                            {"role": "user", "content": user_text},
-                            {"role": "assistant", "content": assistant_text if assistant_text else "[no response recorded]"},
-                        ],
-                        "metadata": {
-                            "session_id": session_id,
-                            "env_name": metadata.get("env_name"),
-                            "policy_name": metadata.get("policy_name"),
-                            "seed": metadata.get("seed"),
-                            "total_reward": total_reward,
-                            "achievements_count": achievements_count,
-                            "model": metadata.get("model"),
-                            "created_at": created_at_dt.isoformat() if created_at_dt else created_at_raw,
-                        },
-                    }
-                    accepted.append(record)
-
-        if not accepted:
-            raise click.ClickException("No sessions matched the provided filters")
-
-        if limit is not None and limit > 0:
-            accepted = accepted[:limit]
-
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        with output_path.open("w", encoding="utf-8") as handle:
-            for item in accepted:
-                handle.write(json.dumps(item, ensure_ascii=False))
-                handle.write("\n")
-
-        click.echo(f"Wrote {len(accepted)} examples -> {output_path}")
-        await tracer.db.close()
+eval_command = eval_core.command
 
-    asyncio.run(_run_filter())
+filter_command = filter_core.command
 
 
 def register_eval(cli: click.Group) -> None: