synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

synth_ai/cli/task_apps.py

@@ -1,23 +1,30 @@
 from __future__ import annotations
 
+import argparse
 import ast
 import asyncio
 import contextlib
+import functools
 import hashlib
 import importlib
 import importlib.util
 import inspect
 import json
 import os
+import shlex
 import shutil
 import signal
+import sqlite3
 import subprocess
 import sys
 import tempfile
 import textwrap
+import time
 import types
+import uuid
 from collections.abc import Callable, Iterable, Iterator, Sequence
 from dataclasses import dataclass
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, cast
 
@@ -25,24 +32,39 @@ try:  # Python 3.11+
     import tomllib as _toml
 except Exception:  # pragma: no cover - fallback
     _toml = None  # type: ignore
-import uuid
 
 import click
 from click.exceptions import Abort
 
+from synth_ai.tracing_v3 import (  # type: ignore[import-untyped]
+    BaseEvent,
+    EnvironmentEvent,
+    RuntimeEvent,
+    SessionEventMarkovBlanketMessage,
+    SessionMessageContent,
+    SessionTimeStep,
+    SessionTracer,
+    TimeRecord,
+)
+from synth_ai.tracing_v3 import (  # type: ignore[import-untyped]
+    SessionTrace as V3SessionTrace,
+)
+
 # ---------------------------------------------------------------------------
 # Dynamic imports to avoid hard dependencies during type checking.
 # ---------------------------------------------------------------------------
 ModalDeploymentConfigType = TaskAppConfigType = TaskAppEntryType = Any
 
 try:  # Resolve base URL defaults lazily
-    _config_module = importlib.import_module("synth_ai.config.base_url")
+    _config_module = cast(
+        Any, importlib.import_module("synth_ai.config.base_url")
+    )
     PROD_BASE_URL_DEFAULT = cast(str, _config_module.PROD_BASE_URL_DEFAULT)
 except Exception:  # pragma: no cover - fallback
     PROD_BASE_URL_DEFAULT = "https://agent-learning.onrender.com"
 
 try:
-    _task_apps_module = importlib.import_module("synth_ai.task.apps")
+    _task_apps_module = cast(Any, importlib.import_module("synth_ai.task.apps"))
     ModalDeploymentConfig = cast(
         type[ModalDeploymentConfigType], _task_apps_module.ModalDeploymentConfig
     )
@@ -53,9 +75,9 @@ except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load task app registry") from exc
 
 try:
-    _task_server_module = importlib.import_module("synth_ai.task.server")
-    create_task_app = _task_server_module.create_task_app
-    run_task_app = _task_server_module.run_task_app
+    _task_server_module = cast(Any, importlib.import_module("synth_ai.task.server"))
+    create_task_app = cast(Callable[..., Any], _task_server_module.create_task_app)
+    run_task_app = cast(Callable[..., Any], _task_server_module.run_task_app)
 except Exception as exc:  # pragma: no cover - critical dependency
     raise RuntimeError("Unable to load task app server utilities") from exc
 
@@ -64,10 +86,12 @@ def _load_demo_directory() -> Path | None:
     """Return the demo task apps directory if available."""
 
     try:
-        module = importlib.import_module("synth_ai.demos.demo_task_apps.core")
-        loader = module.load_demo_dir
+        module = cast(
+            Any, importlib.import_module("synth_ai.demos.demo_task_apps.core")
+        )
+        loader = cast(Callable[[], str | Path | None], module.load_demo_dir)
         demo_dir = loader()
-        if isinstance(demo_dir, (str, Path)):
+        if isinstance(demo_dir, str | Path):
             demo_path = Path(demo_dir)
             if demo_path.exists():
                 return demo_path.resolve()
@@ -105,6 +129,25 @@ DEFAULT_SEARCH_RELATIVE = (
 )
 
 
+def _pearson(xs: Sequence[float], ys: Sequence[float]) -> float | None:
+    if len(xs) != len(ys) or len(xs) < 2:
+        return None
+    mean_x = sum(xs) / len(xs)
+    mean_y = sum(ys) / len(ys)
+    num = 0.0
+    denom_x = 0.0
+    denom_y = 0.0
+    for x, y in zip(xs, ys, strict=False):
+        dx = x - mean_x
+        dy = y - mean_y
+        num += dx * dy
+        denom_x += dx * dx
+        denom_y += dy * dy
+    if denom_x <= 0 or denom_y <= 0:
+        return None
+    return num / (denom_x ** 0.5 * denom_y ** 0.5)
+
+
 @dataclass
 class AppChoice:
     app_id: str
@@ -128,6 +171,171 @@ class AppChoice:
         return entry
 
 
+@dataclass
+class JudgeSpec:
+    name: str
+    fn: Callable[..., Any]
+    kwargs: dict[str, Any]
+
+
+def _parse_datetime_for_trace(value: Any) -> datetime | None:
+    if isinstance(value, datetime):
+        return value if value.tzinfo else value.replace(tzinfo=UTC)
+    if isinstance(value, str):
+        value = value.replace("Z", "+00:00")
+        try:
+            dt = datetime.fromisoformat(value)
+        except ValueError:
+            try:
+                dt = datetime.fromtimestamp(float(value), tz=UTC)
+            except Exception:
+                return None
+        return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
+    if isinstance(value, int | float):
+        return datetime.fromtimestamp(float(value), tz=UTC)
+    return None
+
+
+def _time_record_from_dict(payload: dict[str, Any] | None) -> TimeRecord:
+    payload = payload or {}
+    event_time = payload.get("event_time")
+    if not isinstance(event_time, int | float):
+        try:
+            event_time = float(event_time)
+        except Exception:
+            event_time = float(time.time())
+    message_time = payload.get("message_time")
+    if message_time is not None:
+        try:
+            message_time = int(message_time)
+        except Exception:
+            message_time = None
+    return TimeRecord(event_time=event_time, message_time=message_time)
+
+
+def _event_from_dict(payload: dict[str, Any]) -> BaseEvent:
+    base_kwargs = {
+        "system_instance_id": payload.get("system_instance_id", ""),
+        "time_record": _time_record_from_dict(payload.get("time_record")),
+        "metadata": payload.get("metadata") or {},
+        "event_metadata": payload.get("event_metadata"),
+    }
+    if "actions" in payload:
+        return RuntimeEvent(actions=payload.get("actions") or [], **base_kwargs)
+    if any(key in payload for key in ("reward", "terminated", "truncated")):
+        return EnvironmentEvent(
+            reward=float(payload.get("reward", 0.0) or 0.0),
+            terminated=bool(payload.get("terminated", False)),
+            truncated=bool(payload.get("truncated", False)),
+            system_state_before=payload.get("system_state_before"),
+            system_state_after=payload.get("system_state_after"),
+            **base_kwargs,
+        )
+    return BaseEvent(**base_kwargs)
+
+
+def _markov_message_from_dict(payload: dict[str, Any]) -> SessionEventMarkovBlanketMessage:
+    content_payload = payload.get("content") or {}
+    content = SessionMessageContent(
+        text=content_payload.get("text"),
+        json_payload=content_payload.get("json_payload"),
+    )
+    raw_type = (payload.get("message_type") or "").lower()
+    if raw_type == "observation":
+        normalized_type = "system"
+    elif raw_type == "action":
+        normalized_type = "assistant"
+    elif raw_type in {"user", "assistant", "system", "tool_use", "tool_result"}:
+        normalized_type = raw_type
+    else:
+        normalized_type = "system"
+
+    return SessionEventMarkovBlanketMessage(
+        content=content,
+        message_type=normalized_type,
+        time_record=_time_record_from_dict(payload.get("time_record")),
+        metadata=payload.get("metadata") or {},
+    )
+
+
+def _step_from_dict(payload: dict[str, Any]) -> SessionTimeStep:
+    events = [
+        _event_from_dict(event)
+        for event in payload.get("events", [])
+        if isinstance(event, dict)
+    ]
+    messages = [
+        _markov_message_from_dict(msg)
+        for msg in payload.get("markov_blanket_messages", [])
+        if isinstance(msg, dict)
+    ]
+    timestamp = _parse_datetime_for_trace(payload.get("timestamp")) or datetime.now(UTC)
+    completed_at = _parse_datetime_for_trace(payload.get("completed_at"))
+    return SessionTimeStep(
+        step_id=payload.get("step_id", ""),
+        step_index=int(payload.get("step_index", 0) or 0),
+        timestamp=timestamp,
+        turn_number=payload.get("turn_number"),
+        events=events,
+        markov_blanket_messages=messages,
+        step_metadata=payload.get("step_metadata") or {},
+        completed_at=completed_at,
+    )
+
+
+def _session_trace_from_dict(payload: dict[str, Any]) -> V3SessionTrace | None:
+    if not isinstance(payload, dict):
+        return None
+    steps = [
+        _step_from_dict(step)
+        for step in payload.get("session_time_steps", [])
+        if isinstance(step, dict)
+    ]
+    events = [
+        _event_from_dict(event)
+        for event in payload.get("event_history", [])
+        if isinstance(event, dict)
+    ]
+    markov_history = [
+        _markov_message_from_dict(msg)
+        for msg in payload.get("markov_blanket_message_history", [])
+        if isinstance(msg, dict)
+    ]
+    created_at = _parse_datetime_for_trace(payload.get("created_at")) or datetime.now(UTC)
+    metadata = payload.get("metadata") or {}
+    session_metadata = payload.get("session_metadata")
+    return V3SessionTrace(
+        session_id=payload.get("session_id", ""),
+        created_at=created_at,
+        session_time_steps=steps,
+        event_history=events,
+        markov_blanket_message_history=markov_history,
+        metadata=metadata,
+        session_metadata=session_metadata,
+    )
+
+
+async def _store_trace(
+    tracer: SessionTracer | None,
+    trace_namespace: dict[str, Any] | None,
+    extra_metadata: dict[str, Any] | None = None,
+):
+    if tracer is None or not isinstance(trace_namespace, dict):
+        return
+    session_payload = trace_namespace.get("session_trace")
+    if not isinstance(session_payload, dict):
+        return
+    trace_obj = _session_trace_from_dict(session_payload)
+    if trace_obj is None:
+        return
+    if tracer.db is None:
+        await tracer.initialize()
+    meta = dict(trace_obj.metadata or {})
+    if extra_metadata:
+        meta.update(extra_metadata)
+    trace_obj.metadata = meta
+    await tracer.db.insert_session_trace(trace_obj)
+
 def _temporary_sys_path(paths: Sequence[Path]):
     """Context manager to prepend entries to sys.path temporarily."""
 
@@ -676,36 +884,44 @@ def _build_modal_config_from_ast(modal_call: ast.Call) -> ModalDeploymentConfigT
             elif kw.arg == "pip_packages" and isinstance(kw.value, (ast.List, ast.Tuple)):
                 # Handle pip_packages list/tuple
                 packages: list[str] = []
-                for elt in kw.value.elts:
-                    if isinstance(elt, ast.Constant):
-                        packages.append(elt.value)
+                value_node = kw.value
+                if isinstance(value_node, (ast.List, ast.Tuple)):
+                    for elt in value_node.elts:
+                        if isinstance(elt, ast.Constant):
+                            packages.append(elt.value)
                 kwargs[kw.arg] = tuple(packages)
             elif kw.arg == "extra_local_dirs" and isinstance(kw.value, (ast.List, ast.Tuple)):
                 # Handle extra_local_dirs list/tuple of tuples
                 dirs = []
-                for elt in kw.value.elts:
-                    if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
-                        src = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
-                        dst = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
-                        if src and dst:
-                            dirs.append((src, dst))
+                value_node = kw.value
+                if isinstance(value_node, (ast.List, ast.Tuple)):
+                    for elt in value_node.elts:
+                        if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
+                            src = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
+                            dst = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
+                            if src and dst:
+                                dirs.append((src, dst))
                 kwargs[kw.arg] = tuple(dirs)
             elif kw.arg == "secret_names" and isinstance(kw.value, (ast.List, ast.Tuple)):
                 # Handle secret_names list/tuple
                 secrets = []
-                for elt in kw.value.elts:
-                    if isinstance(elt, ast.Constant):
-                        secrets.append(elt.value)
+                value_node = kw.value
+                if isinstance(value_node, (ast.List, ast.Tuple)):
+                    for elt in value_node.elts:
+                        if isinstance(elt, ast.Constant):
+                            secrets.append(elt.value)
                 kwargs[kw.arg] = tuple(secrets)
             elif kw.arg == "volume_mounts" and isinstance(kw.value, (ast.List, ast.Tuple)):
                 # Handle volume_mounts list/tuple of tuples
                 mounts = []
-                for elt in kw.value.elts:
-                    if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
-                        name = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
-                        mount = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
-                        if name and mount:
-                            mounts.append((name, mount))
+                value_node = kw.value
+                if isinstance(value_node, (ast.List, ast.Tuple)):
+                    for elt in value_node.elts:
+                        if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
+                            name = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
+                            mount = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
+                            if name and mount:
+                                mounts.append((name, mount))
                 kwargs[kw.arg] = tuple(mounts)
 
         return ModalDeploymentConfig(**kwargs)
@@ -832,6 +1048,71 @@ def _import_task_app_module(
     return module
 
 
+@contextlib.contextmanager
+def _safe_import_context() -> Iterator[None]:
+    """Guard module imports against argparse/uvicorn side effects."""
+
+    original_argv = sys.argv[:]
+    sys.argv = [original_argv[0]] if original_argv else ["python"]
+
+    parser_cls = argparse.ArgumentParser
+    old_parse_args = parser_cls.parse_args
+
+    def _parse_noargs(self, args=None, namespace=None):  # type: ignore[override]
+        if args is None:
+            args = []
+        if namespace is None:
+            namespace = argparse.Namespace()
+        try:
+            return old_parse_args(self, args, namespace)
+        except SystemExit:
+            return namespace
+
+    parser_cls.parse_args = _parse_noargs  # type: ignore[assignment]
+
+    uvicorn_run = None
+    run_task_app_orig = None
+    try:
+        import uvicorn  # type: ignore
+
+        uvicorn_run = uvicorn.run
+        uvicorn.run = lambda *args, **kwargs: None  # type: ignore[assignment]
+    except Exception:
+        uvicorn_run = None
+
+    try:
+        _task_server_patch = cast(
+            Any, importlib.import_module("synth_ai.task.server")
+        )
+        run_task_app_orig = cast(Callable[..., Any], _task_server_patch.run_task_app)
+        _task_server_patch.run_task_app = (  # type: ignore[assignment]
+            lambda *args, **kwargs: None
+        )
+    except Exception:
+        run_task_app_orig = None
+
+    try:
+        yield
+    finally:
+        sys.argv = original_argv
+        parser_cls.parse_args = old_parse_args  # type: ignore[assignment]
+        if uvicorn_run is not None:
+            try:
+                import uvicorn  # type: ignore
+
+                uvicorn.run = uvicorn_run  # type: ignore[assignment]
+            except Exception:
+                pass
+        if run_task_app_orig is not None:
+            try:
+                _task_server_patch = cast(
+                    Any, importlib.import_module("synth_ai.task.server")
+                )
+                _task_server_patch.run_task_app = run_task_app_orig  # type: ignore[assignment]
+            except Exception:
+                pass
+
+
 def _load_entry_from_path(
     path: Path, app_id: str, module_search_roots: Sequence[Path] | None = None
 ) -> TaskAppEntryType:
@@ -859,13 +1140,14 @@ def _load_entry_from_path(
 
     for module_name, namespace_root in _possible_module_names(resolved, search_roots):
         try:
-            module = _import_task_app_module(
-                resolved,
-                module_name,
-                namespace_root=namespace_root,
-                sys_path_roots=search_roots,
-                ensure_namespace=True,
-            )
+            with _safe_import_context():
+                module = _import_task_app_module(
+                    resolved,
+                    module_name,
+                    namespace_root=namespace_root,
+                    sys_path_roots=search_roots,
+                    ensure_namespace=True,
+                )
             break
         except Exception as exc:  # pragma: no cover - best-effort fallbacks
             last_error = exc
@@ -874,13 +1156,14 @@ def _load_entry_from_path(
     if module is None:
         hashed_name = f"_synth_task_app_{hashlib.md5(str(resolved).encode(), usedforsecurity=False).hexdigest()}"
         try:
-            module = _import_task_app_module(
-                resolved,
-                hashed_name,
-                namespace_root=None,
-                sys_path_roots=search_roots,
-                ensure_namespace=False,
-            )
+            with _safe_import_context():
+                module = _import_task_app_module(
+                    resolved,
+                    hashed_name,
+                    namespace_root=None,
+                    sys_path_roots=search_roots,
+                    ensure_namespace=False,
+                )
         except Exception as exc:  # pragma: no cover - propagate meaningful error
             detail = last_error or exc
             raise click.ClickException(f"Failed to import {resolved}: {detail}") from detail
@@ -928,7 +1211,10 @@ def _load_entry_from_path(
             if has_required:
                 continue
             try:
-                result = attr()
+                with _safe_import_context():
+                    result = attr()
+            except SystemExit:
+                continue
             except Exception:
                 continue
             if isinstance(result, TaskAppConfig) and result.app_id == app_id:
@@ -1024,21 +1310,173 @@ def _resolve_env_paths_for_script(script_path: Path, explicit: Sequence[str]) ->
     return [env_candidates[choice - 1]]
 
 
+def _path_is_within(child: Path, parent: Path) -> bool:
+    try:
+        child.resolve().relative_to(parent.resolve())
+        return True
+    except Exception:
+        return False
+
+
+@functools.lru_cache(maxsize=16)
+def _is_modal_shim(path_str: str) -> bool:
+    """Return True if the candidate CLI path refers to the synth-ai shim."""
+
+    path = Path(path_str)
+    try:
+        resolved = path.resolve(strict=True)
+    except Exception:
+        resolved = path
+
+    if not resolved.exists() or resolved.is_dir():
+        return False
+
+    snippet = ""
+    try:
+        snippet = resolved.read_bytes()[:4096].decode("utf-8", errors="ignore")
+    except Exception:
+        snippet = ""
+
+    shim_markers = (
+        "synth_ai.cli._modal_wrapper",
+        "from modal.__main__ import main",
+        "import modal.__main__",
+        "run_module('modal.__main__'",
+    )
+    if snippet and any(marker in snippet for marker in shim_markers):
+        return True
+
+    try:
+        size = resolved.stat().st_size
+    except Exception:
+        size = None
+
+    if (
+        size is not None
+        and size < 2048
+        and "python" in (snippet.splitlines() or [""])[0]
+        and (
+            "modal.__main__" in snippet
+            or "modal.__main__" in snippet.replace(" ", "")
+        )
+    ):
+        return True
+
+    virtual_env = os.environ.get("VIRTUAL_ENV")
+    if virtual_env and _path_is_within(resolved, Path(virtual_env)):
+        return True
+
+    if _path_is_within(resolved, REPO_ROOT):
+        return True
+
+    uv_tools_dir = Path.home() / ".local" / "share" / "uv" / "tools"
+    return uv_tools_dir.exists() and _path_is_within(resolved, uv_tools_dir)
+
+
+def _find_modal_executable(modal_cli: str) -> tuple[str | None, str | None]:
+    """Return the first non-shim executable and the first shim discovered on PATH."""
+
+    if not modal_cli:
+        modal_cli = "modal"
+
+    candidate_path = Path(modal_cli).expanduser()
+    if candidate_path.is_absolute() or len(candidate_path.parts) > 1:
+        resolved_candidate = candidate_path
+        if not resolved_candidate.is_absolute():
+            resolved_candidate = (Path.cwd() / resolved_candidate).resolve()
+        else:
+            resolved_candidate = resolved_candidate.resolve()
+        if not resolved_candidate.exists():
+            raise click.ClickException(f"--modal-cli path does not exist: {resolved_candidate}")
+        if not os.access(resolved_candidate, os.X_OK):
+            raise click.ClickException(f"--modal-cli is not executable: {resolved_candidate}")
+        return str(resolved_candidate), None
+
+    path_env = os.environ.get("PATH", "")
+    if not path_env:
+        return None, None
+
+    seen_dirs: set[str] = set()
+    seen_candidates: set[str] = set()
+    shim_path: str | None = None
+
+    for raw_entry in path_env.split(os.pathsep):
+        if not raw_entry:
+            continue
+        try:
+            resolved_entry = str(Path(raw_entry).resolve())
+        except Exception:
+            resolved_entry = os.path.normpath(raw_entry)
+        if resolved_entry in seen_dirs:
+            continue
+        seen_dirs.add(resolved_entry)
+
+        candidate = shutil.which(modal_cli, path=raw_entry)
+        if candidate is None:
+            continue
+        if candidate in seen_candidates:
+            continue
+        seen_candidates.add(candidate)
+
+        if _is_modal_shim(candidate):
+            if shim_path is None:
+                shim_path = candidate
+            continue
+        return candidate, shim_path
+
+    return None, shim_path
+
+
 def _modal_command_prefix(modal_cli: str) -> list[str]:
     """Resolve a command prefix for invoking the Modal CLI within the active environment."""
-    if modal_cli == "modal" and importlib.util.find_spec("modal") is not None:
+
+    force_wrapper_env = os.environ.get("SYNTH_FORCE_MODAL_WRAPPER", "").strip().lower()
+    if force_wrapper_env in {"1", "true", "yes"}:
+        click.secho(
+            "[modal-prefix] SYNTH_FORCE_MODAL_WRAPPER=1 -> using in-process wrapper",
+            fg="yellow",
+        )
         return [sys.executable, "-m", "synth_ai.cli._modal_wrapper"]
 
-    modal_path = shutil.which(modal_cli)
-    if modal_path is not None:
-        return [modal_path]
+    lookup = modal_cli or "modal"
+    spec = importlib.util.find_spec("modal") if lookup == "modal" else None
+
+    preferred, shim_candidate = _find_modal_executable(lookup)
+    if preferred is not None:
+        detail = f"[modal-prefix] modal_cli={lookup} selected={preferred}"
+        if lookup == "modal":
+            detail += f" spec={'yes' if spec else 'no'}"
+        click.secho(detail, fg="cyan")
+        return [preferred]
+
+    if lookup != "modal":
+        raise click.ClickException(f"Modal CLI not found (looked for '{lookup}')")
+
+    if spec is not None:
+        warning = "[modal-prefix] Using synth-ai modal shim; pass --modal-cli /path/to/modal to override."
+        if shim_candidate is not None:
+            warning = (
+                f"[modal-prefix] Using synth-ai modal shim at {shim_candidate}; "
+                "pass --modal-cli /path/to/modal to override."
+            )
+        click.secho(warning, fg="yellow")
+        click.secho(
+            "[modal-prefix] modal_cli=modal selected=module-wrapper spec=yes",
+            fg="yellow",
+        )
+        return [sys.executable, "-m", "synth_ai.cli._modal_wrapper"]
 
-    if modal_cli == "modal":
+    if shim_candidate is not None:
         raise click.ClickException(
-            "Modal CLI not found. Install the 'modal' package in this environment or pass "
-            "--modal-cli with an explicit path."
+            "Modal CLI resolution found the synth-ai shim but the 'modal' package "
+            "is not importable in this environment. Install the official Modal CLI "
+            "or pass --modal-cli with its path."
         )
-    raise click.ClickException(f"Modal CLI not found (looked for '{modal_cli}')")
+
+    raise click.ClickException(
+        "Modal CLI not found. Install the 'modal' package in this environment or pass "
+        "--modal-cli with an explicit path."
+    )
 
 
 def _build_modal_app_wrapper(original_script: Path) -> tuple[Path, Path]:
@@ -1173,8 +1611,15 @@ def _run_modal_script(
     if modal_name and command == "deploy":
         cmd.extend(["--name", modal_name])
     if dry_run:
-        click.echo("Dry run: " + " ".join(cmd))
+        click.echo(
+            "Dry run: " + " ".join(shlex.quote(component) for component in cmd),
+            err=False,
+        )
         return
+    click.secho(
+        "[modal-exec] " + " ".join(shlex.quote(component) for component in cmd),
+        fg="cyan",
+    )
     try:
         # Stream output live for better diagnostics
         proc = subprocess.Popen(
@@ -1429,7 +1874,6 @@ def _run_modal_with_entry(
         inline_secret_values=inline_secret_values,
     )
     cmd = [*_modal_command_prefix(modal_cli), command, str(script_path)]
-
     if modal_name and command == "deploy":
         cmd.extend(["--name", modal_name])
 
@@ -1444,9 +1888,13 @@ def _run_modal_with_entry(
     proc_env["PYTHONPATH"] = os.pathsep.join(list(dict.fromkeys(pythonpath_entries)))
 
     if dry_run:
-        click.echo("Dry run: " + " ".join(cmd))
+        click.echo("Dry run: " + " ".join(shlex.quote(component) for component in cmd))
         script_path.unlink(missing_ok=True)
         return
+    click.secho(
+        "[modal-exec] " + " ".join(shlex.quote(component) for component in cmd),
+        fg="cyan",
+    )
 
     try:
         # Stream output live for better diagnostics
@@ -1531,6 +1979,10 @@ def _parse_env_file(path: Path) -> dict[str, str]:
 
 
 def _interactive_fill_env(env_path: Path) -> Path | None:
+    if not sys.stdin.isatty():
+        raise click.ClickException(
+            "ENVIRONMENT_API_KEY missing. Provide --env-file or run `synth-ai setup` in an interactive shell to create one."
+        )
     existing = _parse_env_file(env_path) if env_path.exists() else {}
 
     def _prompt(label: str, *, default: str = "", required: bool) -> str | None:
@@ -1570,6 +2022,10 @@ def _ensure_env_values(env_paths: list[Path], fallback_dir: Path) -> None:
     if (os.environ.get("ENVIRONMENT_API_KEY") or "").strip():
         return
     target = env_paths[0] if env_paths else (fallback_dir / ".env").resolve()
+    click.echo(
+        "⚠️  ENVIRONMENT_API_KEY not set. Run `uvx synth-ai setup`, "
+        "or pass --env-file pointing at a .env with ENVIRONMENT_API_KEY."
+    )
     result = _interactive_fill_env(target)
     if result is None:
         raise click.ClickException("ENVIRONMENT_API_KEY required to continue")
@@ -1593,7 +2049,7 @@ def _deploy_entry(
             f"Task app '{entry.app_id}' does not define Modal deployment settings"
         )
 
-    env_paths = _determine_env_files(entry, env_file)
+    env_paths = _determine_env_files(entry, env_file, original_path=original_path)
     click.echo("Using env file(s): " + ", ".join(str(p.resolve()) for p in env_paths))
     _run_modal_with_entry(
         entry,
@@ -1620,7 +2076,7 @@ def _modal_serve_entry(
             f"Task app '{entry.app_id}' does not define Modal deployment settings"
         )
 
-    env_paths = _determine_env_files(entry, env_file)
+    env_paths = _determine_env_files(entry, env_file, original_path=original_path)
     click.echo("Using env file(s): " + ", ".join(str(p.resolve()) for p in env_paths))
     _run_modal_with_entry(
         entry,
@@ -1651,6 +2107,255 @@ def list_apps() -> None:
         click.echo(f"- {entry.app_id}{aliases}: {entry.description}")
 
 
+@task_app_group.command("validate")
+@click.argument("app_id", type=str, required=True)
+@click.option(
+    "--url",
+    type=str,
+    default=None,
+    help="Task app URL to validate (if not provided, starts a local server)",
+)
+@click.option(
+    "--port",
+    type=int,
+    default=8765,
+    help="Port to use for temporary server (default: 8765)",
+)
+@click.option(
+    "--api-key",
+    type=str,
+    default=None,
+    envvar="ENVIRONMENT_API_KEY",
+    help="API key for authentication (default: $ENVIRONMENT_API_KEY)",
+)
+@click.option(
+    "--min-instances",
+    type=int,
+    default=10,
+    help="Minimum number of task instances required (default: 10)",
+)
+@click.option(
+    "--verbose",
+    "-v",
+    is_flag=True,
+    help="Show detailed information about the task app",
+)
+@click.option(
+    "--json",
+    "output_json",
+    is_flag=True,
+    help="Output results as JSON",
+)
+def validate_task_app_cmd(
+    app_id: str,
+    url: str | None,
+    port: int,
+    api_key: str | None,
+    min_instances: int,
+    verbose: bool,
+    output_json: bool,
+) -> None:
+    """Validate a task app deployment readiness.
+    
+    This command verifies that a task app is properly configured and ready to run
+    by checking all required HTTP endpoints, authentication, and task availability.
+    
+    By default, it starts a temporary local server for validation. You can also
+    validate a remote deployment by passing --url.
+    
+    \b
+    What gets validated:
+    • Root endpoint (/) responds correctly
+    • Health endpoint (/health) is accessible with proper authentication
+    • Info endpoint (/info) returns valid task metadata  
+    • Task info endpoint (/task_info) provides task instances
+    • Rollout endpoint (/rollout) is registered
+    • At least N task instances are available (default: 10)
+    
+    \b
+    Examples:
+    
+    \b
+    Validate grpo-crafter (starts local server automatically):
+        $ synth-ai task-app validate grpo-crafter
+    
+    \b
+    Validate sokoban with verbose output:
+        $ synth-ai task-app validate sokoban --verbose
+    
+    \b
+    Validate with custom port:
+        $ synth-ai task-app validate sokoban --port 9000
+    
+    \b
+    Validate a remote deployment:
+        $ synth-ai task-app validate grpo-crafter --url https://my-crafter.modal.run
+    
+    \b
+    Require at least 20 task instances:
+        $ synth-ai task-app validate grpo-crafter --min-instances 20
+    
+    \b
+    Get JSON output for automation:
+        $ synth-ai task-app validate sokoban --json
+    
+    \b
+    Common use cases:
+    • Pre-deployment verification: Check task app works before deploying to Modal
+    • CI/CD integration: Use --json flag for automated validation in pipelines
+    • Debug failing deployments: Use --verbose to see detailed endpoint responses
+    • Test API key configuration: Verify authentication is set up correctly
+    """
+    import asyncio
+    import socket
+    import subprocess
+    import tempfile
+    import time
+    
+    # Import the validate_task_app function defined in this module
+    from synth_ai.cli._validate_task_app import validate_task_app  # type: ignore[attr-defined]
+    
+    proc = None
+    task_app_url = url
+    
+    try:
+        # If no URL provided, start a temporary server
+        if not task_app_url:
+            # Find an available port
+            def is_port_available(port: int) -> bool:
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                    try:
+                        s.bind(("", port))
+                        return True
+                    except OSError:
+                        return False
+            
+            while not is_port_available(port):
+                port += 1
+            
+            task_app_url = f"http://localhost:{port}"
+            
+            if not output_json:
+                click.echo(f"Starting temporary {app_id} server on port {port}...")
+            
+            # Start the server in background
+            env = os.environ.copy()
+            if api_key:
+                env["ENVIRONMENT_API_KEY"] = api_key
+            
+            # Create a temporary trace DB and trace dir to avoid prompts
+            import tempfile
+            temp_dir = tempfile.mkdtemp()
+            temp_trace_db = os.path.join(temp_dir, "validate_trace.db")
+            temp_trace_dir = os.path.join(temp_dir, "traces")
+            os.makedirs(temp_trace_dir, exist_ok=True)
+            
+            proc = subprocess.Popen(
+                [
+                    "uv",
+                    "run",
+                    "synth-ai",
+                    "task-app",
+                    "serve",
+                    app_id,
+                    "--port",
+                    str(port),
+                    "--no-reload",
+                    "--trace",
+                    temp_trace_dir,
+                    "--trace-db",
+                    temp_trace_db,
+                ],
+                env=env,
+                stdin=subprocess.PIPE,  # Add stdin to handle any prompts
+                stdout=subprocess.DEVNULL if output_json else subprocess.PIPE,
+                stderr=subprocess.DEVNULL if output_json else subprocess.PIPE,
+                text=True,
+            )
+            
+            # Write empty input to stdin to skip any prompts
+            if proc.stdin:
+                try:
+                    proc.stdin.write("\n")
+                    proc.stdin.flush()
+                    proc.stdin.close()
+                except Exception:
+                    pass
+            
+            # Wait for server to be ready
+            if not output_json:
+                click.echo("Waiting for server to start...")
+            
+            import httpx
+            for _attempt in range(60):  # 30 seconds timeout
+                try:
+                    async def check_health():
+                        async with httpx.AsyncClient(timeout=2.0) as client:
+                            resp = await client.get(f"{task_app_url}/")
+                            return resp.status_code == 200
+                    
+                    if asyncio.run(check_health()):
+                        break
+                except Exception:
+                    pass
+                
+                # Check if process died
+                if proc.poll() is not None:
+                    stderr_output = ""
+                    if proc.stderr and not output_json:
+                        stderr_output = proc.stderr.read()
+                    click.echo(click.style("✗ Server process exited unexpectedly", fg="red"), err=True)
+                    if stderr_output and not output_json:
+                        click.echo(f"Error output:\n{stderr_output}", err=True)
+                    sys.exit(1)
+                
+                time.sleep(0.5)
+            else:
+                click.echo(click.style("✗ Server failed to start within 30 seconds", fg="red"), err=True)
+                sys.exit(1)
+            
+            if not output_json:
+                click.echo(click.style("✓ Server started", fg="green"))
+                click.echo()
+        
+        # Ensure URL doesn't have trailing slash
+        task_app_url = task_app_url.rstrip("/")
+        
+        async def _run() -> tuple[bool, dict[str, Any]]:
+            return await validate_task_app(
+                url=task_app_url,
+                api_key=api_key,
+                min_instances=min_instances,
+                verbose=verbose,
+            )
+        
+        success, results = asyncio.run(_run())
+        
+        if output_json:
+            import json as _json
+            click.echo(_json.dumps(results, indent=2))
+        
+        sys.exit(0 if success else 1)
+    
+    finally:
+        # Cleanup: stop the temporary server
+        if proc is not None:
+            if not output_json:
+                click.echo("\nStopping temporary server...")
+            try:
+                proc.terminate()
+                proc.wait(timeout=5)
+            except Exception:
+                proc.kill()
+        
+        # Cleanup temp trace DB
+        if not url and 'temp_dir' in locals():
+            import contextlib
+            import shutil
+            with contextlib.suppress(Exception):
+                shutil.rmtree(temp_dir, ignore_errors=True)
+
+
 def _load_env_files_into_process(paths: Sequence[str]) -> None:
     for p in paths:
         try:
@@ -1907,7 +2612,9 @@ def serve_task_group(
     )
 
 
-def _determine_env_files(entry: TaskAppEntryType, user_env_files: Sequence[str]) -> list[Path]:
+def _determine_env_files(
+    entry: TaskAppEntryType, user_env_files: Sequence[str], *, original_path: Path | None = None
+) -> list[Path]:
     resolved: list[Path] = []
     for candidate in user_env_files:
         p = Path(candidate).expanduser()
@@ -1917,30 +2624,46 @@ def _determine_env_files(entry: TaskAppEntryType, user_env_files: Sequence[str])
     if resolved:
         return resolved
 
-    # Always prompt for env file selection instead of auto-loading defaults
-    # Look for env files in current working directory first, then repo root
-    cwd = Path.cwd()
-    env_candidates = []
-
-    # Add CWD env files first (prioritized)
-    cwd_env_files = sorted(cwd.glob("**/*.env"))
-    env_candidates.extend(cwd_env_files)
+    declared: list[Path] = []
+    for candidate in getattr(entry, "env_files", ()) or ():
+        try:
+            p = Path(candidate).expanduser()
+        except Exception:
+            continue
+        if p.exists() and p.is_file():
+            declared.append(p)
+    if declared:
+        return declared
 
-    # Add repo root env files
-    repo_env_files = sorted(REPO_ROOT.glob("**/*.env"))
-    # Avoid duplicates
-    for repo_file in repo_env_files:
-        if repo_file not in env_candidates:
-            env_candidates.append(repo_file)
+    def _append_candidate(collection: list[Path], candidate: Path) -> None:
+        if candidate.exists() and candidate.is_file() and candidate not in collection:
+            collection.append(candidate)
 
-    if not env_candidates:
-        raise click.ClickException("No env file found. Pass --env-file explicitly.")
+    auto_candidates: list[Path] = []
 
-    click.echo("Select env file to load:")
-    for idx, path in enumerate(env_candidates, start=1):
-        click.echo(f"  {idx}) {path.resolve()}")
-    choice = click.prompt("Enter choice", type=click.IntRange(1, len(env_candidates)), default=1)
-    return [env_candidates[choice - 1]]
+    search_dirs: list[Path] = []
+    if original_path is not None:
+        search_dirs.append(original_path.parent.resolve())
+        for parent in original_path.parent.resolve().parents:
+            search_dirs.append(parent)
+    cwd = Path.cwd().resolve()
+    if cwd not in search_dirs:
+        search_dirs.append(cwd)
+    repo_root = REPO_ROOT.resolve()
+    if repo_root not in search_dirs:
+        search_dirs.append(repo_root)
+
+    for directory in search_dirs:
+        _append_candidate(auto_candidates, directory / ".env")
+        for candidate in sorted(directory.glob("*.env")):
+            _append_candidate(auto_candidates, candidate)
+
+    if auto_candidates:
+        return [auto_candidates[0]]
+
+    raise click.ClickException(
+        "No .env file discovered automatically. Pass --env-file /path/to/.env or generate one with `uvx synth-ai setup`."
+    )
 
 
 def _ensure_port_free(port: int, host: str, *, force: bool) -> None:
@@ -2242,7 +2965,14 @@ def deploy_app(
 def modal_serve_app(
     app_id: str | None, modal_cli: str, modal_name: str | None, env_file: Sequence[str]
 ) -> None:
-    choice = _select_app_choice(app_id, purpose="modal-serve")
+    click.echo(f"[modal-serve] requested app_id={app_id or '(auto)'} modal_cli={modal_cli}")
+    try:
+        choice = _select_app_choice(app_id, purpose="modal-serve")
+    except SystemExit as exc:  # bubble up with context (legacy argparse would trigger this)
+        raise click.ClickException(
+            f"Legacy CLI intercepted modal-serve (exit {exc.code}). "
+            "Make sure you're running the Click CLI (synth_ai.cli:cli)."
+        ) from exc
 
     if choice.modal_script:
         env_paths = _resolve_env_paths_for_script(choice.modal_script, env_file)
@@ -2251,6 +2981,7 @@ def modal_serve_app(
         return
 
     entry = choice.ensure_entry()
+    click.echo(f"[modal-serve] serving entry {entry.app_id} from {choice.path}")
     _modal_serve_entry(entry, modal_name, modal_cli, env_file, original_path=choice.path)
 
 
@@ -2480,22 +3211,60 @@ def register(cli: click.Group) -> None:
     cli.add_command(serve_command)
     cli.add_command(task_app_group)
     cli.add_command(eval_command)
+    cli.add_command(filter_command)
 
 
-@click.command("eval")
+@click.command(
+    "eval",
+    help="Run one-off rollouts against a task app and print judge/eval summaries.",
+)
 @click.argument("app_id", type=str, required=False)
-@click.option("--config", type=click.Path(), default=None, help="Path to eval TOML (short schema)")
+@click.option(
+    "--config",
+    type=click.Path(),
+    default=None,
+    help="Path to eval TOML (short schema). Auto-discovers the first matching file when omitted.",
+)
 @click.option(
     "--url",
     "task_app_url",
     type=str,
     default=None,
-    help="Base URL of a running task app (skip in-process server)",
+    help="Base URL of a running task app instead of spawning locally (requires --env-file for secrets).",
+)
+@click.option(
+    "--seeds",
+    default="0,1,2,3,4",
+    help="Comma-separated seeds/indices to evaluate. Use negative numbers to wrap around the dataset.",
 )
-@click.option("--seeds", default="0,1,2,3,4", help="Comma-separated seeds/indices to evaluate")
 @click.option("--split", default="train", show_default=True, help="Dataset split to use")
-@click.option("--model", default=None, help="Model identifier (prompted if omitted)")
-@click.option("--env-file", multiple=True, type=click.Path(), help="Env file(s) for keys")
+@click.option(
+    "--model",
+    default=None,
+    help="Model identifier. When omitted the CLI will prompt based on task metadata.",
+)
+@click.option(
+    "--env-file",
+    multiple=True,
+    type=click.Path(),
+    help="Env file(s) to load (API keys, etc.). Required when using --url or remote judges.",
+)
+@click.option(
+    "--trace-db",
+    default="traces/v3/eval_traces.db",
+    show_default=True,
+    help="SQLite/Turso URL for storing rollout traces set to 'none' to disable persistence.",
+)
+@click.option(
+    "--metadata",
+    multiple=True,
+    help="Filter tasks by key=value metadata (e.g., --metadata difficulty=easy)",
+)
+@click.option(
+    "--metadata-sql",
+    default=None,
+    help="SQLite query that returns seeds to evaluate (e.g., SELECT seed FROM tasks WHERE difficulty='easy' LIMIT 5)",
+)
 def eval_command(
     app_id: str | None,
     config: str | None,
@@ -2504,8 +3273,17 @@ def eval_command(
     split: str,
     model: str | None,
     env_file: Sequence[str],
+    trace_db: str,
+    metadata: Sequence[str],
+    metadata_sql: str | None,
 ) -> None:
-    """Run local rollouts against a task app using in-process ASGI and summarize results."""
+    """Run rollouts against a task app and report judge statistics.
+
+    By default the command spins up the selected task app in-process, executes the
+    requested seeds, and prints aggregate scores (official and custom judges). When
+    pointing at a remote `--url`, supply matching `--env-file` values so the CLI can
+    forward authentication headers to the running service.
+    """
     cfg: dict[str, Any] = {}
     config_path: Path | None = None
     if config:
@@ -2534,6 +3312,50 @@ def eval_command(
 
     app_id = app_id or (cfg.get("app_id") if isinstance(cfg.get("app_id"), str) else None)  # type: ignore
 
+    metadata_filters: dict[str, str] = {}
+    cfg_metadata = cfg.get("metadata")
+    if isinstance(cfg_metadata, dict):
+        for key, value in cfg_metadata.items():
+            metadata_filters[str(key)] = str(value)
+    elif isinstance(cfg_metadata, list):
+        for item in cfg_metadata:
+            if isinstance(item, str) and "=" in item:
+                key, value = item.split("=", 1)
+                metadata_filters[key.strip()] = value.strip()
+
+    for item in metadata or ():
+        if "=" not in item:
+            raise click.ClickException(f"Metadata filters must be key=value (got: {item})")
+        key, value = item.split("=", 1)
+        key = key.strip()
+        value = value.strip()
+        if not key or not value:
+            raise click.ClickException(f"Invalid metadata filter: {item}")
+        metadata_filters[key] = value
+
+    metadata_sql_query: str | None = None
+    cfg_metadata_sql = cfg.get("metadata_sql")
+    if isinstance(cfg_metadata_sql, dict):
+        metadata_sql_query = cfg_metadata_sql.get("query") or cfg_metadata_sql.get("sql")
+    elif isinstance(cfg_metadata_sql, str):
+        metadata_sql_query = cfg_metadata_sql
+
+    if metadata_sql:
+        metadata_sql_query = metadata_sql
+    if metadata_sql_query is not None:
+        metadata_sql_query = str(metadata_sql_query)
+
+    trace_db_url: str | None = None
+    trace_db = (trace_db or "").strip()
+    if trace_db and trace_db.lower() not in {"none", "off", "disable"}:
+        if "://" in trace_db:
+            trace_db_url = trace_db
+        else:
+            trace_path = Path(trace_db).expanduser()
+            trace_path.parent.mkdir(parents=True, exist_ok=True)
+            trace_db_url = f"sqlite+aiosqlite:///{trace_path}"
+    trace_tracer: SessionTracer | None = SessionTracer(db_url=trace_db_url, auto_save=True) if trace_db_url else None
+
     # Determine selection params (CLI takes precedence; TOML only fills unset model/seeds/env)
     if cfg.get("model") and not model:
         model = str(cfg["model"])  # type: ignore[index]
@@ -2553,14 +3375,16 @@ def eval_command(
         elif isinstance(ef, list):
             env_file = tuple(str(x) for x in ef)  # type: ignore[assignment]
 
+    choice_for_env: AppChoice | None = None
     entry: TaskAppEntryType | None = None
     if task_app_url is None:
-        choice = _select_app_choice(app_id, purpose="eval")
-        entry = choice.ensure_entry()
+        choice_for_env = _select_app_choice(app_id, purpose="eval")
+        entry = choice_for_env.ensure_entry()
 
     env_paths: list[Path] = []
     if entry is not None:
-        env_paths = _determine_env_files(entry, env_file)
+        original_env_path = choice_for_env.path if choice_for_env is not None else None
+        env_paths = _determine_env_files(entry, env_file, original_path=original_env_path)
     else:
         if not env_file:
             raise click.ClickException("--env-file is required when using --url")
@@ -2583,12 +3407,30 @@ def eval_command(
         app = create_task_app(config)
 
     # Determine supported models
+    inference_meta: dict[str, Any] = {}
     supported: list[str] = []
+    seen_models: set[str] = set()
+
+    def _add_supported_model(candidate: Any) -> None:
+        if not candidate:
+            return
+        text = str(candidate).strip()
+        if not text or text in seen_models:
+            return
+        supported.append(text)
+        seen_models.add(text)
+
     if task_app_url is None:
         try:
-            supported = list((config.base_task_info.inference or {}).get("models") or [])  # type: ignore[union-attr]
+            if hasattr(config, "base_task_info") and config.base_task_info:
+                inf_obj = getattr(config.base_task_info, "inference", None)
+                if inf_obj is not None:
+                    if hasattr(inf_obj, "model_dump"):
+                        inference_meta = dict(inf_obj.model_dump(exclude_none=True))  # type: ignore[attr-defined]
+                    elif isinstance(inf_obj, dict):
+                        inference_meta = dict(inf_obj)
         except Exception:
-            supported = []
+            inference_meta = {}
     else:
         try:
             import httpx as _hx
@@ -2601,38 +3443,38 @@ def eval_command(
                 info = c.get("/info").json()
             inf = info.get("inference") if isinstance(info, dict) else None
             if isinstance(inf, dict):
-                m = inf.get("models")
-                if isinstance(m, list):
-                    supported = [str(x) for x in m]
-                if not supported:
-                    providers = inf.get("providers")
-                    if isinstance(providers, list):
-                        if "openai" in providers:
-                            supported.append("gpt-5")
-                        if "groq" in providers:
-                            supported.append("groq:llama-3.1-70b-versatile")
-                        supported.append("synth:qwen-0.6b")
+                inference_meta = dict(inf)
         except Exception:
-            supported = []
-    if not supported:
-        # Only fall back to local config-derived providers when running in-process
-        if task_app_url is None:
-            try:
-                providers = list((config.base_task_info.inference or {}).get("providers") or [])  # type: ignore[union-attr]
-            except Exception:
-                providers = []
-            if "openai" in providers:
-                supported.append("gpt-5")
-            if "groq" in providers:
-                supported.append("groq:llama-3.1-70b-versatile")
-        # Always include a local synth model option for smoke tests
-        supported.append("synth:qwen-0.6b")
+            inference_meta = {}
+
+    default_model = inference_meta.get("model")
+    if isinstance(default_model, str):
+        _add_supported_model(default_model)
+
+    models_field = inference_meta.get("models")
+    if isinstance(models_field, list):
+        for candidate in models_field:
+            _add_supported_model(candidate)
+
+    supported_models = inference_meta.get("supported_models")
+    if isinstance(supported_models, list):
+        for candidate in supported_models:
+            _add_supported_model(candidate)
+
+    providers = inference_meta.get("providers")
+    if isinstance(providers, list):
+        if "openai" in providers:
+            _add_supported_model("gpt-5")
+        if "groq" in providers:
+            _add_supported_model("groq:llama-3.1-70b-versatile")
+
+    _add_supported_model("synth:qwen-0.6b")
 
     selected_model = model
     if not selected_model:
         if not supported:
             raise click.ClickException(
-                "No supported models; supply --model or add base_task_info.inference.models"
+                "No supported models; supply --model or add base_task_info.inference.model"
             )
         click.echo("Select model to evaluate:")
         for idx, m in enumerate(supported, start=1):
@@ -2652,70 +3494,347 @@ def eval_command(
     if api_key:
         headers["X-API-Key"] = api_key
 
+    # Precompute optional policy overrides from TOML
+    policy_overrides: dict[str, Any] = {}
+    try:
+        # Accept [eval.policy] table or top-level keys for convenience
+        if isinstance(cfg.get("policy"), dict):
+            policy_overrides.update(dict(cfg["policy"]))
+        # Back-compat: allow temperature/max_tokens at top level
+        for k in (
+            "temperature",
+            "max_tokens",
+            "reasoning_effort",
+            "system_hint",
+            "tool_choice",
+            "inference_url",
+        ):
+            if k in cfg and k not in policy_overrides:
+                policy_overrides[k] = cfg.get(k)
+    except Exception:
+        policy_overrides = {}
+
+    raw_concurrency = cfg.get("concurrency")
+    try:
+        concurrency_limit = int(raw_concurrency) if raw_concurrency is not None else 1
+    except Exception:
+        concurrency_limit = 1
+    if concurrency_limit <= 0:
+        concurrency_limit = 1
+    concurrency_limit = min(concurrency_limit, max(1, len(seed_values)))
+
+    judge_specs: list[JudgeSpec] = []
+
+    def _register_judge(name_hint: str | None, judge_cfg: dict[str, Any]) -> None:
+        if not judge_cfg:
+            return
+        judge_module = judge_cfg.get("module")
+        judge_path = judge_cfg.get("path")
+        judge_callable_name = judge_cfg.get("callable") or judge_cfg.get("function")
+        if judge_module and judge_path:
+            raise click.ClickException("Judge config cannot set both 'module' and 'path'")
+        if not judge_module and not judge_path:
+            raise click.ClickException("Judge config requires 'module' or 'path'")
+        try:
+            if judge_module:
+                module = importlib.import_module(str(judge_module))
+            else:
+                path = Path(str(judge_path)).expanduser()
+                if not path.exists():
+                    raise click.ClickException(f"Judge module path not found: {path}")
+                spec = importlib.util.spec_from_file_location(
+                    f"_eval_judge_{path.stem}", path
+                )
+                if not spec or not spec.loader:
+                    raise click.ClickException(f"Failed to load judge module from {path}")
+                module = importlib.util.module_from_spec(spec)
+                sys.modules[spec.name] = module
+                spec.loader.exec_module(module)
+        except click.ClickException:
+            raise
+        except Exception as exc:
+            raise click.ClickException(f"Unable to load judge module: {exc}") from exc
+
+        if judge_callable_name:
+            try:
+                judge_fn = getattr(module, str(judge_callable_name))
+            except AttributeError as exc:
+                raise click.ClickException(
+                    f"Judge callable '{judge_callable_name}' not found in module"
+                ) from exc
+        else:
+            if hasattr(module, "judge"):
+                judge_fn = module.judge
+            else:
+                raise click.ClickException("Judge module must expose 'judge' callable")
+
+        if not callable(judge_fn):
+            raise click.ClickException("Judge callable is not callable")
+
+        judge_kwargs = {
+            k: v
+            for k, v in judge_cfg.items()
+            if k not in {"module", "path", "callable", "function", "name"}
+        }
+        display_name = str(
+            judge_cfg.get("name")
+            or name_hint
+            or f"judge{len(judge_specs) + 1}"
+        )
+        judge_specs.append(JudgeSpec(display_name, judge_fn, judge_kwargs))
+
+    raw_judge_cfg = cfg.get("judge")
+    if isinstance(raw_judge_cfg, dict) and raw_judge_cfg:
+        direct_keys = {"module", "path", "callable", "function", "name"}
+        has_direct_keys = any(key in raw_judge_cfg for key in direct_keys)
+        nested_candidates = [
+            (key, value)
+            for key, value in raw_judge_cfg.items()
+            if isinstance(value, dict)
+        ]
+        if has_direct_keys and not nested_candidates:
+            _register_judge(None, raw_judge_cfg)
+        else:
+            for sub_name, sub_cfg in nested_candidates:
+                _register_judge(sub_name, sub_cfg)
+
+    raw_judges_list = cfg.get("judges")
+    if isinstance(raw_judges_list, list):
+        for _index, entry in enumerate(raw_judges_list, start=1):
+            if isinstance(entry, dict):
+                _register_judge(entry.get("name") or f"judge{len(judge_specs) + 1}", entry)
+
+    records: list[dict[str, Any]] = []
+
     successes = 0
     failures = 0
     # Aggregate outcome stats across successful seeds
     outcome_sum: float = 0.0
     outcome_count: int = 0
     outcome_correct: int = 0
-    if task_app_url is None:
-        transport = httpx.ASGITransport(app=app)  # type: ignore[name-defined]
-        # Newer httpx types consider ASGITransport under httpx._transports; cast to satisfy type checker
-        client = httpx.Client(
-            transport=cast(Any, transport),
-            base_url="http://eval.local",
-            timeout=60.0,
-            headers=headers,
-        )
-    else:
-        client = httpx.Client(base_url=task_app_url, timeout=60.0, headers=headers)
-    try:
-        with contextlib.suppress(Exception):
-            client.get("/task_info")
-        # Precompute optional policy overrides from TOML
-        policy_overrides: dict[str, Any] = {}
-        try:
-            # Accept [eval.policy] table or top-level keys for convenience
-            if isinstance(cfg.get("policy"), dict):
-                policy_overrides.update(dict(cfg["policy"]))
-            # Back-compat: allow temperature/max_tokens at top level
-            for k in (
-                "temperature",
-                "max_tokens",
-                "reasoning_effort",
-                "system_hint",
-                "tool_choice",
-            ):
-                if k in cfg and k not in policy_overrides:
-                    policy_overrides[k] = cfg.get(k)
-        except Exception:
-            policy_overrides = {}
-
-        for seed_val in seed_values:
-            body = {
-                "run_id": str(uuid.uuid4()),
-                "env": {"config": {"split": split, "index": seed_val}, "seed": seed_val},
-                "policy": {
-                    "policy_name": selected_model,
-                    "config": {"model": selected_model, **policy_overrides},
-                },
-                "ops": [],
+
+    def _build_task_rows(taskset: Any) -> dict[int, dict[str, Any]]:
+        rows: dict[int, dict[str, Any]] = {}
+        if not isinstance(taskset, dict):
+            return rows
+
+        scenario_ids = taskset.get("scenario_ids") or []
+        loop_ids = taskset.get("loop_ids") or []
+        thread_ids = taskset.get("thread_ids") or []
+        difficulty_map = taskset.get("difficulty_map") or {}
+
+        max_len = max(len(scenario_ids), len(loop_ids), len(thread_ids))
+        for seed in range(max_len):
+            scenario_id = scenario_ids[seed] if seed < len(scenario_ids) else None
+            loop_id = loop_ids[seed] if seed < len(loop_ids) else None
+            thread_id = thread_ids[seed] if seed < len(thread_ids) else None
+            difficulty = None
+            if isinstance(difficulty_map, dict):
+                if scenario_id and scenario_id in difficulty_map:
+                    difficulty = difficulty_map.get(scenario_id)
+                elif str(seed) in difficulty_map:
+                    difficulty = difficulty_map.get(str(seed))
+
+            rows[seed] = {
+                "seed": seed,
+                "scenario_id": scenario_id,
+                "loop_id": loop_id,
+                "thread_id": thread_id,
+                "difficulty": difficulty,
             }
+        return rows
+
+    def _apply_metadata_filters(
+        rows: dict[int, dict[str, Any]], seeds_list: list[int], filters: dict[str, str]
+    ) -> list[int]:
+        if not filters:
+            return seeds_list
+        filtered: list[int] = []
+        for seed in seeds_list:
+            row = rows.get(seed)
+            if not row:
+                continue
+            include = True
+            for key, expected in filters.items():
+                actual = row.get(key)
+                if actual is None:
+                    include = False
+                    break
+                if str(actual).lower() != expected.lower():
+                    include = False
+                    break
+            if include:
+                filtered.append(seed)
+        return filtered
+
+    def _apply_metadata_sql(
+        rows: dict[int, dict[str, Any]], seeds_list: list[int], query: str
+    ) -> list[int]:
+        """Return seeds that satisfy an arbitrary SQL query.
+
+        The query is executed against an in-memory SQLite table named `tasks`
+        with columns (seed INTEGER, scenario_id TEXT, loop_id TEXT, thread_id TEXT, difficulty TEXT).
+        Any rows whose `seed` value (or first column if `seed` is absent) appear in the result set are retained.
+        """
+        if not query:
+            return seeds_list
+        conn = sqlite3.connect(":memory:")
+        try:
+            cur = conn.cursor()
+            cur.execute(
+                "CREATE TABLE tasks (seed INTEGER, scenario_id TEXT, loop_id TEXT, thread_id TEXT, difficulty TEXT)"
+            )
+            insert_stmt = (
+                "INSERT INTO tasks (seed, scenario_id, loop_id, thread_id, difficulty) VALUES (?,?,?,?,?)"
+            )
+            for seed in seeds_list:
+                row = rows.get(seed, {})
+                cur.execute(
+                    insert_stmt,
+                    [
+                        seed,
+                        row.get("scenario_id"),
+                        row.get("loop_id"),
+                        row.get("thread_id"),
+                        row.get("difficulty"),
+                    ],
+                )
+
+            result = cur.execute(query)
+            fetched = result.fetchall()
+            if not fetched:
+                return []
+            description = result.description or []
+            col_names = [col[0] for col in description]
+            seeds_out: list[int] = []
+            for entry in fetched:
+                value = entry[col_names.index("seed")] if "seed" in col_names else entry[0]
+                try:
+                    seeds_out.append(int(value))
+                except Exception as exc:
+                    raise click.ClickException(
+                        "metadata SQL query must return seed integers"
+                    ) from exc
+            seeds_set = set(seeds_out)
+            return [seed for seed in seeds_list if seed in seeds_set]
+        except sqlite3.Error as exc:
+            raise click.ClickException(f"Failed to execute metadata SQL query: {exc}") from exc
+        finally:
+            conn.close()
+
+    async def _run_eval() -> None:
+        nonlocal successes, failures, outcome_sum, outcome_count, outcome_correct, records, seed_values
+
+        if trace_tracer is not None and trace_tracer.db is None:
+            await trace_tracer.initialize()
+
+        if task_app_url is None:
+            transport = httpx.ASGITransport(app=app)  # type: ignore[name-defined]
+            async_client = httpx.AsyncClient(
+                transport=cast(Any, transport),
+                base_url="http://eval.local",
+                timeout=300.0,
+                follow_redirects=True,
+                headers=headers,
+            )
+        else:
+            async_client = httpx.AsyncClient(
+                base_url=task_app_url,
+                timeout=300.0,
+                follow_redirects=True,
+                headers=headers,
+            )
+
+        try:
+            taskset_payload: dict[str, Any] | None = None
             try:
-                resp = client.post("/rollout", json=body)
-                ok = 200 <= resp.status_code < 300
+                task_info_response = await async_client.get("/task_info")
+            except Exception:
+                task_info_response = None
+            if task_info_response is not None and task_info_response.status_code == 200:
+                with contextlib.suppress(Exception):
+                    payload_json = task_info_response.json()
+                if isinstance(payload_json, dict) and "taskset" in payload_json:
+                    taskset_payload = payload_json.get("taskset")
+                    if not isinstance(taskset_payload, dict):
+                        taskset_payload = None
+                elif isinstance(payload_json, dict):
+                    taskset_payload = payload_json
+
+            available_seeds = list(seed_values)
+            if metadata_sql_query or metadata_filters:
+                if not taskset_payload:
+                    raise click.ClickException(
+                        "Task metadata filters require the task app to expose /task_info metadata"
+                    )
+                rows = _build_task_rows(taskset_payload)
+                if metadata_sql_query:
+                    available_seeds = _apply_metadata_sql(rows, available_seeds, metadata_sql_query)
+                if metadata_filters:
+                    available_seeds = _apply_metadata_filters(rows, available_seeds, metadata_filters)
+                if not available_seeds:
+                    raise click.ClickException("No seeds match the provided metadata filters")
+                seed_values = available_seeds
+
+            semaphore = asyncio.Semaphore(concurrency_limit)
+
+            async def _run_seed(seed_val: int) -> None:
+                nonlocal successes, failures, outcome_sum, outcome_count, outcome_correct, records
+                body = {
+                    "run_id": str(uuid.uuid4()),
+                    "env": {"config": {"split": split, "index": seed_val}, "seed": seed_val},
+                    "policy": {
+                        "policy_name": selected_model,
+                        "config": {"model": selected_model, **policy_overrides},
+                    },
+                    "ops": [],
+                }
+                rollout_elapsed: float | None = None
+                rollout_start = time.perf_counter()
+                try:
+                    async with semaphore:
+                        response = await async_client.post("/rollout", json=body)
+                    rollout_elapsed = time.perf_counter() - rollout_start
+                except Exception as exc:
+                    failures += 1
+                    click.echo(f"seed={seed_val} error={exc}")
+                    return
+
+                ok = 200 <= response.status_code < 300
                 if ok:
                     successes += 1
                 else:
                     failures += 1
 
-                # Print summary with any available metrics/tool calls
-                summary = [f"seed={seed_val}", f"status={resp.status_code}"]
+                summary = [f"seed={seed_val}", f"status={response.status_code}"]
+                data: Any
                 try:
-                    data = resp.json()
+                    data = response.json()
                 except Exception:
                     data = None
+
+                metrics: dict[str, Any] | None = None
+                completion: str | None = None
+                prompt_index: int | None = None
+                prompt_text: str | None = None
+                task_id: str | None = None
+                task_split: str | None = None
+                task_rubric_id: str | None = None
+
+                trace_namespace: dict[str, Any] | None = None
+                session_trace_dict: dict[str, Any] | None = None
+
                 if isinstance(data, dict):
+                    trace_namespace = data.get("trace")
+                    if not isinstance(trace_namespace, dict):
+                        raise RuntimeError(
+                            "rollout response missing trace payload; task app must return tracing_v3 data"
+                        )
+                    session_trace_dict = trace_namespace.get("session_trace")
+                    if not isinstance(session_trace_dict, dict):
+                        raise RuntimeError(
+                            "rollout response trace missing 'session_trace'; ensure the task app is serving the tracing_v3 build"
+                        )
                     metrics = data.get("metrics") if isinstance(data.get("metrics"), dict) else None
                     if metrics:
                         mean_return = metrics.get("mean_return") or metrics.get("total_reward")
@@ -2724,7 +3843,6 @@ def eval_command(
                             summary.append(f"mean_return={mean_return}")
                         if outcome is not None:
                             summary.append(f"outcome={outcome}")
-                            # Aggregate outcome stats
                             try:
                                 val = float(outcome)
                                 outcome_sum += val
@@ -2733,7 +3851,6 @@ def eval_command(
                                     outcome_correct += 1
                             except Exception:
                                 pass
-                    # Try to infer tool call count from first trajectory step
                     trajs = (
                         data.get("trajectories")
                         if isinstance(data.get("trajectories"), list)
@@ -2747,38 +3864,163 @@ def eval_command(
                             tool_calls = step0.get("tool_calls") or step0.get("tools") or []
                             if isinstance(tool_calls, list):
                                 summary.append(f"tool_calls={len(tool_calls)}")
+                            obs = step0.get("obs") if isinstance(step0, dict) else None
+                            if isinstance(obs, dict):
+                                idx_val = obs.get("prompt_index")
+                                if isinstance(idx_val, int):
+                                    prompt_index = idx_val
+                                prompt_raw = obs.get("prompt")
+                                if isinstance(prompt_raw, str):
+                                    prompt_text = prompt_raw
+                                if task_id is None:
+                                    candidate_id = obs.get("task_id")
+                                    if isinstance(candidate_id, str) and candidate_id:
+                                        task_id = candidate_id
+                                if task_split is None:
+                                    candidate_split = obs.get("task_split")
+                                    if isinstance(candidate_split, str) and candidate_split:
+                                        task_split = candidate_split
+                                if task_rubric_id is None:
+                                    candidate_rid = obs.get("task_rubric_id")
+                                    if isinstance(candidate_rid, str) and candidate_rid:
+                                        task_rubric_id = candidate_rid
+                        final = first.get("final") if isinstance(first, dict) else None
+                        if isinstance(final, dict):
+                            final_obs = final.get("observation")
+                            if isinstance(final_obs, dict):
+                                comp_val = final_obs.get("completion")
+                                if isinstance(comp_val, str):
+                                    completion = comp_val
+                                if task_id is None:
+                                    candidate_id = final_obs.get("task_id")
+                                    if isinstance(candidate_id, str) and candidate_id:
+                                        task_id = candidate_id
+                                if task_split is None:
+                                    candidate_split = final_obs.get("task_split")
+                                    if isinstance(candidate_split, str) and candidate_split:
+                                        task_split = candidate_split
+                                if task_rubric_id is None:
+                                    candidate_rid = final_obs.get("task_rubric_id")
+                                    if isinstance(candidate_rid, str) and candidate_rid:
+                                        task_rubric_id = candidate_rid
+                            final_info = final.get("info")
+                            if isinstance(final_info, dict):
+                                if task_id is None:
+                                    candidate_id = final_info.get("task_id")
+                                    if isinstance(candidate_id, str) and candidate_id:
+                                        task_id = candidate_id
+                                if task_split is None:
+                                    candidate_split = final_info.get("task_split")
+                                    if isinstance(candidate_split, str) and candidate_split:
+                                        task_split = candidate_split
+                                if task_rubric_id is None:
+                                    candidate_rid = final_info.get("task_rubric_id")
+                                    if isinstance(candidate_rid, str) and candidate_rid:
+                                        task_rubric_id = candidate_rid
+                    if task_id:
+                        summary.append(f"task_id={task_id}")
                     click.echo(" ".join(summary))
-                    # Print the full response JSON (trace, trajectories, metrics)
                     with contextlib.suppress(Exception):
                         click.echo(json.dumps(data, indent=2))
                 else:
                     click.echo(" ".join(summary))
-            except Exception as exc:
-                failures += 1
-                click.echo(f"seed={seed_val} error={exc}")
 
-    finally:
-        try:
-            client.close()
-        except AttributeError:
-            transport_obj = getattr(client, "_transport", None)
-            if transport_obj and hasattr(transport_obj, "aclose"):
-                try:
-                    asyncio.run(transport_obj.aclose())
-                except RuntimeError:
-                    # Fallback when already inside a running loop (rare for CLI).
-                    new_loop = asyncio.new_event_loop()
+                official_score = None
+                if isinstance(metrics, dict):
+                    for key in ("mean_return", "total_reward", "outcome_score"):
+                        val = metrics.get(key)
+                        if isinstance(val, int | float):
+                            official_score = float(val)
+                            break
+                if official_score is None and isinstance(data, dict):
                     try:
-                        new_loop.run_until_complete(transport_obj.aclose())
-                    finally:
-                        new_loop.close()
-        except Exception:
-            pass
+                        reward_val = data["trajectories"][0]["steps"][0].get("reward")
+                        if isinstance(reward_val, int | float):
+                            official_score = float(reward_val)
+                    except Exception:
+                        pass
+
+                if official_score is not None:
+                    if official_score < 0.0:
+                        official_score = 0.0
+                    elif official_score > 1.0:
+                        official_score = min(1.0, official_score)
+
+                judge_scores: dict[str, float | None] = {}
+                judges_timings: dict[str, float | None] = {}
+                timings: dict[str, Any] = {
+                    "rollout_s": rollout_elapsed,
+                    "judges": judges_timings,
+                }
+                if judge_specs:
+                    for spec in judge_specs:
+                        score_value: float | None = None
+                        judge_elapsed: float | None = None
+                        if completion is not None:
+                            judge_payload = {
+                                "seed": seed_val,
+                                "prompt_index": prompt_index,
+                                "prompt": prompt_text,
+                                "completion": completion,
+                                "metrics": metrics,
+                                "response": data,
+                                "trace": trace_namespace,
+                            }
+                            try:
+                                judge_start = time.perf_counter()
+                                result = spec.fn(judge_payload, **spec.kwargs)
+                                judge_elapsed = time.perf_counter() - judge_start
+                                if isinstance(result, int | float):
+                                    score_value = float(result)
+                            except Exception as exc:
+                                if judge_elapsed is None:
+                                    judge_elapsed = time.perf_counter() - judge_start
+                                click.echo(f"seed={seed_val} judge[{spec.name}]_error={exc}")
+                        judges_timings[spec.name] = judge_elapsed
+                        judge_scores[spec.name] = score_value
+
+                if trace_tracer is not None and trace_namespace:
+                    storage_metadata = {
+                        "eval_seed": seed_val,
+                        "prompt_index": prompt_index,
+                        "task_id": task_id,
+                        "task_split": task_split,
+                        "task_rubric_id": task_rubric_id,
+                        "official_score": official_score,
+                        "judge_scores": judge_scores,
+                        "model": selected_model,
+                        "prompt": prompt_text,
+                        "completion": completion,
+                    }
+                    await _store_trace(trace_tracer, trace_namespace, storage_metadata)
+
+                records.append(
+                    {
+                        "seed": seed_val,
+                        "prompt_index": prompt_index,
+                        "task_id": task_id,
+                        "task_split": task_split,
+                        "task_rubric_id": task_rubric_id,
+                        "official_score": official_score,
+                        "judge_scores": judge_scores,
+                        "timings": timings,
+                    }
+                )
+
+            await asyncio.gather(*[_run_seed(seed_val) for seed_val in seed_values])
+        finally:
+            await async_client.aclose()
+
+    try:
+        asyncio.run(_run_eval())
+    finally:
+        if trace_tracer is not None and trace_tracer.db is not None:
+            asyncio.run(trace_tracer.db.close())
 
     click.echo(
         f"Eval complete: {successes} ok, {failures} failed; model={selected_model}, split={split}"
     )
-    # Print outcome summary if any successes
+
     if outcome_count > 0:
         mean_outcome = outcome_sum / float(outcome_count)
         frac_right = outcome_correct / float(outcome_count)
@@ -2786,6 +4028,285 @@ def eval_command(
             f"Outcome summary: correct={outcome_correct}/{outcome_count} ({frac_right:.2%}), mean_outcome={mean_outcome:.3f}"
         )
 
+    if records:
+        judge_specs = judge_specs or []  # ensure iterable
+        official_scores = [
+            r["official_score"] for r in records if r["official_score"] is not None
+        ]
+        if official_scores:
+            click.echo(f"  Official mean: {sum(official_scores) / len(official_scores):.3f}")
+        else:
+            click.echo("  Official mean: n/a")
+
+        for spec in judge_specs:
+            spec_scores = [
+                record["judge_scores"].get(spec.name)
+                for record in records
+                if record["judge_scores"].get(spec.name) is not None
+            ]
+            if spec_scores:
+                mean_spec = sum(spec_scores) / len(spec_scores)
+                click.echo(f"  [{spec.name}] mean: {mean_spec:.3f}")
+            else:
+                click.echo(f"  [{spec.name}] mean: n/a")
+
+            paired = [
+                (
+                    record["official_score"],
+                    record["judge_scores"].get(spec.name),
+                )
+                for record in records
+                if record["official_score"] is not None
+                and record["judge_scores"].get(spec.name) is not None
+            ]
+            if len(paired) >= 2:
+                corr = _pearson(
+                    [p[0] for p in paired if p[0] is not None],
+                    [p[1] for p in paired if p[1] is not None],
+                )
+                if corr is not None:
+                    click.echo(f"    Pearson r: {corr:.3f}")
+                else:
+                    click.echo("    Pearson r: undefined (zero variance)")
+            else:
+                click.echo("    Pearson r: n/a (need ≥2 paired scores)")
+
+        header = ["Seed", "Prompt", "Official"]
+        header.extend(spec.name for spec in judge_specs)
+        rows: list[list[str]] = []
+        for record in sorted(records, key=lambda r: (r["seed"], r.get("prompt_index") or -1)):
+            seed_val = str(record["seed"])
+            prompt_idx = (
+                str(record["prompt_index"])
+                if record["prompt_index"] is not None
+                else "-"
+            )
+            official_val = (
+                f"{record['official_score']:.3f}"
+                if record["official_score"] is not None
+                else "-"
+            )
+            row = [seed_val, prompt_idx, official_val]
+            for spec in judge_specs:
+                score_val = record["judge_scores"].get(spec.name)
+                row.append(f"{score_val:.3f}" if isinstance(score_val, int | float) else "-")
+            rows.append(row)
+
+        widths = [len(col) for col in header]
+        for row in rows:
+            for idx, cell in enumerate(row):
+                widths[idx] = max(widths[idx], len(cell))
+
+        click.echo("")
+        click.echo("  ".join(h.ljust(widths[idx]) for idx, h in enumerate(header)))
+        click.echo("  ".join("-" * widths[idx] for idx in range(len(header))))
+        for row in rows:
+            click.echo("  ".join(cell.ljust(widths[idx]) for idx, cell in enumerate(row)))
+
+
+
+@click.command(
+    "filter",
+    help="Export filtered tracing sessions to SFT-ready JSONL based on a TOML config.",
+)
+@click.option(
+    "--config",
+    "config_path",
+    type=click.Path(),
+    required=True,
+    help="Path to TOML config describing the input trace DB, score thresholds, and output JSONL.",
+)
+def filter_command(config_path: str) -> None:
+    """Render tracing sessions that match filter rules into SFT JSONL.
+
+    The TOML file should contain a `[filter]` table with at least:
+
+        db = \"path/to/traces.db\"      # sqlite path or URL (sqlite+aiosqlite://...)
+        output = \"ft_data/out.jsonl\"  # destination JSONL
+
+    Optional keys such as `splits`, `task_ids`, `models`, `min_official_score`, or
+    `min_judge_scores.my_judge = 0.7` allow you to narrow the dataset down to
+    high-quality traces. See `customers/agora_single_file/configs/filter_local.toml`
+    for a working example.
+    """
+    if _toml is None:
+        raise click.ClickException("TOML parser not available; install tomli or use Python 3.11+")
+
+    cfg_path = Path(config_path)
+    if not cfg_path.exists():
+        raise click.ClickException(f"Filter config not found: {cfg_path}")
+
+    try:
+        config_data = _toml.loads(cfg_path.read_text(encoding="utf-8"))
+    except Exception as exc:
+        raise click.ClickException(f"Failed to parse TOML '{cfg_path}': {exc}") from exc
+
+    filter_cfg = config_data.get("filter") if isinstance(config_data, dict) else None
+    if not isinstance(filter_cfg, dict):
+        raise click.ClickException("Config must contain a [filter] table")
+
+    db_value = str(filter_cfg.get("db", "traces/v3/eval_traces.db")).strip()
+    if not db_value:
+        raise click.ClickException("filter.db must be provided")
+    if "://" in db_value:
+        db_url = db_value
+    else:
+        db_path = Path(db_value).expanduser()
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        db_url = f"sqlite+aiosqlite:///{db_path}"
+
+    output_value = filter_cfg.get("output")
+    if not output_value:
+        raise click.ClickException("filter.output must be provided")
+    output_path = Path(str(output_value)).expanduser()
+
+    splits = set(filter_cfg.get("splits", []) or [])
+    task_ids = set(filter_cfg.get("task_ids", []) or [])
+    models = set(filter_cfg.get("models", []) or [])
+    min_official = filter_cfg.get("min_official_score")
+    max_official = filter_cfg.get("max_official_score")
+    if min_official is not None:
+        try:
+            min_official = float(min_official)
+        except Exception as err:
+            raise click.ClickException("filter.min_official_score must be numeric") from err
+    if max_official is not None:
+        try:
+            max_official = float(max_official)
+        except Exception as err:
+            raise click.ClickException("filter.max_official_score must be numeric") from err
+    min_judge_scores = filter_cfg.get("min_judge_scores", {}) or {}
+    max_judge_scores = filter_cfg.get("max_judge_scores", {}) or {}
+    try:
+        min_judge_scores = {k: float(v) for k, v in min_judge_scores.items()}
+    except Exception as err:
+        raise click.ClickException("filter.min_judge_scores values must be numeric") from err
+    try:
+        max_judge_scores = {k: float(v) for k, v in max_judge_scores.items()}
+    except Exception as err:
+        raise click.ClickException("filter.max_judge_scores values must be numeric") from err
+    min_created = _parse_datetime_for_trace(filter_cfg.get("min_created_at"))
+    max_created = _parse_datetime_for_trace(filter_cfg.get("max_created_at"))
+    limit = filter_cfg.get("limit")
+    if limit is not None:
+        try:
+            limit = int(limit)
+        except Exception as err:
+            raise click.ClickException("filter.limit must be an integer") from err
+
+    def _score_ok(value: Any, min_val: Any, max_val: Any) -> bool:
+        try:
+            if value is None:
+                return min_val is None
+            value = float(value)
+        except Exception:
+            return False
+        if min_val is not None and value < float(min_val):
+            return False
+        return not (max_val is not None and value > float(max_val))
+
+    async def _run_filter() -> None:
+        tracer = SessionTracer(db_url=db_url, auto_save=False)
+        await tracer.initialize()
+
+        df = await tracer.db.query_traces(
+            "SELECT session_id, created_at, metadata FROM session_traces ORDER BY created_at"
+        )
+        if getattr(df, "empty", True):
+            raise click.ClickException("No traces found in database")
+
+        sessions = df.to_dict("records")
+        accepted: list[dict[str, Any]] = []
+
+        for row in sessions:
+            metadata_raw = row.get("metadata")
+            if isinstance(metadata_raw, str):
+                try:
+                    metadata = json.loads(metadata_raw)
+                except Exception:
+                    metadata = {}
+            elif isinstance(metadata_raw, dict):
+                metadata = dict(metadata_raw)
+            else:
+                metadata = {}
+
+            created_at_raw = row.get("created_at")
+            created_at_dt = _parse_datetime_for_trace(created_at_raw)
+
+            session_id = row.get("session_id")
+
+            if splits and metadata.get("task_split") not in splits:
+                continue
+            if task_ids and metadata.get("task_id") not in task_ids:
+                continue
+            if models and metadata.get("model") not in models:
+                continue
+
+            if min_created and (created_at_dt is None or created_at_dt < min_created):
+                continue
+            if max_created and (created_at_dt is None or created_at_dt > max_created):
+                continue
+
+            if not _score_ok(metadata.get("official_score"), min_official, max_official):
+                continue
+
+            judge_scores = metadata.get("judge_scores") or {}
+            include = True
+            for judge_name, threshold in (min_judge_scores or {}).items():
+                if not _score_ok(judge_scores.get(judge_name), threshold, None):
+                    include = False
+                    break
+            if not include:
+                continue
+            for judge_name, threshold in (max_judge_scores or {}).items():
+                if not _score_ok(judge_scores.get(judge_name), None, threshold):
+                    include = False
+                    break
+            if not include:
+                continue
+
+            prompt = metadata.get("prompt") or ""
+            completion = metadata.get("completion") or ""
+            if not prompt or not completion:
+                continue
+
+            record = {
+                "messages": [
+                    {"role": "user", "content": str(prompt)},
+                    {"role": "assistant", "content": str(completion)},
+                ],
+                "metadata": {
+                    "session_id": session_id,
+                    "task_id": metadata.get("task_id"),
+                    "task_split": metadata.get("task_split"),
+                    "task_rubric_id": metadata.get("task_rubric_id"),
+                    "official_score": metadata.get("official_score"),
+                    "judge_scores": judge_scores,
+                    "model": metadata.get("model"),
+                    "created_at": created_at_dt.isoformat() if created_at_dt else created_at_raw,
+                    "prompt": prompt,
+                    "completion": completion,
+                },
+            }
+            accepted.append(record)
+
+        if not accepted:
+            raise click.ClickException("No sessions matched the provided filters")
+
+        if limit is not None and limit > 0:
+            accepted = accepted[:limit]
+
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with output_path.open("w", encoding="utf-8") as handle:
+            for item in accepted:
+                handle.write(json.dumps(item, ensure_ascii=False))
+                handle.write("\n")
+
+        click.echo(f"Wrote {len(accepted)} examples -> {output_path}")
+        await tracer.db.close()
+
+    asyncio.run(_run_filter())
+
 
 def register_eval(cli: click.Group) -> None:
     cli.add_command(eval_command)