synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/cli/commands/baseline/list.py

@@ -0,0 +1,93 @@
+"""List command for baseline discovery."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import click
+from synth_ai.baseline.config import BaselineConfig
+from synth_ai.baseline.discovery import (
+    BaselineChoice,
+    discover_baseline_files,
+    load_baseline_config_from_file,
+)
+
+
+@click.command("list")
+@click.option(
+    "--tag",
+    multiple=True,
+    help="Filter baselines by tag (can be specified multiple times)",
+)
+@click.option(
+    "--metadata",
+    type=str,
+    help="Filter by metadata key-value pair (format: key=value)",
+)
+@click.option(
+    "--verbose",
+    is_flag=True,
+    help="Show detailed information about each baseline",
+)
+def list_command(tag: tuple[str, ...], metadata: Optional[str], verbose: bool) -> None:
+    """List all available baseline files."""
+    search_roots = [Path.cwd()]
+    choices = discover_baseline_files(search_roots)
+    
+    if not choices:
+        click.echo("No baseline files found.", err=True)
+        click.echo("Create baseline files in examples/baseline/ or */*_baseline.py")
+        return
+    
+    # Load configs for filtering
+    configs: list[tuple[BaselineChoice, BaselineConfig]] = []
+    for choice in choices:
+        try:
+            config = load_baseline_config_from_file(choice.baseline_id, choice.path)
+            configs.append((choice, config))
+        except Exception as e:
+            if verbose:
+                click.echo(f"Warning: Could not load {choice.baseline_id}: {e}", err=True)
+            continue
+    
+    # Apply filters
+    filtered_configs = configs
+    
+    if tag:
+        tag_set = {t.lower() for t in tag}
+        filtered_configs = [
+            (c, config) for c, config in filtered_configs
+            if any(config.matches_tag(t) for t in tag_set)
+        ]
+    
+    if metadata:
+        if "=" not in metadata:
+            raise click.ClickException("--metadata must be in format key=value")
+        key, value = metadata.split("=", 1)
+        filtered_configs = [
+            (c, config) for c, config in filtered_configs
+            if config.matches_metadata(key.strip(), value.strip())
+        ]
+    
+    if not filtered_configs:
+        click.echo("No baselines match the specified filters.")
+        return
+    
+    # Display results
+    click.echo(f"Found {len(filtered_configs)} baseline(s):\n")
+    
+    for choice, config in filtered_configs:
+        click.echo(f"  {config.baseline_id}")
+        click.echo(f"    Name: {config.name}")
+        if config.description:
+            click.echo(f"    Description: {config.description}")
+        if config.tags:
+            click.echo(f"    Tags: {', '.join(config.tags)}")
+        click.echo(f"    Splits: {', '.join(config.splits.keys())}")
+        if verbose:
+            click.echo(f"    Path: {choice.path}")
+            if config.metadata:
+                click.echo(f"    Metadata: {config.metadata}")
+        click.echo()
+

synth_ai/cli/commands/eval/core.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, cast
 
 import click
 from synth_ai.task.config import EvalConfig
+from synth_ai.tracing_v3.session_tracer import SessionTracer
 from synth_ai.utils.task_app_discovery import discover_eval_config_paths
 
 from .errors import (
@@ -199,8 +200,9 @@ def _eval_command_impl(
     if cfg:
         try:
             normalized_cfg = validate_eval_options(cfg)
-            eval_cfg = EvalConfig.from_dict(dict(normalized_cfg))
-            cfg = dict(normalized_cfg)
+            normalized_cfg_dict = dict(normalized_cfg)
+            eval_cfg = EvalConfig.from_dict(normalized_cfg_dict)
+            cfg = normalized_cfg_dict
             click.echo(f"✓ Config validated: {len(eval_cfg.seeds)} seeds, model={eval_cfg.model}")
         except (ValueError, TypeError) as validation_error:
             raise InvalidEvalConfigError(detail=str(validation_error)) from validation_error
@@ -261,11 +263,9 @@ def _eval_command_impl(
             trace_path = Path(trace_db).expanduser()
             trace_path.parent.mkdir(parents=True, exist_ok=True)
             trace_db_url = f"sqlite+aiosqlite:///{trace_path}"
-    trace_tracer = (
-        session_tracer_cls(db_url=trace_db_url, auto_save=True)
-        if trace_db_url and session_tracer_cls is not None
-        else None
-    )
+    trace_tracer: SessionTracer | None = None
+    if trace_db_url and session_tracer_cls is not None:
+        trace_tracer = cast(SessionTracer, session_tracer_cls(db_url=trace_db_url, auto_save=True))
 
     # Determine selection params (CLI takes precedence; TOML only fills unset model/seeds/env)
     if cfg.get("model") and not model:
@@ -723,14 +723,17 @@ def _eval_command_impl(
                     "mode": "eval",  # RolloutMode.EVAL: use inference URLs as-is, no transformations
                 }
                 if env_name:
-                    body["env"]["env_name"] = env_name  # type: ignore[assignment]
-                
+                    env_section = body.get("env")
+                    if isinstance(env_section, dict):
+                        env_section["env_name"] = env_name
+                    else:
+                        body["env"] = {"env_name": env_name}
+
                 # Debug: print the body being sent
                 if seed_val == 0:
                     click.echo(f"[DEBUG] rollout body env: {body['env']}")
                     click.echo(f"[DEBUG] rollout body policy: {body['policy']}")
                     click.echo(f"[DEBUG] rollout body mode: {body.get('mode', 'NOT SET')}")
-                    click.echo(f"[DEBUG] rollout record payload: {body.get('record')}")
                 rollout_elapsed: float | None = None
                 rollout_start = time.perf_counter()
                 try:

synth_ai/cli/commands/filter/core.py

@@ -139,12 +139,23 @@ def _select_messages(message_rows: Sequence[dict[str, Any]]) -> list[dict[str, A
         if msg_type not in {"user", "policy_user_prompt"}:
             continue
 
+        # Look backwards for system prompt
+        system_msg = None
+        for prev in range(index - 1, -1, -1):
+            prev_type = message_rows[prev].get("message_type")
+            if prev_type == "policy_system_prompt":
+                system_msg = message_rows[prev]
+                break
+
         assistant_msg = None
+        tool_call_msg = None
         for follow in range(index + 1, len(message_rows)):
             next_type = message_rows[follow].get("message_type")
-            if next_type in {"assistant", "policy_system_prompt"}:
-                if next_type == "assistant":
-                    assistant_msg = message_rows[follow]
+            if next_type == "assistant":
+                assistant_msg = message_rows[follow]
+                break
+            elif next_type == "policy_tool_call":
+                tool_call_msg = message_rows[follow]
                 break
 
         try:
@@ -157,8 +168,34 @@ def _select_messages(message_rows: Sequence[dict[str, Any]]) -> list[dict[str, A
         if not user_text:
             continue
 
+        messages = []
+        
+        # Add system prompt if found
+        if system_msg is not None:
+            try:
+                system_content_raw = system_msg.get("content")
+                system_content = json.loads(system_content_raw) if isinstance(system_content_raw, str) else system_content_raw
+                system_content = _extract_content(system_content)
+                system_text = _extract_text(system_content)
+                if system_text:
+                    messages.append({"role": "system", "content": system_text})
+            except Exception:
+                pass
+
+        # Add user message
+        user_payload = user_content if isinstance(user_content, list) else user_text
+        messages.append({"role": "user", "content": user_payload})
+
+        # Add assistant/tool call response
         assistant_content = None
-        if assistant_msg is not None:
+        if tool_call_msg is not None:
+            raw = tool_call_msg.get("content")
+            try:
+                assistant_content = json.loads(raw) if isinstance(raw, str) else raw
+            except Exception:
+                assistant_content = raw
+            assistant_content = _extract_content(assistant_content)
+        elif assistant_msg is not None:
             raw = assistant_msg.get("content")
             try:
                 assistant_content = json.loads(raw) if isinstance(raw, str) else raw
@@ -166,22 +203,14 @@ def _select_messages(message_rows: Sequence[dict[str, Any]]) -> list[dict[str, A
                 assistant_content = raw
             assistant_content = _extract_content(assistant_content)
 
-        assistant_text = _extract_text(assistant_content) if assistant_content is not None else ""
-        user_payload = user_content if isinstance(user_content, list) else user_text
         assistant_payload = (
             assistant_content
             if isinstance(assistant_content, list)
-            else (assistant_text or "[no response recorded]")
+            else (_extract_text(assistant_content) if assistant_content is not None else "[no response recorded]")
         )
+        messages.append({"role": "assistant", "content": assistant_payload})
 
-        records.append(
-            {
-                "messages": [
-                    {"role": "user", "content": user_payload},
-                    {"role": "assistant", "content": assistant_payload},
-                ]
-            }
-        )
+        records.append({"messages": messages})
     return records
 
 
@@ -219,7 +248,9 @@ def filter_command(config_path: str) -> None:
     async def _run() -> None:
         tracer = SessionTracer(db_url=db_url, auto_save=False)
         await tracer.initialize()
-        assert tracer.db is not None, "Database should be initialized"
+
+        if tracer.db is None:
+            raise FilterCliError("Database not initialized")
 
         df = await tracer.db.query_traces(
             "SELECT session_id, created_at, metadata FROM session_traces ORDER BY created_at"
@@ -261,6 +292,8 @@ def filter_command(config_path: str) -> None:
             total_reward = None
             achievements_count = None
             if min_official is not None or max_official is not None:
+                if tracer.db is None:
+                    raise FilterCliError("Database not initialized")
                 reward_rows = await tracer.db.query_traces(
                     "SELECT total_reward, achievements_count FROM outcome_rewards WHERE session_id = :session_id",
                     {"session_id": session_id},
@@ -296,6 +329,8 @@ def filter_command(config_path: str) -> None:
             messages_query = (
                 "\n            SELECT message_type, content, timestamp \n            FROM messages \n            WHERE session_id = :session_id\n            ORDER BY timestamp ASC, id ASC\n        "
             )
+            if tracer.db is None:
+                raise FilterCliError("Database not initialized")
             msg_df = await tracer.db.query_traces(messages_query, {"session_id": session_id})
             message_rows = (
                 msg_df.to_dict("records") if hasattr(msg_df, "to_dict") else []
@@ -353,7 +388,8 @@ def filter_command(config_path: str) -> None:
                 handle.write("\n")
 
         click.echo(f"Wrote {len(accepted)} examples -> {output_path}")
-        await tracer.db.close()
+        if tracer.db is not None:
+            await tracer.db.close()
 
     try:
         asyncio.run(_run())

synth_ai/cli/commands/help/core.py

@@ -70,4 +70,3 @@ def register(group: click.Group) -> None:
 
 
 __all__ = ["help_command", "get_command", "register"]
-

synth_ai/cli/commands/smoke/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from .core import command, register
+
+__all__ = ["command", "register"]
+
+