synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/baseline/config.py

@@ -0,0 +1,209 @@
+"""Core dataclasses for baseline configuration and results."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
+
+
+class BaselineTaskRunner:
+    """
+    Base class for task runners.
+    
+    Subclasses should implement `run_task` method for class-based approach,
+    or you can use standalone async functions for function-based approach.
+    """
+    
+    def __init__(
+        self,
+        policy_config: Dict[str, Any],
+        env_config: Dict[str, Any],
+    ):
+        """
+        Initialize task runner with configuration.
+        
+        Args:
+            policy_config: Policy configuration (model, temperature, etc.)
+            env_config: Environment configuration (max_steps, difficulty, etc.)
+        """
+        self.policy_config = policy_config
+        self.env_config = env_config
+    
+    async def run_task(self, seed: int) -> TaskResult:
+        """
+        Execute a single task instance.
+        
+        This method is called for each seed in the selected split.
+        
+        Args:
+            seed: The seed/index for this task instance
+        
+        Returns:
+            TaskResult: Structured result containing success, rewards, metadata, trace
+        """
+        raise NotImplementedError("Subclasses must implement run_task method")
+
+
+@dataclass
+class DataSplit:
+    """Definition of a data split (train/val/test)."""
+    
+    name: str  # "train", "val", "test"
+    seeds: List[int]  # Seed/index values for this split
+    metadata: Dict[str, Any] = field(default_factory=dict)  # Optional metadata
+
+
+@dataclass
+class TaskResult:
+    """Result from a single task execution."""
+    
+    # Required: Seed/index that was evaluated
+    seed: int
+    
+    # Required: Did the task complete successfully?
+    success: bool
+    
+    # Required: Outcome reward for the episode
+    outcome_reward: float
+    
+    # Optional: Event rewards (step-level)
+    event_rewards: List[Dict[str, Any]] = field(default_factory=list)
+    
+    # Optional: Total steps/turns taken
+    total_steps: int = 0
+    
+    # Optional: Metadata (achievements, completion info, etc.)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Error information if success=False
+    error: Optional[str] = None
+    
+    # Optional: v3 trace (SessionTrace dict)
+    trace: Optional[Dict[str, Any]] = None
+
+
+# Type alias for task runner (can be class or function)
+TaskRunnerType = (
+    type[BaselineTaskRunner]
+    | Callable[[int, dict[str, Any], dict[str, Any]], Any]  # Function signature
+)
+
+# Type alias for result aggregator (can be class or function)
+AggregatorType = (
+    type[Any]  # Class with aggregate() method
+    | Callable[[list[TaskResult]], dict[str, Any]]  # Function signature
+)
+
+
+@dataclass
+class BaselineConfig:
+    """Configuration for a baseline file.
+    
+    A baseline file defines how to evaluate a task without requiring
+    a deployed task app. It provides self-contained evaluation logic
+    with first-class support for train/val/test splits.
+    
+    Supports both class-based and function-based task runners:
+    - Class-based: Pass a class that inherits from BaselineTaskRunner
+    - Function-based: Pass an async function with signature:
+      async def task_runner(seed: int, policy_config: Dict[str, Any], 
+                           env_config: Dict[str, Any]) -> TaskResult
+    """
+    
+    # Required: Unique identifier for this baseline config
+    baseline_id: str
+    
+    # Required: Human-readable name
+    name: str
+    
+    # Required: Task runner (class or function)
+    # Class-based: Pass a class inheriting from BaselineTaskRunner
+    #   The class will be instantiated with policy_config and env_config,
+    #   and run_task(seed) will be called for each seed.
+    # Function-based: Pass an async function with signature:
+    #   async def task_runner(seed: int, policy_config: Dict[str, Any], 
+    #                        env_config: Dict[str, Any]) -> TaskResult
+    task_runner: TaskRunnerType
+    
+    # Required: Data splits (train/val/test)
+    splits: Dict[str, DataSplit]
+    
+    # Optional: Description for documentation
+    description: str = ""
+    
+    # Optional: Default policy configuration
+    default_policy_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Default environment configuration
+    default_env_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for filtering/organization
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Tags for filtering and discovery
+    tags: List[str] = field(default_factory=list)
+    
+    # Optional: Custom result aggregator (class or function)
+    # Class-based: Pass a class with aggregate(results: List[TaskResult]) method
+    #   The class will be instantiated and aggregate() called.
+    # Function-based: Pass a function with signature:
+    #   def aggregate_results(results: List[TaskResult]) -> Dict[str, Any]
+    result_aggregator: Optional[AggregatorType] = None
+    
+    # Optional: Path to this baseline file (set by discovery)
+    _source_path: Optional[Path] = None
+    
+    def matches_tag(self, tag: str) -> bool:
+        """Check if baseline matches a tag (case-insensitive)."""
+        return tag.lower() in [t.lower() for t in self.tags]
+    
+    def matches_metadata(self, key: str, value: Any) -> bool:
+        """Check if baseline metadata matches key-value pair."""
+        return self.metadata.get(key) == value
+
+
+@dataclass
+class BaselineResults:
+    """Aggregate results from a baseline evaluation."""
+    
+    # Configuration that was used
+    config: BaselineConfig
+    
+    # Split that was evaluated
+    split_name: str
+    
+    # Per-seed results
+    results: List[TaskResult]
+    
+    # Aggregate metrics
+    aggregate_metrics: Dict[str, Any]
+    
+    # Execution metadata
+    execution_time_seconds: float
+    model_name: str
+    timestamp: str
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary for JSON output."""
+        return {
+            "baseline_id": self.config.baseline_id,
+            "name": self.config.name,
+            "split": self.split_name,
+            "model": self.model_name,
+            "timestamp": self.timestamp,
+            "execution_time_seconds": self.execution_time_seconds,
+            "aggregate_metrics": self.aggregate_metrics,
+            "results": [
+                {
+                    "seed": r.seed,
+                    "success": r.success,
+                    "outcome_reward": r.outcome_reward,
+                    "total_steps": r.total_steps,
+                    "metadata": r.metadata,
+                    "error": r.error,
+                }
+                for r in self.results
+            ],
+        }
+

synth_ai/baseline/discovery.py

@@ -0,0 +1,214 @@
+"""AST-based discovery mechanism for baseline files."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from synth_ai.baseline.config import BaselineConfig
+
+# Search patterns for baseline files
+BASELINE_FILE_PATTERNS = [
+    "**/baseline/*.py",
+    "**/baselines/*.py",
+    "**/*_baseline.py",
+]
+
+# Directories to ignore during discovery
+IGNORE_PATTERNS = {
+    "__pycache__",
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "build",
+    "dist",
+    ".mypy_cache",
+    ".pytest_cache",
+}
+
+
+@dataclass
+class BaselineChoice:
+    """Represents a discovered baseline configuration."""
+    
+    baseline_id: str
+    path: Path
+    lineno: int
+    source: str  # "discovered" or "registered"
+    config: Optional[BaselineConfig] = None
+
+
+class BaselineConfigVisitor(ast.NodeVisitor):
+    """AST visitor to find BaselineConfig instances."""
+    
+    def __init__(self):
+        self.matches: List[Tuple[str, int]] = []  # (baseline_id, lineno)
+    
+    def visit_Assign(self, node: ast.Assign) -> None:
+        """Visit assignment statements looking for BaselineConfig."""
+        if not isinstance(node.value, ast.Call):
+            self.generic_visit(node)
+            return
+        
+        # Check if right-hand side is BaselineConfig(...)
+        func = node.value.func
+        if isinstance(func, ast.Name) and func.id == "BaselineConfig":
+            # Extract baseline_id from constructor args
+            baseline_id = self._extract_baseline_id(node.value)
+            if baseline_id:
+                self.matches.append((baseline_id, node.lineno))
+        
+        self.generic_visit(node)
+    
+    def _extract_baseline_id(self, call_node: ast.Call) -> Optional[str]:
+        """Extract baseline_id from BaselineConfig constructor."""
+        for keyword in call_node.keywords:
+            if keyword.arg == "baseline_id" and isinstance(keyword.value, ast.Constant):
+                return keyword.value.value
+        return None
+
+
+def should_ignore_path(path: Path) -> bool:
+    """Check if a path should be ignored during discovery."""
+    return any(part in IGNORE_PATTERNS for part in path.parts)
+
+
+def discover_baseline_files(search_roots: List[Path]) -> List[BaselineChoice]:
+    """Discover baseline files via AST scanning.
+    
+    Args:
+        search_roots: List of root directories to search in
+    
+    Returns:
+        List of BaselineChoice objects representing discovered baselines
+    """
+    results: List[BaselineChoice] = []
+    seen = set()
+    
+    for root in search_roots:
+        if not root.exists():
+            continue
+        
+        for pattern in BASELINE_FILE_PATTERNS:
+            for path in root.glob(pattern):
+                if should_ignore_path(path):
+                    continue
+                
+                try:
+                    source = path.read_text(encoding="utf-8")
+                    tree = ast.parse(source, filename=str(path))
+                except (OSError, SyntaxError):
+                    continue
+                
+                visitor = BaselineConfigVisitor()
+                visitor.visit(tree)
+                
+                for baseline_id, lineno in visitor.matches:
+                    key = (baseline_id, path.resolve())
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    
+                    results.append(
+                        BaselineChoice(
+                            baseline_id=baseline_id,
+                            path=path.resolve(),
+                            lineno=lineno,
+                            source="discovered",
+                        )
+                    )
+    
+    return results
+
+
+def load_baseline_config_from_file(
+    baseline_id: str,
+    path: Path,
+) -> BaselineConfig:
+    """Load a BaselineConfig from a Python file.
+    
+    Args:
+        baseline_id: The baseline_id to look for
+        path: Path to the Python file
+    
+    Returns:
+        BaselineConfig instance
+    
+    Raises:
+        ValueError: If baseline_id not found or file cannot be loaded
+    """
+    # Load the module
+    spec = importlib.util.spec_from_file_location("baseline_module", path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Cannot load baseline file: {path}")
+    
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except ModuleNotFoundError as e:
+        missing_module = str(e).split("'")[1] if "'" in str(e) else str(e)
+        raise ImportError(
+            f"❌ Missing dependency for baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Missing module: {missing_module}\n"
+            f"   Fix: pip install {missing_module}  (or 'uv add {missing_module}')"
+        ) from e
+    except SyntaxError as e:
+        raise ValueError(
+            f"❌ Syntax error in baseline file '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error at line {e.lineno}: {e.msg}\n"
+            f"   Text: {e.text.strip() if e.text else 'N/A'}\n"
+            f"   Fix: Check the Python syntax in the baseline file"
+        ) from e
+    except Exception as e:
+        error_type = type(e).__name__
+        raise ValueError(
+            f"❌ Failed to load baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error type: {error_type}\n"
+            f"   Message: {str(e)}\n"
+            f"   This may be due to:\n"
+            f"     - Missing dependencies (check imports)\n"
+            f"     - Configuration errors in the baseline file\n"
+            f"     - Environment variables not set\n"
+            f"   Tip: Run with --verbose for more details"
+        ) from e
+    
+    # Find the BaselineConfig instance
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig) and attr.baseline_id == baseline_id:
+            # Set source path for reference
+            attr._source_path = path
+            return attr
+    
+    # Provide helpful error message
+    found_configs = []
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig):
+            found_configs.append(attr.baseline_id)
+    
+    if found_configs:
+        raise ValueError(
+            f"❌ Baseline '{baseline_id}' not found in {path}\n"
+            f"   Found baselines in this file: {', '.join(found_configs)}\n"
+            f"   Fix: Use one of the above baseline IDs or check the baseline_id parameter"
+        )
+    else:
+        raise ValueError(
+            f"❌ No BaselineConfig instances found in {path}\n"
+            f"   Expected to find a BaselineConfig with baseline_id='{baseline_id}'\n"
+            f"   Fix: Ensure the file defines a BaselineConfig instance with baseline_id='{baseline_id}'"
+        )
+

synth_ai/baseline/execution.py

@@ -0,0 +1,146 @@
+"""Execution engine for baseline evaluations."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from synth_ai.baseline.config import (
+    BaselineConfig,
+    BaselineTaskRunner,
+    TaskResult,
+)
+
+
+def default_aggregator(results: List[TaskResult]) -> Dict[str, Any]:
+    """Default result aggregation function.
+    
+    Computes mean, std, min, max, success rate, and other basic metrics.
+    
+    Args:
+        results: List of TaskResult objects from all seeds
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    successful_results = [r for r in results if r.success]
+    outcome_rewards = [r.outcome_reward for r in successful_results]
+    
+    if not outcome_rewards:
+        return {
+            "mean_outcome_reward": 0.0,
+            "std_outcome_reward": 0.0,
+            "min_outcome_reward": 0.0,
+            "max_outcome_reward": 0.0,
+            "success_rate": 0.0,
+            "total_tasks": len(results),
+            "successful_tasks": 0,
+            "failed_tasks": len(results),
+        }
+    
+    mean_reward = sum(outcome_rewards) / len(outcome_rewards)
+    
+    # Calculate standard deviation
+    variance = sum((x - mean_reward) ** 2 for x in outcome_rewards) / len(outcome_rewards)
+    std_reward = variance ** 0.5
+    
+    return {
+        "mean_outcome_reward": mean_reward,
+        "std_outcome_reward": std_reward,
+        "min_outcome_reward": min(outcome_rewards),
+        "max_outcome_reward": max(outcome_rewards),
+        "success_rate": len(successful_results) / len(results),
+        "total_tasks": len(results),
+        "successful_tasks": len(successful_results),
+        "failed_tasks": len(results) - len(successful_results),
+    }
+
+
+def _is_class_based_runner(task_runner: Any) -> bool:
+    """Check if task_runner is a class (not a function)."""
+    return (
+        isinstance(task_runner, type)
+        and issubclass(task_runner, BaselineTaskRunner)
+    )
+
+
+async def run_baseline_evaluation(
+    config: BaselineConfig,
+    seeds: List[int],
+    policy_config: Dict[str, Any],
+    env_config: Dict[str, Any],
+    concurrency: int = 4,
+) -> List[TaskResult]:
+    """Run baseline evaluation for given seeds.
+    
+    Args:
+        config: BaselineConfig instance
+        seeds: List of seeds to evaluate
+        policy_config: Policy configuration (merged from defaults + overrides)
+        env_config: Environment configuration (merged from defaults + overrides)
+        concurrency: Maximum concurrent task executions
+    
+    Returns:
+        List of TaskResult objects, one per seed
+    """
+    # Determine if we're using class-based or function-based runner
+    is_class_based = _is_class_based_runner(config.task_runner)
+    
+    # Instantiate runner if class-based
+    runner_instance: Optional[BaselineTaskRunner] = None
+    if is_class_based:
+        runner_instance = config.task_runner(policy_config, env_config)
+    
+    # Create semaphore for concurrency control
+    semaphore = asyncio.Semaphore(concurrency)
+    
+    async def run_task(seed: int) -> TaskResult:
+        """Execute a single task with error handling."""
+        async with semaphore:
+            try:
+                if is_class_based and runner_instance:
+                    # Class-based: call run_task method
+                    return await runner_instance.run_task(seed)
+                else:
+                    # Function-based: call function directly
+                    task_runner_fn = config.task_runner
+                    return await task_runner_fn(seed, policy_config, env_config)
+            except Exception as exc:
+                # Return error result
+                return TaskResult(
+                    seed=seed,
+                    success=False,
+                    outcome_reward=0.0,
+                    error=str(exc),
+                )
+    
+    # Execute all tasks concurrently
+    results = await asyncio.gather(*[run_task(seed) for seed in seeds])
+    return list(results)
+
+
+def aggregate_results(
+    config: BaselineConfig,
+    results: List[TaskResult],
+) -> Dict[str, Any]:
+    """Aggregate results using custom aggregator or default.
+    
+    Args:
+        config: BaselineConfig instance
+        results: List of TaskResult objects
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    if config.result_aggregator is None:
+        return default_aggregator(results)
+    
+    # Check if aggregator is a class or function
+    if isinstance(config.result_aggregator, type):
+        # Class-based: instantiate and call aggregate()
+        aggregator_instance = config.result_aggregator()
+        return aggregator_instance.aggregate(results)
+    else:
+        # Function-based: call directly
+        return config.result_aggregator(results)
+

synth_ai/cli/__init__.py

@@ -52,9 +52,77 @@ if not _cli_module:
     raise ImportError("synth_ai.cli.root is required for CLI entrypoint")
 cli = _cli_module.cli  # type: ignore[attr-defined]
 
+# Register core commands implemented as standalone modules
+try:
+    from synth_ai.cli.demo import demo_cmd
+    cli.add_command(demo_cmd, name="demo")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register demo command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.setup import setup_cmd
+    cli.add_command(setup_cmd, name="setup")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register setup command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.deploy import deploy_cmd  # type: ignore[attr-defined]
+    cli.add_command(deploy_cmd, name="deploy")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register deploy command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.opencode import opencode_cmd
+    cli.add_command(opencode_cmd, name="opencode")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register opencode command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.codex import codex_cmd
+    cli.add_command(codex_cmd, name="codex")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register codex command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.eval import command as eval_cmd
+    cli.add_command(eval_cmd, name="eval")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register eval command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.claude import claude_cmd
+    cli.add_command(claude_cmd, name="claude")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register claude command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.commands.baseline import command as baseline_cmd
+    from synth_ai.cli.commands.baseline.list import list_command as baseline_list_cmd
+    cli.add_command(baseline_cmd, name="baseline")
+    baseline_cmd.add_command(baseline_list_cmd, name="list")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register baseline command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+
 
 # Register optional subcommands packaged under synth_ai.cli.*
-for _module_path in ("synth_ai.cli.demo", "synth_ai.cli.turso"):
+for _module_path in ("synth_ai.cli.commands.demo", "synth_ai.cli.commands.status", "synth_ai.cli.turso"):
     module = _maybe_import(_module_path)
     if not module:
         continue
@@ -64,27 +132,35 @@ for _module_path in ("synth_ai.cli.demo", "synth_ai.cli.turso"):
     if fn:
         fn(cli)
 
+# Smoke command registration (CLI-only helper)
+try:
+    from synth_ai.cli.commands.smoke import register as register_smoke
+
+    register_smoke(cli)
+except Exception:
+    pass
+
+# Register help command
+_maybe_call("synth_ai.cli.commands.help.core", "register", cli)
+
 # Train CLI lives under synth_ai.api.train
 _maybe_call("synth_ai.api.train", "register", cli)
 
 # Task app group/commands are optional and have richer API surface
 _task_apps_module = _maybe_import("synth_ai.cli.task_apps")
-if _task_apps_module:
-    task_app_group = getattr(_task_apps_module, "task_app_group", None)
-    if task_app_group is not None:
-        cli.add_command(task_app_group, name="task-app")
-        # Expose common aliases when present
-        commands = getattr(task_app_group, "commands", None)
-        if isinstance(commands, dict):
-            for alias, name in (("serve", "serve"), ("deploy", "deploy"), ("modal-serve", "modal-serve")):
-                command = commands.get(name)
-                if command is not None:
-                    cli.add_command(command, name=alias)
-    register_task_apps = _callable_from(_task_apps_module, "register")
-    if register_task_apps:
-        register_task_apps(cli)
-
-# Register TUI command if dependencies allow
-_maybe_call("synth_ai.cli.tui", "register", cli)
+#if _task_apps_module:
+task_app_group = getattr(_task_apps_module, "task_app_group", None)
+if task_app_group is not None:
+    cli.add_command(task_app_group, name="task-app")
+    # Expose common aliases when present
+    commands = getattr(task_app_group, "commands", None)
+    if isinstance(commands, dict):
+        for alias, name in (("serve", "serve"), ("deploy", "deploy"), ("modal-serve", "modal-serve")):
+            command = commands.get(name)
+            if command is not None:
+                cli.add_command(command, name=alias)
+register_task_apps = _callable_from(_task_apps_module, "register")
+if register_task_apps:
+    register_task_apps(cli)
 
 # Top-level 'info' alias removed; use `synth-ai task-app info` instead