synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/baseline/discovery.py

@@ -0,0 +1,214 @@
+"""AST-based discovery mechanism for baseline files."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from synth_ai.baseline.config import BaselineConfig
+
+# Search patterns for baseline files
+BASELINE_FILE_PATTERNS = [
+    "**/baseline/*.py",
+    "**/baselines/*.py",
+    "**/*_baseline.py",
+]
+
+# Directories to ignore during discovery
+IGNORE_PATTERNS = {
+    "__pycache__",
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "build",
+    "dist",
+    ".mypy_cache",
+    ".pytest_cache",
+}
+
+
+@dataclass
+class BaselineChoice:
+    """Represents a discovered baseline configuration."""
+    
+    baseline_id: str
+    path: Path
+    lineno: int
+    source: str  # "discovered" or "registered"
+    config: Optional[BaselineConfig] = None
+
+
+class BaselineConfigVisitor(ast.NodeVisitor):
+    """AST visitor to find BaselineConfig instances."""
+    
+    def __init__(self):
+        self.matches: List[Tuple[str, int]] = []  # (baseline_id, lineno)
+    
+    def visit_Assign(self, node: ast.Assign) -> None:
+        """Visit assignment statements looking for BaselineConfig."""
+        if not isinstance(node.value, ast.Call):
+            self.generic_visit(node)
+            return
+        
+        # Check if right-hand side is BaselineConfig(...)
+        func = node.value.func
+        if isinstance(func, ast.Name) and func.id == "BaselineConfig":
+            # Extract baseline_id from constructor args
+            baseline_id = self._extract_baseline_id(node.value)
+            if baseline_id:
+                self.matches.append((baseline_id, node.lineno))
+        
+        self.generic_visit(node)
+    
+    def _extract_baseline_id(self, call_node: ast.Call) -> Optional[str]:
+        """Extract baseline_id from BaselineConfig constructor."""
+        for keyword in call_node.keywords:
+            if keyword.arg == "baseline_id" and isinstance(keyword.value, ast.Constant):
+                return keyword.value.value
+        return None
+
+
+def should_ignore_path(path: Path) -> bool:
+    """Check if a path should be ignored during discovery."""
+    return any(part in IGNORE_PATTERNS for part in path.parts)
+
+
+def discover_baseline_files(search_roots: List[Path]) -> List[BaselineChoice]:
+    """Discover baseline files via AST scanning.
+    
+    Args:
+        search_roots: List of root directories to search in
+    
+    Returns:
+        List of BaselineChoice objects representing discovered baselines
+    """
+    results: List[BaselineChoice] = []
+    seen = set()
+    
+    for root in search_roots:
+        if not root.exists():
+            continue
+        
+        for pattern in BASELINE_FILE_PATTERNS:
+            for path in root.glob(pattern):
+                if should_ignore_path(path):
+                    continue
+                
+                try:
+                    source = path.read_text(encoding="utf-8")
+                    tree = ast.parse(source, filename=str(path))
+                except (OSError, SyntaxError):
+                    continue
+                
+                visitor = BaselineConfigVisitor()
+                visitor.visit(tree)
+                
+                for baseline_id, lineno in visitor.matches:
+                    key = (baseline_id, path.resolve())
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    
+                    results.append(
+                        BaselineChoice(
+                            baseline_id=baseline_id,
+                            path=path.resolve(),
+                            lineno=lineno,
+                            source="discovered",
+                        )
+                    )
+    
+    return results
+
+
+def load_baseline_config_from_file(
+    baseline_id: str,
+    path: Path,
+) -> BaselineConfig:
+    """Load a BaselineConfig from a Python file.
+    
+    Args:
+        baseline_id: The baseline_id to look for
+        path: Path to the Python file
+    
+    Returns:
+        BaselineConfig instance
+    
+    Raises:
+        ValueError: If baseline_id not found or file cannot be loaded
+    """
+    # Load the module
+    spec = importlib.util.spec_from_file_location("baseline_module", path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Cannot load baseline file: {path}")
+    
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except ModuleNotFoundError as e:
+        missing_module = str(e).split("'")[1] if "'" in str(e) else str(e)
+        raise ImportError(
+            f"❌ Missing dependency for baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Missing module: {missing_module}\n"
+            f"   Fix: pip install {missing_module}  (or 'uv add {missing_module}')"
+        ) from e
+    except SyntaxError as e:
+        raise ValueError(
+            f"❌ Syntax error in baseline file '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error at line {e.lineno}: {e.msg}\n"
+            f"   Text: {e.text.strip() if e.text else 'N/A'}\n"
+            f"   Fix: Check the Python syntax in the baseline file"
+        ) from e
+    except Exception as e:
+        error_type = type(e).__name__
+        raise ValueError(
+            f"❌ Failed to load baseline '{baseline_id}'\n"
+            f"   File: {path}\n"
+            f"   Error type: {error_type}\n"
+            f"   Message: {str(e)}\n"
+            f"   This may be due to:\n"
+            f"     - Missing dependencies (check imports)\n"
+            f"     - Configuration errors in the baseline file\n"
+            f"     - Environment variables not set\n"
+            f"   Tip: Run with --verbose for more details"
+        ) from e
+    
+    # Find the BaselineConfig instance
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig) and attr.baseline_id == baseline_id:
+            # Set source path for reference
+            attr._source_path = path
+            return attr
+    
+    # Provide helpful error message
+    found_configs = []
+    for attr_name in dir(module):
+        if attr_name.startswith("_"):
+            continue
+        attr = getattr(module, attr_name)
+        if isinstance(attr, BaselineConfig):
+            found_configs.append(attr.baseline_id)
+    
+    if found_configs:
+        raise ValueError(
+            f"❌ Baseline '{baseline_id}' not found in {path}\n"
+            f"   Found baselines in this file: {', '.join(found_configs)}\n"
+            f"   Fix: Use one of the above baseline IDs or check the baseline_id parameter"
+        )
+    else:
+        raise ValueError(
+            f"❌ No BaselineConfig instances found in {path}\n"
+            f"   Expected to find a BaselineConfig with baseline_id='{baseline_id}'\n"
+            f"   Fix: Ensure the file defines a BaselineConfig instance with baseline_id='{baseline_id}'"
+        )
+

synth_ai/baseline/execution.py

@@ -0,0 +1,146 @@
+"""Execution engine for baseline evaluations."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from synth_ai.baseline.config import (
+    BaselineConfig,
+    BaselineTaskRunner,
+    TaskResult,
+)
+
+
+def default_aggregator(results: List[TaskResult]) -> Dict[str, Any]:
+    """Default result aggregation function.
+    
+    Computes mean, std, min, max, success rate, and other basic metrics.
+    
+    Args:
+        results: List of TaskResult objects from all seeds
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    successful_results = [r for r in results if r.success]
+    outcome_rewards = [r.outcome_reward for r in successful_results]
+    
+    if not outcome_rewards:
+        return {
+            "mean_outcome_reward": 0.0,
+            "std_outcome_reward": 0.0,
+            "min_outcome_reward": 0.0,
+            "max_outcome_reward": 0.0,
+            "success_rate": 0.0,
+            "total_tasks": len(results),
+            "successful_tasks": 0,
+            "failed_tasks": len(results),
+        }
+    
+    mean_reward = sum(outcome_rewards) / len(outcome_rewards)
+    
+    # Calculate standard deviation
+    variance = sum((x - mean_reward) ** 2 for x in outcome_rewards) / len(outcome_rewards)
+    std_reward = variance ** 0.5
+    
+    return {
+        "mean_outcome_reward": mean_reward,
+        "std_outcome_reward": std_reward,
+        "min_outcome_reward": min(outcome_rewards),
+        "max_outcome_reward": max(outcome_rewards),
+        "success_rate": len(successful_results) / len(results),
+        "total_tasks": len(results),
+        "successful_tasks": len(successful_results),
+        "failed_tasks": len(results) - len(successful_results),
+    }
+
+
+def _is_class_based_runner(task_runner: Any) -> bool:
+    """Check if task_runner is a class (not a function)."""
+    return (
+        isinstance(task_runner, type)
+        and issubclass(task_runner, BaselineTaskRunner)
+    )
+
+
+async def run_baseline_evaluation(
+    config: BaselineConfig,
+    seeds: List[int],
+    policy_config: Dict[str, Any],
+    env_config: Dict[str, Any],
+    concurrency: int = 4,
+) -> List[TaskResult]:
+    """Run baseline evaluation for given seeds.
+    
+    Args:
+        config: BaselineConfig instance
+        seeds: List of seeds to evaluate
+        policy_config: Policy configuration (merged from defaults + overrides)
+        env_config: Environment configuration (merged from defaults + overrides)
+        concurrency: Maximum concurrent task executions
+    
+    Returns:
+        List of TaskResult objects, one per seed
+    """
+    # Determine if we're using class-based or function-based runner
+    is_class_based = _is_class_based_runner(config.task_runner)
+    
+    # Instantiate runner if class-based
+    runner_instance: Optional[BaselineTaskRunner] = None
+    if is_class_based:
+        runner_instance = config.task_runner(policy_config, env_config)
+    
+    # Create semaphore for concurrency control
+    semaphore = asyncio.Semaphore(concurrency)
+    
+    async def run_task(seed: int) -> TaskResult:
+        """Execute a single task with error handling."""
+        async with semaphore:
+            try:
+                if is_class_based and runner_instance:
+                    # Class-based: call run_task method
+                    return await runner_instance.run_task(seed)
+                else:
+                    # Function-based: call function directly
+                    task_runner_fn = config.task_runner
+                    return await task_runner_fn(seed, policy_config, env_config)
+            except Exception as exc:
+                # Return error result
+                return TaskResult(
+                    seed=seed,
+                    success=False,
+                    outcome_reward=0.0,
+                    error=str(exc),
+                )
+    
+    # Execute all tasks concurrently
+    results = await asyncio.gather(*[run_task(seed) for seed in seeds])
+    return list(results)
+
+
+def aggregate_results(
+    config: BaselineConfig,
+    results: List[TaskResult],
+) -> Dict[str, Any]:
+    """Aggregate results using custom aggregator or default.
+    
+    Args:
+        config: BaselineConfig instance
+        results: List of TaskResult objects
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    if config.result_aggregator is None:
+        return default_aggregator(results)
+    
+    # Check if aggregator is a class or function
+    if isinstance(config.result_aggregator, type):
+        # Class-based: instantiate and call aggregate()
+        aggregator_instance = config.result_aggregator()
+        return aggregator_instance.aggregate(results)
+    else:
+        # Function-based: call directly
+        return config.result_aggregator(results)
+

synth_ai/cli/__init__.py

@@ -53,12 +53,72 @@ if not _cli_module:
 cli = _cli_module.cli  # type: ignore[attr-defined]
 
 # Register core commands implemented as standalone modules
+try:
+    from synth_ai.cli.demo import demo_cmd
+    cli.add_command(demo_cmd, name="demo")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register demo command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
 try:
     from synth_ai.cli.setup import setup_cmd
-
     cli.add_command(setup_cmd, name="setup")
-except Exception:
-    pass
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register setup command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.deploy import deploy_cmd  # type: ignore[attr-defined]
+    cli.add_command(deploy_cmd, name="deploy")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register deploy command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.opencode import opencode_cmd
+    cli.add_command(opencode_cmd, name="opencode")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register opencode command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.codex import codex_cmd
+    cli.add_command(codex_cmd, name="codex")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register codex command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.eval import command as eval_cmd
+    cli.add_command(eval_cmd, name="eval")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register eval command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.claude import claude_cmd
+    cli.add_command(claude_cmd, name="claude")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register claude command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
+try:
+    from synth_ai.cli.commands.baseline import command as baseline_cmd
+    from synth_ai.cli.commands.baseline.list import list_command as baseline_list_cmd
+    cli.add_command(baseline_cmd, name="baseline")
+    baseline_cmd.add_command(baseline_list_cmd, name="list")
+except Exception as e:
+    import sys
+    print(f"[DEBUG] Failed to register baseline command: {e}", file=sys.stderr)
+    import traceback
+    traceback.print_exc()
 
 
 # Register optional subcommands packaged under synth_ai.cli.*
@@ -72,6 +132,14 @@ for _module_path in ("synth_ai.cli.commands.demo", "synth_ai.cli.commands.status
     if fn:
         fn(cli)
 
+# Smoke command registration (CLI-only helper)
+try:
+    from synth_ai.cli.commands.smoke import register as register_smoke
+
+    register_smoke(cli)
+except Exception:
+    pass
+
 # Register help command
 _maybe_call("synth_ai.cli.commands.help.core", "register", cli)
 
@@ -80,19 +148,19 @@ _maybe_call("synth_ai.api.train", "register", cli)
 
 # Task app group/commands are optional and have richer API surface
 _task_apps_module = _maybe_import("synth_ai.cli.task_apps")
-if _task_apps_module:
-    task_app_group = getattr(_task_apps_module, "task_app_group", None)
-    if task_app_group is not None:
-        cli.add_command(task_app_group, name="task-app")
-        # Expose common aliases when present
-        commands = getattr(task_app_group, "commands", None)
-        if isinstance(commands, dict):
-            for alias, name in (("serve", "serve"), ("deploy", "deploy"), ("modal-serve", "modal-serve")):
-                command = commands.get(name)
-                if command is not None:
-                    cli.add_command(command, name=alias)
-    register_task_apps = _callable_from(_task_apps_module, "register")
-    if register_task_apps:
-        register_task_apps(cli)
+#if _task_apps_module:
+task_app_group = getattr(_task_apps_module, "task_app_group", None)
+if task_app_group is not None:
+    cli.add_command(task_app_group, name="task-app")
+    # Expose common aliases when present
+    commands = getattr(task_app_group, "commands", None)
+    if isinstance(commands, dict):
+        for alias, name in (("serve", "serve"), ("deploy", "deploy"), ("modal-serve", "modal-serve")):
+            command = commands.get(name)
+            if command is not None:
+                cli.add_command(command, name=alias)
+register_task_apps = _callable_from(_task_apps_module, "register")
+if register_task_apps:
+    register_task_apps(cli)
 
 # Top-level 'info' alias removed; use `synth-ai task-app info` instead

synth_ai/cli/claude.py

@@ -0,0 +1,70 @@
+import os
+import subprocess
+
+import click
+from synth_ai.types import MODEL_NAMES, ModelName
+from synth_ai.urls import BACKEND_URL_SYNTH_RESEARCH_ANTHROPIC
+from synth_ai.utils import find_bin_path, install_bin, resolve_env_var, verify_bin, write_agents_md
+
+
+@click.command("claude")
+@click.option(
+    "--model",
+    "model_name",
+    type=str,
+    default=None
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Prompt for API keys even if cached values exist."
+)
+@click.option(
+    "--url",
+    "override_url",
+    type=str,
+    default=None,
+)
+def claude_cmd(
+    model_name: ModelName | None = None,
+    force: bool = False,
+    override_url: str | None = None
+) -> None:
+
+    while True:
+        bin_path = find_bin_path("claude")
+        if bin_path:
+            break
+        if not install_bin(
+            "Claude Code",
+            ["curl -fsSL https://claude.ai/install.sh | bash"]
+        ):
+            print("Failed to find your installed Claude Code")
+            print("Please install from: https://claude.com/claude-code")
+            return
+    print(f"Using Claude at {bin_path}")
+
+    if not verify_bin(bin_path):
+        print("Failed to verify Claude Code is runnable")
+        return
+
+    write_agents_md()
+    env = os.environ.copy()
+
+    if model_name is not None:
+        if model_name not in MODEL_NAMES:
+            raise ValueError(f"model_name={model_name} is invalid. Valid values for model_name: {MODEL_NAMES}")
+        if override_url:
+            url = f"{override_url.rstrip('/')}/{model_name}"
+            print(f"Using override URL with model: {url}")
+        else:
+            url = f"{BACKEND_URL_SYNTH_RESEARCH_ANTHROPIC}/{model_name}"
+        env["ANTHROPIC_BASE_URL"] = url
+        api_key = resolve_env_var("SYNTH_API_KEY", override_process_env=force)
+        env["ANTHROPIC_AUTH_TOKEN"] = api_key
+        env["SYNTH_API_KEY"] = api_key
+
+    try:
+        subprocess.run(["claude"], check=True, env=env)
+    except subprocess.CalledProcessError:
+        print("Failed to launch Claude Code")

synth_ai/cli/codex.py

@@ -0,0 +1,84 @@
+import os
+import subprocess
+
+import click
+from synth_ai.types import MODEL_NAMES, ModelName
+from synth_ai.urls import BACKEND_URL_SYNTH_RESEARCH_OPENAI
+from synth_ai.utils import find_bin_path, install_bin, resolve_env_var, verify_bin, write_agents_md
+
+
+@click.command("codex")
+@click.option(
+    "--model",
+    "model_name",
+    type=str,
+    default=None
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Prompt for API keys even if cached values exist."
+)
+@click.option(
+    "--url",
+    "override_url",
+    type=str,
+    default=None,
+)
+def codex_cmd(
+    model_name: ModelName | None = None,
+    force: bool = False,
+    override_url: str | None = None
+)-> None:
+
+    while True:
+        bin_path = find_bin_path("codex")
+        if bin_path:
+            break
+        if not install_bin(
+            "Codex",
+            [
+                "brew install codex",
+                "npm install -g @openai/codex"
+            ]
+        ):
+            print("Failed to find your installed Codex")
+            print("Please install from: https://developers.openai.com/codex/cli/")
+            return
+    print(f"Using Codex at {bin_path}")
+
+    if not verify_bin(bin_path):
+        print("Failed to verify Codex is runnable")
+        return
+    
+    write_agents_md()
+    env = os.environ.copy()
+    override_args = []
+    
+    if model_name is not None:
+        if model_name not in MODEL_NAMES:
+            raise ValueError(f"model_name={model_name} is invalid. Valid values for model_name: {MODEL_NAMES}")
+        if override_url:
+            url = override_url
+            print("Using override URL:", url)
+        else:
+            url = BACKEND_URL_SYNTH_RESEARCH_OPENAI
+        provider_config = f'{{name="Synth",base_url="{url}",env_key="OPENAI_API_KEY"}}'
+        config_overrides = [
+            f"model_providers.synth={provider_config}",
+            'model_provider="synth"',
+            f'default_model="{model_name}"'
+        ]
+        override_args = [arg for override in config_overrides for arg in ("-c", override)]
+        env["OPENAI_API_KEY"] = resolve_env_var("SYNTH_API_KEY", override_process_env=force)
+        env["SYNTH_API_KEY"] = env["OPENAI_API_KEY"]
+    
+    try:
+        cmd = ["codex"]
+        if model_name is not None:
+            cmd.extend(["-m", model_name])
+        cmd.extend(override_args)
+        print(" ".join(cmd))
+        subprocess.run(cmd, check=True, env=env)
+    except subprocess.CalledProcessError:
+        print("Failed to run Codex")

synth_ai/cli/commands/__init__.py

@@ -14,4 +14,5 @@ __all__ = [
     "filter",
     "deploy",
     "status",
+    "smoke",
 ]

synth_ai/cli/commands/baseline/__init__.py

@@ -0,0 +1,12 @@
+"""CLI command for baseline evaluation."""
+
+from __future__ import annotations
+
+from .core import command
+from .list import list_command
+
+__all__ = ["command"]
+
+# Register list subcommand
+command.add_command(list_command, name="list")
+