synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/baseline/execution.py

@@ -0,0 +1,146 @@
+"""Execution engine for baseline evaluations."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from synth_ai.baseline.config import (
+    BaselineConfig,
+    BaselineTaskRunner,
+    TaskResult,
+)
+
+
+def default_aggregator(results: List[TaskResult]) -> Dict[str, Any]:
+    """Default result aggregation function.
+    
+    Computes mean, std, min, max, success rate, and other basic metrics.
+    
+    Args:
+        results: List of TaskResult objects from all seeds
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    successful_results = [r for r in results if r.success]
+    outcome_rewards = [r.outcome_reward for r in successful_results]
+    
+    if not outcome_rewards:
+        return {
+            "mean_outcome_reward": 0.0,
+            "std_outcome_reward": 0.0,
+            "min_outcome_reward": 0.0,
+            "max_outcome_reward": 0.0,
+            "success_rate": 0.0,
+            "total_tasks": len(results),
+            "successful_tasks": 0,
+            "failed_tasks": len(results),
+        }
+    
+    mean_reward = sum(outcome_rewards) / len(outcome_rewards)
+    
+    # Calculate standard deviation
+    variance = sum((x - mean_reward) ** 2 for x in outcome_rewards) / len(outcome_rewards)
+    std_reward = variance ** 0.5
+    
+    return {
+        "mean_outcome_reward": mean_reward,
+        "std_outcome_reward": std_reward,
+        "min_outcome_reward": min(outcome_rewards),
+        "max_outcome_reward": max(outcome_rewards),
+        "success_rate": len(successful_results) / len(results),
+        "total_tasks": len(results),
+        "successful_tasks": len(successful_results),
+        "failed_tasks": len(results) - len(successful_results),
+    }
+
+
+def _is_class_based_runner(task_runner: Any) -> bool:
+    """Check if task_runner is a class (not a function)."""
+    return (
+        isinstance(task_runner, type)
+        and issubclass(task_runner, BaselineTaskRunner)
+    )
+
+
+async def run_baseline_evaluation(
+    config: BaselineConfig,
+    seeds: List[int],
+    policy_config: Dict[str, Any],
+    env_config: Dict[str, Any],
+    concurrency: int = 4,
+) -> List[TaskResult]:
+    """Run baseline evaluation for given seeds.
+    
+    Args:
+        config: BaselineConfig instance
+        seeds: List of seeds to evaluate
+        policy_config: Policy configuration (merged from defaults + overrides)
+        env_config: Environment configuration (merged from defaults + overrides)
+        concurrency: Maximum concurrent task executions
+    
+    Returns:
+        List of TaskResult objects, one per seed
+    """
+    # Determine if we're using class-based or function-based runner
+    is_class_based = _is_class_based_runner(config.task_runner)
+    
+    # Instantiate runner if class-based
+    runner_instance: Optional[BaselineTaskRunner] = None
+    if is_class_based:
+        runner_instance = config.task_runner(policy_config, env_config)
+    
+    # Create semaphore for concurrency control
+    semaphore = asyncio.Semaphore(concurrency)
+    
+    async def run_task(seed: int) -> TaskResult:
+        """Execute a single task with error handling."""
+        async with semaphore:
+            try:
+                if is_class_based and runner_instance:
+                    # Class-based: call run_task method
+                    return await runner_instance.run_task(seed)
+                else:
+                    # Function-based: call function directly
+                    task_runner_fn = config.task_runner
+                    return await task_runner_fn(seed, policy_config, env_config)
+            except Exception as exc:
+                # Return error result
+                return TaskResult(
+                    seed=seed,
+                    success=False,
+                    outcome_reward=0.0,
+                    error=str(exc),
+                )
+    
+    # Execute all tasks concurrently
+    results = await asyncio.gather(*[run_task(seed) for seed in seeds])
+    return list(results)
+
+
+def aggregate_results(
+    config: BaselineConfig,
+    results: List[TaskResult],
+) -> Dict[str, Any]:
+    """Aggregate results using custom aggregator or default.
+    
+    Args:
+        config: BaselineConfig instance
+        results: List of TaskResult objects
+    
+    Returns:
+        Dict with aggregate metrics
+    """
+    if config.result_aggregator is None:
+        return default_aggregator(results)
+    
+    # Check if aggregator is a class or function
+    if isinstance(config.result_aggregator, type):
+        # Class-based: instantiate and call aggregate()
+        aggregator_instance = config.result_aggregator()
+        return aggregator_instance.aggregate(results)
+    else:
+        # Function-based: call directly
+        return config.result_aggregator(results)
+

synth_ai/cfgs.py

@@ -0,0 +1,227 @@
+import os
+from pathlib import Path
+from typing import Any, Literal, Optional, cast
+
+from pydantic import BaseModel
+from synth_ai.utils import log_error, log_event
+from synth_ai.utils.apps import validate_modal_app, validate_task_app
+from synth_ai.utils.paths import get_bin_path
+
+
+class LocalDeployCfg(BaseModel):
+    task_app_path: Path
+    env_api_key: str
+    trace: bool = True
+    host: str = "127.0.0.1"
+    port: int = 8000
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        task_app_path: Path,
+        env_api_key: str,
+        trace: bool = True,
+        host: str = "127.0.0.1",
+        port: int = 8000
+    ) -> "LocalDeployCfg":
+        ctx = {
+            "task_app": str(task_app_path),
+            "trace": bool(trace),
+            "host": str(host),
+            "port": int(port),
+        }
+        log_event("info", "creating LocalDeployCfg", ctx=ctx)
+        try:
+            validate_task_app(task_app_path)
+            cfg = cls(
+                task_app_path=task_app_path,
+                env_api_key=env_api_key,
+                trace=trace,
+                host=host,
+                port=port,
+            )
+            log_event("info", "LocalDeployCfg created", ctx=ctx)
+            return cfg
+        except Exception as exc:
+            log_error("LocalDeployCfg creation failed", ctx={**ctx, "error": type(exc).__name__})
+            raise
+
+    @classmethod
+    def create_from_dict(
+        cls,
+        data: dict[str, Any]
+    ) -> "LocalDeployCfg":
+        path = Path(data["task_app_path"])
+        ctx = {
+            "task_app": str(path),
+            "trace": bool(data.get("trace", True)),
+            "host": str(data.get("host", "127.0.0.1")),
+            "port": int(data.get("port", 8000)),
+        }
+        log_event("info", "creating LocalDeployCfg from dict", ctx=ctx)
+        validate_task_app(path)
+        try:
+            cfg = cls(
+                task_app_path=path,
+                env_api_key=str(data.get("env_api_key", "")),
+                trace=bool(data.get("trace", True)),
+                host=str(data.get("host", "127.0.0.1")),
+                port=int(data.get("port", 8000)),
+            )
+            log_event("info", "LocalDeployCfg from dict created", ctx=ctx)
+            return cfg
+        except (KeyError, TypeError, ValueError) as err:
+            log_error("LocalDeployCfg from dict failed", ctx={**ctx, "error": type(err).__name__})
+            raise ValueError(f"Invalid local deploy configuration: {err}") from err
+        
+
+class ModalDeployCfg(BaseModel):
+    task_app_path: Path
+    modal_app_path: Path
+    modal_bin_path: Path
+    synth_api_key: str
+    env_api_key: str
+    cmd_arg: Literal["deploy", "serve"] = "deploy"
+    task_app_name: Optional[str] = None
+    dry_run: bool = False
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        task_app_path: Path,
+        modal_app_path: Path,
+        synth_api_key: str,
+        env_api_key: str,
+        modal_bin_path: Path | None = None,
+        cmd_arg: Literal["deploy", "serve"] = "deploy",
+        task_app_name: Optional[str] = None,
+        dry_run: bool = False,
+    ) -> "ModalDeployCfg":
+        ctx = {
+            "task_app": str(task_app_path),
+            "modal_app": str(modal_app_path),
+            "cmd_arg": cmd_arg,
+            "dry_run": dry_run,
+            "task_app_name": task_app_name,
+            "modal_cli": str(modal_bin_path) if modal_bin_path else None,
+        }
+        log_event("info", "creating ModalDeployCfg", ctx=ctx)
+        modal_bin_path = modal_bin_path or get_bin_path("modal")
+        if modal_bin_path is None:
+            log_error("ModalDeployCfg creation failed", ctx={**ctx, "error": "ModalCLINotFound"})
+            raise ValueError("Modal CLI not found; install `modal` or pass --modal-cli with its path.")
+        try:
+            cfg = cls(
+                task_app_path=validate_task_app(task_app_path),
+                modal_app_path=validate_modal_app(modal_app_path),
+                modal_bin_path=modal_bin_path,
+                synth_api_key=synth_api_key,
+                env_api_key=env_api_key,
+                cmd_arg=cmd_arg,
+                task_app_name=task_app_name,
+                dry_run=dry_run,
+            )
+            log_event("info", "ModalDeployCfg created", ctx=ctx)
+            return cfg
+        except Exception as exc:
+            log_error("ModalDeployCfg creation failed", ctx={**ctx, "error": type(exc).__name__})
+            raise
+    
+    @classmethod
+    def create_from_kwargs(cls, **kwargs: Any) -> "ModalDeployCfg":
+        synth_api_key = kwargs.get("synth_api_key")
+        if not synth_api_key:
+            raise ValueError("synth_api_key must be provided")
+        env_api_key = kwargs.get("env_api_key")
+        if not env_api_key:
+            raise ValueError("env_api_key must be provided")
+
+        cmd_arg = str(kwargs.get("cmd_arg", "deploy")).strip().lower()
+        if cmd_arg not in {"deploy", "serve"}:
+            raise ValueError("`--modal-mode` must be either 'deploy' or 'serve'.")
+
+        dry_run = bool(kwargs.get("dry_run", False))
+        if dry_run and os.getenv("CTX") == "mcp":
+            log_error("ModalDeployCfg create_from_kwargs blocked", ctx={"reason": "dry_run_mcp"})
+            raise ValueError("`synth-ai deploy --runtime modal --dry-run` cannot be used by MCP")
+        if dry_run and cmd_arg == "serve":
+            log_error("ModalDeployCfg create_from_kwargs blocked", ctx={"reason": "dry_run_serve"})
+            raise ValueError("`synth-ai deploy --runtime modal --modal-mode serve` cannot be used with `--dry-run`")
+
+        modal_bin_path_arg = kwargs.get("modal_bin_path")
+        modal_bin_path = Path(str(modal_bin_path_arg)).expanduser() if modal_bin_path_arg else get_bin_path("modal")
+        if modal_bin_path is None or not modal_bin_path.exists():
+            log_error(
+                "ModalDeployCfg create_from_kwargs failed",
+                ctx={"error": "ModalCLINotFound", "provided_modal_cli": str(modal_bin_path_arg or "")},
+            )
+            raise ValueError('Modal binary not found via shutil.which("modal"). Install `modal` or pass --modal-cli with its path.')
+        modal_bin_path = modal_bin_path.resolve()
+
+        task_app_name = kwargs.get("task_app_name")
+        if task_app_name is not None:
+            task_app_name = str(task_app_name).strip() or None
+
+        literal_cmd = cast(Literal["deploy", "serve"], cmd_arg)
+
+        ctx = {
+            "task_app": str(kwargs.get("task_app_path")),
+            "modal_app": str(kwargs.get("modal_app_path")),
+            "cmd_arg": literal_cmd,
+            "dry_run": dry_run,
+            "task_app_name": task_app_name,
+            "modal_cli": str(modal_bin_path) if modal_bin_path else None,
+        }
+        log_event("info", "creating ModalDeployCfg from kwargs", ctx=ctx)
+        try:
+            cfg = cls(
+                task_app_path=validate_task_app(kwargs.get("task_app_path")),
+                modal_app_path=validate_modal_app(kwargs.get("modal_app_path")),
+                modal_bin_path=modal_bin_path,
+                synth_api_key=synth_api_key,
+                env_api_key=env_api_key,
+                cmd_arg=literal_cmd,
+                task_app_name=task_app_name,
+                dry_run=dry_run,
+            )
+            log_event("info", "ModalDeployCfg from kwargs created", ctx=ctx)
+            return cfg
+        except Exception as exc:
+            log_error("ModalDeployCfg from kwargs failed", ctx={**ctx, "error": type(exc).__name__})
+            raise
+
+
+class CloudflareTunnelDeployCfg(BaseModel):
+    task_app_path: Path
+    env_api_key: str
+    host: str = "127.0.0.1"
+    port: int = 8000
+    mode: Literal["quick", "managed"] = "quick"
+    tunnel_token: Optional[str] = None
+    subdomain: Optional[str] = None
+    trace: bool = True
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        task_app_path: Path,
+        env_api_key: str,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        mode: Literal["quick", "managed"] = "quick",
+        subdomain: Optional[str] = None,
+        trace: bool = True,
+    ) -> "CloudflareTunnelDeployCfg":
+        validate_task_app(task_app_path)
+        return cls(
+            task_app_path=task_app_path,
+            env_api_key=env_api_key,
+            host=host,
+            port=port,
+            mode=mode,
+            subdomain=subdomain,
+            trace=trace,
+        )

synth_ai/cli/__init__.py

@@ -5,7 +5,21 @@ and exposes a top-level Click group named `cli` compatible with the
 pyproject entry point `synth_ai.cli:cli`.
 """
 
-from __future__ import annotations
+import importlib
+import sys
+from collections.abc import Callable
+from typing import Any
+
+from synth_ai.cli.claude import claude_cmd
+from synth_ai.cli.codex import codex_cmd
+from synth_ai.cli.commands.baseline import command as baseline_cmd
+from synth_ai.cli.commands.baseline.list import list_command as baseline_list_cmd
+from synth_ai.cli.demo import demo_cmd
+from synth_ai.cli.deploy import deploy_cmd
+from synth_ai.cli.eval import command as eval_cmd
+from synth_ai.cli.mcp import mcp_cmd
+from synth_ai.cli.opencode import opencode_cmd
+from synth_ai.cli.setup import setup_cmd
 
 # Load environment variables from a local .env if present (repo root)
 try:
@@ -17,61 +31,92 @@ except Exception:
     # dotenv is optional at runtime; proceed if unavailable
     pass
 
+def _callable_from(module: Any, attr: str) -> Callable[..., Any] | None:
+    candidate = getattr(module, attr, None)
+    return candidate if callable(candidate) else None
 
-from .root import cli  # new canonical CLI entrypoint
 
-# Register subcommands from this package onto the group
-try:
-    from . import watch as _watch
+def _maybe_import(module_path: str) -> Any | None:
+    try:
+        return importlib.import_module(module_path)
+    except Exception:
+        return None
 
-    _watch.register(cli)
-except Exception:
-    pass
-try:
-    from . import balance as _balance
 
-    _balance.register(cli)
-except Exception:
-    pass
-try:
-    from . import man as _man
+def _maybe_call(module_path: str, attr: str, *args: Any, **kwargs: Any) -> None:
+    module = _maybe_import(module_path)
+    if not module:
+        return
+    fn = _callable_from(module, attr)
+    if fn:
+        fn(*args, **kwargs)
 
-    _man.register(cli)
-except Exception:
-    pass
-try:
-    from . import traces as _traces
 
-    _traces.register(cli)
-except Exception:
-    pass
-try:
-    from . import recent as _recent
+# Apply Typer patch if available
+_maybe_call("synth_ai.cli._typer_patch", "patch_typer_make_metavar")
 
-    _recent.register(cli)
-except Exception:
-    pass
-try:
-    from . import calc as _calc
 
-    _calc.register(cli)
-except Exception:
-    pass
-try:
-    from . import status as _status
+_cli_module = _maybe_import("synth_ai.cli.root")
+if not _cli_module:
+    raise ImportError("synth_ai.cli.root is required for CLI entrypoint")
+cli = _cli_module.cli  # type: ignore[attr-defined]
 
-    _status.register(cli)
-except Exception:
-    pass
-try:
-    from . import demo as _demo
 
-    _demo.register(cli)
-except Exception:
-    pass
+
+# Register core commands implemented as standalone modules
+
+cli.add_command(baseline_cmd, name="baseline")
+baseline_cmd.add_command(baseline_list_cmd, name="list")
+cli.add_command(claude_cmd, name="claude")
+cli.add_command(codex_cmd, name="codex")
+cli.add_command(demo_cmd, name="demo")
+cli.add_command(deploy_cmd, name="deploy")
+cli.add_command(eval_cmd, name="eval")
+cli.add_command(mcp_cmd, name="mcp")
+cli.add_command(opencode_cmd, name="opencode")
+cli.add_command(setup_cmd, name="setup")
+
+
+# Register optional subcommands packaged under synth_ai.cli.*
+for _module_path in ("synth_ai.cli.commands.demo", "synth_ai.cli.commands.status", "synth_ai.cli.turso"):
+    module = _maybe_import(_module_path)
+    if not module:
+        continue
+    sub_name = _module_path.rsplit(".", 1)[-1]
+    setattr(sys.modules[__name__], sub_name, module)
+    fn = _callable_from(module, "register")
+    if fn:
+        fn(cli)
+
+# Smoke command registration (CLI-only helper)
 try:
-    from . import rl_demo as _rl_demo
+    from synth_ai.cli.commands.smoke import register as register_smoke
 
-    _rl_demo.register(cli)
+    register_smoke(cli)
 except Exception:
     pass
+
+# Register help command
+_maybe_call("synth_ai.cli.commands.help.core", "register", cli)
+
+# Train CLI lives under synth_ai.api.train
+_maybe_call("synth_ai.api.train", "register", cli)
+
+# Task app group/commands are optional and have richer API surface
+_task_apps_module = _maybe_import("synth_ai.cli.task_apps")
+#if _task_apps_module:
+task_app_group = getattr(_task_apps_module, "task_app_group", None)
+if task_app_group is not None:
+    cli.add_command(task_app_group, name="task-app")
+    # Expose common aliases when present
+    commands = getattr(task_app_group, "commands", None)
+    if isinstance(commands, dict):
+        for alias, name in (("serve", "serve"), ("deploy", "deploy"), ("modal-serve", "modal-serve")):
+            command = commands.get(name)
+            if command is not None:
+                cli.add_command(command, name=alias)
+register_task_apps = _callable_from(_task_apps_module, "register")
+if register_task_apps:
+    register_task_apps(cli)
+
+# Top-level 'info' alias removed; use `synth-ai task-app info` instead

synth_ai/cli/_modal_wrapper.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import importlib
+import sys
+
+
+def main() -> int:
+    # Apply Typer compatibility patch before Modal CLI bootstraps Click/Typer internals.
+    try:
+        module = importlib.import_module("synth_ai.cli._typer_patch")
+    except Exception:
+        module = None
+    if module is not None:
+        patch = getattr(module, "patch_typer_make_metavar", None)
+        if callable(patch):
+            patch()
+
+    from modal.__main__ import main as modal_main
+
+    # Present ourselves as the upstream `modal` CLI so Typer/Click parsing stays intact.
+    if sys.argv:
+        sys.argv[0] = "modal"
+    else:
+        sys.argv = ["modal"]
+
+    result = modal_main()
+    return result if result is not None else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

synth_ai/cli/_storage.py

@@ -0,0 +1,20 @@
+"""Dynamic loader for tracing storage components.
+
+This avoids hard dependencies on the tracing_v3 storage package during import
+time, which keeps CLI modules usable in constrained environments while still
+allowing type checkers to resolve the symbols dynamically.
+"""
+
+from __future__ import annotations
+
+import importlib
+from collections.abc import Callable
+from typing import Any, cast
+
+
+def load_storage() -> tuple[Any, Any]:
+    """Return (create_storage, StorageConfig) from tracing_v3.storage."""
+    storage_module = cast(Any, importlib.import_module("synth_ai.tracing_v3.storage"))
+    create_storage = cast(Callable[..., Any], storage_module.create_storage)
+    storage_config = storage_module.StorageConfig
+    return create_storage, storage_config

synth_ai/cli/_typer_patch.py

@@ -0,0 +1,47 @@
+from click import Parameter
+
+
+def patch_typer_make_metavar() -> None:
+    try:
+        from typer.main import TyperArgument, TyperOption
+    except Exception:
+        return
+
+    def _call_get_metavar(param_type: object, param: Parameter) -> str | None:
+        getter = getattr(param_type, "get_metavar", None)
+        if getter is None:
+            return None
+        try:
+            return getter(param)
+        except TypeError:
+            try:
+                return getter(param, None)
+            except TypeError:
+                return None
+
+    def _patched_argument_make_metavar(self, ctx=None) -> str:
+        if self.metavar is not None:
+            return self.metavar
+        var = (self.name or "").upper()
+        if not self.required:
+            var = f"[{var}]"
+        type_var = _call_get_metavar(self.type, self)
+        if type_var:
+            var += f":{type_var}"
+        if self.nargs != 1:
+            var += "..."
+        return var
+
+    def _patched_option_make_metavar(self, ctx=None) -> str:
+        if self.metavar is not None:
+            return self.metavar
+        metavar = _call_get_metavar(self.type, self)
+        if not metavar:
+            name = getattr(self.type, "name", "") or ""
+            metavar = name.upper()
+        if self.nargs != 1:
+            metavar += "..."
+        return metavar
+
+    TyperArgument.make_metavar = _patched_argument_make_metavar  # type: ignore[assignment]
+    TyperOption.make_metavar = _patched_option_make_metavar  # type: ignore[assignment]

synth_ai/cli/_validate_task_app.py

@@ -0,0 +1,29 @@
+"""Task app validation utilities - imported by task_apps.py"""
+
+# This module provides the validate_task_app function for CLI use
+# The actual implementation is imported from the task module
+
+import importlib
+from collections.abc import Callable
+from typing import Any
+
+_validators_module: Any | None = None
+validate_task_app: Callable[..., Any] | None = None
+
+try:
+    _validators_module = importlib.import_module("synth_ai.task.validators")
+except Exception:
+    _validators_module = None
+
+if _validators_module is not None:
+    candidate = getattr(_validators_module, "validate_task_app_endpoint", None)
+    if callable(candidate):
+        validate_task_app = candidate
+
+if validate_task_app is None:
+    def _missing_validate_task_app(*_args: Any, **_kwargs: Any) -> None:
+        raise RuntimeError("task validation utilities are unavailable in this environment")
+
+    validate_task_app = _missing_validate_task_app
+
+__all__ = ["validate_task_app"]