synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/cli/demo_apps/mipro/train_cfg.toml

@@ -0,0 +1,92 @@
+# MIPROv2 Prompt Learning for Banking77
+# Local backend configuration targeting the Banking77 intent classification task app.
+
+[prompt_learning]
+algorithm = "mipro"
+task_app_url = "http://127.0.0.1:8102"
+task_app_id = "banking77"
+
+# Initial prompt pattern (pattern-based mode)
+[prompt_learning.initial_prompt]
+id = "banking77_pattern"
+name = "Banking77 Classification Pattern"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "You are an expert banking assistant that classifies customer queries into banking intents. Given a customer message, respond with exactly one intent label from the provided list using the `banking77_classify` tool."
+order = 0
+
+[[prompt_learning.initial_prompt.messages]]
+role = "user"
+pattern = "Customer Query: {query}\n\nClassify this query into one of the banking intents using the tool call."
+order = 1
+
+[prompt_learning.initial_prompt.wildcards]
+query = "REQUIRED"  # Will be provided by task app at runtime
+
+# Policy configuration (model, provider, etc.)
+[prompt_learning.policy]
+inference_mode = "synth_hosted"
+model = "openai/gpt-oss-20b"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 128
+policy_name = "banking77-mipro"  # Required for Banking77 task app
+
+# Training split config
+[prompt_learning.env_config]
+pool = "train"
+
+# MIPROv2-specific configuration
+[prompt_learning.mipro]
+env_name = "banking77"
+num_iterations = 5  # 5 iterations × 2 per iteration = 10 total prompt combos
+num_evaluations_per_iteration = 2  # 2 proposals per iteration
+batch_size = 6
+max_concurrent = 16
+meta_model = "llama-3.3-70b-versatile"
+meta_model_provider = "groq"
+few_shot_score_threshold = 0.85
+max_instructions = 3
+
+# Retry guard for duplicate proposals/candidates
+duplicate_retry_limit = 10
+
+# Seed pools for different phases - tons of data for reliable evaluation
+bootstrap_train_seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]  # 15 seeds for bootstrap
+online_pool = [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]  # 25 seeds for online evaluation
+test_pool = [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]  # 10 held-out seeds for final evaluation
+val_seeds = [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]  # Validation seeds (matches GEPA terminology, used for top-K evaluation)
+reference_pool = [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]  # Reference corpus seeds (for meta-prompt context, same as val_seeds)
+
+# TPE hyperparameters
+[prompt_learning.mipro.tpe]
+gamma = 0.25              # Quantile for splitting good/bad observations
+n_candidates = 32         # Number of candidates to evaluate Expected Improvement
+n_startup_trials = 10     # Uniform random trials before TPE kicks in
+epsilon = 0.1             # Exploration probability (uniform sample)
+alpha = 1.0               # Smoothing parameter for KDE (Laplace smoothing for categorical variables)
+
+# Demo set selection hyperparameters
+[prompt_learning.mipro.demo]
+max_few_shot_examples = 5  # Maximum number of few-shot examples to include
+sets_per_size = 6          # Number of demo sets to generate per size
+include_empty = true       # Whether to include empty demo set (zero-shot)
+
+# Grounding configuration for instruction proposals
+[prompt_learning.mipro.grounding]
+n = 10                     # Number of instruction proposals to generate
+temperature = 0.7          # Temperature for LLM proposal generation
+max_tokens = 600           # Max tokens for proposal generation
+
+# Meta-update configuration
+[prompt_learning.mipro.meta_update]
+enabled = true             # Enable periodic meta-updates
+every_iterations = 3       # Regenerate instructions every N iterations
+topk_success = 5           # Number of top success examples to use for grounding
+topk_failure = 5           # Number of top failure examples to use for grounding
+validate_on_batch = 32     # Batch size for proposal validation
+keep_k = 12                # Maximum number of instruction variants to keep
+dedup_token_overlap = 0.8  # Token overlap threshold for deduplication
+regen_temperature_decay = 0.95  # Temperature decay factor per meta-update
+

synth_ai/cli/demos/__init__.py

@@ -0,0 +1,12 @@
+"""Demo CLI commands.
+
+Commands for running and copying demo task apps.
+"""
+
+from synth_ai.cli.demos.demo import demo_cmd
+
+__all__ = [
+    "demo_cmd",
+]
+
+

synth_ai/cli/demos/demo.py

@@ -0,0 +1,32 @@
+import shutil
+from pathlib import Path
+from typing import Literal, TypeAlias, get_args
+
+import click
+
+from synth_ai.core.paths import REPO_ROOT
+
+DemoType: TypeAlias = Literal[
+    "mipro",
+    "sft",
+    "rl"
+]
+
+
+@click.command()
+@click.argument(
+    "demo_type",
+    type=click.Choice(get_args(DemoType)),
+)
+def demo_cmd(demo_type: DemoType) -> None:
+    src = REPO_ROOT / "synth_ai" / "demos" / demo_type
+    if not src.exists():
+        raise click.ClickException(f"Demo source directory not found: {src}")
+    dst = Path.cwd() / f"demo_{src.name}"
+    if dst.exists():
+        if not click.confirm(f"Destination already exists: {dst}. Overwrite?", abort=True):
+            click.echo("Aborted.")
+            return
+        shutil.rmtree(dst)
+    shutil.copytree(src, dst)
+    click.echo(f"Copied {demo_type.upper()} demo to your CWD: {dst}")

synth_ai/cli/demos/rl_demo.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""
+New RL demo command group kept fully separate from legacy demo.
+
+Usage examples:
+  uvx synth-ai rl_demo check
+  uvx synth-ai rl_demo deploy --app /path/to/math_task_app.py --name synth-math-demo
+  uvx synth-ai rl_demo configure
+  uvx synth-ai rl_demo run --batch-size 4 --group-size 16 --model Qwen/Qwen3-0.6B
+  uvx synth-ai run --config demo_config.toml
+
+For convenience, dotted aliases are also exposed:
+  uvx synth-ai rl_demo.check
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any, cast
+
+import click
+from click.exceptions import Exit
+
+demo_commands = cast(Any, importlib.import_module("synth_ai.cli.demo_apps.core.cli"))
+
+
+def _run_demo_command(func, *args, **kwargs) -> None:
+    """Invoke a demo command and exit via Click on non-zero status codes."""
+
+    try:
+        result = func(*args, **kwargs)
+    except SystemExit as exc:  # pragma: no cover - defensive
+        raise Exit(exc.code if isinstance(exc.code, int) else 1) from exc
+
+    if result is None:
+        return
+    try:
+        code = int(result)
+    except (TypeError, ValueError):
+        return
+    if code != 0:
+        raise Exit(code)
+
+
+def register(cli):
+    @cli.group("rl_demo")
+    def rl_demo():
+        """RL Demo commands (separate from legacy demo)."""
+
+    # Help pyright understand dynamic Click group attributes
+    _rlg = cast(Any, rl_demo)
+
+    @_rlg.command("setup")
+    def rl_setup():
+        _run_demo_command(demo_commands.setup)
+
+    # (prepare command removed; consolidated into configure)
+
+    @_rlg.command("deploy")
+    @click.option("--local", is_flag=True, help="Run local FastAPI instead of Modal deploy")
+    @click.option(
+        "--app",
+        type=click.Path(),
+        default=None,
+        help="Path to Modal app.py for uv run modal deploy",
+    )
+    @click.option("--name", type=str, default="synth-math-demo", help="Modal app name")
+    @click.option(
+        "--script",
+        type=click.Path(),
+        default=None,
+        help="Path to deploy_task_app.sh (optional legacy)",
+    )
+    def rl_deploy(local: bool, app: str | None, name: str, script: str | None):
+        _run_demo_command(
+            demo_commands.deploy,
+            local=local,
+            app=app,
+            name=name,
+            script=script,
+        )
+
+    @_rlg.command("configure")
+    def rl_configure():
+        _run_demo_command(demo_commands.run)
+
+    @_rlg.command("init")
+    @click.option("--template", type=str, default=None, help="Template id to instantiate")
+    @click.option("--dest", type=click.Path(), default=None, help="Destination directory for files")
+    @click.option("--force", is_flag=True, help="Overwrite existing files in destination")
+    def rl_init(template: str | None, dest: str | None, force: bool):
+        _run_demo_command(
+            demo_commands.init,
+            template=template,
+            dest=dest,
+            force=force,
+        )
+
+    @_rlg.command("run")
+    @click.option(
+        "--config", type=click.Path(), default=None, help="Path to TOML config (skip prompt)"
+    )
+    @click.option("--batch-size", type=int, default=None)
+    @click.option("--group-size", type=int, default=None)
+    @click.option("--model", type=str, default=None)
+    @click.option("--timeout", type=int, default=600)
+    @click.option("--dry-run", is_flag=True, help="Print request body and exit")
+    def rl_run(
+        config: str | None,
+        batch_size: int | None,
+        group_size: int | None,
+        model: str | None,
+        timeout: int,
+        dry_run: bool,
+    ):
+        _run_demo_command(
+            demo_commands.run,
+            config=config,
+            batch_size=batch_size,
+            group_size=group_size,
+            model=model,
+            timeout=timeout,
+            dry_run=dry_run,
+        )
+
+    # Dotted aliases (top-level): legacy check → setup
+    @cli.command("rl_demo.check")
+    def rl_check_alias():
+        _run_demo_command(demo_commands.setup)
+
+    @cli.command("rl_demo.setup")
+    def rl_setup_alias():
+        _run_demo_command(demo_commands.setup)
+
+    # (prepare alias removed)
+
+    @cli.command("rl_demo.deploy")
+    @click.option("--local", is_flag=True, help="Run local FastAPI instead of Modal deploy")
+    @click.option(
+        "--app",
+        type=click.Path(),
+        default=None,
+        help="Path to Modal app.py for uv run modal deploy",
+    )
+    @click.option("--name", type=str, default="synth-math-demo", help="Modal app name")
+    @click.option(
+        "--script",
+        type=click.Path(),
+        default=None,
+        help="Path to deploy_task_app.sh (optional legacy)",
+    )
+    def rl_deploy_alias(local: bool, app: str | None, name: str, script: str | None):
+        _run_demo_command(
+            demo_commands.deploy,
+            local=local,
+            app=app,
+            name=name,
+            script=script,
+        )
+
+    @cli.command("rl_demo.configure")
+    def rl_configure_alias():
+        _run_demo_command(demo_commands.run)
+
+    @cli.command("rl_demo.init")
+    @click.option("--template", type=str, default=None, help="Template id to instantiate")
+    @click.option("--dest", type=click.Path(), default=None, help="Destination directory for files")
+    @click.option("--force", is_flag=True, help="Overwrite existing files in destination")
+    def rl_init_alias(template: str | None, dest: str | None, force: bool):
+        _run_demo_command(
+            demo_commands.init,
+            template=template,
+            dest=dest,
+            force=force,
+        )
+
+    @cli.command("rl_demo.run")
+    @click.option(
+        "--config", type=click.Path(), default=None, help="Path to TOML config (skip prompt)"
+    )
+    @click.option("--batch-size", type=int, default=None)
+    @click.option("--group-size", type=int, default=None)
+    @click.option("--model", type=str, default=None)
+    @click.option("--timeout", type=int, default=600)
+    @click.option("--dry-run", is_flag=True, help="Print request body and exit")
+    def rl_run_alias(
+        config: str | None,
+        batch_size: int | None,
+        group_size: int | None,
+        model: str | None,
+        timeout: int,
+        dry_run: bool,
+    ):
+        _run_demo_command(
+            demo_commands.run,
+            config=config,
+            batch_size=batch_size,
+            group_size=group_size,
+            model=model,
+            timeout=timeout,
+            dry_run=dry_run,
+        )
+
+    # Top-level convenience alias: `synth-ai deploy`
+    @cli.command("demo-deploy")
+    @click.option("--local", is_flag=True, help="Run local FastAPI instead of Modal deploy")
+    @click.option(
+        "--app",
+        type=click.Path(),
+        default=None,
+        help="Path to Modal app.py for uv run modal deploy",
+    )
+    @click.option("--name", type=str, default="synth-math-demo", help="Modal app name")
+    @click.option(
+        "--script",
+        type=click.Path(),
+        default=None,
+        help="Path to deploy_task_app.sh (optional legacy)",
+    )
+    def deploy_demo(local: bool, app: str | None, name: str, script: str | None):
+        _run_demo_command(
+            demo_commands.deploy,
+            local=local,
+            app=app,
+            name=name,
+            script=script,
+        )
+
+    @cli.command("run")
+    @click.option(
+        "--config", type=click.Path(), default=None, help="Path to TOML config (skip prompt)"
+    )
+    @click.option("--batch-size", type=int, default=None)
+    @click.option("--group-size", type=int, default=None)
+    @click.option("--model", type=str, default=None)
+    @click.option("--timeout", type=int, default=600)
+    @click.option("--dry-run", is_flag=True, help="Print request body and exit")
+    def run_top(
+        config: str | None,
+        batch_size: int | None,
+        group_size: int | None,
+        model: str | None,
+        timeout: int,
+        dry_run: bool,
+    ):
+        _run_demo_command(
+            demo_commands.run,
+            config=config,
+            batch_size=batch_size,
+            group_size=group_size,
+            model=model,
+            timeout=timeout,
+            dry_run=dry_run,
+        )

synth_ai/cli/deploy.py

@@ -0,0 +1,216 @@
+import asyncio
+from pathlib import Path
+from typing import Any, Literal, TypeAlias, cast, get_args
+
+import click
+
+from synth_ai.cli.lib.apps.task_app import find_task_apps_in_cwd
+from synth_ai.cli.lib.env import get_synth_and_env_keys
+from synth_ai.core.cfgs import CFDeployCfg, LocalDeployCfg, ModalDeployCfg
+from synth_ai.core.integrations.cloudflare import deploy_app_tunnel
+from synth_ai.core.integrations.modal import deploy_app_modal
+from synth_ai.core.paths import print_paths_formatted
+from synth_ai.core.telemetry import flush_logger, log_error, log_info
+from synth_ai.core.uvicorn import deploy_app_uvicorn
+
+RuntimeType: TypeAlias = Literal[
+    "local",
+    "modal",
+    "tunnel"
+]
+
+
+@click.command()
+# --- Arguments ---
+@click.argument(
+    "runtime",
+    type=click.Choice(get_args(RuntimeType)),
+    required=False,
+    default="local"
+)
+@click.argument(
+    "task_app_path",
+    type=click.Path(path_type=Path),
+    required=False
+)
+# --- Universal option(s) ---
+@click.option(
+    "--env", 
+    type=click.Path(
+        exists=True,
+        dir_okay=False,
+        file_okay=True,
+        path_type=Path
+    ),
+    required=True,
+    help="Path to .env file to use (required)"
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    default=False,
+    help="Skip task app validation"
+)
+# --- Local runtime-only options ---
+@click.option(
+    "--trace/--no-trace",
+    default=True,
+    help="Enable or disable trace output"
+)
+@click.option(
+    "--host",
+    default="127.0.0.1", 
+    help="Host to bind to"
+)
+@click.option(
+    "--port",
+    default=8000,
+    type=int,
+    help="Port to bind to"
+)
+@click.option(
+    "--wait/--no-wait",
+    "wait",
+    default=False,
+    help="Wait for deployment to complete (blocking mode). Default is non-blocking (background)"
+)
+# --- Tunnel runtime-only options ---
+@click.option(
+    "--tunnel-mode",
+    type=click.Choice(["quick", "managed"], case_sensitive=False),
+    default="quick",
+    help="Tunnel mode: quick (ephemeral) or managed (stable)"
+)
+@click.option(
+    "--tunnel-subdomain",
+    type=str,
+    default=None,
+    help="Custom subdomain for managed tunnel (e.g., 'my-company')"
+)
+@click.option(
+    "--keep-alive/--background",
+    default=False,
+    help="(Deprecated: use --wait) Keep tunnel alive (blocking mode). Default is background (non-blocking)"
+)
+@click.option(
+    "--wait/--no-wait",
+    "wait",
+    default=False,
+    help="Wait for deployment to complete (blocking mode). Default is non-blocking (background)"
+)
+# --- Modal runtime-only options ---
+@click.option(
+    "--modal-app",
+    help="Enter the path to your Modal app"
+)
+@click.option(
+    "--name",
+    default=None,
+    help="Override Modal app name"
+)
+@click.option(
+    "--modal-mode",
+    default="deploy",
+    help="Mode: deploy or serve"
+)
+@click.option(
+    "--modal-cli",
+    type=click.Path(
+        dir_okay=False,
+        file_okay=True,
+        exists=True,
+        path_type=Path
+    ),
+    default=None,
+    help="Path to Modal CLI",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Print Modal command without executing"
+)
+def deploy_cmd(
+    runtime: RuntimeType,
+    task_app_path: Path | None,
+    **kwargs
+) -> None:
+    ctx: dict[str, Any] = {
+        "runtime": runtime,
+        "task_app_path": str(task_app_path),
+        **kwargs
+    }
+    try:
+        log_info("deploy command hit", ctx=ctx)
+
+        if not task_app_path:
+            available_task_apps = find_task_apps_in_cwd()
+            if len(available_task_apps) == 1:
+                task_app_path, _ = available_task_apps[0]
+                print("Automatically selected task app at", task_app_path)
+            else:
+                if len(available_task_apps) == 0:
+                    print("No task apps found in cwd.")
+                    print("Validate your task app with: synth-ai task-app check [TASK_APP_PATH]")
+                else:
+                    print("Multiple task apps found. Please specify which one to use:")
+                    print_paths_formatted(available_task_apps)
+                print("Usage: synth-ai deploy [RUNTIME] [TASK_APP_PATH]")
+                return None
+        
+        env_file = kwargs.get("env")
+        synth_api_key, env_api_key = get_synth_and_env_keys(env_file)
+
+        # if not kwargs.get("force", False):
+        #     validate_task_app(task_app_path)
+
+        match runtime:
+            case "local":
+                log_info("starting local deploy", ctx=ctx)
+                deploy_app_uvicorn(
+                    LocalDeployCfg.create(  # type: ignore[call-arg, arg-type]
+                    task_app_path=task_app_path,
+                        env_api_key=env_api_key,  # type: ignore[arg-type]
+                    trace = bool(kwargs.get("trace", True)),
+                    host = str(kwargs.get("host", "127.0.0.1")),
+                    port = int(kwargs.get("port", 8000))
+                    )
+                )
+            case "modal":
+                log_info("starting modal deploy", ctx=ctx)
+                deploy_app_modal(ModalDeployCfg.create_from_kwargs(
+                    task_app_path=task_app_path,
+                    synth_api_key=synth_api_key,
+                    env_api_key=env_api_key,
+                    **kwargs
+                ))
+            case "tunnel":
+                log_info("starting tunnel deploy", ctx=ctx)
+                tunnel_mode = kwargs.get("tunnel_mode", "managed")
+                if tunnel_mode == "managed" and not synth_api_key:
+                    raise RuntimeError(
+                        "SYNTH_API_KEY required for managed tunnel mode. "
+                        "Either run synth-ai setup to load automatically or manually load to process environment or pass .env via synth-ai deploy --env .env"
+                    )
+                asyncio.run(deploy_app_tunnel(
+                    CFDeployCfg.create(
+                        task_app_path=task_app_path,
+                        env_api_key=env_api_key,
+                        host=str(kwargs.get("host", "127.0.0.1")),
+                        port=int(kwargs.get("port", 8000)),
+                        mode=cast(Literal["quick", "managed"], tunnel_mode),
+                        subdomain=kwargs.get("tunnel_subdomain"),
+                        trace=bool(kwargs.get("trace", True))
+                    ),
+                    env_file,
+                    keep_alive=bool(kwargs.get("keep_alive", False))
+                ))
+        
+    except Exception as exc:
+        ctx["error"] = type(exc).__name__
+        ctx["error_message"] = str(exc)
+        log_error("deploy command failed", ctx=ctx)
+        click.echo(f"{exc}", err=True)
+    
+    finally:
+        log_info("deploy command completed", ctx=ctx)
+        flush_logger(0.5)

synth_ai/cli/infra/__init__.py

@@ -0,0 +1,14 @@
+"""Infrastructure CLI commands (balance, setup, status, turso, mcp, modal_app).
+
+Commands for managing Synth infrastructure and configuration.
+"""
+
+from synth_ai.cli.infra.mcp import mcp_cmd
+from synth_ai.cli.infra.modal_app import modal_app_cmd
+
+__all__ = [
+    "mcp_cmd",
+    "modal_app_cmd",
+]
+
+