synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/cli/commands/baseline/list.py

@@ -0,0 +1,94 @@
+"""List command for baseline discovery."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from synth_ai.sdk.baseline.config import BaselineConfig
+from synth_ai.sdk.baseline.discovery import (
+    BaselineChoice,
+    discover_baseline_files,
+    load_baseline_config_from_file,
+)
+
+
+@click.command("list")
+@click.option(
+    "--tag",
+    multiple=True,
+    help="Filter baselines by tag (can be specified multiple times)",
+)
+@click.option(
+    "--metadata",
+    type=str,
+    help="Filter by metadata key-value pair (format: key=value)",
+)
+@click.option(
+    "--verbose",
+    is_flag=True,
+    help="Show detailed information about each baseline",
+)
+def list_command(tag: tuple[str, ...], metadata: Optional[str], verbose: bool) -> None:
+    """List all available baseline files."""
+    search_roots = [Path.cwd()]
+    choices = discover_baseline_files(search_roots)
+    
+    if not choices:
+        click.echo("No baseline files found.", err=True)
+        click.echo("Create baseline files in examples/baseline/ or */*_baseline.py")
+        return
+    
+    # Load configs for filtering
+    configs: list[tuple[BaselineChoice, BaselineConfig]] = []
+    for choice in choices:
+        try:
+            config = load_baseline_config_from_file(choice.baseline_id, choice.path)
+            configs.append((choice, config))
+        except Exception as e:
+            if verbose:
+                click.echo(f"Warning: Could not load {choice.baseline_id}: {e}", err=True)
+            continue
+    
+    # Apply filters
+    filtered_configs = configs
+    
+    if tag:
+        tag_set = {t.lower() for t in tag}
+        filtered_configs = [
+            (c, config) for c, config in filtered_configs
+            if any(config.matches_tag(t) for t in tag_set)
+        ]
+    
+    if metadata:
+        if "=" not in metadata:
+            raise click.ClickException("--metadata must be in format key=value")
+        key, value = metadata.split("=", 1)
+        filtered_configs = [
+            (c, config) for c, config in filtered_configs
+            if config.matches_metadata(key.strip(), value.strip())
+        ]
+    
+    if not filtered_configs:
+        click.echo("No baselines match the specified filters.")
+        return
+    
+    # Display results
+    click.echo(f"Found {len(filtered_configs)} baseline(s):\n")
+    
+    for choice, config in filtered_configs:
+        click.echo(f"  {config.baseline_id}")
+        click.echo(f"    Name: {config.name}")
+        if config.description:
+            click.echo(f"    Description: {config.description}")
+        if config.tags:
+            click.echo(f"    Tags: {', '.join(config.tags)}")
+        click.echo(f"    Splits: {', '.join(config.splits.keys())}")
+        if verbose:
+            click.echo(f"    Path: {choice.path}")
+            if config.metadata:
+                click.echo(f"    Metadata: {config.metadata}")
+        click.echo()
+

synth_ai/cli/commands/demo/__init__.py

@@ -0,0 +1,3 @@
+from .core import command, register
+
+__all__ = ["command", "register"]

synth_ai/cli/commands/demo/core.py

@@ -0,0 +1,153 @@
+import importlib
+import os
+import subprocess
+from pathlib import Path
+from typing import Any, cast
+
+import click
+from click.exceptions import Exit
+
+__all__ = ["command", "register"]
+
+_demo_cli = cast(Any, importlib.import_module("synth_ai.cli.demo_apps.core.cli"))
+
+
+def _find_demo_scripts(root: Path) -> list[Path]:
+    if not root.exists():
+        return []
+    return sorted(path for path in root.rglob("run_demo.sh") if path.is_file())
+
+
+def _run_demo_command(func: Any, *args: Any, **kwargs: Any) -> None:
+    """Invoke a demo command and map non-zero exits to Click exits."""
+
+    try:
+        result = func(*args, **kwargs)
+    except SystemExit as exc:  # pragma: no cover - defensive shim
+        raise Exit(exc.code if isinstance(exc.code, int) else 1) from exc
+
+    if result is None:
+        return
+
+    try:
+        code = int(result)
+    except (TypeError, ValueError):
+        return
+    if code != 0:
+        raise Exit(code)
+
+
+@click.group(
+    "demo",
+    invoke_without_command=True,
+    help="Demo helpers for the math RL pipeline.",
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Overwrite existing files in the current directory when initializing demo assets.",
+)
+@click.option("--list", "list_only", is_flag=True, help="List available legacy demos and exit.")
+@click.option("-f", "filter_term", default="", help="Filter legacy demos by substring.")
+@click.pass_context
+def command(ctx: click.Context, force: bool, list_only: bool, filter_term: str) -> None:
+    """Default command: initialize RL demo files into ./synth_demo/ (alias of `demo init`)."""
+    if ctx.invoked_subcommand is not None:
+        return
+
+    if list_only:
+        repo_root = Path(os.getcwd())
+        examples_dir = repo_root / "examples"
+        demos = _find_demo_scripts(examples_dir)
+        if filter_term:
+            term = filter_term.lower()
+            demos = [path for path in demos if term in str(path).lower()]
+
+        if not demos:
+            click.echo("No run_demo.sh scripts found under examples/.")
+            return
+
+        click.echo("Available demos:")
+        for idx, path in enumerate(demos, start=1):
+            click.echo(f" {idx}. {path.relative_to(repo_root)}")
+        click.echo("")
+
+        def _validate_choice(val: str) -> int:
+            try:
+                selection = int(val)
+            except Exception as err:  # pragma: no cover - Click handles prompt errors
+                raise click.BadParameter("Enter a number from the list") from err
+            if selection < 1 or selection > len(demos):
+                raise click.BadParameter(f"Choose a number between 1 and {len(demos)}")
+            return selection
+
+        choice = click.prompt("Select a demo to run", value_proc=_validate_choice)
+        script = demos[choice - 1]
+
+        click.echo("")
+        click.echo(f"🚀 Running {script.relative_to(repo_root)}\n")
+
+        try:
+            subprocess.run(["bash", str(script)], check=True)
+        except subprocess.CalledProcessError as exc:
+            click.echo(f"❌ Demo exited with non-zero status: {exc.returncode}")
+        except KeyboardInterrupt:
+            click.echo("\n🛑 Demo interrupted by user")
+        return
+
+    _run_demo_command(_demo_cli.init, force=force)
+
+
+@command.command("deploy")
+@click.option("--local", is_flag=True, help="Run the local FastAPI app instead of deploying to Modal.")
+@click.option("--app", type=click.Path(), default=None, help="Path to Modal app.py for `uv run modal deploy`.")
+@click.option("--name", type=str, default="synth-math-demo", help="Modal app name.")
+@click.option(
+    "--script",
+    type=click.Path(),
+    default=None,
+    help="Path to deploy_task_app.sh (optional legacy helper).",
+)
+def demo_deploy(local: bool, app: str | None, name: str, script: str | None) -> None:
+    _run_demo_command(
+        _demo_cli.deploy,
+        local=local,
+        app=app,
+        name=name,
+        script=script,
+    )
+
+
+@command.command("configure")
+def demo_configure() -> None:
+    _run_demo_command(_demo_cli.run)
+
+
+@command.command("setup")
+def demo_setup() -> None:
+    _run_demo_command(_demo_cli.setup)
+
+
+@command.command("run")
+@click.option("--batch-size", type=int, default=None)
+@click.option("--group-size", type=int, default=None)
+@click.option("--model", type=str, default=None)
+@click.option("--timeout", type=int, default=600)
+def demo_run(
+    batch_size: int | None,
+    group_size: int | None,
+    model: str | None,
+    timeout: int,
+) -> None:
+    _run_demo_command(
+        _demo_cli.run,
+        batch_size=batch_size,
+        group_size=group_size,
+        model=model,
+        timeout=timeout,
+    )
+
+
+def register(cli: click.Group) -> None:
+    """Attach the demo command group and related aliases to the CLI."""
+    cli.add_command(command)

synth_ai/cli/commands/eval/__init__.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from .errors import EvalCliError
+from .validation import validate_eval_options
+
+__all__ = [
+    "command",
+    "get_command",
+    "EvalCliError",
+    "validate_eval_options",
+]
+
+
+def __getattr__(name: str):
+    if name in {"command", "get_command"}:
+        from .core import command, get_command
+
+        return command if name == "command" else get_command
+    raise AttributeError(name)