synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/cli/commands/filter/validation.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import math
+from collections.abc import MutableMapping
+from typing import Any
+
+__all__ = ["validate_filter_options"]
+
+
+def validate_filter_options(options: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+    """Validate parameters passed to the filter CLI command."""
+    # Coerce optional collections to the expected container types and strip blanks
+    result: dict[str, Any] = dict(options)
+
+    def _coerce_list(key: str) -> None:
+        value = result.get(key)
+        if value is None:
+            result[key] = []
+        elif isinstance(value, list | tuple | set):
+            result[key] = [str(item).strip() for item in value if str(item).strip()]
+        else:
+            result[key] = [str(value).strip()] if str(value).strip() else []
+
+    def _coerce_dict(key: str) -> None:
+        value = result.get(key)
+        if value is None:
+            result[key] = {}
+        elif isinstance(value, MutableMapping):
+            normalized: dict[str, float] = {}
+            for k, v in value.items():
+                if k is None:
+                    continue
+                try:
+                    number = float(v)
+                    if math.isnan(number) or math.isinf(number):
+                        continue
+                    normalized[str(k).strip()] = number
+                except Exception:
+                    continue
+            result[key] = normalized
+        else:
+            result[key] = {}
+
+    _coerce_list("splits")
+    _coerce_list("task_ids")
+    _coerce_list("models")
+    _coerce_dict("min_judge_scores")
+    _coerce_dict("max_judge_scores")
+
+    for duration_key in ("min_official_score", "max_official_score"):
+        value = result.get(duration_key)
+        if value is None or value == "":
+            result[duration_key] = None
+        else:
+            try:
+                result[duration_key] = float(value)
+            except Exception:
+                result[duration_key] = None
+
+    for int_key in ("limit", "offset", "shuffle_seed"):
+        value = result.get(int_key)
+        if value is None or value == "":
+            result[int_key] = None
+        else:
+            try:
+                result[int_key] = int(value)
+            except Exception:
+                result[int_key] = None
+
+    shuffle_value = result.get("shuffle")
+    if isinstance(shuffle_value, str):
+        result["shuffle"] = shuffle_value.strip().lower() in {"1", "true", "yes"}
+    else:
+        result["shuffle"] = bool(shuffle_value)
+
+    # Preserve extra keys (e.g., min_created_at) as-is for downstream handling
+    return result

synth_ai/cli/commands/help/__init__.py

@@ -0,0 +1,177 @@
+"""Help content for CLI commands."""
+
+DEPLOY_HELP = """
+Deploy a Synth AI task app locally or to Modal.
+
+OVERVIEW
+--------
+The deploy command supports two runtimes:
+  • modal: Deploy to Modal's cloud platform (default)
+  • uvicorn: Run locally with FastAPI/Uvicorn
+
+BASIC USAGE
+-----------
+  # Deploy to Modal (production)
+  uvx synth-ai deploy
+
+  # Deploy specific task app
+  uvx synth-ai deploy my-math-app
+
+  # Run locally for development
+  uvx synth-ai deploy --runtime=uvicorn --port 8001
+
+MODAL DEPLOYMENT
+----------------
+Modal deployment requires:
+  1. Modal authentication (run: modal token new)
+  2. ENVIRONMENT_API_KEY (run: uvx synth-ai setup)
+
+Options:
+  --modal-mode [deploy|serve]  Use 'deploy' for production (default), 
+                                'serve' for ephemeral development
+  --name TEXT                   Override Modal app name
+  --dry-run                     Preview the deploy command without executing
+  --env-file PATH               Env file(s) to load (can be repeated)
+
+Examples:
+  # Standard production deployment
+  uvx synth-ai deploy --runtime=modal
+
+  # Deploy with custom name
+  uvx synth-ai deploy --runtime=modal --name my-task-app-v2
+
+  # Preview deployment command
+  uvx synth-ai deploy --dry-run
+
+  # Deploy with custom env file
+  uvx synth-ai deploy --env-file .env.production
+
+LOCAL DEVELOPMENT
+-----------------
+Run locally with auto-reload and tracing:
+
+  uvx synth-ai deploy --runtime=uvicorn --port 8001 --reload
+
+Options:
+  --host TEXT                   Bind address (default: 0.0.0.0)
+  --port INTEGER                Port number (prompted if not provided)
+  --reload/--no-reload          Enable auto-reload on code changes
+  --force/--no-force            Kill existing process on port
+  --trace PATH                  Enable tracing to directory (default: traces/v3)
+  --trace-db PATH               SQLite DB for traces
+
+Examples:
+  # Basic local server
+  uvx synth-ai deploy --runtime=uvicorn
+
+  # Development with auto-reload
+  uvx synth-ai deploy --runtime=uvicorn --reload --port 8001
+
+  # With custom trace directory
+  uvx synth-ai deploy --runtime=uvicorn --trace ./my-traces
+
+TROUBLESHOOTING
+---------------
+Common issues:
+
+1. "ENVIRONMENT_API_KEY is required"
+   → Run: uvx synth-ai setup
+
+2. "Modal CLI not found"
+   → Install: pip install modal
+   → Authenticate: modal token new
+
+3. "Task app not found"
+   → Check app_id matches your task_app.py configuration
+   → Run: uvx synth-ai task-app list (if available)
+
+4. "Port already in use" (uvicorn)
+   → Use --force to kill existing process
+   → Or specify different --port
+
+5. "No env file discovered"
+   → Create .env file with required keys
+   → Or pass --env-file explicitly
+
+ENVIRONMENT VARIABLES
+---------------------
+  SYNTH_API_KEY              Your Synth platform API key
+  ENVIRONMENT_API_KEY        Task environment authentication
+  TASK_APP_BASE_URL          Base URL for deployed task app
+  DEMO_DIR                   Demo directory path
+  SYNTH_DEMO_DIR             Alternative demo directory
+
+For more information: https://docs.usesynth.ai/deploy
+"""
+
+SETUP_HELP = """
+Configure Synth AI credentials and environment.
+
+OVERVIEW
+--------
+The setup command initializes your Synth AI environment by:
+  1. Authenticating with the Synth platform via browser
+  2. Saving your API keys to ~/.synth/config
+  3. Verifying Modal authentication (for deployments)
+  4. Testing connectivity to backend services
+
+USAGE
+-----
+  uvx synth-ai setup
+
+The command will:
+  • Open your browser for authentication (or prompt for manual entry)
+  • Save SYNTH_API_KEY and ENVIRONMENT_API_KEY
+  • Verify Modal is authenticated
+  • Test backend connectivity
+
+WHAT YOU'LL NEED
+----------------
+  • Web browser for authentication
+  • Modal account (for deployments): https://modal.com
+  • Active internet connection
+
+TROUBLESHOOTING
+---------------
+1. "Failed to fetch keys from frontend"
+   → You'll be prompted to enter keys manually
+   → Get keys from: https://www.usesynth.ai/dashboard/settings
+
+2. "Modal authentication status: not authenticated"
+   → Run: modal token new
+   → Then re-run: uvx synth-ai setup
+
+3. Browser doesn't open
+   → Check your default browser settings
+   → Or enter keys manually when prompted
+
+WHERE ARE KEYS STORED?
+----------------------
+Keys are saved to: ~/.synth/config
+
+This file is read automatically by all Synth AI commands.
+You can also use .env files in your project directory.
+
+NEXT STEPS
+----------
+After setup completes:
+  1. Deploy your task app: uvx synth-ai deploy
+  2. Start local development: uvx synth-ai deploy --runtime=uvicorn
+  3. Run training: uvx synth-ai train
+
+For more information: https://docs.usesynth.ai/setup
+"""
+
+COMMAND_HELP = {
+    "deploy": DEPLOY_HELP,
+    "setup": SETUP_HELP,
+}
+
+
+def get_command_help(command: str) -> str | None:
+    """Get detailed help text for a command."""
+    return COMMAND_HELP.get(command)
+
+
+__all__ = ["DEPLOY_HELP", "SETUP_HELP", "COMMAND_HELP", "get_command_help"]
+

synth_ai/cli/commands/help/core.py

@@ -0,0 +1,73 @@
+"""Help command implementation."""
+
+import click
+from click.exceptions import Exit
+
+from . import COMMAND_HELP, get_command_help
+
+
+@click.command("help")
+@click.argument("command_name", type=str, required=False)
+def help_command(command_name: str | None) -> None:
+    """Display detailed help for Synth AI commands.
+    
+    USAGE
+    -----
+      uvx synth-ai help [COMMAND]
+    
+    EXAMPLES
+    --------
+      # List available help topics
+      uvx synth-ai help
+      
+      # Get detailed help for deploy
+      uvx synth-ai help deploy
+      
+      # Get detailed help for setup
+      uvx synth-ai help setup
+    """
+    if not command_name:
+        # Show list of available help topics
+        click.echo("Synth AI - Detailed Help")
+        click.echo("=" * 50)
+        click.echo("\nAvailable help topics:")
+        click.echo("")
+        
+        for cmd in sorted(COMMAND_HELP.keys()):
+            click.echo(f"  • {cmd}")
+        
+        click.echo("\nUsage:")
+        click.echo("  uvx synth-ai help [COMMAND]")
+        click.echo("")
+        click.echo("Examples:")
+        click.echo("  uvx synth-ai help deploy")
+        click.echo("  uvx synth-ai help setup")
+        click.echo("")
+        click.echo("You can also use standard --help flags:")
+        click.echo("  uvx synth-ai deploy --help")
+        click.echo("  uvx synth-ai setup --help")
+        return
+    
+    # Show detailed help for specific command
+    help_text = get_command_help(command_name)
+    if not help_text:
+        click.echo(f"No detailed help available for '{command_name}'", err=True)
+        click.echo(f"\nTry: uvx synth-ai {command_name} --help", err=True)
+        click.echo("Or: uvx synth-ai help (to see available topics)", err=True)
+        raise Exit(1)
+    
+    click.echo(help_text)
+
+
+def get_command() -> click.Command:
+    """Get the help command for registration."""
+    return help_command
+
+
+def register(group: click.Group) -> None:
+    """Register the help command with a Click group."""
+    group.add_command(help_command)
+
+
+__all__ = ["help_command", "get_command", "register"]
+

synth_ai/cli/commands/status/__init__.py

@@ -0,0 +1,64 @@
+"""Status and listing commands for the Synth CLI."""
+
+from __future__ import annotations
+
+import click
+
+from .config import resolve_backend_config
+from .subcommands.files import files_group
+from .subcommands.jobs import jobs_group
+from .subcommands.models import models_group
+from .subcommands.runs import runs_group
+from .subcommands.summary import summary_command
+
+
+def _attach_group(cli: click.Group, group: click.Group, name: str) -> None:
+    """Attach the provided Click group to the CLI if not already present."""
+    if name in cli.commands:
+        return
+    cli.add_command(group, name=name)
+
+
+def register(cli: click.Group) -> None:
+    """Register all status command groups on the provided CLI root."""
+
+    @click.group(help="Inspect training jobs, models, files, and job runs.")
+    @click.option(
+        "--base-url",
+        envvar="SYNTH_STATUS_BASE_URL",
+        default=None,
+        help="Synth backend base URL (defaults to environment configuration).",
+    )
+    @click.option(
+        "--api-key",
+        envvar="SYNTH_STATUS_API_KEY",
+        default=None,
+        help="API key for authenticated requests (falls back to Synth defaults).",
+    )
+    @click.option(
+        "--timeout",
+        default=30.0,
+        show_default=True,
+        type=float,
+        help="HTTP request timeout in seconds.",
+    )
+    @click.pass_context
+    def status(ctx: click.Context, base_url: str | None, api_key: str | None, timeout: float) -> None:
+        """Populate shared backend configuration for subcommands."""
+        cfg = resolve_backend_config(base_url=base_url, api_key=api_key, timeout=timeout)
+        ctx.ensure_object(dict)
+        ctx.obj["status_backend_config"] = cfg
+
+    status.add_command(jobs_group, name="jobs")
+    status.add_command(models_group, name="models")
+    status.add_command(files_group, name="files")
+    status.add_command(runs_group, name="runs")
+    status.add_command(summary_command, name="summary")
+
+    cli.add_command(status, name="status")
+    _attach_group(cli, jobs_group, "jobs")
+    _attach_group(cli, models_group, "models")
+    _attach_group(cli, files_group, "files")
+    _attach_group(cli, runs_group, "runs")
+    if "status-summary" not in cli.commands:
+        cli.add_command(summary_command, name="status-summary")

synth_ai/cli/commands/status/client.py

@@ -0,0 +1,192 @@
+"""Async HTTP client for Synth status and listing endpoints."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from .config import BackendConfig
+from .errors import StatusAPIError
+
+
+class StatusAPIClient:
+    """Thin wrapper around httpx.AsyncClient with convenience methods."""
+
+    def __init__(self, config: BackendConfig) -> None:
+        self._config = config
+        timeout = httpx.Timeout(config.timeout)
+        self._client = httpx.AsyncClient(
+            base_url=config.base_url,
+            headers=config.headers,
+            timeout=timeout,
+        )
+
+    async def __aenter__(self) -> StatusAPIClient:
+        await self._client.__aenter__()
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        await self._client.__aexit__(*args)
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    # Jobs -----------------------------------------------------------------
+
+    async def list_jobs(
+        self,
+        *,
+        status: str | None = None,
+        job_type: str | None = None,
+        created_after: str | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        params: dict[str, Any] = {}
+        if status:
+            params["status"] = status
+        if job_type:
+            params["type"] = job_type
+        if created_after:
+            params["created_after"] = created_after
+        if limit:
+            params["limit"] = limit
+        resp = await self._client.get("/learning/jobs", params=params)
+        return self._json_list(resp, key="jobs")
+
+    async def get_job(self, job_id: str) -> dict[str, Any]:
+        resp = await self._client.get(f"/learning/jobs/{job_id}")
+        return self._json(resp)
+
+    async def get_job_status(self, job_id: str) -> dict[str, Any]:
+        resp = await self._client.get(f"/learning/jobs/{job_id}/status")
+        return self._json(resp)
+
+    async def cancel_job(self, job_id: str) -> dict[str, Any]:
+        resp = await self._client.post(f"/learning/jobs/{job_id}/cancel")
+        return self._json(resp)
+
+    async def get_job_config(self, job_id: str) -> dict[str, Any]:
+        resp = await self._client.get(f"/learning/jobs/{job_id}/config")
+        return self._json(resp)
+
+    async def get_job_metrics(self, job_id: str) -> dict[str, Any]:
+        resp = await self._client.get(f"/learning/jobs/{job_id}/metrics")
+        return self._json(resp)
+
+    async def get_job_timeline(self, job_id: str) -> list[dict[str, Any]]:
+        resp = await self._client.get(f"/learning/jobs/{job_id}/timeline")
+        return self._json_list(resp, key="timeline")
+
+    async def list_job_runs(self, job_id: str) -> list[dict[str, Any]]:
+        resp = await self._client.get(f"/jobs/{job_id}/runs")
+        return self._json_list(resp, key="runs")
+
+    async def get_job_events(
+        self,
+        job_id: str,
+        *,
+        since: str | None = None,
+        limit: int | None = None,
+        after: str | None = None,
+        run_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        params: dict[str, Any] = {}
+        if since:
+            params["since"] = since
+        if limit:
+            params["limit"] = limit
+        if after:
+            params["after"] = after
+        if run_id:
+            params["run"] = run_id
+        resp = await self._client.get(f"/learning/jobs/{job_id}/events", params=params)
+        return self._json_list(resp, key="events")
+
+    # Files ----------------------------------------------------------------
+
+    async def list_files(
+        self,
+        *,
+        purpose: str | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        params: dict[str, Any] = {}
+        if purpose:
+            params["purpose"] = purpose
+        if limit:
+            params["limit"] = limit
+        resp = await self._client.get("/files", params=params)
+        data = self._json(resp)
+        if isinstance(data, dict):
+            for key in ("files", "data", "items"):
+                if isinstance(data.get(key), list):
+                    return list(data[key])
+        if isinstance(data, list):
+            return list(data)
+        return []
+
+    async def get_file(self, file_id: str) -> dict[str, Any]:
+        resp = await self._client.get(f"/files/{file_id}")
+        return self._json(resp)
+
+    # Models ---------------------------------------------------------------
+
+    async def list_models(
+        self,
+        *,
+        limit: int | None = None,
+        model_type: str | None = None,
+    ) -> list[dict[str, Any]]:
+        params: dict[str, Any] = {}
+        if limit:
+            params["limit"] = limit
+        endpoint = "/learning/models/rl" if model_type == "rl" else "/learning/models"
+        resp = await self._client.get(endpoint, params=params)
+        return self._json_list(resp, key="models")
+
+    async def get_model(self, model_id: str) -> dict[str, Any]:
+        resp = await self._client.get(f"/learning/models/{model_id}")
+        return self._json(resp)
+
+    # Helpers --------------------------------------------------------------
+
+    def _json(self, response: httpx.Response) -> dict[str, Any]:
+        try:
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            detail = self._extract_detail(exc.response)
+            raise StatusAPIError(detail, exc.response.status_code if exc.response else None) from exc
+        try:
+            data = response.json()
+        except ValueError as exc:
+            raise StatusAPIError("Backend response was not valid JSON") from exc
+        if isinstance(data, dict):
+            return data
+        return {"data": data}
+
+    def _json_list(self, response: httpx.Response, *, key: str | None = None) -> list[dict[str, Any]]:
+        payload = self._json(response)
+        if key and isinstance(payload.get(key), list):
+            return list(payload[key])
+        if isinstance(payload.get("data"), list):
+            return list(payload["data"])
+        if isinstance(payload.get("results"), list):
+            return list(payload["results"])
+        if isinstance(payload, list):
+            return list(payload)
+        return []
+
+    @staticmethod
+    def _extract_detail(response: httpx.Response | None) -> str:
+        if response is None:
+            return "Backend request failed"
+        try:
+            data = response.json()
+            if isinstance(data, dict):
+                for key in ("detail", "message", "error"):
+                    if data.get(key):
+                        return str(data[key])
+            return response.text
+        except ValueError:
+            return response.text

synth_ai/cli/commands/status/config.py

@@ -0,0 +1,92 @@
+"""Configuration utilities for the status command suite.
+
+Provides helpers to resolve backend URLs, API keys, and request timeouts
+from CLI options and environment variables.
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+
+DEFAULT_TIMEOUT = 30.0
+
+
+def _load_backend_helpers() -> tuple[str, Callable[[], tuple[str, str]] | None]:
+    """Attempt to load shared backend helpers from synth_ai.config.base_url."""
+    try:
+        module = importlib.import_module("synth_ai.config.base_url")
+    except Exception:
+        return "https://agent-learning.onrender.com", None
+
+    default = getattr(module, "PROD_BASE_URL_DEFAULT", "https://agent-learning.onrender.com")
+    getter = getattr(module, "get_backend_from_env", None)
+    return str(default), getter if callable(getter) else None
+
+
+PROD_BASE_URL_DEFAULT, _GET_BACKEND_FROM_ENV = _load_backend_helpers()
+
+
+def _normalize_base_url(raw: str) -> str:
+    """Ensure the configured base URL includes the /api/v1 prefix."""
+    base = raw.rstrip("/") if raw else ""
+    if not base:
+        return raw
+    if base.endswith("/api") or base.endswith("/api/v1") or "/api/" in base:
+        return base
+    return f"{base}/api/v1"
+
+
+def _default_base_url() -> str:
+    """Compute the default backend base URL using env vars or helper module."""
+    for var in ("SYNTH_BACKEND_BASE_URL", "BACKEND_BASE_URL", "SYNTH_BASE_URL"):
+        val = os.getenv(var)
+        if val:
+            return _normalize_base_url(val)
+    if _GET_BACKEND_FROM_ENV:
+        try:
+            base, _ = _GET_BACKEND_FROM_ENV()
+            return _normalize_base_url(base)
+        except Exception:
+            pass
+    return _normalize_base_url(PROD_BASE_URL_DEFAULT)
+
+
+def _resolve_api_key(cli_key: str | None) -> tuple[str | None, str | None]:
+    """Resolve the API key from CLI input or known environment variables."""
+    if cli_key:
+        return cli_key, "--api-key"
+    for var in ("SYNTH_BACKEND_API_KEY", "SYNTH_API_KEY", "DEFAULT_DEV_API_KEY"):
+        val = os.getenv(var)
+        if val:
+            return val, var
+    return None, None
+
+
+@dataclass()
+class BackendConfig:
+    """Configuration bundle shared across status commands."""
+
+    base_url: str
+    api_key: str | None
+    timeout: float = DEFAULT_TIMEOUT
+
+    @property
+    def headers(self) -> dict[str, str]:
+        if not self.api_key:
+            return {}
+        return {"Authorization": f"Bearer {self.api_key}"}
+
+
+def resolve_backend_config(
+    *,
+    base_url: str | None,
+    api_key: str | None,
+    timeout: float | None = None,
+) -> BackendConfig:
+    """Resolve the backend configuration from CLI options/environment."""
+    resolved_url = _normalize_base_url(base_url) if base_url else _default_base_url()
+    key, _ = _resolve_api_key(api_key)
+    return BackendConfig(base_url=resolved_url, api_key=key, timeout=timeout or DEFAULT_TIMEOUT)

synth_ai/cli/commands/status/errors.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+"""
+Custom error hierarchy for status CLI commands.
+"""
+
+
+
+class StatusAPIError(RuntimeError):
+    """Raised when the backend returns a non-success response."""
+
+    def __init__(self, message: str, status_code: int | None = None):
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class StatusCLIError(RuntimeError):
+    """Raised for client-side validation errors."""
+
+    pass