synth-ai 0.2.16__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/cli/commands/status/subcommands/runs.py

@@ -0,0 +1,81 @@
+"""`synth runs` command group."""
+
+from __future__ import annotations
+
+import asyncio
+
+import click
+
+from ..client import StatusAPIClient
+from ..errors import StatusAPIError
+from ..formatters import console, events_panel, print_json, runs_table
+from ..utils import bail, common_options, parse_relative_time, resolve_context_config
+
+
+@click.group("runs", help="Inspect individual job runs/attempts.")
+@click.pass_context
+def runs_group(ctx: click.Context) -> None:  # pragma: no cover - Click wiring
+    ctx.ensure_object(dict)
+
+
+@runs_group.command("list")
+@common_options()
+@click.argument("job_id")
+@click.option("--json", "output_json", is_flag=True)
+@click.pass_context
+def list_runs(
+    ctx: click.Context,
+    base_url: str | None,
+    api_key: str | None,
+    timeout: float,
+    job_id: str,
+    output_json: bool,
+) -> None:
+    cfg = resolve_context_config(ctx, base_url=base_url, api_key=api_key, timeout=timeout)
+
+    async def _run() -> None:
+        try:
+            async with StatusAPIClient(cfg) as client:
+                runs = await client.list_job_runs(job_id)
+                if output_json:
+                    print_json(runs)
+                else:
+                    console.print(runs_table(runs))
+        except StatusAPIError as exc:
+            bail(f"Backend error: {exc}")
+
+    asyncio.run(_run())
+
+
+@runs_group.command("logs")
+@common_options()
+@click.argument("job_id")
+@click.option("--run", "run_id", required=True, help="Run identifier (number or ID) to inspect.")
+@click.option("--since", help="Filter events after the supplied timestamp/relative offset.")
+@click.option("--json", "output_json", is_flag=True)
+@click.pass_context
+def run_logs(
+    ctx: click.Context,
+    base_url: str | None,
+    api_key: str | None,
+    timeout: float,
+    job_id: str,
+    run_id: str,
+    since: str | None,
+    output_json: bool,
+) -> None:
+    cfg = resolve_context_config(ctx, base_url=base_url, api_key=api_key, timeout=timeout)
+    since_filter = parse_relative_time(since)
+
+    async def _run() -> None:
+        try:
+            async with StatusAPIClient(cfg) as client:
+                events = await client.get_job_events(job_id, since=since_filter, run_id=run_id)
+                if output_json:
+                    print_json(events)
+                else:
+                    console.print(events_panel(events))
+        except StatusAPIError as exc:
+            bail(f"Backend error: {exc}")
+
+    asyncio.run(_run())

synth_ai/cli/commands/status/subcommands/summary.py

@@ -0,0 +1,47 @@
+"""`synth status summary` command."""
+
+from __future__ import annotations
+
+import asyncio
+
+import click
+
+from ..client import StatusAPIClient
+from ..errors import StatusAPIError
+from ..formatters import console, files_table, jobs_table, models_table
+from ..utils import common_options, resolve_context_config
+
+
+@click.command("summary", help="Show a condensed overview of recent jobs, models, and files.")
+@common_options()
+@click.option("--limit", default=5, show_default=True, type=int, help="Rows per section.")
+@click.pass_context
+def summary_command(
+    ctx: click.Context,
+    base_url: str | None,
+    api_key: str | None,
+    timeout: float,
+    limit: int,
+) -> None:
+    cfg = resolve_context_config(ctx, base_url=base_url, api_key=api_key, timeout=timeout)
+
+    async def _run() -> tuple[list[dict[str, object]], list[dict[str, object]], list[dict[str, object]]]:
+        async with StatusAPIClient(cfg) as client:
+            try:
+                jobs = await client.list_jobs(limit=limit)
+            except StatusAPIError:
+                jobs = []
+            try:
+                models = await client.list_models(limit=limit)
+            except StatusAPIError:
+                models = []
+            try:
+                files = await client.list_files(limit=limit)
+            except StatusAPIError:
+                files = []
+        return jobs, models, files
+
+    jobs, models, files = asyncio.run(_run())
+    console.print(jobs_table(jobs[:limit]))
+    console.print(models_table(models[:limit]))
+    console.print(files_table(files[:limit]))

synth_ai/cli/commands/status/utils.py

@@ -0,0 +1,114 @@
+"""Shared utilities for status commands."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable, Coroutine
+from datetime import UTC, datetime, timedelta
+from typing import Any, TypeVar
+
+import click
+from rich.console import Console
+
+from .config import DEFAULT_TIMEOUT, BackendConfig, resolve_backend_config
+
+T = TypeVar("T")
+
+console = Console()
+
+
+def parse_relative_time(value: str | None) -> str | None:
+    """Convert relative time expressions (e.g., '5m', '2h', '1d') to ISO strings."""
+    if not value:
+        return None
+    token = value.strip().lower()
+    if not token:
+        return None
+    multiplier = 1.0
+    if token.endswith("ms"):
+        multiplier = 0.001
+        token = token[:-2]
+    elif token.endswith("s"):
+        multiplier = 1.0
+        token = token[:-1]
+    elif token.endswith("m"):
+        multiplier = 60.0
+        token = token[:-1]
+    elif token.endswith("h"):
+        multiplier = 3600.0
+        token = token[:-1]
+    elif token.endswith("d"):
+        multiplier = 86400.0
+        token = token[:-1]
+
+    try:
+        seconds = float(token) * multiplier
+    except ValueError:
+        return value
+
+    dt = datetime.now(UTC) - timedelta(seconds=seconds)
+    return dt.isoformat()
+
+
+def ensure_async(fn: Callable[..., Coroutine[Any, Any, T]]) -> Callable[..., T]:
+    """Decorator to run an async callable via asyncio.run inside Click commands."""
+
+    def wrapper(*args, **kwargs):
+        return asyncio.run(fn(*args, **kwargs))
+
+    return wrapper
+
+
+def resolve_context_config(
+    ctx: click.Context,
+    *,
+    base_url: str | None,
+    api_key: str | None,
+    timeout: float | None,
+) -> BackendConfig:
+    if base_url is not None or api_key is not None or timeout not in (None, DEFAULT_TIMEOUT):
+        return resolve_backend_config(base_url=base_url, api_key=api_key, timeout=timeout)
+    obj = ctx.find_object(dict)
+    if obj and isinstance(obj.get("status_backend_config"), BackendConfig):
+        return obj["status_backend_config"]
+    return resolve_backend_config(base_url=base_url, api_key=api_key, timeout=timeout)
+
+
+def warn(message: str) -> None:
+    console.print(f"[yellow]{message}[/yellow]")
+
+
+def bail(message: str) -> None:
+    raise click.ClickException(message)
+
+
+def common_options() -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """Apply shared backend CLI options to a command."""
+
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        options = [
+            click.option(
+                "--base-url",
+                envvar="SYNTH_STATUS_BASE_URL",
+                default=None,
+                help="Override the Synth backend base URL for this command.",
+            ),
+            click.option(
+                "--api-key",
+                envvar="SYNTH_STATUS_API_KEY",
+                default=None,
+                help="API key for the Synth backend.",
+            ),
+            click.option(
+                "--timeout",
+                default=DEFAULT_TIMEOUT,
+                show_default=True,
+                type=float,
+                help="HTTP request timeout in seconds.",
+            ),
+        ]
+        for option in reversed(options):
+            func = option(func)
+        return func
+
+    return decorator

synth_ai/cli/commands/train/__init__.py

@@ -0,0 +1,53 @@
+from .core import register, train_command
+from .errors import (
+    InvalidJudgeConfigError,
+    InvalidRubricConfigError,
+    TrainCliError,
+)
+from .judge_schemas import (
+    JudgeConfig,
+    JudgeOptionsConfig,
+    JudgeRequestPayload,
+    RubricConfig,
+    RubricWeightsConfig,
+    build_judge_http_options,
+)
+from .judge_validation import (
+    check_for_deprecated_fields,
+    extract_and_validate_judge_rubric,
+    validate_judge_config,
+    validate_rubric_config,
+)
+from .validation import (
+    load_and_validate_rl,
+    load_and_validate_sft,
+    validate_rl_config,
+    validate_sft_config,
+)
+
+__all__ = [
+    # Core
+    "register",
+    "train_command",
+    # Errors
+    "TrainCliError",
+    "InvalidJudgeConfigError",
+    "InvalidRubricConfigError",
+    # SFT/RL validation
+    "validate_sft_config",
+    "validate_rl_config",
+    "load_and_validate_sft",
+    "load_and_validate_rl",
+    # Judge/Rubric schemas
+    "RubricWeightsConfig",
+    "RubricConfig",
+    "JudgeOptionsConfig",
+    "JudgeConfig",
+    "JudgeRequestPayload",
+    "build_judge_http_options",
+    # Judge/Rubric validation
+    "validate_rubric_config",
+    "validate_judge_config",
+    "extract_and_validate_judge_rubric",
+    "check_for_deprecated_fields",
+]

synth_ai/cli/commands/train/core.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import click
+from synth_ai.api.train.cli import (
+    register as _register_with_cli,
+)
+from synth_ai.api.train.cli import (
+    train_command as _train_command,
+)
+
+__all__ = ["register", "train_command"]
+
+
+def register(cli: click.Group) -> None:
+    """Attach the train command to the root CLI."""
+    _register_with_cli(cli)
+
+
+def train_command(*args, **kwargs):
+    """Entrypoint used by the train CLI command."""
+    return _train_command(*args, **kwargs)

synth_ai/cli/commands/train/errors.py

@@ -0,0 +1,117 @@
+from dataclasses import dataclass
+
+
+class TrainCliError(RuntimeError):
+    """Base exception for train CLI failures."""
+
+
+@dataclass(slots=True)
+class TomlParseError(TrainCliError):
+    """Raised when TOML file cannot be parsed."""
+    path: str
+    detail: str
+
+
+@dataclass(slots=True)
+class ConfigNotFoundError(TrainCliError):
+    """Raised when config file is not found."""
+    path: str
+
+
+@dataclass(slots=True)
+class InvalidSFTConfigError(TrainCliError):
+    """Raised when SFT configuration is invalid."""
+    detail: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class InvalidRLConfigError(TrainCliError):
+    """Raised when RL configuration is invalid."""
+    detail: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class MissingAlgorithmError(TrainCliError):
+    """Raised when [algorithm] section is missing or invalid."""
+    detail: str
+
+
+@dataclass(slots=True)
+class MissingModelError(TrainCliError):
+    """Raised when model specification is missing."""
+    detail: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class MissingDatasetError(TrainCliError):
+    """Raised when dataset path is missing for SFT."""
+    detail: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class MissingComputeError(TrainCliError):
+    """Raised when compute configuration is missing or incomplete."""
+    detail: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class UnsupportedAlgorithmError(TrainCliError):
+    """Raised when algorithm type is not supported."""
+    algorithm_type: str
+    expected: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class InvalidHyperparametersError(TrainCliError):
+    """Raised when hyperparameters are invalid."""
+    detail: str
+    parameter: str | None = None
+
+
+@dataclass(slots=True)
+class InvalidTopologyError(TrainCliError):
+    """Raised when topology configuration is invalid."""
+    detail: str
+    hint: str | None = None
+
+
+@dataclass(slots=True)
+class InvalidJudgeConfigError(TrainCliError):
+    """Raised when judge configuration validation fails."""
+    detail: str
+
+    def __str__(self) -> str:
+        return self.detail
+
+
+@dataclass(slots=True)
+class InvalidRubricConfigError(TrainCliError):
+    """Raised when rubric configuration validation fails."""
+    detail: str
+
+    def __str__(self) -> str:
+        return self.detail
+
+
+__all__ = [
+    "TrainCliError",
+    "TomlParseError",
+    "ConfigNotFoundError",
+    "InvalidSFTConfigError",
+    "InvalidRLConfigError",
+    "MissingAlgorithmError",
+    "MissingModelError",
+    "MissingDatasetError",
+    "MissingComputeError",
+    "UnsupportedAlgorithmError",
+    "InvalidHyperparametersError",
+    "InvalidTopologyError",
+    "InvalidJudgeConfigError",
+    "InvalidRubricConfigError",
+]

synth_ai/cli/commands/train/judge_schemas.py

@@ -0,0 +1,199 @@
+"""
+Pydantic schemas for judge/rubric configuration.
+
+These models define the ACTUAL fields used by the backend judge service,
+with all dead code removed. This is the single source of truth for what
+gets sent in HTTP requests.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import Field, model_validator
+from synth_ai.api.train.configs.shared import ExtraModel
+
+__all__ = [
+    "RubricWeightsConfig",
+    "RubricConfig",
+    "JudgeOptionsConfig",
+    "JudgeConfig",
+    "JudgeRequestPayload",
+]
+
+
+class RubricWeightsConfig(ExtraModel):
+    """
+    Reward blending weights (client-side only, not sent to backend).
+    
+    These weights control how env rewards, event judge scores, and outcome
+    judge scores are combined into a final reward signal for policy gradients.
+    
+    Formula:
+        total_reward = (env * env_return) + (event * sum(event_scores)) + (outcome * outcome_score)
+    """
+    env: float = Field(
+        default=1.0,
+        description="Weight for environment rewards (task app native rewards)",
+        ge=0.0,
+    )
+    event: float = Field(
+        default=0.0,
+        description="Weight for per-event judge scores (step-level judging)",
+        ge=0.0,
+    )
+    outcome: float = Field(
+        default=0.0,
+        description="Weight for outcome judge score (episode-level judging)",
+        ge=0.0,
+    )
+
+    @model_validator(mode="after")
+    def _validate_weights_sum(self) -> RubricWeightsConfig:
+        """Ensure at least one weight is non-zero."""
+        if self.env == 0.0 and self.event == 0.0 and self.outcome == 0.0:
+            raise ValueError("At least one reward weight must be non-zero")
+        return self
+
+
+class RubricConfig(ExtraModel):
+    """
+    Top-level rubric configuration.
+    
+    Controls whether rubric-based judging is enabled and how rewards are blended.
+    """
+    enabled: bool = Field(
+        default=False,
+        description="Master switch for rubric-based judging",
+    )
+    weights: RubricWeightsConfig = Field(
+        default_factory=RubricWeightsConfig,
+        description="Reward blending weights (env/event/outcome)",
+    )
+
+
+class JudgeOptionsConfig(ExtraModel):
+    """
+    Judge provider options (sent to backend in HTTP request).
+    
+    These fields are sent in the "options" object of the judge score request.
+    All fields here map directly to the backend JudgeOptions schema.
+    """
+    provider: str = Field(
+        ...,
+        description="Judge provider type ('openai', 'groq', 'gemini')",
+        pattern=r"^(openai|groq|gemini)$",
+    )
+    model: str = Field(
+        ...,
+        description="Model identifier (e.g., 'openai/gpt-oss-120b', 'gpt-5')",
+        min_length=1,
+    )
+    rubric_id: Optional[str] = Field(
+        default=None,
+        description="Base rubric identifier (e.g., 'crafter/bundle@v1')",
+    )
+    event: bool = Field(
+        default=True,
+        description="Enable per-event (step-level) judging",
+    )
+    outcome: bool = Field(
+        default=True,
+        description="Enable outcome (episode-level) judging",
+    )
+    timeout_s: Optional[float] = Field(
+        default=None,
+        description="Request timeout in seconds",
+        gt=0,
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Optional metadata (e.g., {'async': true, 'custom_field': 'value'})",
+    )
+    rubric_overrides: dict[str, Any] = Field(
+        default_factory=dict,
+        description=(
+            "Static rubric criteria overrides (rarely used - TaskInfo overrides take priority). "
+            "Format: {'event': {'criteria': [...]}, 'outcome': {'criteria': [...]}}"
+        ),
+    )
+
+    @model_validator(mode="after")
+    def _validate_at_least_one_enabled(self) -> JudgeOptionsConfig:
+        """Ensure at least one judging type is enabled."""
+        if not self.event and not self.outcome:
+            raise ValueError("At least one of 'event' or 'outcome' must be enabled")
+        return self
+
+
+class JudgeConfig(ExtraModel):
+    """
+    Top-level judge configuration.
+    
+    This is parsed from TOML [judge] section and contains all judge-related settings.
+    """
+    options: JudgeOptionsConfig = Field(
+        ...,
+        description="Judge provider options (sent to backend)",
+    )
+
+
+# HTTP Request Payload Structures (for documentation/type safety)
+
+class JudgeRequestPayload(ExtraModel):
+    """
+    HTTP request payload structure for POST /api/judge/v1/score.
+    
+    This is the ACTUAL payload sent to the backend judge service.
+    Used for type safety and documentation only.
+    """
+    policy_name: str = Field(..., description="Name of the policy being evaluated")
+    task_app: dict[str, Any] = Field(..., description="Task app metadata (id, base_url)")
+    trace: dict[str, Any] = Field(..., description="Tracing v3 payload (event_history, metadata)")
+    options: dict[str, Any] = Field(..., description="Judge options (provider, model, etc.)")
+
+    class Config:
+        extra = "allow"  # Backend might add extra fields
+
+
+# Helper to convert to backend request format
+
+def build_judge_http_options(
+    options_config: JudgeOptionsConfig,
+    *,
+    rubric_overrides_from_task_info: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    """
+    Build the 'options' dict for HTTP request to backend judge.
+    
+    Args:
+        options_config: Validated judge options from TOML
+        rubric_overrides_from_task_info: Dynamic overrides fetched from TaskInfo (takes priority)
+    
+    Returns:
+        Dict ready to send in HTTP request payload
+    """
+    payload = {
+        "provider": options_config.provider,
+        "model": options_config.model,
+        "event": options_config.event,
+        "outcome": options_config.outcome,
+    }
+    
+    # Optional fields
+    if options_config.rubric_id:
+        payload["rubric_id"] = options_config.rubric_id
+    
+    if options_config.timeout_s is not None:
+        payload["timeout_s"] = options_config.timeout_s
+    
+    if options_config.metadata:
+        payload["metadata"] = options_config.metadata
+    
+    # Rubric overrides: TaskInfo takes priority over static config
+    if rubric_overrides_from_task_info:
+        payload["rubric_overrides"] = rubric_overrides_from_task_info
+    elif options_config.rubric_overrides:
+        payload["rubric_overrides"] = options_config.rubric_overrides
+    
+    return payload