synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/cli/commands/baseline/list.py

@@ -1,94 +0,0 @@
-"""List command for baseline discovery."""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Optional
-
-import click
-
-from synth_ai.sdk.baseline.config import BaselineConfig
-from synth_ai.sdk.baseline.discovery import (
-    BaselineChoice,
-    discover_baseline_files,
-    load_baseline_config_from_file,
-)
-
-
-@click.command("list")
-@click.option(
-    "--tag",
-    multiple=True,
-    help="Filter baselines by tag (can be specified multiple times)",
-)
-@click.option(
-    "--metadata",
-    type=str,
-    help="Filter by metadata key-value pair (format: key=value)",
-)
-@click.option(
-    "--verbose",
-    is_flag=True,
-    help="Show detailed information about each baseline",
-)
-def list_command(tag: tuple[str, ...], metadata: Optional[str], verbose: bool) -> None:
-    """List all available baseline files."""
-    search_roots = [Path.cwd()]
-    choices = discover_baseline_files(search_roots)
-    
-    if not choices:
-        click.echo("No baseline files found.", err=True)
-        click.echo("Create baseline files in examples/baseline/ or */*_baseline.py")
-        return
-    
-    # Load configs for filtering
-    configs: list[tuple[BaselineChoice, BaselineConfig]] = []
-    for choice in choices:
-        try:
-            config = load_baseline_config_from_file(choice.baseline_id, choice.path)
-            configs.append((choice, config))
-        except Exception as e:
-            if verbose:
-                click.echo(f"Warning: Could not load {choice.baseline_id}: {e}", err=True)
-            continue
-    
-    # Apply filters
-    filtered_configs = configs
-    
-    if tag:
-        tag_set = {t.lower() for t in tag}
-        filtered_configs = [
-            (c, config) for c, config in filtered_configs
-            if any(config.matches_tag(t) for t in tag_set)
-        ]
-    
-    if metadata:
-        if "=" not in metadata:
-            raise click.ClickException("--metadata must be in format key=value")
-        key, value = metadata.split("=", 1)
-        filtered_configs = [
-            (c, config) for c, config in filtered_configs
-            if config.matches_metadata(key.strip(), value.strip())
-        ]
-    
-    if not filtered_configs:
-        click.echo("No baselines match the specified filters.")
-        return
-    
-    # Display results
-    click.echo(f"Found {len(filtered_configs)} baseline(s):\n")
-    
-    for choice, config in filtered_configs:
-        click.echo(f"  {config.baseline_id}")
-        click.echo(f"    Name: {config.name}")
-        if config.description:
-            click.echo(f"    Description: {config.description}")
-        if config.tags:
-            click.echo(f"    Tags: {', '.join(config.tags)}")
-        click.echo(f"    Splits: {', '.join(config.splits.keys())}")
-        if verbose:
-            click.echo(f"    Path: {choice.path}")
-            if config.metadata:
-                click.echo(f"    Metadata: {config.metadata}")
-        click.echo()
-

synth_ai/cli/commands/eval/errors.py

@@ -1,81 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-
-class EvalCliError(RuntimeError):
-    """Base exception for eval CLI failures."""
-
-
-@dataclass(slots=True)
-class TomlUnavailableError(EvalCliError):
-    hint: str | None = None
-
-
-@dataclass(slots=True)
-class EvalConfigNotFoundError(EvalCliError):
-    path: str
-
-
-@dataclass(slots=True)
-class EvalConfigParseError(EvalCliError):
-    path: str
-    detail: str
-
-
-@dataclass(slots=True)
-class MissingEvalTableError(EvalCliError):
-    """Raised when the eval config lacks an [eval] table."""
-
-
-@dataclass(slots=True)
-class InvalidEvalConfigError(EvalCliError):
-    detail: str
-
-
-@dataclass(slots=True)
-class SeedParseError(EvalCliError):
-    value: str
-
-
-@dataclass(slots=True)
-class MetadataFilterFormatError(EvalCliError):
-    entry: str
-
-
-@dataclass(slots=True)
-class TaskInfoUnavailableError(EvalCliError):
-    """Raised when metadata filters require task info but the task app does not expose it."""
-
-
-@dataclass(slots=True)
-class NoSeedsMatchedError(EvalCliError):
-    hint: str | None = None
-
-
-@dataclass(slots=True)
-class MetadataSQLExecutionError(EvalCliError):
-    query: str
-    detail: str
-
-
-@dataclass(slots=True)
-class MetadataSQLResultError(EvalCliError):
-    query: str
-    detail: str
-
-
-__all__ = [
-    "EvalCliError",
-    "TomlUnavailableError",
-    "EvalConfigNotFoundError",
-    "EvalConfigParseError",
-    "MissingEvalTableError",
-    "InvalidEvalConfigError",
-    "SeedParseError",
-    "MetadataFilterFormatError",
-    "TaskInfoUnavailableError",
-    "NoSeedsMatchedError",
-    "MetadataSQLExecutionError",
-    "MetadataSQLResultError",
-]

synth_ai/cli/commands/status/formatters.py

@@ -1,164 +0,0 @@
-"""Rich-based formatting helpers for status commands."""
-
-from __future__ import annotations
-
-import json
-from collections.abc import Iterable
-from datetime import datetime
-from typing import Any
-
-from rich import box
-from rich.console import Console
-from rich.panel import Panel
-from rich.table import Table
-from rich.text import Text
-
-console = Console()
-
-
-def _format_timestamp(value: Any) -> str:
-    if value in (None, "", 0):
-        return ""
-    if isinstance(value, int | float):
-        try:
-            return datetime.fromtimestamp(float(value)).isoformat()
-        except Exception:
-            return str(value)
-    if isinstance(value, str):
-        try:
-            return datetime.fromisoformat(value.replace("Z", "+00:00")).strftime("%Y-%m-%d %H:%M:%S")
-        except Exception:
-            return value
-    return str(value)
-
-
-def print_json(data: Any) -> None:
-    console.print_json(data=data)
-
-
-def jobs_table(jobs: Iterable[dict[str, Any]]) -> Table:
-    table = Table(title="Training Jobs", box=box.SIMPLE, header_style="bold")
-    table.add_column("ID", style="cyan", overflow="fold")
-    table.add_column("Type", style="magenta")
-    table.add_column("Status")
-    table.add_column("Created", style="green")
-    table.add_column("Updated", style="green")
-    table.add_column("Model", style="yellow", overflow="fold")
-    for job in jobs:
-        status = job.get("status", "unknown")
-        status_color = {
-            "running": "green",
-            "queued": "cyan",
-            "succeeded": "bright_green",
-            "failed": "red",
-            "cancelled": "yellow",
-        }.get(status, "white")
-        table.add_row(
-            str(job.get("job_id") or job.get("id", "")),
-            str(job.get("training_type") or job.get("type", "")),
-            f"[{status_color}]{status}[/{status_color}]",
-            _format_timestamp(job.get("created_at")),
-            _format_timestamp(job.get("updated_at")),
-            str(job.get("model_id") or job.get("model", "")),
-        )
-    return table
-
-
-def job_panel(job: dict[str, Any]) -> Panel:
-    lines = [f"[bold cyan]Job[/bold cyan] {job.get('job_id') or job.get('id')}"]
-    if job.get("name"):
-        lines.append(f"Name: {job['name']}")
-    lines.append(f"Type: {job.get('training_type', job.get('type', ''))}")
-    lines.append(f"Status: {job.get('status', 'unknown')}")
-    if job.get("model_id"):
-        lines.append(f"Model: {job['model_id']}")
-    if job.get("base_model"):
-        lines.append(f"Base Model: {job['base_model']}")
-    lines.append(f"Created: {_format_timestamp(job.get('created_at'))}")
-    lines.append(f"Updated: {_format_timestamp(job.get('updated_at'))}")
-    if config := job.get("config"):
-        lines.append("")
-        lines.append(f"[dim]{json.dumps(config, indent=2, sort_keys=True)}[/dim]")
-    return Panel("\n".join(lines), title="Job Details", border_style="cyan")
-
-
-def runs_table(runs: Iterable[dict[str, Any]]) -> Table:
-    table = Table(title="Job Runs", box=box.SIMPLE, header_style="bold")
-    table.add_column("Run #", justify="right")
-    table.add_column("Engine")
-    table.add_column("Status")
-    table.add_column("Created")
-    table.add_column("Started")
-    table.add_column("Ended")
-    table.add_column("Duration", justify="right")
-    for run in runs:
-        table.add_row(
-            str(run.get("run_number") or run.get("id", "")),
-            str(run.get("engine", "")),
-            str(run.get("status", "unknown")),
-            _format_timestamp(run.get("created_at")),
-            _format_timestamp(run.get("started_at")),
-            _format_timestamp(run.get("ended_at")),
-            str(run.get("duration_seconds") or run.get("duration", "")),
-        )
-    return table
-
-
-def events_panel(events: Iterable[dict[str, Any]]) -> Panel:
-    rendered = []
-    for event in events:
-        ts = _format_timestamp(event.get("timestamp") or event.get("created_at"))
-        level = event.get("level") or event.get("severity", "info")
-        message = event.get("message") or event.get("detail") or ""
-        rendered.append(f"[dim]{ts}[/dim] [{level}] {message}")
-    if not rendered:
-        rendered.append("[dim]No events found.[/dim]")
-    return Panel("\n".join(rendered), title="Job Events", border_style="green")
-
-
-def metrics_table(metrics: dict[str, Any]) -> Table:
-    table = Table(title="Job Metrics", box=box.SIMPLE, header_style="bold")
-    table.add_column("Metric")
-    table.add_column("Value", justify="right")
-    for key, value in metrics.items():
-        if isinstance(value, dict):
-            table.add_row(key, Text(json.dumps(value), overflow="fold"))
-        else:
-            table.add_row(key, str(value))
-    return table
-
-
-def files_table(files: Iterable[dict[str, Any]]) -> Table:
-    table = Table(title="Training Files", box=box.SIMPLE, header_style="bold")
-    table.add_column("ID", overflow="fold")
-    table.add_column("Purpose")
-    table.add_column("Size", justify="right")
-    table.add_column("Created")
-    table.add_column("Filename", overflow="fold")
-    for file in files:
-        table.add_row(
-            str(file.get("file_id") or file.get("id", "")),
-            str(file.get("purpose", "")),
-            str(file.get("bytes", "")),
-            _format_timestamp(file.get("created_at")),
-            str(file.get("filename", "")),
-        )
-    return table
-
-
-def models_table(models: Iterable[dict[str, Any]]) -> Table:
-    table = Table(title="Fine-tuned Models", box=box.SIMPLE, header_style="bold")
-    table.add_column("ID", overflow="fold")
-    table.add_column("Base")
-    table.add_column("Created")
-    table.add_column("Owner")
-    table.add_column("Status")
-    for model in models:
-        table.add_row(
-            str(model.get("id", model.get("name", ""))),
-            str(model.get("base_model") or model.get("base", "")),
-            _format_timestamp(model.get("created_at")),
-            str(model.get("owner") or model.get("organization", "")),
-            str(model.get("status", "")),
-        )
-    return table

synth_ai/cli/commands/status/subcommands/pricing.py

@@ -1,23 +0,0 @@
-from __future__ import annotations
-
-import click
-from rich.table import Table
-
-from synth_ai.core.pricing.model_pricing import MODEL_PRICES  # type: ignore[import-untyped]
-
-from ..formatters import console
-
-
-@click.command("pricing", help="List supported provider/model rates (SDK static table).")
-def pricing_command() -> None:
-    table = Table(title="Supported Models and Rates (USD/token)")
-    table.add_column("Provider", style="cyan", no_wrap=True)
-    table.add_column("Model", style="magenta")
-    table.add_column("Input USD", justify="right")
-    table.add_column("Output USD", justify="right")
-    for provider, models in MODEL_PRICES.items():
-        for model, rates in models.items():
-            table.add_row(provider, model, f"{rates.input_usd:.9f}", f"{rates.output_usd:.9f}")
-    console.print(table)
-
-

synth_ai/cli/commands/status/subcommands/usage.py

@@ -1,203 +0,0 @@
-from __future__ import annotations
-
-import contextlib
-from datetime import UTC, datetime, timedelta
-from typing import Any
-
-import click
-
-from ..client import StatusAPIClient
-from ..errors import StatusAPIError
-from ..formatters import console
-from ..utils import common_options, resolve_context_config
-
-
-def _parse_iso(ts: str | None) -> datetime | None:
-    if not ts:
-        return None
-    try:
-        # Python 3.11 handles 'YYYY-mm-ddTHH:MM:SS.ssssss+00:00' and '...Z'
-        if ts.endswith("Z"):
-            ts = ts.replace("Z", "+00:00")
-        return datetime.fromisoformat(ts)
-    except Exception:
-        return None
-
-
-def _extract_total_usd(events: list[dict[str, Any]]) -> tuple[float, int]:
-    """Return (usd_total, tokens_total) for an arbitrary job's events.
-
-    Strategy:
-    - Prefer a consolidated total from any *.completed event with total_usd
-    - Next, prefer any *.billing.end event with total_usd
-    - Otherwise, combine usage.recorded's usd_tokens with billing.sandboxes' usd
-      and sum token counts if present
-    Works for prompt learning and other job types that follow similar conventions.
-    """
-    total_usd = 0.0
-    token_count = 0
-
-    # Prefer consolidated totals from completion events (any namespace)
-    for e in reversed(events):
-        typ = str(e.get("type") or "").lower()
-        if typ.endswith(".completed"):
-            data = e.get("data") or {}
-            try:
-                total_usd = float(data.get("total_usd") or 0.0)
-            except Exception:
-                total_usd = 0.0
-            # Try common token fields
-            tc = 0
-            for k in ("token_count_total", "token_count"):
-                try:
-                    tc = int(data.get(k) or 0)
-                    if tc:
-                        break
-                except Exception:
-                    pass
-            if not tc:
-                try:
-                    tc = int((data.get("token_count_rollouts") or 0) + (data.get("token_count_mutation") or 0))
-                except Exception:
-                    tc = 0
-            token_count = tc
-            return total_usd, token_count
-
-    # Next, billing.end if present with total_usd
-    for e in reversed(events):
-        typ = str(e.get("type") or "").lower()
-        if typ.endswith("billing.end"):
-            data = e.get("data") or {}
-            try:
-                total_usd = float(data.get("total_usd") or 0.0)
-            except Exception:
-                total_usd = 0.0
-            # token_count may not be present here; fall through to usage tokens calc
-            break
-
-    # Fallback: combine usage + sandboxes (prompt learning style); generic scan
-    usd_tokens = 0.0
-    sandbox_usd = 0.0
-    # token fields observed across tasks
-    token_fields = ("token_count_total", "token_count", "tokens_in", "tokens_out",
-                    "token_count_rollouts", "token_count_mutation")
-    for e in events:
-        typ = str(e.get("type") or "").lower()
-        data = e.get("data") or {}
-        # generic usage-style aggregation
-        if "usage" in typ or typ.endswith("usage.recorded"):
-            with contextlib.suppress(Exception):
-                usd_tokens = float(data.get("usd_tokens") or data.get("usd_estimate") or 0.0)
-            # accumulate tokens if any
-            for k in token_fields:
-                with contextlib.suppress(Exception):
-                    token_count += int(data.get(k) or 0)
-        # sandbox billing
-        if typ.endswith("billing.sandboxes"):
-            with contextlib.suppress(Exception):
-                sandbox_usd += float(data.get("usd") or 0.0)
-    return (total_usd or (usd_tokens + sandbox_usd)), token_count
-
-
-@click.command("usage", help="Show recent usage (daily/weekly/monthly) and remaining budget if provided.")
-@common_options()
-@click.option("--budget-usd", type=float, default=None, help="Optional credit/budget to compute remaining.")
-@click.option("--json", "output_json", is_flag=True, help="Emit machine-readable JSON.")
-@click.pass_context
-def usage_command(
-    ctx: click.Context,
-    base_url: str | None,
-    api_key: str | None,
-    timeout: float,
-    budget_usd: float | None,
-    output_json: bool,
-) -> None:
-    cfg = resolve_context_config(ctx, base_url=base_url, api_key=api_key, timeout=timeout)
-    now = datetime.now(UTC)
-    daily_cutoff = (now - timedelta(days=1)).isoformat()
-    weekly_cutoff = (now - timedelta(days=7)).isoformat()
-    monthly_cutoff = (now - timedelta(days=30)).isoformat()
-
-    async def _run() -> tuple[dict[str, float | int], dict[str, float | int], dict[str, float | int]]:
-        daily = {"usd": 0.0, "tokens": 0, "sandbox_seconds": 0.0}
-        weekly = {"usd": 0.0, "tokens": 0, "sandbox_seconds": 0.0}
-        monthly = {"usd": 0.0, "tokens": 0, "sandbox_seconds": 0.0}
-        async with StatusAPIClient(cfg) as client:
-            try:
-                jobs = await client.list_jobs(created_after=weekly_cutoff)
-            except StatusAPIError as exc:
-                raise click.ClickException(f"Backend error: {exc}") from exc
-            for j in jobs or []:
-                job_id = str(j.get("job_id") or j.get("id") or "")
-                if not job_id:
-                    continue
-                try:
-                    events = await client.get_job_events(job_id, since=weekly_cutoff)
-                except StatusAPIError:
-                    events = []
-                if not events:
-                    continue
-                # Use event timestamps for windowing
-                # Weekly
-                weekly_ev = [e for e in events if (_parse_iso(e.get("created_at")) or now) >= datetime.fromisoformat(weekly_cutoff)]
-                w_usd, w_tok = _extract_total_usd(weekly_ev)
-                weekly["usd"] += w_usd
-                weekly["tokens"] += w_tok
-                # sandbox seconds
-                for e in weekly_ev:
-                    if str(e.get("type") or "").lower().endswith("billing.sandboxes"):
-                        with contextlib.suppress(Exception):
-                            weekly["sandbox_seconds"] += float((e.get("data") or {}).get("seconds") or 0.0)
-                # Daily
-                daily_ev = [e for e in events if (_parse_iso(e.get("created_at")) or now) >= datetime.fromisoformat(daily_cutoff)]
-                d_usd, d_tok = _extract_total_usd(daily_ev)
-                daily["usd"] += d_usd
-                daily["tokens"] += d_tok
-                for e in daily_ev:
-                    if str(e.get("type") or "").lower().endswith("billing.sandboxes"):
-                        with contextlib.suppress(Exception):
-                            daily["sandbox_seconds"] += float((e.get("data") or {}).get("seconds") or 0.0)
-                # Monthly
-                monthly_ev = [e for e in events if (_parse_iso(e.get("created_at")) or now) >= datetime.fromisoformat(monthly_cutoff)]
-                m_usd, m_tok = _extract_total_usd(monthly_ev)
-                monthly["usd"] += m_usd
-                monthly["tokens"] += m_tok
-                for e in monthly_ev:
-                    if str(e.get("type") or "").lower().endswith("billing.sandboxes"):
-                        with contextlib.suppress(Exception):
-                            monthly["sandbox_seconds"] += float((e.get("data") or {}).get("seconds") or 0.0)
-        return daily, weekly, monthly
-
-    daily, weekly, monthly = __import__("asyncio").run(_run())
-
-    if output_json:
-        import json as _json
-        payload: dict[str, Any] = {
-            "daily": {
-                "usd": round(float(daily["usd"]), 4),
-                "tokens": int(daily["tokens"]),
-                "sandbox_hours": round(float(daily["sandbox_seconds"]) / 3600.0, 4),
-            },
-            "weekly": {
-                "usd": round(float(weekly["usd"]), 4),
-                "tokens": int(weekly["tokens"]),
-                "sandbox_hours": round(float(weekly["sandbox_seconds"]) / 3600.0, 4),
-            },
-            "monthly": {
-                "usd": round(float(monthly["usd"]), 4),
-                "tokens": int(monthly["tokens"]),
-                "sandbox_hours": round(float(monthly["sandbox_seconds"]) / 3600.0, 4),
-            },
-        }
-        if budget_usd is not None:
-            payload["remaining_vs_budget"] = round(max(0.0, float(budget_usd) - float(weekly["usd"])), 4)
-        console.print(_json.dumps(payload))
-        return
-
-    console.print(f"Daily usage: ${float(daily['usd']):.2f} | tokens {int(daily['tokens'])} | sandbox {float(daily['sandbox_seconds'])/3600.0:.2f}h")
-    console.print(f"Weekly usage: ${float(weekly['usd']):.2f} | tokens {int(weekly['tokens'])} | sandbox {float(weekly['sandbox_seconds'])/3600.0:.2f}h")
-    console.print(f"Monthly usage: ${float(monthly['usd']):.2f} | tokens {int(monthly['tokens'])} | sandbox {float(monthly['sandbox_seconds'])/3600.0:.2f}h")
-    if budget_usd is not None:
-        remaining = max(0.0, float(budget_usd) - float(weekly["usd"]))
-        console.print(f"Remaining (vs weekly budget ${float(budget_usd):.2f}): ${remaining:.2f}")
-