synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/cli/commands/eval/validation.py

@@ -1,133 +1,60 @@
-from __future__ import annotations
-
-import re
-from collections.abc import MutableMapping
-from typing import Any
-
-__all__ = ["validate_eval_options"]
-
-_SEED_RANGE = re.compile(r"^\s*(-?\d+)\s*-\s*(-?\d+)\s*$")
+"""Validation helpers for eval options."""
 
-
-def _coerce_bool(value: Any) -> bool:
-    if isinstance(value, str):
-        return value.strip().lower() in {"1", "true", "yes", "on"}
-    return bool(value)
-
-
-def _coerce_int(value: Any) -> int | None:
-    if value is None or value == "":
-        return None
-    return int(value)
+from __future__ import annotations
 
 
-def _parse_seeds(value: Any) -> list[int]:
-    if value is None:
+def _parse_seeds(value: str | list[int]) -> list[int]:
+    if isinstance(value, list):
+        return value
+    if not value:
         return []
-    if isinstance(value, str):
-        chunks = [chunk.strip() for chunk in value.split(",") if chunk.strip()]
-    elif isinstance(value, list | tuple | set):
-        chunks = list(value)
-    else:
-        chunks = [value]
-    seeds: list[int] = []
-    for chunk in chunks:
-        if isinstance(chunk, int):
-            seeds.append(chunk)
-        else:
-            text = str(chunk).strip()
-            if not text:
-                continue
-            match = _SEED_RANGE.match(text)
-            if match:
-                start = int(match.group(1))
-                end = int(match.group(2))
-                if start > end:
-                    raise ValueError(f"Invalid seed range '{text}': start must be <= end")
-                seeds.extend(range(start, end + 1))
-            else:
-                seeds.append(int(text))
-    return seeds
-
-
-def _normalize_metadata(value: Any) -> dict[str, str]:
-    if value is None:
-        return {}
-    if isinstance(value, MutableMapping):
-        return {str(k): str(v) for k, v in value.items()}
-    if isinstance(value, list | tuple):
-        result: dict[str, str] = {}
-        for item in value:
-            if isinstance(item, str) and "=" in item:
-                key, val = item.split("=", 1)
-                result[key.strip()] = val.strip()
-        return result
-    if isinstance(value, str) and "=" in value:
-        key, val = value.split("=", 1)
-        return {key.strip(): val.strip()}
-    return {}
+    parts = [part.strip() for part in value.split(",") if part.strip()]
+    return [int(part) for part in parts]
 
 
-def _ensure_list(value: Any) -> list[str] | None:
-    if value is None:
-        return None
-    if isinstance(value, list | tuple | set):
-        return [str(item) for item in value]
-    return [str(value)]
-
-
-def _ensure_dict(value: Any) -> dict[str, Any]:
-    if isinstance(value, MutableMapping):
-        return dict(value)
-    return {}
-
-
-def validate_eval_options(options: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
-    """Validate and normalise eval configuration options."""
-
-    result: dict[str, Any] = dict(options)
-
-    if "seeds" in result:
-        result["seeds"] = _parse_seeds(result.get("seeds"))
-
-    for field in ("max_turns", "max_llm_calls", "concurrency"):
-        try:
-            result[field] = _coerce_int(result.get(field))
-        except Exception as exc:
-            raise ValueError(f"Invalid value for {field}: {result.get(field)}") from exc
+def _parse_metadata(values: list[str]) -> dict[str, str]:
+    if not values:
+        return {}
+    parsed: dict[str, str] = {}
+    for entry in values:
+        if "=" not in entry:
+            raise ValueError("Metadata filter must be key=value")
+        key, value = entry.split("=", 1)
+        parsed[key.strip()] = value.strip()
+    return parsed
+
+
+def _parse_ops(value: str | list[str]) -> list[str]:
+    if isinstance(value, list):
+        return [str(item).strip() for item in value if str(item).strip()]
+    if not value:
+        return []
+    return [part.strip() for part in value.split(",") if part.strip()]
 
-    if result.get("max_llm_calls") is None:
-        result["max_llm_calls"] = 10
-    if result.get("concurrency") is None:
-        result["concurrency"] = 1
 
-    if "return_trace" in result:
-        result["return_trace"] = _coerce_bool(result.get("return_trace"))
+def validate_eval_options(options: dict[str, object]) -> dict[str, object]:
+    normalized = dict(options)
+    seeds = normalized.get("seeds") or ""
+    normalized["seeds"] = _parse_seeds(seeds)  # type: ignore[arg-type]
 
-    metadata_value = result.get("metadata")
-    result["metadata"] = _normalize_metadata(metadata_value)
+    metadata = normalized.get("metadata") or []
+    normalized["metadata"] = _parse_metadata(metadata)  # type: ignore[arg-type]
 
-    if "ops" in result:
-        ops_list = _ensure_list(result.get("ops"))
-        result["ops"] = ops_list
+    for key in ("max_turns", "max_llm_calls", "concurrency"):
+        if key in normalized and normalized[key] not in (None, ""):
+            normalized[key] = int(normalized[key])  # type: ignore[arg-type]
 
-    result["env_config"] = _ensure_dict(result.get("env_config"))
-    result["policy_config"] = _ensure_dict(result.get("policy_config"))
+    if "ops" in normalized:
+        normalized["ops"] = _parse_ops(normalized.get("ops") or "")  # type: ignore[arg-type]
 
-    trace_format = result.get("trace_format")
-    if trace_format is not None:
-        result["trace_format"] = str(trace_format)
+    if "poll" in normalized and normalized["poll"] not in (None, ""):
+        normalized["poll"] = float(normalized["poll"])  # type: ignore[arg-type]
 
-    metadata_sql = result.get("metadata_sql")
-    if metadata_sql is not None and not isinstance(metadata_sql, str):
-        result["metadata_sql"] = str(metadata_sql)
+    if "return_trace" in normalized:
+        value = str(normalized["return_trace"]).lower()
+        normalized["return_trace"] = value in ("true", "1", "yes")
 
-    model = result.get("model")
-    if model is not None:
-        result["model"] = str(model)
+    return normalized
 
-    app_id = result.get("app_id")
-    if app_id is not None:
-        result["app_id"] = str(app_id)
 
-    return result
+__all__ = ["validate_eval_options"]

synth_ai/cli/commands/filter/core.py

@@ -239,8 +239,8 @@ def filter_command(config_path: str) -> None:
     models = set(filter_cfg.models)
     min_official = filter_cfg.min_official_score
     max_official = filter_cfg.max_official_score
-    min_judge_scores = filter_cfg.min_judge_scores
-    max_judge_scores = filter_cfg.max_judge_scores
+    min_verifier_scores = filter_cfg.min_verifier_scores
+    max_verifier_scores = filter_cfg.max_verifier_scores
     min_created = _parse_datetime_for_trace(raw_cfg.get("min_created_at"))
     max_created = _parse_datetime_for_trace(raw_cfg.get("max_created_at"))
     limit = filter_cfg.limit
@@ -311,16 +311,16 @@ def filter_command(config_path: str) -> None:
                 elif min_official is not None:
                     continue
 
-            judge_scores = metadata.get("judge_scores") or {}
+            verifier_scores = metadata.get("verifier_scores") or {}
             include = True
-            for judge_name, threshold in (min_judge_scores or {}).items():
-                if not _score_ok(judge_scores.get(judge_name), threshold, None):
+            for verifier_name, threshold in (min_verifier_scores or {}).items():
+                if not _score_ok(verifier_scores.get(verifier_name), threshold, None):
                     include = False
                     break
             if not include:
                 continue
-            for judge_name, threshold in (max_judge_scores or {}).items():
-                if not _score_ok(judge_scores.get(judge_name), None, threshold):
+            for verifier_name, threshold in (max_verifier_scores or {}).items():
+                if not _score_ok(verifier_scores.get(verifier_name), None, threshold):
                     include = False
                     break
             if not include:

synth_ai/cli/commands/filter/validation.py

@@ -44,8 +44,8 @@ def validate_filter_options(options: MutableMapping[str, Any]) -> MutableMapping
     _coerce_list("splits")
     _coerce_list("task_ids")
     _coerce_list("models")
-    _coerce_dict("min_judge_scores")
-    _coerce_dict("max_judge_scores")
+    _coerce_dict("min_verifier_scores")
+    _coerce_dict("max_verifier_scores")
 
     for duration_key in ("min_official_score", "max_official_score"):
         value = result.get(duration_key)

synth_ai/cli/commands/smoke/core.py

@@ -18,14 +18,13 @@ import httpx
 
 from synth_ai.core.tracing_v3.config import resolve_trace_db_settings
 from synth_ai.core.tracing_v3.turso.daemon import start_sqld
-from synth_ai.sdk.task.client import TaskAppClient
+from synth_ai.sdk.localapi.client import LocalAPIClient
 from synth_ai.sdk.task.contracts import (
     RolloutEnvSpec,
     RolloutMode,
     RolloutPolicySpec,
     RolloutRecordConfig,
     RolloutRequest,
-    RolloutSafetyConfig,
 )
 from synth_ai.sdk.task.validators import (
     normalize_inference_url,
@@ -773,7 +772,7 @@ async def _run_smoke_async(
     mock_backend = (mock_backend or "synthetic").strip().lower()
 
     # Discover environment if not provided
-    async with TaskAppClient(base_url=base, api_key=api_key) as client:
+    async with LocalAPIClient(base_url=base, api_key=api_key) as client:
         # Probe basic info quickly
         try:
             _ = await client.health()
@@ -795,12 +794,6 @@ async def _run_smoke_async(
             click.echo("Could not infer environment name; pass --env-name.", err=True)
             return 2
 
-        # Build ops: alternating agent/env for max_steps
-        ops: list[str] = []
-        for _ in range(max_steps):
-            ops.append("agent")
-            ops.append("env")
-
         # Inference URL: user override > preset > local mock > Synth API default
         synth_base = (os.getenv("SYNTH_API_BASE") or os.getenv("SYNTH_BASE_URL") or "https://api.synth.run").rstrip("/")
         # Avoid double '/api' if base already includes it
@@ -852,7 +845,6 @@ async def _run_smoke_async(
                         run_id=run_id,
                         env=RolloutEnvSpec(env_name=env_name, config={}, seed=i),
                         policy=RolloutPolicySpec(policy_name=policy_name, config=policy_cfg),
-                        ops=ops,
                         record=RolloutRecordConfig(
                             trajectories=True,
                             logprobs=False,
@@ -861,7 +853,6 @@ async def _run_smoke_async(
                             trace_format=("structured" if return_trace else "compact"),
                         ),
                         on_done="reset",
-                        safety=RolloutSafetyConfig(max_ops=max_steps * 4, max_time_s=900.0),
                         training_session_id=None,
                         synth_base_url=synth_base,
                         mode=RolloutMode.RL,
@@ -869,7 +860,6 @@ async def _run_smoke_async(
 
                     try:
                         click.echo(f">> POST /rollout run_id={run_id} env={env_name} policy={policy_name} url={inference_url_with_cid}")
-                        click.echo(f"   ops={ops[:10]}{'...' if len(ops) > 10 else ''}")
                         response = await client.rollout(request)
                     except Exception as exc:
                         click.echo(f"Rollout[{i}:{g}] failed: {type(exc).__name__}: {exc}", err=True)
@@ -888,10 +878,10 @@ async def _run_smoke_async(
                     metrics = response.metrics
                     if inferred_url:
                         click.echo(f"  rollout[{i}:{g}] inference_url: {inferred_url}")
-                    click.echo(f"  rollout[{i}:{g}] episodes={metrics.num_episodes} steps={metrics.num_steps} mean_return={metrics.mean_return:.4f}")
+                    click.echo(f"  rollout[{i}:{g}] episodes={metrics.num_episodes} steps={metrics.num_steps} reward_mean={metrics.reward_mean:.4f}")
 
                     total_steps += int(metrics.num_steps)
-                    if (metrics.mean_return or 0.0) != 0.0:
+                    if (metrics.reward_mean or 0.0) != 0.0:
                         nonzero_returns += 1
                     if response.trace is not None and isinstance(response.trace, dict):
                         v3_traces += 1
@@ -916,8 +906,8 @@ async def _run_smoke_async(
                             metrics_dump = response.metrics.model_dump()
                         except Exception:
                             metrics_dump = {
-                                "episode_returns": getattr(response.metrics, "episode_returns", None),
-                                "mean_return": getattr(response.metrics, "mean_return", None),
+                                "episode_rewards": getattr(response.metrics, "episode_rewards", None),
+                                "reward_mean": getattr(response.metrics, "reward_mean", None),
                                 "num_steps": getattr(response.metrics, "num_steps", None),
                                 "num_episodes": getattr(response.metrics, "num_episodes", None),
                                 "outcome_score": getattr(response.metrics, "outcome_score", None),
@@ -1016,7 +1006,7 @@ async def _run_smoke_async(
             if v3_traces < successes:
                 click.echo("  ⚠ Some rollouts missing v3 traces (trace field)", err=True)
             if total_steps == 0:
-                click.echo("  ⚠ No steps executed; check ops/policy config", err=True)
+                click.echo("  ⚠ No steps executed; check policy config", err=True)
 
             return 0
 

synth_ai/cli/commands/status/__init__.py

@@ -1,66 +1,3 @@
-"""Status and listing commands for the Synth CLI."""
+"""Status command package."""
 
 from __future__ import annotations
-
-import click
-
-from .config import resolve_backend_config
-from .subcommands.files import files_group
-from .subcommands.jobs import jobs_group
-from .subcommands.models import models_group
-from .subcommands.runs import runs_group
-from .subcommands.session import session_status_cmd
-from .subcommands.summary import summary_command
-
-
-def _attach_group(cli: click.Group, group: click.Group, name: str) -> None:
-    """Attach the provided Click group to the CLI if not already present."""
-    if name in cli.commands:
-        return
-    cli.add_command(group, name=name)
-
-
-def register(cli: click.Group) -> None:
-    """Register all status command groups on the provided CLI root."""
-
-    @click.group(help="Inspect training jobs, models, files, and job runs.")
-    @click.option(
-        "--base-url",
-        envvar="SYNTH_STATUS_BASE_URL",
-        default=None,
-        help="Synth backend base URL (defaults to environment configuration).",
-    )
-    @click.option(
-        "--api-key",
-        envvar="SYNTH_STATUS_API_KEY",
-        default=None,
-        help="API key for authenticated requests (falls back to Synth defaults).",
-    )
-    @click.option(
-        "--timeout",
-        default=30.0,
-        show_default=True,
-        type=float,
-        help="HTTP request timeout in seconds.",
-    )
-    @click.pass_context
-    def status(ctx: click.Context, base_url: str | None, api_key: str | None, timeout: float) -> None:
-        """Populate shared backend configuration for subcommands."""
-        cfg = resolve_backend_config(base_url=base_url, api_key=api_key, timeout=timeout)
-        ctx.ensure_object(dict)
-        ctx.obj["status_backend_config"] = cfg
-
-    status.add_command(jobs_group, name="jobs")
-    status.add_command(models_group, name="models")
-    status.add_command(files_group, name="files")
-    status.add_command(runs_group, name="runs")
-    status.add_command(session_status_cmd, name="session")
-    status.add_command(summary_command, name="summary")
-
-    cli.add_command(status, name="status")
-    _attach_group(cli, jobs_group, "jobs")
-    _attach_group(cli, models_group, "models")
-    _attach_group(cli, files_group, "files")
-    _attach_group(cli, runs_group, "runs")
-    if "status-summary" not in cli.commands:
-        cli.add_command(summary_command, name="status-summary")

synth_ai/cli/commands/status/client.py

@@ -1,4 +1,4 @@
-"""Async HTTP client for Synth status and listing endpoints."""
+"""HTTP client for status commands."""
 
 from __future__ import annotations
 
@@ -8,31 +8,43 @@ import httpx
 
 from .config import BackendConfig
 from .errors import StatusAPIError
+from .utils import build_headers
 
 
 class StatusAPIClient:
-    """Thin wrapper around httpx.AsyncClient with convenience methods."""
-
     def __init__(self, config: BackendConfig) -> None:
         self._config = config
-        timeout = httpx.Timeout(config.timeout)
-        self._client = httpx.AsyncClient(
-            base_url=config.base_url,
-            headers=config.headers,
-            timeout=timeout,
-        )
-
-    async def __aenter__(self) -> StatusAPIClient:
-        await self._client.__aenter__()
+        self._client: httpx.AsyncClient | None = None
+
+    async def __aenter__(self) -> "StatusAPIClient":
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                base_url=self._config.base_url,
+                headers=build_headers(self._config.api_key),
+                timeout=self._config.timeout,
+            )
         return self
 
-    async def __aexit__(self, *args: Any) -> None:
-        await self._client.__aexit__(*args)
-
-    async def close(self) -> None:
-        await self._client.aclose()
-
-    # Jobs -----------------------------------------------------------------
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    async def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
+        assert self._client is not None
+        resp = await self._client.get(path, params=params)
+        if resp.status_code >= 400:
+            detail = resp.json().get("detail", "")
+            raise StatusAPIError(detail or "Request failed", status_code=resp.status_code)
+        return resp.json()
+
+    async def _post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        assert self._client is not None
+        resp = await self._client.post(path, json=payload or {})
+        if resp.status_code >= 400:
+            detail = resp.json().get("detail", "")
+            raise StatusAPIError(detail or "Request failed", status_code=resp.status_code)
+        return resp.json()
 
     async def list_jobs(
         self,
@@ -43,94 +55,22 @@ class StatusAPIClient:
         limit: int | None = None,
     ) -> list[dict[str, Any]]:
         params: dict[str, Any] = {}
-        if status:
+        if status is not None:
             params["status"] = status
-        if job_type:
+        if job_type is not None:
             params["type"] = job_type
-        if created_after:
+        if created_after is not None:
             params["created_after"] = created_after
-        if limit:
+        if limit is not None:
             params["limit"] = limit
-        resp = await self._client.get("/learning/jobs", params=params)
-        return self._json_list(resp, key="jobs")
+        payload = await self._get("/learning/jobs", params=params or None)
+        return payload.get("jobs", [])
 
     async def get_job(self, job_id: str) -> dict[str, Any]:
-        resp = await self._client.get(f"/learning/jobs/{job_id}")
-        return self._json(resp)
-
-    async def get_job_status(self, job_id: str) -> dict[str, Any]:
-        resp = await self._client.get(f"/learning/jobs/{job_id}/status")
-        return self._json(resp)
+        return await self._get(f"/learning/jobs/{job_id}")
 
     async def cancel_job(self, job_id: str) -> dict[str, Any]:
-        resp = await self._client.post(f"/learning/jobs/{job_id}/cancel")
-        return self._json(resp)
-
-    async def get_job_config(self, job_id: str) -> dict[str, Any]:
-        resp = await self._client.get(f"/learning/jobs/{job_id}/config")
-        return self._json(resp)
-
-    async def get_job_metrics(self, job_id: str) -> dict[str, Any]:
-        resp = await self._client.get(f"/learning/jobs/{job_id}/metrics")
-        return self._json(resp)
-
-    async def get_job_timeline(self, job_id: str) -> list[dict[str, Any]]:
-        resp = await self._client.get(f"/learning/jobs/{job_id}/timeline")
-        return self._json_list(resp, key="timeline")
-
-    async def list_job_runs(self, job_id: str) -> list[dict[str, Any]]:
-        resp = await self._client.get(f"/jobs/{job_id}/runs")
-        return self._json_list(resp, key="runs")
-
-    async def get_job_events(
-        self,
-        job_id: str,
-        *,
-        since: str | None = None,
-        limit: int | None = None,
-        after: str | None = None,
-        run_id: str | None = None,
-    ) -> list[dict[str, Any]]:
-        params: dict[str, Any] = {}
-        if since:
-            params["since"] = since
-        if limit:
-            params["limit"] = limit
-        if after:
-            params["after"] = after
-        if run_id:
-            params["run"] = run_id
-        resp = await self._client.get(f"/learning/jobs/{job_id}/events", params=params)
-        return self._json_list(resp, key="events")
-
-    # Files ----------------------------------------------------------------
-
-    async def list_files(
-        self,
-        *,
-        purpose: str | None = None,
-        limit: int | None = None,
-    ) -> list[dict[str, Any]]:
-        params: dict[str, Any] = {}
-        if purpose:
-            params["purpose"] = purpose
-        if limit:
-            params["limit"] = limit
-        resp = await self._client.get("/files", params=params)
-        data = self._json(resp)
-        if isinstance(data, dict):
-            for key in ("files", "data", "items"):
-                if isinstance(data.get(key), list):
-                    return list(data[key])
-        if isinstance(data, list):
-            return list(data)
-        return []
-
-    async def get_file(self, file_id: str) -> dict[str, Any]:
-        resp = await self._client.get(f"/files/{file_id}")
-        return self._json(resp)
-
-    # Models ---------------------------------------------------------------
+        return await self._post(f"/learning/jobs/{job_id}/cancel")
 
     async def list_models(
         self,
@@ -138,55 +78,14 @@ class StatusAPIClient:
         limit: int | None = None,
         model_type: str | None = None,
     ) -> list[dict[str, Any]]:
-        params: dict[str, Any] = {}
-        if limit:
-            params["limit"] = limit
-        endpoint = "/learning/models/rl" if model_type == "rl" else "/learning/models"
-        resp = await self._client.get(endpoint, params=params)
-        return self._json_list(resp, key="models")
-
-    async def get_model(self, model_id: str) -> dict[str, Any]:
-        resp = await self._client.get(f"/learning/models/{model_id}")
-        return self._json(resp)
-
-    # Helpers --------------------------------------------------------------
-
-    def _json(self, response: httpx.Response) -> dict[str, Any]:
-        try:
-            response.raise_for_status()
-        except httpx.HTTPStatusError as exc:
-            detail = self._extract_detail(exc.response)
-            raise StatusAPIError(detail, exc.response.status_code if exc.response else None) from exc
-        try:
-            data = response.json()
-        except ValueError as exc:
-            raise StatusAPIError("Backend response was not valid JSON") from exc
-        if isinstance(data, dict):
-            return data
-        return {"data": data}
-
-    def _json_list(self, response: httpx.Response, *, key: str | None = None) -> list[dict[str, Any]]:
-        payload = self._json(response)
-        if key and isinstance(payload.get(key), list):
-            return list(payload[key])
-        if isinstance(payload.get("data"), list):
-            return list(payload["data"])
-        if isinstance(payload.get("results"), list):
-            return list(payload["results"])
-        if isinstance(payload, list):
-            return list(payload)
-        return []
-
-    @staticmethod
-    def _extract_detail(response: httpx.Response | None) -> str:
-        if response is None:
-            return "Backend request failed"
-        try:
-            data = response.json()
-            if isinstance(data, dict):
-                for key in ("detail", "message", "error"):
-                    if data.get(key):
-                        return str(data[key])
-            return response.text
-        except ValueError:
-            return response.text
+        if model_type:
+            payload = await self._get(f"/learning/models/{model_type}")
+            return payload.get("models", [])
+        params = {"limit": limit} if limit is not None else None
+        payload = await self._get("/learning/models", params=params)
+        return payload.get("models", [])
+
+    async def list_job_runs(self, job_id: str) -> list[dict[str, Any]]:
+        payload = await self._get(f"/jobs/{job_id}/runs")
+        return payload.get("runs", [])
+