synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/streaming/types.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Any
+
+
+class StreamType(Enum):
+    """Categories of streaming payloads emitted by training jobs."""
+
+    STATUS = auto()
+    EVENTS = auto()
+    METRICS = auto()
+    TIMELINE = auto()
+
+    @property
+    def endpoint_path(self) -> str:
+        """Return the endpoint suffix used when polling this stream."""
+        return {
+            StreamType.STATUS: "",
+            StreamType.EVENTS: "/events",
+            StreamType.METRICS: "/metrics",
+            StreamType.TIMELINE: "/timeline",
+        }[self]
+
+
+@dataclass(slots=True)
+class StreamMessage:
+    """Unified representation of a streaming payload."""
+
+    stream_type: StreamType
+    timestamp: str
+    job_id: str
+    data: dict[str, Any]
+    seq: int | None = None
+    step: int | None = None
+    phase: str | None = None
+
+    @property
+    def key(self) -> str:
+        """Return a unique identifier used for deduplication."""
+        if self.stream_type is StreamType.EVENTS:
+            return f"event:{self.seq}"
+        if self.stream_type is StreamType.METRICS:
+            name = self.data.get("name", "")
+            if self.step is not None:
+                return f"metric:{name}:{self.step}"
+            ts = (
+                self.timestamp
+                or self.data.get("created_at")
+                or self.data.get("updated_at")
+                or self.data.get("timestamp")
+            )
+            if ts:
+                return f"metric:{name}:{ts}"
+            try:
+                fingerprint = json.dumps(self.data, sort_keys=True, default=str)
+            except Exception:
+                fingerprint = repr(self.data)
+            return f"metric:{name}:{fingerprint}"
+        if self.stream_type is StreamType.TIMELINE:
+            return f"timeline:{self.phase}:{self.timestamp}"
+        # Include status value in key so status changes are always shown
+        status = str(self.data.get("status") or self.data.get("state") or "")
+        return f"status:{status}:{self.timestamp}"
+
+    @classmethod
+    def from_status(cls, job_id: str, status_data: dict[str, Any]) -> StreamMessage:
+        """Create a message representing a job status payload."""
+        return cls(
+            stream_type=StreamType.STATUS,
+            timestamp=status_data.get("updated_at", "") or status_data.get("created_at", ""),
+            job_id=job_id,
+            data=status_data,
+        )
+
+    @classmethod
+    def from_event(cls, job_id: str, event_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a job event."""
+        return cls(
+            stream_type=StreamType.EVENTS,
+            timestamp=event_data.get("created_at", ""),
+            job_id=job_id,
+            data=event_data,
+            seq=event_data.get("seq"),
+        )
+
+    @classmethod
+    def from_metric(cls, job_id: str, metric_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a metric point."""
+        return cls(
+            stream_type=StreamType.METRICS,
+            timestamp=metric_data.get("created_at", ""),
+            job_id=job_id,
+            data=metric_data,
+            step=metric_data.get("step"),
+        )
+
+    @classmethod
+    def from_timeline(cls, job_id: str, timeline_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a status timeline entry."""
+        return cls(
+            stream_type=StreamType.TIMELINE,
+            timestamp=timeline_data.get("created_at", ""),
+            job_id=job_id,
+            data=timeline_data,
+            phase=timeline_data.get("phase"),
+        )
+
+
+__all__ = ["StreamMessage", "StreamType"]

synth_ai/sdk/task/__init__.py

@@ -0,0 +1,164 @@
+"""Task namespace (legacy).
+
+Prefer synth_ai.sdk.localapi.* moving forward. This module remains for backward
+compatibility during the naming transition.
+"""
+
+from .auth import (
+    is_api_key_header_authorized,
+    normalize_environment_api_key,
+    require_api_key_dependency,
+)
+from .client import LocalAPIClient, TaskAppClient
+from .config import EvalConfig, FilterConfig
+from .contracts import (
+    DatasetInfo,
+    InferenceInfo,
+    LimitsInfo,
+    LocalAPIEndpoints,
+    RolloutEnvSpec,
+    RolloutMetrics,
+    RolloutPolicySpec,
+    RolloutRecordConfig,
+    RolloutRequest,
+    RolloutResponse,
+    RolloutSafetyConfig,
+    RubricInfo,
+    RubricSection,
+    TaskAppEndpoints,
+    TaskDescriptor,
+    TaskInfo,
+)
+from .datasets import TaskDatasetRegistry, TaskDatasetSpec
+from .errors import error_payload, http_exception, json_error_response
+from .health import task_app_health
+from .in_process import InProcessTaskApp
+from .in_process_runner import (
+    InProcessJobResult,
+    merge_dot_overrides,
+    resolve_backend_api_base,
+    run_in_process_job,
+    run_in_process_job_sync,
+)
+from .inference_api import InferenceAPIClient
+from .json import to_jsonable
+from .proxy import (
+    extract_message_text,
+    inject_system_hint,
+    parse_tool_call_from_text,
+    prepare_for_groq,
+    prepare_for_openai,
+    synthesize_tool_call_if_missing,
+)
+from .rubrics import (
+    Criterion,
+    Rubric,
+    blend_rubrics,
+    load_rubric,
+    score_events_against_rubric,
+    score_outcome_against_rubric,
+)
+from .server import (
+    LocalAPIConfig,
+    ProxyConfig,
+    RubricBundle,
+    TaskAppConfig,
+    create_task_app,
+    run_server_background,
+    run_task_app,
+)
+from .trace_correlation_helpers import (
+    build_trace_payload,
+    build_trajectory_trace,
+    extract_trace_correlation_id,
+    include_event_history_in_response,
+    include_event_history_in_trajectories,
+    include_trace_correlation_id_in_response,
+    validate_trace_correlation_id,
+    verify_trace_correlation_id_in_response,
+)
+from .validators import (
+    normalize_inference_url,
+    validate_rollout_response_for_rl,
+    validate_task_app_endpoint,
+    validate_task_app_url,
+)
+from .vendors import (
+    get_groq_key_or_503,
+    get_openai_key_or_503,
+    normalize_vendor_keys,
+)
+
+__all__ = [
+    "normalize_inference_url",
+    "validate_rollout_response_for_rl",
+    "validate_task_app_url",
+    "validate_task_app_endpoint",
+    "task_app_health",
+    "EvalConfig",
+    "FilterConfig",
+    "TaskAppEndpoints",
+    "LocalAPIEndpoints",
+    "RolloutEnvSpec",
+    "RolloutPolicySpec",
+    "RolloutRecordConfig",
+    "RolloutSafetyConfig",
+    "RolloutRequest",
+    "RolloutResponse",
+    "RolloutMetrics",
+    "TaskDescriptor",
+    "DatasetInfo",
+    "RubricInfo",
+    "RubricSection",
+    "InferenceInfo",
+    "LimitsInfo",
+    "TaskInfo",
+    "to_jsonable",
+    "normalize_environment_api_key",
+    "is_api_key_header_authorized",
+    "require_api_key_dependency",
+    "normalize_vendor_keys",
+    "get_openai_key_or_503",
+    "get_groq_key_or_503",
+    "prepare_for_openai",
+    "prepare_for_groq",
+    "inject_system_hint",
+    "extract_message_text",
+    "parse_tool_call_from_text",
+    "synthesize_tool_call_if_missing",
+    "TaskDatasetSpec",
+    "TaskDatasetRegistry",
+    "Criterion",
+    "Rubric",
+    "load_rubric",
+    "blend_rubrics",
+    "score_events_against_rubric",
+    "score_outcome_against_rubric",
+    "TaskAppClient",
+    "LocalAPIClient",
+    "error_payload",
+    "http_exception",
+    "json_error_response",
+    "run_task_app",
+    "run_server_background",
+    "create_task_app",
+    "RubricBundle",
+    "ProxyConfig",
+    "TaskAppConfig",
+    "LocalAPIConfig",
+    "InferenceAPIClient",
+    "InProcessTaskApp",
+    "InProcessJobResult",
+    "merge_dot_overrides",
+    "resolve_backend_api_base",
+    "run_in_process_job",
+    "run_in_process_job_sync",
+    "build_trajectory_trace",
+    "build_trace_payload",
+    "extract_trace_correlation_id",
+    "include_event_history_in_response",
+    "include_event_history_in_trajectories",
+    "include_trace_correlation_id_in_response",
+    "validate_trace_correlation_id",
+    "verify_trace_correlation_id_in_response",
+]

synth_ai/sdk/task/apps/__init__.py

@@ -0,0 +1,169 @@
+"""Registry for Task Apps exposed via the shared FastAPI harness.
+
+Prefer synth_ai.sdk.localapi.apps moving forward. This module remains for
+backward compatibility during the naming transition.
+"""
+
+from __future__ import annotations
+
+import importlib
+import sys
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from ..server import TaskAppConfig
+
+
+@dataclass(slots=True)
+class ModalDeploymentConfig:
+    """Modal deployment defaults for a task app."""
+
+    app_name: str
+    python_version: str = "3.11"
+    pip_packages: Sequence[str] = field(default_factory=tuple)
+    apt_packages: Sequence[str] = field(default_factory=tuple)
+    extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    secret_names: Sequence[str] = field(default_factory=tuple)
+    volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    env_vars: dict[str, str] = field(default_factory=dict)
+    timeout: int = 600
+    memory: int = 4096
+    cpu: float = 2.0
+    min_containers: int = 1
+    max_containers: int = 4
+
+
+@dataclass(slots=True)
+class TaskAppEntry:
+    """Metadata describing a registered task app."""
+
+    app_id: str
+    description: str
+    config_factory: Callable[[], TaskAppConfig]
+    aliases: Sequence[str] = field(default_factory=tuple)
+    env_files: Sequence[str] = field(default_factory=tuple)
+    modal: ModalDeploymentConfig | None = None
+
+
+@dataclass(slots=True)
+class LocalAPIEntry:
+    """Metadata describing a registered local API."""
+
+    api_id: str
+    description: str
+    config_factory: Callable[[], TaskAppConfig]
+    aliases: Sequence[str] = field(default_factory=tuple)
+    env_files: Sequence[str] = field(default_factory=tuple)
+    modal: ModalDeploymentConfig | None = None
+
+    @property
+    def app_id(self) -> str:
+        return self.api_id
+
+
+class TaskAppRegistry:
+    """In-memory registry of known task apps."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, TaskAppEntry] = {}
+        self._alias_to_id: dict[str, str] = {}
+
+    def register(self, entry: TaskAppEntry) -> None:
+        if entry.app_id in self._entries:
+            # Allow idempotent registration when modules are imported multiple times.
+            return
+        self._entries[entry.app_id] = entry
+        for alias in entry.aliases:
+            existing = self._alias_to_id.get(alias)
+            if existing and existing != entry.app_id:
+                raise ValueError(f"Alias already registered: {alias}")
+            self._alias_to_id[alias] = entry.app_id
+
+    def get(self, app_id: str) -> TaskAppEntry:
+        resolved = self._alias_to_id.get(app_id, app_id)
+        if resolved not in self._entries:
+            raise KeyError(f"Unknown task app id: {app_id}")
+        return self._entries[resolved]
+
+    def list(self) -> List[TaskAppEntry]:
+        return sorted(self._entries.values(), key=lambda entry: entry.app_id)
+
+    def __iter__(self) -> Iterable[TaskAppEntry]:
+        return iter(self.list())
+
+    def clear(self) -> None:
+        """Clear all registered task apps."""
+        self._entries.clear()
+        self._alias_to_id.clear()
+
+
+registry = TaskAppRegistry()
+
+
+def register_task_app(*, entry: TaskAppEntry) -> None:
+    registry.register(entry)
+
+
+def register_local_api(*, entry: LocalAPIEntry | TaskAppEntry) -> None:
+    if isinstance(entry, LocalAPIEntry):
+        registry.register(
+            TaskAppEntry(
+                app_id=entry.api_id,
+                description=entry.description,
+                config_factory=entry.config_factory,
+                aliases=entry.aliases,
+                env_files=entry.env_files,
+                modal=entry.modal,
+            )
+        )
+        return
+    registry.register(entry)
+
+
+def discover_task_apps_from_cwd() -> None:
+    """Discover and register task apps from the current working directory and subdirectories."""
+    cwd = Path.cwd()
+
+    # Look for task app files in common patterns
+    patterns = [
+        "**/task_app/*.py",
+        "**/task_apps/*.py",
+        "**/*_task_app.py",
+        "**/grpo_crafter.py",
+        "**/math_single_step.py",
+    ]
+
+    discovered_files = []
+    for pattern in patterns:
+        discovered_files.extend(cwd.glob(pattern))
+
+    # Add current directory to Python path temporarily
+    original_path = sys.path.copy()
+    try:
+        sys.path.insert(0, str(cwd))
+
+        for file_path in discovered_files:
+            if file_path.name.startswith("__"):
+                continue
+
+            # Convert file path to module name
+            relative_path = file_path.relative_to(cwd)
+            module_parts = list(relative_path.parts[:-1]) + [relative_path.stem]
+            module_name = ".".join(module_parts)
+
+            try:
+                # Import the module to trigger registration
+                importlib.import_module(module_name)
+            except Exception:
+                # Silently skip modules that can't be imported
+                # This allows for graceful handling of missing dependencies
+                continue
+
+    finally:
+        sys.path[:] = original_path
+
+
+# Note: Task apps are now discovered dynamically by the CLI, not auto-registered
+# This allows for better separation between SDK and example-specific implementations

synth_ai/sdk/task/client.py

@@ -0,0 +1,175 @@
+"""Async HTTP client for interacting with Task Apps.
+
+Prefer synth_ai.sdk.localapi.client moving forward. This module remains for
+backward compatibility during the naming transition.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from typing import Any
+
+import httpx
+from pydantic import BaseModel
+
+from .contracts import RolloutRequest, RolloutResponse, TaskInfo
+from .json import to_jsonable
+
+
+def _prepare_payload(payload: Any) -> Any:
+    if payload is None:
+        return None
+    if isinstance(payload, BaseModel):
+        return payload.model_dump(mode="json", by_alias=True)
+    return to_jsonable(payload)
+
+
+class TaskAppClient:
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str | None = None,
+        *,
+        timeout: float = 600.0,
+        retries: int = 3,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout = timeout
+        self.retries = max(1, retries)
+        self._client: httpx.AsyncClient | None = None
+        self.env = _TaskAppEnvironmentClient(self)
+
+    async def __aenter__(self) -> TaskAppClient:
+        await self._ensure_client()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.aclose()
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                base_url=self.base_url,
+                timeout=httpx.Timeout(self.timeout),
+                follow_redirects=True,
+            )
+        return self._client
+
+    def _headers(self) -> dict[str, str]:
+        headers: dict[str, str] = {}
+        # Primary key
+        primary = (self.api_key or "").strip()
+        if primary:
+            headers["X-API-Key"] = primary
+            # Also set Authorization for clients that read bearer tokens
+            headers.setdefault("Authorization", f"Bearer {primary}")
+        # Include ALL available environment keys via CSV in X-API-Keys
+        keys: list[str] = []
+        if primary:
+            keys.append(primary)
+        aliases = (os.getenv("ENVIRONMENT_API_KEY_ALIASES") or "").strip()
+        if aliases:
+            for part in aliases.split(","):
+                trimmed = part.strip()
+                if trimmed and trimmed not in keys:
+                    keys.append(trimmed)
+        if keys:
+            headers["X-API-Keys"] = ",".join(keys)
+        return headers
+
+    async def aclose(self) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    async def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, Any] | list[tuple[str, Any]] | None = None,
+        json_payload: Any = None,
+    ) -> httpx.Response:
+        client = await self._ensure_client()
+        payload = _prepare_payload(json_payload)
+        headers = self._headers()
+        last_exc: Exception | None = None
+        for attempt in range(self.retries):
+            try:
+                response = await client.request(
+                    method,
+                    path,
+                    headers=headers,
+                    params=params,
+                    json=payload,
+                )
+                response.raise_for_status()
+                return response
+            except httpx.HTTPStatusError as exc:
+                if 500 <= exc.response.status_code < 600 and attempt + 1 < self.retries:
+                    await asyncio.sleep(0.1 * (attempt + 1))
+                    last_exc = exc
+                    continue
+                raise
+            except httpx.HTTPError as exc:
+                last_exc = exc
+                if attempt + 1 >= self.retries:
+                    raise
+                await asyncio.sleep(0.1 * (attempt + 1))
+        if last_exc:  # pragma: no cover - defensive
+            raise last_exc
+        raise RuntimeError("Unreachable code in TaskAppClient._request")
+
+    async def health(self) -> dict[str, Any]:
+        response = await self._request("GET", "/health")
+        return response.json()
+
+    async def info(self) -> dict[str, Any]:
+        response = await self._request("GET", "/info")
+        return response.json()
+
+    async def task_info(self, seeds: list[int] | None = None) -> TaskInfo | list[TaskInfo]:
+        params: list[tuple[str, Any]] | None = None
+        if seeds:
+            params = [("seed", seed) for seed in seeds]
+        response = await self._request("GET", "/task_info", params=params)
+        data = response.json()
+        if isinstance(data, list):
+            return [TaskInfo.model_validate(item) for item in data]
+        return TaskInfo.model_validate(data)
+
+    async def rollout(self, request: RolloutRequest) -> RolloutResponse:
+        response = await self._request("POST", "/rollout", json_payload=request)
+        data = response.json()
+        return RolloutResponse.model_validate(data)
+
+
+class LocalAPIClient(TaskAppClient):
+    """Alias for TaskAppClient with LocalAPI naming."""
+
+
+class _TaskAppEnvironmentClient:
+    def __init__(self, client: TaskAppClient) -> None:
+        self._client = client
+
+    async def initialize(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
+        response = await self._client._request(
+            "POST", f"/env/{env_name}/initialize", json_payload=payload
+        )
+        return response.json()
+
+    async def step(self, env_name: str, payload: dict[str, Any]) -> dict[str, Any]:
+        response = await self._client._request(
+            "POST", f"/env/{env_name}/step", json_payload=payload
+        )
+        return response.json()
+
+    async def terminate(
+        self, env_name: str, payload: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
+        response = await self._client._request(
+            "POST", f"/env/{env_name}/terminate", json_payload=payload or {}
+        )
+        return response.json()