synth-ai 0.2.6.dev1__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/sdk/streaming/types.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Any
+
+
+class StreamType(Enum):
+    """Categories of streaming payloads emitted by training jobs."""
+
+    STATUS = auto()
+    EVENTS = auto()
+    METRICS = auto()
+    TIMELINE = auto()
+
+    @property
+    def endpoint_path(self) -> str:
+        """Return the endpoint suffix used when polling this stream."""
+        return {
+            StreamType.STATUS: "",
+            StreamType.EVENTS: "/events",
+            StreamType.METRICS: "/metrics",
+            StreamType.TIMELINE: "/timeline",
+        }[self]
+
+
+@dataclass(slots=True)
+class StreamMessage:
+    """Unified representation of a streaming payload."""
+
+    stream_type: StreamType
+    timestamp: str
+    job_id: str
+    data: dict[str, Any]
+    seq: int | None = None
+    step: int | None = None
+    phase: str | None = None
+
+    @property
+    def key(self) -> str:
+        """Return a unique identifier used for deduplication."""
+        if self.stream_type is StreamType.EVENTS:
+            return f"event:{self.seq}"
+        if self.stream_type is StreamType.METRICS:
+            name = self.data.get("name", "")
+            if self.step is not None:
+                return f"metric:{name}:{self.step}"
+            ts = (
+                self.timestamp
+                or self.data.get("created_at")
+                or self.data.get("updated_at")
+                or self.data.get("timestamp")
+            )
+            if ts:
+                return f"metric:{name}:{ts}"
+            try:
+                fingerprint = json.dumps(self.data, sort_keys=True, default=str)
+            except Exception:
+                fingerprint = repr(self.data)
+            return f"metric:{name}:{fingerprint}"
+        if self.stream_type is StreamType.TIMELINE:
+            return f"timeline:{self.phase}:{self.timestamp}"
+        # Include status value in key so status changes are always shown
+        status = str(self.data.get("status") or self.data.get("state") or "")
+        return f"status:{status}:{self.timestamp}"
+
+    @classmethod
+    def from_status(cls, job_id: str, status_data: dict[str, Any]) -> StreamMessage:
+        """Create a message representing a job status payload."""
+        return cls(
+            stream_type=StreamType.STATUS,
+            timestamp=status_data.get("updated_at", "") or status_data.get("created_at", ""),
+            job_id=job_id,
+            data=status_data,
+        )
+
+    @classmethod
+    def from_event(cls, job_id: str, event_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a job event."""
+        return cls(
+            stream_type=StreamType.EVENTS,
+            timestamp=event_data.get("created_at", ""),
+            job_id=job_id,
+            data=event_data,
+            seq=event_data.get("seq"),
+        )
+
+    @classmethod
+    def from_metric(cls, job_id: str, metric_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a metric point."""
+        return cls(
+            stream_type=StreamType.METRICS,
+            timestamp=metric_data.get("created_at", ""),
+            job_id=job_id,
+            data=metric_data,
+            step=metric_data.get("step"),
+        )
+
+    @classmethod
+    def from_timeline(cls, job_id: str, timeline_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a status timeline entry."""
+        return cls(
+            stream_type=StreamType.TIMELINE,
+            timestamp=timeline_data.get("created_at", ""),
+            job_id=job_id,
+            data=timeline_data,
+            phase=timeline_data.get("phase"),
+        )
+
+
+__all__ = ["StreamMessage", "StreamType"]

synth_ai/sdk/task/__init__.py

@@ -0,0 +1,164 @@
+"""Task namespace (legacy).
+
+Prefer synth_ai.sdk.localapi.* moving forward. This module remains for backward
+compatibility during the naming transition.
+"""
+
+from .auth import (
+    is_api_key_header_authorized,
+    normalize_environment_api_key,
+    require_api_key_dependency,
+)
+from .client import LocalAPIClient, TaskAppClient
+from .config import EvalConfig, FilterConfig
+from .contracts import (
+    DatasetInfo,
+    InferenceInfo,
+    LimitsInfo,
+    LocalAPIEndpoints,
+    RolloutEnvSpec,
+    RolloutMetrics,
+    RolloutPolicySpec,
+    RolloutRecordConfig,
+    RolloutRequest,
+    RolloutResponse,
+    RolloutSafetyConfig,
+    RubricInfo,
+    RubricSection,
+    TaskAppEndpoints,
+    TaskDescriptor,
+    TaskInfo,
+)
+from .datasets import TaskDatasetRegistry, TaskDatasetSpec
+from .errors import error_payload, http_exception, json_error_response
+from .health import task_app_health
+from .in_process import InProcessTaskApp
+from .in_process_runner import (
+    InProcessJobResult,
+    merge_dot_overrides,
+    resolve_backend_api_base,
+    run_in_process_job,
+    run_in_process_job_sync,
+)
+from .inference_api import InferenceAPIClient
+from .json import to_jsonable
+from .proxy import (
+    extract_message_text,
+    inject_system_hint,
+    parse_tool_call_from_text,
+    prepare_for_groq,
+    prepare_for_openai,
+    synthesize_tool_call_if_missing,
+)
+from .rubrics import (
+    Criterion,
+    Rubric,
+    blend_rubrics,
+    load_rubric,
+    score_events_against_rubric,
+    score_outcome_against_rubric,
+)
+from .server import (
+    LocalAPIConfig,
+    ProxyConfig,
+    RubricBundle,
+    TaskAppConfig,
+    create_task_app,
+    run_server_background,
+    run_task_app,
+)
+from .trace_correlation_helpers import (
+    build_trace_payload,
+    build_trajectory_trace,
+    extract_trace_correlation_id,
+    include_event_history_in_response,
+    include_event_history_in_trajectories,
+    include_trace_correlation_id_in_response,
+    validate_trace_correlation_id,
+    verify_trace_correlation_id_in_response,
+)
+from .validators import (
+    normalize_inference_url,
+    validate_rollout_response_for_rl,
+    validate_task_app_endpoint,
+    validate_task_app_url,
+)
+from .vendors import (
+    get_groq_key_or_503,
+    get_openai_key_or_503,
+    normalize_vendor_keys,
+)
+
+__all__ = [
+    "normalize_inference_url",
+    "validate_rollout_response_for_rl",
+    "validate_task_app_url",
+    "validate_task_app_endpoint",
+    "task_app_health",
+    "EvalConfig",
+    "FilterConfig",
+    "TaskAppEndpoints",
+    "LocalAPIEndpoints",
+    "RolloutEnvSpec",
+    "RolloutPolicySpec",
+    "RolloutRecordConfig",
+    "RolloutSafetyConfig",
+    "RolloutRequest",
+    "RolloutResponse",
+    "RolloutMetrics",
+    "TaskDescriptor",
+    "DatasetInfo",
+    "RubricInfo",
+    "RubricSection",
+    "InferenceInfo",
+    "LimitsInfo",
+    "TaskInfo",
+    "to_jsonable",
+    "normalize_environment_api_key",
+    "is_api_key_header_authorized",
+    "require_api_key_dependency",
+    "normalize_vendor_keys",
+    "get_openai_key_or_503",
+    "get_groq_key_or_503",
+    "prepare_for_openai",
+    "prepare_for_groq",
+    "inject_system_hint",
+    "extract_message_text",
+    "parse_tool_call_from_text",
+    "synthesize_tool_call_if_missing",
+    "TaskDatasetSpec",
+    "TaskDatasetRegistry",
+    "Criterion",
+    "Rubric",
+    "load_rubric",
+    "blend_rubrics",
+    "score_events_against_rubric",
+    "score_outcome_against_rubric",
+    "TaskAppClient",
+    "LocalAPIClient",
+    "error_payload",
+    "http_exception",
+    "json_error_response",
+    "run_task_app",
+    "run_server_background",
+    "create_task_app",
+    "RubricBundle",
+    "ProxyConfig",
+    "TaskAppConfig",
+    "LocalAPIConfig",
+    "InferenceAPIClient",
+    "InProcessTaskApp",
+    "InProcessJobResult",
+    "merge_dot_overrides",
+    "resolve_backend_api_base",
+    "run_in_process_job",
+    "run_in_process_job_sync",
+    "build_trajectory_trace",
+    "build_trace_payload",
+    "extract_trace_correlation_id",
+    "include_event_history_in_response",
+    "include_event_history_in_trajectories",
+    "include_trace_correlation_id_in_response",
+    "validate_trace_correlation_id",
+    "verify_trace_correlation_id_in_response",
+]

synth_ai/sdk/task/apps/__init__.py

@@ -0,0 +1,169 @@
+"""Registry for Task Apps exposed via the shared FastAPI harness.
+
+Prefer synth_ai.sdk.localapi.apps moving forward. This module remains for
+backward compatibility during the naming transition.
+"""
+
+from __future__ import annotations
+
+import importlib
+import sys
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from ..server import TaskAppConfig
+
+
+@dataclass(slots=True)
+class ModalDeploymentConfig:
+    """Modal deployment defaults for a task app."""
+
+    app_name: str
+    python_version: str = "3.11"
+    pip_packages: Sequence[str] = field(default_factory=tuple)
+    apt_packages: Sequence[str] = field(default_factory=tuple)
+    extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    secret_names: Sequence[str] = field(default_factory=tuple)
+    volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    env_vars: dict[str, str] = field(default_factory=dict)
+    timeout: int = 600
+    memory: int = 4096
+    cpu: float = 2.0
+    min_containers: int = 1
+    max_containers: int = 4
+
+
+@dataclass(slots=True)
+class TaskAppEntry:
+    """Metadata describing a registered task app."""
+
+    app_id: str
+    description: str
+    config_factory: Callable[[], TaskAppConfig]
+    aliases: Sequence[str] = field(default_factory=tuple)
+    env_files: Sequence[str] = field(default_factory=tuple)
+    modal: ModalDeploymentConfig | None = None
+
+
+@dataclass(slots=True)
+class LocalAPIEntry:
+    """Metadata describing a registered local API."""
+
+    api_id: str
+    description: str
+    config_factory: Callable[[], TaskAppConfig]
+    aliases: Sequence[str] = field(default_factory=tuple)
+    env_files: Sequence[str] = field(default_factory=tuple)
+    modal: ModalDeploymentConfig | None = None
+
+    @property
+    def app_id(self) -> str:
+        return self.api_id
+
+
+class TaskAppRegistry:
+    """In-memory registry of known task apps."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, TaskAppEntry] = {}
+        self._alias_to_id: dict[str, str] = {}
+
+    def register(self, entry: TaskAppEntry) -> None:
+        if entry.app_id in self._entries:
+            # Allow idempotent registration when modules are imported multiple times.
+            return
+        self._entries[entry.app_id] = entry
+        for alias in entry.aliases:
+            existing = self._alias_to_id.get(alias)
+            if existing and existing != entry.app_id:
+                raise ValueError(f"Alias already registered: {alias}")
+            self._alias_to_id[alias] = entry.app_id
+
+    def get(self, app_id: str) -> TaskAppEntry:
+        resolved = self._alias_to_id.get(app_id, app_id)
+        if resolved not in self._entries:
+            raise KeyError(f"Unknown task app id: {app_id}")
+        return self._entries[resolved]
+
+    def list(self) -> List[TaskAppEntry]:
+        return sorted(self._entries.values(), key=lambda entry: entry.app_id)
+
+    def __iter__(self) -> Iterable[TaskAppEntry]:
+        return iter(self.list())
+
+    def clear(self) -> None:
+        """Clear all registered task apps."""
+        self._entries.clear()
+        self._alias_to_id.clear()
+
+
+registry = TaskAppRegistry()
+
+
+def register_task_app(*, entry: TaskAppEntry) -> None:
+    registry.register(entry)
+
+
+def register_local_api(*, entry: LocalAPIEntry | TaskAppEntry) -> None:
+    if isinstance(entry, LocalAPIEntry):
+        registry.register(
+            TaskAppEntry(
+                app_id=entry.api_id,
+                description=entry.description,
+                config_factory=entry.config_factory,
+                aliases=entry.aliases,
+                env_files=entry.env_files,
+                modal=entry.modal,
+            )
+        )
+        return
+    registry.register(entry)
+
+
+def discover_task_apps_from_cwd() -> None:
+    """Discover and register task apps from the current working directory and subdirectories."""
+    cwd = Path.cwd()
+
+    # Look for task app files in common patterns
+    patterns = [
+        "**/task_app/*.py",
+        "**/task_apps/*.py",
+        "**/*_task_app.py",
+        "**/grpo_crafter.py",
+        "**/math_single_step.py",
+    ]
+
+    discovered_files = []
+    for pattern in patterns:
+        discovered_files.extend(cwd.glob(pattern))
+
+    # Add current directory to Python path temporarily
+    original_path = sys.path.copy()
+    try:
+        sys.path.insert(0, str(cwd))
+
+        for file_path in discovered_files:
+            if file_path.name.startswith("__"):
+                continue
+
+            # Convert file path to module name
+            relative_path = file_path.relative_to(cwd)
+            module_parts = list(relative_path.parts[:-1]) + [relative_path.stem]
+            module_name = ".".join(module_parts)
+
+            try:
+                # Import the module to trigger registration
+                importlib.import_module(module_name)
+            except Exception:
+                # Silently skip modules that can't be imported
+                # This allows for graceful handling of missing dependencies
+                continue
+
+    finally:
+        sys.path[:] = original_path
+
+
+# Note: Task apps are now discovered dynamically by the CLI, not auto-registered
+# This allows for better separation between SDK and example-specific implementations

synth_ai/sdk/task/auth.py

@@ -0,0 +1,165 @@
+"""Authentication helpers shared by Task Apps."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable
+from contextlib import suppress
+from typing import Any
+
+from .errors import http_exception
+
+_API_KEY_ENV = "ENVIRONMENT_API_KEY"
+_DEV_API_KEY_ENVS = ("dev_environment_api_key", "DEV_ENVIRONMENT_API_KEY")
+_API_KEY_HEADER = "x-api-key"
+_API_KEYS_HEADER = "x-api-keys"
+_AUTH_HEADER = "authorization"
+_API_KEY_ALIASES_ENV = (
+    "ENVIRONMENT_API_KEY_ALIASES"  # comma-separated list of additional valid keys
+)
+
+
+def _mask(value: str, *, prefix: int = 4) -> str:
+    if not value:
+        return "<empty>"
+    visible = value[:prefix]
+    return f"{visible}{'…' if len(value) > prefix else ''}"
+
+
+def normalize_environment_api_key() -> str | None:
+    """Ensure `ENVIRONMENT_API_KEY` is populated from dev fallbacks.
+
+    Returns the resolved key (if any) so callers can branch on configuration.
+    """
+
+    key = os.getenv(_API_KEY_ENV)
+    if key:
+        return key
+    for env in _DEV_API_KEY_ENVS:
+        candidate = os.getenv(env)
+        if candidate:
+            os.environ[_API_KEY_ENV] = candidate
+            print(
+                f"[task:auth] {_API_KEY_ENV} set from {env} (prefix={_mask(candidate)})",
+                flush=True,
+            )
+            return candidate
+    return None
+
+
+def allowed_environment_api_keys() -> set[str]:
+    """Return the set of valid environment API keys for this Task App.
+
+    Includes:
+    - The primary ENVIRONMENT_API_KEY (normalized from dev fallbacks if needed)
+    - Any comma-separated aliases from ENVIRONMENT_API_KEY_ALIASES
+    """
+    keys: set[str] = set()
+    primary = normalize_environment_api_key()
+    if primary:
+        keys.add(primary)
+    aliases = (os.getenv(_API_KEY_ALIASES_ENV) or "").strip()
+    if aliases:
+        for part in aliases.split(","):
+            trimmed = part.strip()
+            if trimmed:
+                keys.add(trimmed)
+    return keys
+
+
+def _header_values(request: Any, header: str) -> Iterable[str]:
+    header_lower = header.lower()
+    if request is None:
+        return []
+    headers = getattr(request, "headers", None)
+    if headers:
+        raw = headers.get(header) or headers.get(header_lower)
+        if raw is not None:
+            return [raw]
+    if isinstance(request, dict):
+        raw = request.get(header) or request.get(header_lower)
+        if raw is not None:
+            return [raw]
+    # Support passing explicit header dict via keyword arg on FastAPI route handlers
+    for attr in ("headers", "state"):
+        maybe = getattr(request, attr, None)
+        if isinstance(maybe, dict):
+            raw = maybe.get(header) or maybe.get(header_lower)
+            if raw is not None:
+                return [raw]
+    return []
+
+
+def _split_csv(values: Iterable[str]) -> list[str]:
+    seen: list[str] = []
+    for v in values:
+        if not isinstance(v, str):
+            continue
+        for part in v.split(","):
+            trimmed = part.strip()
+            if trimmed:
+                seen.append(trimmed)
+    return seen
+
+
+def is_api_key_header_authorized(request: Any) -> bool:
+    """Return True if any header-provided key matches any allowed environment key."""
+
+    allowed = allowed_environment_api_keys()
+    if not allowed:
+        return False
+    single = list(_header_values(request, _API_KEY_HEADER))
+    multi = list(_header_values(request, _API_KEYS_HEADER))
+    auths = list(_header_values(request, _AUTH_HEADER))
+    bearer: list[str] = []
+    for a in auths:
+        if isinstance(a, str) and a.lower().startswith("bearer "):
+            bearer.append(a.split(" ", 1)[1].strip())
+    candidates = _split_csv(single + multi + bearer)
+    return any(candidate in allowed for candidate in candidates)
+
+
+def require_api_key_dependency(request: Any) -> None:
+    """FastAPI dependency enforcing Task App authentication headers."""
+
+    allowed = allowed_environment_api_keys()
+    if not allowed:
+        raise http_exception(
+            503, "missing_environment_api_key", "ENVIRONMENT_API_KEY is not configured"
+        )
+    # Build candidate list for verbose diagnostics
+    single = list(_header_values(request, _API_KEY_HEADER))
+    multi = list(_header_values(request, _API_KEYS_HEADER))
+    auths = list(_header_values(request, _AUTH_HEADER))
+    bearer: list[str] = []
+    for a in auths:
+        if isinstance(a, str) and a.lower().startswith("bearer "):
+            bearer.append(a.split(" ", 1)[1].strip())
+    candidates = _split_csv(single + multi + bearer)
+    if not any(candidate in allowed for candidate in candidates):
+        with suppress(Exception):
+            print(
+                {
+                    "task_auth_failed": True,
+                    "allowed_first15": [k[:15] for k in allowed],
+                    "allowed_count": len(allowed),
+                    "got_first15": [c[:15] for c in candidates],
+                    "got_lens": [len(c) for c in candidates],
+                    "have_x_api_key": bool(single),
+                    "have_x_api_keys": bool(multi),
+                    "have_authorization": bool(auths),
+                },
+                flush=True,
+            )
+        # Use 400 to make failures unmistakable during preflight
+        raise http_exception(
+            400,
+            "unauthorised",
+            "API key missing or invalid",
+            extra={
+                "allowed_first15": [k[:15] for k in allowed],
+                "allowed_count": len(allowed),
+                "got_first15": [c[:15] for c in candidates],
+                "got_lens": [len(c) for c in candidates],
+            },
+        )