synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/sdk/streaming/types.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Any
+
+
+class StreamType(Enum):
+    """Categories of streaming payloads emitted by training jobs."""
+
+    STATUS = auto()
+    EVENTS = auto()
+    METRICS = auto()
+    TIMELINE = auto()
+
+    @property
+    def endpoint_path(self) -> str:
+        """Return the endpoint suffix used when polling this stream."""
+        return {
+            StreamType.STATUS: "",
+            StreamType.EVENTS: "/events",
+            StreamType.METRICS: "/metrics",
+            StreamType.TIMELINE: "/timeline",
+        }[self]
+
+
+@dataclass(slots=True)
+class StreamMessage:
+    """Unified representation of a streaming payload."""
+
+    stream_type: StreamType
+    timestamp: str
+    job_id: str
+    data: dict[str, Any]
+    seq: int | None = None
+    step: int | None = None
+    phase: str | None = None
+
+    @property
+    def key(self) -> str:
+        """Return a unique identifier used for deduplication."""
+        if self.stream_type is StreamType.EVENTS:
+            return f"event:{self.seq}"
+        if self.stream_type is StreamType.METRICS:
+            name = self.data.get("name", "")
+            if self.step is not None:
+                return f"metric:{name}:{self.step}"
+            ts = (
+                self.timestamp
+                or self.data.get("created_at")
+                or self.data.get("updated_at")
+                or self.data.get("timestamp")
+            )
+            if ts:
+                return f"metric:{name}:{ts}"
+            try:
+                fingerprint = json.dumps(self.data, sort_keys=True, default=str)
+            except Exception:
+                fingerprint = repr(self.data)
+            return f"metric:{name}:{fingerprint}"
+        if self.stream_type is StreamType.TIMELINE:
+            return f"timeline:{self.phase}:{self.timestamp}"
+        # Include status value in key so status changes are always shown
+        status = str(self.data.get("status") or self.data.get("state") or "")
+        return f"status:{status}:{self.timestamp}"
+
+    @classmethod
+    def from_status(cls, job_id: str, status_data: dict[str, Any]) -> StreamMessage:
+        """Create a message representing a job status payload."""
+        return cls(
+            stream_type=StreamType.STATUS,
+            timestamp=status_data.get("updated_at", "") or status_data.get("created_at", ""),
+            job_id=job_id,
+            data=status_data,
+        )
+
+    @classmethod
+    def from_event(cls, job_id: str, event_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a job event."""
+        return cls(
+            stream_type=StreamType.EVENTS,
+            timestamp=event_data.get("created_at", ""),
+            job_id=job_id,
+            data=event_data,
+            seq=event_data.get("seq"),
+        )
+
+    @classmethod
+    def from_metric(cls, job_id: str, metric_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a metric point."""
+        return cls(
+            stream_type=StreamType.METRICS,
+            timestamp=metric_data.get("created_at", ""),
+            job_id=job_id,
+            data=metric_data,
+            step=metric_data.get("step"),
+        )
+
+    @classmethod
+    def from_timeline(cls, job_id: str, timeline_data: dict[str, Any]) -> StreamMessage:
+        """Create a message describing a status timeline entry."""
+        return cls(
+            stream_type=StreamType.TIMELINE,
+            timestamp=timeline_data.get("created_at", ""),
+            job_id=job_id,
+            data=timeline_data,
+            phase=timeline_data.get("phase"),
+        )
+
+
+__all__ = ["StreamMessage", "StreamType"]

synth_ai/sdk/task/__init__.py

@@ -0,0 +1,151 @@
+from .auth import (
+    is_api_key_header_authorized,
+    normalize_environment_api_key,
+    require_api_key_dependency,
+)
+from .client import TaskAppClient
+from .config import EvalConfig, FilterConfig
+from .contracts import (
+    DatasetInfo,
+    InferenceInfo,
+    LimitsInfo,
+    RolloutEnvSpec,
+    RolloutMetrics,
+    RolloutPolicySpec,
+    RolloutRecordConfig,
+    RolloutRequest,
+    RolloutResponse,
+    RolloutSafetyConfig,
+    RolloutStep,
+    RolloutTrajectory,
+    RubricInfo,
+    RubricSection,
+    TaskAppEndpoints,
+    TaskDescriptor,
+    TaskInfo,
+)
+from .datasets import TaskDatasetRegistry, TaskDatasetSpec
+from .errors import error_payload, http_exception, json_error_response
+from .health import task_app_health
+from .in_process import InProcessTaskApp
+from .in_process_runner import (
+    InProcessJobResult,
+    merge_dot_overrides,
+    resolve_backend_api_base,
+    run_in_process_job,
+    run_in_process_job_sync,
+)
+from .inference_api import InferenceAPIClient
+from .json import to_jsonable
+from .proxy import (
+    extract_message_text,
+    inject_system_hint,
+    parse_tool_call_from_text,
+    prepare_for_groq,
+    prepare_for_openai,
+    synthesize_tool_call_if_missing,
+)
+from .rubrics import (
+    Criterion,
+    Rubric,
+    blend_rubrics,
+    load_rubric,
+    score_events_against_rubric,
+    score_outcome_against_rubric,
+)
+from .server import (
+    ProxyConfig,
+    RubricBundle,
+    TaskAppConfig,
+    create_task_app,
+    run_task_app,
+)
+from .trace_correlation_helpers import (
+    build_trajectory_trace,
+    extract_trace_correlation_id,
+    include_event_history_in_trajectories,
+    include_trace_correlation_id_in_response,
+    validate_trace_correlation_id,
+    verify_trace_correlation_id_in_response,
+)
+from .validators import (
+    normalize_inference_url,
+    validate_rollout_response_for_rl,
+    validate_task_app_endpoint,
+    validate_task_app_url,
+)
+from .vendors import (
+    get_groq_key_or_503,
+    get_openai_key_or_503,
+    normalize_vendor_keys,
+)
+
+__all__ = [
+    "normalize_inference_url",
+    "validate_rollout_response_for_rl",
+    "validate_task_app_url",
+    "validate_task_app_endpoint",
+    "task_app_health",
+    "EvalConfig",
+    "FilterConfig",
+    "TaskAppEndpoints",
+    "RolloutEnvSpec",
+    "RolloutPolicySpec",
+    "RolloutRecordConfig",
+    "RolloutSafetyConfig",
+    "RolloutRequest",
+    "RolloutResponse",
+    "RolloutTrajectory",
+    "RolloutStep",
+    "RolloutMetrics",
+    "TaskDescriptor",
+    "DatasetInfo",
+    "RubricInfo",
+    "RubricSection",
+    "InferenceInfo",
+    "LimitsInfo",
+    "TaskInfo",
+    "to_jsonable",
+    "normalize_environment_api_key",
+    "is_api_key_header_authorized",
+    "require_api_key_dependency",
+    "normalize_vendor_keys",
+    "get_openai_key_or_503",
+    "get_groq_key_or_503",
+    "prepare_for_openai",
+    "prepare_for_groq",
+    "inject_system_hint",
+    "extract_message_text",
+    "parse_tool_call_from_text",
+    "synthesize_tool_call_if_missing",
+    "TaskDatasetSpec",
+    "TaskDatasetRegistry",
+    "Criterion",
+    "Rubric",
+    "load_rubric",
+    "blend_rubrics",
+    "score_events_against_rubric",
+    "score_outcome_against_rubric",
+    "TaskAppClient",
+    "error_payload",
+    "http_exception",
+    "json_error_response",
+    "run_task_app",
+    "create_task_app",
+    "RubricBundle",
+    "ProxyConfig",
+    "TaskAppConfig",
+    "InferenceAPIClient",
+    "InProcessTaskApp",
+    "InProcessJobResult",
+    "merge_dot_overrides",
+    "resolve_backend_api_base",
+    "run_in_process_job",
+    "run_in_process_job_sync",
+    "build_trajectory_trace",
+    "extract_trace_correlation_id",
+    "include_event_history_in_trajectories",
+    "include_trace_correlation_id_in_response",
+    "validate_trace_correlation_id",
+    "verify_trace_correlation_id_in_response",
+]

synth_ai/sdk/task/apps/__init__.py

@@ -0,0 +1,133 @@
+"""Registry for Task Apps exposed via the shared FastAPI harness."""
+
+from __future__ import annotations
+
+import importlib
+import sys
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List
+
+from ..server import TaskAppConfig
+
+
+@dataclass(slots=True)
+class ModalDeploymentConfig:
+    """Modal deployment defaults for a task app."""
+
+    app_name: str
+    python_version: str = "3.11"
+    pip_packages: Sequence[str] = field(default_factory=tuple)
+    apt_packages: Sequence[str] = field(default_factory=tuple)
+    extra_local_dirs: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    secret_names: Sequence[str] = field(default_factory=tuple)
+    volume_mounts: Sequence[tuple[str, str]] = field(default_factory=tuple)
+    env_vars: dict[str, str] = field(default_factory=dict)
+    timeout: int = 600
+    memory: int = 4096
+    cpu: float = 2.0
+    min_containers: int = 1
+    max_containers: int = 4
+
+
+@dataclass(slots=True)
+class TaskAppEntry:
+    """Metadata describing a registered task app."""
+
+    app_id: str
+    description: str
+    config_factory: Callable[[], TaskAppConfig]
+    aliases: Sequence[str] = field(default_factory=tuple)
+    env_files: Sequence[str] = field(default_factory=tuple)
+    modal: ModalDeploymentConfig | None = None
+
+
+class TaskAppRegistry:
+    """In-memory registry of known task apps."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, TaskAppEntry] = {}
+        self._alias_to_id: dict[str, str] = {}
+
+    def register(self, entry: TaskAppEntry) -> None:
+        if entry.app_id in self._entries:
+            # Allow idempotent registration when modules are imported multiple times.
+            return
+        self._entries[entry.app_id] = entry
+        for alias in entry.aliases:
+            existing = self._alias_to_id.get(alias)
+            if existing and existing != entry.app_id:
+                raise ValueError(f"Alias already registered: {alias}")
+            self._alias_to_id[alias] = entry.app_id
+
+    def get(self, app_id: str) -> TaskAppEntry:
+        resolved = self._alias_to_id.get(app_id, app_id)
+        if resolved not in self._entries:
+            raise KeyError(f"Unknown task app id: {app_id}")
+        return self._entries[resolved]
+
+    def list(self) -> List[TaskAppEntry]:
+        return sorted(self._entries.values(), key=lambda entry: entry.app_id)
+
+    def __iter__(self) -> Iterable[TaskAppEntry]:
+        return iter(self.list())
+
+    def clear(self) -> None:
+        """Clear all registered task apps."""
+        self._entries.clear()
+        self._alias_to_id.clear()
+
+
+registry = TaskAppRegistry()
+
+
+def register_task_app(*, entry: TaskAppEntry) -> None:
+    registry.register(entry)
+
+
+def discover_task_apps_from_cwd() -> None:
+    """Discover and register task apps from the current working directory and subdirectories."""
+    cwd = Path.cwd()
+
+    # Look for task app files in common patterns
+    patterns = [
+        "**/task_app/*.py",
+        "**/task_apps/*.py",
+        "**/*_task_app.py",
+        "**/grpo_crafter.py",
+        "**/math_single_step.py",
+    ]
+
+    discovered_files = []
+    for pattern in patterns:
+        discovered_files.extend(cwd.glob(pattern))
+
+    # Add current directory to Python path temporarily
+    original_path = sys.path.copy()
+    try:
+        sys.path.insert(0, str(cwd))
+
+        for file_path in discovered_files:
+            if file_path.name.startswith("__"):
+                continue
+
+            # Convert file path to module name
+            relative_path = file_path.relative_to(cwd)
+            module_parts = list(relative_path.parts[:-1]) + [relative_path.stem]
+            module_name = ".".join(module_parts)
+
+            try:
+                # Import the module to trigger registration
+                importlib.import_module(module_name)
+            except Exception:
+                # Silently skip modules that can't be imported
+                # This allows for graceful handling of missing dependencies
+                continue
+
+    finally:
+        sys.path[:] = original_path
+
+
+# Note: Task apps are now discovered dynamically by the CLI, not auto-registered
+# This allows for better separation between SDK and example-specific implementations

synth_ai/sdk/task/config.py

@@ -0,0 +1,261 @@
+"""Configuration dataclasses for task app CLI commands (eval, filter)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass(slots=True)
+class EvalConfig:
+    """Configuration for 'synth-ai eval' command.
+    
+    Validates and provides defaults for evaluation runs against task apps.
+    """
+    
+    # Required: Task app identifier
+    app_id: str
+    
+    # Required: Model to evaluate
+    model: str
+    
+    # Required: Seeds to run
+    seeds: list[int]
+    
+    # Optional: Task app URL (None = spawn in-process)
+    task_app_url: str | None = None
+    
+    # Optional: Data split to use
+    split: str = "train"
+    
+    # Optional: Maximum turns/steps per episode
+    max_turns: int | None = None
+    
+    # Optional: Maximum LLM calls per episode
+    max_llm_calls: int = 10
+    
+    # Optional: Concurrency for parallel rollouts
+    concurrency: int = 1
+    
+    # Optional: Environment name
+    env_name: str | None = None
+    
+    # Optional: Policy name
+    policy_name: str | None = None
+    
+    # Optional: Trace format ("compact", "full", "structured")
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+    
+    # Optional: Whether to return traces in response
+    return_trace: bool = False
+    
+    # Optional: Operations sequence (if not provided, generates default)
+    ops: list[str] | None = None
+    
+    # Optional: Environment config overrides
+    env_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Policy config overrides
+    policy_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for traces
+    metadata: dict[str, str] = field(default_factory=dict)
+    
+    # Optional: SQL query for metadata filtering
+    metadata_sql: str | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.app_id:
+            raise ValueError("app_id is required")
+        
+        if not self.model:
+            raise ValueError("model is required")
+        
+        if not self.seeds:
+            raise ValueError("seeds list cannot be empty")
+        
+        if not isinstance(self.seeds, list):
+            raise ValueError("seeds must be a list of integers")
+        
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        
+        if self.max_llm_calls < 1:
+            raise ValueError("max_llm_calls must be >= 1")
+        
+        if self.max_turns is not None and self.max_turns < 1:
+            raise ValueError("max_turns must be >= 1")
+        
+        if self.trace_format not in ("compact", "full", "structured"):
+            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
+        """Create EvalConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with eval configuration
+            
+        Returns:
+            Validated EvalConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "app_id": data.get("app_id"),
+            "model": data.get("model"),
+            "seeds": data.get("seeds", []),
+            "task_app_url": data.get("task_app_url"),
+            "split": data.get("split", "train"),
+            "max_turns": data.get("max_turns"),
+            "max_llm_calls": data.get("max_llm_calls", 10),
+            "concurrency": data.get("concurrency", 1),
+            "env_name": data.get("env_name"),
+            "policy_name": data.get("policy_name"),
+            "trace_format": data.get("trace_format", "compact"),
+            "return_trace": data.get("return_trace", False),
+            "ops": data.get("ops"),
+            "env_config": data.get("env_config", {}),
+            "policy_config": data.get("policy_config", {}),
+            "metadata": data.get("metadata", {}),
+            "metadata_sql": data.get("metadata_sql"),
+        }
+        
+        return cls(**config_dict)
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    """Configuration for 'synth-ai filter' command.
+    
+    Validates and provides defaults for filtering traces into SFT datasets.
+    """
+    
+    # Required: Database path or URL
+    db: str
+    
+    # Required: Output JSONL path
+    output: str
+    
+    # Optional: Filter by data splits
+    splits: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by task IDs
+    task_ids: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by models
+    models: list[str] = field(default_factory=list)
+    
+    # Optional: Minimum official score threshold
+    min_official_score: float | None = None
+    
+    # Optional: Maximum official score threshold
+    max_official_score: float | None = None
+    
+    # Optional: Minimum judge scores (judge_name -> min_score)
+    min_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Maximum judge scores (judge_name -> max_score)
+    max_judge_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Limit number of examples
+    limit: int | None = None
+    
+    # Optional: Offset for pagination
+    offset: int | None = None
+    
+    # Optional: Whether to shuffle results
+    shuffle: bool = False
+    
+    # Optional: Random seed for shuffling
+    shuffle_seed: int | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.db:
+            raise ValueError("db (database path or URL) is required")
+        
+        if not self.output:
+            raise ValueError("output (JSONL file path) is required")
+        
+        # Validate output has .jsonl extension
+        output_path = Path(self.output)
+        if output_path.suffix.lower() not in (".jsonl", ".json"):
+            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
+        
+        # Validate score thresholds
+        if (
+            self.min_official_score is not None
+            and self.max_official_score is not None
+            and self.min_official_score > self.max_official_score
+        ):
+            raise ValueError("min_official_score cannot be greater than max_official_score")
+        
+        # Validate limit/offset
+        if self.limit is not None and self.limit < 1:
+            raise ValueError("limit must be >= 1")
+        
+        if self.offset is not None and self.offset < 0:
+            raise ValueError("offset must be >= 0")
+        
+        # Validate shuffle seed requires shuffle
+        if self.shuffle_seed is not None and not self.shuffle:
+            raise ValueError("shuffle_seed requires shuffle=true")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
+        """Create FilterConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with filter configuration
+            
+        Returns:
+            Validated FilterConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "db": data.get("db"),
+            "output": data.get("output"),
+            "splits": data.get("splits", []),
+            "task_ids": data.get("task_ids", []),
+            "models": data.get("models", []),
+            "min_official_score": data.get("min_official_score"),
+            "max_official_score": data.get("max_official_score"),
+            "min_judge_scores": data.get("min_judge_scores", {}),
+            "max_judge_scores": data.get("max_judge_scores", {}),
+            "limit": data.get("limit"),
+            "offset": data.get("offset"),
+            "shuffle": data.get("shuffle", False),
+            "shuffle_seed": data.get("shuffle_seed"),
+        }
+        
+        return cls(**config_dict)
+    
+    def get_db_url(self) -> str:
+        """Convert db path to proper SQLite URL if needed.
+        
+        Returns:
+            Database URL suitable for SQLAlchemy/aiosqlite
+        """
+        db_value = self.db.strip()
+        if "://" in db_value:
+            return db_value
+        else:
+            db_path = Path(db_value).expanduser().resolve()
+            # Ensure parent directory exists
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            return f"sqlite+aiosqlite:///{db_path}"
+    
+    def get_output_path(self) -> Path:
+        """Get resolved output path with parent directory created.
+        
+        Returns:
+            Resolved Path object with parent directory created
+        """
+        output_path = Path(self.output).expanduser().resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+
+