synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/task/config.py

@@ -0,0 +1,256 @@
+"""Configuration dataclasses for task app CLI commands (eval, filter)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass(slots=True)
+class EvalConfig:
+    """Configuration for 'synth-ai eval' command.
+    
+    Validates and provides defaults for evaluation runs against task apps.
+    """
+    
+    # Required: Task app identifier
+    app_id: str
+    
+    # Required: Model to evaluate
+    model: str
+    
+    # Required: Seeds to run
+    seeds: list[int]
+    
+    # Optional: Task app URL (None = spawn in-process)
+    task_app_url: str | None = None
+    
+    # Optional: Data split to use
+    split: str = "train"
+    
+    # Optional: Maximum turns/steps per episode
+    max_turns: int | None = None
+    
+    # Optional: Maximum LLM calls per episode
+    max_llm_calls: int = 10
+    
+    # Optional: Concurrency for parallel rollouts
+    concurrency: int = 1
+    
+    # Optional: Environment name
+    env_name: str | None = None
+    
+    # Optional: Policy name
+    policy_name: str | None = None
+    
+    # Optional: Trace format ("compact", "full", "structured")
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+    
+    # Optional: Whether to return traces in response
+    return_trace: bool = False
+    
+    # Optional: Environment config overrides
+    env_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Policy config overrides
+    policy_config: dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for traces
+    metadata: dict[str, str] = field(default_factory=dict)
+    
+    # Optional: SQL query for metadata filtering
+    metadata_sql: str | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.app_id:
+            raise ValueError("app_id is required")
+        
+        if not self.model:
+            raise ValueError("model is required")
+        
+        if not self.seeds:
+            raise ValueError("seeds list cannot be empty")
+        
+        if not isinstance(self.seeds, list):
+            raise ValueError("seeds must be a list of integers")
+        
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        
+        if self.max_llm_calls < 1:
+            raise ValueError("max_llm_calls must be >= 1")
+        
+        if self.max_turns is not None and self.max_turns < 1:
+            raise ValueError("max_turns must be >= 1")
+        
+        if self.trace_format not in ("compact", "full", "structured"):
+            raise ValueError(f"trace_format must be 'compact', 'full', or 'structured', got: {self.trace_format}")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalConfig:
+        """Create EvalConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with eval configuration
+            
+        Returns:
+            Validated EvalConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "app_id": data.get("app_id"),
+            "model": data.get("model"),
+            "seeds": data.get("seeds", []),
+            "task_app_url": data.get("task_app_url"),
+            "split": data.get("split", "train"),
+            "max_turns": data.get("max_turns"),
+            "max_llm_calls": data.get("max_llm_calls", 10),
+            "concurrency": data.get("concurrency", 1),
+            "env_name": data.get("env_name"),
+            "policy_name": data.get("policy_name"),
+            "trace_format": data.get("trace_format", "compact"),
+            "return_trace": data.get("return_trace", False),
+            "env_config": data.get("env_config", {}),
+            "policy_config": data.get("policy_config", {}),
+            "metadata": data.get("metadata", {}),
+            "metadata_sql": data.get("metadata_sql"),
+        }
+        
+        return cls(**config_dict)
+
+
+@dataclass(slots=True)
+class FilterConfig:
+    """Configuration for 'synth-ai filter' command.
+    
+    Validates and provides defaults for filtering traces into SFT datasets.
+    """
+    
+    # Required: Database path or URL
+    db: str
+    
+    # Required: Output JSONL path
+    output: str
+    
+    # Optional: Filter by data splits
+    splits: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by task IDs
+    task_ids: list[str] = field(default_factory=list)
+    
+    # Optional: Filter by models
+    models: list[str] = field(default_factory=list)
+    
+    # Optional: Minimum official score threshold
+    min_official_score: float | None = None
+    
+    # Optional: Maximum official score threshold
+    max_official_score: float | None = None
+    
+    # Optional: Minimum verifier scores (verifier_name -> min_score)
+    min_verifier_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Maximum verifier scores (verifier_name -> max_score)
+    max_verifier_scores: dict[str, float] = field(default_factory=dict)
+    
+    # Optional: Limit number of examples
+    limit: int | None = None
+    
+    # Optional: Offset for pagination
+    offset: int | None = None
+    
+    # Optional: Whether to shuffle results
+    shuffle: bool = False
+    
+    # Optional: Random seed for shuffling
+    shuffle_seed: int | None = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.db:
+            raise ValueError("db (database path or URL) is required")
+        
+        if not self.output:
+            raise ValueError("output (JSONL file path) is required")
+        
+        # Validate output has .jsonl extension
+        output_path = Path(self.output)
+        if output_path.suffix.lower() not in (".jsonl", ".json"):
+            raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
+        
+        # Validate score thresholds
+        if (
+            self.min_official_score is not None
+            and self.max_official_score is not None
+            and self.min_official_score > self.max_official_score
+        ):
+            raise ValueError("min_official_score cannot be greater than max_official_score")
+        
+        # Validate limit/offset
+        if self.limit is not None and self.limit < 1:
+            raise ValueError("limit must be >= 1")
+        
+        if self.offset is not None and self.offset < 0:
+            raise ValueError("offset must be >= 0")
+        
+        # Validate shuffle seed requires shuffle
+        if self.shuffle_seed is not None and not self.shuffle:
+            raise ValueError("shuffle_seed requires shuffle=true")
+    
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FilterConfig:
+        """Create FilterConfig from a dictionary (e.g. from TOML).
+        
+        Args:
+            data: Dictionary with filter configuration
+            
+        Returns:
+            Validated FilterConfig instance
+        """
+        # Extract known fields
+        config_dict = {
+            "db": data.get("db"),
+            "output": data.get("output"),
+            "splits": data.get("splits", []),
+            "task_ids": data.get("task_ids", []),
+            "models": data.get("models", []),
+            "min_official_score": data.get("min_official_score"),
+            "max_official_score": data.get("max_official_score"),
+            "min_verifier_scores": data.get("min_verifier_scores", {}),
+            "max_verifier_scores": data.get("max_verifier_scores", {}),
+            "limit": data.get("limit"),
+            "offset": data.get("offset"),
+            "shuffle": data.get("shuffle", False),
+            "shuffle_seed": data.get("shuffle_seed"),
+        }
+        
+        return cls(**config_dict)
+    
+    def get_db_url(self) -> str:
+        """Convert db path to proper SQLite URL if needed.
+        
+        Returns:
+            Database URL suitable for SQLAlchemy/aiosqlite
+        """
+        db_value = self.db.strip()
+        if "://" in db_value:
+            return db_value
+        else:
+            db_path = Path(db_value).expanduser().resolve()
+            # Ensure parent directory exists
+            db_path.parent.mkdir(parents=True, exist_ok=True)
+            return f"sqlite+aiosqlite:///{db_path}"
+    
+    def get_output_path(self) -> Path:
+        """Get resolved output path with parent directory created.
+        
+        Returns:
+            Resolved Path object with parent directory created
+        """
+        output_path = Path(self.output).expanduser().resolve()
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+

synth_ai/sdk/task/contracts.py

@@ -0,0 +1,340 @@
+"""Contracts for Task Apps.
+
+Prefer synth_ai.sdk.localapi.contracts moving forward. This module remains for
+backward compatibility during the naming transition.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class RolloutMode(str, Enum):
+    """Mode controls how rollout infrastructure processes inference URLs."""
+    RL = "rl"
+    EVAL = "eval"
+
+
+class OutputMode(str, Enum):
+    """Controls how the policy expects model outputs.
+
+    - TOOL_CALLS: Use function/tool calling (default, current behavior)
+    - TEXT: Plain text in message.content
+    - STRUCTURED: JSON via response_format (OpenAI json_schema, Groq json_object, Gemini responseSchema)
+    """
+    TOOL_CALLS = "tool_calls"
+    TEXT = "text"
+    STRUCTURED = "structured"
+
+
+class StructuredOutputConfig(BaseModel):
+    """Configuration for structured output mode (OutputMode.STRUCTURED).
+
+    Defines the JSON schema that the model must conform to when using structured outputs.
+    This is normalized across providers:
+    - OpenAI: response_format.json_schema
+    - Groq: response_format.json_schema or json_object
+    - Gemini: generationConfig.responseSchema
+    """
+    schema: dict[str, Any] = Field(
+        ...,
+        description="JSON Schema for the expected response structure"
+    )
+    schema_name: str = Field(
+        default="response",
+        description="Name for the schema (required by some providers)"
+    )
+    strict: bool = Field(
+        default=True,
+        description="Whether to enforce strict schema validation (OpenAI strict mode)"
+    )
+
+
+@dataclass(frozen=True)
+class TaskAppEndpoints:
+    """Required Task App endpoints used by RL trainers and clients.
+
+    Task Apps run as lightweight HTTP services (often on Modal) that expose these
+    standard endpoints. Additional endpoints (proxies, debug routes) may be added
+    by individual task apps as needed.
+    """
+
+    root: str = "/"
+    health: str = "/health"
+    info: str = "/info"
+    task_info: str = "/task_info"
+    rollout: str = "/rollout"
+
+
+@dataclass(frozen=True)
+class LocalAPIEndpoints(TaskAppEndpoints):
+    """Alias for TaskAppEndpoints with LocalAPI naming."""
+
+
+# --- Unified rollout schema used by Task App services and SDK utilities ---
+
+
+class RolloutEnvSpec(BaseModel):
+    env_id: str | None = None
+    env_name: str | None = None
+    config: dict[str, Any] = Field(default_factory=dict)
+    seed: int | None = None
+
+
+class RolloutPolicySpec(BaseModel):
+    policy_id: str | None = None
+    policy_name: str | None = None
+    config: dict[str, Any] = Field(default_factory=dict)
+
+    # Output mode configuration (defaults to tool_calls for backward compatibility)
+    output_mode: OutputMode = Field(
+        default=OutputMode.TOOL_CALLS,
+        description="How the policy expects model outputs: tool_calls, text, or structured"
+    )
+    structured_config: StructuredOutputConfig | None = Field(
+        default=None,
+        description="Configuration for structured output mode (required if output_mode=STRUCTURED)"
+    )
+
+
+class RolloutRecordConfig(BaseModel):
+    logprobs: bool = False
+    value: bool = False
+    return_trace: bool = False
+    trace_format: Literal["compact", "full", "structured"] = "compact"
+
+
+class RolloutSafetyConfig(BaseModel):
+    max_time_s: float = 3600.0
+
+
+class RolloutRequest(BaseModel):
+    run_id: str
+    env: RolloutEnvSpec
+    policy: RolloutPolicySpec
+    record: RolloutRecordConfig = RolloutRecordConfig()
+    on_done: str = "reset"
+    safety: RolloutSafetyConfig = RolloutSafetyConfig()
+    training_session_id: str | None = None
+    synth_base_url: str | None = None
+    mode: RolloutMode = RolloutMode.RL  # Default to RL mode for training/optimization
+
+
+class RolloutMetrics(BaseModel):
+    """Metrics from a rollout execution.
+
+    ## Preferred Fields (New - Normalized)
+
+    - `outcome_reward`: The reward for this rollout (PREFERRED)
+    - `event_rewards`: Optional per-step rewards
+
+    ## Legacy Fields (Backward Compatibility)
+
+    - `episode_rewards`, `reward_mean`, `num_steps`: Still supported for backward
+      compatibility. For new implementations, just use `outcome_reward`.
+    - `outcome_score`: Alias for `outcome_reward` (deprecated)
+
+    ## Example - Minimal (New Style)
+
+        metrics = RolloutMetrics(
+            outcome_reward=1.0,  # PREFERRED - just provide the reward
+        )
+
+    ## Example - Full (Backward Compatible)
+
+        metrics = RolloutMetrics(
+            episode_rewards=[1.0],
+            reward_mean=1.0,
+            num_steps=1,
+            outcome_reward=1.0,  # PREFERRED
+        )
+    """
+
+    # =========================================================================
+    # PREFERRED FIELDS (New - Normalized)
+    # =========================================================================
+    outcome_reward: float | None = Field(
+        default=None,
+        description="The reward for this rollout. PREFERRED field for scoring.",
+    )
+    event_rewards: list[float] | None = Field(
+        default=None,
+        description="Optional per-step/event rewards for multi-step tasks.",
+    )
+
+    # =========================================================================
+    # LEGACY FIELDS (Backward Compatibility)
+    # =========================================================================
+    episode_rewards: list[float] = Field(
+        default_factory=list,
+        description="[LEGACY] Per-episode rewards. Use outcome_reward instead.",
+    )
+    reward_mean: float = Field(
+        default=0.0,
+        description="[LEGACY] Mean reward. Use outcome_reward instead.",
+    )
+    num_steps: int = Field(
+        default=1,
+        description="[LEGACY] Step count. Can be derived from event_rewards or trace.",
+    )
+    num_episodes: int = Field(
+        default=1,
+        description="[LEGACY] Episode count. Usually 1 for GEPA tasks.",
+    )
+    outcome_score: float | None = Field(
+        default=None,
+        description="[DEPRECATED] Alias for outcome_reward. Use outcome_reward instead.",
+    )
+    events_score: float | None = Field(
+        default=None,
+        description="[LEGACY] Aggregate event score. Use event_rewards instead.",
+    )
+    details: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Metadata only. Do NOT use details.correct for rewards.",
+    )
+
+
+class RolloutResponse(BaseModel):
+    """Response from a rollout execution.
+
+    ## Key Fields
+
+    - `run_id`: Echo from request (required)
+    - `metrics`: Rollout metrics with `outcome_reward` (required)
+    - `trace`: v3 trace payload (required for verifier scoring)
+
+    ## Canonical Locations (Top-Level)
+
+    - `trace_correlation_id`: Correlation ID for trace recovery (TOP-LEVEL CANONICAL)
+    - `inference_url`: Inference URL used for this rollout (TOP-LEVEL CANONICAL)
+
+    These fields SHOULD be at top-level. The monorepo parses from top-level first,
+    with fallback to nested locations for backward compatibility.
+
+    ## Example
+
+        response = RolloutResponse(
+            run_id=request.run_id,
+            metrics=RolloutMetrics(outcome_reward=1.0),
+            trace=trace_payload,
+            trace_correlation_id="trace_abc123",
+            inference_url="https://api.usesynth.ai/v1/trial-xyz",
+        )
+    """
+
+    run_id: str
+    metrics: RolloutMetrics
+    trace: dict[str, Any] | None = None
+
+    # =========================================================================
+    # CANONICAL LOCATIONS (Top-Level - Preferred for Parsing)
+    # =========================================================================
+    trace_correlation_id: str | None = Field(
+        default=None,
+        description="Correlation ID for trace recovery. TOP-LEVEL CANONICAL location.",
+    )
+    inference_url: str | None = Field(
+        default=None,
+        description="Inference URL used for this rollout. TOP-LEVEL CANONICAL location.",
+    )
+
+    # =========================================================================
+    # LEGACY FIELDS (Backward Compatibility)
+    # =========================================================================
+    branches: dict[str, list[str]] = Field(
+        default_factory=dict,
+        description="[LEGACY] Branch tracking. Usually empty for single-path rollouts.",
+    )
+    aborted: bool = Field(
+        default=False,
+        description="Whether the rollout was aborted early.",
+    )
+    pipeline_metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="[LEGACY] Additional metadata. Prefer top-level fields instead.",
+    )
+
+
+class _ExtraAllowModel(BaseModel):
+    """Base helper that preserves unknown keys while still exposing typed attributes."""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class TaskDescriptor(_ExtraAllowModel):
+    """Human-readable task identifiers shown in UIs and logs."""
+
+    id: str
+    name: str
+    description: str | None = None
+    version: str | None = None
+
+
+class DatasetInfo(_ExtraAllowModel):
+    """Metadata about the prompt/task dataset powering the environment."""
+
+    id: str | None = None
+    name: str | None = None
+    version: str | None = None
+    splits: list[str] | None = None
+    default_split: str | None = None
+    description: str | None = None
+
+
+class RubricCriterion(_ExtraAllowModel):
+    id: str
+    description: str
+    weight: float | None = None
+
+
+class RubricSection(_ExtraAllowModel):
+    name: str
+    criteria: list[RubricCriterion] = Field(default_factory=list)
+
+
+class RubricInfo(_ExtraAllowModel):
+    """Outcome and event scoring definitions used by verifiers."""
+
+    outcome: RubricSection | None = None
+    events: RubricSection | None = None
+
+
+class InferenceInfo(_ExtraAllowModel):
+    """Recommended defaults for policy model routing."""
+
+    model: str | None = None
+    inference_url: str | None = None
+
+
+class LimitsInfo(_ExtraAllowModel):
+    """Operational limits the environment enforces."""
+
+    max_turns: int | None = None
+    max_response_tokens: int | None = None
+    timeout_seconds: int | None = None
+
+
+class TaskInfo(_ExtraAllowModel):
+    """Static metadata describing the capabilities of a Task App task."""
+
+    task: TaskDescriptor
+    dataset: DatasetInfo
+    inference: InferenceInfo
+    limits: LimitsInfo
+    environment: str | None = Field(
+        default=None,
+        description="[DEPRECATED] Legacy field not read by server. Will be removed in future version.",
+    )
+    rubric: RubricInfo | None = Field(
+        default=None,
+        description="[DEPRECATED] Use LocalAPIConfig.rubrics (RubricBundle) instead. Server ignores this field.",
+    )
+    task_metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Task-specific extras (e.g. prompt version info, documentation links).",
+    )

synth_ai/sdk/task/datasets.py

@@ -0,0 +1,108 @@
+"""Dataset registry and helpers shared by Task Apps."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Hashable
+from typing import Any, List
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class TaskDatasetSpec(BaseModel):
+    """Declarative metadata describing a dataset that a Task App exposes."""
+
+    id: str
+    name: str
+    version: str | None = None
+    splits: list[str] = Field(default_factory=list)
+    default_split: str | None = None
+    cardinality: int | None = None
+    description: str | None = None
+
+    @field_validator("default_split")
+    @classmethod
+    def _validate_default_split(cls, value: str | None, info):
+        values = info.data if hasattr(info, "data") else {}  # type: ignore[attr-defined]
+        if value and value not in (values.get("splits") or []):
+            raise ValueError("default_split must be one of splits when provided")
+        return value
+
+
+RegistryLoader = Callable[[TaskDatasetSpec], Any]
+
+
+class TaskDatasetRegistry:
+    """Lightweight registry mapping dataset specs to loader callables."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, tuple[TaskDatasetSpec, RegistryLoader, bool]] = {}
+        self._cache: dict[Hashable, Any] = {}
+
+    def register(
+        self, spec: TaskDatasetSpec, loader: RegistryLoader, *, cache: bool = True
+    ) -> None:
+        """Register a dataset loader and its metadata."""
+
+        self._entries[spec.id] = (spec, loader, cache)
+
+    def describe(self, dataset_id: str) -> TaskDatasetSpec:
+        if dataset_id not in self._entries:
+            raise KeyError(f"Dataset not registered: {dataset_id}")
+        spec, _, _ = self._entries[dataset_id]
+        return spec
+
+    def list(self) -> List[TaskDatasetSpec]:
+        return [entry[0] for entry in self._entries.values()]
+
+    def get(self, spec: TaskDatasetSpec | str) -> Any:
+        """Return dataset materialisation (with optional caching)."""
+
+        if isinstance(spec, str):
+            if spec not in self._entries:
+                raise KeyError(f"Dataset not registered: {spec}")
+            base_spec, loader, cache_enabled = self._entries[spec]
+            effective_spec = base_spec
+        else:
+            if spec.id not in self._entries:
+                raise KeyError(f"Dataset not registered: {spec.id}")
+            base_spec, loader, cache_enabled = self._entries[spec.id]
+            effective_spec = base_spec.model_copy(update=spec.model_dump(exclude_unset=True))
+
+        cache_key: Hashable = (
+            effective_spec.id,
+            effective_spec.version,
+            effective_spec.default_split,
+        )
+        if cache_enabled:
+            if cache_key not in self._cache:
+                self._cache[cache_key] = loader(effective_spec)
+            return self._cache[cache_key]
+        return loader(effective_spec)
+
+    @staticmethod
+    def ensure_split(spec: TaskDatasetSpec, split: str | None) -> str:
+        """Validate that `split` exists on the spec; return a concrete split."""
+
+        if not spec.splits:
+            return split or spec.default_split or "default"
+        if split is None:
+            if spec.default_split:
+                return spec.default_split
+            raise ValueError(f"split must be provided for dataset {spec.id}")
+        if split not in spec.splits:
+            raise ValueError(f"Unknown split '{split}' for dataset {spec.id}")
+        return split
+
+    @staticmethod
+    def normalise_seed(seed: Any, *, cardinality: int | None = None) -> int:
+        """Normalise arbitrary seed input into a bounded non-negative integer."""
+
+        try:
+            value = int(seed)
+        except Exception as exc:  # pragma: no cover - defensive
+            raise ValueError(f"Seed must be convertible to int (got {seed!r})") from exc
+        if value < 0:
+            value = abs(value)
+        if cardinality and cardinality > 0:
+            value = value % cardinality
+        return value