synth-ai 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/api/train/validators.py

@@ -0,0 +1,277 @@
+"""SDK-side validation for training configs - catch errors BEFORE sending to backend."""
+
+from pathlib import Path
+from typing import Any
+
+import click
+
+
+class ConfigValidationError(Exception):
+    """Raised when a training config is invalid."""
+    pass
+
+
+def validate_prompt_learning_config(config_data: dict[str, Any], config_path: Path) -> None:
+    """
+    Validate prompt learning config BEFORE sending to backend.
+    
+    This catches common errors early with clear messages instead of cryptic backend errors.
+    
+    Args:
+        config_data: Parsed TOML/JSON config
+        config_path: Path to config file (for error messages)
+    
+    Raises:
+        ConfigValidationError: If config is invalid
+        click.ClickException: If validation fails (for CLI)
+    """
+    errors: list[str] = []
+    
+    # Check for prompt_learning section
+    pl_section = config_data.get("prompt_learning")
+    if not pl_section:
+        errors.append(
+            "Missing [prompt_learning] section in config. "
+            "Expected: [prompt_learning] with algorithm, task_app_url, etc."
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    if not isinstance(pl_section, dict):
+        errors.append(
+            f"[prompt_learning] must be a table/dict, got {type(pl_section).__name__}"
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    # CRITICAL: Validate algorithm field
+    algorithm = pl_section.get("algorithm")
+    if not algorithm:
+        errors.append(
+            "Missing required field: prompt_learning.algorithm\n"
+            "  Must be one of: 'gepa', 'mipro'\n"
+            "  Example:\n"
+            "    [prompt_learning]\n"
+            "    algorithm = \"gepa\""
+        )
+    elif algorithm not in ("gepa", "mipro"):
+        errors.append(
+            f"Invalid algorithm: '{algorithm}'\n"
+            f"  Must be one of: 'gepa', 'mipro' (Note: MIPRO not yet implemented)\n"
+            f"  Got: '{algorithm}'"
+        )
+    
+    # Validate task_app_url
+    task_app_url = pl_section.get("task_app_url")
+    if not task_app_url:
+        errors.append(
+            "Missing required field: prompt_learning.task_app_url\n"
+            "  Example:\n"
+            "    task_app_url = \"http://127.0.0.1:8102\""
+        )
+    elif not isinstance(task_app_url, str):
+        errors.append(
+            f"task_app_url must be a string, got {type(task_app_url).__name__}"
+        )
+    elif not task_app_url.startswith(("http://", "https://")):
+        errors.append(
+            f"task_app_url must start with http:// or https://, got: '{task_app_url}'"
+        )
+    
+    # Validate initial_prompt if present
+    initial_prompt = pl_section.get("initial_prompt")
+    if initial_prompt:
+        if not isinstance(initial_prompt, dict):
+            errors.append(
+                f"prompt_learning.initial_prompt must be a table/dict, got {type(initial_prompt).__name__}"
+            )
+        else:
+            # Validate messages array
+            messages = initial_prompt.get("messages")
+            if messages is not None:
+                if not isinstance(messages, list):
+                    errors.append(
+                        f"prompt_learning.initial_prompt.messages must be an array, got {type(messages).__name__}"
+                    )
+                elif len(messages) == 0:
+                    errors.append(
+                        "prompt_learning.initial_prompt.messages is empty (must have at least one message)"
+                    )
+    
+    # Validate policy config
+    policy = pl_section.get("policy")
+    if not policy or not isinstance(policy, dict):
+        errors.append("Missing [prompt_learning.policy] section or not a table")
+    else:
+        # Enforce inference_mode
+        mode = str(policy.get("inference_mode", "")).strip().lower()
+        if not mode:
+            errors.append("Missing required field: prompt_learning.policy.inference_mode (must be 'synth_hosted')")
+        elif mode != "synth_hosted":
+            errors.append("prompt_learning.policy.inference_mode must be 'synth_hosted' (bring_your_own unsupported)")
+        # Required fields for synth_hosted
+        provider = (policy.get("provider") or "").strip().lower()
+        model = (policy.get("model") or "").strip()
+        inference_url = (policy.get("inference_url") or "").strip()
+        if not provider:
+            errors.append("Missing required field: prompt_learning.policy.provider")
+        if not model:
+            errors.append("Missing required field: prompt_learning.policy.model")
+        if not inference_url:
+            errors.append("Missing required field: prompt_learning.policy.inference_url")
+        elif not isinstance(inference_url, str) or not inference_url.startswith(("http://", "https://")):
+            errors.append(f"policy.inference_url must start with http:// or https://, got: '{inference_url}'")
+    
+    # Validate algorithm-specific config
+    if algorithm == "gepa":
+        gepa_config = pl_section.get("gepa")
+        if not gepa_config or not isinstance(gepa_config, dict):
+            errors.append("Missing [prompt_learning.gepa] section for GEPA algorithm")
+        else:
+            # Numeric sanity checks
+            def _pos_int(name: str) -> None:
+                val = gepa_config.get(name)
+                if val is not None:
+                    try:
+                        ival = int(val)
+                        if ival <= 0:
+                            errors.append(f"prompt_learning.gepa.{name} must be > 0")
+                    except Exception:
+                        errors.append(f"prompt_learning.gepa.{name} must be an integer")
+            for fld in ("initial_population_size", "num_generations", "children_per_generation", "max_concurrent_rollouts"):
+                _pos_int(fld)
+            # Budget cap
+            if "max_spend_usd" in gepa_config and gepa_config.get("max_spend_usd") is not None:
+                try:
+                    f = float(gepa_config.get("max_spend_usd"))
+                    if f <= 0:
+                        errors.append("prompt_learning.gepa.max_spend_usd must be > 0 when provided")
+                except Exception:
+                    errors.append("prompt_learning.gepa.max_spend_usd must be numeric")
+    
+    elif algorithm == "mipro":
+        # MIPRO is not yet implemented in synth-ai
+        errors.append(
+            "MIPRO algorithm is not yet implemented in synth-ai.\n"
+            "  Please use 'gepa' algorithm for prompt optimization.\n"
+            "  MIPRO support is planned for a future release.\n"
+            "  Example:\n"
+            "    [prompt_learning]\n"
+            "    algorithm = \"gepa\"\n"
+            "    [prompt_learning.gepa]\n"
+            "    # ... gepa configuration"
+        )
+    
+    # Raise all errors at once for better UX
+    if errors:
+        _raise_validation_errors(errors, config_path)
+
+
+def _raise_validation_errors(errors: list[str], config_path: Path) -> None:
+    """Format and raise validation errors."""
+    error_msg = (
+        f"\n❌ Invalid prompt learning config: {config_path}\n\n"
+        f"Found {len(errors)} error(s):\n\n"
+    )
+    
+    for i, error in enumerate(errors, 1):
+        # Indent multi-line errors
+        indented_error = "\n  ".join(error.split("\n"))
+        error_msg += f"{i}. {indented_error}\n\n"
+    
+    error_msg += (
+        "📖 See example configs:\n"
+        "  - examples/blog_posts/gepa/configs/banking77_gepa_local.toml\n"
+        "  - examples/blog_posts/mipro/configs/banking77_mipro_local.toml\n"
+    )
+    
+    raise click.ClickException(error_msg)
+
+
+def validate_rl_config(config_data: dict[str, Any], config_path: Path) -> None:
+    """
+    Validate RL config BEFORE sending to backend.
+    
+    Args:
+        config_data: Parsed TOML/JSON config
+        config_path: Path to config file (for error messages)
+    
+    Raises:
+        ConfigValidationError: If config is invalid
+        click.ClickException: If validation fails (for CLI)
+    """
+    errors: list[str] = []
+    
+    # Check for rl section
+    rl_section = config_data.get("rl") or config_data.get("online_rl")
+    if not rl_section:
+        errors.append(
+            "Missing [rl] or [online_rl] section in config"
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    # Validate algorithm
+    algorithm = rl_section.get("algorithm")
+    if not algorithm:
+        errors.append(
+            "Missing required field: rl.algorithm\n"
+            "  Must be one of: 'grpo', 'ppo', etc."
+        )
+    
+    # Validate task_url
+    task_url = rl_section.get("task_url")
+    if not task_url:
+        errors.append(
+            "Missing required field: rl.task_url"
+        )
+    elif not isinstance(task_url, str):
+        errors.append(
+            f"task_url must be a string, got {type(task_url).__name__}"
+        )
+    
+    if errors:
+        _raise_validation_errors(errors, config_path)
+
+
+def validate_sft_config(config_data: dict[str, Any], config_path: Path) -> None:
+    """
+    Validate SFT config BEFORE sending to backend.
+    
+    Args:
+        config_data: Parsed TOML/JSON config
+        config_path: Path to config file (for error messages)
+    
+    Raises:
+        ConfigValidationError: If config is invalid
+        click.ClickException: If validation fails (for CLI)
+    """
+    errors: list[str] = []
+    
+    # Check for sft section
+    sft_section = config_data.get("sft")
+    if not sft_section:
+        errors.append(
+            "Missing [sft] section in config"
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    # Validate model
+    model = sft_section.get("model")
+    if not model:
+        errors.append(
+            "Missing required field: sft.model"
+        )
+    
+    if errors:
+        _raise_validation_errors(errors, config_path)
+
+
+__all__ = [
+    "ConfigValidationError",
+    "validate_prompt_learning_config",
+    "validate_rl_config",
+    "validate_sft_config",
+]
+

synth_ai/baseline/__init__.py

@@ -0,0 +1,25 @@
+"""Baseline file system for self-contained task evaluation.
+
+This package provides abstractions for defining and executing baseline evaluations
+without requiring deployed task apps. Supports both class-based and function-based
+task runners with first-class train/val/test split support.
+"""
+
+from __future__ import annotations
+
+from synth_ai.baseline.config import (
+    BaselineConfig,
+    BaselineResults,
+    BaselineTaskRunner,
+    DataSplit,
+    TaskResult,
+)
+
+__all__ = [
+    "BaselineConfig",
+    "BaselineTaskRunner",
+    "DataSplit",
+    "TaskResult",
+    "BaselineResults",
+]
+

synth_ai/baseline/config.py

@@ -0,0 +1,209 @@
+"""Core dataclasses for baseline configuration and results."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
+
+
+class BaselineTaskRunner:
+    """
+    Base class for task runners.
+    
+    Subclasses should implement `run_task` method for class-based approach,
+    or you can use standalone async functions for function-based approach.
+    """
+    
+    def __init__(
+        self,
+        policy_config: Dict[str, Any],
+        env_config: Dict[str, Any],
+    ):
+        """
+        Initialize task runner with configuration.
+        
+        Args:
+            policy_config: Policy configuration (model, temperature, etc.)
+            env_config: Environment configuration (max_steps, difficulty, etc.)
+        """
+        self.policy_config = policy_config
+        self.env_config = env_config
+    
+    async def run_task(self, seed: int) -> TaskResult:
+        """
+        Execute a single task instance.
+        
+        This method is called for each seed in the selected split.
+        
+        Args:
+            seed: The seed/index for this task instance
+        
+        Returns:
+            TaskResult: Structured result containing success, rewards, metadata, trace
+        """
+        raise NotImplementedError("Subclasses must implement run_task method")
+
+
+@dataclass
+class DataSplit:
+    """Definition of a data split (train/val/test)."""
+    
+    name: str  # "train", "val", "test"
+    seeds: List[int]  # Seed/index values for this split
+    metadata: Dict[str, Any] = field(default_factory=dict)  # Optional metadata
+
+
+@dataclass
+class TaskResult:
+    """Result from a single task execution."""
+    
+    # Required: Seed/index that was evaluated
+    seed: int
+    
+    # Required: Did the task complete successfully?
+    success: bool
+    
+    # Required: Outcome reward for the episode
+    outcome_reward: float
+    
+    # Optional: Event rewards (step-level)
+    event_rewards: List[Dict[str, Any]] = field(default_factory=list)
+    
+    # Optional: Total steps/turns taken
+    total_steps: int = 0
+    
+    # Optional: Metadata (achievements, completion info, etc.)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Error information if success=False
+    error: Optional[str] = None
+    
+    # Optional: v3 trace (SessionTrace dict)
+    trace: Optional[Dict[str, Any]] = None
+
+
+# Type alias for task runner (can be class or function)
+TaskRunnerType = (
+    type[BaselineTaskRunner]
+    | Callable[[int, dict[str, Any], dict[str, Any]], Any]  # Function signature
+)
+
+# Type alias for result aggregator (can be class or function)
+AggregatorType = (
+    type[Any]  # Class with aggregate() method
+    | Callable[[list[TaskResult]], dict[str, Any]]  # Function signature
+)
+
+
+@dataclass
+class BaselineConfig:
+    """Configuration for a baseline file.
+    
+    A baseline file defines how to evaluate a task without requiring
+    a deployed task app. It provides self-contained evaluation logic
+    with first-class support for train/val/test splits.
+    
+    Supports both class-based and function-based task runners:
+    - Class-based: Pass a class that inherits from BaselineTaskRunner
+    - Function-based: Pass an async function with signature:
+      async def task_runner(seed: int, policy_config: Dict[str, Any], 
+                           env_config: Dict[str, Any]) -> TaskResult
+    """
+    
+    # Required: Unique identifier for this baseline config
+    baseline_id: str
+    
+    # Required: Human-readable name
+    name: str
+    
+    # Required: Task runner (class or function)
+    # Class-based: Pass a class inheriting from BaselineTaskRunner
+    #   The class will be instantiated with policy_config and env_config,
+    #   and run_task(seed) will be called for each seed.
+    # Function-based: Pass an async function with signature:
+    #   async def task_runner(seed: int, policy_config: Dict[str, Any], 
+    #                        env_config: Dict[str, Any]) -> TaskResult
+    task_runner: TaskRunnerType
+    
+    # Required: Data splits (train/val/test)
+    splits: Dict[str, DataSplit]
+    
+    # Optional: Description for documentation
+    description: str = ""
+    
+    # Optional: Default policy configuration
+    default_policy_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Default environment configuration
+    default_env_config: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Metadata for filtering/organization
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    # Optional: Tags for filtering and discovery
+    tags: List[str] = field(default_factory=list)
+    
+    # Optional: Custom result aggregator (class or function)
+    # Class-based: Pass a class with aggregate(results: List[TaskResult]) method
+    #   The class will be instantiated and aggregate() called.
+    # Function-based: Pass a function with signature:
+    #   def aggregate_results(results: List[TaskResult]) -> Dict[str, Any]
+    result_aggregator: Optional[AggregatorType] = None
+    
+    # Optional: Path to this baseline file (set by discovery)
+    _source_path: Optional[Path] = None
+    
+    def matches_tag(self, tag: str) -> bool:
+        """Check if baseline matches a tag (case-insensitive)."""
+        return tag.lower() in [t.lower() for t in self.tags]
+    
+    def matches_metadata(self, key: str, value: Any) -> bool:
+        """Check if baseline metadata matches key-value pair."""
+        return self.metadata.get(key) == value
+
+
+@dataclass
+class BaselineResults:
+    """Aggregate results from a baseline evaluation."""
+    
+    # Configuration that was used
+    config: BaselineConfig
+    
+    # Split that was evaluated
+    split_name: str
+    
+    # Per-seed results
+    results: List[TaskResult]
+    
+    # Aggregate metrics
+    aggregate_metrics: Dict[str, Any]
+    
+    # Execution metadata
+    execution_time_seconds: float
+    model_name: str
+    timestamp: str
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary for JSON output."""
+        return {
+            "baseline_id": self.config.baseline_id,
+            "name": self.config.name,
+            "split": self.split_name,
+            "model": self.model_name,
+            "timestamp": self.timestamp,
+            "execution_time_seconds": self.execution_time_seconds,
+            "aggregate_metrics": self.aggregate_metrics,
+            "results": [
+                {
+                    "seed": r.seed,
+                    "success": r.success,
+                    "outcome_reward": r.outcome_reward,
+                    "total_steps": r.total_steps,
+                    "metadata": r.metadata,
+                    "error": r.error,
+                }
+                for r in self.results
+            ],
+        }
+