synth-ai 0.2.16__py3-none-any.whl → 0.2.19__py3-none-any.whl

synth_ai/api/train/configs/sft.py

@@ -7,7 +7,7 @@ from typing import Any
 from pydantic import Field
 
 from ..utils import load_toml
-from .shared import AlgorithmConfig, ComputeConfig, ExtraModel
+from .shared import AlgorithmConfig, ComputeConfig, ExtraModel, LoraConfig, PolicyConfig
 
 
 class JobConfig(ExtraModel):
@@ -35,6 +35,7 @@ class TrainingConfig(ExtraModel):
     mode: str | None = None
     use_qlora: bool | None = None
     validation: TrainingValidationConfig | None = None
+    lora: LoraConfig | None = None  # NEW: nested LoRA config
 
 
 class HyperparametersParallelism(ExtraModel):
@@ -65,10 +66,12 @@ class HyperparametersConfig(ExtraModel):
 class SFTConfig(ExtraModel):
     algorithm: AlgorithmConfig | None = None
     job: JobConfig
+    policy: PolicyConfig | None = None  # NEW: unified policy section
     compute: ComputeConfig | None = None
     data: SFTDataConfig | None = None
     training: TrainingConfig | None = None
     hyperparameters: HyperparametersConfig = Field(default_factory=HyperparametersConfig)
+    lora: dict[str, Any] | None = None  # DEPRECATED: use training.lora instead
     tags: dict[str, Any] | None = None
 
     def to_dict(self) -> dict[str, Any]:
@@ -76,7 +79,8 @@ class SFTConfig(ExtraModel):
 
     @classmethod
     def from_mapping(cls, data: Mapping[str, Any]) -> SFTConfig:
-        return cls.model_validate(dict(data))
+        """Load SFT config from dict/TOML mapping."""
+        return cls.model_validate(data)
 
     @classmethod
     def from_path(cls, path: Path) -> SFTConfig:

synth_ai/api/train/configs/shared.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, model_validator
 
 
 class ExtraModel(BaseModel):
@@ -15,10 +15,67 @@ class AlgorithmConfig(ExtraModel):
     variety: str
 
 
+class TopologyConfig(ExtraModel):
+    """Compute topology configuration - how GPUs are distributed across processes."""
+    type: str | None = None  # e.g., "single_node_split"
+    gpus_for_vllm: int | None = None
+    gpus_for_training: int | None = None
+    gpus_for_ref: int | None = None
+    tensor_parallel: int | None = None
+    reference_placement: str | None = None  # NEW: e.g., "none", "shared", "dedicated"
+
+
+class LoraConfig(ExtraModel):
+    """LoRA (Low-Rank Adaptation) training configuration."""
+    r: int | None = None  # Rank
+    alpha: int | None = None
+    dropout: float | None = None
+    target_modules: list[str] | None = None
+
+
 class ComputeConfig(ExtraModel):
     gpu_type: str
     gpu_count: int
     nodes: int | None = None
+    topology: TopologyConfig | None = None  # NEW: nested topology
+
+
+class PolicyConfig(ExtraModel):
+    """Unified policy configuration for both SFT and RL.
+    
+    This is the SINGLE SOURCE OF TRUTH for:
+    - What model to use (model_name or source)
+    - How to sample from it (temperature, max_tokens, etc.)
+    - How to train it (trainer_mode, label)
+    """
+    
+    # Model specification (exactly one required)
+    model_name: str | None = None  # e.g., "Qwen/Qwen3-4B"
+    source: str | None = None       # e.g., "ft:abc123" for checkpoints
+    
+    # Sampling parameters (with sensible defaults)
+    max_tokens: int = 512
+    temperature: float = 0.7
+    top_p: float = 0.95
+    top_k: int | None = None
+    repetition_penalty: float = 1.0
+    stop_sequences: list[str] | None = None
+    
+    # Training-specific
+    trainer_mode: str  # "lora", "full", "qlora"
+    label: str         # Model identifier/name
+    
+    # Optional - for distributed inference
+    inference_url: str | None = None
+    
+    @model_validator(mode="after")
+    def _ensure_exactly_one_source(self) -> PolicyConfig:
+        """Ensure exactly one of model_name or source is set."""
+        if not (bool(self.model_name) ^ bool(self.source)):
+            raise ValueError(
+                "Must set exactly one: [policy].model_name OR [policy].source"
+            )
+        return self
 
 
-__all__ = ["ExtraModel", "AlgorithmConfig", "ComputeConfig"]
+__all__ = ["ExtraModel", "AlgorithmConfig", "ComputeConfig", "PolicyConfig", "TopologyConfig", "LoraConfig"]

synth_ai/api/train/task_app.py

@@ -38,7 +38,7 @@ def _health_response_ok(resp: requests.Response | None) -> tuple[bool, str]:
     return False, ""
 
 
-def check_task_app_health(base_url: str, api_key: str, *, timeout: float = 10.0) -> TaskAppHealth:
+def check_task_app_health(base_url: str, api_key: str, *, timeout: float = 30.0) -> TaskAppHealth:
     # Send ALL known environment keys so the server can authorize any valid one
     import os
 

synth_ai/api/train/validators.py

@@ -0,0 +1,277 @@
+"""SDK-side validation for training configs - catch errors BEFORE sending to backend."""
+
+from pathlib import Path
+from typing import Any
+
+import click
+
+
+class ConfigValidationError(Exception):
+    """Raised when a training config is invalid."""
+    pass
+
+
+def validate_prompt_learning_config(config_data: dict[str, Any], config_path: Path) -> None:
+    """
+    Validate prompt learning config BEFORE sending to backend.
+    
+    This catches common errors early with clear messages instead of cryptic backend errors.
+    
+    Args:
+        config_data: Parsed TOML/JSON config
+        config_path: Path to config file (for error messages)
+    
+    Raises:
+        ConfigValidationError: If config is invalid
+        click.ClickException: If validation fails (for CLI)
+    """
+    errors: list[str] = []
+    
+    # Check for prompt_learning section
+    pl_section = config_data.get("prompt_learning")
+    if not pl_section:
+        errors.append(
+            "Missing [prompt_learning] section in config. "
+            "Expected: [prompt_learning] with algorithm, task_app_url, etc."
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    if not isinstance(pl_section, dict):
+        errors.append(
+            f"[prompt_learning] must be a table/dict, got {type(pl_section).__name__}"
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    # CRITICAL: Validate algorithm field
+    algorithm = pl_section.get("algorithm")
+    if not algorithm:
+        errors.append(
+            "Missing required field: prompt_learning.algorithm\n"
+            "  Must be one of: 'gepa', 'mipro'\n"
+            "  Example:\n"
+            "    [prompt_learning]\n"
+            "    algorithm = \"gepa\""
+        )
+    elif algorithm not in ("gepa", "mipro"):
+        errors.append(
+            f"Invalid algorithm: '{algorithm}'\n"
+            f"  Must be one of: 'gepa', 'mipro' (Note: MIPRO not yet implemented)\n"
+            f"  Got: '{algorithm}'"
+        )
+    
+    # Validate task_app_url
+    task_app_url = pl_section.get("task_app_url")
+    if not task_app_url:
+        errors.append(
+            "Missing required field: prompt_learning.task_app_url\n"
+            "  Example:\n"
+            "    task_app_url = \"http://127.0.0.1:8102\""
+        )
+    elif not isinstance(task_app_url, str):
+        errors.append(
+            f"task_app_url must be a string, got {type(task_app_url).__name__}"
+        )
+    elif not task_app_url.startswith(("http://", "https://")):
+        errors.append(
+            f"task_app_url must start with http:// or https://, got: '{task_app_url}'"
+        )
+    
+    # Validate initial_prompt if present
+    initial_prompt = pl_section.get("initial_prompt")
+    if initial_prompt:
+        if not isinstance(initial_prompt, dict):
+            errors.append(
+                f"prompt_learning.initial_prompt must be a table/dict, got {type(initial_prompt).__name__}"
+            )
+        else:
+            # Validate messages array
+            messages = initial_prompt.get("messages")
+            if messages is not None:
+                if not isinstance(messages, list):
+                    errors.append(
+                        f"prompt_learning.initial_prompt.messages must be an array, got {type(messages).__name__}"
+                    )
+                elif len(messages) == 0:
+                    errors.append(
+                        "prompt_learning.initial_prompt.messages is empty (must have at least one message)"
+                    )
+    
+    # Validate policy config
+    policy = pl_section.get("policy")
+    if not policy or not isinstance(policy, dict):
+        errors.append("Missing [prompt_learning.policy] section or not a table")
+    else:
+        # Enforce inference_mode
+        mode = str(policy.get("inference_mode", "")).strip().lower()
+        if not mode:
+            errors.append("Missing required field: prompt_learning.policy.inference_mode (must be 'synth_hosted')")
+        elif mode != "synth_hosted":
+            errors.append("prompt_learning.policy.inference_mode must be 'synth_hosted' (bring_your_own unsupported)")
+        # Required fields for synth_hosted
+        provider = (policy.get("provider") or "").strip().lower()
+        model = (policy.get("model") or "").strip()
+        inference_url = (policy.get("inference_url") or "").strip()
+        if not provider:
+            errors.append("Missing required field: prompt_learning.policy.provider")
+        if not model:
+            errors.append("Missing required field: prompt_learning.policy.model")
+        if not inference_url:
+            errors.append("Missing required field: prompt_learning.policy.inference_url")
+        elif not isinstance(inference_url, str) or not inference_url.startswith(("http://", "https://")):
+            errors.append(f"policy.inference_url must start with http:// or https://, got: '{inference_url}'")
+    
+    # Validate algorithm-specific config
+    if algorithm == "gepa":
+        gepa_config = pl_section.get("gepa")
+        if not gepa_config or not isinstance(gepa_config, dict):
+            errors.append("Missing [prompt_learning.gepa] section for GEPA algorithm")
+        else:
+            # Numeric sanity checks
+            def _pos_int(name: str) -> None:
+                val = gepa_config.get(name)
+                if val is not None:
+                    try:
+                        ival = int(val)
+                        if ival <= 0:
+                            errors.append(f"prompt_learning.gepa.{name} must be > 0")
+                    except Exception:
+                        errors.append(f"prompt_learning.gepa.{name} must be an integer")
+            for fld in ("initial_population_size", "num_generations", "children_per_generation", "max_concurrent_rollouts"):
+                _pos_int(fld)
+            # Budget cap
+            if "max_spend_usd" in gepa_config and gepa_config.get("max_spend_usd") is not None:
+                try:
+                    f = float(gepa_config.get("max_spend_usd"))
+                    if f <= 0:
+                        errors.append("prompt_learning.gepa.max_spend_usd must be > 0 when provided")
+                except Exception:
+                    errors.append("prompt_learning.gepa.max_spend_usd must be numeric")
+    
+    elif algorithm == "mipro":
+        # MIPRO is not yet implemented in synth-ai
+        errors.append(
+            "MIPRO algorithm is not yet implemented in synth-ai.\n"
+            "  Please use 'gepa' algorithm for prompt optimization.\n"
+            "  MIPRO support is planned for a future release.\n"
+            "  Example:\n"
+            "    [prompt_learning]\n"
+            "    algorithm = \"gepa\"\n"
+            "    [prompt_learning.gepa]\n"
+            "    # ... gepa configuration"
+        )
+    
+    # Raise all errors at once for better UX
+    if errors:
+        _raise_validation_errors(errors, config_path)
+
+
+def _raise_validation_errors(errors: list[str], config_path: Path) -> None:
+    """Format and raise validation errors."""
+    error_msg = (
+        f"\n❌ Invalid prompt learning config: {config_path}\n\n"
+        f"Found {len(errors)} error(s):\n\n"
+    )
+    
+    for i, error in enumerate(errors, 1):
+        # Indent multi-line errors
+        indented_error = "\n  ".join(error.split("\n"))
+        error_msg += f"{i}. {indented_error}\n\n"
+    
+    error_msg += (
+        "📖 See example configs:\n"
+        "  - examples/blog_posts/gepa/configs/banking77_gepa_local.toml\n"
+        "  - examples/blog_posts/mipro/configs/banking77_mipro_local.toml\n"
+    )
+    
+    raise click.ClickException(error_msg)
+
+
+def validate_rl_config(config_data: dict[str, Any], config_path: Path) -> None:
+    """
+    Validate RL config BEFORE sending to backend.
+    
+    Args:
+        config_data: Parsed TOML/JSON config
+        config_path: Path to config file (for error messages)
+    
+    Raises:
+        ConfigValidationError: If config is invalid
+        click.ClickException: If validation fails (for CLI)
+    """
+    errors: list[str] = []
+    
+    # Check for rl section
+    rl_section = config_data.get("rl") or config_data.get("online_rl")
+    if not rl_section:
+        errors.append(
+            "Missing [rl] or [online_rl] section in config"
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    # Validate algorithm
+    algorithm = rl_section.get("algorithm")
+    if not algorithm:
+        errors.append(
+            "Missing required field: rl.algorithm\n"
+            "  Must be one of: 'grpo', 'ppo', etc."
+        )
+    
+    # Validate task_url
+    task_url = rl_section.get("task_url")
+    if not task_url:
+        errors.append(
+            "Missing required field: rl.task_url"
+        )
+    elif not isinstance(task_url, str):
+        errors.append(
+            f"task_url must be a string, got {type(task_url).__name__}"
+        )
+    
+    if errors:
+        _raise_validation_errors(errors, config_path)
+
+
+def validate_sft_config(config_data: dict[str, Any], config_path: Path) -> None:
+    """
+    Validate SFT config BEFORE sending to backend.
+    
+    Args:
+        config_data: Parsed TOML/JSON config
+        config_path: Path to config file (for error messages)
+    
+    Raises:
+        ConfigValidationError: If config is invalid
+        click.ClickException: If validation fails (for CLI)
+    """
+    errors: list[str] = []
+    
+    # Check for sft section
+    sft_section = config_data.get("sft")
+    if not sft_section:
+        errors.append(
+            "Missing [sft] section in config"
+        )
+        _raise_validation_errors(errors, config_path)
+        return
+    
+    # Validate model
+    model = sft_section.get("model")
+    if not model:
+        errors.append(
+            "Missing required field: sft.model"
+        )
+    
+    if errors:
+        _raise_validation_errors(errors, config_path)
+
+
+__all__ = [
+    "ConfigValidationError",
+    "validate_prompt_learning_config",
+    "validate_rl_config",
+    "validate_sft_config",
+]
+

synth_ai/auth/credentials.py

@@ -0,0 +1,119 @@
+import contextlib
+import os
+import time
+import webbrowser
+
+import requests
+from requests import RequestException
+from synth_ai.utils.env import resolve_env_var, write_env_var_to_dotenv, write_env_var_to_json
+
+
+def fetch_credentials_from_web_browser_session(
+    browser: bool = True,
+    prod: bool = True
+) -> None:
+    synth_api_key = ''
+    env_api_key = ''
+    org_name = ''
+
+    if browser:
+        origin = "https://www.usesynth.ai" if prod else "http://localhost:3000"
+        init_url = f"{origin}/api/sdk/handshake/init"
+        token_url =f"{origin}/api/sdk/handshake/token"
+
+        print(f"\n🌐 Connecting to {origin} to fetch your Synth credentials")
+
+        # 1. Initialize browser handshake
+        try:
+            init_res = requests.post(init_url, timeout=10)
+        except RequestException as exc:
+            raise RuntimeError(f"Failed to reach handshake init endpoint: {exc}") from exc
+
+        if init_res.status_code != 200:
+            body = init_res.text.strip()
+            raise RuntimeError(f"Handshake init failed ({init_res.status_code}): {body or 'no response body'}")
+
+        try:
+            init_data = init_res.json()
+        except ValueError as exc:
+            raise RuntimeError("Handshake init returned malformed JSON.") from exc
+
+        device_code = str(init_data.get("device_code") or "").strip()
+        verification_uri = str(init_data.get("verification_uri") or "").strip()
+        if not device_code or not verification_uri:
+            raise RuntimeError("Handshake init response missing device_code or verification_uri.")
+
+        try:
+            expires_in = int(init_data.get("expires_in") or 600)
+        except (TypeError, ValueError):
+            expires_in = 120
+        try:
+            interval = max(int(init_data.get("interval") or 3), 1)
+        except (TypeError, ValueError):
+            interval = 3
+
+        # 2. Open browser to verification URL
+        with contextlib.suppress(Exception):
+            webbrowser.open(verification_uri)
+
+        deadline = time.time() + expires_in
+        handshake_data = None
+
+        # 3. Poll handshake token endpoint
+        while time.time() <= deadline:
+            try:
+                handshake_res = requests.post(
+                    token_url,
+                    json={"device_code": device_code},
+                    timeout=10,
+                )
+            except RequestException:
+                time.sleep(interval)
+                continue
+
+            if handshake_res.status_code == 200:
+                try:
+                    handshake_data = handshake_res.json()
+                except ValueError as exc:
+                    raise RuntimeError("Handshake token returned malformed JSON.") from exc
+                break
+
+            if handshake_res.status_code in (404, 410):
+                raise RuntimeError("Handshake failed: device code expired or was revoked.")
+
+            time.sleep(interval)
+
+        if handshake_data is None:
+            raise TimeoutError("Handshake timed out before credentials were returned.")
+
+        # 4. Extract credentials from handshake payload
+        org = handshake_data.get("org")
+        if not isinstance(org, dict):
+            org = {}
+        org_name = str(org.get("name") or "your organization").strip()
+
+        credentials = handshake_data.get("keys")
+        if not isinstance(credentials, dict):
+            credentials = {}
+
+        synth_api_key = str(credentials.get("synth") or "").strip()
+        env_api_key = str(credentials.get("rl_env") or "").strip()
+
+        print(f"\n✅ Connected to {org_name}")
+
+    # Load credentials to process environment and save credentials to .env and ~/synth-ai/config.json
+    if synth_api_key:
+        print("\nLoading SYNTH_API_KEY into process environment")
+        os.environ["SYNTH_API_KEY"] = synth_api_key
+    synth_api_key = resolve_env_var("SYNTH_API_KEY")
+    if env_api_key:
+        print("\nLoading ENVIRONMENT_API_KEY into process environment")
+        os.environ["ENVIRONMENT_API_KEY"] = env_api_key
+    env_api_key = resolve_env_var("ENVIRONMENT_API_KEY")
+
+    if browser:
+        print('')
+        write_env_var_to_json("SYNTH_API_KEY", synth_api_key, "~/.synth-ai/config.json")
+        write_env_var_to_dotenv("SYNTH_API_KEY", synth_api_key)
+        write_env_var_to_json("ENVIRONMENT_API_KEY", env_api_key, "~/.synth-ai/config.json")
+        write_env_var_to_dotenv("ENVIRONMENT_API_KEY", env_api_key)

synth_ai/baseline/__init__.py

@@ -0,0 +1,25 @@
+"""Baseline file system for self-contained task evaluation.
+
+This package provides abstractions for defining and executing baseline evaluations
+without requiring deployed task apps. Supports both class-based and function-based
+task runners with first-class train/val/test split support.
+"""
+
+from __future__ import annotations
+
+from synth_ai.baseline.config import (
+    BaselineConfig,
+    BaselineResults,
+    BaselineTaskRunner,
+    DataSplit,
+    TaskResult,
+)
+
+__all__ = [
+    "BaselineConfig",
+    "BaselineTaskRunner",
+    "DataSplit",
+    "TaskResult",
+    "BaselineResults",
+]
+