synth-ai 0.2.16__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/cli/commands/train/validation.py

@@ -0,0 +1,443 @@
+"""TOML validation logic for train commands (SFT and RL)."""
+
+from collections.abc import MutableMapping
+from pathlib import Path
+from typing import Any
+
+from pydantic import ValidationError
+from synth_ai.api.train.configs.rl import RLConfig
+from synth_ai.api.train.configs.sft import SFTConfig
+from synth_ai.api.train.utils import load_toml
+
+from .errors import (
+    InvalidJudgeConfigError,
+    InvalidRLConfigError,
+    InvalidRubricConfigError,
+    InvalidSFTConfigError,
+    MissingAlgorithmError,
+    MissingComputeError,
+    MissingDatasetError,
+    MissingModelError,
+    TomlParseError,
+    UnsupportedAlgorithmError,
+)
+from .judge_validation import extract_and_validate_judge_rubric
+
+__all__ = [
+    "validate_sft_config",
+    "validate_rl_config",
+    "load_and_validate_sft",
+    "load_and_validate_rl",
+]
+
+
+def validate_sft_config(config: MutableMapping[str, Any]) -> dict[str, Any]:
+    """Validate SFT configuration from TOML.
+    
+    Args:
+        config: Raw configuration dictionary from TOML
+        
+    Returns:
+        Validated configuration dictionary
+        
+    Raises:
+        InvalidSFTConfigError: If validation fails
+        MissingAlgorithmError: If algorithm section is missing or invalid
+        MissingModelError: If model is not specified
+        MissingDatasetError: If dataset path is not specified
+        MissingComputeError: If compute section is missing required fields
+    """
+    # Check for required top-level sections
+    if "algorithm" not in config or not config["algorithm"]:
+        raise MissingAlgorithmError(
+            detail="[algorithm] section is required for SFT configs"
+        )
+    
+    if "job" not in config or not config["job"]:
+        raise InvalidSFTConfigError(
+            detail="[job] section is required for SFT configs"
+        )
+    
+    job = config.get("job", {})
+    if not job.get("model"):
+        raise MissingModelError(
+            detail="[job].model is required (e.g., 'Qwen/Qwen3-4B')"
+        )
+    
+    # Check that at least one dataset source is specified
+    if not (job.get("data") or job.get("data_path")):
+        raise MissingDatasetError(
+            detail="[job].data or [job].data_path must be specified",
+            hint="Provide path to training JSONL file"
+        )
+    
+    # Validate algorithm type, method, and variety
+    algorithm = config.get("algorithm", {})
+    if algorithm.get("type") not in {"offline", None}:
+        raise UnsupportedAlgorithmError(
+            algorithm_type=algorithm.get("type", "unknown"),
+            expected="offline",
+            hint="SFT requires algorithm.type = 'offline'"
+        )
+    
+    method = algorithm.get("method", "")
+    if method and method not in {"sft", "supervised_finetune"}:
+        raise UnsupportedAlgorithmError(
+            algorithm_type=method,
+            expected="sft or supervised_finetune",
+            hint="SFT requires algorithm.method = 'sft' or 'supervised_finetune'"
+        )
+    
+    # Validate variety is present
+    if not algorithm.get("variety"):
+        raise MissingAlgorithmError(
+            detail="[algorithm].variety is required (e.g., 'fft', 'lora', 'qlora')"
+        )
+    
+    # Validate compute section
+    compute = config.get("compute", {})
+    if not compute:
+        raise MissingComputeError(
+            detail="[compute] section is required",
+            hint="Specify gpu_type, gpu_count, and nodes"
+        )
+    
+    if not compute.get("gpu_type"):
+        raise MissingComputeError(
+            detail="[compute].gpu_type is required (e.g., 'H100', 'A100')"
+        )
+    
+    if not compute.get("gpu_count"):
+        raise MissingComputeError(
+            detail="[compute].gpu_count is required"
+        )
+    
+    # Validate using Pydantic model
+    try:
+        validated = SFTConfig.from_mapping(config)
+        return validated.to_dict()
+    except ValidationError as exc:
+        errors = []
+        for error in exc.errors():
+            loc = ".".join(str(x) for x in error["loc"])
+            msg = error["msg"]
+            errors.append(f"  • {loc}: {msg}")
+        raise InvalidSFTConfigError(
+            detail="Pydantic validation failed:\n" + "\n".join(errors)
+        ) from exc
+
+
+def validate_rl_config(config: MutableMapping[str, Any]) -> dict[str, Any]:
+    """Validate RL configuration from TOML.
+    
+    Args:
+        config: Raw configuration dictionary from TOML
+        
+    Returns:
+        Validated configuration dictionary
+        
+    Raises:
+        InvalidRLConfigError: If validation fails
+        MissingAlgorithmError: If algorithm section is missing or invalid
+        MissingModelError: If model is not specified
+        MissingComputeError: If compute section is missing required fields
+    """
+    # Check for required top-level sections
+    if "algorithm" not in config or not config["algorithm"]:
+        raise MissingAlgorithmError(
+            detail="[algorithm] section is required for RL configs"
+        )
+    
+    # Check for model OR policy (policy is the new format)
+    if "policy" not in config and "model" not in config:
+        raise MissingModelError(
+            detail="[policy] or [model] section is required for RL configs"
+        )
+    
+    # Validate algorithm type, method, and variety
+    algorithm = config.get("algorithm", {})
+    if algorithm.get("type") not in {"online", None}:
+        raise UnsupportedAlgorithmError(
+            algorithm_type=algorithm.get("type", "unknown"),
+            expected="online",
+            hint="RL requires algorithm.type = 'online'"
+        )
+    
+    method = algorithm.get("method", "")
+    if method and method not in {"policy_gradient", "ppo", "gspo"}:
+        raise UnsupportedAlgorithmError(
+            algorithm_type=method,
+            expected="policy_gradient",
+            hint="RL requires algorithm.method = 'policy_gradient'"
+        )
+    
+    # Validate variety is present
+    if not algorithm.get("variety"):
+        raise MissingAlgorithmError(
+            detail="[algorithm].variety is required (e.g., 'gspo', 'ppo')"
+        )
+    
+    # Validate model/policy section
+    model = config.get("model", {})
+    policy = config.get("policy", {})
+    
+    # Use policy if available, otherwise fall back to model
+    if policy:
+        if not policy.get("model_name") and not policy.get("source"):
+            raise MissingModelError(
+                detail="[policy].model_name or [policy].source must be specified",
+                hint="Provide base model (e.g., 'Qwen/Qwen3-4B') or source checkpoint"
+            )
+        
+        if not policy.get("trainer_mode"):
+            raise InvalidRLConfigError(
+                detail="[policy].trainer_mode is required (e.g., 'full', 'lora')"
+            )
+        
+        if not policy.get("label"):
+            raise InvalidRLConfigError(
+                detail="[policy].label is required (e.g., 'my-rl-model')",
+                hint="Provide a descriptive label for this model"
+            )
+    elif model:
+        if not model.get("base") and not model.get("source"):
+            raise MissingModelError(
+                detail="[model].base or [model].source must be specified",
+                hint="Provide base model (e.g., 'Qwen/Qwen3-4B') or source checkpoint"
+            )
+        
+        if not model.get("trainer_mode"):
+            raise InvalidRLConfigError(
+                detail="[model].trainer_mode is required (e.g., 'full', 'lora')"
+            )
+        
+        if not model.get("label"):
+            raise InvalidRLConfigError(
+                detail="[model].label is required (e.g., 'my-rl-model')",
+                hint="Provide a descriptive label for this model"
+            )
+    
+    # Validate compute section
+    compute = config.get("compute", {})
+    if not compute:
+        raise MissingComputeError(
+            detail="[compute] section is required",
+            hint="Specify gpu_type and gpu_count"
+        )
+    
+    if not compute.get("gpu_type"):
+        raise MissingComputeError(
+            detail="[compute].gpu_type is required (e.g., 'H100', 'A100')"
+        )
+    
+    if not compute.get("gpu_count"):
+        raise MissingComputeError(
+            detail="[compute].gpu_count is required"
+        )
+    
+    # Check for rollout configuration
+    rollout = config.get("rollout", {})
+    if not rollout:
+        raise InvalidRLConfigError(
+            detail="[rollout] section is required for RL configs",
+            hint="Specify env_name, policy_name, max_turns, etc."
+        )
+    
+    if not rollout.get("env_name"):
+        raise InvalidRLConfigError(
+            detail="[rollout].env_name is required (e.g., 'math', 'crafter')"
+        )
+    
+    if not rollout.get("policy_name"):
+        raise InvalidRLConfigError(
+            detail="[rollout].policy_name is required"
+        )
+    
+    # Validate topology section (can be top-level or under compute)
+    topology = config.get("topology") or compute.get("topology", {})
+    if not topology:
+        raise InvalidRLConfigError(
+            detail="[topology] or [compute.topology] section is required",
+            hint="Specify gpus_for_vllm, gpus_for_training, etc."
+        )
+    
+    # Validate vllm section and tensor_parallel consistency
+    vllm = config.get("vllm", {})
+    topology_tensor_parallel = topology.get("tensor_parallel")
+    vllm_tensor_parallel = vllm.get("tensor_parallel_size")
+    
+    if topology_tensor_parallel and not vllm_tensor_parallel:
+        raise InvalidRLConfigError(
+            detail="Both [topology].tensor_parallel and [vllm].tensor_parallel_size must be provided",
+            hint=f"Add [vllm] section with tensor_parallel_size={topology_tensor_parallel}"
+        )
+    
+    if vllm_tensor_parallel and not topology_tensor_parallel:
+        raise InvalidRLConfigError(
+            detail="Both [topology].tensor_parallel and [vllm].tensor_parallel_size must be provided",
+            hint=f"Add tensor_parallel={vllm_tensor_parallel} to [topology] section"
+        )
+    
+    # Check for training section and its required fields
+    training = config.get("training", {})
+    if training:
+        required_training_fields = {
+            "num_epochs": "number of training epochs",
+            "iterations_per_epoch": "iterations per epoch",
+            "max_turns": "maximum turns",
+            "batch_size": "batch size",
+            "group_size": "group size",
+            "learning_rate": "learning rate",
+            "weight_sync_interval": "weight sync interval",
+            "log_interval": "logging interval",
+        }
+        
+        for field, description in required_training_fields.items():
+            if field not in training:
+                raise InvalidRLConfigError(
+                    detail=f"[training].{field} is required ({description})",
+                    hint=f"Add {field} to the [training] section"
+                )
+        
+        # Validate weight_sync_interval is positive
+        weight_sync_interval = training.get("weight_sync_interval")
+        if weight_sync_interval is not None and weight_sync_interval <= 0:
+            raise InvalidRLConfigError(
+                detail="[training].weight_sync_interval must be a positive integer",
+                hint="Set weight_sync_interval to a value >= 1"
+            )
+        
+        # Ensure weight_sync block exists with proper defaults
+        # Backend requires mode="direct" - always inject it
+        if "weight_sync" not in training:
+            training["weight_sync"] = {
+                "enable": True,
+                "mode": "direct",  # Backend requirement
+                "targets": ["policy"],
+                "interval": training.get("weight_sync_interval", 1),
+            }
+        else:
+            weight_sync = training["weight_sync"]
+            # Always force mode to "direct" (backend requirement)
+            weight_sync["mode"] = "direct"
+            
+            # Validate existing weight_sync block
+            if not weight_sync.get("enable"):
+                raise InvalidRLConfigError(
+                    detail="[training.weight_sync].enable must be true",
+                    hint="Set enable=true in the weight_sync section"
+                )
+            targets = weight_sync.get("targets", [])
+            if not targets or "policy" not in targets:
+                raise InvalidRLConfigError(
+                    detail="[training.weight_sync].targets must include 'policy'",
+                    hint="Add targets=['policy'] to the weight_sync section"
+                )
+            # Inject interval if not present
+            if "interval" not in weight_sync:
+                weight_sync["interval"] = training.get("weight_sync_interval", 1)
+    
+    # Check for evaluation section
+    evaluation = config.get("evaluation", {})
+    if evaluation:
+        required_eval_fields = {
+            "instances": "number of evaluation instances",
+            "every_n_iters": "evaluation frequency",
+            "seeds": "evaluation seeds",
+        }
+        
+        for field, description in required_eval_fields.items():
+            if field not in evaluation:
+                raise InvalidRLConfigError(
+                    detail=f"[evaluation].{field} is required ({description})",
+                    hint=f"Add {field} to the [evaluation] section"
+                )
+    
+    # Inject services section if not present (will be populated at runtime)
+    if "services" not in config:
+        config["services"] = {
+            "task_url": "placeholder",  # Will be resolved at runtime
+        }
+    
+    # Inject reference placement if not present (like builders.py does)
+    # Reference is now under compute.topology.reference_placement
+    if "compute" not in config:
+        config["compute"] = {}
+    if "topology" not in config["compute"]:
+        config["compute"]["topology"] = {}
+    if "reference_placement" not in config["compute"]["topology"]:
+        config["compute"]["topology"]["reference_placement"] = "none"
+    
+    # Validate judge/rubric configuration with formalized Pydantic models
+    # This will emit deprecation warnings for dead fields and validate structure
+    try:
+        rubric_config, judge_config = extract_and_validate_judge_rubric(config)
+        # Validation passed - configs are clean and ready for use
+        # The validated Pydantic models can be used by training code if needed
+    except (InvalidJudgeConfigError, InvalidRubricConfigError) as exc:
+        raise InvalidRLConfigError(
+            detail=f"Judge/Rubric validation failed: {exc.detail}",
+            hint="Check JUDGE_RUBRIC_CLEANUP_GUIDE.md for migration help."
+        ) from exc
+    
+    # Validate using Pydantic model
+    try:
+        validated = RLConfig.from_mapping(config)
+        return validated.to_dict()
+    except ValidationError as exc:
+        errors = []
+        for error in exc.errors():
+            loc = ".".join(str(x) for x in error["loc"])
+            msg = error["msg"]
+            errors.append(f"  • {loc}: {msg}")
+        raise InvalidRLConfigError(
+            detail="Pydantic validation failed:\n" + "\n".join(errors)
+        ) from exc
+
+
+def load_and_validate_sft(config_path: Path) -> dict[str, Any]:
+    """Load and validate an SFT TOML configuration file.
+    
+    Args:
+        config_path: Path to TOML configuration file
+        
+    Returns:
+        Validated configuration dictionary
+        
+    Raises:
+        TomlParseError: If TOML parsing fails
+        InvalidSFTConfigError: If validation fails
+    """
+    try:
+        raw_config = load_toml(config_path)
+    except Exception as exc:
+        raise TomlParseError(
+            path=str(config_path),
+            detail=str(exc)
+        ) from exc
+    
+    return validate_sft_config(raw_config)
+
+
+def load_and_validate_rl(config_path: Path) -> dict[str, Any]:
+    """Load and validate an RL TOML configuration file.
+    
+    Args:
+        config_path: Path to TOML configuration file
+        
+    Returns:
+        Validated configuration dictionary
+        
+    Raises:
+        TomlParseError: If TOML parsing fails
+        InvalidRLConfigError: If validation fails
+    """
+    try:
+        raw_config = load_toml(config_path)
+    except Exception as exc:
+        raise TomlParseError(
+            path=str(config_path),
+            detail=str(exc)
+        ) from exc
+    
+    return validate_rl_config(raw_config)

synth_ai/cli/demo.py

@@ -1,165 +1,5 @@
-#!/usr/bin/env python3
-"""
-CLI: interactive launcher for example demos and RL demo helpers.
-
-- `synth-ai demo` (no subcommand) -> initialize RL demo files into ./synth_demo/
-- `synth-ai demo deploy|configure|run` -> invoke RL demo helpers directly.
-"""
-
 from __future__ import annotations
 
-import importlib
-import os
-import subprocess
-from pathlib import Path
-from typing import Any, cast
-
-import click
-from click.exceptions import Exit
-
-demo_commands = cast(
-    Any, importlib.import_module("synth_ai.demos.core.cli")
-)
-
-
-def _find_demo_scripts(root: Path) -> list[Path]:
-    if not root.exists():
-        return []
-    return sorted([p for p in root.rglob("run_demo.sh") if p.is_file()])
-
-
-def _run_demo_command(func, *args, **kwargs) -> None:
-    """Invoke a demo command and exit via Click on non-zero status codes."""
-
-    try:
-        result = func(*args, **kwargs)
-    except SystemExit as exc:  # pragma: no cover - defensive
-        raise Exit(exc.code or 1) from exc
-
-    if result is None:
-        return
-
-    try:
-        code = int(result)
-    except (TypeError, ValueError):
-        return
-    if code != 0:
-        raise Exit(code)
-
-
-def register(cli):
-    @cli.group("demo", invoke_without_command=True)
-    @click.option(
-        "--force", is_flag=True, help="Overwrite existing files in CWD when initializing demo"
-    )
-    @click.option("--list", "list_only", is_flag=True, help="List available legacy demos and exit")
-    @click.option("-f", "filter_term", default="", help="Filter legacy demos by substring")
-    @click.pass_context
-    def demo(ctx: click.Context, force: bool, list_only: bool, filter_term: str):
-        """Demo helpers.
-
-        - Default (no subcommand): initialize RL demo files into ./synth_demo/ (alias of rl_demo init)
-        - Legacy mode: with --list, find and run examples/*/run_demo.sh
-        - New RL demo subcommands: deploy, configure, run
-        """
-        if ctx.invoked_subcommand is not None:
-            return
-
-        # If explicitly asked to list legacy demos, show interactive picker
-        if list_only:
-            repo_root = Path(os.getcwd())
-            examples_dir = repo_root / "examples"
-            demos = _find_demo_scripts(examples_dir)
-            if filter_term:
-                demos = [p for p in demos if filter_term.lower() in str(p).lower()]
-
-            if not demos:
-                click.echo("No run_demo.sh scripts found under examples/.")
-                return
-
-            click.echo("Available demos:")
-            for idx, p in enumerate(demos, start=1):
-                click.echo(f" {idx}. {p.relative_to(repo_root)}")
-            click.echo("")
-
-            def _validate_choice(val: str) -> int:
-                try:
-                    i = int(val)
-                except Exception as err:
-                    raise click.BadParameter("Enter a number from the list") from err
-                if i < 1 or i > len(demos):
-                    raise click.BadParameter(f"Choose a number between 1 and {len(demos)}")
-                return i
-
-            choice = click.prompt("Select a demo to run", value_proc=_validate_choice)
-            script = demos[choice - 1]
-
-            click.echo("")
-            click.echo(f"🚀 Running {script.relative_to(repo_root)}\n")
-
-            try:
-                subprocess.run(["bash", str(script)], check=True)
-            except subprocess.CalledProcessError as e:
-                click.echo(f"❌ Demo exited with non-zero status: {e.returncode}")
-            except KeyboardInterrupt:
-                click.echo("\n🛑 Demo interrupted by user")
-            return
-
-        # Default: initialize RL demo files via new command
-        _run_demo_command(demo_commands.init, force=force)
-
-    # (prepare command removed; configure now prepares baseline TOML)
-
-    # Help pyright understand dynamic Click group attributes
-    _dg = cast(Any, demo)
-
-    @_dg.command("deploy")
-    @click.option("--local", is_flag=True, help="Run local FastAPI instead of Modal deploy")
-    @click.option(
-        "--app",
-        type=click.Path(),
-        default=None,
-        help="Path to Modal app.py for uv run modal deploy",
-    )
-    @click.option("--name", type=str, default="synth-math-demo", help="Modal app name")
-    @click.option(
-        "--script",
-        type=click.Path(),
-        default=None,
-        help="Path to deploy_task_app.sh (optional legacy)",
-    )
-    def demo_deploy(local: bool, app: str | None, name: str, script: str | None):
-        _run_demo_command(
-            demo_commands.deploy,
-            local=local,
-            app=app,
-            name=name,
-            script=script,
-        )
-
-    @_dg.command("configure")
-    def demo_configure():
-        _run_demo_command(demo_commands.run)
-
-    @_dg.command("setup")
-    def demo_setup():
-        _run_demo_command(demo_commands.setup)
-
-    @_dg.command("run")
-    @click.option("--batch-size", type=int, default=None)
-    @click.option("--group-size", type=int, default=None)
-    @click.option("--model", type=str, default=None)
-    @click.option("--timeout", type=int, default=600)
-    def demo_run(batch_size: int | None, group_size: int | None, model: str | None, timeout: int):
-        _run_demo_command(
-            demo_commands.run,
-            batch_size=batch_size,
-            group_size=group_size,
-            model=model,
-            timeout=timeout,
-        )
+from synth_ai.cli.commands.demo.core import register
 
-    @cli.command("setup")
-    def setup_alias():
-        """Perform SDK handshake and write keys to .env."""
-        _run_demo_command(demo_commands.setup)
+__all__ = ["register"]

synth_ai/cli/deploy/__init__.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from .core import command, get_command
+from .errors import (
+    DeployCliError,
+    EnvFileDiscoveryError,
+    EnvironmentKeyLoadError,
+    EnvKeyPreflightError,
+    MissingEnvironmentApiKeyError,
+    ModalCliResolutionError,
+    ModalExecutionError,
+    TaskAppNotFoundError,
+)
+from .validation import validate_deploy_options
+
+__all__ = [
+    "command",
+    "get_command",
+    "DeployCliError",
+    "MissingEnvironmentApiKeyError",
+    "EnvironmentKeyLoadError",
+    "EnvFileDiscoveryError",
+    "TaskAppNotFoundError",
+    "ModalCliResolutionError",
+    "ModalExecutionError",
+    "EnvKeyPreflightError",
+    "validate_deploy_options",
+]

synth_ai/cli/deploy/core.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from synth_ai.cli.commands.deploy.core import command, get_command
+
+__all__ = ["command", "get_command"]

synth_ai/cli/deploy/errors.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from synth_ai.cli.commands.deploy.errors import (
+    DeployCliError,
+    EnvFileDiscoveryError,
+    EnvironmentKeyLoadError,
+    EnvKeyPreflightError,
+    MissingEnvironmentApiKeyError,
+    ModalCliResolutionError,
+    ModalExecutionError,
+    TaskAppNotFoundError,
+)
+
+__all__ = [
+    "DeployCliError",
+    "MissingEnvironmentApiKeyError",
+    "EnvironmentKeyLoadError",
+    "EnvFileDiscoveryError",
+    "TaskAppNotFoundError",
+    "ModalCliResolutionError",
+    "ModalExecutionError",
+    "EnvKeyPreflightError",
+]

synth_ai/cli/deploy/validation.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from synth_ai.cli.commands.deploy.validation import validate_deploy_options
+
+__all__ = ["validate_deploy_options"]