synth-ai 0.2.10__py3-none-any.whl → 0.2.13.dev1__py3-none-any.whl

synth_ai/__init__.py

@@ -2,6 +2,28 @@
 Synth AI - Software for aiding the best and multiplying the will.
 """
 
+from __future__ import annotations
+
+from importlib import metadata as _metadata
+from importlib.metadata import PackageNotFoundError
+from pathlib import Path
+
+try:  # Prefer the installed package metadata when available
+    __version__ = _metadata.version("synth-ai")
+except PackageNotFoundError:  # Fallback to pyproject version for editable installs
+    try:
+        import tomllib as _toml  # Python 3.11+
+    except ModuleNotFoundError:  # pragma: no cover - legacy interpreter guard
+        import tomli as _toml  # type: ignore[no-redef]
+
+    try:
+        pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
+        with pyproject_path.open("rb") as fh:
+            _pyproject = _toml.load(fh)
+        __version__ = str(_pyproject["project"]["version"])
+    except Exception:
+        __version__ = "0.0.0.dev0"
+
 # Environment exports - moved from synth-env
 from synth_ai.environments import *  # noqa
 import synth_ai.environments as environments  # expose module name for __all__
@@ -21,12 +43,22 @@ try:
 except Exception:
     AsyncOpenAI = OpenAI = None  # type: ignore
 
+# Judge API contract schemas
+from synth_ai.judge_schemas import (
+    JudgeScoreRequest,
+    JudgeScoreResponse,
+    JudgeOptions,
+    JudgeTaskApp,
+    JudgeTracePayload,
+    ReviewPayload,
+    CriterionScorePayload,
+)
+
 # Legacy tracing v1 is not required for v3 usage and can be unavailable in minimal envs.
 tracing = None  # type: ignore
 EventPartitionElement = RewardSignal = SystemTrace = TrainingQuestion = None  # type: ignore
 trace_event_async = trace_event_sync = upload = None  # type: ignore
 
-__version__ = "0.2.6.dev4"
 __all__ = [
     "LM",
     "OpenAI",
@@ -34,4 +66,12 @@ __all__ = [
     "Anthropic",
     "AsyncAnthropic",
     "environments",
+    # Judge API contracts
+    "JudgeScoreRequest",
+    "JudgeScoreResponse",
+    "JudgeOptions",
+    "JudgeTaskApp",
+    "JudgeTracePayload",
+    "ReviewPayload",
+    "CriterionScorePayload",
 ]  # Explicitly define public API (v1 tracing omitted in minimal env)

synth_ai/api/train/builders.py

@@ -1,23 +1,33 @@
 from __future__ import annotations
 
+import importlib
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 import click
-from synth_ai.api.models.supported import (
-    UnsupportedModelError,
-    ensure_allowed_model,
-    normalize_model_identifier,
-)
-from synth_ai.learning.sft.config import prepare_sft_job_payload
+from pydantic import ValidationError
+
+try:
+    _models_module = importlib.import_module("synth_ai.api.models.supported")
+    UnsupportedModelError = _models_module.UnsupportedModelError
+    ensure_allowed_model = _models_module.ensure_allowed_model
+    normalize_model_identifier = _models_module.normalize_model_identifier
+except Exception as exc:  # pragma: no cover - critical dependency
+    raise RuntimeError("Unable to load supported model helpers") from exc
+
+try:
+    prepare_sft_job_payload = importlib.import_module("synth_ai.learning.sft.config").prepare_sft_job_payload
+except Exception as exc:  # pragma: no cover - critical dependency
+    raise RuntimeError("Unable to load SFT payload helpers") from exc
 
 from .supported_algos import (
     AlgorithmValidationError,
     ensure_model_supported_for_algorithm,
     validate_algorithm_config,
 )
-from .utils import TrainError, ensure_api_base, load_toml
+from .utils import TrainError, ensure_api_base
+from .configs import RLConfig, SFTConfig
 
 
 @dataclass(slots=True)
@@ -34,6 +44,16 @@ class SFTBuildResult:
     validation_file: Path | None
 
 
+def _format_validation_error(path: Path, exc: ValidationError) -> str:
+    lines: list[str] = []
+    for error in exc.errors():
+        loc = ".".join(str(part) for part in error.get("loc", ()))
+        msg = error.get("msg", "invalid value")
+        lines.append(f"{loc or '<root>'}: {msg}")
+    details = "\n".join(f"  - {line}" for line in lines) or "  - Invalid configuration"
+    return f"Config validation failed ({path}):\n{details}"
+
+
 def build_rl_payload(
     *,
     config_path: Path,
@@ -42,13 +62,30 @@ def build_rl_payload(
     idempotency: str | None,
     allow_experimental: bool | None = None,
 ) -> RLBuildResult:
-    data = load_toml(config_path)
     try:
-        spec = validate_algorithm_config(data.get("algorithm"), expected_family="rl")
+        rl_cfg = RLConfig.from_path(config_path)
+    except ValidationError as exc:
+        raise click.ClickException(_format_validation_error(config_path, exc)) from exc
+
+    data = rl_cfg.to_dict()
+    # Ensure required [reference] section for backend validators
+    try:
+        ref_cfg = data.get("reference") if isinstance(data, dict) else None
+        if not isinstance(ref_cfg, dict):
+            data["reference"] = {"placement": "none"}
+        else:
+            ref_cfg.setdefault("placement", "none")
+    except Exception:
+        # Defensive: never fail builder due to optional defaults
+        data["reference"] = {"placement": "none"}
+    try:
+        spec = validate_algorithm_config(
+            rl_cfg.algorithm.model_dump(), expected_family="rl"
+        )
     except AlgorithmValidationError as exc:
         raise click.ClickException(str(exc)) from exc
     services = data.get("services") if isinstance(data.get("services"), dict) else {}
-    model_cfg = data.get("model") if isinstance(data.get("model"), dict) else {}
+    model_cfg = rl_cfg.model
 
     final_task_url = (
         overrides.get("task_url")
@@ -61,10 +98,8 @@ def build_rl_payload(
             "Task app URL required (provide --task-url or set services.task_url in TOML)"
         )
 
-    raw_source = model_cfg.get("source") if isinstance(model_cfg, dict) else ""
-    model_source = str(raw_source or "").strip()
-    raw_base = model_cfg.get("base") if isinstance(model_cfg, dict) else ""
-    model_base = str(raw_base or "").strip()
+    model_source = (model_cfg.source or "").strip()
+    model_base = (model_cfg.base or "").strip()
     override_model = (overrides.get("model") or "").strip()
     if override_model:
         model_source = override_model
@@ -122,23 +157,26 @@ def build_rl_payload(
     except Exception:
         pass
 
+    payload_data: dict[str, Any] = {
+        "endpoint_base_url": final_task_url.rstrip("/"),
+        "config": data,
+    }
     payload: dict[str, Any] = {
         "job_type": "rl",
         "compute": data.get("compute", {}),
-        "data": {
-            "endpoint_base_url": final_task_url.rstrip("/"),
-            "config": data,
-        },
+        "data": payload_data,
         "tags": {"source": "train-cli"},
     }
     if model_source:
-        payload["data"]["model"] = model_source
+        payload_data["model"] = model_source
     if model_base:
-        payload["data"]["base_model"] = model_base
+        payload_data["base_model"] = model_base
 
     backend = overrides.get("backend")
     if backend:
-        payload.setdefault("metadata", {})["backend_base_url"] = ensure_api_base(str(backend))
+        metadata_default: dict[str, Any] = {}
+        metadata = cast(dict[str, Any], payload.setdefault("metadata", metadata_default))
+        metadata["backend_base_url"] = ensure_api_base(str(backend))
 
     return RLBuildResult(payload=payload, task_url=final_task_url, idempotency=idempotency)
 
@@ -149,22 +187,23 @@ def build_sft_payload(
     dataset_override: Path | None,
     allow_experimental: bool | None,
 ) -> SFTBuildResult:
-    data = load_toml(config_path)
     try:
-        spec = validate_algorithm_config(data.get("algorithm"), expected_family="sft")
+        sft_cfg = SFTConfig.from_path(config_path)
+    except ValidationError as exc:
+        raise TrainError(_format_validation_error(config_path, exc)) from exc
+
+    data = sft_cfg.to_dict()
+    try:
+        algo_mapping = sft_cfg.algorithm.model_dump() if sft_cfg.algorithm else None
+        spec = validate_algorithm_config(algo_mapping, expected_family="sft")
     except AlgorithmValidationError as exc:
         raise TrainError(str(exc)) from exc
-    job_cfg = data.get("job") if isinstance(data.get("job"), dict) else {}
     data_cfg = data.get("data") if isinstance(data.get("data"), dict) else {}
     hp_cfg = data.get("hyperparameters") if isinstance(data.get("hyperparameters"), dict) else {}
     train_cfg = data.get("training") if isinstance(data.get("training"), dict) else {}
     compute_cfg = data.get("compute") if isinstance(data.get("compute"), dict) else {}
 
-    raw_dataset = (
-        dataset_override
-        or (job_cfg.get("data") if isinstance(job_cfg, dict) else None)
-        or (job_cfg.get("data_path") if isinstance(job_cfg, dict) else None)
-    )
+    raw_dataset = dataset_override or sft_cfg.job.data or sft_cfg.job.data_path
     if not raw_dataset:
         raise TrainError("Dataset not specified; pass --dataset or set [job].data")
     dataset_path = Path(raw_dataset)
@@ -249,9 +288,11 @@ def build_sft_payload(
             "enabled": bool(validation_cfg.get("enabled", True))
         }
 
-    raw_model = str(
-        job_cfg.get("model") if isinstance(job_cfg, dict) else None or data.get("model") or ""
-    ).strip()
+    raw_model = (sft_cfg.job.model or "").strip()
+    if not raw_model:
+        model_block = data.get("model")
+        if isinstance(model_block, str):
+            raw_model = model_block.strip()
     if not raw_model:
         raise TrainError("Model not specified; set [job].model or [model].base in the config")
 

synth_ai/api/train/cli.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
+import importlib
 import os
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
 import click
-from synth_ai.config.base_url import get_backend_from_env
+
+try:
+    _config_module = importlib.import_module("synth_ai.config.base_url")
+    get_backend_from_env = _config_module.get_backend_from_env
+except Exception as exc:  # pragma: no cover - critical dependency
+    raise RuntimeError("Unable to load backend configuration helpers") from exc
 
 from .builders import build_rl_payload, build_sft_payload
 from .config_finder import discover_configs, prompt_for_config
@@ -231,7 +238,8 @@ def train_command(
     ]
     if missing_keys:
         try:
-            from synth_ai.cli.task_apps import _interactive_fill_env
+            _task_apps_module = importlib.import_module("synth_ai.cli.task_apps")
+            _interactive_fill_env = _task_apps_module._interactive_fill_env
         except Exception as exc:  # pragma: no cover - protective fallback
             raise click.ClickException(f"Unable to prompt for env values: {exc}") from exc
 
@@ -386,9 +394,19 @@ def handle_rl(
             verify_url, headers=verify_headers, json_body={"endpoint_base_url": build.task_url}
         )
         try:
-            vjs = vresp.json()
+            parsed_json = vresp.json()
         except Exception:
-            vjs = {"status": vresp.status_code, "text": (vresp.text or "")[:400]}
+            parsed_json = None
+
+        if isinstance(parsed_json, Mapping):
+            vjs: dict[str, Any] = dict(parsed_json)
+        else:
+            vjs = {
+                "status": vresp.status_code,
+                "text": (vresp.text or "")[:400],
+            }
+            if parsed_json is not None:
+                vjs["body"] = parsed_json
     except Exception as _ve:
         raise click.ClickException(
             f"Task app verification call failed: {type(_ve).__name__}: {_ve}"
@@ -404,8 +422,13 @@ def handle_rl(
         # Print concise summary
         try:
             cands = vjs.get("candidates_first15") or []
-            attempts = vjs.get("attempts") or []
-            statuses = [a.get("status") for a in attempts]
+            attempts_raw = vjs.get("attempts")
+            attempts: list[Mapping[str, Any]] = (
+                [a for a in attempts_raw if isinstance(a, Mapping)]
+                if isinstance(attempts_raw, list)
+                else []
+            )
+            statuses = [attempt.get("status") for attempt in attempts]
             click.echo(f"Verification OK (candidates={cands}, statuses={statuses})")
         except Exception:
             pass

synth_ai/api/train/configs/__init__.py

@@ -0,0 +1,44 @@
+"""Typed training config loaders for RL and SFT jobs."""
+
+from .shared import AlgorithmConfig, ComputeConfig
+from .sft import (
+    HyperparametersConfig,
+    HyperparametersParallelism,
+    JobConfig,
+    SFTConfig,
+    SFTDataConfig,
+    TrainingConfig,
+    TrainingValidationConfig,
+)
+from .rl import (
+    EvaluationConfig,
+    JudgeConfig,
+    JudgeOptionsConfig,
+    ModelConfig,
+    RLConfig,
+    RLServicesConfig,
+    RLTrainingConfig,
+    RolloutConfig,
+    WeightSyncConfig,
+)
+
+__all__ = [
+    "AlgorithmConfig",
+    "ComputeConfig",
+    "EvaluationConfig",
+    "HyperparametersConfig",
+    "HyperparametersParallelism",
+    "JobConfig",
+    "JudgeConfig",
+    "JudgeOptionsConfig",
+    "ModelConfig",
+    "RLConfig",
+    "RLServicesConfig",
+    "RLTrainingConfig",
+    "RolloutConfig",
+    "SFTConfig",
+    "SFTDataConfig",
+    "TrainingConfig",
+    "TrainingValidationConfig",
+    "WeightSyncConfig",
+]

synth_ai/api/train/configs/rl.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Mapping
+
+from pydantic import model_validator
+
+from ..utils import load_toml
+from .shared import AlgorithmConfig, ComputeConfig, ExtraModel
+
+
+class RLServicesConfig(ExtraModel):
+    task_url: str
+    judge_url: str | None = None
+
+
+class ModelConfig(ExtraModel):
+    source: str | None = None
+    base: str | None = None
+    trainer_mode: str
+    label: str
+
+    @model_validator(mode="after")
+    def _ensure_exactly_one_source_or_base(self) -> "ModelConfig":
+        if bool(self.source) == bool(self.base):
+            raise ValueError("Config must set exactly one of [model].source or [model].base")
+        return self
+
+
+class RolloutConfig(ExtraModel):
+    env_name: str
+    policy_name: str
+    env_config: dict[str, Any] | None = None
+    policy_config: dict[str, Any] | None = None
+    max_turns: int
+    episodes_per_batch: int
+    max_concurrent_rollouts: int
+    batches_per_step: int | None = None
+    ops: list[str] | None = None
+
+
+class WeightSyncConfig(ExtraModel):
+    enable: bool | None = None
+    targets: list[str] | None = None
+    mode: str | None = None
+    direct: bool | None = None
+    verify_every_k: int | None = None
+
+
+class RLTrainingConfig(ExtraModel):
+    num_epochs: int
+    iterations_per_epoch: int
+    gradient_accumulation_steps: int | None = None
+    max_accumulated_minibatch: int | None = None
+    max_turns: int
+    batch_size: int
+    group_size: int
+    learning_rate: float
+    log_interval: int | None = None
+    weight_sync_interval: int | None = None
+    step_rewards_enabled: bool | None = None
+    step_rewards_mode: str | None = None
+    step_rewards_indicator_lambda: float | None = None
+    step_rewards_beta: float | None = None
+    step_rewards_strategy: str | None = None
+    event_rewards_kind: str | None = None
+    weight_sync: WeightSyncConfig | None = None
+
+
+class EvaluationConfig(ExtraModel):
+    instances: int
+    every_n_iters: int
+    seeds: list[int]
+
+
+class JudgeOptionsConfig(ExtraModel):
+    event: bool | None = None
+    outcome: bool | None = None
+    provider: str | None = None
+    model: str | None = None
+    rubric_id: str | None = None
+    rubric_overrides: dict[str, Any] | None = None
+    tracks: list[str] | None = None
+    weights: dict[str, float] | None = None
+    max_concurrency: int | None = None
+
+
+class JudgeConfig(ExtraModel):
+    type: str | None = None
+    timeout_s: int | None = None
+    options: JudgeOptionsConfig | None = None
+
+
+class RLConfig(ExtraModel):
+    algorithm: AlgorithmConfig
+    services: RLServicesConfig
+    compute: ComputeConfig | None = None
+    topology: dict[str, Any] | None = None
+    vllm: dict[str, Any] | None = None
+    reference: dict[str, Any] | None = None
+    model: ModelConfig
+    lora: dict[str, Any] | None = None
+    rollout: RolloutConfig | None = None
+    evaluation: EvaluationConfig | None = None
+    training: RLTrainingConfig | None = None
+    rubric: dict[str, Any] | None = None
+    judge: JudgeConfig | None = None
+    tags: dict[str, Any] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.model_dump(mode="python", exclude_none=True)
+
+    @classmethod
+    def from_mapping(cls, data: Mapping[str, Any]) -> "RLConfig":
+        return cls.model_validate(dict(data))
+
+    @classmethod
+    def from_path(cls, path: Path) -> "RLConfig":
+        content = load_toml(path)
+        return cls.from_mapping(content)
+
+
+__all__ = [
+    "EvaluationConfig",
+    "JudgeConfig",
+    "JudgeOptionsConfig",
+    "ModelConfig",
+    "RLConfig",
+    "RLServicesConfig",
+    "RLTrainingConfig",
+    "RolloutConfig",
+    "WeightSyncConfig",
+]

synth_ai/api/train/configs/sft.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Mapping
+
+from pydantic import Field
+
+from ..utils import load_toml
+from .shared import AlgorithmConfig, ComputeConfig, ExtraModel
+
+
+class JobConfig(ExtraModel):
+    model: str
+    data: str | None = None
+    data_path: str | None = None
+    poll_seconds: int | None = None
+
+
+class SFTDataConfig(ExtraModel):
+    topology: dict[str, Any] | None = None
+    validation_path: str | None = None
+
+
+class TrainingValidationConfig(ExtraModel):
+    enabled: bool | None = None
+    evaluation_strategy: str | None = None
+    eval_steps: int | None = None
+    save_best_model_at_end: bool | None = None
+    metric_for_best_model: str | None = None
+    greater_is_better: bool | None = None
+
+
+class TrainingConfig(ExtraModel):
+    mode: str | None = None
+    use_qlora: bool | None = None
+    validation: TrainingValidationConfig | None = None
+
+
+class HyperparametersParallelism(ExtraModel):
+    use_deepspeed: bool | None = None
+    deepspeed_stage: int | None = None
+    fsdp: bool | None = None
+    bf16: bool | None = None
+    fp16: bool | None = None
+    activation_checkpointing: bool | None = None
+    tensor_parallel_size: int | None = None
+    pipeline_parallel_size: int | None = None
+
+
+class HyperparametersConfig(ExtraModel):
+    n_epochs: int = 1
+    batch_size: int | None = None
+    global_batch: int | None = None
+    per_device_batch: int | None = None
+    gradient_accumulation_steps: int | None = None
+    sequence_length: int | None = None
+    learning_rate: float | None = None
+    warmup_ratio: float | None = None
+    train_kind: str | None = None
+    weight_decay: float | None = None
+    parallelism: HyperparametersParallelism | None = None
+
+
+class SFTConfig(ExtraModel):
+    algorithm: AlgorithmConfig | None = None
+    job: JobConfig
+    compute: ComputeConfig | None = None
+    data: SFTDataConfig | None = None
+    training: TrainingConfig | None = None
+    hyperparameters: HyperparametersConfig = Field(default_factory=HyperparametersConfig)
+    tags: dict[str, Any] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.model_dump(mode="python", exclude_none=True)
+
+    @classmethod
+    def from_mapping(cls, data: Mapping[str, Any]) -> "SFTConfig":
+        return cls.model_validate(dict(data))
+
+    @classmethod
+    def from_path(cls, path: Path) -> "SFTConfig":
+        content = load_toml(path)
+        return cls.from_mapping(content)
+
+
+__all__ = [
+    "HyperparametersConfig",
+    "HyperparametersParallelism",
+    "JobConfig",
+    "SFTConfig",
+    "SFTDataConfig",
+    "TrainingConfig",
+    "TrainingValidationConfig",
+]

synth_ai/api/train/configs/shared.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ExtraModel(BaseModel):
+    """Base model that tolerates unknown keys so configs keep forward compatibility."""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class AlgorithmConfig(ExtraModel):
+    type: str
+    method: str
+    variety: str
+
+
+class ComputeConfig(ExtraModel):
+    gpu_type: str
+    gpu_count: int
+    nodes: int | None = None
+
+
+__all__ = ["ExtraModel", "AlgorithmConfig", "ComputeConfig"]

synth_ai/api/train/env_resolver.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import importlib
 import os
 from collections.abc import Callable, Iterable, MutableMapping
 from dataclasses import dataclass
@@ -11,6 +12,18 @@ from . import task_app
 from .utils import REPO_ROOT, mask_value, read_env_file, write_env_value
 
 
+def _load_saved_env_path() -> Path | None:
+    try:
+        module = importlib.import_module("synth_ai.demos.demo_task_apps.core")
+        loader = module.load_env_file_path
+        saved_path = loader()
+        if saved_path:
+            return Path(saved_path)
+    except Exception:
+        return None
+    return None
+
+
 @dataclass(slots=True)
 class KeySpec:
     name: str
@@ -156,25 +169,11 @@ def resolve_env(
                 raise click.ClickException(f"Env file not found: {path}")
         resolver = EnvResolver(provided)
     else:
-        # Check for saved .env path from demo command
-        try:
-            from synth_ai.demos.demo_task_apps.core import load_env_file_path
-
-            saved_env_path = load_env_file_path()
-            if saved_env_path:
-                saved_path = Path(saved_env_path)
-                if saved_path.exists():
-                    click.echo(f"Using .env file: {saved_path}")
-                    resolver = EnvResolver([saved_path])
-                else:
-                    # Saved path no longer exists, fall back to prompt
-                    resolver = EnvResolver(_collect_default_candidates(config_path))
-                    resolver.select_new_env()
-            else:
-                resolver = EnvResolver(_collect_default_candidates(config_path))
-                resolver.select_new_env()
-        except Exception:
-            # If import fails or any error, fall back to original behavior
+        saved_path = _load_saved_env_path()
+        if saved_path and saved_path.exists():
+            click.echo(f"Using .env file: {saved_path}")
+            resolver = EnvResolver([saved_path])
+        else:
             resolver = EnvResolver(_collect_default_candidates(config_path))
             resolver.select_new_env()