synth-ai 0.2.9.dev4__py3-none-any.whl → 0.2.9.dev6__py3-none-any.whl

synth_ai/api/models/supported.py

@@ -0,0 +1,376 @@
+"""Catalog of Synth-hosted base models and helpers (core vs experimental)."""
+
+from __future__ import annotations
+
+import os
+import warnings
+from collections.abc import Iterable, Iterator, Sequence
+from dataclasses import dataclass
+
+# ------------------------------------------------------------------------------
+# Model families
+# ------------------------------------------------------------------------------
+
+QWEN3_MODELS: list[str] = [
+    # Core Qwen3 base models
+    "Qwen/Qwen3-0.6B",
+    "Qwen/Qwen3-1.7B",
+    "Qwen/Qwen3-4B",
+    "Qwen/Qwen3-8B",
+    "Qwen/Qwen3-14B",
+    "Qwen/Qwen3-30B-A3B",
+    "Qwen/Qwen3-32B",
+    # Include 4B-2507 and Thinking variants used in RL
+    "Qwen/Qwen3-4B-Thinking-2507",
+    "Qwen/Qwen3-30B-A3B-Thinking-2507",
+    "Qwen/Qwen3-235B-A22B-Thinking-2507",
+]
+
+# Qwen3 Coder family (backend-supported); text-only, SFT/inference
+QWEN3_CODER_MODELS: list[str] = [
+    # Instruct variants used for coding tasks
+    "Qwen/Qwen3-Coder-30B-A3B-Instruct",
+    "Qwen/Qwen3-Coder-480B-A35B-Instruct",
+]
+
+# Training support sets
+RL_SUPPORTED_MODELS: frozenset[str] = frozenset(
+    {
+        "Qwen/Qwen3-1.7B",
+        "Qwen/Qwen3-4B",
+        "Qwen/Qwen3-4B-Thinking-2507",
+        "Qwen/Qwen3-8B",
+        "Qwen/Qwen3-14B",
+        "Qwen/Qwen3-30B-A3B",
+        "Qwen/Qwen3-30B-A3B-Thinking-2507",
+    }
+)
+
+# SFT allowlist includes core Qwen3 plus Coder family
+SFT_SUPPORTED_MODELS: frozenset[str] = frozenset([*QWEN3_MODELS, *QWEN3_CODER_MODELS])
+
+# ------------------------------------------------------------------------------
+# Lifecycle classification (core vs experimental)
+# ------------------------------------------------------------------------------
+
+# Which base models are considered "experimental" by default.
+_EXPERIMENTAL_DEFAULTS: frozenset[str] = frozenset(
+    {
+        # Larger (>= 64B) or bleeding-edge variants are experimental by default.
+        "Qwen/Qwen3-235B-A22B-Thinking-2507",
+        "Qwen/Qwen3-Coder-480B-A35B-Instruct",
+        # Thinking variants can fluctuate more rapidly.
+        "Qwen/Qwen3-30B-A3B-Thinking-2507",
+        "Qwen/Qwen3-4B-Thinking-2507",
+    }
+)
+
+
+def _parse_experimental_env() -> frozenset[str]:
+    raw = os.getenv("SDK_EXPERIMENTAL_MODELS", "").strip()
+    if not raw:
+        return frozenset()
+    return frozenset(s.strip() for s in raw.split(",") if s.strip())
+
+
+# Final experimental set (defaults ∪ optional env override)
+EXPERIMENTAL_MODELS: frozenset[str] = frozenset(_EXPERIMENTAL_DEFAULTS | _parse_experimental_env())
+
+# Build catalog entries for both core and coder families under unified "Qwen3"
+_ALL_QWEN3_IDS: list[str] = [*QWEN3_MODELS, *QWEN3_CODER_MODELS]
+
+CORE_MODELS: frozenset[str] = frozenset(m for m in _ALL_QWEN3_IDS if m not in EXPERIMENTAL_MODELS)
+
+# ------------------------------------------------------------------------------
+# Experimental gating / warnings
+# ------------------------------------------------------------------------------
+
+
+class ExperimentalWarning(UserWarning):
+    """Warning for usage of experimental SDK models/APIs."""
+
+
+def _experimental_enabled() -> bool:
+    # Global toggle to permit experimental usage
+    return os.getenv("SDK_EXPERIMENTAL", "0") == "1"
+
+
+def _warn_if_experimental(model_id: str) -> None:
+    if model_id in EXPERIMENTAL_MODELS:
+        warnings.warn(
+            f"Model '{model_id}' is experimental and may change or be removed.",
+            category=ExperimentalWarning,
+            stacklevel=2,
+        )
+
+
+# ------------------------------------------------------------------------------
+# Model metadata + catalog
+# ------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class SupportedModel:
+    """Metadata describing a supported base model."""
+
+    model_id: str
+    family: str
+    provider: str
+    modalities: tuple[str, ...] = ()
+    training_modes: tuple[str, ...] = ()
+    lifecycle: str = "core"  # "core" | "experimental"
+
+    def as_dict(self) -> dict[str, object]:
+        data: dict[str, object] = {
+            "model_id": self.model_id,
+            "family": self.family,
+            "provider": self.provider,
+            "lifecycle": self.lifecycle,
+        }
+        if self.modalities:
+            data["modalities"] = list(self.modalities)
+        if self.training_modes:
+            data["training_modes"] = list(self.training_modes)
+        return data
+
+
+SUPPORTED_MODELS: tuple[SupportedModel, ...] = tuple(
+    SupportedModel(
+        model_id=model,
+        family="Qwen3",
+        provider="Qwen",
+        modalities=("text",),
+        training_modes=tuple(
+            sorted(
+                {
+                    *(("sft",) if model in SFT_SUPPORTED_MODELS else ()),
+                    *(("rl",) if model in RL_SUPPORTED_MODELS else ()),
+                }
+            )
+        ),
+        lifecycle=("experimental" if model in EXPERIMENTAL_MODELS else "core"),
+    )
+    for model in _ALL_QWEN3_IDS
+)
+
+_BASE_LOOKUP = {model.model_id.lower(): model.model_id for model in SUPPORTED_MODELS}
+SUPPORTED_BASE_MODEL_IDS: frozenset[str] = frozenset(_BASE_LOOKUP.values())
+FINE_TUNED_PREFIXES: tuple[str, ...] = ("ft:", "fft:", "qft:", "rl:")
+_MODEL_BY_ID = {model.model_id: model for model in SUPPORTED_MODELS}
+
+# ------------------------------------------------------------------------------
+# Public API
+# ------------------------------------------------------------------------------
+
+
+class UnsupportedModelError(ValueError):
+    """Raised when a model identifier is not supported by Synth."""
+
+
+def _extract_base_model(candidate: str, *, allow_finetuned_prefixes: bool) -> str | None:
+    cleaned = candidate.strip()
+    lowered = cleaned.lower()
+    base = _BASE_LOOKUP.get(lowered)
+    if base:
+        return base
+    if not allow_finetuned_prefixes or ":" not in cleaned:
+        return None
+
+    segments = cleaned.split(":")
+    for segment in segments[1:]:
+        candidate_base = segment.strip()
+        if not candidate_base:
+            continue
+        base = _BASE_LOOKUP.get(candidate_base.lower())
+        if base:
+            return base
+    return None
+
+
+def ensure_supported_model(
+    model_id: str,
+    *,
+    allow_finetuned_prefixes: bool = True,
+) -> str:
+    """Validate that *model_id* resolves to a supported base model (no lifecycle gate)."""
+    candidate = (model_id or "").strip()
+    if not candidate:
+        raise UnsupportedModelError("Model identifier is empty")
+
+    base = _extract_base_model(candidate, allow_finetuned_prefixes=allow_finetuned_prefixes)
+    if base:
+        return base
+
+    raise UnsupportedModelError(
+        f"Model '{candidate}' is not supported. Call supported_model_ids() for available base models."
+    )
+
+
+def ensure_allowed_model(
+    model_id: str,
+    *,
+    allow_finetuned_prefixes: bool = True,
+    allow_experimental: bool | None = None,
+) -> str:
+    """Validate support + lifecycle; gate experimental unless enabled."""
+    base = ensure_supported_model(model_id, allow_finetuned_prefixes=allow_finetuned_prefixes)
+    is_exp = base in EXPERIMENTAL_MODELS
+    allow_exp = allow_experimental if allow_experimental is not None else _experimental_enabled()
+    if is_exp and not allow_exp:
+        raise UnsupportedModelError(
+            f"Model '{base}' is experimental and disabled. "
+            "Set SDK_EXPERIMENTAL=1 or pass allow_experimental=True."
+        )
+    if is_exp:
+        _warn_if_experimental(base)
+    return base
+
+
+def normalize_model_identifier(
+    model_id: str,
+    *,
+    allow_finetuned_prefixes: bool = True,
+) -> str:
+    """Return a cleaned model identifier suitable for job payloads (no lifecycle gate)."""
+    canonical = ensure_supported_model(model_id, allow_finetuned_prefixes=allow_finetuned_prefixes)
+    cleaned = (model_id or "").strip()
+    if not cleaned:
+        return canonical
+    if cleaned.lower() in _BASE_LOOKUP:
+        return canonical
+    return cleaned
+
+
+def is_supported_model(model_id: str, *, allow_finetuned_prefixes: bool = True) -> bool:
+    """Return True if *model_id* resolves to a supported base model (ignores lifecycle)."""
+    try:
+        ensure_supported_model(model_id, allow_finetuned_prefixes=allow_finetuned_prefixes)
+    except UnsupportedModelError:
+        return False
+    return True
+
+
+def is_experimental_model(model_id: str) -> bool:
+    """Return True if *model_id* is marked experimental."""
+    try:
+        base = ensure_supported_model(model_id, allow_finetuned_prefixes=True)
+    except UnsupportedModelError:
+        return False
+    return base in EXPERIMENTAL_MODELS
+
+
+def is_core_model(model_id: str) -> bool:
+    """Return True if *model_id* is marked core."""
+    try:
+        base = ensure_supported_model(model_id, allow_finetuned_prefixes=True)
+    except UnsupportedModelError:
+        return False
+    return base in CORE_MODELS
+
+
+def iter_supported_models(
+    *,
+    families: Sequence[str] | None = None,
+    include: Sequence[str] | None = None,
+    exclude: Sequence[str] | None = None,
+) -> Iterator[SupportedModel]:
+    """Yield supported models, optionally filtered by family and lifecycle."""
+    include_set = {s.lower() for s in include} if include else None
+    exclude_set = {s.lower() for s in exclude} if exclude else None
+    fam_set = {f.lower() for f in families} if families else None
+
+    for m in SUPPORTED_MODELS:
+        if fam_set is not None and m.family.lower() not in fam_set:
+            continue
+        if include_set is not None and m.lifecycle.lower() not in include_set:
+            continue
+        if exclude_set is not None and m.lifecycle.lower() in exclude_set:
+            continue
+        yield m
+
+
+def list_supported_models(
+    *,
+    families: Sequence[str] | None = None,
+    include: Sequence[str] | None = None,
+    exclude: Sequence[str] | None = None,
+) -> list[SupportedModel]:
+    """Return supported models as a list for easier consumption."""
+    return list(iter_supported_models(families=families, include=include, exclude=exclude))
+
+
+def supported_model_ids(
+    *,
+    families: Sequence[str] | None = None,
+    include: Sequence[str] | None = None,
+    exclude: Sequence[str] | None = None,
+) -> list[str]:
+    """Return just the model identifiers for supported models."""
+    return [m.model_id for m in iter_supported_models(families=families, include=include, exclude=exclude)]
+
+
+def experimental_model_ids(*, families: Sequence[str] | None = None) -> list[str]:
+    """Return identifiers for experimental supported models."""
+    return supported_model_ids(families=families, include=("experimental",))
+
+
+def core_model_ids(*, families: Sequence[str] | None = None) -> list[str]:
+    """Return identifiers for core supported models."""
+    return supported_model_ids(families=families, include=("core",))
+
+
+def format_supported_models(
+    *,
+    families: Sequence[str] | None = None,
+    include: Sequence[str] | None = None,
+    exclude: Sequence[str] | None = None,
+) -> str:
+    """Produce a human readable table of supported models."""
+    rows: Iterable[SupportedModel] = iter_supported_models(families=families, include=include, exclude=exclude)
+    lines = ["model_id | family | provider | lifecycle | modalities | training_modes", "-" * 96]
+    for model in rows:
+        modalities = ",".join(model.modalities) or "-"
+        training = ",".join(model.training_modes) or "-"
+        lines.append(
+            f"{model.model_id} | {model.family} | {model.provider} | {model.lifecycle} | {modalities} | {training}"
+        )
+    return "\n".join(lines)
+
+
+def training_modes_for_model(model_id: str) -> tuple[str, ...]:
+    """Return the supported training modes (e.g., ('sft','rl')) for the given base model."""
+    canonical = ensure_supported_model(model_id, allow_finetuned_prefixes=True)
+    model = _MODEL_BY_ID.get(canonical)
+    if not model:
+        raise UnsupportedModelError(f"Model '{model_id}' is not registered as supported.")
+    return model.training_modes
+
+
+__all__ = [
+    "QWEN3_MODELS",
+    "QWEN3_CODER_MODELS",
+    "RL_SUPPORTED_MODELS",
+    "SFT_SUPPORTED_MODELS",
+    "EXPERIMENTAL_MODELS",
+    "CORE_MODELS",
+    "ExperimentalWarning",
+    "SupportedModel",
+    "SUPPORTED_MODELS",
+    "SUPPORTED_BASE_MODEL_IDS",
+    "FINE_TUNED_PREFIXES",
+    "UnsupportedModelError",
+    "ensure_supported_model",
+    "ensure_allowed_model",
+    "normalize_model_identifier",
+    "is_supported_model",
+    "is_experimental_model",
+    "is_core_model",
+    "iter_supported_models",
+    "list_supported_models",
+    "supported_model_ids",
+    "experimental_model_ids",
+    "core_model_ids",
+    "format_supported_models",
+    "training_modes_for_model",
+]
+

synth_ai/api/train/builders.py

@@ -5,8 +5,19 @@ from pathlib import Path
 from typing import Any
 
 import click
+from synth_ai.api.models.supported import (
+    UnsupportedModelError,
+    ensure_allowed_model,
+    normalize_model_identifier,
+)
+from synth_ai.learning.sft.config import prepare_sft_job_payload
 
-from .utils import ensure_api_base, load_toml, TrainError
+from .supported_algos import (
+    AlgorithmValidationError,
+    ensure_model_supported_for_algorithm,
+    validate_algorithm_config,
+)
+from .utils import TrainError, ensure_api_base, load_toml
 
 
 @dataclass(slots=True)
@@ -29,23 +40,78 @@ def build_rl_payload(
     task_url: str,
     overrides: dict[str, Any],
     idempotency: str | None,
+    allow_experimental: bool | None = None,
 ) -> RLBuildResult:
     data = load_toml(config_path)
+    try:
+        spec = validate_algorithm_config(data.get("algorithm"), expected_family="rl")
+    except AlgorithmValidationError as exc:
+        raise click.ClickException(str(exc)) from exc
     services = data.get("services") if isinstance(data.get("services"), dict) else {}
     model_cfg = data.get("model") if isinstance(data.get("model"), dict) else {}
 
-    final_task_url = (overrides.get("task_url") or task_url or services.get("task_url") or "").strip()
+    final_task_url = (
+        overrides.get("task_url")
+        or task_url
+        or (services.get("task_url") if isinstance(services, dict) else None)
+        or ""
+    ).strip()
     if not final_task_url:
-        raise click.ClickException("Task app URL required (provide --task-url or set services.task_url in TOML)")
+        raise click.ClickException(
+            "Task app URL required (provide --task-url or set services.task_url in TOML)"
+        )
 
-    model_source = (model_cfg.get("source") or "").strip()
-    model_base = (model_cfg.get("base") or "").strip()
+    raw_source = model_cfg.get("source") if isinstance(model_cfg, dict) else ""
+    model_source = str(raw_source or "").strip()
+    raw_base = model_cfg.get("base") if isinstance(model_cfg, dict) else ""
+    model_base = str(raw_base or "").strip()
     override_model = (overrides.get("model") or "").strip()
     if override_model:
         model_source = override_model
         model_base = ""
     if bool(model_source) == bool(model_base):
-        raise click.ClickException("Model section must specify exactly one of [model].source or [model].base")
+        details = (
+            f"Config: {config_path}\n"
+            f"[model].source={model_source!r} | [model].base={model_base!r}"
+        )
+        hint = (
+            "Set exactly one: [model].base for a base model (e.g. 'Qwen/Qwen3-1.7B') "
+            "or [model].source for a fine-tuned model id. Also remove any conflicting "
+            "'[policy].model' entries."
+        )
+        raise click.ClickException(
+            "Invalid model config: exactly one of [model].source or [model].base is required.\n"
+            + details
+            + "\nHint: "
+            + hint
+        )
+
+    try:
+        if model_source:
+            model_source = normalize_model_identifier(model_source)
+        if model_base:
+            model_base = normalize_model_identifier(model_base, allow_finetuned_prefixes=False)
+    except UnsupportedModelError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    base_model_for_training: str | None = None
+    if model_source:
+        base_model_for_training = ensure_allowed_model(
+            model_source,
+            allow_finetuned_prefixes=True,
+            allow_experimental=allow_experimental,
+        )
+    elif model_base:
+        base_model_for_training = ensure_allowed_model(
+            model_base,
+            allow_finetuned_prefixes=False,
+            allow_experimental=allow_experimental,
+        )
+    if base_model_for_training:
+        try:
+            ensure_model_supported_for_algorithm(base_model_for_training, spec)
+        except AlgorithmValidationError as exc:
+            raise click.ClickException(str(exc)) from exc
 
     # Force TOML services.task_url to the effective endpoint to avoid split URLs
     try:
@@ -81,34 +147,53 @@ def build_sft_payload(
     *,
     config_path: Path,
     dataset_override: Path | None,
+    allow_experimental: bool | None,
 ) -> SFTBuildResult:
     data = load_toml(config_path)
+    try:
+        spec = validate_algorithm_config(data.get("algorithm"), expected_family="sft")
+    except AlgorithmValidationError as exc:
+        raise TrainError(str(exc)) from exc
     job_cfg = data.get("job") if isinstance(data.get("job"), dict) else {}
     data_cfg = data.get("data") if isinstance(data.get("data"), dict) else {}
     hp_cfg = data.get("hyperparameters") if isinstance(data.get("hyperparameters"), dict) else {}
     train_cfg = data.get("training") if isinstance(data.get("training"), dict) else {}
     compute_cfg = data.get("compute") if isinstance(data.get("compute"), dict) else {}
 
-    raw_dataset = dataset_override or job_cfg.get("data") or job_cfg.get("data_path")
+    raw_dataset = (
+        dataset_override
+        or (job_cfg.get("data") if isinstance(job_cfg, dict) else None)
+        or (job_cfg.get("data_path") if isinstance(job_cfg, dict) else None)
+    )
     if not raw_dataset:
         raise TrainError("Dataset not specified; pass --dataset or set [job].data")
     dataset_path = Path(raw_dataset)
-    dataset_path = (dataset_path if dataset_path.is_absolute() else (config_path.parent / dataset_path)).resolve()
+    # Resolve relative paths from current working directory, not config directory
+    dataset_path = (
+        dataset_path if dataset_path.is_absolute() else (Path.cwd() / dataset_path)
+    ).resolve()
     if not dataset_path.exists():
         raise TrainError(f"Dataset not found: {dataset_path}")
 
-    validation_path = data_cfg.get("validation_path") if isinstance(data_cfg.get("validation_path"), str) else None
+    validation_path = (
+        data_cfg.get("validation_path")
+        if isinstance(data_cfg, dict)
+        else None
+        if isinstance(data_cfg, dict) and isinstance(data_cfg.get("validation_path"), str)
+        else None
+    )
     validation_file = None
     if validation_path:
         vpath = Path(validation_path)
-        vpath = (vpath if vpath.is_absolute() else (config_path.parent / vpath)).resolve()
+        # Resolve relative paths from current working directory, not config directory
+        vpath = (vpath if vpath.is_absolute() else (Path.cwd() / vpath)).resolve()
         if not vpath.exists():
             click.echo(f"[WARN] Validation dataset {vpath} missing; continuing without validation")
         else:
             validation_file = vpath
 
     hp_block: dict[str, Any] = {
-        "n_epochs": int(hp_cfg.get("n_epochs", 1)),
+        "n_epochs": int(hp_cfg.get("n_epochs", 1) if isinstance(hp_cfg, dict) else 1),
     }
     for key in (
         "batch_size",
@@ -120,20 +205,36 @@ def build_sft_payload(
         "warmup_ratio",
         "train_kind",
     ):
-        if key in hp_cfg:
+        if isinstance(hp_cfg, dict) and key in hp_cfg:
             hp_block[key] = hp_cfg[key]
-    if isinstance(hp_cfg.get("parallelism"), dict):
+    if isinstance(hp_cfg, dict) and isinstance(hp_cfg.get("parallelism"), dict):
         hp_block["parallelism"] = hp_cfg["parallelism"]
 
-    compute_block = {k: compute_cfg[k] for k in ("gpu_type", "gpu_count", "nodes") if k in compute_cfg}
+    compute_block = {
+        k: compute_cfg[k]
+        for k in ("gpu_type", "gpu_count", "nodes")
+        if isinstance(compute_cfg, dict) and k in compute_cfg
+    }
 
     effective = {
         "compute": compute_block,
-        "data": {"topology": data_cfg.get("topology", {}) if isinstance(data_cfg.get("topology"), dict) else {}},
-        "training": {k: v for k, v in train_cfg.items() if k in ("mode", "use_qlora")},
+        "data": {
+            "topology": data_cfg.get("topology", {})
+            if isinstance(data_cfg, dict) and isinstance(data_cfg.get("topology"), dict)
+            else {}
+        },
+        "training": {
+            k: v
+            for k, v in (train_cfg.items() if isinstance(train_cfg, dict) else [])
+            if k in ("mode", "use_qlora")
+        },
     }
 
-    validation_cfg = train_cfg.get("validation") if isinstance(train_cfg.get("validation"), dict) else None
+    validation_cfg = (
+        train_cfg.get("validation")
+        if isinstance(train_cfg, dict) and isinstance(train_cfg.get("validation"), dict)
+        else None
+    )
     if isinstance(validation_cfg, dict):
         hp_block.update(
             {
@@ -144,15 +245,45 @@ def build_sft_payload(
                 "greater_is_better": bool(validation_cfg.get("greater_is_better", False)),
             }
         )
-        effective.setdefault("training", {})["validation"] = {"enabled": bool(validation_cfg.get("enabled", True))}
-
-    payload = {
-        "model": job_cfg.get("model") or data.get("model"),
-        "training_file_id": None,  # populated after upload
-        "training_type": "sft_offline",
-        "hyperparameters": hp_block,
-        "metadata": {"effective_config": effective},
-    }
+        effective.setdefault("training", {})["validation"] = {
+            "enabled": bool(validation_cfg.get("enabled", True))
+        }
+
+    raw_model = str(
+        job_cfg.get("model") if isinstance(job_cfg, dict) else None or data.get("model") or ""
+    ).strip()
+    if not raw_model:
+        raise TrainError("Model not specified; set [job].model or [model].base in the config")
+
+    try:
+        base_model = ensure_allowed_model(
+            raw_model,
+            allow_finetuned_prefixes=False,
+            allow_experimental=allow_experimental,
+        )
+    except UnsupportedModelError as exc:
+        raise TrainError(str(exc)) from exc
+    try:
+        ensure_model_supported_for_algorithm(base_model, spec)
+    except AlgorithmValidationError as exc:
+        raise TrainError(str(exc)) from exc
+
+    try:
+        payload = prepare_sft_job_payload(
+            model=raw_model,
+            training_file=None,
+            hyperparameters=hp_block,
+            metadata={"effective_config": effective},
+            training_type="sft_offline",
+            training_file_field="training_file_id",
+            require_training_file=False,
+            include_training_file_when_none=True,
+            allow_finetuned_prefixes=False,
+        )
+    except UnsupportedModelError as exc:
+        raise TrainError(str(exc)) from exc
+    except ValueError as exc:
+        raise TrainError(str(exc)) from exc
 
     return SFTBuildResult(payload=payload, train_file=dataset_path, validation_file=validation_file)