synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/api/train/env_resolver.py

@@ -1,349 +0,0 @@
-from __future__ import annotations
-
-import importlib
-import os
-from collections.abc import Callable, Iterable, MutableMapping
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, cast
-
-import click
-
-from . import task_app
-from .utils import REPO_ROOT, mask_value, read_env_file, write_env_value
-
-
-def _load_saved_env_path() -> Path | None:
-    try:
-        module = cast(
-            Any, importlib.import_module("synth_ai.demos.demo_task_apps.core")
-        )
-        loader = cast(Callable[[], str | None], module.load_env_file_path)
-        saved_path = loader()
-        if saved_path:
-            return Path(saved_path)
-    except Exception:
-        return None
-    return None
-
-
-@dataclass(slots=True)
-class KeySpec:
-    name: str
-    description: str
-    secret: bool = True
-    allow_modal_secret: bool = False
-    allow_modal_app: bool = False
-    modal_secret_pattern: str | None = None
-    optional: bool = False
-
-
-class EnvResolver:
-    def __init__(self, initial_candidates: list[Path]) -> None:
-        if not initial_candidates:
-            raise click.ClickException("No .env candidates discovered")
-        self._candidates = initial_candidates
-        self._current = initial_candidates[0]
-        self._cache: MutableMapping[Path, dict[str, str]] = {}
-
-    @property
-    def current_path(self) -> Path:
-        return self._current
-
-    def select_new_env(self) -> None:
-        path = prompt_for_env(self._candidates, current=self._current)
-        self._current = path
-        if path not in self._candidates:
-            self._candidates.append(path)
-
-    def get_value(self, key: str) -> str | None:
-        cache = self._cache.get(self._current)
-        if cache is None:
-            cache = read_env_file(self._current)
-            self._cache[self._current] = cache
-        return cache.get(key)
-
-    def set_value(self, key: str, value: str) -> None:
-        cache = self._cache.setdefault(self._current, {})
-        cache[key] = value
-        write_env_value(self._current, key, value)
-
-
-def _collect_default_candidates(config_path: Path | None) -> list[Path]:
-    candidates: list[Path] = []
-    cwd = Path.cwd()
-
-    # Prioritize CWD env files
-    cwd_env = cwd / ".env"
-    if cwd_env.exists():
-        candidates.append(cwd_env.resolve())
-
-    # Search for additional .env files in CWD subdirectories
-    for sub in cwd.glob("**/.env"):
-        try:
-            resolved = sub.resolve()
-        except Exception:
-            continue
-        if resolved in candidates:
-            continue
-        # avoid nested venv caches
-        if any(part in {".venv", "node_modules", "__pycache__"} for part in resolved.parts):
-            continue
-        if len(candidates) >= 20:
-            break
-        candidates.append(resolved)
-
-    # Then config path env file
-    if config_path:
-        cfg_env = config_path.parent / ".env"
-        if cfg_env.exists():
-            candidates.append(cfg_env.resolve())
-
-    # Then repo env files
-    repo_env = REPO_ROOT / ".env"
-    if repo_env.exists():
-        candidates.append(repo_env.resolve())
-    examples_env = REPO_ROOT / "examples" / ".env"
-    if examples_env.exists():
-        candidates.append(examples_env.resolve())
-
-    # Search shallow depth for additional .env files in examples
-    for sub in (REPO_ROOT / "examples").glob("**/.env"):
-        try:
-            resolved = sub.resolve()
-        except Exception:
-            continue
-        if resolved in candidates:
-            continue
-        # avoid nested venv caches
-        if any(part in {".venv", "node_modules", "__pycache__"} for part in resolved.parts):
-            continue
-        if len(candidates) >= 20:
-            break
-        candidates.append(resolved)
-
-    deduped: list[Path] = []
-    for path in candidates:
-        if path not in deduped:
-            deduped.append(path)
-    return deduped
-
-
-def prompt_for_env(candidates: list[Path], *, current: Path | None = None) -> Path:
-    options = list(dict.fromkeys(candidates))  # preserve order, dedupe
-    click.echo("Select an .env file:")
-    for idx, path in enumerate(options, start=1):
-        marker = " (current)" if current and path == current else ""
-        click.echo(f"  {idx}) {path}{marker}")
-    click.echo("  m) Enter path manually")
-    click.echo("  0) Abort")
-
-    choice = click.prompt("Choice", default=None)
-    if choice is None:
-        raise click.ClickException("Selection required")
-    choice = choice.strip().lower()
-    if choice == "0":
-        raise click.ClickException("Aborted by user")
-    if choice in {"m", "manual"}:
-        manual = click.prompt("Enter path to .env", type=str).strip()
-        path = Path(manual).expanduser().resolve()
-        if not path.exists():
-            raise click.ClickException(f"Env file not found: {path}")
-        return path
-    try:
-        idx = int(choice)
-    except ValueError as exc:
-        raise click.ClickException("Invalid selection") from exc
-    if idx < 1 or idx > len(options):
-        raise click.ClickException("Invalid selection")
-    return options[idx - 1]
-
-
-def resolve_env(
-    *,
-    config_path: Path | None,
-    explicit_env_paths: Iterable[str],
-    required_keys: list[KeySpec],
-) -> tuple[Path, dict[str, str]]:
-    provided = [Path(p).expanduser().resolve() for p in explicit_env_paths]
-    if provided:
-        for path in provided:
-            if not path.exists():
-                raise click.ClickException(f"Env file not found: {path}")
-        resolver = EnvResolver(provided)
-    else:
-        saved_path = _load_saved_env_path()
-        if saved_path and saved_path.exists():
-            click.echo(f"Using .env file: {saved_path}")
-            resolver = EnvResolver([saved_path])
-        else:
-            resolver = EnvResolver(_collect_default_candidates(config_path))
-            resolver.select_new_env()
-
-    # Preload selected .env keys into process env so downstream lookups succeed
-    try:
-        current_env_map = read_env_file(resolver.current_path)
-        for k in (
-            "SYNTH_API_KEY",
-            "ENVIRONMENT_API_KEY",
-            "dev_environment_api_key",
-            "DEV_ENVIRONMENT_API_KEY",
-            "TASK_APP_URL",
-            "TASK_APP_BASE_URL",
-        ):
-            v = current_env_map.get(k)
-            if v and not os.environ.get(k):
-                os.environ[k] = v
-    except Exception:
-        pass
-
-    resolved: dict[str, str] = {}
-    for spec in required_keys:
-        value = _resolve_key(resolver, spec)
-        if value:
-            resolved[spec.name] = value
-    return resolver.current_path, resolved
-
-
-def _resolve_key(resolver: EnvResolver, spec: KeySpec) -> str:
-    while True:
-        # Priority: existing environment variable
-        env_val = os.environ.get(spec.name) or resolver.get_value(spec.name)
-        # Allow common aliases in .env to satisfy required keys without extra prompts
-        if not env_val and spec.name == "ENVIRONMENT_API_KEY":
-            for alias in ("dev_environment_api_key", "DEV_ENVIRONMENT_API_KEY"):
-                alt = resolver.get_value(alias)
-                if alt:
-                    env_val = alt
-                    os.environ[spec.name] = alt
-                    click.echo(f"Found {spec.name} via {alias}: {mask_value(alt)}")
-                    break
-        if not env_val and spec.name == "TASK_APP_URL":
-            for alias in ("TASK_APP_BASE_URL",):
-                alt = resolver.get_value(alias)
-                if alt:
-                    env_val = alt
-                    os.environ[spec.name] = alt
-                    click.echo(f"Found {spec.name} via {alias}: {mask_value(alt)}")
-                    break
-        if env_val:
-            click.echo(f"Found {spec.name} in current sources: {mask_value(env_val)}")
-            # Automatically use and persist the value (no prompt)
-            _maybe_persist(resolver, spec, env_val)
-            os.environ[spec.name] = env_val
-            return env_val
-        options: list[tuple[str, Callable[[], str | None]]] = []
-
-        def _enter_manual() -> str:
-            prompt = f"Enter {spec.description}" if spec.description else f"Enter {spec.name}"
-            value = click.prompt(prompt, hide_input=spec.secret).strip()
-            if not value:
-                raise click.ClickException(f"{spec.name} cannot be empty")
-            _maybe_persist(resolver, spec, value)
-            os.environ[spec.name] = value
-            return value
-
-        options.append(("Enter manually", _enter_manual))
-
-        def _pick_env() -> str | None:
-            resolver.select_new_env()
-            return None
-
-        options.append(("Choose another .env", _pick_env))
-
-        if spec.allow_modal_secret:
-            options.append(("Fetch from Modal secret", lambda: _fetch_modal_secret(resolver, spec)))
-        if spec.allow_modal_app:
-            options.append(("Choose Modal app URL", lambda: _fetch_modal_app(spec)))
-        if spec.optional:
-            options.append(("Skip", lambda: ""))
-        options.append(("Abort", lambda: (_raise_abort(spec))))
-
-        click.echo(f"Resolve {spec.name}:")
-        for idx, (label, _) in enumerate(options, start=1):
-            click.echo(f"  {idx}) {label}")
-        choice = click.prompt("Choice", type=int)
-        if choice < 1 or choice > len(options):
-            click.echo("Invalid selection; try again")
-            continue
-        action = options[choice - 1][1]
-        result = action()
-        if result is None:
-            # e.g. user switched env; restart loop
-            continue
-        return result
-
-
-def _maybe_persist(resolver: EnvResolver, spec: KeySpec, value: str) -> None:
-    # Automatically save (no prompt)
-    resolver.set_value(spec.name, value)
-    click.echo(f"Saved {spec.name} to {resolver.current_path}")
-
-
-def _prompt_yes_no(message: str, *, default: bool) -> bool:
-    """Prompt the user for a yes/no answer, accepting numeric variants."""
-
-    default_token = "y" if default else "n"
-    suffix = "[Y/n]" if default else "[y/N]"
-    while True:
-        response = click.prompt(f"{message} {suffix}", default=default_token, show_default=False)
-        normalized = str(response).strip().lower()
-        if normalized in {"", "y", "yes", "1", "true", "t"}:
-            return True
-        if normalized in {"n", "no", "0", "false", "f"}:
-            return False
-        click.echo("Invalid input; enter 'y' or 'n'.")
-
-
-def _fetch_modal_secret(resolver: EnvResolver, spec: KeySpec) -> str | None:
-    try:
-        names = task_app.list_modal_secrets(spec.modal_secret_pattern)
-    except click.ClickException as exc:
-        click.echo(str(exc))
-        return None
-    if not names:
-        click.echo("No Modal secrets matched")
-        return None
-    click.echo(task_app.format_modal_secrets(names))
-    idx = click.prompt("Select secret (0 to cancel)", type=int)
-    if idx == 0:
-        return None
-    if idx < 1 or idx > len(names):
-        click.echo("Invalid selection")
-        return None
-    name = names[idx - 1]
-    value = task_app.get_modal_secret_value(name)
-    _maybe_persist(resolver, spec, value)
-    os.environ[spec.name] = value
-    return value
-
-
-def _fetch_modal_app(spec: KeySpec) -> str | None:
-    try:
-        apps = task_app.list_modal_apps("task-app")
-    except click.ClickException as exc:
-        click.echo(str(exc))
-        return None
-    if not apps:
-        click.echo("No Modal apps matched")
-        return None
-    click.echo(task_app.format_modal_apps(apps))
-    idx = click.prompt("Select app (0 to cancel)", type=int)
-    if idx == 0:
-        return None
-    if idx < 1 or idx > len(apps):
-        click.echo("Invalid selection")
-        return None
-    url = apps[idx - 1].url
-    os.environ[spec.name] = url
-    return url
-
-
-def _raise_abort(spec: KeySpec) -> None:
-    raise click.ClickException(f"Missing required value for {spec.name}")
-
-
-__all__ = [
-    "KeySpec",
-    "resolve_env",
-]

synth_ai/api/train/pollers.py

@@ -1,75 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Any
-
-import click
-
-from .utils import ensure_api_base, fmt_duration, http_get, sleep
-
-
-@dataclass(slots=True)
-class PollOutcome:
-    status: str
-    payload: Mapping[str, Any]
-
-
-class JobPoller:
-    def __init__(
-        self, base_url: str, api_key: str, *, interval: float = 5.0, timeout: float = 3600.0
-    ) -> None:
-        self.base_url = ensure_api_base(base_url)
-        self.api_key = api_key
-        self.interval = interval
-        self.timeout = timeout
-
-    def _headers(self) -> dict[str, str]:
-        return {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json",
-        }
-
-    def poll(self, path: str) -> PollOutcome:
-        elapsed = 0.0
-        status = "unknown"
-        info: Mapping[str, Any] = {}
-        click.echo("Polling job status...")
-        while elapsed <= self.timeout:
-            try:
-                resp = http_get(f"{self.base_url}{path}", headers=self._headers())
-                info = (
-                    resp.json()
-                    if resp.headers.get("content-type", "").startswith("application/json")
-                    else {}
-                )
-                status = (info.get("status") or info.get("state") or "").lower()
-                timestamp = datetime.now().strftime("%H:%M:%S")
-                click.echo(f"[poll] {timestamp} {elapsed:.0f}s status={status}")
-                if status in {"succeeded", "failed", "cancelled", "canceled", "completed"}:
-                    break
-            except Exception as exc:  # pragma: no cover - network failures
-                click.echo(f"[poll] error: {exc}")
-            sleep(self.interval)
-            elapsed += self.interval
-        else:
-            click.echo(f"[poll] timeout after {fmt_duration(self.timeout)}")
-        return PollOutcome(status=status, payload=info)
-
-
-class RLJobPoller(JobPoller):
-    def poll_job(self, job_id: str) -> PollOutcome:
-        return super().poll(f"/rl/jobs/{job_id}")
-
-
-class SFTJobPoller(JobPoller):
-    def poll_job(self, job_id: str) -> PollOutcome:
-        return super().poll(f"/learning/jobs/{job_id}")
-
-
-__all__ = [
-    "PollOutcome",
-    "RLJobPoller",
-    "SFTJobPoller",
-]

synth_ai/api/train/supported_algos.py

@@ -1,147 +0,0 @@
-from __future__ import annotations
-
-import importlib
-from collections.abc import Callable, Mapping
-from dataclasses import dataclass
-from typing import Any, cast
-
-try:
-    _models_module = cast(
-        Any, importlib.import_module("synth_ai.api.models.supported")
-    )
-    RL_SUPPORTED_MODELS = cast(tuple[str, ...], _models_module.RL_SUPPORTED_MODELS)
-    SFT_SUPPORTED_MODELS = cast(tuple[str, ...], _models_module.SFT_SUPPORTED_MODELS)
-    training_modes_for_model = cast(
-        Callable[[str], tuple[str, ...]], _models_module.training_modes_for_model
-    )
-except Exception as exc:  # pragma: no cover - critical dependency
-    raise RuntimeError("Unable to load supported model metadata") from exc
-
-
-@dataclass(frozen=True)
-class AlgorithmSpec:
-    algo_type: str
-    method: str
-    variety: str
-    label: str  # Human readable identifier (e.g. "RL / GSPO")
-    required_training_mode: str  # Expected training mode on the model record (e.g. "rl", "sft")
-
-
-def _normalize(value: object) -> str:
-    if value is None:
-        return ""
-    return str(value).strip().lower()
-
-RL_ALGORITHM_MODEL_IDS: tuple[str, ...] = tuple(sorted(RL_SUPPORTED_MODELS))
-RL_ALGORITHM_MODEL_SET: frozenset[str] = frozenset(RL_ALGORITHM_MODEL_IDS)
-
-SFT_ALGORITHM_MODEL_IDS: tuple[str, ...] = tuple(sorted(SFT_SUPPORTED_MODELS))
-SFT_ALGORITHM_MODEL_SET: frozenset[str] = frozenset(SFT_ALGORITHM_MODEL_IDS)
-
-
-SUPPORTED_ALGORITHMS: tuple[AlgorithmSpec, ...] = (
-    AlgorithmSpec(
-        "online", "policy_gradient", "gspo", "online policy_gradient / gspo", "rl"
-    ),
-    AlgorithmSpec(
-        "offline", "supervised_finetune", "fft", "offline supervised_finetune / fft", "sft"
-    ),
-    AlgorithmSpec("offline", "sft", "fft", "offline sft / fft", "sft"),
-    # Accept explicit LoRA variety for SFT as a first-class alias of FFT SFT.
-    # This allows configs to declare intent with variety="lora" while still
-    # using SFT training mode; actual adapter selection is driven by [training].
-    AlgorithmSpec("offline", "sft", "lora", "offline sft / lora", "sft"),
-)
-
-_SUPPORTED_LOOKUP = {
-    (spec.algo_type, spec.method, spec.variety): spec for spec in SUPPORTED_ALGORITHMS
-}
-
-
-class AlgorithmValidationError(ValueError):
-    """Raised when an algorithm block contains unsupported combinations."""
-
-
-def validate_algorithm_config(
-    algorithm_block: Mapping[str, object] | None,
-    *,
-    expected_family: str | None = None,
-) -> AlgorithmSpec:
-    """Validate the [algorithm] section of a training config.
-
-    Args:
-        algorithm_block: Parsed mapping from the TOML config.
-        expected_family: Optional expected family label ("rl" or "sft").
-
-    Returns:
-        The matched AlgorithmSpec describing the supported combination.
-
-    Raises:
-        AlgorithmValidationError: if the combination is missing or unsupported.
-    """
-
-    if algorithm_block is None:
-        raise AlgorithmValidationError("Missing required [algorithm] section in config.")
-    if not isinstance(algorithm_block, Mapping):
-        raise AlgorithmValidationError("[algorithm] section must be a mapping/object.")
-
-    algo_type = _normalize(algorithm_block.get("type"))
-    method = _normalize(algorithm_block.get("method"))
-    variety = _normalize(algorithm_block.get("variety"))
-
-    key = (algo_type, method, variety)
-    spec = _SUPPORTED_LOOKUP.get(key)
-    if spec is None:
-        supported = "; ".join(
-            f"type='{entry.algo_type}', method='{entry.method}', variety='{entry.variety}'"
-            for entry in SUPPORTED_ALGORITHMS
-        )
-        raise AlgorithmValidationError(
-            "Unsupported algorithm configuration:\n"
-            f"  type={algo_type!r}, method={method!r}, variety={variety!r}\n"
-            f"Supported combinations are: {supported}"
-        )
-
-    if expected_family:
-        expected_family = expected_family.lower()
-        family_map = {
-            ("online", "policy_gradient", "gspo"): "rl",
-            ("offline", "supervised_finetune", "fft"): "sft",
-            ("offline", "sft", "fft"): "sft",
-            ("offline", "sft", "lora"): "sft",
-        }
-        family = family_map.get(key)
-        if family != expected_family:
-            raise AlgorithmValidationError(
-                f"Config contains algorithm {spec.label!r}, "
-                f"but the current command expects {expected_family.upper()}."
-            )
-
-    return spec
-
-
-def ensure_model_supported_for_algorithm(model_id: str, spec: AlgorithmSpec) -> None:
-    """Ensure that the given model supports the training mode required by *spec*."""
-
-    modes = {mode.lower() for mode in training_modes_for_model(model_id)}
-    required = spec.required_training_mode.lower()
-    if required not in modes:
-        if required == "rl":
-            allowed = ", ".join(RL_ALGORITHM_MODEL_IDS)
-        else:
-            allowed = ", ".join(SFT_ALGORITHM_MODEL_IDS)
-        raise AlgorithmValidationError(
-            f"Model '{model_id}' does not support {spec.label} workloads "
-            f"(missing training mode '{required}'). Supported model IDs: {allowed}"
-        )
-
-
-__all__ = [
-    "AlgorithmSpec",
-    "AlgorithmValidationError",
-    "RL_ALGORITHM_MODEL_IDS",
-    "SFT_ALGORITHM_MODEL_IDS",
-    "SUPPORTED_ALGORITHMS",
-    "validate_algorithm_config",
-    "ensure_model_supported_for_algorithm",
-]