synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/api/train/config_finder.py

@@ -1,228 +0,0 @@
-from __future__ import annotations
-
-import json
-import os
-from collections.abc import Iterable
-from dataclasses import dataclass
-from pathlib import Path
-
-import click
-
-from .utils import REPO_ROOT, load_toml, preview_json
-
-_SKIP_DIRS = {".git", "__pycache__", ".venv", "node_modules", "dist", "build"}
-_STATE_FILE = os.path.expanduser("~/.synth-ai/demo.json")
-
-
-@dataclass(slots=True)
-class ConfigCandidate:
-    path: Path
-    train_type: str  # "rl", "sft", or "unknown"
-
-
-def _load_last_config() -> Path | None:
-    """Load the last used training config path from state file."""
-    try:
-        if os.path.isfile(_STATE_FILE):
-            with open(_STATE_FILE) as fh:
-                data = json.load(fh)
-                if isinstance(data, dict):
-                    last_config = data.get("LAST_CONFIG")
-                    if last_config:
-                        path = Path(last_config).resolve()
-                        if path.exists():
-                            return path
-    except Exception:
-        pass
-    return None
-
-
-def _save_last_config(config_path: Path) -> None:
-    """Save the last used training config path to state file."""
-    try:
-        data = {}
-        if os.path.isfile(_STATE_FILE):
-            with open(_STATE_FILE) as fh:
-                data = json.load(fh) or {}
-        if not isinstance(data, dict):
-            data = {}
-        data["LAST_CONFIG"] = str(config_path.resolve())
-        os.makedirs(os.path.dirname(_STATE_FILE), exist_ok=True)
-        with open(_STATE_FILE, "w") as fh:
-            json.dump(data, fh)
-    except Exception:
-        pass
-
-
-def _iter_candidate_paths() -> Iterable[Path]:
-    seen: set[Path] = set()
-
-    # Prioritize current working directory first
-    try:
-        cwd = Path.cwd().resolve()
-    except Exception:
-        cwd = None
-    if cwd and cwd.exists():
-        for path in cwd.rglob("*.toml"):
-            if any(part in _SKIP_DIRS for part in path.parts):
-                continue
-            resolved = path.resolve()
-            if resolved in seen:
-                continue
-            seen.add(resolved)
-            yield resolved
-
-    # Then look in explicit config directories
-    preferred = [
-        REPO_ROOT / "configs",
-        REPO_ROOT / "examples",
-        REPO_ROOT / "training",
-    ]
-    for base in preferred:
-        if not base.exists():
-            continue
-        for path in base.rglob("*.toml"):
-            if any(part in _SKIP_DIRS for part in path.parts):
-                continue
-            resolved = path.resolve()
-            if resolved in seen:
-                continue
-            seen.add(resolved)
-            yield resolved
-
-
-def _infer_config_type(data: dict) -> str:
-    # 1) Strong signals from [algorithm]
-    algo = data.get("algorithm")
-    if isinstance(algo, dict):
-        method = str(algo.get("method") or "").lower()
-        algo_type = str(algo.get("type") or "").lower()
-        variety = str(algo.get("variety") or "").lower()
-
-        # RL indicators
-        if method in {"policy_gradient", "ppo", "reinforce"}:
-            return "rl"
-        if algo_type == "online":
-            return "rl"
-        if variety in {"gspo", "grpo", "ppo"}:
-            return "rl"
-
-        # SFT indicators
-        if method in {"supervised_finetune", "sft"}:
-            return "sft"
-        if algo_type == "offline":
-            return "sft"
-        if variety in {"fft"}:
-            return "sft"
-
-    # 2) Other RL signals
-    if data.get("job_type") == "rl":
-        return "rl"
-    services = data.get("services")
-    if isinstance(services, dict) and ("task_url" in services or "environment" in services):
-        return "rl"
-
-    # 3) Other SFT signals
-    training = data.get("training")
-    if isinstance(training, dict):
-        mode = str(training.get("mode") or "").lower()
-        if mode.startswith("sft") or mode == "sft_offline":
-            return "sft"
-    hyper = data.get("hyperparameters")
-    if isinstance(hyper, dict):
-        kind = str(hyper.get("train_kind") or "").lower()
-        if kind in {"sft", "fft"}:
-            return "sft"
-
-    # 4) Fallback
-    return "unknown"
-
-
-def discover_configs(explicit: list[str], *, requested_type: str | None) -> list[ConfigCandidate]:
-    candidates: list[ConfigCandidate] = []
-    seen: set[Path] = set()
-
-    for raw in explicit:
-        path = Path(raw).expanduser().resolve()
-        if not path.exists():
-            raise click.ClickException(f"Config not found: {path}")
-        data = load_toml(path)
-        cfg_type = _infer_config_type(data)
-        candidates.append(ConfigCandidate(path=path, train_type=cfg_type))
-        seen.add(path)
-
-    if explicit:
-        return candidates
-
-    for path in _iter_candidate_paths():
-        if path in seen:
-            continue
-        try:
-            data = load_toml(path)
-        except Exception:
-            continue
-        cfg_type = _infer_config_type(data)
-        candidates.append(ConfigCandidate(path=path, train_type=cfg_type))
-
-    if requested_type and requested_type != "auto":
-        candidates = [c for c in candidates if c.train_type in {requested_type, "unknown"}]
-
-    # De-dupe by path and keep deterministic ordering by directory depth then name
-    candidates.sort(key=lambda c: (len(c.path.parts), str(c.path)))
-    return candidates
-
-
-def prompt_for_config(
-    candidates: list[ConfigCandidate], *, requested_type: str | None, allow_autoselect: bool = False
-) -> ConfigCandidate:
-    if not candidates:
-        raise click.ClickException("No training configs found. Pass --config explicitly.")
-
-    # Check for last used config and move it to the top if found
-    last_config = _load_last_config()
-    default_idx = 1
-
-    if allow_autoselect and len(candidates) == 1:
-        chosen = candidates[0]
-        _save_last_config(chosen.path)
-        return chosen
-
-    if last_config:
-        for idx, cand in enumerate(candidates):
-            if cand.path.resolve() == last_config:
-                # Move last used config to the front
-                candidates.insert(0, candidates.pop(idx))
-                break
-
-    click.echo("Select a training config:")
-    for idx, cand in enumerate(candidates, start=1):
-        label = cand.train_type if cand.train_type != "unknown" else "?"
-        last_marker = " (last used)" if last_config and cand.path.resolve() == last_config else ""
-        click.echo(f"  {idx}) [{label}] {cand.path}{last_marker}")
-    click.echo("  0) Abort")
-
-    choice = click.prompt("Enter choice", type=int, default=default_idx)
-    if choice == 0:
-        raise click.ClickException("Aborted by user")
-    if choice < 0 or choice > len(candidates):
-        raise click.ClickException("Invalid selection")
-
-    selection = candidates[choice - 1]
-
-    # Save this config as the last used
-    _save_last_config(selection.path)
-
-    try:
-        data = load_toml(selection.path)
-        preview = preview_json({k: data.get(k) for k in list(data.keys())[:4]}, limit=320)
-        click.echo(f"Loaded {selection.path}: {preview}")
-    except Exception:
-        pass
-    return selection
-
-
-__all__ = [
-    "ConfigCandidate",
-    "discover_configs",
-    "prompt_for_config",
-]

synth_ai/api/train/env_resolver.py

@@ -1,347 +0,0 @@
-from __future__ import annotations
-
-import os
-from collections.abc import Callable, Iterable, MutableMapping
-from dataclasses import dataclass
-from pathlib import Path
-
-import click
-
-from . import task_app
-from .utils import REPO_ROOT, mask_value, read_env_file, write_env_value
-
-
-@dataclass(slots=True)
-class KeySpec:
-    name: str
-    description: str
-    secret: bool = True
-    allow_modal_secret: bool = False
-    allow_modal_app: bool = False
-    modal_secret_pattern: str | None = None
-    optional: bool = False
-
-
-class EnvResolver:
-    def __init__(self, initial_candidates: list[Path]) -> None:
-        if not initial_candidates:
-            raise click.ClickException("No .env candidates discovered")
-        self._candidates = initial_candidates
-        self._current = initial_candidates[0]
-        self._cache: MutableMapping[Path, dict[str, str]] = {}
-
-    @property
-    def current_path(self) -> Path:
-        return self._current
-
-    def select_new_env(self) -> None:
-        path = prompt_for_env(self._candidates, current=self._current)
-        self._current = path
-        if path not in self._candidates:
-            self._candidates.append(path)
-
-    def get_value(self, key: str) -> str | None:
-        cache = self._cache.get(self._current)
-        if cache is None:
-            cache = read_env_file(self._current)
-            self._cache[self._current] = cache
-        return cache.get(key)
-
-    def set_value(self, key: str, value: str) -> None:
-        cache = self._cache.setdefault(self._current, {})
-        cache[key] = value
-        write_env_value(self._current, key, value)
-
-
-def _collect_default_candidates(config_path: Path | None) -> list[Path]:
-    candidates: list[Path] = []
-    cwd = Path.cwd()
-
-    # Prioritize CWD env files
-    cwd_env = cwd / ".env"
-    if cwd_env.exists():
-        candidates.append(cwd_env.resolve())
-
-    # Search for additional .env files in CWD subdirectories
-    for sub in cwd.glob("**/.env"):
-        try:
-            resolved = sub.resolve()
-        except Exception:
-            continue
-        if resolved in candidates:
-            continue
-        # avoid nested venv caches
-        if any(part in {".venv", "node_modules", "__pycache__"} for part in resolved.parts):
-            continue
-        if len(candidates) >= 20:
-            break
-        candidates.append(resolved)
-
-    # Then config path env file
-    if config_path:
-        cfg_env = config_path.parent / ".env"
-        if cfg_env.exists():
-            candidates.append(cfg_env.resolve())
-
-    # Then repo env files
-    repo_env = REPO_ROOT / ".env"
-    if repo_env.exists():
-        candidates.append(repo_env.resolve())
-    examples_env = REPO_ROOT / "examples" / ".env"
-    if examples_env.exists():
-        candidates.append(examples_env.resolve())
-
-    # Search shallow depth for additional .env files in examples
-    for sub in (REPO_ROOT / "examples").glob("**/.env"):
-        try:
-            resolved = sub.resolve()
-        except Exception:
-            continue
-        if resolved in candidates:
-            continue
-        # avoid nested venv caches
-        if any(part in {".venv", "node_modules", "__pycache__"} for part in resolved.parts):
-            continue
-        if len(candidates) >= 20:
-            break
-        candidates.append(resolved)
-
-    deduped: list[Path] = []
-    for path in candidates:
-        if path not in deduped:
-            deduped.append(path)
-    return deduped
-
-
-def prompt_for_env(candidates: list[Path], *, current: Path | None = None) -> Path:
-    options = list(dict.fromkeys(candidates))  # preserve order, dedupe
-    click.echo("Select an .env file:")
-    for idx, path in enumerate(options, start=1):
-        marker = " (current)" if current and path == current else ""
-        click.echo(f"  {idx}) {path}{marker}")
-    click.echo("  m) Enter path manually")
-    click.echo("  0) Abort")
-
-    choice = click.prompt("Choice", default=None)
-    if choice is None:
-        raise click.ClickException("Selection required")
-    choice = choice.strip().lower()
-    if choice == "0":
-        raise click.ClickException("Aborted by user")
-    if choice in {"m", "manual"}:
-        manual = click.prompt("Enter path to .env", type=str).strip()
-        path = Path(manual).expanduser().resolve()
-        if not path.exists():
-            raise click.ClickException(f"Env file not found: {path}")
-        return path
-    try:
-        idx = int(choice)
-    except ValueError as exc:
-        raise click.ClickException("Invalid selection") from exc
-    if idx < 1 or idx > len(options):
-        raise click.ClickException("Invalid selection")
-    return options[idx - 1]
-
-
-def resolve_env(
-    *,
-    config_path: Path | None,
-    explicit_env_paths: Iterable[str],
-    required_keys: list[KeySpec],
-) -> tuple[Path, dict[str, str]]:
-    provided = [Path(p).expanduser().resolve() for p in explicit_env_paths]
-    if provided:
-        for path in provided:
-            if not path.exists():
-                raise click.ClickException(f"Env file not found: {path}")
-        resolver = EnvResolver(provided)
-    else:
-        # Check for saved .env path from demo command
-        try:
-            from synth_ai.demos.demo_task_apps.core import load_env_file_path
-
-            saved_env_path = load_env_file_path()
-            if saved_env_path:
-                saved_path = Path(saved_env_path)
-                if saved_path.exists():
-                    click.echo(f"Using .env file: {saved_path}")
-                    resolver = EnvResolver([saved_path])
-                else:
-                    # Saved path no longer exists, fall back to prompt
-                    resolver = EnvResolver(_collect_default_candidates(config_path))
-                    resolver.select_new_env()
-            else:
-                resolver = EnvResolver(_collect_default_candidates(config_path))
-                resolver.select_new_env()
-        except Exception:
-            # If import fails or any error, fall back to original behavior
-            resolver = EnvResolver(_collect_default_candidates(config_path))
-            resolver.select_new_env()
-
-    # Preload selected .env keys into process env so downstream lookups succeed
-    try:
-        current_env_map = read_env_file(resolver.current_path)
-        for k in (
-            "SYNTH_API_KEY",
-            "ENVIRONMENT_API_KEY",
-            "dev_environment_api_key",
-            "DEV_ENVIRONMENT_API_KEY",
-            "TASK_APP_URL",
-            "TASK_APP_BASE_URL",
-        ):
-            v = current_env_map.get(k)
-            if v and not os.environ.get(k):
-                os.environ[k] = v
-    except Exception:
-        pass
-
-    resolved: dict[str, str] = {}
-    for spec in required_keys:
-        value = _resolve_key(resolver, spec)
-        if value:
-            resolved[spec.name] = value
-    return resolver.current_path, resolved
-
-
-def _resolve_key(resolver: EnvResolver, spec: KeySpec) -> str:
-    while True:
-        # Priority: existing environment variable
-        env_val = os.environ.get(spec.name) or resolver.get_value(spec.name)
-        # Allow common aliases in .env to satisfy required keys without extra prompts
-        if not env_val and spec.name == "ENVIRONMENT_API_KEY":
-            for alias in ("dev_environment_api_key", "DEV_ENVIRONMENT_API_KEY"):
-                alt = resolver.get_value(alias)
-                if alt:
-                    env_val = alt
-                    os.environ[spec.name] = alt
-                    click.echo(f"Found {spec.name} via {alias}: {mask_value(alt)}")
-                    break
-        if not env_val and spec.name == "TASK_APP_URL":
-            for alias in ("TASK_APP_BASE_URL",):
-                alt = resolver.get_value(alias)
-                if alt:
-                    env_val = alt
-                    os.environ[spec.name] = alt
-                    click.echo(f"Found {spec.name} via {alias}: {mask_value(alt)}")
-                    break
-        if env_val:
-            click.echo(f"Found {spec.name} in current sources: {mask_value(env_val)}")
-            # Automatically use and persist the value (no prompt)
-            _maybe_persist(resolver, spec, env_val)
-            os.environ[spec.name] = env_val
-            return env_val
-        options: list[tuple[str, Callable[[], str | None]]] = []
-
-        def _enter_manual() -> str:
-            prompt = f"Enter {spec.description}" if spec.description else f"Enter {spec.name}"
-            value = click.prompt(prompt, hide_input=spec.secret).strip()
-            if not value:
-                raise click.ClickException(f"{spec.name} cannot be empty")
-            _maybe_persist(resolver, spec, value)
-            os.environ[spec.name] = value
-            return value
-
-        options.append(("Enter manually", _enter_manual))
-
-        def _pick_env() -> str | None:
-            resolver.select_new_env()
-            return None
-
-        options.append(("Choose another .env", _pick_env))
-
-        if spec.allow_modal_secret:
-            options.append(("Fetch from Modal secret", lambda: _fetch_modal_secret(resolver, spec)))
-        if spec.allow_modal_app:
-            options.append(("Choose Modal app URL", lambda: _fetch_modal_app(spec)))
-        if spec.optional:
-            options.append(("Skip", lambda: ""))
-        options.append(("Abort", lambda: (_raise_abort(spec))))
-
-        click.echo(f"Resolve {spec.name}:")
-        for idx, (label, _) in enumerate(options, start=1):
-            click.echo(f"  {idx}) {label}")
-        choice = click.prompt("Choice", type=int)
-        if choice < 1 or choice > len(options):
-            click.echo("Invalid selection; try again")
-            continue
-        action = options[choice - 1][1]
-        result = action()
-        if result is None:
-            # e.g. user switched env; restart loop
-            continue
-        return result
-
-
-def _maybe_persist(resolver: EnvResolver, spec: KeySpec, value: str) -> None:
-    # Automatically save (no prompt)
-    resolver.set_value(spec.name, value)
-    click.echo(f"Saved {spec.name} to {resolver.current_path}")
-
-
-def _prompt_yes_no(message: str, *, default: bool) -> bool:
-    """Prompt the user for a yes/no answer, accepting numeric variants."""
-
-    default_token = "y" if default else "n"
-    suffix = "[Y/n]" if default else "[y/N]"
-    while True:
-        response = click.prompt(f"{message} {suffix}", default=default_token, show_default=False)
-        normalized = str(response).strip().lower()
-        if normalized in {"", "y", "yes", "1", "true", "t"}:
-            return True
-        if normalized in {"n", "no", "0", "false", "f"}:
-            return False
-        click.echo("Invalid input; enter 'y' or 'n'.")
-
-
-def _fetch_modal_secret(resolver: EnvResolver, spec: KeySpec) -> str | None:
-    try:
-        names = task_app.list_modal_secrets(spec.modal_secret_pattern)
-    except click.ClickException as exc:
-        click.echo(str(exc))
-        return None
-    if not names:
-        click.echo("No Modal secrets matched")
-        return None
-    click.echo(task_app.format_modal_secrets(names))
-    idx = click.prompt("Select secret (0 to cancel)", type=int)
-    if idx == 0:
-        return None
-    if idx < 1 or idx > len(names):
-        click.echo("Invalid selection")
-        return None
-    name = names[idx - 1]
-    value = task_app.get_modal_secret_value(name)
-    _maybe_persist(resolver, spec, value)
-    os.environ[spec.name] = value
-    return value
-
-
-def _fetch_modal_app(spec: KeySpec) -> str | None:
-    try:
-        apps = task_app.list_modal_apps("task-app")
-    except click.ClickException as exc:
-        click.echo(str(exc))
-        return None
-    if not apps:
-        click.echo("No Modal apps matched")
-        return None
-    click.echo(task_app.format_modal_apps(apps))
-    idx = click.prompt("Select app (0 to cancel)", type=int)
-    if idx == 0:
-        return None
-    if idx < 1 or idx > len(apps):
-        click.echo("Invalid selection")
-        return None
-    url = apps[idx - 1].url
-    os.environ[spec.name] = url
-    return url
-
-
-def _raise_abort(spec: KeySpec) -> None:
-    raise click.ClickException(f"Missing required value for {spec.name}")
-
-
-__all__ = [
-    "KeySpec",
-    "resolve_env",
-]

synth_ai/api/train/pollers.py

@@ -1,75 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Any
-
-import click
-
-from .utils import ensure_api_base, fmt_duration, http_get, sleep
-
-
-@dataclass(slots=True)
-class PollOutcome:
-    status: str
-    payload: Mapping[str, Any]
-
-
-class JobPoller:
-    def __init__(
-        self, base_url: str, api_key: str, *, interval: float = 5.0, timeout: float = 3600.0
-    ) -> None:
-        self.base_url = ensure_api_base(base_url)
-        self.api_key = api_key
-        self.interval = interval
-        self.timeout = timeout
-
-    def _headers(self) -> dict[str, str]:
-        return {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json",
-        }
-
-    def poll(self, path: str) -> PollOutcome:
-        elapsed = 0.0
-        status = "unknown"
-        info: Mapping[str, Any] = {}
-        click.echo("Polling job status...")
-        while elapsed <= self.timeout:
-            try:
-                resp = http_get(f"{self.base_url}{path}", headers=self._headers())
-                info = (
-                    resp.json()
-                    if resp.headers.get("content-type", "").startswith("application/json")
-                    else {}
-                )
-                status = (info.get("status") or info.get("state") or "").lower()
-                timestamp = datetime.now().strftime("%H:%M:%S")
-                click.echo(f"[poll] {timestamp} {elapsed:.0f}s status={status}")
-                if status in {"succeeded", "failed", "cancelled", "canceled", "completed"}:
-                    break
-            except Exception as exc:  # pragma: no cover - network failures
-                click.echo(f"[poll] error: {exc}")
-            sleep(self.interval)
-            elapsed += self.interval
-        else:
-            click.echo(f"[poll] timeout after {fmt_duration(self.timeout)}")
-        return PollOutcome(status=status, payload=info)
-
-
-class RLJobPoller(JobPoller):
-    def poll_job(self, job_id: str) -> PollOutcome:
-        return super().poll(f"/rl/jobs/{job_id}")
-
-
-class SFTJobPoller(JobPoller):
-    def poll_job(self, job_id: str) -> PollOutcome:
-        return super().poll(f"/learning/jobs/{job_id}")
-
-
-__all__ = [
-    "PollOutcome",
-    "RLJobPoller",
-    "SFTJobPoller",
-]