synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/utils/modal.py

@@ -0,0 +1,308 @@
+import contextlib
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse, urlunparse
+
+from synth_ai.demos import core as demo_core
+from synth_ai.demos.core import DEFAULT_TASK_APP_SECRET_NAME, DemoEnv
+
+from .env import mask_str
+from .http import http_request
+from .process import popen_capture
+from .user_config import load_user_config
+
+__all__ = [
+    "ensure_modal_installed",
+    "ensure_task_app_ready",
+    "find_asgi_apps",
+    "is_local_demo_url",
+    "is_modal_public_url",
+    "normalize_endpoint_url",
+]
+
+
+def is_modal_public_url(url: str | None) -> bool:
+    try:
+        candidate = (url or "").strip().lower()
+        if not candidate or not (candidate.startswith("http://") or candidate.startswith("https://")):
+            return False
+        return (".modal.run" in candidate) and ("modal.local" not in candidate) and ("pypi-mirror" not in candidate)
+    except Exception:
+        return False
+
+
+def is_local_demo_url(url: str | None) -> bool:
+    try:
+        candidate = (url or "").strip().lower()
+        if not candidate:
+            return False
+        return candidate.startswith("http://127.0.0.1") or candidate.startswith("http://localhost")
+    except Exception:
+        return False
+
+
+def normalize_endpoint_url(url: str) -> str:
+    """Convert loopback URLs to forms accepted by the backend."""
+    if not url:
+        return url
+    try:
+        parsed = urlparse(url)
+        host = parsed.hostname or ""
+        if host in {"127.0.0.1", "::1"}:
+            new_host = "localhost"
+            netloc = new_host
+            if parsed.port:
+                netloc = f"{new_host}:{parsed.port}"
+            if parsed.username:
+                creds = parsed.username
+                if parsed.password:
+                    creds += f":{parsed.password}"
+                netloc = f"{creds}@{netloc}"
+            parsed = parsed._replace(netloc=netloc)
+            return urlunparse(parsed)
+    except Exception:
+        pass
+    return url
+
+
+def find_asgi_apps(root: Path) -> list[Path]:
+    """Recursively search for Python files that declare a Modal ASGI app."""
+    results: list[Path] = []
+    skip_dirs = {
+        ".git",
+        ".hg",
+        ".svn",
+        "node_modules",
+        "dist",
+        "build",
+        "__pycache__",
+        ".ruff_cache",
+        ".mypy_cache",
+        "venv",
+        ".venv",
+    }
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in skip_dirs]
+        for name in filenames:
+            if not name.endswith(".py"):
+                continue
+            path = Path(dirpath) / name
+            try:
+                with path.open("r", encoding="utf-8", errors="ignore") as fh:
+                    txt = fh.read()
+                if ("@asgi_app()" in txt) or ("@modal.asgi_app()" in txt):
+                    results.append(path)
+            except Exception:
+                continue
+
+    def _priority(path: Path) -> tuple[int, str]:
+        rel = str(path.resolve())
+        in_demo = "/synth_demo/" in rel or rel.endswith("/synth_demo/task_app.py")
+        return (0 if in_demo else 1, rel)
+
+    results.sort(key=_priority)
+    return results
+
+
+def ensure_task_app_ready(env: DemoEnv, synth_key: str, *, label: str) -> DemoEnv:
+    persist_path = demo_core.load_demo_dir() or os.getcwd()
+    user_config_map = load_user_config()
+
+    env_key = (env.env_api_key or "").strip()
+    if not env_key:
+        raise RuntimeError(
+            f"[{label}] ENVIRONMENT_API_KEY missing. Run `uvx synth-ai demo deploy` first."
+        )
+
+    template_id = demo_core.load_template_id()
+    allow_local = template_id == "crafter-local"
+
+    task_url = env.task_app_base_url
+    url_ok = is_modal_public_url(task_url) or (allow_local and is_local_demo_url(task_url or ""))
+    if not task_url or not url_ok:
+        resolved = task_url or ""
+        dynamic_lookup_allowed = env.task_app_name and not (
+            allow_local and is_local_demo_url(task_url or "")
+        )
+        if dynamic_lookup_allowed and not is_modal_public_url(resolved):
+            code, out = popen_capture(
+                [
+                    "uv",
+                    "run",
+                    "python",
+                    "-m",
+                    "modal",
+                    "app",
+                    "url",
+                    env.task_app_name,
+                ]
+            )
+            if code == 0 and out:
+                for token in out.split():
+                    if is_modal_public_url(token):
+                        resolved = token.strip().rstrip("/")
+                        break
+        if dynamic_lookup_allowed and not is_modal_public_url(resolved):
+            try:
+                choice = (
+                    input(
+                        f"Resolve URL from Modal for app '{env.task_app_name}'? [Y/n]: "
+                    ).strip().lower()
+                    or "y"
+                )
+            except Exception:
+                choice = "y"
+            if choice.startswith("y"):
+                code, out = popen_capture(
+                    [
+                        "uv",
+                        "run",
+                        "python",
+                        "-m",
+                        "modal",
+                        "app",
+                        "url",
+                        env.task_app_name,
+                    ]
+                )
+                if code == 0 and out:
+                    for token in out.split():
+                        if is_modal_public_url(token):
+                            resolved = token.strip().rstrip("/")
+                            break
+        if not is_modal_public_url(resolved):
+            hint = "Examples: https://<app-name>-fastapi-app.modal.run"
+            if allow_local:
+                hint += " or http://127.0.0.1:8001"
+            print(f"[{label}] Task app URL not configured or not a valid target.")
+            print(hint)
+            entered = input(
+                "Enter Task App base URL (must contain '.modal.run'), or press Enter to abort: "
+            ).strip()
+            if not entered:
+                raise RuntimeError(f"[{label}] Task App URL is required.")
+            entered_clean = entered.rstrip("/")
+            if not (
+                is_modal_public_url(entered_clean)
+                or (allow_local and is_local_demo_url(entered_clean))
+            ):
+                raise RuntimeError(f"[{label}] Valid Task App URL is required.")
+            task_url = entered_clean
+        else:
+            task_url = resolved
+        demo_core.persist_task_url(task_url, name=(env.task_app_name or None), path=persist_path)
+
+    app_name = (env.task_app_name or "").strip()
+    requires_modal_name = is_modal_public_url(task_url)
+    if requires_modal_name and not app_name:
+        fallback = input("Enter Modal app name for the task app (required): ").strip()
+        if not fallback:
+            raise RuntimeError(f"[{label}] Task app name is required.")
+        app_name = fallback
+        demo_core.persist_task_url(task_url, name=app_name, path=persist_path)
+
+    demo_core.persist_task_url(task_url, name=app_name if requires_modal_name else None, path=persist_path)
+    if synth_key:
+        os.environ["SYNTH_API_KEY"] = synth_key
+
+    openai_key = (
+        os.environ.get("OPENAI_API_KEY")
+        or str(user_config_map.get("OPENAI_API_KEY") or "")
+    ).strip()
+    if openai_key:
+        os.environ["OPENAI_API_KEY"] = openai_key
+
+    print(f"[{label}] Verifying rollout health:")
+    try:
+        preview = mask_str(env_key)
+        print(f"[{label}] {preview}")
+    except Exception:
+        pass
+    health_base = task_url.rstrip("/")
+    health_urls = [f"{health_base}/health/rollout", f"{health_base}/health"]
+    rc = 0
+    body: Any = ""
+    for h in health_urls:
+        print(f"[{label}] GET", h)
+        rc, body = http_request("GET", h, headers={"X-API-Key": env_key})
+        if rc == 200:
+            break
+    print(f"[{label}] status: {rc}")
+    try:
+        preview_body = json.dumps(body)[:800] if isinstance(body, dict) else str(body)[:800]
+    except Exception:
+        preview_body = str(body)[:800]
+    print(f"[{label}] body:", preview_body)
+    if rc != 200:
+        print(f"[{label}] Warning: rollout health check failed ({rc}). Response: {body}")
+        with contextlib.suppress(Exception):
+            print(f"[{label}] Sent header X-API-Key → {mask_str(env_key)}")
+    else:
+        print(f"[{label}] Task app rollout health check OK.")
+
+    os.environ["TASK_APP_BASE_URL"] = task_url
+    os.environ["ENVIRONMENT_API_KEY"] = env_key
+    os.environ["TASK_APP_SECRET_NAME"] = DEFAULT_TASK_APP_SECRET_NAME
+    updated_env = demo_core.load_env()
+    updated_env.env_api_key = env_key
+    updated_env.task_app_base_url = task_url
+    updated_env.task_app_name = app_name if requires_modal_name else ""
+    updated_env.task_app_secret_name = DEFAULT_TASK_APP_SECRET_NAME
+    return updated_env
+
+
+def ensure_modal_installed() -> None:
+    """Install the modal package if it is not already available and check authentication."""
+    modal_installed = False
+    try:
+        import importlib.util as import_util
+
+        if import_util.find_spec("modal") is not None:
+            modal_installed = True
+    except Exception:
+        pass
+
+    if not modal_installed:
+        print("modal not found; installing…")
+        try:
+            if shutil.which("uv"):
+                code, out = popen_capture(["uv", "pip", "install", "modal>=1.1.4"])
+            else:
+                code, out = popen_capture([sys.executable, "-m", "pip", "install", "modal>=1.1.4"])
+            if code != 0:
+                print(out)
+                print("Failed to install modal; continuing may fail.")
+                return
+            print("✓ modal installed successfully")
+            modal_installed = True
+        except Exception as exc:
+            print(f"modal install error: {exc}")
+            return
+
+    if modal_installed:
+        try:
+            import importlib.util as import_util
+
+            if import_util.find_spec("modal") is None:
+                print("Warning: modal is still not importable after install attempt.")
+                return
+        except Exception:
+            print("Warning: unable to verify modal installation.")
+            return
+
+    auth_ok, auth_msg = demo_core.modal_auth_status()
+    if auth_ok:
+        print(f"✓ Modal authenticated: {auth_msg}")
+    else:
+        print("\n⚠️  Modal authentication required")
+        print(f"   Status: {auth_msg}")
+        print("\n   To authenticate Modal, run:")
+        print("     modal setup")
+        print("\n   Or set environment variables:")
+        print("     export MODAL_TOKEN_ID=your-token-id")
+        print("     export MODAL_TOKEN_SECRET=your-token-secret")
+        print("\n   You can deploy later after authenticating.\n")

synth_ai/utils/process.py

@@ -0,0 +1,212 @@
+import os
+import signal
+import socket
+import time
+from collections.abc import Iterable
+from typing import Any
+
+__all__ = [
+    "ensure_local_port_available",
+    "popen_capture",
+    "popen_stream",
+    "popen_stream_capture",
+]
+
+
+def popen_capture(
+    cmd: list[str], cwd: str | None = None, env: dict[str, Any] | None = None
+) -> tuple[int, str]:
+    """Execute a subprocess and capture combined stdout/stderr."""
+    import subprocess
+
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+        out, _ = proc.communicate()
+        return int(proc.returncode or 0), out or ""
+    except Exception as exc:
+        return 1, str(exc)
+
+
+def popen_stream(
+    cmd: list[str], cwd: str | None = None, env: dict[str, Any] | None = None
+) -> int:
+    """Stream subprocess output line-by-line to stdout for real-time feedback."""
+    import subprocess
+    import threading
+
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+    except Exception as exc:
+        print(f"Failed to launch {' '.join(cmd)}: {exc}")
+        return 1
+
+    def _pump(stdout) -> None:
+        try:
+            for line in stdout:
+                print(line.rstrip())
+        except Exception:
+            pass
+
+    if proc.stdout is not None:
+        t = threading.Thread(target=_pump, args=(proc.stdout,), daemon=True)
+        t.start()
+        proc.wait()
+        t.join(timeout=1.0)
+    else:
+        proc.wait()
+    return int(proc.returncode or 0)
+
+
+def popen_stream_capture(
+    cmd: list[str], cwd: str | None = None, env: dict[str, Any] | None = None
+) -> tuple[int, str]:
+    """Stream subprocess output to stdout and also capture it into a buffer."""
+    import subprocess
+    import threading
+
+    buf_lines: list[str] = []
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+    except Exception as exc:
+        print(f"Failed to launch {' '.join(cmd)}: {exc}")
+        return 1, ""
+
+    def _pump(stdout) -> None:
+        try:
+            for line in stdout:
+                line = line.rstrip()
+                print(line)
+                buf_lines.append(line)
+        except Exception:
+            pass
+
+    if proc.stdout is not None:
+        t = threading.Thread(target=_pump, args=(proc.stdout,), daemon=True)
+        t.start()
+        proc.wait()
+        t.join(timeout=1.0)
+    else:
+        proc.wait()
+    return int(proc.returncode or 0), "\n".join(buf_lines)
+
+
+def _list_process_ids(port: int) -> list[int]:
+    try:
+        import subprocess
+
+        out = subprocess.run(
+            ["lsof", "-ti", f"TCP:{port}"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if not out.stdout:
+            return []
+        result: list[int] = []
+        for token in out.stdout.strip().splitlines():
+            token = token.strip()
+            if token.isdigit():
+                result.append(int(token))
+        return result
+    except Exception:
+        return []
+
+
+def _terminate_pids(pids: Iterable[int], *, aggressive: bool) -> bool:
+    terminated_any = False
+    for pid in pids:
+        try:
+            os.kill(pid, signal.SIGTERM)
+            terminated_any = True
+        except Exception as exc:
+            print(f"Failed to terminate PID {pid}: {exc}")
+    if terminated_any:
+        time.sleep(1.0)
+
+    if aggressive and pids:
+        still_running = []
+        for pid in pids:
+            try:
+                os.kill(pid, 0)
+            except OSError:
+                continue
+            still_running.append(pid)
+        if still_running:
+            for pid in still_running:
+                try:
+                    os.kill(pid, signal.SIGKILL)
+                except Exception as exc:
+                    print(f"Failed to force terminate PID {pid}: {exc}")
+            time.sleep(0.5)
+    return terminated_any
+
+
+def ensure_local_port_available(host: str, port: int, *, force: bool = False) -> bool:
+    """Ensure ``host:port`` is free before starting a local server."""
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.settimeout(0.5)
+        in_use = sock.connect_ex((host, port)) == 0
+    if not in_use:
+        return True
+
+    print(f"Port {port} on {host} is already in use.")
+    pids = _list_process_ids(port)
+
+    if pids:
+        print("Found processes using this port:")
+        for pid in pids:
+            print(f"  PID {pid}")
+    else:
+        print("Could not automatically identify the owning process.")
+
+    if not force:
+        try:
+            choice = input(f"Stop the existing process on port {port}? [y/N]: ").strip().lower() or "n"
+        except Exception:
+            choice = "n"
+        if not choice.startswith("y"):
+            print("Aborting; stop the running server and try again.")
+            return False
+    else:
+        print("Attempting to terminate the existing process...")
+
+    if pids:
+        _terminate_pids(pids, aggressive=force)
+    else:
+        print("Unable to determine owning process. Please stop it manually and retry.")
+        return False
+
+    for _ in range(10):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.settimeout(0.5)
+            if sock.connect_ex((host, port)) != 0:
+                print("Port is now available.")
+                return True
+        time.sleep(0.5)
+
+    print("Port still in use after terminating processes.")
+    return False

synth_ai/utils/prompts.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from argparse import Namespace
+from collections.abc import Callable, Mapping, MutableMapping
+from typing import Any
+
+__all__ = ["ensure_required_args"]
+
+
+def ensure_required_args(
+    args: Namespace,
+    prompts: Mapping[str, str],
+    *,
+    coerce: Mapping[str, Callable[[Any], Any]] | None = None,
+    defaults: Mapping[str, Any] | None = None,
+) -> Namespace:
+    """Ensure required CLI arguments are populated.
+
+    Legacy helper that historically prompted users. Our tests rely on it to
+    populate defaults and perform simple coercions, so we implement a minimal,
+    non-interactive version that mirrors that behaviour.
+    """
+
+    coerce_map = dict(coerce or {})
+    default_map: MutableMapping[str, Any] = dict(defaults or {})
+
+    for key, label in prompts.items():
+        value = getattr(args, key, None)
+        if value in (None, "") and key in default_map:
+            value = default_map[key]
+        if value in (None, ""):
+            raise ValueError(f"{label} is required")
+        if key in coerce_map:
+            try:
+                value = coerce_map[key](value)
+            except Exception as exc:  # pragma: no cover - defensive
+                raise ValueError(f"Failed to normalize {label}: {exc}") from exc
+        setattr(args, key, value)
+    return args

synth_ai/utils/sqld.py

@@ -0,0 +1,122 @@
+import contextlib
+import logging
+import os
+import shutil
+import socket
+import subprocess
+import tempfile
+
+import click
+
+SQLD_VERSION = "v0.26.2"
+
+
+def find_sqld_binary() -> str | None:
+    """Locate an existing sqld binary on PATH or in common install locations."""
+
+    sqld_path = shutil.which("sqld")
+    if sqld_path:
+        return sqld_path
+    common_paths = [
+        "/usr/local/bin/sqld",
+        "/usr/bin/sqld",
+        os.path.expanduser("~/.local/bin/sqld"),
+        os.path.expanduser("~/bin/sqld"),
+        os.path.expanduser("~/.turso/bin/sqld"),
+    ]
+    for path in common_paths:
+        if os.path.exists(path) and os.access(path, os.X_OK):
+            return path
+    return None
+
+
+def install_sqld() -> str:
+    """Install sqld via the Turso CLI, installing the CLI via Homebrew if needed."""
+
+    click.echo("🔧 sqld not found. Attempting automatic install...")
+
+    turso_cli_path = shutil.which("turso")
+    brew_path = shutil.which("brew")
+
+    if not turso_cli_path:
+        if not brew_path:
+            raise click.ClickException(
+                "Automatic install requires either Homebrew or an existing Turso CLI.\n"
+                "Install manually using one of:\n"
+                "  • brew install tursodatabase/tap/turso\n"
+                "  • curl -sSfL https://get.tur.so/install.sh | bash\n"
+                "Then run 'turso dev' once and re-run this command."
+            )
+
+        click.echo("🧰 Installing Turso CLI via Homebrew (tursodatabase/tap/turso)…")
+        try:
+            subprocess.run(
+                [brew_path, "install", "tursodatabase/tap/turso"],
+                check=True,
+            )
+        except subprocess.CalledProcessError as exc:
+            raise click.ClickException(
+                "Homebrew install failed. Please resolve brew errors and retry."
+            ) from exc
+
+        turso_cli_path = shutil.which("turso")
+        if not turso_cli_path:
+            raise click.ClickException(
+                "Homebrew reported success but the 'turso' binary is not on PATH."
+            )
+
+    click.echo("📥 Downloading sqld via 'turso dev' (this may take a few seconds)…")
+
+    with tempfile.NamedTemporaryFile(prefix="synth_sqld_", suffix=".db", delete=False) as temp_db:
+        temp_db_path = temp_db.name
+
+    env = os.environ.copy()
+    env.setdefault("TURSO_NONINTERACTIVE", "1")
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+
+    cmd = [
+        turso_cli_path,
+        "dev",
+        f"--db-file={temp_db_path}",
+        f"--port={port}",
+    ]
+    proc: subprocess.Popen[str] | None = None
+    stdout_data = ""
+    stderr_data = ""
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            env=env,
+        )
+        try:
+            stdout_data, stderr_data = proc.communicate(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.terminate()
+            try:
+                stdout_data, stderr_data = proc.communicate(timeout=5)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                stdout_data, stderr_data = proc.communicate()
+    finally:
+        if proc and proc.returncode not in (0, None) and (stdout_data or stderr_data):
+            logging.getLogger(__name__).debug(
+                "turso dev stdout: %s\nstderr: %s", stdout_data, stderr_data
+            )
+        with contextlib.suppress(OSError):
+            os.unlink(temp_db_path)
+
+    sqld_path = find_sqld_binary()
+    if sqld_path:
+        click.echo(f"✅ sqld available at {sqld_path}")
+        return sqld_path
+
+    raise click.ClickException(
+        "sqld download did not succeed. Run 'turso dev' manually once, "
+        "ensure it downloads sqld, and try again."
+    )