lumilake-deploy 0.1.0.dev1__py3-none-any.whl

lumilake_deploy/doctor.py

@@ -0,0 +1,218 @@
+"""Validate ``.env`` (and optionally ``.env.flowmesh``) before ``deploy up``.
+
+Mirrors :func:`flowmesh_stack.doctor.run_doctor_checks` in spirit: emits a
+structured report of errors / warnings / notes the operator can read at a
+glance. The primary purpose is to catch a malformed ``.env`` (missing
+required keys, obviously-bad values) before ``deploy up`` brings the
+server up against broken environment.
+"""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+from urllib.parse import urlparse
+
+type FindingLevel = Literal["note", "warning", "error"]
+
+
+@dataclass(frozen=True)
+class DoctorFinding:
+    level: FindingLevel
+    message: str
+
+
+@dataclass
+class DoctorReport:
+    findings: list[DoctorFinding] = field(default_factory=list)
+    callback: Callable[[DoctorFinding], Any] | None = None
+
+    @property
+    def errors(self) -> list[str]:
+        return [f.message for f in self.findings if f.level == "error"]
+
+    @property
+    def warnings(self) -> list[str]:
+        return [f.message for f in self.findings if f.level == "warning"]
+
+    @property
+    def notes(self) -> list[str]:
+        return [f.message for f in self.findings if f.level == "note"]
+
+    def error(self, message: str) -> None:
+        self._add("error", message)
+
+    def warning(self, message: str) -> None:
+        self._add("warning", message)
+
+    def note(self, message: str) -> None:
+        self._add("note", message)
+
+    def _add(self, level: FindingLevel, message: str) -> None:
+        finding = DoctorFinding(level, message)
+        self.findings.append(finding)
+        if self.callback:
+            self.callback(finding)
+
+
+# Keys the server always needs.
+_ALWAYS_REQUIRED: tuple[str, ...] = (
+    "LUMILAKE_SERVER_HOST",
+    "LUMILAKE_SERVER_PORT",
+    "LUMILAKE_RUNTIME_ORCHESTRATOR_URL",
+    "S3_ARCHIVE_PREFIX",
+    "LUMILAKE_IMAGE_TAG",
+)
+
+# Keys required for direct compute mode (when ``LUMID_DATA_URL`` is empty).
+# When LUMID_DATA_URL is set, lumid.data owns these credentials and the
+# server skips direct connections — the keys become optional.
+_DIRECT_MODE_REQUIRED: tuple[str, ...] = (
+    "DATABASE_URL",
+    "S3_URL",
+    "S3_USER_DATA_PREFIX",
+)
+
+_OPTIONAL_KEYS: tuple[str, ...] = (
+    "LUMILAKE_LOG_LEVEL",
+    "LUMILAKE_REGISTRY",
+    "LUMILAKE_OPTIMIZER_TYPE",
+    "LUMILAKE_OPTIMIZER_BATCH_SIZE",
+    "LUMILAKE_STARVATION_LIMIT",
+    "LUMILAKE_BATCH_ACCUMULATION_SECONDS",
+    "LUMILAKE_OPTIMIZER_SUBPROCESS_TIMEOUT_SECONDS",
+    "LUMILAKE_CPU_WORKER_GROUP_SIZE",
+    "LUMILAKE_GPU_WORKER_GROUP_SIZE",
+    "LUMILAKE_FLOWMESH_OUTPUT_DESTINATION",
+    "LUMILAKE_RUNTIME_TOKEN",
+    "LUMILAKE_JOB_MANAGER_TYPE",
+    "LUMILAKE_RUNTIME_MANAGER_TYPE",
+    "LUMILAKE_POLL_TIMEOUT_SECONDS",
+    "LUMILAKE_POLL_INTERVAL_SECONDS",
+    "LUMILAKE_HTTP_TIMEOUT_SECONDS",
+    "LUMILAKE_QUEUE_QUANTUM_MEDIUM",
+    "LUMILAKE_DATA_PROFILE_NUM_TEST_QUERIES",
+    "LUMILAKE_S3_PROFILE_COST_PER_FILE",
+    "LUMILAKE_S3_PROFILE_COST_PER_MIB",
+    "LUMILAKE_SKIP_DOTENV_CHECK",
+    "S3_CERT_FILE",
+    "LUMID_DATA_URL",
+    "LUMID_DATA_TOKEN",
+    "LUMID_DATA_TIMEOUT_SECONDS",
+    "HARDWARE_CPU_REQUIREMENT",
+    "HARDWARE_MEMORY_REQUIREMENT",
+    "HARDWARE_GPU_REQUIREMENT",
+    "HARDWARE_GPU_MEMORY_REQUIREMENT",
+    "LUMILAKE_GPU_DEVICES",
+)
+
+_KNOWN_KEYS: frozenset[str] = frozenset(
+    _ALWAYS_REQUIRED + _DIRECT_MODE_REQUIRED + _OPTIONAL_KEYS
+)
+
+
+def _parse_env_file(path: Path) -> dict[str, str]:
+    """Tolerant ``.env`` parser — accepts ``KEY=value`` or ``KEY="value"``."""
+    values: dict[str, str] = {}
+    for raw in path.read_text().splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        key = key.strip()
+        value = value.strip()
+        if value.startswith('"') and value.endswith('"'):
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _check_required(values: dict[str, str], report: DoctorReport) -> None:
+    for key in _ALWAYS_REQUIRED:
+        if not values.get(key):
+            report.error(f"{key} is required but missing or empty")
+    if not values.get("LUMID_DATA_URL"):
+        for key in _DIRECT_MODE_REQUIRED:
+            if not values.get(key):
+                report.error(
+                    f"{key} is required in direct mode "
+                    "(set ``LUMID_DATA_URL`` to route through lumid.data instead)"
+                )
+
+
+def _check_unknown(values: dict[str, str], report: DoctorReport) -> None:
+    for key in values:
+        if key not in _KNOWN_KEYS:
+            report.warning(f"{key} is unknown — typo, or stale schema?")
+
+
+def _check_data_plane(values: dict[str, str], report: DoctorReport) -> None:
+    """When LUMID_DATA_URL is set, flag the rest of the lumid block to
+    encourage consistent configuration.
+    """
+    if values.get("LUMID_DATA_URL") and not values.get("LUMID_DATA_TIMEOUT_SECONDS"):
+        report.note("LUMID_DATA_TIMEOUT_SECONDS unset; defaulting to 30s.")
+
+
+def _check_s3_url(values: dict[str, str], report: DoctorReport) -> None:
+    if values.get("LUMID_DATA_URL"):
+        return
+    raw_url = values.get("S3_URL", "")
+    if not raw_url:
+        return
+    parsed = urlparse(raw_url)
+    if parsed.scheme != "s3":
+        report.error("S3_URL must use the s3:// scheme")
+    if not parsed.hostname:
+        report.error("S3_URL must include an endpoint host")
+    if not parsed.username or not parsed.password:
+        report.error("S3_URL must include access key and secret")
+    if not parsed.path.lstrip("/"):
+        report.error("S3_URL must include a bucket or bucket/prefix path")
+
+
+def run_env_checks(
+    env_path: Path,
+    callback: Callable[[DoctorFinding], Any] | None = None,
+) -> DoctorReport:
+    """Validate a single lumilake ``.env`` file."""
+    report = DoctorReport(callback=callback)
+    if not env_path.is_file():
+        report.error(
+            f"{env_path} not found. Run ``lumilake deploy init`` to "
+            "create one from ``.env.example``."
+        )
+        return report
+    values = _parse_env_file(env_path)
+    _check_required(values, report)
+    _check_unknown(values, report)
+    _check_data_plane(values, report)
+    _check_s3_url(values, report)
+    if not report.errors:
+        report.note(f"{env_path}: schema looks correct")
+    return report
+
+
+def run_doctor(
+    env_path: Path,
+    *,
+    flowmesh_env_path: Path | None = None,
+    callback: Callable[[DoctorFinding], Any] | None = None,
+) -> DoctorReport:
+    """Top-level entry point. Validates ``.env`` and, when supplied,
+    notes the presence of ``.env.flowmesh`` (full validation belongs to
+    ``flowmesh stack doctor``).
+    """
+    report = run_env_checks(env_path, callback=callback)
+    if flowmesh_env_path is not None:
+        if flowmesh_env_path.is_file():
+            report.note(
+                f"{flowmesh_env_path}: present "
+                "(run ``flowmesh stack doctor`` for FlowMesh-side validation)"
+            )
+        else:
+            report.error(
+                f"{flowmesh_env_path} not found. Run "
+                "``lumilake deploy init --flowmesh`` to create one."
+            )
+    return report

lumilake_deploy/env.py

@@ -0,0 +1,63 @@
+"""``.env``-file helpers for the deploy CLI."""
+
+from pathlib import Path
+
+ENV_FILE_NAME = ".env"
+ENV_TEMPLATE_NAME = ".env.example"
+FLOWMESH_ENV_FILE_NAME = ".env.flowmesh"
+
+
+def _quote(value: str) -> str:
+    """Shell-quote a value for the env file (simple double-quote wrapping)."""
+    escaped = value.replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _line(key: str, value: str) -> str:
+    return f"{key}={_quote(value)}\n"
+
+
+def read_env_value(path: Path, key: str) -> str:
+    """Read a single env var value from a ``.env``-style file.
+
+    Returns an empty string if the key is not found. Strips surrounding
+    double-quotes on the value.
+    """
+    if not path.is_file():
+        return ""
+    prefix = f"{key}="
+    for raw in path.read_text().splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or not line.startswith(prefix):
+            continue
+        value = line[len(prefix) :]
+        if value.startswith('"') and value.endswith('"'):
+            value = value[1:-1]
+        return value
+    return ""
+
+
+def patch_env_value(path: Path, key: str, value: str) -> None:
+    """Replace ``key=...`` in ``path`` if present, append otherwise.
+
+    Used by ``deploy init`` only — never by lifecycle commands.
+    Idempotent: writing the same value is a no-op.
+    """
+    quoted_line = _line(key, value)
+    if not path.is_file():
+        path.write_text(quoted_line)
+        return
+    prefix = f"{key}="
+    out: list[str] = []
+    replaced = False
+    for raw in path.read_text().splitlines(keepends=True):
+        if raw.lstrip().startswith(prefix):
+            out.append(quoted_line)
+            replaced = True
+        else:
+            out.append(raw)
+    if not replaced:
+        if out and not out[-1].endswith("\n"):
+            out[-1] = out[-1] + "\n"
+        out.append(quoted_line)
+    path.write_text("".join(out))

lumilake_deploy/errors.py

@@ -0,0 +1,5 @@
+"""Shared exception types for the deploy orchestration."""
+
+
+class DeployError(RuntimeError):
+    """Raised when the deploy orchestration can't proceed."""

lumilake_deploy/flowmesh.py

@@ -0,0 +1,344 @@
+"""FlowMesh stack lifecycle management for lumilake deploy."""
+
+import subprocess
+import time
+from pathlib import Path
+
+from docker import DockerClient
+from docker.errors import ImageNotFound
+from flowmesh import FlowMesh
+from flowmesh.exceptions import FlowMeshConnectionError
+from flowmesh_cli_stack.utils import (
+    STACK_PATH_KEYS,
+    stack_compose_file,
+    stack_env_example,
+    stack_resource_env_overrides,
+)
+from flowmesh_stack.docker import DockerComposeStack
+from flowmesh_stack.env import ensure_env_file, load_env, parse_env_file
+from flowmesh_stack.node_client import NodeClient
+from flowmesh_stack.paths import ensure_dir, ensure_file, resolve_path
+from flowmesh_stack.workers import create_workers as sdk_create_workers
+from lumilake import envs
+from lumilake.log import init_child_logger
+
+logger = init_child_logger("deploy.flowmesh")
+
+
+def _info(msg: str) -> None:
+    logger.info(msg)
+
+
+def _error(msg: str) -> None:
+    logger.error(msg)
+
+
+# ---------------------------------------------------------------------------
+# Port collision check
+# ---------------------------------------------------------------------------
+
+
+def check_ports(ports: dict[str, int]) -> list[str]:
+    """Return error messages for ports already in use."""
+    errors: list[str] = []
+    for label, port in ports.items():
+        result = subprocess.run(
+            ["ss", "-tln", f"sport = :{port}"],
+            capture_output=True,
+            text=True,
+        )
+        if result.stdout.strip().count("\n") > 0:
+            errors.append(f"Port {port} ({label}) is already in use.")
+    return errors
+
+
+# ---------------------------------------------------------------------------
+# Stack management via SDK
+# ---------------------------------------------------------------------------
+
+_WORKDIR = Path(".flowmesh")
+
+
+def _make_workdir() -> Path:
+    """Create a working directory for FlowMesh compose mounts."""
+    _WORKDIR.mkdir(exist_ok=True)
+    (_WORKDIR / "secrets" / "tls" / "redis").mkdir(parents=True, exist_ok=True)
+    (_WORKDIR / "secrets" / "tls" / "server").mkdir(parents=True, exist_ok=True)
+    wc = _WORKDIR / "configs" / "worker_config.yaml"
+    wc.parent.mkdir(parents=True, exist_ok=True)
+    if not wc.exists():
+        wc.write_text("default_worker_config:\n  hb_interval: 30\n\nworkers: []\n")
+    return _WORKDIR
+
+
+def _ensure_deploy_paths_in_workdir(_base_dir: Path) -> None:
+    """ensure_deploy_paths pinned to _WORKDIR (ignores the cwd-based base_dir)."""
+    base = _WORKDIR.resolve()
+    ensure_dir(
+        resolve_path(envs.REDIS_TLS_DIR, default="./secrets/tls/redis", base_dir=base)
+    )
+    ensure_dir(
+        resolve_path(
+            envs.SERVER_TLS_DIR,
+            default="./secrets/tls/server",
+            base_dir=base,
+        )
+    )
+    ensure_file(
+        resolve_path(
+            envs.SERVER_WORKER_CONFIG,
+            default="./configs/worker_config.yaml",
+            base_dir=base,
+        )
+    )
+
+
+def _load_env(ef: Path) -> None:
+    ensure_env_file(ef, stack_env_example())
+    load_env(ef, base_dir=_WORKDIR.resolve(), path_keys=STACK_PATH_KEYS)
+
+
+_stack = DockerComposeStack(
+    compose_file=stack_compose_file(),
+    env_file_var="STACK_ENV_FILE",
+    load_env=_load_env,
+    ensure_deploy_paths=_ensure_deploy_paths_in_workdir,
+)
+
+
+def env_ports(env_file: str | Path) -> dict[str, int]:
+    """Extract configured ports from env file for collision checking."""
+    env = parse_env_file(Path(env_file))
+    return {
+        "FlowMesh HTTP": int(env["SERVER_HTTP_PORT"]),
+        "FlowMesh gRPC": int(env["SERVER_GRPC_PORT"]),
+        "FlowMesh Redis control": int(env["REDIS_CONTROL_PORT"]),
+        "FlowMesh Redis telemetry": int(env["REDIS_TELEMETRY_PORT"]),
+    }
+
+
+def _server_image_exists(env_file: str | Path) -> bool:
+    """Check if the FlowMesh server image is already available locally."""
+    env = parse_env_file(Path(env_file))
+    registry = env["FLOWMESH_REGISTRY"]
+    version = env["FLOWMESH_VERSION"]
+    tag = f"{registry}/flowmesh_server:{version}"
+    try:
+        client = DockerClient.from_env()
+        client.images.get(tag)
+        return True
+    except ImageNotFound:
+        return False
+
+
+def _slug_env(env_file: Path) -> dict[str, str]:
+    """Resolve the FlowMesh stack slug into compose-bound env vars."""
+    slug = stack_slug(env_file)
+    return {
+        "FLOWMESH_STACK_SLUG": slug,
+        "COMPOSE_PROJECT_NAME": slug,
+        "WORKER_RESULTS_DIR": f"{slug}_results",
+    }
+
+
+def stack_slug(env_file: Path) -> str:
+    """Return the FlowMesh stack slug derived from ``env_file``."""
+    env = parse_env_file(env_file)
+    return stack_resource_env_overrides(env)["FLOWMESH_STACK_SLUG"]
+
+
+def stack_pull(env_file: str | Path) -> None:
+    """Pull FlowMesh Docker images (skipped if already present locally)."""
+    _make_workdir()
+    env_path = Path(env_file).resolve()
+    if _server_image_exists(env_file):
+        _info("FlowMesh images already present locally, skipping pull.")
+        return
+    _info("Pulling FlowMesh images...")
+    _stack.run(["pull"], env_file=env_path, env=_slug_env(env_path))
+
+
+def stack_up(env_file: str | Path) -> None:
+    """Start the FlowMesh Docker Compose stack."""
+    _make_workdir()
+    env_path = Path(env_file).resolve()
+
+    errors = check_ports(env_ports(env_file))
+    if errors:
+        for e in errors:
+            _error(e)
+        raise RuntimeError("Fix port conflicts in .env.flowmesh and retry.")
+
+    # ``--profile root`` is required so compose activates redis_control +
+    # redis_telemetry; the bundled FlowMesh compose puts them under
+    # ``profiles: [root]`` and without this flag they get silently skipped.
+    args = ["--profile", "root", "up", "-d", "--wait"]
+
+    _info("Starting FlowMesh stack...")
+    result = _stack.run(
+        args, env_file=env_path, env=_slug_env(env_path), to_deploy=True
+    )
+    if result.returncode != 0:
+        raise RuntimeError("Failed to start FlowMesh stack.")
+    _info("FlowMesh stack is up.")
+
+
+def destroy_all_workers(env_file: str | Path) -> None:
+    """Destroy every FlowMesh-managed worker via the SDK."""
+    env = parse_env_file(Path(env_file))
+    base_url = f"http://localhost:{env['SERVER_HTTP_PORT']}"
+    # Bundled .env.flowmesh template ships no SERVER_TOKEN; default to
+    # empty (the unauthed local stack accepts an empty token).
+    token = env.get("SERVER_TOKEN", "")
+    _info("Destroying FlowMesh workers...")
+    with NodeClient(base_url=base_url, token=token) as client:
+        client.destroy_all_workers(ignore_unreachable=True)
+
+
+def stack_down(env_file: str | Path) -> None:
+    """Stop the FlowMesh stack."""
+    if not _WORKDIR.exists():
+        return
+    env_path = Path(env_file).resolve()
+    destroy_all_workers(env_path)
+    _info("Stopping FlowMesh stack...")
+    # ``--profile root`` so compose sees redis_control + redis_telemetry
+    # and tears them down too; without it they orphan and block re-up.
+    _stack.run(
+        ["--profile", "root", "down"], env_file=env_path, env=_slug_env(env_path)
+    )
+
+
+def stack_clean(env_file: str | Path) -> None:
+    """Stop the FlowMesh stack and remove volumes."""
+    if not _WORKDIR.exists():
+        return
+    env_path = Path(env_file).resolve()
+    _info("Cleaning FlowMesh stack (removing volumes)...")
+    _stack.run(
+        ["--profile", "root", "down", "-v"],
+        env_file=env_path,
+        env=_slug_env(env_path),
+    )
+
+
+# Aliases used by ``stop.py`` — keep the older verb-first names readable.
+def stop_stack(*, env_file: str | Path) -> None:
+    stack_down(env_file)
+
+
+def clean_stack(*, env_file: str | Path) -> None:
+    stack_clean(env_file)
+
+
+# ---------------------------------------------------------------------------
+# Health check via SDK
+# ---------------------------------------------------------------------------
+
+
+def wait_healthy(env_file: str | Path, timeout: int = 120) -> bool:
+    """Wait for FlowMesh server health using the FlowMesh SDK."""
+    env = parse_env_file(Path(env_file))
+    base_url = f"http://localhost:{env['SERVER_HTTP_PORT']}"
+    _info(f"Waiting for FlowMesh server at {base_url}...")
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            with FlowMesh(base_url=base_url) as client:
+                resp = client.system.health()
+                if resp.ok:
+                    _info("FlowMesh server is healthy.")
+                    return True
+        except (FlowMeshConnectionError, Exception):
+            pass
+        time.sleep(2)
+    _error(f"FlowMesh server not healthy at {base_url} after {timeout}s")
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Worker creation via SDK
+# ---------------------------------------------------------------------------
+
+
+def _pull_worker_image(env_file: str | Path, kind: str) -> None:
+    """Pre-pull the worker image for a given kind ("cpu" or "gpu") on the host.
+
+    FlowMesh's server spawns worker containers via the Docker Engine API
+    (mounted socket). Pulling via the CLI first caches the image locally
+    and lets FlowMesh's subsequent image reference resolve without a pull.
+    """
+    env = parse_env_file(Path(env_file))
+    registry = env["FLOWMESH_REGISTRY"]
+    version = env["FLOWMESH_VERSION"]
+    image = f"{registry}/flowmesh_worker:{version}-{kind}"
+    try:
+        DockerClient.from_env().images.get(image)
+        _info(f"Worker image {image} already present locally; skipping pre-pull.")
+        return
+    except ImageNotFound:
+        pass
+    _info(f"Pre-pulling worker image {image}...")
+    result = subprocess.run(
+        ["docker", "pull", image], check=False, capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"Failed to pull {image}. "
+            f"Ensure `docker login {registry.split('/')[0]}` has been run "
+            f"with a token that can read this image.\n{result.stderr}"
+        )
+
+
+def create_workers(env_file: str | Path, cpu_count: int, gpu_devices: str) -> None:
+    """Create FlowMesh workers via SDK."""
+    env = parse_env_file(Path(env_file))
+    base_url = f"http://localhost:{env['SERVER_HTTP_PORT']}"
+    token = env.get("SERVER_TOKEN", "")
+
+    with NodeClient(base_url=base_url, token=token) as client:
+        if cpu_count > 0:
+            _pull_worker_image(env_file, "cpu")
+            _info(f"Creating {cpu_count} CPU worker(s)...")
+            created = sdk_create_workers(client, kind="cpu", count=cpu_count)
+            for label, resp in created:
+                _info(f"Created {label}: {resp['name']}")
+
+        device_ids = _resolve_gpu_device_ids(gpu_devices)
+        if not device_ids:
+            _info(
+                "No GPUs detected; skipping GPU worker creation. "
+                "Set LUMILAKE_GPU_DEVICES to a comma-separated list "
+                "of indices to force-create."
+            )
+        else:
+            _pull_worker_image(env_file, "gpu")
+            targets = ",".join(device_ids)
+            _info(f"Creating {len(device_ids)} GPU worker(s): {targets}...")
+            created = sdk_create_workers(
+                client,
+                kind="gpu",
+                count=len(device_ids),
+                targets=targets,
+            )
+            for label, resp in created:
+                _info(f"Created {label}: {resp['name']}")
+
+
+def _resolve_gpu_device_ids(gpu_devices: str) -> list[str]:
+    """Resolve a LUMILAKE_GPU_DEVICES-style string into explicit device ids."""
+    normalized = gpu_devices.strip()
+    if normalized and normalized.lower() != "all":
+        return [tok.strip() for tok in normalized.split(",") if tok.strip()]
+    try:
+        result = subprocess.run(
+            ["nvidia-smi", "--query-gpu=index", "--format=csv,noheader"],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+    except FileNotFoundError:
+        return []
+    if result.returncode != 0:
+        return []
+    return [line.strip() for line in result.stdout.splitlines() if line.strip()]