lumilake-deploy 0.1.0.dev1__py3-none-any.whl

lumilake_deploy/__init__.py

@@ -0,0 +1,11 @@
+"""Lumilake deploy orchestration implementation.
+
+Public entry points called from ``cli/commands/deploy.py``:
+
+- :func:`setup.run_setup` — stand up the stack from ``.env``.
+- :func:`stop.run_stop` — stop all services (optionally purge volumes).
+- :func:`update_flowmesh.run_update` — re-lock + install latest FlowMesh packages.
+
+External binaries (``docker``, ``uv``) are invoked via subprocess; the
+Python layer owns the control flow, parsing, and state.
+"""

lumilake_deploy/_demo_data.py

@@ -0,0 +1,139 @@
+import os
+import re
+import sys
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+
+@dataclass(frozen=True)
+class S3Config:
+    endpoint: str
+    access_key: str
+    secret_key: str
+    bucket: str
+    secure: bool
+    cert_file: str | None
+
+
+def load_env_file(env_file: Path | None) -> dict[str, str]:
+    if env_file is None:
+        return {}
+    if not env_file.is_file():
+        raise FileNotFoundError(f"env file not found: {env_file}")
+    pat = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$")
+    out: dict[str, str] = {}
+    for raw in env_file.read_text().splitlines():
+        line = raw.split("#", 1)[0].strip()
+        if not line:
+            continue
+        m = pat.match(line)
+        if not m:
+            continue
+        key, value = m.group(1), m.group(2)
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
+            value = value[1:-1]
+        out[key] = value
+    return out
+
+
+def resolve_env(
+    env_file: Path | None,
+    overrides: dict[str, str | None] | None = None,
+) -> dict[str, str]:
+    merged: dict[str, str] = {}
+    merged.update(load_env_file(env_file))
+    for key, value in os.environ.items():
+        if value:
+            merged[key] = value
+    if overrides:
+        for key, override in overrides.items():
+            if override is not None:
+                merged[key] = override
+    return merged
+
+
+def require_env(env: dict[str, str], keys: Iterable[str]) -> None:
+    missing = [k for k in keys if not env.get(k)]
+    if missing:
+        raise SystemExit(
+            f"missing required env variable(s): {', '.join(missing)} "
+            "(set in --env-file or process env)"
+        )
+
+
+def parse_s3_url(raw: str, cert_file: str | None = None) -> S3Config:
+    parsed = urlparse(raw)
+    if parsed.scheme != "s3":
+        raise SystemExit(f"S3_URL must use s3:// scheme: {raw!r}")
+    if not parsed.hostname or not parsed.username or not parsed.password:
+        raise SystemExit(
+            "S3_URL must include credentials and host, e.g. "
+            "s3://access:secret@host:port/bucket"
+        )
+    bucket = parsed.path.lstrip("/").split("/", 1)[0]
+    if not bucket:
+        raise SystemExit("S3_URL must include a bucket in the path")
+    endpoint = parsed.hostname
+    if parsed.port:
+        endpoint = f"{endpoint}:{parsed.port}"
+    return S3Config(
+        endpoint=endpoint,
+        access_key=unquote(parsed.username),
+        secret_key=unquote(parsed.password),
+        bucket=bucket,
+        secure=bool(cert_file),
+        cert_file=cert_file,
+    )
+
+
+def make_minio_client(cfg: S3Config) -> Any:
+    try:
+        from minio import Minio
+    except ImportError as exc:
+        raise SystemExit(
+            "minio package is required. Install via "
+            "`uv sync --all-packages` or `pip install minio`."
+        ) from exc
+
+    http_client = None
+    if cfg.cert_file:
+        import certifi
+        import urllib3
+
+        http_client = urllib3.PoolManager(
+            cert_reqs="CERT_REQUIRED",
+            ca_certs=cfg.cert_file or certifi.where(),
+        )
+    return Minio(
+        endpoint=cfg.endpoint,
+        access_key=cfg.access_key,
+        secret_key=cfg.secret_key,
+        secure=cfg.secure,
+        http_client=http_client,
+    )
+
+
+def find_default_env_file(start: Path | None = None) -> Path | None:
+    cur = (start or Path.cwd()).resolve()
+    for parent in [cur, *cur.parents]:
+        candidate = parent / ".env"
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def info(msg: str) -> None:
+    print(msg, file=sys.stderr, flush=True)
+
+
+def human_bytes(n: int) -> str:
+    units = ["B", "KiB", "MiB", "GiB", "TiB"]
+    val = float(n)
+    for unit in units:
+        if val < 1024.0:
+            return f"{val:.1f} {unit}"
+        val /= 1024.0
+    return f"{val:.1f} PiB"

lumilake_deploy/assets/.env.example

@@ -0,0 +1,82 @@
+# Lumilake server env template.
+#
+# Copy to ``.env`` (run ``lumilake deploy init``) and edit. The packaged
+# compose file consumes this via ``env_file: .env``; local runs load it via
+# ``load_dotenv(".env")``.
+#
+# Run ``lumilake deploy doctor`` after editing to validate.
+
+# ── Server ────────────────────────────────────────────────────────────────────
+LUMILAKE_LOG_LEVEL="INFO"
+LUMILAKE_SERVER_HOST="0.0.0.0"
+LUMILAKE_SERVER_PORT="9000"
+
+# ── Scheduler ─────────────────────────────────────────────────────────────────
+LUMILAKE_OPTIMIZER_BATCH_SIZE="10"
+LUMILAKE_STARVATION_LIMIT="3"
+LUMILAKE_BATCH_ACCUMULATION_SECONDS="0"
+LUMILAKE_OPTIMIZER_SUBPROCESS_TIMEOUT_SECONDS="60"
+LUMILAKE_CPU_WORKER_GROUP_SIZE="1"
+LUMILAKE_GPU_WORKER_GROUP_SIZE="1"
+# GPU devices Lumilake assigns to FlowMesh workers, one worker per
+# device. Distinct from ``.env.flowmesh``'s ``CUDA_VISIBLE_DEVICES``,
+# which scopes what the FlowMesh server container itself sees. Set to a
+# specific index (e.g. "0") or a comma-separated subset ("0,2") for the
+# free GPUs on your host; "all" expands to every nvidia-smi-detected
+# GPU. Leave blank to skip GPU worker creation.
+LUMILAKE_GPU_DEVICES=""
+
+# ── Runtime (FlowMesh) ────────────────────────────────────────────────────────
+# The orchestrator URL the server dispatches workflows to. When using the
+# bundled FlowMesh stack (``lumilake deploy init --flowmesh``), this is
+# the http://127.0.0.1:<flowmesh.server_port> from ``.env.flowmesh``.
+LUMILAKE_RUNTIME_ORCHESTRATOR_URL="http://127.0.0.1:18000"
+LUMILAKE_RUNTIME_TOKEN=""
+
+# Worker output delivery: "local" or "http". Both work identically for
+# Lumilake-emitted upstream paths.
+LUMILAKE_FLOWMESH_OUTPUT_DESTINATION="local"
+
+# ── Data plane (lumid.data, agent retrievals only) --------------------------
+# Only DataRetrievalOps with ``type: agent`` route through lumid.data;
+# SQL and S3 retrievals always go direct against ``DATABASE_URL`` /
+# ``S3_URL`` below regardless of ``LUMID_DATA_URL``.
+# LUMID_DATA_URL="http://127.0.0.1:9102"
+# LUMID_DATA_TOKEN=""
+# LUMID_DATA_TIMEOUT_SECONDS="30"
+
+# ── Compute database -------------------------------------------------------
+# PostgreSQL connection used by every SQL ``DataRetrievalOp``.
+DATABASE_URL="postgresql://lumilake:lumilake_password@127.0.0.1:15432/lumilake"
+
+# ── Compute S3 -------------------------------------------------------------
+# S3-compatible connection used by every S3 ``DataRetrievalOp`` and the
+# job-record archive (see ``S3_ARCHIVE_PREFIX``).
+S3_URL="s3://lumilake:lumilake_password@127.0.0.1:19100/lumilake-demo"
+S3_USER_DATA_PREFIX="lumilake-demo"
+# S3_CERT_FILE=""
+
+# Archive ``bucket/prefix`` for job records + run artifacts. Shares the
+# ``S3_*`` connection; only the prefix is configurable.
+S3_ARCHIVE_PREFIX="lumilake-archive/artifacts"
+
+# ── Tuning (optional) --------------------------------------------------------
+# LUMILAKE_POLL_TIMEOUT_SECONDS=""
+# LUMILAKE_POLL_INTERVAL_SECONDS=""
+# LUMILAKE_HTTP_TIMEOUT_SECONDS="300"
+# LUMILAKE_QUEUE_QUANTUM_MEDIUM=""
+# LUMILAKE_DATA_PROFILE_NUM_TEST_QUERIES="1"
+# LUMILAKE_S3_PROFILE_COST_PER_FILE="0.05"
+# LUMILAKE_S3_PROFILE_COST_PER_MIB="0.01"
+
+# ── Hardware Requirements (optional) ─────────────────────────────────────────
+# HARDWARE_CPU_REQUIREMENT="8"
+# HARDWARE_MEMORY_REQUIREMENT="16Gi"
+# HARDWARE_GPU_REQUIREMENT="1"
+# HARDWARE_GPU_MEMORY_REQUIREMENT=""
+
+# ── Docker ───────────────────────────────────────────────────────────────────
+# `dev` is the rolling main image. Pin to `vX.Y.Z` for production.
+LUMILAKE_IMAGE_TAG="dev"
+# Trust-bearing override of the server-image registry (see docs/ENV.md).
+# LUMILAKE_REGISTRY="ghcr.io/mlsys-io"

lumilake_deploy/assets/__init__.py

@@ -0,0 +1,35 @@
+"""Packaged deploy assets (compose file, env template).
+
+Resolved through :mod:`importlib.resources` so the deploy CLI works the
+same from a workspace checkout or a PyPI install — operators don't have
+to be in (or even have) a Lumilake source tree.
+"""
+
+from importlib import resources
+from pathlib import Path
+
+
+class AssetNotFoundError(FileNotFoundError):
+    """Raised when a packaged asset cannot be resolved."""
+
+
+def asset_path(*parts: str) -> Path:
+    """Return a usable filesystem path for an asset inside this package."""
+    resource = resources.files(__name__)
+    for part in parts:
+        resource /= part
+    try:
+        with resources.as_file(resource) as path:
+            return Path(path)
+    except FileNotFoundError as exc:
+        raise AssetNotFoundError(str(resource)) from exc
+
+
+def env_example_path() -> Path:
+    """Path to the bundled ``.env.example`` template."""
+    return asset_path(".env.example")
+
+
+def compose_path() -> Path:
+    """Path to the bundled ``compose.yml`` (docker-compose) file."""
+    return asset_path("compose.yml")

lumilake_deploy/assets/compose.yml

@@ -0,0 +1,40 @@
+x-logging: &default-logging
+  driver: json-file
+  options:
+    max-size: "10m"
+    max-file: "3"
+
+x-logging-verbose: &verbose-logging
+  driver: json-file
+  options:
+    max-size: "50m"
+    max-file: "3"
+
+services:
+  server:
+    image: ${LUMILAKE_REGISTRY:-ghcr.io/mlsys-io}/lumilake_server:${LUMILAKE_IMAGE_TAG:-dev}
+    # Use the locally-tagged image when present (so ``lumilake deploy
+    # build`` outputs are honored); ``lumilake deploy pull`` resolves
+    # rolling :dev/:latest tags explicitly when an update is wanted.
+    pull_policy: missing
+    container_name: lumilake-server
+    profiles: ["server"]
+    network_mode: host
+    env_file: .env
+    environment:
+      # Env vars are injected via env_file; no .env inside the container.
+      LUMILAKE_SKIP_DOTENV_CHECK: "1"
+    volumes:
+      # Surface the host's S3 cert at the same path inside the container
+      # so ``S3_CERT_FILE`` resolves. Mounting ``/dev/null`` is a no-op
+      # when the variable is unset (direct-mode without TLS verification).
+      - ${S3_CERT_FILE:-/dev/null}:${S3_CERT_FILE:-/dev/null}:ro
+    healthcheck:
+      test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen(f'http://localhost:{os.environ.get(\"LUMILAKE_SERVER_PORT\", \"9000\")}/healthz')"]
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      # Cover envs.validate() + DB pool init before unhealthy.
+      start_period: 40s
+    restart: unless-stopped
+    logging: *verbose-logging

lumilake_deploy/containers.py

@@ -0,0 +1,48 @@
+"""Service-to-container mapping for the Lumilake stack.
+
+FlowMesh names containers and volumes from ``FLOWMESH_STACK_SLUG`` (the
+slugged form of ``FLOWMESH_STACK_SUFFIX``). ``lumilake deploy init
+--flowmesh`` sets the suffix to ``lumilake``, so the SDK can't hardcode
+``flowmesh_node_*`` — it has to look the slug up from ``.env.flowmesh``
+in the operator's deployment directory.
+"""
+
+from pathlib import Path
+
+from . import flowmesh as fm_mod
+from .env import FLOWMESH_ENV_FILE_NAME
+
+SERVICE_NAMES: tuple[str, ...] = (
+    "server",
+    "flowmesh",
+    "flowmesh-redis",
+    "flowmesh-redis-telemetry",
+)
+
+
+def container_names(deploy_dir: Path) -> dict[str, str]:
+    """Map each service name to the docker container name.
+
+    Looks up ``FLOWMESH_STACK_SLUG`` from ``deploy_dir/.env.flowmesh``
+    when present; falls back to ``flowmesh_node`` so the SDK keeps
+    pre-init behavior intact.
+    """
+    env_fm = deploy_dir / FLOWMESH_ENV_FILE_NAME
+    slug = fm_mod.stack_slug(env_fm) if env_fm.is_file() else "flowmesh_node"
+    return {
+        "server": "lumilake-server",
+        "flowmesh": f"{slug}_server",
+        "flowmesh-redis": f"{slug}_redis_control",
+        "flowmesh-redis-telemetry": f"{slug}_redis_telemetry",
+    }
+
+
+def flowmesh_state_volumes(deploy_dir: Path) -> tuple[str, ...]:
+    """Volume names FlowMesh creates for its postgres + redis state."""
+    env_fm = deploy_dir / FLOWMESH_ENV_FILE_NAME
+    slug = fm_mod.stack_slug(env_fm) if env_fm.is_file() else "flowmesh_node"
+    return (
+        f"{slug}_postgres_data",
+        f"{slug}_redis_control_data",
+        f"{slug}_redis_telemetry_data",
+    )

lumilake_deploy/docker_client.py

@@ -0,0 +1,175 @@
+"""Thin wrappers over ``docker-py`` for container lifecycle ops."""
+
+import datetime as dt
+from collections.abc import Iterator
+from functools import cache
+
+import docker
+from docker.errors import APIError, DockerException, ImageNotFound
+from docker.models.containers import Container
+from docker.models.volumes import Volume
+
+from .errors import DeployError
+
+
+@cache
+def get_docker_client() -> docker.DockerClient:
+    """Return a cached ``DockerClient`` built from the environment."""
+    try:
+        return docker.from_env()
+    except DockerException as exc:
+        raise DeployError(
+            "Cannot connect to Docker. Ensure the daemon is running and your "
+            "user is in the docker group (sudo usermod -aG docker $USER)."
+        ) from exc
+
+
+def engine_is_up() -> bool:
+    """Return True when the Docker daemon is reachable.
+
+    ``ping()`` is an explicit probe — we don't wrap it in a check-first
+    pattern because "is the daemon alive" has no underlying resource to
+    list. DockerException here is the documented "unreachable" signal,
+    not a swallow; anything else would be a library contract violation.
+    """
+    try:
+        return bool(get_docker_client().ping())
+    except (DockerException, DeployError):
+        return False
+
+
+def image_exists(tag: str) -> bool:
+    """Return True when ``tag`` is present in the local image store."""
+    try:
+        get_docker_client().images.get(tag)
+        return True
+    except ImageNotFound:
+        return False
+
+
+def image_pull(tag: str) -> None:
+    """Pull ``tag`` from its registry into the local image store."""
+    client = get_docker_client()
+    try:
+        client.images.pull(tag)
+    except (APIError, ImageNotFound) as exc:
+        raise DeployError(f"Failed to pull image {tag}: {exc}") from exc
+
+
+def _find_container(name: str) -> Container | None:
+    matches = get_docker_client().containers.list(all=True, filters={"name": name})
+    for container in matches:
+        if container.name == name:
+            return container
+    return None
+
+
+def container_exists(name: str) -> bool:
+    return _find_container(name) is not None
+
+
+def container_stop(name: str, *, timeout: int = 30) -> bool:
+    """Stop ``name`` if present. Returns True on success, False if missing."""
+    container = _find_container(name)
+    if container is None:
+        return False
+    try:
+        container.stop(timeout=timeout)
+    except APIError as exc:
+        raise DeployError(f"Failed to stop container {name}: {exc}") from exc
+    return True
+
+
+def container_restart(name: str, *, timeout: int = 30) -> None:
+    container = _find_container(name)
+    if container is None:
+        raise DeployError(f"Container {name} not found")
+    try:
+        container.restart(timeout=timeout)
+    except APIError as exc:
+        raise DeployError(f"Failed to restart container {name}: {exc}") from exc
+
+
+def container_logs_tail(
+    name: str,
+    *,
+    tail: int = 30,
+    since: dt.datetime | None = None,
+    timestamps: bool = False,
+) -> str:
+    """Return the last ``tail`` log lines (or empty string if missing).
+
+    ``since`` restricts output to entries after that timestamp.
+    ``timestamps`` prepends an RFC3339 timestamp to each line.
+    """
+    container = _find_container(name)
+    if container is None:
+        return ""
+    try:
+        data = container.logs(tail=tail, since=since, timestamps=timestamps)
+    except APIError as exc:
+        raise DeployError(f"Failed to read logs for {name}: {exc}") from exc
+    return data.decode(errors="replace") if isinstance(data, bytes) else str(data)
+
+
+def container_logs_stream(
+    name: str,
+    *,
+    since: dt.datetime | None = None,
+    timestamps: bool = False,
+) -> Iterator[bytes]:
+    """Stream follow-mode logs from the container as byte chunks."""
+    container = _find_container(name)
+    if container is None:
+        raise DeployError(f"Container {name} not found")
+    try:
+        yield from container.logs(
+            stream=True, follow=True, since=since, timestamps=timestamps
+        )
+    except APIError as exc:
+        raise DeployError(f"Failed to stream logs for {name}: {exc}") from exc
+
+
+def container_health_status(name: str) -> str:
+    """Return ``healthy`` / ``unhealthy`` / ``starting``, or "" when the
+    container is missing or has no healthcheck configured."""
+    container = _find_container(name)
+    if container is None:
+        return ""
+    health = container.attrs["State"].get("Health")
+    return "" if health is None else str(health["Status"])
+
+
+def container_status(name: str) -> str:
+    """Return the container's lifecycle state, or ``missing`` when absent."""
+    container = _find_container(name)
+    if container is None:
+        return "missing"
+    return container.status
+
+
+def _find_volume(name: str) -> Volume | None:
+    matches = get_docker_client().volumes.list(filters={"name": name})
+    for volume in matches:
+        if volume.name == name:
+            return volume
+    return None
+
+
+def volume_exists(name: str) -> bool:
+    return _find_volume(name) is not None
+
+
+def volume_remove(name: str) -> bool:
+    """Remove ``name`` if present. Returns True on success, False if missing.
+
+    Raises ``DeployError`` on API errors (e.g. volume still in use).
+    """
+    volume = _find_volume(name)
+    if volume is None:
+        return False
+    try:
+        volume.remove()
+    except APIError as exc:
+        raise DeployError(f"Failed to remove volume {name}: {exc}") from exc
+    return True