omicslab 2.1.0__tar.gz → 2.2.0__tar.gz

{omicslab-2.1.0 → omicslab-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omicslab
-Version: 2.1.0
+Version: 2.2.0
 Summary: Python SDK and CLI for the OmicsLab Platform
 Project-URL: Homepage, https://platform.omicslab.io
 Project-URL: Repository, https://github.com/vieomics/omicslab-platform

omicslab-2.2.0/scripts/bootstrap-env.sh

@@ -0,0 +1,224 @@
+#!/bin/bash
+# bootstrap-env.sh — Install pixi + omicslab SDK from a public tools bucket.
+#
+# Environment variables:
+#   SDK_TOOLS_BASE  — tools bucket base URL (default: R2 production). Override
+#                     to point at any public S3-compatible provider (e.g.
+#                     local S3: http://localhost:4566/omicslab/data/tools).
+#   OMICSLAB_HOME   — where pixi.toml/lock live (default: $HOME)
+#   PIXI_TOML       — manifest filename (default: pixi.toml, or {region}.pixi.toml)
+#   PIXI_LOCK       — lock filename (default: pixi.lock, or {region}.pixi.lock)
+#   PIXI_VERSION    — pixi release to fetch from the tools bucket
+#                     (default: v0.76.1)
+#   INSTALL_DOCKER  — set to 1 for studio jobs to install the docker engine
+#                     (client + daemon) and start its daemon (default: 0)
+#
+# The script is idempotent: re-running it after a successful install is a
+# no-op (pixi binary present, environment installed, SDK already loaded).
+#
+# It:
+#   1. Installs pixi from the tools bucket (pixi/bin/pixi-<version>-linux-amd64.tar.gz)
+#      into an absolute path under $HOME — never hits pixi.sh / GitHub, so it
+#      works on air-gapped runners.
+#   2. Downloads the wheel-free manifest from the same bucket.
+#   3. Runs pixi install --frozen (all conda + pypi deps, both cached in the
+#      bucket channel/pypi mirrors).
+#   4. Downloads omicslab-latest.whl, extracts version from wheel metadata,
+#      renames to a valid PEP 427 filename, pip installs.
+#   5. Grants the pixi interpreter CAP_NET_BIND_SERVICE (falling back to
+#      lowering ip_unprivileged_port_start) so the studio TLS proxy can bind
+#      the privileged port 443.
+set -euo pipefail
+
+TOOLS_BASE="${SDK_TOOLS_BASE:-https://s3-hcm-r2.s3cloud.vn/omicslab/data/tools}"
+OMICSLAB_HOME="${OMICSLAB_HOME:-$HOME}"
+PIXI_TOML="${PIXI_TOML:-pixi.toml}"
+PIXI_LOCK="${PIXI_LOCK:-pixi.lock}"
+PIXI_VERSION="${PIXI_VERSION:-v0.76.1}"
+RETRIES=3
+
+# ── 1. pixi (from the tools bucket, absolute path under $HOME) ────────────
+PIXI_BIN="${HOME}/.pixi/bin/pixi"
+if [ ! -x "${PIXI_BIN}" ]; then
+    PIXI_URL="$TOOLS_BASE/pixi/bin/pixi-${PIXI_VERSION}-linux-amd64.tar.gz"
+    echo "Installing pixi ${PIXI_VERSION} from: $PIXI_URL"
+    mkdir -p "$(dirname "${PIXI_BIN}")"
+    curl -fsSL "$PIXI_URL" -o /tmp/pixi.tar.gz
+    tar -xzf /tmp/pixi.tar.gz -C "$(dirname "${PIXI_BIN}")"
+    rm -f /tmp/pixi.tar.gz
+else
+    echo "pixi already installed at ${PIXI_BIN}"
+fi
+export PATH="$(dirname "${PIXI_BIN}"):$PATH"
+
+# ── 1b. FUSE helper for mountpoint-s3 (non-root mounts) ────────────────────
+# mount-s3 performs unprivileged FUSE mounts via `fusermount3`, which must be
+# root-owned + setuid. The pixi env ships a non-setuid copy under sbin/ (also
+# not on PATH), so use the distro package when it is missing. Best-effort: on
+# air-gapped hosts without apt, mounts still work when run as root.
+if ! command -v fusermount3 >/dev/null 2>&1; then
+    if command -v apt-get >/dev/null 2>&1; then
+        SUDO=""
+        if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
+            SUDO="sudo -n"
+        fi
+        echo "Installing fuse3 (fusermount3 helper for mountpoint-s3)..."
+        $SUDO apt-get update -qq >/dev/null 2>&1 || true
+        $SUDO apt-get install -y -qq fuse3 >/dev/null 2>&1 || \
+            echo "WARNING: fuse3 install failed; mountpoint-s3 mounts will require root"
+    else
+        echo "WARNING: no apt-get; mountpoint-s3 non-root mounts will require root"
+    fi
+fi
+
+# ── 1c. docker engine (studio jobs) ────────────────────────────────────────
+# Studio jobs start their session container with `docker run`. On the
+# Docker-simulated VM the host daemon answers `docker info` through the
+# mounted /var/run/docker.sock, so only the client is needed; a real cloud VM
+# (CloudFly / GreenNode) has no host socket, so install the engine (client +
+# daemon), start the daemon, and let the job user reach its socket. Gated by
+# INSTALL_DOCKER so analysis jobs pay nothing for it. Best-effort: on
+# air-gapped hosts without apt the install is skipped and `job run --studio`
+# reports the unreachable daemon.
+if [ "${INSTALL_DOCKER:-0}" = "1" ]; then
+    SUDO=""
+    if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
+        SUDO="sudo -n"
+    fi
+    if ! command -v docker >/dev/null 2>&1; then
+        if command -v apt-get >/dev/null 2>&1; then
+            echo "Installing docker engine (client + daemon)..."
+            $SUDO apt-get update -qq >/dev/null 2>&1 || true
+            $SUDO apt-get install -y -qq docker.io >/dev/null 2>&1 || \
+                echo "WARNING: docker install failed; the studio session cannot start"
+        else
+            echo "WARNING: no apt-get; cannot install docker for the studio session"
+        fi
+    fi
+    # Start the daemon: systemd where available, else a detached dockerd.
+    # Check for the socket (not `docker info`) so a permission problem on the
+    # socket is not mistaken for a stopped daemon and does not spawn a second
+    # dockerd next to the systemd unit.
+    if command -v systemctl >/dev/null 2>&1; then
+        $SUDO systemctl enable --now docker >/dev/null 2>&1 || true
+    fi
+    if [ ! -S /var/run/docker.sock ] && command -v dockerd >/dev/null 2>&1; then
+        echo "Starting dockerd..."
+        $SUDO dockerd >/dev/null 2>&1 &
+    fi
+    for _ in $(seq 1 15); do
+        [ -S /var/run/docker.sock ] && break
+        sleep 1
+    done
+    # The daemon socket is root:docker mode 0660. The studio agent runs as the
+    # (possibly non-root) SSH user and reuses this very session, so `usermod`
+    # membership alone would only apply to a later login. Grant the socket to
+    # the job user's own group for the current session instead — least
+    # privilege, never world-writable. The studio container itself is started
+    # with --security-opt no-new-privileges and without the host socket mounted,
+    # so the web app inside it cannot reach this daemon.
+    if [ "$(id -u)" -ne 0 ]; then
+        $SUDO usermod -aG docker "$USER" >/dev/null 2>&1 || true
+        $SUDO chgrp "$(id -gn)" /var/run/docker.sock >/dev/null 2>&1 || true
+        $SUDO chmod 660 /var/run/docker.sock >/dev/null 2>&1 || true
+    fi
+    if docker info >/dev/null 2>&1; then
+        echo "docker daemon ready"
+    else
+        echo "WARNING: docker daemon not reachable after install"
+    fi
+fi
+
+# ── 2. manifest (from the tools bucket) ────────────────────────────────────
+TOML_URL="$TOOLS_BASE/pixi/config/$PIXI_TOML"
+LOCK_URL="$TOOLS_BASE/pixi/config/$PIXI_LOCK"
+echo "Fetching manifest:"
+echo "  TOML: $TOML_URL"
+echo "  LOCK: $LOCK_URL"
+mkdir -p "$OMICSLAB_HOME"
+curl -fsSL "$TOML_URL" -o "$OMICSLAB_HOME/pixi.toml"
+curl -fsSL "$LOCK_URL" -o "$OMICSLAB_HOME/pixi.lock"
+
+# ── 3. pixi install + SDK wheel (with retries) ────────────────────────────
+cd "$OMICSLAB_HOME"
+for i in $(seq 1 $RETRIES); do
+    if "${PIXI_BIN}" install --frozen; then
+        break
+    fi
+    echo "bootstrap-retry-$i"
+    sleep 5
+done
+
+# Download the stable alias and resolve the real version from wheel
+# metadata so pip gets a valid PEP 427 filename.
+echo "Installing omicslab SDK wheel..."
+curl -fsSL "$TOOLS_BASE/omicslab-latest.whl" -o /tmp/omicslab.whl
+VERSION=$("${PIXI_BIN}" run python3 -c "
+import zipfile, os
+z = zipfile.ZipFile('/tmp/omicslab.whl')
+meta = [n for n in z.namelist() if n.endswith('.dist-info/METADATA')][0]
+ver = [l.split(': ',1)[1] for l in z.read(meta).decode().splitlines() if l.startswith('Version:')][0]
+os.rename('/tmp/omicslab.whl', f'/tmp/omicslab-{ver}-py3-none-any.whl')
+print(ver)
+")
+WHEEL_PATH="/tmp/omicslab-${VERSION}-py3-none-any.whl"
+WHEEL_SHA256=$(sha256sum "$WHEEL_PATH" | awk '{print $1}')
+if "${PIXI_BIN}" run python -m pip install --no-deps "$WHEEL_PATH"; then
+    # Record the installed wheel so the runner auto-updater can tell whether
+    # the tools bucket has a newer build (by SHA-256, not just version).
+    RECORD_DIR="${HOME}/.omicslab"
+    mkdir -p "$RECORD_DIR"
+    cat > "$RECORD_DIR/install.json" <<JSON
+{
+  "version": "${VERSION}",
+  "sha256": "${WHEEL_SHA256}",
+  "wheel": "omicslab-${VERSION}-py3-none-any.whl",
+  "tools_base": "${TOOLS_BASE}",
+  "installed_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+}
+JSON
+    chmod 600 "$RECORD_DIR/install.json"
+    rm -f "$WHEEL_PATH"
+    echo "SDK ${VERSION} installed successfully"
+else
+    echo "Bootstrap failed after $RETRIES attempts"
+    exit 1
+fi
+
+# ── 4. studio direct access: allow binding the privileged port 443 ─────────
+# A studio session's in-VM TLS proxy binds 0.0.0.0:443 so
+# {job_id}.<studio-domain> resolves straight to this host. An unprivileged
+# user cannot bind ports below 1024 by default, so grant the pixi interpreter
+# CAP_NET_BIND_SERVICE — a targeted, per-binary capability with no system-wide
+# effect. Where setcap is unavailable (or the filesystem rejects capabilities)
+# fall back to lowering the kernel's unprivileged port floor. This is
+# provider-agnostic: it runs on every Linux host the SDK is bootstrapped on.
+# Best-effort — `job run --studio` still starts, and the proxy logs an
+# actionable error at startup if 443 turns out not to be bindable.
+PIXI_PY="$(readlink -f "${OMICSLAB_HOME}/.pixi/envs/default/bin/python" 2>/dev/null || true)"
+if [ ! -x "${PIXI_PY}" ]; then
+    PIXI_PY="$("${PIXI_BIN}" run python -c 'import sys; print(sys.executable)' 2>/dev/null | tail -n1)"
+    PIXI_PY="$(readlink -f "${PIXI_PY}" 2>/dev/null || true)"
+fi
+if [ -z "${PIXI_PY}" ] || [ ! -x "${PIXI_PY}" ]; then
+    echo "WARNING: could not locate the pixi python; studio proxy may be unable to bind 443"
+else
+    SUDO_CAP=""
+    if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
+        SUDO_CAP="sudo -n"
+    fi
+    if ! command -v setcap >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
+        $SUDO_CAP apt-get update -qq >/dev/null 2>&1 || true
+        $SUDO_CAP apt-get install -y -qq libcap2-bin >/dev/null 2>&1 || true
+    fi
+    if command -v setcap >/dev/null 2>&1 && \
+        $SUDO_CAP setcap cap_net_bind_service=+ep "${PIXI_PY}" 2>/dev/null; then
+        echo "Granted CAP_NET_BIND_SERVICE to ${PIXI_PY}"
+    elif $SUDO_CAP sysctl -w net.ipv4.ip_unprivileged_port_start=0 >/dev/null 2>&1; then
+        echo "net.ipv4.ip_unprivileged_port_start=0" | \
+            $SUDO_CAP tee /etc/sysctl.d/99-omicslab-studio.conf >/dev/null 2>&1 || true
+        echo "Lowered net.ipv4.ip_unprivileged_port_start to 0"
+    else
+        echo "WARNING: could not enable binding port 443; direct studio access may be unavailable"
+    fi
+fi

omicslab-2.2.0/src/omicslab/_version.py

@@ -0,0 +1 @@
+__version__ = "2.2.0"

omicslab-2.2.0/src/omicslab/cli/job_commands.py

@@ -0,0 +1,441 @@
+"""``omicslab job run`` — the single job-execution command.
+
+``job run`` IS the job. On a cloud VM it runs directly; on SLURM it is the
+sbatch payload running on the compute node. It runs the native engine
+(:mod:`omicslab.job._run`) synchronously, self-reports RUNNING / incremental
+logs / terminal status over REST, and exits with the job's exit code. There
+is no controller, no daemon and no rendered ``job.sh`` on the execution path.
+
+Usage::
+
+    omicslab job run --analysis --id <id> --work-dir <dir>
+    omicslab job run --studio    --id <id> --work-dir <dir>
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+
+import typer
+
+from .utils import error, info, success
+
+logger = logging.getLogger(__name__)
+
+job_app = typer.Typer(
+    name="job",
+    help="Run jobs (analysis / studio) — self-managed, synchronous",
+    no_args_is_help=True,
+)
+
+
+def _snapshot_argv(*, job_id: str, output_dir: str, request: dict) -> list[str]:
+    """Build the ``omicslab studio snapshot`` argv for a queued request.
+
+    The request is queued by the backend and delivered over the status poll
+    (the cloud agent has no WebSocket channel). The agent runs the snapshot
+    locally, so no SSH credential is needed.
+    """
+    argv = [
+        "studio",
+        "snapshot",
+        "--container",
+        f"omicslab-studio-{job_id}",
+        "--output-dir",
+        output_dir,
+        "--filename",
+        str(request.get("filename") or f"studio-snapshot-{job_id}.tar.zst"),
+        "--upload",
+        "--workspace-id",
+        str(request.get("workspace_id") or ""),
+    ]
+    if request.get("key"):
+        argv += ["--key", str(request["key"])]
+    if request.get("op_id"):
+        argv += ["--op-id", str(request["op_id"])]
+    return argv
+
+
+# The backend marks a job FAILED when no status/log update arrives for 10
+# minutes (service_job/tasks/sweep.py), treating it as compute-disconnected. An
+# interactive studio session can sit idle for far longer than that (code-server
+# emits no logs while the user is not typing), so ``job run`` must prove the VM
+# is alive on a periodic cadence. RUNNING → RUNNING is a no-op transition on the
+# backend, so re-reporting RUNNING is safe.
+JOB_HEARTBEAT_INTERVAL_SECONDS = 120
+
+
+def _job_heartbeat_loop(
+    job_client,
+    job_id: str,
+    stop_event,
+    interval: int = JOB_HEARTBEAT_INTERVAL_SECONDS,
+) -> None:
+    """Post RUNNING every ``interval`` seconds until ``stop_event`` is set.
+
+    Refreshes the backend's ``runner:job:last_update`` heartbeat so a quiet job
+    is not swept as disconnected. Failures are swallowed — a heartbeat POST must
+    never take the job down.
+    """
+    while not stop_event.wait(interval):
+        try:
+            job_client.post(f"/jobs/{job_id}/status/", json_data={"status": "running"})
+        except Exception:
+            logger.debug("heartbeat failed for job %s", job_id)
+
+
+# Cadence of the periodic ``[system]`` log line (CPU / memory / disk). A fast
+# cadence is useful locally, but on production it floods the job log/API over a
+# long interactive session, so use 30s for local (http) and 5 minutes for
+# production (https) — the same split as the runner client (job/client.py).
+JOB_SYSTEM_MONITOR_INTERVAL_SECONDS = 30
+JOB_SYSTEM_MONITOR_INTERVAL_PROD_SECONDS = 300
+
+
+def _system_monitor_interval(server_url: str | None) -> int:
+    """Seconds between ``[system]`` lines: fast locally, 5 min in production.
+
+    ``OMICSLAB_SYSTEM_MONITOR_INTERVAL`` overrides both (used by tests/ops).
+    """
+    override = os.environ.get("OMICSLAB_SYSTEM_MONITOR_INTERVAL", "")
+    if override:
+        try:
+            return max(1, int(override))
+        except ValueError:
+            pass
+    if (server_url or "").startswith("http://"):
+        return JOB_SYSTEM_MONITOR_INTERVAL_SECONDS
+    return JOB_SYSTEM_MONITOR_INTERVAL_PROD_SECONDS
+
+
+def _start_system_monitor(
+    job_dir: str,
+    stop_event,
+    emit,
+    slurm_job_id: str | None = None,
+    interval: int = JOB_SYSTEM_MONITOR_INTERVAL_SECONDS,
+) -> threading.Thread:
+    """Start a daemon thread emitting periodic ``[system]`` log lines.
+
+    Writes into the job's ``job.log`` and forwards each line to ``emit`` (the
+    ``job run`` log callback that POSTs it to the API) so the platform shows
+    live CPU / memory / disk. Uses SLURM ``sstat`` on a compute node, else
+    psutil. Returns the started thread.
+    """
+    from ..job._run._monitor import (
+        format_slurm_stats,
+        format_system_stats,
+        monitor_slurm_job,
+        monitor_system,
+    )
+
+    log_file = os.path.join(job_dir, "job.log")
+    if slurm_job_id:
+        target = monitor_slurm_job
+        kwargs = {
+            "job_id": slurm_job_id,
+            "log_file": log_file,
+            "callback": lambda stats: emit(format_slurm_stats(stats)),
+        }
+    else:
+        target = monitor_system
+        kwargs = {
+            "log_file": log_file,
+            "callback": lambda stats: emit(format_system_stats(stats)),
+        }
+
+    thread = threading.Thread(
+        target=target,
+        kwargs={**kwargs, "interval": interval, "stop_event": stop_event},
+        name="job-system-monitor",
+        daemon=True,
+    )
+    thread.start()
+    return thread
+
+
+@job_app.command("run")
+def run_job(
+    id: str | None = typer.Option(
+        None,
+        "--id",
+        "-j",
+        help="Job ID to execute",
+    ),
+    work_dir: str | None = typer.Option(
+        None,
+        "--work-dir",
+        "-w",
+        help="Job working directory (contains params.json). Executed in place.",
+    ),
+    studio: bool = typer.Option(
+        False,
+        "--studio",
+        help="Run a studio job (interactive container session). Default: analysis.",
+    ),
+    server_url: str | None = typer.Option(
+        None,
+        "--server-url",
+        "-s",
+        help="Server API base URL (e.g. http://omx-backend:8000/api/v1)",
+    ),
+    api_token: str | None = typer.Option(
+        None,
+        "--token",
+        envvar="OMICSLAB_TOKEN",
+        help="API token for authentication",
+    ),
+    no_report: bool = typer.Option(
+        False,
+        "--no-report",
+        help="Skip API reporting (writes status/log locally; for testing)",
+    ),
+):
+    """Execute a job synchronously and self-report its lifecycle.
+
+    ``job run --id <id> --work-dir <dir>`` runs a Nextflow / tool analysis
+    job in place; ``job run --studio --id <id> --work-dir <dir>`` runs an
+    interactive studio container session. Both are self-managed: they POST
+    RUNNING, stream incremental log lines, watch for CANCELLING and post the
+    terminal status over REST as they run. No job.sh, no background submit,
+    no monitor loop.
+    """
+    from datetime import datetime as _dt
+
+    kind = "studio" if studio else "analysis"
+
+    if not work_dir:
+        error("Provide --work-dir <dir> (the staged job directory with params.json).")
+        raise typer.Exit(1)
+
+    job_dir = os.path.abspath(work_dir)
+    os.makedirs(job_dir, exist_ok=True)
+    params_file = os.path.join(job_dir, "params.json")
+    if not os.path.isfile(params_file):
+        error(f"Params file not found: {params_file}")
+        raise typer.Exit(1)
+    try:
+        with open(params_file) as f:
+            params_dict = json.load(f)
+    except json.JSONDecodeError as e:
+        error(f"Invalid JSON in {params_file}: {e}")
+        raise typer.Exit(1)
+
+    # ── Resolve the job id ─────────────────────────────────────────────
+    # The staged work-dir is ``<jobs_dir>/<job_id>`` so its basename is the
+    # job id; this lets the runner dispatch ``job run --work-dir <dir>``
+    # without threading ``--id`` through the sbatch wrapper.
+    if not id:
+        id = os.path.basename(job_dir.rstrip("/")) or None
+    if not id:
+        error("Provide --id <job-id>.")
+        raise typer.Exit(1)
+
+    # ── Resolve server URL / token (staged creds win over CLI/env) ──
+    if not no_report:
+        from ..config import Config
+
+        config = Config()
+        if not api_token:
+            api_token = params_dict.get("OMICSLAB_TOKEN") or config.get_token()
+            if not api_token:
+                error(
+                    "No API token. Provide --token, set OMICSLAB_TOKEN, or stage "
+                    "OMICSLAB_TOKEN in params.json."
+                )
+                raise typer.Exit(1)
+        if not server_url:
+            server_url = params_dict.get("API_BASE_URL") or config.get_base_url()
+        if not server_url:
+            error(
+                "Server URL required. Provide --server-url, stage API_BASE_URL "
+                "in params.json, or set it via 'omicslab auth login'."
+            )
+            raise typer.Exit(1)
+
+    # ── API client for self-reporting ──
+    client = None
+    if not no_report:
+        from ..client import Client
+
+        resolved_url = server_url.rstrip("/")
+        if not resolved_url.endswith("/api/v1"):
+            resolved_url += "/api/v1"
+        client = Client(token=api_token, base_url=resolved_url)
+
+    if client:
+        try:
+            client.post(f"/jobs/{id}/status/", json_data={"status": "running"})
+        except Exception as e:
+            logger.warning("Failed to report running: %s", e)
+
+    start = _dt.now()
+
+    def _post_log(line: str):
+        if client is None:
+            return
+        try:
+            client.post(f"/jobs/{id}/log/", json_data={"lines": [line]})
+        except Exception:
+            pass
+
+    # ── Watch for CANCELLING / snapshot requests while the engine runs ──
+    cancel_flag = threading.Event()
+    cancelled = threading.Event()
+    job_client = client
+
+    def _launch_snapshot(request: dict) -> None:
+        """Run a backend-queued snapshot locally (the agent owns the container)."""
+        import shutil
+        import subprocess
+        import sys as _sys
+
+        from ..job._run._log import append as _append_log
+
+        out_dir = os.path.join(job_dir, "snapshots")
+        os.makedirs(out_dir, exist_ok=True)
+        env = dict(os.environ)
+        if api_token:
+            env["OMICSLAB_TOKEN"] = api_token
+        if server_url:
+            env["OMICSLAB_BASE_URL"] = server_url
+        exe = shutil.which("omicslab") or _sys.argv[0]
+        argv = [exe, *_snapshot_argv(job_id=id, output_dir=out_dir, request=request)]
+        logger.info(
+            "Snapshot requested for job %s; launching local studio snapshot", id
+        )
+        log_path = os.path.join(out_dir, "snapshot.log")
+        try:
+            proc = subprocess.Popen(
+                argv,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+                close_fds=True,
+                env=env,
+            )
+        except Exception as exc:  # noqa: BLE001 — surface it, never crash the agent
+            _append_log(job_dir, f"[snapshot] failed to start: {exc}", _post_log)
+            return
+
+        def _stream_snapshot_log(proc=proc, log_path=log_path) -> None:
+            # Tee the snapshot output: keep the on-VM snapshot.log and mirror it
+            # into the job log (and the API) so snapshot progress is visible in
+            # the platform, not only on the VM.
+            try:
+                with open(log_path, "a") as log:
+                    for raw in iter(proc.stdout.readline, b""):
+                        line = raw.decode("utf-8", "replace").rstrip("\n")
+                        log.write(line + "\n")
+                        log.flush()
+                        _append_log(job_dir, f"[snapshot] {line}", _post_log)
+            except Exception:
+                logger.debug("snapshot log stream ended", exc_info=True)
+            finally:
+                try:
+                    proc.stdout.close()
+                except Exception:
+                    pass
+
+        threading.Thread(
+            target=_stream_snapshot_log,
+            name=f"snapshot-log-{id[:8]}",
+            daemon=True,
+        ).start()
+
+    def _watch_cancel():
+        while not cancel_flag.is_set():
+            if job_client is None:
+                return
+            try:
+                resp = job_client.get(f"/jobs/{id}/")
+            except Exception:
+                resp = None
+            if isinstance(resp, dict):
+                if kind == "studio" and resp.get("snapshot_request"):
+                    try:
+                        _launch_snapshot(resp["snapshot_request"])
+                    except Exception:
+                        logger.warning(
+                            "Failed to launch requested snapshot for job %s", id
+                        )
+                if resp.get("status") == "cancelling":
+                    cancelled.set()
+                    # Signal the whole process group so the child pipeline is killed.
+                    import os as _os
+                    import signal as _signal
+
+                    try:
+                        _os.killpg(_os.getpgid(_os.getpid()), _signal.SIGTERM)
+                    except (ProcessLookupError, OSError):
+                        pass
+                    return
+            time.sleep(5)
+
+    if job_client is not None:
+        threading.Thread(target=_watch_cancel, daemon=True).start()
+        threading.Thread(
+            target=_job_heartbeat_loop,
+            args=(job_client, id, cancel_flag),
+            name=f"job-heartbeat-{id[:8]}",
+            daemon=True,
+        ).start()
+        _start_system_monitor(
+            job_dir,
+            cancel_flag,
+            _post_log,
+            slurm_job_id=os.environ.get("SLURM_JOB_ID")
+            or os.environ.get("SLURM_JOBID"),
+            interval=_system_monitor_interval(server_url),
+        )
+
+    info(f"Executing {kind} job {id}")
+    try:
+        if kind == "studio":
+            from ..job._run.studio import run_studio
+
+            exit_code = run_studio(id, params_dict, on_log=_post_log, job_dir=job_dir)
+        else:
+            from ..job._run.analysis import run_analysis
+
+            exit_code = run_analysis(id, params_dict, on_log=_post_log, job_dir=job_dir)
+    except Exception as e:
+        logger.exception("Job %s failed", id)
+        error(f"Execution failed: {e}")
+        exit_code = 1
+
+    cancel_flag.set()
+    if client is not None:
+        status = (
+            "cancelled"
+            if (cancelled.is_set() or exit_code == 130)
+            else ("completed" if exit_code == 0 else "failed")
+        )
+        payload = {"status": status}
+        elapsed = int((_dt.now() - start).total_seconds())
+        payload["running_time"] = elapsed
+        try:
+            client.post(f"/jobs/{id}/status/", json_data=payload)
+        except Exception as e:
+            logger.warning("Failed to report final status: %s", e)
+
+        # Keep the work-dir for non-https (dev/local) servers; job run owns
+        # cleanup in production.
+        if server_url and server_url.startswith("https://"):
+            try:
+                from ..job.executor import JobExecutor
+
+                executor = JobExecutor()
+                executor.register_job_dir(id, job_dir)
+                executor.cleanup(id)
+            except Exception as e:
+                logger.warning("Failed to cleanup work-dir: %s", e)
+
+    if exit_code not in (0, 130):
+        raise typer.Exit(exit_code)
+    success(f"Job {id} completed")