omicslab 2.0.0__tar.gz → 2.2.0__tar.gz

{omicslab-2.0.0 → omicslab-2.2.0}/Makefile

@@ -1,6 +1,4 @@
 # === Config (all overridable via environment / CLI) ===
-ENDPOINT_URL ?= http://localhost:4566
-REGION_NAME  ?= us-east-1
 
 S3_BUCKET_NAME ?= omicslab
 S3_PREFIX      ?= data/tools

{omicslab-2.0.0 → omicslab-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omicslab
-Version: 2.0.0
+Version: 2.2.0
 Summary: Python SDK and CLI for the OmicsLab Platform
 Project-URL: Homepage, https://platform.omicslab.io
 Project-URL: Repository, https://github.com/vieomics/omicslab-platform

{omicslab-2.0.0 → omicslab-2.2.0}/README.developer.md

@@ -191,7 +191,7 @@ This:
 4. Polls `http://localhost:8000/api/v1/health` until healthy.
 
 Notes:
-- **Storage is S3-compatible (SeaweedFS).** `ENDPOINT_URL=http://localhost:4566`
+- **Storage is S3-compatible (SeaweedFS).** `AWS_ENDPOINT_URL=http://localhost:4566`
   (from `.env.test`) with credentials `test`/`testtest` (`AWS_ACCESS_KEY_ID` /
   `AWS_SECRET_ACCESS_KEY`).
 - `APP_ENV=test` is set so the backend keeps the DB hermetic and uses test env
@@ -308,7 +308,7 @@ CI re-upload the wheel whenever SDK code changes without regenerating locks,
 and refresh the pixi cache only when `sdk/envs/pixi.toml` changes (see the
 gated `upload-bootstrap-env` target in the root `Makefile`).
 
-Config overrides (env vars): `ENDPOINT_URL`, `S3_ENDPOINT_URL`, `S3_BUCKET_NAME`, `S3_PREFIX`, `S3_REGION`.
+Config overrides (env vars): `AWS_ENDPOINT_URL`, `AWS_REGION`, `S3_ENDPOINT_URL`, `S3_BUCKET_NAME`, `S3_PREFIX`, `S3_REGION`.
 
 ### 8.1 Runner SDK self-update
 
@@ -489,7 +489,7 @@ Cloud tests are slow (VM boot + pixi install); use generous timeouts.
 - **Token scope == access.** An org-scoped token can't touch other orgs. Bootstrap
   fresh users/orgs for isolated scenarios.
 - **Storage is S3-compatible (SeaweedFS)** at `:4566`. Use `AWS_ACCESS_KEY_ID=test`,
-  `AWS_SECRET_ACCESS_KEY=testtest`, `ENDPOINT_URL=http://localhost:4566`.
+  `AWS_SECRET_ACCESS_KEY=testtest`, `AWS_ENDPOINT_URL=http://localhost:4566`.
 - **Empty credential listing**: the backend now returns `[]` (via `ok_json_list`)
   even when no credentials exist; older behavior returned an empty body.
 - **Single-get routes** for org/workspace/analysis don't exist; verify via lists.

omicslab-2.2.0/scripts/bootstrap-env.sh

@@ -0,0 +1,224 @@
+#!/bin/bash
+# bootstrap-env.sh — Install pixi + omicslab SDK from a public tools bucket.
+#
+# Environment variables:
+#   SDK_TOOLS_BASE  — tools bucket base URL (default: R2 production). Override
+#                     to point at any public S3-compatible provider (e.g.
+#                     local S3: http://localhost:4566/omicslab/data/tools).
+#   OMICSLAB_HOME   — where pixi.toml/lock live (default: $HOME)
+#   PIXI_TOML       — manifest filename (default: pixi.toml, or {region}.pixi.toml)
+#   PIXI_LOCK       — lock filename (default: pixi.lock, or {region}.pixi.lock)
+#   PIXI_VERSION    — pixi release to fetch from the tools bucket
+#                     (default: v0.76.1)
+#   INSTALL_DOCKER  — set to 1 for studio jobs to install the docker engine
+#                     (client + daemon) and start its daemon (default: 0)
+#
+# The script is idempotent: re-running it after a successful install is a
+# no-op (pixi binary present, environment installed, SDK already loaded).
+#
+# It:
+#   1. Installs pixi from the tools bucket (pixi/bin/pixi-<version>-linux-amd64.tar.gz)
+#      into an absolute path under $HOME — never hits pixi.sh / GitHub, so it
+#      works on air-gapped runners.
+#   2. Downloads the wheel-free manifest from the same bucket.
+#   3. Runs pixi install --frozen (all conda + pypi deps, both cached in the
+#      bucket channel/pypi mirrors).
+#   4. Downloads omicslab-latest.whl, extracts version from wheel metadata,
+#      renames to a valid PEP 427 filename, pip installs.
+#   5. Grants the pixi interpreter CAP_NET_BIND_SERVICE (falling back to
+#      lowering ip_unprivileged_port_start) so the studio TLS proxy can bind
+#      the privileged port 443.
+set -euo pipefail
+
+TOOLS_BASE="${SDK_TOOLS_BASE:-https://s3-hcm-r2.s3cloud.vn/omicslab/data/tools}"
+OMICSLAB_HOME="${OMICSLAB_HOME:-$HOME}"
+PIXI_TOML="${PIXI_TOML:-pixi.toml}"
+PIXI_LOCK="${PIXI_LOCK:-pixi.lock}"
+PIXI_VERSION="${PIXI_VERSION:-v0.76.1}"
+RETRIES=3
+
+# ── 1. pixi (from the tools bucket, absolute path under $HOME) ────────────
+PIXI_BIN="${HOME}/.pixi/bin/pixi"
+if [ ! -x "${PIXI_BIN}" ]; then
+    PIXI_URL="$TOOLS_BASE/pixi/bin/pixi-${PIXI_VERSION}-linux-amd64.tar.gz"
+    echo "Installing pixi ${PIXI_VERSION} from: $PIXI_URL"
+    mkdir -p "$(dirname "${PIXI_BIN}")"
+    curl -fsSL "$PIXI_URL" -o /tmp/pixi.tar.gz
+    tar -xzf /tmp/pixi.tar.gz -C "$(dirname "${PIXI_BIN}")"
+    rm -f /tmp/pixi.tar.gz
+else
+    echo "pixi already installed at ${PIXI_BIN}"
+fi
+export PATH="$(dirname "${PIXI_BIN}"):$PATH"
+
+# ── 1b. FUSE helper for mountpoint-s3 (non-root mounts) ────────────────────
+# mount-s3 performs unprivileged FUSE mounts via `fusermount3`, which must be
+# root-owned + setuid. The pixi env ships a non-setuid copy under sbin/ (also
+# not on PATH), so use the distro package when it is missing. Best-effort: on
+# air-gapped hosts without apt, mounts still work when run as root.
+if ! command -v fusermount3 >/dev/null 2>&1; then
+    if command -v apt-get >/dev/null 2>&1; then
+        SUDO=""
+        if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
+            SUDO="sudo -n"
+        fi
+        echo "Installing fuse3 (fusermount3 helper for mountpoint-s3)..."
+        $SUDO apt-get update -qq >/dev/null 2>&1 || true
+        $SUDO apt-get install -y -qq fuse3 >/dev/null 2>&1 || \
+            echo "WARNING: fuse3 install failed; mountpoint-s3 mounts will require root"
+    else
+        echo "WARNING: no apt-get; mountpoint-s3 non-root mounts will require root"
+    fi
+fi
+
+# ── 1c. docker engine (studio jobs) ────────────────────────────────────────
+# Studio jobs start their session container with `docker run`. On the
+# Docker-simulated VM the host daemon answers `docker info` through the
+# mounted /var/run/docker.sock, so only the client is needed; a real cloud VM
+# (CloudFly / GreenNode) has no host socket, so install the engine (client +
+# daemon), start the daemon, and let the job user reach its socket. Gated by
+# INSTALL_DOCKER so analysis jobs pay nothing for it. Best-effort: on
+# air-gapped hosts without apt the install is skipped and `job run --studio`
+# reports the unreachable daemon.
+if [ "${INSTALL_DOCKER:-0}" = "1" ]; then
+    SUDO=""
+    if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
+        SUDO="sudo -n"
+    fi
+    if ! command -v docker >/dev/null 2>&1; then
+        if command -v apt-get >/dev/null 2>&1; then
+            echo "Installing docker engine (client + daemon)..."
+            $SUDO apt-get update -qq >/dev/null 2>&1 || true
+            $SUDO apt-get install -y -qq docker.io >/dev/null 2>&1 || \
+                echo "WARNING: docker install failed; the studio session cannot start"
+        else
+            echo "WARNING: no apt-get; cannot install docker for the studio session"
+        fi
+    fi
+    # Start the daemon: systemd where available, else a detached dockerd.
+    # Check for the socket (not `docker info`) so a permission problem on the
+    # socket is not mistaken for a stopped daemon and does not spawn a second
+    # dockerd next to the systemd unit.
+    if command -v systemctl >/dev/null 2>&1; then
+        $SUDO systemctl enable --now docker >/dev/null 2>&1 || true
+    fi
+    if [ ! -S /var/run/docker.sock ] && command -v dockerd >/dev/null 2>&1; then
+        echo "Starting dockerd..."
+        $SUDO dockerd >/dev/null 2>&1 &
+    fi
+    for _ in $(seq 1 15); do
+        [ -S /var/run/docker.sock ] && break
+        sleep 1
+    done
+    # The daemon socket is root:docker mode 0660. The studio agent runs as the
+    # (possibly non-root) SSH user and reuses this very session, so `usermod`
+    # membership alone would only apply to a later login. Grant the socket to
+    # the job user's own group for the current session instead — least
+    # privilege, never world-writable. The studio container itself is started
+    # with --security-opt no-new-privileges and without the host socket mounted,
+    # so the web app inside it cannot reach this daemon.
+    if [ "$(id -u)" -ne 0 ]; then
+        $SUDO usermod -aG docker "$USER" >/dev/null 2>&1 || true
+        $SUDO chgrp "$(id -gn)" /var/run/docker.sock >/dev/null 2>&1 || true
+        $SUDO chmod 660 /var/run/docker.sock >/dev/null 2>&1 || true
+    fi
+    if docker info >/dev/null 2>&1; then
+        echo "docker daemon ready"
+    else
+        echo "WARNING: docker daemon not reachable after install"
+    fi
+fi
+
+# ── 2. manifest (from the tools bucket) ────────────────────────────────────
+TOML_URL="$TOOLS_BASE/pixi/config/$PIXI_TOML"
+LOCK_URL="$TOOLS_BASE/pixi/config/$PIXI_LOCK"
+echo "Fetching manifest:"
+echo "  TOML: $TOML_URL"
+echo "  LOCK: $LOCK_URL"
+mkdir -p "$OMICSLAB_HOME"
+curl -fsSL "$TOML_URL" -o "$OMICSLAB_HOME/pixi.toml"
+curl -fsSL "$LOCK_URL" -o "$OMICSLAB_HOME/pixi.lock"
+
+# ── 3. pixi install + SDK wheel (with retries) ────────────────────────────
+cd "$OMICSLAB_HOME"
+for i in $(seq 1 $RETRIES); do
+    if "${PIXI_BIN}" install --frozen; then
+        break
+    fi
+    echo "bootstrap-retry-$i"
+    sleep 5
+done
+
+# Download the stable alias and resolve the real version from wheel
+# metadata so pip gets a valid PEP 427 filename.
+echo "Installing omicslab SDK wheel..."
+curl -fsSL "$TOOLS_BASE/omicslab-latest.whl" -o /tmp/omicslab.whl
+VERSION=$("${PIXI_BIN}" run python3 -c "
+import zipfile, os
+z = zipfile.ZipFile('/tmp/omicslab.whl')
+meta = [n for n in z.namelist() if n.endswith('.dist-info/METADATA')][0]
+ver = [l.split(': ',1)[1] for l in z.read(meta).decode().splitlines() if l.startswith('Version:')][0]
+os.rename('/tmp/omicslab.whl', f'/tmp/omicslab-{ver}-py3-none-any.whl')
+print(ver)
+")
+WHEEL_PATH="/tmp/omicslab-${VERSION}-py3-none-any.whl"
+WHEEL_SHA256=$(sha256sum "$WHEEL_PATH" | awk '{print $1}')
+if "${PIXI_BIN}" run python -m pip install --no-deps "$WHEEL_PATH"; then
+    # Record the installed wheel so the runner auto-updater can tell whether
+    # the tools bucket has a newer build (by SHA-256, not just version).
+    RECORD_DIR="${HOME}/.omicslab"
+    mkdir -p "$RECORD_DIR"
+    cat > "$RECORD_DIR/install.json" <<JSON
+{
+  "version": "${VERSION}",
+  "sha256": "${WHEEL_SHA256}",
+  "wheel": "omicslab-${VERSION}-py3-none-any.whl",
+  "tools_base": "${TOOLS_BASE}",
+  "installed_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+}
+JSON
+    chmod 600 "$RECORD_DIR/install.json"
+    rm -f "$WHEEL_PATH"
+    echo "SDK ${VERSION} installed successfully"
+else
+    echo "Bootstrap failed after $RETRIES attempts"
+    exit 1
+fi
+
+# ── 4. studio direct access: allow binding the privileged port 443 ─────────
+# A studio session's in-VM TLS proxy binds 0.0.0.0:443 so
+# {job_id}.<studio-domain> resolves straight to this host. An unprivileged
+# user cannot bind ports below 1024 by default, so grant the pixi interpreter
+# CAP_NET_BIND_SERVICE — a targeted, per-binary capability with no system-wide
+# effect. Where setcap is unavailable (or the filesystem rejects capabilities)
+# fall back to lowering the kernel's unprivileged port floor. This is
+# provider-agnostic: it runs on every Linux host the SDK is bootstrapped on.
+# Best-effort — `job run --studio` still starts, and the proxy logs an
+# actionable error at startup if 443 turns out not to be bindable.
+PIXI_PY="$(readlink -f "${OMICSLAB_HOME}/.pixi/envs/default/bin/python" 2>/dev/null || true)"
+if [ ! -x "${PIXI_PY}" ]; then
+    PIXI_PY="$("${PIXI_BIN}" run python -c 'import sys; print(sys.executable)' 2>/dev/null | tail -n1)"
+    PIXI_PY="$(readlink -f "${PIXI_PY}" 2>/dev/null || true)"
+fi
+if [ -z "${PIXI_PY}" ] || [ ! -x "${PIXI_PY}" ]; then
+    echo "WARNING: could not locate the pixi python; studio proxy may be unable to bind 443"
+else
+    SUDO_CAP=""
+    if [ "$(id -u)" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
+        SUDO_CAP="sudo -n"
+    fi
+    if ! command -v setcap >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
+        $SUDO_CAP apt-get update -qq >/dev/null 2>&1 || true
+        $SUDO_CAP apt-get install -y -qq libcap2-bin >/dev/null 2>&1 || true
+    fi
+    if command -v setcap >/dev/null 2>&1 && \
+        $SUDO_CAP setcap cap_net_bind_service=+ep "${PIXI_PY}" 2>/dev/null; then
+        echo "Granted CAP_NET_BIND_SERVICE to ${PIXI_PY}"
+    elif $SUDO_CAP sysctl -w net.ipv4.ip_unprivileged_port_start=0 >/dev/null 2>&1; then
+        echo "net.ipv4.ip_unprivileged_port_start=0" | \
+            $SUDO_CAP tee /etc/sysctl.d/99-omicslab-studio.conf >/dev/null 2>&1 || true
+        echo "Lowered net.ipv4.ip_unprivileged_port_start to 0"
+    else
+        echo "WARNING: could not enable binding port 443; direct studio access may be unavailable"
+    fi
+fi

omicslab-2.2.0/src/omicslab/_version.py

@@ -0,0 +1 @@
+__version__ = "2.2.0"