omicslab 2.0.0__tar.gz → 2.1.0__tar.gz

{omicslab-2.0.0 → omicslab-2.1.0}/Makefile

@@ -1,6 +1,4 @@
 # === Config (all overridable via environment / CLI) ===
-ENDPOINT_URL ?= http://localhost:4566
-REGION_NAME  ?= us-east-1
 
 S3_BUCKET_NAME ?= omicslab
 S3_PREFIX      ?= data/tools

{omicslab-2.0.0 → omicslab-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omicslab
-Version: 2.0.0
+Version: 2.1.0
 Summary: Python SDK and CLI for the OmicsLab Platform
 Project-URL: Homepage, https://platform.omicslab.io
 Project-URL: Repository, https://github.com/vieomics/omicslab-platform

{omicslab-2.0.0 → omicslab-2.1.0}/README.developer.md

@@ -191,7 +191,7 @@ This:
 4. Polls `http://localhost:8000/api/v1/health` until healthy.
 
 Notes:
-- **Storage is S3-compatible (SeaweedFS).** `ENDPOINT_URL=http://localhost:4566`
+- **Storage is S3-compatible (SeaweedFS).** `AWS_ENDPOINT_URL=http://localhost:4566`
   (from `.env.test`) with credentials `test`/`testtest` (`AWS_ACCESS_KEY_ID` /
   `AWS_SECRET_ACCESS_KEY`).
 - `APP_ENV=test` is set so the backend keeps the DB hermetic and uses test env
@@ -308,7 +308,7 @@ CI re-upload the wheel whenever SDK code changes without regenerating locks,
 and refresh the pixi cache only when `sdk/envs/pixi.toml` changes (see the
 gated `upload-bootstrap-env` target in the root `Makefile`).
 
-Config overrides (env vars): `ENDPOINT_URL`, `S3_ENDPOINT_URL`, `S3_BUCKET_NAME`, `S3_PREFIX`, `S3_REGION`.
+Config overrides (env vars): `AWS_ENDPOINT_URL`, `AWS_REGION`, `S3_ENDPOINT_URL`, `S3_BUCKET_NAME`, `S3_PREFIX`, `S3_REGION`.
 
 ### 8.1 Runner SDK self-update
 
@@ -489,7 +489,7 @@ Cloud tests are slow (VM boot + pixi install); use generous timeouts.
 - **Token scope == access.** An org-scoped token can't touch other orgs. Bootstrap
   fresh users/orgs for isolated scenarios.
 - **Storage is S3-compatible (SeaweedFS)** at `:4566`. Use `AWS_ACCESS_KEY_ID=test`,
-  `AWS_SECRET_ACCESS_KEY=testtest`, `ENDPOINT_URL=http://localhost:4566`.
+  `AWS_SECRET_ACCESS_KEY=testtest`, `AWS_ENDPOINT_URL=http://localhost:4566`.
 - **Empty credential listing**: the backend now returns `[]` (via `ok_json_list`)
   even when no credentials exist; older behavior returned an empty body.
 - **Single-get routes** for org/workspace/analysis don't exist; verify via lists.

omicslab-2.1.0/src/omicslab/_version.py

@@ -0,0 +1 @@
+__version__ = "2.1.0"

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/cli/main.py

@@ -670,7 +670,8 @@ def upload(
                         },
                     )
                     success(
-                        f"Uploaded {local.name} → {remote_key} ({file_size / 1024 / 1024:.1f} MB)"
+                        f"Uploaded {local.name} → {remote_key} "
+                        f"({file_size / 1024 / 1024:.1f} MB) using presigned URL"
                     )
                 except Exception as e:
                     try:
@@ -759,7 +760,8 @@ def upload(
 
                     if resp.is_success:
                         success(
-                            f"Uploaded {local.name} → {remote_key} ({file_size / 1024 / 1024:.1f} MB)"
+                            f"Uploaded {local.name} → {remote_key} "
+                            f"({file_size / 1024 / 1024:.1f} MB) using presigned URL"
                         )
                         break
 
@@ -793,6 +795,8 @@ def upload(
         info("No files found to upload")
         return
 
+    info(f"Uploading {len(file_list)} files using presigned URLs...")
+
     from rich.progress import (
         BarColumn,
         DownloadColumn,
@@ -880,11 +884,11 @@ def upload(
     if uploaded[0] < len(file_list):
         fail(
             f"Upload incomplete: {uploaded[0]}/{len(file_list)} files uploaded "
-            f"({total_bytes / 1024 / 1024:.1f} MB)"
+            f"({total_bytes / 1024 / 1024:.1f} MB) using presigned URLs"
         )
     success(
         f"Uploaded {uploaded[0]}/{len(file_list)} files "
-        f"({total_bytes / 1024 / 1024:.1f} MB)"
+        f"({total_bytes / 1024 / 1024:.1f} MB) using presigned URLs"
     )
 
 
@@ -1014,6 +1018,8 @@ def download(
             info("No files found in this path.")
             return
 
+        info(f"Downloading {len(download_items)} files using presigned URLs...")
+
         downloaded = 0
         failed = 0
         progress = Progress(
@@ -1075,11 +1081,12 @@ def download(
         if failed:
             success(
                 f"Downloaded {downloaded}/{len(download_items)} files "
-                f"({failed} failed) to {dest_dir}"
+                f"({failed} failed) to {dest_dir} using presigned URLs"
             )
         else:
             success(
-                f"Downloaded {downloaded}/{len(download_items)} files to {dest_dir}"
+                f"Downloaded {downloaded}/{len(download_items)} files to {dest_dir} "
+                f"using presigned URLs"
             )
         return
 
@@ -1127,7 +1134,8 @@ def download(
                     progress.update(task, advance=len(chunk))
 
         success(
-            f"Downloaded {full_key} → {dest} ({dest.stat().st_size / 1024 / 1024:.1f} MB)"
+            f"Downloaded {full_key} → {dest} "
+            f"({dest.stat().st_size / 1024 / 1024:.1f} MB) using presigned URL"
         )
     except typer.Exit:
         raise

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/_run/_data_prep.py

@@ -3,15 +3,17 @@
 Implements the analysis branch of ``omicslab job prepare-data``: scans
 ``params.json`` for ``s3://`` values (except ``outdir``), resolves each
 bucket to a workspace via the API, and decides how to make the data
-available locally — public buckets are FUSE-mounted, private buckets in
-sandbox mode are downloaded, marketplace/cross-workspace buckets are
-mounted with source credentials. ``outdir`` is always rewritten to a
+available locally — public buckets are FUSE-mounted and private buckets are
+downloaded via presigned URLs (``storage_mode="sandbox"``, the platform
+default for every job). Clients running under the future
+EXPORT_JOB_IAM_KEYS safeguard may carry real credentials and mount instead.
+``outdir`` is always rewritten to a
 local path, preserving the original S3 path for post-job upload.
 
 CSV/TSV file params (samplesheets) are additionally localized *inside*:
 every ``s3://`` cell is rewritten to the local path of that object, and the
-param is repointed at the rewritten copy. On mounted jobs (marketplace
-compute) the cells point into the mount; on sandbox jobs the referenced
+param is repointed at the rewritten copy. On mounted jobs (credential-export
+safeguard path) the cells point into the mount; on sandbox jobs the referenced
 objects are downloaded next to the localized file first.
 """
 
@@ -78,6 +80,10 @@ def _download_object(
 
     Used by sandbox jobs, which carry no S3 credentials; the presigned URL is
     minted with the job's API token (``OMICSLAB_TOKEN``).
+
+    ``key_path`` is the raw parsed ``s3://bucket/data/...`` key; the storage
+    API re-adds the ``data/`` prefix internally, so it is stripped here (same
+    convention as ``resolve_s3_path``) before minting the URL.
     """
     api_base = os.environ.get("API_BASE_URL", "")
     token = os.environ.get("OMICSLAB_TOKEN", "")
@@ -93,7 +99,9 @@ def _download_object(
 
         client = Client(token=token, base_url=api_base)
         try:
-            result = client.storage.get_download_url(workspace_id, key_path)
+            result = client.storage.get_download_url(
+                workspace_id, key_path.removeprefix("data/")
+            )
         finally:
             client.close()
         url = result.get("url") if isinstance(result, dict) else result
@@ -123,9 +131,9 @@ def prepare_data(params_file: str, job_dir: str) -> None:
 
       **Public workspace**: mounts read-only via s3fs ``--public``.
       **Private workspace + sandbox mode**: downloads via ``omicslab download``.
-      **Private workspace + real creds (marketplace compute)**: rewrites path;
-      the workspace bucket is mounted read-only with the job's credentials
-      (STS tokens or long-lived provider keys).
+      **Private workspace + real creds (EXPORT_JOB_IAM_KEYS safeguard path)**:
+      rewrites path; the workspace bucket is mounted read-only with the job's
+      credentials (STS tokens or long-lived provider keys).
       **Bucket not found** (e.g. ``s3://igenomes``): keeps S3 path as-is (Nextflow handles it).
 
     ``outdir`` is always rewritten to ``{{ job_dir }}/outdir`` (local only).
@@ -340,7 +348,7 @@ def prepare_data(params_file: str, job_dir: str) -> None:
             else:
                 info(f"Public data already mounted: {public_mount}")
 
-    # === Mount cross-workspace private buckets (marketplace, source credentials) ===
+    # === Mount cross-workspace private buckets (EXPORT_JOB_IAM_KEYS path) ===
     for cm in cross_ws_mounts:
         os.makedirs(cm["mount_point"], exist_ok=True)
         mount_cmd = [
@@ -396,12 +404,12 @@ def prepare_data(params_file: str, job_dir: str) -> None:
                 # Single object: fetch it to the exact path the param points at.
                 local_path, mode, workspace_id = _local_for(bucket, key_path)
                 if mode == "sandbox" and local_path and not os.path.isfile(local_path):
-                    info(f"Downloading {value} → {local_path}")
+                    info(f"Downloading {value} → {local_path} using presigned URL")
                     _download_object(local_path, key_path, workspace_id, job_dir)
                 continue
             target = os.path.join(job_dir, "workspace", "data")
             os.makedirs(target, exist_ok=True)
-            info(f"Downloading {value} → {target}")
+            info(f"Downloading {value} → {target} using presigned URLs")
             result = subprocess.run(
                 ["omicslab", "download", "-y", "-r", value, target],
                 capture_output=True,

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/_run/_exec.py

@@ -46,7 +46,13 @@ def write_omicslab_config(
         tool_params = params.get("params", {})
         if not isinstance(tool_params, dict) or not tool_params:
             tool_params = params
-    filtered = {k: v for k, v in tool_params.items() if not isinstance(v, (dict, list))}
+    # ``None`` is not a valid Groovy config literal (``None is not defined``),
+    # so schema-default nulls must be omitted and left to the pipeline defaults.
+    filtered = {
+        k: v
+        for k, v in tool_params.items()
+        if v is not None and not isinstance(v, (dict, list))
+    }
 
     _nextflow_internal_prefixes = (
         "$projectDir",

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/_run/_git.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import os
+import shutil
 import subprocess
 import time
 from urllib.parse import urlparse
@@ -55,6 +56,33 @@ def is_nf_pipeline(params: dict) -> bool:
     return False
 
 
+def _is_git_clone(path: str) -> bool:
+    """True when ``path`` is a usable git work tree, not a partial clone."""
+    if not os.path.isdir(os.path.join(path, ".git")):
+        return False
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--verify", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            cwd=path,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return False
+    return result.returncode == 0
+
+
+def _remove_dir(path: str) -> None:
+    """Remove ``path`` if it exists (including a symlink, dangling or not)."""
+    if not os.path.lexists(path):
+        return
+    if os.path.islink(path):
+        os.remove(path)
+        return
+    shutil.rmtree(path, ignore_errors=True)
+
+
 def clone_repo(
     git_url: str,
     tag: str,
@@ -64,7 +92,10 @@ def clone_repo(
 ) -> str:
     """Clone a git repository at a specific tag.
 
-    If the target directory already exists, skips cloning.
+    Idempotent: an existing checkout is reused. A directory left behind by an
+    interrupted clone (or a racing job) is cleared before cloning, otherwise
+    ``git clone`` fails with "destination path already exists and is not an
+    empty directory".
     Git authentication uses the ``.git-credentials`` file written by
     ``omicslab job setup-auth`` — no embedded tokens in URLs.
 
@@ -82,9 +113,11 @@ def clone_repo(
     Raises:
         RuntimeError: If git clone fails after all retries.
     """
-    if os.path.isdir(target_dir):
+    if _is_git_clone(target_dir):
         print(f"Repository already cloned at {target_dir}")
     else:
+        # Drop a partial/abandoned checkout so the clone has a clean target.
+        _remove_dir(target_dir)
         print(f"Cloning {git_url} into {target_dir} (tag={tag})")
         last_error = None
         for attempt in range(1, MAX_RETRIES + 1):
@@ -113,6 +146,15 @@ def clone_repo(
                 last_error = f"git clone timed out after 120s: {git_url}"
                 print(f"Clone attempt {attempt}/{MAX_RETRIES} timed out")
 
+            # A concurrent job may have completed the clone while we retried.
+            if _is_git_clone(target_dir):
+                print(f"Repository already cloned at {target_dir}")
+                last_error = None
+                break
+
+            # Clear the partial checkout so the next attempt starts clean.
+            _remove_dir(target_dir)
+
             if attempt < MAX_RETRIES:
                 delay = RETRY_DELAY_BASE * attempt
                 print(f"Retrying in {delay}s...")

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/_run/_nextflow.py

@@ -2,6 +2,18 @@ from __future__ import annotations
 
 import os
 
+# RAM reserved for the Nextflow parent controller. With ``executor = "local"``
+# the controller and the pipeline processes share the same instance, so process
+# memory requests must stay below the instance's usable memory or Nextflow
+# refuses to schedule them, e.g.
+# "Process requirement exceeds available memory -- req: 16 GB; avail: 15.3 GB".
+CONTROLLER_MEMORY_RESERVE_GB = 1
+
+
+def process_memory_gb(base_memory: int) -> int:
+    """Process memory request minus the controller reserve, never below 1 GB."""
+    return max(int(base_memory) - CONTROLLER_MEMORY_RESERVE_GB, 1)
+
 
 def write_runtime_config(
     output_path: str,
@@ -16,9 +28,20 @@ def write_runtime_config(
     Credentials are NOT written to this file — Nextflow reads them natively
     from ``AWS_ACCESS_KEY_ID`` / ``AWS_SECRET_ACCESS_KEY`` environment variables.
 
+    ``resourceLimits`` stays at the job ``memory``; each process memory request
+    is reduced by ``CONTROLLER_MEMORY_RESERVE_GB`` because the Nextflow parent
+    controller runs on the same instance as the pipeline processes
+    (``executor = "local"``).
+
     Written with 0o600 permissions to prevent credential leakage.
     Process labels follow nf-core conventions.
     """
+    mem_default = process_memory_gb(2)
+    mem_single = process_memory_gb(2)
+    mem_low = process_memory_gb(4)
+    mem_medium = process_memory_gb(16)
+    mem_high = process_memory_gb(32)
+    mem_high_memory = process_memory_gb(200)
     endpoint = os.environ.get("AWS_ENDPOINT_URL", "")
     region = os.environ.get("AWS_REGION", "") or os.environ.get("AWS_REGION_NAME", "")
     has_aws_creds = bool(os.environ.get("AWS_ACCESS_KEY_ID", ""))
@@ -36,15 +59,8 @@ def write_runtime_config(
 
     config = f"""\
 {aws_block}\
-singularity.pullTimeout = 8.h
-
-docker {{
-    pullTimeout = 30.min
-    retry {{
-        enabled = true
-        maxRetries = 3
-        delay = 5000
-    }}
+apptainer {{
+    pullTimeout = 8.h
 }}
 
 timeline {{
@@ -69,6 +85,9 @@ dag {{
 }}
 
 process {{
+    // Reserve {CONTROLLER_MEMORY_RESERVE_GB}.GB for the Nextflow parent controller
+    // sharing this instance (executor = local); resourceLimits stays at the
+    // job memory and only the process requests are reduced.
     resourceLimits = [
         cpus: {cpu},
         memory: {memory}.GB,
@@ -76,34 +95,34 @@ process {{
     ]
 
     cpus   = {{ 1      * task.attempt }}
-    memory = {{ 2.GB   * task.attempt }}
+    memory = {{ {mem_default}.GB   * task.attempt }}
     time   = {{ 4.h    * task.attempt }}
 
     withLabel:process_single {{
         cpus   = {{ 1                   }}
-        memory = {{ 2.GB * task.attempt }}
+        memory = {{ {mem_single}.GB * task.attempt }}
         time   = {{ 4.h  * task.attempt }}
     }}
     withLabel:process_low {{
         cpus   = {{ 2     * task.attempt }}
-        memory = {{ 4.GB  * task.attempt }}
+        memory = {{ {mem_low}.GB  * task.attempt }}
         time   = {{ 4.h   * task.attempt }}
     }}
     withLabel:process_medium {{
         cpus   = {{ 8     * task.attempt }}
-        memory = {{ 16.GB * task.attempt }}
+        memory = {{ {mem_medium}.GB * task.attempt }}
         time   = {{ 8.h   * task.attempt }}
     }}
     withLabel:process_high {{
         cpus   = {{ 16    * task.attempt }}
-        memory = {{ 32.GB * task.attempt }}
+        memory = {{ {mem_high}.GB * task.attempt }}
         time   = {{ 16.h  * task.attempt }}
     }}
     withLabel:process_long {{
         time   = {{ 20.h  * task.attempt }}
     }}
     withLabel:process_high_memory {{
-        memory = {{ 200.GB * task.attempt }}
+        memory = {{ {mem_high_memory}.GB * task.attempt }}
     }}
 }}
 """

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/_run/_storage.py

@@ -25,6 +25,32 @@ def _has_any_aws_creds() -> bool:
     )
 
 
+def _count_files(source_dir: str) -> int:
+    """Number of files under ``source_dir`` (0 when unreadable)."""
+    try:
+        return sum(len(files) for _, _, files in os.walk(source_dir))
+    except OSError:
+        return 0
+
+
+def _failure_detail(result) -> str:
+    """Best-effort error text from a captured subprocess result.
+
+    The CLI reports errors on stdout, so stderr alone is often empty (storage
+    quota and allocation errors in particular) — fall back to stdout, then to
+    the exit code.
+    """
+    for stream in (getattr(result, "stderr", None), getattr(result, "stdout", None)):
+        if isinstance(stream, bytes):
+            text = stream.decode("utf-8", errors="replace")
+        else:
+            text = str(stream or "")
+        text = text.strip()
+        if text:
+            return text[-500:]
+    return f"exit code {getattr(result, 'returncode', '?')}"
+
+
 def resolve_bucket_mount_info(bucket: str) -> dict | None:
     """Resolve an S3 bucket to its workspace mount configuration via the API.
 
@@ -227,14 +253,17 @@ def upload_outputs(
     user_email: str = "",
     *,
     aws_credentials: bool | None = None,
+    on_log: callable | None = None,
 ) -> bool:
-    """Upload job outputs to S3, auto-detecting marketplace vs sandbox mode.
+    """Upload job outputs to S3, auto-detecting credentials vs presigned mode.
 
-    **Marketplace mode**: If ``AWS_ACCESS_KEY_ID`` is set, uses
+    **Credential path** (``EXPORT_JOB_IAM_KEYS`` safeguard, or a standalone SDK
+    run with its own AWS env): if ``AWS_ACCESS_KEY_ID`` is set, uses
     ``aws s3 sync`` with the endpoint auto-resolved from the S3 path.
 
-    **Sandbox mode**: Otherwise, uses ``omicslab upload`` which resolves
-    the S3 path to a workspace and uploads via presigned URLs.
+    **Sandbox mode** (the platform default): otherwise, uses ``omicslab
+    upload`` which resolves the S3 path to a workspace and uploads via
+    presigned URLs.
 
     Args:
         source_dir: Local directory with output files.
@@ -245,6 +274,9 @@ def upload_outputs(
         user_email: Optional user email for audit logging.
         aws_credentials: Explicit decision for direct S3 upload. ``None`` keeps
             the legacy environment-based detection.
+        on_log: Optional callback receiving the transfer-method log lines
+            (``using presigned URLs`` / ``using workspace credentials``) so the
+            job log shows which path was used.
 
     Returns:
         True if the upload succeeded.
@@ -258,6 +290,16 @@ def upload_outputs(
         logger.warning("Invalid S3 path: %s", dest_path)
         return False
 
+    def _emit(message: str) -> None:
+        if on_log is None:
+            return
+        try:
+            on_log(message)
+        except Exception:  # noqa: BLE001 — logging must never break the upload
+            pass
+
+    file_count = _count_files(source_dir)
+
     # Auto-resolve endpoint from workspace if not explicitly provided
     if not endpoint:
         endpoint = _resolve_workspace_endpoint(bucket)
@@ -269,6 +311,8 @@ def upload_outputs(
         aws_credentials = _has_aws_creds()
 
     if aws_credentials:
+        method = "workspace credentials (aws s3 sync)"
+        _emit(f"Uploading {file_count} file(s) using {method}...")
         try:
             aws_cmd = ["aws", "s3", "sync", source_dir, s3_target]
             if endpoint:
@@ -276,12 +320,18 @@ def upload_outputs(
             result = subprocess.run(aws_cmd, capture_output=True, timeout=300)
             if result.returncode == 0:
                 logger.info("Synced %s → %s", source_dir, s3_target)
+                _emit(f"Uploaded {file_count} file(s) using {method} → {s3_target}")
                 success = True
             else:
-                logger.warning("aws s3 sync failed: %s", result.stderr.decode())
+                detail = _failure_detail(result)
+                logger.warning("aws s3 sync failed: %s", detail)
+                _emit(f"Upload failed using {method}: {detail}")
         except (subprocess.TimeoutExpired, FileNotFoundError):
-            pass
+            logger.warning("aws s3 sync command failed")
+            _emit(f"Upload failed using {method}: command unavailable")
     else:
+        method = "presigned URLs"
+        _emit(f"Uploading {file_count} file(s) using {method}...")
         try:
             result = subprocess.run(
                 ["omicslab", "upload", "-y", source_dir, dest_path],
@@ -290,11 +340,15 @@ def upload_outputs(
             )
             if result.returncode == 0:
                 logger.info("Uploaded %s → %s", source_dir, dest_path)
+                _emit(f"Uploaded {file_count} file(s) using {method} → {dest_path}")
                 success = True
             else:
-                logger.warning("omicslab upload failed: %s", result.stderr.decode())
+                detail = _failure_detail(result)
+                logger.warning("omicslab upload failed: %s", detail)
+                _emit(f"Upload failed using {method}: {detail}")
         except (subprocess.TimeoutExpired, FileNotFoundError):
             logger.warning("omicslab upload command failed")
+            _emit(f"Upload failed using {method}: command unavailable")
 
     if success and job_id:
         _log_job_output_upload(dest_path, source_dir, job_id, user_id, user_email)

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/_run/analysis.py

@@ -54,10 +54,12 @@ def _promote_workspace_env(ws_info: dict) -> None:
 def _workspace_has_creds(params: dict) -> bool:
     """True when the job carries real workspace S3 credentials.
 
-    Marketplace cloud-VM jobs receive STS tokens when the provider exposes an
-    assume-role ARN, otherwise the marketplace provider's long-lived keys.
-    Runner/HPC jobs run in sandbox mode and carry no S3 credentials (presigned
-    URLs only).
+    By default the platform never sends S3 keys to compute — every job runs
+    with presigned URLs only. Credentials appear only under the future
+    EXPORT_JOB_IAM_KEYS safeguard: marketplace cloud-VM jobs may then receive
+    STS tokens when the provider exposes an assume-role ARN, otherwise the
+    marketplace provider's long-lived keys. Runner/HPC jobs never receive S3
+    credentials.
     """
     ws_id = params.get("workspace_id", "")
     workspaces = params.get("workspaces")
@@ -71,12 +73,12 @@ def _workspace_has_creds(params: dict) -> bool:
 def _aws_upload_enabled(params: dict) -> bool:
     """Whether outputs may be uploaded over S3 with the job's own credentials.
 
-    Only jobs the backend actually gave workspace credentials to (marketplace
-    cloud-VM compute: STS tokens or the provider's long-lived keys) upload via
-    ``aws s3 sync``. Sandbox jobs (runners/HPC, personal or organization cloud)
-    never receive S3 credentials and must use the presigned ``omicslab upload``
-    path. The legacy ``_has_aws_creds`` check preserves standalone SDK runs that
-    configure AWS credentials themselves.
+    Only jobs the backend explicitly gave workspace credentials to (the
+    EXPORT_JOB_IAM_KEYS safeguard path: STS tokens or the provider's
+    long-lived keys) upload via ``aws s3 sync``. By default every job is a
+    sandbox job and must use the presigned ``omicslab upload`` path. The legacy
+    ``_has_aws_creds`` check preserves standalone SDK runs that configure AWS
+    credentials themselves.
     """
     if _workspace_has_creds(params):
         return True
@@ -426,6 +428,7 @@ def _run_nextflow(
             outdir_s3_full,
             endpoint="",
             aws_credentials=_aws_upload_enabled(params),
+            on_log=_log,
         )
         return 0 if ok else 1
     return 0
@@ -511,6 +514,7 @@ def _run_tool(
             outdir_s3_full,
             endpoint="",
             aws_credentials=aws_credentials,
+            on_log=_log,
         )
         return 0 if ok else 1
     return 0

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/client.py

@@ -617,13 +617,22 @@ class RunnerClient:
                 await asyncio.sleep(self.job_poll_interval)
 
     def _reap_terminal_jobs(self) -> None:
-        """Move terminal jobs from ``_dispatched_jobs`` to ``_completed_jobs``."""
+        """Move terminal jobs from ``_dispatched_jobs`` to ``_completed_jobs``.
+
+        Also drops the job from the executor's SLURM tracking map: the
+        supervisor detects completion via the ``exitcode`` file and never
+        polls ``get_status`` (which is what normally prunes that map), so
+        leaving the entry behind made ``is_idle()`` report the runner as busy
+        forever and blocked SDK self-updates.
+        """
         for job_id in list(self._dispatched_jobs):
             if not self.running:
                 break
             if self._job_is_terminal(job_id):
                 self._dispatched_jobs.pop(job_id, None)
                 self._completed_jobs.add(job_id)
+                if self.executor is not None:
+                    self.executor.forget_slurm_job(job_id)
 
     def _job_is_terminal(self, job_id: str) -> bool:
         """True when a dispatched job has reached a terminal state.

{omicslab-2.0.0 → omicslab-2.1.0}/src/omicslab/job/executor.py

@@ -170,6 +170,17 @@ class JobExecutor:
         finally:
             self._slurm_jobs.pop(job_id, None)
 
+    def forget_slurm_job(self, job_id: str) -> None:
+        """Drop a finished job from the SLURM tracking map (no scancel).
+
+        ``get_status`` prunes terminal jobs when it is polled, but the runner
+        controller can reap a job from its own dispatch map via the
+        ``exitcode`` file without ever calling ``get_status`` — leaving a stale
+        entry here that makes ``is_idle()`` report the runner as busy forever
+        and blocks SDK self-updates.
+        """
+        self._slurm_jobs.pop(job_id, None)
+
     # ── Status / log / exit code (filesystem, job run self-manages) ─────
 
     def get_status(self, job_id: str) -> dict: