scitex-dev 0.62.2__tar.gz → 0.62.4__tar.gz

scitex_dev-0.62.4/.github/ci/build-in-sif.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# Runs INSIDE the approved, versioned CI SIF through exec-in-sif.sh.
+# Install the build frontend into job-owned writable scratch, then run the
+# normal isolated PEP517 build of this checkout's wheel and sdist.
+# Missing interpreters, failed builds or broken wheel entry points fail loud.
+set -euo pipefail
+
+V="${1:-3.12}"
+VENV="/opt/venv-$V"
+PY="$VENV/bin/python"
+test -x "$PY" || {
+    echo "::error::baked python missing in $VENV — rebuild the SIF: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+export LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+# Writable scratch (the runner's TMPDIR=~/.cache/tmp is a host path that does
+# NOT resolve inside the container). Node-local /tmp is writable + ephemeral.
+TMPDIR="/tmp/build-scitex_dev-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-$V"
+export TMPDIR
+# `${TMPDIR:?}` — see run-in-sif.sh for the reasoning and the measurement.
+# Short version: `rm -rf ""` exits 0 SILENTLY on GNU coreutils, so an empty name
+# would delete nothing, fail nothing, and leave the rest of the script
+# addressing paths off the filesystem root. `:?` aborts instead.
+rm -rf "${TMPDIR:?build scratch path is empty — refusing to rm -rf it}"
+mkdir -p "$TMPDIR/site" "$TMPDIR/uv-cache"
+
+# The compute-node $HOME is RO inside the container — point every cache the
+# installer might touch at the writable scratch (else uv/pip die creating
+# ~/.cache).
+source "$(dirname "${BASH_SOURCE[0]}")/release-context.sh"
+scitex_release_context "$TMPDIR" "$VENV"
+echo "build: py=$(scitex_release_run "$PY" -V) target=$TMPDIR/site"
+
+# Install the PEP 517 build frontend into the writable target with UV, then
+# build with its UV installer. Clean dist/ first so only the freshly
+# built artifacts are uploaded.
+scitex_release_run uv pip install --python "$PY" --target="$TMPDIR/site" build
+
+# The entry-point gate below imports the wheel against $TMPDIR/site ahead of
+# the SIF-baked site, so seed it with the declared runtime floor. The SIF
+# image predates any release cut after it was built (e.g. scitex-logging
+# 0.2.3, which provides the mandatory auditor provider); without this the
+# gate audits the new wheel against the image's stale deps and refuses a
+# correct artifact for environmental reasons. UV only (no pip fallback).
+scitex_release_run uv pip install --python "$PY" --target="$TMPDIR/site" "scitex-logging>=0.2.3"
+
+rm -rf dist
+scitex_release_run env PYTHONPATH="$TMPDIR/site" "$PY" -m build --installer uv --outdir dist
+
+echo "=== built artifacts ==="
+ls -l dist
+# fail-loud: refuse to continue the pipeline with an empty dist/.
+test -n "$(ls -A dist 2>/dev/null)" || {
+    echo "::error::python -m build produced no artifacts in dist/"
+    exit 1
+}
+
+# --- Release gate: every declared entry point must IMPORT from the WHEEL ---
+#
+# A dangling entry point does not fail the build on its own: the wheel
+# uploads, `pip install` succeeds, and the breakage lands in the USER's
+# tooling. `pytest11` is imported by pytest at startup, so a dangling target
+# aborts EVERY pytest run in the installed environment before collection.
+# This step moves that failure back into the build.
+#
+# The audit runs against the freshly built WHEEL (unpacked to a temp dir),
+# not against pyproject.toml — a correct declaration can still point at a
+# module the build dropped, which is exactly the bug a source-only check
+# would miss. The SIF has scitex-dev's runtime deps installed, so a genuine
+# import failure means the ARTIFACT is broken, not the environment.
+echo "=== entry-point import gate (built wheel) ==="
+WHEEL="$(ls dist/*.whl | head -n 1)"
+test -n "$WHEEL" || {
+    echo "::error::no wheel in dist/ to audit"
+    exit 1
+}
+scitex_release_run env PYTHONPATH="$PWD/src:$TMPDIR/site" "$PY" - "$WHEEL" <<'PYGATE'
+import sys
+
+from scitex_dev._release.entrypoint_imports import (
+    audit_wheel_entry_point_imports,
+)
+
+report = audit_wheel_entry_point_imports(sys.argv[1], "scitex-dev")
+sys.stdout.write(report.report() + "\n")
+if not report.is_clean:
+    sys.stdout.write(
+        "::error::the built wheel declares entry points that do NOT import; "
+        "publishing it would break `pytest` (and console scripts) for every "
+        "user who installs it\n"
+    )
+    raise SystemExit(1)
+PYGATE
+
+# One immutable release identity accompanies the complete, checked wheel/sdist.
+scitex_release_run "$PY" .github/ci/release-identity.py write \
+    --tag "${RELEASE_TAG:?RELEASE_TAG must name the resolved release tag}" \
+    --commit "${RELEASE_COMMIT:?RELEASE_COMMIT must name the checked-out commit}"

scitex_dev-0.62.4/.github/ci/publish-in-sif.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Runs INSIDE the reused scitex-ci SIF (apptainer exec — invoked via
+# exec-in-sif.sh). Publishes ./dist/* to PyPI via MANUAL OIDC Trusted
+# Publishing, then twine upload.
+#
+# This CPU job uses the versioned SIF and a manual OIDC token exchange over
+# HTTPS. Separate Docker/service runners do not determine this job's route:
+#
+#   1. Ask the GitHub Actions OIDC provider for a JWT with audience=pypi,
+#      using the per-job ACTIONS_ID_TOKEN_REQUEST_{TOKEN,URL} env vars (present
+#      because the publish job declares `permissions: id-token: write`).
+#      apptainer exec (no --cleanenv) passes those host env vars into the SIF.
+#   2. Exchange that JWT at PyPI's mint-token endpoint for a short-lived,
+#      scope-limited PyPI API token.
+#   3. twine upload dist/* with TWINE_USERNAME=__token__ and that minted token.
+#
+# PyPI verifies the Trusted Publisher identity during token minting:
+# project=scitex-dev, owner=scitex-ai, repo=scitex-dev,
+# workflow=pypi-publish-and-github-release-on-tag.yml. Configuration or
+# identity mismatches fail the exchange; this header does not certify trust.
+#
+# curl, python and (after a --target install) twine all live in the SIF.
+#
+# Fail-loud (operator directive): every step asserts non-empty output and
+# `set -euo pipefail`; any failure is a HARD error with the exact cause, never
+# a silent skip.
+set -euo pipefail
+
+V="${1:-3.12}"
+VENV="/opt/venv-$V"
+PY="$VENV/bin/python"
+test -x "$PY" || {
+    echo "::error::baked python missing in $VENV — rebuild the SIF: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+export LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+# dist/ must already hold the artifacts (downloaded by the publish job before
+# this script runs). Fail loud if empty.
+if [ ! -d dist ] || [ -z "$(ls -A dist 2>/dev/null)" ]; then
+    echo "::error::dist/ is empty — nothing to publish (download the build artifact first)"
+    exit 1
+fi
+echo "=== dist to publish ==="
+ls -l dist
+
+# --- writable scratch (compute-node HOME is RO inside the container) ---
+TMPDIR="/tmp/publish-scitex_dev-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-$V"
+export TMPDIR
+# `${TMPDIR:?}` — see run-in-sif.sh for the reasoning and the measurement.
+# Short version: `rm -rf ""` exits 0 SILENTLY on GNU coreutils, so an empty name
+# would delete nothing, fail nothing, and leave the rest of the script
+# addressing paths off the filesystem root. `:?` aborts instead.
+rm -rf "${TMPDIR:?publish scratch path is empty — refusing to rm -rf it}"
+mkdir -p "$TMPDIR/site" "$TMPDIR/uv-cache"
+source "$(dirname "${BASH_SOURCE[0]}")/release-context.sh"
+scitex_release_context "$TMPDIR" "$VENV"
+
+# --- step 1: request the OIDC JWT (audience=pypi) from GitHub ---
+: "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?ACTIONS_ID_TOKEN_REQUEST_TOKEN not set — the publish job needs 'permissions: id-token: write'}"
+: "${ACTIONS_ID_TOKEN_REQUEST_URL:?ACTIONS_ID_TOKEN_REQUEST_URL not set — the publish job needs 'permissions: id-token: write'}"
+
+echo "=== minting OIDC JWT (audience=pypi) ==="
+JWT="$(scitex_release_run curl -fsS \
+    -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
+    "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=pypi" |
+    scitex_release_run "$PY" -c 'import sys,json; print(json.load(sys.stdin)["value"])')"
+test -n "$JWT" || {
+    echo "::error::OIDC JWT request returned an empty token"
+    exit 1
+}
+echo "OIDC JWT obtained (length=${#JWT})"
+
+# Only the existing protected-main caller may publish on the company runner.
+# The token remains RAM-only; this local claim check does not verify its signature.
+printf '%s' "$JWT" | scitex_release_run "$PY" .github/ci/release-identity.py oidc \
+    --commit "${GITHUB_SHA:?GITHUB_SHA must identify the dispatch workflow source}"
+
+# --- step 2: exchange the JWT for a short-lived PyPI API token ---
+echo "=== exchanging JWT at PyPI mint-token endpoint ==="
+MINT_RESP="$(scitex_release_run curl -sS -X POST https://pypi.org/_/oidc/mint-token \
+    -d "{\"token\":\"${JWT}\"}")"
+MINTED="$(printf '%s' "$MINT_RESP" |
+    scitex_release_run "$PY" -c 'import sys,json; d=json.load(sys.stdin); print(d.get("token",""))')"
+if [ -z "$MINTED" ]; then
+    # Error codes identify trust/configuration failures without echoing an
+    # arbitrary provider response, which may contain credential fields.
+    echo "::error::PyPI mint-token returned no token."
+    printf '%s' "$MINT_RESP" |
+        scitex_release_run "$PY" -c 'import sys,json,re
+d=json.load(sys.stdin)
+codes={e.get("code", "unknown") for e in d.get("errors", []) if isinstance(e, dict)}
+safe=sorted(c for c in codes if isinstance(c, str) and re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", c))
+print("PyPI mint-token error codes: " + (", ".join(safe) or "unknown"))' \
+            2>/dev/null || echo "PyPI mint-token error codes: unreadable"
+    exit 1
+fi
+echo "PyPI token minted (length=${#MINTED})"
+
+# --- step 3: install twine into the writable target, then upload ---
+echo "=== installing twine (--target) ==="
+scitex_release_run uv pip install --python "$PY" --target="$TMPDIR/site" twine
+
+echo "=== twine upload dist/* ==="
+scitex_release_run env PYTHONPATH="$TMPDIR/site" \
+    TWINE_USERNAME="__token__" TWINE_PASSWORD="$MINTED" \
+    "$PY" -m twine upload --config-file /dev/null \
+    --non-interactive --disable-progress-bar dist/*
+
+echo "PUBLISH-OK: scitex-dev dist/* uploaded to PyPI via manual OIDC trusted publishing"

scitex_dev-0.62.4/.github/ci/release-identity.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Bind a complete wheel and sdist to one checked-out release commit."""
+
+import argparse
+import base64
+import csv
+import hashlib
+import io
+import json
+import os
+import re
+import stat
+import subprocess
+import tarfile
+import zipfile
+from email.parser import BytesParser
+from pathlib import Path, PurePosixPath
+
+MAX_BYTES = 256 * 1024 * 1024
+MAX_MEMBERS = 20000
+MANIFEST = "release-manifest.json"
+
+
+def require(condition, reason):
+    if not condition:
+        raise ValueError(reason)
+
+
+def safe_member(name):
+    value = PurePosixPath(name)
+    require(name and not value.is_absolute() and ".." not in value.parts
+            and "\\" not in name and "\x00" not in name, "archive-path-refused")
+
+
+def identity(value):
+    return (value.st_dev, value.st_ino, value.st_mode, value.st_size,
+            value.st_mtime_ns, value.st_ctime_ns)
+
+
+def regular_snapshot(path):
+    fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+    try:
+        before = os.fstat(fd)
+        require(stat.S_ISREG(before.st_mode) and before.st_size <= MAX_BYTES,
+                "artifact-kind-or-size-refused")
+        with os.fdopen(os.dup(fd), "rb") as stream:
+            body = stream.read(MAX_BYTES + 1)
+        after = os.fstat(fd)
+        require(len(body) == before.st_size and identity(before) == identity(after)
+                == identity(path.lstat()), "artifact-changed")
+        return body
+    finally:
+        os.close(fd)
+
+
+def metadata(body, version):
+    value = BytesParser().parsebytes(body)
+    require(value["Name"] == "scitex-dev" and value["Version"] == version,
+            "distribution-identity-mismatch")
+
+
+def wheel_check(body, version):
+    with zipfile.ZipFile(io.BytesIO(body)) as archive:
+        members = archive.infolist()
+        names = [member.filename for member in members]
+        require(0 < len(members) <= MAX_MEMBERS and len(set(names)) == len(names),
+                "wheel-membership-refused")
+        require(sum(member.file_size for member in members) <= MAX_BYTES,
+                "wheel-expanded-size-refused")
+        for member in members:
+            safe_member(member.filename)
+            mode = member.external_attr >> 16
+            require(not member.is_dir() and (not stat.S_IFMT(mode)
+                    or stat.S_ISREG(mode)), "wheel-member-kind-refused")
+        prefix = f"scitex_dev-{version}.dist-info/"
+        metadata(archive.read(prefix + "METADATA"), version)
+        records = list(csv.reader(io.StringIO(archive.read(prefix + "RECORD").decode())))
+        require(all(len(row) == 3 for row in records), "wheel-record-shape-refused")
+        require(len(records) == len(names) and len({row[0] for row in records}) == len(names)
+                and {row[0] for row in records} == set(names), "wheel-record-membership-refused")
+        for name, digest, size in records:
+            payload = archive.read(name)
+            if name == prefix + "RECORD":
+                require(digest == size == "", "wheel-record-self-refused")
+            else:
+                expected = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode()
+                require(digest == "sha256=" + expected and size == str(len(payload)),
+                        "wheel-record-byte-mismatch")
+        require(any(name.startswith("scitex_dev/") for name in names), "wheel-package-missing")
+
+
+def sdist_check(body, version):
+    prefix = f"scitex_dev-{version}/"
+    with tarfile.open(fileobj=io.BytesIO(body), mode="r:gz") as archive:
+        members = archive.getmembers()
+        names = [member.name for member in members]
+        require(0 < len(members) <= MAX_MEMBERS and len(set(names)) == len(names),
+                "sdist-membership-refused")
+        require(sum(member.size for member in members) <= MAX_BYTES,
+                "sdist-expanded-size-refused")
+        for member in members:
+            safe_member(member.name)
+            require((member.name == prefix.rstrip("/") or member.name.startswith(prefix))
+                    and (member.isreg() or member.isdir()), "sdist-member-kind-or-root-refused")
+        metadata(archive.extractfile(prefix + "PKG-INFO").read(), version)
+
+
+def artifact_manifest(root, tag, commit):
+    require(re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag) is not None,
+            "release-tag-refused")
+    require(re.fullmatch(r"[a-f0-9]{40}", commit) is not None, "release-commit-refused")
+    actual = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True, timeout=5).strip()
+    require(actual == commit, "checkout-commit-mismatch")
+    version = tag[1:]
+    names = [f"scitex_dev-{version}-py3-none-any.whl", f"scitex_dev-{version}.tar.gz"]
+    require(root.is_dir() and not root.is_symlink(), "distribution-root-refused")
+    require({path.name for path in root.iterdir()} == set(names),
+            "distribution-membership-mismatch")
+    rows = []
+    for name in names:
+        body = regular_snapshot(root / name)
+        (wheel_check if name.endswith(".whl") else sdist_check)(body, version)
+        rows.append({"name": name, "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest()})
+    return {"schema": 1, "project": "scitex-dev", "tag": tag,
+            "version": version, "commit": commit, "files": rows}
+
+
+def verify_oidc(jwt, commit):
+    """Check declared claims locally; PyPI verifies the signed JWT itself."""
+    require(re.fullmatch(r"[a-f0-9]{40}", commit) is not None, "workflow-commit-refused")
+    parts = jwt.split(".")
+    require(len(jwt) <= 65536 and len(parts) == 3
+            and all(re.fullmatch(r"[A-Za-z0-9_-]+", part) for part in parts),
+            "oidc-token-shape-refused")
+    claims = json.loads(base64.urlsafe_b64decode(parts[1] + "=" * (-len(parts[1]) % 4)))
+    require(isinstance(claims, dict), "oidc-claims-shape-refused")
+    workflow = "scitex-ai/scitex-dev/.github/workflows/pypi-publish-and-github-release-on-tag.yml@refs/heads/main"
+    expected = {
+        "iss": "https://token.actions.githubusercontent.com", "aud": "pypi",
+        "repository": "scitex-ai/scitex-dev", "repository_owner": "scitex-ai",
+        "sub": "repo:scitex-ai/scitex-dev:environment:pypi", "environment": "pypi",
+        "ref": "refs/heads/main", "sha": commit, "event_name": "workflow_dispatch",
+        "workflow_ref": workflow,
+        "runner_environment": "self-hosted",
+    }
+    require(all(claims.get(key) == value for key, value in expected.items())
+            and ("job_workflow_ref" not in claims or claims["job_workflow_ref"] == workflow),
+            "oidc-publisher-claims-mismatch")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("operation", choices=["write", "verify", "oidc"])
+    parser.add_argument("--tag")
+    parser.add_argument("--commit", required=True)
+    parser.add_argument("--dist", default="dist")
+    args = parser.parse_args()
+    if args.operation == "oidc":
+        import sys
+
+        verify_oidc(sys.stdin.read(65537), args.commit)
+        print("OIDC-CLAIMS-OK: signature verification remains PyPI's responsibility")
+        return
+    require(args.tag is not None, "release-tag-required")
+    root = Path(args.dist)
+    result = artifact_manifest(root, args.tag, args.commit)
+    path = root.parent / MANIFEST
+    if args.operation == "write":
+        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644)
+        with os.fdopen(fd, "w") as stream:
+            json.dump(result, stream, sort_keys=True)
+            stream.write("\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+    else:
+        require(json.loads(regular_snapshot(path)) == result, "release-manifest-mismatch")
+    print(json.dumps({"verified": True, "tag": result["tag"], "commit": result["commit"],
+                      "files": result["files"]}, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

scitex_dev-0.62.4/.github/ci/run-in-sif.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# Runs INSIDE the approved, versioned CI SIF. $1 = Python3.11/3.12/3.13.
+# Resolve THIS checkout's complete [all,dev] requirements into job-owned
+# writable scratch. The target precedes the image's baked packages, so the
+# selected source and declared dependencies determine the test environment.
+# A --target install permits use of the image's read-only interpreter.
+# Missing interpreters or failed resolution are hard errors.
+set -euo pipefail
+
+V="${1:?python version arg required (3.11/3.12/3.13)}"
+VENV="/opt/venv-$V"
+test -x "$VENV/bin/python" || {
+    echo "::error::baked python missing in $VENV — rebuild the SIF: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+# The central registered image profile declares these exact capabilities.
+# No highest-version search, home binary or host fallback is accepted.
+PGBIN="${SCITEX_CI_PG_BIN:?registered image PostgreSQL directory is required}"
+PG_CAPABILITY_VERSION="${SCITEX_CI_PG_VERSION:?registered image PostgreSQL version is required}"
+PSQL="$PGBIN/psql"
+
+export LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+# The audit protocol uses canonical level prefixes. A supported debug logger
+# format adds bracketed source locations; those are deliberately not exempted
+# by the strict finding classifier. State the CI format before any imports.
+export SCITEX_LOGGING_FORMAT=default SCITEX_LOG_FORMAT=default
+
+# Real writable scratch. The runner profile exports TMPDIR=~/.cache/tmp, a host
+# path that does NOT resolve inside the container; tests (tmp_path) and the
+# install target both need a working, writable tmp. Node-local /tmp is writable
+# + ephemeral and per-version-isolated so concurrent matrix legs don't collide.
+export TMPDIR="/tmp/ci-scitex_dev-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-$V"
+# `${TMPDIR:?}` AND NOT `$TMPDIR`.
+#
+# The line above cannot produce an empty value TODAY — the `/tmp/…` prefix is a
+# literal. The guard pins that. scitex-agent-container's wrappers started from
+# this exact shape and later moved the name into a helper function in another
+# file, at which point "always non-empty" stopped being visible from the
+# deletion site; nothing there would have noticed.
+#
+# And `rm -rf ""` IS NOT A SAFE NO-OP. Measured, GNU coreutils 9.4: `-f` treats
+# the empty operand as a nonexistent file, so it exits 0 SILENTLY — `set -euo
+# pipefail` catches nothing, and the script CONTINUES with TMPDIR="". Every
+# later use is then a path off the filesystem root: `"$TMPDIR/site"` is `/site`.
+# The empty value is dangerous precisely because it is quiet.
+rm -rf "${TMPDIR:?ci scratch path is empty — refusing to rm -rf it}"
+mkdir -p "$TMPDIR/site" "$TMPDIR/uv-cache"
+
+# Set owned state before the installer or any application/plugin import.
+source "$(dirname "${BASH_SOURCE[0]}")/release-context.sh"
+scitex_release_context "$TMPDIR" "$VENV"
+
+# Verify the declared image server/client closure in the neutral environment,
+# before dependency resolution or a test/plugin can run.
+scitex_release_run "$VENV/bin/python" -I -S -B \
+    "$(dirname "${BASH_SOURCE[0]}")/verify-postgres-capability.py" \
+    --bin-dir "$PGBIN" --expected-version "$PG_CAPABILITY_VERSION"
+
+# The HPC compute-node $HOME is READ-ONLY inside the container, so uv/pip cannot
+# create their default caches under ~/.cache — point them at the writable
+# scratch instead (else `uv pip install` dies: "failed to create directory
+# ~/.cache/uv: File exists / read-only").
+export UV_CACHE_DIR="$TMPDIR/uv-cache"
+export XDG_CACHE_HOME="$TMPDIR"
+export PIP_CACHE_DIR="$TMPDIR/pip-cache"
+
+# Headless matplotlib — no DISPLAY on the compute node; force the Agg backend so
+# pyplot imports + figure rendering in the test suite never try to open a GUI.
+export MPLBACKEND=Agg
+
+# Dedicated, stable matplotlib config/cache dir for this matrix leg. Without
+# pinning it, MPLCONFIGDIR defaults to $XDG_CACHE_HOME/matplotlib which is COLD
+# every CI run; the xdist workers (one per core, see below) then each cold-start
+# matplotlib and RACE to build fontList.json in that shared dir.
+# A partial/contended cache makes some renders fall back to a different font, so
+# scitex-dev's reproducibility tests (validate_recipe renders the SAME recipe
+# twice and compares) see render1 != render2 → spurious MSE-over-threshold
+# failures (e.g. TestValidateRecipe, max channel diff 255). One stable dir +
+# a single warm-up below (build the cache ONCE, pre-fork) removes the race.
+export MPLCONFIGDIR="$TMPDIR/mpl"
+mkdir -p "$MPLCONFIGDIR"
+
+# A VIRTUAL_ENV leaked from the runner profile (~/.env-3.11) is a broken symlink
+# in here; unset it so no tool (uv, pip) tries to follow it.
+unset VIRTUAL_ENV || true
+
+# venv bin on PATH (this matrix leg's python3 + pip); PYTHONPATH points at the
+# writable target so imports + coverage use the freshly-installed checkout.
+export PATH="$VENV/bin:$PATH"
+
+echo "py=$(scitex_release_run "$VENV/bin/python" -V) target=$TMPDIR/site"
+
+# The full declared test environment is mandatory. A reduced-extra fallback can
+# make the same commit pass or fail depending on resolver timing, so it is not
+# a valid CI environment.
+scitex_release_run uv pip install --python "$VENV/bin/python" --target="$TMPDIR/site" -e ".[all,dev]"
+
+PGDIR="$TMPDIR/postgres"
+mkdir -p "$PGDIR"
+cleanup_postgres() {
+    scitex_release_run "$PGBIN/pg_ctl" -D "$PGDIR/data" -m immediate stop >/dev/null 2>&1 || true
+}
+trap cleanup_postgres EXIT
+scitex_release_run "$PGBIN/initdb" -D "$PGDIR/data" -A trust --encoding=UTF8 -U postgres >"$PGDIR/initdb.log" 2>&1 || {
+    echo "::error::PostgreSQL initdb failed in the verified CI image"
+    tail -40 "$PGDIR/initdb.log"
+    exit 1
+}
+scitex_release_run "$PGBIN/pg_ctl" -D "$PGDIR/data" -o "-k $PGDIR -h '' -p 5432" -w -t 60 start >"$PGDIR/start.log" 2>&1 || {
+    echo "::error::throwaway PostgreSQL cluster failed to start"
+    tail -40 "$PGDIR/start.log"
+    exit 1
+}
+PGHOST_ENC="$(printf '%s' "$PGDIR" | sed 's|/|%2F|g')"
+export SCITEX_STORE_DSN="postgresql://postgres@${PGHOST_ENC}/postgres"
+scitex_release_run env "PGPASSFILE=$TMPDIR/refused-pgpass" "$PSQL" -X -h "$PGDIR" -p 5432 -U postgres -d postgres -Atqc 'select 1' | grep -qx 1 || {
+    echo "::error::throwaway PostgreSQL cluster failed its readiness query"
+    exit 1
+}
+echo "postgres=$($PGBIN/postgres --version) socket=$PGDIR readiness=verified"
+
+# Only this verified socket-only cluster replaces the refused import-time
+# endpoint. Keep every other application setting in the literal allowlist.
+scitex_test_run() {
+    scitex_release_run env \
+        "PYTHONPATH=$TMPDIR/site:$PWD/src" \
+        "MPLBACKEND=Agg" "MPLCONFIGDIR=$MPLCONFIGDIR" \
+        "PGHOST=$PGDIR" "PGPORT=5432" "PGUSER=postgres" "PGDATABASE=postgres" \
+        "PGPASSFILE=$TMPDIR/refused-pgpass" "SCITEX_STORE_DSN=$SCITEX_STORE_DSN" "$@"
+}
+
+# Parallelise with pytest-xdist (baked in [dev]/[all,dev] as pytest-xdist>=3).
+# scitex-dev's suite is ~2460 tests; single-process it overran the job's old
+# 30-min cap (2300 passed in ~28 min, cancelled at 96%). Each xdist worker is
+# a SEPARATE PROCESS, so matplotlib's global rcParams / pyplot state and the
+# scitex-dev style-stack are naturally isolated per worker — the safe way to
+# parallelise a matplotlib-heavy suite.
+#
+# Worker count: use ALL cores. Each matrix leg now runs on its own dedicated
+# self-hosted node (one runner per node: scitex-dev-01/02/03), so there is no
+# co-tenant to yield half the box to — the old nproc//2 cap left 2x the cores
+# idle. nice/ionice (below) handles the "yield to higher-priority work if the
+# node is ever shared" concern instead of statically reserving half the CPUs.
+# Floor 4. pyproject addopts carries `-v`; override to `-q` here — 2460 verbose
+# lines x workers bloats the CI log and adds measurable overhead.
+NPROC="$(nproc 2>/dev/null || echo 4)"
+WORKERS=$NPROC
+[ "$WORKERS" -lt 4 ] && WORKERS=4
+echo "xdist workers=$WORKERS (nproc=$NPROC)"
+
+# Warm the matplotlib font cache ONCE, single-process, before xdist forks the
+# workers. This builds $MPLCONFIGDIR/fontlist-*.json a single time so every
+# worker reads a complete, consistent cache instead of racing to build it
+# concurrently (the source of the render1!=render2 reproducibility flakes).
+# Fail-loud: if matplotlib can't even build its font cache, CI must surface it.
+# matplotlib may not be a dependency of this package; only warm the
+# font cache when it's importable (no-op otherwise — never fail the run
+# on an optional warm-up).
+if scitex_test_run python -c "import matplotlib" 2>/dev/null; then
+  scitex_test_run python -c "import matplotlib; matplotlib.use('Agg'); from matplotlib import font_manager; font_manager.fontManager; import matplotlib.pyplot as plt; f=plt.figure(); f.canvas.draw(); print('mpl font cache warmed at', matplotlib.get_cachedir())"
+else
+  echo "matplotlib not importable — skipping font-cache warm-up (not a dep)"
+fi
+
+# Distribution: `--dist load` (per-TEST round-robin), NOT `--dist loadscope`.
+# loadscope pins an entire MODULE's tests to ONE worker — and scitex-dev's heavy
+# suites are big SINGLE modules (e.g. tests/integration/test_all_plotters_*.py
+# parametrize one test over all 47 plotters, ~28 s each). loadscope therefore
+# ran all ~50+ cases of such a module SERIALLY on one worker (~25 min) while the
+# rest idled. There are NO module/session/class-scoped fixtures in those heavy
+# modules and the root conftest's autouse `_close_figures` resets pyplot state
+# after EVERY test, so loadscope's "same worker per module" buys nothing here —
+# it only serialized. `load` spreads the parametrized cases across ALL workers.
+#
+# nice -n 19 ionice -c 3: run at the lowest CPU + idle I/O priority so that if
+# this node is ever shared with interactive/dev work, CI grabs otherwise-idle
+# cores but YIELDS the CPU and disk to any higher-priority process — "all
+# available CPUs, with priority handling". The shell remains alive so its EXIT
+# trap always stops the private PostgreSQL cluster.
+scitex_test_run nice -n 19 ionice -c 3 \
+    python -m pytest tests/ -n "$WORKERS" --dist load -q \
+    --cov=src/scitex_dev --cov-report=xml --cov-report=term \
+    -p no:cacheprovider