scitex-storage 0.2.0__tar.gz

scitex_storage-0.2.0/.github/ci/build-in-sif.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Runs INSIDE the reused scitex-ci SIF (apptainer exec — invoked via
+# exec-in-sif.sh). Builds scitex-storage's wheel + sdist into ./dist/.
+# Adapted from scitex-ssh's .github/ci/build-in-sif.sh (the fleet's
+# established sibling convention) — only the TMPDIR label and echoed package
+# name changed; logic otherwise unchanged.
+#
+# WHY build in the SIF: the self-hosted Spartan runner has no Python on the
+# bare node (the whole reason the old `actions/setup-python@v5` step failed:
+# "version 3.x not found for this OS"). The SIF bakes python 3.11/3.12/3.13 +
+# pip + uv at /opt/venv-<ver>, exactly like the working pytest-matrix CI.
+#
+# `python -m build` needs the `build` frontend, which is NOT baked in the SIF
+# (only scitex-dev[all,dev] deps are). Mirror run-in-sif.sh: install `build`
+# into a writable --target on node-local /tmp and put it on PYTHONPATH. The
+# SIF's /opt/venv-* are root-owned + RO and the compute-node HOME is RO inside
+# the container, so a normal install fails Permission denied — a --target on
+# writable scratch sidesteps both.
+#
+# Fail-loud (operator directive): a missing interpreter or a failed build is a
+# HARD error, never a silent fallback.
+set -euo pipefail
+
+V="${1:-3.12}"
+VENV="/opt/venv-$V"
+PY="$VENV/bin/python"
+test -x "$PY" || {
+    echo "::error::baked python missing in $VENV — rebuild the SIF: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+export LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+# Writable scratch (the runner's TMPDIR=~/.cache/tmp is a host path that does
+# NOT resolve inside the container). Node-local /tmp is writable + ephemeral.
+TMPDIR="/tmp/build-scitex_storage-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-$V"
+export TMPDIR
+rm -rf "$TMPDIR"
+mkdir -p "$TMPDIR/site" "$TMPDIR/uv-cache"
+
+# The compute-node $HOME is RO inside the container — point every cache the
+# installer might touch at the writable scratch (else uv/pip die creating
+# ~/.cache).
+export UV_CACHE_DIR="$TMPDIR/uv-cache"
+export XDG_CACHE_HOME="$TMPDIR"
+export PIP_CACHE_DIR="$TMPDIR/pip-cache"
+
+# A VIRTUAL_ENV leaked from the runner profile (~/.env-3.11) is a broken
+# symlink in here; unset it so no tool follows it.
+unset VIRTUAL_ENV || true
+
+export PATH="$VENV/bin:$PATH"
+echo "build: py=$("$PY" -V) target=$TMPDIR/site"
+
+# Install the PEP 517 build frontend into the writable target (uv fast path,
+# pip safety net), then build with it. Clean dist/ first so only the freshly
+# built artifacts are uploaded.
+uv pip install --python "$PY" --target="$TMPDIR/site" build ||
+    "$PY" -m pip install --target="$TMPDIR/site" build
+
+export PYTHONPATH="$TMPDIR/site${PYTHONPATH:+:$PYTHONPATH}"
+
+rm -rf dist
+"$PY" -m build --outdir dist
+
+echo "=== built artifacts ==="
+ls -l dist
+# fail-loud: refuse to continue the pipeline with an empty dist/.
+test -n "$(ls -A dist 2>/dev/null)" || {
+    echo "::error::python -m build produced no artifacts in dist/"
+    exit 1
+}

scitex_storage-0.2.0/.github/ci/exec-in-sif.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Outer apptainer-exec wrapper for scitex-storage's self-hosted (Spartan) CI.
+# Copied verbatim from scitex-ssh's .github/ci/exec-in-sif.sh (the fleet's
+# established sibling convention) — no package-specific content in this file.
+#
+# Runs ON THE RUNNER (outside the SIF). Resolves the apptainer shim + SIF image
+# from the repo Actions Variables, then `apptainer exec`s the SIF and hands off
+# to an INNER script (run inside the container). Keeps every workflow job's YAML
+# down to one line — `bash .github/ci/exec-in-sif.sh <inner-script> [args...]` —
+# and concentrates all the HPC/SIF plumbing (shim PATH, ~-expansion, scratch,
+# binds) in one version-controlled place.
+#
+# Required env (set by the workflow from repo Actions Variables):
+#   SCITEX_CI_APPTAINER   path to the apptainer shim   (e.g. ~/.env-3.11/bin/apptainer)
+#   SCITEX_CI_SIF         path to the CI SIF image     (e.g. ~/.scitex/dev/containers/ci-cpu.sif)
+#
+# Usage:
+#   bash .github/ci/exec-in-sif.sh run-in-sif.sh 3.12
+#
+# Fail-loud (operator directive): a missing shim or SIF is a HARD error — never
+# a silent fallback to a bare-runner install.
+set -euo pipefail
+
+INNER="${1:?inner script name required (relative to .github/ci/)}"
+shift || true
+
+# The runner's job shell is --noprofile --norc (no Lmod), so the apptainer shim
+# must be put on PATH explicitly; it execs the real Apptainer binary directly.
+# ~-expand the Actions-Variable paths: a quoted "~/…" is NOT tilde-expanded by
+# the shell, so substitute a leading ~ with $HOME ourselves.
+APPTAINER="${SCITEX_CI_APPTAINER:?SCITEX_CI_APPTAINER not set (repo Actions Variable)}"
+SIF="${SCITEX_CI_SIF:?SCITEX_CI_SIF not set (repo Actions Variable)}"
+APPTAINER="${APPTAINER/#\~/$HOME}"
+SIF="${SIF/#\~/$HOME}"
+export PATH="$HOME/.env-3.11/bin:$PATH"
+
+[ -x "$APPTAINER" ] || {
+    echo "::error::apptainer shim not executable at $APPTAINER"
+    exit 1
+}
+[ -f "$SIF" ] || {
+    echo "::error::CI SIF missing at $SIF — rebuild it: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+# apptainer scratch on the shared FS — keeps HOME clean.
+export APPTAINER_TMPDIR="/data/gpfs/projects/punim0264/ywatanabe/ci/apptainer-tmp"
+mkdir -p "$APPTAINER_TMPDIR"
+
+# --bind punim0264: $HOME/.scitex is a symlink into punim0264; bind it so the
+# symlink resolves inside the container. --pwd "$PWD" keeps the checkout as cwd.
+exec "$APPTAINER" exec --pwd "$PWD" --bind /data/gpfs/projects/punim0264 \
+    "$SIF" bash ".github/ci/$INNER" "$@"

scitex_storage-0.2.0/.github/ci/publish-in-sif.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Runs INSIDE the reused scitex-ci SIF (apptainer exec — invoked via
+# exec-in-sif.sh). Publishes ./dist/* to PyPI via MANUAL OIDC Trusted
+# Publishing, then twine upload.
+# Adapted from scitex-ssh's .github/ci/publish-in-sif.sh (the fleet's
+# established sibling convention) — only the TMPDIR label and package name in
+# the trust-config comment / final echo changed; logic otherwise unchanged.
+#
+# WHY manual OIDC (not pypa/gh-action-pypi-publish): that action is a Docker
+# container action. Self-hosted Spartan compute nodes have NO Docker, so the
+# action cannot run there. PyPI Trusted Publishing is just an OIDC token
+# exchange over plain HTTPS, so we do it by hand:
+#
+#   1. Ask the GitHub Actions OIDC provider for a JWT with audience=pypi,
+#      using the per-job ACTIONS_ID_TOKEN_REQUEST_{TOKEN,URL} env vars (present
+#      because the publish job declares `permissions: id-token: write`).
+#      apptainer exec (no --cleanenv) passes those host env vars into the SIF.
+#   2. Exchange that JWT at PyPI's mint-token endpoint for a short-lived,
+#      scope-limited PyPI API token.
+#   3. twine upload dist/* with TWINE_USERNAME=__token__ and that minted token.
+#
+# REQUIRES a Trusted Publisher to already be configured on PyPI for
+# (project=scitex-storage, owner=scitex-ai, repo=scitex-storage,
+#  workflow=release.yml, environment=pypi). This is a ONE-TIME, PyPI-side
+# (pypi.org/manage/) operator action — NOT something this script, or any CI
+# job, can self-configure. If it is not yet registered, mint-token below
+# returns an empty token and this script fails loud with PyPI's raw error body.
+#
+# curl, python and (after a --target install) twine all live in the SIF.
+#
+# Fail-loud (operator directive): every step asserts non-empty output and
+# `set -euo pipefail`; any failure is a HARD error with the exact cause, never
+# a silent skip.
+set -euo pipefail
+
+V="${1:-3.12}"
+VENV="/opt/venv-$V"
+PY="$VENV/bin/python"
+test -x "$PY" || {
+    echo "::error::baked python missing in $VENV — rebuild the SIF: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+export LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+# dist/ must already hold the artifacts (downloaded by the publish job before
+# this script runs). Fail loud if empty.
+if [ ! -d dist ] || [ -z "$(ls -A dist 2>/dev/null)" ]; then
+    echo "::error::dist/ is empty — nothing to publish (download the build artifact first)"
+    exit 1
+fi
+echo "=== dist to publish ==="
+ls -l dist
+
+# --- writable scratch (compute-node HOME is RO inside the container) ---
+TMPDIR="/tmp/publish-scitex_storage-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-$V"
+export TMPDIR
+rm -rf "$TMPDIR"
+mkdir -p "$TMPDIR/site" "$TMPDIR/uv-cache"
+export UV_CACHE_DIR="$TMPDIR/uv-cache"
+export XDG_CACHE_HOME="$TMPDIR"
+export PIP_CACHE_DIR="$TMPDIR/pip-cache"
+unset VIRTUAL_ENV || true
+export PATH="$VENV/bin:$PATH"
+
+# --- step 1: request the OIDC JWT (audience=pypi) from GitHub ---
+: "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?ACTIONS_ID_TOKEN_REQUEST_TOKEN not set — the publish job needs 'permissions: id-token: write'}"
+: "${ACTIONS_ID_TOKEN_REQUEST_URL:?ACTIONS_ID_TOKEN_REQUEST_URL not set — the publish job needs 'permissions: id-token: write'}"
+
+echo "=== minting OIDC JWT (audience=pypi) ==="
+JWT="$(curl -fsS \
+    -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \
+    "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=pypi" |
+    "$PY" -c 'import sys,json; print(json.load(sys.stdin)["value"])')"
+test -n "$JWT" || {
+    echo "::error::OIDC JWT request returned an empty token"
+    exit 1
+}
+echo "OIDC JWT obtained (length=${#JWT})"
+
+# --- step 2: exchange the JWT for a short-lived PyPI API token ---
+echo "=== exchanging JWT at PyPI mint-token endpoint ==="
+MINT_RESP="$(curl -sS -X POST https://pypi.org/_/oidc/mint-token \
+    -d "{\"token\":\"${JWT}\"}")"
+MINTED="$(printf '%s' "$MINT_RESP" |
+    "$PY" -c 'import sys,json; d=json.load(sys.stdin); print(d.get("token",""))')"
+if [ -z "$MINTED" ]; then
+    # Surface PyPI's error body VERBATIM (the JWT is NOT echoed) so a trust
+    # misconfiguration is diagnosable — this is the decisive failure mode for
+    # the fleet. Print the raw response unconditionally (most reliable on an
+    # error path); pretty-print is best-effort on top.
+    echo "::error::PyPI mint-token returned no token."
+    echo "--- PyPI mint-token response body (raw) ---"
+    printf '%s\n' "$MINT_RESP"
+    echo "--- (pretty, best-effort) ---"
+    printf '%s' "$MINT_RESP" |
+        "$PY" -c 'import sys,json; print(json.dumps(json.load(sys.stdin), indent=2))' \
+            2>/dev/null || true
+    exit 1
+fi
+echo "PyPI token minted (length=${#MINTED})"
+
+# --- step 3: install twine into the writable target, then upload ---
+echo "=== installing twine (--target) ==="
+uv pip install --python "$PY" --target="$TMPDIR/site" twine ||
+    "$PY" -m pip install --target="$TMPDIR/site" twine
+export PYTHONPATH="$TMPDIR/site${PYTHONPATH:+:$PYTHONPATH}"
+
+echo "=== twine upload dist/* ==="
+TWINE_USERNAME="__token__" TWINE_PASSWORD="$MINTED" \
+    "$PY" -m twine upload --non-interactive --disable-progress-bar dist/*
+
+echo "PUBLISH-OK: scitex-storage dist/* uploaded to PyPI via manual OIDC trusted publishing"

scitex_storage-0.2.0/.github/ci/run-in-sif.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Runs INSIDE the reused scitex-ci SIF (apptainer exec). $1 = python version.
+# Adapted from scitex-ssh's .github/ci/run-in-sif.sh (the fleet's established
+# sibling convention) — package name / import path / cov target swapped for
+# scitex-storage; logic otherwise unchanged.
+#
+# WHY a layered install (not the bare PYTHONPATH=src trick scitex-dev uses):
+# the shared ci-cpu.sif bakes scitex-dev[all,dev] DEPS, NOT scitex-storage's.
+# So we install THIS checkout + its [all,dev] extras (WITH dependency
+# resolution) into a writable --target dir and prepend that on PYTHONPATH. The
+# SIF still supplies the heavy shared base (pip/uv, the python interpreters,
+# scitex-dev's deps), so only scitex-storage's own thin dep set is fetched per
+# run.
+#
+# --target (not a plain `-e .`): the SIF's /opt/venv-* are root-owned + RO and
+# the HPC compute-node HOME is RO inside the container, so a normal site install
+# fails Permission denied. A writable target on node-local /tmp sidesteps both.
+#
+# Fail-loud: a missing interpreter or a failed install is a hard error.
+set -euo pipefail
+
+V="${1:?python version arg required (3.11/3.12/3.13)}"
+VENV="/opt/venv-$V"
+test -x "$VENV/bin/python" || {
+    echo "::error::baked python missing in $VENV — rebuild the SIF: scitex-container apptainer build ci-cpu"
+    exit 1
+}
+
+export LC_ALL=C.UTF-8 LANG=C.UTF-8
+
+# Real writable scratch. The runner profile exports TMPDIR=~/.cache/tmp, a host
+# path that does NOT resolve inside the container; tests (tmp_path) and the
+# install target both need a working, writable tmp. Node-local /tmp is writable
+# + ephemeral and per-version-isolated so concurrent matrix legs don't collide.
+export TMPDIR="/tmp/ci-scitex_storage-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-0}-$V"
+rm -rf "$TMPDIR"
+mkdir -p "$TMPDIR/site" "$TMPDIR/uv-cache"
+
+# The HPC compute-node $HOME is READ-ONLY inside the container, so uv/pip cannot
+# create their default caches under ~/.cache — point them at the writable
+# scratch instead (else `uv pip install` dies: "failed to create directory
+# ~/.cache/uv: File exists / read-only").
+export UV_CACHE_DIR="$TMPDIR/uv-cache"
+export XDG_CACHE_HOME="$TMPDIR"
+export PIP_CACHE_DIR="$TMPDIR/pip-cache"
+
+# A VIRTUAL_ENV leaked from the runner profile (~/.env-3.11) is a broken symlink
+# in here; unset it so no tool (uv, pip) tries to follow it.
+unset VIRTUAL_ENV || true
+
+# venv bin on PATH (this matrix leg's python3 + pip); PYTHONPATH points at the
+# writable target so imports + coverage use the freshly-installed checkout.
+export PATH="$VENV/bin:$PATH"
+
+echo "py=$("$VENV/bin/python" -V) target=$TMPDIR/site"
+
+# Install scitex-storage + its [all,dev] extras WITH deps into the writable
+# target. Fallback chain so a packaging hiccup in an optional extra doesn't
+# strand CI: [all,dev] → [dev] → bare. uv first (fast resolver), pip as a
+# final safety net.
+uv pip install --python "$VENV/bin/python" --target="$TMPDIR/site" -e ".[all,dev]" ||
+    uv pip install --python "$VENV/bin/python" --target="$TMPDIR/site" -e ".[dev]" ||
+    uv pip install --python "$VENV/bin/python" --target="$TMPDIR/site" -e "." ||
+    pip install --target="$TMPDIR/site" -e ".[dev]"
+
+export PYTHONPATH="$TMPDIR/site:$PWD/src${PYTHONPATH:+:$PYTHONPATH}"
+
+# Parallelise with pytest-xdist if available (no-op fallback to plain -q if
+# pytest-xdist isn't in scitex-storage's [dev] extras). Worker count: use ALL
+# cores (each matrix leg runs on its own dedicated self-hosted node, no
+# co-tenant). Floor 4.
+NPROC="$(nproc 2>/dev/null || echo 4)"
+WORKERS=$NPROC
+[ "$WORKERS" -lt 4 ] && WORKERS=4
+echo "xdist workers=$WORKERS (nproc=$NPROC)"
+
+XDIST_ARGS=()
+if python -c "import xdist" 2>/dev/null; then
+    XDIST_ARGS=(-n "$WORKERS" --dist load)
+else
+    echo "pytest-xdist not importable — running single-process"
+fi
+
+# nice -n 19 ionice -c 3: lowest CPU + idle I/O priority so CI yields to any
+# higher-priority process if the node is ever shared. exec replaces the shell so
+# signals/exit code propagate to the runner step.
+exec nice -n 19 ionice -c 3 \
+    python -m pytest tests/ "${XDIST_ARGS[@]}" -q \
+    --cov=src/scitex_storage --cov-report=xml --cov-report=term \
+    -p no:cacheprovider

scitex_storage-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+name: ci
+
+# Org-level reusable-workflow caller (scitex-ai/.github, fleet standard as of
+# 2026-07-10 — see scitex-container / scitex-todo for the same pattern).
+# Four thin `uses:` calls into scitex-ai/.github reusable workflows instead
+# of inlining the job bodies; centrally maintained.
+#
+# Job ids are chosen so the resulting required-status-check context is
+# predictable: "<job id> / <reusable job name>".
+
+on:
+  pull_request:
+    branches: [main, develop]
+  push:
+    branches: [main, develop]
+  schedule:
+    - cron: "0 17 * * *" # nightly 17:00 UTC (~02:00 JST), off-peak
+  workflow_dispatch:
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pytest-matrix:
+    # Required-status-check context: "pytest-matrix / pytest-matrix-on-ubuntu-py<X.YY>"
+    uses: scitex-ai/.github/.github/workflows/pytest-matrix.yml@main
+    secrets: inherit
+
+  import-smoke:
+    # Required-status-check context: "import-smoke / import-smoke-on-ubuntu-py3-12"
+    uses: scitex-ai/.github/.github/workflows/import-smoke.yml@main
+    secrets: inherit
+
+  quality-audit:
+    # Required-status-check context: "quality-audit / audit"
+    uses: scitex-ai/.github/.github/workflows/quality-audit.yml@main
+    secrets: inherit
+
+  rtd-sphinx-build:
+    # Required-status-check context: "rtd-sphinx-build / docs-sphinx"
+    uses: scitex-ai/.github/.github/workflows/rtd-sphinx-build.yml@main
+    secrets: inherit
+
+  # Self-contained (not a shared-org `uses:` call, unlike the four jobs
+  # above) -- `scan` shells out to `fd` and `find-duplicates` to `fclones`
+  # (see src/scitex_storage/_scan.py / _duplicates.py), and `pytest-matrix`
+  # above doesn't install either binary, so the tests that exercise them
+  # for real stay skipped there. This job installs both (fast: apt for
+  # `fd-find`, the upstream release .deb for `fclones` -- no `cargo build`
+  # in CI) and reruns the suite so the real integration paths get CI
+  # coverage too.
+  fd-fclones-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install fd + fclones
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y fd-find
+          FCLONES_VERSION=0.35.0
+          curl -fsSL -o /tmp/fclones.deb \
+            "https://github.com/pkolaczk/fclones/releases/download/v${FCLONES_VERSION}/fclones_${FCLONES_VERSION}-1_amd64.deb"
+          sudo apt-get install -y /tmp/fclones.deb
+          fdfind --version
+          fclones --version
+      - name: Install scitex-storage[dev]
+        run: pip install -e '.[dev]'
+      - name: Run tests with fd/fclones on PATH
+        run: pytest "$GITHUB_WORKSPACE/tests/" -q

scitex_storage-0.2.0/.github/workflows/cla.yml

@@ -0,0 +1,26 @@
+name: CLA Assistant
+
+# Org-level reusable-workflow caller (scitex-ai/.github, fleet standard).
+# Defaults (default_branch=main, owner_allowlist="bot*,ywatanabe1989",
+# owner_login=ywatanabe1989) already match this repo — no `with:` overrides
+# needed.
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened, closed, synchronize]
+  push:
+    branches: [main, develop]
+    tags: ['v*']
+
+permissions:
+  actions: write
+  contents: write
+  pull-requests: write
+  statuses: write
+
+jobs:
+  call:
+    uses: scitex-ai/.github/.github/workflows/cla.yml@main
+    secrets: inherit

scitex_storage-0.2.0/.github/workflows/pypi-publish-and-github-release-on-tag.yml

@@ -0,0 +1,191 @@
+name: release
+
+# Tag-driven release pipeline — SELF-HOSTED (Spartan CPU) via the scitex-ci SIF.
+# Copied verbatim (structure + comments) from scitex-ssh's
+# .github/workflows/pypi-publish-and-github-release-on-tag.yml — the fleet's
+# established sibling convention for a scitex-* package's first (and every
+# subsequent) PyPI release — and adapted for scitex-storage (package name,
+# PyPI project url). See that repo for the canonical reference.
+#
+# The intended operator flow is:
+#
+#     git tag vX.Y.Z && git push origin vX.Y.Z
+#
+# Everything else fires automatically:
+#
+#   1. test          — matrix pytest INSIDE the SIF. Must pass or the pipeline
+#                      halts; nothing is published until tests are green on the
+#                      tagged commit.
+#   2. build         — wheel + sdist built INSIDE the SIF, uploaded as artifact.
+#   3. publish       — PyPI via MANUAL OIDC trusted publishing INSIDE the SIF.
+#   4. release       — GitHub Release with CHANGELOG notes + artifacts.
+#
+# develop→main promotion is a DELIBERATE, separate PR (operator policy);
+# this pipeline never auto-syncs main.
+#
+# WHY the SIF (operator standardization 2026-06-18, fleet canary 2026-06-21):
+# the Spartan compute runners have NO Python on the bare node, so the old
+# `actions/setup-python@v5` step failed at init ("version 3.x not found for
+# this OS"). The working pytest-matrix CI runs INSIDE the pre-built, reused
+# scitex-ci SIF (`apptainer exec ci-cpu.sif`), which bakes python 3.11/3.12/
+# 3.13 + pip + uv + fd at /opt/venv-<ver>. This release pipeline mirrors that:
+#   - test/build run `bash .github/ci/exec-in-sif.sh <inner>.sh <pyver>`.
+#   - publish CANNOT use pypa/gh-action-pypi-publish (a Docker action; the
+#     compute nodes have no Docker), so it does MANUAL OIDC trusted publishing
+#     inside the SIF (.github/ci/publish-in-sif.sh): GitHub OIDC JWT →
+#     PyPI mint-token → twine upload.
+#   - actions/checkout, upload/download-artifact are JS actions and run fine
+#     on the self-hosted runner.
+# fail-loud: a missing SIF/shim/interpreter is a HARD error, never a
+# bare-runner fallback.
+#
+# PREREQUISITE (one-time, PyPI-side, NOT part of this workflow): a Trusted
+# Publisher must be registered on PyPI for
+# (project=scitex-storage, owner=scitex-ai, repo=scitex-storage,
+#  workflow=release.yml, environment=pypi) BEFORE the first tag is pushed —
+# this is an operator action on pypi.org/manage/, not something this
+# workflow (or any CI job) can self-configure. See CHANGELOG.md /
+# scitex-todo card scitex-storage-pypi-release for status.
+#
+# runs-on driven by repo Actions Variable CI_RUNS_ON; SIF + apptainer paths
+# come from SCITEX_CI_SIF / SCITEX_CI_APPTAINER (same as pytest-matrix).
+#
+# Manual re-run: the workflow also accepts a `version` input via
+# workflow_dispatch so you can re-publish without re-tagging.
+
+on:
+  push:
+    tags: ['v*']
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Tag to publish (e.g. v0.2.0) — must already exist on the repo'
+        required: true
+        type: string
+
+env:
+  SCITEX_CI_APPTAINER: ${{ vars.SCITEX_CI_APPTAINER }}
+  SCITEX_CI_SIF: ${{ vars.SCITEX_CI_SIF }}
+
+jobs:
+  test:
+    runs-on: ${{ fromJSON(vars.CI_RUNS_ON || '["self-hosted","Linux","X64","scitex-ci"]') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - name: Resolve version
+        id: ver
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            v="${GITHUB_REF#refs/tags/}"
+          else
+            v="${{ inputs.version }}"
+          fi
+          echo "version=$v" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.ver.outputs.version }}
+          fetch-depth: 0
+      # SIF-based self-hosted test run — mirrors pytest-matrix exactly. No
+      # setup-python (the bare Spartan node has no Python tool cache; that step
+      # failed "version not found for this OS"); no apt fd-install (fd is baked
+      # in the SIF). exec-in-sif.sh apptainer-execs the reused ci-cpu.sif and
+      # run-in-sif.sh layers THIS (tagged) checkout + [all,dev] into a writable
+      # target, then runs pytest. fail-loud on a missing SIF/shim.
+      - name: Run tests in the reused CI SIF (apptainer exec — deps layered, not on the bare runner)
+        run: bash .github/ci/exec-in-sif.sh run-in-sif.sh ${{ matrix.python-version }}
+
+  build:
+    needs: test
+    runs-on: ${{ fromJSON(vars.CI_RUNS_ON || '["self-hosted","Linux","X64","scitex-ci"]') }}
+    outputs:
+      version: ${{ steps.ver.outputs.version }}
+    steps:
+      - name: Resolve version
+        id: ver
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            v="${GITHUB_REF#refs/tags/}"
+          else
+            v="${{ inputs.version }}"
+          fi
+          echo "version=$v" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.ver.outputs.version }}
+          fetch-depth: 0
+      # Build INSIDE the SIF (no setup-python on the bare Spartan node).
+      # build-in-sif.sh installs the `build` frontend into a writable --target
+      # inside the SIF, then `python -m build` → dist/. fail-loud on empty dist.
+      - name: Build wheel + sdist in the CI SIF (apptainer exec)
+        run: bash .github/ci/exec-in-sif.sh build-in-sif.sh 3.12
+      # upload-artifact is a JS action — runs fine on the self-hosted runner.
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ${{ fromJSON(vars.CI_RUNS_ON || '["self-hosted","Linux","X64","scitex-ci"]') }}
+    environment:
+      name: pypi
+      url: https://pypi.org/p/scitex-storage
+    # id-token: write is what populates ACTIONS_ID_TOKEN_REQUEST_{TOKEN,URL},
+    # which publish-in-sif.sh uses to mint the OIDC JWT. Keep it.
+    permissions:
+      id-token: write
+    steps:
+      # Need the repo checked out so .github/ci/exec-in-sif.sh +
+      # publish-in-sif.sh are on disk (the build job uploaded only dist/).
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      # Manual OIDC trusted publishing INSIDE the SIF — pypa/gh-action-pypi-
+      # publish is a Docker action and the Spartan compute nodes have no Docker.
+      # publish-in-sif.sh: GitHub OIDC JWT (audience=pypi) → PyPI mint-token →
+      # twine upload dist/*. apptainer exec (no --cleanenv) passes the
+      # ACTIONS_ID_TOKEN_REQUEST_* env vars into the SIF. fail-loud at each step.
+      # REQUIRES a Trusted Publisher already configured on PyPI for this
+      # (project, owner, repo, workflow, environment) tuple — see the
+      # PREREQUISITE note at the top of this file.
+      - name: Publish to PyPI via manual OIDC trusted publishing in the SIF
+        run: bash .github/ci/exec-in-sif.sh publish-in-sif.sh 3.12
+
+  release:
+    needs: [build, publish]
+    runs-on: ${{ fromJSON(vars.CI_RUNS_ON || '["self-hosted","Linux","X64","scitex-ci"]') }}
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build.outputs.version }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      - name: Extract release notes from CHANGELOG.md
+        run: |
+          VERSION="${{ needs.build.outputs.version }}"
+          VERSION="${VERSION#v}"
+          awk -v v="$VERSION" '
+            $0 ~ "^## \\[" v "\\]" {p=1; next}
+            p && /^## \[/ {exit}
+            p
+          ' CHANGELOG.md > release-notes.md || true
+          if [ ! -s release-notes.md ]; then
+            echo "Release v${VERSION}. See CHANGELOG.md for details." > release-notes.md
+          fi
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create "${{ needs.build.outputs.version }}" \
+            --title "${{ needs.build.outputs.version }}" \
+            --notes-file release-notes.md \
+            dist/*

scitex_storage-0.2.0/.gitignore

@@ -0,0 +1,43 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.egg-info/
+*.egg
+dist/
+build/
+.eggs/
+*.whl
+.venv/
+venv/
+env/
+.env
+.tox/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+*.swp
+*.swo
+*~
+.DS_Store
+.idea/
+.vscode/
+
+# direnv + git worktrees live in-tree but are never committed.
+.envrc
+.worktrees/
+
+# scitex-dev local state: ignore everything under .scitex/ except the
+# tracked dev/config.yaml and the CLI-audit noun/verb dict (both are audit
+# inputs CI must see on a fresh checkout).
+.scitex/*
+!.scitex/dev/
+.scitex/dev/*
+!.scitex/dev/config.yaml
+!.scitex/dev/cli-audit-dict.yaml
+
+# Sphinx local build output (the in-wheel src/<pkg>/_sphinx_html bundle IS tracked)
+docs/sphinx/_build/
+
+# Operator's private brainstorm notes — never committed (see repo instructions).
+GITIGNORED/