xenium-splitter 0.2.1__tar.gz

xenium_splitter-0.2.1/.github/workflows/publish.yml

@@ -0,0 +1,129 @@
+name: Publish To PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pypi-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Test And Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      RELEASE_TAG: ${{ github.ref_name }}
+    steps:
+      - name: Check Out Release Tag
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Validate Release Tag
+        run: |
+          python -m pip install packaging
+          python - <<'PY'
+          import os
+          from packaging.version import Version
+
+          version = Version(os.environ["RELEASE_TAG"].removeprefix("v"))
+          if version.local is not None or version.is_devrelease:
+              raise SystemExit("Release tags must not contain local or development versions")
+          print(f"Release version: {version}")
+          PY
+
+      - name: Install Build And Test Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]" build twine
+
+      - name: Run Tests
+        run: python -m pytest
+
+      - name: Build Distributions
+        run: python -m build
+
+      - name: Check Distribution Metadata And Versions
+        run: |
+          python -m twine check dist/*
+          python - <<'PY'
+          import os
+          import tarfile
+          import zipfile
+          from email.parser import BytesParser
+          from pathlib import Path
+          from packaging.version import Version
+
+          expected = Version(os.environ["RELEASE_TAG"].removeprefix("v"))
+          wheels = list(Path("dist").glob("*.whl"))
+          sources = list(Path("dist").glob("*.tar.gz"))
+          if len(wheels) != 1 or len(sources) != 1:
+              raise SystemExit("Expected exactly one wheel and one source distribution")
+          with zipfile.ZipFile(wheels[0]) as archive:
+              metadata_path = next(
+                  name for name in archive.namelist() if name.endswith(".dist-info/METADATA")
+              )
+              wheel_metadata = BytesParser().parsebytes(archive.read(metadata_path))
+          with tarfile.open(sources[0]) as archive:
+              metadata_member = next(
+                  member for member in archive.getmembers()
+                  if member.name.count("/") == 1 and member.name.endswith("/PKG-INFO")
+              )
+              source_metadata = BytesParser().parsebytes(
+                  archive.extractfile(metadata_member).read()
+              )
+          for metadata in (wheel_metadata, source_metadata):
+              actual = Version(metadata["Version"])
+              if actual != expected:
+                  raise SystemExit(f"Tag version {expected} does not match build version {actual}")
+          print(f"Both distributions match release tag: {expected}")
+          PY
+
+      - name: Smoke Test Installed Wheel
+        working-directory: ${{ runner.temp }}
+        run: |
+          python -m venv "$RUNNER_TEMP/wheel-smoke"
+          "$RUNNER_TEMP/wheel-smoke/bin/python" -m pip install "$GITHUB_WORKSPACE"/dist/*.whl
+          "$RUNNER_TEMP/wheel-smoke/bin/python" -m pip check
+          "$RUNNER_TEMP/wheel-smoke/bin/xenium-splitter" --help
+          "$RUNNER_TEMP/wheel-smoke/bin/xenium-recalc-diffexp" --help
+          "$RUNNER_TEMP/wheel-smoke/bin/xenium-validate-transcripts-zarr" --help
+
+      - name: Upload Release Distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: distributions
+          path: dist/*
+          if-no-files-found: error
+
+  publish:
+    name: Publish To PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment:
+      name: pypi
+      url: https://pypi.org/project/xenium-splitter/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download Release Distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: distributions
+          path: dist
+
+      - name: Publish With Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@release/v1

xenium_splitter-0.2.1/.gitignore

@@ -0,0 +1,23 @@
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+.venv/
+venv/
+
+# local outputs
+output/
+
+# large raw data
+data/
+
+data_out/

xenium_splitter-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

xenium_splitter-0.2.1/PKG-INFO

@@ -0,0 +1,419 @@
+Metadata-Version: 2.4
+Name: xenium-splitter
+Version: 0.2.1
+Summary: Split Xenium outputs by LASSO regions and optionally process matching H&E images.
+Author: xenium-splitter contributors
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy<2,>=1.26
+Requires-Dist: opencv-python-headless<4.12,>=4.8
+Requires-Dist: pandas>=2.2
+Requires-Dist: pyarrow>=15.0
+Requires-Dist: h5py>=3.10
+Requires-Dist: zarr<=3.0.0
+Requires-Dist: Pillow>=10.0
+Requires-Dist: shapely>=2.0
+Requires-Dist: tifffile>=2024.5.22
+Requires-Dist: imagecodecs>=2024.1.1
+Requires-Dist: typer>=0.12
+Provides-Extra: parquet
+Provides-Extra: svs
+Requires-Dist: openslide-python>=1.3.1; extra == "svs"
+Provides-Extra: hdf5
+Provides-Extra: zarr
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: ruff>=0.6.0; extra == "dev"
+Dynamic: license-file
+
+# xenium-splitter
+
+`xenium-splitter` splits Xenium output files by user-defined LASSO regions and writes one output
+directory per region. It can also process a matching H&E image, applying polygon masks and
+cropping to each region.
+
+## Features
+
+- CLI-first workflow.
+- LASSO region parsing from GeoJSON/JSON or CSV/TSV.
+- Tabular Xenium split (CSV/TSV/TXT/Parquet) using detected coordinate columns.
+- Coordinate re-basing in each region output (region origin becomes `(0, 0)`).
+- Image masking + cropping for raster images (`.png`, `.jpg`, `.jpeg`, `.tif`, `.tiff`, `.ome.tif`, `.ome.tiff`).
+- Optional SVS handling for H&E: load, split, and optionally convert to OME-TIFF.
+- `show-regions` preview command: render labeled LASSO regions and dotted region bounding boxes on low-res H&E/morphology images.
+- Metadata README generation with run parameters and file-level metrics.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+Zarr, HDF5, and Parquet support are included in the default installation.
+
+Optional extras:
+
+```bash
+pip install -e ".[svs,dev]"
+```
+
+The `svs` extra adds SVS image support and requires native OpenSlide libraries.
+The `dev` extra adds testing and linting tools. The legacy `parquet`, `hdf5`, and
+`zarr` extras remain accepted for compatibility but are no longer necessary.
+
+For versioning and publishing instructions, see [release.md](release.md).
+
+## CLI Usage
+
+### RAM Estimate
+
+```bash
+xenium-splitter estimate-ram \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.csv \
+  --output-dir /path/to/output
+```
+
+Estimates peak RAM using the on-disk archive sizes directly inside `--input-dir`:
+
+```text
+PeakRAM(GB) = 2.5 + 0.010 * TranscriptZarrMB + 0.007 * CellZarrMB
+```
+
+`TranscriptZarrMB` is the size of `transcripts.zarr.zip`; `CellZarrMB` is the size
+of `cells.zarr.zip`. Both files are required. MB means 1,000,000 bytes, not MiB.
+The estimator reads file sizes only; it does not load or extract the archives.
+It accepts all the same options, defaults, and required paths as `split`; replace
+`split` with `estimate-ram` in a split invocation. This includes repeated
+`--include-glob` options, image/analysis switches, and `-v`/`--verbose`.
+`--images-only` and `--skip-images` remain mutually exclusive.
+Only the two archive sizes affect this equation: processing options, LASSO
+geometry, H&E, and the output path are not modeled. No output files or directories
+are created.
+
+This is an approximate peak estimate, not a guaranteed memory limit or a scheduler
+allocation recommendation. Image layout, region geometry, and processing options
+are not modeled; allow additional headroom when requesting RAM.
+
+### Options
+
+**Required:**
+
+| Option         | Description                                                                                              |
+| -------------- | -------------------------------------------------------------------------------------------------------- |
+| `--input-dir`  | Xenium output directory (the `outs/` folder or equivalent).                                              |
+| `--lasso-file` | LASSO region file — GeoJSON/JSON or CSV/TSV (see [LASSO Input Expectations](#lasso-input-expectations)). |
+| `--output-dir` | Destination directory; one `region_<id>/` subfolder is created per region.                               |
+
+**Optional — H&E image:**
+
+| Option                 | Default | Description                                                                            |
+| ---------------------- | ------- | -------------------------------------------------------------------------------------- |
+| `--he-image`           | None    | External H&E image to split alongside Xenium outputs (TIFF, OME-TIFF, SVS, PNG, JPEG). |
+| `--convert-svs-to-ome` | off     | When `--he-image` is an SVS file, also write a full-slide OME-TIFF per region.         |
+
+> **Note:** xenium-splitter does **not** perform H&E registration. The H&E image is assumed to already be spatially aligned with the Xenium coordinate space. No registration file, affine transform, or warping is applied. If your H&E requires registration to the morphology image, that step must be completed before running this tool. The tool also assumes the H&E image uses the **same pixel size** as the Xenium morphology data — it reads `pixel_size_um` from `experiment.xenium` and uses that value to convert polygon coordinates to pixel coordinates for both. If your H&E has a different pixel resolution, the crop will be incorrectly positioned.
+
+**Optional — image processing:**
+
+| Option                                   | Default            | Description                                                                                                                                                                                                                             |
+| ---------------------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--squash-layers` / `--no-squash-layers` | `--squash-layers`  | Flatten multi-layer TIFF stacks to 2D/RGB before cropping.                                                                                                                                                                              |
+| `--skip-images` / `--process-images`     | `--process-images` | Skip all image cropping and masking (useful for fast data-only runs).                                                                                                                                                                   |
+| `--images-only`                          | off                | Process **only** image files; skip all tabular data, zarr, CFM, and diffexp stages. Useful when running images separately to keep RAM available, or for a re-run after a prior data-only pass. Mutually exclusive with `--skip-images`. |
+| `--overlays` / `--no-overlays`           | `--no-overlays`    | Write annotated FOV grid overlay images alongside each region `morphology_mip.ome.tif`.                                                                                                                                                 |
+
+**Optional — data filtering:**
+
+| Option               | Default          | Description                                                                        |
+| -------------------- | ---------------- | ---------------------------------------------------------------------------------- |
+| `--include-glob`     | None (all files) | Glob pattern to restrict which files are read from `--input-dir`; repeatable.      |
+| `--copy-transcripts` | off              | Copy `transcripts.zarr.zip` verbatim to each region instead of filtering/rebasing. |
+
+**Optional — analysis outputs:**
+
+| Option                                                                 | Default                            | Description                                                                       |
+| ---------------------------------------------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------- |
+| `--recalculate-diffexp` / `--skip-diffexp-recalc`                      | `--recalculate-diffexp`            | Recompute `analysis/diffexp` CSVs from the filtered matrix and clustering labels. |
+| `--write-cell-feature-matrix-zarr` / `--skip-cell-feature-matrix-zarr` | `--write-cell-feature-matrix-zarr` | Write `cell_feature_matrix.zarr.zip` in each region output.                       |
+
+**Optional — logging:**
+
+| Option             | Default | Description                           |
+| ------------------ | ------- | ------------------------------------- |
+| `-v` / `--verbose` | off     | Enable debug-level logging to stderr. |
+
+### Examples
+
+**Minimal split — tabular data and morphology only:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output
+```
+
+**Fast data-only run — skip all image processing:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.csv \
+  --output-dir /path/to/output \
+  --skip-images -v
+```
+
+**Include a pre-aligned H&E TIFF alongside Xenium outputs:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --he-image /path/to/aligned_he.ome.tif
+```
+
+**Include an SVS H&E and also produce per-region OME-TIFF exports:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --he-image /path/to/he.svs \
+  --convert-svs-to-ome
+```
+
+**Write FOV grid overlays for visual QC of region alignment:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --overlays
+```
+
+**Process only a subset of files matching a glob pattern:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --include-glob "cells*" \
+  --include-glob "transcripts*" \
+  --skip-images
+```
+
+**Copy transcripts verbatim (no filtering or coordinate rebasing):**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --copy-transcripts
+```
+
+**Images-only re-run after a prior data-only pass (minimise peak RAM):**
+
+```bash
+# Step 1 — data only (low RAM):
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --skip-images
+
+# Step 2 — images only (reserves all RAM for image loading):
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --images-only
+```
+
+**Images-only with a large SVS H&E:**
+
+```bash
+xenium-splitter split \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-dir /path/to/output \
+  --images-only \
+  --he-image /path/to/he.svs
+```
+
+### Temp Cleanup
+
+Remove temporary working directories created under the OS temp folder
+(`.../xenium_splitter`):
+
+```bash
+xenium-splitter clean-temp
+```
+
+### Region Preview (show-regions)
+
+Create a quick visual QC image (max `2000 x 2000`) that overlays LASSO regions and
+region bounding boxes on a source tissue image.
+
+Behavior:
+
+- Source image preference: H&E first, morphology fallback.
+- Region outline color: purple.
+- Dotted region bounding box color:
+  - H&E source: black.
+  - Morphology source: green.
+- Region labels: each `region_id` label is drawn once per bounding box in the top-inside corner with a small margin.
+- Label font size scales with generated image size for readability.
+
+CLI options:
+
+| Option               | Description                                                                    |
+| -------------------- | ------------------------------------------------------------------------------ |
+| `--input-dir`        | Xenium input directory used to discover source images and `experiment.xenium`. |
+| `--lasso-file`       | LASSO region file (GeoJSON/JSON or CSV/TSV).                                   |
+| `--output-image`     | Output preview image path (for example `regions_preview.png`).                 |
+| `--he-image`         | Optional explicit H&E image path; when provided, this is used directly.        |
+| `--max-dimension-px` | Longest output edge in pixels (max allowed value is `2000`, default `2000`).   |
+
+Examples:
+
+```bash
+xenium-splitter show-regions \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.geojson \
+  --output-image /path/to/output/regions_preview.png
+```
+
+```bash
+xenium-splitter show-regions \
+  --input-dir /path/to/xenium_output \
+  --lasso-file regions.csv \
+  --he-image /path/to/aligned_he.svs \
+  --output-image /path/to/output/regions_preview.png
+```
+
+## LASSO Input Expectations
+
+### GeoJSON / JSON
+
+- `FeatureCollection` with polygon-like geometries (`Polygon` or `MultiPolygon`).
+- Region id is read from `id`, `region_id`, `name`, or generated automatically.
+
+### CSV / TSV
+
+Two supported layouts:
+
+1. Point list with columns: `region_id`, `x`, `y`.
+2. WKT with columns: `region_id`, `polygon_wkt`.
+
+## Output Layout
+
+For each region, a directory is created:
+
+```text
+output/
+  region_<id>/
+    ...split files mirroring input names...
+  run_metadata_README.md
+```
+
+## RAM Requirements
+
+Memory usage is driven by three independent factors: transcript table size, image size, and zarr processing. They do not all peak simultaneously, but on large datasets they can overlap significantly. Note, official benchmarking has not been done on this yet. Once benchmarks are available, they will be added to this section.
+
+### Without image processing (`--skip-images`)
+
+The dominant cost is the full transcript table, which is loaded entirely into RAM as a pandas DataFrame before being filtered per region.
+
+| Dataset scale    | Approx. transcripts | Recommended RAM |
+| ---------------- | ------------------- | --------------- |
+| Small (< 1 mm²)  | ~500 K              | 4 GB            |
+| Medium (~10 mm²) | ~5 M                | 8 GB            |
+| Large (~50 mm²)  | ~50 M               | 16–32 GB        |
+
+The cell/barcode tables and CFM zarr add a modest additional cost (typically < 1 GB).
+
+### With image processing (default)
+
+Image memory cost depends on format and adds on top of the transcript cost above.
+
+**Morphology TIFF / OME-TIFF — always full-image load:**
+The entire morphology image is read into RAM before cropping; windowed reads are not used for TIFF. A temporary copy is created during Z-stack squashing (max-projection), so peak is approximately 2× the raw array size.
+
+| Image dimensions | Channels / planes  | RAM (peak ~2×) |
+| ---------------- | ------------------ | -------------- |
+| 5 K × 5 K        | 1 plane, uint16    | ~100 MB        |
+| 10 K × 10 K      | 1 plane, uint16    | ~400 MB        |
+| 20 K × 20 K      | 7 Z-planes, uint16 | ~11 GB         |
+| 30 K × 30 K      | 7 Z-planes, uint16 | ~25 GB         |
+
+**H&E as TIFF / OME-TIFF — full-image load:**
+Same behaviour as morphology: the complete image is loaded before any cropping.
+
+| Image dimensions | Channels  | RAM (peak ~2×) |
+| ---------------- | --------- | -------------- |
+| 10 K × 20 K      | RGB uint8 | ~1.2 GB        |
+| 25 K × 45 K      | RGB uint8 | ~6.8 GB        |
+
+**H&E as SVS — windowed, memory-efficient:**
+OpenSlide reads only the polygon bounding box per region; the full slide is never loaded. A typical crop (5 K × 5 K px) uses ~100 MB regardless of source file size. **SVS is the recommended format for large H&E images.**
+
+### Practical guidance
+
+| Scenario                                      | Recommended minimum RAM |
+| --------------------------------------------- | ----------------------- |
+| Data only, small dataset                      | 8 GB                    |
+| Data only, large dataset (≥ 50 M transcripts) | 32–64 GB                |
+| Data + morphology TIFF, medium dataset        | 16 GB                   |
+| Data + H&E TIFF (25 K × 45 K), medium dataset | 32 GB                   |
+| Data + H&E TIFF (25 K × 45 K), large dataset  | 64 GB                   |
+| Data + H&E SVS (any size), medium dataset     | 16 GB                   |
+| Data + H&E SVS (any size), large dataset      | 32–48 GB                |
+
+> **Tip:** Run `--skip-images` first for a fast data-only pass. Re-run with images only if needed. For large H&E files, convert to SVS or use a tiled OME-TIFF with a viewer that supports windowed reads before passing to this tool.
+
+## Current Scope and Notes
+
+- Files without recognized coordinate columns (for tabular) are skipped and documented in metadata.
+- Unknown binary formats are currently skipped.
+- SVS requires `openslide-python` and system OpenSlide libraries.
+
+## FOV Recalculation Rules
+
+When rebuilding transcript FOV assignments, `xenium-splitter` reads
+`instrument_sw_version` from `experiment.xenium` and uses these FOV dimensions
+(in pixels):
+
+- `< 1.2`: `4240` rows x `2960` cols
+- `>= 1.2`: `3520` rows x `2960` cols
+
+The FOV overlap is fixed at `128` pixels.
+
+- X stride: `2960 - 128 = 2832`
+- Y stride (`< 1.2`): `4240 - 128 = 4112`
+- Y stride (`>= 1.2`): `3520 - 128 = 3392`
+
+Potential FOV counts in each direction are computed from rebased transcript
+pixel coordinates (`x_px`, `y_px`) as:
+
+- `potential_fov_x = floor(max(x_px) / x_stride) + 1`
+- `potential_fov_y = floor(max(y_px) / y_stride) + 1`
+
+where `x_px = floor(x_um / pixel_size_um)` and `y_px = floor(y_um / pixel_size_um)`.
+
+For example, with `instrument_sw_version: 1.1.2.4`, the splitter uses the
+`< 1.2` rule: `4240 x 2960` FOVs, overlap `128`, strides `4112` (Y) and `2832` (X).
+
+## Development
+
+```bash
+pytest
+ruff check .
+```