mcmicroprep 0.1.0__tar.gz

mcmicroprep-0.1.0/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.1
+Name: mcmicroprep
+Version: 0.1.0
+Summary: 
+Author: Ajit Johnson Nirmal
+Author-email: ajitjohnson.n@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: lxml (>=6.0.0,<7.0.0)
+Requires-Dist: ome-types (>=0.6.1,<0.7.0)
+Requires-Dist: pandas (>=2.3.1,<3.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
+Description-Content-Type: text/markdown
+
+# 🧪 mcmicroprep 🚀
+
+A **command-line tool** for preparing multiplexed imaging datasets (🦠 Olympus, 🩸 RareCyte) for the MCMICRO Nextflow pipeline.
+
+
+## 🛠️ Installation
+
+1. **Prerequisites**
+
+   - Conda or Miniconda installed 🐍
+   - Python **3.10+** environment 🌟
+   - SLURM & Nextflow (`labsyspharm/mcmicro`) on your `$PATH`
+
+2. **Create Conda env**
+
+   ```bash
+   conda create -n mcmicroprep python=3.12
+   conda activate mcmicroprep
+   ```
+
+3. **Install package**
+
+   ```bash
+   pip install mcmicroprep
+   ```
+
+## 📁 Expected Dataset Structure
+
+Your dataset root should contain one subdirectory per slide. Structures vary by vendor:
+
+### 🦠 Olympus
+
+Each *slide directory* must contain at least one `*_frames/` folder —-- this is the minimum required structure. Additional files or folders may be present and do not need to be removed.
+
+```
+.DATASET FOLDER
+├── slide1/
+│   ├── image1_frames/
+│   ├── image2_frames/
+├── slide2/
+└── slideN/
+```
+
+After running for **Olympus**: each slide/image folder would be as follows
+
+```
+slide1/
+├── raw/                   # image1_frames/, image2_frames/
+├── misc_files/            # JSON, logs
+├── batch_submission.sh    # pipeline wrapper
+├── mcmicro_template.sh    # Nextflow template
+├── base.config
+├── markers.csv
+└── params.yml            
+```
+
+### 🩸 RareCyte
+
+Slide dirs may contain `*.rcpnl` at any depth: —-- this is the minimum required structure. Additional files or folders may be present and do not need to be removed.
+
+```
+/path/to/dataset/
+├── slide1/
+│   ├── img001.rcpnl
+│   ├── subA/img002.rcpnl
+│   └── other files
+└── slideN/
+```
+
+After running for **RareCyte**:
+
+```
+slide1/
+├── raw/                   # all .rcpnl files
+│   ├── img001.rcpnl
+│   └── img002.rcpnl
+├── misc_files/            # CSV, text
+├── batch_submission.sh
+├── mcmicro_template.sh
+├── base.config
+├── markers.csv
+└── params.yml             
+```
+
+## 🚀 Usage
+
+> **Note**: Configured for the HMS O2 cluster (SLURM). Generalize by editing SLURM directives in `templates/common/`.
+
+### 🦠 Olympus
+
+```bash
+preparemcmicro \
+  --microscope olympus \
+  --image-root /path/to/dataset
+```
+
+### 🩸 RareCyte
+
+```bash
+preparemcmicro \
+  --microscope rarecyte \
+  --image-root /path/to/dataset
+```
+
+## 🛠️ Next Steps for Users
+
+1. ✏️ **Edit **`` in each slide directory to include your experiment-specific cycle-to-marker mappings.
+2. 📤 **Upload** the entire processed dataset folder to the O2 cluster if you ran this locally.
+3. 🚀 **Start the job** on O2:
+   ```bash
+   cd /n/scratch/users/USERNAME/<DATASET FOLDER>
+   bash batch_submission.sh --dataset_path /n/scratch/users/USERNAME/<DATASET FOLDER>
+   ```
+
+Happy processing! 🔬
+
+

mcmicroprep-0.1.0/README.md

@@ -0,0 +1,116 @@
+# 🧪 mcmicroprep 🚀
+
+A **command-line tool** for preparing multiplexed imaging datasets (🦠 Olympus, 🩸 RareCyte) for the MCMICRO Nextflow pipeline.
+
+
+## 🛠️ Installation
+
+1. **Prerequisites**
+
+   - Conda or Miniconda installed 🐍
+   - Python **3.10+** environment 🌟
+   - SLURM & Nextflow (`labsyspharm/mcmicro`) on your `$PATH`
+
+2. **Create Conda env**
+
+   ```bash
+   conda create -n mcmicroprep python=3.12
+   conda activate mcmicroprep
+   ```
+
+3. **Install package**
+
+   ```bash
+   pip install mcmicroprep
+   ```
+
+## 📁 Expected Dataset Structure
+
+Your dataset root should contain one subdirectory per slide. Structures vary by vendor:
+
+### 🦠 Olympus
+
+Each *slide directory* must contain at least one `*_frames/` folder —-- this is the minimum required structure. Additional files or folders may be present and do not need to be removed.
+
+```
+.DATASET FOLDER
+├── slide1/
+│   ├── image1_frames/
+│   ├── image2_frames/
+├── slide2/
+└── slideN/
+```
+
+After running for **Olympus**: each slide/image folder would be as follows
+
+```
+slide1/
+├── raw/                   # image1_frames/, image2_frames/
+├── misc_files/            # JSON, logs
+├── batch_submission.sh    # pipeline wrapper
+├── mcmicro_template.sh    # Nextflow template
+├── base.config
+├── markers.csv
+└── params.yml            
+```
+
+### 🩸 RareCyte
+
+Slide dirs may contain `*.rcpnl` at any depth: —-- this is the minimum required structure. Additional files or folders may be present and do not need to be removed.
+
+```
+/path/to/dataset/
+├── slide1/
+│   ├── img001.rcpnl
+│   ├── subA/img002.rcpnl
+│   └── other files
+└── slideN/
+```
+
+After running for **RareCyte**:
+
+```
+slide1/
+├── raw/                   # all .rcpnl files
+│   ├── img001.rcpnl
+│   └── img002.rcpnl
+├── misc_files/            # CSV, text
+├── batch_submission.sh
+├── mcmicro_template.sh
+├── base.config
+├── markers.csv
+└── params.yml             
+```
+
+## 🚀 Usage
+
+> **Note**: Configured for the HMS O2 cluster (SLURM). Generalize by editing SLURM directives in `templates/common/`.
+
+### 🦠 Olympus
+
+```bash
+preparemcmicro \
+  --microscope olympus \
+  --image-root /path/to/dataset
+```
+
+### 🩸 RareCyte
+
+```bash
+preparemcmicro \
+  --microscope rarecyte \
+  --image-root /path/to/dataset
+```
+
+## 🛠️ Next Steps for Users
+
+1. ✏️ **Edit **`` in each slide directory to include your experiment-specific cycle-to-marker mappings.
+2. 📤 **Upload** the entire processed dataset folder to the O2 cluster if you ran this locally.
+3. 🚀 **Start the job** on O2:
+   ```bash
+   cd /n/scratch/users/USERNAME/<DATASET FOLDER>
+   bash batch_submission.sh --dataset_path /n/scratch/users/USERNAME/<DATASET FOLDER>
+   ```
+
+Happy processing! 🔬
+

mcmicroprep-0.1.0/mcmicroprep/companion_script.py

@@ -0,0 +1,105 @@
+# %%
+import dataclasses
+import lxml.etree
+import ome_types
+import pandas as pd
+import pathlib
+import pydantic
+import re
+import sys
+import uuid
+
+
+# %%
+@pydantic.validate_arguments
+@dataclasses.dataclass
+class AlignInfo:
+    version: str
+    unknown1: str
+    unknown2: str
+    unknown3: str
+    unknown4: str
+    pixel_size_x: float
+    pixel_size_y: float
+    tile_size_x: int
+    tile_size_y: int
+
+
+# in_path = pathlib.Path(sys.argv[1])
+# align_info_raw = re.split(r"\s+", (in_path / "AlignInfo").read_text().rstrip())
+# replaced by ajit
+def generate_companion_ome(in_path):
+    in_path = pathlib.Path(in_path)
+    align_info_raw = re.split(r"\s+", (in_path / "AlignInfo").read_text().rstrip())
+    assert align_info_raw[0] == "SAlignInfo2"
+    align_info = AlignInfo(*align_info_raw)
+
+    tree = lxml.etree.parse(in_path / "ScanSpace")
+    points = pd.read_xml(in_path / "ScanSpace", xpath="./vPoints/point")
+    # vInvSrtPrm maps point indices to field indices (.tif numbers).
+    points["field"] = [int(index.text) for index in tree.find("vInvSrtPrm")]
+    points = points.sort_values("field")
+    num_channels = int(tree.find("vOMs").attrib["n"])
+    channel_indices = list(range(num_channels))
+
+    ome = ome_types.OME()
+    field_groups = points.groupby("iAI", sort=False)
+    for _, plane_data in field_groups:
+        # Ensure we have all channels and they are already sorted.
+        assert list(plane_data.iOM) == channel_indices
+        channels = []
+        planes = []
+        tiff_data_blocks = []
+        for p in plane_data.itertuples():
+            channels.append(ome_types.model.Channel())
+            planes.append(
+                ome_types.model.Plane(
+                    position_x=p.fStgX,
+                    position_x_unit="µm",
+                    position_y=p.fStgY,
+                    position_y_unit="µm",
+                    the_z=0,
+                    the_t=0,
+                    the_c=p.iOM,
+                ),
+            )
+            tiff_data_blocks.append(
+                ome_types.model.TiffData(
+                    uuid=ome_types.model.TiffData.UUID(
+                        file_name=f"{p.field}.tif",
+                        value=uuid.uuid4().urn,
+                    ),
+                    first_c=p.iOM,
+                    plane_count=1,
+                ),
+            )
+        pixels = ome_types.model.Pixels(
+            dimension_order="XYCZT",
+            type="uint16",
+            size_x=align_info.tile_size_x,
+            size_y=align_info.tile_size_y,
+            size_z=1,
+            size_c=num_channels,
+            size_t=1,
+            physical_size_x=align_info.pixel_size_x,
+            physical_size_x_unit="µm",
+            physical_size_y=align_info.pixel_size_y,
+            physical_size_y_unit="µm",
+            channels=channels,
+            planes=planes,
+            tiff_data_blocks=tiff_data_blocks,
+        )
+        image = ome_types.model.Image(pixels=pixels)
+        ome.images.append(image)
+
+    with open(in_path / "image.companion.ome", "w", encoding="utf-8") as f:
+        f.write(ome.to_xml())
+
+
+def main(frames_path):
+    generate_companion_ome(frames_path)
+
+if __name__ == "__main__":
+    import sys
+    main(sys.argv[1])
+

mcmicroprep-0.1.0/mcmicroprep/core.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+import shutil
+from mcmicroprep.microscopes.base import MicroscopeHandler
+
+
+class CorePipeline:
+    def __init__(self, handler: MicroscopeHandler, image_root: Path):
+        self.handler = handler
+        self.image_root = Path(image_root)
+        self.templates = Path(__file__).parent / "templates"
+
+    def run(self):
+        # 1️⃣ Organize raw imagery in-place
+        self.handler.restructure_raw(self.image_root)
+        # 2️⃣ Enforce dataset structure (expect image_root to be the dataset root)
+        # No additional moving of slide folders; image_root should contain slide dirs directly.
+        # 3️⃣ Generate boilerplate
+        self._generate_boilerplate()
+
+    def _generate_boilerplate(self):
+        # Copy common boilerplate templates
+        common = self.templates / "common"
+        for fname in [
+            "batch_submission.sh",
+            "mcmicro_template.sh",
+            "base.config",
+            "markers.csv",
+        ]:
+            src = common / fname
+            dst = self.image_root / fname
+            if not dst.exists():
+                shutil.copy(src, dst)
+                if dst.suffix == ".sh":
+                    dst.chmod(dst.stat().st_mode | 0o111)
+
+        # Copy microscope-specific params YAML
+        params_src = self.templates / "params" / f"{self.handler.name}.yml"
+        params_dst = self.image_root / "params.yml"
+        if params_src.exists() and not params_dst.exists():
+            shutil.copy(params_src, params_dst)

mcmicroprep-0.1.0/mcmicroprep/microscopes/base.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+class MicroscopeHandler(ABC):
+    """
+    Base class for microscope-specific handlers.
+    """
+    name: str  # e.g. 'olympus'
+
+    @abstractmethod
+    def restructure_raw(self, image_root: Path):
+        """Reorganize raw files into raw/ and misc_files/ in place."""
+        pass
+
+    @abstractmethod
+    def params_template(self) -> dict:
+        """(Optional) Return dict for param overrides; not used for file-based params."""
+        pass

mcmicroprep-0.1.0/mcmicroprep/microscopes/olympus.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+from pathlib import Path
+from mcmicroprep.microscopes.base import MicroscopeHandler
+
+
+class OlympusHandler(MicroscopeHandler):
+    name = "olympus"
+
+    def restructure_raw(self, image_root: Path):
+        for slide in image_root.iterdir():
+            if not slide.is_dir() or slide.name in [
+                "dataset",
+                "templates",
+                "microscopes",
+            ]:
+                continue
+            raw = slide / "raw"
+            misc = slide / "misc_files"
+            raw.mkdir(exist_ok=True)
+            misc.mkdir(exist_ok=True)
+
+            for item in list(slide.iterdir()):
+                if item.is_dir() and item.name.endswith("_frames"):
+                    item.rename(raw / item.name)
+                elif item.name not in ["raw", "misc_files"]:
+                    item.rename(misc / item.name)
+
+            # Only run companion script if missing the OME file
+            for frame in raw.iterdir():
+                ome = frame / "image.companion.ome"
+                if not ome.exists():
+                    # Invoke installed module, not script in cwd
+                    subprocess.run(
+                        [
+                            sys.executable,
+                            "-m",
+                            "mcmicroprep.companion_script",
+                            str(frame),
+                        ],
+                        check=True,
+                    )
+
+    def params_template(self) -> dict:
+        # Not used when file-based params selection is preferred
+        return {}

mcmicroprep-0.1.0/mcmicroprep/microscopes/rarecyte.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+from pathlib import Path
+from mcmicroprep.microscopes.base import MicroscopeHandler
+
+
+class RareCyteHandler(MicroscopeHandler):
+    name = "rarecyte"
+
+    def restructure_raw(self, image_root: Path):
+        for slide in image_root.iterdir():
+            if not slide.is_dir() or slide.name in ["templates", "microscopes"]:
+                continue
+            raw = slide / "raw"
+            misc = slide / "misc_files"
+            raw.mkdir(exist_ok=True)
+            misc.mkdir(exist_ok=True)
+
+            # Flatten: move all .rcpnl files directly into raw/
+            for path in slide.rglob("*.rcpnl"):
+                if raw in path.parents or misc in path.parents:
+                    continue
+                dest = raw / path.name
+                path.rename(dest)
+
+            # Move all other top-level items to misc_files/
+            for item in list(slide.iterdir()):
+                if item.name not in ["raw", "misc_files"]:
+                    item.rename(misc / item.name)
+
+    def params_template(self) -> dict:
+        return {}

mcmicroprep-0.1.0/mcmicroprep/preparemcmicro.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import argparse
+from pathlib import Path
+from mcmicroprep.core import CorePipeline
+from mcmicroprep.microscopes.olympus import OlympusHandler
+from mcmicroprep.microscopes.rarecyte import RareCyteHandler
+
+HANDLERS = {
+    "olympus": OlympusHandler,
+    "rarecyte": RareCyteHandler,
+    # add more handlers here
+}
+
+
+def main():
+    parser = argparse.ArgumentParser("Prepare data for MCMicro")
+    parser.add_argument(
+        "--microscope",
+        choices=HANDLERS,
+        required=True,
+        help="Microscope vendor (e.g., olympus, rarecyte)",
+    )
+    parser.add_argument(
+        "--image-root", required=True, help="Root folder containing slide directories"
+    )
+    args = parser.parse_args()
+
+    handler = HANDLERS[args.microscope]()
+    pipeline = CorePipeline(handler, Path(args.image_root))
+    pipeline.run()
+
+
+if __name__ == "__main__":
+    main()

mcmicroprep-0.1.0/mcmicroprep/templates/common/base.config

@@ -0,0 +1,28 @@
+process {
+  withName:illumination   {
+    cpus   = 4
+    time   = '12h'
+  }
+  withName:ashlar         {
+    cpus   = 1
+    time   = '36h'
+    queue  = 'medium'
+  }
+   withName: 'segmentation:worker'         {
+    time   = '2h'
+    queue = 'gpu_quad'
+  }
+  withName:s3seg          {
+    cpus   = 4
+    time   = '2h'
+  }
+  withName:mcquant        {
+    cpus   = 1
+    queue  = 'medium'
+    time   = '96h'
+  }
+}
+
+report {
+  enabled = false
+}

mcmicroprep-0.1.0/mcmicroprep/templates/common/batch_submission.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Parse arguments
+OPTS=$(getopt -o ""   --long dataset_path:,mcmicro_template:,markers_csv:,params_yml:   -n 'submission.sh' -- "$@")
+
+if [ $? != 0 ]; then
+  echo "Failed parsing options." >&2
+  exit 1
+fi
+
+eval set -- "$OPTS"
+
+dataset_path=""
+mcmicro_template=""
+markers_csv=""
+params_yml=""
+
+# Extract arguments
+while true; do
+  case "$1" in
+    --dataset_path ) dataset_path="$2"; shift 2 ;;
+    --mcmicro_template ) mcmicro_template="$2"; shift 2 ;;
+    --markers_csv ) markers_csv="$2"; shift 2 ;;
+    --params_yml ) params_yml="$2"; shift 2 ;;
+    -- ) shift; break ;;
+    * ) break ;;
+  esac
+done
+
+# Check dataset_path
+if [[ -z "$dataset_path" || ! -d "$dataset_path" ]]; then
+  echo "Error: dataset_path is required and must exist: $dataset_path"
+  exit 1
+fi
+
+# Assign default paths if optional ones are not provided
+if [[ -z "$mcmicro_template" ]]; then
+  mcmicro_template="${dataset_path%/}/mcmicro_template.sh"
+  echo "No mcmicro_template provided. Using default: $mcmicro_template"
+fi
+
+if [[ -z "$markers_csv" ]]; then
+  markers_csv="${dataset_path%/}/markers.csv"
+  echo "No markers_csv provided. Using default: $markers_csv"
+fi
+
+if [[ -z "$params_yml" ]]; then
+  params_yml="${dataset_path%/}/params.yml"
+  echo "No params_yml provided. Using default: $params_yml"
+fi
+
+# Check that all necessary files exist now
+if [[ ! -f "$mcmicro_template" ]]; then
+  echo "Error: mcmicro_template not found at $mcmicro_template"
+  exit 1
+fi
+
+if [[ ! -f "$markers_csv" ]]; then
+  echo "Error: markers_csv not found at $markers_csv"
+  exit 1
+fi
+
+if [[ ! -f "$params_yml" ]]; then
+  echo "Error: params_yml not found at $params_yml"
+  exit 1
+fi
+
+# Submit jobs for each raw folder
+for raw_folder in "$dataset_path"/raw/*_frames; do
+  if [ -d "$raw_folder" ]; then
+    echo "Submitting job for: $raw_folder"
+    sbatch "$mcmicro_template" "$raw_folder"
+  fi
+done

mcmicroprep-0.1.0/mcmicroprep/templates/common/markers.csv

@@ -0,0 +1,5 @@
+cycle,marker_name
+1,DNA
+1,Marker1
+1,Marker2
+1,Marker3

mcmicroprep-0.1.0/mcmicroprep/templates/common/mcmicro_template.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#SBATCH -p short
+#SBATCH -J nextflow_O2
+#SBATCH -t 0-12:00
+#SBATCH --mem=1G
+#SBATCH --mail-type=END
+
+SAMPLEDIR=$1
+SAMPLEID=$(basename $SAMPLEDIR)
+
+module purge
+module load java
+
+/n/groups/lsp/mcmicro/tools/o2/config_pre_reg.sh -s -u $SAMPLEDIR > $SAMPLEDIR/memory.config
+
+nextflow run labsyspharm/mcmicro -profile O2,WSI,GPU --in $SAMPLEDIR -w /n/scratch/users/${USER:0:1}/$USER/work -c $(dirname "$SAMPLEDIR")/base.config -c $SAMPLEDIR/memory.config -publish_dir_mode link

mcmicroprep-0.1.0/mcmicroprep/templates/params/olympus.yml

@@ -0,0 +1,18 @@
+workflow:
+  start-at: illumination
+  stop-at: quantification
+  background: false
+  segmentation-channel: 1 1
+  multi-formats: .ome
+  tma: false
+options:
+  ashlar: --flip-y --filter-sigma 1
+  unmicst: --channel 1 --scalingFactor 0.5 --tool unmicst-duo --outlier 99.99
+  s3seg: --maxima-footprint-size 15 --area-max 50000 --expand-size 6 --pixelSize 0.325 --mean-intensity-min 80
+  mcquant: --masks nucleiRing.ome.tif cellRing.ome.tif cytoRing.ome.tif
+  coreograph: --channel 1
+modules:
+  watershed:
+    name: s3seg
+    container: labsyspharm/s3segmenter
+    version: 1.5.5-large

mcmicroprep-0.1.0/mcmicroprep/templates/params/rarecyte.yml

@@ -0,0 +1,16 @@
+workflow:
+  start-at: illumination
+  stop-at: quantification
+  background: false
+  segmentation-channel: 1 1
+  tma: false
+options:
+  unmicst: --channel 1 --scalingFactor 0.5 --tool unmicst-duo --outlier 99.99
+  s3seg: --maxima-footprint-size 15 --area-max 50000 --expand-size 6 --pixelSize 0.325 --mean-intensity-min 80
+  mcquant: --masks nucleiRing.ome.tif cellRing.ome.tif cytoRing.ome.tif
+  coreograph: --channel 1
+modules:
+  watershed:
+    name: s3seg
+    container: labsyspharm/s3segmenter
+    version: 1.5.5-large

mcmicroprep-0.1.0/mcmicroprep/templates/params_base.yml

@@ -0,0 +1,10 @@
+# Base parameters for MCMicro workflow
+illumination_correction:
+  method: default
+segmentation:
+  method: default
+markers:
+  # cycle: marker_name
+  # e.g., 1: DAPI
+channels:
+  # channel-specific settings go here

mcmicroprep-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "mcmicroprep"
+version = "0.1.0"
+description = ""
+authors = ["Ajit Johnson Nirmal <ajitjohnson.n@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+lxml = "^6.0.0"
+ome-types = "^0.6.1"
+pandas = "^2.3.1"
+pydantic = "^2.11.7"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+preparemcmicro = "mcmicroprep.preparemcmicro:main"