cellarr 0.3.2__tar.gz → 0.4.1.dev1__tar.gz

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/.github/workflows/pypi-publish.yml

@@ -14,10 +14,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.9
+    - name: Set up Python 3.11
       uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.11
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/.github/workflows/pypi-test.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
+        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
 
     name: Python ${{ matrix.python-version }}
     steps:

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: '^docs/conf.py'
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
   - id: trailing-whitespace
   - id: check-added-large-files
@@ -33,7 +33,7 @@ repos:
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.6.8
+  rev: v0.8.2
   hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Version 0.4.0
+
+- chore: Remove Python 3.8 (EOL).
+- precommit: Replace docformatter with ruff's formatter.
+
 ## Version 0.3.2
 
 - Functionality to iterate over samples and cells.

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cellarr
-Version: 0.3.2
+Version: 0.4.1.dev1
 Summary: TileDB-based array storage for genomics data collections.
 Home-page: https://github.com/BiocPy/cellarr
 Author: Jayaram Kancherla
@@ -10,7 +10,7 @@ Project-URL: Documentation, https://github.com/BiocPy/cellarr
 Platform: any
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 License-File: LICENSE.txt
 Requires-Dist: importlib-metadata; python_version < "3.8"

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/setup.cfg

@@ -21,7 +21,7 @@ packages = find_namespace:
 include_package_data = True
 package_dir = 
 	=src
-python_requires = >=3.8
+python_requires = >=3.9
 install_requires = 
 	importlib-metadata; python_version<"3.8"
 	anndata

cellarr-0.4.1.dev1/src/cellarr/slurm/README.md

@@ -0,0 +1,58 @@
+
+# manifest json
+
+```json
+{
+    "files": [
+        "/path/to/dataset1.h5ad",
+        "/path/to/dataset2.h5ad"
+    ],
+    "matrix_options": [
+        {
+            "matrix_name": "counts",
+            "dtype": "uint32"
+        },
+        {
+            "matrix_name": "normalized",
+            "dtype": "float32"
+        }
+    ],
+    "gene_options": {
+        "feature_column": "index"
+    },
+    "sample_options": {
+        "metadata": {
+            "sample_1": {
+                "condition": "control",
+                "batch": "1"
+            },
+            "sample_2": {
+                "condition": "treatment",
+                "batch": "1"
+            }
+        }
+    },
+    "cell_options": {
+        "column_types": {
+            "cell_type": "ascii",
+            "quality_score": "float32"
+        },
+        "metadata": {
+            "processing_date": ["2024-01-01", "2024-01-02", ...]
+        }
+    }
+}
+```
+
+
+Run
+
+```sh
+
+python build_cellarr_steps.py \
+    --input-manifest manifest.json \
+    --output-dir /path/to/output \
+    --memory-per-job 64 \
+    --cpus-per-task 4
+
+```

cellarr-0.4.1.dev1/src/cellarr/slurm/build_cellarr_steps.py

@@ -0,0 +1,287 @@
+import argparse
+import json
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import anndata
+import numpy as np
+
+from cellarr.buildutils_tiledb_array import create_tiledb_array
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+class SlurmBuilder:
+    """SLURM-based builder for CellArrDataset."""
+
+    def __init__(
+        self,
+        output_dir: str,
+        log_dir: str,
+        temp_dir: str,
+        memory_gb: int = 64,
+        time_hours: int = 24,
+        cpus_per_task: int = 4,
+    ):
+        """Initialize the SLURM builder.
+
+        Args:
+            output_dir:
+                Path to final output directory.
+
+            log_dir:
+                Path to store SLURM logs.
+
+            temp_dir:
+                Path for temporary files.
+
+            memory_gb:
+                Memory per job in GB.
+
+            time_hours:
+                Time limit per job in hours.
+
+            cpus_per_task:
+                CPUs per task.
+        """
+        self.output_dir = Path(output_dir)
+        self.log_dir = Path(log_dir)
+        self.temp_dir = Path(temp_dir)
+        self.memory_gb = memory_gb
+        self.time_hours = time_hours
+        self.cpus_per_task = cpus_per_task
+
+    def create_slurm_script(
+        self, job_name: str, python_script: str, args: Dict, dependencies: Optional[str] = None
+    ) -> str:
+        """Create a SLURM job submission script."""
+        script = f"""#!/bin/bash
+#SBATCH --job-name={job_name}
+#SBATCH --output={self.log_dir}/{job_name}_%j.out
+#SBATCH --error={self.log_dir}/{job_name}_%j.err
+#SBATCH --time={self.time_hours}:00:00
+#SBATCH --mem={self.memory_gb}G
+#SBATCH --cpus-per-task={self.cpus_per_task}
+"""
+        if dependencies:
+            script += f"#SBATCH --dependency={dependencies}\n"
+
+        script += f"""
+source ~/.bashrc
+conda activate cellarr_env  # MODIFY AS NEEDED
+
+python {python_script} '{json.dumps(args)}'
+"""
+        script_path = self.log_dir / f"{job_name}_submit.sh"
+        with open(script_path, "w") as f:
+            f.write(script)
+        return script_path
+
+    def create_array_script(
+        self, job_name: str, python_script: str, args: Dict, n_tasks: int, dependencies: Optional[str] = None
+    ) -> str:
+        """Create a SLURM array job submission script."""
+        script = f"""#!/bin/bash
+#SBATCH --job-name={job_name}
+#SBATCH --output={self.log_dir}/{job_name}_%A_%a.out
+#SBATCH --error={self.log_dir}/{job_name}_%A_%a.err
+#SBATCH --time={self.time_hours}:00:00
+#SBATCH --mem={self.memory_gb}G
+#SBATCH --cpus-per-task={self.cpus_per_task}
+#SBATCH --array=0-{n_tasks-1}
+"""
+        if dependencies:
+            script += f"#SBATCH --dependency={dependencies}\n"
+
+        script += f"""
+source ~/.bashrc
+conda activate cellarr_env # MODIFY AS NEEDED
+
+python {python_script} '{json.dumps(args)}'
+"""
+        script_path = self.log_dir / f"{job_name}_array.sh"
+        with open(script_path, "w") as f:
+            f.write(script)
+        return script_path
+
+    def submit_job(self, script_path: str) -> str:
+        """Submit a SLURM job and return job ID."""
+        result = subprocess.run(["sbatch", script_path], capture_output=True, text=True)
+        return result.stdout.strip().split()[-1]
+
+    def submit_gene_annotation_job(self, files: List[str], gene_options: Dict) -> str:
+        """Submit gene annotation processing job."""
+        args = {
+            "files": files,
+            "output_dir": str(self.output_dir),
+            "gene_options": gene_options,
+            "temp_dir": str(self.temp_dir / "gene_annotation"),
+        }
+
+        script_path = self.create_slurm_script(
+            job_name="cellarr_gene_annot", python_script="process_gene_annotation.py", args=args
+        )
+        return self.submit_job(script_path)
+
+    def submit_sample_metadata_job(self, files: List[str], sample_options: Dict, dependency: str) -> str:
+        """Submit sample metadata processing job."""
+        args = {
+            "files": files,
+            "output_dir": str(self.output_dir),
+            "sample_options": sample_options,
+            "temp_dir": str(self.temp_dir / "sample_metadata"),
+        }
+
+        script_path = self.create_slurm_script(
+            job_name="cellarr_sample_meta",
+            python_script="process_sample_metadata.py",
+            args=args,
+            # dependencies=f"afterok:{dependency}",
+        )
+        return self.submit_job(script_path)
+
+    def submit_cell_metadata_job(self, files: List[str], cell_options: Dict, dependency: str) -> str:
+        """Submit cell metadata processing job."""
+        args = {
+            "files": files,
+            "output_dir": str(self.output_dir),
+            "cell_options": cell_options,
+            "temp_dir": str(self.temp_dir / "cell_metadata"),
+        }
+
+        script_path = self.create_slurm_script(
+            job_name="cellarr_cell_meta",
+            python_script="process_cell_metadata.py",
+            args=args,
+            # dependencies=f"afterok:{dependency}",
+        )
+        return self.submit_job(script_path)
+
+    def submit_matrix_processing(self, files: List[str], matrix_options: Dict, dependency: str) -> Tuple[str, str]:
+        """Submit matrix processing as SLURM array job."""
+        # Calculate cell offsets for each file
+        offset = 0
+        file_infos = []
+        for file in files:
+            file_infos.append({"file": file, "offset": offset})
+            adata = anndata.read_h5ad(file, backed="r")
+            offset += adata.n_obs
+
+        # Create matrix TileDB array
+        matrix_uri = str(self.output_dir / "assays" / matrix_options["matrix_name"])
+        create_tiledb_array(
+            matrix_uri,
+            matrix_attr_name=matrix_options.get("matrix_attr_name", "data"),
+            matrix_dim_dtype=np.dtype(matrix_options.get("dtype", "float32")),
+        )
+
+        # Prepare array job arguments
+        array_args = {
+            "output_dir": str(self.output_dir),
+            "temp_dir": str(self.temp_dir / f"matrix_{matrix_options['matrix_name']}"),
+            "matrix_options": matrix_options,
+            "gene_annotation_file": str(self.temp_dir / "gene_annotation/gene_set.json"),
+            "file_infos": file_infos,
+        }
+
+        # Submit array job
+        array_script = self.create_array_script(
+            job_name=f"matrix_{matrix_options['matrix_name']}",
+            python_script="process_matrix.py",
+            args=array_args,
+            n_tasks=len(files),
+            dependencies=f"afterok:{dependency}",
+        )
+        array_job_id = self.submit_job(array_script)
+
+        # Submit finalization job
+        final_args = {
+            "output_dir": str(self.output_dir),
+            "temp_dir": str(self.temp_dir / f"matrix_{matrix_options['matrix_name']}"),
+            "matrix_options": matrix_options,
+            "file_infos": file_infos,
+        }
+
+        final_script = self.create_slurm_script(
+            job_name=f"finalize_matrix_{matrix_options['matrix_name']}",
+            python_script="finalize_matrix.py",
+            args=final_args,
+            dependencies=f"afterok:{array_job_id}",
+        )
+        final_job_id = self.submit_job(final_script)
+
+        return array_job_id, final_job_id
+
+    def submit_final_assembly(self, matrix_names: List[str], dependencies: List[str]) -> str:
+        """Submit final assembly job."""
+        args = {"input_dir": str(self.output_dir), "output_dir": str(self.output_dir), "matrix_names": matrix_names}
+
+        script_path = self.create_slurm_script(
+            job_name="cellarr_final_assembly",
+            python_script="final_assembly.py",
+            args=args,
+            dependencies=f"afterok:{','.join(dependencies)}",
+        )
+        return self.submit_job(script_path)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build CellArrDataset using SLURM steps")
+    parser.add_argument("--input-manifest", required=True, help="Path to JSON manifest file")
+    parser.add_argument("--output-dir", required=True, help="Base output directory")
+    parser.add_argument("--memory-per-job", type=int, default=64, help="Memory in GB per job")
+    parser.add_argument("--time-per-job", type=int, default=24, help="Time in hours per job")
+    parser.add_argument("--cpus-per-task", type=int, default=4, help="CPUs per task")
+    args = parser.parse_args()
+
+    # Create directories
+    base_dir = Path(args.output_dir)
+    log_dir = base_dir / "logs"
+    temp_dir = base_dir / "temp"
+    final_dir = base_dir / "final"
+
+    for d in [log_dir, temp_dir, final_dir]:
+        d.mkdir(parents=True, exist_ok=True)
+
+    # Read manifest
+    with open(args.input_manifest) as f:
+        manifest = json.load(f)
+
+    # Initialize builder
+    builder = SlurmBuilder(
+        output_dir=str(final_dir),
+        log_dir=str(log_dir),
+        temp_dir=str(temp_dir),
+        memory_gb=args.memory_per_job,
+        time_hours=args.time_per_job,
+        cpus_per_task=args.cpus_per_task,
+    )
+
+    # Submit jobs
+    gene_job_id = builder.submit_gene_annotation_job(manifest["files"], manifest.get("gene_options", {}))
+
+    sample_job_id = builder.submit_sample_metadata_job(
+        manifest["files"], manifest.get("sample_options", {}), gene_job_id
+    )
+
+    cell_job_id = builder.submit_cell_metadata_job(manifest["files"], manifest.get("cell_options", {}), sample_job_id)
+
+    # Process matrices
+    matrix_options = manifest.get("matrix_options", [{"matrix_name": "counts"}])
+    if not isinstance(matrix_options, list):
+        matrix_options = [matrix_options]
+
+    matrix_job_ids = []
+    for matrix_opt in matrix_options:
+        _, final_id = builder.submit_matrix_processing(manifest["files"], matrix_opt, f"{cell_job_id},{gene_job_id}")
+        matrix_job_ids.append(final_id)
+
+    # Submit final assembly
+    builder.submit_final_assembly([opt["matrix_name"] for opt in matrix_options], matrix_job_ids)
+
+
+if __name__ == "__main__":
+    main()

cellarr-0.4.1.dev1/src/cellarr/slurm/build_cellarr_steps_bkp.py

@@ -0,0 +1,185 @@
+import argparse
+import json
+import os
+import subprocess
+from pathlib import Path
+from typing import Dict, Optional
+
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def create_slurm_script(
+    job_name: str,
+    log_dir: str,
+    python_script: str,
+    args: Dict,
+    memory_gb: int = 64,
+    time_hours: int = 24,
+    cpus_per_task: int = 4,
+    dependencies: Optional[str] = None,
+) -> str:
+    """Create SLURM job submission script."""
+    script = f"""#!/bin/bash
+#SBATCH --job-name={job_name}
+#SBATCH --output={log_dir}/{job_name}_%j.out
+#SBATCH --error={log_dir}/{job_name}_%j.err
+#SBATCH --time={time_hours}:00:00
+#SBATCH --mem={memory_gb}G
+#SBATCH --cpus-per-task={cpus_per_task}
+"""
+    if dependencies:
+        script += f"#SBATCH --dependency={dependencies}\n"
+
+    script += """
+source ~/.bashrc
+conda activate cellarr_env  # Modify as needed for your environment
+
+python {python_script} '{json.dumps(args)}'
+"""
+    script_path = os.path.join(log_dir, f"{job_name}_submit.sh")
+    with open(script_path, "w") as f:
+        f.write(script)
+    return script_path
+
+
+def submit_slurm_job(script_path: str, dependency: Optional[str] = None) -> str:
+    """Submit SLURM job and return job ID."""
+    cmd = ["sbatch"]
+    if dependency:
+        cmd.extend(["--dependency", dependency])
+    cmd.append(script_path)
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    job_id = result.stdout.strip().split()[-1]
+    return job_id
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Build CellArrDataset using SLURM steps")
+    parser.add_argument("--input-manifest", required=True, help="Path to JSON manifest file")
+    parser.add_argument("--output-dir", required=True, help="Base output directory")
+    parser.add_argument("--memory-per-job", type=int, default=64, help="Memory in GB per job")
+    parser.add_argument("--time-per-job", type=int, default=24, help="Time in hours per job")
+    parser.add_argument("--cpus-per-task", type=int, default=4, help="CPUs per task")
+    args = parser.parse_args()
+
+    # Create output directories
+    base_dir = Path(args.output_dir)
+    log_dir = base_dir / "logs"
+    temp_dir = base_dir / "temp"
+    final_dir = base_dir / "final"
+
+    for d in [log_dir, temp_dir, final_dir]:
+        d.mkdir(parents=True, exist_ok=True)
+
+    # Read manifest file
+    with open(args.input_manifest) as f:
+        manifest = json.load(f)
+
+    # Step 1: Gene Annotation Job
+    gene_args = {
+        "files": manifest["files"],
+        "output_dir": str(final_dir),
+        "gene_options": manifest.get("gene_options", {}),
+        "temp_dir": str(temp_dir / "gene_annotation"),
+    }
+
+    gene_script = create_slurm_script(
+        job_name="cellarr_gene_annot",
+        log_dir=str(log_dir),
+        python_script="process_gene_annotation.py",
+        args=gene_args,
+        memory_gb=args.memory_per_job,
+        cpus_per_task=args.cpus_per_task,
+    )
+    gene_job_id = submit_slurm_job(gene_script)
+
+    # Step 2: Sample Metadata Job
+    sample_args = {
+        "files": manifest["files"],
+        "output_dir": str(final_dir),
+        "sample_options": manifest.get("sample_options", {}),
+        "temp_dir": str(temp_dir / "sample_metadata"),
+    }
+
+    sample_script = create_slurm_script(
+        job_name="cellarr_sample_meta",
+        log_dir=str(log_dir),
+        python_script="process_sample_metadata.py",
+        args=sample_args,
+        memory_gb=args.memory_per_job,
+        cpus_per_task=args.cpus_per_task,
+        # dependencies=f"afterok:{gene_job_id}",
+    )
+    sample_job_id = submit_slurm_job(sample_script)
+
+    # Step 3: Cell Metadata Job
+    cell_args = {
+        "files": manifest["files"],
+        "output_dir": str(final_dir),
+        "cell_options": manifest.get("cell_options", {}),
+        "temp_dir": str(temp_dir / "cell_metadata"),
+    }
+
+    cell_script = create_slurm_script(
+        job_name="cellarr_cell_meta",
+        log_dir=str(log_dir),
+        python_script="process_cell_metadata.py",
+        args=cell_args,
+        memory_gb=args.memory_per_job,
+        cpus_per_task=args.cpus_per_task,
+        # dependencies=f"afterok:{sample_job_id}",
+    )
+    cell_job_id = submit_slurm_job(cell_script)
+
+    # Step 4: Matrix Jobs (one per matrix type)
+    matrix_job_ids = []
+    matrix_options = manifest.get("matrix_options", [{"matrix_name": "counts"}])
+    if not isinstance(matrix_options, list):
+        matrix_options = [matrix_options]
+
+    for matrix_opt in matrix_options:
+        matrix_args = {
+            "files": manifest["files"],
+            "output_dir": str(final_dir),
+            "matrix_options": matrix_opt,
+            "gene_annotation_file": str(temp_dir / "gene_annotation/gene_set.json"),
+            "temp_dir": str(temp_dir / f"matrix_{matrix_opt['matrix_name']}"),
+        }
+
+        matrix_script = create_slurm_script(
+            job_name=f"cellarr_matrix_{matrix_opt['matrix_name']}",
+            log_dir=str(log_dir),
+            python_script="process_matrix.py",
+            args=matrix_args,
+            memory_gb=args.memory_per_job * 2,  # More memory for matrix processing
+            cpus_per_task=args.cpus_per_task,
+            dependencies=f"afterok:{cell_job_id},{gene_job_id}",
+        )
+        job_id = submit_slurm_job(matrix_script)
+        matrix_job_ids.append(job_id)
+
+    # Step 5: Final Assembly Job
+    final_args = {
+        "input_dir": str(final_dir),
+        "output_dir": str(final_dir),
+        "matrix_names": [opt["matrix_name"] for opt in matrix_options],
+    }
+
+    final_script = create_slurm_script(
+        job_name="cellarr_final_assembly",
+        log_dir=str(log_dir),
+        python_script="final_assembly.py",
+        args=final_args,
+        memory_gb=args.memory_per_job,
+        cpus_per_task=args.cpus_per_task,
+        dependencies=f"afterok:{','.join(matrix_job_ids)}",
+    )
+    submit_slurm_job(final_script)
+
+
+if __name__ == "__main__":
+    main()

cellarr-0.4.1.dev1/src/cellarr/slurm/final_assembly.py

@@ -0,0 +1,20 @@
+import json
+
+from cellarr import CellArrDataset
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def final_assembly(args_json: str):
+    args = json.loads(args_json)
+
+    # Perform any final optimizations or validations
+    dataset = CellArrDataset(dataset_path=args["output_dir"], assay_uri=args["matrix_names"])
+
+    # Save final metadata
+    metadata = {"shape": dataset.shape, "matrices": args["matrix_names"]}
+
+    with open(f"{args['output_dir']}/metadata.json", "w") as f:
+        json.dump(metadata, f)

cellarr-0.4.1.dev1/src/cellarr/slurm/finalize_matrix.py

@@ -0,0 +1,45 @@
+import json
+import sys
+from pathlib import Path
+
+from cellarr.buildutils_tiledb_array import optimize_tiledb_array
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def finalize_matrix(args_json: str):
+    """Finalize the matrix after all array jobs complete."""
+    args = json.loads(args_json)
+
+    # Verify all tasks completed
+    completed_dir = Path(args["temp_dir"]) / "completed"
+    expected_tasks = len(args["file_infos"])
+    completed_tasks = len(list(completed_dir.glob("task_*.json")))
+
+    if completed_tasks != expected_tasks:
+        raise RuntimeError(f"Expected {expected_tasks} tasks but only {completed_tasks} completed")
+
+    # Optimize the TileDB array
+    matrix_uri = f"{args['output_dir']}/assays/{args['matrix_options']['matrix_name']}"
+    optimize_tiledb_array(matrix_uri)
+
+    # Save completion metadata
+    with open(f"{args['temp_dir']}/matrix_metadata.json", "w") as f:
+        json.dump(
+            {
+                "matrix_name": args["matrix_options"]["matrix_name"],
+                "files_processed": len(args["file_infos"]),
+                "uri": matrix_uri,
+            },
+            f,
+        )
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python finalize_matrix.py '<json_args>'")
+        sys.exit(1)
+
+    finalize_matrix(sys.argv[1])

cellarr-0.4.1.dev1/src/cellarr/slurm/process_cell_metadata.py

@@ -0,0 +1,68 @@
+import json
+import sys
+
+import pandas as pd
+
+from cellarr import utils_anndata as uad
+from cellarr.buildutils_tiledb_frame import (
+    create_tiledb_frame_from_dataframe,
+)
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def process_cell_metadata(args_json: str):
+    """Process and create cell metadata store.
+
+    Creates cell metadata including:
+    - Sample mapping
+    - Cell indices within samples
+    - Original cell annotations from input files
+    """
+    args = json.loads(args_json)
+
+    # Extract cell metadata with specific column subset if provided
+    cell_meta_columns = args.get("cell_options", {}).get("column_types", {})
+    files_cache = uad.extract_anndata_info(
+        args["files"], obs_subset_columns=list(cell_meta_columns.keys()) if cell_meta_columns else None
+    )
+
+    # Get cell counts
+    cell_counts = uad.scan_for_cellcounts(files_cache)
+
+    # Create sample mapping for each cell
+    sample_per_cell = []
+    cell_index_in_sample = []
+    sample_names = [f"sample_{idx+1}" for idx in range(len(args["files"]))]
+
+    for idx, count in enumerate(cell_counts):
+        sample_per_cell.extend([sample_names[idx]] * count)
+        cell_index_in_sample.extend(range(count))
+
+    # Create base cell metadata
+    cell_metadata = pd.DataFrame(
+        {"cellarr_sample": sample_per_cell, "cellarr_cell_index_in_sample": cell_index_in_sample}
+    )
+
+    # Add original cell annotations from input files
+    if cell_meta_columns:
+        original_meta = uad.scan_for_cellmetadata(files_cache)
+        if not original_meta.empty:
+            # Ensure index alignment
+            original_meta.reset_index(drop=True, inplace=True)
+            cell_metadata = pd.concat([cell_metadata, original_meta], axis=1)
+
+    # Create TileDB store
+    create_tiledb_frame_from_dataframe(
+        f"{args['output_dir']}/cell_metadata", cell_metadata, column_types=cell_meta_columns
+    )
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python process_cell_metadata.py '<json_args>'")
+        sys.exit(1)
+
+    process_cell_metadata(sys.argv[1])

cellarr-0.4.1.dev1/src/cellarr/slurm/process_gene_annotation.py

@@ -0,0 +1,33 @@
+import json
+
+import pandas as pd
+
+from cellarr import utils_anndata as uad
+from cellarr.buildutils_tiledb_frame import create_tiledb_frame_from_dataframe
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def process_gene_annotation(args_json: str):
+    args = json.loads(args_json)
+
+    # Extract gene information from all files
+    files_cache = uad.extract_anndata_info(
+        args["files"], var_feature_column=args["gene_options"].get("feature_column", "index")
+    )
+
+    # Scan for features
+    gene_set = uad.scan_for_features(files_cache)
+    gene_set = sorted(gene_set)
+
+    # Create gene annotation dataframe
+    gene_annotation = pd.DataFrame({"cellarr_gene_index": gene_set}, index=gene_set)
+
+    # Save gene set for later use
+    with open(f"{args['temp_dir']}/gene_set.json", "w") as f:
+        json.dump(gene_set, f)
+
+    # Create TileDB store
+    create_tiledb_frame_from_dataframe(f"{args['output_dir']}/gene_annotation", gene_annotation)

cellarr-0.4.1.dev1/src/cellarr/slurm/process_matrix.py

@@ -0,0 +1,50 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+from cellarr import utils_anndata as uad
+from cellarr.buildutils_tiledb_array import write_csr_matrix_to_tiledb
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def process_matrix_file(args_json: str):
+    """Process a single file for matrix creation."""
+    args = json.loads(args_json)
+
+    # Get SLURM array task ID
+    task_id = int(os.environ.get("SLURM_ARRAY_TASK_ID", 0))
+
+    # Load gene set mapping
+    with open(args["gene_annotation_file"]) as f:
+        gene_set = json.load(f)
+    gene_map = {gene: idx for idx, gene in enumerate(gene_set)}
+
+    # Get file and offset for this task
+    file_info = args["file_infos"][task_id]
+    input_file = file_info["file"]
+    row_offset = file_info["offset"]
+
+    # Process the file
+    matrix = uad.remap_anndata(input_file, gene_map, layer_matrix_name=args["matrix_options"]["matrix_name"])
+
+    # Write to TileDB
+    matrix_uri = f"{args['output_dir']}/assays/{args['matrix_options']['matrix_name']}"
+    write_csr_matrix_to_tiledb(matrix_uri, matrix[args["matrix_options"]["matrix_name"]], row_offset=row_offset)
+
+    # Save task completion marker
+    task_marker = Path(args["temp_dir"]) / "completed" / f"task_{task_id}.json"
+    task_marker.parent.mkdir(exist_ok=True)
+    with open(task_marker, "w") as f:
+        json.dump({"file": input_file, "cells_processed": matrix[args["matrix_options"]["matrix_name"]].shape[0]}, f)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python process_matrix.py '<json_args>'")
+        sys.exit(1)
+
+    process_matrix_file(sys.argv[1])

cellarr-0.4.1.dev1/src/cellarr/slurm/process_matrix_all.py

@@ -0,0 +1,40 @@
+import json
+
+import numpy as np
+
+from cellarr import utils_anndata as uad
+from cellarr.buildutils_tiledb_array import create_tiledb_array, write_csr_matrix_to_tiledb
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def process_matrix(args_json: str):
+    args = json.loads(args_json)
+
+    # Load gene set
+    with open(args["gene_annotation_file"]) as f:
+        gene_set = json.load(f)
+
+    # Create gene set mapping
+    gene_map = {gene: idx for idx, gene in enumerate(gene_set)}
+
+    # Create TileDB array
+    create_tiledb_array(
+        f"{args['output_dir']}/assays/{args['matrix_options']['matrix_name']}",
+        matrix_dim_dtype=np.dtype(args["matrix_options"].get("dtype", "float32")),
+    )
+
+    # Process each file
+    offset = 0
+    for file in args["files"]:
+        matrix = uad.remap_anndata(file, gene_map, layer_matrix_name=args["matrix_options"]["matrix_name"])
+
+        write_csr_matrix_to_tiledb(
+            f"{args['output_dir']}/assays/{args['matrix_options']['matrix_name']}",
+            matrix[args["matrix_options"]["matrix_name"]],
+            row_offset=offset,
+        )
+
+        offset += matrix[args["matrix_options"]["matrix_name"]].shape[0]

cellarr-0.4.1.dev1/src/cellarr/slurm/process_sample_metadata.py

@@ -0,0 +1,76 @@
+import json
+import sys
+
+import pandas as pd
+
+from cellarr import utils_anndata as uad
+from cellarr.buildutils_tiledb_frame import create_tiledb_frame_from_dataframe
+
+__author__ = "Jayaram Kancherla"
+__copyright__ = "Jayaram Kancherla"
+__license__ = "MIT"
+
+
+def process_sample_metadata(args_json: str):
+    """Process and create sample metadata store.
+
+    Creates sample metadata including:
+    - Basic sample information
+    - Cell counts per sample
+    - Original gene sets
+    - Sample index information
+    """
+    args = json.loads(args_json)
+
+    # Extract information from files
+    files_cache = uad.extract_anndata_info(args["files"], var_feature_column=args.get("feature_column", "index"))
+
+    # Get cell counts for each sample
+    cell_counts = uad.scan_for_cellcounts(files_cache)
+
+    # Create basic sample metadata
+    sample_names = [f"sample_{idx+1}" for idx in range(len(args["files"]))]
+    sample_metadata = pd.DataFrame(
+        {"cellarr_sample": sample_names, "cellarr_cell_counts": cell_counts, "cellarr_filename": args["files"]}
+    )
+
+    # Add sample indices for efficient slicing
+    counter = sample_metadata["cellarr_cell_counts"].shift(1)
+    counter.iloc[0] = 0
+    sample_metadata["cellarr_sample_start_index"] = counter.cumsum().astype(int)
+
+    # Calculate end indices
+    ends = sample_metadata["cellarr_sample_start_index"].shift(-1)
+    ends.iloc[-1] = int(sample_metadata["cellarr_cell_counts"].sum())
+    ends = ends - 1
+    sample_metadata["cellarr_sample_end_index"] = ends.astype(int)
+
+    # Add original gene sets for each sample
+    gene_sets = uad.scan_for_features(files_cache, unique=False)
+    sample_metadata["cellarr_original_gene_set"] = [",".join(genes) for genes in gene_sets]
+
+    # Add any custom metadata if provided in options
+    custom_metadata = args.get("sample_options", {}).get("metadata", {})
+    for sample, metadata in custom_metadata.items():
+        for key, value in metadata.items():
+            if key not in sample_metadata.columns:
+                sample_metadata[key] = None
+            sample_idx = sample_metadata.index[sample_metadata["cellarr_sample"] == sample][0]
+            sample_metadata.at[sample_idx, key] = value
+
+    # Create TileDB store
+    create_tiledb_frame_from_dataframe(f"{args['output_dir']}/sample_metadata", sample_metadata)
+
+    # Save metadata for subsequent steps
+    metadata = {"num_samples": len(sample_names), "total_cells": int(sample_metadata["cellarr_cell_counts"].sum())}
+
+    with open(f"{args['temp_dir']}/sample_metadata.json", "w") as f:
+        json.dump(metadata, f)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python process_sample_metadata.py '<json_args>'")
+        sys.exit(1)
+
+    process_sample_metadata(sys.argv[1])

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/src/cellarr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cellarr
-Version: 0.3.2
+Version: 0.4.1.dev1
 Summary: TileDB-based array storage for genomics data collections.
 Home-page: https://github.com/BiocPy/cellarr
 Author: Jayaram Kancherla
@@ -10,7 +10,7 @@ Project-URL: Documentation, https://github.com/BiocPy/cellarr
 Platform: any
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 License-File: LICENSE.txt
 Requires-Dist: importlib-metadata; python_version < "3.8"

{cellarr-0.3.2 → cellarr-0.4.1.dev1}/src/cellarr.egg-info/SOURCES.txt

@@ -45,6 +45,17 @@ src/cellarr.egg-info/dependency_links.txt
 src/cellarr.egg-info/not-zip-safe
 src/cellarr.egg-info/requires.txt
 src/cellarr.egg-info/top_level.txt
+src/cellarr/slurm/README.md
+src/cellarr/slurm/__init__.py
+src/cellarr/slurm/build_cellarr_steps.py
+src/cellarr/slurm/build_cellarr_steps_bkp.py
+src/cellarr/slurm/final_assembly.py
+src/cellarr/slurm/finalize_matrix.py
+src/cellarr/slurm/process_cell_metadata.py
+src/cellarr/slurm/process_gene_annotation.py
+src/cellarr/slurm/process_matrix.py
+src/cellarr/slurm/process_matrix_all.py
+src/cellarr/slurm/process_sample_metadata.py
 tests/conftest.py
 tests/test_anndata_utils.py
 tests/test_build.py