crispyx 0.0.1__tar.gz

crispyx-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Du Jinhong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crispyx-0.0.1/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: crispyx
+Version: 0.0.1
+Summary: Tools for scalable CRISPR screen analysis on on-disk AnnData objects
+Author: Streamlining CRISPR Team
+License-File: LICENSE
+Requires-Dist: anndata>=0.9
+Requires-Dist: numpy>=1.23
+Requires-Dist: numba>=0.59
+Requires-Dist: pandas>=1.5
+Requires-Dist: scipy>=1.10
+Requires-Dist: h5py>=3.0
+Requires-Dist: joblib>=1.0
+Requires-Dist: scikit-learn>=1.0
+Requires-Dist: scanpy>=1.9.2
+Requires-Dist: seaborn>=0.12
+Requires-Dist: matplotlib>=3.5
+Requires-Dist: tqdm>=4.50
+Provides-Extra: test
+Requires-Dist: filelock; extra == "test"
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: statsmodels>=0.14; extra == "test"
+Requires-Dist: pydeseq2>=0.4; extra == "test"
+Provides-Extra: benchmark
+Requires-Dist: pertpy>=0.4; extra == "benchmark"
+Requires-Dist: pyyaml>=6.0; extra == "benchmark"
+Requires-Dist: tqdm>=4.65; extra == "benchmark"
+Requires-Dist: psutil>=5.9; extra == "benchmark"
+Provides-Extra: docs
+Requires-Dist: sphinx>=6.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=1.2; extra == "docs"
+Dynamic: license-file

crispyx-0.0.1/README.md

@@ -0,0 +1,75 @@
+# crispyx
+
+## Motivation
+
+Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, which can require 30–100+ GB of RAM and makes many screens impractical to analyse on commodity hardware or shared HPC nodes with per-job memory limits.
+
+**crispyx** solves this by streaming data directly from on-disk AnnData (`.h5ad`) files. Quality control, normalisation, pseudo-bulk aggregation, and differential expression all operate without materialising the full matrix in memory, so even the largest screens can be processed with modest resources.
+
+## Features
+
+- **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; all without loading the full matrix into memory
+- **Pseudo-bulk aggregation** – Average log expression and pseudo-bulk count matrices for effect size estimation
+- **Differential expression** – t-test, Wilcoxon rank-sum, and negative binomial GLM with apeGLM LFC shrinkage; multi-core support and adaptive memory management
+- **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
+- **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
+- **Data preparation utilities** – Edit backed metadata without loading X; standardise gene names; normalise perturbation labels; auto-detect metadata columns
+- **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`; Docker and Singularity support
+
+## Quick Start
+
+```python
+import crispyx as cx
+
+# Open dataset without loading into memory
+adata = cx.read_h5ad_ondisk("data/demo_benchmark.h5ad")
+
+# Quality control with adaptive thresholds
+adata = cx.pp.qc_summary(
+    adata,
+    perturbation_column="perturbation",
+    min_genes=5,
+    min_cells_per_perturbation=5,
+)
+
+# Differential expression
+adata = cx.tl.rank_genes_groups(
+    adata,
+    perturbation_column="perturbation",
+    method="wilcoxon",  # or "t-test", "nb_glm"
+)
+
+# Access results
+print(adata.uns["rank_genes_groups"])
+de_results = adata.uns["rank_genes_groups"].load()
+```
+
+For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, plotting, data preparation utilities), see the [Usage Guide](docs/usage.rst) and the [tutorial notebook](docs/crispyx_tutorial.ipynb).
+
+## Installation
+
+```bash
+pip install -e .
+```
+
+## Benchmarking
+
+```bash
+cd benchmarking
+./run_benchmark.sh config/Adamson.yaml       # single dataset
+./run_benchmark.sh config/*.yaml             # all datasets
+```
+
+See [benchmarking/README.md](benchmarking/README.md) for configuration options and output structure.
+
+## Testing
+
+```bash
+pytest
+```
+
+## Documentation
+
+```bash
+sphinx-build docs docs/_build
+```

crispyx-0.0.1/pyproject.toml

@@ -0,0 +1,60 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "crispyx"
+version = "0.0.1"
+description = "Tools for scalable CRISPR screen analysis on on-disk AnnData objects"
+authors = [{name = "Streamlining CRISPR Team"}]
+dependencies = [
+    "anndata>=0.9",
+    "numpy>=1.23",
+    "numba>=0.59",
+    "pandas>=1.5",
+    "scipy>=1.10",
+    "h5py>=3.0",
+    "joblib>=1.0",
+    "scikit-learn>=1.0",
+    "scanpy>=1.9.2",
+    "seaborn>=0.12",
+    "matplotlib>=3.5",
+    "tqdm>=4.50",
+]
+
+[project.optional-dependencies]
+test = [
+    "filelock",
+    "pytest",
+    "statsmodels>=0.14",
+    "pydeseq2>=0.4",
+]
+benchmark = [
+    "pertpy>=0.4",
+    "pyyaml>=6.0",
+    "tqdm>=4.65",
+    "psutil>=5.9",
+]
+docs = [
+    "sphinx>=6.0",
+    "sphinx-rtd-theme>=1.2",
+]
+
+[tool.pytest.ini_options]
+addopts = "-q"
+filterwarnings = [
+    # Ignore h5py deprecation warnings about numpy.product (third-party issue)
+    "ignore:`product` is deprecated as of NumPy 1.25.0:DeprecationWarning",
+    # Ignore pydeseq2 FutureWarnings about dtype incompatibility (third-party issue)
+    "ignore:Setting an item of incompatible dtype is deprecated:FutureWarning",
+    # Ignore UserWarnings from third-party libraries during tests
+    "ignore:Some cells have zero counts:UserWarning",
+    "ignore:Every gene contains at least one zero:UserWarning",
+    "ignore:The dispersion trend curve fitting did not converge:UserWarning",
+    # Ignore AnnData implicit modification warnings (third-party issue)
+    "ignore:Transforming to str index:anndata.ImplicitModificationWarning",
+    # Ignore AnnData old-format warnings from test fixture files written without encoding metadata
+    "ignore:.*was written without encoding metadata:anndata.OldFormatWarning",
+    # Ignore matplotlib layout warnings in plotting tests
+    "ignore:Tight layout not applied:UserWarning",
+]

crispyx-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

crispyx-0.0.1/src/crispyx/__init__.py

@@ -0,0 +1,155 @@
+"""Streamlined CRISPR screen analysis toolkit with Scanpy-style entry points."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("crispyx")
+except PackageNotFoundError:
+    __version__ = "0.0.0.dev"
+
+# ---------------------------------------------------------------------------
+# Public API re-exports
+# ---------------------------------------------------------------------------
+
+from .data import (
+    AnnData,
+    OverlapResult,
+    compute_overlap,
+    convert_to_csc,
+    convert_to_csr,
+    detect_gene_symbol_column,
+    detect_perturbation_column,
+    ensure_gene_symbol_column,
+    infer_columns,
+    load_obs,
+    load_var,
+    normalize_total_log1p,
+    normalise_perturbation_labels,
+    read_h5ad_ondisk,
+    read_backed,
+    resolve_data_path,
+    standardise_gene_names,
+    write_obs,
+    write_var,
+)
+from .de import (
+    RankGenesGroupsResult,
+    nb_glm_test,
+    shrink_lfc,
+    t_test,
+    wilcoxon_test,
+)
+from .profiling import (
+    Profiler,
+    MemoryProfiler,
+    TimingProfiler,
+    plot_benchmark_comparison,
+)
+from .plotting import (
+    materialize_rank_genes_groups,
+    plot_ma,
+    plot_overlap_heatmap,
+    plot_pca,
+    plot_pca_loadings,
+    plot_pca_variance_ratio,
+    plot_qc_perturbation_counts,
+    plot_qc_summary,
+    plot_top_genes_bar,
+    plot_umap,
+    plot_volcano,
+    rank_genes_groups_df,
+)
+from .pseudobulk import (
+    compute_average_log_expression,
+    compute_pseudobulk_expression,
+)
+from .qc import (
+    filter_cells_by_gene_count,
+    filter_genes_by_cell_count,
+    filter_perturbations_by_cell_count,
+    quality_control_summary,
+)
+
+# ---------------------------------------------------------------------------
+# Scanpy-style namespace singletons: cx.pp, cx.pb, cx.tl, cx.pl
+# ---------------------------------------------------------------------------
+
+from ._namespaces import (
+    _PlottingNamespace,
+    _PreprocessingNamespace,
+    _PseudobulkNamespace,
+    _ToolsNamespace,
+)
+
+pp = _PreprocessingNamespace()
+pb = _PseudobulkNamespace()
+tl = _ToolsNamespace()
+pl = _PlottingNamespace()
+
+# ---------------------------------------------------------------------------
+# __all__
+# ---------------------------------------------------------------------------
+
+__all__ = [
+    "__version__",
+    # Namespace singletons
+    "pp",
+    "pb",
+    "tl",
+    "pl",
+    # Quality control
+    "filter_cells_by_gene_count",
+    "filter_genes_by_cell_count",
+    "filter_perturbations_by_cell_count",
+    "quality_control_summary",
+    # Pseudo-bulk
+    "compute_average_log_expression",
+    "compute_pseudobulk_expression",
+    # Differential expression
+    "RankGenesGroupsResult",
+    "t_test",
+    "wilcoxon_test",
+    "nb_glm_test",
+    "shrink_lfc",
+    # Data utilities
+    "AnnData",
+    "ensure_gene_symbol_column",
+    "read_h5ad_ondisk",
+    "read_backed",
+    "resolve_data_path",
+    "normalize_total_log1p",
+    "convert_to_csc",
+    "convert_to_csr",
+    "load_obs",
+    "load_var",
+    "write_obs",
+    "write_var",
+    "standardise_gene_names",
+    "normalise_perturbation_labels",
+    "detect_perturbation_column",
+    "detect_gene_symbol_column",
+    "infer_columns",
+    "OverlapResult",
+    "compute_overlap",
+    # Profiling
+    "Profiler",
+    "MemoryProfiler",
+    "TimingProfiler",
+    "plot_benchmark_comparison",
+    # Plotting
+    "materialize_rank_genes_groups",
+    "rank_genes_groups_df",
+    "plot_pca",
+    "plot_pca_variance_ratio",
+    "plot_pca_loadings",
+    "plot_umap",
+    "plot_volcano",
+    "plot_ma",
+    "plot_top_genes_bar",
+    "plot_qc_perturbation_counts",
+    "plot_qc_summary",
+    "plot_overlap_heatmap",
+]
+

crispyx-0.0.1/src/crispyx/_checkpoint.py

@@ -0,0 +1,252 @@
+"""Checkpoint and progress utilities for streaming DE tests.
+
+This module provides atomic checkpointing and progress tracking for
+resumable differential expression tests.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import h5py
+import numpy as np
+
+if TYPE_CHECKING:
+    from tqdm import tqdm
+
+logger = logging.getLogger(__name__)
+
+# Check for tqdm availability
+try:
+    from tqdm import tqdm as _tqdm
+    HAS_TQDM = True
+except ImportError:
+    HAS_TQDM = False
+
+
+def _write_checkpoint_atomic(
+    checkpoint_path: Path,
+    data: dict,
+) -> None:
+    """Write checkpoint data atomically using temp file + rename.
+    
+    This ensures checkpoint file is never corrupted on crash.
+    """
+    # Ensure parent directory exists
+    checkpoint_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    # Write to a temporary file in the same directory
+    tmp_path = checkpoint_path.with_suffix(".tmp")
+    try:
+        with open(tmp_path, "w") as f:
+            json.dump(data, f, indent=2)
+        # Atomic rename
+        os.rename(tmp_path, checkpoint_path)
+    except Exception:
+        # Clean up temp file on error
+        if tmp_path.exists():
+            try:
+                tmp_path.unlink()
+            except Exception:
+                pass
+        raise
+
+
+def _read_checkpoint(checkpoint_path: Path) -> dict | None:
+    """Read checkpoint file, returning None if missing or corrupted.
+    
+    Returns
+    -------
+    dict or None
+        Checkpoint data if valid, None if file is missing or corrupted.
+    """
+    if not checkpoint_path.exists():
+        return None
+    try:
+        with open(checkpoint_path, "r") as f:
+            data = json.load(f)
+        # Validate required fields
+        if not isinstance(data, dict):
+            return None
+        if "completed" not in data or "total" not in data:
+            return None
+        return data
+    except (json.JSONDecodeError, IOError, OSError):
+        return None
+
+
+def _scan_h5ad_completed(
+    h5ad_path: Path,
+    all_candidates: list[str],
+    result_dataset: str = "uns/rank_genes_groups/full/scores",
+) -> list[str]:
+    """Scan h5ad file to detect completed perturbations by non-zero/non-NaN rows.
+    
+    This is a fallback when checkpoint file is missing or corrupted.
+    
+    Parameters
+    ----------
+    h5ad_path
+        Path to the output h5ad file.
+    all_candidates
+        List of all perturbation labels (in order).
+    result_dataset
+        HDF5 dataset path to check for results. Should have shape (n_groups, n_genes).
+        
+    Returns
+    -------
+    list[str]
+        List of perturbation labels that have been completed.
+    """
+    completed = []
+    if not h5ad_path.exists():
+        return completed
+    
+    try:
+        with h5py.File(h5ad_path, "r") as f:
+            # Try to access the result dataset
+            if result_dataset in f:
+                ds = f[result_dataset]
+                n_groups = ds.shape[0]
+                for idx in range(min(n_groups, len(all_candidates))):
+                    row = ds[idx, :]
+                    # Check if row has any non-NaN, non-zero values
+                    if np.any(np.isfinite(row) & (row != 0)):
+                        completed.append(all_candidates[idx])
+            else:
+                # Try alternative: check layers in X matrix
+                if "X" in f:
+                    X = f["X"]
+                    if hasattr(X, "shape") and len(X.shape) == 2:
+                        n_groups = X.shape[0]
+                        for idx in range(min(n_groups, len(all_candidates))):
+                            row = X[idx, :]
+                            if np.any(np.isfinite(row) & (row != 0)):
+                                completed.append(all_candidates[idx])
+    except Exception as e:
+        logger.warning(f"Failed to scan h5ad for completed perturbations: {e}")
+    
+    return completed
+
+
+def _get_resumable_candidates(
+    checkpoint_path: Path,
+    h5ad_path: Path,
+    all_candidates: list[str],
+    retry_failed: bool = True,
+) -> tuple[list[str], list[str], list[str]]:
+    """Get candidates to process, accounting for previous progress.
+    
+    Parameters
+    ----------
+    checkpoint_path
+        Path to the progress JSON file.
+    h5ad_path
+        Path to the output h5ad file.
+    all_candidates
+        List of all perturbation labels to process.
+    retry_failed
+        If True, previously failed perturbations will be retried.
+        
+    Returns
+    -------
+    tuple[list[str], list[str], list[str]]
+        (candidates_to_run, completed, failed)
+        - candidates_to_run: perturbations that need to be processed
+        - completed: perturbations already completed
+        - failed: perturbations that failed (for logging)
+    """
+    checkpoint = _read_checkpoint(checkpoint_path)
+    
+    if checkpoint is not None:
+        completed = checkpoint.get("completed", [])
+        failed = checkpoint.get("failed", [])
+        logger.info(f"Resuming: {len(completed)}/{len(all_candidates)} already completed")
+        if failed:
+            logger.info(f"  {len(failed)} previously failed perturbations")
+    else:
+        # Checkpoint missing or corrupted - try scanning h5ad
+        if h5ad_path.exists():
+            logger.warning(
+                f"Checkpoint file missing or corrupted at {checkpoint_path}. "
+                f"Scanning h5ad file to detect completed perturbations..."
+            )
+            completed = _scan_h5ad_completed(h5ad_path, all_candidates)
+            failed = []
+            if completed:
+                logger.info(f"Detected {len(completed)} completed perturbations from h5ad scan")
+        else:
+            completed = []
+            failed = []
+    
+    # Determine which candidates to run
+    completed_set = set(completed)
+    failed_set = set(failed) if not retry_failed else set()
+    
+    candidates_to_run = [
+        c for c in all_candidates 
+        if c not in completed_set and c not in failed_set
+    ]
+    
+    return candidates_to_run, completed, failed
+
+
+def _get_checkpoint_interval(n_perturbations: int, checkpoint_interval: int | None) -> int:
+    """Determine checkpoint interval based on dataset size.
+    
+    Parameters
+    ----------
+    n_perturbations
+        Total number of perturbations.
+    checkpoint_interval
+        User-specified interval, or None for auto.
+        
+    Returns
+    -------
+    int
+        Number of perturbations to process between checkpoints.
+    """
+    if checkpoint_interval is not None:
+        return max(1, checkpoint_interval)
+    # Auto: every 1 for small datasets, every 10 for larger ones
+    if n_perturbations < 100:
+        return 1
+    elif n_perturbations < 1000:
+        return 10
+    else:
+        return 50
+
+
+class _DummyProgress:
+    """Dummy progress bar that does nothing (for when verbose=False)."""
+    
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, *args):
+        pass
+    
+    def update(self, n: int = 1):
+        pass
+    
+    def set_postfix(self, **kwargs):
+        pass
+
+
+def _create_progress_context(
+    total: int,
+    desc: str,
+    verbose: bool,
+) -> "_tqdm | _DummyProgress":
+    """Create a progress bar context manager.
+    
+    Returns tqdm progress bar if verbose=True and tqdm is available,
+    otherwise returns a dummy context manager.
+    """
+    if verbose and HAS_TQDM and total > 0:
+        return _tqdm(total=total, desc=desc, unit="perturbation")
+    return _DummyProgress()