pysccdc 0.1.0__py3-none-any.whl

pysccdc/__init__.py

@@ -0,0 +1,54 @@
+"""pysccdc -- pure-Python port of the R package **scCDC**.
+
+scCDC (Wang *et al.*, *Genome Biology* 2024) is an entropy-based,
+*gene-specific* ambient-RNA decontamination method for scRNA-seq and
+snRNA-seq.  Instead of correcting every gene, scCDC detects the small
+set of **Global Contamination-causing Genes (GCGs)** and corrects only
+those -- avoiding the over-correction of lowly/non-contaminating genes
+seen in DecontX / SoupX / CellBender.
+
+The pipeline:
+
+1. :func:`ContaminationDetection` -- per cluster, compute the Shannon
+   entropy of every gene's count distribution, fit the expected
+   entropy-vs-expression curve with an outlier-trimmed smoothing spline,
+   score the *entropy divergence* (expected - observed) and flag genes
+   significant in > ``restriction_factor`` of clusters as GCGs.
+2. :func:`ContaminationQuantification` -- a dataset-level contamination
+   ratio from the GCGs.
+3. :func:`ContaminationCorrection` -- for each GCG, a Youden-index
+   threshold on the per-cell count distributions decides how much to
+   subtract (floored at zero); non-GCGs are untouched.
+
+All functions are AnnData-friendly and depend only on
+numpy / scipy / pandas / anndata / scikit-learn.
+"""
+from __future__ import annotations
+
+from . import datasets
+from .correction import (
+    ContaminationCorrection,
+    ContaminationQuantification,
+    simple_roc,
+    youden_threshold,
+)
+from .detection import ContaminationDetection, generate_curve
+from .entropy import matrix_entropy, vector_entropy
+from .spline import SmoothSpline, smooth_spline
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ContaminationDetection",
+    "ContaminationCorrection",
+    "ContaminationQuantification",
+    "generate_curve",
+    "vector_entropy",
+    "matrix_entropy",
+    "SmoothSpline",
+    "smooth_spline",
+    "simple_roc",
+    "youden_threshold",
+    "datasets",
+    "__version__",
+]

pysccdc/_anndata.py

@@ -0,0 +1,72 @@
+"""AnnData <-> dense-matrix glue used across :mod:`pysccdc`.
+
+scCDC reasons in the Seurat layout (genes x cells); AnnData stores
+cells x genes.  These helpers convert and pick out the count layer /
+cluster labels so the rest of the package can work with plain NumPy.
+"""
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+__all__ = ["counts_matrix", "get_clusters", "to_dense"]
+
+
+def to_dense(x) -> np.ndarray:
+    """Return a dense 2-D ``float64`` array from dense or sparse input."""
+    if hasattr(x, "toarray"):
+        x = x.toarray()
+    return np.asarray(x, dtype=np.float64)
+
+
+def counts_matrix(
+    adata, layer: Optional[str] = None
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Extract the (genes x cells) count matrix from an AnnData.
+
+    Parameters
+    ----------
+    adata
+        :class:`anndata.AnnData` with raw counts.
+    layer
+        ``adata.layers`` key; ``None`` uses ``adata.X``.
+
+    Returns
+    -------
+    counts : ndarray
+        Dense ``genes x cells`` matrix (transposed from AnnData).
+    genes : ndarray
+        ``var_names``.
+    cells : ndarray
+        ``obs_names``.
+    """
+    mat = adata.layers[layer] if layer is not None else adata.X
+    counts = to_dense(mat).T  # genes x cells
+    genes = np.asarray(adata.var_names)
+    cells = np.asarray(adata.obs_names)
+    return counts, genes, cells
+
+
+def get_clusters(adata, cluster_key: Optional[str] = None) -> np.ndarray:
+    """Return the per-cell cluster labels as a string array.
+
+    When ``cluster_key`` is ``None`` the first categorical/object column
+    of ``adata.obs`` is used.
+    """
+    if cluster_key is None:
+        for col in adata.obs.columns:
+            ser = adata.obs[col]
+            if isinstance(ser.dtype, pd.CategoricalDtype) or \
+                    ser.dtype == object:
+                cluster_key = col
+                break
+        if cluster_key is None:
+            raise ValueError(
+                "No categorical column found in adata.obs; "
+                "pass cluster_key explicitly."
+            )
+    if cluster_key not in adata.obs.columns:
+        raise KeyError(f"cluster_key '{cluster_key}' not in adata.obs.")
+    return adata.obs[cluster_key].astype(str).to_numpy()

pysccdc/correction.py

@@ -0,0 +1,357 @@
+"""GCG-specific decontamination -- :func:`ContaminationCorrection`.
+
+Port of scCDC's ``Contamination_Correction.R`` (and the contamination
+ratio of ``Contamination_Quantification.R``).
+
+For each GCG, scCDC
+
+1. ranks clusters by their mean **log-normalized** expression of the
+   gene and takes the lowest-expressing qualified cluster as the first
+   *eGCG-negative* reference;
+2. computes, for every cluster, an **AUROC** of the per-cell counts of
+   that cluster vs. the reference cluster (``Cal_AUCs``);
+3. splits clusters into *eGCG-positive* (AUROC >= ``auc_thres``) and
+   *eGCG-negative* (AUROC < ``auc_thres``);
+4. pools the counts of the eGCG-negative cells with those of the
+   **least** eGCG-positive cluster, builds a ROC of that pooled vector
+   labelled neg(0)/pos(1), and takes the count value maximising the
+   **Youden index** (sensitivity + specificity - 1) as the subtraction
+   threshold (``Cal_thres``);
+5. subtracts ``round(threshold)`` from every cell's count of that gene,
+   floored at zero.
+
+Only the GCGs are touched; all other genes pass through unchanged --
+scCDC's anti-over-correction design.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Sequence, Union
+
+import numpy as np
+import pandas as pd
+
+from ._anndata import counts_matrix, get_clusters
+
+__all__ = [
+    "simple_roc",
+    "youden_threshold",
+    "ContaminationCorrection",
+    "ContaminationQuantification",
+]
+
+
+# ----------------------------------------------------------------------
+def _log_normalize(counts: np.ndarray, scale_factor: float = 1e4) -> np.ndarray:
+    """Seurat ``LogNormalize``: ``log1p(count / colsum * scale_factor)``.
+
+    ``counts`` is genes x cells; column sums are per-cell library sizes.
+    """
+    lib = counts.sum(axis=0)
+    lib[lib == 0] = 1.0
+    return np.log1p(counts / lib[None, :] * scale_factor)
+
+
+def _roc_curve(scores: np.ndarray, labels: np.ndarray):
+    """ROC thresholds / sensitivities / specificities (pROC, direction '<').
+
+    ``direction='<'`` in pROC means the positive class is predicted to
+    have *higher* scores.  pROC inserts ``+/-Inf`` boundary thresholds
+    and uses midpoints between consecutive unique scores; we replicate
+    that so the Youden-optimal threshold matches exactly.
+    """
+    scores = np.asarray(scores, dtype=float)
+    labels = np.asarray(labels)
+    n_pos = np.sum(labels == 1)
+    n_neg = np.sum(labels == 0)
+
+    uniq = np.unique(scores)
+    if uniq.size == 0:
+        return np.array([np.inf]), np.array([0.0]), np.array([1.0])
+    # pROC threshold grid: -Inf, midpoints of consecutive uniques, +Inf
+    if uniq.size == 1:
+        thr = np.array([-np.inf, np.inf])
+    else:
+        mids = (uniq[:-1] + uniq[1:]) / 2.0
+        thr = np.concatenate(([-np.inf], mids, [np.inf]))
+
+    sens = np.empty(thr.size)
+    spec = np.empty(thr.size)
+    pos_scores = scores[labels == 1]
+    neg_scores = scores[labels == 0]
+    for i, t in enumerate(thr):
+        # predict positive when score >= t
+        tp = np.sum(pos_scores >= t)
+        tn = np.sum(neg_scores < t)
+        sens[i] = tp / n_pos if n_pos > 0 else 0.0
+        spec[i] = tn / n_neg if n_neg > 0 else 0.0
+    return thr, sens, spec
+
+
+def simple_roc(exp: np.ndarray, cls: np.ndarray) -> float:
+    """AUROC of ``exp`` separating the two classes in ``cls`` (pROC '<').
+
+    The positive class is the label ``1``.  Ties contribute 0.5 each
+    (Mann-Whitney U / pROC convention).
+    """
+    exp = np.asarray(exp, dtype=float)
+    cls = np.asarray(cls)
+    pos = exp[cls == 1]
+    neg = exp[cls == 0]
+    n_pos, n_neg = pos.size, neg.size
+    if n_pos == 0 or n_neg == 0:
+        return float("nan")
+    # rank-sum AUC with tie handling
+    allv = np.concatenate([pos, neg])
+    order = np.argsort(allv, kind="mergesort")
+    ranks = np.empty(allv.size, dtype=float)
+    sv = allv[order]
+    i = 0
+    while i < sv.size:
+        j = i
+        while j + 1 < sv.size and sv[j + 1] == sv[i]:
+            j += 1
+        ranks[order[i:j + 1]] = (i + j) / 2.0 + 1.0
+        i = j + 1
+    rank_pos = ranks[:n_pos].sum()
+    auc = (rank_pos - n_pos * (n_pos + 1) / 2.0) / (n_pos * n_neg)
+    return float(auc)
+
+
+def youden_threshold(neg: np.ndarray, pos: np.ndarray) -> float:
+    """Youden-index optimal cut count separating ``neg`` from ``pos``.
+
+    Mirrors scCDC's ``Cal_thres`` -- builds the ROC of the pooled counts
+    labelled 0(neg)/1(pos) and returns the pROC threshold that maximises
+    ``sensitivity + specificity - 1``.
+    """
+    scores = np.concatenate([neg, pos]).astype(float)
+    labels = np.concatenate([
+        np.zeros(len(neg), dtype=int), np.ones(len(pos), dtype=int)
+    ])
+    thr, sens, spec = _roc_curve(scores, labels)
+    yd = sens + spec - 1.0
+    best = thr[int(np.argmax(yd))]
+    return float(best)
+
+
+# ----------------------------------------------------------------------
+def _cal_aucs(counts, lognorm, labels, gene_idx, qualified):
+    """``Cal_AUCs`` -- AUROC of each cluster vs the lowest-expressing one.
+
+    Returns ``(aucs_by_cluster, eGCG_neg_reference_cluster)``.
+    """
+    # mean log-normalized expression per cluster (Seurat AverageExpression
+    # on the LogNormalize 'data' slot used by Cal_AUCs to *order* clusters)
+    clusters = np.unique(labels)
+    mean_exp = {}
+    for cl in clusters:
+        mask = labels == cl
+        # AverageExpression = expm1(mean(log-data)) then implicitly compared;
+        # ordering by the log-data mean is monotone-equivalent.
+        mean_exp[cl] = lognorm[gene_idx, mask].mean()
+    order = sorted(clusters, key=lambda c: mean_exp[c])
+    qset = set(qualified)
+    ref = next((c for c in order if c in qset), order[0])
+
+    g_counts = counts[gene_idx]
+    ref_vals = g_counts[labels == ref]
+    aucs = {}
+    for cl in order:
+        cl_vals = g_counts[labels == cl]
+        scores = np.concatenate([cl_vals, ref_vals])
+        cls = np.concatenate([
+            np.ones(len(cl_vals), dtype=int),
+            np.zeros(len(ref_vals), dtype=int),
+        ])
+        aucs[cl] = simple_roc(scores, cls)
+    # preserve the expression-sorted order
+    return order, aucs, ref
+
+
+def _cal_thres(counts, labels, gene_idx, order, aucs, auc_thres):
+    """``Cal_thres`` -- the per-gene Youden subtraction threshold."""
+    auc_arr = np.array([aucs[c] for c in order])
+    if np.sum(auc_arr >= auc_thres) == 0:
+        auc_thres = np.nanmax(auc_arr)
+
+    neg_cls = [c for c in order if aucs[c] < auc_thres]
+    pos_cls = [c for c in order if aucs[c] >= auc_thres]
+    if len(pos_cls) == 0:  # degenerate -> nothing to subtract
+        return 0.0
+    low_pos = pos_cls[0]
+
+    g_counts = counts[gene_idx]
+    neg_mask = np.isin(labels, neg_cls) if neg_cls else np.zeros(
+        labels.shape, dtype=bool
+    )
+    pos_mask = labels == low_pos
+    neg_vals = g_counts[neg_mask]
+    pos_vals = g_counts[pos_mask]
+    if neg_vals.size == 0 or pos_vals.size == 0:
+        return 0.0
+    return youden_threshold(neg_vals, pos_vals)
+
+
+def _cont_level(counts, lognorm, labels, gene_idx, order, aucs, auc_thres):
+    """``Cal_Cont_level`` -- per-gene contamination ratio.
+
+    Total expression of the gene in eGCG-negative cells divided by the
+    total expression of *all* genes in those same cells.
+    """
+    auc_arr = np.array([aucs[c] for c in order])
+    if np.sum(auc_arr >= auc_thres) == 0:
+        auc_thres = np.nanmax(auc_arr)
+    neg_cls = [c for c in order if aucs[c] < auc_thres]
+    if not neg_cls:
+        return 0.0
+    neg_mask = np.isin(labels, neg_cls)
+    gene_total = counts[gene_idx, neg_mask].sum()
+    all_total = counts[:, neg_mask].sum()
+    if all_total == 0:
+        return 0.0
+    return float(gene_total / all_total)
+
+
+# ----------------------------------------------------------------------
+def ContaminationCorrection(
+    adata,
+    cont_genes: Union[Sequence[str], pd.DataFrame],
+    cluster_key: Optional[str] = None,
+    auc_thres: float = 0.9,
+    min_cell: int = 50,
+    layer: Optional[str] = None,
+    copy: bool = True,
+    corrected_layer: str = "Corrected",
+):
+    """Decontaminate the GCG counts of an AnnData (scCDC ``ContaminationCorrection``).
+
+    Parameters
+    ----------
+    adata
+        AnnData with raw integer counts (in ``X`` or ``layer``).
+    cont_genes
+        GCG names; either a sequence, or the DataFrame returned by
+        :func:`~pysccdc.ContaminationDetection` (its index is used).
+    cluster_key
+        ``adata.obs`` column with cluster labels (auto-detected if None).
+    auc_thres
+        AUROC threshold splitting eGCG-positive / -negative clusters.
+    min_cell
+        Clusters smaller than this are not used as references.
+    layer
+        Count layer; ``None`` -> ``adata.X``.
+    copy
+        Return a corrected copy (``True``) or modify ``adata`` in place.
+    corrected_layer
+        Name of the layer that receives the corrected matrix.
+
+    Returns
+    -------
+    anndata.AnnData
+        AnnData with a ``layers[corrected_layer]`` holding the corrected
+        counts.  ``.uns['sccdc']`` records the per-gene thresholds.
+    """
+    if isinstance(cont_genes, pd.DataFrame):
+        cont_genes = list(cont_genes.index)
+    cont_genes = list(cont_genes)
+
+    out = adata.copy() if copy else adata
+    counts, genes, _ = counts_matrix(out, layer=layer)
+    labels = get_clusters(out, cluster_key)
+    labels = np.asarray(labels)
+
+    gene_pos = {g: i for i, g in enumerate(genes)}
+    missing = [g for g in cont_genes if g not in gene_pos]
+    if missing:
+        raise KeyError(f"cont_genes not in adata.var_names: {missing}")
+
+    uniq, cnts = np.unique(labels, return_counts=True)
+    qualified = [c for c, n in zip(uniq, cnts) if n >= min_cell]
+    if not qualified:
+        raise ValueError(f"No cluster reaches min_cell={min_cell}.")
+
+    lognorm = _log_normalize(counts)
+
+    corrected = counts.copy()
+    thresholds = {}
+    for g in cont_genes:
+        gi = gene_pos[g]
+        order, aucs, _ref = _cal_aucs(
+            counts, lognorm, labels, gi, qualified
+        )
+        thr = _cal_thres(counts, labels, gi, order, aucs, auc_thres)
+        thresholds[g] = thr
+        corrected[gi] = np.maximum(counts[gi] - round(thr), 0.0)
+
+    # write back in AnnData orientation (cells x genes)
+    out.layers[corrected_layer] = corrected.T
+    info = out.uns.get("sccdc", {})
+    info = dict(info)
+    info["thresholds"] = thresholds
+    info["GCGs"] = cont_genes
+    info["auc_thres"] = auc_thres
+    out.uns["sccdc"] = info
+    return out
+
+
+# ----------------------------------------------------------------------
+def ContaminationQuantification(
+    adata,
+    cont_genes: Union[Sequence[str], pd.DataFrame],
+    cluster_key: Optional[str] = None,
+    auc_thres: float = 0.9,
+    min_cell: int = 50,
+    top10_gcg: bool = False,
+    layer: Optional[str] = None,
+    return_per_gene: bool = False,
+):
+    """Dataset-level contamination ratio from the GCGs (scCDC ``ContaminationQuantification``).
+
+    For each GCG the contamination level is its total expression in the
+    eGCG-negative cells divided by the total expression of all genes in
+    those cells; the dataset ratio is the **maximum** across GCGs.  A
+    ratio above ~3e-4 indicates high contamination (scCDC heuristic).
+
+    Parameters
+    ----------
+    adata, cont_genes, cluster_key, auc_thres, min_cell, layer
+        As in :func:`ContaminationCorrection`.
+    top10_gcg
+        Use only the first 10 GCGs (scCDC option).
+    return_per_gene
+        Also return the per-gene ratio :class:`pandas.Series`.
+
+    Returns
+    -------
+    float or tuple
+        The maximum contamination ratio, or ``(max_ratio, per_gene)``
+        when ``return_per_gene`` is set.
+    """
+    if isinstance(cont_genes, pd.DataFrame):
+        cont_genes = list(cont_genes.index)
+    cont_genes = list(cont_genes)
+    if top10_gcg and len(cont_genes) >= 10:
+        cont_genes = cont_genes[:10]
+
+    counts, genes, _ = counts_matrix(adata, layer=layer)
+    labels = np.asarray(get_clusters(adata, cluster_key))
+    gene_pos = {g: i for i, g in enumerate(genes)}
+
+    uniq, cnts = np.unique(labels, return_counts=True)
+    qualified = [c for c, n in zip(uniq, cnts) if n >= min_cell]
+    if not qualified:
+        raise ValueError(f"No cluster reaches min_cell={min_cell}.")
+
+    lognorm = _log_normalize(counts)
+    ratios = {}
+    for g in cont_genes:
+        gi = gene_pos[g]
+        order, aucs, _ref = _cal_aucs(counts, lognorm, labels, gi, qualified)
+        ratios[g] = _cont_level(
+            counts, lognorm, labels, gi, order, aucs, auc_thres
+        )
+    per_gene = pd.Series(ratios, name="contamination_ratio")
+    max_ratio = float(per_gene.max()) if len(per_gene) else 0.0
+    if return_per_gene:
+        return max_ratio, per_gene
+    return max_ratio

pysccdc/datasets.py

@@ -0,0 +1,119 @@
+"""Synthetic example data for :mod:`pysccdc`.
+
+:func:`simulate_contaminated` builds a small clustered count matrix in
+which a handful of genes are deliberately spiked with *ambient*
+contamination -- each contaminating gene is a marker of one cluster but
+also leaks low counts into every other cluster.  This is the input used
+by the R-parity test suite.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+
+__all__ = ["simulate_contaminated"]
+
+
+def simulate_contaminated(
+    n_clusters: int = 4,
+    n_cells_per_cluster: int = 200,
+    n_genes: int = 120,
+    n_contaminating: int = 4,
+    contamination_strength: float = 6.0,
+    random_state: Optional[int] = 0,
+):
+    """Simulate a clustered scRNA-seq count matrix with spiked GCGs.
+
+    Parameters
+    ----------
+    n_clusters
+        Number of cell clusters.
+    n_cells_per_cluster
+        Cells per cluster.
+    n_genes
+        Total genes (markers + housekeeping + contaminating).
+    n_contaminating
+        Number of deliberately-spiked contaminating genes (GCGs).
+    contamination_strength
+        Mean ambient leakage count of each contaminating gene into
+        non-source clusters.
+    random_state
+        Seed for reproducibility.
+
+    Returns
+    -------
+    anndata.AnnData
+        ``X`` holds integer counts (cells x genes); ``obs['cluster']``
+        the cluster labels; ``uns['true_GCGs']`` the spiked gene names.
+    """
+    import anndata as ad
+    import pandas as pd
+
+    rng = np.random.default_rng(random_state)
+    n_cells = n_clusters * n_cells_per_cluster
+
+    labels = np.repeat(
+        [f"c{i}" for i in range(n_clusters)], n_cells_per_cluster
+    )
+
+    gene_names = [f"gene{i}" for i in range(n_genes)]
+    counts = np.zeros((n_cells, n_genes), dtype=np.int64)
+
+    # ---- housekeeping / background genes -----------------------------
+    # genuinely variable expression -> a broad spread of count values,
+    # i.e. HIGH entropy for their mean expression (non-contaminating).
+    n_bg = n_genes - n_contaminating
+    for j in range(n_bg):
+        # negative-binomial-like over-dispersion gives a wide count
+        # distribution (high entropy) typical of real genes
+        rate = rng.gamma(shape=1.2, scale=rng.uniform(1.0, 8.0),
+                         size=n_cells)
+        counts[:, j] = rng.poisson(rate)
+
+    # add cluster-marker structure to some background genes so clusters
+    # are biologically distinct (these are NOT contaminating) -- still a
+    # broad (high-entropy) distribution within the marker cluster.
+    markers_per_cluster = max(1, n_bg // (n_clusters * 4))
+    for ci in range(n_clusters):
+        cmask = labels == f"c{ci}"
+        for k in range(markers_per_cluster):
+            j = ci * markers_per_cluster + k
+            if j >= n_bg:
+                break
+            rate = rng.gamma(shape=2.0, scale=12.0, size=cmask.sum())
+            counts[cmask, j] += rng.poisson(rate)
+
+    # ---- contaminating genes (the true GCGs) -------------------------
+    # A GCG is a strong marker of ONE cluster but leaks *ambient* RNA
+    # into every other cluster.  The ambient leakage is near-constant
+    # low counts (it reflects the soup composition, not real biology),
+    # so the gene's overall count distribution is unusually CONCENTRATED
+    # for its mean expression -> entropy BELOW the fitted curve, the
+    # positive entropy divergence scCDC keys on.
+    true_gcgs = []
+    for k in range(n_contaminating):
+        j = n_bg + k
+        gene_names[j] = f"CONT{k}"
+        true_gcgs.append(f"CONT{k}")
+        source = k % n_clusters
+        src_mask = labels == f"c{source}"
+        # high, fairly uniform expression in the source cluster
+        counts[src_mask, j] = rng.poisson(45.0, size=src_mask.sum())
+        # ambient leakage everywhere else: a near-constant low count
+        # (Bernoulli presence x a tight integer level) -> low entropy
+        other = ~src_mask
+        lvl = int(round(contamination_strength))
+        present = rng.random(other.sum()) < 0.85
+        counts[other, j] += np.where(present, lvl, lvl - 1)
+
+    obs = pd.DataFrame(
+        {"cluster": pd.Categorical(labels)},
+        index=[f"cell{i}" for i in range(n_cells)],
+    )
+    var = pd.DataFrame(index=gene_names)
+    adata = ad.AnnData(
+        X=counts.astype(np.float64), obs=obs, var=var
+    )
+    adata.uns["true_GCGs"] = true_gcgs
+    return adata