pypando 0.1.0__py3-none-any.whl

pypando/__init__.py

@@ -0,0 +1,117 @@
+"""py-Pando — pure-Python port of Pando (Fleck et al., Nature 2023).
+
+Pando infers gene regulatory networks from paired single-cell RNA and ATAC
+data: for every target gene it regresses expression on TF expression x
+accessibility of TF-binding peaks near the gene.
+
+Functional API (mirrors the R package)::
+
+    import pypando as pp
+    grn = pp.initiate_grn(rna, atac, annotation)
+    grn = pp.find_motifs(grn, genome="hg38.fa")
+    grn = pp.infer_grn(grn, genes=targets)
+    grn = pp.find_modules(grn)
+    pp.coef(grn); pp.gof(grn); pp.NetworkModules(grn)
+"""
+from . import data
+from .annotation import annotation_from_gtf
+from .core import Pando
+from .grn import (
+    find_modules,
+    find_motifs,
+    find_peaks_near_genes,
+    fit_grn_models,
+    format_coefs,
+    infer_grn,
+    initiate_grn,
+    region_background,
+    regions_near_genes,
+    subset_regions,
+)
+from .graph import get_network_graph
+from .models import (
+    fit_bagging_ridge,
+    fit_bayesian_ridge,
+    fit_brms,
+    fit_cvglmnet,
+    fit_glm,
+    fit_glmnet,
+    fit_model,
+    fit_xgb,
+)
+from .motifs import PFM, load_motif2tf, load_pando_motifs
+from .objects import (
+    GRNData,
+    Modules,
+    Network,
+    Regions,
+    RegulatoryNetwork,
+    DefaultNetwork,
+    GetGRN,
+    GetNetwork,
+    NetworkFeatures,
+    NetworkGraph,
+    NetworkModules,
+    NetworkParams,
+    NetworkRegions,
+    NetworkTFs,
+    Params,
+    coef,
+    gof,
+)
+from .objects import GetAssaySummary, aggregate_assay, create_GRNData
+from .plots import plot_gof, plot_module_metrics, plot_network_graph
+from .utils import aggregate_matrix, sparse_cor
+
+__all__ = [
+    "Pando",
+    "annotation_from_gtf",
+    "initiate_grn",
+    "find_motifs",
+    "find_peaks_near_genes",
+    "infer_grn",
+    "fit_grn_models",
+    "format_coefs",
+    "find_modules",
+    "region_background",
+    "regions_near_genes",
+    "subset_regions",
+    "get_network_graph",
+    "fit_model",
+    "fit_glm",
+    "fit_glmnet",
+    "fit_cvglmnet",
+    "fit_xgb",
+    "fit_bagging_ridge",
+    "fit_bayesian_ridge",
+    "fit_brms",
+    "PFM",
+    "load_pando_motifs",
+    "load_motif2tf",
+    "GRNData",
+    "RegulatoryNetwork",
+    "Regions",
+    "Network",
+    "Modules",
+    "GetGRN",
+    "GetNetwork",
+    "DefaultNetwork",
+    "NetworkFeatures",
+    "NetworkTFs",
+    "NetworkRegions",
+    "NetworkModules",
+    "NetworkParams",
+    "NetworkGraph",
+    "Params",
+    "coef",
+    "gof",
+    "aggregate_matrix",
+    "aggregate_assay",
+    "GetAssaySummary",
+    "create_GRNData",
+    "plot_gof",
+    "plot_module_metrics",
+    "plot_network_graph",
+    "sparse_cor",
+]
+__version__ = "0.1.0"

pypando/_blas.py

@@ -0,0 +1,177 @@
+"""R's QR (``dqrdc2`` / ``dqrls``) compiled with numba, calling the BLAS.
+
+R's Fortran routines call the BLAS level-1 kernels ``dnrm2``, ``ddot``,
+``daxpy`` and ``dscal``. Floating-point results of those kernels depend on
+their summation order, so for decisions that sit on a knife edge (a column
+is "negligible" if its residual norm falls below ``1e-11`` times its
+original norm) the only way to reproduce R is to make the same calls in the
+same order. This module does that through SciPy's BLAS (the library NumPy
+and SciPy are linked to); with the same BLAS build as R's, the QR is
+bit-identical to R's.
+"""
+from __future__ import annotations
+
+import ctypes
+
+import numpy as np
+
+__all__ = ["dqrdc2_blas", "dqrls_blas", "available"]
+
+try:
+    import numba
+    from numba import njit
+    import scipy.linalg.cython_blas as _cb
+
+    def _fn(name, restype, *argtypes):
+        cap = _cb.__pyx_capi__[name]
+        ctypes.pythonapi.PyCapsule_GetName.restype = ctypes.c_char_p
+        ctypes.pythonapi.PyCapsule_GetName.argtypes = [ctypes.py_object]
+        ctypes.pythonapi.PyCapsule_GetPointer.restype = ctypes.c_void_p
+        ctypes.pythonapi.PyCapsule_GetPointer.argtypes = [ctypes.py_object, ctypes.c_char_p]
+        ptr = ctypes.pythonapi.PyCapsule_GetPointer(cap, ctypes.pythonapi.PyCapsule_GetName(cap))
+        return ctypes.CFUNCTYPE(restype, *argtypes)(ptr)
+
+    _P = ctypes.c_void_p
+    _dnrm2 = _fn("dnrm2", ctypes.c_double, _P, _P, _P)
+    _ddot = _fn("ddot", ctypes.c_double, _P, _P, _P, _P, _P)
+    _daxpy = _fn("daxpy", None, _P, _P, _P, _P, _P, _P)
+    _dscal = _fn("dscal", None, _P, _P, _P, _P)
+    available = True
+except Exception:  # pragma: no cover - numba or the capsules missing
+    available = False
+
+
+if available:
+
+    @njit(cache=False)
+    def _dqrdc2(x, tol, jpvt, qraux, work, ib, db):
+        # x: Fortran-ordered (n, p). ib: int32 scratch[4], db: float64 scratch[2]
+        n, p = x.shape
+        xp = x.ctypes.data
+        ld = n
+        step = 8  # sizeof(double)
+        one = ib.ctypes.data  # ib[0] = 1 (incx)
+        nn = ib.ctypes.data + 4  # ib[1] = length
+        ib[0] = 1
+        if n > 0:
+            for j in range(p):
+                ib[1] = n
+                v = _dnrm2(nn, xp + (j * ld) * step, one)
+                qraux[j] = v
+                work[j, 0] = v
+                work[j, 1] = v if v != 0.0 else 1.0
+        lup = min(n, p)
+        k = p + 1
+        for l in range(lup):
+            while not ((l + 1) >= k or qraux[l] >= work[l, 1] * tol):
+                for i in range(n):
+                    t = x[i, l]
+                    for j in range(l + 1, p):
+                        x[i, j - 1] = x[i, j]
+                    x[i, p - 1] = t
+                i0 = jpvt[l]
+                t = qraux[l]
+                tt = work[l, 0]
+                ttt = work[l, 1]
+                for j in range(l + 1, p):
+                    jpvt[j - 1] = jpvt[j]
+                    qraux[j - 1] = qraux[j]
+                    work[j - 1, 0] = work[j, 0]
+                    work[j - 1, 1] = work[j, 1]
+                jpvt[p - 1] = i0
+                qraux[p - 1] = t
+                work[p - 1, 0] = tt
+                work[p - 1, 1] = ttt
+                k -= 1
+            if (l + 1) != n:
+                m = n - l
+                ib[1] = m
+                col_l = xp + (l * ld + l) * step
+                nrmxl = _dnrm2(nn, col_l, one)
+                if nrmxl != 0.0:
+                    if x[l, l] != 0.0:
+                        nrmxl = abs(nrmxl) if x[l, l] >= 0 else -abs(nrmxl)
+                    db[0] = 1.0 / nrmxl
+                    _dscal(nn, db.ctypes.data, col_l, one)
+                    x[l, l] = 1.0 + x[l, l]
+                    for j in range(l + 1, p):
+                        col_j = xp + (j * ld + l) * step
+                        t = -_ddot(nn, col_l, one, col_j, one) / x[l, l]
+                        db[1] = t
+                        _daxpy(nn, db.ctypes.data + 8, col_l, one, col_j, one)
+                        if qraux[j] != 0.0:
+                            r = abs(x[l, j]) / qraux[j]
+                            tt = 1.0 - r * r
+                            if tt < 0.0:
+                                tt = 0.0
+                            t = tt
+                            if abs(t) >= 1e-6:
+                                qraux[j] = qraux[j] * np.sqrt(t)
+                            else:
+                                ib[2] = n - l - 1
+                                qraux[j] = _dnrm2(ib.ctypes.data + 8, xp + (j * ld + l + 1) * step, one)
+                                work[j, 0] = qraux[j]
+                    qraux[l] = x[l, l]
+                    x[l, l] = -nrmxl
+        return min(k - 1, n)
+
+    @njit(cache=False)
+    def _dqrsl_b(x, k, qraux, y, b, ib, db):
+        # qty (job digit b) then b (job digit c), as dqrsl with job 1110
+        n = x.shape[0]
+        xp = x.ctypes.data
+        step = 8
+        ib[0] = 1
+        one = ib.ctypes.data
+        nn = ib.ctypes.data + 4
+        qty = y.copy()
+        qp = qty.ctypes.data
+        ju = min(k, n - 1)
+        for j in range(ju):
+            if qraux[j] != 0.0:
+                temp = x[j, j]
+                x[j, j] = qraux[j]
+                ib[1] = n - j
+                col = xp + (j * n + j) * step
+                t = -_ddot(nn, col, one, qp + j * step, one) / x[j, j]
+                db[1] = t
+                _daxpy(nn, db.ctypes.data + 8, col, one, qp + j * step, one)
+                x[j, j] = temp
+        for i in range(k):
+            b[i] = qty[i]
+        bp = b.ctypes.data
+        for jj in range(k):
+            j = k - jj - 1
+            if x[j, j] == 0.0:
+                return j + 1  # info
+            b[j] = b[j] / x[j, j]
+            if j > 0:
+                db[1] = -b[j]
+                ib[1] = j
+                _daxpy(nn, db.ctypes.data + 8, xp + (j * n) * step, one, bp, one)
+        return 0
+
+
+def dqrdc2_blas(x: np.ndarray, tol: float):
+    """Same contract as :func:`pypando.rstats.dqrdc2`, BLAS-exact."""
+    X = np.array(x, dtype=np.float64, order="F", copy=True)
+    p = X.shape[1]
+    jpvt = np.arange(p, dtype=np.int64)
+    qraux = np.zeros(p)
+    work = np.zeros((p, 2))
+    ib = np.zeros(4, dtype=np.int32)
+    db = np.zeros(2)
+    k = _dqrdc2(X, float(tol), jpvt, qraux, work, ib, db)
+    return X, int(k), qraux, jpvt
+
+
+def dqrls_blas(x: np.ndarray, y: np.ndarray, tol: float):
+    """Same contract as :func:`pypando.rstats.dqrls`, BLAS-exact."""
+    qr, k, qraux, pivot = dqrdc2_blas(x, tol)
+    p = qr.shape[1]
+    b = np.zeros(p)
+    if k > 0:
+        ib = np.zeros(4, dtype=np.int32)
+        db = np.zeros(2)
+        _dqrsl_b(qr, k, qraux, np.ascontiguousarray(y, dtype=np.float64), b, ib, db)
+    return {"coefficients": b, "rank": k, "pivot": pivot, "qr": qr, "qraux": qraux}

pypando/_wilcox.py

@@ -0,0 +1,77 @@
+"""One-sample Wilcoxon signed rank test as in R 4.4 (``stats::wilcox.test``).
+
+R 4.4.x (the reference environment, R 4.4.3) drops zeros, ranks
+``abs(x)`` without rounding (``digits.rank = Inf``) and uses the exact
+null distribution when ``n < 50`` with no ties and no zeros; otherwise the
+normal approximation with tie correction and continuity correction.
+R >= 4.6 changed the zero handling; this follows 4.4.
+"""
+from __future__ import annotations
+
+import math
+
+import numpy as np
+from scipy import stats
+
+__all__ = ["wilcox_signed_rank_pvalue", "psignrank"]
+
+
+def _csignrank_counts(n: int) -> np.ndarray:
+    """Number of subsets of {1..n} with each sum 0..n(n+1)/2."""
+    u = n * (n + 1) // 2
+    c = np.zeros(u + 1, dtype=object)
+    c[0] = 1
+    for k in range(1, n + 1):
+        c[k:] = c[k:] + c[:-k] if k <= u else c[k:]
+    return c
+
+
+def psignrank(q: float, n: int, lower_tail: bool = True) -> float:
+    """``stats::psignrank`` (exact)."""
+    q = math.floor(q + 1e-7)
+    u = n * (n + 1) // 2
+    if q < 0:
+        return 0.0 if lower_tail else 1.0
+    if q >= u:
+        return 1.0 if lower_tail else 0.0
+    c = _csignrank_counts(n)
+    total = 2**n
+    low = sum(c[: q + 1])
+    p = low / total
+    return float(p) if lower_tail else float((total - low) / total)
+
+
+def wilcox_signed_rank_pvalue(x, mu: float = 0.0, correct: bool = True, exact=None) -> float:
+    """``wilcox.test(x, mu = mu)$p.value``, two-sided, R 4.4 semantics."""
+    x = np.asarray(x, dtype=np.float64)
+    x = x[~np.isnan(x)]
+    if len(x) < 1:
+        raise ValueError("not enough (non-missing) 'x' observations")
+    x = x - mu
+    zeroes = bool(np.any(x == 0))
+    if zeroes:
+        x = x[x != 0]
+    n = float(len(x))
+    if exact is None:
+        exact = n < 50
+    r = stats.rankdata(np.abs(x), method="average")
+    statistic = float(r[x > 0].sum())
+    ties = len(r) != len(np.unique(r))
+    if exact and not ties and not zeroes:
+        ni = int(n)
+        if statistic > n * (n + 1) / 4:
+            p = psignrank(statistic - 1, ni, lower_tail=False)
+        else:
+            p = psignrank(statistic, ni)
+        return min(2 * p, 1.0)
+    _, nties = np.unique(r, return_counts=True)
+    z = statistic - n * (n + 1) / 4
+    sigma = math.sqrt(max(0.0, n * (n + 1) * (2 * n + 1) / 24 - float(np.sum(nties.astype(float) ** 3 - nties)) / 48))
+    correction = math.copysign(0.5, z) if (correct and z != 0) else 0.0
+    # IEEE arithmetic as in R: sigma = 0 (e.g. all values were zeros and were
+    # dropped) gives z = NaN (0/0) or +-Inf, and the p-value NaN or 0.
+    with np.errstate(divide="ignore", invalid="ignore"):
+        z = np.float64(z - correction) / np.float64(sigma)
+    if np.isnan(z):
+        return float("nan")
+    return float(2 * min(stats.norm.cdf(z), stats.norm.sf(z)))

pypando/annotation.py

@@ -0,0 +1,56 @@
+"""Gene annotation in the shape Pando reads (Signac's ``Annotation(object)``).
+
+Pando uses a gene annotation for two things: a gene's extent (the union of
+its exon / CDS / UTR rows; Signac's intronic ``gap`` rows are dropped) and
+the exons it removes from the candidate regions. Signac builds this table
+with ``GetGRangesFromEnsDb``; :func:`annotation_from_gtf` builds the same
+table from a GENCODE or Ensembl GTF/GFF3 file.
+"""
+from __future__ import annotations
+
+import gzip
+import re
+from typing import Sequence
+
+import pandas as pd
+
+__all__ = ["annotation_from_gtf"]
+
+# Signac::GetGRangesFromEnsDb default biotypes (GENCODE >= v32 calls lincRNA "lncRNA")
+DEFAULT_BIOTYPES = ("protein_coding", "lncRNA", "lincRNA", "rRNA", "processed_transcript")
+_TYPES = {"exon": "exon", "CDS": "cds", "five_prime_UTR": "utr", "three_prime_UTR": "utr", "UTR": "utr"}
+_ATTR = re.compile(r'(\w+)[ =]"?([^";]+)"?')
+
+
+def annotation_from_gtf(path, biotypes: Sequence[str] | None = DEFAULT_BIOTYPES, chr_prefix: str | None = None) -> pd.DataFrame:
+    """Signac-style annotation from a GTF / GFF3 (optionally gzipped).
+
+    Returns columns ``seqnames, start, end, strand, tx_id, gene_name, gene_id,
+    gene_biotype, type`` with ``type`` in ``exon``, ``cds``, ``utr``, for genes
+    whose biotype is in ``biotypes`` (``None`` keeps all). ``chr_prefix``
+    (e.g. ``"chr"``) is prepended to sequence names that lack it, for
+    Ensembl files used with UCSC-style peaks.
+    """
+    opener = gzip.open if str(path).endswith(".gz") else open
+    keep = None if biotypes is None else set(biotypes)
+    rows = []
+    with opener(path, "rt") as fh:
+        for line in fh:
+            if line.startswith("#"):
+                continue
+            f = line.rstrip("\n").split("\t")
+            if len(f) < 9 or f[2] not in _TYPES:
+                continue
+            attrs = dict(_ATTR.findall(f[8]))
+            biotype = attrs.get("gene_type") or attrs.get("gene_biotype") or ""
+            if keep is not None and biotype not in keep:
+                continue
+            chrom = f[0]
+            if chr_prefix and not chrom.startswith(chr_prefix):
+                chrom = chr_prefix + chrom
+            rows.append((chrom, int(f[3]), int(f[4]), f[6],
+                         (attrs.get("transcript_id") or "").split(".")[0],
+                         attrs.get("gene_name", ""), (attrs.get("gene_id") or "").split(".")[0],
+                         biotype, _TYPES[f[2]]))
+    return pd.DataFrame(rows, columns=["seqnames", "start", "end", "strand", "tx_id", "gene_name",
+                                       "gene_id", "gene_biotype", "type"])

pypando/core.py

@@ -0,0 +1,48 @@
+"""Class API: the same workflow as method chaining on one object."""
+from __future__ import annotations
+
+from . import grn as _grn
+from .graph import get_network_graph
+from .objects import GRNData, coef, gof, NetworkModules
+
+
+class Pando:
+    """Method-chaining wrapper around :class:`~pypando.objects.GRNData`.
+
+    >>> p = Pando(rna, atac, annotation).find_motifs(genome="hg38.fa").infer_grn(genes=targets).find_modules()
+    >>> p.coefs.head()
+    """
+
+    def __init__(self, rna, atac, annotation, **initiate_kwargs):
+        self.data: GRNData = _grn.initiate_grn(rna, atac, annotation, **initiate_kwargs)
+
+    def find_motifs(self, **kwargs) -> "Pando":
+        _grn.find_motifs(self.data, **kwargs)
+        return self
+
+    def infer_grn(self, **kwargs) -> "Pando":
+        _grn.infer_grn(self.data, **kwargs)
+        return self
+
+    def find_modules(self, **kwargs) -> "Pando":
+        _grn.find_modules(self.data, **kwargs)
+        return self
+
+    def network_graph(self, **kwargs) -> "Pando":
+        get_network_graph(self.data, **kwargs)
+        return self
+
+    @property
+    def coefs(self):
+        return coef(self.data)
+
+    @property
+    def gof(self):
+        return gof(self.data)
+
+    @property
+    def modules(self):
+        return NetworkModules(self.data)
+
+    def __repr__(self):
+        return repr(self.data)

pypando/data/__init__.py

@@ -0,0 +1,52 @@
+"""Pando's bundled data sets.
+
+Shipped with the wheel: the motif collection (:func:`pypando.motifs.load_pando_motifs`),
+the motif-to-TF table (:func:`pypando.motifs.load_motif2tf`) and the Ensembl v93
+GRCh38 annotation used by ``peak_to_gene_method="GREAT"`` (:func:`ensdb_v93_hg38`).
+
+Downloaded on first use (pooch, checksum-pinned), because they are too large for a
+wheel: the phastCons 20-mammal conserved elements and the SCREEN cCREs that Pando
+offers as ``regions`` for :func:`pypando.initiate_grn`.
+"""
+from __future__ import annotations
+
+from importlib import resources
+
+import pandas as pd
+
+__all__ = ["ensdb_v93_hg38", "phastcons_hg38", "screen_ccre_hg38"]
+
+_ASSET_BASE = "https://github.com/omicverse/py-Pando/releases/download/data-v1/"
+_ASSETS = {
+    "phastcons20_hg38.tsv.gz": "sha256:4e7e30bb0d2727a3ff5409fa8e8e1948d4543a3a635b0925ce785c3d8ceb5950",
+    "screen_ccre_hg38.tsv.gz": "sha256:13815dd04916f6a31702b943cc49adf047647cef5451c9268191adff059f466d",
+}
+
+
+def _read(name: str) -> pd.DataFrame:
+    with resources.files(__name__).joinpath(name).open("rb") as fh:
+        return pd.read_csv(fh, sep="\t", compression="gzip", keep_default_na=False, low_memory=False)
+
+
+def ensdb_v93_hg38() -> pd.DataFrame:
+    """``EnsDb.Hsapiens.v93.annot.UCSC.hg38`` (one row per transcript, gene ranges)."""
+    df = _read("ensdb_v93_hg38.tsv.gz")
+    for c in ("start", "end"):
+        df[c] = df[c].astype("int64")
+    return df
+
+
+def _fetch(name: str) -> str:
+    import pooch
+
+    return pooch.retrieve(_ASSET_BASE + name, known_hash=_ASSETS[name], progressbar=False)
+
+
+def phastcons_hg38() -> pd.DataFrame:
+    """``phastConsElements20Mammals.UCSC.hg38`` (2,244,053 conserved elements)."""
+    return pd.read_csv(_fetch("phastcons20_hg38.tsv.gz"), sep="\t", keep_default_na=False)
+
+
+def screen_ccre_hg38() -> pd.DataFrame:
+    """``SCREEN.ccRE.UCSC.hg38`` (926,535 candidate cis-regulatory elements)."""
+    return pd.read_csv(_fetch("screen_ccre_hg38.tsv.gz"), sep="\t", keep_default_na=False)