sclab 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

sclab/preprocess/_transfer_metadata.py

@@ -1,6 +1,6 @@
 from collections import Counter
 from functools import partial
-from typing import Literal
+from typing import Callable, Literal
 
 import numpy as np
 import pandas as pd
@@ -23,45 +23,79 @@ def transfer_metadata(
     min_neighs: int = 5,
     weight_by: Literal["connectivity", "distance", "constant"] = "connectivity",
 ):
-    D: csr_matrix = adata.obsp["distances"]
-    C: csr_matrix = adata.obsp["connectivities"]
-    D = D.tocsr()
-
-    match weight_by:
-        case "connectivity":
-            W = C.tocsr()
-        case "distance":
-            W = D.tocsr()
-            W.data = 1.0 / W.data
-        case "constant":
-            W = D.tocsr()
-            W.data[:] = 1.0
-        case _:
-            raise ValueError(f"Unsupported weight_by {weight_by}")
-
-    meta_values: pd.Series
-    new_values: pd.Series
+    new_values, new_values_err = _propagate_metadata(
+        adata,
+        column=column,
+        periodic=periodic,
+        vmin=vmin,
+        vmax=vmax,
+        min_neighs=min_neighs,
+        weight_by=weight_by,
+        mask=adata.obs[group_key] != source_group,
+    )
+
+    adata.obs[f"transferred_{new_values.name}"] = new_values
+    adata.obs[f"transferred_{new_values_err.name}"] = new_values_err
+
+
+def propagate_metadata(
+    adata: AnnData,
+    column: str,
+    periodic: bool = False,
+    vmin: float = 0,
+    vmax: float = 1,
+    min_neighs: int = 5,
+    weight_by: Literal["connectivity", "distance", "constant"] = "connectivity",
+):
+    new_values, new_values_err = _propagate_metadata(
+        adata,
+        column=column,
+        periodic=periodic,
+        vmin=vmin,
+        vmax=vmax,
+        min_neighs=min_neighs,
+        weight_by=weight_by,
+    )
+
+    mask = adata.obs[column].isna()
+    adata.obs.loc[mask, column] = new_values.loc[mask]
+    adata.obs.loc[mask, new_values_err.name] = new_values_err.loc[mask]
+
+
+def _propagate_metadata(
+    adata: AnnData,
+    column: str,
+    periodic: bool = False,
+    vmin: float = 0,
+    vmax: float = 1,
+    min_neighs: int = 5,
+    weight_by: Literal["connectivity", "distance", "constant"] = "connectivity",
+    mask: np.ndarray | pd.Series | None = None,
+) -> tuple[pd.Series, pd.Series]:
+    D, W = _get_neighbors_and_weights(adata, weight_by=weight_by)
 
+    assign_value_fn: Callable
     series = adata.obs[column]
     if isinstance(series.dtype, pd.CategoricalDtype) or is_bool_dtype(series.dtype):
         assign_value_fn = _assign_categorical
-        new_column = f"transferred_{column}"
-        new_column_err = f"transferred_{column}_proportion"
     elif is_numeric_dtype(series.dtype) and periodic:
         assign_value_fn = partial(_assign_numerical_periodic, vmin=vmin, vmax=vmax)
-        new_column = f"transferred_{column}"
-        new_column_err = f"transferred_{column}_error"
     elif is_numeric_dtype(series.dtype):
         assign_value_fn = _assign_numerical
-        new_column = f"transferred_{column}"
-        new_column_err = f"transferred_{column}_error"
     else:
         raise ValueError(f"Unsupported dtype {series.dtype} for column {column}")
 
-    meta_values = series.copy()
-    meta_values[adata.obs[group_key] != source_group] = np.nan
-    new_values = pd.Series(index=series.index, dtype=series.dtype, name=new_column)
-    new_values_err = pd.Series(index=series.index, dtype=float, name=new_column_err)
+    if isinstance(series.dtype, pd.CategoricalDtype) or is_bool_dtype(series.dtype):
+        column_err = f"{column}_proportion"
+    else:
+        column_err = f"{column}_error"
+
+    meta_values: pd.Series = series.copy()
+    if mask is not None:
+        meta_values[mask] = pd.NA
+
+    new_values = pd.Series(index=series.index, dtype=series.dtype, name=column)
+    new_values_err = pd.Series(index=series.index, dtype=float, name=column_err)
 
     for i, (d, w) in tqdm(enumerate(zip(D, W)), total=D.shape[0]):
         if not pd.isna(meta_values.iloc[i]):
@@ -86,8 +120,33 @@ def transfer_metadata(
         new_values.iloc[i] = assigned_value
         new_values_err.iloc[i] = assigned_value_err
 
-    adata.obs[new_column] = new_values.copy()
-    adata.obs[new_column_err] = new_values_err.copy()
+    new_values = pd.concat([new_values, meta_values], axis=1).bfill(axis=1).iloc[:, 0]
+
+    return new_values, new_values_err
+
+
+def _get_neighbors_and_weights(
+    adata: AnnData,
+    weight_by: Literal["connectivity", "distance", "constant"] = "connectivity",
+):
+    D: csr_matrix = adata.obsp["distances"].copy()
+    C: csr_matrix = adata.obsp["connectivities"].copy()
+    D = D.tocsr()
+    W: csr_matrix
+
+    match weight_by:
+        case "connectivity":
+            W = C.tocsr().copy()
+        case "distance":
+            W = D.tocsr().copy()
+            W.data = 1.0 / W.data
+        case "constant":
+            W = D.tocsr().copy()
+            W.data[:] = 1.0
+        case _:
+            raise ValueError(f"Unsupported weight_by {weight_by}")
+
+    return D, W
 
 
 def _assign_categorical(values: pd.Series, weights: NDArray):

sclab/tools/cellflow/pseudotime/_pseudotime.py

@@ -280,6 +280,7 @@ def estimate_periodic_pseudotime_start(
     time_key: str = "pseudotime",
     bandwidth: float = 1 / 64,
     show_plot: bool = False,
+    nth_root: int = 1,
 ):
     # TODO: Test implementation
     pseudotime = adata.obs[time_key].values.copy()
@@ -316,7 +317,10 @@ def estimate_periodic_pseudotime_start(
     roots = (x[idx] + x[1:][idx]) / 2
     heights = yp[idx]
 
-    max_peak_x = roots[heights.argmin()]
+    roots = roots[heights.argsort()]
+    heights = heights[heights.argsort()]
+
+    max_peak_x = roots[nth_root - 1]
 
     if show_plot:
         plt.hist(

sclab/tools/cellflow/utils/periodic_genes.py

@@ -0,0 +1,106 @@
+import numpy as np
+import pandas as pd
+from anndata import AnnData
+from numpy.typing import NDArray
+from scipy.signal import get_window, periodogram
+from scipy.sparse import spmatrix
+
+from sclab.tools.utils import aggregate_and_filter
+
+
+def periodic_genes(
+    adata: AnnData,
+    time_key: str,
+    tmin: float,
+    tmax: float,
+    period: float,
+    n: int,
+    min_pct_power_below: float = 0.75,
+    layer: str | None = None,
+):
+    times = adata.obs[time_key].values.copy()
+    if layer is None or layer == "X":
+        X = adata.X
+    else:
+        X = adata.layers[layer]
+
+    _assert_integer_counts(X)
+
+    tmp_adata = AnnData(X, obs=adata.obs[[time_key]], var=adata.var[[]])
+
+    w = (tmax - tmin) / n
+    bins = np.arange(-w / 2 + tmin, tmax, w)
+    labels = list(map(lambda x: f"{x:.2f}", bins[:-1] + w / 2))
+
+    times[times >= bins.max()] = times[times >= bins.max()] - tmax
+    tmp_adata.obs["timepoint"] = pd.cut(times, bins=bins, labels=labels)
+    aggregated = aggregate_and_filter(
+        tmp_adata,
+        "timepoint",
+        replicas_per_group=1,
+        make_stats=False,
+        make_dummies=False,
+    )
+    log_cnts = np.log1p(aggregated.X)
+    profiles = pd.DataFrame(log_cnts, index=labels, columns=aggregated.var_names)
+    ps = power_spectrum_df(profiles)
+    pp = pct_power_below(ps, 1 / period)
+
+    adata.varm["profile"] = profiles.T
+    adata.varm["periodogram"] = ps.T
+    adata.var["pct_power_below"] = pp
+    adata.var["periodic"] = pp > min_pct_power_below
+
+
+def _assert_integer_counts(X: spmatrix | NDArray):
+    message = "Periodic genes requires raw integer counts. E.g. `layer = 'counts'`."
+    if isinstance(X, spmatrix):
+        assert all(X.data % 1 == 0), message
+    else:
+        assert all(X % 1 == 0), message
+
+
+def infer_dt_from_index(idx: pd.Index) -> float:
+    # Works for numeric or datetime indexes
+    if isinstance(idx, pd.DatetimeIndex):
+        dt = np.median(np.diff(idx.view("i8"))) / 1e9  # seconds
+    else:
+        dt = float(np.median(np.diff(idx.values.astype(float))))
+    return dt
+
+
+def power_spectrum_df(X: pd.DataFrame, window: str = "hann", detrend: str = "constant"):
+    # X: rows=timepoints, columns=variables
+    Xd = X - X.mean()  # remove DC so percent computations are stable
+    dt = infer_dt_from_index(X.index) if X.index.size > 1 else 1.0
+    fs = 1.0 / dt
+    win = get_window(window, X.shape[0], fftbins=True)
+
+    # Build a tidy dataframe of periodograms for all columns
+    out = {}
+    for c in Xd.columns:
+        f, Pxx = periodogram(
+            Xd[c].values,
+            fs=fs,
+            window=win,
+            detrend=detrend,
+            scaling="spectrum",  # integrates to variance
+            return_onesided=True,
+        )
+        out[c] = Pxx
+    ps = pd.DataFrame(out, index=pd.Index(f, name="frequency"))
+    return ps  # units: (data units)^2, integrates (sum * df) to variance per column
+
+
+def pct_power_below(ps: pd.DataFrame, max_freq: float) -> pd.Series:
+    # ps is spectrum from power_spectrum_df (one-sided, DC included but we demeaned)
+    # Compute integrals via the rectangle rule: sum * df (df = freq spacing)
+    if len(ps.index) < 2:
+        return pd.Series({c: np.nan for c in ps.columns}, name="pct_power_at_low_freq")
+    df = ps.index[1] - ps.index[0]
+    mask_low = ps.index <= max_freq
+    num: pd.Series = ps.loc[mask_low].sum() * df
+    den: pd.Series = ps.sum() * df
+    s = num / den
+    s.name = "pct_power_at_low_freq"
+    return s

sclab/tools/differential_expression/__init__.py

@@ -1,5 +1,7 @@
 from ._pseudobulk_edger import pseudobulk_edger
+from ._pseudobulk_limma import pseudobulk_limma
 
 __all__ = [
     "pseudobulk_edger",
+    "pseudobulk_limma",
 ]

sclab/tools/differential_expression/_pseudobulk_edger.py

@@ -12,9 +12,9 @@ def pseudobulk_edger(
     cell_identity_key: str | None = None,
     batch_key: str | None = None,
     layer: str | None = None,
-    replicas_per_group: int = 10,
+    replicas_per_group: int = 5,
     min_cells_per_group: int = 30,
-    bootstrap_sampling: bool = True,
+    bootstrap_sampling: bool = False,
     use_cells: dict[str, list[str]] | None = None,
     aggregate: bool = True,
     verbosity: int = 0,
@@ -134,7 +134,7 @@ def pseudobulk_edger(
 
         try:
             R(f"""
-                outs <- fit_model(aggr_adata, "{gk}", "{cell_identity_key}", "{batch_key}", verbosity = {verbosity})
+                outs <- fit_edger_model(aggr_adata, "{gk}", "{cell_identity_key}", "{batch_key}", verbosity = {verbosity})
                 fit <- outs$fit
                 y <- outs$y
             """)
@@ -214,51 +214,58 @@ suppressPackageStartupMessages({
     library(MAST)
 })
 
-fit_model <- function(adata_, group_key, cell_identity_key = "None", batch_key = "None", verbosity = 0){
+fit_edger_model <- function(adata_, group_key, cell_identity_key = "None", batch_key = "None", verbosity = 0){
 
     if (verbosity > 0){
         cat("Group key:", group_key, "\n")
         cat("Cell identity key:", cell_identity_key, "\n")
     }
 
-    # create an edgeR object with counts and grouping factor
-    y <- DGEList(assay(adata_, "X"), group = colData(adata_)[[group_key]])
-    # filter out genes with low counts
-    if (verbosity > 1){
-        cat("Dimensions before subsetting:", dim(y), "\n")
-    }
-    keep <- filterByExpr(y)
-    y <- y[keep, , keep.lib.sizes=FALSE]
-    if (verbosity > 1){
-        cat("Dimensions after subsetting:", dim(y), "\n")
-    }
-
-    # normalize
-    y <- calcNormFactors(y)
     # create a vector that is concatentation of condition and cell type that we will later use with contrasts
     if (cell_identity_key == "None"){
         group <- colData(adata_)[[group_key]]
     } else {
         group <- paste0(colData(adata_)[[group_key]], "_", colData(adata_)[[cell_identity_key]])
     }
+
     if (verbosity > 1){
         cat("Group(s):", group, "\n")
     }
 
-    replica <- colData(adata_)$replica
+    group   <- factor(group)
+    replica <- factor(colData(adata_)$replica)
 
     # create a design matrix
     if (batch_key == "None"){
         design <- model.matrix(~ 0 + group + replica)
     } else {
-        batch <- colData(adata_)[[batch_key]]
+        batch  <- factor(colData(adata_)[[batch_key]])
         design <- model.matrix(~ 0 + group + replica + batch)
     }
+    colnames(design) <- make.names(colnames(design))
+
+    # create an edgeR object with counts and grouping factor
+    y <- DGEList(assay(adata_, "X"), group = group)
+
+    # filter out genes with low counts
+    if (verbosity > 1){
+        cat("Dimensions before subsetting:", dim(y), "\n")
+    }
+
+    keep <- filterByExpr(y, design = design)
+    y <- y[keep, , keep.lib.sizes=FALSE]
+    if (verbosity > 1){
+        cat("Dimensions after subsetting:", dim(y), "\n")
+    }
+
+    # normalize
+    y <- calcNormFactors(y)
 
     # estimate dispersion
     y <- estimateDisp(y, design = design)
     # fit the model
     fit <- glmQLFit(y, design)
+
     return(list("fit"=fit, "design"=design, "y"=y))
 }
 """
@@ -282,9 +289,7 @@ def _try_imports():
     except ModuleNotFoundError:
         message = (
             "edger_pseudobulk requires rpy2 and anndata2ri to be installed.\n"
-            "or\n"
-            "$ pip install rpy2 sclab-tools[r]\n"
-            "or\n"
+            "please install with one of the following:\n"
             "$ pip install rpy2 anndata2ri\n"
             "or\n"
             "$ conda install -c conda-forge rpy2 anndata2ri\n"

sclab/tools/differential_expression/_pseudobulk_helpers.py

@@ -18,6 +18,8 @@ def aggregate_and_filter(
     min_cells_per_group: int = 30,
     bootstrap_sampling: bool = False,
     use_cells: dict[str, list[str]] | None = None,
+    make_stats: bool = True,
+    make_dummies: bool = True,
 ) -> AnnData:
     """
     Aggregate and filter cells in an AnnData object into cell populations.
@@ -40,6 +42,10 @@ def aggregate_and_filter(
         Whether to use bootstrap sampling to create replicas. Defaults to False.
     use_cells : dict[str, list[str]], optional
         If not None, only use the specified cells. Defaults to None.
+    make_stats : bool, optional
+        Whether to create expression statistics for each group. Defaults to True.
+    make_dummies : bool, optional
+        Whether to make categorical columns into dummies. Defaults to True.
 
     Returns
     -------
@@ -61,7 +67,12 @@ def aggregate_and_filter(
         _prepare_categorical_column(adata, cell_identity_key)
         cell_identity_dtype = adata.obs[cell_identity_key].dtype
 
-    var_dataframe = _create_var_dataframe(adata, layer, grouping_keys, groups_to_drop)
+    if make_stats:
+        var_dataframe = _create_var_dataframe(
+            adata, layer, grouping_keys, groups_to_drop
+        )
+    else:
+        var_dataframe = pd.DataFrame(index=adata.var_names)
 
     data = {}
     meta = {}
@@ -100,6 +111,7 @@ def aggregate_and_filter(
     data = pd.DataFrame(data).T
     meta = pd.DataFrame(meta).T
     meta["replica"] = meta["replica"].astype("category")
+    meta["replica_size"] = meta["replica_size"].astype(int)
     meta[group_key] = meta[group_key].astype(group_dtype)
     if cell_identity_key is not None:
         meta[cell_identity_key] = meta[cell_identity_key].astype(cell_identity_dtype)
@@ -110,7 +122,8 @@ def aggregate_and_filter(
         var=var_dataframe,
     )
 
-    _join_dummies(aggr_adata, group_key)
+    if make_dummies:
+        _join_dummies(aggr_adata, group_key)
 
     return aggr_adata