sclab 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

sclab/tools/differential_expression/_pseudobulk_limma.py

@@ -0,0 +1,257 @@
+import pandas as pd
+from anndata import AnnData
+
+from ._pseudobulk_helpers import aggregate_and_filter
+
+
+def pseudobulk_limma(
+    adata_: AnnData,
+    group_key: str,
+    condition_group: str | list[str] | None = None,
+    reference_group: str | None = None,
+    cell_identity_key: str | None = None,
+    batch_key: str | None = None,
+    layer: str | None = None,
+    replicas_per_group: int = 5,
+    min_cells_per_group: int = 30,
+    bootstrap_sampling: bool = False,
+    use_cells: dict[str, list[str]] | None = None,
+    aggregate: bool = True,
+    verbosity: int = 0,
+) -> dict[str, pd.DataFrame]:
+    _try_imports()
+    import anndata2ri  # noqa: F401
+    import rpy2.robjects as robjects
+    from rpy2.rinterface_lib.embedded import RRuntimeError  # noqa: F401
+    from rpy2.robjects import pandas2ri  # noqa: F401
+    from rpy2.robjects.conversion import localconverter  # noqa: F401
+
+    R = robjects.r
+
+    if aggregate:
+        aggr_adata = aggregate_and_filter(
+            adata_,
+            group_key,
+            cell_identity_key,
+            layer,
+            replicas_per_group,
+            min_cells_per_group,
+            bootstrap_sampling,
+            use_cells,
+        )
+    else:
+        aggr_adata = adata_.copy()
+
+    with localconverter(anndata2ri.converter):
+        R.assign("aggr_adata", aggr_adata)
+
+    # defines the R function for fitting the model with limma
+    R(_fit_model_r_script)
+
+    if condition_group is None:
+        condition_group_list = aggr_adata.obs[group_key].unique()
+    elif isinstance(condition_group, str):
+        condition_group_list = [condition_group]
+    else:
+        condition_group_list = condition_group
+
+    if cell_identity_key is not None:
+        cids = aggr_adata.obs[cell_identity_key].unique()
+    else:
+        cids = [""]
+
+    tt_dict = {}
+    for condition_group in condition_group_list:
+        if reference_group is not None and condition_group == reference_group:
+            continue
+
+        if verbosity > 0:
+            print(f"Fitting model for {condition_group}...")
+
+        if reference_group is not None:
+            gk = group_key
+        else:
+            gk = f"{group_key}_{condition_group}"
+
+        try:
+            R(f"""
+                outs <- fit_limma_model(aggr_adata, "{gk}", "{cell_identity_key}", verbosity = {verbosity})
+                fit <- outs$fit
+                v <- outs$v
+            """)
+
+        except RRuntimeError as e:
+            print("Error fitting model for", condition_group)
+            print("Error:", e)
+            print("Skipping...", flush=True)
+            continue
+
+        if reference_group is None:
+            new_contrasts_tuples = [
+                (
+                    condition_group,  # common prefix
+                    "",  # condition group
+                    "not",  # reference group
+                    cid,  # cell identity
+                )
+                for cid in cids
+            ]
+
+        else:
+            new_contrasts_tuples = [
+                (
+                    "",  # common prefix
+                    condition_group,  # condition group
+                    reference_group,  # reference group
+                    cid,  # cell identity
+                )
+                for cid in cids
+            ]
+
+        new_contrasts = [
+            f"group{cnd}{prefix}_{cid}".strip("_")
+            + "-"
+            + f"group{ref}{prefix}_{cid}".strip("_")
+            for prefix, cnd, ref, cid in new_contrasts_tuples
+        ]
+
+        for contrast, contrast_tuple in zip(new_contrasts, new_contrasts_tuples):
+            prefix, cnd, ref, cid = contrast_tuple
+
+            if ref == "not":
+                cnd, ref = "", "rest"
+
+            contrast_key = f"{prefix}{cnd}_vs_{ref}"
+            if cid:
+                contrast_key = f"{cell_identity_key}:{cid}|{contrast_key}"
+
+            if verbosity > 0:
+                print(f"Computing contrast: {contrast_key}... ({contrast})")
+
+            R(f"myContrast <- makeContrasts('{contrast}', levels = v$design)")
+            R("fit2 <- contrasts.fit(fit, myContrast)")
+            R("fit2 <- eBayes(fit2)")
+            R("tt <- topTable(fit2, n = Inf)")
+            tt: pd.DataFrame = pandas2ri.rpy2py(R("tt"))
+            tt.index.name = "gene_ids"
+
+            genes = tt.index
+            cnd, ref = [c[5:] for c in contrast.split("-")]
+            tt["pct_expr_cnd"] = aggr_adata.var[f"pct_expr_{cnd}"].loc[genes]
+            tt["pct_expr_ref"] = aggr_adata.var[f"pct_expr_{ref}"].loc[genes]
+            tt["num_expr_cnd"] = aggr_adata.var[f"num_expr_{cnd}"].loc[genes]
+            tt["num_expr_ref"] = aggr_adata.var[f"num_expr_{ref}"].loc[genes]
+            tt["tot_expr_cnd"] = aggr_adata.var[f"tot_expr_{cnd}"].loc[genes]
+            tt["tot_expr_ref"] = aggr_adata.var[f"tot_expr_{ref}"].loc[genes]
+            tt["mean_cnd"] = tt["tot_expr_cnd"] / tt["num_expr_cnd"]
+            tt["mean_ref"] = tt["tot_expr_ref"] / tt["num_expr_ref"]
+            tt_dict[contrast_key] = tt
+
+    return tt_dict
+
+
+_fit_model_r_script = """
+suppressPackageStartupMessages({
+    library(edgeR)
+    library(limma)
+    library(MAST)
+})
+
+fit_limma_model <- function(adata_, group_key, cell_identity_key = "None", batch_key = "None", verbosity = 0){
+
+    if (verbosity > 0){
+        cat("Group key:", group_key, "\n")
+        cat("Cell identity key:", cell_identity_key, "\n")
+    }
+
+    # create a vector that is concatentation of condition and cell type that we will later use with contrasts
+    if (cell_identity_key == "None"){
+        group <- colData(adata_)[[group_key]]
+    } else {
+        group <- paste0(colData(adata_)[[group_key]], "_", colData(adata_)[[cell_identity_key]])
+    }
+
+    if (verbosity > 1){
+        cat("Group(s):", group, "\n")
+    }
+
+    group   <- factor(group)
+    replica <- factor(colData(adata_)$replica)
+
+    # create a design matrix
+    if (batch_key == "None"){
+        design <- model.matrix(~ 0 + group + replica)
+    } else {
+        batch  <- factor(colData(adata_)[[batch_key]])
+        design <- model.matrix(~ 0 + group + replica + batch)
+    }
+    colnames(design) <- make.names(colnames(design))
+
+    # create an edgeR object with counts and grouping factor
+    y <- DGEList(assay(adata_, "X"), group = group)
+
+    # filter out genes with low counts
+    if (verbosity > 1){
+        cat("Dimensions before subsetting:", dim(y), "\n")
+    }
+
+    keep <- filterByExpr(y, design = design)
+    y <- y[keep, , keep.lib.sizes=FALSE]
+    if (verbosity > 1){
+        cat("Dimensions after subsetting:", dim(y), "\n")
+    }
+
+    # normalize
+    y <- calcNormFactors(y)
+
+    # Apply voom transformation to prepare for linear modeling
+    v <- voom(y, design = design)
+
+    # fit the linear model
+    fit <- lmFit(v, design)
+    ne <- limma::nonEstimable(design)
+    if (!is.null(ne) && verbosity > 0) cat("Non-estimable:", ne, "\n")
+    fit <- eBayes(fit)
+
+    return(list("fit"=fit, "design"=design, "v"=v))
+}
+"""
+
+
+def _try_imports():
+    try:
+        import rpy2.robjects as robjects
+        from rpy2.robjects.packages import PackageNotInstalledError, importr
+
+        robjects.r("options(warn=-1)")
+        import anndata2ri  # noqa: F401
+        from rpy2.rinterface_lib.embedded import RRuntimeError  # noqa: F401
+        from rpy2.robjects import numpy2ri, pandas2ri  # noqa: F401
+        from rpy2.robjects.conversion import localconverter  # noqa: F401
+
+        importr("edgeR")
+        importr("limma")
+        importr("MAST")
+        importr("SingleCellExperiment")
+
+    except ModuleNotFoundError:
+        message = (
+            "pseudobulk_limma requires rpy2 and anndata2ri to be installed.\n"
+            "please install with one of the following:\n"
+            "$ pip install rpy2 anndata2ri\n"
+            "or\n"
+            "$ conda install -c conda-forge rpy2 anndata2ri\n"
+        )
+        print(message)
+        raise ModuleNotFoundError(message)
+
+    except PackageNotInstalledError:
+        message = (
+            "pseudobulk_limma requires the following R packages to be installed: limma, edgeR, MAST, and SingleCellExperiment.\n"
+            "> \n"
+            "> if (!require('BiocManager', quietly = TRUE)) install.packages('BiocManager');\n"
+            "> BiocManager::install(c('limma', 'edgeR', 'MAST', 'SingleCellExperiment'));\n"
+            "> \n"
+        )
+        print(message)
+        raise ImportError(message)

sclab/tools/imputation/_alra.py

@@ -0,0 +1,135 @@
+import gc
+
+import numpy as np
+from numpy import float32
+from numpy.typing import NDArray
+from scipy.sparse import csc_matrix, csr_matrix, issparse
+
+
+def _alra_on_ndarray(
+    data: NDArray | csr_matrix,
+) -> tuple[NDArray[float32], NDArray[float32]]:
+    """
+    Run ALRA on the given data.
+
+    Parameters
+    ----------
+    data : NDArray | csr_matrix
+        Input data to impute.
+
+    Returns
+    -------
+    data_aprx : NDArray
+        Approximated data.
+    data_alra : NDArray
+        Imputed data.
+    """
+    import rpy2.robjects as robjects
+    import rpy2.robjects.numpy2ri
+    from rpy2.robjects.packages import importr
+
+    rpy2.robjects.numpy2ri.activate()
+    R = robjects.r
+    alra = importr("ALRA")
+
+    if issparse(data):
+        data = np.ascontiguousarray(data.todense("C"), dtype=np.float32)
+
+    # convert to R object
+    r_X = R.matrix(data, nrow=data.shape[0], ncol=data.shape[1])
+    # run ALRA
+    r_res = alra.alra(r_X, 0, 10, 0.001)
+    # retrieve imputed data
+    r_K = r_res[0]  # rank k
+    r_T = r_res[1]  # rank k thresholded
+    r_S = r_res[2]  # rank k thresholded scaled
+    # convert back to numpy array
+    data_aprx = np.array(r_K, dtype=float32)
+    data_thrs = np.array(r_T, dtype=float32)
+    data_alra = np.array(r_S, dtype=float32)
+
+    # clean up
+    del (r_X, r_res, r_K, r_T, r_S)
+    R("gc()")
+    gc.collect()
+
+    return data_aprx, data_thrs, data_alra
+
+
+def _fix_alra_scale(
+    input_data: NDArray | csr_matrix | csc_matrix,
+    thrs_data: NDArray,
+    target_data: NDArray,
+) -> NDArray:
+    # Convert sparse -> dense
+    if issparse(input_data):
+        input_data = input_data.toarray("C")
+        input_data = input_data.astype(np.float32)
+        input_data = np.ascontiguousarray(input_data, dtype=np.float32)
+
+    n_cells, n_genes = input_data.shape
+
+    # per-gene nonzero means/sds (match R: sample sd ddof=1)
+    input_means = np.full(n_genes, fill_value=np.nan)
+    input_stds = np.full(n_genes, fill_value=np.nan)
+    thrs_means = np.full(n_genes, fill_value=np.nan)
+    thrs_stds = np.full(n_genes, fill_value=np.nan)
+    v: NDArray
+
+    for i, e in enumerate(input_data.T):
+        v = e[e > 0]
+
+        if v.size == 0:
+            continue
+        input_means[i] = v.mean()
+
+        if v.size == 1:
+            continue
+        input_stds[i] = v.std(ddof=1)
+
+    for i, e in enumerate(thrs_data.T):
+        v = e[e > 0]
+
+        if v.size == 0:
+            continue
+        thrs_means[i] = v.mean()
+
+        if v.size == 1:
+            continue
+        thrs_stds[i] = v.std(ddof=1)
+
+    # columns to scale (mirror R's toscale)
+    toscale = (
+        ~np.isnan(thrs_stds)
+        & ~np.isnan(input_stds)
+        & ~((thrs_stds == 0) & (input_stds == 0))
+        & ~(thrs_stds == 0)
+    )
+
+    # affine params
+    a = np.full(n_genes, fill_value=1.0)
+    b = np.full(n_genes, fill_value=0.0)
+    a[toscale] = input_stds[toscale] / thrs_stds[toscale]
+    b[toscale] = input_means[toscale] - a[toscale] * thrs_means[toscale]
+
+    # apply to target matrix (only columns in toscale)
+    out = target_data.copy()
+    out[:, toscale] = out[:, toscale] * a[toscale] + b[toscale]
+
+    # keep zeros as zeros
+    out[thrs_data == 0] = 0
+
+    # clip negatives to zero
+    out[out < 0] = 0
+
+    # restore originally observed positives that became zero
+    mask = (input_data > 0) & (out == 0)
+    out[mask] = input_data[mask]
+
+    return out
+
+
+__all__ = [
+    "_alra_on_ndarray",
+    "_fix_alra_scale",
+]

sclab/tools/utils/__init__.py

@@ -0,0 +1,5 @@
+from ._aggregate_and_filter import aggregate_and_filter
+
+__all__ = [
+    "aggregate_and_filter",
+]

sclab/tools/utils/_aggregate_and_filter.py

@@ -0,0 +1,290 @@
+import random
+
+import numpy as np
+import pandas as pd
+from anndata import AnnData
+from numpy import ndarray
+from scipy.sparse import csr_matrix, issparse
+
+
+# code inspired from
+# https://www.sc-best-practices.org/conditions/differential_gene_expression.html
+def aggregate_and_filter(
+    adata: AnnData,
+    group_key: str = "batch",
+    cell_identity_key: str | None = None,
+    layer: str | None = None,
+    replicas_per_group: int = 3,
+    min_cells_per_group: int = 30,
+    bootstrap_sampling: bool = False,
+    use_cells: dict[str, list[str]] | None = None,
+    make_stats: bool = True,
+    make_dummies: bool = True,
+) -> AnnData:
+    """
+    Aggregate and filter cells in an AnnData object into cell populations.
+
+    Parameters
+    ----------
+    adata : AnnData
+        AnnData object to aggregate and filter.
+    group_key : str, optional
+        Key to group cells by. Defaults to 'batch'.
+    cell_identity_key : str, optional
+        Key to use to identify cell identities. Defaults to None.
+    layer : str, optional
+        Layer in AnnData object to use for aggregation. Defaults to None.
+    replicas_per_group : int, optional
+        Number of replicas to create for each group. Defaults to 3.
+    min_cells_per_group : int, optional
+        Minimum number of cells required for a group to be included. Defaults to 30.
+    bootstrap_sampling : bool, optional
+        Whether to use bootstrap sampling to create replicas. Defaults to False.
+    use_cells : dict[str, list[str]], optional
+        If not None, only use the specified cells. Defaults to None.
+    make_stats : bool, optional
+        Whether to create expression statistics for each group. Defaults to True.
+    make_dummies : bool, optional
+        Whether to make categorical columns into dummies. Defaults to True.
+
+    Returns
+    -------
+    AnnData
+        AnnData object with aggregated and filtered cells.
+    """
+    adata = _prepare_dataset(adata, use_cells)
+
+    grouping_keys = [group_key]
+    if cell_identity_key is not None:
+        grouping_keys.append(cell_identity_key)
+
+    groups_to_drop = _get_groups_to_drop(adata, grouping_keys, min_cells_per_group)
+
+    _prepare_categorical_column(adata, group_key)
+    group_dtype = adata.obs[group_key].dtype
+
+    if cell_identity_key is not None:
+        _prepare_categorical_column(adata, cell_identity_key)
+        cell_identity_dtype = adata.obs[cell_identity_key].dtype
+
+    if make_stats:
+        var_dataframe = _create_var_dataframe(
+            adata, layer, grouping_keys, groups_to_drop
+        )
+    else:
+        var_dataframe = pd.DataFrame(index=adata.var_names)
+
+    data = {}
+    meta = {}
+    groups = adata.obs.groupby(grouping_keys, observed=True).groups
+    for group, group_idxs in groups.items():
+        if not isinstance(group, tuple):
+            group = (group,)
+
+        if not _including(group, groups_to_drop):
+            continue
+
+        sample_id = "_".join(group)
+        match group:
+            case (gid, cid):
+                group_metadata = {group_key: gid, cell_identity_key: cid}
+            case (gid,):
+                group_metadata = {group_key: gid}
+
+        adata_group = adata[group_idxs]
+        indices = _get_replica_idxs(adata_group, replicas_per_group, bootstrap_sampling)
+        for i, rep_idx in enumerate(indices):
+            replica_number = i + 1
+            replica_size = len(rep_idx)
+            replica_sample_id = f"{sample_id}_rep{replica_number}"
+
+            adata_group_replica = adata_group[rep_idx]
+            X = _get_layer(adata_group_replica, layer)
+
+            data[replica_sample_id] = np.array(X.sum(axis=0)).flatten()
+            meta[replica_sample_id] = {
+                **group_metadata,
+                "replica": str(replica_number),
+                "replica_size": replica_size,
+            }
+
+    data = pd.DataFrame(data).T
+    meta = pd.DataFrame(meta).T
+    meta["replica"] = meta["replica"].astype("category")
+    meta["replica_size"] = meta["replica_size"].astype(int)
+    meta[group_key] = meta[group_key].astype(group_dtype)
+    if cell_identity_key is not None:
+        meta[cell_identity_key] = meta[cell_identity_key].astype(cell_identity_dtype)
+
+    aggr_adata = AnnData(
+        data.values,
+        obs=meta,
+        var=var_dataframe,
+    )
+
+    if make_dummies:
+        _join_dummies(aggr_adata, group_key)
+
+    return aggr_adata
+
+
+def _prepare_dataset(
+    adata: AnnData,
+    use_cells: dict[str, list[str]] | None,
+) -> AnnData:
+    if use_cells is not None:
+        for key, value in use_cells.items():
+            adata = adata[adata.obs[key].isin(value)]
+
+    return adata.copy()
+
+
+def _get_groups_to_drop(
+    adata: AnnData,
+    grouping_keys: str | list[str],
+    min_cells_per_group: int,
+):
+    group_sizes = adata.obs.groupby(grouping_keys, observed=True).size()
+    groups_to_drop = group_sizes[group_sizes < min_cells_per_group].index.to_list()
+
+    if len(groups_to_drop) > 0:
+        print("Dropping the following samples:")
+
+    groups_to_drop = groups_to_drop + [
+        (g,) for g in groups_to_drop if not isinstance(g, tuple)
+    ]
+
+    return groups_to_drop
+
+
+def _prepare_categorical_column(adata: AnnData, column: str) -> None:
+    if not isinstance(adata.obs[column].dtype, pd.CategoricalDtype):
+        adata.obs[column] = adata.obs[column].astype("category")
+
+
+def _create_var_dataframe(
+    adata: AnnData,
+    layer: str,
+    grouping_keys: list[str],
+    groups_to_drop: list[str],
+):
+    columns = _get_var_dataframe_columns(adata, grouping_keys, groups_to_drop)
+    var_dataframe = pd.DataFrame(index=adata.var_names, columns=columns, dtype=float)
+
+    groups = adata.obs.groupby(grouping_keys, observed=True).groups
+    for group, idx in groups.items():
+        if not isinstance(group, tuple):
+            group = (group,)
+
+        if not _including(group, groups_to_drop):
+            continue
+
+        sample_id = "_".join(group)
+        rest_id = f"not{sample_id}"
+
+        adata_subset = adata[idx]
+        rest_subset = adata[~adata.obs_names.isin(idx)]
+
+        X = _get_layer(adata_subset, layer, dense=True)
+        Y = _get_layer(rest_subset, layer, dense=True)
+
+        var_dataframe[f"pct_expr_{sample_id}"] = (X > 0).mean(axis=0)
+        var_dataframe[f"pct_expr_{rest_id}"] = (Y > 0).mean(axis=0)
+        var_dataframe[f"num_expr_{sample_id}"] = (X > 0).sum(axis=0)
+        var_dataframe[f"num_expr_{rest_id}"] = (Y > 0).sum(axis=0)
+        var_dataframe[f"tot_expr_{sample_id}"] = X.sum(axis=0)
+        var_dataframe[f"tot_expr_{rest_id}"] = Y.sum(axis=0)
+
+    return var_dataframe
+
+
+def _get_var_dataframe_columns(
+    adata: AnnData, grouping_keys: list[str], groups_to_drop: list[str]
+) -> list[str]:
+    columns = []
+
+    groups = adata.obs.groupby(grouping_keys, observed=True).groups
+    for group, _ in groups.items():
+        if not isinstance(group, tuple):
+            group = (group,)
+
+        if not _including(group, groups_to_drop):
+            continue
+
+        sample_id = "_".join(group)
+        rest_id = f"not{sample_id}"
+
+        columns.extend(
+            [
+                f"pct_expr_{sample_id}",
+                f"pct_expr_{rest_id}",
+                f"num_expr_{sample_id}",
+                f"num_expr_{rest_id}",
+                f"tot_expr_{sample_id}",
+                f"tot_expr_{rest_id}",
+            ]
+        )
+
+    return columns
+
+
+def _including(group: tuple | str, groups_to_drop: list[str]) -> bool:
+    match group:
+        case (gid, cid):
+            if isinstance(cid, float) and np.isnan(cid):
+                return False
+
+        case (gid,) | gid:
+            ...
+
+    if gid in groups_to_drop:
+        return False
+
+    return True
+
+
+def _get_replica_idxs(
+    adata_group: AnnData,
+    replicas_per_group: int,
+    bootstrap_sampling: bool,
+):
+    group_size = adata_group.n_obs
+    indices = list(adata_group.obs_names)
+    if bootstrap_sampling:
+        indices = np.array(
+            [
+                np.random.choice(indices, size=group_size, replace=True)
+                for _ in range(replicas_per_group)
+            ]
+        )
+
+    else:
+        random.shuffle(indices)
+        indices = np.array_split(np.array(indices), replicas_per_group)
+
+    return indices
+
+
+def _get_layer(adata: AnnData, layer: str | None, dense: bool = False):
+    X: ndarray | csr_matrix
+
+    if layer is None or layer == "X":
+        X = adata.X
+    else:
+        X = adata.layers[layer]
+
+    if dense:
+        if issparse(X):
+            X = np.asarray(X.todense())
+        else:
+            X = np.asarray(X)
+
+    return X
+
+
+def _join_dummies(aggr_adata: AnnData, group_key: str) -> None:
+    dummies = pd.get_dummies(aggr_adata.obs[group_key], prefix=group_key).astype(str)
+    dummies = dummies.astype(str).apply(lambda s: s.map({"True": "", "False": "not"}))
+    dummies = dummies + aggr_adata.obs[group_key].cat.categories
+
+    aggr_adata.obs = aggr_adata.obs.join(dummies)