sclab 0.1.7__py3-none-any.whl → 0.3.4__py3-none-any.whl

sclab/tools/differential_expression/_pseudobulk_edger.py

@@ -0,0 +1,309 @@
+import pandas as pd
+from anndata import AnnData
+
+from ._pseudobulk_helpers import aggregate_and_filter
+
+
+def pseudobulk_edger(
+    adata_: AnnData,
+    group_key: str,
+    condition_group: str | list[str] | None = None,
+    reference_group: str | None = None,
+    cell_identity_key: str | None = None,
+    batch_key: str | None = None,
+    layer: str | None = None,
+    replicas_per_group: int = 5,
+    min_cells_per_group: int = 30,
+    bootstrap_sampling: bool = False,
+    use_cells: dict[str, list[str]] | None = None,
+    aggregate: bool = True,
+    verbosity: int = 0,
+) -> dict[str, pd.DataFrame]:
+    """
+    Fits a model using edgeR and computes top tags for a given condition vs
+    reference group.
+
+    Parameters
+    ----------
+    adata_ : AnnData
+        Annotated data matrix.
+    group_key : str
+        Key in AnnData object to use to group cells.
+    condition_group : str | list[str] | None, optional
+        Condition group to compare to reference group. If None, each group will be
+        contrasted to the corresponding reference group.
+    reference_group : str | None, optional
+        Reference group to compare condition group(s) to. If None, the condition group
+        is compared to the rest of the cells.
+    cell_identity_key : str | None, optional
+        If provided, separate contrasts will be computed for each identity. Defaults to None.
+    layer : str | None, optional
+        Layer in AnnData object to use. EdgeR requires raw counts. Defaults to None.
+    replicas_per_group : int, optional
+        Number of replicas to create for each group. Defaults to 10.
+    min_cells_per_group : int, optional
+        Minimum number of cells required for a group to be included. Defaults to 30.
+    bootstrap_sampling : bool, optional
+        Whether to use bootstrap sampling to create replicas. Defaults to True.
+    use_cells : dict[str, list[str]] | None, optional
+        If not None, only use the specified cells. Defaults to None. Dictionary key
+        is a categorical variable in the obs dataframe and the dictionary value is a
+        list of categories to include.
+    aggregate : bool, optional
+        Whether to aggregate cells before fitting the model. EdgeR requires a small
+        number of samples, so if adata_ is a single-cell experiment, the cells should
+        be aggregated. Defaults to True.
+    verbosity : int, optional
+        Verbosity level. Defaults to 0.
+
+    Returns
+    -------
+    dict[str, pd.DataFrame]
+        Dictionary of dataframes, one for each contrast, with the following columns:
+
+        * gene_ids : str
+            Gene IDs.
+        * logFC : float
+            Log2 fold change.
+        * logCPM : float
+            Log2 CPM.
+        * F: float
+            F-statistic.
+        * PValue : float
+            p-value.
+        * FDR : float
+            False discovery rate.
+        * pct_expr_cnd : float
+            Percentage of cells in condition group expressing the gene.
+        * pct_expr_ref : float
+            Percentage of cells in reference group expressing the gene.
+    """
+    _try_imports()
+    import anndata2ri  # noqa: F401
+    import rpy2.robjects as robjects
+    from rpy2.rinterface_lib.embedded import RRuntimeError  # noqa: F401
+    from rpy2.robjects import pandas2ri  # noqa: F401
+    from rpy2.robjects.conversion import localconverter  # noqa: F401
+
+    R = robjects.r
+
+    if aggregate:
+        aggr_adata = aggregate_and_filter(
+            adata_,
+            group_key,
+            cell_identity_key,
+            layer,
+            replicas_per_group,
+            min_cells_per_group,
+            bootstrap_sampling,
+            use_cells,
+        )
+    else:
+        aggr_adata = adata_.copy()
+
+    with localconverter(anndata2ri.converter):
+        R.assign("aggr_adata", aggr_adata)
+
+    # defines the R function for fitting the model with edgeR
+    R(_fit_model_r_script)
+
+    if condition_group is None:
+        condition_group_list = aggr_adata.obs[group_key].unique()
+    elif isinstance(condition_group, str):
+        condition_group_list = [condition_group]
+    else:
+        condition_group_list = condition_group
+
+    if cell_identity_key is not None:
+        cids = aggr_adata.obs[cell_identity_key].unique()
+    else:
+        cids = [""]
+
+    tt_dict = {}
+    for condition_group in condition_group_list:
+        if reference_group is not None and condition_group == reference_group:
+            continue
+
+        if verbosity > 0:
+            print(f"Fitting model for {condition_group}...")
+
+        if reference_group is not None:
+            gk = group_key
+        else:
+            gk = f"{group_key}_{condition_group}"
+
+        try:
+            R(f"""
+                outs <- fit_edger_model(aggr_adata, "{gk}", "{cell_identity_key}", "{batch_key}", verbosity = {verbosity})
+                fit <- outs$fit
+                y <- outs$y
+            """)
+
+        except RRuntimeError as e:
+            print("Error fitting model for", condition_group)
+            print("Error:", e)
+            print("Skipping...", flush=True)
+            continue
+
+        if reference_group is None:
+            new_contrasts_tuples = [
+                (
+                    condition_group,  # common prefix
+                    "",  # condition group
+                    "not",  # reference group
+                    cid,  # cell identity
+                )
+                for cid in cids
+            ]
+
+        else:
+            new_contrasts_tuples = [
+                (
+                    "",  # common prefix
+                    condition_group,  # condition group
+                    reference_group,  # reference group
+                    cid,  # cell identity
+                )
+                for cid in cids
+            ]
+
+        new_contrasts = [
+            f"group{cnd}{prefix}_{cid}".strip("_")
+            + "-"
+            + f"group{ref}{prefix}_{cid}".strip("_")
+            for prefix, cnd, ref, cid in new_contrasts_tuples
+        ]
+
+        for contrast, contrast_tuple in zip(new_contrasts, new_contrasts_tuples):
+            prefix, cnd, ref, cid = contrast_tuple
+
+            if ref == "not":
+                cnd, ref = "", "rest"
+
+            contrast_key = f"{prefix}{cnd}_vs_{ref}"
+            if cid:
+                contrast_key = f"{cell_identity_key}:{cid}|{contrast_key}"
+
+            if verbosity > 0:
+                print(f"Computing contrast: {contrast_key}... ({contrast})")
+
+            R(f"myContrast <- makeContrasts('{contrast}', levels = y$design)")
+            R("qlf <- glmQLFTest(fit, contrast=myContrast)")
+            R("tt <- topTags(qlf, n = Inf)$table")
+            tt: pd.DataFrame = pandas2ri.rpy2py(R("tt"))
+            tt.index.name = "gene_ids"
+
+            genes = tt.index
+            cnd, ref = [c[5:] for c in contrast.split("-")]
+            tt["pct_expr_cnd"] = aggr_adata.var[f"pct_expr_{cnd}"].loc[genes]
+            tt["pct_expr_ref"] = aggr_adata.var[f"pct_expr_{ref}"].loc[genes]
+            tt["num_expr_cnd"] = aggr_adata.var[f"num_expr_{cnd}"].loc[genes]
+            tt["num_expr_ref"] = aggr_adata.var[f"num_expr_{ref}"].loc[genes]
+            tt["tot_expr_cnd"] = aggr_adata.var[f"tot_expr_{cnd}"].loc[genes]
+            tt["tot_expr_ref"] = aggr_adata.var[f"tot_expr_{ref}"].loc[genes]
+            tt["mean_cnd"] = tt["tot_expr_cnd"] / tt["num_expr_cnd"]
+            tt["mean_ref"] = tt["tot_expr_ref"] / tt["num_expr_ref"]
+            tt_dict[contrast_key] = tt
+
+    return tt_dict
+
+
+_fit_model_r_script = """
+suppressPackageStartupMessages({
+    library(edgeR)
+    library(MAST)
+})
+
+fit_edger_model <- function(adata_, group_key, cell_identity_key = "None", batch_key = "None", verbosity = 0){
+
+    if (verbosity > 0){
+        cat("Group key:", group_key, "\n")
+        cat("Cell identity key:", cell_identity_key, "\n")
+    }
+
+    # create a vector that is concatentation of condition and cell type that we will later use with contrasts
+    if (cell_identity_key == "None"){
+        group <- colData(adata_)[[group_key]]
+    } else {
+        group <- paste0(colData(adata_)[[group_key]], "_", colData(adata_)[[cell_identity_key]])
+    }
+
+    if (verbosity > 1){
+        cat("Group(s):", group, "\n")
+    }
+
+    group   <- factor(group)
+    replica <- factor(colData(adata_)$replica)
+
+    # create a design matrix
+    if (batch_key == "None"){
+        design <- model.matrix(~ 0 + group + replica)
+    } else {
+        batch  <- factor(colData(adata_)[[batch_key]])
+        design <- model.matrix(~ 0 + group + replica + batch)
+    }
+    colnames(design) <- make.names(colnames(design))
+
+    # create an edgeR object with counts and grouping factor
+    y <- DGEList(assay(adata_, "X"), group = group)
+
+    # filter out genes with low counts
+    if (verbosity > 1){
+        cat("Dimensions before subsetting:", dim(y), "\n")
+    }
+
+    keep <- filterByExpr(y, design = design)
+    y <- y[keep, , keep.lib.sizes=FALSE]
+    if (verbosity > 1){
+        cat("Dimensions after subsetting:", dim(y), "\n")
+    }
+
+    # normalize
+    y <- calcNormFactors(y)
+
+    # estimate dispersion
+    y <- estimateDisp(y, design = design)
+    # fit the model
+    fit <- glmQLFit(y, design)
+
+    return(list("fit"=fit, "design"=design, "y"=y))
+}
+"""
+
+
+def _try_imports():
+    try:
+        import rpy2.robjects as robjects
+        from rpy2.robjects.packages import PackageNotInstalledError, importr
+
+        robjects.r("options(warn=-1)")
+        import anndata2ri  # noqa: F401
+        from rpy2.rinterface_lib.embedded import RRuntimeError  # noqa: F401
+        from rpy2.robjects import numpy2ri, pandas2ri  # noqa: F401
+        from rpy2.robjects.conversion import localconverter  # noqa: F401
+
+        importr("edgeR")
+        importr("MAST")
+        importr("SingleCellExperiment")
+
+    except ModuleNotFoundError:
+        message = (
+            "edger_pseudobulk requires rpy2 and anndata2ri to be installed.\n"
+            "please install with one of the following:\n"
+            "$ pip install rpy2 anndata2ri\n"
+            "or\n"
+            "$ conda install -c conda-forge rpy2 anndata2ri\n"
+        )
+        print(message)
+        raise ModuleNotFoundError(message)
+
+    except PackageNotInstalledError:
+        message = (
+            "edger_pseudobulk requires the following R packages to be installed: edgeR, MAST, and SingleCellExperiment.\n"
+            "> \n"
+            "> if (!require('BiocManager', quietly = TRUE)) install.packages('BiocManager');\n"
+            "> BiocManager::install(c('edgeR', 'MAST', 'SingleCellExperiment'));\n"
+            "> \n"
+        )
+        print(message)
+        raise ImportError(message)

sclab/tools/differential_expression/_pseudobulk_helpers.py

@@ -0,0 +1,290 @@
+import random
+
+import numpy as np
+import pandas as pd
+from anndata import AnnData
+from numpy import ndarray
+from scipy.sparse import csr_matrix, issparse
+
+
+# code inspired from
+# https://www.sc-best-practices.org/conditions/differential_gene_expression.html
+def aggregate_and_filter(
+    adata: AnnData,
+    group_key: str = "batch",
+    cell_identity_key: str | None = None,
+    layer: str | None = None,
+    replicas_per_group: int = 3,
+    min_cells_per_group: int = 30,
+    bootstrap_sampling: bool = False,
+    use_cells: dict[str, list[str]] | None = None,
+    make_stats: bool = True,
+    make_dummies: bool = True,
+) -> AnnData:
+    """
+    Aggregate and filter cells in an AnnData object into cell populations.
+
+    Parameters
+    ----------
+    adata : AnnData
+        AnnData object to aggregate and filter.
+    group_key : str, optional
+        Key to group cells by. Defaults to 'batch'.
+    cell_identity_key : str, optional
+        Key to use to identify cell identities. Defaults to None.
+    layer : str, optional
+        Layer in AnnData object to use for aggregation. Defaults to None.
+    replicas_per_group : int, optional
+        Number of replicas to create for each group. Defaults to 3.
+    min_cells_per_group : int, optional
+        Minimum number of cells required for a group to be included. Defaults to 30.
+    bootstrap_sampling : bool, optional
+        Whether to use bootstrap sampling to create replicas. Defaults to False.
+    use_cells : dict[str, list[str]], optional
+        If not None, only use the specified cells. Defaults to None.
+    make_stats : bool, optional
+        Whether to create expression statistics for each group. Defaults to True.
+    make_dummies : bool, optional
+        Whether to make categorical columns into dummies. Defaults to True.
+
+    Returns
+    -------
+    AnnData
+        AnnData object with aggregated and filtered cells.
+    """
+    adata = _prepare_dataset(adata, use_cells)
+
+    grouping_keys = [group_key]
+    if cell_identity_key is not None:
+        grouping_keys.append(cell_identity_key)
+
+    groups_to_drop = _get_groups_to_drop(adata, grouping_keys, min_cells_per_group)
+
+    _prepare_categorical_column(adata, group_key)
+    group_dtype = adata.obs[group_key].dtype
+
+    if cell_identity_key is not None:
+        _prepare_categorical_column(adata, cell_identity_key)
+        cell_identity_dtype = adata.obs[cell_identity_key].dtype
+
+    if make_stats:
+        var_dataframe = _create_var_dataframe(
+            adata, layer, grouping_keys, groups_to_drop
+        )
+    else:
+        var_dataframe = pd.DataFrame(index=adata.var_names)
+
+    data = {}
+    meta = {}
+    groups = adata.obs.groupby(grouping_keys, observed=True).groups
+    for group, group_idxs in groups.items():
+        if not isinstance(group, tuple):
+            group = (group,)
+
+        if not _including(group, groups_to_drop):
+            continue
+
+        sample_id = "_".join(group)
+        match group:
+            case (gid, cid):
+                group_metadata = {group_key: gid, cell_identity_key: cid}
+            case (gid,):
+                group_metadata = {group_key: gid}
+
+        adata_group = adata[group_idxs]
+        indices = _get_replica_idxs(adata_group, replicas_per_group, bootstrap_sampling)
+        for i, rep_idx in enumerate(indices):
+            replica_number = i + 1
+            replica_size = len(rep_idx)
+            replica_sample_id = f"{sample_id}_rep{replica_number}"
+
+            adata_group_replica = adata_group[rep_idx]
+            X = _get_layer(adata_group_replica, layer)
+
+            data[replica_sample_id] = np.array(X.sum(axis=0)).flatten()
+            meta[replica_sample_id] = {
+                **group_metadata,
+                "replica": str(replica_number),
+                "replica_size": replica_size,
+            }
+
+    data = pd.DataFrame(data).T
+    meta = pd.DataFrame(meta).T
+    meta["replica"] = meta["replica"].astype("category")
+    meta["replica_size"] = meta["replica_size"].astype(int)
+    meta[group_key] = meta[group_key].astype(group_dtype)
+    if cell_identity_key is not None:
+        meta[cell_identity_key] = meta[cell_identity_key].astype(cell_identity_dtype)
+
+    aggr_adata = AnnData(
+        data.values,
+        obs=meta,
+        var=var_dataframe,
+    )
+
+    if make_dummies:
+        _join_dummies(aggr_adata, group_key)
+
+    return aggr_adata
+
+
+def _prepare_dataset(
+    adata: AnnData,
+    use_cells: dict[str, list[str]] | None,
+) -> AnnData:
+    if use_cells is not None:
+        for key, value in use_cells.items():
+            adata = adata[adata.obs[key].isin(value)]
+
+    return adata.copy()
+
+
+def _get_groups_to_drop(
+    adata: AnnData,
+    grouping_keys: str | list[str],
+    min_cells_per_group: int,
+):
+    group_sizes = adata.obs.groupby(grouping_keys, observed=True).size()
+    groups_to_drop = group_sizes[group_sizes < min_cells_per_group].index.to_list()
+
+    if len(groups_to_drop) > 0:
+        print("Dropping the following samples:")
+
+    groups_to_drop = groups_to_drop + [
+        (g,) for g in groups_to_drop if not isinstance(g, tuple)
+    ]
+
+    return groups_to_drop
+
+
+def _prepare_categorical_column(adata: AnnData, column: str) -> None:
+    if not isinstance(adata.obs[column].dtype, pd.CategoricalDtype):
+        adata.obs[column] = adata.obs[column].astype("category")
+
+
+def _create_var_dataframe(
+    adata: AnnData,
+    layer: str,
+    grouping_keys: list[str],
+    groups_to_drop: list[str],
+):
+    columns = _get_var_dataframe_columns(adata, grouping_keys, groups_to_drop)
+    var_dataframe = pd.DataFrame(index=adata.var_names, columns=columns, dtype=float)
+
+    groups = adata.obs.groupby(grouping_keys, observed=True).groups
+    for group, idx in groups.items():
+        if not isinstance(group, tuple):
+            group = (group,)
+
+        if not _including(group, groups_to_drop):
+            continue
+
+        sample_id = "_".join(group)
+        rest_id = f"not{sample_id}"
+
+        adata_subset = adata[idx]
+        rest_subset = adata[~adata.obs_names.isin(idx)]
+
+        X = _get_layer(adata_subset, layer, dense=True)
+        Y = _get_layer(rest_subset, layer, dense=True)
+
+        var_dataframe[f"pct_expr_{sample_id}"] = (X > 0).mean(axis=0)
+        var_dataframe[f"pct_expr_{rest_id}"] = (Y > 0).mean(axis=0)
+        var_dataframe[f"num_expr_{sample_id}"] = (X > 0).sum(axis=0)
+        var_dataframe[f"num_expr_{rest_id}"] = (Y > 0).sum(axis=0)
+        var_dataframe[f"tot_expr_{sample_id}"] = X.sum(axis=0)
+        var_dataframe[f"tot_expr_{rest_id}"] = Y.sum(axis=0)
+
+    return var_dataframe
+
+
+def _get_var_dataframe_columns(
+    adata: AnnData, grouping_keys: list[str], groups_to_drop: list[str]
+) -> list[str]:
+    columns = []
+
+    groups = adata.obs.groupby(grouping_keys, observed=True).groups
+    for group, _ in groups.items():
+        if not isinstance(group, tuple):
+            group = (group,)
+
+        if not _including(group, groups_to_drop):
+            continue
+
+        sample_id = "_".join(group)
+        rest_id = f"not{sample_id}"
+
+        columns.extend(
+            [
+                f"pct_expr_{sample_id}",
+                f"pct_expr_{rest_id}",
+                f"num_expr_{sample_id}",
+                f"num_expr_{rest_id}",
+                f"tot_expr_{sample_id}",
+                f"tot_expr_{rest_id}",
+            ]
+        )
+
+    return columns
+
+
+def _including(group: tuple | str, groups_to_drop: list[str]) -> bool:
+    match group:
+        case (gid, cid):
+            if isinstance(cid, float) and np.isnan(cid):
+                return False
+
+        case (gid,) | gid:
+            ...
+
+    if gid in groups_to_drop:
+        return False
+
+    return True
+
+
+def _get_replica_idxs(
+    adata_group: AnnData,
+    replicas_per_group: int,
+    bootstrap_sampling: bool,
+):
+    group_size = adata_group.n_obs
+    indices = list(adata_group.obs_names)
+    if bootstrap_sampling:
+        indices = np.array(
+            [
+                np.random.choice(indices, size=group_size, replace=True)
+                for _ in range(replicas_per_group)
+            ]
+        )
+
+    else:
+        random.shuffle(indices)
+        indices = np.array_split(np.array(indices), replicas_per_group)
+
+    return indices
+
+
+def _get_layer(adata: AnnData, layer: str | None, dense: bool = False):
+    X: ndarray | csr_matrix
+
+    if layer is None or layer == "X":
+        X = adata.X
+    else:
+        X = adata.layers[layer]
+
+    if dense:
+        if issparse(X):
+            X = np.asarray(X.todense())
+        else:
+            X = np.asarray(X)
+
+    return X
+
+
+def _join_dummies(aggr_adata: AnnData, group_key: str) -> None:
+    dummies = pd.get_dummies(aggr_adata.obs[group_key], prefix=group_key).astype(str)
+    dummies = dummies.astype(str).apply(lambda s: s.map({"True": "", "False": "not"}))
+    dummies = dummies + aggr_adata.obs[group_key].cat.categories
+
+    aggr_adata.obs = aggr_adata.obs.join(dummies)