biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/scripts/scrna/TopExpressingGenes.R

@@ -1,175 +1,209 @@
-source("{{biopipen_dir}}/utils/misc.R")
-
 library(Seurat)
-library(tibble)
-library(enrichR)
 library(rlang)
 library(dplyr)
-
-setEnrichrSite("Enrichr")
+library(tidyselect)
+library(biopipen.utils)
 
 srtfile <- {{in.srtobj | r}}
 outdir <- {{out.outdir | r}}
+joboutdir <- {{job.outdir | r}}
 mutaters <- {{ envs.mutaters | r }}
 ident <- {{ envs.ident | r }}
-group.by <- {{ envs["group-by"] | r }}  # nolint
+group_by <- {{ envs.group_by | default: envs["group-by"] | default: None | r }}  # nolint
 each <- {{ envs.each | r }}
-prefix_each <- {{ envs.prefix_each | r }}
-section <- {{ envs.section | r }}
 dbs <- {{ envs.dbs | r }}
 n <- {{ envs.n | r }}
+enrich_style <- {{ envs.enrich_style | r }}
+sset <- {{ envs.subset | r }}
+enrich_plots_defaults <- {{ envs.enrich_plots_defaults | r }}
+enrich_plots <- {{ envs.enrich_plots | r }}
 cases <- {{ envs.cases | r: todot = "-" }}  # nolint
 
 set.seed(8525)
+log <- get_logger()
+reporter <- get_reporter()
 
-print("- Loading Seurat object ...")
-srtobj <- readRDS(srtfile)
+log$info("Reading Seurat object ...")
+srtobj <- read_obj(srtfile)
+assay <- DefaultAssay(srtobj)
 
-print("- Mutate meta data if needed ...")
-if (!is.null(mutaters) && length(mutaters)) {
+if (!is.null(mutaters) && length(mutaters) > 0) {
+    log$info("Mutating meta data ...")
     srtobj@meta.data <- srtobj@meta.data %>%
         mutate(!!!lapply(mutaters, parse_expr))
 }
 
-print("- Expanding cases ...")
-if (is.null(cases) || length(cases) == 0) {
-    cases <- list(
-        DEFAULT = list(
-            ident = ident,
-            group.by = group.by,
-            each = each,
-            prefix_each = prefix_each,
-            section = section,
-            dbs = dbs,
-            n = n
-        )
-    )
-} else {
-    cases <- lapply(cases, function(cs) {
-        list_setdefault(
-            cs,
-            ident = ident,
-            group.by = group.by,
-            each = each,
-            prefix_each = prefix_each,
-            section = section,
-            dbs = dbs,
-            n = n
+enrich_plots <- lapply(enrich_plots, function(x) {
+    list_update(enrich_plots_defaults, x)
+})
+defaults <- list(
+    ident = ident,
+    group_by = group_by,
+    each = each,
+    dbs = dbs,
+    n = n,
+    enrich_style = enrich_style,
+    enrich_plots = enrich_plots,
+    enrich_plots_defaults = enrich_plots_defaults,
+    subset = sset
+)
+
+cases <- expand_cases(cases, defaults, default_case = "Top Expressing Genes", post = function(name, case) {
+    outcases <- list()
+    if (is.null(case$each) || is.na(case$each) || nchar(case$each) == 0 || isFALSE(each)) {
+        case$enrich_plots <- lapply(
+            case$enrich_plots,
+            function(x) { list_update(case$enrich_plots_defaults, x) }
         )
-    })
-}
+        case$enrich_plots_defaults <- NULL
 
-# Expand each and ident
-newcases <- list()
-for (name in names(cases)) {  # nolint
-    case <- cases[[name]]
-    if (is.null(case$each) && !is.null(case$ident)) {
-        newcases[[paste0(case$section, ":", name)]] <- case
-    } else if (is.null(case$each)) {
-        idents <- srtobj@meta.data %>%
-            pull(case$group.by) %>%
-            unique() %>%
-            na.omit()
-        for (ident in idents) {
-            key <- paste0(name, ":", ident)
-            newcases[[key]] <- case
-            newcases[[key]]$ident <- ident
-        }
+        outcases[[name]] <- case
     } else {
-        eachs <- srtobj@meta.data %>% pull(case$each) %>% unique() %>% na.omit()
+        eachs <- if (!is.null(case$subset)) {
+            srtobj@meta.data %>%
+                filter(!!parse_expr(case$subset)) %>%
+                pull(case$each) %>% na.omit() %>% unique() %>% as.vector()
+        } else {
+            srtobj@meta.data %>%
+                pull(case$each) %>% na.omit() %>% unique() %>% as.vector()
+        }
+
+        if (length(cases) == 0 && name == "Top Expressing Genes") {
+            name <- case$each
+        }
+
         for (each in eachs) {
-            by <- make.names(paste0(".", name, "_", each))
-            srtobj@meta.data <- srtobj@meta.data %>% mutate(
-                !!sym(by) := if_else(
-                    !!sym(case$each) == each,
-                    !!sym(case$group.by),
-                    NA
-                )
-            )
-            if (is.null(case$ident)) {
-                idents <- srtobj@meta.data %>%
-                    pull(case$group.by) %>%
-                    unique() %>%
-                    na.omit()
-                for (ident in idents) {
-                    kname <- if (name == "DEFAULT") "" else paste0("-", name)
-                    key <- paste0(each, kname, ":", ident)
-                    if (case$prefix_each) {
-                        key <- paste0(case$each, "-", key)
-                    }
-                    newcases[[key]] <- case
-                    newcases[[key]]$ident <- ident
-                    newcases[[key]]$group.by <- by  # nolint
-                }
+            newname <- paste0(name, " - ", each)
+            newcase <- case
+            newcase$each_name <- case$each
+            newcase$each <- each
+
+            if (!is.null(case$subset)) {
+                newcase$subset <- paste0(case$subset, " & ", bQuote(case$each), " == '", each, "'")
             } else {
-                key <- paste0(case$each, ":", each)
-                if (name != "DEFAULT") {
-                    key <- paste0(key, " - ", name)
-                }
-                newcases[[key]] <- case
+                newcase$subset <- paste0(bQuote(case$each), " == '", each, "'")
             }
+
+            newcase$enrich_plots <- lapply(
+                case$enrich_plots,
+                function(x) { list_update(case$enrich_plots_defaults, x) }
+            )
+            newcase$enrich_plots_defaults <- NULL
+
+            outcases[[newname]] <- newcase
         }
     }
-}
-cases <- newcases
-
-do_enrich <- function(expr, odir) {
-    print("  Saving expressions ...")
-    write.table(
-        expr %>% as.data.frame() %>% rownames_to_column("Gene"),
-        file.path(odir, "expr.txt"),
-        sep = "\t",
-        row.names = TRUE,
-        col.names = TRUE,
-        quote = FALSE
+
+    outcases
+})
+
+log$info("Running cases ...")
+
+process_markers <- function(markers, info, case) {
+    # Save markers
+    write.table(markers, file.path(info$prefix, "top_genes.tsv"), sep = "\t", quote = FALSE, row.names = FALSE)
+    reporter$add2(
+        list(
+            name = "Table",
+            contents = list(
+                list(kind = "descr", content = "Showing top expressing genes ordered by their expression descendingly."),
+                list(kind = "table", src = file.path(info$prefix, "top_genes.tsv"), data = list(nrows = 100))
+            )
+        ),
+        hs = c(info$section, info$name),
+        hs2 = paste0("Top Genes"),
+        ui = "tabs"
     )
-    write.table(
-        expr %>% as.data.frame() %>% rownames_to_column("Gene") %>% head(n),
-        file.path(odir, "exprn.txt"),
-        sep = "\t",
-        row.names = TRUE,
-        col.names = TRUE,
-        quote = FALSE
+
+    enrich <- RunEnrichment(
+        markers$gene,
+        dbs = case$dbs, style = case$enrich_style)
+
+    write.table(enrich, file.path(info$prefix, "enrich.tsv"), sep = "\t", quote = FALSE, row.names = FALSE)
+    reporter$add2(
+        list(
+            name = "Table",
+            contents = list(list(kind = "table", src = file.path(info$prefix, "enrich.tsv"), data = list(nrows = 100)))
+        ),
+        hs = c(info$section, info$name),
+        hs2 = "Enrichment Analysis",
+        ui = "tabs"
     )
 
-    print("  Running enrichment ...")
-    enriched <- enrichr(rownames(head(expr, n)), dbs)  # nolint
-    for (db in dbs) {
-        write.table(
-            enriched[[db]],
-            file.path(odir, paste0("Enrichr-", db, ".txt")),
-            sep = "\t",
-            row.names = FALSE,
-            col.names = TRUE,
-            quote = FALSE
-        )
-        png(
-            file.path(odir, paste0("Enrichr-", db, ".png")),
-            res = 100, height = 1000, width = 1000
-        )
-        print(plotEnrich(enriched[[db]], showTerms = 20, title = db))  # nolint
-        dev.off()
+    # Visualize enriched terms
+    if (length(case$enrich_plots) > 0) {
+        for (db in case$dbs) {
+            plots <- list()
+            for (plotname in names(case$enrich_plots)) {
+                plotargs <- case$enrich_plots[[plotname]]
+                plotargs$data <- enrich[enrich$Database == db, , drop = FALSE]
+
+                p <- do_call(VizEnrichment, plotargs)
+
+                outprefix <- file.path(info$prefix, paste0("enrich.", slugify(db), ".", slugify(plotname)))
+                if (plotargs$plot_type == "bar") {
+                    attr(p, "height") <- attr(p, "height") / 1.5
+                }
+                save_plot(p, outprefix, plotargs$devpars, formats = "png")
+                plots[[length(plots) + 1]] <- reporter$image(outprefix, c(), FALSE)
+            }
+            reporter$add2(
+                list(name = db, contents = plots),
+                hs = c(info$section, info$name),
+                hs2 = "Enrichment Analysis",
+                ui = "tabs"
+            )
+        }
     }
 }
 
-do_case <- function(casename) {
-    print(paste("- Running for case:", casename))
-    case <- cases[[casename]]
-    parts <- unlist(strsplit(casename, ":"))
-    section <- parts[1]
-    casename <- paste(parts[-1], collapse = ":")
 
-    print("  Calculating average expression ...")
+run_case <- function(name) {
+    log$info("Case: {name} ...")
+    case <- cases[[name]]
+
+    log$info("- Subsetting cells and calculating average expression ...")
+    if (!is.null(case$subset)) {
+        subobj <- filter(srtobj, !!parse_expr(case$subset))
+    } else {
+        subobj <- srtobj
+    }
+    case$group_by <- case$group_by %||% GetIdentityColumn(srtobj)
+    if (is.null(case$ident)) {
+        case$ident <- as.character(unique(subobj@meta.data[[case$group_by]]))
+    }
     avgexpr <- AverageExpression(
-        srtobj,
-        group.by = case$group.by
-    )$RNA[, case$ident, drop = FALSE]
-    avgexpr <- avgexpr[order(-avgexpr), , drop = FALSE]
+        subobj,
+        group_by = case$group_by,
+        assays = assay
+    )[[assay]]
+    # https://github.com/satijalab/seurat/issues/7893
+    colnames(avgexpr) <- as.character(unique(subobj@meta.data[[case$group_by]]))
+    avgexpr <- avgexpr[, case$ident, drop = FALSE]
 
-    odir <- file.path(outdir, section, casename)
-    dir.create(odir, recursive = TRUE, showWarnings = FALSE)
+    for (idt in case$ident) {
+        log$info("- Processing {idt} ...")
+        info <- case_info(paste0(name, "::", idt), outdir, create = TRUE)
+        expr <- avgexpr[, idt, drop = FALSE]
+        expr <- expr[order(expr, decreasing = TRUE), , drop = FALSE]
+        expr <- expr[1:min(case$n, nrow(expr)), , drop = FALSE]
+        expr <- as.data.frame(expr)
+        expr$gene <- rownames(expr)
+        colnames(expr) <- c("avg_expr", "gene")
+        expr <- expr[, c("gene", "avg_expr"), drop = FALSE]
 
-    do_enrich(avgexpr, odir)
+        log$info("  Performing enrichment analysis ...")
+        process_markers(expr, info, case = list(
+            ident = idt,
+            dbs = case$dbs,
+            enrich_style = case$enrich_style,
+            enrich_plots = case$enrich_plots
+        ))
+    }
+
+    invisible()
 }
 
-sapply(sort(names(cases)), do_case)
+sapply(names(cases), run_case)
+
+reporter$save(joboutdir)

biopipen/scripts/scrna/celltypist-wrapper.py

@@ -0,0 +1,195 @@
+from argparse import ArgumentParser
+from typing import Union
+import numpy as np
+import pandas as pd
+import scanpy as sc
+import celltypist
+from celltypist.classifier import logger, AnnData, Model, Classifier
+
+parser = ArgumentParser(description="Run CellTypist")
+parser.add_argument(
+    "-i", "--input", required=True, help="Input H5AD file with AnnData object"
+)
+parser.add_argument("-o", "--output", required=True, help="Output file")
+parser.add_argument("-m", "--model", required=True, help="Model file")
+parser.add_argument(
+    "-v", "--majority_voting", action="store_true", help="Majority voting"
+)
+parser.add_argument(
+    "-c",
+    "--over_clustering",
+    required=False,
+    default=None,
+    help="Over clustering. Error if the column does not exist.",
+)
+
+
+def classifier_init(
+    self, filename="", model="", transpose=False, gene_file=None, cell_file=None
+):
+    """Celltypist check if adata is in the range of log1p normalized data to 10000
+    counts per cell. Otherwise it will use the raw data if available. However, in
+    some cases, the raw data has invalid feature names (var_names) which causes errors.
+    Here we check if the feature names of raw data is valid with intersection with
+    model features, if not, we will use the adata.X instead of adata.raw.X
+    """
+    if isinstance(model, str):
+        model = Model.load(model)
+    self.model = model
+    if not filename:
+        logger.warn("📭 No input file provided to the classifier")
+        return
+    if isinstance(filename, str):
+        self.filename = filename
+        logger.info(f"📁 Input file is '{self.filename}'")
+        logger.info("⏳ Loading data")
+    if isinstance(filename, str) and filename.endswith(
+        (".csv", ".txt", ".tsv", ".tab", ".mtx", ".mtx.gz")
+    ):
+        self.adata = sc.read(self.filename)
+        if transpose:
+            self.adata = self.adata.transpose()
+        if self.filename.endswith((".mtx", ".mtx.gz")):
+            if (gene_file is None) or (cell_file is None):
+                raise FileNotFoundError(
+                    "🛑 Missing `gene_file` and/or `cell_file`. Please provide both "
+                    "arguments together with the input mtx file"
+                )
+            genes_mtx = pd.read_csv(gene_file, header=None)[0].values
+            cells_mtx = pd.read_csv(cell_file, header=None)[0].values
+            if len(genes_mtx) != self.adata.n_vars:
+                raise ValueError(
+                    f"🛑 The number of genes in {gene_file} does not match the number "
+                    f"of genes in {self.filename}"
+                )
+            if len(cells_mtx) != self.adata.n_obs:
+                raise ValueError(
+                    f"🛑 The number of cells in {cell_file} does not match the number "
+                    f"of cells in {self.filename}"
+                )
+            self.adata.var_names = genes_mtx
+            self.adata.obs_names = cells_mtx
+        if not float(self.adata.X[:1000].max()).is_integer():
+            logger.warn(
+                "⚠️ Warning: the input file seems not a raw count matrix. The "
+                "prediction result may not be accurate"
+            )
+        if (
+            (self.adata.n_vars >= 100000)
+            or (len(self.adata.var_names[0]) >= 30)
+            or (
+                len(
+                    self.adata.obs_names.intersection(
+                        ["GAPDH", "ACTB", "CALM1", "PTPRC", "MALAT1"]
+                    )
+                )
+                >= 1
+            )
+        ):
+            logger.warn(
+                "⚠️ The input matrix is detected to be a gene-by-cell matrix, will "
+                "transpose it"
+            )
+            self.adata = self.adata.transpose()
+        self.adata.var_names_make_unique()
+        sc.pp.normalize_total(self.adata, target_sum=1e4)
+        sc.pp.log1p(self.adata)
+        self.indata = self.adata.X
+        self.indata_genes = self.adata.var_names
+        self.indata_names = self.adata.obs_names
+    elif isinstance(filename, AnnData) or (
+        isinstance(filename, str) and filename.endswith(".h5ad")
+    ):
+        self.adata = sc.read(filename) if isinstance(filename, str) else filename
+        self.adata.var_names_make_unique()
+        # When to use raw.X?
+        # 1. if adata.raw exists
+        # 2. if adata.raw.var_names has intersection with model genes
+        # 3. if adata.X is not in the expected range
+        use_raw = self.adata.raw and (
+            self.adata.X[:1000].min() < 0 or self.adata.X[:1000].max() > 9.22
+        ) and np.isin(
+            self.adata.raw.var_names, self.model.classifier.features
+        ).sum() > 0
+
+        if use_raw:
+            if not self.adata.raw:
+                raise ValueError(
+                    "🛑 Invalid expression matrix in `.X`, expect log1p normalized "
+                    "expression to 10000 counts per cell"
+                )
+            elif (self.adata.raw.X[:1000].min() < 0) or (
+                self.adata.raw.X[:1000].max() > 9.22
+            ):
+                raise ValueError(
+                    "🛑 Invalid expression matrix in both `.X` and `.raw.X`, expect "
+                    "log1p normalized expression to 10000 counts per cell"
+                )
+            else:
+                logger.info(
+                    "👀 Invalid expression matrix in `.X`, expect log1p normalized "
+                    "expression to 10000 counts per cell; will use `.raw.X` instead"
+                )
+                self.indata = self.adata.raw.X
+                self.indata_genes = self.adata.raw.var_names
+                self.indata_names = self.adata.raw.obs_names
+        else:
+            self.indata = self.adata.X
+            self.indata_genes = self.adata.var_names
+            self.indata_names = self.adata.obs_names
+        if np.abs(np.expm1(self.indata[0]).sum() - 10000) > 1:
+            logger.warn(
+                "⚠️ Warning: invalid expression matrix, expect ALL genes and log1p "
+                "normalized expression to 10000 counts per cell. The prediction result "
+                "may not be accurate"
+            )
+    else:
+        raise ValueError(
+            "🛑 Invalid input. Supported types: .csv, .txt, .tsv, .tab, .mtx, .mtx.gz "
+            "and .h5ad, or AnnData loaded in memory"
+        )
+
+    logger.info(
+        f"🔬 Input data has {self.indata.shape[0]} cells and {len(self.indata_genes)} "
+        "genes"
+    )
+
+
+if __name__ == "__main__":
+    Classifier.__init__ = classifier_init  # type: ignore
+
+    args = parser.parse_args()
+    adata = sc.read_h5ad(args.input)
+    over_clustering = args.over_clustering
+    if over_clustering and over_clustering not in adata.obs.columns:
+        raise ValueError(
+            f"Over clustering column '{over_clustering}' not found in AnnData object."
+        )
+    if "neighbors" in adata.uns and "params" in adata.uns["neighbors"]:
+        adata.uns["neighbors"]["params"].setdefault("n_neighbors", 15)
+
+    annotated = celltypist.annotate(
+        adata,
+        model=args.model,
+        majority_voting=args.majority_voting,
+        over_clustering=over_clustering,
+    )
+
+    out_adata = annotated.to_adata()
+    # leave as is
+    # if over_clustering and args.majority_voting:
+    #     # rename majority_voting column to over_clustering
+    #     out_adata.obs[over_clustering] = out_adata.obs["majority_voting"]
+
+    if args.output.endswith(".h5ad"):
+        try:
+            out_adata._raw._var.rename(  # type: ignore
+                columns={"_index": "features"}, inplace=True
+            )
+            del out_adata.raw
+        except (KeyError, AttributeError):
+            pass
+
+        out_adata.write(args.output)
+    else:
+        out_adata.obs.to_csv(args.output, sep="\t", index=True)