biopipen 0.34.6__tar.gz → 0.34.8__tar.gz

{biopipen-0.34.6 → biopipen-0.34.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: biopipen
-Version: 0.34.6
+Version: 0.34.8
 Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
 License: MIT
 Author: pwwang

biopipen-0.34.8/biopipen/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.34.8"

{biopipen-0.34.6 → biopipen-0.34.8}/biopipen/ns/scrna.py

@@ -2706,6 +2706,7 @@ class PseudoBulkDEG(Proc):
             analysis.
 
     Envs:
+        ncores (type=int): Number of cores to use for parallelization.
         mutaters (type=json): Mutaters to mutate the metadata of the
             seurat object. Keys are the new column names and values are the
             expressions to mutate the columns. These new columns can be
@@ -2715,6 +2716,9 @@ class PseudoBulkDEG(Proc):
         each: The column name in metadata to separate the cells into different cases.
             When specified, the case will be expanded to multiple cases for
             each value in the column.
+        cache (type=auto): Where to cache the results.
+            If `True`, cache to `outdir` of the job. If `False`, don't cache.
+            Otherwise, specify the directory to cache to.
         subset: An expression in string to subset the cells.
         aggregate_by: The column names in metadata to aggregate the cells.
         layer: The layer to pull and aggregate the data.
@@ -2844,7 +2848,9 @@ class PseudoBulkDEG(Proc):
     lang = config.lang.rscript
     script = "file://../scripts/scrna/PseudoBulkDEG.R"
     envs = {
+        "ncores": config.misc.ncores,
         "mutaters": {},
+        "cache": config.path.tmpdir,
         "each": None,
         "subset": None,
         "aggregate_by": None,

{biopipen-0.34.6 → biopipen-0.34.8}/biopipen/scripts/scrna/MarkersFinder.R

@@ -165,10 +165,12 @@ post_casing <- function(name, case) {
 
         if (length(cases) == 0 && name == "Marker Discovery") {
             name <- case$each
+        } else {
+            name <- paste0(name, " (", case$each, ")")
         }
 
         for (each in eachs) {
-            newname <- paste0(name, " - ", each)
+            newname <- paste0(name, "::", each)
             newcase <- case
 
             newcase$original_case <- name
@@ -266,20 +268,22 @@ process_markers <- function(markers, info, case) {
         ui = "tabs"
     )
 
-    for (plotname in names(case$marker_plots)) {
-        plotargs <- case$marker_plots[[plotname]]
-        plotargs$degs <- markers
-        rownames(plotargs$degs) <- make.unique(markers$gene)
-        plotargs$outprefix <- file.path(info$prefix, paste0("markers.", slugify(plotname)))
-        do_call(VizDEGs, plotargs)
-        reporter$add2(
-            list(
-                name = plotname,
-                contents = list(reporter$image(plotargs$outprefix, plotargs$more_formats, plotargs$save_code))),
-            hs = c(info$section, info$name),
-            hs2 = ifelse(is.null(case$ident), "Markers", paste0("Markers (", case$ident, ")")),
-            ui = "tabs"
-        )
+    if (nrow(markers) > 0) {
+        for (plotname in names(case$marker_plots)) {
+            plotargs <- case$marker_plots[[plotname]]
+            plotargs$degs <- markers
+            rownames(plotargs$degs) <- make.unique(markers$gene)
+            plotargs$outprefix <- file.path(info$prefix, paste0("markers.", slugify(plotname)))
+            do_call(VizDEGs, plotargs)
+            reporter$add2(
+                list(
+                    name = plotname,
+                    contents = list(reporter$image(plotargs$outprefix, plotargs$more_formats, plotargs$save_code))),
+                hs = c(info$section, info$name),
+                hs2 = ifelse(is.null(case$ident), "Markers", paste0("Markers (", case$ident, ")")),
+                ui = "tabs"
+            )
+        }
     }
 
     # Do enrichment analysis
@@ -397,6 +401,10 @@ process_allmarkers <- function(markers, plotcases, casename, groupname) {
         plotargs <- plotcases[[plotname]]
         plotargs$degs <- markers
         plotargs$outprefix <- file.path(info$prefix, slugify(plotname))
+        if (identical(plotargs$plot_type, "heatmap")) {
+            plotargs$show_row_names = plotargs$show_row_names %||% TRUE
+            plotargs$show_column_names = plotargs$show_column_names %||% TRUE
+        }
         do_call(VizDEGs, plotargs)
         reporter$add2(
             list(
@@ -545,7 +553,9 @@ run_case <- function(name) {
                 attr(markers, "group_by") <- each
                 attr(markers, "ident_1") <- NULL
                 attr(markers, "ident_2") <- NULL
-                process_allmarkers(markers, allmarker_plots, name, each)
+                if (!is.null(markers) && nrow(markers) > 0) {
+                    process_allmarkers(markers, allmarker_plots, name, each)
+                }
             }
 
             if (length(overlaps) > 0) {
@@ -555,7 +565,7 @@ run_case <- function(name) {
 
         }
 
-        if (!is.null(enriches)) {
+        if (!is.null(enriches) && length(enriches) > 0) {
             log$info("- Summarizing enrichments in subcases (by each: {each}) ...")
             if (!is.data.frame(enriches)) {
                 each_levels <- names(enriches)
@@ -571,7 +581,7 @@ run_case <- function(name) {
                 enriches[[each]] <- factor(enriches[[each]], levels = each_levels)
             }
 
-            if (length(allenrich_plots) > 0) {
+            if (length(allenrich_plots) > 0 && !is.null(enriches) && nrow(enriches) > 0) {
                 log$info("- Visualizing all enrichments together ...")
                 process_allenriches(enriches, allenrich_plots, name, each)
             }
@@ -634,7 +644,9 @@ run_case <- function(name) {
         ))
 
         if (!is.null(original_case) && !is.null(cases[[original_case]])) {
-            markers[[each_name]] <- each
+            if (nrow(markers) > 0) {
+                markers[[each_name]] <- each
+            }
             cases[[original_case]]$markers[[each]] <<- markers
             cases[[original_case]]$enriches[[each]] <<- enrich
         }

{biopipen-0.34.6 → biopipen-0.34.8}/biopipen/scripts/scrna/PseudoBulkDEG.R

@@ -8,7 +8,9 @@ outdir <- {{out.outdir | r}}
 joboutdir <- {{job.outdir | r}}
 each <- {{envs.each | r}}
 subset <- {{envs.subset | r}}
+ncores <- {{envs.ncores | r}}
 mutaters <- {{envs.mutaters | r}}
+cache <- {{ envs.cache | r }}
 aggregate_by <- {{envs.aggregate_by | r}}
 layer <- {{envs.layer | r}}
 assay <- {{envs.assay | r}}
@@ -35,6 +37,7 @@ overlaps <- {{ envs.overlaps | r }}
 cases <- {{envs.cases | r}}
 
 aggregate_by <- unique(c(aggregate_by, group_by, paired_by, each))
+if (isTRUE(cache)) { cache <- joboutdir }
 
 log <- get_logger()
 reporter <- get_reporter()
@@ -74,10 +77,12 @@ defaults <- list(
     ident_1 = ident_1,
     ident_2 = ident_2,
     dbs = dbs,
+    ncores = ncores,
     sigmarkers = sigmarkers,
     enrich_style = enrich_style,
     paired_by = paired_by,
     tool = tool,
+    cache = cache,
     allmarker_plots_defaults = allmarker_plots_defaults,
     allmarker_plots = allmarker_plots,
     allenrich_plots_defaults = allenrich_plots_defaults,
@@ -131,12 +136,14 @@ expand_each <- function(name, case) {
 
         if (length(cases) == 0 && name == "DEG Analysis") {
             name <- case$each
+        } else {
+            name <- paste0(name, " (", case$each, ")")
         }
 
         case$aggregate_by <- unique(c(case$aggregate_by, case$group_by, case$paired_by, case$each))
 
         for (each in eachs) {
-            newname <- paste0(case$each, "::", each)
+            newname <- paste0(name, "::", each)
             newcase <- case
 
             newcase$original_case <- name
@@ -179,6 +186,7 @@ expand_each <- function(name, case) {
         if (length(case$overlaps) > 0 || length(case$allmarker_plots) > 0 || length(case$allenrich_plots) > 0) {
             ovcase <- case
 
+            ovcase$allexprs <- list()
             ovcase$markers <- list()
             ovcase$allmarker_plots <- lapply(
                 ovcase$allmarker_plots,
@@ -212,7 +220,52 @@ process_markers <- function(markers, info, case) {
     # markers <- markers %>%
     #     mutate(gene = as.character(gene)) %>%
     #     arrange(p_val_adj, desc(abs(avg_log2FC)))
+
+    empty <- if (case$enrich_style == "enrichr") {
+        data.frame(
+            Database = character(0),
+            Term = character(0),
+            Overlap = character(0),
+            P.value = numeric(0),
+            Adjusted.P.value = numeric(0),
+            Odds.Ratio = numeric(0),
+            Combined.Score = numeric(0),
+            Genes = character(0),
+            Rank = numeric(0)
+        )
+    } else {  # clusterProfiler
+        data.frame(
+            ID = character(0),
+            Description = character(0),
+            GeneRatio = character(0),
+            BgRatio = character(0),
+            Count = integer(0),
+            pvalue = numeric(0),
+            p.adjust = numeric(0),
+            qvalue = numeric(0),
+            geneID = character(0),
+            Database = character(0)
+        )
+    }
+    if (is.null(markers) || nrow(markers) == 0) {
+        if (case$error) {
+            stop("Error: No markers found in case '", info$name, "'.")
+        } else {
+            log$warn("! Warning: No markers found in case '", info$name, "'.")
+            reporter$add2(
+                list(
+                    name = "Warning",
+                    contents = list(list(kind = "error", content = "No markers found.", kind_ = "warning"))),
+                hs = c(info$section, info$name),
+                hs2 = "DEG Analysis",
+                ui = "tabs"
+            )
+            return(empty)
+        }
+    }
     markers$gene <- as.character(markers$gene)
+    markers$p_val_adj <- as.numeric(markers$p_val_adj)
+    markers$log2FC <- as.numeric(markers$log2FC)
     markers <- markers[order(markers$p_val_adj, -abs(markers$log2FC)), ]
 
     # Save markers
@@ -287,7 +340,7 @@ process_markers <- function(markers, info, case) {
             stop("Error: Not enough significant DEGs with '", case$sigmarkers, "' in case '", info$name, "' found (< 5) for enrichment analysis.")
         } else {
             message <- paste0("Not enough significant DEGs with '", case$sigmarkers, "' found (< 5) for enrichment analysis.")
-            log$warn("  ! Error: {message}")
+            log$warn("! Error: {message}")
             reporter$add2(
                 list(
                     name = "Warning",
@@ -345,7 +398,7 @@ process_markers <- function(markers, info, case) {
             if (case$error) {
                 stop("Error: ", e$message)
             } else {
-                log$warn("  ! Error: {e$message}")
+                log$warn("! Error: {e$message}")
                 reporter$add2(
                     list(
                         name = "Warning",
@@ -478,6 +531,7 @@ process_overlaps <- function(markers, ovcases, casename, groupname) {
 
 run_case <- function(name) {
     case <- cases[[name]]
+    log$info("----------------------------------------")
     log$info("Case: {name} ...")
 
     case <- extract_vars(
@@ -485,18 +539,21 @@ run_case <- function(name) {
         "dbs", "sigmarkers", "allmarker_plots", "allenrich_plots", "marker_plots", "enrich_plots",
         "overlaps", "original_case", "markers", "enriches", "each_name", "each", "enrich_style",
         "aggregate_by", "subset", "layer", "assay", "group_by", "ident_1", "ident_2", "original_subset",
-        "paired_by", "tool", "error",
+        "paired_by", "tool", "error", "ncores", "cache", "allexprs",
         allow_nonexisting = TRUE
     )
 
     if (!is.null(markers) || !is.null(enriches)) {
-        if (!is.null(markers)) {  # It is the overlap/allmarker case
-            log$info("- Summarizing DEGs in subcases (by each: {each}) ...")
+        if (!is.null(markers) && length(markers) > 0) {
+            log$info("Summarizing DEGs in subcases (by each: {each}) ...")
             # handle the overlaps / allmarkers analysis here
             if (!is.data.frame(markers)) {
                 each_levels <- names(markers)
                 markers <- do_call(rbind, lapply(each_levels, function(x) {
                     markers_df <- markers[[x]]
+                    if (is.null(markers_df) || nrow(markers_df) == 0) {
+                        return(NULL)
+                    }
                     if (nrow(markers_df) > 0) {
                         markers_df[[each]] <- x
                     } else {
@@ -508,17 +565,17 @@ run_case <- function(name) {
             }
             # gene, p_val, avg_log2FC, pct.1, pct.2, p_val_adj, diff_pct, <each>
 
+            if (!is.data.frame(allexprs)) {
+                meta <- do_call(rbind, lapply(allexprs, attr, "meta"))
+                allexprs <- do_call(cbind, allexprs)
+            } else {
+                meta <- attr(allexprs, "meta")
+            }
+
             if (length(allmarker_plots) > 0) {
-                log$info("- Visualizing all DEGs together ...")
-                exprs <- AggregateExpressionPseudobulk(
-                    srtobj, aggregate_by = aggregate_by, layer = layer, assay = assay,
-                    subset = original_subset, log = log
-                )
-                attr(markers, "object") <- AggregateExpressionPseudobulk(
-                    srtobj, aggregate_by = aggregate_by, layer = layer, assay = assay,
-                    subset = original_subset, log = log
-                )
-                attr(markers, "meta") <- attr(exprs, "meta")
+                log$info("Visualizing all DEGs together ...")
+                attr(markers, "object") <- allexprs
+                attr(markers, "meta") <- meta
                 attr(markers, "group_by") <- each
                 attr(markers, "paired_by") <- paired_by
                 attr(markers, "ident_1") <- NULL
@@ -527,18 +584,21 @@ run_case <- function(name) {
             }
 
             if (length(overlaps) > 0) {
-                log$info("- Visualizing overlaps between subcases ...")
+                log$info("Visualizing overlaps between subcases ...")
                 process_overlaps(markers, overlaps, name, each)
             }
 
         }
 
-        if (!is.null(enriches)) {
-            log$info("- Summarizing enrichments in subcases (by each: {each}) ...")
+        if (!is.null(enriches) && length(enriches) > 0) {
+            log$info("Summarizing enrichments in subcases (by each: {each}) ...")
             if (!is.data.frame(enriches)) {
                 each_levels <- names(enriches)
                 enriches <- do_call(rbind, lapply(each_levels, function(x) {
                     enrich_df <- enriches[[x]]
+                    if (is.null(enrich_df) || nrow(enrich_df) == 0) {
+                        return(NULL)
+                    }
                     if (nrow(enrich_df) > 0) {
                         enrich_df[[each]] <- x
                     } else {
@@ -546,11 +606,13 @@ run_case <- function(name) {
                     }
                     enrich_df
                 }))
-                enriches[[each]] <- factor(enriches[[each]], levels = each_levels)
+                if (!is.null(enriches) && nrow(enriches) > 0) {
+                    enriches[[each]] <- factor(enriches[[each]], levels = each_levels)
+                }
             }
 
-            if (length(allenrich_plots) > 0) {
-                log$info("- Visualizing all enrichments together ...")
+            if (length(allenrich_plots) > 0 && !is.null(enriches) && nrow(enriches) > 0) {
+                log$info("Visualizing all enrichments together ...")
                 process_allenriches(enriches, allenrich_plots, name, each)
             }
         }
@@ -558,16 +620,36 @@ run_case <- function(name) {
         return(invisible())
     }
 
+    info <- case_info(name, outdir, create = TRUE)
     exprs <- AggregateExpressionPseudobulk(
         srtobj, aggregate_by = aggregate_by, layer = layer, assay = assay,
         subset = subset, log = log
     )
-    markers <- RunDEGAnalysis(
-        exprs, group_by = group_by, ident_1 = ident_1, ident_2 = ident_2,
-        paired_by = paired_by, tool = tool, log = log
+    markers <- tryCatch(
+        {
+            RunDEGAnalysis(
+                exprs, group_by = group_by, ident_1 = ident_1, ident_2 = ident_2,
+                paired_by = paired_by, tool = tool, log = log, ncores = ncores,
+                cache = cache
+            )
+        }, error = function(e) {
+            if (error) {
+                stop("Error: ", e$message)
+            } else {
+                log$warn("! Error: {e$message}")
+                reporter$add2(
+                    list(
+                        name = "Warning",
+                        contents = list(list(kind = "error", content = e$message, kind_ = "warning"))),
+                    hs = c(info$section, info$name),
+                    hs2 = "DEG Analysis",
+                    ui = "tabs"
+                )
+                return(invisible())
+            }
+        }
     )
 
-    info <- case_info(name, outdir, create = TRUE)
     enrich <- process_markers(markers, info = info, case = list(
         dbs = dbs,
         sigmarkers = sigmarkers,
@@ -579,9 +661,12 @@ run_case <- function(name) {
     ))
 
     if (!is.null(original_case) && !is.null(cases[[original_case]])) {
-        markers[[each_name]] <- each
+        if (!is.null(markers)) {
+            markers[[each_name]] <- each
+        }
         cases[[original_case]]$markers[[each]] <<- markers
         cases[[original_case]]$enriches[[each]] <<- enrich
+        cases[[original_case]]$allexprs[[each]] <<- exprs
     }
 
     invisible()

{biopipen-0.34.6 → biopipen-0.34.8}/biopipen/scripts/scrna/ScFGSEA.R

@@ -82,11 +82,13 @@ expand_each <- function(name, case) {
         }
 
         if (length(cases) == 0 && name == "GSEA") {
-            name <- case$each
+            prefix <- case$each
+        } else {
+            prefix <- paste0(name, " (", case$each, ")")
         }
 
         for (each in eachs) {
-            newname <- paste0(case$each, "::", each)
+            newname <- paste0(prefix, "::", each)
             newcase <- case
 
             newcase$original_case <- paste0(name, " (all ", case$each,")")
@@ -142,6 +144,11 @@ do_case <- function(name) {
 
     if (!is.null(case$gseas)) {
 
+        if (length(case$gseas) == 0) {
+            log$warn("  No GSEA results found for case {name}. Skipping.")
+            return(invisible(NULL))
+        }
+
         each_levels <- names(case$gseas)
         gseas <- do_call(rbind, lapply(each_levels, function(x) {
             gsea_df <- case$gseas[[x]]
@@ -240,25 +247,16 @@ do_case <- function(name) {
         quote = FALSE
     )
     if (all(is.na(ranks))) {
-        if (length(allclasses) < 100) {
-            log$warn("  Ignoring this case because all gene ranks are NA and there are <100 cells.")
-            reporter$add2(
-                list(
-                    kind = "error",
-                    content = paste0("Not enough cells (n = ", length(allclasses), ") to run fgsea.")
-                ),
-                hs = c(info$section, info$name)
-            )
-            return(NULL)
-        } else {
-            stop(paste0(
-                "All gene ranks are NA (# cells = ",
-                length(allclasses),
-                "). ",
-                "It's probably due to high missing rate in the data. ",
-                "You may want to try a different `envs$method` for pre-ranking."
-            ))
-        }
+        log$warn("  All gene ranks are NA. It's probably due to high missing rate in the data.")
+        log$warn("  Case ignored, you may also try a different ranking method.")
+        reporter$add2(
+            list(
+                kind = "error",
+                content = "All gene ranks are NA. It's probably due to high missing rate in the data."
+            ),
+            hs = c(info$section, info$name)
+        )
+        return(invisible(NULL))
     }
 
     # run fgsea

{biopipen-0.34.6 → biopipen-0.34.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "biopipen"
-version = "0.34.6"
+version = "0.34.8"
 description = "Bioinformatics processes/pipelines that can be run from `pipen run`"
 authors = ["pwwang <pwwang@pwwang.com>"]
 license = "MIT"

{biopipen-0.34.6 → biopipen-0.34.8}/setup.py

@@ -86,7 +86,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'biopipen',
-    'version': '0.34.6',
+    'version': '0.34.8',
     'description': 'Bioinformatics processes/pipelines that can be run from `pipen run`',
     'long_description': 'None',
     'author': 'pwwang',

biopipen-0.34.6/biopipen/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.34.6"