biopipen 0.27.3__tar.gz → 0.27.5__tar.gz

{biopipen-0.27.3 → biopipen-0.27.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: biopipen
-Version: 0.27.3
+Version: 0.27.5
 Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
 License: MIT
 Author: pwwang
@@ -20,4 +20,3 @@ Requires-Dist: pipen-filters (>=0.12,<0.13)
 Requires-Dist: pipen-poplog (>=0.1.2,<0.2.0)
 Requires-Dist: pipen-runinfo (>=0.6,<0.7) ; extra == "runinfo"
 Requires-Dist: pipen-verbose (>=0.11,<0.12)
-Requires-Dist: pyyaml-include (==1.*)

biopipen-0.27.5/biopipen/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.27.5"

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/core/testing.py

@@ -51,15 +51,16 @@ class PipelineSucceeded:
         pipen._succeeded = succeeded
 
 
-def get_pipeline(testfile, loglevel="debug", **kwargs):
+def get_pipeline(testfile, loglevel="debug", enable_report=False, **kwargs):
     """Get a pipeline for a test file"""
     name, workdir, outdir = _get_test_dirs(testfile, False)
+    report_plugin_prefix = "+" if enable_report else "-"
     kws = {
         "name": name,
         "workdir": workdir,
         "outdir": outdir,
         "loglevel": loglevel,
-        "plugins": [PipelineSucceeded, "-report"],
+        "plugins": [PipelineSucceeded, f"{report_plugin_prefix}report"],
     }
     kws.update(kwargs)
     return Pipen(**kws)

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/ns/delim.py

@@ -113,7 +113,7 @@ class SampleInfo(Proc):
         "exclude_cols": None,
         "defaults": {
             "on": "Sample",
-            "distinct": None,
+            # "distinct": None,
             "group": None,
             "na_group": False,
             "each": None,

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/ns/plot.py

@@ -114,3 +114,39 @@ class Heatmap(Proc):
         "globals": "",
     }
     script = "file://../scripts/plot/Heatmap.R"
+
+
+class ROC(Proc):
+    """Plot ROC curve using [`plotROC`](https://cran.r-project.org/web/packages/plotROC/vignettes/examples.html).
+
+    Input:
+        infile: The input file for data, tab-separated.
+            The first column should be ids of the records (this is optional if `envs.noids` is True).
+            The second column should be the labels of the records (1 for positive, 0 for negative).
+            If they are not binary, you can specify the positive label by `envs.pos_label`.
+            From the third column, it should be the scores of the different models.
+
+    Output:
+        outfile: The output figure file
+
+    Envs:
+        noids: Whether the input file has ids (first column) or not.
+        pos_label: The positive label.
+        ci: Whether to use `geom_rocci()` instead of `geom_roc()`.
+        devpars: The parameters for `png()`
+        args: Additional arguments for `geom_roc()` or `geom_rocci()` if `envs.ci` is True.
+        style_roc: Arguments for `style_roc()`
+    """  # noqa: E501
+    input = "infile:file"
+    output = "outfile:file:{{in.infile | stem}}.roc.png"
+    lang = config.lang.rscript
+    envs = {
+        "noids": False,
+        "pos_label": 1,
+        "ci": False,
+        "devpars": {"res": 100, "width": 750, "height": 600},
+        "args": {"labels": False},
+        "style_roc": {},
+        "show_auc": True,
+    }
+    script = "file://../scripts/plot/ROC.R"

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/ns/scrna.py

@@ -122,6 +122,9 @@ class SeuratPreparing(Proc):
             genes.
             ///
 
+        cell_qc_per_sample (flag): Whether to perform cell QC per sample or not.
+            If `True`, the cell QC will be performed per sample, and the QC will be
+            applied to each sample before merging.
         gene_qc (ns): Filter genes.
             `gene_qc` is applied after `cell_qc`.
             - min_cells: The minimum number of cells that a gene must be
@@ -222,6 +225,7 @@ class SeuratPreparing(Proc):
     envs = {
         "ncores": config.misc.ncores,
         "cell_qc": None,  # "nFeature_RNA > 200 & percent.mt < 5",
+        "cell_qc_per_sample": False,
         "gene_qc": {"min_cells": 0, "excludes": []},
         "use_sct": False,
         "no_integration": False,
@@ -413,7 +417,7 @@ class SeuratClusterStats(Proc):
         nCells_All = { }
         ```
 
-        ![nCells_All](https://pwwang.github.io/immunopipe/processes/images/SeuratClusterStats_nCells_All.png){: width="80%" }
+        ![nCells_All](https://pwwang.github.io/immunopipe/latest/processes/images/SeuratClusterStats_nCells_All.png){: width="80%" }
 
         ### Number of cells in each cluster by groups
 
@@ -422,7 +426,7 @@ class SeuratClusterStats(Proc):
         nCells_Sample = { group-by = "Sample" }
         ```
 
-        ![nCells_Sample](https://pwwang.github.io/immunopipe/processes/images/SeuratClusterStats_nCells_Sample.png){: width="80%" }
+        ![nCells_Sample](https://pwwang.github.io/immunopipe/latest/processes/images/SeuratClusterStats_nCells_Sample.png){: width="80%" }
 
         ### Violin plots for the gene expressions
 
@@ -435,8 +439,8 @@ class SeuratClusterStats(Proc):
         vlnplots_1 = { features = ["FOXP3", "IL2RA"], pt-size = 0, kind = "vln" }
         ```
 
-        ![vlnplots](https://pwwang.github.io/immunopipe/processes/images/SeuratClusterStats_vlnplots.png){: width="80%" }
-        ![vlnplots_1](https://pwwang.github.io/immunopipe/processes/images/SeuratClusterStats_vlnplots_1.png){: width="80%" }
+        ![vlnplots](https://pwwang.github.io/immunopipe/latest/processes/images/SeuratClusterStats_vlnplots.png){: width="80%" }
+        ![vlnplots_1](https://pwwang.github.io/immunopipe/latest/processes/images/SeuratClusterStats_vlnplots_1.png){: width="80%" }
 
         ### Dimension reduction plot with labels
 
@@ -447,7 +451,7 @@ class SeuratClusterStats(Proc):
         repel = true
         ```
 
-        ![dimplots](https://pwwang.github.io/immunopipe/processes/images/SeuratClusterStats_dimplots.png){: width="80%" }
+        ![dimplots](https://pwwang.github.io/immunopipe/latest/processes/images/SeuratClusterStats_dimplots.png){: width="80%" }
 
     Input:
         srtobj: The seurat object loaded by `SeuratClustering`
@@ -857,7 +861,7 @@ class CellsDistribution(Proc):
         group_order = [ "Tumor", "Normal" ]
         ```
 
-        ![CellsDistribution_example](https://pwwang.github.io/immunopipe/processes/images/CellsDistribution_example.png)
+        ![CellsDistribution_example](https://pwwang.github.io/immunopipe/latest/processes/images/CellsDistribution_example.png)
 
     Input:
         srtobj: The seurat object in RDS format
@@ -1483,14 +1487,17 @@ class SeuratTo10X(Proc):
         srtobj: The seurat object in RDS
 
     Output:
-        outdir: The output directory
+        outdir: The output directory.
+            When `envs.split_by` is specified, the subdirectories will be
+            created for each distinct value of the column.
+            Otherwise, the matrices will be written to the output directory.
 
     Envs:
         version: The version of 10X format
     """
     input = "srtobj:file"
     output = "outdir:dir:{{in.srtobj | stem}}"
-    envs = {"version": "3"}
+    envs = {"version": "3", "split_by": None}
     lang = config.lang.rscript
     script = "file://../scripts/scrna/SeuratTo10X.R"
 
@@ -1870,7 +1877,7 @@ class RadarPlots(Proc):
 
         Then we will have a radar plots like this:
 
-        ![Radar plots](https://pwwang.github.io/immunopipe/processes/images/RadarPlots-default.png)
+        ![Radar plots](https://pwwang.github.io/immunopipe/latest/processes/images/RadarPlots-default.png)
 
         We can use `each` to separate the cells into different cases:
 
@@ -1882,7 +1889,7 @@ class RadarPlots(Proc):
 
         Then we will have two radar plots, one for `Pre` and one for `Post`:
 
-        ![Radar plots](https://pwwang.github.io/immunopipe/processes/images/RadarPlots-each.png)
+        ![Radar plots](https://pwwang.github.io/immunopipe/latest/processes/images/RadarPlots-each.png)
 
         Using `cluster_order` to change the order of the clusters and show only the first 3 clusters:
 
@@ -1893,7 +1900,7 @@ class RadarPlots(Proc):
         breaks = [0, 50, 100]  # also change the breaks
         ```
 
-        ![Radar plots cluster_order](https://pwwang.github.io/immunopipe/processes/images/RadarPlots-cluster_order.png)
+        ![Radar plots cluster_order](https://pwwang.github.io/immunopipe/latest/processes/images/RadarPlots-cluster_order.png)
 
 
         /// Attention

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/ns/scrna_metabolic_landscape.py

@@ -22,11 +22,11 @@ class MetabolicPathwayActivity(Proc):
     For each subset, a heatmap and a violin plot will be generated.
     The heatmap shows the pathway activities for each group and each metabolic pathway
 
-    ![MetabolicPathwayActivity_heatmap](https://pwwang.github.io/immunopipe/processes/images/MetabolicPathwayActivity_heatmap.png){: width="80%"}
+    ![MetabolicPathwayActivity_heatmap](https://pwwang.github.io/immunopipe/latest/processes/images/MetabolicPathwayActivity_heatmap.png){: width="80%"}
 
     The violin plot shows the distribution of the pathway activities for each group
 
-    ![MetabolicPathwayActivity_violin](https://pwwang.github.io/immunopipe/processes/images/MetabolicPathwayActivity_violin.png){: width="45%"}
+    ![MetabolicPathwayActivity_violin](https://pwwang.github.io/immunopipe/latest/processes/images/MetabolicPathwayActivity_violin.png){: width="45%"}
 
     Envs:
         ntimes (type=int): Number of times to do the permutation
@@ -294,7 +294,7 @@ class MetabolicPathwayHeterogeneity(Proc):
     The heterogeneity can be reflected by the NES values and the p-values in
     different groups for the metabolic pathways.
 
-    ![MetabolicPathwayHeterogeneity](https://pwwang.github.io/immunopipe/processes/images/MetabolicPathwayHeterogeneity.png)
+    ![MetabolicPathwayHeterogeneity](https://pwwang.github.io/immunopipe/latest/processes/images/MetabolicPathwayHeterogeneity.png)
 
 
     Envs:

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/ns/snp.py

@@ -71,3 +71,68 @@ class PlinkSimulation(Proc):
         "sample_prefix": None,
     }
     script = "file://../scripts/snp/PlinkSimulation.py"
+
+
+class MatrixEQTL(Proc):
+    """Run Matrix eQTL
+
+    See also <https://www.bios.unc.edu/research/genomic_software/Matrix_eQTL/>
+
+    Input:
+        geno: Genotype matrix file with rows representing SNPs and columns
+            representing samples.
+        expr: Expression matrix file with rows representing genes and columns
+            representing samples.
+        cov: Covariate matrix file with rows representing covariates and columns
+            representing samples.
+
+    Output:
+        alleqtls: Matrix eQTL output file
+        cisqtls: The cis-eQTL file if `snppos` and `genepos` are provided.
+            Otherwise it'll be empty.
+
+    Envs:
+        model (choice): The model to use.
+            - `linear`: Linear model
+            - `modelLINEAR`: Same as `linear`
+            - `anova`: ANOVA model
+            - `modelANOVA`: Same as `anova`
+        pval (type=float): P-value threshold for eQTLs
+        transp (type=float): P-value threshold for trans-eQTLs.
+            If cis-eQTLs are not enabled (`snppos` and `genepos` are not set),
+            this defaults to 1e-5.
+            If cis-eQTLs are enabled, this defaults to `None`, which will disable
+            trans-eQTL analysis.
+        fdr (flag): Do FDR calculation or not (save memory if not).
+        snppos: The path of the SNP position file.
+            It could be a BED, GFF, VCF or a tab-delimited file with
+            `snp`, `chr`, `pos` as the first 3 columns.
+        genepos: The path of the gene position file.
+            It could be a BED or GFF file.
+        dist (type=int): Distance threshold for cis-eQTLs.
+        transpose_geno (flag): If set, the genotype matrix (`in.geno`)
+            will be transposed.
+        transpose_expr (flag): If set, the expression matrix (`in.expr`)
+            will be transposed.
+        transpose_cov (flag): If set, the covariate matrix (`in.cov`)
+            will be transposed.
+    """
+    input = "geno:file, expr:file, cov:file"
+    output = [
+        "alleqtls:file:{{in.geno | stem}}.alleqtls.txt",
+        "cisqtls:file:{{in.geno | stem}}.cisqtls.txt",
+    ]
+    lang = config.lang.rscript
+    envs = {
+        "model": "linear",
+        "pval": 1e-3,
+        "transp": None,
+        "fdr": False,
+        "snppos": None,
+        "genepos": config.ref.refgene,
+        "dist": 250000,
+        "transpose_geno": False,
+        "transpose_expr": False,
+        "transpose_cov": False,
+    }
+    script = "file://../scripts/snp/MatrixEQTL.R"

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/ns/tcr.py

@@ -923,7 +923,7 @@ class CloneResidency(Proc):
 
     - Residency plots showing the residency of clones in the two groups
 
-        ![CloneResidency_residency](https://pwwang.github.io/immunopipe/processes/images/CloneResidency.png)
+        ![CloneResidency_residency](https://pwwang.github.io/immunopipe/latest/processes/images/CloneResidency.png)
 
         The points in the plot are jittered to avoid overplotting. The x-axis is the residency in the first group and
         the y-axis is the residency in the second group. The size of the points are relative to the normalized size of
@@ -943,7 +943,7 @@ class CloneResidency(Proc):
 
     - Venn diagrams showing the overlap of the clones in the two groups
 
-        ![CloneResidency_venn](https://pwwang.github.io/immunopipe/processes/images/CloneResidency_venn.png){: width="60%"}
+        ![CloneResidency_venn](https://pwwang.github.io/immunopipe/latest/processes/images/CloneResidency_venn.png){: width="60%"}
 
     Input:
         immdata: The data loaded by `immunarch::repLoad()`
@@ -1259,7 +1259,7 @@ class TCRClusterStats(Proc):
         by = "Sample"
         ```
 
-        ![Cluster_size](https://pwwang.github.io/immunopipe/processes/images/TCRClusteringStats_cluster_size.png){: width="80%"}
+        ![Cluster_size](https://pwwang.github.io/immunopipe/latest/processes/images/TCRClusteringStats_cluster_size.png){: width="80%"}
 
         ### Shared clusters
 
@@ -1269,7 +1269,7 @@ class TCRClusterStats(Proc):
         heatmap_meta = ["region"]
         ```
 
-        ![Shared_clusters](https://pwwang.github.io/immunopipe/processes/images/TCRClusteringStats_shared_clusters.png){: width="80%"}
+        ![Shared_clusters](https://pwwang.github.io/immunopipe/latest/processes/images/TCRClusteringStats_shared_clusters.png){: width="80%"}
 
         ### Sample diversity
 
@@ -1278,11 +1278,11 @@ class TCRClusterStats(Proc):
         method = "gini"
         ```
 
-        ![Sample_diversity](https://pwwang.github.io/immunopipe/processes/images/TCRClusteringStats_sample_diversity.png){: width="80%"}
+        ![Sample_diversity](https://pwwang.github.io/immunopipe/latest/processes/images/TCRClusteringStats_sample_diversity.png){: width="80%"}
 
         Compared to the sample diversity using TCR clones:
 
-        ![Sample_diversity](https://pwwang.github.io/immunopipe/processes/images/Immunarch_sample_diversity.png){: width="80%"}
+        ![Sample_diversity](https://pwwang.github.io/immunopipe/latest/processes/images/Immunarch_sample_diversity.png){: width="80%"}
 
     Input:
         immfile: The immunarch object with TCR clusters attached

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/scripts/delim/SampleInfo.R

@@ -113,14 +113,14 @@ for (name in names(stats)) {
     if (stat$plot == "boxplot" || stat$plot == "box") {
         p <- ggplot(data, aes(x=!!group, y=!!sym(stat$on), fill=!!group)) +
             geom_boxplot(position = "dodge") +
-            scale_fill_biopipen() +
+            scale_fill_biopipen(alpha = .6) +
             xlab("")
     } else if (stat$plot == "violin" ||
                stat$plot == "violinplot" ||
                stat$plot == "vlnplot") {
         p <- ggplot(data, aes(x = !!group, y = !!sym(stat$on), fill=!!group)) +
             geom_violin(position = "dodge") +
-            scale_fill_biopipen() +
+            scale_fill_biopipen(alpha = .6) +
             xlab("")
     } else if (
         (grepl("violin", stat$plot) || grepl("vln", stat$plot)) &&
@@ -129,12 +129,12 @@ for (name in names(stats)) {
         p <- ggplot(data, aes(x = !!group, y = !!sym(stat$on), fill = !!group)) +
             geom_violin(position = "dodge") +
             geom_boxplot(width = 0.1, position = position_dodge(0.9), fill="white") +
-            scale_fill_biopipen() +
+            scale_fill_biopipen(alpha = .6) +
             xlab("")
     } else if (stat$plot == "histogram" || stat$plot == "hist") {
         p <- ggplot(data, aes(x = !!sym(stat$on), fill = !!group)) +
             geom_histogram(bins = 10, position = "dodge", alpha = 0.8, color = "white") +
-            scale_fill_biopipen()
+            scale_fill_biopipen(alpha = .6)
     } else if (stat$plot == "pie" || stat$plot == "piechart") {
         if (is.null(stat$each)) {
             data <- data %>% distinct(!!group, .keep_all = TRUE)
@@ -157,7 +157,7 @@ for (name in names(stats)) {
                 fill="#EEEEEE",
                 size=4
             ) +
-            scale_fill_biopipen(name = group) +
+            scale_fill_biopipen(alpha = .6, name = group) +
             ggtitle(paste0("# ", stat$on))
     } else if (stat$plot == "bar" || stat$plot == "barplot") {
         if (is.null(stat$each)) {
@@ -169,7 +169,7 @@ for (name in names(stats)) {
             data,
             aes(x = !!group, y = !!sym(count_on), fill = !!group)) +
             geom_bar(stat = "identity") +
-            scale_fill_biopipen() +
+            scale_fill_biopipen(alpha = .6) +
             ylab(paste0("# ", stat$on))
     } else {
         stop("Unknown plot type: ", stat$plot)

biopipen-0.27.5/biopipen/scripts/plot/ROC.R

@@ -0,0 +1,88 @@
+
+source("{{biopipen_dir}}/utils/misc.R")
+
+library(rlang)
+library(ggplot2)
+library(plotROC)
+
+infile <- {{in.infile | r}}
+outfile <- {{out.outfile | r}}
+joboutdir <- {{job.outdir | r}}
+noids <- {{envs.noids | r}}
+pos_label <- {{envs.pos_label | r}}
+ci <- {{envs.ci | r}}
+devpars <- {{envs.devpars | r}}
+show_auc <- {{envs.show_auc | r}}
+args <- {{envs.args | r: todot="-"}}
+style_roc_args <- {{envs.style_roc | r: todot="-"}}
+if (!is.null(style_roc_args$theme)) {
+    style_roc_args$theme <- eval(parse(text=style_roc_args$theme))
+}
+
+data <- read.table(infile, header=TRUE, sep="\t", row.names = NULL, check.names = FALSE, stringsAsFactors=FALSE)
+if (!noids) {
+    data <- data[, -1]
+}
+
+# Normalize the first column (labels) into 0 and 1.
+# If they are not 0/1, use pos_label to determine the positive class.
+label_col <- colnames(data)[1]
+if (is.character(data[[label_col]])) {
+    data[[label_col]] <- as.numeric(data[[label_col]] == pos_label)
+}
+
+models <- colnames(data)[2:ncol(data)]
+
+if (length(models) > 1) {
+    # pivot longer the models, and put the model names into the column 'model'
+    data <- melt_roc(data, label_col, colnames(data)[2:ncol(data)])
+} else {
+    data <- data.frame(
+        D = data[[label_col]],
+        M = data[[models]],
+        name = rep(models, nrow(data))
+    )
+}
+
+# Plot the ROC curve
+p <- ggplot(data, aes(d = D, m = M, color = name))
+
+if (isTRUE(ci)) {
+    p <- p + do.call(geom_rocci, args)
+} else {
+    p <- p + do.call(geom_roc, args)
+}
+
+p <- p + do.call(style_roc, style_roc_args)
+p <- p + scale_color_biopipen()
+
+if (length(models) > 1) {
+    p <- p + theme(legend.title = element_blank())
+} else {
+    p <- p + theme(legend.position = "none")
+}
+
+aucs = calc_auc(p)
+write.table(aucs, file=file.path(joboutdir, "aucs.tsv"), sep="\t", quote=FALSE, row.names=FALSE)
+
+if (show_auc) {
+    aucs = split(aucs$AUC, aucs$name)
+    if (length(aucs) > 1) {
+        # Add AUC values to the legend items
+        p <- p +
+            scale_color_manual(
+                values = pal_biopipen()(length(models)),
+                labels = sapply(models, function(m) paste(m, " (AUC =", round(aucs[[m]], 2), ")")),
+                breaks = models)
+    } else {
+        p <- p +
+            geom_text(
+                x = 0.8, y = 0.2, label = paste("AUC =", round(unlist(aucs), 2)),
+                color = "black", size = 4)
+    }
+}
+
+devpars$filename <- outfile
+do.call(png, devpars)
+print(p)
+dev.off()

{biopipen-0.27.3 → biopipen-0.27.5}/biopipen/scripts/scrna/SeuratClusterStats-features.R

@@ -81,7 +81,7 @@ do_one_features = function(name) {
     if (case$kind %in% c("ridge", "ridgeplot")) {
         case$kind = "ridge"
         if (is.null(case$cols)) {
-            case$cols = pal_biopipen()(32)
+            case$cols = pal_biopipen()(n_uidents)
         }
         excluded_args = c(excluded_args, "split.by", "reduction")
         fn = RidgePlot