biopipen 0.21.2__py3-none-any.whl → 0.22.1__py3-none-any.whl

biopipen/scripts/tcr/TCRClusterStats.R

@@ -6,13 +6,16 @@ library(dplyr)
 library(rlang)
 library(immunarch)
 library(ggprism)
+library(slugify)
 
 immfile = {{in.immfile | quote}}
 outdir = {{out.outdir | quote}}
 cluster_size_envs = {{envs.cluster_size | r}}
 shared_clusters_envs = {{envs.shared_clusters | r}}
 sample_diversity_envs = {{envs.sample_diversity | r}}
+joboutdir = {{job.outdir | r}}
 
+log_info("Expanding analysis cases ...")
 expand_cases = function(envs) {
     cases = envs$cases
     envs$cases = NULL
@@ -51,8 +54,9 @@ shared_clusters_cases = expand_cases(shared_clusters_envs)
 sample_diversity_cases = expand_cases(sample_diversity_envs)
 
 cluster_size_distribution = function(name) {
-    print(paste0("- Working on cluster size distribution: ", name))
-    odir = file.path(outdir, "ClusterSizeDistribution", name)
+    log_info("- Working on cluster size distribution: {name}")
+
+    odir = file.path(outdir, "ClusterSizeDistribution", slugify(name, tolower = FALSE))
     dir.create(odir, showWarnings = FALSE, recursive = TRUE)
     case = cluster_size_cases[[name]]
 
@@ -75,16 +79,28 @@ cluster_size_distribution = function(name) {
         ggs = c(
             "theme_prism()",
             "scale_y_continuous(trans='log10')",
-            "labs(x='TCR cluster size', y='Count')"
+            "labs(x='TCR cluster size', y='Count')",
+            "scale_fill_biopipen()"
         ),
         devpars = case$devpars,
         outfile = outplot
     )
+
+    add_report(
+        list(
+            src = outplot,
+            name = ifelse(name == "DEFAULT", FALSE, name),
+            descr = paste0("Cluster size distribution for each ", case$by)
+        ),
+        ui = "table_of_images",
+        h1 = "Cluster Size Distribution"
+    )
 }
 
 shared_clusters = function(name) {
-    print(paste0("- Working on shared clusters: ", name))
-    odir = file.path(outdir, "SharedClusters", name)
+    log_info("- Working on shared clusters: {name}")
+
+    odir = file.path(outdir, "SharedClusters", slugify(name, tolower = FALSE))
     dir.create(odir, showWarnings = FALSE, recursive = TRUE)
     case = shared_clusters_cases[[name]]
     if (!is.null(case$grouping)) {
@@ -139,12 +155,21 @@ shared_clusters = function(name) {
         ),
         devpars = case$devpars,
         outfile = file.path(odir, "shared_clusters.png")
+    )
 
+    add_report(
+        list(
+            src = file.path(odir, "shared_clusters.png"),
+            name = ifelse(name == "DEFAULT", FALSE, name),
+            descr = paste0("Shared TCR clusters across samples")
+        ),
+        ui = "table_of_images",
+        h1 = "Shared TCR Clusters"
     )
 }
 
 shared_clusters_by_grouping = function(name) {
-    odir = file.path(outdir, "SharedClusters", name)
+    odir = file.path(outdir, "SharedClusters", slugify(name, tolower = FALSE))
     case = shared_clusters_cases[[name]]
 
     data = list()
@@ -176,12 +201,47 @@ shared_clusters_by_grouping = function(name) {
         devpars = case$devpars,
         outfile = outfile
     )
+
+    add_report(
+        list(
+            src = outfile,
+            name = ifelse(name == "DEFAULT", FALSE, name),
+            descr = paste0("Shared TCR clusters across ", grouping)
+        ),
+        ui = "table_of_images",
+        h1 = "Shared TCR Clusters"
+    )
 }
 
 
+div_methods = list(
+    gini = list(
+        name = "The Gini coefficient",
+        descr = "The Gini coefficient is a measure of statistical dispersion intended to represent the income or wealth distribution of a nation's residents, and is the most commonly used measurement of inequality."
+    ),
+    gini.simp = list(
+        name = "The Gini-Simpson index",
+        descr = "The Gini-Simpson index is a measure of diversity. It is one of the most commonly used in ecology. It is also known as the Simpson index, the Simpson concentration index, the Simpson dominance index, or the Simpson diversity index."
+    ),
+    inv.simp = list(
+        name = "The inverse Simpson index",
+        descr = "It is the effective number of types that is obtained when
+                 the weighted arithmetic mean is used to quantify average
+                 proportional abundance of types in the dataset of interest."
+    ),
+    div = list(
+        name = "The true diversity",
+        descr = "It refers to the number of equally abundant types needed
+                 for the average proportional abundance of the types to
+                 equal that observed in the dataset of interest where all
+                 types may not be equally abundant."
+    )
+)
+
 sample_diversity = function(name) {
-    print(paste0("- Working on sample diversity: ", name))
-    odir = file.path(outdir, "SampleDiversity", name)
+    log_info("- Working on sample diversity: {name}")
+
+    odir = file.path(outdir, "SampleDiversity", slugify(name, tolower = FALSE))
     dir.create(odir, showWarnings = FALSE, recursive = TRUE)
     case = sample_diversity_cases[[name]]
 
@@ -192,7 +252,19 @@ sample_diversity = function(name) {
     outfile = file.path(odir, "diversity.txt")
     outplot = file.path(odir, "diversity.png")
     div = repDiversity(data, .method = case$method)
-    write.table(div, outfile, row.names=TRUE, col.names=TRUE, quote=FALSE, sep="\t")
+    write.table(
+        if (ncol(div) == 1) {
+            as.data.frame(div) %>% rownames_to_column("Sample")
+        } else {
+            div
+        },
+        outfile,
+        row.names=TRUE,
+        col.names=TRUE,
+        quote=FALSE,
+        sep="\t"
+    )
+
     if (case$method == "gini") {
         div = as.data.frame(div) %>% rownames_to_column("Sample")
         colnames(div)[2] = "gini"
@@ -201,7 +273,8 @@ sample_diversity = function(name) {
         mapping = aes(x = Sample, y = gini, fill = Sample)
         ggs = c(
             "theme_prism(axis_text_angle = 90)",
-            "labs(title='Gini coefficient', subtitle='Sample diversity estimation using the Gini coefficient')"
+            "labs(title='Gini coefficient', subtitle='Sample diversity estimation using the Gini coefficient')",
+            "scale_fill_biopipen()"
         )
         if (is.null(case$by) || length(case$by) == 0) {
 
@@ -225,7 +298,6 @@ sample_diversity = function(name) {
             devpars = case$devpars,
             outfile = outplot
         )
-
     } else {
         if (is.null(case$by) || length(case$by) == 0) {
             p = vis(div)
@@ -243,6 +315,41 @@ sample_diversity = function(name) {
         print(p)
         dev.off()
     }
+
+    add_report(
+        list(
+            ui = "flat",
+            label = "Diversity Plot",
+            contents = list(
+                list(
+                    kind = "descr",
+                    content = paste(
+                        div_methods[[case$method]]$name,
+                        ifelse(
+                            is.null(case$by) || length(case$by) == 0,
+                            "",
+                            paste0(" grouped by ", paste(case$by, collapse = ", "))
+                        ),
+                        div_methods[[case$method]]$descr
+                    )
+                ),
+                list(
+                    kind = "image",
+                    src = outplot
+                )
+            )
+        ),
+        list(
+            ui = "flat",
+            label = "Diversity Table",
+            contents = list(
+                list(kind = "table", src = outfile, data = list(index_col = 0))
+            )
+        ),
+        ui = "tabs",
+        h2 = ifelse(name == "DEFAULT", "#", name),
+        h1 = "Sample Diversity using TCR clusters"
+    )
 }
 
 
@@ -250,14 +357,20 @@ sample_diversity = function(name) {
     # main
     # --------------------------------------------------
     # Load immunarch data
+    log_info("Loading immunarch data ...")
     immdata = readRDS(immfile)
 
     # Cluster size distribution
+    log_info("Performing cluster size distribution analysis ...")
     sapply(names(cluster_size_cases), cluster_size_distribution)
 
     # Shared clusters
+    log_info("Performing shared clusters analysis ...")
     sapply(names(shared_clusters_cases), shared_clusters)
 
     # Diversity
+    log_info("Performing sample diversity analysis ...")
     sapply(names(sample_diversity_cases), sample_diversity)
+
+    save_report(joboutdir)
 }

biopipen/scripts/tcr/TCRClustering.R

@@ -3,11 +3,13 @@
 # python = Sys.which({{envs.python | r}})
 # Sys.setenv(RETICULATE_PYTHON = python)
 # library(reticulate)
+source("{{biopipen_dir}}/utils/single_cell.R")
 
 library(immunarch)
 library(dplyr)
 library(tidyr)
 library(tibble)
+library(glue)
 
 immfile = {{in.immfile | r}}
 outdir = normalizePath({{job.outdir | r}})
@@ -17,6 +19,7 @@ tool = {{envs.tool | r}}
 python = {{envs.python | r}}
 on_multi = {{envs.on_multi | r}}
 args = {{envs.args | r}}
+prefix = {{envs.prefix | r}}
 
 setwd(outdir)
 
@@ -26,17 +29,13 @@ if (on_multi) {
 } else {
     seqdata = immdata$data
 }
+if (is.null(prefix)) { prefix = immdata$prefix }
+if (is.null(prefix)) { prefix = "" }
 
 get_cdr3aa_df = function() {
-    out = NULL
-    for (sample in names(immdata$data)) {
-        tmpdf = immdata$data[[sample]] %>%
-            select(Barcode, CDR3.aa) %>%
-            separate_rows(Barcode, sep = ";") %>%
-            mutate(Barcode = paste0(sample, "_", Barcode))
-        out = bind_rows(out, tmpdf)
-    }
-    out
+    expand_immdata(immdata, cell_id = "Barcode") %>%
+        mutate(Barcode = glue(paste0(prefix, "{Barcode}"))) %>%
+        select(Barcode, CDR3.aa)
 }
 cdr3aa_df = get_cdr3aa_df()
 

biopipen/scripts/tcr/TESSA.R

@@ -1,8 +1,10 @@
 source("{{biopipen_dir}}/utils/misc.R")
+source("{{biopipen_dir}}/utils/single_cell.R")
 
 library(glue)
 library(dplyr)
 library(tidyr)
+library(tibble)
 library(immunarch)
 library(Seurat)
 library(ggplot2)
@@ -11,7 +13,9 @@ library(ggprism)
 immfile <- {{in.immdata | r}}
 exprfile <- {{in.srtobj | r}}
 outfile <- {{out.outfile | r}}
+joboutdir <- {{job.outdir | r}}
 python <- {{envs.python | r}}
+prefix <- {{envs.prefix | r}}
 within_sample <- {{envs.within_sample | r}}
 assay <- {{envs.assay | r}}
 predefined_b <- {{envs.predefined_b | r}}
@@ -27,35 +31,22 @@ if (!dir.exists(tessa_dir)) dir.create(tessa_dir)
 
 ### Start preparing input files for TESSA
 # Prepare input files
-print("Preparing TCR input file ...")
-immdata <- readRDS(immfile)
-
-has_VJ <- "V.name" %in% colnames(immdata$data[[1]]) && "J.name" %in% colnames(immdata$data[[1]])
-# Merge all samples
-tcrdata <- do_call(rbind, lapply(seq_len(nrow(immdata$meta)), function(i) {
-    # Clones  Proportion   CDR3.aa                       Barcode
-    # 5      4 0.008583691 CAVRDTGNTPLVF;CASSEYSNQPQHF   GTTCGGGCACTTACGA-1;TCTCTAAGTACCAGTT-1
-    # 6      4 0.008583691 CALTQAAGNKLTF;CASRPEDLRGQPQHF GCTTGAAGTCGGCACT-1;TACTCGCTCCTAAGTG-1
-    if (has_VJ) {
-        cldata = immdata$data[[i]][, c("Barcode", "CDR3.aa", "V.name", "J.name")]
-    } else {
-        cldata = immdata$data[[i]][, c("Barcode", "CDR3.aa")]
-    }
-    # # A tibble: 4 × 5
-    # Sample                  Patient     Timepoint Tissue
-    # <chr>                   <chr>       <chr>     <chr>
-    # 1 MC1685Pt011-Baseline-PB MC1685Pt011 Baseline  PB
-    mdata = as.list(immdata$meta[i, , drop=FALSE])
-    for (mname in names(mdata)) {
-        assign(mname, mdata[[mname]])
-    }
+log_info("Preparing TCR input file ...")
+# If immfile endswith .rds, then it is an immunarch object
+if (endsWith(tolower(immfile), ".rds")) {
+    immdata <- readRDS(immfile)
+    if (is.null(prefix)) { prefix = immdata$prefix }
+    if (is.null(prefix)) { prefix = "" }
+    tcrdata <- expand_immdata(immdata) %>%
+        mutate(Barcode = glue(paste0(prefix, "{Barcode}")))
+    rm(immdata)
+} else {
+    tcrdata <- read.table(immfile, sep="\t", header=TRUE, row.names=1) %>%
+        rownames_to_column("Barcode")
+}
+
+has_VJ <- "V.name" %in% colnames(tcrdata) && "J.name" %in% colnames(tcrdata)
 
-    cldata %>%
-        separate_rows(Barcode, sep=";") %>%
-        # Just in case there are duplicated barcodes
-        distinct(Barcode, .keep_all = TRUE) %>%
-        mutate(Barcode = glue("{{envs.prefix}}{Barcode}"), sample = Sample)
-}))
 if (has_VJ) {
     tcrdata <- tcrdata %>% dplyr::mutate(
         v_gene = sub("-\\d+$", "", V.name),
@@ -65,18 +56,18 @@ if (has_VJ) {
         cdr3 = CDR3.aa,
         v_gene,
         j_gene,
-        sample
+        sample = Sample
     )
 } else {
     tcrdata <- tcrdata %>% dplyr::select(
         contig_id = Barcode,
         cdr3 = CDR3.aa,
-        sample
+        sample = Sample
     )
 }
 
 
-print("Preparing expression input file ...")
+log_info("Preparing expression input file ...")
 is_seurat <- endsWith(tolower(exprfile), ".rds")
 is_gz <- endsWith(tolower(exprfile), ".gz")
 
@@ -94,31 +85,34 @@ cell_ids <- intersect(tcrdata$contig_id, colnames(expr))
 unused_tcr_cells <- setdiff(tcrdata$contig_id, cell_ids)
 unused_expr_cells <- setdiff(colnames(expr), cell_ids)
 if (length(unused_tcr_cells) > 0) {
-    warning(glue("{length(unused_tcr_cells)}/{nrow(tcrdata)} TCR cells are not used."), immediate. = TRUE)
+    log_warn(glue("{length(unused_tcr_cells)}/{nrow(tcrdata)} TCR cells are not used."))
 }
 if (length(unused_expr_cells) > 0) {
-    warning(glue("{length(unused_expr_cells)}/{ncol(expr)} expression cells are not used."), immediate. = TRUE)
+    log_warn(glue("{length(unused_expr_cells)}/{ncol(expr)} expression cells are not used."))
 }
 if (length(cell_ids) == 0) {
-    stop("No common cells between TCR and expression data. Are you using the correct prefix?")
+    stop(paste0(
+        "No common cells between TCR and expression data. ",
+        "Are you using the correct `envs.prefix` here or in `ImmunarchLoading`?"
+    ))
 }
 tcrdata <- tcrdata[tcrdata$contig_id %in% cell_ids, , drop=FALSE]
 expr <- as.matrix(expr)[, tcrdata$contig_id, drop=FALSE]
 
 # Write input files
-print("Writing input files ...")
+log_info("Writing input files ...")
 write.table(tcrdata, file.path(tessa_dir, "tcrdata.txt"), sep=",", quote=FALSE, row.names=FALSE)
 write.table(expr, file.path(tessa_dir, "exprdata.txt"), sep=",", quote=FALSE, row.names=TRUE, col.names=TRUE)
 
 ### End preparing input files for TESSA
 
 ### Start running TESSA
-print("Running TESSA ...")
+log_info("Running TESSA ...")
 
 # The original TESSA uses a python wrapper to run the encoder and tessa model
 # here we run those two steps directly here
 
-print("- Running encoder ...")
+log_info("- Running encoder ...")
 cmd_encoder <- paste(
     python,
     file.path(tessa_srcdir, "BriseisEncoder.py"),
@@ -140,14 +134,14 @@ if (has_VJ) {
         file.path(tessa_dir, "tcr_vj.txt")
     )
 }
-print(paste("- ", cmd_encoder))
+log_info(paste("- ", cmd_encoder))
 
 rc <- system(cmd_encoder)
 if (rc != 0) {
     stop("Error: Failed to run encoder.")
 }
 
-print("- Running TESSA model ...")
+log_info("- Running TESSA model ...")
 source(file.path(tessa_srcdir, "real_data.R"))
 
 tessa <- run_tessa(
@@ -162,7 +156,7 @@ tessa <- run_tessa(
 )
 
 # Save TESSA results
-print("Saving TESSA results ...")
+log_info("Saving TESSA results ...")
 if (is_seurat) {
     cells <- rownames(sobj@meta.data)
     sobj@meta.data <- sobj@meta.data %>%
@@ -187,7 +181,7 @@ if (is_seurat) {
 }
 
 # Post analysis
-print("Post analysis ...")
+log_info("Post analysis ...")
 plot_tessa(tessa, result_dir)
 plot_Tessa_clusters(tessa, result_dir)
 
@@ -201,3 +195,34 @@ p <- tessa$meta %>%
 png(file.path(result_dir, "Cluster_size_dist.png"), width=8, height=8, units="in", res=100)
 print(p)
 dev.off()
+
+add_report(
+    list(
+        src = file.path(result_dir, "Cluster_size_dist.png"),
+        descr = "Histogram of cluster size distribution"
+    ),
+    list(
+        src = file.path(result_dir, "clone_size.png"),
+        descr = "Center cluster size vs. non-center cluster size"
+    ),
+    list(
+        src = file.path(result_dir, "exp_TCR_pair_plot.png"),
+        descr = "Expression-TCR distance plot"
+    ),
+    list(
+        src = file.path(result_dir, "TCR_dist_density.png"),
+        descr = "TCR distance density plot"
+    ),
+    list(
+        src = file.path(result_dir, "TCR_explore.png"),
+        descr = "Exploratory plot at the TCR level"
+    ),
+    list(
+        src = file.path(result_dir, "TCR_explore_clusters.png"),
+        descr = "TESSA clusters"
+    ),
+    h1 = "TESSA Results",
+    ui = "table_of_images"
+)
+
+save_report(joboutdir)

biopipen/utils/misc.R

@@ -1,12 +1,13 @@
 # Misc utilities for R
 library(logger)
+library(jsonlite)
 
 .logger_layout <- layout_glue_generator(
     format = '{sprintf("%-7s", level)} [{format(time, "%Y-%m-%d %H:%M:%S")}] {msg}'
 )
 log_layout(.logger_layout)
 log_appender(appender_stdout)
-log_errors()
+tryCatch(log_errors(), error = function(e) {})
 
 .isBQuoted <- function(x) {
     # Check if x is backtick-quoted
@@ -112,3 +113,97 @@ list_update <- function(x, y) {
     }
     x
 }
+
+#’ Biopipen palette
+#’ @param alpha Alpha value
+#’ @return A palette function
+#' @export
+pal_biopipen <- function(alpha = 1) {
+    if (alpha > 1L | alpha <= 0L) stop("alpha must be in (0, 1]")
+    colors <- c(
+        "#ec3f3f", "#009e73", "#008ad8", "#cc79a7",
+        "#e69f00", "#50cada", "#f0e442", "#a76ce7",
+        "#ff864d", "#45e645", "#3699b5", "#ffdcda",
+        "#d55e00", "#778ba6", "#c37b35", "#bc28ff"
+    )
+    colors <- scales::alpha(colors, alpha)
+    function(n) {
+        if (n <= length(colors)) {
+            colors[1:n]
+        } else {
+            out_colors <- colors
+            out_alpha <- 1.0
+            while(length(out_colors) < n) {
+                out_alpha <- out_alpha - 0.3
+                out_colors <- c(out_colors, scales::alpha(colors, out_alpha))
+            }
+            out_colors[1:n]
+        }
+    }
+}
+
+scale_color_biopipen <- function(alpha = 1, ...) {
+    ggplot2::discrete_scale("colour", "biopipen", pal_biopipen(alpha), ...)
+}
+
+scale_colour_biopipen <- scale_color_biopipen
+
+scale_fill_biopipen <- function(alpha = 1, ...) {
+    ggplot2::discrete_scale("fill", "biopipen", pal_biopipen(alpha), ...)
+}
+
+.report <- list(
+    # h1 => list(
+    #   h2 => list(
+    #       h3#1 => list(ui1 => list(content11, content12)),
+    #       h3#2 => list(ui2 => list(content21, content22))
+    #   )
+    # )
+)
+
+add_report <- function(..., h1, h2 = "#", h3 = "#", ui = "flat") {
+    if (is.null(.report[[h1]])) {
+        .report[[h1]] <<- list()
+    }
+    if (is.null(.report[[h1]][[h2]])) {
+        .report[[h1]][[h2]] <<- list()
+    }
+    if (is.null(.report[[h1]][[h2]][[h3]])) {
+        .report[[h1]][[h2]][[h3]] <<- list()
+    }
+    if (is.null(.report[[h1]][[h2]][[h3]][[ui]])) {
+        .report[[h1]][[h2]][[h3]][[ui]] <<- list()
+    }
+    content = list(...)
+    for (i in seq_along(content)) {
+        .report[[h1]][[h2]][[h3]][[ui]] <<- c(
+            .report[[h1]][[h2]][[h3]][[ui]],
+            list(content[[i]])
+        )
+    }
+}
+
+save_report <- function(path, clear = TRUE) {
+    if (dir.exists(path)) {
+        path <- file.path(path, "report.json")
+    }
+
+    writeLines(toJSON(.report, pretty = TRUE, auto_unbox = TRUE), path)
+    if (clear) {
+        .report <<- list()
+    }
+}
+
+
+# Escape html
+html_escape <- function(text) {
+    if (is.null(text)) {
+        return("")
+    }
+    text = gsub("&", "&amp;", text)
+    text = gsub("<", "&lt;", text)
+    text = gsub(">", "&gt;", text)
+    text = gsub("\"", "&quot;", text)
+    text = gsub("'", "&#039;", text)
+    text
+}

{biopipen-0.21.2.dist-info → biopipen-0.22.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: biopipen
-Version: 0.21.2
+Version: 0.22.1
 Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
 License: MIT
 Author: pwwang