biopipen 0.21.2__py3-none-any.whl → 0.22.1__py3-none-any.whl

biopipen/scripts/tcr/CDR3AAPhyschem.R

@@ -1,3 +1,4 @@
+source("{{biopipen_dir}}/utils/misc.R")
 library(dplyr)
 library(tidyr)
 library(tibble)
@@ -8,10 +9,12 @@ library(hash)
 library(glmnet)
 library(broom.mixed)
 library(stringr)
+library(slugify)
 
 immdatafile = {{in.immdata | quote}}
 srtobjfile = {{in.srtobj | r}}
 outdir = {{out.outdir | quote}}
+joboutdir = {{job.outdir | quote}}
 group_name = {{envs.group | r}}
 comparison = {{envs.comparison | r}}
 prefix = {{envs.prefix | r}}
@@ -141,7 +144,7 @@ for (i in 1:3){
 }
 
 # Loading metadata from srtobjfile
-print("Loading metadata from srtobjfile")
+log_info("Loading metadata from srtobjfile")
 if (is.null(srtobjfile)) {
     metadata = NULL
 } else {
@@ -161,7 +164,7 @@ if (is.null(srtobjfile)) {
     }
 }
 
-print("Loading immdata from immdatafile")
+log_info("Loading immdata from immdatafile")
 immdata = readRDS(immdatafile)
 
 
@@ -190,7 +193,7 @@ merge_data = function(sam) {
     if (!is.null(prefix) && nchar(prefix) > 0) {
         # Replace the placeholder like {Sample} with the data in other columns
         # in the same row
-        sdata = sdata %>% mutate(.prefix_len = nchar(glue("{{envs.prefix}}")))
+        sdata = sdata %>% mutate(.prefix_len = nchar(glue(prefix)))
         # Remove the prefix in the rownames of sdata
         rownames(sdata) = substring(rownames(sdata), sdata$.prefix_len + 1)
         sdata = sdata %>% select(-.prefix_len)
@@ -215,17 +218,17 @@ merge_data = function(sam) {
 
 # Expanded and merged with metadata
 # Now we are able to select the cells using group and comparison
-print("Merging data with metadata for each sample")
+log_info("Merging data with metadata for each sample")
 merged = NULL
 for (sam in immdata$meta$Sample) {
-    print(glue("- For sample {sam}"))
+    log_info("- For sample {sam}")
     md = merge_data(sam)
     merged = if (is.null(merged)) md else rbind(merged, md)
 }
 
 # Statistics about the cell numbers with groups avaiable in metadata
 # !!group_name, TotalCells, AvailCells, AvailCellsPct
-print("Calculating statistics")
+log_info("Calculating statistics")
 if (is.null(subset_cols)) {
     stats = merged %>%
         # group by group_name
@@ -259,9 +262,45 @@ if (is.null(subset_cols)) {
 }
 
 # save the stats
-write.table(stats, file = file.path(outdir, "stats.txt"), sep = "\t", quote = FALSE, row.names = FALSE)
+write.table(
+    stats,
+    file = file.path(outdir, "stats.txt"),
+    sep = "\t",
+    quote = FALSE,
+    row.names = FALSE,
+)
+
+add_report(
+    list(
+        kind = "descr",
+        content = "Statistics about the cells mapped to the comparison groups. Columns:"
+    ),
+    list(
+        kind = "list",
+        items = c(
+            "_Group: The group name in the comparison, or null, if cells are not mapped to any group",
+            "TotalCells: The total number of cells. This number should be the same for all groups",
+            "CellsPerGroup: The number of cells in the mapped group",
+            paste0(
+                "AvailCellsPerGroup: The number of cells with CDR3 length between ",
+                CDR3_MINLEN,
+                " and ",
+                CDR3_MAXLEN,
+                " for each group. These cells are used for the analysis"
+            ),
+            "AvailCellsPct: The percentage of AvailCellsPerGroup over CellsPerGroup"
+        )
+    ),
+    list(
+        kind = "table",
+        src = file.path(outdir, "stats.txt")
+    ),
+    h1 = "Available Cells"
+)
 
-print("Add amino acid features")
+
+
+log_info("Add amino acid features")
 merged = merged %>%
     filter(!is.na(.Group) & length >= CDR3_MINLEN & length <= CDR3_MAXLEN) %>%
     add_percentAA() %>%
@@ -269,13 +308,13 @@ merged = merged %>%
 
 
 do_one_subset = function(s) {
-    print(paste("Processing subset", s))
+    log_info(paste("Processing subset", s))
     if (is.null(s)) {
         data = merged
         odir = file.path(outdir, "ALL")
     } else {
         data = merged %>% filter(.Subset == s)
-        odir = file.path(outdir, s)
+        odir = file.path(outdir, slugify(s))
     }
     dir.create(odir, recursive = TRUE, showWarnings = FALSE)
 
@@ -342,6 +381,55 @@ do_one_subset = function(s) {
     print(g)
     dev.off()
 
+    add_report(
+        list(
+            kind = "descr",
+            content = "Estimated coefficients for each feature and position in the CDR3"
+        ),
+        h1 = ifelse(
+            is.null(s),
+            "Estimated OR (per s.d.)",
+            paste0(paste(subset_cols, collapse = ", "), " - ", s)
+        ),
+        h2 = ifelse(
+            is.null(s),
+            "#",
+            "Estimated OR (per s.d.)"
+        )
+    )
+
+    add_report(
+        list(
+            name = "Plot",
+            contents = list(
+                list(
+                    kind = "image",
+                    src = file.path(odir, "estimated_coefficients.png")
+                )
+            )
+        ),
+        list(
+            name = "Estimates",
+            contents = list(
+                list(
+                    kind = "table",
+                    src = file.path(odir, "estimates.txt")
+                )
+            )
+        ),
+        h1 = ifelse(
+            is.null(s),
+            "Estimated OR (per s.d.)",
+            paste0(paste(subset_cols, collapse = ", "), " - ", s)
+        ),
+        h2 = ifelse(
+            is.null(s),
+            "#",
+            "Estimated OR (per s.d.)"
+        ),
+        ui = "tabs"
+    )
+
     # distributions
     data$mid_hydro = sapply(data$midseq, function(x) get_feat_score(x, AA_MAPS[[2]]))
     data$smid_hydro = scale(data$mid_hydro)[,1]
@@ -372,6 +460,29 @@ do_one_subset = function(s) {
     png(file.path(odir, "distribution.png"), width=1000, height=1000, res=100)
     print(g)
     dev.off()
+
+    add_report(
+        list(
+            kind = "table_image",
+            descr = paste0(
+                "Hydrophobicity values are averaged over the CDR3 for each TCR and ",
+                "then scaled to have a mean of 0 and a variance of 1. ",
+                "Horizontal lines depict the mean for each population"
+            ),
+            src = file.path(odir, "distribution.png")
+        ),
+        h1 = ifelse(
+            is.null(s),
+            "Hydrophobicity Distribution",
+            paste0(paste(subset_cols, collapse = ", "), " - ", s)
+        ),
+        h2 = ifelse(
+            is.null(s),
+            "#",
+            "Hydrophobicity Distribution"
+        )
+    )
+
 }
 
 if (is.null(subset_cols)) {
@@ -380,3 +491,5 @@ if (is.null(subset_cols)) {
     subsets = na.omit(unique(merged$.Subset))
     sapply(subsets, do_one_subset)
 }
+
+save_report(joboutdir)

biopipen/scripts/tcr/CloneResidency.R

@@ -10,6 +10,7 @@ library(ggplot2)
 library(ggprism)
 library(ggVennDiagram)
 library(UpSetR)
+library(slugify)
 
 theme_set(theme_prism())
 
@@ -17,6 +18,7 @@ theme_set(theme_prism())
 immfile <- {{ in.immdata | quote }}
 metafile <- {{ in.metafile | r }}
 outdir <- {{ out.outdir | quote }}
+joboutdir <- {{ job.outdir | quote }}
 
 subject_key <- {{ envs.subject | r }}
 group_key <- {{ envs.group | r }}
@@ -29,7 +31,7 @@ cases <- {{ envs.cases | r }}
 
 # Fill up cases using `envs.xxx` if not provided and compose a DEFAULT case
 # if no cases are provided
-print("- Preparing cases...")
+log_info("Preparing cases...")
 if (is.null(cases) || length(cases) == 0) {
     cases <- list(
         DEFAULT = list(
@@ -60,11 +62,11 @@ if (is.null(cases) || length(cases) == 0) {
     }
 }
 
-print("- Preparing data ...")
-print("  Loading immdata ...")
+log_info("Preparing data ...")
+log_info("- Loading immdata ...")
 immdata <- readRDS(immfile)
 
-print("  Expanding immdata$data to cell-level ...")
+log_info("- Expanding immdata$data to cell-level ...")
 cldata <- do_call(rbind, lapply(names(immdata$data), function(name) {
     dat <- immdata$data[[name]] %>% separate_rows(Barcode, sep = ";") # Split barcodes
     dat$Sample <- name
@@ -76,7 +78,7 @@ cldata <- do_call(rbind, lapply(names(immdata$data), function(name) {
 }))
 
 if (!is.null(metafile)) {
-    print("  Loading metafile ...")
+    log_info("- Loading metafile ...")
     # Check if extension is rds/RDS, if so, it should be a Seurat object
     if (endsWith(metafile, ".rds") || endsWith(metafile, ".RDS")) {
         meta <- readRDS(metafile)@meta.data
@@ -86,14 +88,14 @@ if (!is.null(metafile)) {
         )
     }
 
-    print("  Merging metafile to cldata ...")
+    log_info("- Merging metafile to cldata ...")
     cldata <- cbind(
         cldata,
         meta[cldata$Barcode, setdiff(colnames(meta), colnames(cldata)), drop = FALSE]
     )
 }
 
-print("- Applying mutaters ...")
+log_info("Applying mutaters ...")
 if (!is.null(mutaters) && length(mutaters) > 0) {
     cldata <- cldata %>% mutate(!!!lapply(mutaters, parse_expr))
 }
@@ -115,7 +117,7 @@ get_groups <- function(order) {
 }
 
 perpare_case <- function(casename, case) {
-    print(paste("- Preparing case:", casename, "..."))
+    log_info("Preparing case: {casename} ...")
     # Check if required keys are provided
     if (is.null(case$subject) || length(case$subject) == 0) {
         stop(paste("`subject` is required for case:", casename))
@@ -133,13 +135,12 @@ perpare_case <- function(casename, case) {
             ))
         } else if (!any(has_comma)) {
             if (length(case$order) > 2) {
-                warning(
+                log_warn(
                     paste0(
                         "- Order of groups in case:", casename,
                         " is not recommended, please use comma to separate groups. \n",
                         "Instead of `['A', 'B', 'C']`, use `['A,B', 'A,C', 'B,C']`."
-                    ),
-                    immediate. = TRUE
+                    )
                 )
                 case$order <- sapply(
                     combn(case$order, 2, simplify = FALSE),
@@ -162,19 +163,19 @@ perpare_case <- function(casename, case) {
 
     # Create case-specific directories
     # Scatter plots
-    scatter_dir <- file.path(outdir, casename, "scatter")
+    scatter_dir <- file.path(outdir, slugify(casename, tolower = FALSE), "scatter")
     dir.create(scatter_dir, recursive = TRUE, showWarnings = FALSE)
 
     # Venn diagrams
-    venn_dir <- file.path(outdir, casename, "venn")
+    venn_dir <- file.path(outdir, slugify(casename, tolower = FALSE), "venn")
     dir.create(venn_dir, recursive = TRUE, showWarnings = FALSE)
 
     # Upset plots
-    upset_dir <- file.path(outdir, casename, "upset")
+    upset_dir <- file.path(outdir, slugify(casename, tolower = FALSE), "upset")
     dir.create(upset_dir, recursive = TRUE, showWarnings = FALSE)
 
     # Counts
-    counts_dir <- file.path(outdir, casename, "counts")
+    counts_dir <- file.path(outdir, slugify(casename, tolower = FALSE), "counts")
     dir.create(counts_dir, recursive = TRUE, showWarnings = FALSE)
 
     case
@@ -376,6 +377,7 @@ plot_upset <- function(counts, singletons) {
 }
 
 handle_subject <- function(i, subjects, casename, case) {
+    casedir = file.path(outdir, slugify(casename, tolower = FALSE))
     # Generate a residency table
     # |    CDR3.aa    | Tumor | Normal |
     # | SEABESRWEFAEF | 0     | 10     |
@@ -387,7 +389,7 @@ handle_subject <- function(i, subjects, casename, case) {
         as.character() %>%
         paste(collapse = "-")
 
-    print(paste("- Handling", i, case$subject, "..."))
+    log_info("Handling {i} {case$subject} ...")
 
     if (!is.null(case$subset)) {
         counts <- cldata %>% filter(!!parse_expr(case$subset))
@@ -412,10 +414,7 @@ handle_subject <- function(i, subjects, casename, case) {
         case$order <- sapply(combn(groups, 2, simplify = FALSE), function(x) paste(x, collapse = ","))
     }
     if (length(unique(counts[[case$group]])) < 2) {
-        warning(
-            paste0("-", casename, ", Subject doesn't have enough groups:", subject),
-            immediate. = TRUE
-        )
+        log_warn("{casename}, Subject doesn't have enough groups: {subject}")
         return()
     }
     singletons = counts %>%
@@ -437,7 +436,7 @@ handle_subject <- function(i, subjects, casename, case) {
 
     # Save samples to group_by so they can be aligned accordingly in the report
     if (!is.null(section)) {
-        group_dir <- file.path(outdir, casename, "section")
+        group_dir <- file.path(casedir, "section")
         dir.create(group_dir, showWarnings = FALSE)
 
         sgroups <- subject_row %>%
@@ -445,60 +444,90 @@ handle_subject <- function(i, subjects, casename, case) {
             pull(section) %>%
             unique() %>%
             paste(collapse = "-")
-        group_file <- file.path(group_dir, paste0(sgroups, ".txt"))
+        group_file <- file.path(group_dir, paste0(slugify(sgroups), ".txt"))
         cat(subject, file = group_file, sep = "\n", append = TRUE)
     }
 
     # Save counts
-    counts_dir <- file.path(outdir, casename, "counts")
+    counts_dir <- file.path(casedir, "counts")
+    countfile <- file.path(counts_dir, paste0(slugify(subject), ".txt"))
     write.table(
         counts,
-        file = file.path(counts_dir, paste0(subject, ".txt")),
+        file = countfile,
         sep = "\t",
         row.names = TRUE,
         col.names = TRUE,
         quote = FALSE
     )
+    add_report(
+        list(kind = "table", src = countfile, ds_name = subject),
+        h1 = ifelse(casename == "DEFAULT", "Clone Size Tables", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Clone size Tables"),
+        ui = "dropdown_switcher"
+    )
 
     # scatter plot
     # Make plots B ~ A, C ~ B, and C ~ A for order A, B, C
     # combns <- combn(groups, 2, simplify = FALSE)
-    scatter_dir <- file.path(outdir, casename, "scatter")
+    scatter_dir <- file.path(casedir, "scatter")
     for (j in seq_along(case$order)) {
         pair <- strsplit(case$order[j], ",")[[1]]
         if (length(setdiff(pair, groups)) > 0) {
-            warning(
+            log_warn(
                 paste0(
                     "- One of the comparisons doesn't exist in case (", casename,
                     ") for subject (", subject, "): ",
                     case$order[j]
-                ),
-                immediate. = TRUE
+                )
             )
             next
         }
         scatter_p <- plot_scatter(counts, subject, pair[1], pair[2])
         scatter_png <- file.path(
             scatter_dir,
-            paste0("scatter_", subject, "_", pair[1], "_", pair[2], ".png")
+            paste0("scatter_", slugify(subject), "_", slugify(pair[1]), "_", slugify(pair[2]), ".png")
         )
         png(scatter_png, res = 100, height = 800, width = 1000)
         print(scatter_p)
         dev.off()
+
+        add_report(
+            list(
+                name = paste0(subject, " (", pair[1], " - ", pair[2], ")"),
+                src = scatter_png
+            ),
+            h1 = ifelse(casename == "DEFAULT", "Residency Plots", casename),
+            h2 = ifelse(casename == "DEFAULT", "#", "Residency Plots"),
+            ui = "table_of_images:3"
+        )
     }
 
     # upset/venn
-    venn_dir <- file.path(outdir, casename, "venn")
-    venn_png <- file.path(venn_dir, paste0("venn_", subject, ".png"))
+    venn_dir <- file.path(casedir, "venn")
+    venn_png <- file.path(venn_dir, paste0("venn_", slugify(subject), ".png"))
     png(venn_png, res = 100, height = 600, width = 800)
     print(plot_venndg(counts, groups, singletons))
     dev.off()
 
-    upset_dir <- file.path(outdir, casename, "upset")
-    upset_png <- file.path(upset_dir, paste0("upset_", subject, ".png"))
+    add_report(
+        list(src = venn_png),
+        h1 = ifelse(casename == "DEFAULT", "Overlapping Clones (Venn Diagram)", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Overlapping Clones (Venn Diagram)"),
+        ui = "table_of_images:3"
+    )
+
+    upset_dir <- file.path(casedir, "upset")
+    upset_png <- file.path(upset_dir, paste0("upset_", slugify(subject), ".png"))
     png(upset_png, res = 100, height = 600, width = 800)
     print(plot_upset(counts, singletons))
     dev.off()
+
+    add_report(
+        list(src = upset_png),
+        h1 = ifelse(casename == "DEFAULT", "Overlapping Clones (UpSet Plots)", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Overlapping Clones (UpSet Plots)"),
+        ui = "table_of_images:3"
+    )
 }
 
 handle_case <- function(casename, case) {
@@ -513,9 +542,60 @@ handle_case <- function(casename, case) {
             distinct(!!!syms(case$subject)) %>%
             drop_na()
     }
+    add_report(
+        list(ds_name = "Select a subject ..."),
+        h1 = ifelse(casename == "DEFAULT", "Clone Size Tables", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Clone size Tables"),
+        ui = "dropdown_switcher"
+    )
+    add_report(
+        list(
+            kind = "descr",
+            content = paste0(
+                "The residency plots showing the clones of paired samples (x-axis and y-axis). ",
+                "The size of the dot represents the relative abundance of the clone. ",
+                "The color of the dot represents the type of the clone: "
+            )
+        ),
+        list(
+            kind = "list",
+            items = c(
+                "Collapsed (clones that are less abundant in the y-axis sample)",
+                "Dual (clones that are equally abundant in both samples)",
+                "Expanded (clones that are more abundant in the y-axis sample)",
+                "(x-axis sample) Multiplet (clones that are only present in the x-axis sample, with multiple cells)",
+                "(x-axis sample) Singleton (clones that are only present in the x-axis sample, with a single cell)",
+                "(y-axis sample) Multiplet (clones that are only present in the y-axis sample, with multiple cells)",
+                "(y-axis sample) Singleton (clones that are only present in the y-axis sample, with a single cell)"
+            )
+        ),
+        h1 = ifelse(casename == "DEFAULT", "Residency Plots", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Residency Plots"),
+        ui = "flat"
+    )
+    add_report(
+        list(
+            kind = "descr",
+            content = "For samples in each subject, showing the overlapping clones between samples in Venn diagrams."
+        ),
+        h1 = ifelse(casename == "DEFAULT", "Overlapping Clones (Venn Diagram)", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Overlapping Clones (Venn Diagram)"),
+        ui = "flat"
+    )
+    add_report(
+        list(
+            kind = "descr",
+            content = "For samples in each subject, showing the overlapping clones between samples in UpSet plots."
+        ),
+        h1 = ifelse(casename == "DEFAULT", "Overlapping Clones (UpSet Plots)", casename),
+        h2 = ifelse(casename == "DEFAULT", "#", "Overlapping Clones (UpSet Plots)"),
+        ui = "flat"
+    )
     sapply(seq_len(nrow(subjects)), handle_subject, subjects, casename, case)
 }
 
 for (casename in names(cases)) {
     handle_case(casename, cases[[casename]])
-}
+}
+
+save_report(joboutdir)

biopipen/scripts/tcr/Immunarch-basic.R

@@ -70,8 +70,22 @@ do_one_case_basic = function(name, case, method) {
     }
     ofig = file.path(odir, paste0(name, ".png"))
     png(ofig, width = case$devpars$width, height = case$devpars$height, res = case$devpars$res)
-    print(p)
+    print(p + scale_fill_biopipen())
     dev.off()
+
+    add_report(
+        list(
+            src = ofig,
+            name = if (name == "DEFAULT") NULL else name
+        ),
+        h1 = "Exploratory Analysis",
+        h2 = switch(method,
+            len = "CDR3 length distribution",
+            volume = "Clonotype volume (Number of clonotypes)",
+            count = "Clonotype abundances"
+        ),
+        ui = "table_of_images"
+    )
 }
 
 # Do cases
@@ -82,11 +96,11 @@ do_cases_basic = function(cases, method) {
     }
 }
 
-volumes = fill_up_cases_basic(volumes)
-do_cases_basic(volumes, "volume")
-
 lens = fill_up_cases_basic(lens)
 do_cases_basic(lens, "len")
 
+volumes = fill_up_cases_basic(volumes)
+do_cases_basic(volumes, "volume")
+
 counts = fill_up_cases_basic(counts)
 do_cases_basic(counts, "count")

biopipen/scripts/tcr/Immunarch-clonality.R

@@ -82,6 +82,20 @@ do_one_case_clonality = function(name, case, method) {
     png(ofig, width = case$devpars$width, height = case$devpars$height, res = case$devpars$res)
     print(p)
     dev.off()
+
+    add_report(
+        list(
+            src = ofig,
+            name = if (name == "DEFAULT") NULL else name
+        ),
+        h1 = "Clonality Analysis",
+        h2 = switch(method,
+            top = "Top Clones",
+            rare = "Rare Clones",
+            homeo = "Clonal Space Homeostasis"
+        ),
+        ui = "table_of_images"
+    )
 }
 
 # Do cases