biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/scripts/tcr/Immunarch.R

@@ -1,5 +1,5 @@
-source("{{biopipen_dir}}/utils/misc.R")
-source("{{biopipen_dir}}/utils/single_cell.R")
+{{ biopipen_dir | joinpaths: "utils", "misc.R" | source_r }}
+{{ biopipen_dir | joinpaths: "utils", "single_cell.R" | source_r }}
 # Basic analysis and clonality
 # TODO: How about TRA chain?
 library(rlang)
@@ -11,19 +11,29 @@ library(dplyr)
 library(glue)
 library(tidyr)
 library(tibble)
+library(logger)
 
+log_info("Loading arguments ...")
 theme_set(theme_prism())
 
 immfile = {{ in.immdata | r }}
 metafile = {{ in.metafile | r }}
 outdir = {{ out.outdir | r }}
+joboutdir = {{ job.outdir | r }}
 mutaters = {{ envs.mutaters | r }}
 prefix = {{ envs.prefix | r }}
 
+log_info("Loading immdata ...")
 immdata = readRDS(immfile)
+
+if (is.null(prefix)) { prefix = immdata$prefix }
+if (is.null(prefix)) { prefix = "" }
+
+log_info("Expanding immdata ...")
 exdata = expand_immdata(immdata)
 
-if (endsWith(metafile, ".rds") || endsWith(metafile, ".RDS")) {
+log_info("Loading metadata if provided ...")
+if (!is.null(metafile) && (endsWith(metafile, ".rds") || endsWith(metafile, ".RDS"))) {
     meta = readRDS(metafile)@meta.data
 } else if (!is.null(metafile) && nchar(metafile) > 0) {
     meta = read.table(metafile, sep = "\t", header = TRUE, row.names = 1)
@@ -31,6 +41,7 @@ if (endsWith(metafile, ".rds") || endsWith(metafile, ".RDS")) {
     meta = NULL
 }
 
+log_info("Merging metadata if provided ...")
 if (!is.null(meta)) {
     cell_ids = glue_data(exdata, paste0(prefix, "{Barcode}"))
     dup_names = intersect(colnames(meta), colnames(exdata))
@@ -43,8 +54,10 @@ if (!is.null(meta)) {
 }
 rm(meta)
 
+log_info("Mutating data if `envs.mutaters` is provided ...")
 if (!is.null(mutaters) && length(mutaters) > 0) {
     exdata = mutate(exdata, !!!lapply(mutaters, parse_expr))
+    immdata = immdata_from_expanded(exdata)
 }
 
 n_samples = length(immdata$data)
@@ -52,39 +65,78 @@ n_samples = length(immdata$data)
 ##################
 # Basic analysis #
 ##################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-basic.R" %}
+volumes = {{envs.volumes | r}}
+lens = {{envs.lens | r}}
+counts = {{envs.counts | r}}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-basic.R" | source_r }}
 
 ##################
 # Clonality      #
 ##################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-clonality.R" %}
+top_clones = {{envs.top_clones | r}}
+rare_clones = {{envs.rare_clones | r}}
+hom_clones = {{envs.hom_clones | r}}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-clonality.R" | source_r }}
 
 ##################
 # Overlap        #
 ##################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-overlap.R" %}
+overlaps = {{ envs.overlaps | r: todot="-" }}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-overlap.R" | source_r }}
 
 ##################
 # Gene usage     #
 ##################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-geneusage.R" %}
+gene_usages = {{ envs.gene_usages | r: todot="-" }}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-geneusage.R" | source_r }}
 
 ##################
 # Spectratyping  #
 ##################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-spectratyping.R" %}
+spects = {{ envs.spects | r }}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-spectratyping.R" | source_r }}
 
 ########################
 # Diversity estimation #
 ########################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-diversity.R" %}
+div_method = {{envs.divs.method | default: "gini" | r}}
+div_by = {{envs.divs.by | default: None | r}}
+div_plot_type = {{envs.divs.plot_type | default: "bar" | r}}
+div_order = {{envs.divs.order | default: [] | r}}
+div_args = {{envs.divs.args | default: {} | r: todot="-"}}
+div_test = {{envs.divs.test | default: None | r}}
+div_cases = {{envs.divs.cases | default: {} | r: todot="-"}}
+div_devpars = {{envs.divs.devpars | default: None | r}}
+div_separate_by = {{envs.divs.separate_by | default: None | r}}
+div_split_by = {{envs.divs.split_by | default: None | r}}
+div_split_order = {{envs.divs.split_order | default: None | r}}
+div_align_x = {{envs.divs.align_x | default: False | r}}
+div_align_y = {{envs.divs.align_y | default: False | r}}
+div_subset = {{envs.divs.subset | default: None | r}}
+div_log = {{envs.divs.log | default: False | r}}
+div_ncol = {{envs.divs.ncol | default: 2 | r}}
+div_ymin = {{envs.divs.ymin | default: None | r}}
+div_ymax = {{envs.divs.ymax | default: None | r}}
+
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "immunarch-patched.R" | source_r }}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-diversity.R" | source_r }}
 
 ######################
 # Clonotype tracking #
 ######################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-tracking.R" %}
+trackings = {{ envs.trackings | r }}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-tracking.R" | source_r }}
 
 ######################
 # K-mer analysis     #
 ######################
-{% include biopipen_dir + "/scripts/tcr/Immunarch-kmer.R" %}
+kmers = {{ envs.kmers | r: todot="-" }}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-kmer.R" | source_r }}
+
+######################
+# VJ junction        #
+######################
+vj_juncs <- {{envs.vj_junc | r}}
+{{ biopipen_dir | joinpaths: "scripts", "tcr", "Immunarch-vjjunc.R" | source_r }}
+
+save_report(joboutdir)

biopipen/scripts/tcr/Immunarch2VDJtools.R

@@ -3,8 +3,8 @@ library(dplyr)
 library(tidyr)
 library(stringr)
 
-immfile = {{in.immdata | quote}}
-outdir = {{out.outdir | quote}}
+immfile = {{in.immdata | r}}
+outdir = {{out.outdir | r}}
 
 immdata = readRDS(immfile)
 

biopipen/scripts/tcr/ImmunarchFilter.R

@@ -1,4 +1,4 @@
-source("{{biopipen_dir}}/utils/misc.R")
+{{ biopipen_dir | joinpaths: "utils", "misc.R" | source_r }}
 
 library(dplyr)
 library(glue)
@@ -6,15 +6,15 @@ library(tidyr)
 library(tibble)
 library(immunarch)
 
-immfile = {{in.immdata | quote}}
+immfile = {{in.immdata | r}}
 {% if in.filterfile %}
 filters = {{in.filterfile | toml_load | r}}
 {% else %}
 filters = {{envs.filters | r}}
 {% endif %}
 metacols = {{envs.metacols | r}}
-outfile = {{out.outfile | quote}}
-groupfile = {{out.groupfile | quote}}
+outfile = {{out.outfile | r}}
+groupfile = {{out.groupfile | r}}
 
 immdata0 = readRDS(immfile)
 groupname = filters$name

biopipen/scripts/tcr/ImmunarchLoading.R

@@ -1,4 +1,5 @@
-source("{{biopipen_dir}}/utils/misc.R")
+{{ biopipen_dir | joinpaths: "utils", "misc.R" | source_r }}
+{{ biopipen_dir | joinpaths: "utils", "single_cell.R" | source_r }}
 
 # Loading 10x data into immunarch
 library(immunarch)
@@ -8,12 +9,13 @@ library(tibble)
 library(glue)
 library(bracer)
 
-metafile = {{ in.metafile | quote }}
-rdsfile = {{ out.rdsfile | quote }}
-metatxt = {{ out.metatxt | quote }}
-tmpdir = {{ envs.tmpdir | quote }}
-mode = {{ envs.mode | quote }}
-metacols = {{ envs.metacols | r}}
+metafile = {{ in.metafile | r }}
+rdsfile = {{ out.rdsfile | r }}
+metatxt = {{ out.metatxt | r }}
+tmpdir = {{ envs.tmpdir | r }}
+mode = {{ envs.mode | r }}
+extracols = {{ envs.extracols | r}}
+prefix = {{ envs.prefix | r }}
 
 metadata = read.table(
     metafile,
@@ -34,6 +36,11 @@ if (!"TCRData" %in% colnames(metadata)) {
 ## Helpers
 
 get_contig_annofile = function(dir, sample, warn=TRUE) {
+    if (is.na(dir) || !is.character(dir) || nchar(dir) == 0 || dir == "NA") {
+        warning(paste0("No path found for sample: ", sample), immediate. = TRUE)
+        return (NULL)
+    }
+
     annofilepat = paste0(
         "*", "{all,filtered}", "_contig_annotations.csv*"  # .gz
     )
@@ -86,6 +93,9 @@ for (i in seq_len(nrow(metadata))) {
     sample = as.character(metadata[i, "Sample"])
 
     annofile = get_contig_annofile(metadata[i, "TCRData"], sample)
+    if (is.null(annofile)) {
+        next
+    }
     anno = read.delim2(annofile, sep=",", header=TRUE, stringsAsFactors=FALSE)
     # Add cdr1, cdr2, fwr1, fwr2, etc columns
     if (!"cdr1" %in% colnames(anno)) {
@@ -134,6 +144,7 @@ for (i in seq_len(nrow(metadata))) {
     # file.symlink(normalizePath(annofile), file.path(datadir, paste0(sample, ext)))
 }
 
+log_info("Loading TCR data ...")
 immdata = repLoad(datadir, .mode=mode)
 if (mode == "single") {
     data = immdata$data
@@ -164,27 +175,25 @@ immdata$meta = left_join(
     by = "Sample"
 )
 
+immdata$prefix = prefix
+
 saveRDS(immdata, file=rdsfile)
 
-metadf = do_call(rbind, lapply(seq_len(nrow(immdata$meta)), function(i) {
-    # Clones  Proportion   CDR3.aa                       Barcode
-    # 5      4 0.008583691 CAVRDTGNTPLVF;CASSEYSNQPQHF   GTTCGGGCACTTACGA-1;TCTCTAAGTACCAGTT-1
-    # 6      4 0.008583691 CALTQAAGNKLTF;CASRPEDLRGQPQHF GCTTGAAGTCGGCACT-1;TACTCGCTCCTAAGTG-1
-    cldata = immdata$data[[i]][, unique(c(metacols, "Barcode"))]
-    # # A tibble: 4 × 5
-    # Sample                  Patient     Timepoint Tissue
-    # <chr>                   <chr>       <chr>     <chr>
-    # 1 MC1685Pt011-Baseline-PB MC1685Pt011 Baseline  PB
-    mdata = as.list(immdata$meta[i, , drop=FALSE])
-    for (mname in names(mdata)) {
-        assign(mname, mdata[[mname]])
-    }
-
-    cldata %>%
-        separate_rows(Barcode, sep=";") %>%
-        distinct(Barcode, .keep_all = TRUE) %>%
-        mutate(Barcode = glue("{{envs.prefix}}{Barcode}")) %>%
-        column_to_rownames("Barcode")
-
-}))
-write.table(metadf, metatxt, sep="\t", quote=FALSE, row.names=TRUE, col.names=TRUE)
+log_info("Saving cell-level data ...")
+exdata <- expand_immdata(immdata, cell_id = "Barcode") %>%
+    distinct(Sample, Barcode, .keep_all = TRUE) %>%
+    mutate(Barcode = glue(paste0(prefix, "{Barcode}"))) %>%
+    select(any_of(c(
+        colnames(immdata$meta),
+        "Barcode",
+        "CDR3.aa",
+        "Clones",
+        "Proportion",
+        "V.name",
+        "D.name",
+        "J.name",
+        extracols
+    ))) %>%
+    column_to_rownames("Barcode")
+
+write.table(exdata, metatxt, sep="\t", quote=FALSE, row.names=TRUE, col.names=TRUE)

biopipen/scripts/tcr/SampleDiversity.R

@@ -1,4 +1,4 @@
-source("{{biopipen_dir}}/utils/misc.R")
+{{ biopipen_dir | joinpaths: "utils", "misc.R" | source_r }}
 library(immunarch)
 
 immdatafile = {{in.immdata | r}}

biopipen/scripts/tcr/ScRepCombiningExpression.R

@@ -0,0 +1,40 @@
+library(scRepertoire)
+library(Seurat)
+library(biopipen.utils)
+
+screpfile <- {{in.screpfile | r}}
+srtobjfile <- {{in.srtobj | r}}
+outfile <- {{out.outfile | r}}
+cloneCall <- {{envs.cloneCall | r}}
+chain <- {{envs.chain | r}}
+group_by <- {{envs.group_by | default: envs["group-by"] | default: None | r}}
+proportion <- {{envs.proportion | r}}
+filterNA <- {{envs.filterNA | r}}
+cloneSize <- {{envs.cloneSize | r}}
+addLabel <- {{envs.addLabel | r}}
+
+log <- get_logger()
+
+log$info("Loading scRepertoire object ...")
+screp <- read_obj(screpfile)
+
+log$info("Loading Seurat object ...")
+srtobj <- read_obj(srtobjfile)
+
+log$info("Combining expression data ...")
+
+obj <- combineExpression(
+    input.data = screp,
+    sc.data = srtobj,
+    cloneCall = cloneCall,
+    chain = chain,
+    group.by = group_by,
+    proportion = proportion,
+    filterNA = filterNA,
+    cloneSize = unlist(cloneSize),
+    addLabel = addLabel
+)
+obj$VDJ_Presence <- !is.na(obj$CTaa)
+
+log$info("Saving combined object ...")
+save_obj(obj, outfile)

biopipen/scripts/tcr/ScRepLoading.R

@@ -0,0 +1,166 @@
+library(rlang)
+library(bracer)
+library(scRepertoire)
+library(biopipen.utils)
+
+metafile <- {{in.metafile | r}}
+outfile <- {{out.outfile | r}}
+combineTCR_args <- {{envs.combineTCR | r}}
+combineBCR_args <- {{envs.combineBCR | r}}
+type <- {{envs.type | r}}
+exclude <- {{envs.exclude | r}}
+format <- {{envs.format | r}}
+tmpdir <- {{envs.tmpdir | r}}
+
+type = toupper(type)
+if (length(exclude) == 1) {
+    exclude <- strsplit(exclude, ",")[[1]]
+}
+
+log <- get_logger()
+
+log$info("Loading metadata ...")
+metadata <- read.table(metafile, header = TRUE, sep = "\t", row.names = NULL, check.names = FALSE)
+if (type == "AUTO") {
+    if ("TCRData" %in% colnames(metadata) && "BCRData" %in% colnames(metadata)) {
+        log$warn("Both TCRData and BCRData columns found in metadata. Defaulting to TCR.")
+        type <- "TCR"
+    } else if ("TCRData" %in% colnames(metadata)) {
+        type <- "TCR"
+    } else if ("BCRData" %in% colnames(metadata)) {
+        type <- "BCR"
+    } else {
+        stop("Error: Neither TCRData nor BCRData column found in metadata.")
+    }
+}
+
+data_column <- ifelse(type == "TCR", "TCRData", "BCRData")
+combine_fn <- ifelse(type == "TCR", combineTCR, combineBCR)
+combine_args <- if (type == "TCR") { combineTCR_args } else { combineBCR_args }
+
+stopifnot("Error: Column `Sample` is not found in metafile." = "Sample" %in% colnames(metadata))
+if (!data_column %in% colnames(metadata)) {
+    stop(paste0("Error: Column `", data_column, "` is not found in metafile."))
+}
+rownames(metadata) <- metadata$Sample
+
+.gunzip <- function(input, output) {
+    # Open connections
+    con_in <- gzfile(input, "rt")   # "rt" = read text mode
+    con_out <- file(output, "wt")  # "wt" = write text mode
+
+    # Read line by line and write
+    while(length(line <- readLines(con_in, n = 10, warn = FALSE)) > 0) {
+        writeLines(line, con_out)
+    }
+
+    # Close connections
+    close(con_in)
+    close(con_out)
+}
+
+get_file_name <- function(fmt) {
+    if (is.null(fmt)) { return("filtered_contig_annotations.csv") }
+    fmt <- tolower(fmt)
+    if (fmt == "10x") { return("filtered_contig_annotations.csv") }
+    if (fmt == "airr") { return("airr_rearrangement.tsv") }
+    if (fmt == "bd") { return("Contigs_AIRR.tsv") }
+    if (fmt == "dandelion") { return("all_contig_dandelion.tsv") }
+    if (fmt == "immcantation") { return("data.tsv") }
+    if (fmt == "json") { return("contigs.json") }
+    if (fmt == "parsebio") { return("barcode_report.tsv") }
+    if (fmt == "mixcr") { return("clones.tsv") }
+    if (fmt == "omniscope") { return("contigs.csv") }
+    if (fmt == "trust4") { return("barcode_report.tsv") }
+    if (fmt == "wat3r") { return("barcode_results.csv") }
+
+    stop("Unsupported format: ", fmt)
+}
+
+get_format <- function(filename) {
+    if (identical(filename, "filtered_contig_annotations.csv")) { return("10X") }
+    if (identical(filename, "airr_rearrangement.tsv")) { return("AIRR") }
+    if (identical(filename, "Contigs_AIRR.tsv")) { return("BD") }
+    if (identical(filename, "all_contig_dandelion.tsv")) { return("Dandelion") }
+    if (identical(filename, "data.tsv")) { return("Immcantation") }
+    if (endsWith(filename, ".json")) { return("JSON") }
+    if (identical(filename, "barcode_report.tsv")) { return("ParseBio") }
+    if (identical(filename, "clones.tsv")) { return("MiXCR") }
+    if (identical(filename, "contigs.csv")) { return("Omniscope") }
+    # if (identical(filename, "barcode_report.tsv")) { return("TRUST4") }
+    if (identical(filename, "barcode_results.csv")) { return("WAT3R") }
+
+    return("10X")
+}
+
+# helper function
+get_contig_dir <- function(input, sample, fmt) {
+    if (is.na(input) || !is.character(input) || nchar(input) == 0 || input == "NA") {
+        warning(paste0("No path found for sample: ", sample), immediate. = TRUE)
+        return(list(NULL, fmt))
+    }
+    if (!file.exists(input)) {
+        stop(paste0("Input path does not exist for sample: ", sample, ": ", input))
+    }
+    if (dir.exists(input)) {
+        return(list(input, fmt))
+    }
+    # file
+    filedir <- file.path(tmpdir, slugify(sample))
+    dir.create(filedir, recursive = TRUE, showWarnings = FALSE)
+
+    # if it is gzipped
+    if (grepl("\\.gz$", input)) {
+        flatfile <- file.path(filedir, sub("\\.gz$", "", basename(input)))
+        .gunzip(input, flatfile)
+        input <- flatfile
+    }
+
+    fmt <- fmt %||% get_format(basename(input))
+    filename <- get_file_name(fmt)
+    file.symlink(input, file.path(filedir, filename))
+
+    return(list(filedir, fmt))
+}
+
+load_contig <- function(input, sample, fmt) {
+    log$info("- Sample: {sample}")
+    dirfmt <- get_contig_dir(input, sample, fmt)
+    dir <- dirfmt[[1]]
+    fmt <- dirfmt[[2]]
+    if (is.null(dir)) { return(NULL) }
+    x <- loadContigs(dir, format = fmt %||% "10X")
+    x <- x[[1]]
+    x$sample <- NULL
+    if (identical(fmt %||% "10X", "10X") && colnames(x)[1] == "X") {
+        x$X <- NULL
+    }
+
+    x
+}
+
+
+log$info("Reading {type} data ...")
+contig_list <- lapply(seq_len(nrow(metadata)), function(i) {
+    sample <- as.character(metadata$Sample[i])
+    path <- metadata[[data_column]][i]
+    load_contig(path, sample, fmt = format)
+})
+names(contig_list) <- as.character(metadata$Sample)
+contig_list <- contig_list[!sapply(contig_list, is.null)]
+
+log$info("Combining {type} data and adding meta data ...")
+if (isTRUE(combine_args$samples)) {
+    combine_args$samples <- names(contig_list)
+}
+combine_args$input.data <- contig_list
+screp_data <- do_call(combine_fn, combine_args)
+for (col in colnames(metadata)) {
+    if (col %in% exclude) { next }
+    screp_data <- addVariable(screp_data, col, metadata[names(screp_data), col])
+}
+
+rm(contig_list, combine_args)
+
+log$info("Saving {type} data ...")
+save_obj(screp_data, outfile)