biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/scripts/tcr/TESSA.R

@@ -1,23 +1,28 @@
-source("{{biopipen_dir}}/utils/misc.R")
-
 library(glue)
 library(dplyr)
 library(tidyr)
-library(immunarch)
+library(tibble)
 library(Seurat)
-library(ggplot2)
-library(ggprism)
+library(biopipen.utils)
 
-immfile <- {{in.immdata | r}}
-exprfile <- {{in.srtobj | r}}
+screpdata <- {{in.screpdata | r}}
 outfile <- {{out.outfile | r}}
+joboutdir <- {{job.outdir | r}}
 python <- {{envs.python | r}}
 within_sample <- {{envs.within_sample | r}}
 assay <- {{envs.assay | r}}
 predefined_b <- {{envs.predefined_b | r}}
 max_iter <- {{envs.max_iter | int}}
 save_tessa <- {{envs.save_tessa | r}}
-tessa_srcdir <- "{{biopipen_dir}}/scripts/tcr/TESSA_source"
+
+log <- get_logger()
+reporter <- get_reporter()
+
+# In case this script is running in the cloud and <biopipen_dir> can not be found in there
+# In stead, we use the python command, which is associated with the cloud environment,
+# to get the biopipen directory
+biopipen_dir <- get_biopipen_dir(python)
+tessa_srcdir <- file.path(biopipen_dir, "scripts", "tcr", "TESSA_source")
 
 outdir <- dirname(outfile)
 result_dir <- file.path(outdir, "result")
@@ -27,98 +32,51 @@ if (!dir.exists(tessa_dir)) dir.create(tessa_dir)
 
 ### Start preparing input files for TESSA
 # Prepare input files
-print("Preparing TCR input file ...")
-immdata <- readRDS(immfile)
-
-has_VJ <- "V.name" %in% colnames(immdata$data[[1]]) && "J.name" %in% colnames(immdata$data[[1]])
-# Merge all samples
-tcrdata <- do_call(rbind, lapply(seq_len(nrow(immdata$meta)), function(i) {
-    # Clones  Proportion   CDR3.aa                       Barcode
-    # 5      4 0.008583691 CAVRDTGNTPLVF;CASSEYSNQPQHF   GTTCGGGCACTTACGA-1;TCTCTAAGTACCAGTT-1
-    # 6      4 0.008583691 CALTQAAGNKLTF;CASRPEDLRGQPQHF GCTTGAAGTCGGCACT-1;TACTCGCTCCTAAGTG-1
-    if (has_VJ) {
-        cldata = immdata$data[[i]][, c("Barcode", "CDR3.aa", "V.name", "J.name")]
-    } else {
-        cldata = immdata$data[[i]][, c("Barcode", "CDR3.aa")]
-    }
-    # # A tibble: 4 × 5
-    # Sample                  Patient     Timepoint Tissue
-    # <chr>                   <chr>       <chr>     <chr>
-    # 1 MC1685Pt011-Baseline-PB MC1685Pt011 Baseline  PB
-    mdata = as.list(immdata$meta[i, , drop=FALSE])
-    for (mname in names(mdata)) {
-        assign(mname, mdata[[mname]])
-    }
-
-    cldata %>%
-        separate_rows(Barcode, sep=";") %>%
-        # Just in case there are duplicated barcodes
-        distinct(Barcode, .keep_all = TRUE) %>%
-        mutate(Barcode = glue("{{envs.prefix}}{Barcode}"), sample = Sample)
-}))
-if (has_VJ) {
-    tcrdata <- tcrdata %>% dplyr::mutate(
-        v_gene = sub("-\\d+$", "", V.name),
-        j_gene = sub("-\\d+$", "", J.name)
-    ) %>% dplyr::select(
-        contig_id = Barcode,
-        cdr3 = CDR3.aa,
-        v_gene,
-        j_gene,
-        sample
-    )
-} else {
-    tcrdata <- tcrdata %>% dplyr::select(
-        contig_id = Barcode,
-        cdr3 = CDR3.aa,
-        sample
-    )
-}
-
-
-print("Preparing expression input file ...")
-is_seurat <- endsWith(tolower(exprfile), ".rds")
-is_gz <- endsWith(tolower(exprfile), ".gz")
-
-if (is_seurat) {
-    sobj <- readRDS(exprfile)
-    expr <- GetAssayData(sobj, slot = "data", assay = assay)
-} else if (is_gz) {
-    expr <- read.table(gzfile(exprfile), sep="\t", header=TRUE, row.names=1)
-} else {
-    expr <- read.table(exprfile, sep="\t", header=TRUE, row.names=1)
-}
-
+log$info("Reading input file ...")
+sobj <- read_obj(screpdata)
+
+log$info("Preparing TCR input file ...")
+# If immfile endswith .rds, then it is an immunarch object
+tcrdata <- sobj@meta.data %>%
+    rownames_to_column("contig_id") %>%
+    select(contig_id, CTaa, CTgene, sample = Sample) %>%
+    filter(!is.na(CTaa) & !is.na(CTgene)) %>%
+    separate(CTaa, into = c(NA, "cdr3"), sep = "_", remove = TRUE) %>%
+    filter(!is.na(cdr3) & cdr3 != "NA" & cdr3 != "nan") %>%
+    separate(CTgene, into = c(NA, "vjgene"), sep = "_", remove = TRUE) %>%
+    separate(vjgene, into = c("v_gene", NA, "j_gene", NA), sep = "\\.", remove = TRUE) %>%
+    mutate(v_gene = sub("-\\d+$", "", v_gene), j_gene = sub("-\\d+$", "", j_gene))
+
+log$info("Preparing expression input file ...")
+expr <- GetAssayData(sobj, layer = "data")
 cell_ids <- intersect(tcrdata$contig_id, colnames(expr))
 # Warning about unused cells
-unused_tcr_cells <- setdiff(tcrdata$contig_id, cell_ids)
 unused_expr_cells <- setdiff(colnames(expr), cell_ids)
-if (length(unused_tcr_cells) > 0) {
-    warning(glue("{length(unused_tcr_cells)}/{nrow(tcrdata)} TCR cells are not used."), immediate. = TRUE)
-}
 if (length(unused_expr_cells) > 0) {
-    warning(glue("{length(unused_expr_cells)}/{ncol(expr)} expression cells are not used."), immediate. = TRUE)
+    log$warn(glue("{length(unused_expr_cells)}/{ncol(expr)} cells without TCR data are not used."))
 }
 if (length(cell_ids) == 0) {
-    stop("No common cells between TCR and expression data. Are you using the correct prefix?")
+    stop(
+        "No TCR data found in the Seurat object. ",
+        "Please use scRepertiore::combineExpression() to generate the Seurat object with TCR data."
+    )
 }
-tcrdata <- tcrdata[tcrdata$contig_id %in% cell_ids, , drop=FALSE]
 expr <- as.matrix(expr)[, tcrdata$contig_id, drop=FALSE]
 
 # Write input files
-print("Writing input files ...")
+log$info("Writing input files ...")
 write.table(tcrdata, file.path(tessa_dir, "tcrdata.txt"), sep=",", quote=FALSE, row.names=FALSE)
 write.table(expr, file.path(tessa_dir, "exprdata.txt"), sep=",", quote=FALSE, row.names=TRUE, col.names=TRUE)
 
 ### End preparing input files for TESSA
 
 ### Start running TESSA
-print("Running TESSA ...")
+log$info("Running TESSA ...")
 
 # The original TESSA uses a python wrapper to run the encoder and tessa model
 # here we run those two steps directly here
 
-print("- Running encoder ...")
+log$info("- Running encoder ...")
 cmd_encoder <- paste(
     python,
     file.path(tessa_srcdir, "BriseisEncoder.py"),
@@ -133,21 +91,22 @@ cmd_encoder <- paste(
     "-output_log",
     file.path(tessa_dir, "tcr_encoder.log")
 )
-if (has_VJ) {
-    cmd_encoder <- paste(
-        cmd_encoder,
-        "-output_VJ",
-        file.path(tessa_dir, "tcr_vj.txt")
-    )
-}
-print(paste("- ", cmd_encoder))
+cmd_encoder <- paste(
+    cmd_encoder,
+    "-output_VJ",
+    file.path(tessa_dir, "tcr_vj.txt")
+)
+
+print("Running:")
+print(cmd_encoder)
+log$debug(paste("- ", cmd_encoder))
 
 rc <- system(cmd_encoder)
 if (rc != 0) {
     stop("Error: Failed to run encoder.")
 }
 
-print("- Running TESSA model ...")
+log$info("- Running TESSA model ...")
 source(file.path(tessa_srcdir, "real_data.R"))
 
 tessa <- run_tessa(
@@ -162,42 +121,67 @@ tessa <- run_tessa(
 )
 
 # Save TESSA results
-print("Saving TESSA results ...")
-if (is_seurat) {
-    cells <- rownames(sobj@meta.data)
-    sobj@meta.data <- sobj@meta.data %>%
-        mutate(
-            TESSA_Cluster = tessa$meta[
-                match(cells, tessa$meta$barcode),
-                "cluster_number"
-            ]
-        ) %>%
-        add_count(TESSA_Cluster, name = "TESSA_Cluster_Size")
-    rownames(sobj@meta.data) <- cells
-
-    if (save_tessa) {
-        sobj@misc$tessa <- tessa
-    }
-    saveRDS(sobj, outfile)
-} else {
-    out <- tessa$meta %>%
-        dplyr::select(barcode, TESSA_Cluster = cluster_number) %>%
-        add_count(TESSA_Cluster, name = "TESSA_Cluster_Size")
-    write.table(out, outfile, sep="\t", quote=FALSE, row.names=FALSE, col.names=TRUE)
+log$info("Saving TESSA results ...")
+cells <- rownames(sobj@meta.data)
+sobj@meta.data <- sobj@meta.data %>%
+    mutate(
+        TESSA_Cluster = tessa$meta[
+            match(cells, tessa$meta$barcode),
+            "cluster_number"
+        ]
+    ) %>%
+    add_count(TESSA_Cluster, name = "TESSA_Cluster_Size")
+rownames(sobj@meta.data) <- cells
+
+if (save_tessa) {
+    sobj@misc$tessa <- tessa
 }
+save_obj(sobj, outfile)
 
 # Post analysis
-print("Post analysis ...")
+log$info("Post analysis ...")
 plot_tessa(tessa, result_dir)
 plot_Tessa_clusters(tessa, result_dir)
 
 p <- tessa$meta %>%
     dplyr::select(barcode, TESSA_Cluster = cluster_number) %>%
     add_count(TESSA_Cluster, name = "TESSA_Cluster_Size") %>%
-    ggplot(aes(x = TESSA_Cluster_Size)) +
-    geom_histogram(binwidth = 1) +
-    theme_prism()
+    plotthis::Histogram(x = "TESSA_Cluster_Size")
+
+res <- 100
+height <- attr(p, "height") * res
+width <- attr(p, "width") * res
+prefix <- file.path(result_dir, "Cluster_size_dist")
+save_plot(p, prefix, devpars = list(width = width, height = height, res = res))
+
+reporter$add(
+    list(
+        src = file.path(result_dir, "Cluster_size_dist.png"),
+        descr = "Histogram of cluster size distribution",
+        download = file.path(result_dir, "Cluster_size_dist.pdf")
+    ),
+    list(
+        src = file.path(result_dir, "clone_size.png"),
+        descr = "Center cluster size vs. non-center cluster size"
+    ),
+    list(
+        src = file.path(result_dir, "exp_TCR_pair_plot.png"),
+        descr = "Expression-TCR distance plot"
+    ),
+    list(
+        src = file.path(result_dir, "TCR_dist_density.png"),
+        descr = "TCR distance density plot"
+    ),
+    list(
+        src = file.path(result_dir, "TCR_explore.png"),
+        descr = "Exploratory plot at the TCR level"
+    ),
+    list(
+        src = file.path(result_dir, "TCR_explore_clusters.png"),
+        descr = "TESSA clusters"
+    ),
+    h1 = "TESSA Results",
+    ui = "table_of_images"
+)
 
-png(file.path(result_dir, "Cluster_size_dist.png"), width=8, height=8, units="in", res=100)
-print(p)
-dev.off()
+reporter$save(joboutdir)

biopipen/scripts/tcr/VJUsage.R

@@ -1,9 +1,9 @@
 
-infile = {{in.infile | quote}}
-outprefix = {{out.outfile | prefix | replace: ".fancyvj.wt", "" | quote}}
-vdjtools = {{ envs.vdjtools | quote }}
-vdjtools_patch = {{ envs.vdjtools_patch | quote }}
-joboutdir = {{job.outdir | quote}}
+infile = {{in.infile | r}}
+outprefix = {{out.outfile | prefix | replace: ".fancyvj.wt", "" | r}}
+vdjtools = {{ envs.vdjtools | r }}
+vdjtools_patch = {{ envs.vdjtools_patch | r }}
+joboutdir = {{job.outdir | r}}
 
 command = sprintf(
     "cd %s && bash %s %s PlotFancyVJUsage --plot-type png %s %s",

biopipen/scripts/tcr/immunarch-patched.R

@@ -0,0 +1,142 @@
+library(immunarch)
+
+vis.immunr_gini <- function(.data, .by = NA, .meta = NA,
+                            .errorbars = c(0.025, 0.975), .errorbars.off = FALSE,
+                            .points = TRUE, .test = TRUE, .signif.label.size = 3.5,
+                            .legend = NA, .plot.type = "bar", ...) {
+  # repDiversity(..., .method = "gini") generates a matrix
+  .data = data.frame(Sample = rownames(.data), Value = .data[, 1])
+  if (.plot.type == "bar") {
+    vis_bar(
+        .data = .data, .by = .by, .meta = .meta,
+        .errorbars = .errorbars, .errorbars.off = .errorbars.off, .stack = FALSE,
+        .points = .points, .test = .test, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, "Gini coefficient"),
+        .title = "Gini coefficient", .subtitle = "Sample diversity estimation using the Gini coefficient",
+        .legend = .legend, .leg.title = NA
+    )
+  } else {
+    vis_box(
+        .data = .data, .by = .by, .meta = .meta, .test = .test,
+        .points = .points, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, "Gini coefficient"),
+        .title = "Gini coefficient", .subtitle = "Sample diversity estimation using the Gini coefficient",
+        .legend = .legend, .leg.title = NA, .melt = FALSE
+    )
+  }
+}
+
+vis.immunr_div <- function(.data, .by = NA, .meta = NA,
+                            .errorbars = c(0.025, 0.975), .errorbars.off = FALSE,
+                            .points = TRUE, .test = TRUE, .signif.label.size = 3.5,
+                            .legend = NA, .plot.type = "bar", ...) {
+  # repDiversity(..., .method = "gini") generates a matrix
+  if (.plot.type == "bar") {
+    immunarch:::vis.immunr_div(.data = .data,.by = .by, .meta = .meta,
+        .errorbars = .errorbars, .errorbars.off = .errorbars.off, .stack = FALSE,
+        .points = .points, .test = .test, .signif.label.size = .signif.label.size,
+        .legend = .legend)
+  } else {
+    vis_box(
+        .data = .data, .by = .by, .meta = .meta, .test = .test,
+        .points = .points, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, "Effective number of clonoypes"),
+        .title = "True diversity", .subtitle = "Sample diversity estimation using the true diversity index",
+        .legend = NA, .leg.title = NA, .melt = FALSE
+    )
+  }
+}
+
+vis.immunr_chao1 <- function(.data, .by = NA, .meta = NA,
+                            .errorbars = c(0.025, 0.975), .errorbars.off = FALSE,
+                            .points = TRUE, .test = TRUE, .signif.label.size = 3.5,
+                            .legend = NA, .plot.type = "bar", ...) {
+  # repDiversity(..., .method = "gini") generates a matrix
+  if (.plot.type == "bar") {
+    immunarch:::vis.immunr_chao1(.data = .data,.by = .by, .meta = .meta,
+        .errorbars = .errorbars, .errorbars.off = .errorbars.off, .stack = FALSE,
+        .points = .points, .test = .test, .signif.label.size = .signif.label.size,
+        .legend = .legend)
+  } else {
+    .data <- data.frame(Sample = row.names(.data), Value = .data[, 1])
+    vis_box(
+        .data = .data, .by = .by, .meta = .meta, .test = .test,
+        .points = .points, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, "Chao1"),
+        .title = "Chao1", .subtitle = "Sample diversity estimation using Chao1",
+        .legend = NA, .leg.title = NA, .melt = FALSE
+    )
+  }
+}
+
+vis.immunr_ginisimp <- function(.data, .by = NA, .meta = NA,
+                            .errorbars = c(0.025, 0.975), .errorbars.off = FALSE,
+                            .points = TRUE, .test = TRUE, .signif.label.size = 3.5,
+                            .legend = NA, .plot.type = "bar", ...) {
+  # repDiversity(..., .method = "gini") generates a matrix
+  if (.plot.type == "bar") {
+    immunarch:::vis.immunr_ginisimp(.data = .data,.by = .by, .meta = .meta,
+        .errorbars = .errorbars, .errorbars.off = .errorbars.off, .stack = FALSE,
+        .points = .points, .test = .test, .signif.label.size = .signif.label.size,
+        .legend = .legend)
+  } else {
+    vis_box(
+        .data = .data, .by = .by, .meta = .meta, .test = .test,
+        .points = .points, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, "Gini-Simpson index"),
+        .title = "Gini-Simpson index", .subtitle = "Sample diversity estimation using the Gini-Simpson index",
+        .legend = .legend, .leg.title = NA, .melt = FALSE
+    )
+  }
+}
+
+vis.immunr_invsimp <- function(.data, .by = NA, .meta = NA,
+                            .errorbars = c(0.025, 0.975), .errorbars.off = FALSE,
+                            .points = TRUE, .test = TRUE, .signif.label.size = 3.5,
+                            .legend = NA, .plot.type = "bar", ...) {
+  # repDiversity(..., .method = "gini") generates a matrix
+  if (.plot.type == "bar") {
+    immunarch:::vis.immunr_invsimp(.data = .data,.by = .by, .meta = .meta,
+        .errorbars = .errorbars, .errorbars.off = .errorbars.off, .stack = FALSE,
+        .points = .points, .test = .test, .signif.label.size = .signif.label.size,
+        .legend = .legend)
+  } else {
+    vis_box(
+        .data = .data, .by = .by, .meta = .meta, .test = .test,
+        .points = .points, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, "Inverse Simpson index"),
+        .title = "Inverse Simpson index", .subtitle = "Sample diversity estimation using the inverse Simpson index",
+        .legend = .legend, .leg.title = NA, .melt = FALSE
+    )
+  }
+}
+
+vis.immunr_dxx <- function(.data, .by = NA, .meta = NA,
+                            .errorbars = c(0.025, 0.975), .errorbars.off = FALSE,
+                            .points = TRUE, .test = TRUE, .signif.label.size = 3.5,
+                            .legend = NA, .plot.type = "bar", ...) {
+  # repDiversity(..., .method = "gini") generates a matrix
+  if (.plot.type == "bar") {
+    immunarch:::vis.immunr_dxx(.data = .data,.by = .by, .meta = .meta,
+        .errorbars = .errorbars, .errorbars.off = .errorbars.off, .stack = FALSE,
+        .points = .points, .test = .test, .signif.label.size = .signif.label.size,
+        .legend = .legend)
+  } else {
+    perc_value <- round(.data[1, 2][1])
+    .data <- data.frame(Sample = row.names(.data), Value = .data[, 1])
+    vis_box(
+        .data = .data, .by = .by, .meta = .meta, .test = .test,
+        .points = .points, .signif.label.size = .signif.label.size,
+        .defgroupby = "Sample", .grouping.var = "Group",
+        .labs = c(NA, paste0("D", perc_value)),
+        .title = paste0("D", perc_value, " diversity index"), .subtitle = paste0("Number of clonotypes occupying the ", perc_value, "% of repertoires"),
+        .legend = .legend, .leg.title = NA, .melt = FALSE
+    )
+  }
+}

biopipen/scripts/tcr/vdjtools-patch.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # run the command and capture the stdout
-out=$(command $@)
+out=$(command "$@")
 
 echo "$out"
 

biopipen/scripts/vcf/BcftoolsAnnotate.py

@@ -0,0 +1,91 @@
+from os import path
+from contextlib import suppress
+from pathlib import PosixPath  # noqa: F401
+
+from biopipen.utils.reference import tabix_index
+from biopipen.utils.misc import logger
+from biopipen.scripts.vcf.bcftools_utils import run_bcftools
+
+infile: str = {{in.infile | quote}}  # pyright: ignore # noqa: E999
+annfile: str = {{in.annfile | quote}}  # pyright: ignore
+outfile: str = {{out.outfile | quote}}  # pyright: ignore
+joboutdir: str = {{job.outdir | quote}}  # pyright: ignore
+envs: dict = {{envs | dict | repr}}  # pyright: ignore
+
+bcftools = envs.pop("bcftools")
+tabix = envs.pop("tabix")
+ncores = envs.pop("ncores")
+columns = envs.pop("columns")
+remove = envs.pop("remove")
+header = envs.pop("header")
+gz = envs.pop("gz")
+index = envs.pop("index")
+
+if isinstance(columns, list):
+    columns = ",".join(columns)
+
+if "c" in envs:
+    logger.warning(r"Ignoring envs\[c], use envs\[columns] instead.")
+    del envs["c"]
+
+if isinstance(remove, list):
+    remove = ",".join(remove)
+
+if "x" in envs:
+    logger.warning(r"Ignoring envs\[x], use envs\[remove] instead.")
+    del envs["x"]
+
+envs_has_annfile = "a" in envs or "annotations" in envs
+headerfile = path.join(joboutdir, "header.txt")
+if header:
+    with open(headerfile, "w") as fh:
+        fh.writelines(header)
+
+if annfile and envs_has_annfile:
+    logger.warning(
+        r"Ignoring envs\[a/annotations] because in.annfile is provided."
+    )
+    with suppress(KeyError):
+        del envs["a"]
+    with suppress(KeyError):
+        del envs["annotations"]
+elif not annfile and envs_has_annfile:
+    annfile = envs.pop("annotations", None) or envs.pop("a", None)
+
+
+if index and not gz:
+    logger.warning("Forcing envs.gz to True because envs.index is True.")
+    gz = True
+
+envs[""] = [bcftools, "annotate"]
+envs["o"] = outfile
+envs["threads"] = ncores
+
+if "O" not in envs and "output-type" not in envs and "output_type" not in envs:
+    envs["O"] = "z" if gz else "v"
+
+if columns:
+    envs["columns"] = columns
+    if not annfile:
+        raise ValueError(
+            "envs.columns specified but no in.annfile/envs.annfile provided."
+        )
+    envs["_"] = tabix_index(infile, "vcf", tabix=tabix)
+
+if remove:
+    envs["remove"] = remove
+    # no need to index it
+    envs["_"] = infile
+
+if "columns" not in envs and "remove" not in envs:
+    logger.warning(
+        "No columns/remove specified, no columns will be carried over or removed."
+    )
+
+if annfile:
+    envs["annotations"] = tabix_index(annfile, "vcf", tabix=tabix)
+
+if header:
+    envs["header_lines"] = headerfile
+
+run_bcftools(envs, bcftools=bcftools, index=index, tabix=tabix)

biopipen/scripts/vcf/BcftoolsFilter.py

@@ -0,0 +1,90 @@
+from pathlib import Path, PosixPath  # noqa: F401
+
+from biopipen.utils.misc import logger
+from biopipen.scripts.vcf.bcftools_utils import run_bcftools
+
+infile: str | Path = {{in.infile | quote}}  # pyright: ignore # noqa: #999
+outfile: str = {{out.outfile | quote}}  # pyright: ignore
+outdir = Path(outfile).parent
+
+envs: dict = {{envs | dict | repr}}  # pyright: ignore
+bcftools = envs.pop("bcftools")
+tabix = envs.pop("tabix")
+keep = envs.pop("keep")
+ncores = envs.pop("ncores")
+includes = envs.pop("includes")
+excludes = envs.pop("excludes")
+gz = envs.pop("gz")
+index = envs.pop("index")
+
+# a.vcf.gz -> a
+# a.vcf -> a
+stem = Path(infile).stem
+if stem.endswith(".vcf"):
+    stem = stem[:-4]
+# .vcf.gz
+# .gz
+ext = ".vcf.gz" if index or gz else '.vcf'
+
+
+def normalize_expr(expr, flag, prev_n_filters=0):
+    out = {}
+    if not expr:
+        return out
+    if isinstance(expr, list):
+        for ex in expr:
+            out[f"FILTER_{flag.upper()}_{len(out) + 1 + prev_n_filters}"] = (ex, flag)
+    elif isinstance(expr, dict):
+        for name, ex in expr.items():
+            out[name] = (ex, flag)
+    else: # str
+        out[f"FILTER_{flag.upper()}_{len(out) + 1 + prev_n_filters}"] = (expr, flag)
+    return out
+
+
+def handle_filter(vcf, fname, filt, flag, final):
+    logger.info("- Handling filter %s: %s ...", fname, filt)
+
+    arguments = envs.copy()
+    arguments[flag] = filt
+    arguments["_"] = vcf
+    arguments["o"] = outfile if final else outdir / f"{stem}.{fname}{ext}"
+    if keep:
+        arguments["s"] = fname
+
+    run_bcftools(arguments, bcftools=bcftools, index=index and final, tabix=tabix)
+
+    if final:
+        flagfile = outdir.joinpath(f"{stem}.{fname}{ext}")
+        if flagfile.is_symlink():
+            flagfile.unlink()
+        outdir.joinpath(f"{stem}.{fname}{ext}").symlink_to(outfile)
+
+    return arguments["o"]
+
+
+includes = normalize_expr(includes, "include")
+excludes = normalize_expr(excludes, "exclude", len(includes))
+includes.update(excludes)
+
+if index and not gz:
+    logger.warning("Forcing envs.gz to True because envs.index is True.")
+    gz = True
+
+envs[""] = [bcftools, "filter"]
+envs["_"] = infile
+envs["o"] = outfile
+envs["threads"] = ncores
+
+if "O" not in envs and "output-type" not in envs and "output_type" not in envs:
+    envs["O"] = "z" if gz else "v"
+
+if keep:
+    envs["soft_filter"] = "+"
+
+if "m" not in envs and "mode" not in envs:
+    envs["m"] = "+"
+
+# bcftools can be only done once at one filter
+for i, (fname, (filt, flag)) in enumerate(includes.items()):
+    infile = handle_filter(infile, fname, filt, flag, i == len(includes) - 1)

biopipen/scripts/vcf/BcftoolsMerge.py

@@ -0,0 +1,31 @@
+from biopipen.utils.reference import tabix_index
+from biopipen.utils.misc import logger
+from biopipen.scripts.vcf.bcftools_utils import run_bcftools
+
+infiles: list = {{in.infiles | each: as_path}}  # pyright: ignore # noqa: E999
+outfile = {{out.outfile | repr}}  # pyright: ignore
+joboutdir = {{job.outdir | repr}}  # pyright: ignore
+envs: dict = {{envs | dict | repr}}  # pyright: ignore
+
+bcftools = envs.pop("bcftools")
+tabix = envs.pop("tabix")
+ncores = envs.pop("ncores")
+gz = envs.pop("gz")
+index = envs.pop("index")
+
+envs.setdefault("force-single", True)
+envs.setdefault("missing-to-ref", True)
+
+if index and not gz:
+    logger.warning("Forcing envs.gz to True because envs.index is True.")
+    gz = True
+
+if "O" not in envs and "output-type" not in envs and "output_type" not in envs:
+    envs["O"] = "z" if gz else "v"
+
+envs[""] = [bcftools, "merge"]
+envs["o"] = outfile
+envs["threads"] = ncores
+envs["_"] = infiles
+
+run_bcftools(envs, bcftools=bcftools, index=index, tabix=tabix)