biopipen 0.31.5__tar.gz → 0.31.6__tar.gz

{biopipen-0.31.5 → biopipen-0.31.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: biopipen
-Version: 0.31.5
+Version: 0.31.6
 Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
 License: MIT
 Author: pwwang

biopipen-0.31.6/biopipen/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.31.6"

{biopipen-0.31.5 → biopipen-0.31.6}/biopipen/ns/regulatory.py

@@ -212,3 +212,75 @@ class MotifAffinityTest(Proc):
         "atsnp_args": {"padj_cutoff": True, "padj": "BH", "p": "pval_diff"},
     }
     script = "file://../scripts/regulatory/MotifAffinityTest.R"
+
+
+class VariantMotifPlot(Proc):
+    """A plot with a genomic region surrounding a genomic variant, and
+    potentially disrupted motifs.
+
+    Currently only SNVs are supported.
+
+    Input:
+        infile: File containing the variants and motifs.
+            It is a TAB-delimited file with the following columns:
+            - chrom: The chromosome of the SNV. Alias: chr, seqnames.
+            - start: The start position of the SNV, no matter 0- or 1-based.
+            - end: The end position of the SNV, which will be used as the position of the SNV.
+            - strand: Indicating the direction of the surrounding sequence matching the motif.
+            - SNP_id: The name of the SNV.
+            - REF: The reference allele of the SNV.
+            - ALT: The alternative allele of the SNV.
+            - providerId: The motif id. It can be specified by `envs.motif_col`.
+            - providerName: The name of the motif provider. Optional.
+            - Regulator: The regulator name. Optional, can be specified by `envs.regulator_col`.
+            - motifPos: The position of the motif, relative to the position of the SNV.
+                For example, '-8, 4' means the motif is 8 bp upstream and 4 bp downstream of the SNV.
+
+    Envs:
+        genome: The genome assembly.
+            Used to fetch the sequences around the variants by package, for example, `BSgenome.Hsapiens.UCSC.hg19` is required if
+            `hg19`. If it is an organism other than human, please specify the full name of the package, for example, `BSgenome.Mmusculus.UCSC.mm10`.
+        motifdb: The path to the motif database. This is required.
+            It should be in the format of MEME motif database.
+            Databases can be downloaded here: <https://meme-suite.org/meme/doc/download.html>.
+            See also introduction to the databases: <https://meme-suite.org/meme/db/motifs>.
+            [universalmotif](https://github.com/bjmt/universalmotif) is required to read the motif database.
+        motif_col: The column name in the motif file containing the motif names.
+            If this is not provided, `envs.regulator_col` and `envs.regmotifs` are required,
+            which are used to infer the motif names from the regulator names.
+        regulator_col: The column name in the motif file containing the regulator names.
+            Both `motif_col` and `regulator_col` should be the direct column names or
+            the index (1-based) of the columns.
+            If no `regulator_col` is provided, no regulator information is written in
+            the output. Otherwise, the regulator information is written in the output in
+            the `Regulator` column.
+        regmotifs: The path to the regulator-motif mapping file.
+            It must have header and the columns `Motif` or `Model` for motif names and
+            `TF`, `Regulator` or `Transcription factor`  for regulator names.
+        notfound (choice): What to do if a motif is not found in the database,
+            or a regulator is not found in the regulator-motif mapping (envs.regmotifs)
+            file.
+            - error: Report error and stop the process.
+            - ignore: Ignore the motif and continue.
+        devpars (ns): The default device parameters for the plot.
+            - width (type=int): The width of the plot.
+            - height (type=int): The height of the plot.
+            - res (type=int): The resolution of the plot.
+        plot_vars (type=auto): The variants (SNP_id) to plot.
+            A list of variant names to plot or a string with the variant names separated by comma.
+            When not specified, all variants are plotted.
+    """  # noqa: E501
+    input = "infile:file"
+    output = "outdir:dir:{{in.infile | stem}}.vmplots"
+    lang = config.lang.rscript
+    envs = {
+        "genome": config.ref.genome,
+        "motifdb": config.ref.tf_motifdb,
+        "motif_col": "providerId",
+        "regulator_col": None,
+        "regmotifs": config.ref.tf_motifs,
+        "notfound": "error",
+        "devpars": {"width": 800, "height": None, "res": 100},
+        "plot_vars": None,
+    }
+    script = "file://../scripts/regulatory/VariantMotifPlot.R"

{biopipen-0.31.5 → biopipen-0.31.6}/biopipen/ns/vcf.py

@@ -335,6 +335,8 @@ class TruvariBench(Proc):
     """Run `truvari bench` to compare a VCF with CNV calls and
     base CNV standards
 
+    Requires truvari v4+
+
     See https://github.com/ACEnglish/truvari/wiki/bench
 
     Input:
@@ -358,7 +360,7 @@ class TruvariBench(Proc):
         "truvari": config.exe.truvari,
         "ref": config.ref.reffa,
         "refdist": 500,
-        "pctsim": 0.7,
+        "pctseq": 0.7,
         "pctsize": 0.7,
         "pctovl": 0.0,
         "typeignore": False,
@@ -402,7 +404,7 @@ class TruvariBenchSummary(Proc):
     output = "outdir:dir:truvari_bench.summary"
     lang = config.lang.rscript
     envs = {
-        "plots": ["call cnt", "base cnt", "precision", "recall", "f1"],
+        "plots": ["comp cnt", "base cnt", "precision", "recall", "f1"],
         "devpars": None,
     }
     script = "file://../scripts/vcf/TruvariBenchSummary.R"
@@ -414,6 +416,8 @@ class TruvariConsistency(Proc):
 
     See https://github.com/ACEnglish/truvari/wiki/consistency
 
+    Requires truvari v4+
+
     Input:
         vcfs: The vcf files with CNV calls
 

biopipen-0.31.6/biopipen/scripts/regulatory/MotifAffinityTest.R

@@ -0,0 +1,84 @@
+# Script for regulatory.MotifAffinityTest
+{{ biopipen_dir | joinpaths: "utils", "misc.R" | source_r }}
+{{ biopipen_dir | joinpaths: "scripts", "regulatory", "motifs-common.R" | source_r }}
+
+library(BiocParallel)
+library(BSgenome)
+
+motiffile <- {{in.motiffile | r}}
+varfile <- {{in.varfile | r}}
+outdir <- {{out.outdir | r}}
+ncores <- {{envs.ncores | r}}
+tool <- {{envs.tool | r}}
+bcftools <- {{envs.bcftools | r}}
+genome <- {{envs.genome | r}}
+motif_col <- {{envs.motif_col | r}}
+regulator_col <- {{envs.regulator_col | r}}
+notfound <- {{envs.notfound | r}}
+motifdb <- {{envs.motifdb | r}}
+regmotifs <- {{envs.regmotifs | r}}
+devpars <- {{envs.devpars | r}}
+plot_nvars <- {{envs.plot_nvars | r}}
+plots <- {{envs.plots | r}}
+cutoff <- {{envs.cutoff | r}}
+
+if (is.null(motifdb) || !file.exists(motifdb)) {
+    stop("Motif database (envs.motifdb) is required and must exist")
+}
+
+if (is.null(genome)) {
+    stop("Reference genome (envs.ref) is required and must exist")
+}
+
+if (is.null(motiffile) || !file.exists(motiffile)) {
+    stop("Motif file (in.motiffile) is required and must exist")
+}
+
+if (is.null(varfile) || !file.exists(varfile)) {
+    stop("Variant file (in.varfile) is required and must exist")
+}
+
+if (is.null(motif_col) && is.null(regulator_col)) {
+    stop("Either motif (envs.motif_col) or regulator (envs.regulator_col) column must be provided")
+}
+
+log_info("Reading input regulator/motif file ...")
+in_motifs <- read.table(motiffile, header=TRUE, sep="\t", stringsAsFactors=FALSE, check.names = FALSE)
+
+log_info("Ensuring motifs and regulators in the input data ...")
+in_motifs <- ensure_regulator_motifs(in_motifs, outdir, motif_col, regulator_col, regmotifs, notfound = notfound)
+genome_pkg <- get_genome_pkg(genome)
+
+log_info("Reading variant file ...")
+if (grepl("\\.vcf$", varfile) || grepl("\\.vcf\\.gz$", varfile)) {
+    log_info("Converting VCF file to BED file ...")
+    varfile_bed <- file.path(outdir, gsub("\\.vcf(\\.gz)?$", ".bed", basename(varfile)))
+    cmd <- c(
+        bcftools, "query",
+        "-f", "%CHROM\\t%POS0\\t%END\\t%ID\\t0\\t+\\t%REF\\t%ALT{0}\\n",
+        "-i", 'FILTER="PASS" || FILTER="." || FILTER=""',
+        "-o", varfile_bed,
+        varfile
+    )
+    run_command(cmd, fg = TRUE)
+
+    varfile <- varfile_bed
+}
+
+# `chrom`, `start`, `end`, `name`, `score`, `strand`, `ref`, `alt`.
+snpinfo <- read.table(varfile, header=FALSE, stringsAsFactors=FALSE)
+colnames(snpinfo) <- c("chrom", "start", "end", "name", "score", "strand", "ref", "alt")
+
+log_info("Reading motif database ...")
+mdb <- read_meme_to_motifdb(motifdb, in_motifs, motif_col, regulator_col, notfound, outdir)
+
+tool <- tolower(tool)
+tool <- match.arg(tool, c("motifbreakr", "atsnp"))
+
+if (tool == "motifbreakr") {
+    motifbreakr_args <- {{envs.motifbreakr_args | r}}
+    {{ biopipen_dir | joinpaths: "scripts", "regulatory", "MotifAffinityTest_MotifBreakR.R" | source_r }}
+} else {  # atsnp
+    atsnp_args <- {{envs.atsnp_args | r}}
+    {{ biopipen_dir | joinpaths: "scripts", "regulatory", "MotifAffinityTest_AtSNP.R" | source_r }}
+}

{biopipen-0.31.5 → biopipen-0.31.6}/biopipen/scripts/regulatory/MotifAffinityTest_AtSNP.R

@@ -1,36 +1,6 @@
 library(atSNP)
 library(rtracklayer)
 
-log_info("Converting universalmotif object to motif_library ...")
-
-motifdb_names <- sapply(meme, function(m) m@name)
-motifs <- check_motifs(motifdb_names)
-meme <- filter_motifs(meme, name = motifs)
-# Get the right order of motif names
-motifs <- sapply(meme, function(m) m@name)
-
-# used for atSNP
-mdb <- lapply(meme, function(m) t(m@motif))
-names(mdb) <- motifs
-
-# compose one used for plotting using motifbreakR
-motifdb_matrices <- lapply(meme, function(m) m@motif)
-names(motifdb_matrices) <- motifs
-motifdb_meta <- do.call(rbind, lapply(meme, function(m) {
-    ats <- attributes(m)
-    ats$dataSource <- basename(motifdb)
-    ats$class <- NULL
-    ats$motif <- NULL
-    ats$gapinfo <- NULL
-    ats$sequenceCount <- ats$nsites
-    ats$providerId <- ats$name
-    ats$providerName <- ats$name
-    ats$organism <- if (is.null(ats$organism) || length(ats$organism) == 0) "Unknown" else ats$organism
-    unlist(ats)
-}))
-rownames(motifdb_meta) <- motifs
-pmotifs <- MotifDb:::MotifList(motifdb_matrices, tbl.metadata = motifdb_meta)
-
 log_info("Converting snpinfo to atSNP object ...")
 
 # c("chrom", "start", "end", "name", "score", "strand", "ref", "alt", "ref_seq", "alt_seq")
@@ -53,7 +23,9 @@ write.table(
     file = atsnp_bed,
     sep = "\t", quote = FALSE, row.names = FALSE, col.names = TRUE
 )
-k <- max(sapply(mdb, nrow))
+
+motif_lib <- motifdb_to_motiflib(mdb)
+k <- max(sapply(motif_lib, nrow))
 snps <- LoadSNPData(
     atsnp_bed,
     genome.lib = genome_pkg,
@@ -62,13 +34,12 @@ snps <- LoadSNPData(
     half.window.size = k
 )
 
-# run motifbreakR
 log_info("Running atSNP ...")
-atsnp_scores <- ComputeMotifScore(mdb, snps, ncores = ncores)
+atsnp_scores <- ComputeMotifScore(motif_lib, snps, ncores = ncores)
 
 log_info("Calculating p values ...")
 atsnp_result <- ComputePValues(
-    motif.lib = mdb,
+    motif.lib = motif_lib,
     snp.info = snps,
     motif.scores = atsnp_scores$motif.scores,
     ncores = ncores,
@@ -101,7 +72,7 @@ atsnp_result$motifPos <- sapply(1:nrow(atsnp_result), function(i) {
     paste(c(atsnp_result$ref_start[i] - k, atsnp_result$ref_end[i] - k), collapse = ",")
 })
 if (!is.null(regulator_col)) {
-    atsnp_result$Regulator <- in_motifs[
+    atsnp_result$geneSymbol <- atsnp_result$Regulator <- in_motifs[
         match(atsnp_result$providerId, in_motifs[[motif_col]]),
         regulator_col,
         drop = TRUE
@@ -120,7 +91,30 @@ atsnp_result$alleleDiff <- -atsnp_result[[cutoff_col]]
 atsnp_result$effect <- "strong"
 atsnp_result$motifPos <- lapply(atsnp_result$motifPos, function(x) as.integer(unlist(strsplit(x, ","))))
 atsnp_result <- makeGRangesFromDataFrame(atsnp_result, keep.extra.columns = TRUE, starts.in.df.are.0based = TRUE)
+genome(atsnp_result) <- genome
 attributes(atsnp_result)$genome.package <- genome_pkg
-attributes(atsnp_result)$motifs <- pmotifs
+attributes(atsnp_result)$motifs <- mdb
+
+if (is.null(plots) || length(plots) == 0) {
+    atsnp_result <- atsnp_result[order(-abs(atsnp_result$alleleDiff)), , drop = FALSE]
+    atsnp_result <- atsnp_result[1:min(plot_nvars, length(atsnp_result)), , drop = FALSE]
+    variants <- unique(atsnp_result$SNP_id)
+} else {
+    variants <- names(plots)
+}
+for (variant in variants) {
+    log_info("- Variant: {variant}")
+    if (is.null(plots[[variant]])) {
+        plots[[variant]] <- list(devpars = devpars, which = "TRUE")
+    }
+    if (is.null(plots[[variant]]$which)) {
+        plots[[variant]]$which <- "TRUE"
+    }
+    if (is.null(plots[[variant]]$devpars)) {
+        plots[[variant]]$devpars <- devpars
+    }
+    res <- atsnp_result[atsnp_result$SNP_id == variant, , drop = FALSE]
+    res <- subset(res, subset = eval(parse(text = plots[[variant]]$which)))
 
-plot_variant(atsnp_result)
+    plot_variant_motifs(res, variant, plots[[variant]]$devpars, outdir)
+}

{biopipen-0.31.5 → biopipen-0.31.6}/biopipen/scripts/regulatory/MotifAffinityTest_MotifBreakR.R

@@ -1,30 +1,6 @@
 library(motifbreakR)
-bsgenome <- getBSgenome(genome_pkg)
-
-log_info("Converting universalmotif object to MotifDb object ...")
-
-motifdb_names <- sapply(meme, function(m) m@name)
-motifs <- check_motifs(motifdb_names)
-meme <- filter_motifs(meme, name = motifs)
-# Get the right order of motif names
-motifs <- sapply(meme, function(m) m@name)
-motifdb_matrices <- lapply(meme, function(m) m@motif)
-names(motifdb_matrices) <- motifs
 
-motifdb_meta <- do.call(rbind, lapply(meme, function(m) {
-    ats <- attributes(m)
-    ats$dataSource <- basename(motifdb)
-    ats$class <- NULL
-    ats$motif <- NULL
-    ats$gapinfo <- NULL
-    ats$sequenceCount <- ats$nsites
-    ats$providerId <- ats$name
-    ats$providerName <- ats$name
-    ats$organism <- if (is.null(ats$organism) || length(ats$organism) == 0) "Unknown" else ats$organism
-    unlist(ats)
-}))
-rownames(motifdb_meta) <- motifs
-mdb <- MotifDb:::MotifList(motifdb_matrices, tbl.metadata = motifdb_meta)
+bsgenome <- getBSgenome(genome_pkg)
 
 # `chrom`, `start`, `end`, `name`, `score`, `strand`, `ref`, `alt`.
 is_indel <- nchar(snpinfo$ref) != 1 | nchar(snpinfo$alt) != 1
@@ -93,4 +69,27 @@ write.table(
 )
 rm(results_to_save)
 
-plot_variant(results)
+log_info("Plotting variants ...")
+if (is.null(plots) || length(plots) == 0) {
+    results <- results[order(-abs(results$alleleDiff)), , drop = FALSE]
+    results <- results[1:min(plot_nvars, length(results)), , drop = FALSE]
+    variants <- unique(results$SNP_id)
+} else {
+    variants <- names(plots)
+}
+for (variant in variants) {
+    log_info("- Variant: {variant}")
+    if (is.null(plots[[variant]])) {
+        plots[[variant]] <- list(devpars = devpars, which = "TRUE")
+    }
+    if (is.null(plots[[variant]]$which)) {
+        plots[[variant]]$which <- "TRUE"
+    }
+    if (is.null(plots[[variant]]$devpars)) {
+        plots[[variant]]$devpars <- devpars
+    }
+    res <- results[results$SNP_id == variant, , drop = FALSE]
+    res <- subset(res, subset = eval(parse(text = plots[[variant]]$which)))
+
+    plot_variant_motifs(res, variant, plots[[variant]]$devpars, outdir)
+}

biopipen-0.31.6/biopipen/scripts/regulatory/VariantMotifPlot.R

@@ -0,0 +1,76 @@
+{{ biopipen_dir | joinpaths: "utils", "misc.R" | source_r }}
+{{ biopipen_dir | joinpaths: "scripts", "regulatory", "motifs-common.R" | source_r }}
+
+library(BSgenome)
+library(GenomicRanges)
+
+infile <- {{in.infile | r}}
+outdir <- {{out.outdir | r}}
+genome <- {{envs.genome | r}}
+motifdb <- {{envs.motifdb | r}}
+motif_col <- {{envs.motif_col | r}}
+regulator_col <- {{envs.regulator_col | r}}
+regmotifs <- {{envs.regmotifs | r}}
+notfound <- {{envs.notfound | r}}
+devpars <- {{envs.devpars | r}}
+plot_vars <- {{envs.plot_vars | r}}
+
+if (is.null(motifdb) || !file.exists(motifdb)) {
+    stop("Motif database (envs.motifdb) is required and must exist")
+}
+
+if (is.null(genome)) {
+    stop("Reference genome (envs.ref) is required and must exist")
+}
+
+if (is.null(motif_col) && is.null(regulator_col)) {
+    stop("Either motif (envs.motif_col) or regulator (envs.regulator_col) column must be provided")
+}
+
+log_info("Reading input data ...")
+indata <- read.table(infile, header=TRUE, sep="\t", stringsAsFactors=FALSE, check.names = FALSE)
+
+log_info("Ensuring regulators in the input data ...")
+indata <- ensure_regulator_motifs(indata, outdir, motif_col, regulator_col, regmotifs, notfound = notfound)
+genome_pkg <- get_genome_pkg(genome)
+
+log_info("Reading motif database ...")
+meme <- read_meme_to_motifdb(motifdb, indata, motif_col, regulator_col, notfound, outdir)
+
+log_info("Composing motifbreakR results from input data ...")
+indata$chr <- indata$chrom %||% indata$chr %||% indata$seqnames
+indata$seqnames <- NULL
+indata$strand <- indata$strand %||% "+"
+indata$varType <- indata$varType %||% "SNV"
+indata$geneSymbol <- indata$geneSymbol %||% indata$Regulator
+indata$providerId <- indata$providerId %||% indata$motif
+indata$providerName <- indata$providerName %||% indata$providerId
+indata$dataSource <- indata$dataSource %||% strsplit(basename(motifdb), "\\.")[[1]][1]
+indata$effect <- indata$effect %||% "strong"
+indata$altPos <- indata$altPos %||% 1
+indata$alleleDiff <- indata$alleleDiff %||% indata$score %||% 0
+
+# check other required columns
+for (col in c("start", "end", "SNP_id", "REF", "ALT", "motifPos")) {
+    if (!(col %in% colnames(indata))) {
+        stop("Column '", col, "' is required in the input data")
+    }
+}
+indata$motifPos <- lapply(indata$motifPos, function(x) as.integer(unlist(strsplit(x, ","))))
+indata <- makeGRangesFromDataFrame(indata, keep.extra.columns = TRUE, starts.in.df.are.0based = TRUE)
+genome(indata) <- genome
+attributes(indata)$genome.package <- genome_pkg
+attributes(indata)$motifs <- meme
+
+log_info("Plotting variants ...")
+if (is.null(plot_vars)) {
+    plot_vars <- unique(indata$SNP_id)
+} else if (length(plot_vars) > 1) {
+    plot_vars <- unique(plot_vars)
+} else {
+    plot_vars <- strsplit(plot_vars, ",")[[1]]
+}
+for (pvar in plot_vars) {
+    log_info("- Variant: {pvar}")
+    plot_variant_motifs(indata, pvar, devpars, outdir)
+}