biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/scripts/bam/BamMerge.py

@@ -1,8 +1,8 @@
 from pathlib import Path
-from biopipen.utils.misc import run_command
+from biopipen.utils.misc import run_command, logger
 
-bamfiles = {{in.bamfiles | repr}}  # pyright: ignore
-outfile = Path({{out.outfile | repr}})  # pyright: ignore
+bamfiles = {{in.bamfiles | default: [] | each: str}}  # pyright: ignore # noqa
+outfile = Path({{out.outfile | quote}})  # pyright: ignore
 ncores = {{envs.ncores | int}}  # pyright: ignore
 tool = {{envs.tool | quote}}  # pyright: ignore
 samtools = {{envs.samtools | quote}}  # pyright: ignore
@@ -18,7 +18,7 @@ if should_index and not should_sort:
 
 def use_samtools():
     """Use samtools to merge bam files"""
-    print("Using samtools")
+    logger.info("Using samtools ...")
     ofile = (
         outfile
         if not should_sort
@@ -43,11 +43,11 @@ def use_samtools():
         *merge_args,
         *bamfiles,
     ]
-    print("- Merging")
+    logger.info("- Merging the bam files ...")
     run_command(cmd)
 
     if should_sort:
-        print("- Sorting")
+        logger.info("- Sorting the merged bam file ...")
         for key in ["-o", "-@", "--threads"]:
             if key in sort_args:
                 raise ValueError(
@@ -67,16 +67,14 @@ def use_samtools():
         run_command(cmd)
 
     if should_index:
-        print("- Indexing")
+        logger.info("- Indexing the output bam file ...")
         cmd = [samtools, "index", "-@", ncores, outfile]
         run_command(cmd)
 
-    print("Done")
-
 
 def use_sambamba():
     """Use sambamba to merge bam files"""
-    print("Using sambamba")
+    logger.info("Using sambamba ...")
     ofile = (
         outfile
         if not should_sort
@@ -90,11 +88,11 @@ def use_sambamba():
             )
 
     cmd = [sambamba, "merge", "-t", ncores, *merge_args, ofile, *bamfiles]
-    print("- Merging")
+    logger.info("- Merging the bam files ...")
     run_command(cmd)
 
     if should_sort:
-        print("- Sorting")
+        logger.info("- Sorting the merged bam file ...")
         for key in ["-t", "--nthreads", "-o", "--out"]:
             if key in sort_args:
                 raise ValueError(
@@ -115,12 +113,10 @@ def use_sambamba():
         run_command(cmd)
 
     if should_index:
-        print("- Indexing")
+        logger.info("- Indexing the output bam file ...")
         cmd = [sambamba, "index", "-t", ncores, outfile]
         run_command(cmd)
 
-    print("Done")
-
 
 if __name__ == "__main__":
     if tool == "samtools":

biopipen/scripts/bam/BamSampling.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+from biopipen.utils.misc import run_command, logger
+
+# using:
+# samtools view --subsample 0.1 --subsample-seed 1234 --threads 4 -b -o out.bam in.bam
+
+bamfile = {{ in.bamfile | quote }} # pyright: ignore # noqa
+outfile = Path({{ out.outfile | quote }}) # pyright: ignore
+ncores = {{ envs.ncores | int }} # pyright: ignore
+samtools = {{ envs.samtools | repr }} # pyright: ignore
+tool = {{ envs.tool | repr }} # pyright: ignore
+fraction: float = {{ envs.fraction | repr }} # pyright: ignore
+seed = {{ envs.seed | int }} # pyright: ignore
+should_index = {{ envs.index | repr }} # pyright: ignore
+should_sort = {{ envs.sort | repr }} # pyright: ignore
+sort_args = {{ envs.sort_args | repr }} # pyright: ignore
+
+if should_index and not should_sort:
+    raise ValueError("Indexing requires sorting")
+
+if fraction is None:
+    raise ValueError("'envs.fraction' must be provided.")
+
+if tool != "samtools":
+    raise ValueError(
+        f"Tool {tool} is not supported. "
+        "Currently only samtools is supported."
+    )
+
+if fraction > 1:
+    # calculate the fraction based on the number of reads
+    logger.info("Converting fraction > 1 to a fraction of reads.")
+    cmd = [
+        samtools,
+        "view",
+        "--threads",
+        ncores,
+        "-c",
+        bamfile
+    ]
+    nreads = run_command(cmd, stdout="return").strip()  # type: ignore
+    fraction = fraction / float(int(nreads))
+
+ofile = (
+    outfile
+    if not should_sort
+    else outfile.with_stem(f"{outfile.stem}.unsorted")
+)
+
+cmd = [
+    samtools,
+    "view",
+    "--subsample",
+    fraction,
+    "--subsample-seed",
+    seed,
+    "--threads",
+    ncores,
+    "-b",
+    "-o",
+    ofile,
+    bamfile
+]
+run_command(cmd, fg=True)
+
+if should_sort:
+    logger.info("Sorting the output bam file.")
+    for key in ["-o", "-@", "--threads"]:
+        if key in sort_args:
+            raise ValueError(
+                f"envs.sort_args cannot contain {key}, "
+                "which is managed by the script"
+            )
+
+    cmd = [
+        samtools,
+        "sort",
+        "-@",
+        ncores,
+        *sort_args,
+        "-o",
+        outfile,
+        ofile
+    ]
+    run_command(cmd, fg=True)
+
+if should_index:
+    logger.info("Indexing the output bam file.")
+    cmd = [samtools, "index", "-@", ncores, outfile]
+    run_command(cmd, fg=True)

biopipen/scripts/bam/BamSort.py

@@ -0,0 +1,141 @@
+from hashlib import md5
+from pathlib import Path
+from biopipen.utils.misc import run_command, dict_to_cli_args
+
+infile: str = {{ in.bamfile | quote }} # pyright: ignore # noqa
+outfile = Path({{ out.outfile | quote }}) # pyright: ignore
+args: dict = {{ envs | dict | repr }} # pyright: ignore
+ncores = args.pop("ncores")
+tool = args.pop("tool")
+samtools = args.pop("samtools")
+sambamba = args.pop("sambamba")
+tmpdir = args.pop("tmpdir")
+byname = args.pop("byname")
+should_index = args.pop("index")
+sig = md5(infile.encode()).hexdigest()
+tmpdir = Path(tmpdir).joinpath(
+    f"biopipen_BamSort_{{job.index}}_{sig}_{Path(infile).name}"
+)
+tmpdir.mkdir(parents=True, exist_ok=True)
+tmpdir = str(tmpdir)
+
+
+def use_samtools():
+    """Use samtools to sort/index bam file.
+
+    Usage: samtools sort [options...] [in.bam]
+    Options:
+    -l INT     Set compression level, from 0 (uncompressed) to 9 (best)
+    -u         Output uncompressed data (equivalent to -l 0)
+    -m INT     Set maximum memory per thread; suffix K/M/G recognized [768M]
+    -M         Use minimiser for clustering unaligned/unplaced reads
+    -K INT     Kmer size to use for minimiser [20]
+    -n         Sort by read name (not compatible with samtools index command)
+    -t TAG     Sort by value of TAG. Uses position as secondary index (or read name if -n is set)
+    -o FILE    Write final output to FILE rather than standard output
+    -T PREFIX  Write temporary files to PREFIX.nnnn.bam
+        --no-PG
+                Do not add a PG line
+        --template-coordinate
+                Sort by template-coordinate
+        --input-fmt-option OPT[=VAL]
+                Specify a single input file format option in the form
+                of OPTION or OPTION=VALUE
+    -O, --output-fmt FORMAT[,OPT[=VAL]]...
+                Specify output format (SAM, BAM, CRAM)
+        --output-fmt-option OPT[=VAL]
+                Specify a single output file format option in the form
+                of OPTION or OPTION=VALUE
+        --reference FILE
+                Reference sequence FASTA FILE [null]
+    -@, --threads INT
+                Number of additional threads to use [0]
+        --write-index
+                Automatically index the output files [off]
+        --verbosity INT
+                Set level of verbosity
+    """  # noqa
+    sargs = args.copy()
+    sargs["n"] = byname
+    sargs["T"] = f"{tmpdir}/tmp"
+    sargs["threads"] = ncores
+
+    if should_index:
+        sargs["write-index"] = True
+        # https://github.com/samtools/samtools/issues/1196
+        sargs["o"] = f"{outfile}##idx##{outfile}.bai"
+    else:
+        sargs["o"] = outfile
+
+    n_outfmt = sum(["O" in sargs, "output-fmt" in sargs])
+    if n_outfmt > 1:
+        raise ValueError(
+            "envs.args cannot contain both 'O' and 'output-fmt'"
+        )
+    if n_outfmt == 0:
+        sargs["O"] = "BAM"
+
+    cmd = [
+        samtools,
+        "sort",
+        *dict_to_cli_args(sargs),
+        infile,
+    ]
+    run_command(cmd)
+
+
+def use_sambamba():
+    """Use sambamba to sort/index bam file.
+
+    sambamba 0.8.2
+    by Artem Tarasov and Pjotr Prins (C) 2012-2021
+        LDC 1.28.1 / DMD v2.098.1 / LLVM12.0.0 / bootstrap LDC - the LLVM D compiler (1.28.1)
+
+    Usage: sambamba-sort [options] <input.bam>
+
+    Options: -m, --memory-limit=LIMIT
+                approximate total memory limit for all threads (by default 2GB)
+            --tmpdir=TMPDIR
+                directory for storing intermediate files; default is system directory for temporary files
+            -o, --out=OUTPUTFILE
+                output file name; if not provided, the result is written to a file with .sorted.bam extension
+            -n, --sort-by-name
+                sort by read name instead of coordinate (lexicographical order)
+            --sort-picard
+                sort by query name like in picard
+            -N, --natural-sort
+                sort by read name instead of coordinate (so-called 'natural' sort as in samtools)
+            -M, --match-mates
+                pull mates of the same alignment together when sorting by read name
+            -l, --compression-level=COMPRESSION_LEVEL
+                level of compression for sorted BAM, from 0 to 9
+            -u, --uncompressed-chunks
+                write sorted chunks as uncompressed BAM (default is writing with compression level 1), that might be faster in some cases but uses more disk space
+            -p, --show-progress
+                show progressbar in STDERR
+            -t, --nthreads=NTHREADS
+                use specified number of threads
+            -F, --filter=FILTER
+                keep only reads that satisfy FILTER
+    """  # noqa
+    sargs = args.copy()
+    sargs["nthreads"] = ncores
+    sargs["n"] = byname
+    sargs["tmpdir"] = tmpdir
+    sargs["o"] = outfile
+    cmd = [
+        sambamba,
+        "sort",
+        *dict_to_cli_args(sargs, sep="="),
+        infile,
+    ]
+    run_command(cmd)
+
+
+if __name__ == "__main__":
+    if tool == "samtools":
+        use_samtools()
+    elif tool == "sambamba":
+        use_sambamba()
+    else:
+        raise ValueError(f"Unknown tool: {tool}")

biopipen/scripts/bam/BamSplitChroms.py

@@ -2,12 +2,12 @@ from pathlib import Path
 from biopipen.utils.misc import run_command
 from biopipen.utils.reference import bam_index
 
-bamfile = {{in.bamfile | quote}}  # pyright: ignore
-outdir = {{out.outdir | quote}}  # pyright: ignore
-tool = {{envs.tool | quote}}  # pyright: ignore
-samtools = {{envs.samtools | quote}}  # pyright: ignore
-sambamba = {{envs.sambamba | quote}}  # pyright: ignore
-ncores = {{envs.ncores | repr}}  # pyright: ignore
+bamfile: str = {{in.bamfile | quote}}  # pyright: ignore  # noqa
+outdir: str = {{out.outdir | quote}}  # pyright: ignore
+tool: str = {{envs.tool | quote}}  # pyright: ignore
+samtools: str = {{envs.samtools | quote}}  # pyright: ignore
+sambamba: str = {{envs.sambamba | quote}}  # pyright: ignore
+ncores: int = {{envs.ncores | repr}}  # pyright: ignore
 keep_other_sq = {{envs.keep_other_sq | repr}}  # pyright: ignore
 chroms_to_keep = {{envs.chroms | repr}}  # pyright: ignore
 should_index = {{envs.index | bool}}  # pyright: ignore
@@ -17,13 +17,13 @@ def _remove_other_sq(infile, chrom, outfile):
     exe = samtools if tool == "samtools" else sambamba
     print("\nRemoving other chromosomes in @SQ in header")
     header_cmd = [exe, "view", "-H", infile]
-    header_p = run_command(
+    header_p = run_command(  # type: ignore
         header_cmd,
         stdout=True,
         wait=False,
         print_command=True,
     )
-    header = header_p.stdout.read().decode().strip().splitlines()
+    header = header_p.stdout.read().decode().strip().splitlines()  # type: ignore
     new_header = []
     for line in header:
         if line.startswith("@SQ"):
@@ -63,7 +63,7 @@ def use_samtools():
             "| grep '^@SQ' | cut -f 2 | cut -d ':' -f 2"
         )
         p = run_command(cmd, stdout=True, wait=False)
-        chroms = p.stdout.read().decode().strip().splitlines()
+        chroms = p.stdout.read().decode().strip().splitlines()  # type: ignore
     else:
         print("\nUsing provided chromosomes")
         chroms = chroms_to_keep
@@ -121,7 +121,7 @@ def use_sambamba():
             "| grep '^@SQ' | cut -f 2 | cut -d ':' -f 2"
         )
         p = run_command(cmd, stdout=True, wait=False)
-        chroms = p.stdout.read().decode().splitlines()
+        chroms = p.stdout.read().decode().splitlines()  # type: ignore
     else:
         print("\nUsing provided chromosomes")
         chroms = chroms_to_keep

biopipen/scripts/bam/BamSubsetByBed.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+from biopipen.utils.misc import run_command, logger
+
+# using:
+# samtools view --subsample 0.1 --subsample-seed 1234 --threads 4 -b -o out.bam in.bam
+
+bamfile = {{ in.bamfile | quote }} # pyright: ignore # noqa
+bedfile = {{ in.bedfile | quote }} # pyright: ignore # noqa
+outfile = Path({{ out.outfile | quote }}) # pyright: ignore
+ncores = {{ envs.ncores | int }} # pyright: ignore
+samtools = {{ envs.samtools | repr }} # pyright: ignore
+tool = {{ envs.tool | repr }} # pyright: ignore
+should_index = {{ envs.index | repr }} # pyright: ignore
+
+if tool != "samtools":
+    raise ValueError(
+        f"Tool {tool} is not supported. "
+        "Currently only samtools is supported."
+    )
+
+cmd = [
+    samtools,
+    "view",
+    "--target-file",
+    bedfile,
+    "-b",
+    "--threads",
+    ncores,
+    "-o",
+    outfile,
+    bamfile
+]
+run_command(cmd, fg=True)
+
+if should_index:
+    logger.info("Indexing the output bam file.")
+    cmd = [samtools, "index", "-@", ncores, outfile]
+    run_command(cmd, fg=True)

biopipen/scripts/bam/CNAClinic.R

@@ -1,13 +1,40 @@
-source("{{biopipen_dir}}/utils/misc.R")
 library(parallel)
 library(dplyr)
+library(biopipen.utils)
 library(CNAclinic)
 
+# https://github.com/sdchandra/CNAclinic/issues/4
+.reorderByChrom.patched <- function(x){
+    chromosome <- as.character(x$chromosome)
+    chromosome[which(chromosome == "X")] <- "23"
+    chromosome[which(chromosome == "Y")] <- "24"
+    chromosome[which(chromosome == "MT")] <- "25"
+
+    x$chromosome <- as.numeric(chromosome)
+    # Error in xtfrm.data.frame(x) : cannot xtfrm data frames
+    # x <- x[order(x["chromosome"], x["start"]), ]
+    x <- x[order(x[, "chromosome"], x[, "start"]), ]
+
+    x$chromosome <- as.character(x$chromosome)
+    # Replace 23 by X:
+    x$chromosome[which(x$chromosome == "23")] <- "X"
+
+    # Replace 24 by Y
+    x$chromosome[which(x$chromosome == "24")] <- "Y"
+
+    # Replace 25 by MT
+    x$chromosome[which(x$chromosome == "25")] <- "MT"
+
+    return(x)
+}
+
+monkey_patch("CNAclinic", ".reorderByChrom", .reorderByChrom.patched)
+
 metafile = {{in.metafile | r}}
 outdir = {{out.outdir | r}}
 ncores = {{envs.ncores | int}}
 binsizer = {{envs.binsizer | r}}
-binsize = {{envs.binsize | int}}
+binsize = {{envs.binsize | r}}
 seed = {{envs.seed | int}}
 genome = {{envs.genome | r}}
 run_args = {{envs.run_args | r}}
@@ -29,7 +56,11 @@ if (("Group" %in% metacols) && !("Patient" %in% metacols)) {
 }
 
 if (!("Binsizer" %in% metacols) && is.null(binsizer) && is.null(binsize)) {
-    stop("The metadata file must have a column named 'Binsizer' or the `envs.binsizer` must be specified")
+    stop(
+        "The metadata file must have a column named 'Binsizer' or ",
+        "the `envs.binsizer` must be specified when no `envs.binsize` is provided. ",
+        "The Binsizer column should indicate which samples are to be used for binsize selection."
+    )
 }
 
 # add missing columns
@@ -108,7 +139,7 @@ do_one_sample = function(i) {
         bamfile,
         sample,
         refSamples=refSamples,
-        binSize=binsize
+        binSize=binsize / 1000
     )
 
     run_args_i = run_args
@@ -118,7 +149,12 @@ do_one_sample = function(i) {
 
     plot_args_i = plot_args
     plot_args_i$object = CNAData
-    genomewide_plot = do_call(plotSampleData, plot_args_i)
+    genomewide_plot <- tryCatch({
+        do_call(plotSampleData, plot_args_i)
+    }, error = function(e) {
+        message("Error in plotting genomewide data for sample ", sample, ": ", e$message)
+        return(ggplot2::ggplot() + ggplot2::labs(title = paste("Error in plotting genomewide data for sample", sample)))
+    })
 
     odir = file.path(outdir, sample)
     dir.create(odir, recursive = TRUE, showWarnings = FALSE)