biopipen 0.24.1__tar.gz → 1.1.4__tar.gz

biopipen-1.1.4/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: biopipen
+Version: 1.1.4
+Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
+License: MIT
+Author: pwwang
+Author-email: pwwang@pwwang.com
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: log2file
+Provides-Extra: runinfo
+Requires-Dist: datar[pandas] (>=0.15.8,<0.16.0)
+Requires-Dist: pipen-board[report] (>=1.1,<2.0)
+Requires-Dist: pipen-cli-run (>=1.0,<2.0)
+Requires-Dist: pipen-deprecated (>=1.0,<2.0)
+Requires-Dist: pipen-filters (>=1.1.2,<2.0.0)
+Requires-Dist: pipen-log2file (>=1.1,<2.0) ; extra == "log2file"
+Requires-Dist: pipen-poplog (>=1.1,<2.0)
+Requires-Dist: pipen-runinfo (>=1.1,<2.0) ; extra == "runinfo"
+Requires-Dist: pipen-verbose (>=1.1,<2.0)

biopipen-1.1.4/biopipen/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.1.4"

{biopipen-0.24.1 → biopipen-1.1.4}/biopipen/core/config.toml

@@ -1,11 +1,17 @@
 # Executables or binaries
 [exe]
+# BeEM: https://github.com/kad-ecoli/BeEM
+beem = "BeEM"
 # bedtools to handle bed files
 bedtools = "bedtools"
 # bcftools to handle bcf/vcf files
 bcftools = "bcftools"
+# calculate_rmsd: https://github.com/charnley/rmsd
+calculate_rmsd = "calculate_rmsd"
 # cellranger
 cellranger = "cellranger"
+# cellsnp-lite
+cellsnp_lite = "cellsnp-lite"
 # Control-FREEC to call cnvs
 freec = "freec"
 # liftover coordinates across genomes
@@ -13,6 +19,8 @@ liftover = "liftOver"
 # gatk, installed via conda
 gatk = "gatk"
 gatk4 = "gatk"
+# google cloud sdk
+gcloud = "gcloud"
 # vdjtools, installed via conda
 vdjtools = "vdjtools"
 # cnvkit.py
@@ -23,10 +31,20 @@ cnvpytor = "cnvpytor"
 cnvnator2vcf = "cnvnator2VCF.pl"
 # convert
 convert = "convert"
+# fimo from meme
+fimo = "fimo"
+# MAXIT: https://sw-tools.rcsb.org/apps/MAXIT/
+maxit = "maxit"
+# MQuad: https://github.com/single-cell-genetics/MQuad
+mquad = "mquad"
 # wget
 wget = "wget"
 # aria2c
 aria2c = "aria2c"
+# plink
+plink = "plink"
+# plink2
+plink2 = "plink2"
 # tabix
 tabix = "tabix"
 # sambamba
@@ -84,6 +102,10 @@ genome = ""
 # Database file for scType
 # https://github.com/IanevskiAleksandr/sc-type/
 sctype_db = ""
+# TF Motif database
+tf_motifdb = ""
+# TF motif pairs
+tf_motifs = ""
 
 [misc]
 # Number of cores used for each job

{biopipen-0.24.1 → biopipen-1.1.4}/biopipen/core/filters.py

@@ -1,13 +1,16 @@
 """Additional filters for pipen"""
 from __future__ import annotations
 
+import re
 import shlex
 from pathlib import Path
 from typing import Any, List, Mapping
 
-from argx import Namespace
+from argx import Namespace  # pyright: ignore[reportPrivateImportUsage]
 from liquid.filters.manager import FilterManager
-from pipen_report.filters import register_component, render_ui, _tag
+from pipen_report.filters import register_component, _tag
+
+# from .defaults import BIOPIPEN_DIR
 
 filtermanager = FilterManager()
 
@@ -15,7 +18,7 @@ filtermanager = FilterManager()
 @filtermanager.register
 def dict_to_cli_args(
     dic: Mapping[str, Any],
-    exclude: List[str] = None,
+    exclude: List[str] | None = None,
     prefix: str | None = None,
     sep: str | None = " ",
     dup_key: bool = True,
@@ -28,7 +31,7 @@ def dict_to_cli_args(
 
     Args:
         dic: The dict to convert
-        exclude: The keys to exclude
+        exclude: The keys to exclude before conversion (e.g. dashify)
         prefix: The prefix of the keys after conversion
             Defaults to `None`, mean `-` for short keys and `--` for long keys
         sep: The separator between key and value
@@ -118,7 +121,7 @@ def dict_to_cli_args(
 def r(
     obj: Any,
     ignoreintkey: bool = True,
-    todot: str = None,
+    todot: str | None = None,
     sortkeys: bool = False,
     skip: int = 0,
     _i: int = 0,
@@ -169,8 +172,10 @@ def r(
             return "TRUE"
         if obj.upper() == "FALSE":
             return "FALSE"
-        if obj.upper() == "NA" or obj.upper() == "NULL":
+        if obj.upper() == "NA" or obj.upper() == "NULL" or obj == "None":
             return obj.upper()
+        if re.match(r"^\d+:\d+$", obj):
+            return obj
         if obj.startswith("r:") or obj.startswith("R:"):
             return str(obj)[2:]
         return repr(str(obj))
@@ -221,68 +226,31 @@ def r(
     return repr(obj)
 
 
-@register_component("fgsea")
-def _render_fgsea(
-    cont: Mapping[str, Any],
-    job: Mapping[str, Any],
-    level: int,
-    na_arg: str = "10",
-) -> str:
-    """Render fgsea report"""
-    # cont["dir"] is required
-    n_pathways = int(na_arg)
-    pathways = []
-    with Path(cont["dir"]).joinpath("fgsea.txt").open() as f:
-        next(f)  # skip header
-        for line in f:
-            pathway, _ = line.split("\t", 1)
-            pathways.append(pathway)
-            if len(pathways) >= n_pathways:
-                break
-
-    components = [
-        # Summary
-        {
-            "title": "Enrichment Analysis Summary",
-            "ui": "tabs",
-            "contents": [
-                {
-                    "title": "Plot",
-                    "ui": "flat",
-                    "contents": [
-                        {
-                            "kind": "image",
-                            "src": str(Path(cont["dir"]).joinpath("gsea_table.png")),
-                        }
-                    ],
-                },
-                {
-                    "title": "Table",
-                    "ui": "flat",
-                    "contents": [
-                        {
-                            "kind": "table",
-                            "src": str(Path(cont["dir"]).joinpath("fgsea.txt")),
-                        }
-                    ],
-                },
-            ]
-        },
-        # Pathways
-        {
-            "title": f"Enriched Pathways (Top {n_pathways})",
-            "ui": "table_of_images",
-            "contents": [
-                {
-                    "src": str(Path(cont["dir"]) / f"fgsea_{pw.replace('/', '-')}.png"),
-                    "title": pw,
-                }
-                for pw in pathways
-            ]
-        },
-    ]
-
-    return render_ui(components, "accordion", job, level)
+@filtermanager.register
+def source_r(path: str | Path, chdir: bool = False) -> str:
+    """Source an R script.
+
+    In addition to generating `source(path)`, we also include the mtime for the script
+    to trigger the job not cached when the script is updated.
+
+    If your process is used in a cloud environment, it is recommended to
+    use the `read` filter to load the script content instead of sourcing it using
+    the `source` function in R to void the path issue (path could be different
+    in different environments).
+
+    Args:
+        path: The path to the R script
+
+    Returns:
+        The R code to source the script
+    """
+    path = Path(path)
+    mtime = int(path.stat().st_mtime)
+    return (
+        f"# Last modified: {mtime}\n"
+        # f"biopipen_dir = {r(BIOPIPEN_DIR)}\n"
+        f"source('{path}', chdir = {r(chdir)})"
+    )
 
 
 @register_component("pdf")
@@ -312,72 +280,3 @@ def _render_gsea(
     """Render gsea report"""
     # cont["dir"] is required
     raise NotImplementedError()
-
-
-@register_component("enrichr")
-def _render_enrichr(
-    cont: Mapping[str, Any],
-    job: Mapping[str, Any],
-    level: int,
-) -> str:
-    """Render enrichr report"""
-    # cont["dir"] is required
-    dbs = [sumfile.stem[8:] for sumfile in Path(cont["dir"]).glob("Enrichr-*.txt")]
-    components = []
-
-    for db in dbs:
-        enrichr_plot = Path(cont["dir"]).joinpath(f"Enrichr-{db}.png")
-        if enrichr_plot.exists():
-            components.append(
-                {
-                    "title": db,
-                    "ui": "tabs",
-                    "contents": [
-                        {
-                            "title": "Plot",
-                            "ui": "flat",
-                            "contents": [
-                                {
-                                    "kind": "image",
-                                    "src": str(
-                                        Path(cont["dir"]).joinpath(f"Enrichr-{db}.png")
-                                    ),
-                                }
-                            ],
-                        },
-                        {
-                            "title": "Table",
-                            "ui": "flat",
-                            "contents": [
-                                {
-                                    "kind": "table",
-                                    "src": str(
-                                        Path(cont["dir"]).joinpath(f"Enrichr-{db}.txt")
-                                    ),
-                                }
-                            ],
-                        },
-                    ],
-                }
-            )
-        else:
-            components.append(
-                {
-                    "title": db,
-                    "ui": "tabs",
-                    "contents": [
-                        {
-                            "title": "Error",
-                            "ui": "flat",
-                            "contents": [
-                                {
-                                    "kind": "error",
-                                    "content": "No enriched terms found.",
-                                }
-                            ],
-                        },
-                    ],
-                }
-            )
-
-    return render_ui(components, "accordion", job, level)

{biopipen-0.24.1 → biopipen-1.1.4}/biopipen/core/proc.py

@@ -1,7 +1,9 @@
 """Provides a base class for the processes to subclass"""
-from diot import Diot
+from __future__ import annotations
+
+from diot import Diot  # type: ignore
 from liquid.defaults import SEARCH_PATHS
-from pipen import Proc as PipenProc
+from pipen import Proc as PipenProc  # type: ignore
 from pipen_filters.filters import FILTERS
 
 from .filters import filtermanager
@@ -23,7 +25,7 @@ class Proc(PipenProc):
     template_opts = {
         "globals": {**FILTERS, "biopipen_dir": str(BIOPIPEN_DIR)},
         "filters": {**FILTERS, **filtermanager.filters},
-        "search_paths": SEARCH_PATHS + [str(REPORT_DIR)],
+        "search_paths": SEARCH_PATHS + [str(REPORT_DIR)],  # type: ignore
     }
 
     plugin_opts = {

biopipen-1.1.4/biopipen/core/testing.py

@@ -0,0 +1,122 @@
+"""Provide utilities for testing."""
+import tempfile
+from functools import wraps
+from pathlib import Path
+
+from pipen import Pipen
+
+TESTING_INDEX_INIT = 1
+TESTING_PARENT_DIR = Path(__file__).parent.parent.parent.joinpath("tests", "running")
+TESTING_PARENT_DIR.mkdir(parents=True, exist_ok=True)
+TESTING_DIR = str(TESTING_PARENT_DIR.joinpath("biopipen-tests-%(index)s"))
+RSCRIPT_DIR = TESTING_PARENT_DIR.joinpath("biopipen-tests-rscripts")
+RSCRIPT_DIR.mkdir(exist_ok=True)
+
+
+def _find_testing_index(new):
+    """Find the next available testing index"""
+    index = TESTING_INDEX_INIT
+    while True:
+        dir = TESTING_DIR % {"index": index}
+        if not Path(dir).exists():
+            if new:
+                break
+            else:
+                return max(index - 1, TESTING_INDEX_INIT)
+        index += 1
+    return index
+
+
+def _get_test_dirs(testfile, new):
+    """Get the workdir and outdir for a test pipeline"""
+    index = _find_testing_index(new)
+    workdir = TESTING_DIR % {"index": index}
+    procname = Path(testfile).parent.stem
+    nsname = Path(testfile).parent.parent.stem
+    name = f"{nsname}.{procname}"
+    outdir = f"{workdir}/{nsname}/output"
+    workdir = f"{workdir}/{procname}/pipen"
+    Path(workdir).mkdir(parents=True, exist_ok=True)
+    Path(outdir).mkdir(parents=True, exist_ok=True)
+    return name, workdir, outdir
+
+
+def get_pipeline(testfile, loglevel="debug", enable_report=False, **kwargs):
+    """Get a pipeline for a test file"""
+    name, workdir, outdir = _get_test_dirs(testfile, False)
+    report_plugin_prefix = "+" if enable_report else "-"
+    plugins = kwargs.pop("plugins", [])
+    if any("report" in p for p in plugins if isinstance(p, str)):
+        raise ValueError(
+            "Do not pass `report` plugin to `get_pipeline(plugins=[...])`, "
+            "use `enable_report` instead."
+        )
+    plugins.append(f"{report_plugin_prefix}report")
+    kws = {
+        "name": name,
+        "workdir": workdir,
+        "outdir": outdir,
+        "loglevel": loglevel,
+        "plugins": plugins,
+    }
+    kws.update(kwargs)
+    return Pipen(**kws)
+
+
+def _run_rcode(rcode: str) -> str:
+    """Run R code and return the output"""
+    import hashlib
+    import textwrap
+    import subprocess as sp
+
+    # Use sha256 of rcode to name the file
+    rcode_hash = hashlib.sha256(rcode.encode()).hexdigest()
+    script_file = RSCRIPT_DIR.joinpath(f"rcode-{rcode_hash}.R")
+    script_file.write_text(rcode)
+    p = sp.Popen(["Rscript", str(script_file)], stdout=sp.PIPE, stderr=sp.PIPE)
+    out, err = p.communicate()
+    if p.returncode != 0:
+        out = (
+            f"R codefile:\n  {script_file}\n"
+            f"Error:\n{textwrap.indent(err.decode(), '  ')}"
+        )
+        return out
+
+    return out.decode().strip()
+
+
+def r_test(mem: callable) -> callable:
+    """A decorator to test R code"""
+    @wraps(mem)
+    def decorator(self, *args, **kwargs):
+        rcode = mem(self, *args, **kwargs)
+        source = getattr(self, "SOURCE_FILE", None)
+        expect = (
+            "expect <- function(expr, ...) {\n"
+            "  if (!expr) {\n"
+            "    msg <- lapply(\n"
+            "      list(...),\n"
+            "      function(x) { ifelse(is.null(x), 'NULL', x) }\n"
+            "    )\n"
+            "    stop(paste0(unlist(msg), collapse = ' '))\n"
+            "  }\n"
+            "}\n"
+        )
+        rcode = f"{expect}\n\n{rcode}\n\ncat('PASSED')\n"
+        if source is not None:
+            if not isinstance(source, (list, tuple)):
+                source = [source]
+
+            libs = "\n".join([f"suppressWarnings(source('{s}'))" for s in source])
+            rcode = f'{libs}\n\n{rcode}'
+
+        out = _run_rcode(rcode)
+        self.assertEqual(
+            out,
+            "PASSED",
+            "\n-----------------------------\n"
+            f"{out}"
+            "\n-----------------------------\n"
+        )
+
+    return decorator

{biopipen-0.24.1 → biopipen-1.1.4}/biopipen/ns/bam.py

@@ -4,6 +4,9 @@ from ..core.proc import Proc
 from ..core.config import config
 
 
+# +-------------------------------------------------------------------+
+# | CNV callers                                                       |
+# +-------------------------------------------------------------------+
 class CNVpytor(Proc):
     """Detect CNV using CNVpytor
 
@@ -17,7 +20,6 @@ class CNVpytor(Proc):
 
     Envs:
         cnvpytor: Path to cnvpytor
-        cnvnator2vcf: Path to CNVnator2VCF.pl to convert the result to VCF file
         samtools: Path to samtools, used to index bam file in case it's not
         ncores: Number of cores to use (`-j` for cnvpytor)
         refdir: The directory containing the fasta file for each chromosome
@@ -27,21 +29,19 @@ class CNVpytor(Proc):
         binsizes: The binsizes
         snp: How to read snp data
         filters: The filters to filter the result
-            See - https://github.com/abyzovlab/CNVpytor/blob/master
-            /GettingStarted.md#predicting-cnv-regions
+            See - https://github.com/abyzovlab/CNVpytor/blob/master/GettingStarted.md#predicting-cnv-regions
         mask_snps: Whether mask 1000 Genome snps
         baf_nomask: Do not use P mask in BAF histograms
 
     Requires:
         cnvpytor:
            - check: {{proc.envs.cnvpytor}} --version
-    """
+    """  # noqa: E501
     input = "bamfile:file, snpfile:file"
     output = "outdir:dir:{{in.bamfile | stem}}.cnvpytor"
     lang = config.lang.python
     envs = {
         "cnvpytor": config.exe.cnvpytor,
-        "cnvnator2vcf": config.exe.cnvnator2vcf,
         "samtools": config.exe.samtools,
         "ncores": config.misc.ncores,
         "refdir": config.ref.refdir,
@@ -152,7 +152,7 @@ class CNAClinic(Proc):
             A list of sample names
             A float number (0 < x <= 1), the fraction of samples to use
             A integer number (x > 1), the number of samples to use
-        binsize: Directly use this binsize for CNAClinic, in kbp.
+        binsize: Directly use this binsize for CNAClinic, in bp.
         genome: The genome assembly
         run_args: The arguments for CNAClinic::runSegmentation
         plot_args: The arguments for CNAClinic::plotSampleData
@@ -183,6 +183,9 @@ class CNAClinic(Proc):
     }
 
 
+# +-------------------------------------------------------------------+
+# | Bam processing tools                                              |
+# +-------------------------------------------------------------------+
 class BamSplitChroms(Proc):
     """Split bam file by chromosomes
 
@@ -262,3 +265,142 @@ class BamMerge(Proc):
         "sort_args": [],
     }
     script = "file://../scripts/bam/BamMerge.py"
+
+
+class BamSampling(Proc):
+    """Keeping only a fraction of read pairs from a bam file
+
+    Input:
+        bamfile: The bam file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        ncores: Number of cores to use
+        samtools: Path to samtools executable
+        tool: The tool to use, currently only "samtools" is supported
+        fraction (type=float): The fraction of reads to keep.
+            If `0 < fraction <= 1`, it's the fraction of reads to keep.
+            If `fraction > 1`, it's the number of reads to keep.
+            Note that when fraction > 1, you may not get the exact number
+            of reads specified but a close number.
+        seed: The seed for random number generator
+        index: Whether to index the output bam file
+        sort: Whether to sort the output bam file
+        sort_args: The arguments for sorting bam file using `samtools sort`.
+            These keys are not allowed: `-o`, `-@`,
+            and `--threads`, as they are managed by the script.
+    """
+    input = "bamfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}.sampled{{envs.fraction}}.bam"
+    lang = config.lang.python
+    envs = {
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "tool": "samtools",
+        "fraction": None,
+        "seed": 8525,
+        "index": True,
+        "sort": True,
+        "sort_args": [],
+    }
+    script = "file://../scripts/bam/BamSampling.py"
+
+
+class BamSubsetByBed(Proc):
+    """Subset bam file by the regions in a bed file
+
+    Input:
+        bamfile: The bam file
+        bedfile: The bed file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        ncores: Number of cores to use
+        samtools: Path to samtools executable
+        tool: The tool to use, currently only "samtools" is supported
+        index: Whether to index the output bam file
+    """
+    input = "bamfile:file, bedfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}-subset.bam"
+    lang = config.lang.python
+    envs = {
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "tool": "samtools",
+        "index": True,
+    }
+    script = "file://../scripts/bam/BamSubsetByBed.py"
+
+
+class BamSort(Proc):
+    """Sort bam file
+
+    Input:
+        bamfile: The bam file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        tool (choice): The tool to use.
+            - samtools: Use `samtools`
+            - sambamba: Use `sambamba`
+        ncores (type=int): Number of cores to use
+        samtools: Path to samtools executable
+        sambamba: Path to sambamba executable
+        tmpdir: The temporary directory to use
+        byname (flag): Whether to sort by read name
+        index (flag): Whether to index the output bam file
+            The index file will be created in the same directory as the output
+            bam file
+        <more>: Other arguments passed to the sorting tool
+            See `samtools sort` or `sambamba sort`
+    """
+    input = "bamfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}.sorted.bam"
+    lang = config.lang.python
+    envs = {
+        "tool": "samtools",
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "sambamba": config.exe.sambamba,
+        "tmpdir": config.path.tmpdir,
+        "byname": False,
+        "index": True,
+    }
+    script = "file://../scripts/bam/BamSort.py"
+
+
+class SamtoolsView(Proc):
+    """View bam file using samtools, mostly used for filtering
+
+    This is a wrapper for `samtools view` command.
+    It will create a new bam file with the same name as the input bam file.
+
+    Input:
+        bamfile: The bam file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        ncores: Number of cores to use
+        samtools: Path to samtools executable
+        index: Whether to index the output bam file
+            Requires the input bam file to be sorted.
+        <more>: Other arguments passed to the view tool
+            See `samtools view` or `sambamba view`.
+    """
+    input = "bamfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}.bam"
+    lang = config.lang.python
+    envs = {
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "index": True,
+    }
+    script = "file://../scripts/bam/SamtoolsView.py"