biopipen 0.27.9__tar.gz → 1.1.4__tar.gz

biopipen-1.1.4/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: biopipen
+Version: 1.1.4
+Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
+License: MIT
+Author: pwwang
+Author-email: pwwang@pwwang.com
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: log2file
+Provides-Extra: runinfo
+Requires-Dist: datar[pandas] (>=0.15.8,<0.16.0)
+Requires-Dist: pipen-board[report] (>=1.1,<2.0)
+Requires-Dist: pipen-cli-run (>=1.0,<2.0)
+Requires-Dist: pipen-deprecated (>=1.0,<2.0)
+Requires-Dist: pipen-filters (>=1.1.2,<2.0.0)
+Requires-Dist: pipen-log2file (>=1.1,<2.0) ; extra == "log2file"
+Requires-Dist: pipen-poplog (>=1.1,<2.0)
+Requires-Dist: pipen-runinfo (>=1.1,<2.0) ; extra == "runinfo"
+Requires-Dist: pipen-verbose (>=1.1,<2.0)

biopipen-1.1.4/biopipen/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.1.4"

{biopipen-0.27.9 → biopipen-1.1.4}/biopipen/core/config.toml

@@ -1,11 +1,17 @@
 # Executables or binaries
 [exe]
+# BeEM: https://github.com/kad-ecoli/BeEM
+beem = "BeEM"
 # bedtools to handle bed files
 bedtools = "bedtools"
 # bcftools to handle bcf/vcf files
 bcftools = "bcftools"
+# calculate_rmsd: https://github.com/charnley/rmsd
+calculate_rmsd = "calculate_rmsd"
 # cellranger
 cellranger = "cellranger"
+# cellsnp-lite
+cellsnp_lite = "cellsnp-lite"
 # Control-FREEC to call cnvs
 freec = "freec"
 # liftover coordinates across genomes
@@ -13,6 +19,8 @@ liftover = "liftOver"
 # gatk, installed via conda
 gatk = "gatk"
 gatk4 = "gatk"
+# google cloud sdk
+gcloud = "gcloud"
 # vdjtools, installed via conda
 vdjtools = "vdjtools"
 # cnvkit.py
@@ -23,12 +31,20 @@ cnvpytor = "cnvpytor"
 cnvnator2vcf = "cnvnator2VCF.pl"
 # convert
 convert = "convert"
+# fimo from meme
+fimo = "fimo"
+# MAXIT: https://sw-tools.rcsb.org/apps/MAXIT/
+maxit = "maxit"
+# MQuad: https://github.com/single-cell-genetics/MQuad
+mquad = "mquad"
 # wget
 wget = "wget"
 # aria2c
 aria2c = "aria2c"
 # plink
 plink = "plink"
+# plink2
+plink2 = "plink2"
 # tabix
 tabix = "tabix"
 # sambamba
@@ -86,6 +102,10 @@ genome = ""
 # Database file for scType
 # https://github.com/IanevskiAleksandr/sc-type/
 sctype_db = ""
+# TF Motif database
+tf_motifdb = ""
+# TF motif pairs
+tf_motifs = ""
 
 [misc]
 # Number of cores used for each job

{biopipen-0.27.9 → biopipen-1.1.4}/biopipen/core/filters.py

@@ -1,13 +1,16 @@
 """Additional filters for pipen"""
 from __future__ import annotations
 
+import re
 import shlex
 from pathlib import Path
 from typing import Any, List, Mapping
 
-from argx import Namespace
+from argx import Namespace  # pyright: ignore[reportPrivateImportUsage]
 from liquid.filters.manager import FilterManager
-from pipen_report.filters import register_component, render_ui, _tag
+from pipen_report.filters import register_component, _tag
+
+# from .defaults import BIOPIPEN_DIR
 
 filtermanager = FilterManager()
 
@@ -15,7 +18,7 @@ filtermanager = FilterManager()
 @filtermanager.register
 def dict_to_cli_args(
     dic: Mapping[str, Any],
-    exclude: List[str] = None,
+    exclude: List[str] | None = None,
     prefix: str | None = None,
     sep: str | None = " ",
     dup_key: bool = True,
@@ -28,7 +31,7 @@ def dict_to_cli_args(
 
     Args:
         dic: The dict to convert
-        exclude: The keys to exclude
+        exclude: The keys to exclude before conversion (e.g. dashify)
         prefix: The prefix of the keys after conversion
             Defaults to `None`, mean `-` for short keys and `--` for long keys
         sep: The separator between key and value
@@ -118,7 +121,7 @@ def dict_to_cli_args(
 def r(
     obj: Any,
     ignoreintkey: bool = True,
-    todot: str = None,
+    todot: str | None = None,
     sortkeys: bool = False,
     skip: int = 0,
     _i: int = 0,
@@ -169,8 +172,10 @@ def r(
             return "TRUE"
         if obj.upper() == "FALSE":
             return "FALSE"
-        if obj.upper() == "NA" or obj.upper() == "NULL":
+        if obj.upper() == "NA" or obj.upper() == "NULL" or obj == "None":
             return obj.upper()
+        if re.match(r"^\d+:\d+$", obj):
+            return obj
         if obj.startswith("r:") or obj.startswith("R:"):
             return str(obj)[2:]
         return repr(str(obj))
@@ -221,69 +226,31 @@ def r(
     return repr(obj)
 
 
-@register_component("fgsea")
-def _render_fgsea(
-    cont: Mapping[str, Any],
-    job: Mapping[str, Any],
-    level: int,
-    na_arg: str = "10",
-) -> str:
-    """Render fgsea report"""
-    # cont["dir"] is required
-    n_pathways = int(na_arg)
-    pathways = []
-    with Path(cont["dir"]).joinpath("fgsea.txt").open() as f:
-        next(f)  # skip header
-        for line in f:
-            items = line.strip().split("\t")
-            pathways.append((items[0], items[-1]))
-            if len(pathways) >= n_pathways:
-                break
-
-    components = [
-        # Summary
-        {
-            "title": "Enrichment Analysis Summary",
-            "ui": "tabs",
-            "contents": [
-                {
-                    "title": "Plot",
-                    "ui": "flat",
-                    "contents": [
-                        {
-                            "kind": "image",
-                            "src": str(Path(cont["dir"]).joinpath("gsea_table.png")),
-                        }
-                    ],
-                },
-                {
-                    "title": "Table",
-                    "ui": "flat",
-                    "contents": [
-                        {
-                            "kind": "table",
-                            "src": str(Path(cont["dir"]).joinpath("fgsea.txt")),
-                            "data": {"excluded": {"slug"}},
-                        }
-                    ],
-                },
-            ]
-        },
-        # Pathways
-        {
-            "title": f"Enriched Pathways (Top {n_pathways})",
-            "ui": "table_of_images",
-            "contents": [
-                {
-                    "src": str(Path(cont["dir"]) / f"fgsea_{slug}.png"),
-                    "title": pw,
-                }
-                for pw, slug in pathways
-            ]
-        },
-    ]
-
-    return render_ui(components, "accordion", job, level)
+@filtermanager.register
+def source_r(path: str | Path, chdir: bool = False) -> str:
+    """Source an R script.
+
+    In addition to generating `source(path)`, we also include the mtime for the script
+    to trigger the job not cached when the script is updated.
+
+    If your process is used in a cloud environment, it is recommended to
+    use the `read` filter to load the script content instead of sourcing it using
+    the `source` function in R to void the path issue (path could be different
+    in different environments).
+
+    Args:
+        path: The path to the R script
+
+    Returns:
+        The R code to source the script
+    """
+    path = Path(path)
+    mtime = int(path.stat().st_mtime)
+    return (
+        f"# Last modified: {mtime}\n"
+        # f"biopipen_dir = {r(BIOPIPEN_DIR)}\n"
+        f"source('{path}', chdir = {r(chdir)})"
+    )
 
 
 @register_component("pdf")
@@ -313,72 +280,3 @@ def _render_gsea(
     """Render gsea report"""
     # cont["dir"] is required
     raise NotImplementedError()
-
-
-@register_component("enrichr")
-def _render_enrichr(
-    cont: Mapping[str, Any],
-    job: Mapping[str, Any],
-    level: int,
-) -> str:
-    """Render enrichr report"""
-    # cont["dir"] is required
-    dbs = [sumfile.stem[8:] for sumfile in Path(cont["dir"]).glob("Enrichr-*.txt")]
-    components = []
-
-    for db in dbs:
-        enrichr_plot = Path(cont["dir"]).joinpath(f"Enrichr-{db}.png")
-        if enrichr_plot.exists():
-            components.append(
-                {
-                    "title": db,
-                    "ui": "tabs",
-                    "contents": [
-                        {
-                            "title": "Plot",
-                            "ui": "flat",
-                            "contents": [
-                                {
-                                    "kind": "image",
-                                    "src": str(
-                                        Path(cont["dir"]).joinpath(f"Enrichr-{db}.png")
-                                    ),
-                                }
-                            ],
-                        },
-                        {
-                            "title": "Table",
-                            "ui": "flat",
-                            "contents": [
-                                {
-                                    "kind": "table",
-                                    "src": str(
-                                        Path(cont["dir"]).joinpath(f"Enrichr-{db}.txt")
-                                    ),
-                                }
-                            ],
-                        },
-                    ],
-                }
-            )
-        else:
-            components.append(
-                {
-                    "title": db,
-                    "ui": "tabs",
-                    "contents": [
-                        {
-                            "title": "Error",
-                            "ui": "flat",
-                            "contents": [
-                                {
-                                    "kind": "error",
-                                    "content": "No enriched terms found.",
-                                }
-                            ],
-                        },
-                    ],
-                }
-            )
-
-    return render_ui(components, "accordion", job, level)

{biopipen-0.27.9 → biopipen-1.1.4}/biopipen/core/proc.py

@@ -1,7 +1,9 @@
 """Provides a base class for the processes to subclass"""
-from diot import Diot
+from __future__ import annotations
+
+from diot import Diot  # type: ignore
 from liquid.defaults import SEARCH_PATHS
-from pipen import Proc as PipenProc
+from pipen import Proc as PipenProc  # type: ignore
 from pipen_filters.filters import FILTERS
 
 from .filters import filtermanager
@@ -23,7 +25,7 @@ class Proc(PipenProc):
     template_opts = {
         "globals": {**FILTERS, "biopipen_dir": str(BIOPIPEN_DIR)},
         "filters": {**FILTERS, **filtermanager.filters},
-        "search_paths": SEARCH_PATHS + [str(REPORT_DIR)],
+        "search_paths": SEARCH_PATHS + [str(REPORT_DIR)],  # type: ignore
     }
 
     plugin_opts = {

{biopipen-0.27.9 → biopipen-1.1.4}/biopipen/core/testing.py

@@ -6,7 +6,8 @@ from pathlib import Path
 from pipen import Pipen
 
 TESTING_INDEX_INIT = 1
-TESTING_PARENT_DIR = Path(tempfile.gettempdir())
+TESTING_PARENT_DIR = Path(__file__).parent.parent.parent.joinpath("tests", "running")
+TESTING_PARENT_DIR.mkdir(parents=True, exist_ok=True)
 TESTING_DIR = str(TESTING_PARENT_DIR.joinpath("biopipen-tests-%(index)s"))
 RSCRIPT_DIR = TESTING_PARENT_DIR.joinpath("biopipen-tests-rscripts")
 RSCRIPT_DIR.mkdir(exist_ok=True)
@@ -44,12 +45,19 @@ def get_pipeline(testfile, loglevel="debug", enable_report=False, **kwargs):
     """Get a pipeline for a test file"""
     name, workdir, outdir = _get_test_dirs(testfile, False)
     report_plugin_prefix = "+" if enable_report else "-"
+    plugins = kwargs.pop("plugins", [])
+    if any("report" in p for p in plugins if isinstance(p, str)):
+        raise ValueError(
+            "Do not pass `report` plugin to `get_pipeline(plugins=[...])`, "
+            "use `enable_report` instead."
+        )
+    plugins.append(f"{report_plugin_prefix}report")
     kws = {
         "name": name,
         "workdir": workdir,
         "outdir": outdir,
         "loglevel": loglevel,
-        "plugins": [f"{report_plugin_prefix}report"],
+        "plugins": plugins,
     }
     kws.update(kwargs)
     return Pipen(**kws)
@@ -96,7 +104,12 @@ def r_test(mem: callable) -> callable:
         )
         rcode = f"{expect}\n\n{rcode}\n\ncat('PASSED')\n"
         if source is not None:
-            rcode = f'suppressWarnings(source("{self.SOURCE_FILE}"))\n\n{rcode}'
+            if not isinstance(source, (list, tuple)):
+                source = [source]
+
+            libs = "\n".join([f"suppressWarnings(source('{s}'))" for s in source])
+            rcode = f'{libs}\n\n{rcode}'
+
         out = _run_rcode(rcode)
         self.assertEqual(
             out,

{biopipen-0.27.9 → biopipen-1.1.4}/biopipen/ns/bam.py

@@ -4,6 +4,9 @@ from ..core.proc import Proc
 from ..core.config import config
 
 
+# +-------------------------------------------------------------------+
+# | CNV callers                                                       |
+# +-------------------------------------------------------------------+
 class CNVpytor(Proc):
     """Detect CNV using CNVpytor
 
@@ -17,7 +20,6 @@ class CNVpytor(Proc):
 
     Envs:
         cnvpytor: Path to cnvpytor
-        cnvnator2vcf: Path to CNVnator2VCF.pl to convert the result to VCF file
         samtools: Path to samtools, used to index bam file in case it's not
         ncores: Number of cores to use (`-j` for cnvpytor)
         refdir: The directory containing the fasta file for each chromosome
@@ -27,21 +29,19 @@ class CNVpytor(Proc):
         binsizes: The binsizes
         snp: How to read snp data
         filters: The filters to filter the result
-            See - https://github.com/abyzovlab/CNVpytor/blob/master
-            /GettingStarted.md#predicting-cnv-regions
+            See - https://github.com/abyzovlab/CNVpytor/blob/master/GettingStarted.md#predicting-cnv-regions
         mask_snps: Whether mask 1000 Genome snps
         baf_nomask: Do not use P mask in BAF histograms
 
     Requires:
         cnvpytor:
            - check: {{proc.envs.cnvpytor}} --version
-    """
+    """  # noqa: E501
     input = "bamfile:file, snpfile:file"
     output = "outdir:dir:{{in.bamfile | stem}}.cnvpytor"
     lang = config.lang.python
     envs = {
         "cnvpytor": config.exe.cnvpytor,
-        "cnvnator2vcf": config.exe.cnvnator2vcf,
         "samtools": config.exe.samtools,
         "ncores": config.misc.ncores,
         "refdir": config.ref.refdir,
@@ -152,7 +152,7 @@ class CNAClinic(Proc):
             A list of sample names
             A float number (0 < x <= 1), the fraction of samples to use
             A integer number (x > 1), the number of samples to use
-        binsize: Directly use this binsize for CNAClinic, in kbp.
+        binsize: Directly use this binsize for CNAClinic, in bp.
         genome: The genome assembly
         run_args: The arguments for CNAClinic::runSegmentation
         plot_args: The arguments for CNAClinic::plotSampleData
@@ -183,6 +183,9 @@ class CNAClinic(Proc):
     }
 
 
+# +-------------------------------------------------------------------+
+# | Bam processing tools                                              |
+# +-------------------------------------------------------------------+
 class BamSplitChroms(Proc):
     """Split bam file by chromosomes
 
@@ -262,3 +265,142 @@ class BamMerge(Proc):
         "sort_args": [],
     }
     script = "file://../scripts/bam/BamMerge.py"
+
+
+class BamSampling(Proc):
+    """Keeping only a fraction of read pairs from a bam file
+
+    Input:
+        bamfile: The bam file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        ncores: Number of cores to use
+        samtools: Path to samtools executable
+        tool: The tool to use, currently only "samtools" is supported
+        fraction (type=float): The fraction of reads to keep.
+            If `0 < fraction <= 1`, it's the fraction of reads to keep.
+            If `fraction > 1`, it's the number of reads to keep.
+            Note that when fraction > 1, you may not get the exact number
+            of reads specified but a close number.
+        seed: The seed for random number generator
+        index: Whether to index the output bam file
+        sort: Whether to sort the output bam file
+        sort_args: The arguments for sorting bam file using `samtools sort`.
+            These keys are not allowed: `-o`, `-@`,
+            and `--threads`, as they are managed by the script.
+    """
+    input = "bamfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}.sampled{{envs.fraction}}.bam"
+    lang = config.lang.python
+    envs = {
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "tool": "samtools",
+        "fraction": None,
+        "seed": 8525,
+        "index": True,
+        "sort": True,
+        "sort_args": [],
+    }
+    script = "file://../scripts/bam/BamSampling.py"
+
+
+class BamSubsetByBed(Proc):
+    """Subset bam file by the regions in a bed file
+
+    Input:
+        bamfile: The bam file
+        bedfile: The bed file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        ncores: Number of cores to use
+        samtools: Path to samtools executable
+        tool: The tool to use, currently only "samtools" is supported
+        index: Whether to index the output bam file
+    """
+    input = "bamfile:file, bedfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}-subset.bam"
+    lang = config.lang.python
+    envs = {
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "tool": "samtools",
+        "index": True,
+    }
+    script = "file://../scripts/bam/BamSubsetByBed.py"
+
+
+class BamSort(Proc):
+    """Sort bam file
+
+    Input:
+        bamfile: The bam file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        tool (choice): The tool to use.
+            - samtools: Use `samtools`
+            - sambamba: Use `sambamba`
+        ncores (type=int): Number of cores to use
+        samtools: Path to samtools executable
+        sambamba: Path to sambamba executable
+        tmpdir: The temporary directory to use
+        byname (flag): Whether to sort by read name
+        index (flag): Whether to index the output bam file
+            The index file will be created in the same directory as the output
+            bam file
+        <more>: Other arguments passed to the sorting tool
+            See `samtools sort` or `sambamba sort`
+    """
+    input = "bamfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}.sorted.bam"
+    lang = config.lang.python
+    envs = {
+        "tool": "samtools",
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "sambamba": config.exe.sambamba,
+        "tmpdir": config.path.tmpdir,
+        "byname": False,
+        "index": True,
+    }
+    script = "file://../scripts/bam/BamSort.py"
+
+
+class SamtoolsView(Proc):
+    """View bam file using samtools, mostly used for filtering
+
+    This is a wrapper for `samtools view` command.
+    It will create a new bam file with the same name as the input bam file.
+
+    Input:
+        bamfile: The bam file
+
+    Output:
+        outfile: The output bam file
+
+    Envs:
+        ncores: Number of cores to use
+        samtools: Path to samtools executable
+        index: Whether to index the output bam file
+            Requires the input bam file to be sorted.
+        <more>: Other arguments passed to the view tool
+            See `samtools view` or `sambamba view`.
+    """
+    input = "bamfile:file"
+    output = "outfile:file:{{in.bamfile | stem}}.bam"
+    lang = config.lang.python
+    envs = {
+        "ncores": config.misc.ncores,
+        "samtools": config.exe.samtools,
+        "index": True,
+    }
+    script = "file://../scripts/bam/SamtoolsView.py"

{biopipen-0.27.9 → biopipen-1.1.4}/biopipen/ns/bed.py

@@ -163,3 +163,78 @@ class BedtoolsMerge(Proc):
         "bedtools": config.exe.bedtools,
     }
     script = "file://../scripts/bed/BedtoolsMerge.py"
+
+
+class BedtoolsIntersect(Proc):
+    """Find the intersection of two BED files, using `bedtools intersect`
+
+    See <https://bedtools.readthedocs.io/en/latest/content/tools/intersect.html>
+
+    Input:
+        afile: The first BED file
+        bfile: The second BED file
+
+    Output:
+        outfile: The output BED file
+
+    Envs:
+        bedtools: The path to bedtools
+        sort: Sort `afile` and `bfile` before intersecting.
+            By default, `-sorted` is used, assuming the input files are sorted.
+            If error occurs, try to set `sort` to `True`.
+        chrsize: Alias for `g` in `bedtools intersect`.
+        postcmd: The command to be executed for the output file after intersecting.
+            You can use `$infile`, `$outfile`, and `$outdir` to refer to the input,
+            output, and output directory, respectively.
+        <more>: Other options to be passed to `bedtools intersect`
+    """  # noqa: E501
+    input = "afile:file", "bfile:file"
+    output = "outfile:file:{{in.afile | stem0}}_{{in.bfile | stem0}}-intersect.bt"
+    lang = config.lang.python
+    envs = {
+        "bedtools": config.exe.bedtools,
+        "sort": False,
+        "chrsize": config.ref.chrsize,
+        "postcmd": None,
+    }
+    script = "file://../scripts/bed/BedtoolsIntersect.py"
+
+
+class BedtoolsMakeWindows(Proc):
+    """Make windows from a BED file or genome size file, using `bedtools makewindows`.
+
+    Input:
+        infile: The input BED file or a genome size file
+            Type will be detected by the number of columns in the file.
+            If it has 3+ columns, it is treated as a BED file, otherwise
+            a genome size file.
+
+    Output:
+        outfile: The output BED file
+
+    Envs:
+        bedtools: The path to bedtools
+        window (type=int): The size of the windows
+        step (type=int): The step size of the windows
+        nwin (type=int): The number of windows to be generated
+            Exclusive with `window` and `step`.
+            Either `nwin` or `window` and `step` should be provided.
+        reverse (flag): Reverse numbering of windows in the output
+        name (choice): How to name the generated windows/regions
+            - none: Do not add any name
+            - src: Use the source interval's name
+            - winnum: Use the window number
+            - srcwinnum: Use the source interval's name and window number
+    """  # noqa: E501
+    input = "infile:file"
+    output = "outfile:file:{{in.infile | stem}}_windows.bed"
+    lang = config.lang.python
+    envs = {
+        "bedtools": config.exe.bedtools,
+        "window": None,
+        "step": None,
+        "nwin": None,
+        "reverse": False,
+        "name": "none",
+    }
+    script = "file://../scripts/bed/BedtoolsMakeWindows.py"