biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/ns/vcf.py

@@ -335,6 +335,8 @@ class TruvariBench(Proc):
     """Run `truvari bench` to compare a VCF with CNV calls and
     base CNV standards
 
+    Requires truvari v4+
+
     See https://github.com/ACEnglish/truvari/wiki/bench
 
     Input:
@@ -358,7 +360,7 @@ class TruvariBench(Proc):
         "truvari": config.exe.truvari,
         "ref": config.ref.reffa,
         "refdist": 500,
-        "pctsim": 0.7,
+        "pctseq": 0.7,
         "pctsize": 0.7,
         "pctovl": 0.0,
         "typeignore": False,
@@ -402,7 +404,7 @@ class TruvariBenchSummary(Proc):
     output = "outdir:dir:truvari_bench.summary"
     lang = config.lang.rscript
     envs = {
-        "plots": ["call cnt", "base cnt", "precision", "recall", "f1"],
+        "plots": ["comp cnt", "base cnt", "precision", "recall", "f1"],
         "devpars": None,
     }
     script = "file://../scripts/vcf/TruvariBenchSummary.R"
@@ -414,6 +416,8 @@ class TruvariConsistency(Proc):
 
     See https://github.com/ACEnglish/truvari/wiki/consistency
 
+    Requires truvari v4+
+
     Input:
         vcfs: The vcf files with CNV calls
 
@@ -439,3 +443,233 @@ class TruvariConsistency(Proc):
     envs = {"truvari": config.exe.truvari, "heatmap": {}}
     script = "file://../scripts/vcf/TruvariConsistency.R"
     plugin_opts = {"report": "file://../reports/vcf/TruvariConsistency.svelte"}
+
+
+class BcftoolsAnnotate(Proc):
+    """Add or remove annotations from VCF files
+
+    See also: <https://samtools.github.io/bcftools/bcftools.html#annotate>
+
+    Input:
+        infile: The input VCF file
+        annfile: The annotation file.
+            Currently only VCF files are supported.
+
+    Output:
+        outfile: The VCF file with annotations added or removed.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile and annfile
+        annfile: The annotation file. If `in.annfile` is provided,
+            this is ignored
+        ncores (type=int): Number of cores (`--threads`) to use
+        columns (auto): Comma-separated or list of columns or tags to carry over from
+            the annotation file. Overrides `-c, --columns`
+        remove (auto): Remove the specified columns from the input file
+        header (list): Headers to be added
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools annotate`
+            See also <https://samtools.github.io/bcftools/bcftools.html#annotate>
+            Note that the underscore `_` will be replaced with dash `-` in the
+            argument name.
+    """
+    input = "infile:file, annfile:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "annfile": None,
+        "columns": [],
+        "remove": [],
+        "header": [],
+        "gz": True,
+        "index": True,
+        "ncores": config.misc.ncores,
+    }
+    script = "file://../scripts/vcf/BcftoolsAnnotate.py"
+
+
+class BcftoolsFilter(Proc):
+    """Apply fixed threshold filters to VCF files
+
+    Input:
+        infile: The input VCF file
+
+    Output:
+        outfile: The filtered VCF file. If the `in.infile` is gzipped, this is
+            gzipped as well.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        keep: Whether we should keep the filtered variants or not.
+            If True, the filtered variants will be kept in the output file, but
+            with a new FILTER.
+        includes: and
+        excludes: include/exclude only sites for which EXPRESSION is true.
+            See: <https://samtools.github.io/bcftools/bcftools.html#expressions>
+            If provided, `envs.include/exclude` will be ignored.
+            If `str`/`list` used, The filter names will be `Filter_<type>_<index>`.
+            A dict is used where keys are filter names and values are expressions
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools filter`
+            See also <https://samtools.github.io/bcftools/bcftools.html#filter>
+    """
+    input = "infile:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "keep": True,
+        "includes": None,
+        "excludes": None,
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsFilter.py"
+
+
+class BcftoolsSort(Proc):
+    """Sort VCF files using `bcftools sort`.
+
+    `bcftools sort` is used to sort VCF files by chromosome and position based on the
+    order of contigs in the header.
+
+    Here we provide a chrsize file to first sort the contigs in the header and then
+    sort the VCF file using `bcftools sort`.
+
+    Input:
+        infile: The input VCF file
+
+    Output:
+        outfile: The sorted VCF file.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        chrsize: The chromosome size file, from which the chromosome order is used
+            to sort the contig in the header first.
+            If not provided, `bcftools sort` will be used directly.
+        notfound (choice): What if the contig in the VCF file is not found in the
+            `chrsize` file.
+            - error: Report error
+            - remove: Remove the contig from the header.
+                Note that if there are records with the removed contig, an error will
+                be raised by `bcftools sort`
+            - start: Move the contig to the start of the contigs from `chrsize`
+            - end: Move the contig to the end of the contigs from `chrsize`
+        <more>: Other arguments for `bcftools sort`. For example `max_mem`.
+            See also <https://samtools.github.io/bcftools/bcftools.html#sort>
+    """
+    input = "infile:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "chrsize": config.ref.chrsize,
+        "notfound": "remove",
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsSort.py"
+
+
+class BcftoolsMerge(Proc):
+    """Merge multiple VCF files using `bcftools merge`.
+
+    Input:
+        infiles: The input VCF files
+
+    Output:
+        outfile: The merged VCF file.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools merge`.
+            See also <https://samtools.github.io/bcftools/bcftools.html#merge>
+    """
+    input = "infiles:files"
+    output = (
+        "outfile:file:{{in.infiles | first | stem | append: '_etc_merged'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsMerge.py"
+
+
+class BcftoolsView(Proc):
+    """View, subset and filter VCF files by position and filtering expression.
+
+    Also convert between VCF and BCF.
+
+    Input:
+        infile: The input VCF file
+        regions_file: The region file used to subset the input VCF file.
+        samples_file: The samples file used to subset the input VCF file.
+
+    Output:
+        outfile: The output VCF file.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        regions_file: The region file used to subset the input VCF file.
+            If `in.regions_file` is provided, this is ignored.
+        samples_file: The samples file used to subset the input VCF file.
+            If `in.samples_file` is provided, this is ignored.
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools view`.
+            See also https://samtools.github.io/bcftools/bcftools.html#view
+            Note that the underscore `_` will be replaced with dash `-` in the
+            argument name.
+    """
+    input = "infile:file, regions_file:file, samples_file:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "regions_file": None,
+        "samples_file": None,
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsView.py"

biopipen/ns/web.py

@@ -13,8 +13,11 @@ class Download(Proc):
         outfile: The file downloaded
 
     Envs:
-        tool: Which tool to use to download the data
-            wget, aria2c or python's urllib
+        tool (choice): Which tool to use to download the data
+            - wget: Use wget
+            - aria2c: Use aria2c
+            - urllib: Use python's urllib
+            - aria: Alias for aria2c
         wget: Path to wget
         aria2c: Path to aria2c
         args: The arguments to pass to the tool
@@ -28,8 +31,17 @@ class Download(Proc):
     """
     input = "url"
     output = (
+        # Need to replace http:// and https:// to avoid cloudpathlib.AnyPath to get
+        # the basename for something like "https://example.com/data/?file=datafile.txt"
+        # as data, but "?file=datafile.txt"
         "outfile:file:"
-        "{{in.url | basename | replace: '%2E', '.' | slugify: separator='.'}}"
+        """{{in.url
+            | replace: 'http://', ''
+            | replace: 'https://', ''
+            | basename
+            | url_decode
+            | slugify: separator='.', lowercase=False, regex_pattern='[^-a-zA-Z0-9_]+'
+        }}"""
     )
     lang = config.lang.python
     envs = {
@@ -43,7 +55,10 @@ class Download(Proc):
 
 
 class DownloadList(Proc):
-    """Download data from URLs in a file
+    """Download data from URLs in a file.
+
+    This does not work by iterating over the URLs in the file. The whole file is
+    passed to `wget` or `aria2c` at once.
 
     Input:
         urlfile: The file containing the URLs to download data from
@@ -52,8 +67,11 @@ class DownloadList(Proc):
         outdir: The directory containing the downloaded files
 
     Envs:
-        tool: Which tool to use to download the data
-            wget, aria2c or python's urllib
+        tool (choice): Which tool to use to download the data
+            - wget: Use wget
+            - aria2c: Use aria2c
+            - urllib: Use python's urllib
+            - aria: Alias for aria2c
         wget: Path to wget
         aria2c: Path to aria2c
         args: The arguments to pass to the tool
@@ -76,3 +94,76 @@ class DownloadList(Proc):
         "ncores": config.misc.ncores,
     }
     script = "file://../scripts/web/DownloadList.py"
+
+
+class GCloudStorageDownloadFile(Proc):
+    """Download file from Google Cloud Storage
+
+    Before using this, make sure you have the `gcloud` tool installed and
+    logged in with the appropriate credentials using `gcloud auth login`.
+
+    Also make sure you have [`google-crc32c`](https://pypi.org/project/google-crc32c/)
+    installed to verify the integrity of the downloaded files.
+
+    Input:
+        url: The URL to download data from.
+            It should be in the format gs://bucket/path/to/file
+
+    Output:
+        outfile: The file downloaded
+
+    Envs:
+        gcloud: Path to gcloud
+        args (ns): Other arguments to pass to the `gcloud storage cp` command
+            - do_not_decompress (flag): Do not decompress the file.
+            - <more>: More arguments to pass to the `gcloud storage cp` command
+                See `gcloud storage cp --help` for more information
+    """
+    input = "url:var"
+    output = "outfile:file:{{in.url | replace: 'gs://', '/' | basename}}"
+    lang = config.lang.python
+    envs = {
+        "gcloud": config.exe.gcloud,
+        "args": {"do_not_decompress": True},
+    }
+    script = "file://../scripts/web/GCloudStorageDownloadFile.py"
+
+
+class GCloudStorageDownloadBucket(Proc):
+    """Download all files from a Google Cloud Storage bucket
+
+    Before using this, make sure you have the `gcloud` tool installed and
+    logged in with the appropriate credentials using `gcloud auth login`.
+
+    Note that this will not use the `--recursive` flag of `gcloud storage cp`.
+    The files will be listed and downloaded one by one so that they can be parallelized.
+
+    Also make sure you have [`google-crc32c`](https://pypi.org/project/google-crc32c/)
+    installed to verify the integrity of the downloaded files.
+
+    Input:
+        url: The URL to download data from.
+            It should be in the format gs://bucket
+
+    Output:
+        outdir: The directory containing the downloaded files
+
+    Envs:
+        gcloud: Path to gcloud
+        keep_structure (flag): Keep the directory structure of the bucket
+        ncores (type=int): The number of cores to use to download the files in parallel
+        args (ns): Other arguments to pass to the `gcloud storage cp` command
+            - do_not_decompress (flag): Do not decompress the file.
+            - <more>: More arguments to pass to the `gcloud storage cp` command
+                See `gcloud storage cp --help` for more information
+    """
+    input = "url:var"
+    output = "outdir:dir:{{in.url | replace: 'gs://', ''}}"
+    lang = config.lang.python
+    envs = {
+        "gcloud": config.exe.gcloud,
+        "keep_structure": True,
+        "ncores": config.misc.ncores,
+        "args": {"do_not_decompress": True},
+    }
+    script = "file://../scripts/web/GCloudStorageDownloadBucket.py"

biopipen/reports/bam/CNVpytor.svelte

@@ -4,19 +4,16 @@
     import { Tabs, Tab, TabContent } from "$ccs";
 </script>
 
-{% for case in envs.cases %}
-<h1>{{case}}</h1>
-
-{%  for binsize in envs.cases[case].binsizes %}
-<h2>Binsize: {{binsize}}</h2>
+{% for binsize in envs.binsizes %}
+<h1>Binsize: {{binsize}}</h1>
 
 {% from_ os.path import join, basename %}
 {% assign manplots = [] %}
 {% assign circplots = [] %}
 {% assign samples = [] %}
 {% for job in jobs %}
-{%  set manplot = job.out.outdir | joinpaths: case, "manhattan."+str(binsize)+".*.png" | glob %}
-{%  set circplot = job.out.outdir | joinpaths: case, "circular."+str(binsize)+".*.png" | glob %}
+{%  set manplot = job.out.outdir | glob: "manhattan."+str(binsize)+".*.png" %}
+{%  set circplot = job.out.outdir | glob: "circular."+str(binsize)+".*.png" %}
 {%  set _ = manplots.append(manplot[0]) %}
 {%  if len(circplot) > 0 %}
 {%      set _ = circplots.append(circplot[0]) %}
@@ -45,6 +42,4 @@
     </div>
 </Tabs>
 
-{%  endfor %}
-
 {% endfor %}

biopipen/reports/cellranger/CellRangerCount.svelte

@@ -0,0 +1,18 @@
+{% from "utils/misc.liq" import report_jobs, table_of_images -%}
+<script>
+    import { Iframe } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    <Iframe
+        src="{{job.out.outdir}}/outs/web_summary.html"
+        width="100%"
+        frameborder="0"
+        style="min-height: 60vh" />
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+    <h1>{{job.out.outdir | basename | escape}}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/cellranger/CellRangerSummary.svelte

@@ -0,0 +1,16 @@
+{% from "utils/misc.liq" import report_jobs -%}
+
+<script>
+    import { Image, DataTable, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {{ job | render_job: h=h }}
+{%- endmacro -%}
+
+
+{%- macro head_job(job) -%}
+    <h1>{{job.out.outdir | stem | escape}}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/cellranger/CellRangerVdj.svelte

@@ -0,0 +1,18 @@
+{% from "utils/misc.liq" import report_jobs, table_of_images -%}
+<script>
+    import { Iframe } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    <Iframe
+        src="{{job.out.outdir}}/outs/web_summary.html"
+        width="100%"
+        frameborder="0"
+        style="min-height: 60vh" />
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+    <h1>{{job.out.outdir | basename | escape}}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/cnvkit/CNVkitDiagram.svelte

@@ -4,7 +4,7 @@
 </script>
 
 {%- macro report_job(job, h=1) -%}
-    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+    {%- for pngfile in job.out.outdir | glob: '*.png' -%}
     <h{{h+1}}>{{pngfile | stem0 | title}}</h{{h+1}}>
     <Image src="{{pngfile}}" />
     {%- endfor -%}

biopipen/reports/cnvkit/CNVkitHeatmap.svelte

@@ -4,7 +4,7 @@
 </script>
 
 {%- macro report_job(job, h=1) -%}
-    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+    {%- for pngfile in job.out.outdir | glob: '*.png' -%}
     <h{{h+1}}>{{pngfile | stem0 | title}}</h{{h+1}}>
     <Image src="{{pngfile}}" />
     {%- endfor -%}

biopipen/reports/cnvkit/CNVkitScatter.svelte

@@ -4,7 +4,7 @@
 </script>
 
 {%- macro report_job(job, h=1) -%}
-    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+    {%- for pngfile in job.out.outdir | glob: '*.png' -%}
     <h{{h+1}}>{{pngfile | stem0 | title}}</h{{h+1}}>
     <Image src="{{pngfile}}" />
     {%- endfor -%}

biopipen/reports/common.svelte

@@ -0,0 +1,15 @@
+{% from "utils/misc.liq" import report_jobs, table_of_images -%}
+<script>
+    import { Image, DataTable, Descr } from "$libs";
+    import { Tabs, Tab, TabContent, UnorderedList, ListItem, InlineNotification } from "$ccs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {{ job | render_job: h=h }}
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+    <h1>{{job.in | attr: "values" | call | first | stem0 | escape}}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/protein/ProdigySummary.svelte

@@ -0,0 +1,16 @@
+{% from "utils/misc.liq" import report_jobs -%}
+
+<script>
+    import { Image, DataTable, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {{ job | render_job: h=h }}
+{%- endmacro -%}
+
+
+{%- macro head_job(job) -%}
+    <h1>{{job.out.outdir | stem | escape}}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/scrna/CellsDistribution.svelte

@@ -1,50 +1,15 @@
 {% from "utils/misc.liq" import report_jobs, table_of_images -%}
 <script>
-    import { Image } from "$libs";
+    import { Image, DataTable, Descr } from "$libs";
+    import { Tabs, Tab, TabContent } from "$ccs";
 </script>
 
 {%- macro report_job(job, h=1) -%}
-    {%- set secdirs = job.out.outdir | glob: "*" -%}
-    {%- if len(secdirs) == 1 -%}
-        {%- set secname = secdirs | first | basename -%}
-        {%- if secdirs[0] | joinpaths: "venn.png" | exists -%}
-            {%- if secname == "DEFAULT" -%}
-                <h{{h}}>Case overlapping</h{{h}}>
-            {%- else -%}
-                <h{{h}}>{{ secname | escape }} - Case overlapping</h{{h}}>
-            {%- endif -%}
-            {{ table_of_images(
-                [joinpaths(secdirs[0], "venn.png"), joinpaths(secdirs[0], "upset.png")],
-                ["Venn plot", "Upset plot"]) }}
-        {%- endif -%}
-        {%- for plotfile in secdirs[0] | glob: "case-*.png" -%}
-            {%- if secname == "DEFAULT" -%}
-                <h{{h}}>{{ plotfile | stem | escape }}</h{{h}}>
-            {%- else -%}
-                <h{{h}}>{{ secname | escape }} - {{ plotfile | stem | escape }}</h{{h}}>
-            {%- endif -%}
-            <Image src={{plotfile | quote}} />
-        {%- endfor -%}
-    {%- else -%}
-        {%- for secdir in secdirs -%}
-            {%- set sec = secdir | basename -%}
-            <h{{h}}>{{sec | escape}}</h{{h}}>
-            {%- if secdir | joinpaths: "venn.png" | exists -%}
-                <h{{h+1}}>Case overlapping</h{{h+1}}>
-                {{ table_of_images(
-                    [joinpaths(secdir, "venn.png"), joinpaths(secdir, "upset.png")],
-                    ["Venn plot", "Upset plot"]) }}
-            {%- endif -%}
-            {%- for plotfile in secdir | glob: "case-*.png" -%}
-                <h{{h+1}}>{{ plotfile | stem }}</h{{h+1}}>
-                <Image src={{plotfile | quote}} />
-            {%- endfor -%}
-        {%- endfor -%}
-    {%- endif -%}
+    {{ job | render_job: h=h }}
 {%- endmacro -%}
 
 {%- macro head_job(job) -%}
-<h1>{{job.in.srtobj | stem0 | escape}}</h1>
+    <h1>{{job.in.srtobj | stem0 | escape}}</h1>
 {%- endmacro -%}
 
 {{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/scrna/DimPlots.svelte

@@ -4,7 +4,7 @@
 </script>
 
 {%- macro report_job(job, h=1) -%}
-{% set images = job.out.outdir | joinpaths: "*.png" | glob %}
+{% set images = job.out.outdir | glob: "*.png" %}
 {{ table_of_images(images) }}
 {%- endmacro -%}