biopipen 0.28.1__py3-none-any.whl → 0.29.1__py3-none-any.whl

biopipen/ns/stats.py

@@ -73,11 +73,103 @@ class ChowTest(Proc):
     script = "file://../scripts/stats/ChowTest.R"
 
 
+class Mediation(Proc):
+    """Mediation analysis.
+
+    The flowchart of mediation analysis:
+
+    ![Mediation Analysis](https://library.virginia.edu/sites/default/files/inline-images/mediation_flowchart-1.png)
+
+    Reference:
+        - <https://library.virginia.edu/data/articles/introduction-to-mediation-analysis>
+        - <https://en.wikipedia.org/wiki/Mediation_(statistics)>
+        - <https://tilburgsciencehub.com/topics/analyze/regression/linear-regression/mediation-analysis/>
+        - <https://ademos.people.uic.edu/Chapter14.html>
+
+    Input:
+        infile: The input data file. The rows are samples and the columns are
+            features. It must be tab-delimited.
+            ```
+            Sample   F1   F2   F3   ...   Fn
+            S1       1.2  3.4  5.6        7.8
+            S2       2.3  4.5  6.7        8.9
+            ...
+            Sm       5.6  7.8  9.0        1.2
+            ```
+        fmlfile: The formula file.
+            ```
+            Case   M   Y   X   Cov     Model_M    Model_Y
+            Case1  F1  F2  F3  F4,F5   glm        lm
+            ...
+            ```
+            Where Y is the outcome variable, X is the predictor variable, M is the
+            mediator variable, and Case is the case name. Model_M and Model_Y are the
+            models for M and Y, respectively.
+            `envs.cases` will be ignored if this is provided.
+
+    Output:
+        outfile: The output file.
+            Columns to help understand the results:
+            Total Effect: a total effect of X on Y (without M) (`Y ~ X`).
+            ADE: A Direct Effect of X on Y after taking into account a mediation effect of M (`Y ~ X + M`).
+            ACME: The Mediation Effect, the total effect minus the direct effect,
+            which equals to a product of a coefficient of X in the second step and a coefficient of M in the last step.
+            The goal of mediation analysis is to obtain this indirect effect and see if it's statistically significant.
+
+    Envs:
+        ncores (type=int): Number of cores to use for parallelization for cases.
+        sims (type=int): Number of Monte Carlo draws for nonparametric bootstrap or quasi-Bayesian approximation.
+            Will be passed to `mediation::mediate` function.
+        args (ns): Other arguments passed to `mediation::mediate` function.
+            - <more>: More arguments passed to `mediation::mediate` function.
+                See: <https://rdrr.io/cran/mediation/man/mediate.html>
+        padj (choice): The method for (ACME) p-value adjustment.
+            - none: No p-value adjustment (no Padj column in outfile).
+            - holm: Holm-Bonferroni method.
+            - hochberg: Hochberg method.
+            - hommel: Hommel method.
+            - bonferroni: Bonferroni method.
+            - BH: Benjamini-Hochberg method.
+            - BY: Benjamini-Yekutieli method.
+            - fdr: FDR correction method.
+        cases (type=json): The cases for mediation analysis.
+            Ignored if `in.fmlfile` is provided.
+            A json/dict with case names as keys and values as a dict of M, Y, X, Cov, Model_M, Model_Y.
+            For example:
+            ```json
+            {
+                "Case1": {
+                    "M": "F1",
+                    "Y": "F2",
+                    "X": "F3",
+                    "Cov": "F4,F5",
+                    "Model_M": "glm",
+                    "Model_Y": "lm"
+                },
+                ...
+            }
+            ```
+        transpose_input (flag): Whether to transpose the input file.
+    """  # noqa: E501
+    input = "infile:file, fmlfile:file"
+    output = "outfile:file:{{in.infile | stem}}.mediation.txt"
+    lang = config.lang.rscript
+    envs = {
+        "ncores": config.misc.ncores,
+        "sims": 1000,
+        "args": {},
+        "padj": "none",
+        "cases": {},
+        "transpose_input": False,
+    }
+    script = "file://../scripts/stats/Mediation.R"
+
+
 class LiquidAssoc(Proc):
     """Liquid association tests.
 
     See Also https://github.com/gundt/fastLiquidAssociation
-    Requieres https://github.com/pwwang/fastLiquidAssociation
+    Requires https://github.com/pwwang/fastLiquidAssociation
 
     Input:
         infile: The input data file. The rows are samples and the columns are
@@ -275,8 +367,10 @@ class MetaPvalue(Proc):
 
     Envs:
         id_cols: The column names used in all `in.infiles` as ID columns. Multiple
-            columns can be specified by comma-seperated values. For example, `ID1,ID2`.
-            If `id_expr` is specified, this should be a single column name for the new
+            columns can be specified by comma-seperated values. For example, `ID1,ID2`,
+            where `ID1` is the ID column in the first file and `ID2` is the ID column
+            in the second file.
+            If `id_exprs` is specified, this should be a single column name for the new
             ID column in each `in.infiles` and the final `out.outfile`.
         id_exprs: The R expressions for each `in.infiles` to get ID column(s).
         pval_cols: The column names used in all `in.infiles` as p-value columns.
@@ -294,6 +388,8 @@ class MetaPvalue(Proc):
             - votep: Vote counting method.
             - wilkinsonp: Wilkinson's method.
             - invchisq: Inverse chi-square method.
+        keep_single (flag): Whether to keep the original p-value when there is only one
+            p-value.
         na: The method to handle NA values. -1 to skip the record. Otherwise NA
             will be replaced by the given value.
         padj (choice): The method for p-value adjustment.
@@ -315,6 +411,74 @@ class MetaPvalue(Proc):
         "pval_cols": None,
         "method": "fisher",
         "na": -1,
+        "keep_single": True,
         "padj": "none",
     }
     script = "file://../scripts/stats/MetaPvalue.R"
+
+
+class MetaPvalue1(Proc):
+    """Calulation of meta p-values.
+
+    Unlike `MetaPvalue`, this process only accepts one input file.
+
+    The p-values will be grouped by the ID columns and combined by the selected method.
+
+    Input:
+        infile: The input file.
+            The file is a tab-delimited file with multiple
+            columns. There should be ID column(s) to group the rows where
+            p-value column(s) to be combined.
+
+    Output:
+        outfile: The output file. It is a tab-delimited file with the first column as
+            the ID and the second column as the combined p-value.
+            ```
+            ID  ID1 ...  Pval   Padj
+            a   x   ...  0.123  0.123
+            b   y   ...  0.123  0.123
+            ...
+            ```
+
+    Envs:
+        id_cols: The column names used in `in.infile` as ID columns. Multiple
+            columns can be specified by comma-seperated values. For example, `ID1,ID2`.
+        pval_col: The column name used in `in.infile` as p-value column.
+        method (choice): The method used to calculate the meta-pvalue.
+            - fisher: Fisher's method.
+            - sumlog: Sum of logarithms (same as Fisher's method)
+            - logitp: Logit method.
+            - sumz: Sum of z method (Stouffer's method).
+            - meanz: Mean of z method.
+            - meanp: Mean of p method.
+            - invt: Inverse t method.
+            - sump: Sum of p method (Edgington's method).
+            - votep: Vote counting method.
+            - wilkinsonp: Wilkinson's method.
+            - invchisq: Inverse chi-square method.
+        na: The method to handle NA values. -1 to skip the record. Otherwise NA
+            will be replaced by the given value.
+        keep_single (flag): Whether to keep the original p-value when there is only one
+            p-value.
+        padj (choice): The method for p-value adjustment.
+            - none: No p-value adjustment (no Padj column in outfile).
+            - holm: Holm-Bonferroni method.
+            - hochberg: Hochberg method.
+            - hommel: Hommel method.
+            - bonferroni: Bonferroni method.
+            - BH: Benjamini-Hochberg method.
+            - BY: Benjamini-Yekutieli method.
+            - fdr: FDR correction method.
+    """
+    input = "infile:file"
+    output = "outfile:file:{{in.infile | stem}}.metapval.txt"
+    lang = config.lang.rscript
+    envs = {
+        "id_cols": None,
+        "pval_col": None,
+        "method": "fisher",
+        "na": -1,
+        "keep_single": True,
+        "padj": "none",
+    }
+    script = "file://../scripts/stats/MetaPvalue1.R"

biopipen/ns/vcf.py

@@ -439,3 +439,199 @@ class TruvariConsistency(Proc):
     envs = {"truvari": config.exe.truvari, "heatmap": {}}
     script = "file://../scripts/vcf/TruvariConsistency.R"
     plugin_opts = {"report": "file://../reports/vcf/TruvariConsistency.svelte"}
+
+
+class BcftoolsAnnotate(Proc):
+    """Add or remove annotations from VCF files
+
+    See also: <https://samtools.github.io/bcftools/bcftools.html#annotate>
+
+    Input:
+        infile: The input VCF file
+        annfile: The annotation file.
+            Currently only VCF files are supported.
+
+    Output:
+        outfile: The VCF file with annotations added or removed.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile and annfile
+        annfile: The annotation file. If `in.annfile` is provided,
+            this is ignored
+        ncores (type=int): Number of cores (`--threads`) to use
+        columns (auto): Comma-separated or list of columns or tags to carry over from
+            the annotation file. Overrides `-c, --columns`
+        remove (auto): Remove the specified columns from the input file
+        header (type=list): Headers to be added
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools annotate`
+            See also <https://samtools.github.io/bcftools/bcftools.html#annotate>
+            Note that the underscore `_` will be replaced with dash `-` in the
+            argument name.
+    """
+    input = "infile:file, annfile:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "annfile": None,
+        "columns": [],
+        "remove": [],
+        "header": [],
+        "gz": True,
+        "index": True,
+        "ncores": config.misc.ncores,
+    }
+    script = "file://../scripts/vcf/BcftoolsAnnotate.py"
+
+
+class BcftoolsFilter(Proc):
+    """Apply fixed threshold filters to VCF files
+
+    Input:
+        infile: The input VCF file
+
+    Output:
+        outfile: The filtered VCF file. If the `in.infile` is gzipped, this is
+            gzipped as well.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        keep: Whether we should keep the filtered variants or not.
+            If True, the filtered variants will be kept in the output file, but
+            with a new FILTER.
+        includes: and
+        excludes: include/exclude only sites for which EXPRESSION is true.
+            See: <https://samtools.github.io/bcftools/bcftools.html#expressions>
+            If provided, `envs.include/exclude` will be ignored.
+            If `str`/`list` used, The filter names will be `Filter_<type>_<index>`.
+            A dict is used where keys are filter names and values are expressions
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools filter`
+            See also <https://samtools.github.io/bcftools/bcftools.html#filter>
+    """
+    input = "infile:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "keep": True,
+        "includes": None,
+        "excludes": None,
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsFilter.py"
+
+
+class BcftoolsSort(Proc):
+    """Sort VCF files using `bcftools sort`.
+
+    `bcftools sort` is used to sort VCF files by chromosome and position based on the
+    order of contigs in the header.
+
+    Here we provide a chrsize file to first sort the contigs in the header and then
+    sort the VCF file using `bcftools sort`.
+
+    Input:
+        infile: The input VCF file
+
+    Output:
+        outfile: The sorted VCF file.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        chrsize: The chromosome size file, from which the chromosome order is used
+            to sort the contig in the header first.
+            If not provided, `bcftools sort` will be used directly.
+        notfound (choice): What if the contig in the VCF file is not found in the
+            `chrsize` file.
+            - error: Report error
+            - remove: Remove the contig from the header.
+                Note that if there are records with the removed contig, an error will
+                be raised by `bcftools sort`
+            - start: Move the contig to the start of the contigs from `chrsize`
+            - end: Move the contig to the end of the contigs from `chrsize`
+        <more>: Other arguments for `bcftools sort`. For example `max_mem`.
+            See also <https://samtools.github.io/bcftools/bcftools.html#sort>
+    """
+    input = "infile:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "chrsize": config.ref.chrsize,
+        "notfound": "remove",
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsSort.py"
+
+
+class BcftoolsView(Proc):
+    """View, subset and filter VCF files by position and filtering expression.
+
+    Also convert between VCF and BCF.
+
+    Input:
+        infile: The input VCF file
+        regions_file: The region file used to subset the input VCF file.
+        samples_file: The samples file used to subset the input VCF file.
+
+    Output:
+        outfile: The output VCF file.
+
+    Envs:
+        bcftools: Path to bcftools
+        tabix: Path to tabix, used to index infile/outfile
+        ncores (type=int): Number of cores (`--threads`) to use
+        regions_file: The region file used to subset the input VCF file.
+            If `in.regions_file` is provided, this is ignored.
+        samples_file: The samples file used to subset the input VCF file.
+            If `in.samples_file` is provided, this is ignored.
+        gz (flag): Whether to gzip the output file
+        index (flag): Whether to index the output file (tbi) (`envs.gz` forced to True)
+        <more>: Other arguments for `bcftools view`.
+            See also https://samtools.github.io/bcftools/bcftools.html#view
+            Note that the underscore `_` will be replaced with dash `-` in the
+            argument name.
+    """
+    input = "infile:file, regions_file:file, samples_file:file"
+    output = (
+        "outfile:file:{{in.infile | stem: 'gz'}}.vcf"
+        "{{'.gz' if envs.index or envs.gz else ''}}"
+    )
+    lang = config.lang.python
+    envs = {
+        "bcftools": config.exe.bcftools,
+        "tabix": config.exe.tabix,
+        "ncores": config.misc.ncores,
+        "regions_file": None,
+        "samples_file": None,
+        "gz": True,
+        "index": True,
+    }
+    script = "file://../scripts/vcf/BcftoolsView.py"

biopipen/reports/snp/PlinkCallRate.svelte

@@ -0,0 +1,24 @@
+{% from "utils/misc.liq" import report_jobs -%}
+<script>
+    import { Image, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    <h{{h+1}}>Sample Call Rate</h{{h+1}}>
+    {%- for pngfile in job.out.outdir | joinpaths: '*.samplecr.png' | glob -%}
+    <Descr>Cutoff: {{envs.samplecr}}</Descr>
+    <Image src="{{pngfile}}" />
+    {%- endfor -%}
+
+    <h{{h+1}}>Variant Call Rate</h{{h+1}}>
+    {%- for pngfile in job.out.outdir | joinpaths: '*.varcr.png' | glob -%}
+    <Descr>Cutoff: {{envs.varcr}}</Descr>
+    <Image src="{{pngfile}}" />
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+<h1>Sample: {{job.in.cnrfile | stem0 }}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/snp/PlinkFreq.svelte

@@ -0,0 +1,18 @@
+{% from "utils/misc.liq" import report_jobs -%}
+<script>
+    import { Image, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+        {% set metric_col = pngfile | stem | ext0 %}
+        <h{{h+1}}>{{metric_col}} distribution</h{{h+1}}>
+        <Image src="{{pngfile}}" />
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+<h1>Sample: {{job.in.cnrfile | stem0 }}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/snp/PlinkHWE.svelte

@@ -0,0 +1,18 @@
+{% from "utils/misc.liq" import report_jobs -%}
+<script>
+    import { Image, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+    <h{{h+1}}>Distribution</h{{h+1}}>
+    <Descr>Cutoff: {{envs.cutoff}}</Descr>
+    <Image src="{{pngfile}}" />
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+<h1>Sample: {{job.in.cnrfile | stem0 }}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/snp/PlinkHet.svelte

@@ -0,0 +1,18 @@
+{% from "utils/misc.liq" import report_jobs -%}
+<script>
+    import { Image, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+    <h{{h+1}}>Distribution</h{{h+1}}>
+    <Descr>Cutoff: [mean - {{envs.cutoff}} x sd, mean + {{envs.cutoff}} x sd]</Descr>
+    <Image src="{{pngfile}}" />
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+<h1>Sample: {{job.in.cnrfile | stem0 }}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}

biopipen/reports/snp/PlinkIBD.svelte

@@ -0,0 +1,18 @@
+{% from "utils/misc.liq" import report_jobs -%}
+<script>
+    import { Image, Descr } from "$libs";
+</script>
+
+{%- macro report_job(job, h=1) -%}
+    {%- for pngfile in job.out.outdir | joinpaths: '*.png' | glob -%}
+    <h{{h+1}}>Heatmap</h{{h+1}}>
+    <Descr>PI_HAT threshold = {{envs.pihat}}</Descr>
+    <Image src="{{pngfile}}" />
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro head_job(job) -%}
+<h1>Sample: {{job.in.cnrfile | stem0 }}</h1>
+{%- endmacro -%}
+
+{{ report_jobs(jobs, head_job, report_job) }}