biopipen 0.28.1__py3-none-any.whl → 0.29.1__py3-none-any.whl

biopipen/scripts/vcf/BcftoolsFilter.py

@@ -0,0 +1,90 @@
+from pathlib import Path, PosixPath  # noqa: F401
+
+from biopipen.utils.misc import logger
+from biopipen.scripts.vcf.bcftools_utils import run_bcftools
+
+infile = {{in.infile | repr}}  # pyright: ignore # noqa: #999
+outfile = {{out.outfile | repr}}  # pyright: ignore
+outdir = Path(outfile).parent
+
+envs = {{envs | dict | repr}}  # pyright: ignore
+bcftools = envs.pop("bcftools")
+tabix = envs.pop("tabix")
+keep = envs.pop("keep")
+ncores = envs.pop("ncores")
+includes = envs.pop("includes")
+excludes = envs.pop("excludes")
+gz = envs.pop("gz")
+index = envs.pop("index")
+
+# a.vcf.gz -> a
+# a.vcf -> a
+stem = Path(infile).stem
+if stem.endswith(".vcf"):
+    stem = stem[:-4]
+# .vcf.gz
+# .gz
+ext = ".vcf.gz" if index or gz else '.vcf'
+
+
+def normalize_expr(expr, flag, prev_n_filters=0):
+    out = {}
+    if not expr:
+        return out
+    if isinstance(expr, list):
+        for ex in expr:
+            out[f"FILTER_{flag.upper()}_{len(out) + 1 + prev_n_filters}"] = (ex, flag)
+    elif isinstance(expr, dict):
+        for name, ex in expr.items():
+            out[name] = (ex, flag)
+    else: # str
+        out[f"FILTER_{flag.upper()}_{len(out) + 1 + prev_n_filters}"] = (expr, flag)
+    return out
+
+
+def handle_filter(vcf, fname, filt, flag, final):
+    logger.info("- Handling filter %s: %s ...", fname, filt)
+
+    arguments = envs.copy()
+    arguments[flag] = filt
+    arguments["_"] = vcf
+    arguments["o"] = outfile if final else outdir / f"{stem}.{fname}{ext}"
+    if keep:
+        arguments["s"] = fname
+
+    run_bcftools(arguments, bcftools=bcftools, index=index and final, tabix=tabix)
+
+    if final:
+        flagfile = outdir.joinpath(f"{stem}.{fname}{ext}")
+        if flagfile.is_symlink():
+            flagfile.unlink()
+        outdir.joinpath(f"{stem}.{fname}{ext}").symlink_to(outfile)
+
+    return arguments["o"]
+
+
+includes = normalize_expr(includes, "include")
+excludes = normalize_expr(excludes, "exclude", len(includes))
+includes.update(excludes)
+
+if index and not gz:
+    logger.warning("Forcing envs.gz to True because envs.index is True.")
+    gz = True
+
+envs[""] = [bcftools, "filter"]
+envs["_"] = infile
+envs["o"] = outfile
+envs["threads"] = ncores
+
+if "O" not in envs and "output-type" not in envs and "output_type" not in envs:
+    envs["O"] = "z" if gz else "v"
+
+if keep:
+    envs["soft_filter"] = "+"
+
+if "m" not in envs and "mode" not in envs:
+    envs["m"] = "+"
+
+# bcftools can be only done once at one filter
+for i, (fname, (filt, flag)) in enumerate(includes.items()):
+    infile = handle_filter(infile, fname, filt, flag, i == len(includes) - 1)

biopipen/scripts/vcf/BcftoolsSort.py

@@ -0,0 +1,113 @@
+from typing import Literal
+from pathlib import Path, PosixPath  # noqa: F401
+
+from biopipen.utils.misc import run_command, logger
+from biopipen.scripts.vcf.bcftools_utils import run_bcftools
+
+infile = {{in.infile | quote}}  # pyright: ignore # noqa: E999
+outfile = {{out.outfile | quote}}  # pyright: ignore
+envs = {{envs | dict | repr}}  # pyright: ignore
+
+outdir = Path(outfile).parent
+bcftools = envs.pop("bcftools")
+tabix = envs.pop("tabix")
+ncores = envs.pop("ncores")
+gz = envs.pop("gz")
+index = envs.pop("index")
+chrsize = envs.pop("chrsize")
+notfound = envs.pop("notfound")
+
+if chrsize:
+    class Contig:
+        def __init__(self, name: str, length: str):
+            self.name = name
+            self.length = length
+
+        def __str__(self) -> str:
+            return f"##contig=<ID={self.name},length={self.length}>"
+
+    def parse_header(header_file: Path) -> tuple[list[str], dict[str, Contig]]:
+        hlines = []
+        ctgs = {}
+        with open(header_file) as fh:
+            for line in fh:
+                if line.startswith("##contig"):
+                    ctg = line.strip().split("##contig=<ID=")[1].split(",length=")
+                    ctgs[ctg[0]] = Contig(ctg[0], ctg[1].replace(">", ""))
+                else:
+                    hlines.append(line.strip())
+        return hlines, ctgs
+
+    def match_contigs(
+        ctgs: dict[str, Contig],
+        chroms: list[str],
+        notfound: Literal["error", "remove", "start", "end"],
+    ) -> list[str]:
+        if (
+            ctgs
+            and chroms
+            and all(chrom.startswith("chr") for chrom in chroms)
+            and not any(chrom.startswith("chr") for chrom in ctgs)
+        ):
+            logger.warning(
+                "Removing 'chr' prefix from chromosomes in envs.chrsize file, "
+                "because the input VCF file does not have 'chr' prefix."
+            )
+            chroms = [chrom[3:] for chrom in chroms]
+
+        new_ctgs = []
+        for chrom in chroms:
+            if chrom in ctgs:
+                new_ctgs.append(str(ctgs[chrom]))
+                del ctgs[chrom]
+
+        if ctgs:
+            if notfound == "error":
+                raise ValueError(
+                    "Chromosomes not found in envs.chrsize file: "
+                    f"{', '.join(ctgs.keys())}"
+                )
+            elif notfound == "start":
+                new_ctgs = [str(ctg) for ctg in ctgs.values()] + new_ctgs
+            elif notfound == "end":
+                new_ctgs = new_ctgs + [str(ctg) for ctg in ctgs.values()]
+
+        return new_ctgs
+
+    chroms = []
+    with Path(chrsize).expanduser().open() as fh:
+        for line in fh:
+            chrom = line.strip().split()[0]
+            chroms.append(chrom)
+
+    header_file = outdir / "header.txt"
+    run_command(f'{bcftools} view -h {infile} > {header_file}', fg=True)
+    header_lines, contigs = parse_header(header_file)
+    new_contigs = match_contigs(contigs, chroms, notfound=notfound)
+    header_lines = [header_lines[0], *new_contigs, *header_lines[1:]]
+    reheader_file = outdir / "reheader.txt"
+    with open(reheader_file, "w") as fh:
+        fh.writelines([f"{line}\n" for line in header_lines])
+
+    reheader_vcf = outdir / f"{Path(infile).stem}_reheader.vcf"
+    run_command([
+        bcftools, "reheader",
+        "--header", reheader_file,
+        "-o", reheader_vcf,
+        infile
+    ], fg=True)
+
+    infile = reheader_vcf
+
+envs[""] = [bcftools, "sort"]
+envs["_"] = infile
+envs["o"] = outfile
+
+if index and not gz:
+    logger.warning("Forcing envs.gz to True because envs.index is True.")
+    gz = True
+
+if "O" not in envs and "output-type" not in envs and "output_type" not in envs:
+    envs["O"] = "z" if gz else "v"
+
+run_bcftools(envs, bcftools=bcftools, index=index, tabix=tabix)

biopipen/scripts/vcf/BcftoolsView.py

@@ -0,0 +1,73 @@
+from contextlib import suppress
+# In case there are paths passed to envs
+from pathlib import PosixPath  # noqa: F401
+
+from biopipen.utils.misc import logger
+from biopipen.utils.reference import tabix_index
+from biopipen.scripts.vcf.bcftools_utils import run_bcftools
+
+infile = {{in.infile | repr}}  # pyright: ignore # noqa: #999
+regions_file = {{in.regions_file | repr}}  # pyright: ignore
+samples_file = {{in.samples_file | repr}}  # pyright: ignore
+outfile = {{out.outfile | repr}}  # pyright: ignore
+envs: dict = {{envs | dict | repr}}  # pyright: ignore
+
+bcftools = envs.pop("bcftools")
+tabix = envs.pop("tabix")
+ncores = envs.pop("ncores")
+gz = envs.pop("gz")
+index = envs.pop("index")
+
+if regions_file:
+    if "R" in envs or "regions_file" in envs or "regions-file" in envs:
+        logger.warning(
+            "Ignoring envs\[regions_file/regions-file/R] "
+            "because in.regionsfile is provided."
+        )
+        with suppress(KeyError):
+            del envs["regions_file"]
+        with suppress(KeyError):
+            del envs["regions-file"]
+        with suppress(KeyError):
+            del envs["R"]
+elif "R" in envs or "regions_file" in envs or "regions-file" in envs:
+    regions_file = (
+        envs.pop("regions_file", None)
+        or envs.pop("regions-file", None)
+        or envs.pop("R", None)
+    )
+
+if samples_file:
+    if "S" in envs or "samples_file" in envs or "samples-file" in envs:
+        logger.warning(
+            "Ignoring envs[samples_file/samples-file/S] "
+            "because in.samples_file is provided."
+        )
+        with suppress(KeyError):
+            del envs["samples_file"]
+        with suppress(KeyError):
+            del envs["samples-file"]
+        with suppress(KeyError):
+            del envs["S"]
+elif "S" in envs or "samples_file" in envs or "samples-file" in envs:
+    samples_file = (
+        envs.pop("samples_file", None)
+        or envs.pop("samples-file", None)
+        or envs.pop("S", None)
+    )
+
+if index and not gz:
+    logger.warning("Forcing envs.gz to True because envs.index is True.")
+    gz = True
+
+if "O" not in envs and "output-type" not in envs and "output_type" not in envs:
+    envs["O"] = "z" if gz else "v"
+
+envs[""] = [bcftools, "view"]
+envs["_"] = tabix_index(infile, "vcf", tabix=tabix)
+envs["o"] = outfile
+envs["threads"] = ncores
+envs["regions_file"] = regions_file
+envs["samples_file"] = samples_file
+
+run_bcftools(envs, bcftools=bcftools, index=index, tabix=tabix)

biopipen/scripts/vcf/VcfFix_utils.py

@@ -63,7 +63,7 @@ def fix_vcffile(vcffile, outfile, fixes):
         else:
             modify_fixes.append(fix)
 
-    inopen = gzip.open if vcffile.endswith(".gz") else open
+    inopen = gzip.open if str(vcffile).endswith(".gz") else open
     with inopen(vcffile, "rt") as fin, open(outfile, "w") as fout:
         for line in fin:
             obj = line_to_obj(line)

biopipen/scripts/vcf/bcftools_utils.py

@@ -0,0 +1,52 @@
+"""Utilities for bcftools"""
+
+from biopipen.utils.misc import run_command, dict_to_cli_args
+from biopipen.utils.reference import tabix_index
+
+
+def bcftools_version(bcftools: str) -> tuple[int, ...]:
+    """Get the version of bcftools
+
+    Args:
+        bcftools (str): Path to bcftools
+
+    Returns:
+        tuple[int, ...]: The version of bcftools
+    """
+    bversion = (
+        run_command([bcftools, "version"], stdout="return")
+        .splitlines()[0]  # bcftools 1.20
+        .replace("bcftools", "")
+        .strip()  # 1.20
+        .split(".")
+    )
+    return tuple(map(int, bversion))
+
+
+def run_bcftools(
+    args: dict[str, object],
+    bcftools: str,
+    index: bool,
+    tabix: str
+) -> None:
+    """Run bcftools with the given arguments
+
+    Args:
+        args: Arguments to pass to bcftools
+        bcftools (str): Path to bcftools
+        index (bool): Whether to index the output
+        tabix (str): Path to tabix
+    """
+    if not index:
+        run_command(dict_to_cli_args(args, dashify=True), fg=True)
+    else:
+        bversion = bcftools_version(bcftools)
+        if bversion >= (1, 20):
+            # requires bcftools 1.20+
+            # '--write-index tbi' not working
+            # it has to be '--write-index=tbi'
+            args["write_index=tbi"] = True
+            run_command(dict_to_cli_args(args, dashify=True), fg=True)
+        else:
+            run_command(dict_to_cli_args(args, dashify=True), fg=True)
+            tabix_index(args["o"], "vcf", tmpdir=False, tabix=tabix)

biopipen/utils/gene.R

@@ -1,49 +1,95 @@
-library(mygene)
-library(dplyr)
+suppressPackageStartupMessages({
+    library(rlang)
+    library(dplyr)
+    library(mygene)
+})
 
-gene_name_conversion = function(
+
+#@' Convert gene names between different formats
+#@'
+#@' @param genes A character/integer vector of gene names/ids
+#@' @param species A character vector of species names
+#@' @param infmt A character vector of input gene name formats
+#@'   See the available scopes at
+#@'   https://docs.mygene.info/en/latest/doc/data.html#available-fields
+#@'   You can use ensg as a shortcut for ensembl.gene
+#@' @param outfmt A character vector of output gene name formats
+#@' @param dup How to deal with duplicate gene names found.
+#@'   "first": keep the first one (default), sorted by score descendingly
+#@'   "last": keep the last one, sorted by score descendingly
+#@'   "all": keep all of them, each will be a separate row
+#@'   "<X>": combine them into a single string, separated by X
+#@' @param notfound How to deal with gene names that are not found
+#@'   "error": stop with an error message
+#@'   "use-query": use the query gene name as the converted gene name
+#@'   "skip": skip the gene names that are not found
+#@'   "ignore": Same as "skip"
+#@'   "na": use NA as the converted gene name (default)
+#@' @param suppress_messages Whether to suppress the warning messages
+#@' @return A tibble with the query gene names and the converted gene names
+#@'   When a gene name is not found, the converted name will be NA
+#@'   When duplicate gene names are found, the one with the highest score will be kept
+#@' @export
+gene_name_conversion <- function(
     genes,
-    species,
     infmt,
     outfmt,
-    notfound
+    dup = "first",
+    species = "human",
+    notfound = "na",
+    suppress_messages = FALSE
 ) {
-    out = queryMany(
-        genes,
-        scopes=infmt,
-        fields=outfmt,
-        species=species
-    ) %>% as.data.frame() %>% group_by(
-        query
-    ) %>% arrange(
-        desc(X_score)
-    ) %>% slice_head(n=1) %>% select(
-        -c(X_id, X_score)
-    )
-
-    if ("notfound" %in% colnames(out)) {
-        out = out %>% select(-c("notfound"))
+    notfound <- arg_match(notfound, c("error", "use-query", "skip", "ignore", "na"))
+
+    if (infmt %in% c("ensg", "ensmusg")) { infmt = "ensembl.gene" }
+    if (outfmt %in% c("ensg", "ensmusg")) { outfmt = "ensembl.gene" }
+
+    orig_genes <- genes
+    if (infmt == "ensembl.gene") {
+        # Remove version numbers from ensembl gene ids
+        genes <- gsub("\\..*", "", genes)
+    }
+    query_df <- tibble(query = genes, orig = orig_genes)
+
+    if (suppress_messages) {
+        capture.output(suppressWarnings(suppressMessages({
+            out <- queryMany(genes, scopes=infmt, fields=outfmt, species=species) %>%
+                as_tibble()
+        })))
+    } else {
+        out <- queryMany(genes, scopes=infmt, fields=outfmt, species=species) %>%
+            as_tibble()
+    }
+
+    if (nrow(out) == 0) {
+        return(tibble(query = orig_genes, converted = NA_character_))
     }
 
-    if (length(outfmt) == 1 && "," %in% outfmt) {
-        outfmt = trimws(unlist(strsplit(outfmt, ",", fixed=TRUE)))
+    if (dup == "first") {
+        out = out %>% group_by(query) %>% arrange(desc(X_score)) %>%
+            slice_head(n=1) %>% ungroup() %>% dplyr::select(all_of(c("query", outfmt)))
+    } else if (dup == "last") {
+        out = out %>% group_by(query) %>% arrange(X_score) %>%
+            slice_head(n=1) %>% ungroup() %>% dplyr::select(all_of(c("query", outfmt)))
+    } else if (dup != "all") {
+        out = out %>% group_by(query) %>% arrange(desc(X_score)) %>%
+            summarise(!!sym(outfmt) := paste(unique(!!sym(outfmt)), collapse=dup))
     }
+    out <- query_df %>%
+        left_join(out, by="query") %>%
+        dplyr::select(-"query") %>%
+        dplyr::select(query = orig, everything())
 
-    out = tibble(query=genes) %>% left_join(out, by="query")
-    if (notfound == "use-query") {
-        out = out %>% mutate(
-            across(
-                outfmt,
-                function(col, query) if_else(is.na(col), query, col),
-                query=query
-            )
-        )
-    } else if (notfound == "error" && any(is.na(out[[outfmt[1]]]))) {
-        nagenes = out %>% filter(is.na(.[[outfmt[1]]])) %>% pull("query")
-        stop(paste("Query genes not found:", paste(nagenes, collapse=",")))
-    } else if (notfound == "skip") {
-        out = out %>% filter(!is.na(.[[outfmt[1]]]))
+    if (notfound == "error") {
+        if (any(is.na(out[[outfmt]]))) {
+            nagenes = out %>% filter(is.na(.[[outfmt]])) %>% pull("query")
+            stop(paste("Query genes not found:", paste(nagenes, collapse=",")))
+        }
+    } else if (notfound == "use-query") {
+        out = out %>% mutate(!!sym(outfmt) := coalesce(!!sym(outfmt), query))
+    } else if (notfound == "skip" || notfound == "ignore") {
+        out = out %>% filter(!is.na(!!sym(outfmt)))
     }
 
-    return out
+    return(out)
 }

biopipen/utils/gene.py

@@ -1,86 +1,134 @@
 """Do gene name conversion"""
+from __future__ import annotations
+
+import re
+import contextlib
+import pandas as pd
 from mygene import MyGeneInfo
-from datar.all import (
-    c,
-    f,
-    group_by,
-    desc,
-    arrange,
-    slice_head,
-    tibble,
-    left_join,
-    mutate,
-    is_na,
-    across,
-    if_else,
-    filter_,
-    pull,
-    select,
-)
 
 mygene = MyGeneInfo()
 
 
-class QueryGenesNotFound(Exception):
+class QueryGenesNotFound(ValueError):
     """When genes cannot be found"""
 
 
 def gene_name_conversion(
-    genes,
-    species,
-    infmt,
-    outfmt,
-    notfound,
+    genes: list[str],
+    infmt: str | list[str],
+    outfmt: str,
+    dup: str = "first",
+    species: str = "human",
+    notfound: str = "na",
+    suppress_messages: bool = False,
 ):
     """Convert gene names using MyGeneInfo
 
     Args:
-        genes: A sequence of genes
-        species: The species to limit the query
-            Supported: human, mouse, rat, fruitfly, nematode, zebrafish,
-            thale-cress, frog and pig
-
-        infmt: What's the original gene name format
-            Available fields
-            https://docs.mygene.info/en/latest/doc/query_service.html#available-fields
-        outfmt: What's the target gene name format
-        notfound: What to do if a conversion cannot be done.
-            use-query: Ignore the conversion and use the original name
-            skip: Ignore the conversion and skip the entire row in input file
-            error: Report error
+        genes: A character/integer vector of gene names/ids
+        species: A character vector of species names
+        infmt: A character vector of input gene name formats
+            See the available scopes at
+            https://docs.mygene.info/en/latest/doc/data.html#available-fields
+            You can use ensg as a shortcut for ensembl.gene
+        outfmt: A character vector of output gene name formats
+        dup: How to deal with duplicate gene names found.
+            first: keep the first one (default), sorted by score descendingly
+            last: keep the last one, sorted by score descendingly
+            all: keep all of them, each will be a separate row
+            <X>: combine them into a single string, separated by X
+        notfound: How to deal with gene names that are not found
+            error: stop with an error message
+            use-query: use the query gene name as the converted gene name
+            skip: skip the gene names that are not found
+            ignore: Same as "skip"
+            na: use NA as the converted gene name (default)
+        suppress_messages: Suppress the messages while querying
 
     Returns:
-        A dataframe with two columns, query and `outfmt`.
+        A dataframe with the query gene names and the converted gene names
+        When a gene name is not found, the converted name will be "NA"
+        When duplicate gene names are found, the one with the highest score will be kept
     """
-    out = (
-        mygene.querymany(
+    notfound = notfound.lower()
+    if notfound not in ("error", "use-query", "skip", "ignore", "na"):
+        raise ValueError(
+            "`notfound` of `gene_name_conversion` must be one of "
+            "'error', 'use-query', 'skip', 'ignore', 'na'"
+        )
+
+    if infmt in ["ensg", "ensmusg"]:
+        infmt = "ensembl.gene"
+    if outfmt in ["ensg", "ensmusg"]:
+        outfmt = "ensembl.gene"
+
+    orig_genes = genes[:]
+    if infmt == "ensembl.gene":
+        # Remove version numbers from ensembl gene ids
+        genes = [re.sub("\\..*", "", gene) for gene in genes]
+
+    query_df = pd.DataFrame({"query": genes, "orig": orig_genes})
+
+    if suppress_messages:
+        with contextlib.redirect_stdout(None):
+            out = mygene.querymany(
+                genes,
+                scopes=infmt,
+                fields=outfmt,
+                species=species,
+                as_dataframe=True,
+                df_index=False,
+            )
+    else:
+        out = mygene.querymany(
             genes,
             scopes=infmt,
             fields=outfmt,
+            species=species,
             as_dataframe=True,
             df_index=False,
-            species=species,
         )
-        >> group_by(f.query)
-        >> arrange(desc(f._score))
-        >> slice_head(1)
-        >> select(~c(f._id, f._score, f.notfound))
-    )
-    if isinstance(outfmt, str):
-        outfmt = [of.strip() for of in outfmt.split(",")]
-    out = tibble(query=genes) >> left_join(out, by=f.query)
-    if notfound == "use-query":
-        out = out >> mutate(
-            across(
-                outfmt,
-                lambda col, query: if_else(is_na(col), query, col),
-                query=f.query,
-            )
+
+    if out.shape[0] == 0:
+        return pd.DataFrame({"query": genes, "converted": ["NA"] * len(genes)})
+
+    if dup == "first":
+        out = (
+            out
+            .sort_values("_score", ascending=False)
+            .groupby("query")
+            .head(1)
+            .reset_index(drop=True)
         )
-    elif notfound == "error" and any(is_na(out[outfmt[0]])):
-        nagenes = out >> filter_(is_na(f[outfmt[0]])) >> pull(f.query)
-        raise QueryGenesNotFound(nagenes)
-    elif notfound == "skip":
-        out = out >> filter_(~is_na(f[outfmt[0]]))
+    elif dup == "last":
+        out = (
+            out
+            .sort_values("_score", ascending=False)
+            .groupby("query")
+            .tail(1)
+            .reset_index(drop=True)
+        )
+    elif dup != "all":
+        out = (
+            out
+            .sort_values("_score", ascending=False)
+            .groupby("query")
+            .agg({outfmt: lambda x: f"{dup}".join([str(x) for x in x.unique()])})
+            .reset_index()
+        )
+
+    out = pd.merge(query_df, out, on="query", how="left")
+    out = out.drop(columns=["query"]).rename(columns={"orig": "query"})
+
+    if notfound == "error":
+        if out[outfmt].isnull().any():
+            nagenes = out[out[outfmt].isnull()]["query"].tolist()
+            raise QueryGenesNotFound(f"Query genes not found: {','.join(nagenes)}")
+    elif notfound == "use-query":
+        out[outfmt] = out[outfmt].combine_first(out["query"])
+    elif notfound in ["skip", "ignore"]:
+        out = out.dropna(subset=[outfmt])
+    else:  # notfound == "na"
+        out[outfmt] = out[outfmt].fillna("NA")
 
     return out