biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/scripts/web/GCloudStorageDownloadBucket.py

@@ -0,0 +1,82 @@
+import concurrent.futures
+from pathlib import Path
+
+from biopipen.utils.misc import run_command, dict_to_cli_args, logger
+from biopipen.scripts.web.gcloud_common import (
+    is_logged_in,
+    is_valid_gs_bucket_url,
+    get_file_path,
+)
+
+url: str = {{in.url | quote}}  # pyright: ignore  # noqa: E999
+outdir = Path({{out.outdir | repr}})  # pyright: ignore
+gcloud: str = {{envs.gcloud | quote}}  # pyright: ignore
+keep_structure = {{envs.keep_structure | repr}}  # pyright: ignore
+ncores: int = {{envs.ncores | repr}}  # pyright: ignore
+args: dict = {{envs.args | repr}}  # pyright: ignore
+
+if not is_valid_gs_bucket_url(url):
+    raise Exception(
+        f"Invalid Google Cloud Storage URL for a bucket: {url}. "
+        "URL should be in the format gs://bucket"
+    )
+
+if not is_logged_in(gcloud):
+    raise Exception(
+        "You need to be logged in to gcloud to download files. "
+        "Please run `gcloud auth login` first."
+    )
+
+
+def create_folders(folder_lines):
+    for folder_line in folder_lines:
+        folder_path = get_file_path(folder_line)
+        folder = outdir / folder_path
+        folder.mkdir(parents=True, exist_ok=True)
+
+
+def download_file(i: int, line: str, total: int):
+    path = get_file_path(line)
+
+    if total <= 50:
+        logger.info(f"Downloading {path}")
+    elif 50 < total <= 500:
+        if i % 10 == 0:
+            logger.info(f"Downloading {i}/{total} ...")
+    else:
+        if i % 100 == 0:
+            logger.info(f"Downloading {i}/{total} ...")
+
+    if keep_structure:
+        target = (outdir / path)
+    else:
+        name = Path(path).name
+        target = outdir / name
+        if target.exists():
+            new_name = f"g{i}-{name}"
+            logger.warning(f"{name} already exists. Renaming to {new_name}.")
+            target = outdir / new_name
+
+    gs_args = args.copy()
+    gs_args[""] = [gcloud, "storage", "cp", line, target]
+    run_command(dict_to_cli_args(gs_args, dashify=True), fg=True)
+
+
+def download_bucket():
+    out = run_command([gcloud, "storage", "ls", "--recursive", url], stdout="RETURN")
+    # remove empty lines and skip the root
+    out = list(filter(None, out.splitlines()[1:]))  # type: ignore
+    if keep_structure:
+        # create folders first
+        logger.info(f"Creating folders to keep structure.")
+        folder_lines = [line[:-2] for line in out if line.endswith("/:")]
+        create_folders(folder_lines)
+
+    out = [line for line in out if not line.endswith("/:")]
+    length = len(out)
+    with concurrent.futures.ProcessPoolExecutor(max_workers=ncores) as executor:
+        executor.map(download_file, range(length), out, [length] * length)
+
+
+if __name__ == "__main__":
+    download_bucket()

biopipen/scripts/web/GCloudStorageDownloadFile.py

@@ -0,0 +1,23 @@
+from biopipen.utils.misc import run_command, dict_to_cli_args
+from biopipen.scripts.web.gcloud_common import is_logged_in, is_valid_gs_file_url
+
+url: str = {{in.url | repr}}  # pyright: ignore  # noqa: E999
+outfile = {{out.outfile | repr}}  # pyright: ignore
+gcloud: str = {{envs.gcloud | repr}}  # pyright: ignore
+args: dict = {{envs.args | repr}}  # pyright: ignore
+
+if not is_valid_gs_file_url(url):
+    raise Exception(
+        f"Invalid Google Cloud Storage URL for a file: {url}. "
+        "URL should be in the format gs://bucket/path/to/file"
+    )
+
+if not is_logged_in(gcloud):
+    raise Exception(
+        "You need to be logged in to gcloud to download files. "
+        "Please run `gcloud auth login` first."
+    )
+
+args[""] = [gcloud, "storage", "cp", url, outfile]
+
+run_command(dict_to_cli_args(args, dashify=True), fg=True)

biopipen/scripts/web/gcloud_common.py

@@ -0,0 +1,49 @@
+"""Provides common functions for interacting with Google Cloud Storage."""
+from biopipen.utils.misc import run_command
+
+
+def is_logged_in(gcloud: str) -> bool:
+    """Check if the user is logged in to Google Cloud Storage.
+
+    Args:
+        gcloud: Path to the `gcloud` executable.
+
+    Returns:
+        bool: True if the user is logged in, False otherwise.
+    """
+    out = run_command([gcloud, "auth", "list"], stdout="RETURN")
+    return "ACTIVE" in out  # type: ignore
+
+
+def is_valid_gs_bucket_url(url: str) -> bool:
+    """Check if a URL is a valid Google Cloud Storage bucket URL.
+
+    Such as `gs://bucket`.
+    """
+    if not url.startswith("gs://"):
+        return False
+
+    url = url.rstrip("/")
+    return "/" not in url[5:]
+
+
+def get_file_path(url: str) -> str:
+    """Get the file path from a Google Cloud Storage file URL, without bucket.
+
+    For example: gs://bucket/path/to/file -> path/to/file
+
+    Args:
+        url: The Google Cloud Storage file URL.
+
+    Returns:
+        str: The file path.
+    """
+    return url[5:].split("/", 1)[1]
+
+
+def is_valid_gs_file_url(url: str) -> bool:
+    """Check if a URL is a valid Google Cloud Storage file URL.
+
+    Such as `gs://bucket/path/to/file`.
+    """
+    return url.startswith("gs://")

biopipen/utils/gene.py

@@ -1,86 +1,134 @@
 """Do gene name conversion"""
+from __future__ import annotations
+
+import re
+import contextlib
+import pandas as pd
 from mygene import MyGeneInfo
-from datar.all import (
-    c,
-    f,
-    group_by,
-    desc,
-    arrange,
-    slice_head,
-    tibble,
-    left_join,
-    mutate,
-    is_na,
-    across,
-    if_else,
-    filter_,
-    pull,
-    select,
-)
 
 mygene = MyGeneInfo()
 
 
-class QueryGenesNotFound(Exception):
+class QueryGenesNotFound(ValueError):
     """When genes cannot be found"""
 
 
 def gene_name_conversion(
-    genes,
-    species,
-    infmt,
-    outfmt,
-    notfound,
+    genes: list[str],
+    infmt: str | list[str],
+    outfmt: str,
+    dup: str = "first",
+    species: str = "human",
+    notfound: str = "na",
+    suppress_messages: bool = False,
 ):
     """Convert gene names using MyGeneInfo
 
     Args:
-        genes: A sequence of genes
-        species: The species to limit the query
-            Supported: human, mouse, rat, fruitfly, nematode, zebrafish,
-            thale-cress, frog and pig
-
-        infmt: What's the original gene name format
-            Available fields
-            https://docs.mygene.info/en/latest/doc/query_service.html#available-fields
-        outfmt: What's the target gene name format
-        notfound: What to do if a conversion cannot be done.
-            use-query: Ignore the conversion and use the original name
-            skip: Ignore the conversion and skip the entire row in input file
-            error: Report error
+        genes: A character/integer vector of gene names/ids
+        species: A character vector of species names
+        infmt: A character vector of input gene name formats
+            See the available scopes at
+            https://docs.mygene.info/en/latest/doc/data.html#available-fields
+            You can use ensg as a shortcut for ensembl.gene
+        outfmt: A character vector of output gene name formats
+        dup: How to deal with duplicate gene names found.
+            first: keep the first one (default), sorted by score descendingly
+            last: keep the last one, sorted by score descendingly
+            all: keep all of them, each will be a separate row
+            <X>: combine them into a single string, separated by X
+        notfound: How to deal with gene names that are not found
+            error: stop with an error message
+            use-query: use the query gene name as the converted gene name
+            skip: skip the gene names that are not found
+            ignore: Same as "skip"
+            na: use NA as the converted gene name (default)
+        suppress_messages: Suppress the messages while querying
 
     Returns:
-        A dataframe with two columns, query and `outfmt`.
+        A dataframe with the query gene names and the converted gene names
+        When a gene name is not found, the converted name will be "NA"
+        When duplicate gene names are found, the one with the highest score will be kept
     """
-    out = (
-        mygene.querymany(
+    notfound = notfound.lower()
+    if notfound not in ("error", "use-query", "skip", "ignore", "na"):
+        raise ValueError(
+            "`notfound` of `gene_name_conversion` must be one of "
+            "'error', 'use-query', 'skip', 'ignore', 'na'"
+        )
+
+    if infmt in ["ensg", "ensmusg"]:
+        infmt = "ensembl.gene"
+    if outfmt in ["ensg", "ensmusg"]:
+        outfmt = "ensembl.gene"
+
+    orig_genes = genes[:]
+    if infmt == "ensembl.gene":
+        # Remove version numbers from ensembl gene ids
+        genes = [re.sub("\\..*", "", gene) for gene in genes]
+
+    query_df = pd.DataFrame({"query": genes, "orig": orig_genes})
+
+    if suppress_messages:
+        with contextlib.redirect_stdout(None):
+            out = mygene.querymany(
+                genes,
+                scopes=infmt,
+                fields=outfmt,
+                species=species,
+                as_dataframe=True,
+                df_index=False,
+            )
+    else:
+        out = mygene.querymany(
             genes,
             scopes=infmt,
             fields=outfmt,
+            species=species,
             as_dataframe=True,
             df_index=False,
-            species=species,
         )
-        >> group_by(f.query)
-        >> arrange(desc(f._score))
-        >> slice_head(1)
-        >> select(~c(f._id, f._score, f.notfound))
-    )
-    if isinstance(outfmt, str):
-        outfmt = [of.strip() for of in outfmt.split(",")]
-    out = tibble(query=genes) >> left_join(out, by=f.query)
-    if notfound == "use-query":
-        out = out >> mutate(
-            across(
-                outfmt,
-                lambda col, query: if_else(is_na(col), query, col),
-                query=f.query,
-            )
+
+    if out.shape[0] == 0:
+        return pd.DataFrame({"query": genes, "converted": ["NA"] * len(genes)})
+
+    if dup == "first":
+        out = (
+            out
+            .sort_values("_score", ascending=False)
+            .groupby("query")
+            .head(1)
+            .reset_index(drop=True)
         )
-    elif notfound == "error" and any(is_na(out[outfmt[0]])):
-        nagenes = out >> filter_(is_na(f[outfmt[0]])) >> pull(f.query)
-        raise QueryGenesNotFound(nagenes)
-    elif notfound == "skip":
-        out = out >> filter_(~is_na(f[outfmt[0]]))
+    elif dup == "last":
+        out = (
+            out
+            .sort_values("_score", ascending=False)
+            .groupby("query")
+            .tail(1)
+            .reset_index(drop=True)
+        )
+    elif dup != "all":
+        out = (
+            out
+            .sort_values("_score", ascending=False)
+            .groupby("query")
+            .agg({outfmt: lambda x: f"{dup}".join([str(x) for x in x.unique()])})
+            .reset_index()
+        )
+
+    out = pd.merge(query_df, out, on="query", how="left")
+    out = out.drop(columns=["query"]).rename(columns={"orig": "query"})
+
+    if notfound == "error":
+        if out[outfmt].isnull().any():
+            nagenes = out[out[outfmt].isnull()]["query"].tolist()
+            raise QueryGenesNotFound(f"Query genes not found: {','.join(nagenes)}")
+    elif notfound == "use-query":
+        out[outfmt] = out[outfmt].combine_first(out["query"])
+    elif notfound in ["skip", "ignore"]:
+        out = out.dropna(subset=[outfmt])
+    else:  # notfound == "na"
+        out[outfmt] = out[outfmt].fillna("NA")
 
     return out

biopipen/utils/misc.py

@@ -1,30 +1,126 @@
 from __future__ import annotations
 from pathlib import Path
 
+import os
 import sys
-from typing import List
+import logging
+from subprocess import Popen
+from typing import List, Callable, Any
 from biopipen.core.filters import dict_to_cli_args  # noqa: F401
 
+logger = logging.getLogger("biopipen_job")
+logger.setLevel(logging.DEBUG)
+_handler = logging.StreamHandler(sys.stdout)
+# Use same log format as in R
+# {sprintf("%-7s", level)} [{format(time, "%Y-%m-%d %H:%M:%S")}] {msg}
+# so the logs can be populated by pipen-poplog
+_handler.setFormatter(
+    logging.Formatter(
+        "%(levelname)-7s [%(asctime)s] %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+)
+logger.addHandler(_handler)
+
+
+def require_package(
+    package: str,
+    version: str | None = None,
+    python: str | None = None,
+) -> None:
+    """Require a Python package to be installed with optional version check.
+
+    The version specifier should follow the format used by pip, e.g., '>=1.2.3'.
+    Multiple version specifiers can be separated by commas, e.g., '>=1.2.3,<2.0.0'.
 
-def exec_code(code, global_vars=None, local_vars=None, return_var=None):
-    global_vars = global_vars or {}
-    local_vars = local_vars or {}
-    exec(code, global_vars, local_vars)
-
-    if return_var is not None:
-        return local_vars[return_var]
-
-    return None
+    Args:
+        package (str): The name of the package to check.
+        version (str | None): The version specifier string.
+        python (str | None): The Python interpreter to use.
+    """
+    if not python:
+        import importlib
+        from importlib.metadata import version as get_version
+        from packaging.specifiers import SpecifierSet
+
+        try:
+            importlib.import_module(package)
+        except ImportError:
+            raise ImportError(f"Package '{package}' is required but not installed.")
+
+        if version:
+            installed_version = get_version(package)
+            specifier = SpecifierSet(version)
+            if installed_version not in specifier:
+                raise ImportError(
+                    f"Package '{package}' version '{installed_version}' does not "
+                    f"satisfy the requirement '{package}{version}'."
+                )
+    else:
+        import subprocess
+        from packaging.specifiers import SpecifierSet
+
+        # Check if package is installed using the specified Python interpreter
+        try:
+            result = subprocess.run(
+                [python, "-c", f"import {package}"],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            if result.returncode != 0:
+                raise ImportError(
+                    f"Package '{package}' is required but not installed in {python}."
+                )
+        except subprocess.TimeoutExpired:
+            raise ImportError(
+                f"Timeout while checking if package '{package}' is "
+                f"installed in {python}."
+            )
+        except FileNotFoundError:
+            raise ImportError(f"Python interpreter '{python}' not found.")
+
+        if version:
+            # Get the installed version
+            try:
+                version_cmd = (
+                    f"from importlib.metadata import version; "
+                    f"print(version('{package}'))"
+                )
+                result = subprocess.run(
+                    [python, "-c", version_cmd],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                )
+                if result.returncode != 0:
+                    raise ImportError(
+                        f"Failed to get version of package '{package}' "
+                        f"in {python}."
+                    )
+                installed_version = result.stdout.strip()
+                specifier = SpecifierSet(version)
+                if installed_version not in specifier:
+                    raise ImportError(
+                        f"Package '{package}' version '{installed_version}' "
+                        f"in {python} does not satisfy the requirement "
+                        f"'{package}{version}'."
+                    )
+            except subprocess.TimeoutExpired:
+                raise ImportError(
+                    f"Timeout while checking version of package '{package}' "
+                    f"in {python}."
+                )
 
 
 def run_command(
-    cmd: str | List[str],
+    cmd: str | List[Any],
     fg: bool = False,
     wait: bool = True,
     print_command: bool = True,
-    print_command_handler: callable = print,
+    print_command_handler: Callable = print,
     **kwargs,
-):
+) -> Popen | str:
     """Run a command.
 
     Args:
@@ -41,7 +137,7 @@ def run_command(
         The `Popen` object, or str when `stdout` is `RETURN` or `return`.
     """
     import shlex
-    from subprocess import Popen, PIPE, STDOUT
+    from subprocess import PIPE, STDOUT
 
     if isinstance(cmd, list):
         cmd = [str(c) for c in cmd]
@@ -49,9 +145,12 @@ def run_command(
     if print_command:
         print_command_handler("RUNNING COMMAND:")
         if isinstance(cmd, str):
-            print_command_handler(f"  {cmd}")
+            print_command_handler(f"  {cmd}\n")
         else:
-            print_command_handler(f"  {shlex.join(cmd)}")
+            print_command_handler(f"  {shlex.join(cmd)}\n")
+        # flush the output if print_command_handler is print
+        if print_command_handler is print:
+            sys.stdout.flush()
 
     if isinstance(cmd, str):
         kwargs["shell"] = True
@@ -60,6 +159,7 @@ def run_command(
         kwargs["stdin"] = PIPE
 
     return_stdout = False
+    stdout_file = None
     if kwargs.get("stdout") is True:
         kwargs["stdout"] = PIPE
     elif kwargs.get("stdout") in ("RETURN", "return"):
@@ -68,7 +168,8 @@ def run_command(
     elif isinstance(kwargs.get("stdout"), (str, Path)):
         if isinstance(kwargs["stdout"], str):
             kwargs["stdout"] = Path(kwargs["stdout"])
-        kwargs["stdout"] = kwargs["stdout"].open("w")
+        stdout_file = kwargs["stdout"].open("w")
+        kwargs["stdout"] = stdout_file
         kwargs["close_fds"] = True
 
     if kwargs.get("stderr") is True:
@@ -76,6 +177,10 @@ def run_command(
     elif kwargs.get("stderr") in ("STDOUT", "stdout"):
         kwargs["stderr"] = STDOUT
 
+    # Enable line buffering for stdout/stderr when redirecting to files or pipes
+    if kwargs.get("bufsize") == 1:
+        kwargs.setdefault("universal_newlines", True)
+
     if fg:
         if kwargs.get("stdout") or kwargs.get("stderr"):
             raise ValueError(
@@ -85,18 +190,39 @@ def run_command(
         kwargs["stderr"] = sys.stderr
         kwargs["universal_newlines"] = True
 
+    if "env" in kwargs:
+        kwargs["env"] = {**os.environ, **kwargs["env"]}
+
     try:
         p = Popen(cmd, **kwargs)
     except Exception as e:
-        raise RuntimeError(f"Failed to run command: {e}")
+        raise RuntimeError(
+            f"Failed to run command: {e}\n"
+            f"Command (list): {cmd}\n"
+            f"Command (str): {shlex.join(cmd)}"
+        )
 
     if fg or wait or return_stdout:
         rc = p.wait()
         if rc != 0:
-            raise RuntimeError(f"Failed to run command: {cmd}")
+            if stdout_file:
+                stdout_file.close()
+            if return_stdout and p.stdout:
+                p.stdout.close()
+            raise RuntimeError(
+                f"Failed to run command: rc={rc}\n"
+                f"Command (list): {cmd}\n"
+                f"Command (str): {shlex.join(cmd)}"
+            )
 
         if return_stdout:
-            return p.stdout.read().decode()
+            try:
+                return p.stdout.read().decode()  # type: ignore
+            finally:
+                p.stdout.close()  # type: ignore
+
+        if stdout_file:
+            stdout_file.close()
 
         return p