gentroutils 1.5.0__py3-none-any.whl → 1.6.0.dev1__py3-none-any.whl

gentroutils/commands/utils.py

@@ -1,152 +0,0 @@
-"""Ütility functions for the CLI."""
-
-import asyncio
-import logging
-import sys
-import time
-from functools import wraps
-from pathlib import Path
-from tempfile import NamedTemporaryFile
-from urllib.parse import urlparse
-
-import click
-from google.cloud import storage
-
-logger = logging.getLogger("gentroutils")
-
-
-def set_log_file(ctx: click.Context, param: click.Option, log_file: str) -> str:
-    """Set logging file based on provided `log-file` flag.
-
-    This is a callback function called by the click.Option [--log-file] flag.
-    In case of the `log_file` being path to the GCP bucket the returned value
-    will be the local temporary file path. both log file paths (remote and local)
-    will be stored in the click context object for further reference at the end of the CLI run.
-
-
-    Args:
-        ctx (click.Context): click context
-        param (click.Option): click option
-        log_file (str): log file path
-
-    Raises:
-        click.BadParameter: If the log file is a directory or the URI scheme is not GCS.
-
-    Returns:
-        str: log file path
-    """
-    ctx.ensure_object(dict)
-    if not log_file:
-        return ""
-    logger.info("Extracting log file from the %s", param)
-    upload_to_gcp = False
-
-    if "://" in log_file:
-        upload_to_gcp = True
-    ctx.obj["upload_to_gcp"] = upload_to_gcp
-
-    if upload_to_gcp:
-        parsed_uri = urlparse(log_file)
-        if parsed_uri.scheme != "gs":
-            raise click.BadParameter("Only GCS is supported for logging upload")
-        tmp_file = NamedTemporaryFile(delete=False)
-        logger.info("Logging to temporary file %s", tmp_file.name)
-        handler = logging.FileHandler(tmp_file.name)
-        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-        handler.setFormatter(formatter)
-        handler.setLevel(logging.DEBUG)
-        logger.addHandler(handler)
-        ctx.obj["local_log_file"] = tmp_file.name
-        ctx.obj["local_log_file_obj"] = tmp_file
-        ctx.obj["gcp_log_file"] = log_file
-        return tmp_file.name
-
-    else:
-        local_file = Path(log_file)
-        if local_file.exists() and local_file.is_dir():
-            raise click.BadParameter("Log file is a directory")
-        if local_file.exists() and local_file.is_file():
-            local_file.unlink()
-        if not local_file.exists():
-            local_file.parent.mkdir(parents=True, exist_ok=True)
-            local_file.touch()
-        logger.info("Logging to %s", local_file)
-        handler = logging.FileHandler(local_file)
-        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-        handler.setFormatter(formatter)
-        handler.setLevel(logging.DEBUG)
-        logger.addHandler(handler)
-        return str(local_file)
-
-
-def teardown_cli(ctx: click.Context) -> None:
-    """Teardown the gentroutils cli.
-
-    This function is used to as a teardown function for the CLI.
-    This will upload the log file to the GCP bucket if the `upload_to_gcp` flag is set in the context object.
-
-    Args:
-        ctx (click.Context): click context
-    """
-    if "upload_to_gcp" in ctx.obj and ctx.obj["upload_to_gcp"]:
-        gcp_file = ctx.obj["gcp_log_file"]
-        local_file = ctx.obj["local_log_file"]
-        with open(local_file, "r") as f:
-            content = f.read()
-        try:
-            client = storage.Client()
-            bucket_name = urlparse(gcp_file).netloc
-            bucket = client.bucket(bucket_name=bucket_name)
-            file_name = urlparse(gcp_file).path.lstrip("/")
-            blob = bucket.blob(file_name)
-            logger.info("Uploading %s to %s", local_file, gcp_file)
-            if ctx.obj["dry_run"]:
-                logger.info("Dry run, skipping the upload of the log file")
-            else:
-                blob.upload_from_string(content)
-                ctx.obj["local_log_file_obj"].close()
-        except Exception as e:
-            msg = f"Failed to upload log file to GCP {e}"
-            logger.error(click.style(msg, fg="red"))
-    logger.info("Finished, elapsed time %s seconds", time.time() - ctx.obj["execution_start"])
-
-
-def set_log_lvl(_: click.Context, param: click.Option, value: int) -> int:
-    """Set logging level based on the number of provided `v` flags.
-
-    This is a callback function called by the click.Option [-v] flag.
-    For example
-    `-vv` - DEBUG
-    `-v`  - INFO
-    `no flag - ERROR
-
-    Args:
-        param (click.Option): click option
-        value (int): logging level
-
-    Returns:
-        int: logging level
-    """
-    logger.info("Extracting log level from the %s", param)
-    log_lvls = {0: logging.ERROR, 1: logging.INFO, 2: logging.DEBUG}
-    log_lvl = log_lvls.get(value, logging.DEBUG)
-    handler = logging.StreamHandler(sys.stdout)
-    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-    handler.setFormatter(formatter)
-    handler.setLevel(log_lvl)
-    logger.addHandler(handler)
-    return log_lvl
-
-
-def coro(f):
-    """Corutine wrapper for synchronous functions."""
-
-    @wraps(f)
-    def wrapper(*args, **kwargs):
-        """Wrapper around the synchronous function."""
-        return asyncio.run(f(*args, **kwargs))
-
-    return wrapper
-
-
-__all__ = ["set_log_file", "set_log_lvl", "coro", "logger", "teardown_cli"]

gentroutils/commands/validate_gwas_curation.py

@@ -1,165 +0,0 @@
-"""Validate gwas catalog manual curation file."""
-
-from __future__ import annotations
-
-import logging
-import re
-import sys
-from enum import Enum
-from pathlib import Path
-from typing import TypeVar
-
-import click
-import great_expectations as gx
-from click import Argument, BadParameter
-from great_expectations import expectations as gxe
-
-T = TypeVar("T")
-
-
-logger = logging.getLogger("gentroutils")
-DATASOURCE_NAME = "GWAS Catalog curation"
-
-
-class Lnum(Enum):
-    """List convertable enum."""
-
-    @classmethod
-    def as_list(cls) -> list[T]:
-        """Convert enum to list of strings."""
-        return list(map(lambda c: c.value, cls))
-
-
-class ColumnSet(Lnum):
-    """Expected column names for curation file."""
-
-    STUDY_ID = "studyId"
-    STUDY_TYPE = "studyType"
-    FLAG = "analysisFlag"
-    QUALITY_CONTROL = "qualityControl"
-    IS_CURATED = "isCurated"
-    PUBMED = "pubmedId"
-    PUBLICATION_TITLE = "publicationTitle"
-    TRAIT = "traitFromSource"
-
-
-class StudyType(Lnum):
-    """Expected studyType column values."""
-
-    NO_LICENCE = "no_licence"
-    PQTL = "pQTL"
-
-
-class AnalysisFlag(Lnum):
-    """Expected analysisFlag column values."""
-
-    CC = "Case-case study"
-    EXWAS = "ExWAS"
-    GXE = "GxE"
-    GXG = "GxG"
-    METABOLITE = "Metabolite"
-    MULTIVARIATE = "Multivariate analysis"
-    NON_ADDITIVE = "Non-additive model"
-
-
-class IsCurated(Lnum):
-    """Expected isCurated column values."""
-
-    YES = True
-
-
-def _validate_input_file_name(_: click.Context, param: Argument, value: str) -> str:
-    """Assert file comes from local fs and exists."""
-    logger.debug("Validating %s variable with %s value", param, value)
-    import os
-
-    logger.info(os.getcwd())
-    pattern = re.compile(r"^[\w*/.-]*$")
-    _match = pattern.fullmatch(value)
-    if not _match:
-        logger.error("%s is not a local file.", value)
-        raise BadParameter("Provided path is not local.")
-    p = Path(value)
-    if p.is_dir():
-        logger.error("%s is a directory.", value)
-        raise BadParameter("Provided path is a directory.")
-    if not p.exists():
-        logger.error("%s does not exit.", value)
-        raise BadParameter("Provided path does not exist.")
-    return str(p)
-
-
-def split_source_path(source_path: str) -> tuple[Path, str]:
-    """Split the source path into directory name and filename"""
-    p = Path(source_path)
-    return p.parent, p.name
-
-
-@click.command(name="validate-gwas-curation")
-@click.argument("source_path", type=click.UNPROCESSED, callback=_validate_input_file_name)
-@click.pass_context
-def validate_gwas_curation(ctx: click.Context, source_path: str) -> None:  # noqa: DOC101, D0C103
-    """Validate GWAS catalog manual curation file.
-
-    \b
-    gentroutils -vvv validate-gwas-curation GWAS_Catalog_study_curation.tsv
-
-    """
-    logger.info("Using %s as curation input.", source_path)
-
-    dry_run = ctx.obj["dry_run"]
-    if dry_run:
-        logger.info("Running in --dry-run mode, exitting.")
-        sys.exit(0)
-
-    logger.info("Building great expectations context...")
-    context = gx.get_context(mode="ephemeral")
-    directory, file = split_source_path(source_path)
-    data_source = context.data_sources.add_pandas_filesystem(name=DATASOURCE_NAME, base_directory=directory)
-
-    logger.info("Using %s datasource.", DATASOURCE_NAME)
-    logger.debug("Adding csv asset from %s", file)
-    file_tsv_asset = data_source.add_csv_asset(name="manual_curation", sep="\t", header=0)
-    logger.debug("Adding batch definion path %s", file)
-    batch_definition = file_tsv_asset.add_batch_definition_path(name="manual_curation", path=file)
-
-    logger.info("Building expectation suite...")
-
-    suite = gx.ExpectationSuite(name="Curation Validation")
-    context.suites.add(suite)
-    suite.add_expectation(gxe.ExpectTableColumnsToMatchSet(column_set=ColumnSet.as_list(), exact_match=True))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.PUBMED.value, type_="int"))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.PUBLICATION_TITLE.value, type_="str"))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.TRAIT.value, type_="str"))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.STUDY_ID.value, type_="str"))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.STUDY_TYPE.value, type_="str"))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.FLAG.value, type_="str"))
-    suite.add_expectation(
-        gxe.ExpectColumnDistinctValuesToBeInSet(column=ColumnSet.STUDY_TYPE.value, value_set=StudyType.as_list())
-    )
-    suite.add_expectation(gxe.ExpectColumnDistinctValuesToBeInSet(column=ColumnSet.FLAG.value, value_set=AnalysisFlag.as_list()))
-    suite.add_expectation(gxe.ExpectColumnValueLengthsToEqual(column=ColumnSet.PUBMED.value, value=8))
-    suite.add_expectation(gxe.ExpectColumnValuesToMatchRegex(column=ColumnSet.STUDY_ID.value, regex=r"^GCST\d+$"))
-    suite.add_expectation(gxe.ExpectColumnValuesToNotBeNull(column=ColumnSet.STUDY_ID.value))
-    suite.add_expectation(gxe.ExpectColumnValuesToBeUnique(column=ColumnSet.STUDY_ID.value))
-    suite.save()
-    logger.info("Building validation definition...")
-    validation_definition = gx.ValidationDefinition(data=batch_definition, suite=suite, name="Curation Validation")
-    result = validation_definition.run()
-
-    logger.info(
-        click.style("Validation succeded" if result["success"] else "Validation failed", "green" if result["success"] else "red")
-    )
-    if not result["success"]:
-        for res in result["results"]:
-            if not res["success"]:
-                logger.error(
-                    "Expectation %s for column %s run with %s ",
-                    res["expectation_config"]["type"],
-                    res["expectation_config"]["kwargs"]["column"]
-                    if "column" in res["expectation_config"]["kwargs"]
-                    else res["expectation_config"]["kwargs"]["column_set"],
-                    "succeded" if res["success"] else "failed",
-                )
-                logger.error(res)
-        sys.exit(1)

gentroutils-1.5.0.dist-info/METADATA

@@ -1,135 +0,0 @@
-Metadata-Version: 2.4
-Name: gentroutils
-Version: 1.5.0
-Summary: Open Targets python genetics utility CLI tools
-Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
-License-Expression: Apache-2.0
-License-File: LICENSE
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Healthcare Industry
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: Unix
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
-Requires-Python: <3.13,>=3.10
-Requires-Dist: click>=8.1.7
-Requires-Dist: google-cloud-storage>=2.18.1
-Requires-Dist: great-expectations>=1.3.4
-Requires-Dist: pyfiglet>=1.0.2
-Requires-Dist: requests>=2.32.3
-Description-Content-Type: text/markdown
-
-# gentroutils
-[![checks](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml/badge.svg?branch=dev)](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml)
-![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
-[![release](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml/badge.svg)](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml)
-
-Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
-
-## Installation
-
-```
-pip install gentroutils
-```
-
-## Available commands
-
-To see all available commands after installation run
-
-```{bash}
-gentroutils --help
-```
-
-### Updating gwas catalog metadata
-
-To update gwas catalog metadata run folliwing command
-
-```bash
-gentroutils  -vvv -q gs://ot_orchestration/tests/gentroutils/log.txt  update-gwas-curation-metadata \
--f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://ot_orchestration/tests/gentroutils/gwas-catalog-associations_ontology-annotated.tsv \
--f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://ot_orchestration/tests/gentroutils/gwas-catalog-download-studies-v1.0.3.1.txt \
--f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://ot_orchestration/tests/gentroutils/gwas-catalog-download-ancestries-v1.0.3.1.txt \
--g https://www.ebi.ac.uk/gwas/api/search/stats
-```
-
-The command `update-gwas-curation-metadata` fetches the data from the ftp server and transfers them to the gcp without intermediate temporary files. The download(s) and upload(s) are made asyncronously.
-
-### Validate gwas catalog curation file
-
-To validate gwas catalog curation file after manual curation to see if all expectation tests are passing.
-
-```bash
-gentroutils -vvv validate-gwas-curation GWAS_Catalog_study_curation.tsv
-```
-
-validation is only allowed on the local file, the curation should follow format requirements defined in [OT curation](https://github.com/opentargets/curation/blob/master/genetics/GWAS_Catalog_study_curation.tsv)
-
-
-### Validate manual curation
-
-To validate the manually curated file, run the following command
-
-```bash
-gentroutils -vvv validate-gwas-curation tests/data/manual_curation/correct_curation.tsv
-```
-
-The command will validate the file and return the results of the validation if the issues are found.
-
-## Read before running
-
-The logs from the command are saved under the `-q` log file, if specified `gcp` log file, then the file will be uploaded after the command has run.
-
-To test the command run it with `-d` == `--dry-run`, this will just mark the input and output destinations.
-To allow for full logs to be transmitted to the log file, use `-vvv` to increase the verbosity of the logs
-
-> [!NOTE]
-> Change the path to the output `gcp` files to make sure they are saved under requested path
-
-> [!WARNING]
-> Please read before running the command!:
->
-> * The above command has some default values set for the input and output files, make sure you test them in `--dry-run` so the existing files will not get overwritten!
-> * Make sure to run `gcloud auth application-default login` to allow to use Google Cloud Python SDK before running the command
-
-
-
-## Contribute
-
-To be able to contribute to the project you need to set it up. This project
-runs on:
-
-- [x] python 3.10.8
-- [x] uv (dependency manager)
-
-To set up the project run
-
-```{bash}
-make dev
-```
-
-The command will install above dependencies (initial requirements are curl and bash) if not present and
-install all python dependencies listed in `pyproject.toml`. Finally the command will install `pre-commit` hooks
-requred to be run before the commit is created.
-
-The project has additional `dev` dependencies that include the list of packages used for testing purposes.
-All of the `dev` depnendencies are automatically installed by `uv`.
-
-To see all available dev commands
-
-Run following command to see all available dev commands
-
-```{bash}
-make help
-```
-
-### Manual testing of CLI module
-
-To check CLI execution manually you need to run
-
-```{bash}
-uv run gentroutils
-```

gentroutils-1.5.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-gentroutils/__init__.py,sha256=lCl9VmPVU2hkZa_kh5NipCrNufbcTH00iKF-6WvE0gg,1319
-gentroutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gentroutils/commands/__init__.py,sha256=940jwl8BvK0gTN2mRb_js8Kl74rgff-benDSEO5MuBc,326
-gentroutils/commands/update_gwas_curation_metadata.py,sha256=zOIv5l_84kBlqyXAXJHnwH5OrPKVlObuHl6WhwlZC90,10903
-gentroutils/commands/utils.py,sha256=XEZogKn6ZQKaSwKms7UtJU5h_eGrrHM-FofCoU2k884,5312
-gentroutils/commands/validate_gwas_curation.py,sha256=CWlQ8uI8JAK9l4MyjFNmjDLeMOolrcA3neVaBdAGN80,6148
-gentroutils-1.5.0.dist-info/METADATA,sha256=LYbG6KpN19EN1O3ZCGhXOv_yRAYVPj1odQ5742ELoDQ,5154
-gentroutils-1.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-gentroutils-1.5.0.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
-gentroutils-1.5.0.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
-gentroutils-1.5.0.dist-info/RECORD,,

gentroutils-1.5.0.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-gentroutils = gentroutils:cli