gentroutils 0.2.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

gentroutils/__init__.py

@@ -8,8 +8,9 @@ import time
 import click
 import pyfiglet
 
-from gentroutils.commands.update_gwas_curation_metadata import (
+from gentroutils.commands import (
     update_gwas_curation_metadata_command,
+    validate_gwas_curation,
 )
 from gentroutils.commands.utils import set_log_file, set_log_lvl, teardown_cli
 
@@ -40,5 +41,6 @@ def cli(ctx: click.Context, **kwargs: dict[str, str]) -> None:
 
 
 cli.add_command(update_gwas_curation_metadata_command)
+cli.add_command(validate_gwas_curation)
 
 __all__ = ["cli"]

gentroutils/commands/__init__.py

@@ -3,5 +3,9 @@
 from gentroutils.commands.update_gwas_curation_metadata import (
     update_gwas_curation_metadata_command,
 )
+from gentroutils.commands.validate_gwas_curation import validate_gwas_curation
 
-__all__ = ["update_gwas_curation_metadata_command"]
+__all__ = [
+    "update_gwas_curation_metadata_command",
+    "validate_gwas_curation",
+]

gentroutils/commands/update_gwas_curation_metadata.py

@@ -90,10 +90,7 @@ async def update_gwas_curation_metadata_command(
             MAX_CONCURRENT_CONNECTIONS,
         )
         sys.exit(1)
-    uri_map = [
-        {"input": urlparse(ftp_file), "output": urlparse(gcp_file)}
-        for ftp_file, gcp_file in file_to_transfer
-    ]
+    uri_map = [{"input": urlparse(ftp_file), "output": urlparse(gcp_file)} for ftp_file, gcp_file in file_to_transfer]
     transfer_tasks = generate_transfer_tasks(uri_map, dry_run)
 
     # capture latest release metadata
@@ -117,9 +114,7 @@ async def update_gwas_curation_metadata_command(
     logger.info("gwas_curation_update step completed.")
 
 
-def generate_transfer_tasks(
-    uri_map: list[dict[str, ParseResult]], dry_run: bool
-) -> list[asyncio.Task[None]]:
+def generate_transfer_tasks(uri_map: list[dict[str, ParseResult]], dry_run: bool) -> list[asyncio.Task[None]]:
     """Generate transfer tasks.
 
     Args:
@@ -196,9 +191,7 @@ def generate_transfer_tasks(
     return transfer_tasks
 
 
-async def sync_from_http_to_gcp(
-    url: str, gcp_bucket: str, gcp_prefix: str, gcp_file: str, *, dry_run: bool = True
-) -> None:
+async def sync_from_http_to_gcp(url: str, gcp_bucket: str, gcp_prefix: str, gcp_file: str, *, dry_run: bool = True) -> None:
     """Sync file from HTTP and upload to GCP.
 
     This function fetches the data from the provided HTTP URL and uploads the content
@@ -271,7 +264,8 @@ async def sync_from_ftp_to_gcp(
             gcp_file,
         )
         return
-    with FTP(ftp_server) as ftp:
+    with FTP() as ftp:
+        ftp.connect(ftp_server)
         ftp.login()
         bucket = storage.Client().bucket(gcp_bucket)
         gcp_path = f"{gcp_prefix}/{gcp_file}" if gcp_prefix else gcp_file
@@ -282,9 +276,7 @@ async def sync_from_ftp_to_gcp(
         if dir_match:
             logger.info("Found release date!: %s", dir_match.group("release_date"))
         buffer = io.BytesIO()
-        logger.info(
-            "Retrieving data from: ftp://%s/%s/%s.", ftp_server, ftp_prefix, ftp_file
-        )
+        logger.info("Retrieving data from: ftp://%s/%s/%s.", ftp_server, ftp_prefix, ftp_file)
         ftp.retrbinary(f"RETR {ftp_file}", lambda x: buffer.write(x))
         content = buffer.getvalue().decode("utf-8")
         buffer.close()

gentroutils/commands/utils.py

@@ -52,9 +52,7 @@ def set_log_file(ctx: click.Context, param: click.Option, log_file: str) -> str:
         tmp_file = NamedTemporaryFile(delete=False)
         logger.info("Logging to temporary file %s", tmp_file.name)
         handler = logging.FileHandler(tmp_file.name)
-        formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-        )
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         handler.setFormatter(formatter)
         handler.setLevel(logging.DEBUG)
         logger.addHandler(handler)
@@ -74,9 +72,7 @@ def set_log_file(ctx: click.Context, param: click.Option, log_file: str) -> str:
             local_file.touch()
         logger.info("Logging to %s", local_file)
         handler = logging.FileHandler(local_file)
-        formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-        )
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         handler.setFormatter(formatter)
         handler.setLevel(logging.DEBUG)
         logger.addHandler(handler)
@@ -112,9 +108,7 @@ def teardown_cli(ctx: click.Context) -> None:
         except Exception as e:
             msg = f"Failed to upload log file to GCP {e}"
             logger.error(click.style(msg, fg="red"))
-    logger.info(
-        "Finished, elapsed time %s seconds", time.time() - ctx.obj["execution_start"]
-    )
+    logger.info("Finished, elapsed time %s seconds", time.time() - ctx.obj["execution_start"])
 
 
 def set_log_lvl(_: click.Context, param: click.Option, value: int) -> int:
@@ -137,9 +131,7 @@ def set_log_lvl(_: click.Context, param: click.Option, value: int) -> int:
     log_lvls = {0: logging.ERROR, 1: logging.INFO, 2: logging.DEBUG}
     log_lvl = log_lvls.get(value, logging.DEBUG)
     handler = logging.StreamHandler(sys.stdout)
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
     handler.setFormatter(formatter)
     handler.setLevel(log_lvl)
     logger.addHandler(handler)

gentroutils/commands/validate_gwas_curation.py

@@ -0,0 +1,165 @@
+"""Validate gwas catalog manual curation file."""
+
+from __future__ import annotations
+
+import logging
+import re
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import TypeVar
+
+import click
+import great_expectations as gx
+from click import Argument, BadParameter
+from great_expectations import expectations as gxe
+
+T = TypeVar("T")
+
+
+logger = logging.getLogger("gentroutils")
+DATASOURCE_NAME = "GWAS Catalog curation"
+
+
+class Lnum(Enum):
+    """List convertable enum."""
+
+    @classmethod
+    def as_list(cls) -> list[T]:
+        """Convert enum to list of strings."""
+        return list(map(lambda c: c.value, cls))
+
+
+class ColumnSet(Lnum):
+    """Expected column names for curation file."""
+
+    STUDY_ID = "studyId"
+    STUDY_TYPE = "studyType"
+    FLAG = "analysisFlag"
+    QUALITY_CONTROL = "qualityControl"
+    IS_CURATED = "isCurated"
+    PUBMED = "pubmedId"
+    PUBLICATION_TITLE = "publicationTitle"
+    TRAIT = "traitFromSource"
+
+
+class StudyType(Lnum):
+    """Expected studyType column values."""
+
+    NO_LICENCE = "no_licence"
+    PQTL = "pQTL"
+
+
+class AnalysisFlag(Lnum):
+    """Expected analysisFlag column values."""
+
+    CC = "Case-case study"
+    EXWAS = "ExWAS"
+    GXE = "GxE"
+    GXG = "GxG"
+    METABOLITE = "Metabolite"
+    MULTIVARIATE = "Multivariate analysis"
+    NON_ADDITIVE = "Non-additive model"
+
+
+class IsCurated(Lnum):
+    """Expected isCurated column values."""
+
+    YES = True
+
+
+def _validate_input_file_name(_: click.Context, param: Argument, value: str) -> str:
+    """Assert file comes from local fs and exists."""
+    logger.debug("Validating %s variable with %s value", param, value)
+    import os
+
+    logger.info(os.getcwd())
+    pattern = re.compile(r"^[\w*/.-]*$")
+    _match = pattern.fullmatch(value)
+    if not _match:
+        logger.error("%s is not a local file.", value)
+        raise BadParameter("Provided path is not local.")
+    p = Path(value)
+    if p.is_dir():
+        logger.error("%s is a directory.", value)
+        raise BadParameter("Provided path is a directory.")
+    if not p.exists():
+        logger.error("%s does not exit.", value)
+        raise BadParameter("Provided path does not exist.")
+    return str(p)
+
+
+def split_source_path(source_path: str) -> tuple[Path, str]:
+    """Split the source path into directory name and filename"""
+    p = Path(source_path)
+    return p.parent, p.name
+
+
+@click.command(name="validate-gwas-curation")
+@click.argument("source_path", type=click.UNPROCESSED, callback=_validate_input_file_name)
+@click.pass_context
+def validate_gwas_curation(ctx: click.Context, source_path: str) -> None:  # noqa: DOC101, D0C103
+    """Validate GWAS catalog manual curation file.
+
+    \b
+    gentroutils -vvv validate-gwas-curation GWAS_Catalog_study_curation.tsv
+
+    """
+    logger.info("Using %s as curation input.", source_path)
+
+    dry_run = ctx.obj["dry_run"]
+    if dry_run:
+        logger.info("Running in --dry-run mode, exitting.")
+        sys.exit(0)
+
+    logger.info("Building great expectations context...")
+    context = gx.get_context(mode="ephemeral")
+    directory, file = split_source_path(source_path)
+    data_source = context.data_sources.add_pandas_filesystem(name=DATASOURCE_NAME, base_directory=directory)
+
+    logger.info("Using %s datasource.", DATASOURCE_NAME)
+    logger.debug("Adding csv asset from %s", file)
+    file_tsv_asset = data_source.add_csv_asset(name="manual_curation", sep="\t", header=0)
+    logger.debug("Adding batch definion path %s", file)
+    batch_definition = file_tsv_asset.add_batch_definition_path(name="manual_curation", path=file)
+
+    logger.info("Building expectation suite...")
+
+    suite = gx.ExpectationSuite(name="Curation Validation")
+    context.suites.add(suite)
+    suite.add_expectation(gxe.ExpectTableColumnsToMatchSet(column_set=ColumnSet.as_list(), exact_match=True))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.PUBMED.value, type_="int"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.PUBLICATION_TITLE.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.TRAIT.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.STUDY_ID.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.STUDY_TYPE.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.FLAG.value, type_="str"))
+    suite.add_expectation(
+        gxe.ExpectColumnDistinctValuesToBeInSet(column=ColumnSet.STUDY_TYPE.value, value_set=StudyType.as_list())
+    )
+    suite.add_expectation(gxe.ExpectColumnDistinctValuesToBeInSet(column=ColumnSet.FLAG.value, value_set=AnalysisFlag.as_list()))
+    suite.add_expectation(gxe.ExpectColumnValueLengthsToEqual(column=ColumnSet.PUBMED.value, value=8))
+    suite.add_expectation(gxe.ExpectColumnValuesToMatchRegex(column=ColumnSet.STUDY_ID.value, regex=r"^GCST\d+$"))
+    suite.add_expectation(gxe.ExpectColumnValuesToNotBeNull(column=ColumnSet.STUDY_ID.value))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeUnique(column=ColumnSet.STUDY_ID.value))
+    suite.save()
+    logger.info("Building validation definition...")
+    validation_definition = gx.ValidationDefinition(data=batch_definition, suite=suite, name="Curation Validation")
+    result = validation_definition.run()
+
+    logger.info(
+        click.style("Validation succeded" if result["success"] else "Validation failed", "green" if result["success"] else "red")
+    )
+    if not result["success"]:
+        for res in result["results"]:
+            if not res["success"]:
+                logger.error(
+                    "Expectation %s for column %s run with %s ",
+                    res["expectation_config"]["type"],
+                    res["expectation_config"]["kwargs"]["column"]
+                    if "column" in res["expectation_config"]["kwargs"]
+                    else res["expectation_config"]["kwargs"]["column_set"],
+                    "succeded" if res["success"] else "failed",
+                )
+                logger.error(res)
+        sys.exit(1)

{gentroutils-0.2.0.dist-info → gentroutils-1.5.0.dist-info}/METADATA

@@ -1,9 +1,10 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: gentroutils
-Version: 0.2.0
+Version: 1.5.0
 Summary: Open Targets python genetics utility CLI tools
 Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
-License: Apache-2.0
+License-Expression: Apache-2.0
+License-File: LICENSE
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Healthcare Industry
 Classifier: Intended Audience :: Science/Research
@@ -11,18 +12,21 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
-Requires-Python: >=3.10
+Requires-Python: <3.13,>=3.10
 Requires-Dist: click>=8.1.7
 Requires-Dist: google-cloud-storage>=2.18.1
+Requires-Dist: great-expectations>=1.3.4
 Requires-Dist: pyfiglet>=1.0.2
 Requires-Dist: requests>=2.32.3
 Description-Content-Type: text/markdown
 
 # gentroutils
-
-[![Tests](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml/badge.svg?event=push)](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml)
+[![checks](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml/badge.svg?branch=dev)](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml)
 ![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
+[![release](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml/badge.svg)](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml)
 
 Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
 
@@ -54,6 +58,29 @@ gentroutils  -vvv -q gs://ot_orchestration/tests/gentroutils/log.txt  update-gwa
 
 The command `update-gwas-curation-metadata` fetches the data from the ftp server and transfers them to the gcp without intermediate temporary files. The download(s) and upload(s) are made asyncronously.
 
+### Validate gwas catalog curation file
+
+To validate gwas catalog curation file after manual curation to see if all expectation tests are passing.
+
+```bash
+gentroutils -vvv validate-gwas-curation GWAS_Catalog_study_curation.tsv
+```
+
+validation is only allowed on the local file, the curation should follow format requirements defined in [OT curation](https://github.com/opentargets/curation/blob/master/genetics/GWAS_Catalog_study_curation.tsv)
+
+
+### Validate manual curation
+
+To validate the manually curated file, run the following command
+
+```bash
+gentroutils -vvv validate-gwas-curation tests/data/manual_curation/correct_curation.tsv
+```
+
+The command will validate the file and return the results of the validation if the issues are found.
+
+## Read before running
+
 The logs from the command are saved under the `-q` log file, if specified `gcp` log file, then the file will be uploaded after the command has run.
 
 To test the command run it with `-d` == `--dry-run`, this will just mark the input and output destinations.
@@ -68,13 +95,14 @@ To allow for full logs to be transmitted to the log file, use `-vvv` to increase
 > * The above command has some default values set for the input and output files, make sure you test them in `--dry-run` so the existing files will not get overwritten!
 > * Make sure to run `gcloud auth application-default login` to allow to use Google Cloud Python SDK before running the command
 
+
+
 ## Contribute
 
 To be able to contribute to the project you need to set it up. This project
 runs on:
 
 - [x] python 3.10.8
-- [x] rye (package manager)
 - [x] uv (dependency manager)
 
 To set up the project run
@@ -88,7 +116,7 @@ install all python dependencies listed in `pyproject.toml`. Finally the command
 requred to be run before the commit is created.
 
 The project has additional `dev` dependencies that include the list of packages used for testing purposes.
-All of the `dev` depnendencies are automatically installed by `rye`.
+All of the `dev` depnendencies are automatically installed by `uv`.
 
 To see all available dev commands
 
@@ -103,5 +131,5 @@ make help
 To check CLI execution manually you need to run
 
 ```{bash}
-rye run gentroutils
+uv run gentroutils
 ```

gentroutils-1.5.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+gentroutils/__init__.py,sha256=lCl9VmPVU2hkZa_kh5NipCrNufbcTH00iKF-6WvE0gg,1319
+gentroutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gentroutils/commands/__init__.py,sha256=940jwl8BvK0gTN2mRb_js8Kl74rgff-benDSEO5MuBc,326
+gentroutils/commands/update_gwas_curation_metadata.py,sha256=zOIv5l_84kBlqyXAXJHnwH5OrPKVlObuHl6WhwlZC90,10903
+gentroutils/commands/utils.py,sha256=XEZogKn6ZQKaSwKms7UtJU5h_eGrrHM-FofCoU2k884,5312
+gentroutils/commands/validate_gwas_curation.py,sha256=CWlQ8uI8JAK9l4MyjFNmjDLeMOolrcA3neVaBdAGN80,6148
+gentroutils-1.5.0.dist-info/METADATA,sha256=LYbG6KpN19EN1O3ZCGhXOv_yRAYVPj1odQ5742ELoDQ,5154
+gentroutils-1.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gentroutils-1.5.0.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
+gentroutils-1.5.0.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
+gentroutils-1.5.0.dist-info/RECORD,,

{gentroutils-0.2.0.dist-info → gentroutils-1.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.26.3
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

gentroutils-0.2.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-gentroutils/__init__.py,sha256=aHDzbBMrnsgdcO_FfsYCbbPXProynwB7_2nfyc4UGp8,1281
-gentroutils/commands/__init__.py,sha256=avkqzwa1ck__rLVN0Wqfpr3eHtKS6TvyPeeaHcguJuw,210
-gentroutils/commands/update_gwas_curation_metadata.py,sha256=4Pb2YdEnfulQklFh0KBvAOBnylCsDIAye7Keq2dC0mY,10937
-gentroutils/commands/utils.py,sha256=zYIzu47f-_a3nBeVXRR5xg5QiklrwES8uYNNhjed7gA,5384
-gentroutils-0.2.0.dist-info/METADATA,sha256=lMJ2JdqokHojQaY-hWhs9IvCJ4ei4vBpOfsOAfgBw4E,4061
-gentroutils-0.2.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-gentroutils-0.2.0.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
-gentroutils-0.2.0.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
-gentroutils-0.2.0.dist-info/RECORD,,