gentroutils 0.1.5__py3-none-any.whl → 1.5.0__py3-none-any.whl

gentroutils/__init__.py

@@ -8,8 +8,9 @@ import time
 import click
 import pyfiglet
 
-from gentroutils.commands.update_gwas_curation_metadata import (
+from gentroutils.commands import (
     update_gwas_curation_metadata_command,
+    validate_gwas_curation,
 )
 from gentroutils.commands.utils import set_log_file, set_log_lvl, teardown_cli
 
@@ -40,5 +41,6 @@ def cli(ctx: click.Context, **kwargs: dict[str, str]) -> None:
 
 
 cli.add_command(update_gwas_curation_metadata_command)
+cli.add_command(validate_gwas_curation)
 
 __all__ = ["cli"]

gentroutils/commands/__init__.py

@@ -3,5 +3,9 @@
 from gentroutils.commands.update_gwas_curation_metadata import (
     update_gwas_curation_metadata_command,
 )
+from gentroutils.commands.validate_gwas_curation import validate_gwas_curation
 
-__all__ = ["update_gwas_curation_metadata_command"]
+__all__ = [
+    "update_gwas_curation_metadata_command",
+    "validate_gwas_curation",
+]

gentroutils/commands/update_gwas_curation_metadata.py

@@ -21,27 +21,15 @@ MAX_CONCURRENT_CONNECTIONS = 10
 CURATED_INPUTS = (
     (
         "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_associations_ontology_annotated.tsv",
+        "gs://gwas_catalog_inputs/gwas_catalog_associations_ontology_annotated.tsv",
     ),
     (
         "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_studies.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-studies-v1.0.3.1.tsv",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_studies.tsv",
+        "gs://gwas_catalog_inputs/gwas_catalog_download_studies.tsv",
     ),
     (
         "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_ancestries.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_ancestries.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/harmonised_list.txt",
-        "gs://gwas_catalog_data/curated_inputs/harmonised_list.txt",
+        "gs://gwas_catalog_inputs/gwas_catalog_download_ancestries.tsv",
     ),
 )
 
@@ -75,9 +63,7 @@ async def update_gwas_curation_metadata_command(
     This is the script to fetch the latest GWAS Catalog data files that include:
     - [x] gwas-catalog-associations_ontology-annotated.tsv - list of associations with ontology annotations by GWAS Catalog
     - [x] gwas-catalog-download-studies-v1.0.3.1.txt - list of published studies by GWAS Catalog
-    - [x] gwas-catalog-unpublished-studies-v1.0.3.1.tsv - list of unpublished studies by GWAS Catalog
     - [x] gwas-catalog-download-ancestries-v1.0.3.1.txt - list of published studies by GWAS Catalog
-    - [x] gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv - list of unpublished studies by GWAS Catalog
 
     \b
     By default all GWAS Catalog data files are uploaded from GWAS Catalog FTP server to Open Targets GCP bucket.
@@ -86,13 +72,9 @@ async def update_gwas_curation_metadata_command(
 
     \b
     gentroutils --log-file gs://gwas_catalog_data/curated_inputs/log.txt update-gwas-curation-metadata \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://gwas_catalog_data/curated_inputs/gwas_catalog_associations_ontology_annotated.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_studies.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-studies-v1.0.3.1.tsv gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_studies.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_ancestries.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_ancestries.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/harmonised_list.txt gs://gwas_catalog_data/curated_inputs/harmonised_list.txt \\
-    -f https://raw.githubusercontent.com/opentargets/curation/master/genetics/GWAS_Catalog_study_curation.tsv gs://gwas_catalog_data/manifests/gwas_catalog_study_curation.tsv \\
+    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://gwas_catalog_data/gwas_catalog_associations_ontology_annotated.tsv \\
+    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://gwas_catalog_inputs/gwas_catalog_download_studies.tsv \\
+    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://gwas_catalog_inputs/gwas_catalog_download_ancestries.tsv \\
     -g https://www.ebi.ac.uk/gwas/api/search/stats
 
 
@@ -108,10 +90,7 @@ async def update_gwas_curation_metadata_command(
             MAX_CONCURRENT_CONNECTIONS,
         )
         sys.exit(1)
-    uri_map = [
-        {"input": urlparse(ftp_file), "output": urlparse(gcp_file)}
-        for ftp_file, gcp_file in file_to_transfer
-    ]
+    uri_map = [{"input": urlparse(ftp_file), "output": urlparse(gcp_file)} for ftp_file, gcp_file in file_to_transfer]
     transfer_tasks = generate_transfer_tasks(uri_map, dry_run)
 
     # capture latest release metadata
@@ -135,9 +114,7 @@ async def update_gwas_curation_metadata_command(
     logger.info("gwas_curation_update step completed.")
 
 
-def generate_transfer_tasks(
-    uri_map: list[dict[str, ParseResult]], dry_run: bool
-) -> list[asyncio.Task[None]]:
+def generate_transfer_tasks(uri_map: list[dict[str, ParseResult]], dry_run: bool) -> list[asyncio.Task[None]]:
     """Generate transfer tasks.
 
     Args:
@@ -214,9 +191,7 @@ def generate_transfer_tasks(
     return transfer_tasks
 
 
-async def sync_from_http_to_gcp(
-    url: str, gcp_bucket: str, gcp_prefix: str, gcp_file: str, *, dry_run: bool = True
-) -> None:
+async def sync_from_http_to_gcp(url: str, gcp_bucket: str, gcp_prefix: str, gcp_file: str, *, dry_run: bool = True) -> None:
     """Sync file from HTTP and upload to GCP.
 
     This function fetches the data from the provided HTTP URL and uploads the content
@@ -289,7 +264,8 @@ async def sync_from_ftp_to_gcp(
             gcp_file,
         )
         return
-    with FTP(ftp_server) as ftp:
+    with FTP() as ftp:
+        ftp.connect(ftp_server)
         ftp.login()
         bucket = storage.Client().bucket(gcp_bucket)
         gcp_path = f"{gcp_prefix}/{gcp_file}" if gcp_prefix else gcp_file
@@ -300,9 +276,7 @@ async def sync_from_ftp_to_gcp(
         if dir_match:
             logger.info("Found release date!: %s", dir_match.group("release_date"))
         buffer = io.BytesIO()
-        logger.info(
-            "Retrieving data from: ftp://%s/%s/%s.", ftp_server, ftp_prefix, ftp_file
-        )
+        logger.info("Retrieving data from: ftp://%s/%s/%s.", ftp_server, ftp_prefix, ftp_file)
         ftp.retrbinary(f"RETR {ftp_file}", lambda x: buffer.write(x))
         content = buffer.getvalue().decode("utf-8")
         buffer.close()

gentroutils/commands/utils.py

@@ -6,6 +6,7 @@ import sys
 import time
 from functools import wraps
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from urllib.parse import urlparse
 
 import click
@@ -39,33 +40,43 @@ def set_log_file(ctx: click.Context, param: click.Option, log_file: str) -> str:
         return ""
     logger.info("Extracting log file from the %s", param)
     upload_to_gcp = False
+
     if "://" in log_file:
         upload_to_gcp = True
+    ctx.obj["upload_to_gcp"] = upload_to_gcp
+
     if upload_to_gcp:
         parsed_uri = urlparse(log_file)
-        ctx.obj["gcp_log_file"] = log_file
         if parsed_uri.scheme != "gs":
             raise click.BadParameter("Only GCS is supported for logging upload")
-        log_file = parsed_uri.path.strip("/")
-        ctx.obj["local_log_file"] = log_file
-    ctx.obj["upload_to_gcp"] = upload_to_gcp
-
-    local_file = Path(log_file)
-    if local_file.exists() and local_file.is_dir():
-        raise click.BadParameter("Log file is a directory")
-    if local_file.exists() and local_file.is_file():
-        local_file.unlink()
-    if not local_file.exists():
-        local_file.touch()
-    logger.info("Logging to %s", local_file)
-    handler = logging.FileHandler(local_file)
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
-    handler.setFormatter(formatter)
-    handler.setLevel(logging.DEBUG)
-    logger.addHandler(handler)
-    return str(local_file)
+        tmp_file = NamedTemporaryFile(delete=False)
+        logger.info("Logging to temporary file %s", tmp_file.name)
+        handler = logging.FileHandler(tmp_file.name)
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        handler.setFormatter(formatter)
+        handler.setLevel(logging.DEBUG)
+        logger.addHandler(handler)
+        ctx.obj["local_log_file"] = tmp_file.name
+        ctx.obj["local_log_file_obj"] = tmp_file
+        ctx.obj["gcp_log_file"] = log_file
+        return tmp_file.name
+
+    else:
+        local_file = Path(log_file)
+        if local_file.exists() and local_file.is_dir():
+            raise click.BadParameter("Log file is a directory")
+        if local_file.exists() and local_file.is_file():
+            local_file.unlink()
+        if not local_file.exists():
+            local_file.parent.mkdir(parents=True, exist_ok=True)
+            local_file.touch()
+        logger.info("Logging to %s", local_file)
+        handler = logging.FileHandler(local_file)
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+        handler.setFormatter(formatter)
+        handler.setLevel(logging.DEBUG)
+        logger.addHandler(handler)
+        return str(local_file)
 
 
 def teardown_cli(ctx: click.Context) -> None:
@@ -80,16 +91,24 @@ def teardown_cli(ctx: click.Context) -> None:
     if "upload_to_gcp" in ctx.obj and ctx.obj["upload_to_gcp"]:
         gcp_file = ctx.obj["gcp_log_file"]
         local_file = ctx.obj["local_log_file"]
-        client = storage.Client()
-        bucket_name = urlparse(gcp_file).netloc
-        bucket = client.bucket(bucket_name=bucket_name)
-        blob = bucket.blob(Path(local_file).name)
-        logger.info("Uploading %s to %s", local_file, gcp_file)
-        blob.upload_from_filename(local_file)
-        Path(local_file).unlink()
-    logger.info(
-        "Finished, elapsed time %s seconds", time.time() - ctx.obj["execution_start"]
-    )
+        with open(local_file, "r") as f:
+            content = f.read()
+        try:
+            client = storage.Client()
+            bucket_name = urlparse(gcp_file).netloc
+            bucket = client.bucket(bucket_name=bucket_name)
+            file_name = urlparse(gcp_file).path.lstrip("/")
+            blob = bucket.blob(file_name)
+            logger.info("Uploading %s to %s", local_file, gcp_file)
+            if ctx.obj["dry_run"]:
+                logger.info("Dry run, skipping the upload of the log file")
+            else:
+                blob.upload_from_string(content)
+                ctx.obj["local_log_file_obj"].close()
+        except Exception as e:
+            msg = f"Failed to upload log file to GCP {e}"
+            logger.error(click.style(msg, fg="red"))
+    logger.info("Finished, elapsed time %s seconds", time.time() - ctx.obj["execution_start"])
 
 
 def set_log_lvl(_: click.Context, param: click.Option, value: int) -> int:
@@ -112,9 +131,7 @@ def set_log_lvl(_: click.Context, param: click.Option, value: int) -> int:
     log_lvls = {0: logging.ERROR, 1: logging.INFO, 2: logging.DEBUG}
     log_lvl = log_lvls.get(value, logging.DEBUG)
     handler = logging.StreamHandler(sys.stdout)
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
     handler.setFormatter(formatter)
     handler.setLevel(log_lvl)
     logger.addHandler(handler)

gentroutils/commands/validate_gwas_curation.py

@@ -0,0 +1,165 @@
+"""Validate gwas catalog manual curation file."""
+
+from __future__ import annotations
+
+import logging
+import re
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import TypeVar
+
+import click
+import great_expectations as gx
+from click import Argument, BadParameter
+from great_expectations import expectations as gxe
+
+T = TypeVar("T")
+
+
+logger = logging.getLogger("gentroutils")
+DATASOURCE_NAME = "GWAS Catalog curation"
+
+
+class Lnum(Enum):
+    """List convertable enum."""
+
+    @classmethod
+    def as_list(cls) -> list[T]:
+        """Convert enum to list of strings."""
+        return list(map(lambda c: c.value, cls))
+
+
+class ColumnSet(Lnum):
+    """Expected column names for curation file."""
+
+    STUDY_ID = "studyId"
+    STUDY_TYPE = "studyType"
+    FLAG = "analysisFlag"
+    QUALITY_CONTROL = "qualityControl"
+    IS_CURATED = "isCurated"
+    PUBMED = "pubmedId"
+    PUBLICATION_TITLE = "publicationTitle"
+    TRAIT = "traitFromSource"
+
+
+class StudyType(Lnum):
+    """Expected studyType column values."""
+
+    NO_LICENCE = "no_licence"
+    PQTL = "pQTL"
+
+
+class AnalysisFlag(Lnum):
+    """Expected analysisFlag column values."""
+
+    CC = "Case-case study"
+    EXWAS = "ExWAS"
+    GXE = "GxE"
+    GXG = "GxG"
+    METABOLITE = "Metabolite"
+    MULTIVARIATE = "Multivariate analysis"
+    NON_ADDITIVE = "Non-additive model"
+
+
+class IsCurated(Lnum):
+    """Expected isCurated column values."""
+
+    YES = True
+
+
+def _validate_input_file_name(_: click.Context, param: Argument, value: str) -> str:
+    """Assert file comes from local fs and exists."""
+    logger.debug("Validating %s variable with %s value", param, value)
+    import os
+
+    logger.info(os.getcwd())
+    pattern = re.compile(r"^[\w*/.-]*$")
+    _match = pattern.fullmatch(value)
+    if not _match:
+        logger.error("%s is not a local file.", value)
+        raise BadParameter("Provided path is not local.")
+    p = Path(value)
+    if p.is_dir():
+        logger.error("%s is a directory.", value)
+        raise BadParameter("Provided path is a directory.")
+    if not p.exists():
+        logger.error("%s does not exit.", value)
+        raise BadParameter("Provided path does not exist.")
+    return str(p)
+
+
+def split_source_path(source_path: str) -> tuple[Path, str]:
+    """Split the source path into directory name and filename"""
+    p = Path(source_path)
+    return p.parent, p.name
+
+
+@click.command(name="validate-gwas-curation")
+@click.argument("source_path", type=click.UNPROCESSED, callback=_validate_input_file_name)
+@click.pass_context
+def validate_gwas_curation(ctx: click.Context, source_path: str) -> None:  # noqa: DOC101, D0C103
+    """Validate GWAS catalog manual curation file.
+
+    \b
+    gentroutils -vvv validate-gwas-curation GWAS_Catalog_study_curation.tsv
+
+    """
+    logger.info("Using %s as curation input.", source_path)
+
+    dry_run = ctx.obj["dry_run"]
+    if dry_run:
+        logger.info("Running in --dry-run mode, exitting.")
+        sys.exit(0)
+
+    logger.info("Building great expectations context...")
+    context = gx.get_context(mode="ephemeral")
+    directory, file = split_source_path(source_path)
+    data_source = context.data_sources.add_pandas_filesystem(name=DATASOURCE_NAME, base_directory=directory)
+
+    logger.info("Using %s datasource.", DATASOURCE_NAME)
+    logger.debug("Adding csv asset from %s", file)
+    file_tsv_asset = data_source.add_csv_asset(name="manual_curation", sep="\t", header=0)
+    logger.debug("Adding batch definion path %s", file)
+    batch_definition = file_tsv_asset.add_batch_definition_path(name="manual_curation", path=file)
+
+    logger.info("Building expectation suite...")
+
+    suite = gx.ExpectationSuite(name="Curation Validation")
+    context.suites.add(suite)
+    suite.add_expectation(gxe.ExpectTableColumnsToMatchSet(column_set=ColumnSet.as_list(), exact_match=True))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.PUBMED.value, type_="int"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.PUBLICATION_TITLE.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.TRAIT.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.STUDY_ID.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.STUDY_TYPE.value, type_="str"))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeOfType(column=ColumnSet.FLAG.value, type_="str"))
+    suite.add_expectation(
+        gxe.ExpectColumnDistinctValuesToBeInSet(column=ColumnSet.STUDY_TYPE.value, value_set=StudyType.as_list())
+    )
+    suite.add_expectation(gxe.ExpectColumnDistinctValuesToBeInSet(column=ColumnSet.FLAG.value, value_set=AnalysisFlag.as_list()))
+    suite.add_expectation(gxe.ExpectColumnValueLengthsToEqual(column=ColumnSet.PUBMED.value, value=8))
+    suite.add_expectation(gxe.ExpectColumnValuesToMatchRegex(column=ColumnSet.STUDY_ID.value, regex=r"^GCST\d+$"))
+    suite.add_expectation(gxe.ExpectColumnValuesToNotBeNull(column=ColumnSet.STUDY_ID.value))
+    suite.add_expectation(gxe.ExpectColumnValuesToBeUnique(column=ColumnSet.STUDY_ID.value))
+    suite.save()
+    logger.info("Building validation definition...")
+    validation_definition = gx.ValidationDefinition(data=batch_definition, suite=suite, name="Curation Validation")
+    result = validation_definition.run()
+
+    logger.info(
+        click.style("Validation succeded" if result["success"] else "Validation failed", "green" if result["success"] else "red")
+    )
+    if not result["success"]:
+        for res in result["results"]:
+            if not res["success"]:
+                logger.error(
+                    "Expectation %s for column %s run with %s ",
+                    res["expectation_config"]["type"],
+                    res["expectation_config"]["kwargs"]["column"]
+                    if "column" in res["expectation_config"]["kwargs"]
+                    else res["expectation_config"]["kwargs"]["column_set"],
+                    "succeded" if res["success"] else "failed",
+                )
+                logger.error(res)
+        sys.exit(1)

gentroutils-1.5.0.dist-info/METADATA

@@ -0,0 +1,135 @@
+Metadata-Version: 2.4
+Name: gentroutils
+Version: 1.5.0
+Summary: Open Targets python genetics utility CLI tools
+Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: <3.13,>=3.10
+Requires-Dist: click>=8.1.7
+Requires-Dist: google-cloud-storage>=2.18.1
+Requires-Dist: great-expectations>=1.3.4
+Requires-Dist: pyfiglet>=1.0.2
+Requires-Dist: requests>=2.32.3
+Description-Content-Type: text/markdown
+
+# gentroutils
+[![checks](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml/badge.svg?branch=dev)](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml)
+![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
+[![release](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml/badge.svg)](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml)
+
+Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
+
+## Installation
+
+```
+pip install gentroutils
+```
+
+## Available commands
+
+To see all available commands after installation run
+
+```{bash}
+gentroutils --help
+```
+
+### Updating gwas catalog metadata
+
+To update gwas catalog metadata run folliwing command
+
+```bash
+gentroutils  -vvv -q gs://ot_orchestration/tests/gentroutils/log.txt  update-gwas-curation-metadata \
+-f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://ot_orchestration/tests/gentroutils/gwas-catalog-associations_ontology-annotated.tsv \
+-f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://ot_orchestration/tests/gentroutils/gwas-catalog-download-studies-v1.0.3.1.txt \
+-f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://ot_orchestration/tests/gentroutils/gwas-catalog-download-ancestries-v1.0.3.1.txt \
+-g https://www.ebi.ac.uk/gwas/api/search/stats
+```
+
+The command `update-gwas-curation-metadata` fetches the data from the ftp server and transfers them to the gcp without intermediate temporary files. The download(s) and upload(s) are made asyncronously.
+
+### Validate gwas catalog curation file
+
+To validate gwas catalog curation file after manual curation to see if all expectation tests are passing.
+
+```bash
+gentroutils -vvv validate-gwas-curation GWAS_Catalog_study_curation.tsv
+```
+
+validation is only allowed on the local file, the curation should follow format requirements defined in [OT curation](https://github.com/opentargets/curation/blob/master/genetics/GWAS_Catalog_study_curation.tsv)
+
+
+### Validate manual curation
+
+To validate the manually curated file, run the following command
+
+```bash
+gentroutils -vvv validate-gwas-curation tests/data/manual_curation/correct_curation.tsv
+```
+
+The command will validate the file and return the results of the validation if the issues are found.
+
+## Read before running
+
+The logs from the command are saved under the `-q` log file, if specified `gcp` log file, then the file will be uploaded after the command has run.
+
+To test the command run it with `-d` == `--dry-run`, this will just mark the input and output destinations.
+To allow for full logs to be transmitted to the log file, use `-vvv` to increase the verbosity of the logs
+
+> [!NOTE]
+> Change the path to the output `gcp` files to make sure they are saved under requested path
+
+> [!WARNING]
+> Please read before running the command!:
+>
+> * The above command has some default values set for the input and output files, make sure you test them in `--dry-run` so the existing files will not get overwritten!
+> * Make sure to run `gcloud auth application-default login` to allow to use Google Cloud Python SDK before running the command
+
+
+
+## Contribute
+
+To be able to contribute to the project you need to set it up. This project
+runs on:
+
+- [x] python 3.10.8
+- [x] uv (dependency manager)
+
+To set up the project run
+
+```{bash}
+make dev
+```
+
+The command will install above dependencies (initial requirements are curl and bash) if not present and
+install all python dependencies listed in `pyproject.toml`. Finally the command will install `pre-commit` hooks
+requred to be run before the commit is created.
+
+The project has additional `dev` dependencies that include the list of packages used for testing purposes.
+All of the `dev` depnendencies are automatically installed by `uv`.
+
+To see all available dev commands
+
+Run following command to see all available dev commands
+
+```{bash}
+make help
+```
+
+### Manual testing of CLI module
+
+To check CLI execution manually you need to run
+
+```{bash}
+uv run gentroutils
+```

gentroutils-1.5.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+gentroutils/__init__.py,sha256=lCl9VmPVU2hkZa_kh5NipCrNufbcTH00iKF-6WvE0gg,1319
+gentroutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gentroutils/commands/__init__.py,sha256=940jwl8BvK0gTN2mRb_js8Kl74rgff-benDSEO5MuBc,326
+gentroutils/commands/update_gwas_curation_metadata.py,sha256=zOIv5l_84kBlqyXAXJHnwH5OrPKVlObuHl6WhwlZC90,10903
+gentroutils/commands/utils.py,sha256=XEZogKn6ZQKaSwKms7UtJU5h_eGrrHM-FofCoU2k884,5312
+gentroutils/commands/validate_gwas_curation.py,sha256=CWlQ8uI8JAK9l4MyjFNmjDLeMOolrcA3neVaBdAGN80,6148
+gentroutils-1.5.0.dist-info/METADATA,sha256=LYbG6KpN19EN1O3ZCGhXOv_yRAYVPj1odQ5742ELoDQ,5154
+gentroutils-1.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gentroutils-1.5.0.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
+gentroutils-1.5.0.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
+gentroutils-1.5.0.dist-info/RECORD,,

{gentroutils-0.1.5.dist-info → gentroutils-1.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

gentroutils-0.1.5.dist-info/METADATA

@@ -1,71 +0,0 @@
-Metadata-Version: 2.3
-Name: gentroutils
-Version: 0.1.5
-Summary: Add your description here
-Author-email: Szymon Szyszkowski <ss60@mib117351s.internal.sanger.ac.uk>
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: click>=8.1.7
-Requires-Dist: google-cloud-storage>=2.18.1
-Requires-Dist: pyfiglet>=1.0.2
-Requires-Dist: requests>=2.32.3
-Description-Content-Type: text/markdown
-
-# gentroutils
-
-[![Tests](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml/badge.svg?event=push)](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml)
-![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
-
-Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
-
-## Installation
-
-```
-pip install gentroutils
-```
-
-## Available commands
-
-To see all available commands after installation run
-
-```{bash}
-gentroutils --help
-```
-
-## Contribute
-
-To be able to contribute to the project you need to set it up. This project
-runs on:
-
-- [x] python 3.10.8
-- [x] rye (package manager)
-- [x] uv (dependency manager)
-
-To set up the project run
-
-```{bash}
-make dev
-```
-
-The command will install above dependencies (initial requirements are curl and bash) if not present and
-install all python dependencies listed in `pyproject.toml`. Finally the command will install `pre-commit` hooks
-requred to be run before the commit is created.
-
-The project has additional `dev` dependencies that include the list of packages used for testing purposes.
-All of the `dev` depnendencies are automatically installed by `rye`.
-
-To see all available dev commands
-
-Run following command to see all available dev commands
-
-```{bash}
-make help
-```
-
-### Manual testing of CLI module
-
-To check CLI execution manually you need to run
-
-```{bash}
-rye run gentroutils
-```

gentroutils-0.1.5.dist-info/RECORD

@@ -1,9 +0,0 @@
-gentroutils/__init__.py,sha256=aHDzbBMrnsgdcO_FfsYCbbPXProynwB7_2nfyc4UGp8,1281
-gentroutils/commands/__init__.py,sha256=avkqzwa1ck__rLVN0Wqfpr3eHtKS6TvyPeeaHcguJuw,210
-gentroutils/commands/update_gwas_curation_metadata.py,sha256=7pBBkB6JF3VfT12xiP78MT_pmn0Wv4CF7Tm5TPgBXf8,12525
-gentroutils/commands/utils.py,sha256=9Wyptjww9hiAufCFILdnjdDOE6X6TdtyTWJOTkoIRqg,4316
-gentroutils-0.1.5.dist-info/METADATA,sha256=9PFlHuJakF2bnJfF9d6kPepH0jdRJM3g70GevV5Q7fM,1795
-gentroutils-0.1.5.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-gentroutils-0.1.5.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
-gentroutils-0.1.5.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
-gentroutils-0.1.5.dist-info/RECORD,,