gentroutils 0.1.5__py3-none-any.whl → 0.2.0__py3-none-any.whl

gentroutils/commands/update_gwas_curation_metadata.py

@@ -21,27 +21,15 @@ MAX_CONCURRENT_CONNECTIONS = 10
 CURATED_INPUTS = (
     (
         "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_associations_ontology_annotated.tsv",
+        "gs://gwas_catalog_inputs/gwas_catalog_associations_ontology_annotated.tsv",
     ),
     (
         "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_studies.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-studies-v1.0.3.1.tsv",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_studies.tsv",
+        "gs://gwas_catalog_inputs/gwas_catalog_download_studies.tsv",
     ),
     (
         "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_ancestries.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv",
-        "gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_ancestries.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/harmonised_list.txt",
-        "gs://gwas_catalog_data/curated_inputs/harmonised_list.txt",
+        "gs://gwas_catalog_inputs/gwas_catalog_download_ancestries.tsv",
     ),
 )
 
@@ -75,9 +63,7 @@ async def update_gwas_curation_metadata_command(
     This is the script to fetch the latest GWAS Catalog data files that include:
     - [x] gwas-catalog-associations_ontology-annotated.tsv - list of associations with ontology annotations by GWAS Catalog
     - [x] gwas-catalog-download-studies-v1.0.3.1.txt - list of published studies by GWAS Catalog
-    - [x] gwas-catalog-unpublished-studies-v1.0.3.1.tsv - list of unpublished studies by GWAS Catalog
     - [x] gwas-catalog-download-ancestries-v1.0.3.1.txt - list of published studies by GWAS Catalog
-    - [x] gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv - list of unpublished studies by GWAS Catalog
 
     \b
     By default all GWAS Catalog data files are uploaded from GWAS Catalog FTP server to Open Targets GCP bucket.
@@ -86,13 +72,9 @@ async def update_gwas_curation_metadata_command(
 
     \b
     gentroutils --log-file gs://gwas_catalog_data/curated_inputs/log.txt update-gwas-curation-metadata \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://gwas_catalog_data/curated_inputs/gwas_catalog_associations_ontology_annotated.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_studies.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-studies-v1.0.3.1.tsv gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_studies.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://gwas_catalog_data/curated_inputs/gwas_catalog_download_ancestries.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-unpublished-ancestries-v1.0.3.1.tsv gs://gwas_catalog_data/curated_inputs/gwas_catalog_unpublished_ancestries.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/summary_statistics/harmonised_list.txt gs://gwas_catalog_data/curated_inputs/harmonised_list.txt \\
-    -f https://raw.githubusercontent.com/opentargets/curation/master/genetics/GWAS_Catalog_study_curation.tsv gs://gwas_catalog_data/manifests/gwas_catalog_study_curation.tsv \\
+    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://gwas_catalog_data/gwas_catalog_associations_ontology_annotated.tsv \\
+    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://gwas_catalog_inputs/gwas_catalog_download_studies.tsv \\
+    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://gwas_catalog_inputs/gwas_catalog_download_ancestries.tsv \\
     -g https://www.ebi.ac.uk/gwas/api/search/stats
 
 

gentroutils/commands/utils.py

@@ -6,6 +6,7 @@ import sys
 import time
 from functools import wraps
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from urllib.parse import urlparse
 
 import click
@@ -39,33 +40,47 @@ def set_log_file(ctx: click.Context, param: click.Option, log_file: str) -> str:
         return ""
     logger.info("Extracting log file from the %s", param)
     upload_to_gcp = False
+
     if "://" in log_file:
         upload_to_gcp = True
+    ctx.obj["upload_to_gcp"] = upload_to_gcp
+
     if upload_to_gcp:
         parsed_uri = urlparse(log_file)
-        ctx.obj["gcp_log_file"] = log_file
         if parsed_uri.scheme != "gs":
             raise click.BadParameter("Only GCS is supported for logging upload")
-        log_file = parsed_uri.path.strip("/")
-        ctx.obj["local_log_file"] = log_file
-    ctx.obj["upload_to_gcp"] = upload_to_gcp
-
-    local_file = Path(log_file)
-    if local_file.exists() and local_file.is_dir():
-        raise click.BadParameter("Log file is a directory")
-    if local_file.exists() and local_file.is_file():
-        local_file.unlink()
-    if not local_file.exists():
-        local_file.touch()
-    logger.info("Logging to %s", local_file)
-    handler = logging.FileHandler(local_file)
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
-    handler.setFormatter(formatter)
-    handler.setLevel(logging.DEBUG)
-    logger.addHandler(handler)
-    return str(local_file)
+        tmp_file = NamedTemporaryFile(delete=False)
+        logger.info("Logging to temporary file %s", tmp_file.name)
+        handler = logging.FileHandler(tmp_file.name)
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+        handler.setFormatter(formatter)
+        handler.setLevel(logging.DEBUG)
+        logger.addHandler(handler)
+        ctx.obj["local_log_file"] = tmp_file.name
+        ctx.obj["local_log_file_obj"] = tmp_file
+        ctx.obj["gcp_log_file"] = log_file
+        return tmp_file.name
+
+    else:
+        local_file = Path(log_file)
+        if local_file.exists() and local_file.is_dir():
+            raise click.BadParameter("Log file is a directory")
+        if local_file.exists() and local_file.is_file():
+            local_file.unlink()
+        if not local_file.exists():
+            local_file.parent.mkdir(parents=True, exist_ok=True)
+            local_file.touch()
+        logger.info("Logging to %s", local_file)
+        handler = logging.FileHandler(local_file)
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+        handler.setFormatter(formatter)
+        handler.setLevel(logging.DEBUG)
+        logger.addHandler(handler)
+        return str(local_file)
 
 
 def teardown_cli(ctx: click.Context) -> None:
@@ -80,13 +95,23 @@ def teardown_cli(ctx: click.Context) -> None:
     if "upload_to_gcp" in ctx.obj and ctx.obj["upload_to_gcp"]:
         gcp_file = ctx.obj["gcp_log_file"]
         local_file = ctx.obj["local_log_file"]
-        client = storage.Client()
-        bucket_name = urlparse(gcp_file).netloc
-        bucket = client.bucket(bucket_name=bucket_name)
-        blob = bucket.blob(Path(local_file).name)
-        logger.info("Uploading %s to %s", local_file, gcp_file)
-        blob.upload_from_filename(local_file)
-        Path(local_file).unlink()
+        with open(local_file, "r") as f:
+            content = f.read()
+        try:
+            client = storage.Client()
+            bucket_name = urlparse(gcp_file).netloc
+            bucket = client.bucket(bucket_name=bucket_name)
+            file_name = urlparse(gcp_file).path.lstrip("/")
+            blob = bucket.blob(file_name)
+            logger.info("Uploading %s to %s", local_file, gcp_file)
+            if ctx.obj["dry_run"]:
+                logger.info("Dry run, skipping the upload of the log file")
+            else:
+                blob.upload_from_string(content)
+                ctx.obj["local_log_file_obj"].close()
+        except Exception as e:
+            msg = f"Failed to upload log file to GCP {e}"
+            logger.error(click.style(msg, fg="red"))
     logger.info(
         "Finished, elapsed time %s seconds", time.time() - ctx.obj["execution_start"]
     )

gentroutils-0.2.0.dist-info/METADATA

@@ -0,0 +1,107 @@
+Metadata-Version: 2.3
+Name: gentroutils
+Version: 0.2.0
+Summary: Open Targets python genetics utility CLI tools
+Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
+License: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.7
+Requires-Dist: google-cloud-storage>=2.18.1
+Requires-Dist: pyfiglet>=1.0.2
+Requires-Dist: requests>=2.32.3
+Description-Content-Type: text/markdown
+
+# gentroutils
+
+[![Tests](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml/badge.svg?event=push)](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml)
+![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
+
+Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
+
+## Installation
+
+```
+pip install gentroutils
+```
+
+## Available commands
+
+To see all available commands after installation run
+
+```{bash}
+gentroutils --help
+```
+
+### Updating gwas catalog metadata
+
+To update gwas catalog metadata run folliwing command
+
+```bash
+gentroutils  -vvv -q gs://ot_orchestration/tests/gentroutils/log.txt  update-gwas-curation-metadata \
+-f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://ot_orchestration/tests/gentroutils/gwas-catalog-associations_ontology-annotated.tsv \
+-f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://ot_orchestration/tests/gentroutils/gwas-catalog-download-studies-v1.0.3.1.txt \
+-f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://ot_orchestration/tests/gentroutils/gwas-catalog-download-ancestries-v1.0.3.1.txt \
+-g https://www.ebi.ac.uk/gwas/api/search/stats
+```
+
+The command `update-gwas-curation-metadata` fetches the data from the ftp server and transfers them to the gcp without intermediate temporary files. The download(s) and upload(s) are made asyncronously.
+
+The logs from the command are saved under the `-q` log file, if specified `gcp` log file, then the file will be uploaded after the command has run.
+
+To test the command run it with `-d` == `--dry-run`, this will just mark the input and output destinations.
+To allow for full logs to be transmitted to the log file, use `-vvv` to increase the verbosity of the logs
+
+> [!NOTE]
+> Change the path to the output `gcp` files to make sure they are saved under requested path
+
+> [!WARNING]
+> Please read before running the command!:
+>
+> * The above command has some default values set for the input and output files, make sure you test them in `--dry-run` so the existing files will not get overwritten!
+> * Make sure to run `gcloud auth application-default login` to allow to use Google Cloud Python SDK before running the command
+
+## Contribute
+
+To be able to contribute to the project you need to set it up. This project
+runs on:
+
+- [x] python 3.10.8
+- [x] rye (package manager)
+- [x] uv (dependency manager)
+
+To set up the project run
+
+```{bash}
+make dev
+```
+
+The command will install above dependencies (initial requirements are curl and bash) if not present and
+install all python dependencies listed in `pyproject.toml`. Finally the command will install `pre-commit` hooks
+requred to be run before the commit is created.
+
+The project has additional `dev` dependencies that include the list of packages used for testing purposes.
+All of the `dev` depnendencies are automatically installed by `rye`.
+
+To see all available dev commands
+
+Run following command to see all available dev commands
+
+```{bash}
+make help
+```
+
+### Manual testing of CLI module
+
+To check CLI execution manually you need to run
+
+```{bash}
+rye run gentroutils
+```

gentroutils-0.2.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+gentroutils/__init__.py,sha256=aHDzbBMrnsgdcO_FfsYCbbPXProynwB7_2nfyc4UGp8,1281
+gentroutils/commands/__init__.py,sha256=avkqzwa1ck__rLVN0Wqfpr3eHtKS6TvyPeeaHcguJuw,210
+gentroutils/commands/update_gwas_curation_metadata.py,sha256=4Pb2YdEnfulQklFh0KBvAOBnylCsDIAye7Keq2dC0mY,10937
+gentroutils/commands/utils.py,sha256=zYIzu47f-_a3nBeVXRR5xg5QiklrwES8uYNNhjed7gA,5384
+gentroutils-0.2.0.dist-info/METADATA,sha256=lMJ2JdqokHojQaY-hWhs9IvCJ4ei4vBpOfsOAfgBw4E,4061
+gentroutils-0.2.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+gentroutils-0.2.0.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
+gentroutils-0.2.0.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
+gentroutils-0.2.0.dist-info/RECORD,,

{gentroutils-0.1.5.dist-info → gentroutils-0.2.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.26.3
 Root-Is-Purelib: true
 Tag: py3-none-any

gentroutils-0.1.5.dist-info/METADATA

@@ -1,71 +0,0 @@
-Metadata-Version: 2.3
-Name: gentroutils
-Version: 0.1.5
-Summary: Add your description here
-Author-email: Szymon Szyszkowski <ss60@mib117351s.internal.sanger.ac.uk>
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: click>=8.1.7
-Requires-Dist: google-cloud-storage>=2.18.1
-Requires-Dist: pyfiglet>=1.0.2
-Requires-Dist: requests>=2.32.3
-Description-Content-Type: text/markdown
-
-# gentroutils
-
-[![Tests](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml/badge.svg?event=push)](https://github.com/opentargets/gentroutils/actions/workflows/test.yaml)
-![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
-
-Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
-
-## Installation
-
-```
-pip install gentroutils
-```
-
-## Available commands
-
-To see all available commands after installation run
-
-```{bash}
-gentroutils --help
-```
-
-## Contribute
-
-To be able to contribute to the project you need to set it up. This project
-runs on:
-
-- [x] python 3.10.8
-- [x] rye (package manager)
-- [x] uv (dependency manager)
-
-To set up the project run
-
-```{bash}
-make dev
-```
-
-The command will install above dependencies (initial requirements are curl and bash) if not present and
-install all python dependencies listed in `pyproject.toml`. Finally the command will install `pre-commit` hooks
-requred to be run before the commit is created.
-
-The project has additional `dev` dependencies that include the list of packages used for testing purposes.
-All of the `dev` depnendencies are automatically installed by `rye`.
-
-To see all available dev commands
-
-Run following command to see all available dev commands
-
-```{bash}
-make help
-```
-
-### Manual testing of CLI module
-
-To check CLI execution manually you need to run
-
-```{bash}
-rye run gentroutils
-```

gentroutils-0.1.5.dist-info/RECORD

@@ -1,9 +0,0 @@
-gentroutils/__init__.py,sha256=aHDzbBMrnsgdcO_FfsYCbbPXProynwB7_2nfyc4UGp8,1281
-gentroutils/commands/__init__.py,sha256=avkqzwa1ck__rLVN0Wqfpr3eHtKS6TvyPeeaHcguJuw,210
-gentroutils/commands/update_gwas_curation_metadata.py,sha256=7pBBkB6JF3VfT12xiP78MT_pmn0Wv4CF7Tm5TPgBXf8,12525
-gentroutils/commands/utils.py,sha256=9Wyptjww9hiAufCFILdnjdDOE6X6TdtyTWJOTkoIRqg,4316
-gentroutils-0.1.5.dist-info/METADATA,sha256=9PFlHuJakF2bnJfF9d6kPepH0jdRJM3g70GevV5Q7fM,1795
-gentroutils-0.1.5.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-gentroutils-0.1.5.dist-info/entry_points.txt,sha256=IvxZyBBD71Ota0aPMtVaJzI9OSX5_f-iH4ZJx6sY53w,48
-gentroutils-0.1.5.dist-info/licenses/LICENSE,sha256=RFhQPdSOiMTguUX7JSoIuTxA7HVzCbj_p8WU36HjUQQ,10947
-gentroutils-0.1.5.dist-info/RECORD,,