gentroutils 1.5.0__py3-none-any.whl → 1.6.0.dev1__py3-none-any.whl

gentroutils-1.6.0.dev1.dist-info/METADATA

@@ -0,0 +1,274 @@
+Metadata-Version: 2.4
+Name: gentroutils
+Version: 1.6.0.dev1
+Summary: Open Targets python genetics utility CLI tools
+Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.13
+Requires-Dist: aioftp>=0.25.1
+Requires-Dist: aiohttp>=3.11.18
+Requires-Dist: google-cloud-storage>=3.1.1
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: opentargets-otter>=25.0.2
+Requires-Dist: polars>=1.31.0
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: tqdm>=4.67.1
+Description-Content-Type: text/markdown
+
+# gentroutils
+
+[![checks](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml/badge.svg?branch=dev)](https://github.com/opentargets/gentroutils/actions/workflows/pr.yaml)
+![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)
+[![release](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml/badge.svg)](https://github.com/opentargets/gentroutils/actions/workflows/release.yaml)
+
+Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
+
+## Installation
+
+```
+pip install gentroutils
+```
+
+## Available commands
+
+To see all available commands after installation run
+
+```{bash}
+gentroutils --help
+```
+
+## Usage
+
+To run a single step run
+```{bash}
+uv run gentroutils -s gwas_catalog_release  # After cloning the repository
+gentroutils -s gwas_catalog_release -c otter_config.yaml # When installed by pip
+```
+
+The `gentroutils` repository uses the [otter](https://github.com/opentargets/otter) framework to build the set of tasks to run. The current implementation of tasks can be found in the `config.yaml` file in the root of the repository. To run gentroutils installed via `pip` you need to define the otter config that looks like the `config.yaml` file.
+
+<details>
+<summary>Example config</summary>
+
+For the top level fields refer to the [otter documentation](https://opentargets.github.io/otter/otter.config.html)
+
+```yaml
+---
+work_path: ./work
+log_level: DEBUG
+scratchpad:
+steps:
+  gwas_catalog_release:
+    - name: crawl release metadata
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json"
+      promote: "true"
+    - name: fetch associations
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-associations_ontology-annotated.tsv"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv"
+      promote: true
+    - name: fetch studies
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-download-studies-v1.0.3.1.txt"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_download_studies.tsv"
+      promote: true
+    - name: fetch ancestries
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-download-ancestries-v1.0.3.1.txt"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_download_ancestries.tsv"
+      promote: true
+    - name: curation study
+      requires:
+        - fetch studies
+      previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
+      studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
+      destination_template: ./work/curation_{release_date}.tsv
+      promote: true
+```
+
+The config above defines the steps that are run in parallel by the `otter` framework.
+
+</details>
+
+### Available tasks
+
+The list of tasks (defined in the `config.yaml` file) that can be run are:
+
+#### Crawl release metadata
+
+```yaml
+- name: crawl release metadata
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json"
+      promote: "true"
+```
+
+This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
+
+> [!NOTE] 
+> **Task parameters**
+>
+> - The `stats_uri` is used to fetch the latest release date and other metadata.
+> - The `destination_template` is where the metadata will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. By default it searches for the release directly in the stats_uri json output.
+> - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/gentroutils/latest/stats.json` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+
+---
+
+### Fetch associations
+
+```yaml
+- name: fetch associations
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-associations_ontology-annotated.tsv"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv"
+      promote: true
+```
+
+This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
+
+> [!NOTE] 
+> **Task parameters**
+>
+> - The `stats_uri` is used to fetch the latest release date and other metadata.
+> - The `source_template` is the URL of the GWAS Catalog associations file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
+> - The `destination_template` is where the associations file will be saved, and it also uses the `{release_date}` placeholder. The release date is fetched from the `stats_uri` endpoint.
+> - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_associations_ontology_annotated.tsv` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+
+---
+
+### Fetch studies
+
+```yaml
+- name: fetch studies
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-download-studies-v1.0.3.1.txt"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_download_studies.tsv"
+      promote: true
+```
+
+This task fetches the GWAS Catalog studies file from the specified FTP server and saves it to the specified destination.
+
+> [!NOTE] 
+> **Task parameters**
+>
+> - The `stats_uri` is used to fetch the latest release date and other metadata.
+> - The `source_template` is the URL of the GWAS Catalog studies file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
+> - The `destination_template` is where the studies file will be saved, and it also uses the `{release_date}` placeholder. The release date is fetched from the `stats_uri` endpoint.
+> - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+
+---
+
+### Fetch ancestries
+
+```yaml
+- name: fetch ancestries
+      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-download-ancestries-v1.0.3.1.txt"
+      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_download_ancestries.tsv"
+      promote: true
+```
+
+This task fetches the GWAS Catalog ancestries file from the specified FTP server and saves it to the specified destination.
+
+> [!NOTE] 
+> **Task parameters**
+>
+> - The `stats_uri` is used to fetch the latest release date and other metadata.
+> - The `source_template` is the URL of the GWAS Catalog ancestries file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
+> - The `destination_template` is where the ancestries file will be saved, and it also uses the `{release_date}` placeholder. The release date is fetched from the `stats_uri` endpoint.
+> - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_ancestries.tsv` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+
+---
+
+### Curation
+
+```yaml
+- name: curation study
+      requires:
+        - fetch studies
+      previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
+      studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
+      destination_template: gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv
+      promote: true
+```
+
+This task is used to build the GWAS Catalog curation file that is later used as a template for manual curation. It requires the `fetch studies` task to be completed before it can run. This is due to the fact that the curation file is build based on the list of studies fetched from `download studies` file.
+
+> [!NOTE] 
+> **Task parameters**
+>
+> - The `requires` field specifies that this task depends on the `fetch studies` task, meaning it will only run after the studies have been fetched.
+> - The `previous_curation` field is used to specify the path to the previous curation file. This is used to build the new curation file based on the previous one.
+> - The `studies` field is the path to the studies file that was fetched in the `fetch studies` task. This file is used to build the curation file.
+> - The `destination_template` is where the curation file will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
+> - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+
+---
+
+## Curation process
+
+The base of the curation process for GWAS Catalog data is defined in the [docs/gwas_catalog_curation.md](docs/gwas_catalog_curation.md). The original solution uses R script to prepare the data for curation and then manually curates the data. The solution proposed in the `curation` task autommates the preparation of the data for curation and provides a template for manual curation. The manual curation process is still required, but the data preparation is automated.
+
+The automated process includes:
+
+1. Reading `download studies` file with the list of studies that are currently comming from the latest GWAS Catalog release.
+2. Reading `previous curation` file that contains the list of the curated studies from the previous release.
+3. Comparing the two datasets with following logic:
+   - In case the study is present in the `previous curation` and `download studies`, the study is marked as `curated`
+   * In case the study is present in the `download studies` but not in the `previous curation`, the study is marked as `new`
+   * In case the study is present in the `previous curation` but not in the `download studies`, the study is marked as `removed`
+4. The output of the curation process is a file that contains the list of studies with their status (curated, new, removed) and the fields that are required for manual curation. The output file is saved to the `destination_template` path specified in the task configuration. The file is saved under `gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv` path.
+5. The output file is then promoted to the latest release path `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` so that it can be used for manual curation.
+6. The manual curation process is then performed on the `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` file. The manual curation process is not automated and requires manual intervention. The output from the manual curation process should be saved then to the `gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv` and `gs://gwas_catalog_inputs/curation/{release_date}/curated/GWAS_Catalog_study_curation.tsv` file. This file is then used for the [Open Targets Staging Dags](https://github.com/opentargets/orchestration).
+
+---
+
+## Contribute
+
+To be able to contribute to the project you need to set it up. This project
+runs on:
+
+- [x] python 3.13
+- [x] uv (dependency manager)
+
+To set up the project run
+
+```{bash}
+make dev
+```
+
+The command will install above dependencies (initial requirements are curl and bash) if not present and
+install all python dependencies listed in `pyproject.toml`. Finally the command will install `pre-commit` hooks
+required to be run before the commit is created.
+
+The project has additional `dev` dependencies that include the list of packages used for testing purposes.
+All of the `dev` dependencies are automatically installed by `uv`.
+
+To see all available dev commands
+
+Run following command to see all available dev commands
+
+```{bash}
+make help
+```
+
+### Manual testing of CLI module
+
+To check CLI execution manually you need to run
+
+```{bash}
+uv run gentroutils
+```
+---
+
+This software was developed as part of the Open Targets project. For more
+information please see: http://www.opentargets.org

gentroutils-1.6.0.dev1.dist-info/RECORD

@@ -0,0 +1,22 @@
+gentroutils/__init__.py,sha256=jPZFw8osF4Ih87CAdLjaaMptfJUyqoSzLa6eNAwsuqA,281
+gentroutils/errors.py,sha256=VhsQe0KMc7MMbuzZxTwi2gATs-zOnnGmpl_H_m4lwNg,1664
+gentroutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gentroutils/transfer.py,sha256=qycMPWQi8RRvZZUAsRXmDdxE1crnpZwCuc3d5q4w89M,3889
+gentroutils/io/path/__init__.py,sha256=oBNiw3xr84QAFejVgRIW_PttJ2ZQHZV2uYtIklXlecw,184
+gentroutils/io/path/ftp.py,sha256=kus0GS9Bcm3IzooL8duu7fdtMDr0-Sm7LCtpiUd1Kxs,1585
+gentroutils/io/path/gcs.py,sha256=itEEyUBQLqCBV-F7JF7ZJHUn5OBObmTCsrEfib_V7p8,1514
+gentroutils/io/transfer/__init__.py,sha256=2KasKyCMTpnUvU2SVzZds7ggYBS5ReXn8q3K_TE0YYA,310
+gentroutils/io/transfer/ftp_to_gcs.py,sha256=SGIpxTaQoiIeW2p9JHIMckNWhFpz1Ne4YqwoiYqdWdo,2107
+gentroutils/io/transfer/model.py,sha256=-REhDC8nIT_RoOlRwHpvNfqN_khOm7OnlAi8-U7qX1k,1198
+gentroutils/io/transfer/polars_to_gcs.py,sha256=l1vWDLVw5OuAjPRWEvA69w5V5cz36ARGqCOA9R9iecM,762
+gentroutils/parsers/__init__.py,sha256=HtgvopQ3Xx_cjC2lA3Tp81Rd5-k4CUJGJu0GD7W9r0o,59
+gentroutils/parsers/curation.py,sha256=34GvUTD9SmUpk_CEMoqJbpDyBVW3kKOyMLMiZ4nc9e0,7357
+gentroutils/tasks/__init__.py,sha256=dN9Od2_I504AZjLwBixj04M0h6dmmxGRV7FMwHHEdzM,3537
+gentroutils/tasks/crawl.py,sha256=njCveYWvJz6CHWQfjbciGp57yZcsKuFiZqVWR57_XeU,6838
+gentroutils/tasks/curation.py,sha256=cJZQmaD-44rej-8K6dUW6IlzArZt-YcxtVVTZdKh-Fk,4539
+gentroutils/tasks/fetch.py,sha256=jwqIQ49P--63X1EyKXIUf2iBtv3QRsuNgQWEBwlvgCk,6444
+gentroutils-1.6.0.dev1.dist-info/METADATA,sha256=hD71G_Cz5eFNYbWUXBgaE6hFGwHEx4pHOb4WUXmJd6A,14941
+gentroutils-1.6.0.dev1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gentroutils-1.6.0.dev1.dist-info/entry_points.txt,sha256=KjODdAGrWKebI3ogqs9r6snAJ_DtLGpm1jREbZ6OXGs,49
+gentroutils-1.6.0.dev1.dist-info/licenses/LICENSE,sha256=8rMKYP7K5vL-KA2WmdBkUBz4iFRveUOWUMAFP8uc3P0,10945
+gentroutils-1.6.0.dev1.dist-info/RECORD,,

gentroutils-1.6.0.dev1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gentroutils = gentroutils:main

{gentroutils-1.5.0.dist-info → gentroutils-1.6.0.dev1.dist-info}/licenses/LICENSE

@@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-Copyright [yyyy] [name of copyright owner]
+Copyright 2025 [name of copyright owner]
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

gentroutils/commands/__init__.py

@@ -1,11 +0,0 @@
-"""CLI submodules for gentroutils package."""
-
-from gentroutils.commands.update_gwas_curation_metadata import (
-    update_gwas_curation_metadata_command,
-)
-from gentroutils.commands.validate_gwas_curation import validate_gwas_curation
-
-__all__ = [
-    "update_gwas_curation_metadata_command",
-    "validate_gwas_curation",
-]

gentroutils/commands/update_gwas_curation_metadata.py

@@ -1,287 +0,0 @@
-"""Update gwas catalog metadata."""
-
-from __future__ import annotations
-
-import asyncio
-import io
-import logging
-import re
-import sys
-from ftplib import FTP
-from urllib.parse import ParseResult, urlparse
-
-import click
-import requests
-from google.cloud import storage
-
-from gentroutils.commands.utils import coro
-
-logger = logging.getLogger("gentroutils")
-MAX_CONCURRENT_CONNECTIONS = 10
-CURATED_INPUTS = (
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv",
-        "gs://gwas_catalog_inputs/gwas_catalog_associations_ontology_annotated.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt",
-        "gs://gwas_catalog_inputs/gwas_catalog_download_studies.tsv",
-    ),
-    (
-        "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt",
-        "gs://gwas_catalog_inputs/gwas_catalog_download_ancestries.tsv",
-    ),
-)
-
-
-@click.command(name="update-gwas-curation-metadata")
-@click.option(
-    "--file-to-transfer",
-    "-f",
-    metavar="<ftp_file|http(s) file> <gcp_file>",
-    type=(str, str),
-    multiple=True,
-    default=CURATED_INPUTS,
-)
-@click.option(
-    "--gwas-catalog-release-info-url",
-    "-g",
-    metavar="<url>",
-    default="https://www.ebi.ac.uk/gwas/api/search/stats",
-    type=click.STRING,
-)
-@click.pass_context
-@coro
-async def update_gwas_curation_metadata_command(
-    ctx: click.Context,
-    file_to_transfer: list[tuple[str, str]],
-    gwas_catalog_release_info_url: str,
-) -> None:
-    """Update GWAS Catalog metadata directly to cloud bucket.
-
-    \b
-    This is the script to fetch the latest GWAS Catalog data files that include:
-    - [x] gwas-catalog-associations_ontology-annotated.tsv - list of associations with ontology annotations by GWAS Catalog
-    - [x] gwas-catalog-download-studies-v1.0.3.1.txt - list of published studies by GWAS Catalog
-    - [x] gwas-catalog-download-ancestries-v1.0.3.1.txt - list of published studies by GWAS Catalog
-
-    \b
-    By default all GWAS Catalog data files are uploaded from GWAS Catalog FTP server to Open Targets GCP bucket.
-    The script also captures the latest release metadata from GWAS Catalog release info url.
-    One can overwrite this script to sync data files from FTP or HTTP(s) to GCP bucket. The example usage is as follows:
-
-    \b
-    gentroutils --log-file gs://gwas_catalog_data/curated_inputs/log.txt update-gwas-curation-metadata \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-associations_ontology-annotated.tsv gs://gwas_catalog_data/gwas_catalog_associations_ontology_annotated.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-studies-v1.0.3.1.txt gs://gwas_catalog_inputs/gwas_catalog_download_studies.tsv \\
-    -f ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/gwas-catalog-download-ancestries-v1.0.3.1.txt gs://gwas_catalog_inputs/gwas_catalog_download_ancestries.tsv \\
-    -g https://www.ebi.ac.uk/gwas/api/search/stats
-
-
-    To preserve the logs from this command, you can specify the log file path using `--log-file` option. The log file can point to local or GCP path.
-    Currently only FTP and HTTP(s) protocols are supported for input and GCP protocol is supported for output.
-    """
-    # we always want to have the logs from this command uploaded to the target bucket
-    logger.debug("Running gwas_curation_update step.")
-    dry_run = ctx.obj["dry_run"]
-    if len(file_to_transfer) > MAX_CONCURRENT_CONNECTIONS:
-        logger.error(
-            "File transfer limit exceeded! Max %s connections allowed.",
-            MAX_CONCURRENT_CONNECTIONS,
-        )
-        sys.exit(1)
-    uri_map = [{"input": urlparse(ftp_file), "output": urlparse(gcp_file)} for ftp_file, gcp_file in file_to_transfer]
-    transfer_tasks = generate_transfer_tasks(uri_map, dry_run)
-
-    # capture latest release metadata
-    with requests.get(gwas_catalog_release_info_url) as response:
-        if not response.ok:
-            logger.error("Failed to fetch release info.")
-            sys.exit(1)
-        release_info = response.json()
-        for key, value in release_info.items():
-            logger.debug("%s: %s", key, value)
-
-        efo_version = release_info.get("efoversion")
-        logger.info("Diseases were mapped to %s EFO release.", efo_version)
-        logger.info("EFO version: %s", efo_version)
-        ensembl_build = release_info.get("ensemblbuild")
-        logger.info("Genes were mapped to v%s Ensembl release.", ensembl_build)
-
-    results = await asyncio.gather(*transfer_tasks)
-    if not dry_run:
-        logger.info("Transferred %s files.", len(results))
-    logger.info("gwas_curation_update step completed.")
-
-
-def generate_transfer_tasks(uri_map: list[dict[str, ParseResult]], dry_run: bool) -> list[asyncio.Task[None]]:
-    """Generate transfer tasks.
-
-    Args:
-        uri_map (list[dict[str, ParseResult]]): list of transferable tasks, each should have `input` and `output` keys.
-        dry_run (bool): dry run flag.
-
-    Returns:
-        list[asyncio.Task[None]]: list of asyncio tasks.
-    """
-    ftp_transfer_list = []
-    http_transfer_list = []
-    for uri in uri_map:
-        if uri["input"].scheme != "ftp" and not uri["input"].scheme.startswith("http"):
-            logger.error("Only FTP and HTTP(s) protocols is supported at input.")
-            sys.exit(1)
-        if uri["output"].scheme != "gs":
-            logger.error("Only GCP protocol is supported at output.")
-            sys.exit(1)
-        in_server = uri["input"].netloc
-        out_server = uri["output"].netloc
-        in_prefix = "/".join(uri["input"].path.strip("/").split("/")[:-1])
-        in_file = uri["input"].path.strip("/").split("/")[-1]
-        out_prefix = "/".join(uri["output"].path[1:-1].split("/")[:-1])
-        out_bucket = uri["output"].path.split("/")[-1]
-        if uri["input"].scheme == "ftp":
-            ftp_transfer_list.append(
-                {
-                    "ftp_server": in_server,
-                    "ftp_prefix": in_prefix,
-                    "ftp_filename": in_file,
-                    "gcp_bucket": out_server,
-                    "gcp_prefix": out_prefix,
-                    "gcp_filename": out_bucket,
-                }
-            )
-        if uri["input"].scheme.startswith("http"):
-            http_transfer_list.append(
-                {
-                    "http_url": uri["input"].geturl(),
-                    "gcp_bucket": out_server,
-                    "gcp_prefix": out_prefix,
-                    "gcp_filename": out_bucket,
-                }
-            )
-    transfer_tasks = []
-    for transfer_obj in ftp_transfer_list:
-        transfer_tasks.append(
-            asyncio.create_task(
-                sync_from_ftp_to_gcp(
-                    transfer_obj["ftp_server"],
-                    transfer_obj["ftp_prefix"],
-                    transfer_obj["ftp_filename"],
-                    transfer_obj["gcp_bucket"],
-                    transfer_obj["gcp_prefix"],
-                    transfer_obj["gcp_filename"],
-                    dry_run=dry_run,
-                )
-            )
-        )
-
-    for transfer_obj in http_transfer_list:
-        transfer_tasks.append(
-            asyncio.create_task(
-                sync_from_http_to_gcp(
-                    transfer_obj["http_url"],
-                    transfer_obj["gcp_bucket"],
-                    transfer_obj["gcp_prefix"],
-                    transfer_obj["gcp_filename"],
-                    dry_run=dry_run,
-                )
-            )
-        )
-
-    return transfer_tasks
-
-
-async def sync_from_http_to_gcp(url: str, gcp_bucket: str, gcp_prefix: str, gcp_file: str, *, dry_run: bool = True) -> None:
-    """Sync file from HTTP and upload to GCP.
-
-    This function fetches the data from the provided HTTP URL and uploads the content
-    directly to provided GCP bucket blob.
-
-    Args:
-        url (str): HTTP URL to fetch the data.
-        gcp_bucket (str): GCP bucket name.
-        gcp_prefix (str): GCP prefix.
-        gcp_file (str): GCP file name.
-        dry_run (bool, optional): Dry run flag. Defaults to True.
-    """
-    if dry_run:
-        logger.info(
-            "Attempting to transfer data from %s to gs://%s/%s/%s.",
-            url,
-            gcp_bucket,
-            gcp_prefix,
-            gcp_file,
-        )
-        return
-    logger.info("Retriving data from: %s.", url)
-    response = requests.get(url)
-    if not response.ok:
-        logger.error("Failed to fetch data from %s.", url)
-        return
-
-    content = response.content
-    bucket = storage.Client().bucket(gcp_bucket)
-    gcp_path = f"{gcp_prefix}/{gcp_file}" if gcp_prefix else gcp_file
-
-    blob = bucket.blob(gcp_path)
-    logger.info("Uploading the data to: gs://%s/%s.", gcp_bucket, gcp_path)
-    blob.upload_from_string(content.decode("utf-8"))
-
-
-async def sync_from_ftp_to_gcp(
-    ftp_server: str,
-    ftp_prefix: str,
-    ftp_file: str,
-    gcp_bucket: str,
-    gcp_prefix: str,
-    gcp_file: str,
-    *,
-    dry_run: bool = True,
-) -> None:
-    """Fetch files from FTP and upload to GCP.
-
-    This function fetches the data from the provided FTP server and uploads the content directly
-    to the provided GCP bucket blob.
-
-    Args:
-        ftp_server (str): FTP server.
-        ftp_prefix (str): FTP prefix.
-        ftp_file (str): FTP file name.
-        gcp_bucket (str): GCP bucket name.
-        gcp_prefix (str): GCP prefix.
-        gcp_file (str): GCP file name.
-        dry_run (bool, optional): Dry run flag. Defaults to True.
-
-    """
-    if dry_run:
-        logger.info(
-            "Attempting to transfer data from ftp://%s/%s/%s to gs://%s/%s/%s.",
-            ftp_server,
-            ftp_prefix,
-            ftp_file,
-            gcp_bucket,
-            gcp_prefix,
-            gcp_file,
-        )
-        return
-    with FTP() as ftp:
-        ftp.connect(ftp_server)
-        ftp.login()
-        bucket = storage.Client().bucket(gcp_bucket)
-        gcp_path = f"{gcp_prefix}/{gcp_file}" if gcp_prefix else gcp_file
-        blob = bucket.blob(gcp_path)
-        logger.info("Changing directory to %s.", ftp_prefix)
-        ftp.cwd(ftp_prefix)
-        dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", ftp.pwd())
-        if dir_match:
-            logger.info("Found release date!: %s", dir_match.group("release_date"))
-        buffer = io.BytesIO()
-        logger.info("Retrieving data from: ftp://%s/%s/%s.", ftp_server, ftp_prefix, ftp_file)
-        ftp.retrbinary(f"RETR {ftp_file}", lambda x: buffer.write(x))
-        content = buffer.getvalue().decode("utf-8")
-        buffer.close()
-        logger.info("Uploading data to: gs://%s/%s.", gcp_bucket, gcp_path)
-        blob.upload_from_string("".join(content))
-
-
-__all__ = ["update_gwas_curation_metadata_command"]