gentroutils 1.5.0__py3-none-any.whl → 1.6.0.dev2__py3-none-any.whl

gentroutils/tasks/crawl.py

@@ -0,0 +1,156 @@
+"""Module to handle the crawling of GWAS Catalog release information."""
+
+import tempfile
+from functools import cached_property
+from pathlib import Path
+from typing import Annotated, Any, Self
+
+from loguru import logger
+from otter.storage import get_remote_storage
+from otter.task.model import Spec, Task, TaskContext
+from otter.task.task_reporter import report
+from pydantic import AfterValidator, computed_field
+
+from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, _requires_release_date_template
+
+
+class CrawlSpec(Spec):
+    """Configuration fields for the release crawler task.
+
+    The `CrawlSpec` defines the parameters needed to crawl the GWAS Catalog release information.
+    It includes the `stats_uri` that provides the release statistics and the `destination_prefix` where the
+    release information will be stored.
+
+    Examples:
+    ---
+    >>> cs = CrawlSpec(
+    ...     name="crawl gwas catalog release information",
+    ...     stats_uri="https://www.ebi.ac.uk/gwas/api/search/stats",
+    ...     destination_template="gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json",
+    ...     promote=True
+    ... )
+    >>> cs.name
+    'crawl gwas catalog release information'
+    >>> cs.stats_uri
+    'https://www.ebi.ac.uk/gwas/api/search/stats'
+    >>> cs.destination_template
+    'gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json'
+    >>> cs.promote
+    True
+    >>> rs = GwasCatalogReleaseInfo(
+    ...     date="2023-10-01",
+    ...     associations=1000,
+    ...     studies=200,
+    ...     sumstats=300,
+    ...     snps=400,
+    ...     ensemblbuild="114.0",
+    ...     dbsnpbuild="1.0.0",
+    ...     efoversion="1.0.0",
+    ...     genebuild="GRCh38",
+    ... )
+    >>> cs.substituted_destinations(rs)
+    ['gs://gwas_catalog_inputs/gentroutils/20231001/stats.json', 'gs://gwas_catalog_inputs/gentroutils/latest/stats.json']
+
+
+    ### Example configuration for the crawl task in a YAML file.
+    .. code-block:: yaml
+
+        steps:
+            - crawl gwas catalog release information:
+                destination: gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json
+                stats_uri: https://www.ebi.ac.uk/gwas/api/search/stats
+                promote: true
+    """
+
+    name: str = "crawl gwas catalog release information"
+    """The name of the crawl task."""
+
+    stats_uri: str = "https://www.ebi.ac.uk/gwas/api/search/stats"
+    """The URI to crawl the release statistics information from."""
+
+    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    """The destination path to save the release information.
+       This path should always be a template string that includes `{release_date}`.
+       For example, `gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json`.
+
+       The `release_date` will be substituted with the actual release date or `latest` literal from the stats_uri endpoint.
+    """
+
+    promote: bool = True
+    """Whether to promote the release information as the latest release.
+
+    Given the destination: `gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json`
+
+       * If set to `False` the task will upload the release information
+       only to the specified destination with `release_date` substituted by the value from the stats_uri endpoint.
+       Resulting in following destinations:
+            * `gs://gwas_catalog_inputs/gentroutils/20231001/stats.json`
+
+       * If set to `True`, the task will also upload the release information
+        to the destination with `release_date` substituted to `latest` literal, effectively
+        promoting the release as the latest release.
+    """
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def destinations(self) -> list[TemplateDestination]:
+        """Get the list of destinations templates where the release information will be saved.
+
+        Returns:
+            list[TemplateDestination]: A list of TemplateDestination objects with the formatted destination paths.
+
+        Depending on the `promote` flag this property will return:
+            * If `promote` is `False`, it will return a single destination template.
+            * If `promote` is `True`, it will return two destinations:
+                1. The destination template with the release date substituted.
+                2. The destination with the release date substituted to `latest`.
+        """
+        d1 = self.destination_template
+        if self.promote:
+            d2 = self.destination_template.format(release_date="latest")
+            return [TemplateDestination(d1, False), TemplateDestination(d2, True)]
+        return [TemplateDestination(d1, False)]
+
+    def substituted_destinations(self, release_info: GwasCatalogReleaseInfo) -> list[str]:
+        """Safely parse the destination name to ensure it is valid."""
+        substitutions = {"release_date": release_info.strfmt("%Y%m%d")}
+        return [
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+        ]
+
+    def model_post_init(self, __context: Any) -> None:
+        """Method to ensure the scratchpad is set to ignore missing replacements."""
+        self.scratchpad_ignore_missing = True
+
+
+class Crawl(Task):
+    """Task to crawl the GWAS Catalog release information."""
+
+    def __init__(self, spec: CrawlSpec, context: TaskContext) -> None:
+        super().__init__(spec, context)
+        self.spec: CrawlSpec
+
+    def _write_release_info(self, release_info: GwasCatalogReleaseInfo) -> Self:
+        """Write the release information to the specified GCP blob."""
+        with tempfile.NamedTemporaryFile() as source:
+            logger.info(f"Writing release information to {source.name}")
+            with open(source.name, "w") as source_file:
+                source_file.write(release_info.model_dump_json(indent=2, by_alias=False))
+                source_file.flush()
+                destinations = self.spec.substituted_destinations(release_info)
+                logger.info(f"Destinations for release information: {destinations}")
+                for destination in destinations:
+                    storage = get_remote_storage(destination)
+                    storage.upload(Path(source.name), destination)
+                    logger.info(f"Release information written to {destination}")
+            return self
+
+    @report
+    def run(self) -> Self:
+        """Crawl the release information."""
+        logger.info(f"Crawling release information from {self.spec.stats_uri}")
+        release_info = GwasCatalogReleaseInfo.from_uri(self.spec.stats_uri)
+        logger.info("Crawling completed successfully.")
+        self._write_release_info(release_info)
+        logger.info("Writing release information completed successfully.")
+        return self

gentroutils/tasks/curation.py

@@ -0,0 +1,110 @@
+"""Module to handle the business logic for the GWAS Catalog curation task."""
+
+from __future__ import annotations
+
+from datetime import date
+from functools import cached_property
+from typing import Annotated, Any, Self
+
+from otter.task.model import Spec, Task, TaskContext
+from otter.task.task_reporter import report
+from pydantic import AfterValidator, computed_field
+
+from gentroutils.io.transfer.polars_to_gcs import PolarsDataFrameToGCSTransferableObject
+from gentroutils.parsers.curation import GWASCatalogCuration
+from gentroutils.tasks import TemplateDestination, _requires_release_date_template
+from gentroutils.transfer import TransferManager
+
+
+class CurationSpec(Spec):
+    """Configuration fields for the curation task.
+
+    The `CurationSpec` defines the parameters needed to curate GWAS Catalog data.
+    It includes the `previous_curation`, which is the path to the previous curation data,
+    and the `studies`, which is the path to the studies data.
+
+    Examples:
+    ---
+    >>> cs = CurationSpec(
+    ...     name="curate gwas catalog data",
+    ...     previous_curation="gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv",
+    ...     studies="gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv",
+    ...     destination_template="gs://gwas_catalog_inputs/{release_date}/pending/curation.tsv"
+    ... )
+    >>> cs.name
+    'curate gwas catalog data'
+    >>> cs.previous_curation
+    'gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv'
+    >>> cs.studies
+    'gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv'
+    >>> cs.destination_template
+    'gs://gwas_catalog_inputs/{release_date}/pending/curation.tsv'
+    """
+
+    name: str = "curate gwas catalog data"
+    """The name of the curation task."""
+
+    previous_curation: str
+    """The path to the previous curation data."""
+
+    studies: str
+    """The path to the studies data."""
+
+    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    """The destination path for the curation data."""
+
+    promote: bool = False
+    """Whether to promote the curation data to the latest version."""
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def destinations(self) -> list[TemplateDestination]:
+        """Get the list of destinations templates where the release information will be saved.
+
+        Returns:
+            list[TemplateDestination]: A list of TemplateDestination objects with the formatted destination paths.
+
+        Depending on the `promote` flag this property will return:
+            * If `promote` is `False`, it will return a single destination template.
+            * If `promote` is `True`, it will return two destinations:
+                1. The destination template with the release date substituted.
+                2. The destination with the release date substituted to `latest`.
+        """
+        d1 = TemplateDestination(self.destination_template, False)
+        if self.promote:
+            d2 = d1.format({"release_date": "latest"})
+            return [d1, d2]
+        return [d1]
+
+    def substituted_destinations(self, release_date: date) -> list[str]:
+        """Safely parse the destination name to ensure it is valid."""
+        substitutions = {"release_date": release_date.strftime("%Y%m%d")}
+        return [
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+        ]
+
+    def model_post_init(self, __context: Any) -> None:
+        """Method to ensure the scratchpad is set to ignore missing replacements."""
+        self.scratchpad_ignore_missing = True
+
+
+class Curation(Task):
+    """Task for curating GWAS Catalog data."""
+
+    def __init__(self, spec: CurationSpec, context: TaskContext) -> None:
+        """Initialize the Curation task with the given specification and context."""
+        super().__init__(spec, context)
+        self.spec: CurationSpec
+
+    @report
+    def run(self) -> Self:
+        """Run the curation task."""
+        release_date = date.today()
+        destinations = self.spec.substituted_destinations(release_date)
+        curation = GWASCatalogCuration.from_prev_curation(self.spec.previous_curation, self.spec.studies)
+        transfer_objects = [
+            PolarsDataFrameToGCSTransferableObject(source=curation.result, destination=d) for d in destinations
+        ]
+        TransferManager().transfer(transfer_objects)
+
+        return self

gentroutils/tasks/fetch.py

@@ -0,0 +1,141 @@
+"""Module to handle the fetching of GWAS Catalog release files."""
+
+from functools import cached_property
+from typing import Annotated, Any, Self
+
+from loguru import logger
+from otter.task.model import Spec, Task, TaskContext
+from otter.task.task_reporter import report
+from pydantic import AfterValidator, computed_field
+
+from gentroutils.io.transfer import FTPtoGCPTransferableObject
+from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, _requires_release_date_template
+from gentroutils.transfer import TransferManager
+
+MAX_CONCURRENT_CONNECTIONS = 10
+
+
+class FetchSpec(Spec):
+    """Configuration fields for the fetch task.
+
+    The task downloads single file based on the `source_template` and uploads it to the `destination_template`.
+
+    The `FetchSpec` defines the parameters needed to fetch the GWAS Catalog release files.
+    These should be files that reside in the `https://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/` directory.
+
+    To make sure that we download the latest release and persist the release date,
+    we need to make a single request to the `stats_uri` endpoint, which returns the latest release date.
+    (We are not using the https://ftp.ebi.ac.uk/pub/databases/gwas/releases/latest/` endpoint, rather
+    the endpoint with `https://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/` format to
+    download the files.
+
+
+    Examples:
+    ---
+    >>> fs = FetchSpec(
+    ...     name="fetch associations",
+    ...     stats_uri="https://www.ebi.ac.uk/gwas/api/search/stats",
+    ...     source_template="ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-associations_ontology-annotated.tsv",
+    ...     destination_template="gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv",
+    ...     promote=True
+    ... )
+    >>> fs.name
+    'fetch associations'
+    >>> fs.stats_uri
+    'https://www.ebi.ac.uk/gwas/api/search/stats'
+    >>> fs.source_template
+    'ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-associations_ontology-annotated.tsv'
+    >>> fs.destination_template
+    'gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv'
+    >>> fs.promote
+    True
+    """
+
+    name: str = "fetch gwas catalog data"
+    """The name of the task."""
+
+    stats_uri: str = "https://www.ebi.ac.uk/gwas/api/search/stats"
+    """The URI to crawl the release statistics information from."""
+
+    source_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    """The template URI of the file to download."""
+
+    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    """The template URI to upload the file to."""
+
+    promote: bool = False
+    """Whether to promote the release information as the latest release.
+
+    Given the destination: `gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json`
+
+       * If set to `False` the task will upload the release information
+       only to the specified destination with `release_date` substituted by the value from the stats_uri endpoint.
+       Resulting in following destinations:
+            * `gs://gwas_catalog_inputs/gentroutils/20231001/stats.json`
+
+       * If set to `True`, the task will also upload the release information
+        to the destination with `release_date` substituted to `latest` literal, effectively
+        promoting the release as the latest release.
+    """
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def destinations(self) -> list[TemplateDestination]:
+        """Get the list of destinations templates where the release information will be saved.
+
+        Returns:
+            list[TemplateDestination]: A list of TemplateDestination objects with the formatted destination paths.
+
+        Depending on the `promote` flag this property will return:
+            * If `promote` is `False`, it will return a single destination template.
+            * If `promote` is `True`, it will return two destinations:
+                1. The destination template with the release date substituted.
+                2. The destination with the release date substituted to `latest`.
+        """
+        d1 = TemplateDestination(self.destination_template, False)
+        if self.promote:
+            d2 = d1.format({"release_date": "latest"})
+            return [d1, d2]
+        return [d1]
+
+    def substituted_destinations(self, release_info: GwasCatalogReleaseInfo) -> list[str]:
+        """Safely parse the destination name to ensure it is valid."""
+        substitutions = {"release_date": release_info.strfmt("%Y%m%d")}
+        return [
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+        ]
+
+    def substituted_sources(self, release_info: GwasCatalogReleaseInfo) -> list[str]:
+        """Safely parse the source name to ensure it is valid."""
+        substitutions = {"release_date": release_info.strfmt("%Y/%m/%d")}
+        if self.promote:
+            return [self.source_template.format(**substitutions)] * 2
+        return [self.source_template.format(**substitutions)]
+
+    def model_post_init(self, __context: Any) -> None:
+        """Method to ensure the scratchpad is set to ignore missing replacements."""
+        self.scratchpad_ignore_missing = True
+
+
+class Fetch(Task):
+    """Task to fetch files from the GWAS Catalog release directory file."""
+
+    def __init__(self, spec: FetchSpec, context: TaskContext) -> None:
+        super().__init__(spec, context)
+        self.spec: FetchSpec
+
+    @report
+    def run(self) -> Self:
+        """Fetch the file from the remote to local."""
+        logger.info(f"Fetching file from {self.spec.source_template}")
+        release_info = GwasCatalogReleaseInfo.from_uri(self.spec.stats_uri)
+        logger.info(f"Release information: {release_info}")
+        destinations = self.spec.substituted_destinations(release_info)
+        sources = self.spec.substituted_sources(release_info)
+        transferable_objects = [
+            FTPtoGCPTransferableObject(source=s, destination=d) for s, d in zip(sources, destinations, strict=True)
+        ]
+        logger.info(f"Transferable objects: {transferable_objects}")
+        TransferManager().transfer(transferable_objects)
+        logger.success("File transferred successfully.")
+        return self

gentroutils/transfer.py

@@ -0,0 +1,81 @@
+"""Transfer module."""
+
+import asyncio
+from collections.abc import Sequence
+from typing import cast
+
+import tqdm
+from loguru import logger
+
+from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
+from gentroutils.io.transfer import FTPtoGCPTransferableObject, PolarsDataFrameToGCSTransferableObject
+from gentroutils.io.transfer.model import TransferableObject
+
+
+class TransferManager:
+    """Manager class for handling the transfer of various transferable objects.
+
+    This class provides static methods to `transfer` to move files or objects.
+    Currently it supports:
+
+        - FTP to Google Cloud Storage (GCP) transfers using `FTPtoGCPTransferableObject`.
+        - Polars DataFrame to GCS transfers using `PolarsDataFrameToGCSTransferableObject`.
+
+    """
+
+    @staticmethod
+    async def transfer_ftp_to_gcp(transferable_objects: Sequence[FTPtoGCPTransferableObject]) -> None:
+        """Update GWAS Catalog metadata directly to cloud bucket.
+
+        This method transfers files from FTP to Google Cloud Storage (GCS) using the provided
+        FTPtoGCPTransferableObject instances.
+        It fetches the data for the file provided in the local FTP path, collects the
+        data asynchronously to buffer, and uploads it to the provided GCP bucket blob.
+
+        Args:
+            transferable_objects (Sequence[FTPtoGCPTransferableObject]): A sequence of FTPtoGCPTransferableObject instances.
+
+        """
+        # we always want to have the logs from this command uploaded to the target bucket
+        transfer_tasks = [asyncio.create_task(x.transfer()) for x in transferable_objects]
+        for f in tqdm.tqdm(asyncio.as_completed(transfer_tasks), total=len(transfer_tasks), desc="Downloading"):
+            await f
+        logger.info("gwas_curation_update step completed.")
+
+    @staticmethod
+    async def transfer_polars_to_gcs(transferable_objects: Sequence[PolarsDataFrameToGCSTransferableObject]) -> None:
+        """Transfer Polars DataFrames to Google Cloud Storage.
+
+        This method transfers Polars DataFrames to GCS using the provided
+        PolarsDataFrameToGCSTransferableObject instances.
+
+        Args:
+            transferable_objects (Sequence[PolarsDataFrameToGCSTransferableObject]): A sequence of PolarsDataFrameToGCSTransferableObject instances.
+
+        """
+        transfer_tasks = [asyncio.create_task(x.transfer()) for x in transferable_objects]
+        for f in tqdm.tqdm(asyncio.as_completed(transfer_tasks), total=len(transfer_tasks), desc="Uploading"):
+            await f
+        logger.info("Polars DataFrame transfer to GCS completed.")
+
+    def transfer(self, transferable_objects: Sequence[TransferableObject]) -> None:
+        """Transfer method that handles different types of transferable objects.
+
+        Main method to manage the transfer of various transferable objects.
+
+        Args:
+            transferable_objects (Sequence[TransferableObject]): A sequence of TransferableObject instances.
+
+        Raises:
+            GentroutilsError: If the list of transferable objects is empty or if the objects are not instances of the expected types.
+        """
+        if not transferable_objects:
+            raise GentroutilsError(GentroutilsErrorMessage.EMPTY_TRANSFERABLE_OBJECTS)
+        elif all(isinstance(c, FTPtoGCPTransferableObject) for c in transferable_objects):
+            ftp_objects = cast(Sequence[FTPtoGCPTransferableObject], transferable_objects)
+            asyncio.run(self.transfer_ftp_to_gcp(ftp_objects))
+        elif all(isinstance(c, PolarsDataFrameToGCSTransferableObject) for c in transferable_objects):
+            polars_objects = cast(Sequence[PolarsDataFrameToGCSTransferableObject], transferable_objects)
+            asyncio.run(self.transfer_polars_to_gcs(polars_objects))
+        else:
+            raise GentroutilsError(GentroutilsErrorMessage.INVALID_TRANSFERABLE_OBJECTS)