gentroutils 1.5.0__py3-none-any.whl → 1.6.0.dev1__py3-none-any.whl

gentroutils/__init__.py

@@ -1,46 +1,11 @@
-"""Cli for gentroutils."""
+"""Main entry point for the gentroutils package."""
 
-from __future__ import annotations
+from otter import Runner
 
-import logging
-import time
 
-import click
-import pyfiglet
-
-from gentroutils.commands import (
-    update_gwas_curation_metadata_command,
-    validate_gwas_curation,
-)
-from gentroutils.commands.utils import set_log_file, set_log_lvl, teardown_cli
-
-logger = logging.getLogger("gentroutils")
-logger.setLevel(logging.DEBUG)
-
-
-@click.group()
-@click.option("-d", "--dry-run", is_flag=True, default=False)
-@click.option(
-    "-v",
-    count=True,
-    default=0,
-    callback=set_log_lvl,
-    help="Increase verbosity of the logging. Can be used multiple times. The default log level is ERROR, -v is INFO, -vv is DEBUG",
-)
-@click.option("-q", "--log-file", callback=set_log_file, required=False)
-@click.pass_context
-def cli(ctx: click.Context, **kwargs: dict[str, str]) -> None:
-    r"""Gentroutils Command Line Interface."""
-    ascii_art = pyfiglet.Figlet(font="serifcap").renderText("Gentroutils")
-    click.echo(click.style(ascii_art, fg="blue"))
-    ctx.max_content_width = 200
-    ctx.ensure_object(dict)
-    ctx.obj["dry_run"] = kwargs["dry_run"]
-    ctx.obj["execution_start"] = time.time()
-    ctx.call_on_close(lambda: teardown_cli(ctx))
-
-
-cli.add_command(update_gwas_curation_metadata_command)
-cli.add_command(validate_gwas_curation)
-
-__all__ = ["cli"]
+def main() -> None:
+    """Main function to start the gentroutils otter runner."""
+    runner = Runner("gentroutils")
+    runner.start()
+    runner.register_tasks("gentroutils.tasks")
+    runner.run()

gentroutils/errors.py

@@ -0,0 +1,39 @@
+"""Gentroutils exceptions module."""
+
+from enum import Enum
+
+
+class GentroutilsErrorMessage(Enum):
+    """Base class for all exceptions in the gentroutils module."""
+
+    UNSUPPORTED_URL_SCHEME = "Unsupported URL scheme: {scheme}"
+    BUCKET_NAME_MISSING = "Bucket name is missing in the URL: {url}"
+    FILE_NAME_MISSING = "File name is missing in the URL: {url}"
+    GCS_CLIENT_INITIALIZATION_FAILED = "Failed to initialize Google Cloud Storage client: {error}"
+    FTP_SERVER_MISSING = "FTP server is missing in the URL: {url}"
+    INVALID_TRANSFERABLE_OBJECTS = (
+        "Invalid transferable objects provided. Expected FTPtoGCPTransferableObject instances."
+    )
+    DOWNLOAD_STUDIES_EMPTY = "List of downloaded studies from GWAS Catalog release is empty: {path}"
+    PREVIOUS_CURATION_EMPTY = "Previous curation data is empty: {path}"
+    FAILED_TO_FETCH = "Failed to fetch the release information from the {uri}"
+    MISSING_RELEASE_DATE_TEMPLATE = (
+        "The destination must contain a template for the release date, e.g. some/path/{release_date}/file.txt."
+    )
+    EMPTY_TRANSFERABLE_OBJECTS = "Transferable objects list cannot be empty."
+
+
+class GentroutilsError(Exception):
+    """Base class for the gentroutils exceptions."""
+
+    def __init__(self, message: GentroutilsErrorMessage, **kwargs: str) -> None:
+        """Initialize the GentroutilsError exception.
+
+        Args:
+            message (GentroutilsErrorMessage): The error message.
+            **kwargs (str): Additional arguments to format the message.
+        """
+        super().__init__(message.value.format(**kwargs))
+
+
+__all__ = ["GentroutilsError", "GentroutilsErrorMessage"]

gentroutils/io/path/__init__.py

@@ -0,0 +1,6 @@
+"""Module for handling various path types in gentroutils."""
+
+from gentroutils.io.path.ftp import FTPPath
+from gentroutils.io.path.gcs import GCSPath
+
+__all__ = ["FTPPath", "GCSPath"]

gentroutils/io/path/ftp.py

@@ -0,0 +1,48 @@
+"""FTP path representation."""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
+
+
+class FTPPath:
+    """A class to represent a path in a cloud storage system."""
+
+    # Supported URL schemes
+    SUPPORTED_SCHEMES = ["ftp"]
+
+    def __init__(self, uri: str) -> None:
+        """Initialize the FTPPath object.
+
+        Args:
+            uri (str): The path to object in ftp server.
+
+        Raises:
+            GentroutilsError: If the URL scheme is not supported or if the server or filename is missing.
+        """
+        self.uri = uri
+        # NOTE: The urlparse matches to following tuple
+        # ('scheme', 'netloc', 'path', 'params', 'query', 'fragment')
+        parsed_url = urlparse(uri)
+
+        if parsed_url.scheme not in self.SUPPORTED_SCHEMES:
+            raise GentroutilsError(GentroutilsErrorMessage.UNSUPPORTED_URL_SCHEME, scheme=parsed_url.scheme)
+
+        self.server = parsed_url.netloc
+        if not self.server:
+            raise GentroutilsError(GentroutilsErrorMessage.FTP_SERVER_MISSING, url=uri)
+
+        self.filename = parsed_url.path.split("/")[-1]
+        if not self.filename:
+            raise GentroutilsError(GentroutilsErrorMessage.FILE_NAME_MISSING, url=uri)
+        self.base_dir = "/".join(parsed_url.path.split("/")[0:-1])
+
+    def __repr__(self) -> str:
+        """Return the string representation of the CloudPath object.
+
+        Returns:
+            str: The string representation of the CloudPath object.
+        """
+        return self.uri

gentroutils/io/path/gcs.py

@@ -0,0 +1,45 @@
+"""Google Cloud Storage (GCS) path representation."""
+
+from urllib.parse import urlparse
+
+from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
+
+
+class GCSPath:
+    """A class to represent a path in a cloud storage system."""
+
+    # Supported URL schemes
+    SUPPORTED_SCHEMES = ["gs"]
+
+    def __init__(self, uri: str) -> None:
+        """Initialize the CloudPath object.
+
+        Args:
+            uri (str): The path to the cloud storage object.
+
+        Raises:
+            GentroutilsError: If the URL scheme is not supported or if the bucket or object is missing.
+        """
+        self.uri = uri
+        # NOTE: The urlparse matches to following tuple
+        # ('scheme', 'netloc', 'path', 'params', 'query', 'fragment')
+        parsed_url = urlparse(uri)
+
+        if parsed_url.scheme not in self.SUPPORTED_SCHEMES:
+            raise GentroutilsError(GentroutilsErrorMessage.UNSUPPORTED_URL_SCHEME, scheme=parsed_url.scheme)
+
+        self.bucket = parsed_url.netloc
+        if not self.bucket:
+            raise GentroutilsError(GentroutilsErrorMessage.BUCKET_NAME_MISSING, url=uri)
+
+        self.object = parsed_url.path.lstrip("/").rstrip("/")
+        if not self.object:
+            raise GentroutilsError(GentroutilsErrorMessage.FILE_NAME_MISSING, url=uri)
+
+    def __repr__(self) -> str:
+        """Return the string representation of the CloudPath object.
+
+        Returns:
+            str: The string representation of the CloudPath object.
+        """
+        return self.uri

gentroutils/io/transfer/__init__.py

@@ -0,0 +1,6 @@
+"""Module for handling transfer operations in gentroutils."""
+
+from gentroutils.io.transfer.ftp_to_gcs import FTPtoGCPTransferableObject
+from gentroutils.io.transfer.polars_to_gcs import PolarsDataFrameToGCSTransferableObject
+
+__all__ = ["FTPtoGCPTransferableObject", "PolarsDataFrameToGCSTransferableObject"]

gentroutils/io/transfer/ftp_to_gcs.py

@@ -0,0 +1,49 @@
+"""Transfer files from FTP to Google Cloud Storage (GCS)."""
+
+import io
+import re
+from typing import Annotated
+
+import aioftp
+from google.cloud import storage  # type: ignore[attr-defined]
+from loguru import logger
+from pydantic import AfterValidator
+
+from gentroutils.io.path import FTPPath, GCSPath
+from gentroutils.io.transfer.model import TransferableObject
+
+
+class FTPtoGCPTransferableObject(TransferableObject):
+    """A class to represent an object that can be transferred from FTP to GCP."""
+
+    source: Annotated[str, AfterValidator(lambda x: str(FTPPath(x)))]
+    destination: Annotated[str, AfterValidator(lambda x: str(GCSPath(x)))]
+
+    async def transfer(self) -> None:
+        """Transfer files from FTP to GCP.
+
+        This function fetches the data for the file provided in the local FTP path, collects the
+        data asynchronously to buffer, and uploads it to the provided GCP bucket blob.
+        """
+        logger.info(f"Attempting to transfer data from {self.source} to {self.destination}.")
+        gcs_obj = GCSPath(self.destination)
+        ftp_obj = FTPPath(self.source)
+
+        async with aioftp.Client.context(ftp_obj.server, user="anonymous", password="anonymous") as ftp:  # noqa: S106
+            bucket = storage.Client().bucket(gcs_obj.bucket)
+            blob = bucket.blob(gcs_obj.object)
+            logger.info(f"Changing directory to {ftp_obj.base_dir}.")
+            await ftp.change_directory(ftp_obj.base_dir)
+            pwd = await ftp.get_current_directory()
+            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(pwd))
+            if dir_match:
+                logger.info(f"Found release date!: {dir_match.group('release_date')}")
+            buffer = io.BytesIO()
+            stream = await ftp.download_stream(ftp_obj.filename)
+            async with stream:
+                async for block in stream.iter_by_block():
+                    buffer.write(block)
+            buffer.seek(0)
+            content = buffer.getvalue().decode("utf-8")
+            buffer.close()
+            blob.upload_from_string("".join(content))

gentroutils/io/transfer/model.py

@@ -0,0 +1,36 @@
+"""Base implementation for transferable objects in gentroutils."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class TransferableObject(BaseModel):
+    """Base class for transferable objects in gentroutils.
+
+    Each transferable object should implement the `transfer` method to define how the object is transferred.
+    Also each object should have the:
+
+        - `source`: The source location of the object.
+        - `destination`: The destination location where the object will be transferred.
+    """
+
+    source: Any
+    destination: Any
+
+    def __repr__(self) -> str:
+        """Return a string representation of the transferable object."""
+        return f"{self.__class__.__name__}(source={self.source}, destination={self.destination})"
+
+    def __str__(self) -> str:
+        """Return a string representation of the transferable object."""
+        return self.__repr__()
+
+    def transfer(self):
+        """Transfer the object to the destination."""
+        raise NotImplementedError("Implement in derivative class.")
+
+    class Config:
+        """Configuration that ensures that the derivative classes can have arbitrary types."""
+
+        arbitrary_types_allowed = True

gentroutils/io/transfer/polars_to_gcs.py

@@ -0,0 +1,20 @@
+"""Module for transferring Polars DataFrames to Google Cloud Storage (GCS)."""
+
+import polars as pl
+from loguru import logger
+
+from gentroutils.io.transfer.model import TransferableObject
+
+
+class PolarsDataFrameToGCSTransferableObject(TransferableObject):
+    """A TransferableObject for transferring Polars DataFrames to Google Cloud Storage."""
+
+    source: pl.DataFrame
+    destination: str
+
+    async def transfer(self) -> None:
+        """Transfer the Polars DataFrame to the specified GCS destination."""
+        # Convert Polars DataFrame to CSV and upload to GCS
+        logger.info(f"Transferring Polars DataFrame to {self.destination}.")
+        self.source.write_csv(self.destination)
+        logger.info(f"Uploading DataFrame to {self.destination}")

gentroutils/parsers/__init__.py

@@ -0,0 +1 @@
+"""Module for parsing and handling data in gentroutils."""

gentroutils/parsers/curation.py

@@ -0,0 +1,168 @@
+"""Module to handle the curation of GWAS Catalog data."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+import polars as pl
+from loguru import logger
+
+from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
+
+
+class CurationSchema(StrEnum):
+    """Enum to define the schema for curation tasks."""
+
+    STUDY_ID = "studyId"
+    """The unique identifier for a study."""
+    STUDY_TYPE = "studyType"
+    """The type of study (e.g., gwas)."""
+    ANALYSIS_FLAG = "analysisFlag"
+    """Flag indicating the type of analysis performed."""
+    QUALITY_CONTROL = "qualityControl"
+    """Quality control status of the study."""
+    IS_CURATED = "isCurated"
+    """Flag indicating whether the study has been curated."""
+    PUBMED_ID = "pubmedId"
+    """The PubMed identifier for the study."""
+    PUBLICATION_TITLE = "publicationTitle"
+    """The title of the publication associated with the study."""
+    TRAIT_FROM_SOURCE = "traitFromSource"
+    """The trait as reported in the source data."""
+
+    @classmethod
+    def columns(cls) -> list[str]:
+        """Get the list of columns defined in the schema."""
+        return [member.value for member in cls]
+
+    @classmethod
+    def extended_columns(cls) -> list[str]:
+        """Get the list of columns defined in the schema, including additional metadata."""
+        return [*cls.columns(), "status"]
+
+
+class DownloadStudiesSchema(StrEnum):
+    """Enum to define the columns for the download studies task."""
+
+    STUDY_ID = "studyId"
+    """The unique identifier for a study."""
+    TRAIT_FROM_SOURCE = "traitFromSource"
+    """The trait as reported in the source data."""
+    PUBMED_ID = "pubmedId"
+    """The PubMed identifier for the study."""
+    PUBLICATION_TITLE = "publicationTitle"
+    """The title of the publication associated with the study."""
+
+    @classmethod
+    def mapping(cls) -> dict[str, str]:
+        """Get the mapping of columns to their respective names."""
+        return {
+            "STUDY ACCESSION": cls.STUDY_ID,
+            "STUDY": cls.PUBLICATION_TITLE,
+            "PUBMED ID": cls.PUBMED_ID,
+            "DISEASE/TRAIT": cls.TRAIT_FROM_SOURCE,
+        }
+
+    @classmethod
+    def columns(cls) -> list[str]:
+        """Get the list of columns defined in the schema."""
+        return [member.value for member in cls]
+
+
+class CuratedStudyStatus(StrEnum):
+    """Enum to define the status of a curated study."""
+
+    REMOVED = "removed"
+    """The study has been removed from the GWAS Catalog."""
+    NEW = "new"
+    """The study is new in the GWAS Catalog."""
+    CURATED = "curated"
+    """The study has been curated and is still in the GWAS Catalog."""
+
+
+class GWASCatalogCuration:
+    """Class to handle the curation of GWAS Catalog data."""
+
+    def __init__(self, previous_curation: pl.DataFrame, studies: pl.DataFrame):
+        """Initialize the GWASCatalogCuration with previous curation and studies data."""
+        logger.debug("Initializing GWASCatalogCuration with previous curation and studies data.")
+        self.previous_curation = previous_curation
+        logger.debug("Previous curation data loaded with shape: {}", previous_curation.shape)
+        self.studies = studies
+        logger.debug("Studies data loaded with shape: {}", studies.shape)
+
+    @classmethod
+    def from_prev_curation(cls, previous_curation_path: str, download_studies_path: str) -> GWASCatalogCuration:
+        """Create a GWASCatalogCuration instance from previous curation and studies."""
+        previous_curation_df = pl.read_csv(
+            previous_curation_path,
+            separator="\t",
+            has_header=True,
+            columns=CurationSchema.columns(),
+        )
+        if previous_curation_df.is_empty():
+            raise GentroutilsError(GentroutilsErrorMessage.PREVIOUS_CURATION_EMPTY, path=previous_curation_path)
+        studies_df = pl.read_csv(
+            download_studies_path,
+            separator="\t",
+            quote_char="`",
+            has_header=True,
+            columns=list(DownloadStudiesSchema.mapping().keys()),
+        )
+        if studies_df.is_empty():
+            raise GentroutilsError(GentroutilsErrorMessage.DOWNLOAD_STUDIES_EMPTY, path=download_studies_path)
+        studies_df = studies_df.rename(mapping=DownloadStudiesSchema.mapping())
+        return cls(previous_curation_df, studies_df)
+
+    @property
+    def result(self) -> pl.DataFrame:
+        """Curate the GWAS Catalog data."""
+        # Studies that are curated but were removed from the GWAS Catalog
+        removed_studies = self.previous_curation.join(self.studies, on=CurationSchema.STUDY_ID, how="anti").select(
+            CurationSchema.STUDY_ID, pl.lit(CuratedStudyStatus.REMOVED).alias("status")
+        )
+        logger.debug("Removed studies identified: {}", removed_studies.shape[0])
+
+        # studies that are curated and still in the GWAS Catalog
+        curated_studies = self.previous_curation.join(self.studies, on=CurationSchema.STUDY_ID, how="inner").select(
+            CurationSchema.STUDY_ID, pl.lit(CuratedStudyStatus.CURATED).alias("status")
+        )
+        logger.debug("Curated studies identified: {}", curated_studies.shape[0])
+
+        # Combine all previous studies with updated status information.
+        prev_studies = pl.concat([removed_studies, curated_studies], how="vertical")
+        logger.debug("Previous studies after combining removed and curated: {}", prev_studies.shape[0])
+
+        # Bring back the information from the previous curation
+        prev_studies = (
+            prev_studies.join(self.previous_curation, on=CurationSchema.STUDY_ID, how="full", coalesce=True)
+            .with_columns(pl.coalesce(CurationSchema.IS_CURATED, pl.lit(False)).alias(CurationSchema.IS_CURATED))
+            .select(*CurationSchema.extended_columns())
+        )
+        logger.debug("Previous studies after bringing previous curation data: {}", prev_studies.shape[0])
+
+        assert all(prev_studies.select(CurationSchema.STUDY_ID).is_unique()), "Study IDs must be unique after merging."
+
+        # Studies that are new in the GWAS Catalog
+        new_studies = self.studies.join(self.previous_curation, on=CurationSchema.STUDY_ID, how="anti").select(
+            CurationSchema.STUDY_ID,
+            pl.lit(None).alias(CurationSchema.STUDY_TYPE),
+            pl.lit(None).alias(CurationSchema.ANALYSIS_FLAG),
+            pl.lit(None).alias(CurationSchema.QUALITY_CONTROL),
+            pl.lit(False).alias(CurationSchema.IS_CURATED),
+            CurationSchema.PUBMED_ID,
+            CurationSchema.PUBLICATION_TITLE,
+            CurationSchema.TRAIT_FROM_SOURCE,
+            pl.lit(CuratedStudyStatus.NEW).alias("status"),
+        )
+        logger.debug("New studies identified: {}", new_studies.shape[0])
+
+        # Union of new studies and previously curated studies
+        all_studies = pl.concat([prev_studies, new_studies], how="vertical")
+        logger.debug("All studies after combining new and previous: {}", all_studies.shape[0])
+
+        # Ensure the contract on the output dataframe
+        assert all(all_studies.select(CurationSchema.STUDY_ID).is_unique()), "Study IDs must be unique after merging."
+        assert all_studies.shape[0] == all_studies.shape[0], "The number of studies must match after merging."
+
+        return all_studies.select(CurationSchema.extended_columns())

gentroutils/tasks/__init__.py

@@ -0,0 +1,90 @@
+"""Gentroutils otter tasks."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import date
+
+import aiohttp
+from loguru import logger
+from pydantic import AliasPath, BaseModel, Field
+
+from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
+
+
+def _requires_release_date_template(path: str) -> str:
+    """Ensure that the destination path contains a template for the release date."""
+    if "{release_date}" not in path:
+        raise GentroutilsError(GentroutilsErrorMessage.MISSING_RELEASE_DATE_TEMPLATE, release_date="{release_date}")
+    return path
+
+
+@dataclass
+class TemplateDestination:
+    """A destination that can be formatted with a release date."""
+
+    destination: str
+    """The destination path that can be formatted with a release date."""
+    is_substituted: bool = False
+    """Whether the destination template has already been substituted."""
+
+    def format(self, substitutions: dict[str, str]) -> TemplateDestination:
+        """Format the destination with the given substitutions.
+
+        This method returns a new TemplateDestination object (not a copy of the current one) with the formatted destination.
+        """
+        return TemplateDestination(self.destination.format(**substitutions), True)
+
+
+class GwasCatalogReleaseInfo(BaseModel):
+    """Model to hold GWAS Catalog release information."""
+
+    release_date: date = Field(validation_alias=AliasPath("date"))
+    """Release date of the GWAS Catalog."""
+
+    number_of_associations: int = Field(validation_alias=AliasPath("associations"))
+    """Number of associations in the GWAS Catalog."""
+
+    number_of_studies: int = Field(validation_alias=AliasPath("studies"))
+    """Number of studies in the GWAS Catalog."""
+
+    number_of_sumstats: int = Field(validation_alias=AliasPath("sumstats"))
+    """Number of summary statistics in the GWAS Catalog."""
+
+    number_of_snps: int = Field(validation_alias=AliasPath("snps"))
+    """Number of SNPs in the GWAS Catalog."""
+
+    ensembl_build: str = Field(validation_alias=AliasPath("ensemblbuild"))
+    """Ensembl version used in the GWAS Catalog."""
+
+    dbsnp_build: str = Field(validation_alias=AliasPath("dbsnpbuild"))
+    """dbSNP version used in the GWAS Catalog."""
+
+    efo_version: str = Field(validation_alias=AliasPath("efoversion"))
+    """EFO version used in the GWAS Catalog."""
+
+    gene_build: str = Field(validation_alias=AliasPath("genebuild"))
+    """Gene build version used in the GWAS Catalog."""
+
+    def strfmt(self, format: str = "%Y%m%d") -> str:
+        """Return a string representation of the release information."""
+        return self.release_date.strftime(format)
+
+    @staticmethod
+    async def _get_release_info(uri: str) -> GwasCatalogReleaseInfo:
+        """Get the release information from the specified URI."""
+        headers = {"Accept": "application/json"}
+        async with aiohttp.ClientSession(headers=headers) as session:
+            async with session.get(uri) as response:
+                release_info = await response.json()
+                return GwasCatalogReleaseInfo(**release_info)
+
+    @classmethod
+    def from_uri(cls, uri: str) -> GwasCatalogReleaseInfo:
+        """Fetch the release information from the specified URI."""
+        try:
+            return asyncio.run(cls._get_release_info(uri))
+        except aiohttp.ClientError as e:
+            logger.error(f"Error fetching release info: {e}")
+            raise GentroutilsError(GentroutilsErrorMessage.FAILED_TO_FETCH, uri=uri)