gentroutils 2.0.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

gentroutils/io/transfer/ftp_to_gcs.py

@@ -32,12 +32,24 @@ class FTPtoGCPTransferableObject(TransferableObject):
         async with aioftp.Client.context(ftp_obj.server, user="anonymous", password="anonymous") as ftp:  # noqa: S106
             bucket = storage.Client().bucket(gcs_obj.bucket)
             blob = bucket.blob(gcs_obj.object)
-            logger.info(f"Changing directory to {ftp_obj.base_dir}.")
-            await ftp.change_directory(ftp_obj.base_dir)
-            pwd = await ftp.get_current_directory()
-            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(pwd))
+            logger.info(f"Searching for the release date in the provided ftp path: {ftp_obj.base_dir}.")
+            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(ftp_obj.base_dir))
+
             if dir_match:
-                logger.info(f"Found release date!: {dir_match.group('release_date')}")
+                logger.info(f"Found release date to search in the ftp {dir_match.group('release_date')}.")
+                release_date = dir_match.group("release_date")
+                try:
+                    await ftp.change_directory(ftp_obj.base_dir)
+                except aioftp.StatusCodeError as e:
+                    logger.error(f"Failed to change directory to {ftp_obj.base_dir}: {e}")
+                    logger.warning("Attempting to load the `latest` release.")
+                    ftp_obj = FTPPath(self.source.replace(release_date, "latest"))
+                    try:
+                        await ftp.change_directory(ftp_obj.base_dir)
+                    except aioftp.StatusCodeError as e:
+                        logger.error(f"Failed to find the latest release under {ftp_obj}")
+                        raise
+
             buffer = io.BytesIO()
             stream = await ftp.download_stream(ftp_obj.filename)
             async with stream:

gentroutils/tasks/__init__.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections import defaultdict
 from dataclasses import dataclass
 from datetime import date
 
@@ -13,7 +14,12 @@ from pydantic import AliasPath, BaseModel, Field
 from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
 
 
-def _requires_release_date_template(path: str) -> str:
+class KeepMissing(defaultdict[str, str]):
+    def __missing__(self, key):
+        return "{" + key + "}"
+
+
+def destination_validator(path: str) -> str:
     """Ensure that the destination path contains a template for the release date."""
     if "{release_date}" not in path:
         raise GentroutilsError(GentroutilsErrorMessage.MISSING_RELEASE_DATE_TEMPLATE, release_date="{release_date}")
@@ -34,7 +40,7 @@ class TemplateDestination:
 
         This method returns a new TemplateDestination object (not a copy of the current one) with the formatted destination.
         """
-        return TemplateDestination(self.destination.format(**substitutions), True)
+        return TemplateDestination(self.destination.format_map(KeepMissing(**substitutions)), True)
 
 
 class GwasCatalogReleaseInfo(BaseModel):
@@ -83,6 +89,7 @@ class GwasCatalogReleaseInfo(BaseModel):
     @classmethod
     def from_uri(cls, uri: str) -> GwasCatalogReleaseInfo:
         """Fetch the release information from the specified URI."""
+        logger.debug(f"Fetching release info from {uri}")
         try:
             return asyncio.run(cls._get_release_info(uri))
         except aiohttp.ClientError as e:

gentroutils/tasks/crawl.py

@@ -1,7 +1,6 @@
 """Module to handle the crawling of GWAS Catalog release information."""
 
 import tempfile
-from functools import cached_property
 from pathlib import Path
 from typing import Annotated, Any, Self
 
@@ -9,9 +8,10 @@ from loguru import logger
 from otter.storage import get_remote_storage
 from otter.task.model import Spec, Task, TaskContext
 from otter.task.task_reporter import report
-from pydantic import AfterValidator, computed_field
+from pydantic import AfterValidator
 
-from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, _requires_release_date_template
+from gentroutils.io.path import GCSPath
+from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, destination_validator
 
 
 class CrawlSpec(Spec):
@@ -68,7 +68,7 @@ class CrawlSpec(Spec):
     stats_uri: str = "https://www.ebi.ac.uk/gwas/api/search/stats"
     """The URI to crawl the release statistics information from."""
 
-    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    destination_template: Annotated[str, AfterValidator(destination_validator)]
     """The destination path to save the release information.
        This path should always be a template string that includes `{release_date}`.
        For example, `gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json`.
@@ -91,8 +91,6 @@ class CrawlSpec(Spec):
         promoting the release as the latest release.
     """
 
-    @computed_field  # type: ignore[prop-decorator]
-    @cached_property
     def destinations(self) -> list[TemplateDestination]:
         """Get the list of destinations templates where the release information will be saved.
 
@@ -105,17 +103,17 @@ class CrawlSpec(Spec):
                 1. The destination template with the release date substituted.
                 2. The destination with the release date substituted to `latest`.
         """
-        d1 = self.destination_template
+        d1 = TemplateDestination(self.destination_template, False)
         if self.promote:
-            d2 = self.destination_template.format(release_date="latest")
-            return [TemplateDestination(d1, False), TemplateDestination(d2, True)]
-        return [TemplateDestination(d1, False)]
+            d2 = d1.format({"release_date": "latest"})
+            return [d1, d2]
+        return [d1]
 
     def substituted_destinations(self, release_info: GwasCatalogReleaseInfo) -> list[str]:
         """Safely parse the destination name to ensure it is valid."""
         substitutions = {"release_date": release_info.strfmt("%Y%m%d")}
         return [
-            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations()
         ]
 
     def model_post_init(self, __context: Any) -> None:
@@ -141,9 +139,10 @@ class Crawl(Task):
                 logger.info(f"Destinations for release information: {destinations}")
                 for destination in destinations:
                     storage = get_remote_storage(destination)
+                    assert "gs://" in destination, f"Invalid GCS path in destination template: {destination}"
                     storage.upload(Path(source.name), destination)
                     logger.info(f"Release information written to {destination}")
-            return self
+        return self
 
     @report
     def run(self) -> Self:

gentroutils/tasks/curation.py

@@ -3,16 +3,16 @@
 from __future__ import annotations
 
 from datetime import date
-from functools import cached_property
 from typing import Annotated, Any, Self
 
+from loguru import logger
 from otter.task.model import Spec, Task, TaskContext
 from otter.task.task_reporter import report
-from pydantic import AfterValidator, computed_field
+from pydantic import AfterValidator
 
 from gentroutils.io.transfer.polars_to_gcs import PolarsDataFrameToGCSTransferableObject
 from gentroutils.parsers.curation import GWASCatalogCuration
-from gentroutils.tasks import TemplateDestination, _requires_release_date_template
+from gentroutils.tasks import TemplateDestination, destination_validator
 from gentroutils.transfer import TransferManager
 
 
@@ -50,14 +50,12 @@ class CurationSpec(Spec):
     studies: str
     """The path to the studies data."""
 
-    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    destination_template: Annotated[str, AfterValidator(destination_validator)]
     """The destination path for the curation data."""
 
     promote: bool = False
     """Whether to promote the curation data to the latest version."""
 
-    @computed_field  # type: ignore[prop-decorator]
-    @cached_property
     def destinations(self) -> list[TemplateDestination]:
         """Get the list of destinations templates where the release information will be saved.
 
@@ -80,7 +78,7 @@ class CurationSpec(Spec):
         """Safely parse the destination name to ensure it is valid."""
         substitutions = {"release_date": release_date.strftime("%Y%m%d")}
         return [
-            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations()
         ]
 
     def model_post_init(self, __context: Any) -> None:
@@ -99,9 +97,13 @@ class Curation(Task):
     @report
     def run(self) -> Self:
         """Run the curation task."""
+        logger.info("Starting curation task.")
         release_date = date.today()
+        logger.debug(f"Using release date: {release_date}")
         destinations = self.spec.substituted_destinations(release_date)
+        logger.debug(f"Destinations for curation data: {destinations}")
         curation = GWASCatalogCuration.from_prev_curation(self.spec.previous_curation, self.spec.studies)
+        logger.debug(f"Curation result preview:\n{curation.result.head()}")
         transfer_objects = [
             PolarsDataFrameToGCSTransferableObject(source=curation.result, destination=d) for d in destinations
         ]

gentroutils/tasks/fetch.py

@@ -1,15 +1,14 @@
 """Module to handle the fetching of GWAS Catalog release files."""
 
-from functools import cached_property
 from typing import Annotated, Any, Self
 
 from loguru import logger
 from otter.task.model import Spec, Task, TaskContext
 from otter.task.task_reporter import report
-from pydantic import AfterValidator, computed_field
+from pydantic import AfterValidator
 
 from gentroutils.io.transfer import FTPtoGCPTransferableObject
-from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, _requires_release_date_template
+from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, destination_validator
 from gentroutils.transfer import TransferManager
 
 MAX_CONCURRENT_CONNECTIONS = 10
@@ -57,10 +56,10 @@ class FetchSpec(Spec):
     stats_uri: str = "https://www.ebi.ac.uk/gwas/api/search/stats"
     """The URI to crawl the release statistics information from."""
 
-    source_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    source_template: Annotated[str, AfterValidator(destination_validator)]
     """The template URI of the file to download."""
 
-    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    destination_template: Annotated[str, AfterValidator(destination_validator)]
     """The template URI to upload the file to."""
 
     promote: bool = False
@@ -78,8 +77,6 @@ class FetchSpec(Spec):
         promoting the release as the latest release.
     """
 
-    @computed_field  # type: ignore[prop-decorator]
-    @cached_property
     def destinations(self) -> list[TemplateDestination]:
         """Get the list of destinations templates where the release information will be saved.
 
@@ -102,7 +99,7 @@ class FetchSpec(Spec):
         """Safely parse the destination name to ensure it is valid."""
         substitutions = {"release_date": release_info.strfmt("%Y%m%d")}
         return [
-            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations()
         ]
 
     def substituted_sources(self, release_info: GwasCatalogReleaseInfo) -> list[str]:

{gentroutils-2.0.0.dist-info → gentroutils-3.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gentroutils
-Version: 2.0.0
+Version: 3.1.0
 Summary: Open Targets python genetics utility CLI tools
 Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
 License-Expression: Apache-2.0
@@ -15,10 +15,11 @@ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Requires-Python: >=3.13
 Requires-Dist: aioftp>=0.25.1
 Requires-Dist: aiohttp>=3.11.18
+Requires-Dist: gcsfs>=2025.7.0
 Requires-Dist: google-cloud-storage>=3.1.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: opentargets-otter>=25.0.2
-Requires-Dist: polars>=1.31.0
+Requires-Dist: polars[fsspec]>=1.31.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: tqdm>=4.67.1
 Description-Content-Type: text/markdown
@@ -33,7 +34,7 @@ Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
 
 ## Installation
 
-```
+```{bash}
 pip install gentroutils
 ```
 
@@ -48,6 +49,7 @@ gentroutils --help
 ## Usage
 
 To run a single step run
+
 ```{bash}
 uv run gentroutils -s gwas_catalog_release  # After cloning the repository
 gentroutils -s gwas_catalog_release -c otter_config.yaml # When installed by pip
@@ -60,6 +62,11 @@ The `gentroutils` repository uses the [otter](https://github.com/opentargets/ott
 
 For the top level fields refer to the [otter documentation](https://opentargets.github.io/otter/otter.config.html)
 
+> [!NOTE]
+> All `destination_template` must point to the Google Cloud Storage (GCS) bucket objects.
+> All `source_template` must point to the FTP server paths.
+> In case this is not enforced, the user may experience silent failures.
+
 ```yaml
 ---
 work_path: ./work
@@ -91,7 +98,7 @@ steps:
         - fetch studies
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
-      destination_template: ./work/curation_{release_date}.tsv
+      destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
       promote: true
 ```
 
@@ -114,7 +121,7 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
@@ -135,7 +142,7 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
@@ -203,7 +210,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
 
 This task is used to build the GWAS Catalog curation file that is later used as a template for manual curation. It requires the `fetch studies` task to be completed before it can run. This is due to the fact that the curation file is build based on the list of studies fetched from `download studies` file.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `requires` field specifies that this task depends on the `fetch studies` task, meaning it will only run after the studies have been fetched.
@@ -268,6 +275,7 @@ To check CLI execution manually you need to run
 ```{bash}
 uv run gentroutils
 ```
+
 ---
 
 This software was developed as part of the Open Targets project. For more

{gentroutils-2.0.0.dist-info → gentroutils-3.1.0.dist-info}/RECORD

@@ -6,17 +6,17 @@ gentroutils/io/path/__init__.py,sha256=oBNiw3xr84QAFejVgRIW_PttJ2ZQHZV2uYtIklXle
 gentroutils/io/path/ftp.py,sha256=kus0GS9Bcm3IzooL8duu7fdtMDr0-Sm7LCtpiUd1Kxs,1585
 gentroutils/io/path/gcs.py,sha256=itEEyUBQLqCBV-F7JF7ZJHUn5OBObmTCsrEfib_V7p8,1514
 gentroutils/io/transfer/__init__.py,sha256=2KasKyCMTpnUvU2SVzZds7ggYBS5ReXn8q3K_TE0YYA,310
-gentroutils/io/transfer/ftp_to_gcs.py,sha256=SGIpxTaQoiIeW2p9JHIMckNWhFpz1Ne4YqwoiYqdWdo,2107
+gentroutils/io/transfer/ftp_to_gcs.py,sha256=O4wDNVyAt24_1zIkfGFdNQ-er1CtJ7J5rdj0bxrG7SI,2793
 gentroutils/io/transfer/model.py,sha256=-REhDC8nIT_RoOlRwHpvNfqN_khOm7OnlAi8-U7qX1k,1198
 gentroutils/io/transfer/polars_to_gcs.py,sha256=l1vWDLVw5OuAjPRWEvA69w5V5cz36ARGqCOA9R9iecM,762
 gentroutils/parsers/__init__.py,sha256=HtgvopQ3Xx_cjC2lA3Tp81Rd5-k4CUJGJu0GD7W9r0o,59
 gentroutils/parsers/curation.py,sha256=34GvUTD9SmUpk_CEMoqJbpDyBVW3kKOyMLMiZ4nc9e0,7357
-gentroutils/tasks/__init__.py,sha256=dN9Od2_I504AZjLwBixj04M0h6dmmxGRV7FMwHHEdzM,3537
-gentroutils/tasks/crawl.py,sha256=njCveYWvJz6CHWQfjbciGp57yZcsKuFiZqVWR57_XeU,6838
-gentroutils/tasks/curation.py,sha256=cJZQmaD-44rej-8K6dUW6IlzArZt-YcxtVVTZdKh-Fk,4539
-gentroutils/tasks/fetch.py,sha256=jwqIQ49P--63X1EyKXIUf2iBtv3QRsuNgQWEBwlvgCk,6444
-gentroutils-2.0.0.dist-info/METADATA,sha256=HSEEAgtTAbstKEhM8tPkQIq3Yg_vxeu9w14NU1wwX9Q,14936
-gentroutils-2.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-gentroutils-2.0.0.dist-info/entry_points.txt,sha256=KjODdAGrWKebI3ogqs9r6snAJ_DtLGpm1jREbZ6OXGs,49
-gentroutils-2.0.0.dist-info/licenses/LICENSE,sha256=8rMKYP7K5vL-KA2WmdBkUBz4iFRveUOWUMAFP8uc3P0,10945
-gentroutils-2.0.0.dist-info/RECORD,,
+gentroutils/tasks/__init__.py,sha256=WvMdT_m0F1Kos-FWaVlXLTCuRs-xQD1cRHtHEIZ_U4c,3745
+gentroutils/tasks/crawl.py,sha256=kHLT5EBFUT6yubNKPshHKNrlDMuQ7RM9c22g44VN734,6766
+gentroutils/tasks/curation.py,sha256=qBI0PF6Bby-J-113VM2xgW4a7dq8L99rRq8YMo6dDww,4675
+gentroutils/tasks/fetch.py,sha256=cDP6FCzSCIRq_MZGXZEVQtcLOG6TsOzIg6z_s2U6dkM,6289
+gentroutils-3.1.0.dist-info/METADATA,sha256=8ERSzEvSQV7HsySDmB6wLc6Qm7IUvVmHnV-OnzwXYjg,15272
+gentroutils-3.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gentroutils-3.1.0.dist-info/entry_points.txt,sha256=KjODdAGrWKebI3ogqs9r6snAJ_DtLGpm1jREbZ6OXGs,49
+gentroutils-3.1.0.dist-info/licenses/LICENSE,sha256=8rMKYP7K5vL-KA2WmdBkUBz4iFRveUOWUMAFP8uc3P0,10945
+gentroutils-3.1.0.dist-info/RECORD,,