gentroutils 2.0.0__tar.gz → 3.0.0__tar.gz

gentroutils-3.0.0/.github/workflows/build.yaml

@@ -0,0 +1,87 @@
+
+name: build-artifacts
+
+on:
+  workflow_dispatch:
+
+env:
+  GCP_PROJECT_ID: "open-targets-genetics-dev"
+  GCP_REGION: "europe-west1"
+  TAG: "${{ github.ref_name }}"
+  REPO: "${{ github.event.repository_name }}"
+
+
+jobs:
+  push-to-ghcr-and-gar:
+    name: Build docker image and push to GHCR and GAR
+    runs-on: ubuntu-22.04
+
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+
+    steps:
+      - id: prepare
+        name: Prepare the action and log details
+        shell: bash
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: |
+          TAG=$(echo $TAG | sed 's/^v//')
+          echo "TAG=$TAG" >> $GITHUB_ENV
+          echo "The tag for this build is $TAG"
+          echo "The repo name is: $REPO"
+          echo "Github context:\n$GITHUB_CONTEXT"
+
+      - id: checkout
+        name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - id: auth-ghcr
+        name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: auth-google
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: ${{ env.GCP_PROJECT_ID }}
+          workload_identity_provider: projects/234703259993/locations/global/workloadIdentityPools/github/providers/opentargets
+          access_token_lifetime: 300s
+
+      - id: auth-gar
+        name: Login to Google Artifact Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GCP_REGION }}-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth-google.outputs.access_token }}
+
+      - id: push
+        name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest
+            ghcr.io/${{ github.repository }}:${{ env.TAG }}
+            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPO }}/${{ env.REPO }}:latest
+            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPO }}:${{ env.TAG }}
+
+      - id: generate-attestations
+        name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPO }}/${{ env.REPO }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

{gentroutils-2.0.0 → gentroutils-3.0.0}/CHANGELOG.md

@@ -1,6 +1,35 @@
 # CHANGELOG
 
 
+## v3.0.0 (2025-08-28)
+
+### Bug Fixes
+
+- Ensure otter scratchpad works
+  ([`b781b1f`](https://github.com/opentargets/gentroutils/commit/b781b1f587f94807daa3abde0dbb785121904e68))
+
+### Build System
+
+- Update dependencies
+  ([`e9e05c3`](https://github.com/opentargets/gentroutils/commit/e9e05c31ec6ea443d1cb8af28f3583c6486ecb52))
+
+### Chores
+
+- Format
+  ([`876400b`](https://github.com/opentargets/gentroutils/commit/876400b3d8b5a09c198a8b13db2ffc598a1f218a))
+
+- Remove R session files
+  ([`ffe093d`](https://github.com/opentargets/gentroutils/commit/ffe093d494ec7c469b8b9f99978bfa002426c189))
+
+- Update readme
+  ([`06a21db`](https://github.com/opentargets/gentroutils/commit/06a21dbdb0b706e06537f80d378f8837041b3906))
+
+### Continuous Integration
+
+- Add build command
+  ([`2e67fcd`](https://github.com/opentargets/gentroutils/commit/2e67fcd1d7478b96dc698d5e54e2a077529fcb61))
+
+
 ## v2.0.0 (2025-08-28)
 
 ### Chores
@@ -16,6 +45,16 @@
 
 Co-authored-by: Szymon Szyszkowski <69353402+project-defiant@users.noreply.github.com>
 
+### Continuous Integration
+
+- Update ci
+  ([`81c39b2`](https://github.com/opentargets/gentroutils/commit/81c39b2d02f8fe318d8921b46c636a3361093263))
+
+### Features
+
+- 2.0.0
+  ([`3268c38`](https://github.com/opentargets/gentroutils/commit/3268c383cf6dfac4be7748ad6b48e8ded5f6157c))
+
 
 ## v1.6.0-dev.2 (2025-08-12)
 

{gentroutils-2.0.0 → gentroutils-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gentroutils
-Version: 2.0.0
+Version: 3.0.0
 Summary: Open Targets python genetics utility CLI tools
 Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
 License-Expression: Apache-2.0
@@ -15,10 +15,11 @@ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Requires-Python: >=3.13
 Requires-Dist: aioftp>=0.25.1
 Requires-Dist: aiohttp>=3.11.18
+Requires-Dist: gcsfs>=2025.7.0
 Requires-Dist: google-cloud-storage>=3.1.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: opentargets-otter>=25.0.2
-Requires-Dist: polars>=1.31.0
+Requires-Dist: polars[fsspec,gcs]>=1.31.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: tqdm>=4.67.1
 Description-Content-Type: text/markdown
@@ -48,6 +49,7 @@ gentroutils --help
 ## Usage
 
 To run a single step run
+
 ```{bash}
 uv run gentroutils -s gwas_catalog_release  # After cloning the repository
 gentroutils -s gwas_catalog_release -c otter_config.yaml # When installed by pip
@@ -60,6 +62,11 @@ The `gentroutils` repository uses the [otter](https://github.com/opentargets/ott
 
 For the top level fields refer to the [otter documentation](https://opentargets.github.io/otter/otter.config.html)
 
+> [!NOTE]
+> All `destination_template` must point to the Google Cloud Storage (GCS) bucket objects.
+> All `source_template` must point to the FTP server paths.
+> In case this is not enforced, the user may experience silent failures.
+
 ```yaml
 ---
 work_path: ./work
@@ -91,7 +98,7 @@ steps:
         - fetch studies
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
-      destination_template: ./work/curation_{release_date}.tsv
+      destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
       promote: true
 ```
 
@@ -114,8 +121,7 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `destination_template` is where the metadata will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. By default it searches for the release directly in the stats_uri json output.
@@ -135,8 +141,7 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog associations file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -157,8 +162,7 @@ This task fetches the GWAS Catalog associations file from the specified FTP serv
 
 This task fetches the GWAS Catalog studies file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog studies file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -179,8 +183,7 @@ This task fetches the GWAS Catalog studies file from the specified FTP server an
 
 This task fetches the GWAS Catalog ancestries file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog ancestries file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -203,7 +206,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
 
 This task is used to build the GWAS Catalog curation file that is later used as a template for manual curation. It requires the `fetch studies` task to be completed before it can run. This is due to the fact that the curation file is build based on the list of studies fetched from `download studies` file.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `requires` field specifies that this task depends on the `fetch studies` task, meaning it will only run after the studies have been fetched.
@@ -268,6 +271,7 @@ To check CLI execution manually you need to run
 ```{bash}
 uv run gentroutils
 ```
+
 ---
 
 This software was developed as part of the Open Targets project. For more

{gentroutils-2.0.0 → gentroutils-3.0.0}/README.md

@@ -23,6 +23,7 @@ gentroutils --help
 ## Usage
 
 To run a single step run
+
 ```{bash}
 uv run gentroutils -s gwas_catalog_release  # After cloning the repository
 gentroutils -s gwas_catalog_release -c otter_config.yaml # When installed by pip
@@ -35,6 +36,11 @@ The `gentroutils` repository uses the [otter](https://github.com/opentargets/ott
 
 For the top level fields refer to the [otter documentation](https://opentargets.github.io/otter/otter.config.html)
 
+> [!NOTE]
+> All `destination_template` must point to the Google Cloud Storage (GCS) bucket objects.
+> All `source_template` must point to the FTP server paths.
+> In case this is not enforced, the user may experience silent failures.
+
 ```yaml
 ---
 work_path: ./work
@@ -66,7 +72,7 @@ steps:
         - fetch studies
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
-      destination_template: ./work/curation_{release_date}.tsv
+      destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
       promote: true
 ```
 
@@ -89,8 +95,7 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `destination_template` is where the metadata will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. By default it searches for the release directly in the stats_uri json output.
@@ -110,8 +115,7 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog associations file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -132,8 +136,7 @@ This task fetches the GWAS Catalog associations file from the specified FTP serv
 
 This task fetches the GWAS Catalog studies file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog studies file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -154,8 +157,7 @@ This task fetches the GWAS Catalog studies file from the specified FTP server an
 
 This task fetches the GWAS Catalog ancestries file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
-> **Task parameters**
+> [!NOTE] > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog ancestries file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -178,7 +180,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
 
 This task is used to build the GWAS Catalog curation file that is later used as a template for manual curation. It requires the `fetch studies` task to be completed before it can run. This is due to the fact that the curation file is build based on the list of studies fetched from `download studies` file.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `requires` field specifies that this task depends on the `fetch studies` task, meaning it will only run after the studies have been fetched.
@@ -243,6 +245,7 @@ To check CLI execution manually you need to run
 ```{bash}
 uv run gentroutils
 ```
+
 ---
 
 This software was developed as part of the Open Targets project. For more

gentroutils-3.0.0/config.yaml

@@ -0,0 +1,40 @@
+---
+work_path: ./work
+log_level: DEBUG
+scratchpad:
+  gc_stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+  gc_bucket: "gs://gwas_catalog_inputs"
+  gc_ftp: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases"
+
+steps:
+  gwas_catalog_release:
+    # - name: crawl release metadata
+    #   stats_uri: ${gc_stats_uri}
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/stats.json'
+    #   promote: true
+
+    # - name: fetch associations
+    #   stats_uri: ${gc_stats_uri}
+    #   source_template: '${gc_ftp}/{release_date}/gwas-catalog-associations_ontology-annotated.tsv'
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv'
+    #   promote: true
+
+    # - name: fetch studies
+    #   stats_uri: ${gc_stats_uri}
+    #   source_template: '${gc_ftp}/{release_date}/gwas-catalog-download-studies-v1.0.3.1.txt'
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_download_studies.tsv'
+    #   promote: true
+
+    # - name: fetch ancestries
+    #   stats_uri: ${gc_stats_uri}
+    #   source_template: '${gc_ftp}/{release_date}/gwas-catalog-download-ancestries-v1.0.3.1.txt'
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_download_ancestries.tsv'
+    #   promote: true
+
+    - name: curation study
+      # requires:
+      #   - fetch studies
+      previous_curation: '${gc_bucket}/curation/latest/curated/GWAS_Catalog_study_curation.tsv'
+      studies: '${gc_bucket}/gentroutils/latest/gwas_catalog_download_studies.tsv'
+      destination_template: '${gc_bucket}/curation/{release_date}/raw/GWAS_Catalog_study_curation.tsv'
+      promote: true

{gentroutils-2.0.0 → gentroutils-3.0.0}/pyproject.toml

@@ -1,17 +1,18 @@
 [project]
 authors = [{ name = "Szymon Szyszkowski", email = "ss60@sanger.ac.uk" }]
 name = "gentroutils"
-version = "2.0.0"
+version = "3.0.0"
 description = "Open Targets python genetics utility CLI tools"
 dependencies = [
     "aiohttp>=3.11.18",
     "aioftp>=0.25.1",
-    "polars>=1.31.0",
+    "polars[fsspec,gcs]>=1.31.0",
     "pydantic>=2.10.6",
     "loguru>=0.7.3",
     "tqdm>=4.67.1",
     "opentargets-otter>=25.0.2",
     "google-cloud-storage>=3.1.1",
+    "gcsfs>=2025.7.0",
 ]
 readme = "README.md"
 requires-python = ">=3.13"
@@ -75,6 +76,12 @@ allow-direct-references = true
 [tool.hatch.build.targets.wheel]
 packages = ["src/gentroutils"]
 
+
+# Ignore polars x GCS depencency not imported in code
+[tool.deptry.per_rule_ignores]
+DEP002 = ["gcsfs"]
+
+
 # test configuration
 [tool.pytest.ini_options]
 markers = ["integration_test: Intergration tests", "unit_test: Unit tests"]

{gentroutils-2.0.0 → gentroutils-3.0.0}/src/gentroutils/io/transfer/ftp_to_gcs.py

@@ -32,12 +32,24 @@ class FTPtoGCPTransferableObject(TransferableObject):
         async with aioftp.Client.context(ftp_obj.server, user="anonymous", password="anonymous") as ftp:  # noqa: S106
             bucket = storage.Client().bucket(gcs_obj.bucket)
             blob = bucket.blob(gcs_obj.object)
-            logger.info(f"Changing directory to {ftp_obj.base_dir}.")
-            await ftp.change_directory(ftp_obj.base_dir)
-            pwd = await ftp.get_current_directory()
-            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(pwd))
+            logger.info(f"Searching for the release date in the provided ftp path: {ftp_obj.base_dir}.")
+            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(ftp_obj.base_dir))
+
             if dir_match:
-                logger.info(f"Found release date!: {dir_match.group('release_date')}")
+                logger.info(f"Found release date to search in the ftp {dir_match.group('release_date')}.")
+                release_date = dir_match.group("release_date")
+                try:
+                    await ftp.change_directory(ftp_obj.base_dir)
+                except aioftp.StatusCodeError as e:
+                    logger.error(f"Failed to change directory to {ftp_obj.base_dir}: {e}")
+                    logger.warning("Attempting to load the `latest` release.")
+                    ftp_obj = FTPPath(self.source.replace(release_date, "latest"))
+                    try:
+                        await ftp.change_directory(ftp_obj.base_dir)
+                    except aioftp.StatusCodeError as e:
+                        logger.error(f"Failed to find the latest release under {ftp_obj}")
+                        raise
+
             buffer = io.BytesIO()
             stream = await ftp.download_stream(ftp_obj.filename)
             async with stream:

{gentroutils-2.0.0 → gentroutils-3.0.0}/src/gentroutils/tasks/__init__.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections import defaultdict
 from dataclasses import dataclass
 from datetime import date
 
@@ -13,7 +14,12 @@ from pydantic import AliasPath, BaseModel, Field
 from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
 
 
-def _requires_release_date_template(path: str) -> str:
+class KeepMissing(defaultdict[str, str]):
+    def __missing__(self, key):
+        return "{" + key + "}"
+
+
+def destination_validator(path: str) -> str:
     """Ensure that the destination path contains a template for the release date."""
     if "{release_date}" not in path:
         raise GentroutilsError(GentroutilsErrorMessage.MISSING_RELEASE_DATE_TEMPLATE, release_date="{release_date}")
@@ -34,7 +40,7 @@ class TemplateDestination:
 
         This method returns a new TemplateDestination object (not a copy of the current one) with the formatted destination.
         """
-        return TemplateDestination(self.destination.format(**substitutions), True)
+        return TemplateDestination(self.destination.format_map(KeepMissing(**substitutions)), True)
 
 
 class GwasCatalogReleaseInfo(BaseModel):
@@ -83,6 +89,7 @@ class GwasCatalogReleaseInfo(BaseModel):
     @classmethod
     def from_uri(cls, uri: str) -> GwasCatalogReleaseInfo:
         """Fetch the release information from the specified URI."""
+        logger.debug(f"Fetching release info from {uri}")
         try:
             return asyncio.run(cls._get_release_info(uri))
         except aiohttp.ClientError as e:

{gentroutils-2.0.0 → gentroutils-3.0.0}/src/gentroutils/tasks/crawl.py

@@ -1,7 +1,6 @@
 """Module to handle the crawling of GWAS Catalog release information."""
 
 import tempfile
-from functools import cached_property
 from pathlib import Path
 from typing import Annotated, Any, Self
 
@@ -9,9 +8,10 @@ from loguru import logger
 from otter.storage import get_remote_storage
 from otter.task.model import Spec, Task, TaskContext
 from otter.task.task_reporter import report
-from pydantic import AfterValidator, computed_field
+from pydantic import AfterValidator
 
-from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, _requires_release_date_template
+from gentroutils.io.path import GCSPath
+from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, destination_validator
 
 
 class CrawlSpec(Spec):
@@ -68,7 +68,7 @@ class CrawlSpec(Spec):
     stats_uri: str = "https://www.ebi.ac.uk/gwas/api/search/stats"
     """The URI to crawl the release statistics information from."""
 
-    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    destination_template: Annotated[str, AfterValidator(destination_validator)]
     """The destination path to save the release information.
        This path should always be a template string that includes `{release_date}`.
        For example, `gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json`.
@@ -91,8 +91,6 @@ class CrawlSpec(Spec):
         promoting the release as the latest release.
     """
 
-    @computed_field  # type: ignore[prop-decorator]
-    @cached_property
     def destinations(self) -> list[TemplateDestination]:
         """Get the list of destinations templates where the release information will be saved.
 
@@ -105,17 +103,17 @@ class CrawlSpec(Spec):
                 1. The destination template with the release date substituted.
                 2. The destination with the release date substituted to `latest`.
         """
-        d1 = self.destination_template
+        d1 = TemplateDestination(self.destination_template, False)
         if self.promote:
-            d2 = self.destination_template.format(release_date="latest")
-            return [TemplateDestination(d1, False), TemplateDestination(d2, True)]
-        return [TemplateDestination(d1, False)]
+            d2 = d1.format({"release_date": "latest"})
+            return [d1, d2]
+        return [d1]
 
     def substituted_destinations(self, release_info: GwasCatalogReleaseInfo) -> list[str]:
         """Safely parse the destination name to ensure it is valid."""
         substitutions = {"release_date": release_info.strfmt("%Y%m%d")}
         return [
-            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations()
         ]
 
     def model_post_init(self, __context: Any) -> None:
@@ -141,9 +139,10 @@ class Crawl(Task):
                 logger.info(f"Destinations for release information: {destinations}")
                 for destination in destinations:
                     storage = get_remote_storage(destination)
+                    assert "gs://" in destination, f"Invalid GCS path in destination template: {destination}"
                     storage.upload(Path(source.name), destination)
                     logger.info(f"Release information written to {destination}")
-            return self
+        return self
 
     @report
     def run(self) -> Self:

{gentroutils-2.0.0 → gentroutils-3.0.0}/src/gentroutils/tasks/curation.py

@@ -3,16 +3,16 @@
 from __future__ import annotations
 
 from datetime import date
-from functools import cached_property
 from typing import Annotated, Any, Self
 
+from loguru import logger
 from otter.task.model import Spec, Task, TaskContext
 from otter.task.task_reporter import report
-from pydantic import AfterValidator, computed_field
+from pydantic import AfterValidator
 
 from gentroutils.io.transfer.polars_to_gcs import PolarsDataFrameToGCSTransferableObject
 from gentroutils.parsers.curation import GWASCatalogCuration
-from gentroutils.tasks import TemplateDestination, _requires_release_date_template
+from gentroutils.tasks import TemplateDestination, destination_validator
 from gentroutils.transfer import TransferManager
 
 
@@ -50,14 +50,12 @@ class CurationSpec(Spec):
     studies: str
     """The path to the studies data."""
 
-    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    destination_template: Annotated[str, AfterValidator(destination_validator)]
     """The destination path for the curation data."""
 
     promote: bool = False
     """Whether to promote the curation data to the latest version."""
 
-    @computed_field  # type: ignore[prop-decorator]
-    @cached_property
     def destinations(self) -> list[TemplateDestination]:
         """Get the list of destinations templates where the release information will be saved.
 
@@ -80,7 +78,7 @@ class CurationSpec(Spec):
         """Safely parse the destination name to ensure it is valid."""
         substitutions = {"release_date": release_date.strftime("%Y%m%d")}
         return [
-            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations()
         ]
 
     def model_post_init(self, __context: Any) -> None:
@@ -99,9 +97,13 @@ class Curation(Task):
     @report
     def run(self) -> Self:
         """Run the curation task."""
+        logger.info("Starting curation task.")
         release_date = date.today()
+        logger.debug(f"Using release date: {release_date}")
         destinations = self.spec.substituted_destinations(release_date)
+        logger.debug(f"Destinations for curation data: {destinations}")
         curation = GWASCatalogCuration.from_prev_curation(self.spec.previous_curation, self.spec.studies)
+        logger.debug(f"Curation result preview:\n{curation.result.head()}")
         transfer_objects = [
             PolarsDataFrameToGCSTransferableObject(source=curation.result, destination=d) for d in destinations
         ]

{gentroutils-2.0.0 → gentroutils-3.0.0}/src/gentroutils/tasks/fetch.py

@@ -1,15 +1,14 @@
 """Module to handle the fetching of GWAS Catalog release files."""
 
-from functools import cached_property
 from typing import Annotated, Any, Self
 
 from loguru import logger
 from otter.task.model import Spec, Task, TaskContext
 from otter.task.task_reporter import report
-from pydantic import AfterValidator, computed_field
+from pydantic import AfterValidator
 
 from gentroutils.io.transfer import FTPtoGCPTransferableObject
-from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, _requires_release_date_template
+from gentroutils.tasks import GwasCatalogReleaseInfo, TemplateDestination, destination_validator
 from gentroutils.transfer import TransferManager
 
 MAX_CONCURRENT_CONNECTIONS = 10
@@ -57,10 +56,10 @@ class FetchSpec(Spec):
     stats_uri: str = "https://www.ebi.ac.uk/gwas/api/search/stats"
     """The URI to crawl the release statistics information from."""
 
-    source_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    source_template: Annotated[str, AfterValidator(destination_validator)]
     """The template URI of the file to download."""
 
-    destination_template: Annotated[str, AfterValidator(_requires_release_date_template)]
+    destination_template: Annotated[str, AfterValidator(destination_validator)]
     """The template URI to upload the file to."""
 
     promote: bool = False
@@ -78,8 +77,6 @@ class FetchSpec(Spec):
         promoting the release as the latest release.
     """
 
-    @computed_field  # type: ignore[prop-decorator]
-    @cached_property
     def destinations(self) -> list[TemplateDestination]:
         """Get the list of destinations templates where the release information will be saved.
 
@@ -102,7 +99,7 @@ class FetchSpec(Spec):
         """Safely parse the destination name to ensure it is valid."""
         substitutions = {"release_date": release_info.strfmt("%Y%m%d")}
         return [
-            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations
+            d.format(substitutions).destination if not d.is_substituted else d.destination for d in self.destinations()
         ]
 
     def substituted_sources(self, release_info: GwasCatalogReleaseInfo) -> list[str]:

{gentroutils-2.0.0 → gentroutils-3.0.0}/tests/io/transfer/test_ftp_to_gcs.py

@@ -70,14 +70,13 @@ class TestFTPtoGCPTransferableObject:
 
         # Create and execute transfer
         obj = FTPtoGCPTransferableObject(
-            source="ftp://example.com/data/file.txt", destination="gs://test-bucket/file.txt"
+            source="ftp://example.com/2025/12/12/file.txt", destination="gs://test-bucket/file.txt"
         )
         await obj.transfer()
 
         # Verify FTP operations
         mock_ftp_context.assert_called_once_with("example.com", user="anonymous", password="anonymous")  # noqa: S106
-        mock_ftp_client.change_directory.assert_called_once_with("/data")
-        mock_ftp_client.get_current_directory.assert_called_once()
+        mock_ftp_client.change_directory.assert_called()
         mock_ftp_client.download_stream.assert_called_once_with("file.txt")
 
         # Verify GCS operations

{gentroutils-2.0.0 → gentroutils-3.0.0}/tests/tasks/test_crawl_task.py

@@ -88,7 +88,7 @@ class TestCrawlSpec:
 
     def test_crawl_spec_destinations_with_promote(self, crawl_spec):
         """Test destinations property when promote=True."""
-        destinations = crawl_spec.destinations
+        destinations = crawl_spec.destinations()
         assert len(destinations) == 2
         assert destinations[0].destination == "gs://test-bucket/gwas/{release_date}/stats.json"
         assert destinations[0].is_substituted is False
@@ -97,7 +97,7 @@ class TestCrawlSpec:
 
     def test_crawl_spec_destinations_without_promote(self, crawl_spec_no_promote):
         """Test destinations property when promote=False."""
-        destinations = crawl_spec_no_promote.destinations
+        destinations = crawl_spec_no_promote.destinations()
         assert len(destinations) == 1
         assert destinations[0].destination == "gs://test-bucket/gwas/{release_date}/stats.json"
         assert destinations[0].is_substituted is False

{gentroutils-2.0.0 → gentroutils-3.0.0}/tests/tasks/test_curation_task.py

@@ -28,10 +28,11 @@ class TestCurationSpec:
         assert curation_spec.studies == "gs://test-bucket/studies.json"
         assert curation_spec.destination_template == "gs://test-bucket/{release_date}/curation.json"
         assert curation_spec.promote is True
-        assert curation_spec.destinations[0].destination == "gs://test-bucket/{release_date}/curation.json"
-        assert curation_spec.destinations[0].is_substituted is False
-        assert curation_spec.destinations[1].destination == "gs://test-bucket/latest/curation.json"
-        assert curation_spec.destinations[1].is_substituted is True
+        destinations = curation_spec.destinations()
+        assert destinations[0].destination == "gs://test-bucket/{release_date}/curation.json"
+        assert destinations[0].is_substituted is False
+        assert destinations[1].destination == "gs://test-bucket/latest/curation.json"
+        assert destinations[1].is_substituted is True
 
     def test_curation_spec_requires_release_date_template(self):
         """Test that CurationSpec validates release date template."""

{gentroutils-2.0.0 → gentroutils-3.0.0}/tests/tasks/test_fetch_task.py

@@ -46,11 +46,12 @@ class TestFetchSpec:
         assert fetch_spec.source_template == "https://example.com/{release_date}/data.json"
         assert fetch_spec.destination_template == "gs://test-bucket/{release_date}/data.json"
         assert fetch_spec.promote is True
-        assert len(fetch_spec.destinations) == 2
-        assert fetch_spec.destinations[0].destination == "gs://test-bucket/{release_date}/data.json"
-        assert fetch_spec.destinations[0].is_substituted is False
-        assert fetch_spec.destinations[1].destination == "gs://test-bucket/latest/data.json"
-        assert fetch_spec.destinations[1].is_substituted is True
+        destinations = fetch_spec.destinations()
+        assert len(destinations) == 2
+        assert destinations[0].destination == "gs://test-bucket/{release_date}/data.json"
+        assert destinations[0].is_substituted is False
+        assert destinations[1].destination == "gs://test-bucket/latest/data.json"
+        assert destinations[1].is_substituted is True
 
     def test_initialization_no_promote(self):
         """Test FetchSpec initialization with promote = False."""
@@ -65,9 +66,11 @@ class TestFetchSpec:
         assert fetch_spec.source_template == "https://example.com/{release_date}/data.json"
         assert fetch_spec.destination_template == "gs://test-bucket/{release_date}/data.json"
         assert fetch_spec.promote is False
-        assert len(fetch_spec.destinations) == 1
-        assert fetch_spec.destinations[0].destination == "gs://test-bucket/{release_date}/data.json"
-        assert fetch_spec.destinations[0].is_substituted is False
+
+        destinations = fetch_spec.destinations()
+        assert len(destinations) == 1
+        assert destinations[0].destination == "gs://test-bucket/{release_date}/data.json"
+        assert destinations[0].is_substituted is False
 
     def test_requires_release_date_template(self):
         """Test that FetchSpec validates release date template."""

{gentroutils-2.0.0 → gentroutils-3.0.0}/uv.lock

@@ -457,6 +457,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b9/5c/a3d95dc1ec6cdeb032d789b552ecc76effa3557ea9186e1566df6aac18df/fs-2.4.16-py2.py3-none-any.whl", hash = "sha256:660064febbccda264ae0b6bace80a8d1be9e089e0a5eb2427b7d517f9a91545c", size = 135261, upload-time = "2022-05-02T09:25:52.363Z" },
 ]
 
+[[package]]
+name = "fsspec"
+version = "2025.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8b/02/0835e6ab9cfc03916fe3f78c0956cfcdb6ff2669ffa6651065d5ebf7fc98/fsspec-2025.7.0.tar.gz", hash = "sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58", size = 304432, upload-time = "2025-07-15T16:05:21.19Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597, upload-time = "2025-07-15T16:05:19.529Z" },
+]
+
 [[package]]
 name = "gcloud-storage-emulator"
 version = "0.5.0"
@@ -471,17 +480,36 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f1/46/b70e4ee5df71b4253c146c3792703c1e63c66c1e63d9bd64dbc07a1c456e/gcloud_storage_emulator-0.5.0-py3-none-any.whl", hash = "sha256:69fe95ab57fc45acdeba7b8dfc44c12b9e7921e640776f7ec79d675d753b03cf", size = 19959, upload-time = "2021-09-29T09:03:26.099Z" },
 ]
 
+[[package]]
+name = "gcsfs"
+version = "2025.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiohttp" },
+    { name = "decorator" },
+    { name = "fsspec" },
+    { name = "google-auth" },
+    { name = "google-auth-oauthlib" },
+    { name = "google-cloud-storage" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5b/d7/5eafe9f09f1bb09433a473cef7984cd52c398592c8fd09974e0ad87cfea4/gcsfs-2025.7.0.tar.gz", hash = "sha256:ad3ff66cf189ae8fc375ac8a2af409003dbca02357621cb94a66e457e02ba420", size = 82659, upload-time = "2025-07-15T16:49:21.647Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/21/f5/54bccbee01efbc25581db6aafefb6f6c277d880930f7a083b10052382463/gcsfs-2025.7.0-py2.py3-none-any.whl", hash = "sha256:653503331d58cb02bb34e725d4595d166e93f7f2f3ff88e4c66ef535ae66eae5", size = 36815, upload-time = "2025-07-15T16:49:20.333Z" },
+]
+
 [[package]]
 name = "gentroutils"
-version = "2.0.0"
+version = "3.0.0"
 source = { editable = "." }
 dependencies = [
     { name = "aioftp" },
     { name = "aiohttp" },
+    { name = "gcsfs" },
     { name = "google-cloud-storage" },
     { name = "loguru" },
     { name = "opentargets-otter" },
-    { name = "polars" },
+    { name = "polars", extra = ["fsspec"] },
     { name = "pydantic" },
     { name = "tqdm" },
 ]
@@ -526,10 +554,11 @@ dev = [
 requires-dist = [
     { name = "aioftp", specifier = ">=0.25.1" },
     { name = "aiohttp", specifier = ">=3.11.18" },
+    { name = "gcsfs", specifier = ">=2025.7.0" },
     { name = "google-cloud-storage", specifier = ">=3.1.1" },
     { name = "loguru", specifier = ">=0.7.3" },
     { name = "opentargets-otter", specifier = ">=25.0.2" },
-    { name = "polars", specifier = ">=1.31.0" },
+    { name = "polars", extras = ["fsspec", "gcs"], specifier = ">=1.31.0" },
     { name = "pydantic", specifier = ">=2.10.6" },
     { name = "tqdm", specifier = ">=4.67.1" },
 ]
@@ -624,6 +653,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" },
 ]
 
+[[package]]
+name = "google-auth-oauthlib"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "google-auth" },
+    { name = "requests-oauthlib" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fb/87/e10bf24f7bcffc1421b84d6f9c3377c30ec305d082cd737ddaa6d8f77f7c/google_auth_oauthlib-1.2.2.tar.gz", hash = "sha256:11046fb8d3348b296302dd939ace8af0a724042e8029c1b872d87fabc9f41684", size = 20955, upload-time = "2025-04-22T16:40:29.172Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ac/84/40ee070be95771acd2f4418981edb834979424565c3eec3cd88b6aa09d24/google_auth_oauthlib-1.2.2-py3-none-any.whl", hash = "sha256:fd619506f4b3908b5df17b65f39ca8d66ea56986e5472eb5978fd8f3786f00a2", size = 19072, upload-time = "2025-04-22T16:40:28.174Z" },
+]
+
 [[package]]
 name = "google-cloud-core"
 version = "2.4.3"
@@ -1038,6 +1080,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d2/47/ca494f6f2366f17bd2a79b6c1468f6c441a447b017fbdf600aa40b5d27ac/numpy_typing_compat-2.3.20250730-py3-none-any.whl", hash = "sha256:9ab0cd4bb1b4c31debf0bd745554c735a07a7bb3cc3801dc934b2b5856549612", size = 6057, upload-time = "2025-07-30T01:36:06.027Z" },
 ]
 
+[[package]]
+name = "oauthlib"
+version = "3.3.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" },
+]
+
 [[package]]
 name = "opentargets-otter"
 version = "25.0.4"
@@ -1155,6 +1206,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b8/53/4eaaa4f219add46594db21a05a9a5629ec6af20bd859a90668d5a1448abc/polars-1.32.2-cp39-abi3-win_arm64.whl", hash = "sha256:cd390364f6f3927474bd0aed255103195b9d2b3eef0f0c5bb429db5e6311615e", size = 34059100, upload-time = "2025-08-07T10:50:26.445Z" },
 ]
 
+[package.optional-dependencies]
+fsspec = [
+    { name = "fsspec" },
+]
+
 [[package]]
 name = "pre-commit"
 version = "4.3.0"
@@ -1541,6 +1597,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" },
 ]
 
+[[package]]
+name = "requests-oauthlib"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "oauthlib" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" },
+]
+
 [[package]]
 name = "requests-toolbelt"
 version = "1.0.0"

gentroutils-2.0.0/config.yaml

@@ -1,32 +0,0 @@
----
-work_path: ./work
-log_level: DEBUG
-scratchpad:
-steps:
-  gwas_catalog_release:
-    - name: crawl release metadata
-      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
-      destination_template: 'gs://gwas_catalog_inputs/gentroutils/{release_date}/stats.json'
-      promote: 'true'
-    - name: fetch associations
-      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
-      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-associations_ontology-annotated.tsv"
-      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv"
-      promote: true
-    - name: fetch studies
-      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
-      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-download-studies-v1.0.3.1.txt"
-      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_download_studies.tsv"
-      promote: true
-    - name: fetch ancestries
-      stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
-      source_template: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/{release_date}/gwas-catalog-download-ancestries-v1.0.3.1.txt"
-      destination_template: "gs://gwas_catalog_inputs/gentroutils/{release_date}/gwas_catalog_download_ancestries.tsv"
-      promote: true
-    - name: curation study
-      requires:
-        - fetch studies
-      previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
-      studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
-      destination_template: ./work/curation_{release_date}.tsv
-      promote: true