gentroutils 2.0.0__tar.gz → 3.1.0__tar.gz

gentroutils-3.1.0/.github/workflows/build.yaml

@@ -0,0 +1,90 @@
+
+name: build-artifacts
+
+on:
+  workflow_dispatch:
+
+env:
+  GCP_PROJECT_ID: "open-targets-genetics-dev"
+  GCP_REGION: "europe-west1"
+  TAG: "${{ github.ref_name }}"
+  REPO: "${{ github.event.repository.name }}"
+
+
+jobs:
+  push-to-ghcr-and-gar:
+    if: startsWith(github.ref, 'refs/tags/')
+    name: Build docker image and push to GHCR and GAR
+    runs-on: ubuntu-22.04
+
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+
+    steps:
+      - id: prepare
+        name: Prepare the action and log details
+        shell: bash
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: |
+          TAG=$(echo $TAG | sed 's/^v//')
+          echo "TAG=$TAG" >> $GITHUB_ENV
+          echo "The tag for this build is $TAG"
+          echo "The repo name is: $REPO"
+          echo "Github context:\n$GITHUB_CONTEXT"
+
+      - id: checkout
+        name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - id: auth-ghcr
+        name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: auth-google
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          token_format: access_token
+          project_id: ${{ env.GCP_PROJECT_ID }}
+          service_account: github-actions@open-targets-genetics-dev.iam.gserviceaccount.com
+          workload_identity_provider: projects/234703259993/locations/global/workloadIdentityPools/github/providers/opentargets
+          access_token_lifetime: 300s
+
+      - id: auth-gar
+        name: Login to Google Artifact Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GCP_REGION }}-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth-google.outputs.access_token }}
+
+      - id: push
+        name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:latest
+            ghcr.io/${{ github.repository }}:${{ env.TAG }}
+            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/opentargets/${{ env.REPO }}:latest
+            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/opentargets/${{ env.REPO }}:${{ env.TAG }}
+
+      - id: generate-attestations
+        name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/opentargets/${{ env.REPO }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

{gentroutils-2.0.0 → gentroutils-3.1.0}/.github/workflows/tag.yaml

@@ -4,12 +4,16 @@ on:
     branches:
       - dev
       - main
+    tags-ignore: # Prevent running the action on the tag
+      - '*'
 
 
 jobs:
   create-tag:
-    # NOTE: only trigger the workflow when the commit is not from the GitHubActions bot (prevent self-triggering)
-    if: github.event.commits[0].author.name != 'github-actions[botg]'
+    # NOTE: only trigger the workflow only when human user pushed to the branch (prevent self-triggering)
+    if: |
+      !contains(fromJSON('["github-actions[bot]", "semantic-release"]'), github.actor) &&
+      github.event.head_commit.author.name != 'semantic-release'
     runs-on: ubuntu-latest
     concurrency: release
     environment: DEV

{gentroutils-2.0.0 → gentroutils-3.1.0}/CHANGELOG.md

@@ -1,6 +1,209 @@
 # CHANGELOG
 
 
+## v3.1.0 (2025-09-02)
+
+### Chores
+
+- Trigger release process ([#36](https://github.com/opentargets/gentroutils/pull/36),
+  [`a90fdc7`](https://github.com/opentargets/gentroutils/commit/a90fdc7cd26dcb1263590f93f79bff6ccc867868))
+
+* fix: update auth
+
+* 3.0.1-dev.1
+
+Automatically generated by python-semantic-release
+
+* ci: add service account to impersonate
+
+* 3.0.1-dev.2
+
+* ci: prevent running create-tag on tag
+
+* 3.0.1-dev.3
+
+* ci: prevent running create-tag by semantic-release
+
+* fix: workflow file
+
+* 3.0.1-dev.4
+
+* chore: update readme
+
+* ci: run artifact build only from tag
+
+* 3.0.1-dev.5
+
+* ci: prevent tag action to run after semvar
+
+* 3.0.1-dev.6
+
+* build: remove obscured gcs scope from polars
+
+* feat: rebuild docker image
+
+* feat: add docker build command
+
+* 3.1.0-dev.1
+
+* ci: fix container name
+
+* 3.1.0-dev.2
+
+* ci: change image name structure for gcs
+
+* 3.1.0-dev.3
+
+* ci: update path to attestations
+
+* 3.1.0-dev.4
+
+* 3.1.0-dev.5
+
+---------
+
+Co-authored-by: Szymon Szyszkowski <69353402+project-defiant@users.noreply.github.com>
+
+Co-authored-by: semantic-release <semantic-release>
+
+Co-authored-by: project-defiant <szymonszyszkowski@gmail.com>
+
+
+## v3.1.0-dev.5 (2025-08-29)
+
+### Continuous Integration
+
+- Update path to attestations
+  ([`febea43`](https://github.com/opentargets/gentroutils/commit/febea4366fe94de09b82154d92b90baadfb08871))
+
+
+## v3.1.0-dev.4 (2025-08-29)
+
+### Continuous Integration
+
+- Update path to attestations
+  ([`55a80f1`](https://github.com/opentargets/gentroutils/commit/55a80f11838d6569f4919b730e291be665f33dad))
+
+
+## v3.1.0-dev.3 (2025-08-29)
+
+### Continuous Integration
+
+- Change image name structure for gcs
+  ([`88dbbbd`](https://github.com/opentargets/gentroutils/commit/88dbbbd7637bf7c9ec8c405a9811ad9e29416d48))
+
+
+## v3.1.0-dev.2 (2025-08-29)
+
+### Continuous Integration
+
+- Fix container name
+  ([`9c64ae1`](https://github.com/opentargets/gentroutils/commit/9c64ae1e4d43b1861625ef673df8edd7b5127b48))
+
+
+## v3.1.0-dev.1 (2025-08-29)
+
+### Build System
+
+- Remove obscured gcs scope from polars
+  ([`895bfed`](https://github.com/opentargets/gentroutils/commit/895bfed2486c6ca5123cc6908760020d832638ec))
+
+### Features
+
+- Add docker build command
+  ([`8f42913`](https://github.com/opentargets/gentroutils/commit/8f42913c6c41b160a28ca230cb801dd2da0bebf1))
+
+- Rebuild docker image
+  ([`94f2a49`](https://github.com/opentargets/gentroutils/commit/94f2a49ccb25a8c9eb9238cd56ea75024e26ff50))
+
+
+## v3.0.1-dev.6 (2025-08-29)
+
+### Continuous Integration
+
+- Prevent tag action to run after semvar
+  ([`847e36b`](https://github.com/opentargets/gentroutils/commit/847e36ba4a5848116c2f6311849a1960dd55b34c))
+
+
+## v3.0.1-dev.5 (2025-08-29)
+
+### Chores
+
+- Update readme
+  ([`9e75c35`](https://github.com/opentargets/gentroutils/commit/9e75c35a4ed6113dae76ed9b0a67762fbcd882e3))
+
+### Continuous Integration
+
+- Run artifact build only from tag
+  ([`4906b38`](https://github.com/opentargets/gentroutils/commit/4906b38c49d51a7b8a6a28cb12638ecc9a6fdc5e))
+
+
+## v3.0.1-dev.4 (2025-08-29)
+
+### Bug Fixes
+
+- Workflow file
+  ([`f840d55`](https://github.com/opentargets/gentroutils/commit/f840d555bbf4a8415c71b56b34d72393af8c8ebf))
+
+### Continuous Integration
+
+- Prevent running create-tag by semantic-release
+  ([`57fb068`](https://github.com/opentargets/gentroutils/commit/57fb068a21bf86d048376ab0f4678694abeb2e71))
+
+
+## v3.0.1-dev.3 (2025-08-29)
+
+### Continuous Integration
+
+- Prevent running create-tag on tag
+  ([`963f657`](https://github.com/opentargets/gentroutils/commit/963f657221f2f08f20f1b20a371ad884d584bf0a))
+
+
+## v3.0.1-dev.2 (2025-08-29)
+
+### Continuous Integration
+
+- Add service account to impersonate
+  ([`05d5eb1`](https://github.com/opentargets/gentroutils/commit/05d5eb133efc9a5e3103397ff33233b543c1d2e2))
+
+
+## v3.0.1-dev.1 (2025-08-28)
+
+### Bug Fixes
+
+- Update auth
+  ([`a95b566`](https://github.com/opentargets/gentroutils/commit/a95b566a164d31ad36383fb99d7c88a5aec27b70))
+
+
+## v3.0.0 (2025-08-28)
+
+### Bug Fixes
+
+- Ensure otter scratchpad works
+  ([`b781b1f`](https://github.com/opentargets/gentroutils/commit/b781b1f587f94807daa3abde0dbb785121904e68))
+
+### Build System
+
+- Update dependencies
+  ([`e9e05c3`](https://github.com/opentargets/gentroutils/commit/e9e05c31ec6ea443d1cb8af28f3583c6486ecb52))
+
+### Chores
+
+- Format
+  ([`876400b`](https://github.com/opentargets/gentroutils/commit/876400b3d8b5a09c198a8b13db2ffc598a1f218a))
+
+- Remove R session files
+  ([`ffe093d`](https://github.com/opentargets/gentroutils/commit/ffe093d494ec7c469b8b9f99978bfa002426c189))
+
+- Update readme
+  ([`06a21db`](https://github.com/opentargets/gentroutils/commit/06a21dbdb0b706e06537f80d378f8837041b3906))
+
+### Continuous Integration
+
+- Add build command
+  ([`2e67fcd`](https://github.com/opentargets/gentroutils/commit/2e67fcd1d7478b96dc698d5e54e2a077529fcb61))
+
+
 ## v2.0.0 (2025-08-28)
 
 ### Chores
@@ -16,6 +219,16 @@
 
 Co-authored-by: Szymon Szyszkowski <69353402+project-defiant@users.noreply.github.com>
 
+### Continuous Integration
+
+- Update ci
+  ([`81c39b2`](https://github.com/opentargets/gentroutils/commit/81c39b2d02f8fe318d8921b46c636a3361093263))
+
+### Features
+
+- 2.0.0
+  ([`3268c38`](https://github.com/opentargets/gentroutils/commit/3268c383cf6dfac4be7748ad6b48e8ded5f6157c))
+
 
 ## v1.6.0-dev.2 (2025-08-12)
 

gentroutils-3.1.0/Dockerfile

@@ -0,0 +1,29 @@
+# Description: Dockerfile for the gentroutils package
+#
+# To run locally, you must have a credentials file for GCP. Assuming you do,
+# you can run the following command:
+#
+# docker run -v /path/to/credentials.json:/app/credentials.json \
+#     -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json  \
+#     gentroutuls -s gwas_catalog_release
+# By default the image uses the `config.yaml` file provided in the repository.
+FROM rust:slim-trixie AS rust-builder
+FROM python:3.13.7-slim-trixie
+
+# Copy Rustc and Cargo from the rust-builder stage
+# These are needed to install polars without compiling rust from source
+COPY --from=rust-builder /usr/local/cargo/bin/rustc /usr/local/bin/rustc
+COPY --from=rust-builder /usr/local/cargo/bin/cargo /usr/local/bin/cargo
+
+# Copy Python source
+COPY src /app/src
+COPY pyproject.toml /app/pyproject.toml
+COPY uv.lock /app/uv.lock
+COPY README.md /app/README.md
+COPY config.yaml /app/config.yaml
+
+# Build the executable
+WORKDIR /app
+RUN python -m pip install .
+
+ENTRYPOINT [ "gentroutils", "-c", "/app/config.yaml" ]

{gentroutils-2.0.0 → gentroutils-3.1.0}/Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 .PHONY: $(shell sed -n -e '/^$$/ { n ; /^[^ .\#][^ ]*:/ { s/:.*$$// ; p ; } ; }' $(MAKEFILE_LIST))
-VERSION := $$(grep '^version' pyproject.toml | sed 's%version = "\(.*\)"%\1%')
+VERSION := $$(grep '^version' pyproject.toml | head -1 | sed 's%version = "\(.*\)"%\1%')
 APP_NAME := $$(grep '^name' pyproject.toml | head -1 | sed 's%name = "\(.*\)"%\1%')
 
 .DEFAULT_GOAL := help
@@ -41,3 +41,6 @@ check: lint format test type-check dep-check ## run all checks
 
 help: ## This is help
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+build-docker: ## build docker image
+	docker build -t $(APP_NAME):$(VERSION) --no-cache -f Dockerfile .

{gentroutils-2.0.0 → gentroutils-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gentroutils
-Version: 2.0.0
+Version: 3.1.0
 Summary: Open Targets python genetics utility CLI tools
 Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
 License-Expression: Apache-2.0
@@ -15,10 +15,11 @@ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Requires-Python: >=3.13
 Requires-Dist: aioftp>=0.25.1
 Requires-Dist: aiohttp>=3.11.18
+Requires-Dist: gcsfs>=2025.7.0
 Requires-Dist: google-cloud-storage>=3.1.1
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: opentargets-otter>=25.0.2
-Requires-Dist: polars>=1.31.0
+Requires-Dist: polars[fsspec]>=1.31.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: tqdm>=4.67.1
 Description-Content-Type: text/markdown
@@ -33,7 +34,7 @@ Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
 
 ## Installation
 
-```
+```{bash}
 pip install gentroutils
 ```
 
@@ -48,6 +49,7 @@ gentroutils --help
 ## Usage
 
 To run a single step run
+
 ```{bash}
 uv run gentroutils -s gwas_catalog_release  # After cloning the repository
 gentroutils -s gwas_catalog_release -c otter_config.yaml # When installed by pip
@@ -60,6 +62,11 @@ The `gentroutils` repository uses the [otter](https://github.com/opentargets/ott
 
 For the top level fields refer to the [otter documentation](https://opentargets.github.io/otter/otter.config.html)
 
+> [!NOTE]
+> All `destination_template` must point to the Google Cloud Storage (GCS) bucket objects.
+> All `source_template` must point to the FTP server paths.
+> In case this is not enforced, the user may experience silent failures.
+
 ```yaml
 ---
 work_path: ./work
@@ -91,7 +98,7 @@ steps:
         - fetch studies
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
-      destination_template: ./work/curation_{release_date}.tsv
+      destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
       promote: true
 ```
 
@@ -114,7 +121,7 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
@@ -135,7 +142,7 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
@@ -203,7 +210,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
 
 This task is used to build the GWAS Catalog curation file that is later used as a template for manual curation. It requires the `fetch studies` task to be completed before it can run. This is due to the fact that the curation file is build based on the list of studies fetched from `download studies` file.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `requires` field specifies that this task depends on the `fetch studies` task, meaning it will only run after the studies have been fetched.
@@ -268,6 +275,7 @@ To check CLI execution manually you need to run
 ```{bash}
 uv run gentroutils
 ```
+
 ---
 
 This software was developed as part of the Open Targets project. For more

{gentroutils-2.0.0 → gentroutils-3.1.0}/README.md

@@ -8,7 +8,7 @@ Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
 
 ## Installation
 
-```
+```{bash}
 pip install gentroutils
 ```
 
@@ -23,6 +23,7 @@ gentroutils --help
 ## Usage
 
 To run a single step run
+
 ```{bash}
 uv run gentroutils -s gwas_catalog_release  # After cloning the repository
 gentroutils -s gwas_catalog_release -c otter_config.yaml # When installed by pip
@@ -35,6 +36,11 @@ The `gentroutils` repository uses the [otter](https://github.com/opentargets/ott
 
 For the top level fields refer to the [otter documentation](https://opentargets.github.io/otter/otter.config.html)
 
+> [!NOTE]
+> All `destination_template` must point to the Google Cloud Storage (GCS) bucket objects.
+> All `source_template` must point to the FTP server paths.
+> In case this is not enforced, the user may experience silent failures.
+
 ```yaml
 ---
 work_path: ./work
@@ -66,7 +72,7 @@ steps:
         - fetch studies
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
-      destination_template: ./work/curation_{release_date}.tsv
+      destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
       promote: true
 ```
 
@@ -89,7 +95,7 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
@@ -110,7 +116,7 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
@@ -178,7 +184,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
 
 This task is used to build the GWAS Catalog curation file that is later used as a template for manual curation. It requires the `fetch studies` task to be completed before it can run. This is due to the fact that the curation file is build based on the list of studies fetched from `download studies` file.
 
-> [!NOTE] 
+> [!NOTE]
 > **Task parameters**
 >
 > - The `requires` field specifies that this task depends on the `fetch studies` task, meaning it will only run after the studies have been fetched.
@@ -243,6 +249,7 @@ To check CLI execution manually you need to run
 ```{bash}
 uv run gentroutils
 ```
+
 ---
 
 This software was developed as part of the Open Targets project. For more

gentroutils-3.1.0/config.yaml

@@ -0,0 +1,40 @@
+---
+work_path: ./work
+log_level: DEBUG
+scratchpad:
+  gc_stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+  gc_bucket: "gs://gwas_catalog_inputs"
+  gc_ftp: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases"
+
+steps:
+  gwas_catalog_release:
+    # - name: crawl release metadata
+    #   stats_uri: ${gc_stats_uri}
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/stats.json'
+    #   promote: true
+
+    # - name: fetch associations
+    #   stats_uri: ${gc_stats_uri}
+    #   source_template: '${gc_ftp}/{release_date}/gwas-catalog-associations_ontology-annotated.tsv'
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv'
+    #   promote: true
+
+    # - name: fetch studies
+    #   stats_uri: ${gc_stats_uri}
+    #   source_template: '${gc_ftp}/{release_date}/gwas-catalog-download-studies-v1.0.3.1.txt'
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_download_studies.tsv'
+    #   promote: true
+
+    # - name: fetch ancestries
+    #   stats_uri: ${gc_stats_uri}
+    #   source_template: '${gc_ftp}/{release_date}/gwas-catalog-download-ancestries-v1.0.3.1.txt'
+    #   destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_download_ancestries.tsv'
+    #   promote: true
+
+    - name: curation study
+      # requires:
+      #   - fetch studies
+      previous_curation: '${gc_bucket}/curation/latest/curated/GWAS_Catalog_study_curation.tsv'
+      studies: '${gc_bucket}/gentroutils/latest/gwas_catalog_download_studies.tsv'
+      destination_template: '${gc_bucket}/curation/{release_date}/raw/GWAS_Catalog_study_curation.tsv'
+      promote: true

{gentroutils-2.0.0 → gentroutils-3.1.0}/pyproject.toml

@@ -1,17 +1,18 @@
 [project]
 authors = [{ name = "Szymon Szyszkowski", email = "ss60@sanger.ac.uk" }]
 name = "gentroutils"
-version = "2.0.0"
+version = "3.1.0"
 description = "Open Targets python genetics utility CLI tools"
 dependencies = [
     "aiohttp>=3.11.18",
     "aioftp>=0.25.1",
-    "polars>=1.31.0",
+    "polars[fsspec]>=1.31.0",
     "pydantic>=2.10.6",
     "loguru>=0.7.3",
     "tqdm>=4.67.1",
     "opentargets-otter>=25.0.2",
     "google-cloud-storage>=3.1.1",
+    "gcsfs>=2025.7.0",
 ]
 readme = "README.md"
 requires-python = ">=3.13"
@@ -75,6 +76,12 @@ allow-direct-references = true
 [tool.hatch.build.targets.wheel]
 packages = ["src/gentroutils"]
 
+
+# Ignore polars x GCS depencency not imported in code
+[tool.deptry.per_rule_ignores]
+DEP002 = ["gcsfs"]
+
+
 # test configuration
 [tool.pytest.ini_options]
 markers = ["integration_test: Intergration tests", "unit_test: Unit tests"]

{gentroutils-2.0.0 → gentroutils-3.1.0}/src/gentroutils/io/transfer/ftp_to_gcs.py

@@ -32,12 +32,24 @@ class FTPtoGCPTransferableObject(TransferableObject):
         async with aioftp.Client.context(ftp_obj.server, user="anonymous", password="anonymous") as ftp:  # noqa: S106
             bucket = storage.Client().bucket(gcs_obj.bucket)
             blob = bucket.blob(gcs_obj.object)
-            logger.info(f"Changing directory to {ftp_obj.base_dir}.")
-            await ftp.change_directory(ftp_obj.base_dir)
-            pwd = await ftp.get_current_directory()
-            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(pwd))
+            logger.info(f"Searching for the release date in the provided ftp path: {ftp_obj.base_dir}.")
+            dir_match = re.match(r"^.*(?P<release_date>\d{4}\/\d{2}\/\d{2}){1}$", str(ftp_obj.base_dir))
+
             if dir_match:
-                logger.info(f"Found release date!: {dir_match.group('release_date')}")
+                logger.info(f"Found release date to search in the ftp {dir_match.group('release_date')}.")
+                release_date = dir_match.group("release_date")
+                try:
+                    await ftp.change_directory(ftp_obj.base_dir)
+                except aioftp.StatusCodeError as e:
+                    logger.error(f"Failed to change directory to {ftp_obj.base_dir}: {e}")
+                    logger.warning("Attempting to load the `latest` release.")
+                    ftp_obj = FTPPath(self.source.replace(release_date, "latest"))
+                    try:
+                        await ftp.change_directory(ftp_obj.base_dir)
+                    except aioftp.StatusCodeError as e:
+                        logger.error(f"Failed to find the latest release under {ftp_obj}")
+                        raise
+
             buffer = io.BytesIO()
             stream = await ftp.download_stream(ftp_obj.filename)
             async with stream:

{gentroutils-2.0.0 → gentroutils-3.1.0}/src/gentroutils/tasks/__init__.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+from collections import defaultdict
 from dataclasses import dataclass
 from datetime import date
 
@@ -13,7 +14,12 @@ from pydantic import AliasPath, BaseModel, Field
 from gentroutils.errors import GentroutilsError, GentroutilsErrorMessage
 
 
-def _requires_release_date_template(path: str) -> str:
+class KeepMissing(defaultdict[str, str]):
+    def __missing__(self, key):
+        return "{" + key + "}"
+
+
+def destination_validator(path: str) -> str:
     """Ensure that the destination path contains a template for the release date."""
     if "{release_date}" not in path:
         raise GentroutilsError(GentroutilsErrorMessage.MISSING_RELEASE_DATE_TEMPLATE, release_date="{release_date}")
@@ -34,7 +40,7 @@ class TemplateDestination:
 
         This method returns a new TemplateDestination object (not a copy of the current one) with the formatted destination.
         """
-        return TemplateDestination(self.destination.format(**substitutions), True)
+        return TemplateDestination(self.destination.format_map(KeepMissing(**substitutions)), True)
 
 
 class GwasCatalogReleaseInfo(BaseModel):
@@ -83,6 +89,7 @@ class GwasCatalogReleaseInfo(BaseModel):
     @classmethod
     def from_uri(cls, uri: str) -> GwasCatalogReleaseInfo:
         """Fetch the release information from the specified URI."""
+        logger.debug(f"Fetching release info from {uri}")
         try:
             return asyncio.run(cls._get_release_info(uri))
         except aiohttp.ClientError as e: