gentroutils 3.0.0__tar.gz → 4.0.0__tar.gz

{gentroutils-3.0.0 → gentroutils-4.0.0}/.github/workflows/build.yaml

@@ -8,11 +8,12 @@ env:
   GCP_PROJECT_ID: "open-targets-genetics-dev"
   GCP_REGION: "europe-west1"
   TAG: "${{ github.ref_name }}"
-  REPO: "${{ github.event.repository_name }}"
+  REPO: "${{ github.event.repository.name }}"
 
 
 jobs:
   push-to-ghcr-and-gar:
+    if: startsWith(github.ref, 'refs/tags/')
     name: Build docker image and push to GHCR and GAR
     runs-on: ubuntu-22.04
 
@@ -54,7 +55,9 @@ jobs:
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v2
         with:
+          token_format: access_token
           project_id: ${{ env.GCP_PROJECT_ID }}
+          service_account: github-actions@open-targets-genetics-dev.iam.gserviceaccount.com
           workload_identity_provider: projects/234703259993/locations/global/workloadIdentityPools/github/providers/opentargets
           access_token_lifetime: 300s
 
@@ -75,13 +78,13 @@ jobs:
           tags: |
             ghcr.io/${{ github.repository }}:latest
             ghcr.io/${{ github.repository }}:${{ env.TAG }}
-            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPO }}/${{ env.REPO }}:latest
-            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPO }}:${{ env.TAG }}
+            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/opentargets/${{ env.REPO }}:latest
+            ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/opentargets/${{ env.REPO }}:${{ env.TAG }}
 
       - id: generate-attestations
         name: Generate artifact attestation
         uses: actions/attest-build-provenance@v1
         with:
-          subject-name: ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPO }}/${{ env.REPO }}
+          subject-name: ${{ env.GCP_REGION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/opentargets/${{ env.REPO }}
           subject-digest: ${{ steps.push.outputs.digest }}
           push-to-registry: true

{gentroutils-3.0.0 → gentroutils-4.0.0}/.github/workflows/tag.yaml

@@ -4,12 +4,16 @@ on:
     branches:
       - dev
       - main
+    tags-ignore: # Prevent running the action on the tag
+      - '*'
 
 
 jobs:
   create-tag:
-    # NOTE: only trigger the workflow when the commit is not from the GitHubActions bot (prevent self-triggering)
-    if: github.event.commits[0].author.name != 'github-actions[botg]'
+    # NOTE: only trigger the workflow only when human user pushed to the branch (prevent self-triggering)
+    if: |
+      !contains(fromJSON('["github-actions[bot]", "semantic-release"]'), github.actor) &&
+      github.event.head_commit.author.name != 'semantic-release'
     runs-on: ubuntu-latest
     concurrency: release
     environment: DEV

{gentroutils-3.0.0 → gentroutils-4.0.0}/CHANGELOG.md

@@ -1,6 +1,246 @@
 # CHANGELOG
 
 
+## v4.0.0 (2026-02-03)
+
+
+## v4.0.0-dev.1 (2026-02-03)
+
+### Features
+
+- Updete dependencies
+  ([`b6af4d2`](https://github.com/opentargets/gentroutils/commit/b6af4d28605e7c687f5ec15cae7187c64e834cb0))
+
+
+## v3.2.0 (2026-02-03)
+
+### Chores
+
+- Update uv lock
+  ([`6f13fc0`](https://github.com/opentargets/gentroutils/commit/6f13fc0055ee9a49a215166d3cccb31747602a4f))
+
+
+## v3.2.0-dev.2 (2026-02-03)
+
+### Bug Fixes
+
+- Output tsv file instead of csv
+  ([`aff71b1`](https://github.com/opentargets/gentroutils/commit/aff71b16b6c4d273cc851050a793d3798bae27ac))
+
+- Test
+  ([`f9dd890`](https://github.com/opentargets/gentroutils/commit/f9dd890efc32ab969fbcd14eb0da14e40678e8fb))
+
+- Test for curation
+  ([`b853358`](https://github.com/opentargets/gentroutils/commit/b85335815d7a22745c61404f81b612a14cce06d5))
+
+- Test for curation
+  ([`22138ab`](https://github.com/opentargets/gentroutils/commit/22138ab31f7551a4b161f6f1885b7975d57a0ac7))
+
+### Chores
+
+- Cleanup
+  ([`68a3f66`](https://github.com/opentargets/gentroutils/commit/68a3f6607a4a1b61441c1369f2a9d3b4babec30c))
+
+- Fix glob pattern
+  ([`404b8ca`](https://github.com/opentargets/gentroutils/commit/404b8ca71b95764529ebb3df7c39881a0a12ff5e))
+
+- Handle mutliple sumstat files
+  ([`1fc8902`](https://github.com/opentargets/gentroutils/commit/1fc8902171a8f6edac407b790c3bcbe691792f96))
+
+- Update
+  ([`e69575b`](https://github.com/opentargets/gentroutils/commit/e69575b5a6c802b78b959314b84348d7969eeaeb))
+
+- Update readme
+  ([`12f274c`](https://github.com/opentargets/gentroutils/commit/12f274c5158b3986ba2511791fc2289b24d9aa40))
+
+
+## v3.2.0-dev.1 (2025-11-05)
+
+### Chores
+
+- Uncomment config
+  ([`30c4d68`](https://github.com/opentargets/gentroutils/commit/30c4d68e79a35d2c5c83cd17a15f63906ef834d6))
+
+### Features
+
+- **associations**: Allow zip file transfer from ftp
+  ([`662a635`](https://github.com/opentargets/gentroutils/commit/662a63593cd5f340a768974041461cc65e1566b9))
+
+
+## v3.1.0 (2025-09-02)
+
+### Chores
+
+- Trigger release process ([#36](https://github.com/opentargets/gentroutils/pull/36),
+  [`a90fdc7`](https://github.com/opentargets/gentroutils/commit/a90fdc7cd26dcb1263590f93f79bff6ccc867868))
+
+* fix: update auth
+
+* 3.0.1-dev.1
+
+Automatically generated by python-semantic-release
+
+* ci: add service account to impersonate
+
+* 3.0.1-dev.2
+
+* ci: prevent running create-tag on tag
+
+* 3.0.1-dev.3
+
+* ci: prevent running create-tag by semantic-release
+
+* fix: workflow file
+
+* 3.0.1-dev.4
+
+* chore: update readme
+
+* ci: run artifact build only from tag
+
+* 3.0.1-dev.5
+
+* ci: prevent tag action to run after semvar
+
+* 3.0.1-dev.6
+
+* build: remove obscured gcs scope from polars
+
+* feat: rebuild docker image
+
+* feat: add docker build command
+
+* 3.1.0-dev.1
+
+* ci: fix container name
+
+* 3.1.0-dev.2
+
+* ci: change image name structure for gcs
+
+* 3.1.0-dev.3
+
+* ci: update path to attestations
+
+* 3.1.0-dev.4
+
+* 3.1.0-dev.5
+
+---------
+
+Co-authored-by: Szymon Szyszkowski <69353402+project-defiant@users.noreply.github.com>
+
+Co-authored-by: semantic-release <semantic-release>
+
+Co-authored-by: project-defiant <szymonszyszkowski@gmail.com>
+
+
+## v3.1.0-dev.5 (2025-08-29)
+
+### Continuous Integration
+
+- Update path to attestations
+  ([`febea43`](https://github.com/opentargets/gentroutils/commit/febea4366fe94de09b82154d92b90baadfb08871))
+
+
+## v3.1.0-dev.4 (2025-08-29)
+
+### Continuous Integration
+
+- Update path to attestations
+  ([`55a80f1`](https://github.com/opentargets/gentroutils/commit/55a80f11838d6569f4919b730e291be665f33dad))
+
+
+## v3.1.0-dev.3 (2025-08-29)
+
+### Continuous Integration
+
+- Change image name structure for gcs
+  ([`88dbbbd`](https://github.com/opentargets/gentroutils/commit/88dbbbd7637bf7c9ec8c405a9811ad9e29416d48))
+
+
+## v3.1.0-dev.2 (2025-08-29)
+
+### Continuous Integration
+
+- Fix container name
+  ([`9c64ae1`](https://github.com/opentargets/gentroutils/commit/9c64ae1e4d43b1861625ef673df8edd7b5127b48))
+
+
+## v3.1.0-dev.1 (2025-08-29)
+
+### Build System
+
+- Remove obscured gcs scope from polars
+  ([`895bfed`](https://github.com/opentargets/gentroutils/commit/895bfed2486c6ca5123cc6908760020d832638ec))
+
+### Features
+
+- Add docker build command
+  ([`8f42913`](https://github.com/opentargets/gentroutils/commit/8f42913c6c41b160a28ca230cb801dd2da0bebf1))
+
+- Rebuild docker image
+  ([`94f2a49`](https://github.com/opentargets/gentroutils/commit/94f2a49ccb25a8c9eb9238cd56ea75024e26ff50))
+
+
+## v3.0.1-dev.6 (2025-08-29)
+
+### Continuous Integration
+
+- Prevent tag action to run after semvar
+  ([`847e36b`](https://github.com/opentargets/gentroutils/commit/847e36ba4a5848116c2f6311849a1960dd55b34c))
+
+
+## v3.0.1-dev.5 (2025-08-29)
+
+### Chores
+
+- Update readme
+  ([`9e75c35`](https://github.com/opentargets/gentroutils/commit/9e75c35a4ed6113dae76ed9b0a67762fbcd882e3))
+
+### Continuous Integration
+
+- Run artifact build only from tag
+  ([`4906b38`](https://github.com/opentargets/gentroutils/commit/4906b38c49d51a7b8a6a28cb12638ecc9a6fdc5e))
+
+
+## v3.0.1-dev.4 (2025-08-29)
+
+### Bug Fixes
+
+- Workflow file
+  ([`f840d55`](https://github.com/opentargets/gentroutils/commit/f840d555bbf4a8415c71b56b34d72393af8c8ebf))
+
+### Continuous Integration
+
+- Prevent running create-tag by semantic-release
+  ([`57fb068`](https://github.com/opentargets/gentroutils/commit/57fb068a21bf86d048376ab0f4678694abeb2e71))
+
+
+## v3.0.1-dev.3 (2025-08-29)
+
+### Continuous Integration
+
+- Prevent running create-tag on tag
+  ([`963f657`](https://github.com/opentargets/gentroutils/commit/963f657221f2f08f20f1b20a371ad884d584bf0a))
+
+
+## v3.0.1-dev.2 (2025-08-29)
+
+### Continuous Integration
+
+- Add service account to impersonate
+  ([`05d5eb1`](https://github.com/opentargets/gentroutils/commit/05d5eb133efc9a5e3103397ff33233b543c1d2e2))
+
+
+## v3.0.1-dev.1 (2025-08-28)
+
+### Bug Fixes
+
+- Update auth
+  ([`a95b566`](https://github.com/opentargets/gentroutils/commit/a95b566a164d31ad36383fb99d7c88a5aec27b70))
+
+
 ## v3.0.0 (2025-08-28)
 
 ### Bug Fixes

gentroutils-4.0.0/Dockerfile

@@ -0,0 +1,29 @@
+# Description: Dockerfile for the gentroutils package
+#
+# To run locally, you must have a credentials file for GCP. Assuming you do,
+# you can run the following command:
+#
+# docker run -v /path/to/credentials.json:/app/credentials.json \
+#     -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json  \
+#     gentroutuls -s gwas_catalog_release
+# By default the image uses the `config.yaml` file provided in the repository.
+FROM rust:slim-trixie AS rust-builder
+FROM python:3.13.7-slim-trixie
+
+# Copy Rustc and Cargo from the rust-builder stage
+# These are needed to install polars without compiling rust from source
+COPY --from=rust-builder /usr/local/cargo/bin/rustc /usr/local/bin/rustc
+COPY --from=rust-builder /usr/local/cargo/bin/cargo /usr/local/bin/cargo
+
+# Copy Python source
+COPY src /app/src
+COPY pyproject.toml /app/pyproject.toml
+COPY uv.lock /app/uv.lock
+COPY README.md /app/README.md
+COPY config.yaml /app/config.yaml
+
+# Build the executable
+WORKDIR /app
+RUN python -m pip install .
+
+ENTRYPOINT [ "gentroutils", "-c", "/app/config.yaml" ]

{gentroutils-3.0.0 → gentroutils-4.0.0}/Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 .PHONY: $(shell sed -n -e '/^$$/ { n ; /^[^ .\#][^ ]*:/ { s/:.*$$// ; p ; } ; }' $(MAKEFILE_LIST))
-VERSION := $$(grep '^version' pyproject.toml | sed 's%version = "\(.*\)"%\1%')
+VERSION := $$(grep '^version' pyproject.toml | head -1 | sed 's%version = "\(.*\)"%\1%')
 APP_NAME := $$(grep '^name' pyproject.toml | head -1 | sed 's%name = "\(.*\)"%\1%')
 
 .DEFAULT_GOAL := help
@@ -41,3 +41,6 @@ check: lint format test type-check dep-check ## run all checks
 
 help: ## This is help
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+build-docker: ## build docker image
+	docker build -t $(APP_NAME):$(VERSION) --no-cache -f Dockerfile .

{gentroutils-3.0.0 → gentroutils-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gentroutils
-Version: 3.0.0
+Version: 4.0.0
 Summary: Open Targets python genetics utility CLI tools
 Author-email: Szymon Szyszkowski <ss60@sanger.ac.uk>
 License-Expression: Apache-2.0
@@ -12,14 +12,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
-Requires-Python: >=3.13
+Requires-Python: <=3.13,>3.11
 Requires-Dist: aioftp>=0.25.1
 Requires-Dist: aiohttp>=3.11.18
 Requires-Dist: gcsfs>=2025.7.0
 Requires-Dist: google-cloud-storage>=3.1.1
 Requires-Dist: loguru>=0.7.3
-Requires-Dist: opentargets-otter>=25.0.2
-Requires-Dist: polars[fsspec,gcs]>=1.31.0
+Requires-Dist: opentargets-otter>=25.0.15
+Requires-Dist: polars[fsspec]>=1.31.0
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: tqdm>=4.67.1
 Description-Content-Type: text/markdown
@@ -34,7 +34,7 @@ Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
 
 ## Installation
 
-```
+```{bash}
 pip install gentroutils
 ```
 
@@ -99,6 +99,7 @@ steps:
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
       destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
+      summary_statistics_glob: gs://gwas_catalog_inputs/raw_summary_statistics/*.h.tsv.gz
       promote: true
 ```
 
@@ -121,7 +122,8 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `destination_template` is where the metadata will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. By default it searches for the release directly in the stats_uri json output.
@@ -141,7 +143,8 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog associations file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -162,7 +165,8 @@ This task fetches the GWAS Catalog associations file from the specified FTP serv
 
 This task fetches the GWAS Catalog studies file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog studies file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -183,7 +187,8 @@ This task fetches the GWAS Catalog studies file from the specified FTP server an
 
 This task fetches the GWAS Catalog ancestries file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog ancestries file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -201,6 +206,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
       destination_template: gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv
+      summary_statistics_glob: gs://gwas_catalog_inputs/raw_summary_statistics/*.h.tsv.gz
       promote: true
 ```
 
@@ -214,24 +220,26 @@ This task is used to build the GWAS Catalog curation file that is later used as
 > - The `studies` field is the path to the studies file that was fetched in the `fetch studies` task. This file is used to build the curation file.
 > - The `destination_template` is where the curation file will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
 > - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+> The `summary_statistics_glob` field is used to specify the glob pattern to list all synced summary statistics files from GCS. This is used to identify which studies have summary statistics available.
 
 ---
 
 ## Curation process
 
-The base of the curation process for GWAS Catalog data is defined in the [docs/gwas_catalog_curation.md](docs/gwas_catalog_curation.md). The original solution uses R script to prepare the data for curation and then manually curates the data. The solution proposed in the `curation` task autommates the preparation of the data for curation and provides a template for manual curation. The manual curation process is still required, but the data preparation is automated.
+The base of the curation process for GWAS Catalog data is defined in the [docs/gwas_catalog_curation.md](docs/gwas_catalog_curation.md). The original solution uses R script to prepare the data for curation and then manually curates the data. The solution proposed in the `curation` task automates the preparation of the data for curation and provides a template for manual curation. The manual curation process is still required, but the data preparation is automated.
 
 The automated process includes:
 
 1. Reading `download studies` file with the list of studies that are currently comming from the latest GWAS Catalog release.
 2. Reading `previous curation` file that contains the list of the curated studies from the previous release.
-3. Comparing the two datasets with following logic:
+3. Listing all synced summary statistics files from the `summary_statistics_glob` parameter to identify which studies have summary statistics available. Note that this can be more then the list of studies in the `download studies` file as syncing also involves the unpublished studies.
+4. Comparing the three datasets with following logic:
    - In case the study is present in the `previous curation` and `download studies`, the study is marked as `curated`
-   * In case the study is present in the `download studies` but not in the `previous curation`, the study is marked as `new`
-   * In case the study is present in the `previous curation` but not in the `download studies`, the study is marked as `removed`
-4. The output of the curation process is a file that contains the list of studies with their status (curated, new, removed) and the fields that are required for manual curation. The output file is saved to the `destination_template` path specified in the task configuration. The file is saved under `gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv` path.
-5. The output file is then promoted to the latest release path `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` so that it can be used for manual curation.
-6. The manual curation process is then performed on the `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` file. The manual curation process is not automated and requires manual intervention. The output from the manual curation process should be saved then to the `gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv` and `gs://gwas_catalog_inputs/curation/{release_date}/curated/GWAS_Catalog_study_curation.tsv` file. This file is then used for the [Open Targets Staging Dags](https://github.com/opentargets/orchestration).
+   - In case the study is present in the `download studies` but not in the `previous curation`, the study is marked as `to_curate` or `has_no_sumstats` depending on the presence of summary statistics files
+   - In case the study is present in the `previous curation` but not in the `download studies`, the study is marked as `removed`
+5. The output of the curation process is a file that contains the list of studies with their status (curated, new, removed) and the fields that are required for manual curation. The output file is saved to the `destination_template` path specified in the task configuration. The file is saved under `gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv` path.
+6. The output file is then promoted to the latest release path `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` so that it can be used for manual curation.
+7. The manual curation process is then performed on the `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` file. The manual curation process is not automated and requires manual intervention. The output from the manual curation process should be saved then to the `gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv` and `gs://gwas_catalog_inputs/curation/{release_date}/curated/GWAS_Catalog_study_curation.tsv` file. This file is then used for the [Open Targets Staging Dags](https://github.com/opentargets/orchestration).
 
 ---
 

{gentroutils-3.0.0 → gentroutils-4.0.0}/README.md

@@ -8,7 +8,7 @@ Set of Command Line Interface tools to process Open Targets Genetics GWAS data.
 
 ## Installation
 
-```
+```{bash}
 pip install gentroutils
 ```
 
@@ -73,6 +73,7 @@ steps:
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
       destination_template: gs://gwas_catalog_inputs/gentroutils/curation/{release_date}/GWAS_Catalog_study_curation.tsv
+      summary_statistics_glob: gs://gwas_catalog_inputs/raw_summary_statistics/*.h.tsv.gz
       promote: true
 ```
 
@@ -95,7 +96,8 @@ The list of tasks (defined in the `config.yaml` file) that can be run are:
 
 This task fetches the latest GWAS Catalog release metadata from the `https://www.ebi.ac.uk/gwas/api/search/stats` endpoint and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `destination_template` is where the metadata will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. By default it searches for the release directly in the stats_uri json output.
@@ -115,7 +117,8 @@ This task fetches the latest GWAS Catalog release metadata from the `https://www
 
 This task fetches the GWAS Catalog associations file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog associations file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -136,7 +139,8 @@ This task fetches the GWAS Catalog associations file from the specified FTP serv
 
 This task fetches the GWAS Catalog studies file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog studies file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -157,7 +161,8 @@ This task fetches the GWAS Catalog studies file from the specified FTP server an
 
 This task fetches the GWAS Catalog ancestries file from the specified FTP server and saves it to the specified destination.
 
-> [!NOTE] > **Task parameters**
+> [!NOTE]
+> **Task parameters**
 >
 > - The `stats_uri` is used to fetch the latest release date and other metadata.
 > - The `source_template` is the URL of the GWAS Catalog ancestries file, which uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
@@ -175,6 +180,7 @@ This task fetches the GWAS Catalog ancestries file from the specified FTP server
       previous_curation: gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv
       studies: gs://gwas_catalog_inputs/gentroutils/latest/gwas_catalog_download_studies.tsv
       destination_template: gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv
+      summary_statistics_glob: gs://gwas_catalog_inputs/raw_summary_statistics/*.h.tsv.gz
       promote: true
 ```
 
@@ -188,24 +194,26 @@ This task is used to build the GWAS Catalog curation file that is later used as
 > - The `studies` field is the path to the studies file that was fetched in the `fetch studies` task. This file is used to build the curation file.
 > - The `destination_template` is where the curation file will be saved, and it uses the `{release_date}` placeholder to specify the release date dynamically. The release date is fetched from the `stats_uri` endpoint.
 > - The `promote` field is set to `true`, which means the output will be promoted to the latest release. Meaning that the file will be saved under `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` after the task is completed. If the `promote` field is set to `false`, the file will not be promoted and will be saved under the specified path with the release date.
+> The `summary_statistics_glob` field is used to specify the glob pattern to list all synced summary statistics files from GCS. This is used to identify which studies have summary statistics available.
 
 ---
 
 ## Curation process
 
-The base of the curation process for GWAS Catalog data is defined in the [docs/gwas_catalog_curation.md](docs/gwas_catalog_curation.md). The original solution uses R script to prepare the data for curation and then manually curates the data. The solution proposed in the `curation` task autommates the preparation of the data for curation and provides a template for manual curation. The manual curation process is still required, but the data preparation is automated.
+The base of the curation process for GWAS Catalog data is defined in the [docs/gwas_catalog_curation.md](docs/gwas_catalog_curation.md). The original solution uses R script to prepare the data for curation and then manually curates the data. The solution proposed in the `curation` task automates the preparation of the data for curation and provides a template for manual curation. The manual curation process is still required, but the data preparation is automated.
 
 The automated process includes:
 
 1. Reading `download studies` file with the list of studies that are currently comming from the latest GWAS Catalog release.
 2. Reading `previous curation` file that contains the list of the curated studies from the previous release.
-3. Comparing the two datasets with following logic:
+3. Listing all synced summary statistics files from the `summary_statistics_glob` parameter to identify which studies have summary statistics available. Note that this can be more then the list of studies in the `download studies` file as syncing also involves the unpublished studies.
+4. Comparing the three datasets with following logic:
    - In case the study is present in the `previous curation` and `download studies`, the study is marked as `curated`
-   * In case the study is present in the `download studies` but not in the `previous curation`, the study is marked as `new`
-   * In case the study is present in the `previous curation` but not in the `download studies`, the study is marked as `removed`
-4. The output of the curation process is a file that contains the list of studies with their status (curated, new, removed) and the fields that are required for manual curation. The output file is saved to the `destination_template` path specified in the task configuration. The file is saved under `gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv` path.
-5. The output file is then promoted to the latest release path `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` so that it can be used for manual curation.
-6. The manual curation process is then performed on the `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` file. The manual curation process is not automated and requires manual intervention. The output from the manual curation process should be saved then to the `gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv` and `gs://gwas_catalog_inputs/curation/{release_date}/curated/GWAS_Catalog_study_curation.tsv` file. This file is then used for the [Open Targets Staging Dags](https://github.com/opentargets/orchestration).
+   - In case the study is present in the `download studies` but not in the `previous curation`, the study is marked as `to_curate` or `has_no_sumstats` depending on the presence of summary statistics files
+   - In case the study is present in the `previous curation` but not in the `download studies`, the study is marked as `removed`
+5. The output of the curation process is a file that contains the list of studies with their status (curated, new, removed) and the fields that are required for manual curation. The output file is saved to the `destination_template` path specified in the task configuration. The file is saved under `gs://gwas_catalog_inputs/curation/{release_date}/raw/gwas_catalog_study_curation.tsv` path.
+6. The output file is then promoted to the latest release path `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` so that it can be used for manual curation.
+7. The manual curation process is then performed on the `gs://gwas_catalog_inputs/curation/latest/raw/gwas_catalog_study_curation.tsv` file. The manual curation process is not automated and requires manual intervention. The output from the manual curation process should be saved then to the `gs://gwas_catalog_inputs/curation/latest/curated/GWAS_Catalog_study_curation.tsv` and `gs://gwas_catalog_inputs/curation/{release_date}/curated/GWAS_Catalog_study_curation.tsv` file. This file is then used for the [Open Targets Staging Dags](https://github.com/opentargets/orchestration).
 
 ---
 

gentroutils-4.0.0/config.yaml

@@ -0,0 +1,41 @@
+---
+work_path: ./work
+log_level: DEBUG
+scratchpad:
+  gc_stats_uri: "https://www.ebi.ac.uk/gwas/api/search/stats"
+  gc_bucket: "gs://gwas_catalog_inputs"
+  gc_ftp: "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases"
+
+steps:
+  gwas_catalog_release:
+    - name: crawl release metadata
+      stats_uri: ${gc_stats_uri}
+      destination_template: '${gc_bucket}/gentroutils/{release_date}/stats.json'
+      promote: true
+
+    - name: fetch studies
+      stats_uri: ${gc_stats_uri}
+      source_template: '${gc_ftp}/{release_date}/gwas-catalog-download-studies-v1.0.3.1.txt'
+      destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_download_studies.tsv'
+      promote: true
+
+    - name: fetch ancestries
+      stats_uri: ${gc_stats_uri}
+      source_template: '${gc_ftp}/{release_date}/gwas-catalog-download-ancestries-v1.0.3.1.txt'
+      destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_download_ancestries.tsv'
+      promote: true
+
+    - name: fetch associations
+      stats_uri: ${gc_stats_uri}
+      source_template: '${gc_ftp}/{release_date}/gwas-catalog-associations_ontology-annotated-full.zip'
+      destination_template: '${gc_bucket}/gentroutils/{release_date}/gwas_catalog_associations_ontology_annotated.tsv'
+      promote: true
+
+    - name: curation study
+      requires:
+        - fetch studies
+      previous_curation: '${gc_bucket}/curation/latest/curated/GWAS_Catalog_study_curation.tsv'
+      studies: '${gc_bucket}/gentroutils/latest/gwas_catalog_download_studies.tsv'
+      summary_statistics_glob: '${gc_bucket}/raw_summary_statistics/**.h.tsv.gz'
+      destination_template: '${gc_bucket}/curation/{release_date}/raw/GWAS_Catalog_study_curation.tsv'
+      promote: true

{gentroutils-3.0.0 → gentroutils-4.0.0}/pyproject.toml

@@ -1,21 +1,21 @@
 [project]
 authors = [{ name = "Szymon Szyszkowski", email = "ss60@sanger.ac.uk" }]
 name = "gentroutils"
-version = "3.0.0"
+version = "4.0.0"
 description = "Open Targets python genetics utility CLI tools"
 dependencies = [
     "aiohttp>=3.11.18",
     "aioftp>=0.25.1",
-    "polars[fsspec,gcs]>=1.31.0",
+    "polars[fsspec]>=1.31.0",
     "pydantic>=2.10.6",
     "loguru>=0.7.3",
     "tqdm>=4.67.1",
-    "opentargets-otter>=25.0.2",
+    "opentargets-otter>=25.0.15",
     "google-cloud-storage>=3.1.1",
     "gcsfs>=2025.7.0",
 ]
 readme = "README.md"
-requires-python = ">=3.13"
+requires-python = ">3.11,<=3.13"
 license = "Apache-2.0"
 classifiers = [
     "Development Status :: 3 - Alpha",
@@ -50,7 +50,6 @@ dev = [
     "gcloud-storage-emulator>=0.5.0",
     "types-requests>=2.32.0.20240712",
     "pyftpdlib>=2.0.1",
-    "python-semantic-release>=9.19.1",
     "pandas-stubs>=2.2.3.250308",
     "ipython>=8.36.0",
     "pytest-asyncio>=1.1.0",