PyStormTracker 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pystormtracker-0.4.0/.dockerignore +16 -0
- pystormtracker-0.4.0/.github/workflows/ci.yml +243 -0
- pystormtracker-0.4.0/.github/workflows/docker-publish.yml +119 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/.github/workflows/python-publish.yml +13 -3
- pystormtracker-0.4.0/.gitignore +40 -0
- pystormtracker-0.4.0/.python-version +1 -0
- pystormtracker-0.4.0/ARCHITECTURE.md +97 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/CITATION.cff +4 -4
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/Dockerfile +7 -19
- pystormtracker-0.4.0/PKG-INFO +254 -0
- pystormtracker-0.4.0/README.md +205 -0
- pystormtracker-0.4.0/ROADMAP.md +41 -0
- pystormtracker-0.4.0/data/test/tracks/era5_vo_2.5x2.5_1e-4_v0.0.2_imilast.txt +38347 -0
- pystormtracker-0.4.0/docs/architecture.md +2 -0
- pystormtracker-0.4.0/docs/benchmark.md +2 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/docs/conf.py +3 -2
- pystormtracker-0.4.0/docs/index.md +27 -0
- pystormtracker-0.4.0/docs/readme.md +2 -0
- pystormtracker-0.4.0/docs/roadmap.md +2 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/pyproject.toml +33 -11
- pystormtracker-0.4.0/src/pystormtracker/__init__.py +20 -0
- pystormtracker-0.4.0/src/pystormtracker/cli.py +148 -0
- pystormtracker-0.4.0/src/pystormtracker/hodges/__init__.py +0 -0
- pystormtracker-0.4.0/src/pystormtracker/hodges/tracker.py +33 -0
- pystormtracker-0.4.0/src/pystormtracker/io/__init__.py +0 -0
- pystormtracker-0.4.0/src/pystormtracker/io/imilast.py +103 -0
- pystormtracker-0.4.0/src/pystormtracker/io/loader.py +72 -0
- pystormtracker-0.4.0/src/pystormtracker/models/__init__.py +5 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/src/pystormtracker/models/center.py +4 -8
- pystormtracker-0.4.0/src/pystormtracker/models/tracker.py +27 -0
- pystormtracker-0.4.0/src/pystormtracker/models/tracks.py +428 -0
- pystormtracker-0.4.0/src/pystormtracker/simple/__init__.py +5 -0
- pystormtracker-0.4.0/src/pystormtracker/simple/concurrent.py +131 -0
- pystormtracker-0.4.0/src/pystormtracker/simple/detector.py +266 -0
- pystormtracker-0.4.0/src/pystormtracker/simple/kernels.py +148 -0
- pystormtracker-0.4.0/src/pystormtracker/simple/linker.py +175 -0
- pystormtracker-0.4.0/src/pystormtracker/simple/tracker.py +130 -0
- pystormtracker-0.4.0/src/pystormtracker/utils/__init__.py +0 -0
- pystormtracker-0.4.0/src/pystormtracker/utils/benchmark.py +52 -0
- {pystormtracker-0.3.2/tests → pystormtracker-0.4.0/src/pystormtracker/utils}/data_utils.py +28 -8
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/tests/test_center.py +12 -12
- pystormtracker-0.3.2/tests/test_stormtracker.py → pystormtracker-0.4.0/tests/test_cli.py +2 -3
- pystormtracker-0.4.0/tests/test_integration.py +299 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/tests/test_simple_detector.py +12 -9
- pystormtracker-0.4.0/tests/test_simple_linker.py +32 -0
- pystormtracker-0.4.0/tests/test_simple_tracker.py +40 -0
- pystormtracker-0.4.0/tests/test_tracks.py +247 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/uv.lock +587 -197
- pystormtracker-0.3.2/.github/workflows/ci.yml +0 -148
- pystormtracker-0.3.2/.github/workflows/docker-publish.yml +0 -101
- pystormtracker-0.3.2/.gitignore +0 -73
- pystormtracker-0.3.2/.pre-commit-config.yaml +0 -23
- pystormtracker-0.3.2/PKG-INFO +0 -199
- pystormtracker-0.3.2/README.md +0 -160
- pystormtracker-0.3.2/docs/index.md +0 -21
- pystormtracker-0.3.2/src/pystormtracker/__init__.py +0 -4
- pystormtracker-0.3.2/src/pystormtracker/models/__init__.py +0 -6
- pystormtracker-0.3.2/src/pystormtracker/models/grid.py +0 -36
- pystormtracker-0.3.2/src/pystormtracker/models/time.py +0 -14
- pystormtracker-0.3.2/src/pystormtracker/models/tracks.py +0 -202
- pystormtracker-0.3.2/src/pystormtracker/simple/__init__.py +0 -4
- pystormtracker-0.3.2/src/pystormtracker/simple/detector.py +0 -271
- pystormtracker-0.3.2/src/pystormtracker/simple/linker.py +0 -135
- pystormtracker-0.3.2/src/pystormtracker/stormtracker.py +0 -244
- pystormtracker-0.3.2/tests/test_integration.py +0 -199
- pystormtracker-0.3.2/tests/test_simple_linker.py +0 -54
- pystormtracker-0.3.2/tests/test_tracks.py +0 -124
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/.github/dependabot.yml +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/.readthedocs.yaml +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/LICENSE +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/codecov.yml +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/data/test/tracks/era5_msl_2.5x2.5_v0.0.2_imilast.txt +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/docs/IntercomparisonProtocol.pdf +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/tests/__init__.py +0 -0
- {pystormtracker-0.3.2 → pystormtracker-0.4.0}/tests/conftest.py +0 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
paths:
|
|
6
|
+
- 'src/**'
|
|
7
|
+
- 'tests/**'
|
|
8
|
+
- 'pyproject.toml'
|
|
9
|
+
- 'uv.lock'
|
|
10
|
+
- 'Dockerfile'
|
|
11
|
+
- '.github/workflows/ci.yml'
|
|
12
|
+
push:
|
|
13
|
+
branches:
|
|
14
|
+
- main
|
|
15
|
+
- release/**
|
|
16
|
+
tags:
|
|
17
|
+
- v*
|
|
18
|
+
paths:
|
|
19
|
+
- 'src/**'
|
|
20
|
+
- 'tests/**'
|
|
21
|
+
- 'pyproject.toml'
|
|
22
|
+
- 'uv.lock'
|
|
23
|
+
- 'Dockerfile'
|
|
24
|
+
- '.github/workflows/ci.yml'
|
|
25
|
+
release:
|
|
26
|
+
types: [published]
|
|
27
|
+
workflow_dispatch:
|
|
28
|
+
|
|
29
|
+
concurrency:
|
|
30
|
+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}
|
|
31
|
+
cancel-in-progress: ${{ github.ref_type != 'tag' }}
|
|
32
|
+
|
|
33
|
+
permissions:
|
|
34
|
+
contents: read
|
|
35
|
+
|
|
36
|
+
jobs:
|
|
37
|
+
ruff-lint:
|
|
38
|
+
runs-on: ubuntu-24.04
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/checkout@v6
|
|
41
|
+
with:
|
|
42
|
+
ref: ${{ github.ref }}
|
|
43
|
+
fetch-depth: 0
|
|
44
|
+
- name: Set up uv
|
|
45
|
+
uses: astral-sh/setup-uv@v7
|
|
46
|
+
with:
|
|
47
|
+
enable-cache: true
|
|
48
|
+
- name: Lint with Ruff
|
|
49
|
+
run: uv run ruff check .
|
|
50
|
+
|
|
51
|
+
ruff-format:
|
|
52
|
+
runs-on: ubuntu-24.04
|
|
53
|
+
steps:
|
|
54
|
+
- uses: actions/checkout@v6
|
|
55
|
+
with:
|
|
56
|
+
ref: ${{ github.ref }}
|
|
57
|
+
fetch-depth: 0
|
|
58
|
+
- name: Set up uv
|
|
59
|
+
uses: astral-sh/setup-uv@v7
|
|
60
|
+
with:
|
|
61
|
+
enable-cache: true
|
|
62
|
+
- name: Check formatting with Ruff
|
|
63
|
+
run: uv run ruff format --check .
|
|
64
|
+
|
|
65
|
+
mypy-typecheck:
|
|
66
|
+
runs-on: ubuntu-24.04
|
|
67
|
+
steps:
|
|
68
|
+
- uses: actions/checkout@v6
|
|
69
|
+
with:
|
|
70
|
+
ref: ${{ github.ref }}
|
|
71
|
+
fetch-depth: 0
|
|
72
|
+
- name: Set up uv
|
|
73
|
+
uses: astral-sh/setup-uv@v7
|
|
74
|
+
with:
|
|
75
|
+
enable-cache: true
|
|
76
|
+
- name: Install system dependencies
|
|
77
|
+
run: sudo apt-get update && sudo apt-get install -y libopenmpi-dev
|
|
78
|
+
- name: Install dependencies
|
|
79
|
+
run: uv sync --frozen --group dev
|
|
80
|
+
- name: Type check with Mypy
|
|
81
|
+
run: uv run mypy src/ tests/
|
|
82
|
+
|
|
83
|
+
unit-tests:
|
|
84
|
+
name: unit-tests (Python ${{ matrix.python-version }}${{ matrix.min-deps && ', min-deps' || '' }})
|
|
85
|
+
runs-on: ubuntu-24.04
|
|
86
|
+
strategy:
|
|
87
|
+
fail-fast: false
|
|
88
|
+
matrix:
|
|
89
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
90
|
+
min-deps: [false]
|
|
91
|
+
include:
|
|
92
|
+
- python-version: "3.11"
|
|
93
|
+
min-deps: true
|
|
94
|
+
steps:
|
|
95
|
+
- uses: actions/checkout@v6
|
|
96
|
+
with:
|
|
97
|
+
ref: ${{ github.ref }}
|
|
98
|
+
fetch-depth: 0
|
|
99
|
+
- name: Set up uv
|
|
100
|
+
uses: astral-sh/setup-uv@v7
|
|
101
|
+
with:
|
|
102
|
+
enable-cache: true
|
|
103
|
+
python-version: ${{ matrix.python-version }}
|
|
104
|
+
- name: Install system dependencies
|
|
105
|
+
run: sudo apt-get update && sudo apt-get install -y libopenmpi-dev
|
|
106
|
+
- name: Install dependencies
|
|
107
|
+
if: ${{ !matrix.min-deps }}
|
|
108
|
+
run: uv sync --frozen --group dev
|
|
109
|
+
- name: Install dependencies (min-deps)
|
|
110
|
+
if: ${{ matrix.min-deps }}
|
|
111
|
+
run: uv sync --group dev --resolution lowest-direct
|
|
112
|
+
- name: Run Unit Tests
|
|
113
|
+
if: matrix.python-version != '3.13' || matrix.min-deps
|
|
114
|
+
run: uv run pytest -vv
|
|
115
|
+
|
|
116
|
+
- name: Run Unit Tests with Coverage
|
|
117
|
+
if: matrix.python-version == '3.13' && !matrix.min-deps
|
|
118
|
+
run: uv run pytest -vv --cov=pystormtracker --cov-report=term-missing --cov-report=xml
|
|
119
|
+
|
|
120
|
+
- name: Upload coverage reports to Codecov
|
|
121
|
+
if: matrix.python-version == '3.13' && !matrix.min-deps
|
|
122
|
+
uses: codecov/codecov-action@v5
|
|
123
|
+
with:
|
|
124
|
+
files: ./coverage.xml
|
|
125
|
+
flags: unit
|
|
126
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
127
|
+
|
|
128
|
+
integration-tests:
|
|
129
|
+
name: integration-tests (Python ${{ matrix.python-version }}, ${{ matrix.arch }})
|
|
130
|
+
runs-on: ${{ matrix.os }}
|
|
131
|
+
strategy:
|
|
132
|
+
fail-fast: false
|
|
133
|
+
matrix:
|
|
134
|
+
include:
|
|
135
|
+
- arch: amd64
|
|
136
|
+
os: ubuntu-24.04
|
|
137
|
+
python-version: "3.13"
|
|
138
|
+
- arch: arm64
|
|
139
|
+
os: ubuntu-24.04-arm
|
|
140
|
+
python-version: "3.13"
|
|
141
|
+
steps:
|
|
142
|
+
- uses: actions/checkout@v6
|
|
143
|
+
with:
|
|
144
|
+
ref: ${{ github.ref }}
|
|
145
|
+
fetch-depth: 0
|
|
146
|
+
- name: Set up uv
|
|
147
|
+
uses: astral-sh/setup-uv@v7
|
|
148
|
+
with:
|
|
149
|
+
enable-cache: true
|
|
150
|
+
python-version: ${{ matrix.python-version }}
|
|
151
|
+
- name: Install system dependencies
|
|
152
|
+
run: sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev
|
|
153
|
+
- name: Install dependencies
|
|
154
|
+
run: uv sync --frozen --group dev
|
|
155
|
+
- name: Run Integration Tests
|
|
156
|
+
if: matrix.arch != 'amd64'
|
|
157
|
+
run: uv run pytest -vv tests/test_integration.py --run-integration
|
|
158
|
+
|
|
159
|
+
- name: Run Integration Tests with Coverage
|
|
160
|
+
if: matrix.arch == 'amd64'
|
|
161
|
+
run: uv run pytest -vv --cov=pystormtracker --cov-report=term-missing --cov-report=xml tests/test_integration.py --run-integration
|
|
162
|
+
|
|
163
|
+
- name: Upload coverage reports to Codecov
|
|
164
|
+
if: matrix.arch == 'amd64'
|
|
165
|
+
uses: codecov/codecov-action@v5
|
|
166
|
+
with:
|
|
167
|
+
files: ./coverage.xml
|
|
168
|
+
flags: integration
|
|
169
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
170
|
+
|
|
171
|
+
docker-build:
|
|
172
|
+
name: docker-build
|
|
173
|
+
needs: [ruff-lint, ruff-format, mypy-typecheck, unit-tests, integration-tests]
|
|
174
|
+
# Only run on merges to main, release branches, tags, releases, or manual dispatch
|
|
175
|
+
if: |
|
|
176
|
+
github.event_name != 'pull_request' &&
|
|
177
|
+
(github.ref == 'refs/heads/main' ||
|
|
178
|
+
startsWith(github.ref, 'refs/heads/release/') ||
|
|
179
|
+
startsWith(github.ref, 'refs/tags/v') ||
|
|
180
|
+
github.event_name == 'release' ||
|
|
181
|
+
github.event_name == 'workflow_dispatch')
|
|
182
|
+
runs-on: ubuntu-latest
|
|
183
|
+
steps:
|
|
184
|
+
- name: Checkout repository
|
|
185
|
+
uses: actions/checkout@v6
|
|
186
|
+
with:
|
|
187
|
+
ref: ${{ github.ref }}
|
|
188
|
+
fetch-depth: 0
|
|
189
|
+
|
|
190
|
+
- name: Set up Docker Buildx
|
|
191
|
+
uses: docker/setup-buildx-action@v4
|
|
192
|
+
|
|
193
|
+
- name: Build and load Docker image
|
|
194
|
+
uses: docker/build-push-action@v7
|
|
195
|
+
with:
|
|
196
|
+
context: .
|
|
197
|
+
push: false
|
|
198
|
+
load: true
|
|
199
|
+
platforms: linux/amd64
|
|
200
|
+
tags: "${{ github.repository_owner }}/${{ vars.DOCKER_IMAGE_NAME }}:${{ github.sha }}"
|
|
201
|
+
cache-from: type=gha,scope=docker-build
|
|
202
|
+
cache-to: type=gha,mode=max,scope=docker-build
|
|
203
|
+
|
|
204
|
+
- name: Smoke test Docker image
|
|
205
|
+
run: |
|
|
206
|
+
# Test CLI help
|
|
207
|
+
docker run --rm ${{ github.repository_owner }}/${{ vars.DOCKER_IMAGE_NAME }}:${{ github.sha }} --help
|
|
208
|
+
# Test library import
|
|
209
|
+
docker run --rm --entrypoint python ${{ github.repository_owner }}/${{ vars.DOCKER_IMAGE_NAME }}:${{ github.sha }} -c "import pystormtracker as pst; print('Import success')"
|
|
210
|
+
|
|
211
|
+
- name: Run Trivy vulnerability scanner
|
|
212
|
+
uses: aquasecurity/trivy-action@0.35.0
|
|
213
|
+
with:
|
|
214
|
+
image-ref: "${{ github.repository_owner }}/${{ vars.DOCKER_IMAGE_NAME }}:${{ github.sha }}"
|
|
215
|
+
format: "table"
|
|
216
|
+
exit-code: "0"
|
|
217
|
+
ignore-unfixed: true
|
|
218
|
+
vuln-type: "os,library"
|
|
219
|
+
severity: "CRITICAL,HIGH"
|
|
220
|
+
|
|
221
|
+
pypi-build:
|
|
222
|
+
name: pypi-build
|
|
223
|
+
needs: [ruff-lint, ruff-format, mypy-typecheck, unit-tests, integration-tests]
|
|
224
|
+
# Only run on merges to main, release branches, tags, releases, or manual dispatch
|
|
225
|
+
if: |
|
|
226
|
+
github.event_name != 'pull_request' &&
|
|
227
|
+
(github.ref == 'refs/heads/main' ||
|
|
228
|
+
startsWith(github.ref, 'refs/heads/release/') ||
|
|
229
|
+
startsWith(github.ref, 'refs/tags/v') ||
|
|
230
|
+
github.event_name == 'release' ||
|
|
231
|
+
github.event_name == 'workflow_dispatch')
|
|
232
|
+
runs-on: ubuntu-latest
|
|
233
|
+
steps:
|
|
234
|
+
- uses: actions/checkout@v6
|
|
235
|
+
with:
|
|
236
|
+
ref: ${{ github.ref }}
|
|
237
|
+
fetch-depth: 0
|
|
238
|
+
- name: Set up uv
|
|
239
|
+
uses: astral-sh/setup-uv@v7
|
|
240
|
+
with:
|
|
241
|
+
enable-cache: true
|
|
242
|
+
- name: Build release distributions
|
|
243
|
+
run: uv build --wheel --sdist
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
name: Docker Publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_run:
|
|
5
|
+
workflows: ["CI"]
|
|
6
|
+
types: [completed]
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
11
|
+
cancel-in-progress: false
|
|
12
|
+
|
|
13
|
+
env:
|
|
14
|
+
# Publish to ORG on release, else to OWNER (personal) for merge to main/manual
|
|
15
|
+
DOCKER_HUB_REPO: docker.io/${{ (github.event_name == 'release' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'release')) && vars.DOCKER_ORG_NAME || github.repository_owner }}/${{ vars.DOCKER_IMAGE_NAME }}
|
|
16
|
+
GHCR_REPO: ghcr.io/${{ (github.event_name == 'release' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'release')) && vars.DOCKER_ORG_NAME || github.repository_owner }}/${{ vars.DOCKER_IMAGE_NAME }}
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
build-and-push:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
# Only run if CI succeeded (for workflow_run) or if it's a manual trigger.
|
|
22
|
+
# Added head_repository check for security in trusted context.
|
|
23
|
+
if: |
|
|
24
|
+
(github.event_name == 'workflow_run' &&
|
|
25
|
+
github.event.workflow_run.conclusion == 'success' &&
|
|
26
|
+
github.event.workflow_run.head_repository.full_name == github.repository &&
|
|
27
|
+
(github.event.workflow_run.head_branch == 'main' || github.event.workflow_run.event == 'release')) ||
|
|
28
|
+
github.event_name == 'workflow_dispatch'
|
|
29
|
+
permissions:
|
|
30
|
+
actions: read
|
|
31
|
+
contents: write
|
|
32
|
+
packages: write
|
|
33
|
+
id-token: write
|
|
34
|
+
attestations: write
|
|
35
|
+
artifact-metadata: write
|
|
36
|
+
steps:
|
|
37
|
+
- name: Checkout repository
|
|
38
|
+
uses: actions/checkout@v6
|
|
39
|
+
with:
|
|
40
|
+
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
|
|
41
|
+
fetch-depth: 0
|
|
42
|
+
|
|
43
|
+
- name: Set up QEMU
|
|
44
|
+
uses: docker/setup-qemu-action@v4
|
|
45
|
+
|
|
46
|
+
- name: Set up Docker Buildx
|
|
47
|
+
uses: docker/setup-buildx-action@v4
|
|
48
|
+
|
|
49
|
+
- name: Log in to Docker Hub
|
|
50
|
+
uses: docker/login-action@v4
|
|
51
|
+
with:
|
|
52
|
+
username: ${{ vars.DOCKER_HUB_USERNAME }}
|
|
53
|
+
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
|
|
54
|
+
|
|
55
|
+
- name: Log in to GHCR
|
|
56
|
+
uses: docker/login-action@v4
|
|
57
|
+
with:
|
|
58
|
+
registry: ghcr.io
|
|
59
|
+
username: ${{ github.repository_owner }}
|
|
60
|
+
password: ${{ secrets.GHCR_PAT }}
|
|
61
|
+
|
|
62
|
+
- name: Extract Docker metadata
|
|
63
|
+
id: meta
|
|
64
|
+
uses: docker/metadata-action@v6
|
|
65
|
+
with:
|
|
66
|
+
images: |
|
|
67
|
+
${{ env.DOCKER_HUB_REPO }}
|
|
68
|
+
${{ env.GHCR_REPO }}
|
|
69
|
+
# Fix: Tell the metadata action the real ref, otherwise it defaults to 'main'
|
|
70
|
+
ref: ${{ github.event.workflow_run.head_branch || github.ref }}
|
|
71
|
+
tags: |
|
|
72
|
+
# Tag with 'edge' only for main branch builds
|
|
73
|
+
type=edge,branch=main,priority=700
|
|
74
|
+
# Semver tags for releases (includes 'latest')
|
|
75
|
+
type=semver,pattern=latest,priority=1000
|
|
76
|
+
type=semver,pattern={{version}},priority=900
|
|
77
|
+
type=semver,pattern={{major}}.{{minor}},priority=900
|
|
78
|
+
type=semver,pattern={{major}},enable=${{ !startsWith(github.ref_name, 'v0') }},priority=900
|
|
79
|
+
# Branch tag for all branches except main
|
|
80
|
+
type=ref,event=branch,enable=${{ github.ref_name != 'main' }},priority=600
|
|
81
|
+
# Always tag with short SHA
|
|
82
|
+
type=sha,format=short,prefix=,priority=100
|
|
83
|
+
|
|
84
|
+
- name: Build and push Docker image
|
|
85
|
+
id: push
|
|
86
|
+
uses: docker/build-push-action@v7
|
|
87
|
+
with:
|
|
88
|
+
context: .
|
|
89
|
+
push: true
|
|
90
|
+
provenance: false
|
|
91
|
+
sbom: false
|
|
92
|
+
platforms: linux/amd64,linux/arm64
|
|
93
|
+
tags: ${{ steps.meta.outputs.tags }}
|
|
94
|
+
labels: ${{ steps.meta.outputs.labels }}
|
|
95
|
+
cache-from: type=gha,scope=docker-build
|
|
96
|
+
cache-to: type=gha,mode=max,scope=docker-build
|
|
97
|
+
|
|
98
|
+
- name: Attest Provenance (Docker Hub)
|
|
99
|
+
uses: actions/attest@v4
|
|
100
|
+
with:
|
|
101
|
+
subject-name: ${{ env.DOCKER_HUB_REPO }}
|
|
102
|
+
subject-digest: ${{ steps.push.outputs.digest }}
|
|
103
|
+
push-to-registry: true
|
|
104
|
+
|
|
105
|
+
- name: Generate SBOM
|
|
106
|
+
uses: anchore/sbom-action@v0
|
|
107
|
+
with:
|
|
108
|
+
path: ./
|
|
109
|
+
artifact-name: sbom.cyclonedx.json
|
|
110
|
+
output-file: sbom.cyclonedx.json
|
|
111
|
+
format: cyclonedx-json
|
|
112
|
+
|
|
113
|
+
- name: Attest SBOM (Docker Hub)
|
|
114
|
+
uses: actions/attest@v4
|
|
115
|
+
with:
|
|
116
|
+
subject-name: ${{ env.DOCKER_HUB_REPO }}
|
|
117
|
+
subject-digest: ${{ steps.push.outputs.digest }}
|
|
118
|
+
sbom-path: 'sbom.cyclonedx.json'
|
|
119
|
+
push-to-registry: true
|
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
name: Upload Python Package
|
|
5
5
|
|
|
6
6
|
on:
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
workflow_run:
|
|
8
|
+
workflows: ["CI"]
|
|
9
|
+
types: [completed]
|
|
9
10
|
|
|
10
11
|
concurrency:
|
|
11
12
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
@@ -17,6 +18,13 @@ permissions:
|
|
|
17
18
|
jobs:
|
|
18
19
|
release-build:
|
|
19
20
|
runs-on: ubuntu-latest
|
|
21
|
+
# Only run if CI succeeded AND it was a release event.
|
|
22
|
+
# Added head_repository check for security.
|
|
23
|
+
if: |
|
|
24
|
+
github.event_name == 'workflow_run' &&
|
|
25
|
+
github.event.workflow_run.conclusion == 'success' &&
|
|
26
|
+
github.event.workflow_run.event == 'release' &&
|
|
27
|
+
github.event.workflow_run.head_repository.full_name == github.repository
|
|
20
28
|
permissions:
|
|
21
29
|
contents: read
|
|
22
30
|
id-token: write
|
|
@@ -25,6 +33,7 @@ jobs:
|
|
|
25
33
|
steps:
|
|
26
34
|
- uses: actions/checkout@v6
|
|
27
35
|
with:
|
|
36
|
+
ref: ${{ github.event.workflow_run.head_sha }}
|
|
28
37
|
fetch-depth: 0
|
|
29
38
|
|
|
30
39
|
- name: Set up uv
|
|
@@ -64,7 +73,8 @@ jobs:
|
|
|
64
73
|
id: get_version
|
|
65
74
|
run: |
|
|
66
75
|
# Strips 'v' prefix from tag_name (e.g. v0.2.1 -> 0.2.1)
|
|
67
|
-
|
|
76
|
+
# In workflow_run for a release, head_branch contains the tag name.
|
|
77
|
+
VERSION=${{ github.event.workflow_run.head_branch }}
|
|
68
78
|
echo "version=${VERSION#v}" >> $GITHUB_OUTPUT
|
|
69
79
|
|
|
70
80
|
- name: Retrieve release distributions
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.so
|
|
5
|
+
|
|
6
|
+
# Python Environments & Caches
|
|
7
|
+
.venv/
|
|
8
|
+
env/
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
.pytest_cache/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
.cache/
|
|
13
|
+
|
|
14
|
+
# Distribution / packaging
|
|
15
|
+
build/
|
|
16
|
+
dist/
|
|
17
|
+
sdist/
|
|
18
|
+
|
|
19
|
+
# Unit test / coverage reports
|
|
20
|
+
htmlcov/
|
|
21
|
+
.tox/
|
|
22
|
+
.coverage
|
|
23
|
+
.coverage.*
|
|
24
|
+
coverage.xml
|
|
25
|
+
|
|
26
|
+
# Sphinx documentation
|
|
27
|
+
docs/_build/
|
|
28
|
+
|
|
29
|
+
# IPython intermediate checkpoints
|
|
30
|
+
.ipynb_checkpoints
|
|
31
|
+
|
|
32
|
+
# Data and Track files
|
|
33
|
+
*.nc
|
|
34
|
+
*.txt
|
|
35
|
+
!data/test/tracks/*.txt
|
|
36
|
+
*.pickle
|
|
37
|
+
|
|
38
|
+
# IDE and Project Tooling
|
|
39
|
+
.vscode/
|
|
40
|
+
worktrees/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# PyStormTracker Architecture
|
|
2
|
+
|
|
3
|
+
This document describes the modern, high-performance architecture of PyStormTracker, detailing how it leverages vectorization and decoupled components to process massive climate datasets efficiently.
|
|
4
|
+
|
|
5
|
+
## 1. High-Level Design Philosophy
|
|
6
|
+
|
|
7
|
+
PyStormTracker is built for scale and extensibility. The architecture is centered around three core principles:
|
|
8
|
+
1. **Unified API (Tracker Protocol):** A structural interface that allows the CLI and Python API to support multiple tracking algorithms (e.g., `SimpleTracker`, `HodgesTracker`) interchangeably.
|
|
9
|
+
2. **Centralized Threshold Management:** The `SimpleDetector` is responsible for managing variable-specific detection thresholds (e.g., `1e-4` for vorticity), ensuring consistent behavior across different parallel backends.
|
|
10
|
+
3. **Vectorization & JIT:** Heavy mathematical operations are offloaded to **Numba** JIT-compiled kernels and **NumPy** broadcasting, bypassing Python's loop overhead and Global Interpreter Lock (GIL).
|
|
11
|
+
4. **Hybrid Parallelism:** The architecture parallelizes the computationally intensive **Detection** phase while centralizing the **Linking** phase to ensure perfect serial-parallel consistency.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## 2. Modern Core Components
|
|
16
|
+
|
|
17
|
+
### 2.1 Array-Backed Data Models (`Tracks`, `Track`, `Center`)
|
|
18
|
+
The data models utilize a contiguous memory paradigm:
|
|
19
|
+
* **`Tracks`**: The central container holding contiguous 1D NumPy arrays for `track_ids`, `times`, `lats`, `lons`, and a dictionary of scientific variables.
|
|
20
|
+
* **`Track`**: A lightweight "view" into the `Tracks` arrays for a specific ID.
|
|
21
|
+
* **`Center`**: A simple dataclass used strictly for iteration or final data export.
|
|
22
|
+
|
|
23
|
+
**Benefits:** By avoiding the creation of millions of Python objects, memory usage is minimized, and data serialization between parallel processes is nearly instantaneous. Raw NumPy arrays also enable extremely fast distance calculations via C-level broadcasting.
|
|
24
|
+
|
|
25
|
+
### 2.2 Shared DataLoader
|
|
26
|
+
Data loading is encapsulated in a dedicated `DataLoader` class (`io/loader.py`). This component handles:
|
|
27
|
+
* **Format Abstraction**: Seamlessly detects and opens NetCDF (via `h5netcdf` or `netcdf4`) and GRIB (via `cfgrib`) files.
|
|
28
|
+
* **Variable Mapping**: Automatically maps common variable aliases (e.g., `msl`/`slp`, `vo`/`rv`) and coordinate names (`latitude`/`lat`), allowing the same tracking logic to work across different data providers.
|
|
29
|
+
* **Contiguous I/O**: Performs single-block contiguous reads from disk, bypassing HDF5 lock contention.
|
|
30
|
+
|
|
31
|
+
### 2.3 Vectorized Linker (`SimpleLinker`)
|
|
32
|
+
Trajectory construction uses NumPy broadcasting to calculate Haversine distance matrices between existing track tails and new storm centers. By sorting points spatially before matching, the Linker ensures deterministic, greedy nearest-neighbor linking.
|
|
33
|
+
|
|
34
|
+
### 2.4 Parallel Pipeline (Gather-then-Link)
|
|
35
|
+
To ensure that parallel results are bit-wise identical to serial runs, PyStormTracker uses a hybrid parallel strategy:
|
|
36
|
+
1. **Parallel Detection**: Assigned time chunks are distributed across Dask or MPI workers. Each worker runs Numba kernels to find centers and returns raw coordinate arrays.
|
|
37
|
+
2. **Centralized Linking**: The main process gathers the raw detections from all workers and performs a single sequential link.
|
|
38
|
+
|
|
39
|
+
**Why this works:** In storm tracking, the **Detection** phase (finding local extrema in 3D grids) consumes >95% of the runtime. The **Linking** phase (connecting coordinate lists) is extremely fast once vectorized. Centralizing the link eliminates the complex "merging" bugs found in tree-reduction strategies while maintaining near-perfect parallel scaling.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## 3. The `Tracker` Protocol
|
|
44
|
+
|
|
45
|
+
The `Tracker` Protocol (defined in `src/pystormtracker/models/tracker.py`) provides a standardized interface for all tracking algorithms:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
import pystormtracker as pst
|
|
49
|
+
|
|
50
|
+
# Instantiate any compliant tracker
|
|
51
|
+
tracker = pst.SimpleTracker()
|
|
52
|
+
|
|
53
|
+
# Standardized .track() method
|
|
54
|
+
tracks = tracker.track(
|
|
55
|
+
infile="era5_msl.nc",
|
|
56
|
+
varname="msl",
|
|
57
|
+
start_time="2025-01-01",
|
|
58
|
+
backend="dask"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# Standardized export
|
|
62
|
+
tracks.write("output.txt", format="imilast")
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## 4. Future Architectural Direction
|
|
68
|
+
|
|
69
|
+
To further optimize scalability and memory efficiency for native-resolution climate datasets (e.g., 0.25° ERA5), the architecture is evolving towards deeper integration with the scientific Python ecosystem:
|
|
70
|
+
|
|
71
|
+
* **Idiomatic Xarray (`apply_ufunc`):** Transitioning away from custom MPI/Dask chunking in favor of Xarray's native `apply_ufunc(..., dask="parallelized")`. This delegates chunk management and distributed execution entirely to Xarray/Dask, reducing custom orchestration code.
|
|
72
|
+
* **Lazy Evaluation & Thread Topology:** Shifting from eager chunk-loading to lazy, frame-by-frame memory access to eliminate out-of-memory risks on large domains. Concurrently, strictly pinning Numba thread topologies to prevent CPU oversubscription in multi-process backends.
|
|
73
|
+
* **Tree-based Linking:** Upgrading the current NumPy-broadcasting linker to utilize C-level tree structures (e.g., `scipy.spatial.cKDTree`), breaking the $O(N^2)$ scaling barrier for extremely long or dense trajectory sequences.
|
|
74
|
+
|
|
75
|
+
For more details on specific planned implementations, see the [Roadmap](ROADMAP.md).
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## 5. Performance Benchmarks
|
|
80
|
+
|
|
81
|
+
To quantify the efficiency gains of the modern array-backed JIT architecture, a comprehensive performance comparison was conducted between the legacy object-oriented system (`v0.3.3`) and the current implementation.
|
|
82
|
+
|
|
83
|
+
Detailed execution timings (breaking down Detection, Linking, Export, and I/O Overhead) across Serial, Dask, and MPI backends for both standard and high-resolution ERA5 datasets are available in the [Benchmark Report](benchmark/BENCHMARK.md).
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## Appendix: Evolution from Legacy Architecture
|
|
88
|
+
|
|
89
|
+
The current architecture represents a fundamental shift from the legacy nested-object design used in earlier versions.
|
|
90
|
+
|
|
91
|
+
| Feature | Legacy Architecture (v0.3.x and earlier) | Modern Architecture (v0.4.0+) |
|
|
92
|
+
| :--- | :--- | :--- |
|
|
93
|
+
| **Data Storage** | Nested lists of `Center` and `Track` objects. | Flat, C-contiguous NumPy arrays. |
|
|
94
|
+
| **Parallelism** | Threads (bottlenecked by GIL). | Processes/MPI (true concurrent I/O). |
|
|
95
|
+
| **Linking Strategy** | Tree-reduction (prone to boundary splits). | Parallel Detect + Centralized Link (perfect matching). |
|
|
96
|
+
| **Linker** | $O(N^2)$ nested Python loops. | Vectorized NumPy matrix broadcasting. |
|
|
97
|
+
| **I/O** | Many small lazy-loaded chunks. | Contiguous shared `DataLoader`. |
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
cff-version: 1.2.0
|
|
2
|
-
title: PyStormTracker
|
|
2
|
+
title: mwyau/PyStormTracker
|
|
3
3
|
message: "If you use this software, please cite it as below."
|
|
4
4
|
type: software
|
|
5
5
|
authors:
|
|
@@ -12,15 +12,15 @@ identifiers:
|
|
|
12
12
|
value: 10.5281/zenodo.18764813
|
|
13
13
|
repository-code: 'https://github.com/mwyau/PyStormTracker'
|
|
14
14
|
url: 'https://pystormtracker.readthedocs.io/'
|
|
15
|
-
abstract: A
|
|
15
|
+
abstract: A High-Performance Cyclone Tracker in Python
|
|
16
16
|
keywords:
|
|
17
17
|
- cyclone tracking
|
|
18
18
|
- climate variability
|
|
19
19
|
- dask
|
|
20
20
|
- mpi
|
|
21
21
|
license: BSD-3-Clause
|
|
22
|
-
version: 0.
|
|
23
|
-
date-released: '2026-03-
|
|
22
|
+
version: 0.4.0
|
|
23
|
+
date-released: '2026-03-10'
|
|
24
24
|
preferred-citation:
|
|
25
25
|
type: article
|
|
26
26
|
authors:
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# --- Build Stage ---
|
|
2
|
-
FROM python:3.
|
|
2
|
+
FROM python:3.13-slim AS builder
|
|
3
3
|
|
|
4
4
|
# Prevent uv from creating a virtualenv that might be hard to move
|
|
5
5
|
ENV UV_COMPILE_BYTECODE=1 \
|
|
@@ -8,43 +8,31 @@ ENV UV_COMPILE_BYTECODE=1 \
|
|
|
8
8
|
|
|
9
9
|
WORKDIR /app
|
|
10
10
|
|
|
11
|
-
# Install build dependencies
|
|
12
|
-
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
13
|
-
libeccodes-dev \
|
|
14
|
-
gcc \
|
|
15
|
-
libc6-dev \
|
|
16
|
-
&& rm -rf /var/lib/apt/lists/*
|
|
17
|
-
|
|
18
11
|
# Install uv
|
|
19
12
|
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
|
|
20
13
|
|
|
21
14
|
# 1. Copy only dependency files for better layer caching.
|
|
22
|
-
# This ensures that we only re-download dependencies if these files change.
|
|
23
15
|
COPY pyproject.toml uv.lock ./
|
|
24
16
|
|
|
25
17
|
# 2. Install third-party dependencies first (including grib extra).
|
|
26
|
-
#
|
|
27
|
-
RUN uv
|
|
18
|
+
# Use cache mount for uv to persist downloads and build artifacts.
|
|
19
|
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
20
|
+
uv sync --frozen --no-dev --no-install-workspace --extra grib --no-editable
|
|
28
21
|
|
|
29
22
|
# 3. Copy only necessary source files for the final installation step.
|
|
30
23
|
COPY src/ ./src/
|
|
31
24
|
COPY README.md ./
|
|
32
25
|
|
|
33
26
|
# 4. Final installation of the project package itself.
|
|
34
|
-
|
|
35
|
-
|
|
27
|
+
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
28
|
+
uv sync --frozen --no-dev --extra grib --no-editable
|
|
36
29
|
|
|
37
30
|
|
|
38
31
|
# --- Runtime Stage ---
|
|
39
|
-
FROM python:3.
|
|
32
|
+
FROM python:3.13-slim
|
|
40
33
|
|
|
41
34
|
WORKDIR /app
|
|
42
35
|
|
|
43
|
-
# Install ONLY the runtime shared libraries
|
|
44
|
-
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
45
|
-
libeccodes0 \
|
|
46
|
-
&& rm -rf /var/lib/apt/lists/*
|
|
47
|
-
|
|
48
36
|
# Create data directory for mounting
|
|
49
37
|
RUN mkdir /data && chmod 777 /data
|
|
50
38
|
|