pystilt 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pystilt-0.1.0a1/LICENSE +21 -0
- pystilt-0.1.0a1/PKG-INFO +350 -0
- pystilt-0.1.0a1/README.md +298 -0
- pystilt-0.1.0a1/pyproject.toml +157 -0
- pystilt-0.1.0a1/setup.cfg +4 -0
- pystilt-0.1.0a1/src/pystilt.egg-info/PKG-INFO +350 -0
- pystilt-0.1.0a1/src/pystilt.egg-info/SOURCES.txt +116 -0
- pystilt-0.1.0a1/src/pystilt.egg-info/dependency_links.txt +1 -0
- pystilt-0.1.0a1/src/pystilt.egg-info/entry_points.txt +2 -0
- pystilt-0.1.0a1/src/pystilt.egg-info/requires.txt +31 -0
- pystilt-0.1.0a1/src/pystilt.egg-info/top_level.txt +1 -0
- pystilt-0.1.0a1/src/stilt/__init__.py +64 -0
- pystilt-0.1.0a1/src/stilt/__main__.py +12 -0
- pystilt-0.1.0a1/src/stilt/cli.py +629 -0
- pystilt-0.1.0a1/src/stilt/collections.py +583 -0
- pystilt-0.1.0a1/src/stilt/config/__init__.py +50 -0
- pystilt-0.1.0a1/src/stilt/config/fields.py +60 -0
- pystilt-0.1.0a1/src/stilt/config/footprint.py +61 -0
- pystilt-0.1.0a1/src/stilt/config/meteorology.py +119 -0
- pystilt-0.1.0a1/src/stilt/config/model.py +242 -0
- pystilt-0.1.0a1/src/stilt/config/params.py +483 -0
- pystilt-0.1.0a1/src/stilt/config/runtime.py +70 -0
- pystilt-0.1.0a1/src/stilt/config/spatial.py +67 -0
- pystilt-0.1.0a1/src/stilt/config/transforms.py +83 -0
- pystilt-0.1.0a1/src/stilt/errors.py +107 -0
- pystilt-0.1.0a1/src/stilt/execution/__init__.py +40 -0
- pystilt-0.1.0a1/src/stilt/execution/backends/__init__.py +20 -0
- pystilt-0.1.0a1/src/stilt/execution/backends/factory.py +36 -0
- pystilt-0.1.0a1/src/stilt/execution/backends/kubernetes.py +150 -0
- pystilt-0.1.0a1/src/stilt/execution/backends/local.py +138 -0
- pystilt-0.1.0a1/src/stilt/execution/backends/protocol.py +80 -0
- pystilt-0.1.0a1/src/stilt/execution/backends/slurm.py +305 -0
- pystilt-0.1.0a1/src/stilt/execution/entrypoints.py +86 -0
- pystilt-0.1.0a1/src/stilt/execution/execute.py +229 -0
- pystilt-0.1.0a1/src/stilt/execution/phases.py +313 -0
- pystilt-0.1.0a1/src/stilt/execution/tasks.py +145 -0
- pystilt-0.1.0a1/src/stilt/footprint.py +1114 -0
- pystilt-0.1.0a1/src/stilt/hysplit/__init__.py +5 -0
- pystilt-0.1.0a1/src/stilt/hysplit/bin/linux_x64/hycs_std +0 -0
- pystilt-0.1.0a1/src/stilt/hysplit/bin/macos_x64/hycs_std +0 -0
- pystilt-0.1.0a1/src/stilt/hysplit/bin/version +1 -0
- pystilt-0.1.0a1/src/stilt/hysplit/control.py +169 -0
- pystilt-0.1.0a1/src/stilt/hysplit/data/ASCDATA.CFG +6 -0
- pystilt-0.1.0a1/src/stilt/hysplit/data/LANDUSE.ASC +180 -0
- pystilt-0.1.0a1/src/stilt/hysplit/data/ROUGLEN.ASC +180 -0
- pystilt-0.1.0a1/src/stilt/hysplit/data/TERRAIN.ASC +180 -0
- pystilt-0.1.0a1/src/stilt/hysplit/driver.py +366 -0
- pystilt-0.1.0a1/src/stilt/hysplit/namelist.py +85 -0
- pystilt-0.1.0a1/src/stilt/index/__init__.py +15 -0
- pystilt-0.1.0a1/src/stilt/index/base.py +369 -0
- pystilt-0.1.0a1/src/stilt/index/factory.py +44 -0
- pystilt-0.1.0a1/src/stilt/index/postgres.py +197 -0
- pystilt-0.1.0a1/src/stilt/index/protocol.py +182 -0
- pystilt-0.1.0a1/src/stilt/index/rebuild.py +136 -0
- pystilt-0.1.0a1/src/stilt/index/sql.py +311 -0
- pystilt-0.1.0a1/src/stilt/index/sqlite.py +202 -0
- pystilt-0.1.0a1/src/stilt/index/updates.py +79 -0
- pystilt-0.1.0a1/src/stilt/meteorology.py +344 -0
- pystilt-0.1.0a1/src/stilt/model.py +503 -0
- pystilt-0.1.0a1/src/stilt/observations/__init__.py +79 -0
- pystilt-0.1.0a1/src/stilt/observations/apply.py +90 -0
- pystilt-0.1.0a1/src/stilt/observations/chemistry.py +149 -0
- pystilt-0.1.0a1/src/stilt/observations/geometry.py +86 -0
- pystilt-0.1.0a1/src/stilt/observations/observation.py +54 -0
- pystilt-0.1.0a1/src/stilt/observations/operators.py +27 -0
- pystilt-0.1.0a1/src/stilt/observations/receptors.py +252 -0
- pystilt-0.1.0a1/src/stilt/observations/scenes.py +166 -0
- pystilt-0.1.0a1/src/stilt/observations/selection.py +399 -0
- pystilt-0.1.0a1/src/stilt/observations/sensors/__init__.py +7 -0
- pystilt-0.1.0a1/src/stilt/observations/sensors/base.py +96 -0
- pystilt-0.1.0a1/src/stilt/observations/sensors/column.py +56 -0
- pystilt-0.1.0a1/src/stilt/observations/sensors/point.py +38 -0
- pystilt-0.1.0a1/src/stilt/observations/uncertainty.py +75 -0
- pystilt-0.1.0a1/src/stilt/observations/weighting.py +170 -0
- pystilt-0.1.0a1/src/stilt/py.typed +2 -0
- pystilt-0.1.0a1/src/stilt/receptors.py +597 -0
- pystilt-0.1.0a1/src/stilt/selection.py +189 -0
- pystilt-0.1.0a1/src/stilt/service/__init__.py +5 -0
- pystilt-0.1.0a1/src/stilt/service/kubernetes.py +313 -0
- pystilt-0.1.0a1/src/stilt/simulation.py +512 -0
- pystilt-0.1.0a1/src/stilt/storage/__init__.py +58 -0
- pystilt-0.1.0a1/src/stilt/storage/files.py +196 -0
- pystilt-0.1.0a1/src/stilt/storage/layout.py +108 -0
- pystilt-0.1.0a1/src/stilt/storage/project.py +150 -0
- pystilt-0.1.0a1/src/stilt/storage/store.py +326 -0
- pystilt-0.1.0a1/src/stilt/trajectory.py +343 -0
- pystilt-0.1.0a1/src/stilt/transforms.py +157 -0
- pystilt-0.1.0a1/src/stilt/visualization.py +669 -0
- pystilt-0.1.0a1/tests/test_apply_vertical_operator.py +145 -0
- pystilt-0.1.0a1/tests/test_chemistry.py +85 -0
- pystilt-0.1.0a1/tests/test_cli.py +1050 -0
- pystilt-0.1.0a1/tests/test_column_sensor.py +115 -0
- pystilt-0.1.0a1/tests/test_config.py +510 -0
- pystilt-0.1.0a1/tests/test_errors.py +108 -0
- pystilt-0.1.0a1/tests/test_footprint.py +1111 -0
- pystilt-0.1.0a1/tests/test_hysplit.py +612 -0
- pystilt-0.1.0a1/tests/test_hysplit_release_assignment.py +208 -0
- pystilt-0.1.0a1/tests/test_integration.py +325 -0
- pystilt-0.1.0a1/tests/test_meteorology.py +502 -0
- pystilt-0.1.0a1/tests/test_model.py +1958 -0
- pystilt-0.1.0a1/tests/test_observation_integration.py +49 -0
- pystilt-0.1.0a1/tests/test_observation_receptors.py +224 -0
- pystilt-0.1.0a1/tests/test_observation_scenes.py +105 -0
- pystilt-0.1.0a1/tests/test_observation_selection.py +190 -0
- pystilt-0.1.0a1/tests/test_observation_spatial_selection.py +144 -0
- pystilt-0.1.0a1/tests/test_observations.py +97 -0
- pystilt-0.1.0a1/tests/test_pkg.py +48 -0
- pystilt-0.1.0a1/tests/test_point_sensor.py +87 -0
- pystilt-0.1.0a1/tests/test_receptors.py +457 -0
- pystilt-0.1.0a1/tests/test_runtime.py +37 -0
- pystilt-0.1.0a1/tests/test_service_and_transforms.py +104 -0
- pystilt-0.1.0a1/tests/test_service_kubernetes.py +135 -0
- pystilt-0.1.0a1/tests/test_simulation.py +621 -0
- pystilt-0.1.0a1/tests/test_trajectory.py +386 -0
- pystilt-0.1.0a1/tests/test_uncertainty.py +89 -0
- pystilt-0.1.0a1/tests/test_utils.py +29 -0
- pystilt-0.1.0a1/tests/test_visualization.py +424 -0
- pystilt-0.1.0a1/tests/test_weighting.py +70 -0
pystilt-0.1.0a1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 James Mineau
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pystilt-0.1.0a1/PKG-INFO
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pystilt
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: A Python implementation of the STILT Lagrangian atmospheric transport model.
|
|
5
|
+
Author-email: James Mineau <jameskmineau@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/jmineau/PYSTILT
|
|
8
|
+
Project-URL: Documentation, https://jmineau.github.io/PYSTILT/
|
|
9
|
+
Project-URL: Repository, https://github.com/jmineau/PYSTILT
|
|
10
|
+
Project-URL: Issues, https://github.com/jmineau/PYSTILT/issues
|
|
11
|
+
Keywords: stilt,hysplit,atmospheric-transport,lagrangian,footprint,meteorology
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: arlmet>=0.1.0a1
|
|
25
|
+
Requires-Dist: fsspec>=2024.2
|
|
26
|
+
Requires-Dist: netcdf4>=1.6.5
|
|
27
|
+
Requires-Dist: numpy>=1.24
|
|
28
|
+
Requires-Dist: pandas>=2.0
|
|
29
|
+
Requires-Dist: pyarrow>=14.0
|
|
30
|
+
Requires-Dist: pydantic>=2.0
|
|
31
|
+
Requires-Dist: pydantic-settings>=2.0
|
|
32
|
+
Requires-Dist: pyyaml>=6.0
|
|
33
|
+
Requires-Dist: scipy>=1.10
|
|
34
|
+
Requires-Dist: shapely>=2.0
|
|
35
|
+
Requires-Dist: typer>=0.9
|
|
36
|
+
Requires-Dist: typing-extensions>=4.0
|
|
37
|
+
Requires-Dist: xarray>=2023.6
|
|
38
|
+
Provides-Extra: projection
|
|
39
|
+
Requires-Dist: pyproj>=3.5; extra == "projection"
|
|
40
|
+
Provides-Extra: visualization
|
|
41
|
+
Requires-Dist: matplotlib>=3.7; extra == "visualization"
|
|
42
|
+
Provides-Extra: cloud
|
|
43
|
+
Requires-Dist: gcsfs>=2024.2; extra == "cloud"
|
|
44
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == "cloud"
|
|
45
|
+
Requires-Dist: kubernetes>=28.0; extra == "cloud"
|
|
46
|
+
Requires-Dist: s3fs>=2024.2; extra == "cloud"
|
|
47
|
+
Provides-Extra: complete
|
|
48
|
+
Requires-Dist: pystilt[projection]; extra == "complete"
|
|
49
|
+
Requires-Dist: pystilt[visualization]; extra == "complete"
|
|
50
|
+
Requires-Dist: pystilt[cloud]; extra == "complete"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
# PYSTILT
|
|
54
|
+
|
|
55
|
+
[](https://github.com/jmineau/PYSTILT/actions/workflows/tests.yml)
|
|
56
|
+
[](https://github.com/jmineau/PYSTILT/actions/workflows/docs.yml)
|
|
57
|
+
[](https://github.com/jmineau/PYSTILT/actions/workflows/quality.yml)
|
|
58
|
+
[](https://codecov.io/gh/jmineau/PYSTILT)
|
|
59
|
+
[](https://badge.fury.io/py/pystilt)
|
|
60
|
+
[](https://pypi.org/project/pystilt/)
|
|
61
|
+
[](https://opensource.org/licenses/MIT)
|
|
62
|
+
[](https://github.com/astral-sh/ruff)
|
|
63
|
+
[](https://github.com/microsoft/pyright)
|
|
64
|
+
|
|
65
|
+
PYSTILT is a Python implementation of the [STILT](https://uataq.github.io/stilt/) Lagrangian atmospheric transport model.
|
|
66
|
+
It runs backward trajectories with [HYSPLIT](https://www.ready.noaa.gov/HYSPLIT.php) and computes receptor footprints
|
|
67
|
+
that map where upwind surface fluxes influence a measurement.
|
|
68
|
+
|
|
69
|
+
The project is in alpha and focused on a unified execution model that works for one-off runs,
|
|
70
|
+
large batch runs, and streaming queue workers.
|
|
71
|
+
|
|
72
|
+
## Status
|
|
73
|
+
|
|
74
|
+
PYSTILT is alpha software (v0.1.0a1). No backward compatibility guarantees before v1.0.
|
|
75
|
+
|
|
76
|
+
The core transport is stable: HYSPLIT execution, trajectory and footprint generation,
|
|
77
|
+
numerical R-STILT parity, and the local and SLURM execution paths are all exercised by the
|
|
78
|
+
test suite. The public API may change while the package settles.
|
|
79
|
+
|
|
80
|
+
## Choose a workflow
|
|
81
|
+
|
|
82
|
+
- **One-off transport runs** for local analysis and notebooks:
|
|
83
|
+
use `Model.run()` or `stilt run`.
|
|
84
|
+
- **Queue-backed batch or service runs** for HPC/cloud execution:
|
|
85
|
+
use `Model.register_pending()`, `stilt register`, `stilt pull-worker`, and
|
|
86
|
+
`stilt serve` with a PostgreSQL-backed queue index configured via
|
|
87
|
+
`PYSTILT_DB_URL`.
|
|
88
|
+
- **Observation-driven workflows** for science-facing code:
|
|
89
|
+
use `stilt.observations` to turn normalized observations into `Receptor`
|
|
90
|
+
objects before feeding them into the same runtime.
|
|
91
|
+
|
|
92
|
+
## Roadmap
|
|
93
|
+
|
|
94
|
+
PYSTILT draws design and science inspiration from two sister projects:
|
|
95
|
+
[X-STILT](https://github.com/uataq/X-STILT) for column and satellite science workflows, and
|
|
96
|
+
[stiltctl](https://github.com/jmineau/air-tracker-stiltctl) for cloud-native execution
|
|
97
|
+
patterns. The tables below track what has been absorbed and what remains in scope.
|
|
98
|
+
See the full [roadmap](https://jmineau.github.io/PYSTILT/roadmap.html) for more details.
|
|
99
|
+
|
|
100
|
+
### Execution and orchestration (from stiltctl)
|
|
101
|
+
|
|
102
|
+
| Feature | Status |
|
|
103
|
+
|---|---|
|
|
104
|
+
| Pull-mode queue workers (`stilt pull-worker`) | Implemented |
|
|
105
|
+
| Long-lived streaming mode (`stilt serve`) | Implemented |
|
|
106
|
+
| PostgreSQL-backed simulation registry | Implemented |
|
|
107
|
+
| Scene-based submission grouping | Implemented |
|
|
108
|
+
| Thin CLI → Model → worker call path | Implemented |
|
|
109
|
+
| Kubernetes worker deployment | Partial |
|
|
110
|
+
| Cloud object store outputs (GCS, S3) | In scope |
|
|
111
|
+
|
|
112
|
+
### Column and satellite science (from X-STILT)
|
|
113
|
+
|
|
114
|
+
Full X-STILT feature parity is not a goal. PYSTILT absorbs X-STILT's observation-layer
|
|
115
|
+
design and column-weighting concepts without trying to replicate every script.
|
|
116
|
+
|
|
117
|
+
| Feature | Status |
|
|
118
|
+
|---|---|
|
|
119
|
+
| `stilt.observations` layer (`Observation`, `Scene`, sensor families) | Implemented |
|
|
120
|
+
| Column receptor support | Implemented |
|
|
121
|
+
| Vertical operator particle transforms (AK / pressure weighting) | Implemented |
|
|
122
|
+
| First-order lifetime decay transform | Implemented |
|
|
123
|
+
| Declarative per-footprint transforms in config | Implemented |
|
|
124
|
+
| Slant-column receptor support | In scope (pending HYSPLIT validation) |
|
|
125
|
+
| Additional transform types | In scope |
|
|
126
|
+
| Specific sensor adapters (OCO-2/3, TROPOMI, TCCON) | Deferred |
|
|
127
|
+
| Inventory coupling and background estimation | Deferred |
|
|
128
|
+
|
|
129
|
+
## Installation
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
pip install pystilt
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
For Slurm, Kubernetes, projections, plotting, and cloud object stores:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
pip install "pystilt[complete]"
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Quickstart: one-off run
|
|
142
|
+
|
|
143
|
+
Define a receptor, configure meteorology and footprint grid, then run:
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
import pandas as pd
|
|
147
|
+
import stilt
|
|
148
|
+
|
|
149
|
+
receptor = stilt.Receptor(
|
|
150
|
+
time=pd.Timestamp("2023-07-15 18:00", tz="UTC"),
|
|
151
|
+
latitude=40.766,
|
|
152
|
+
longitude=-111.848,
|
|
153
|
+
altitude=10,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
model = stilt.Model(
|
|
157
|
+
project="./my_project",
|
|
158
|
+
receptors=[receptor],
|
|
159
|
+
config=stilt.ModelConfig(
|
|
160
|
+
n_hours=-24,
|
|
161
|
+
numpar=100,
|
|
162
|
+
mets={
|
|
163
|
+
"hrrr": stilt.MetConfig(
|
|
164
|
+
directory="/data/hrrr",
|
|
165
|
+
file_format="hrrr_%Y%m%d.arl",
|
|
166
|
+
file_tres="1h",
|
|
167
|
+
)
|
|
168
|
+
},
|
|
169
|
+
footprints={
|
|
170
|
+
"default": stilt.FootprintConfig(
|
|
171
|
+
grid=stilt.Grid(
|
|
172
|
+
xmin=-113.0,
|
|
173
|
+
xmax=-110.5,
|
|
174
|
+
ymin=40.0,
|
|
175
|
+
ymax=42.0,
|
|
176
|
+
xres=0.01,
|
|
177
|
+
yres=0.01,
|
|
178
|
+
)
|
|
179
|
+
)
|
|
180
|
+
},
|
|
181
|
+
),
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
handle = model.run()
|
|
185
|
+
handle.wait()
|
|
186
|
+
|
|
187
|
+
sim = list(model.simulations.values())[0]
|
|
188
|
+
traj = sim.trajectories
|
|
189
|
+
foot = sim.get_footprint("default")
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Quickstart: queue/service runtime
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
# Queue workers require a PostgreSQL-backed queue index.
|
|
196
|
+
export PYSTILT_DB_URL=postgresql://user:pass@host:5432/pystilt
|
|
197
|
+
|
|
198
|
+
# Initialize project files (config.yaml and receptors.csv)
|
|
199
|
+
stilt init ./my_project
|
|
200
|
+
|
|
201
|
+
# Run with local workers (blocks until complete)
|
|
202
|
+
stilt run ./my_project --backend local --n-workers 8
|
|
203
|
+
|
|
204
|
+
# Register one grouped scene submission
|
|
205
|
+
stilt register ./my_project --scene-id daily_2026_04_13
|
|
206
|
+
|
|
207
|
+
# Drain queue from worker processes (batch mode)
|
|
208
|
+
stilt pull-worker ./my_project
|
|
209
|
+
|
|
210
|
+
# Long-lived queue workers (streaming mode)
|
|
211
|
+
stilt serve ./my_project
|
|
212
|
+
|
|
213
|
+
# Check project status
|
|
214
|
+
stilt status ./my_project
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
The same queue model is available in Python:
|
|
218
|
+
|
|
219
|
+
```python
|
|
220
|
+
import stilt
|
|
221
|
+
from stilt.execution import pull_simulations
|
|
222
|
+
|
|
223
|
+
model = stilt.Model(project="./my_project")
|
|
224
|
+
model.register_pending(scene_id="daily_2026_04_14")
|
|
225
|
+
pull_simulations(model, follow=False) # batch mode
|
|
226
|
+
print(model.status(scene_id="daily_2026_04_14"))
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
In all modes, workers claim simulations from the PostgreSQL-backed index and
|
|
230
|
+
write terminal state directly back to the same registry.
|
|
231
|
+
|
|
232
|
+
## Quickstart: observation layer
|
|
233
|
+
|
|
234
|
+
PYSTILT also includes a narrow science-facing layer in `stilt.observations`.
|
|
235
|
+
It is designed to sit above `Receptor`, not replace the transport/runtime core.
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
import stilt
|
|
239
|
+
from stilt.observations import PointSensor
|
|
240
|
+
|
|
241
|
+
sensor = PointSensor(name="tower", supported_species=("co2",))
|
|
242
|
+
observations = [
|
|
243
|
+
sensor.make_observation(
|
|
244
|
+
time="2023-01-01 12:00:00",
|
|
245
|
+
latitude=40.77,
|
|
246
|
+
longitude=-111.85,
|
|
247
|
+
altitude=30.0,
|
|
248
|
+
observation_id="tower-001",
|
|
249
|
+
)
|
|
250
|
+
]
|
|
251
|
+
|
|
252
|
+
[scene] = sensor.group_scenes(observations)
|
|
253
|
+
receptors = [sensor.build_receptor(obs) for obs in scene.observations]
|
|
254
|
+
|
|
255
|
+
model = stilt.Model(project="./my_project") # existing project config on disk
|
|
256
|
+
model.register_pending(receptors=receptors, scene_id=scene.id)
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
Direct `Observation(...)` construction is still available when you already
|
|
260
|
+
have a separate product-specific normalization layer. The sensor helper just
|
|
261
|
+
keeps the common path less repetitive.
|
|
262
|
+
|
|
263
|
+
This layer currently focuses on:
|
|
264
|
+
|
|
265
|
+
- normalized `Observation` and `Scene` objects
|
|
266
|
+
- geometry/operator metadata
|
|
267
|
+
- generic point/column sensor families
|
|
268
|
+
- observation-to-receptor conversion
|
|
269
|
+
|
|
270
|
+
See `docs/advanced/observations.rst` for the intended workflow boundary.
|
|
271
|
+
|
|
272
|
+
## Declarative transforms
|
|
273
|
+
|
|
274
|
+
Per-footprint transforms can be declared in config instead of embedded as
|
|
275
|
+
ad hoc callbacks.
|
|
276
|
+
|
|
277
|
+
```yaml
|
|
278
|
+
footprints:
|
|
279
|
+
column:
|
|
280
|
+
grid: slv
|
|
281
|
+
transforms:
|
|
282
|
+
- kind: vertical_operator
|
|
283
|
+
mode: ak_pwf
|
|
284
|
+
levels: [0.0, 1000.0, 2000.0]
|
|
285
|
+
values: [0.2, 0.5, 0.3]
|
|
286
|
+
coordinate: xhgt
|
|
287
|
+
- kind: first_order_lifetime
|
|
288
|
+
lifetime_hours: 4.0
|
|
289
|
+
time_column: time
|
|
290
|
+
time_unit: min
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
The built-in transform interface is intentionally small:
|
|
294
|
+
|
|
295
|
+
- vertical operator weighting
|
|
296
|
+
- first-order lifetime decay
|
|
297
|
+
- runtime typed transforms for more advanced Python workflows
|
|
298
|
+
|
|
299
|
+
## Accessing results
|
|
300
|
+
|
|
301
|
+
```python
|
|
302
|
+
import pandas as pd
|
|
303
|
+
|
|
304
|
+
for sim in model.simulations.values():
|
|
305
|
+
traj = sim.trajectories
|
|
306
|
+
foot = sim.get_footprint("default")
|
|
307
|
+
|
|
308
|
+
# Load footprints across all matching simulations
|
|
309
|
+
footprints = model.footprints["default"].load(
|
|
310
|
+
time_range=("2023-01-01", "2023-01-31")
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
coords = [(-111.9, 40.7), (-111.8, 40.8)]
|
|
314
|
+
time_bins = pd.interval_range(
|
|
315
|
+
start=pd.Timestamp("2023-01-01 00:00", tz="UTC"),
|
|
316
|
+
end=pd.Timestamp("2023-01-02 00:00", tz="UTC"),
|
|
317
|
+
freq="1h",
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
for footprint in footprints:
|
|
321
|
+
hourly = footprint.aggregate(coords=coords, time_bins=time_bins)
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
If a footprint is tracked as `complete-empty`, no NetCDF file is expected for that footprint.
|
|
325
|
+
The model APIs treat it as a successful terminal outcome while skipping missing file loads.
|
|
326
|
+
|
|
327
|
+
## R-STILT parity
|
|
328
|
+
|
|
329
|
+
PYSTILT footprints match the [uataq/stilt](https://github.com/uataq/stilt) R
|
|
330
|
+
implementation on **numerical values** at `rtol=1e-7` per cell, validated by
|
|
331
|
+
end-to-end fidelity scenarios against a **pinned upstream commit**.
|
|
332
|
+
NetCDF output is not byte-compatible with R-STILT;
|
|
333
|
+
it should be read as generic CF-1.8 NetCDF.
|
|
334
|
+
See [STILT-R.md](STILT-R.md) for more details.
|
|
335
|
+
|
|
336
|
+
## Documentation
|
|
337
|
+
|
|
338
|
+
Full documentation is available at [https://jmineau.github.io/PYSTILT/](https://jmineau.github.io/PYSTILT/)
|
|
339
|
+
|
|
340
|
+
## Contributing
|
|
341
|
+
|
|
342
|
+
Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
|
343
|
+
|
|
344
|
+
## License
|
|
345
|
+
|
|
346
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
347
|
+
|
|
348
|
+
## Author
|
|
349
|
+
|
|
350
|
+
**James Mineau** - [jmineau](https://github.com/jmineau)
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
# PYSTILT
|
|
2
|
+
|
|
3
|
+
[](https://github.com/jmineau/PYSTILT/actions/workflows/tests.yml)
|
|
4
|
+
[](https://github.com/jmineau/PYSTILT/actions/workflows/docs.yml)
|
|
5
|
+
[](https://github.com/jmineau/PYSTILT/actions/workflows/quality.yml)
|
|
6
|
+
[](https://codecov.io/gh/jmineau/PYSTILT)
|
|
7
|
+
[](https://badge.fury.io/py/pystilt)
|
|
8
|
+
[](https://pypi.org/project/pystilt/)
|
|
9
|
+
[](https://opensource.org/licenses/MIT)
|
|
10
|
+
[](https://github.com/astral-sh/ruff)
|
|
11
|
+
[](https://github.com/microsoft/pyright)
|
|
12
|
+
|
|
13
|
+
PYSTILT is a Python implementation of the [STILT](https://uataq.github.io/stilt/) Lagrangian atmospheric transport model.
|
|
14
|
+
It runs backward trajectories with [HYSPLIT](https://www.ready.noaa.gov/HYSPLIT.php) and computes receptor footprints
|
|
15
|
+
that map where upwind surface fluxes influence a measurement.
|
|
16
|
+
|
|
17
|
+
The project is in alpha and focused on a unified execution model that works for one-off runs,
|
|
18
|
+
large batch runs, and streaming queue workers.
|
|
19
|
+
|
|
20
|
+
## Status
|
|
21
|
+
|
|
22
|
+
PYSTILT is alpha software (v0.1.0a1). No backward compatibility guarantees before v1.0.
|
|
23
|
+
|
|
24
|
+
The core transport is stable: HYSPLIT execution, trajectory and footprint generation,
|
|
25
|
+
numerical R-STILT parity, and the local and SLURM execution paths are all exercised by the
|
|
26
|
+
test suite. The public API may change while the package settles.
|
|
27
|
+
|
|
28
|
+
## Choose a workflow
|
|
29
|
+
|
|
30
|
+
- **One-off transport runs** for local analysis and notebooks:
|
|
31
|
+
use `Model.run()` or `stilt run`.
|
|
32
|
+
- **Queue-backed batch or service runs** for HPC/cloud execution:
|
|
33
|
+
use `Model.register_pending()`, `stilt register`, `stilt pull-worker`, and
|
|
34
|
+
`stilt serve` with a PostgreSQL-backed queue index configured via
|
|
35
|
+
`PYSTILT_DB_URL`.
|
|
36
|
+
- **Observation-driven workflows** for science-facing code:
|
|
37
|
+
use `stilt.observations` to turn normalized observations into `Receptor`
|
|
38
|
+
objects before feeding them into the same runtime.
|
|
39
|
+
|
|
40
|
+
## Roadmap
|
|
41
|
+
|
|
42
|
+
PYSTILT draws design and science inspiration from two sister projects:
|
|
43
|
+
[X-STILT](https://github.com/uataq/X-STILT) for column and satellite science workflows, and
|
|
44
|
+
[stiltctl](https://github.com/jmineau/air-tracker-stiltctl) for cloud-native execution
|
|
45
|
+
patterns. The tables below track what has been absorbed and what remains in scope.
|
|
46
|
+
See the full [roadmap](https://jmineau.github.io/PYSTILT/roadmap.html) for more details.
|
|
47
|
+
|
|
48
|
+
### Execution and orchestration (from stiltctl)
|
|
49
|
+
|
|
50
|
+
| Feature | Status |
|
|
51
|
+
|---|---|
|
|
52
|
+
| Pull-mode queue workers (`stilt pull-worker`) | Implemented |
|
|
53
|
+
| Long-lived streaming mode (`stilt serve`) | Implemented |
|
|
54
|
+
| PostgreSQL-backed simulation registry | Implemented |
|
|
55
|
+
| Scene-based submission grouping | Implemented |
|
|
56
|
+
| Thin CLI → Model → worker call path | Implemented |
|
|
57
|
+
| Kubernetes worker deployment | Partial |
|
|
58
|
+
| Cloud object store outputs (GCS, S3) | In scope |
|
|
59
|
+
|
|
60
|
+
### Column and satellite science (from X-STILT)
|
|
61
|
+
|
|
62
|
+
Full X-STILT feature parity is not a goal. PYSTILT absorbs X-STILT's observation-layer
|
|
63
|
+
design and column-weighting concepts without trying to replicate every script.
|
|
64
|
+
|
|
65
|
+
| Feature | Status |
|
|
66
|
+
|---|---|
|
|
67
|
+
| `stilt.observations` layer (`Observation`, `Scene`, sensor families) | Implemented |
|
|
68
|
+
| Column receptor support | Implemented |
|
|
69
|
+
| Vertical operator particle transforms (AK / pressure weighting) | Implemented |
|
|
70
|
+
| First-order lifetime decay transform | Implemented |
|
|
71
|
+
| Declarative per-footprint transforms in config | Implemented |
|
|
72
|
+
| Slant-column receptor support | In scope (pending HYSPLIT validation) |
|
|
73
|
+
| Additional transform types | In scope |
|
|
74
|
+
| Specific sensor adapters (OCO-2/3, TROPOMI, TCCON) | Deferred |
|
|
75
|
+
| Inventory coupling and background estimation | Deferred |
|
|
76
|
+
|
|
77
|
+
## Installation
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pip install pystilt
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
For Slurm, Kubernetes, projections, plotting, and cloud object stores:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install "pystilt[complete]"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Quickstart: one-off run
|
|
90
|
+
|
|
91
|
+
Define a receptor, configure meteorology and footprint grid, then run:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
import pandas as pd
|
|
95
|
+
import stilt
|
|
96
|
+
|
|
97
|
+
receptor = stilt.Receptor(
|
|
98
|
+
time=pd.Timestamp("2023-07-15 18:00", tz="UTC"),
|
|
99
|
+
latitude=40.766,
|
|
100
|
+
longitude=-111.848,
|
|
101
|
+
altitude=10,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
model = stilt.Model(
|
|
105
|
+
project="./my_project",
|
|
106
|
+
receptors=[receptor],
|
|
107
|
+
config=stilt.ModelConfig(
|
|
108
|
+
n_hours=-24,
|
|
109
|
+
numpar=100,
|
|
110
|
+
mets={
|
|
111
|
+
"hrrr": stilt.MetConfig(
|
|
112
|
+
directory="/data/hrrr",
|
|
113
|
+
file_format="hrrr_%Y%m%d.arl",
|
|
114
|
+
file_tres="1h",
|
|
115
|
+
)
|
|
116
|
+
},
|
|
117
|
+
footprints={
|
|
118
|
+
"default": stilt.FootprintConfig(
|
|
119
|
+
grid=stilt.Grid(
|
|
120
|
+
xmin=-113.0,
|
|
121
|
+
xmax=-110.5,
|
|
122
|
+
ymin=40.0,
|
|
123
|
+
ymax=42.0,
|
|
124
|
+
xres=0.01,
|
|
125
|
+
yres=0.01,
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
},
|
|
129
|
+
),
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
handle = model.run()
|
|
133
|
+
handle.wait()
|
|
134
|
+
|
|
135
|
+
sim = list(model.simulations.values())[0]
|
|
136
|
+
traj = sim.trajectories
|
|
137
|
+
foot = sim.get_footprint("default")
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Quickstart: queue/service runtime
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
# Queue workers require a PostgreSQL-backed queue index.
|
|
144
|
+
export PYSTILT_DB_URL=postgresql://user:pass@host:5432/pystilt
|
|
145
|
+
|
|
146
|
+
# Initialize project files (config.yaml and receptors.csv)
|
|
147
|
+
stilt init ./my_project
|
|
148
|
+
|
|
149
|
+
# Run with local workers (blocks until complete)
|
|
150
|
+
stilt run ./my_project --backend local --n-workers 8
|
|
151
|
+
|
|
152
|
+
# Register one grouped scene submission
|
|
153
|
+
stilt register ./my_project --scene-id daily_2026_04_13
|
|
154
|
+
|
|
155
|
+
# Drain queue from worker processes (batch mode)
|
|
156
|
+
stilt pull-worker ./my_project
|
|
157
|
+
|
|
158
|
+
# Long-lived queue workers (streaming mode)
|
|
159
|
+
stilt serve ./my_project
|
|
160
|
+
|
|
161
|
+
# Check project status
|
|
162
|
+
stilt status ./my_project
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
The same queue model is available in Python:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
import stilt
|
|
169
|
+
from stilt.execution import pull_simulations
|
|
170
|
+
|
|
171
|
+
model = stilt.Model(project="./my_project")
|
|
172
|
+
model.register_pending(scene_id="daily_2026_04_14")
|
|
173
|
+
pull_simulations(model, follow=False) # batch mode
|
|
174
|
+
print(model.status(scene_id="daily_2026_04_14"))
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
In all modes, workers claim simulations from the PostgreSQL-backed index and
|
|
178
|
+
write terminal state directly back to the same registry.
|
|
179
|
+
|
|
180
|
+
## Quickstart: observation layer
|
|
181
|
+
|
|
182
|
+
PYSTILT also includes a narrow science-facing layer in `stilt.observations`.
|
|
183
|
+
It is designed to sit above `Receptor`, not replace the transport/runtime core.
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
import stilt
|
|
187
|
+
from stilt.observations import PointSensor
|
|
188
|
+
|
|
189
|
+
sensor = PointSensor(name="tower", supported_species=("co2",))
|
|
190
|
+
observations = [
|
|
191
|
+
sensor.make_observation(
|
|
192
|
+
time="2023-01-01 12:00:00",
|
|
193
|
+
latitude=40.77,
|
|
194
|
+
longitude=-111.85,
|
|
195
|
+
altitude=30.0,
|
|
196
|
+
observation_id="tower-001",
|
|
197
|
+
)
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
[scene] = sensor.group_scenes(observations)
|
|
201
|
+
receptors = [sensor.build_receptor(obs) for obs in scene.observations]
|
|
202
|
+
|
|
203
|
+
model = stilt.Model(project="./my_project") # existing project config on disk
|
|
204
|
+
model.register_pending(receptors=receptors, scene_id=scene.id)
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Direct `Observation(...)` construction is still available when you already
|
|
208
|
+
have a separate product-specific normalization layer. The sensor helper just
|
|
209
|
+
keeps the common path less repetitive.
|
|
210
|
+
|
|
211
|
+
This layer currently focuses on:
|
|
212
|
+
|
|
213
|
+
- normalized `Observation` and `Scene` objects
|
|
214
|
+
- geometry/operator metadata
|
|
215
|
+
- generic point/column sensor families
|
|
216
|
+
- observation-to-receptor conversion
|
|
217
|
+
|
|
218
|
+
See `docs/advanced/observations.rst` for the intended workflow boundary.
|
|
219
|
+
|
|
220
|
+
## Declarative transforms
|
|
221
|
+
|
|
222
|
+
Per-footprint transforms can be declared in config instead of embedded as
|
|
223
|
+
ad hoc callbacks.
|
|
224
|
+
|
|
225
|
+
```yaml
|
|
226
|
+
footprints:
|
|
227
|
+
column:
|
|
228
|
+
grid: slv
|
|
229
|
+
transforms:
|
|
230
|
+
- kind: vertical_operator
|
|
231
|
+
mode: ak_pwf
|
|
232
|
+
levels: [0.0, 1000.0, 2000.0]
|
|
233
|
+
values: [0.2, 0.5, 0.3]
|
|
234
|
+
coordinate: xhgt
|
|
235
|
+
- kind: first_order_lifetime
|
|
236
|
+
lifetime_hours: 4.0
|
|
237
|
+
time_column: time
|
|
238
|
+
time_unit: min
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
The built-in transform interface is intentionally small:
|
|
242
|
+
|
|
243
|
+
- vertical operator weighting
|
|
244
|
+
- first-order lifetime decay
|
|
245
|
+
- runtime typed transforms for more advanced Python workflows
|
|
246
|
+
|
|
247
|
+
## Accessing results
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
import pandas as pd
|
|
251
|
+
|
|
252
|
+
for sim in model.simulations.values():
|
|
253
|
+
traj = sim.trajectories
|
|
254
|
+
foot = sim.get_footprint("default")
|
|
255
|
+
|
|
256
|
+
# Load footprints across all matching simulations
|
|
257
|
+
footprints = model.footprints["default"].load(
|
|
258
|
+
time_range=("2023-01-01", "2023-01-31")
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
coords = [(-111.9, 40.7), (-111.8, 40.8)]
|
|
262
|
+
time_bins = pd.interval_range(
|
|
263
|
+
start=pd.Timestamp("2023-01-01 00:00", tz="UTC"),
|
|
264
|
+
end=pd.Timestamp("2023-01-02 00:00", tz="UTC"),
|
|
265
|
+
freq="1h",
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
for footprint in footprints:
|
|
269
|
+
hourly = footprint.aggregate(coords=coords, time_bins=time_bins)
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
If a footprint is tracked as `complete-empty`, no NetCDF file is expected for that footprint.
|
|
273
|
+
The model APIs treat it as a successful terminal outcome while skipping missing file loads.
|
|
274
|
+
|
|
275
|
+
## R-STILT parity
|
|
276
|
+
|
|
277
|
+
PYSTILT footprints match the [uataq/stilt](https://github.com/uataq/stilt) R
|
|
278
|
+
implementation on **numerical values** at `rtol=1e-7` per cell, validated by
|
|
279
|
+
end-to-end fidelity scenarios against a **pinned upstream commit**.
|
|
280
|
+
NetCDF output is not byte-compatible with R-STILT;
|
|
281
|
+
it should be read as generic CF-1.8 NetCDF.
|
|
282
|
+
See [STILT-R.md](STILT-R.md) for more details.
|
|
283
|
+
|
|
284
|
+
## Documentation
|
|
285
|
+
|
|
286
|
+
Full documentation is available at [https://jmineau.github.io/PYSTILT/](https://jmineau.github.io/PYSTILT/)
|
|
287
|
+
|
|
288
|
+
## Contributing
|
|
289
|
+
|
|
290
|
+
Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
|
291
|
+
|
|
292
|
+
## License
|
|
293
|
+
|
|
294
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
295
|
+
|
|
296
|
+
## Author
|
|
297
|
+
|
|
298
|
+
**James Mineau** - [jmineau](https://github.com/jmineau)
|