gmpas 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gmpas-0.4.2/.github/workflows/tests.yml +68 -0
- gmpas-0.4.2/.gitignore +30 -0
- gmpas-0.4.2/LICENSE +21 -0
- gmpas-0.4.2/PKG-INFO +121 -0
- gmpas-0.4.2/README.md +88 -0
- gmpas-0.4.2/docs/REMAPPING.md +354 -0
- gmpas-0.4.2/docs/cluster.md +31 -0
- gmpas-0.4.2/docs/command-line.md +46 -0
- gmpas-0.4.2/docs/configuration.md +8 -0
- gmpas-0.4.2/docs/hfun-viewer.svg +51 -0
- gmpas-0.4.2/docs/installation.md +51 -0
- gmpas-0.4.2/docs/layout.md +10 -0
- gmpas-0.4.2/docs/mcp-server.md +53 -0
- gmpas-0.4.2/docs/mesh-generation.svg +51 -0
- gmpas-0.4.2/docs/mesh-viewer.svg +48 -0
- gmpas-0.4.2/docs/notebook.md +43 -0
- gmpas-0.4.2/docs/preprocessing.md +258 -0
- gmpas-0.4.2/docs/remapping-workflow.svg +64 -0
- gmpas-0.4.2/docs/status.md +44 -0
- gmpas-0.4.2/docs/testing.md +12 -0
- gmpas-0.4.2/docs/viewer.svg +50 -0
- gmpas-0.4.2/docs/why.md +33 -0
- gmpas-0.4.2/environment.yml +39 -0
- gmpas-0.4.2/examples/README.md +63 -0
- gmpas-0.4.2/examples/hfun_concentric.py +161 -0
- gmpas-0.4.2/examples/hfun_uniform.py +27 -0
- gmpas-0.4.2/pyproject.toml +67 -0
- gmpas-0.4.2/src/gmpas/__init__.py +49 -0
- gmpas-0.4.2/src/gmpas/accessor.py +139 -0
- gmpas-0.4.2/src/gmpas/cli.py +811 -0
- gmpas-0.4.2/src/gmpas/config.py +333 -0
- gmpas-0.4.2/src/gmpas/dashboard.py +262 -0
- gmpas-0.4.2/src/gmpas/data.py +101 -0
- gmpas-0.4.2/src/gmpas/mesh.py +697 -0
- gmpas-0.4.2/src/gmpas/paths.py +54 -0
- gmpas-0.4.2/src/gmpas/plot.py +279 -0
- gmpas-0.4.2/src/gmpas/prep/__init__.py +26 -0
- gmpas-0.4.2/src/gmpas/prep/generate.py +510 -0
- gmpas-0.4.2/src/gmpas/prep/hfun.py +279 -0
- gmpas-0.4.2/src/gmpas/prep/hfunview.py +263 -0
- gmpas-0.4.2/src/gmpas/prep/layout.py +428 -0
- gmpas-0.4.2/src/gmpas/prep/meshview.py +224 -0
- gmpas-0.4.2/src/gmpas/py.typed +0 -0
- gmpas-0.4.2/src/gmpas/raster.py +103 -0
- gmpas-0.4.2/src/gmpas/remap.py +496 -0
- gmpas-0.4.2/src/gmpas/scrip.py +160 -0
- gmpas-0.4.2/src/gmpas/series.py +260 -0
- gmpas-0.4.2/src/gmpas/style.py +88 -0
- gmpas-0.4.2/src/gmpas/viewer.py +1167 -0
- gmpas-0.4.2/tests/conftest.py +169 -0
- gmpas-0.4.2/tests/test_cli.py +104 -0
- gmpas-0.4.2/tests/test_config.py +206 -0
- gmpas-0.4.2/tests/test_dashboard.py +213 -0
- gmpas-0.4.2/tests/test_data.py +325 -0
- gmpas-0.4.2/tests/test_generate.py +554 -0
- gmpas-0.4.2/tests/test_hfun.py +296 -0
- gmpas-0.4.2/tests/test_mesh.py +542 -0
- gmpas-0.4.2/tests/test_plot.py +111 -0
- gmpas-0.4.2/tests/test_prep.py +134 -0
- gmpas-0.4.2/tests/test_raster.py +158 -0
- gmpas-0.4.2/tests/test_remap.py +542 -0
- gmpas-0.4.2/tests/test_remap_integration.py +67 -0
- gmpas-0.4.2/tests/test_scrip.py +129 -0
- gmpas-0.4.2/tests/test_style.py +79 -0
- gmpas-0.4.2/tests/test_version.py +36 -0
- gmpas-0.4.2/tests/test_viewer.py +314 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
# Pull requests and pushes to main. Not tags: a tag points at a commit that
|
|
4
|
+
# has already been tested here, so re-running proves nothing new.
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: [main]
|
|
8
|
+
pull_request:
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
# a second push to the same branch supersedes the first
|
|
13
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
14
|
+
cancel-in-progress: true
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
name: pytest · ubuntu · python 3.12
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
defaults:
|
|
21
|
+
run:
|
|
22
|
+
shell: bash -el {0} # -l so the conda env is actually active
|
|
23
|
+
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v4
|
|
26
|
+
|
|
27
|
+
# conda-forge, not pip: cartopy and netCDF4 are painful to build from
|
|
28
|
+
# source, and ESMF_RegridWeightGen is an executable pip cannot install at
|
|
29
|
+
# all. Installing the real environment.yml is what makes a green check
|
|
30
|
+
# here mean the same thing as a green run on a laptop.
|
|
31
|
+
- name: Install the conda environment
|
|
32
|
+
uses: mamba-org/setup-micromamba@v2
|
|
33
|
+
with:
|
|
34
|
+
environment-file: environment.yml
|
|
35
|
+
environment-name: gmpas
|
|
36
|
+
# environment.yml says `python >=3.10`, which would let the solver
|
|
37
|
+
# drift onto whatever is newest. Pin the version CI is meant to test.
|
|
38
|
+
create-args: python=3.12
|
|
39
|
+
cache-environment: true
|
|
40
|
+
|
|
41
|
+
- name: Install gmpas
|
|
42
|
+
# --no-deps: every dependency came from conda above. Letting pip
|
|
43
|
+
# resolve them again can shadow the conda build of a compiled package
|
|
44
|
+
# with a wheel, which is how a working environment silently breaks.
|
|
45
|
+
run: pip install -e . --no-deps
|
|
46
|
+
|
|
47
|
+
- name: Report what is actually installed
|
|
48
|
+
# a failure three steps down is much easier to read with this above it.
|
|
49
|
+
# ESMF_RegridWeightGen is *expected* to be missing here -- environment.yml
|
|
50
|
+
# deliberately does not install it (see docs/REMAPPING.md and issue 34),
|
|
51
|
+
# and every remap test that would shell out to it mocks the call instead.
|
|
52
|
+
run: |
|
|
53
|
+
python -c "import gmpas; print('gmpas', gmpas.__version__)"
|
|
54
|
+
python -c "import numpy, scipy, xarray, netCDF4, matplotlib, cartopy; print('numpy', numpy.__version__, 'scipy', scipy.__version__, 'xarray', xarray.__version__, 'netCDF4', netCDF4.__version__, 'matplotlib', matplotlib.__version__, 'cartopy', cartopy.__version__)"
|
|
55
|
+
which ESMF_RegridWeightGen ncremap || echo "remap tools NOT on PATH"
|
|
56
|
+
|
|
57
|
+
# No ruff step yet, deliberately. `ruff check src tests` currently reports
|
|
58
|
+
# 42 pre-existing violations (mostly E501 in the embedded viewer page), so
|
|
59
|
+
# gating on it would mean this workflow is red from its first run for
|
|
60
|
+
# reasons unrelated to whether the code works. Tracked separately; turn it
|
|
61
|
+
# on once the tree is clean.
|
|
62
|
+
|
|
63
|
+
- name: Run the test suite
|
|
64
|
+
# Agg: the runner is headless, and a rendering test that tries for a
|
|
65
|
+
# display fails in a way that looks nothing like the real problem
|
|
66
|
+
env:
|
|
67
|
+
MPLBACKEND: Agg
|
|
68
|
+
run: pytest -q --durations=10
|
gmpas-0.4.2/.gitignore
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.so
|
|
5
|
+
*.egg-info/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
.venv/
|
|
9
|
+
venv/
|
|
10
|
+
|
|
11
|
+
# Tooling
|
|
12
|
+
.pytest_cache/
|
|
13
|
+
.ruff_cache/
|
|
14
|
+
.coverage
|
|
15
|
+
|
|
16
|
+
# IDE
|
|
17
|
+
.vscode/
|
|
18
|
+
.idea/
|
|
19
|
+
*.swp
|
|
20
|
+
|
|
21
|
+
# Data — never commit model output or generated figures
|
|
22
|
+
*.nc
|
|
23
|
+
*.npz
|
|
24
|
+
*.png
|
|
25
|
+
|
|
26
|
+
# ESMF_RegridWeightGen writes these into its cwd on every run
|
|
27
|
+
PET*.RegridWeightGen.Log
|
|
28
|
+
|
|
29
|
+
# OS
|
|
30
|
+
.DS_Store
|
gmpas-0.4.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 nmathewa
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
gmpas-0.4.2/PKG-INFO
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gmpas
|
|
3
|
+
Version: 0.4.2
|
|
4
|
+
Summary: Fast plotting of MPAS output on its own native variable-resolution mesh
|
|
5
|
+
Project-URL: Homepage, https://github.com/nmathewa/gmpas-lib
|
|
6
|
+
Project-URL: Issues, https://github.com/nmathewa/gmpas-lib/issues
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: atmosphere,mpas,unstructured,visualization,voronoi
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Requires-Dist: netcdf4>=1.6
|
|
17
|
+
Requires-Dist: numpy>=1.24
|
|
18
|
+
Requires-Dist: scipy>=1.10
|
|
19
|
+
Requires-Dist: xarray>=2023.1
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: cartopy>=0.22; extra == 'dev'
|
|
22
|
+
Requires-Dist: matplotlib>=3.7; extra == 'dev'
|
|
23
|
+
Requires-Dist: pillow>=9.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
26
|
+
Provides-Extra: plot
|
|
27
|
+
Requires-Dist: cartopy>=0.22; extra == 'plot'
|
|
28
|
+
Requires-Dist: matplotlib>=3.7; extra == 'plot'
|
|
29
|
+
Requires-Dist: pillow>=9.0; extra == 'plot'
|
|
30
|
+
Provides-Extra: test
|
|
31
|
+
Requires-Dist: pytest>=7.0; extra == 'test'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# gmpas
|
|
35
|
+
|
|
36
|
+
[](https://github.com/nmathewa/gmpas-lib/actions/workflows/tests.yml)
|
|
37
|
+
[](docs/status.md)
|
|
38
|
+
[](docs/installation.md)
|
|
39
|
+
[](LICENSE)
|
|
40
|
+
|
|
41
|
+
Fast plotting of MPAS output **on its own native mesh** — no regridding, so
|
|
42
|
+
variable resolution is preserved exactly as the model carries it. Plus the
|
|
43
|
+
other end of the pipeline: designing a mesh, building it with JIGSAW, and
|
|
44
|
+
looking at either before or after it exists.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import gmpas
|
|
48
|
+
|
|
49
|
+
ds = gmpas.open_mpas("diag.2019-09-01_00.00.00.nc", mesh="maritime.region.nc")
|
|
50
|
+
|
|
51
|
+
ds.mpas.plot("mslp") # cell field, filled Voronoi polygons
|
|
52
|
+
ds.mpas.plot("u") # edge field, drawn on the cell faces themselves
|
|
53
|
+
ds.mpas.plot_mesh() # where the mesh actually refines, in km
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`mesh=` may be omitted when the file carries its own mesh information, or when
|
|
57
|
+
a mesh file with a matching cell count sits beside it.
|
|
58
|
+
|
|
59
|
+
## Installation
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
conda env create -f environment.yml && conda activate gmpas && pip install -e . --no-deps
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Pure pip works too where wheels exist:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install -e ".[dev]"
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Two workflows need external programs, which pip cannot provide. Conservative
|
|
72
|
+
remapping needs ESMF (`conda install -c conda-forge esmf nco`); mesh generation
|
|
73
|
+
needs [JIGSAW](https://github.com/dengwirda/jigsaw) and, for the final step,
|
|
74
|
+
MPI and PnetCDF. Skip both if you only plot and view.
|
|
75
|
+
|
|
76
|
+
Check it landed:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
gmpas --version && pytest -q
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Full detail, including the extras and what each one pulls in:
|
|
83
|
+
[docs/installation.md](docs/installation.md).
|
|
84
|
+
|
|
85
|
+
## Usage
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
gmpas info history.2012-02-25_12.00.00.nc
|
|
89
|
+
gmpas plot history.2012-02-25_12.00.00.nc precipw -o pw.png
|
|
90
|
+
gmpas view /path/to/run/
|
|
91
|
+
gmpas remap history.*.nc -o out/
|
|
92
|
+
gmpas prep view mesh.nc
|
|
93
|
+
gmpas prep hfun hfun.py --check
|
|
94
|
+
gmpas prep generate hfun.py -o mesh/ # needs $JIGSAWDIR and $MKGRIDFILE
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Any path may be a file, a directory, or a glob; a directory or glob is read as
|
|
98
|
+
one time series across files, which is how MPAS writes output. Running `gmpas`
|
|
99
|
+
with no arguments prints the whole list with examples.
|
|
100
|
+
|
|
101
|
+
Everything except `prep` is **postprocessing** — it opens a run and renders,
|
|
102
|
+
remaps or exports it. `prep` is the other end, for work that happens before
|
|
103
|
+
there is any output.
|
|
104
|
+
|
|
105
|
+
## Documentation
|
|
106
|
+
|
|
107
|
+
| | |
|
|
108
|
+
|---|---|
|
|
109
|
+
| [Why gmpas exists](docs/why.md) | the problem with lat-lon tooling, and why this is fast |
|
|
110
|
+
| [Installation](docs/installation.md) | conda, pip, extras, and the external programs |
|
|
111
|
+
| [Command line](docs/command-line.md) | every command and its flags |
|
|
112
|
+
| [In a notebook](docs/notebook.md) | the accessor, and using the pieces directly |
|
|
113
|
+
| [Preprocessing](docs/preprocessing.md) | `prep view`, `prep hfun`, `prep generate` — mesh design and JIGSAW |
|
|
114
|
+
| [Conservative remapping](docs/REMAPPING.md) | the whole terminal workflow, and two MPAS traps |
|
|
115
|
+
| [On a cluster](docs/cluster.md) | port forwarding, and the two variables that matter |
|
|
116
|
+
| [Configuration](docs/configuration.md) | `GMPAS_CACHE_DIR` and `GMPAS_DATA_DIR` |
|
|
117
|
+
| [Examples](examples/) | ready-to-edit `hfun.py` templates |
|
|
118
|
+
| [Tests](docs/testing.md) | what the suite covers |
|
|
119
|
+
| [Layout](docs/layout.md) | what lives in which module |
|
|
120
|
+
| [Differences from the MCP server](docs/mcp-server.md) | what changed on the way to a package |
|
|
121
|
+
| [Status](docs/status.md) | what is implemented and what is not |
|
gmpas-0.4.2/README.md
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# gmpas
|
|
2
|
+
|
|
3
|
+
[](https://github.com/nmathewa/gmpas-lib/actions/workflows/tests.yml)
|
|
4
|
+
[](docs/status.md)
|
|
5
|
+
[](docs/installation.md)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
Fast plotting of MPAS output **on its own native mesh** — no regridding, so
|
|
9
|
+
variable resolution is preserved exactly as the model carries it. Plus the
|
|
10
|
+
other end of the pipeline: designing a mesh, building it with JIGSAW, and
|
|
11
|
+
looking at either before or after it exists.
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
import gmpas
|
|
15
|
+
|
|
16
|
+
ds = gmpas.open_mpas("diag.2019-09-01_00.00.00.nc", mesh="maritime.region.nc")
|
|
17
|
+
|
|
18
|
+
ds.mpas.plot("mslp") # cell field, filled Voronoi polygons
|
|
19
|
+
ds.mpas.plot("u") # edge field, drawn on the cell faces themselves
|
|
20
|
+
ds.mpas.plot_mesh() # where the mesh actually refines, in km
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
`mesh=` may be omitted when the file carries its own mesh information, or when
|
|
24
|
+
a mesh file with a matching cell count sits beside it.
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
conda env create -f environment.yml && conda activate gmpas && pip install -e . --no-deps
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Pure pip works too where wheels exist:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install -e ".[dev]"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Two workflows need external programs, which pip cannot provide. Conservative
|
|
39
|
+
remapping needs ESMF (`conda install -c conda-forge esmf nco`); mesh generation
|
|
40
|
+
needs [JIGSAW](https://github.com/dengwirda/jigsaw) and, for the final step,
|
|
41
|
+
MPI and PnetCDF. Skip both if you only plot and view.
|
|
42
|
+
|
|
43
|
+
Check it landed:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
gmpas --version && pytest -q
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Full detail, including the extras and what each one pulls in:
|
|
50
|
+
[docs/installation.md](docs/installation.md).
|
|
51
|
+
|
|
52
|
+
## Usage
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
gmpas info history.2012-02-25_12.00.00.nc
|
|
56
|
+
gmpas plot history.2012-02-25_12.00.00.nc precipw -o pw.png
|
|
57
|
+
gmpas view /path/to/run/
|
|
58
|
+
gmpas remap history.*.nc -o out/
|
|
59
|
+
gmpas prep view mesh.nc
|
|
60
|
+
gmpas prep hfun hfun.py --check
|
|
61
|
+
gmpas prep generate hfun.py -o mesh/ # needs $JIGSAWDIR and $MKGRIDFILE
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Any path may be a file, a directory, or a glob; a directory or glob is read as
|
|
65
|
+
one time series across files, which is how MPAS writes output. Running `gmpas`
|
|
66
|
+
with no arguments prints the whole list with examples.
|
|
67
|
+
|
|
68
|
+
Everything except `prep` is **postprocessing** — it opens a run and renders,
|
|
69
|
+
remaps or exports it. `prep` is the other end, for work that happens before
|
|
70
|
+
there is any output.
|
|
71
|
+
|
|
72
|
+
## Documentation
|
|
73
|
+
|
|
74
|
+
| | |
|
|
75
|
+
|---|---|
|
|
76
|
+
| [Why gmpas exists](docs/why.md) | the problem with lat-lon tooling, and why this is fast |
|
|
77
|
+
| [Installation](docs/installation.md) | conda, pip, extras, and the external programs |
|
|
78
|
+
| [Command line](docs/command-line.md) | every command and its flags |
|
|
79
|
+
| [In a notebook](docs/notebook.md) | the accessor, and using the pieces directly |
|
|
80
|
+
| [Preprocessing](docs/preprocessing.md) | `prep view`, `prep hfun`, `prep generate` — mesh design and JIGSAW |
|
|
81
|
+
| [Conservative remapping](docs/REMAPPING.md) | the whole terminal workflow, and two MPAS traps |
|
|
82
|
+
| [On a cluster](docs/cluster.md) | port forwarding, and the two variables that matter |
|
|
83
|
+
| [Configuration](docs/configuration.md) | `GMPAS_CACHE_DIR` and `GMPAS_DATA_DIR` |
|
|
84
|
+
| [Examples](examples/) | ready-to-edit `hfun.py` templates |
|
|
85
|
+
| [Tests](docs/testing.md) | what the suite covers |
|
|
86
|
+
| [Layout](docs/layout.md) | what lives in which module |
|
|
87
|
+
| [Differences from the MCP server](docs/mcp-server.md) | what changed on the way to a package |
|
|
88
|
+
| [Status](docs/status.md) | what is implemented and what is not |
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
# Conservative remapping
|
|
2
|
+
|
|
3
|
+
A terminal workflow. gmpas writes the grid file and checks the answer; a real
|
|
4
|
+
remapper computes the weights.
|
|
5
|
+
|
|
6
|
+
## What you need installed
|
|
7
|
+
|
|
8
|
+
`ESMF_RegridWeightGen` (from `esmf`) and, optionally, `ncremap` (from `nco`).
|
|
9
|
+
Both are executables rather than Python packages, so they cannot come from pip
|
|
10
|
+
and are not declared in `pyproject.toml`.
|
|
11
|
+
|
|
12
|
+
**gmpas does not install either itself, on purpose.** `environment.yml` brings
|
|
13
|
+
in only gmpas's own Python dependencies. On an HPC site, load the site's own
|
|
14
|
+
builds instead — `module load esmf`, `module load nco` — rather than adding
|
|
15
|
+
conda-forge copies to this environment: a site build is tuned for the local
|
|
16
|
+
MPI/interconnect, and a second copy here would compete with it on
|
|
17
|
+
`PATH`/`LD_LIBRARY_PATH` rather than help, whether gmpas shells out to it
|
|
18
|
+
(ESMF) or you run it yourself (`ncremap`) — see
|
|
19
|
+
[issue 34](https://github.com/nmathewa/gmpas-lib/issues/34). On a machine with
|
|
20
|
+
no module system, `conda install -c conda-forge esmf nco` into a separate
|
|
21
|
+
environment works fine — just don't add it to the one gmpas itself runs in. A
|
|
22
|
+
`command not found: ESMF_RegridWeightGen` means neither is loaded.
|
|
23
|
+
|
|
24
|
+

|
|
25
|
+
|
|
26
|
+
```mermaid
|
|
27
|
+
flowchart TD
|
|
28
|
+
M["MPAS mesh file"] -->|gmpas scrip| S["src.scrip.nc"]
|
|
29
|
+
T["target grid"] -->|write dst SCRIP| D["dst.scrip.nc"]
|
|
30
|
+
S --> W["ESMF_RegridWeightGen -m conserve"]
|
|
31
|
+
D --> W
|
|
32
|
+
W --> MAP["map.nc<br/><i>reuse for every field and step</i>"]
|
|
33
|
+
MAP --> A["apply: dst = S · src"]
|
|
34
|
+
H["history.nc"] --> A
|
|
35
|
+
A --> C["check the integral"]
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Why gmpas does not compute the weights
|
|
39
|
+
|
|
40
|
+
Spherical polygon intersection — great-circle edges, degenerate cells, poles,
|
|
41
|
+
the antimeridian — is hard to get right, and ESMF and TempestRemap have both
|
|
42
|
+
been doing it correctly for years.
|
|
43
|
+
|
|
44
|
+
And the obvious shortcut does not work. Supersampling a target cell and
|
|
45
|
+
counting which source cells the samples land in **converges towards**
|
|
46
|
+
conservation as the sample count grows, with error falling as `1/sqrt(N)`, but
|
|
47
|
+
never actually preserves the cell integral. That is not conservation, and
|
|
48
|
+
shipping it under that name would be worse than not offering it.
|
|
49
|
+
|
|
50
|
+
## One command
|
|
51
|
+
|
|
52
|
+
Put `target_domain` next to the run, optionally `include_fields`,
|
|
53
|
+
`exclude_fields` and `mesh_file`, then:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
gmpas remap 'history.*.nc' -o out/
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
It reads the configuration, builds the weights once (reusing `map_*.nc` if it
|
|
60
|
+
is already there), and writes **one output file per input file** — a few
|
|
61
|
+
hundred history files concatenated into a single netCDF would be unwieldy, and
|
|
62
|
+
one-in-one-out keeps the valid time in the filename.
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
[1/3] configuration from /scratch/run
|
|
66
|
+
found target_domain
|
|
67
|
+
found include_fields
|
|
68
|
+
absent exclude_fields
|
|
69
|
+
found mesh_file
|
|
70
|
+
input : 193 file(s)
|
|
71
|
+
mesh : init.nc (from mesh_file)
|
|
72
|
+
target: 534 x 267 cells, lon 80.0 .. 160.0, lat -20.0 .. 20.0, 0.149813 deg
|
|
73
|
+
fields: 20 selected of 119 available
|
|
74
|
+
|
|
75
|
+
[2/3] weights
|
|
76
|
+
writing src.scrip.nc and dst.scrip.nc
|
|
77
|
+
normalised 12,774 longitudes onto [0, 2pi)
|
|
78
|
+
weights ready in 18.2 s (77 MB)
|
|
79
|
+
|
|
80
|
+
[3/3] remapping 193 file(s) -> out/
|
|
81
|
+
not remapped — u: on nEdges — needs edge weights
|
|
82
|
+
conservation error 0.0e+00 (0 is exact)
|
|
83
|
+
[ 1/193] history.2012-02-25_00.00.00.remap.nc 18 fields, 347 slabs, 5.7s, eta 18m
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Where the mesh comes from
|
|
87
|
+
|
|
88
|
+
MPAS output streams are user-configured, so a history file need not carry
|
|
89
|
+
`verticesOnCell` at all. The mesh is resolved in this order:
|
|
90
|
+
|
|
91
|
+
1. `--mesh` on the command line
|
|
92
|
+
2. a `mesh_file` in the config directory, holding one line: the path
|
|
93
|
+
3. the data file itself, if it is self-describing
|
|
94
|
+
4. a mesh sitting beside the data with a matching cell count
|
|
95
|
+
|
|
96
|
+
`mesh_file` is usually the right answer for a real run — set once, not passed
|
|
97
|
+
on every command.
|
|
98
|
+
|
|
99
|
+
### What cannot be remapped by cell weights
|
|
100
|
+
|
|
101
|
+
Cell weights carry cell fields. MPAS keeps velocity as `u` on edges and
|
|
102
|
+
vorticity on vertices, and those need their own weight files, so they are
|
|
103
|
+
reported rather than silently dropped:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
not remapped — u: on nEdges — needs edge weights
|
|
107
|
+
not remapped — vorticity: on nVertices — needs vertex weights
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
For winds, `uReconstructZonal` / `uReconstructMeridional` are already at cell
|
|
111
|
+
centres and remap normally.
|
|
112
|
+
|
|
113
|
+
### Running it in parallel
|
|
114
|
+
|
|
115
|
+
Files are independent, so they convert in parallel:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
gmpas remap 'history.*.nc' -o out/ -j 64
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
**Give `-j` explicitly.** Without it the worker count is detected — from
|
|
122
|
+
`SLURM_CPUS_PER_TASK`, the other schedulers' variables, then the process
|
|
123
|
+
affinity mask, and only then `os.cpu_count()` — which beats using the size of
|
|
124
|
+
the machine, but is still only as reliable as what the site sets. The command
|
|
125
|
+
always reports which source it used, and that line is worth reading:
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
64 worker(s) (of 64 from SLURM_CPUS_PER_TASK) detection worked
|
|
129
|
+
1 worker(s) (of 1 from NCPUS) detection was misled
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`NCPUS` in particular is set to `1` by some login profiles regardless of the
|
|
133
|
+
allocation, which silently pins a large job to one worker. `-j` overrides
|
|
134
|
+
whatever was detected:
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
64 worker(s) (asked for 64; 1 available from NCPUS)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Measured on 8 files, 936 slabs, on a 10-core laptop:
|
|
141
|
+
|
|
142
|
+
| workers | wall | per file |
|
|
143
|
+
|---|---|---|
|
|
144
|
+
| 1 | 14 s | 1.73 s |
|
|
145
|
+
| 2 | 8 s | 1.01 s |
|
|
146
|
+
| 4 | 5 s | 0.65 s |
|
|
147
|
+
| 8 | 5 s | 0.58 s |
|
|
148
|
+
|
|
149
|
+
It flattens past the core count because the work is largely netCDF reads. On a
|
|
150
|
+
parallel filesystem the ceiling is I/O bandwidth rather than cores, so very
|
|
151
|
+
high `-j` will not keep scaling — worth measuring on a subset before
|
|
152
|
+
committing a whole run.
|
|
153
|
+
|
|
154
|
+
Under `fork` the weights are inherited copy-on-write rather than re-read, which
|
|
155
|
+
matters at high core counts: ~15 MB of index arrays re-loaded in 256 workers
|
|
156
|
+
would be several gigabytes of duplication for read-only data.
|
|
157
|
+
|
|
158
|
+
A file that fails is reported and the run continues; the exit status is
|
|
159
|
+
non-zero if any failed.
|
|
160
|
+
|
|
161
|
+
### A caveat on ESMF
|
|
162
|
+
|
|
163
|
+
ESMF 8.9.1 on macOS segfaults intermittently — measured at roughly **one run
|
|
164
|
+
in five** on byte-identical inputs that succeed the other four times. Weight
|
|
165
|
+
generation is a one-off, so `gmpas remap` retries up to four times and says so
|
|
166
|
+
when it does. A reproducible crash is different and usually means the padded
|
|
167
|
+
`grid_corners` problem below.
|
|
168
|
+
|
|
169
|
+
## The steps, individually
|
|
170
|
+
|
|
171
|
+
### 1. Write the source grid
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
gmpas scrip history.2012-02-25_12.00.00.nc -o src.scrip.nc
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Works on any file carrying mesh information — an `init.nc`, a `*.grid.nc`, or
|
|
178
|
+
a history file that carries its own mesh. It reports the coverage as a sanity
|
|
179
|
+
check (a global mesh should say 100%) and tells you if any longitudes had to be
|
|
180
|
+
normalised.
|
|
181
|
+
|
|
182
|
+
### 2. Write the target grid
|
|
183
|
+
|
|
184
|
+
Describe it in a `target_domain` file beside the run:
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
nlat = 267
|
|
188
|
+
nlon = 534
|
|
189
|
+
startlat = -20.0
|
|
190
|
+
endlat = 20.0
|
|
191
|
+
startlon = 80.0
|
|
192
|
+
endlon = 160.0
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
`startlat`/`endlat` are the **domain edges** and `nlat` counts cells across
|
|
196
|
+
them, so the spacing is `(endlat - startlat) / nlat` and centres sit half a
|
|
197
|
+
cell inside each edge — the grid then covers exactly the box you asked for.
|
|
198
|
+
|
|
199
|
+
The extent is not fussy beyond that. Target cells falling outside the source
|
|
200
|
+
mesh come back unmapped, and a target narrower than the mesh simply crops;
|
|
201
|
+
both are expected. `gmpas target` prints the covered extent next to the
|
|
202
|
+
requested one so any shift is visible rather than assumed.
|
|
203
|
+
|
|
204
|
+
Then, from that directory:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
gmpas target -o dst.scrip.nc
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
`gmpas target` reads the file from the working directory — no path needed —
|
|
211
|
+
reports the resulting grid, and writes the SCRIP. Pass a data file too and it
|
|
212
|
+
lists which fields would be remapped:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
gmpas target history.2012-02-25_12.00.00.nc -o dst.scrip.nc
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Cell areas are written as exact solid angles, `dlon x (sin(north) -
|
|
219
|
+
sin(south))`, not the `dlon x dlat x cos(lat)` approximation — a remapper
|
|
220
|
+
compares them against its own and the difference shows up as conservation
|
|
221
|
+
error.
|
|
222
|
+
|
|
223
|
+
`ncremap -g dst.scrip.nc -G latlon=180,360` will also generate a target, and
|
|
224
|
+
a SCRIP file is only six arrays if you would rather write one directly.
|
|
225
|
+
|
|
226
|
+
### Choosing fields
|
|
227
|
+
|
|
228
|
+
Two more optional files in the same directory, one variable name per line:
|
|
229
|
+
|
|
230
|
+
```
|
|
231
|
+
include_fields remap only these
|
|
232
|
+
exclude_fields remap everything but these
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Blank lines, `#` comments and stray trailing spaces are all fine — these get
|
|
236
|
+
hand-edited. A name in **both** files is contradictory: **include wins**, and
|
|
237
|
+
a warning names the fields, because silently dropping something explicitly
|
|
238
|
+
asked for leaves output missing with nothing to explain it.
|
|
239
|
+
|
|
240
|
+
### 3. Generate the weights — once per mesh pair
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
ESMF_RegridWeightGen -s src.scrip.nc -d dst.scrip.nc -w map.nc \
|
|
244
|
+
-m conserve --src_regional --dst_regional --ignore_unmapped
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
`--src_regional` / `--dst_regional` matter for a mesh that does not cover the
|
|
248
|
+
sphere; `--ignore_unmapped` lets destination cells outside the source domain
|
|
249
|
+
pass through unmapped rather than aborting.
|
|
250
|
+
|
|
251
|
+
Alternatives, all producing a SCRIP-format weight file:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
ncremap -s src.scrip.nc -g dst.nc -m map.nc -a aave # first-order, the E3SM route
|
|
255
|
+
GenerateOfflineMap --in_mesh a.g --out_mesh b.g --ov_mesh ov.g --out_map map.nc
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
`-m conserve` is first-order; `-m conserve2nd` is second-order and less
|
|
259
|
+
diffusive. TempestRemap adds higher order and `--mono` monotonicity.
|
|
260
|
+
|
|
261
|
+
**Weights depend only on the two grids.** Generate once, then reuse for every
|
|
262
|
+
variable, level and timestep of that run.
|
|
263
|
+
|
|
264
|
+
### 4. Apply them
|
|
265
|
+
|
|
266
|
+
A weight file carries `row`, `col`, `S`, plus `area_a`, `area_b`, `frac_a`,
|
|
267
|
+
`frac_b`. Applying it is a sparse matrix multiply:
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
import numpy as np, xarray as xr
|
|
271
|
+
|
|
272
|
+
w = xr.open_dataset("map.nc")
|
|
273
|
+
dst = np.zeros(w.sizes["n_b"])
|
|
274
|
+
np.add.at(dst, w.row.values - 1, w.S.values * src[w.col.values - 1])
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
The `- 1` is not optional: SCRIP indices are 1-based.
|
|
278
|
+
|
|
279
|
+
`ncremap` will also do this for you:
|
|
280
|
+
|
|
281
|
+
```bash
|
|
282
|
+
ncremap -m map.nc history.nc remapped.nc
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
### 5. Check that it conserved
|
|
286
|
+
|
|
287
|
+
The step worth never skipping, because every failure mode here produces a
|
|
288
|
+
plausible-looking wrong answer rather than an error:
|
|
289
|
+
|
|
290
|
+
```python
|
|
291
|
+
I_src = (src * w.area_a.values * w.frac_a.values).sum()
|
|
292
|
+
I_dst = (dst * w.area_b.values).sum()
|
|
293
|
+
assert abs(I_dst - I_src) / abs(I_src) < 1e-12
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
**Do not multiply the destination by `frac_b`.** With ESMF's default
|
|
297
|
+
`norm_type=dstarea` the weights already carry the destination coverage
|
|
298
|
+
fraction, so multiplying again double counts it — that mistake reported a 0.2%
|
|
299
|
+
error on weights that were exact.
|
|
300
|
+
|
|
301
|
+
Measured on a 413,788-cell regional mesh to a 0.25° lat-lon grid:
|
|
302
|
+
|
|
303
|
+
| field | relative error |
|
|
304
|
+
|---|---|
|
|
305
|
+
| constant 1.0 | 1.1e-16 |
|
|
306
|
+
| smooth analytic (float64) | 0.0 |
|
|
307
|
+
| `precipw`, `t2m` | 0.0 |
|
|
308
|
+
| nearest-neighbour, for contrast | 2.1e-05 |
|
|
309
|
+
|
|
310
|
+
For partially covered destination cells, the raw `dst` is what conserves;
|
|
311
|
+
`dst / frac_b` is what you plot.
|
|
312
|
+
|
|
313
|
+
## Two traps specific to MPAS
|
|
314
|
+
|
|
315
|
+
Both were found by testing, and both break `mpas_tools` output as readily as
|
|
316
|
+
anything else. `gmpas scrip` handles both.
|
|
317
|
+
|
|
318
|
+
### Files mix longitude conventions
|
|
319
|
+
|
|
320
|
+
A real history file stored `lonCell` on `[0, 2π)` reaching 3.23 rad (185°E)
|
|
321
|
+
while storing `lonVertex` on `[-π, π)` — so 12,774 vertices near the dateline
|
|
322
|
+
were negative. A cell's centre and its own corners then sit on different
|
|
323
|
+
branches, and the polygon handed to a remapper is nonsense.
|
|
324
|
+
`mpas_tools.scrip.from_mpas` refuses such a file outright:
|
|
325
|
+
|
|
326
|
+
```
|
|
327
|
+
ValueError: lonVertex is not in the desired range (0, 2pi)
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
`gmpas scrip` normalises onto `[0, 2π)` and reports how many values it moved.
|
|
331
|
+
|
|
332
|
+
### grid_corners must be trimmed
|
|
333
|
+
|
|
334
|
+
MPAS declares `maxEdges` generously — one mesh declares 10 and uses at most 6.
|
|
335
|
+
Written at the declared width, every unused column becomes a degenerate corner
|
|
336
|
+
repeated on every cell, and **ESMF 8.9.1 segfaults on `-m conserve`**:
|
|
337
|
+
|
|
338
|
+
```
|
|
339
|
+
exit 139, no weight file, nothing useful in the log
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
Confirmed at 2°, 1° and 0.25° targets; nearest-neighbour works either way, so
|
|
343
|
+
it is specific to the conservative path. Trimmed to 6 it completes. `gmpas
|
|
344
|
+
scrip` writes `grid_corners = max(nEdgesOnCell)`.
|
|
345
|
+
|
|
346
|
+
## What is not here
|
|
347
|
+
|
|
348
|
+
Applying weights from the command line is not implemented — see
|
|
349
|
+
[issue 2](https://github.com/nmathewa/gmpas-lib/issues/2). The snippet above is
|
|
350
|
+
what it would do.
|
|
351
|
+
|
|
352
|
+
`uxarray` is not yet an option: its remapping is nearest-neighbour, inverse
|
|
353
|
+
distance and bilinear only. Conservative exists through its YAC backend, but
|
|
354
|
+
that integration is still in progress.
|