tlpr2s 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tlpr2s-0.2.0/PKG-INFO +143 -0
- tlpr2s-0.2.0/README.md +105 -0
- tlpr2s-0.2.0/pyproject.toml +25 -0
- tlpr2s-0.2.0/setup.cfg +4 -0
- tlpr2s-0.2.0/src/tlpr2s/__init__.py +11 -0
- tlpr2s-0.2.0/src/tlpr2s/annual.py +77 -0
- tlpr2s-0.2.0/src/tlpr2s/arrays.py +56 -0
- tlpr2s-0.2.0/src/tlpr2s/cli.py +42 -0
- tlpr2s-0.2.0/src/tlpr2s/daac.py +92 -0
- tlpr2s-0.2.0/src/tlpr2s/daac_products.py +930 -0
- tlpr2s-0.2.0/src/tlpr2s/features.py +4 -0
- tlpr2s-0.2.0/src/tlpr2s/model_download.py +47 -0
- tlpr2s-0.2.0/src/tlpr2s/plotting.py +57 -0
- tlpr2s-0.2.0/src/tlpr2s/public.py +11 -0
- tlpr2s-0.2.0/src/tlpr2s/retrieval.py +108 -0
- tlpr2s-0.2.0/src/tlpr2s/ssmis_source.py +65 -0
- tlpr2s-0.2.0/src/tlpr2s.egg-info/PKG-INFO +143 -0
- tlpr2s-0.2.0/src/tlpr2s.egg-info/SOURCES.txt +26 -0
- tlpr2s-0.2.0/src/tlpr2s.egg-info/dependency_links.txt +1 -0
- tlpr2s-0.2.0/src/tlpr2s.egg-info/entry_points.txt +2 -0
- tlpr2s-0.2.0/src/tlpr2s.egg-info/requires.txt +34 -0
- tlpr2s-0.2.0/src/tlpr2s.egg-info/top_level.txt +1 -0
- tlpr2s-0.2.0/tests/test_annual.py +35 -0
- tlpr2s-0.2.0/tests/test_arrays.py +22 -0
- tlpr2s-0.2.0/tests/test_calibration.py +6 -0
- tlpr2s-0.2.0/tests/test_daac.py +13 -0
- tlpr2s-0.2.0/tests/test_huggingface.py +27 -0
- tlpr2s-0.2.0/tests/test_public_api.py +14 -0
tlpr2s-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tlpr2s
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: 2D-array SSMIS F16 transfer-learning precipitation retrieval and swath statistics
|
|
5
|
+
Author: TLP-R2S research team
|
|
6
|
+
Keywords: precipitation,SSMIS,satellite,XGBoost,remote sensing
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: numpy>=1.24
|
|
10
|
+
Requires-Dist: pandas>=1.5
|
|
11
|
+
Requires-Dist: scipy>=1.10
|
|
12
|
+
Requires-Dist: xgboost>=2.0
|
|
13
|
+
Provides-Extra: hub
|
|
14
|
+
Requires-Dist: huggingface_hub<2,>=0.24; extra == "hub"
|
|
15
|
+
Provides-Extra: all
|
|
16
|
+
Requires-Dist: huggingface_hub<2,>=0.24; extra == "all"
|
|
17
|
+
Requires-Dist: matplotlib>=3.7; extra == "all"
|
|
18
|
+
Requires-Dist: cartopy>=0.22; extra == "all"
|
|
19
|
+
Requires-Dist: xarray>=2023.1; extra == "all"
|
|
20
|
+
Requires-Dist: netCDF4>=1.6; extra == "all"
|
|
21
|
+
Requires-Dist: h5py>=3.8; extra == "all"
|
|
22
|
+
Provides-Extra: plot
|
|
23
|
+
Requires-Dist: matplotlib>=3.7; extra == "plot"
|
|
24
|
+
Requires-Dist: cartopy>=0.22; extra == "plot"
|
|
25
|
+
Provides-Extra: io
|
|
26
|
+
Requires-Dist: xarray>=2023.1; extra == "io"
|
|
27
|
+
Requires-Dist: netCDF4>=1.6; extra == "io"
|
|
28
|
+
Requires-Dist: h5py>=3.8; extra == "io"
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: huggingface_hub<2,>=0.24; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
32
|
+
Requires-Dist: build; extra == "dev"
|
|
33
|
+
Requires-Dist: twine; extra == "dev"
|
|
34
|
+
Requires-Dist: matplotlib>=3.7; extra == "dev"
|
|
35
|
+
Requires-Dist: xarray>=2023.1; extra == "dev"
|
|
36
|
+
Requires-Dist: netCDF4>=1.6; extra == "dev"
|
|
37
|
+
Requires-Dist: h5py>=3.8; extra == "dev"
|
|
38
|
+
|
|
39
|
+
# tlpr2s — SSMIS F16 transfer-learning rain and snow retrieval
|
|
40
|
+
|
|
41
|
+
**Python 3.10+ · 2D S4 swath inputs · original XGBoost models and CDF calibration · Hugging Face hosting · orbit visualization · annual grid · full legacy DAAC-oriented NetCDF**
|
|
42
|
+
|
|
43
|
+
The scientific model artifacts are hosted separately at [BuddhaSubedi/TLP-R2S](https://huggingface.co/BuddhaSubedi/TLP-R2S) (approximately 6.43 GB). **They are not included in the Python wheel.** The first model initialization downloads and caches the five original files; subsequent runs reuse the cache. No model download occurs when simply importing `tlpr2s`. The model repo must be public, or the user must authenticate with Hugging Face.
|
|
44
|
+
|
|
45
|
+
> **Scientific status:** This repository is an editable integration release. Real-orbit parity against the original pipeline and NASA DAAC acceptance have not been independently verified. The package is not NASA-endorsed. Do not publish it as validated before testing the actual trained artifacts.
|
|
46
|
+
|
|
47
|
+
## Installation
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
python -m pip install -e '.[all,dev]'
|
|
51
|
+
python -m pytest -q
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
For a public release after publishing on PyPI, users would run `pip install 'tlpr2s[all]'`. The package name's availability and release are not guaranteed.
|
|
55
|
+
|
|
56
|
+
## Download models (once)
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
tlpr2s download-models
|
|
60
|
+
# Better reproducibility: pin an immutable model repository commit SHA
|
|
61
|
+
# tlpr2s download-models --revision FULL_COMMIT_SHA
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The downloader retrieves ONLY these files:
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
xgb_tl_classifier_final.json
|
|
68
|
+
xgb_tl_rain_rate_final.json
|
|
69
|
+
xgb_tl_snow_rate_final_1.json
|
|
70
|
+
cdf_match_for_rain_ssmis.npz
|
|
71
|
+
cdf_match_for_snow_ssmis.npz
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Alternative offline/HPC approach:** download on a login node, or copy the five original files to `/scratch/models/tlpr2s/`, then set `TLPR2S_MODEL_DIR=/scratch/models/tlpr2s` or pass `model_dir=...`. To use a cached model offline, call `PrecipitationRetriever(local_files_only=True)`. To prohibit downloads altogether, call `PrecipitationRetriever(download=False, model_dir=...)`. To select a particular model revision, pass `revision='FULL_COMMIT_SHA'` or set `TLPR2S_MODEL_REVISION`.
|
|
75
|
+
|
|
76
|
+
## Python API — S4-aligned 2D arrays, not full orbit files
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from tlpr2s import FEATURES, predict, plot_orbit, plot_annual, save_nc
|
|
80
|
+
|
|
81
|
+
# inputs: dict with 30 predictor keys; each is 2D (npix,nscan),
|
|
82
|
+
# except Month and Day may be scalar. The 11 lower-frequency SSMIS
|
|
83
|
+
# TB channels and 17 ERA5 fields MUST already be mapped to the S4 grid.
|
|
84
|
+
# See docs/INPUT_FORMAT.md for the exact feature names.
|
|
85
|
+
result = predict(inputs, revision='FULL_COMMIT_SHA')
|
|
86
|
+
|
|
87
|
+
# Result arrays have same (npix,nscan) shape.
|
|
88
|
+
phase = result.phase # 0=clear, 1=rain, 2=snow
|
|
89
|
+
rain = result.rain # mm/hr
|
|
90
|
+
snow = result.snow # mm/hr
|
|
91
|
+
precip = result.surface_precip
|
|
92
|
+
|
|
93
|
+
# Latitude and longitude must be 2D arrays on the same S4 grid.
|
|
94
|
+
plot_orbit(latitude, longitude, precip, vmax=15, save='orbit.png')
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
You may also use `PrecipitationRetriever(revision='FULL_COMMIT_SHA')` once and call `.predict(inputs)` on many orbits to avoid reloading multi-GB models for each orbit. The `predict` convenience function initializes the model each call, which is inefficient for batch processing.
|
|
98
|
+
|
|
99
|
+
### Annual grid and visualization
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from tlpr2s import AnnualAccumulator, plot_annual
|
|
103
|
+
acc = AnnualAccumulator(resolution=0.25)
|
|
104
|
+
for latitude, longitude, result in retrieved_orbits:
|
|
105
|
+
acc.add(latitude, longitude, result.rain, result.snow)
|
|
106
|
+
annual = acc.result()
|
|
107
|
+
annual.save('annual_stats.npz')
|
|
108
|
+
plot_annual(annual, field='mean_precip', save='annual_mean.png')
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Grid is **721 x 1440**: latitudes −90 to +90 inclusive, longitudes −180 to +179.75, with +180 and −180 assigned to the same column. These are swath-sample means and counts, **not unbiased annual accumulation**. See `docs/SCIENTIFIC_LIMITATIONS.md`.
|
|
112
|
+
|
|
113
|
+
### Save the full legacy DAAC-oriented NetCDF
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from tlpr2s import save_nc
|
|
117
|
+
save_nc('TLP-R2S_orbit.nc', result=result,
|
|
118
|
+
latitude=latitude, longitude=longitude,
|
|
119
|
+
input_arrays=original_s4_fields,
|
|
120
|
+
ssmis_full_filename=original_ssmis_path.name,
|
|
121
|
+
ssmis_source_path=original_ssmis_path,
|
|
122
|
+
model_version='TLP-R2S_V01')
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The writer preserves the original product layout, SSMIS TB and ERA5 fields, and actual source scan-time metadata. **A complete source SSMIS granule is required** to recover authentic times; a filename alone is insufficient. The NetCDF schema is DAAC-oriented, not formally certified. See `docs/DAAC_NETCDF.md`.
|
|
126
|
+
|
|
127
|
+
## Command-line tools
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
tlpr2s download-models
|
|
131
|
+
tlpr2s retrieve prepared_s4_arrays.npz --output orbit.npz
|
|
132
|
+
tlpr2s plot-orbit orbit.npz --output orbit.png
|
|
133
|
+
tlpr2s annual orbit1.npz orbit2.npz --output annual.npz
|
|
134
|
+
tlpr2s plot-annual annual.npz --output annual.png
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Publish on GitHub / PyPI
|
|
138
|
+
|
|
139
|
+
See `docs/PUBLISH_GITHUB_PYPI.md`, `docs/RELEASE_GUIDE.md`, `docs/INPUT_FORMAT.md`, and `examples/`. Keep model artifacts **outside** the GitHub repository and wheel. Review licenses, model provenance, original algorithm numerical parity, and the DAAC's product requirements before publication. The example `CITATION.cff` and `LICENSE-TO-SELECT.txt` need owner review.
|
|
140
|
+
|
|
141
|
+
## GitHub upload checklist
|
|
142
|
+
|
|
143
|
+
Start with [docs/GITHUB_UPLOAD_STEPS.md](docs/GITHUB_UPLOAD_STEPS.md) and [docs/MODEL_STORAGE_AND_PROVENANCE.md](docs/MODEL_STORAGE_AND_PROVENANCE.md). The model artifacts are hosted on Hugging Face and must **not** be committed to GitHub.
|
tlpr2s-0.2.0/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# tlpr2s — SSMIS F16 transfer-learning rain and snow retrieval
|
|
2
|
+
|
|
3
|
+
**Python 3.10+ · 2D S4 swath inputs · original XGBoost models and CDF calibration · Hugging Face hosting · orbit visualization · annual grid · full legacy DAAC-oriented NetCDF**
|
|
4
|
+
|
|
5
|
+
The scientific model artifacts are hosted separately at [BuddhaSubedi/TLP-R2S](https://huggingface.co/BuddhaSubedi/TLP-R2S) (approximately 6.43 GB). **They are not included in the Python wheel.** The first model initialization downloads and caches the five original files; subsequent runs reuse the cache. No model download occurs when simply importing `tlpr2s`. The model repo must be public, or the user must authenticate with Hugging Face.
|
|
6
|
+
|
|
7
|
+
> **Scientific status:** This repository is an editable integration release. Real-orbit parity against the original pipeline and NASA DAAC acceptance have not been independently verified. The package is not NASA-endorsed. Do not publish it as validated before testing the actual trained artifacts.
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
python -m pip install -e '.[all,dev]'
|
|
13
|
+
python -m pytest -q
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
For a public release after publishing on PyPI, users would run `pip install 'tlpr2s[all]'`. The package name's availability and release are not guaranteed.
|
|
17
|
+
|
|
18
|
+
## Download models (once)
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
tlpr2s download-models
|
|
22
|
+
# Better reproducibility: pin an immutable model repository commit SHA
|
|
23
|
+
# tlpr2s download-models --revision FULL_COMMIT_SHA
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The downloader retrieves ONLY these files:
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
xgb_tl_classifier_final.json
|
|
30
|
+
xgb_tl_rain_rate_final.json
|
|
31
|
+
xgb_tl_snow_rate_final_1.json
|
|
32
|
+
cdf_match_for_rain_ssmis.npz
|
|
33
|
+
cdf_match_for_snow_ssmis.npz
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**Alternative offline/HPC approach:** download on a login node, or copy the five original files to `/scratch/models/tlpr2s/`, then set `TLPR2S_MODEL_DIR=/scratch/models/tlpr2s` or pass `model_dir=...`. To use a cached model offline, call `PrecipitationRetriever(local_files_only=True)`. To prohibit downloads altogether, call `PrecipitationRetriever(download=False, model_dir=...)`. To select a particular model revision, pass `revision='FULL_COMMIT_SHA'` or set `TLPR2S_MODEL_REVISION`.
|
|
37
|
+
|
|
38
|
+
## Python API — S4-aligned 2D arrays, not full orbit files
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from tlpr2s import FEATURES, predict, plot_orbit, plot_annual, save_nc
|
|
42
|
+
|
|
43
|
+
# inputs: dict with 30 predictor keys; each is 2D (npix,nscan),
|
|
44
|
+
# except Month and Day may be scalar. The 11 lower-frequency SSMIS
|
|
45
|
+
# TB channels and 17 ERA5 fields MUST already be mapped to the S4 grid.
|
|
46
|
+
# See docs/INPUT_FORMAT.md for the exact feature names.
|
|
47
|
+
result = predict(inputs, revision='FULL_COMMIT_SHA')
|
|
48
|
+
|
|
49
|
+
# Result arrays have same (npix,nscan) shape.
|
|
50
|
+
phase = result.phase # 0=clear, 1=rain, 2=snow
|
|
51
|
+
rain = result.rain # mm/hr
|
|
52
|
+
snow = result.snow # mm/hr
|
|
53
|
+
precip = result.surface_precip
|
|
54
|
+
|
|
55
|
+
# Latitude and longitude must be 2D arrays on the same S4 grid.
|
|
56
|
+
plot_orbit(latitude, longitude, precip, vmax=15, save='orbit.png')
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
You may also use `PrecipitationRetriever(revision='FULL_COMMIT_SHA')` once and call `.predict(inputs)` on many orbits to avoid reloading multi-GB models for each orbit. The `predict` convenience function initializes the model each call, which is inefficient for batch processing.
|
|
60
|
+
|
|
61
|
+
### Annual grid and visualization
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from tlpr2s import AnnualAccumulator, plot_annual
|
|
65
|
+
acc = AnnualAccumulator(resolution=0.25)
|
|
66
|
+
for latitude, longitude, result in retrieved_orbits:
|
|
67
|
+
acc.add(latitude, longitude, result.rain, result.snow)
|
|
68
|
+
annual = acc.result()
|
|
69
|
+
annual.save('annual_stats.npz')
|
|
70
|
+
plot_annual(annual, field='mean_precip', save='annual_mean.png')
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Grid is **721 x 1440**: latitudes −90 to +90 inclusive, longitudes −180 to +179.75, with +180 and −180 assigned to the same column. These are swath-sample means and counts, **not unbiased annual accumulation**. See `docs/SCIENTIFIC_LIMITATIONS.md`.
|
|
74
|
+
|
|
75
|
+
### Save the full legacy DAAC-oriented NetCDF
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from tlpr2s import save_nc
|
|
79
|
+
save_nc('TLP-R2S_orbit.nc', result=result,
|
|
80
|
+
latitude=latitude, longitude=longitude,
|
|
81
|
+
input_arrays=original_s4_fields,
|
|
82
|
+
ssmis_full_filename=original_ssmis_path.name,
|
|
83
|
+
ssmis_source_path=original_ssmis_path,
|
|
84
|
+
model_version='TLP-R2S_V01')
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The writer preserves the original product layout, SSMIS TB and ERA5 fields, and actual source scan-time metadata. **A complete source SSMIS granule is required** to recover authentic times; a filename alone is insufficient. The NetCDF schema is DAAC-oriented, not formally certified. See `docs/DAAC_NETCDF.md`.
|
|
88
|
+
|
|
89
|
+
## Command-line tools
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
tlpr2s download-models
|
|
93
|
+
tlpr2s retrieve prepared_s4_arrays.npz --output orbit.npz
|
|
94
|
+
tlpr2s plot-orbit orbit.npz --output orbit.png
|
|
95
|
+
tlpr2s annual orbit1.npz orbit2.npz --output annual.npz
|
|
96
|
+
tlpr2s plot-annual annual.npz --output annual.png
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Publish on GitHub / PyPI
|
|
100
|
+
|
|
101
|
+
See `docs/PUBLISH_GITHUB_PYPI.md`, `docs/RELEASE_GUIDE.md`, `docs/INPUT_FORMAT.md`, and `examples/`. Keep model artifacts **outside** the GitHub repository and wheel. Review licenses, model provenance, original algorithm numerical parity, and the DAAC's product requirements before publication. The example `CITATION.cff` and `LICENSE-TO-SELECT.txt` need owner review.
|
|
102
|
+
|
|
103
|
+
## GitHub upload checklist
|
|
104
|
+
|
|
105
|
+
Start with [docs/GITHUB_UPLOAD_STEPS.md](docs/GITHUB_UPLOAD_STEPS.md) and [docs/MODEL_STORAGE_AND_PROVENANCE.md](docs/MODEL_STORAGE_AND_PROVENANCE.md). The model artifacts are hosted on Hugging Face and must **not** be committed to GitHub.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
[project]
|
|
5
|
+
name = "tlpr2s"
|
|
6
|
+
version = "0.2.0"
|
|
7
|
+
description = "2D-array SSMIS F16 transfer-learning precipitation retrieval and swath statistics"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
authors = [{name = "TLP-R2S research team"}]
|
|
11
|
+
keywords = ["precipitation", "SSMIS", "satellite", "XGBoost", "remote sensing"]
|
|
12
|
+
dependencies = ["numpy>=1.24", "pandas>=1.5", "scipy>=1.10", "xgboost>=2.0"]
|
|
13
|
+
[project.optional-dependencies]
|
|
14
|
+
hub = ["huggingface_hub>=0.24,<2"]
|
|
15
|
+
all = ["huggingface_hub>=0.24,<2", "matplotlib>=3.7", "cartopy>=0.22", "xarray>=2023.1", "netCDF4>=1.6", "h5py>=3.8"]
|
|
16
|
+
plot = ["matplotlib>=3.7", "cartopy>=0.22"]
|
|
17
|
+
io = ["xarray>=2023.1", "netCDF4>=1.6", "h5py>=3.8"]
|
|
18
|
+
dev = ["huggingface_hub>=0.24,<2", "pytest>=7", "build", "twine", "matplotlib>=3.7", "xarray>=2023.1", "netCDF4>=1.6", "h5py>=3.8"]
|
|
19
|
+
[project.scripts]
|
|
20
|
+
tlpr2s = "tlpr2s.cli:main"
|
|
21
|
+
[tool.setuptools.packages.find]
|
|
22
|
+
where = ["src"]
|
|
23
|
+
[tool.pytest.ini_options]
|
|
24
|
+
testpaths = ["tests"]
|
|
25
|
+
|
tlpr2s-0.2.0/setup.cfg
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""TLP-R2S: SSMIS F16 S4-swath precipitation retrieval."""
|
|
2
|
+
from .model_download import download_models
|
|
3
|
+
from .features import FEATURES
|
|
4
|
+
from .arrays import feature_frame, save_swath, load_swath
|
|
5
|
+
from .retrieval import PrecipitationRetriever, RetrievalResult
|
|
6
|
+
from .annual import AnnualAccumulator, AnnualStatistics, aggregate_swaths
|
|
7
|
+
from .plotting import plot_orbit, plot_annual
|
|
8
|
+
from .daac import DAAC_compatible_nc
|
|
9
|
+
from .public import predict, save_nc
|
|
10
|
+
__version__ = "0.2.0"
|
|
11
|
+
__all__ = ["download_models", "FEATURES", "feature_frame", "save_swath", "load_swath", "PrecipitationRetriever", "RetrievalResult", "predict", "plot_orbit", "plot_annual", "save_nc", "DAAC_compatible_nc", "AnnualAccumulator", "AnnualStatistics", "aggregate_swaths"]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Streaming swath-to-grid statistics. A sampling mean, not an annual accumulation."""
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
import numpy as np
|
|
4
|
+
from .arrays import load_swath
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class AnnualStatistics:
|
|
8
|
+
latitude: np.ndarray
|
|
9
|
+
longitude: np.ndarray
|
|
10
|
+
mean_rain: np.ndarray
|
|
11
|
+
mean_snow: np.ndarray
|
|
12
|
+
mean_precip: np.ndarray
|
|
13
|
+
observation_count: np.ndarray
|
|
14
|
+
swath_coverage_count: np.ndarray
|
|
15
|
+
@property
|
|
16
|
+
def shape(self):return self.mean_precip.shape
|
|
17
|
+
def save(self,path):
|
|
18
|
+
np.savez_compressed(path,**vars(self))
|
|
19
|
+
@classmethod
|
|
20
|
+
def load(cls,path):
|
|
21
|
+
with np.load(path,allow_pickle=False) as z:return cls(**{k:z[k] for k in cls.__dataclass_fields__})
|
|
22
|
+
|
|
23
|
+
class AnnualAccumulator:
|
|
24
|
+
"""Periodic longitude grid and inclusive polar latitude nodes.
|
|
25
|
+
|
|
26
|
+
Default grid: latitude [-90,90] inclusive (721 points), longitude
|
|
27
|
+
[-180,180) (1440 points). -180 and +180 map to the SAME column.
|
|
28
|
+
Observations are assigned to nearest grid node. Each pixel counts once;
|
|
29
|
+
annual means are sampling means, not time-weighted accumulations.
|
|
30
|
+
"""
|
|
31
|
+
def __init__(self,resolution=0.25):
|
|
32
|
+
nlat=180/resolution;nlon=360/resolution
|
|
33
|
+
if int(nlat)!=nlat or int(nlon)!=nlon:
|
|
34
|
+
raise ValueError('resolution must divide 180 and 360')
|
|
35
|
+
self.resolution=float(resolution)
|
|
36
|
+
self.nlat=int(nlat)+1
|
|
37
|
+
self.nlon=int(nlon)
|
|
38
|
+
n=self.nlat*self.nlon
|
|
39
|
+
self.count=np.zeros(n,dtype=np.int64)
|
|
40
|
+
self.coverage=np.zeros(n,dtype=np.int64)
|
|
41
|
+
self.rain_sum=np.zeros(n,dtype=np.float64)
|
|
42
|
+
self.snow_sum=np.zeros(n,dtype=np.float64)
|
|
43
|
+
def add(self,latitude,longitude,rain,snow):
|
|
44
|
+
lat=np.asarray(latitude);lon=np.asarray(longitude)
|
|
45
|
+
r=np.asarray(rain);s=np.asarray(snow)
|
|
46
|
+
if lat.ndim!=2 or any(v.shape!=lat.shape for v in (lon,r,s)):
|
|
47
|
+
raise ValueError('All swath arrays must have identical 2D shape')
|
|
48
|
+
good=(np.isfinite(lat)&np.isfinite(lon)&np.isfinite(r)&np.isfinite(s)
|
|
49
|
+
&(lat>=-90)&(lat<=90)&(r>=0)&(s>=0))
|
|
50
|
+
if not good.any():return 0
|
|
51
|
+
# Round to nearest latitude node, including both poles.
|
|
52
|
+
la=np.clip(np.floor((lat[good]+90)/self.resolution+0.5).astype(np.int64),0,self.nlat-1)
|
|
53
|
+
# Periodic longitude: -180 and +180 map to column 0.
|
|
54
|
+
lo=(np.floor(((lon[good]+180)%360)/self.resolution+0.5).astype(np.int64))%self.nlon
|
|
55
|
+
idx=la*self.nlon+lo
|
|
56
|
+
self.count+=np.bincount(idx,minlength=self.count.size)
|
|
57
|
+
self.rain_sum+=np.bincount(idx,weights=r[good],minlength=self.count.size)
|
|
58
|
+
self.snow_sum+=np.bincount(idx,weights=s[good],minlength=self.count.size)
|
|
59
|
+
self.coverage[np.unique(idx)]+=1
|
|
60
|
+
return int(good.sum())
|
|
61
|
+
def add_file(self,path):
|
|
62
|
+
d=load_swath(path)
|
|
63
|
+
return self.add(d['latitude'],d['longitude'],d['rain'],d['snow'])
|
|
64
|
+
def result(self):
|
|
65
|
+
shape=(self.nlat,self.nlon)
|
|
66
|
+
def mean(total):
|
|
67
|
+
return np.divide(total,self.count,out=np.full_like(total,np.nan),where=self.count>0).reshape(shape)
|
|
68
|
+
lat=-90+np.arange(self.nlat)*self.resolution
|
|
69
|
+
lon=-180+np.arange(self.nlon)*self.resolution
|
|
70
|
+
return AnnualStatistics(lat,lon,mean(self.rain_sum),mean(self.snow_sum),mean(self.rain_sum+self.snow_sum),self.count.reshape(shape).copy(),self.coverage.reshape(shape).copy())
|
|
71
|
+
|
|
72
|
+
def aggregate_swaths(paths,resolution=0.25):
|
|
73
|
+
acc=AnnualAccumulator(resolution)
|
|
74
|
+
paths=list(paths)
|
|
75
|
+
if not paths:raise ValueError('No swath files provided')
|
|
76
|
+
for p in paths:acc.add_file(p)
|
|
77
|
+
return acc.result()
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""2D swath input validation. No files are required for inference."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pandas as pd
|
|
4
|
+
from .features import FEATURES
|
|
5
|
+
|
|
6
|
+
def feature_frame(data, feature_names=None):
|
|
7
|
+
"""Return (DataFrame, (height,width)) for dict of 2D features or (30,H,W) array.
|
|
8
|
+
|
|
9
|
+
All 30 features must be present in the exact training order. Month/Day may
|
|
10
|
+
be scalar, while other features must be 2D arrays of identical shape.
|
|
11
|
+
"""
|
|
12
|
+
if isinstance(data, np.ndarray):
|
|
13
|
+
a=np.asarray(data)
|
|
14
|
+
if a.ndim != 3 or a.shape[0] != len(FEATURES):
|
|
15
|
+
raise ValueError(f'Stacked input must have shape ({len(FEATURES)}, H, W)')
|
|
16
|
+
if feature_names is not None and list(feature_names) != FEATURES:
|
|
17
|
+
raise ValueError('feature_names must match the trained 30-feature order')
|
|
18
|
+
data={name:a[i] for i,name in enumerate(FEATURES)}
|
|
19
|
+
if not isinstance(data,dict):
|
|
20
|
+
raise TypeError('Input must be dict[str, 2D array] or ndarray (30,H,W)')
|
|
21
|
+
missing=[k for k in FEATURES if k not in data]
|
|
22
|
+
if missing:raise ValueError(f'Missing model features: {missing}')
|
|
23
|
+
extras=set(data)-set(FEATURES)
|
|
24
|
+
if extras:raise ValueError(f'Unexpected model features: {sorted(extras)}')
|
|
25
|
+
shape=None
|
|
26
|
+
for k in FEATURES:
|
|
27
|
+
a=np.asarray(data[k])
|
|
28
|
+
if a.ndim==2:
|
|
29
|
+
if shape is None:shape=a.shape
|
|
30
|
+
elif shape!=a.shape:raise ValueError(f'{k} shape {a.shape} != {shape}')
|
|
31
|
+
elif not (k in ('Month','Day') and a.ndim==0):
|
|
32
|
+
raise ValueError(f'{k} must be a 2D array'+(' or scalar' if k in ('Month','Day') else ''))
|
|
33
|
+
if shape is None or min(shape)<1:raise ValueError('No nonempty 2D features')
|
|
34
|
+
columns={}
|
|
35
|
+
for k in FEATURES:
|
|
36
|
+
a=np.asarray(data[k],dtype=np.float32)
|
|
37
|
+
if a.ndim==0:a=np.full(shape,a.item(),dtype=np.float32)
|
|
38
|
+
if k in FEATURES[:11]:a=np.where(a<0,np.nan,a)
|
|
39
|
+
columns[k]=a.ravel(order='C')
|
|
40
|
+
return pd.DataFrame(columns)[FEATURES],shape
|
|
41
|
+
|
|
42
|
+
def save_swath(path,latitude,longitude,phase,rain,snow):
|
|
43
|
+
"""Save a complete retrieved swath as portable NPZ."""
|
|
44
|
+
lat=np.asarray(latitude);lon=np.asarray(longitude)
|
|
45
|
+
if lat.shape!=lon.shape or any(np.asarray(v).shape!=lat.shape for v in (phase,rain,snow)):
|
|
46
|
+
raise ValueError('Latitude, longitude, phase, rain and snow must have identical 2D shape')
|
|
47
|
+
np.savez_compressed(path,latitude=lat,longitude=lon,phase=phase,rain=rain,snow=snow)
|
|
48
|
+
|
|
49
|
+
def load_swath(path):
|
|
50
|
+
with np.load(path,allow_pickle=False) as z:
|
|
51
|
+
required=('latitude','longitude','phase','rain','snow')
|
|
52
|
+
if any(k not in z for k in required):raise ValueError(f'NPZ must contain {required}')
|
|
53
|
+
d={k:z[k] for k in required}
|
|
54
|
+
if d['latitude'].ndim!=2 or any(v.shape!=d['latitude'].shape for v in d.values()):
|
|
55
|
+
raise ValueError('All swath variables must be matching 2D arrays')
|
|
56
|
+
return d
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Command line tools for saved array inputs, swath plots and annual gridding."""
|
|
2
|
+
import argparse
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import numpy as np
|
|
5
|
+
from .arrays import save_swath,load_swath
|
|
6
|
+
from .annual import aggregate_swaths,AnnualStatistics
|
|
7
|
+
|
|
8
|
+
def main(argv=None):
|
|
9
|
+
p=argparse.ArgumentParser(prog='tlpr2s',description='SSMIS 2D precipitation retrieval and mapping')
|
|
10
|
+
sub=p.add_subparsers(dest='cmd',required=True)
|
|
11
|
+
r=sub.add_parser('retrieve',help='Retrieve from NPZ with 30 feature arrays and latitude/longitude')
|
|
12
|
+
r.add_argument('input');r.add_argument('--models',default=None);r.add_argument('--cdf',default=None);r.add_argument('--output',required=True);r.add_argument('--threads',type=int,default=1)
|
|
13
|
+
o=sub.add_parser('plot-orbit');o.add_argument('input');o.add_argument('--output',required=True);o.add_argument('--field',choices=['rain','snow','surface_precip','phase'],default='surface_precip');o.add_argument('--vmax',type=float,default=15)
|
|
14
|
+
a=sub.add_parser('annual');a.add_argument('files',nargs='+');a.add_argument('--output',required=True);a.add_argument('--resolution',type=float,default=.25)
|
|
15
|
+
q=sub.add_parser('plot-annual');q.add_argument('input');q.add_argument('--output',required=True);q.add_argument('--field',default='mean_precip')
|
|
16
|
+
d=sub.add_parser('download-models',help='Download the five official Hugging Face artifacts into the local cache')
|
|
17
|
+
d.add_argument('--revision',default=None,help='Pin a model repository commit SHA or tag')
|
|
18
|
+
d.add_argument('--cache-dir',default=None)
|
|
19
|
+
d.add_argument('--offline',action='store_true',help='Use cache only')
|
|
20
|
+
args=p.parse_args(argv)
|
|
21
|
+
if args.cmd=='download-models':
|
|
22
|
+
from .model_download import download_models
|
|
23
|
+
print(download_models(revision=args.revision,cache_dir=args.cache_dir,local_files_only=args.offline))
|
|
24
|
+
elif args.cmd=='retrieve':
|
|
25
|
+
from .retrieval import PrecipitationRetriever
|
|
26
|
+
with np.load(args.input,allow_pickle=False) as z:
|
|
27
|
+
if 'latitude' not in z or 'longitude' not in z:raise ValueError('Need latitude and longitude in NPZ')
|
|
28
|
+
lat=z['latitude'];lon=z['longitude'];data={k:z[k] for k in z.files if k not in ('latitude','longitude')}
|
|
29
|
+
result=PrecipitationRetriever(args.models,args.cdf,args.threads).predict(data)
|
|
30
|
+
save_swath(args.output,lat,lon,result.phase,result.rain,result.snow)
|
|
31
|
+
elif args.cmd=='plot-orbit':
|
|
32
|
+
from .plotting import plot_orbit
|
|
33
|
+
d=load_swath(args.input)
|
|
34
|
+
field=d['rain']+d['snow'] if args.field=='surface_precip' else d[args.field]
|
|
35
|
+
plot_orbit(d['latitude'],d['longitude'],field,title=args.field,save=args.output,vmax=args.vmax)
|
|
36
|
+
elif args.cmd=='annual':
|
|
37
|
+
stats=aggregate_swaths(args.files,args.resolution);stats.save(args.output)
|
|
38
|
+
print(f'Saved {args.output}; valid pixels: {int(stats.observation_count.sum())}')
|
|
39
|
+
elif args.cmd=='plot-annual':
|
|
40
|
+
from .plotting import plot_annual
|
|
41
|
+
plot_annual(AnnualStatistics.load(args.input),args.field,save=args.output)
|
|
42
|
+
if __name__=='__main__':main()
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Public interface for full legacy NASA DAAC-oriented SSMIS F16 NetCDF schema.
|
|
2
|
+
|
|
3
|
+
This writer requires the actual source granule to preserve genuine S4 scan times.
|
|
4
|
+
A full source filename is required and checked against the supplied file path.
|
|
5
|
+
"""
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import numpy as np
|
|
8
|
+
from .daac_products import netcdf, SSMIS_CHANNELS, ERA5_MAT_FIELDS, ERA5_REFERENCE_MAT_FIELDS
|
|
9
|
+
from .ssmis_source import SOURCE_RE
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _as_2d(value, name, shape=None):
|
|
13
|
+
arr = np.asarray(value)
|
|
14
|
+
if arr.ndim != 2:
|
|
15
|
+
raise ValueError(f'{name} must be 2D, got {arr.shape}')
|
|
16
|
+
if shape is not None and arr.shape != shape:
|
|
17
|
+
raise ValueError(f'{name} shape {arr.shape} != {shape}')
|
|
18
|
+
return arr
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def DAAC_compatible_nc(output_path, *, result, latitude, longitude,
|
|
22
|
+
input_arrays, ssmis_full_filename, ssmis_source_path,
|
|
23
|
+
source_identifier='2D-array retrieval', model_version='TLP-R2S_V01'):
|
|
24
|
+
"""Save the complete original TLP-R2S DAAC-oriented NetCDF product.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
output_path : str or Path
|
|
29
|
+
Destination .nc file.
|
|
30
|
+
result : RetrievalResult
|
|
31
|
+
Inference output with phase, rain, snow arrays.
|
|
32
|
+
latitude, longitude : 2D array
|
|
33
|
+
SSMIS S4 coordinates, shaped (npix,nscan).
|
|
34
|
+
input_arrays : dict
|
|
35
|
+
2D arrays for ALL original SSMIS brightness-temperature and ERA5 fields.
|
|
36
|
+
Required SSMIS keys: TbS1, TbS2, TbS3, TbS4. Each is a
|
|
37
|
+
channel-stacked 3D array (npix,nscan,nchannels), OR provide individual
|
|
38
|
+
2D channel maps under keys such as '19.35V', '183.31±1H'.
|
|
39
|
+
ERA5 fields are named ERA5_tciw etc. Reference ERA5_msr and
|
|
40
|
+
ERA5_mtpr are optional, as in the original writer.
|
|
41
|
+
ssmis_full_filename : str
|
|
42
|
+
Complete original SSMIS F16 L1C granule filename; validated exactly.
|
|
43
|
+
ssmis_source_path : str or Path
|
|
44
|
+
Actual original SSMIS NetCDF file, used to read genuine S4 scan times.
|
|
45
|
+
Metadata cannot be fully populated from a filename alone.
|
|
46
|
+
source_identifier : str
|
|
47
|
+
Provenance identifier for input 2D arrays.
|
|
48
|
+
model_version : str
|
|
49
|
+
Retrieval model version.
|
|
50
|
+
|
|
51
|
+
Returns
|
|
52
|
+
-------
|
|
53
|
+
pathlib.Path
|
|
54
|
+
Written NetCDF path.
|
|
55
|
+
"""
|
|
56
|
+
if not isinstance(input_arrays, dict):
|
|
57
|
+
raise TypeError('input_arrays must be a dictionary of mapped S4 fields')
|
|
58
|
+
name=Path(ssmis_full_filename).name
|
|
59
|
+
if name != str(ssmis_full_filename):
|
|
60
|
+
raise ValueError('ssmis_full_filename must be a filename, not a path')
|
|
61
|
+
if SOURCE_RE.fullmatch(name) is None:
|
|
62
|
+
raise ValueError(f'Full SSMIS F16 source filename does not match original pattern: {name}')
|
|
63
|
+
source=Path(ssmis_source_path)
|
|
64
|
+
if source.name != name:
|
|
65
|
+
raise ValueError(f'Source file basename {source.name!r} differs from full filename {name!r}')
|
|
66
|
+
if not source.is_file():
|
|
67
|
+
raise FileNotFoundError(f'SSMIS source file required for scan-time metadata: {source}')
|
|
68
|
+
lat=_as_2d(latitude,'latitude')
|
|
69
|
+
lon=_as_2d(longitude,'longitude',lat.shape)
|
|
70
|
+
phase=_as_2d(result.phase,'phase',lat.shape)
|
|
71
|
+
rain=_as_2d(result.rain,'rain',lat.shape)
|
|
72
|
+
snow=_as_2d(result.snow,'snow',lat.shape)
|
|
73
|
+
x=dict(input_arrays)
|
|
74
|
+
x['LatS4']=lat
|
|
75
|
+
x['LonS4']=lon
|
|
76
|
+
for swath,channels in SSMIS_CHANNELS.items():
|
|
77
|
+
key='Tb'+swath
|
|
78
|
+
if key in x:
|
|
79
|
+
continue
|
|
80
|
+
absent=[ch for ch,_ in channels if ch not in x]
|
|
81
|
+
if absent:
|
|
82
|
+
raise KeyError(f'Missing {key} or mapped 2D channel fields: {absent}')
|
|
83
|
+
x[key]=np.stack([_as_2d(x[ch],ch,lat.shape) for ch,_ in channels],axis=-1)
|
|
84
|
+
missing=[key for key in ERA5_MAT_FIELDS.values() if key not in x]
|
|
85
|
+
if missing:
|
|
86
|
+
raise KeyError(f'Full DAAC product requires original ERA5 input fields: {missing}')
|
|
87
|
+
for key in list(ERA5_MAT_FIELDS.values())+list(ERA5_REFERENCE_MAT_FIELDS.values()):
|
|
88
|
+
if key in x: _as_2d(x[key],key,lat.shape)
|
|
89
|
+
output=Path(output_path)
|
|
90
|
+
netcdf(output, x, phase, rain, snow, source_identifier,
|
|
91
|
+
source_ssmis=source, model_version=model_version)
|
|
92
|
+
return output
|