ngiab-data-preprocess 4.3.0__tar.gz → 4.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/PKG-INFO +105 -59
- ngiab_data_preprocess-4.5.0/README.md +267 -0
- ngiab_data_preprocess-4.5.0/ciroh-bgsafe.png +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/create_realization.py +71 -180
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/dataset_utils.py +44 -10
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/file_paths.py +4 -4
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/graph_utils.py +0 -2
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/subset.py +38 -7
- ngiab_data_preprocess-4.5.0/modules/data_sources/lstm-catchment-template.yml +17 -0
- ngiab_data_preprocess-4.3.0/modules/data_sources/em-realization-template.json → ngiab_data_preprocess-4.5.0/modules/data_sources/lstm-realization-template.json +5 -8
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/ngen-routing-template.yaml +1 -1
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/source_validation.py +15 -2
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/js/data_processing.js +31 -55
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/js/main.js +0 -1
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/views.py +16 -2
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_cli/__main__.py +10 -12
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_cli/arguments.py +2 -3
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_preprocess.egg-info/PKG-INFO +105 -59
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_preprocess.egg-info/SOURCES.txt +3 -3
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_preprocess.egg-info/requires.txt +1 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/pyproject.toml +2 -1
- ngiab_data_preprocess-4.3.0/README.md +0 -222
- ngiab_data_preprocess-4.3.0/modules/data_sources/em-catchment-template.yml +0 -10
- ngiab_data_preprocess-4.3.0/modules/data_sources/em-config.yml +0 -60
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/.github/workflows/build_only.yml +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/.github/workflows/publish.yml +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/.gitignore +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/LICENSE +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/dask_utils.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/datasets.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/forcings.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/gpkg_utils.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_processing/s3fs_utils.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/cfe-nowpm-realization-template.json +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/cfe-template.ini +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/forcing_template.nc +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/noah-owp-modular-init.namelist.input +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/template.sql +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/data_sources/triggers.sql +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/__init__.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/__main__.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/css/console.css +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/css/main.css +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/css/toggle.css +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/js/console.js +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/resources/loading.gif +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/static/resources/screenshot.jpg +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/map_app/templates/index.html +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_cli/custom_logging.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_cli/forcing_cli.py +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_preprocess.egg-info/dependency_links.txt +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_preprocess.egg-info/entry_points.txt +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/modules/ngiab_data_preprocess.egg-info/top_level.txt +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/setup.cfg +0 -0
- {ngiab_data_preprocess-4.3.0 → ngiab_data_preprocess-4.5.0}/tests/test_nan_impute.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ngiab_data_preprocess
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.5.0
|
|
4
4
|
Summary: Graphical Tools for creating Next Gen Water model input data.
|
|
5
5
|
Author-email: Josh Cunningham <jcunningham8@ua.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
|
|
@@ -32,6 +32,7 @@ Requires-Dist: colorama==0.4.6
|
|
|
32
32
|
Requires-Dist: bokeh==3.5.1
|
|
33
33
|
Requires-Dist: boto3
|
|
34
34
|
Requires-Dist: numcodecs<0.16.0
|
|
35
|
+
Requires-Dist: scipy>=1.15.3
|
|
35
36
|
Provides-Extra: eval
|
|
36
37
|
Requires-Dist: ngiab_eval; extra == "eval"
|
|
37
38
|
Provides-Extra: plot
|
|
@@ -40,55 +41,92 @@ Dynamic: license-file
|
|
|
40
41
|
|
|
41
42
|
# NGIAB Data Preprocess
|
|
42
43
|
|
|
43
|
-
This repository contains tools for preparing data to run a [
|
|
44
|
+
This repository contains tools for preparing data to run a [NextGen](https://github.com/NOAA-OWP/ngen)-based simulation using [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra). The tools allow you to select a catchment of interest on an interactive map, choose a date range, and prepare the data with just a few clicks!
|
|
44
45
|
|
|
45
46
|

|
|
46
47
|
|
|
48
|
+
| | |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
|  | Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama (NA22NWS4320003). |
|
|
51
|
+
|
|
47
52
|
## Table of Contents
|
|
48
53
|
|
|
49
54
|
1. [What does this tool do?](#what-does-this-tool-do)
|
|
50
|
-
2. [
|
|
55
|
+
2. [Limitations](#limitations)
|
|
56
|
+
- [Custom realizations](#custom-realizations)
|
|
57
|
+
- [Calibration](#calibration)
|
|
51
58
|
- [Evaluation](#evaluation)
|
|
52
59
|
- [Visualisation](#visualisation)
|
|
53
60
|
3. [Requirements](#requirements)
|
|
54
|
-
4. [Installation and
|
|
61
|
+
4. [Installation and running](#installation-and-running)
|
|
55
62
|
- [Running without install](#running-without-install)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
63
|
+
- [For uv installation](#for-uv-installation)
|
|
64
|
+
- [For legacy pip installation](#for-legacy-pip-installation)
|
|
65
|
+
- [Development installation](#development-installation)
|
|
66
|
+
5. [Map interface documentation](#map-interface-documentation)
|
|
67
|
+
- [Running the map interface app](#running-the-map-interface-app)
|
|
68
|
+
- [Using the map interace](#using-the-map-interface)
|
|
69
|
+
6. [CLI documentation](#cli-documentation)
|
|
70
|
+
- [Running the CLI](#running-the-cli)
|
|
60
71
|
- [Arguments](#arguments)
|
|
61
|
-
- [Usage
|
|
72
|
+
- [Usage notes](#usage-notes)
|
|
62
73
|
- [Examples](#examples)
|
|
74
|
+
7. [Realization information](#realization-information)
|
|
75
|
+
- [NOAH + CFE](#noah--cfe)
|
|
63
76
|
|
|
64
77
|
## What does this tool do?
|
|
65
78
|
|
|
66
|
-
This tool prepares data to run a
|
|
79
|
+
This tool prepares data to run a NextGen-based simulation by creating a run package that can be used with NGIAB.
|
|
67
80
|
It uses geometry and model attributes from the [v2.2 hydrofabric](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/conus/conus_nextgen.gpkg) more information on [all data sources here](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/hfv2.2-data_model.html).
|
|
68
81
|
The raw forcing data is [nwm retrospective v3 forcing](https://noaa-nwm-retrospective-3-0-pds.s3.amazonaws.com/index.html#CONUS/zarr/forcing/) data or the [AORC 1km gridded data](https://noaa-nws-aorc-v1-1-1km.s3.amazonaws.com/index.html) depending on user input
|
|
69
82
|
|
|
70
|
-
1. **
|
|
71
|
-
2.
|
|
72
|
-
3. Creates **configuration files**
|
|
83
|
+
1. **Subsets** (delineates) everything upstream of your point of interest (catchment, gage, flowpath etc) from the hydrofabric. This subset is output as a geopackage (.gpkg).
|
|
84
|
+
2. Calculates **forcings** as a weighted mean of the gridded NWM or AORC forcings. Weights are calculated using [exact extract](https://isciences.github.io/exactextract/) and computed with numpy.
|
|
85
|
+
3. Creates **configuration files** for a default NGIAB model run.
|
|
73
86
|
- realization.json - ngen model configuration
|
|
74
87
|
- troute.yaml - routing configuration.
|
|
75
88
|
- **per catchment** model configuration
|
|
76
|
-
4. Optionally
|
|
89
|
+
4. Optionally performs a non-interactive [Docker-based NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra) run.
|
|
90
|
+
|
|
91
|
+
## Limitations
|
|
92
|
+
This tool cannot do the following:
|
|
93
|
+
|
|
94
|
+
### Custom realizations
|
|
95
|
+
This tool currently only outputs a single, default realization, which is described in "[Realization information](#realization-information)". Support for additional model configurations is planned, but not currently available.
|
|
96
|
+
|
|
97
|
+
### Calibration
|
|
98
|
+
If available, this repository will download [calibrated parameters](https://communityhydrofabric.s3.us-east-1.amazonaws.com/index.html#hydrofabrics/community/gage_parameters/) from the [Community Hydrofabric](https://github.com/CIROH-UA/community_hf_patcher) AWS S3 bucket.
|
|
99
|
+
However, many gages and catchments will not have such parameters available. In these cases, Data Preprocess will output realizations with default values.
|
|
77
100
|
|
|
78
|
-
|
|
101
|
+
For automatic calibration, please see [ngiab-cal](https://github.com/CIROH-UA/ngiab-cal), which is under active development.
|
|
79
102
|
|
|
80
103
|
### Evaluation
|
|
81
|
-
For automatic evaluation using [
|
|
104
|
+
For automatic evaluation using [TEEHR](https://github.com/RTIInternational/teehr), please run [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra) interactively using the `guide.sh` script.
|
|
82
105
|
|
|
83
106
|
### Visualisation
|
|
84
107
|
For automatic interactive visualisation, please run [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra) interactively using the `guide.sh` script
|
|
85
108
|
|
|
86
|
-
|
|
109
|
+
# Requirements
|
|
87
110
|
|
|
88
|
-
|
|
111
|
+
This tool is **officially supported** on **macOS** and **Ubuntu** (tested on 22.04 & 24.04). To use it on Windows, please install [**WSL**](https://learn.microsoft.com/en-us/windows/wsl/install).
|
|
89
112
|
|
|
90
|
-
|
|
91
|
-
|
|
113
|
+
It is also **highly recommended** to use [Astral UV](https://docs.astral.sh/uv/) to install and run this tool. Installing the project via `pip` without the use of a virtual environment creates a **severe risk** of dependency conflicts.
|
|
114
|
+
|
|
115
|
+
# Installation and running
|
|
116
|
+
|
|
117
|
+
### Running without install
|
|
118
|
+
This package supports pipx and uvx, which means you can run the tool without installing it. No virtual environment needed, just UV.
|
|
119
|
+
```bash
|
|
120
|
+
# Run these from anywhere!
|
|
121
|
+
uvx --from ngiab-data-preprocess cli --help # Running the CLI
|
|
122
|
+
uvx ngiab-prep --help # Alias for the CLI
|
|
123
|
+
uvx --from ngiab-data-preprocess map_app # Running the map interface
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### For uv installation
|
|
127
|
+
|
|
128
|
+
<details>
|
|
129
|
+
<summary>Click here to expand</summary>
|
|
92
130
|
|
|
93
131
|
```bash
|
|
94
132
|
# Install UV
|
|
@@ -111,16 +149,10 @@ uv run map_app
|
|
|
111
149
|
|
|
112
150
|
UV automatically detects any virtual environments in the current directory and will use them when you use `uv run`.
|
|
113
151
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
# run this from anywhere
|
|
118
|
-
uvx --from ngiab_data_preprocess cli --help
|
|
119
|
-
# for the map
|
|
120
|
-
uvx --from ngiab_data_preprocess map_app
|
|
121
|
-
```
|
|
152
|
+
</details>
|
|
153
|
+
|
|
154
|
+
### For legacy pip installation
|
|
122
155
|
|
|
123
|
-
## For legacy pip installation
|
|
124
156
|
<details>
|
|
125
157
|
<summary>Click here to expand</summary>
|
|
126
158
|
|
|
@@ -142,7 +174,7 @@ python -m map_app
|
|
|
142
174
|
```
|
|
143
175
|
</details>
|
|
144
176
|
|
|
145
|
-
|
|
177
|
+
### Development installation
|
|
146
178
|
|
|
147
179
|
<details>
|
|
148
180
|
<summary>Click to expand installation steps</summary>
|
|
@@ -168,11 +200,17 @@ To install and run the tool, follow these steps:
|
|
|
168
200
|
```
|
|
169
201
|
</details>
|
|
170
202
|
|
|
171
|
-
|
|
203
|
+
# Map interface documentation
|
|
204
|
+
|
|
205
|
+
## Running the map interface app
|
|
172
206
|
|
|
173
|
-
Running the
|
|
207
|
+
Running the `map_app` tool will open the app in a new browser tab.
|
|
208
|
+
|
|
209
|
+
Install-free: `uvx --from ngiab-data-preprocess map_app`
|
|
210
|
+
Installed with uv: `uv run map_app`
|
|
211
|
+
|
|
212
|
+
## Using the map interface
|
|
174
213
|
|
|
175
|
-
To use the tool:
|
|
176
214
|
1. Select the catchment you're interested in on the map.
|
|
177
215
|
2. Pick the time period you want to simulate.
|
|
178
216
|
3. Click the following buttons in order:
|
|
@@ -184,7 +222,12 @@ Once all the steps are finished, you can run NGIAB on the folder shown underneat
|
|
|
184
222
|
|
|
185
223
|
**Note:** When using the tool, the default output will be stored in the `~/ngiab_preprocess_output/<your-input-feature>/` folder. There is no overwrite protection on the folders.
|
|
186
224
|
|
|
187
|
-
# CLI
|
|
225
|
+
# CLI documentation
|
|
226
|
+
|
|
227
|
+
## Running the CLI
|
|
228
|
+
|
|
229
|
+
Install-free: `uvx ngiab-prep`
|
|
230
|
+
Installed with uv: `uv run cli`
|
|
188
231
|
|
|
189
232
|
## Arguments
|
|
190
233
|
|
|
@@ -201,11 +244,11 @@ Once all the steps are finished, you can run NGIAB on the folder shown underneat
|
|
|
201
244
|
- `-o OUTPUT_NAME`, `--output_name OUTPUT_NAME`: Name of the output folder.
|
|
202
245
|
- `--source` : The datasource you want to use, either `nwm` for retrospective v3 or `aorc`. Default is `nwm`
|
|
203
246
|
- `-D`, `--debug`: Enable debug logging.
|
|
204
|
-
- `--run`: Automatically run
|
|
205
|
-
- `--validate`: Run every missing step required to run
|
|
206
|
-
- `-a`, `--all`: Run all operations
|
|
247
|
+
- `--run`: Automatically run [NGIAB's docker distribution](https://github.com/CIROH-UA/NGIAB-CloudInfra) against the output folder.
|
|
248
|
+
- `--validate`: Run every missing step required to run NGIAB.
|
|
249
|
+
- `-a`, `--all`: Run all operations. Equivalent to `-sfr` and `--run`.
|
|
207
250
|
|
|
208
|
-
## Usage
|
|
251
|
+
## Usage notes
|
|
209
252
|
- If your input has a prefix of `gage-`, you do not need to pass `-g`.
|
|
210
253
|
- The `-l`, `-g`, `-s`, `-f`, `-r` flags can be combined like normal CLI flags. For example, to subset, generate forcings, and create a realization, you can use `-sfr` or `-s -f -r`.
|
|
211
254
|
- When using the `--all` flag, it automatically sets `subset`, `forcings`, `realization`, and `run` to `True`.
|
|
@@ -213,50 +256,53 @@ Once all the steps are finished, you can run NGIAB on the folder shown underneat
|
|
|
213
256
|
|
|
214
257
|
## Examples
|
|
215
258
|
|
|
216
|
-
|
|
259
|
+
1. Prepare everything for an NGIAB run at a given gage:
|
|
217
260
|
```bash
|
|
218
|
-
|
|
219
|
-
# add --run or replace -sfr with --all to run
|
|
261
|
+
uvx ngiab-prep -i gage-10154200 -sfr --start 2022-01-01 --end 2022-02-28
|
|
262
|
+
# add --run or replace -sfr with --all to run NGIAB, too
|
|
220
263
|
# to name the folder, add -o folder_name
|
|
221
264
|
```
|
|
222
265
|
|
|
223
|
-
|
|
266
|
+
2. Subset the hydrofabric using a catchment ID or VPU:
|
|
224
267
|
```bash
|
|
225
|
-
|
|
226
|
-
|
|
268
|
+
uvx ngiab-prep -i cat-7080 -s
|
|
269
|
+
uvx ngiab-prep --vpu 01 -s
|
|
227
270
|
```
|
|
228
271
|
|
|
229
|
-
|
|
272
|
+
3. Generate forcings using a single catchment ID:
|
|
230
273
|
```bash
|
|
231
|
-
|
|
274
|
+
uvx ngiab-prep -i cat-5173 -f --start 2022-01-01 --end 2022-02-28
|
|
232
275
|
```
|
|
233
276
|
|
|
234
|
-
|
|
277
|
+
4. Create realization using a latitude/longitude pair and output to a named folder:
|
|
235
278
|
```bash
|
|
236
|
-
|
|
279
|
+
uvx ngiab-prep -i 33.22,-87.54 -l -r --start 2022-01-01 --end 2022-02-28 -o custom_output
|
|
237
280
|
```
|
|
238
281
|
|
|
239
|
-
|
|
282
|
+
5. Perform all operations using a latitude/longitude pair:
|
|
240
283
|
```bash
|
|
241
|
-
|
|
284
|
+
uvx ngiab-prep -i 33.22,-87.54 -l -s -f -r --start 2022-01-01 --end 2022-02-28
|
|
242
285
|
```
|
|
243
286
|
|
|
244
|
-
|
|
287
|
+
6. Subset the hydrofabric using a gage ID:
|
|
245
288
|
```bash
|
|
246
|
-
|
|
289
|
+
uvx ngiab-prep -i 10154200 -g -s
|
|
247
290
|
# or
|
|
248
|
-
|
|
291
|
+
uvx ngiab-prep -i gage-10154200 -s
|
|
249
292
|
```
|
|
250
293
|
|
|
251
|
-
|
|
294
|
+
7. Generate forcings using a single gage ID:
|
|
252
295
|
```bash
|
|
253
|
-
|
|
296
|
+
uvx ngiab-prep -i 01646500 -g -f --start 2022-01-01 --end 2022-02-28
|
|
254
297
|
```
|
|
255
298
|
|
|
256
|
-
|
|
257
|
-
```bash
|
|
258
|
-
python -m ngiab_data_cli -i cat-5173 -a --start 2022-01-01 --end 2022-02-28
|
|
259
|
-
```
|
|
299
|
+
# Realization information
|
|
260
300
|
|
|
301
|
+
This tool currently offers one default realization.
|
|
261
302
|
|
|
303
|
+
## NOAH + CFE
|
|
262
304
|
|
|
305
|
+
[This realization](https://github.com/CIROH-UA/NGIAB_data_preprocess/blob/main/modules/data_sources/cfe-nowpm-realization-template.json) is intended to be roughly comparable to earlier versions of the National Water Model.
|
|
306
|
+
- [NOAH-OWP-Modular](https://github.com/NOAA-OWP/NOAH-OWP-Modular): A refactoring of Noah-MP, a land-surface model. Used to model groundwater properties.
|
|
307
|
+
- [Conceptual Functional Equivalent (CFE)](https://github.com/NOAA-OWP/CFE): A simplified conceptual approximation of versions 1.2, 2.0, and 2.1 of the National Water Model. Used to model precipitation and evaporation.
|
|
308
|
+
- [SLoTH](https://github.com/NOAA-OWP/SLoTH): A module used to feed through unchanged values. In this default configuration, it simply forces certain soil moisture and ice fraction properties to zero.
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
# NGIAB Data Preprocess
|
|
2
|
+
|
|
3
|
+
This repository contains tools for preparing data to run a [NextGen](https://github.com/NOAA-OWP/ngen)-based simulation using [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra). The tools allow you to select a catchment of interest on an interactive map, choose a date range, and prepare the data with just a few clicks!
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
| | |
|
|
8
|
+
| --- | --- |
|
|
9
|
+
|  | Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama (NA22NWS4320003). |
|
|
10
|
+
|
|
11
|
+
## Table of Contents
|
|
12
|
+
|
|
13
|
+
1. [What does this tool do?](#what-does-this-tool-do)
|
|
14
|
+
2. [Limitations](#limitations)
|
|
15
|
+
- [Custom realizations](#custom-realizations)
|
|
16
|
+
- [Calibration](#calibration)
|
|
17
|
+
- [Evaluation](#evaluation)
|
|
18
|
+
- [Visualisation](#visualisation)
|
|
19
|
+
3. [Requirements](#requirements)
|
|
20
|
+
4. [Installation and running](#installation-and-running)
|
|
21
|
+
- [Running without install](#running-without-install)
|
|
22
|
+
- [For uv installation](#for-uv-installation)
|
|
23
|
+
- [For legacy pip installation](#for-legacy-pip-installation)
|
|
24
|
+
- [Development installation](#development-installation)
|
|
25
|
+
5. [Map interface documentation](#map-interface-documentation)
|
|
26
|
+
- [Running the map interface app](#running-the-map-interface-app)
|
|
27
|
+
- [Using the map interace](#using-the-map-interface)
|
|
28
|
+
6. [CLI documentation](#cli-documentation)
|
|
29
|
+
- [Running the CLI](#running-the-cli)
|
|
30
|
+
- [Arguments](#arguments)
|
|
31
|
+
- [Usage notes](#usage-notes)
|
|
32
|
+
- [Examples](#examples)
|
|
33
|
+
7. [Realization information](#realization-information)
|
|
34
|
+
- [NOAH + CFE](#noah--cfe)
|
|
35
|
+
|
|
36
|
+
## What does this tool do?
|
|
37
|
+
|
|
38
|
+
This tool prepares data to run a NextGen-based simulation by creating a run package that can be used with NGIAB.
|
|
39
|
+
It uses geometry and model attributes from the [v2.2 hydrofabric](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/conus/conus_nextgen.gpkg) more information on [all data sources here](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/hfv2.2-data_model.html).
|
|
40
|
+
The raw forcing data is [nwm retrospective v3 forcing](https://noaa-nwm-retrospective-3-0-pds.s3.amazonaws.com/index.html#CONUS/zarr/forcing/) data or the [AORC 1km gridded data](https://noaa-nws-aorc-v1-1-1km.s3.amazonaws.com/index.html) depending on user input
|
|
41
|
+
|
|
42
|
+
1. **Subsets** (delineates) everything upstream of your point of interest (catchment, gage, flowpath etc) from the hydrofabric. This subset is output as a geopackage (.gpkg).
|
|
43
|
+
2. Calculates **forcings** as a weighted mean of the gridded NWM or AORC forcings. Weights are calculated using [exact extract](https://isciences.github.io/exactextract/) and computed with numpy.
|
|
44
|
+
3. Creates **configuration files** for a default NGIAB model run.
|
|
45
|
+
- realization.json - ngen model configuration
|
|
46
|
+
- troute.yaml - routing configuration.
|
|
47
|
+
- **per catchment** model configuration
|
|
48
|
+
4. Optionally performs a non-interactive [Docker-based NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra) run.
|
|
49
|
+
|
|
50
|
+
## Limitations
|
|
51
|
+
This tool cannot do the following:
|
|
52
|
+
|
|
53
|
+
### Custom realizations
|
|
54
|
+
This tool currently only outputs a single, default realization, which is described in "[Realization information](#realization-information)". Support for additional model configurations is planned, but not currently available.
|
|
55
|
+
|
|
56
|
+
### Calibration
|
|
57
|
+
If available, this repository will download [calibrated parameters](https://communityhydrofabric.s3.us-east-1.amazonaws.com/index.html#hydrofabrics/community/gage_parameters/) from the [Community Hydrofabric](https://github.com/CIROH-UA/community_hf_patcher) AWS S3 bucket.
|
|
58
|
+
However, many gages and catchments will not have such parameters available. In these cases, Data Preprocess will output realizations with default values.
|
|
59
|
+
|
|
60
|
+
For automatic calibration, please see [ngiab-cal](https://github.com/CIROH-UA/ngiab-cal), which is under active development.
|
|
61
|
+
|
|
62
|
+
### Evaluation
|
|
63
|
+
For automatic evaluation using [TEEHR](https://github.com/RTIInternational/teehr), please run [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra) interactively using the `guide.sh` script.
|
|
64
|
+
|
|
65
|
+
### Visualisation
|
|
66
|
+
For automatic interactive visualisation, please run [NGIAB](https://github.com/CIROH-UA/NGIAB-CloudInfra) interactively using the `guide.sh` script
|
|
67
|
+
|
|
68
|
+
# Requirements
|
|
69
|
+
|
|
70
|
+
This tool is **officially supported** on **macOS** and **Ubuntu** (tested on 22.04 & 24.04). To use it on Windows, please install [**WSL**](https://learn.microsoft.com/en-us/windows/wsl/install).
|
|
71
|
+
|
|
72
|
+
It is also **highly recommended** to use [Astral UV](https://docs.astral.sh/uv/) to install and run this tool. Installing the project via `pip` without the use of a virtual environment creates a **severe risk** of dependency conflicts.
|
|
73
|
+
|
|
74
|
+
# Installation and running
|
|
75
|
+
|
|
76
|
+
### Running without install
|
|
77
|
+
This package supports pipx and uvx, which means you can run the tool without installing it. No virtual environment needed, just UV.
|
|
78
|
+
```bash
|
|
79
|
+
# Run these from anywhere!
|
|
80
|
+
uvx --from ngiab-data-preprocess cli --help # Running the CLI
|
|
81
|
+
uvx ngiab-prep --help # Alias for the CLI
|
|
82
|
+
uvx --from ngiab-data-preprocess map_app # Running the map interface
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### For uv installation
|
|
86
|
+
|
|
87
|
+
<details>
|
|
88
|
+
<summary>Click here to expand</summary>
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
# Install UV
|
|
92
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
93
|
+
# It can be installed via pip if that fails
|
|
94
|
+
# pip install uv
|
|
95
|
+
|
|
96
|
+
# Create a virtual environment in the current directory
|
|
97
|
+
uv venv
|
|
98
|
+
|
|
99
|
+
# Install the tool in the virtual environment
|
|
100
|
+
uv pip install ngiab_data_preprocess
|
|
101
|
+
|
|
102
|
+
# To run the cli
|
|
103
|
+
uv run cli --help
|
|
104
|
+
|
|
105
|
+
# To run the map
|
|
106
|
+
uv run map_app
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
UV automatically detects any virtual environments in the current directory and will use them when you use `uv run`.
|
|
110
|
+
|
|
111
|
+
</details>
|
|
112
|
+
|
|
113
|
+
### For legacy pip installation
|
|
114
|
+
|
|
115
|
+
<details>
|
|
116
|
+
<summary>Click here to expand</summary>
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
# If you're installing this on jupyterhub / 2i2c you HAVE TO DEACTIVATE THE CONDA ENV
|
|
120
|
+
(notebook) jovyan@jupyter-user:~$ conda deactivate
|
|
121
|
+
jovyan@jupyter-user:~$
|
|
122
|
+
# The interactive map won't work on 2i2c
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
# This tool is likely to not work without a virtual environment
|
|
127
|
+
python3 -m venv .venv
|
|
128
|
+
source .venv/bin/activate
|
|
129
|
+
# installing and running the tool
|
|
130
|
+
pip install 'ngiab_data_preprocess'
|
|
131
|
+
python -m map_app
|
|
132
|
+
# CLI instructions at the bottom of the README
|
|
133
|
+
```
|
|
134
|
+
</details>
|
|
135
|
+
|
|
136
|
+
### Development installation
|
|
137
|
+
|
|
138
|
+
<details>
|
|
139
|
+
<summary>Click to expand installation steps</summary>
|
|
140
|
+
|
|
141
|
+
To install and run the tool, follow these steps:
|
|
142
|
+
|
|
143
|
+
1. Clone the repository:
|
|
144
|
+
```bash
|
|
145
|
+
git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
|
|
146
|
+
cd NGIAB_data_preprocess
|
|
147
|
+
```
|
|
148
|
+
2. Create a virtual environment:
|
|
149
|
+
```bash
|
|
150
|
+
uv venv
|
|
151
|
+
```
|
|
152
|
+
3. Install the tool:
|
|
153
|
+
```bash
|
|
154
|
+
uv pip install -e .
|
|
155
|
+
```
|
|
156
|
+
4. Run the map app:
|
|
157
|
+
```bash
|
|
158
|
+
uv run map_app
|
|
159
|
+
```
|
|
160
|
+
</details>
|
|
161
|
+
|
|
162
|
+
# Map interface documentation
|
|
163
|
+
|
|
164
|
+
## Running the map interface app
|
|
165
|
+
|
|
166
|
+
Running the `map_app` tool will open the app in a new browser tab.
|
|
167
|
+
|
|
168
|
+
Install-free: `uvx --from ngiab-data-preprocess map_app`
|
|
169
|
+
Installed with uv: `uv run map_app`
|
|
170
|
+
|
|
171
|
+
## Using the map interface
|
|
172
|
+
|
|
173
|
+
1. Select the catchment you're interested in on the map.
|
|
174
|
+
2. Pick the time period you want to simulate.
|
|
175
|
+
3. Click the following buttons in order:
|
|
176
|
+
1) Create subset gpkg
|
|
177
|
+
2) Create Forcing from Zarrs
|
|
178
|
+
3) Create Realization
|
|
179
|
+
|
|
180
|
+
Once all the steps are finished, you can run NGIAB on the folder shown underneath the subset button.
|
|
181
|
+
|
|
182
|
+
**Note:** When using the tool, the default output will be stored in the `~/ngiab_preprocess_output/<your-input-feature>/` folder. There is no overwrite protection on the folders.
|
|
183
|
+
|
|
184
|
+
# CLI documentation
|
|
185
|
+
|
|
186
|
+
## Running the CLI
|
|
187
|
+
|
|
188
|
+
Install-free: `uvx ngiab-prep`
|
|
189
|
+
Installed with uv: `uv run cli`
|
|
190
|
+
|
|
191
|
+
## Arguments
|
|
192
|
+
|
|
193
|
+
- `-h`, `--help`: Show the help message and exit.
|
|
194
|
+
- `-i INPUT_FEATURE`, `--input_feature INPUT_FEATURE`: ID of feature to subset. Providing a prefix will automatically convert to catid, e.g., cat-5173 or gage-01646500 or wb-1234.
|
|
195
|
+
- `--vpu VPU_ID` : The id of the vpu to subset e.g 01. 10 = 10L + 10U and 03 = 03N + 03S + 03W. `--help` will display all the options.
|
|
196
|
+
- `-l`, `--latlon`: Use latitude and longitude instead of catid. Expects comma-separated values via the CLI, e.g., `python -m ngiab_data_cli -i 54.33,-69.4 -l -s`.
|
|
197
|
+
- `-g`, `--gage`: Use gage ID instead of catid. Expects a single gage ID via the CLI, e.g., `python -m ngiab_data_cli -i 01646500 -g -s`.
|
|
198
|
+
- `-s`, `--subset`: Subset the hydrofabric to the given feature.
|
|
199
|
+
- `-f`, `--forcings`: Generate forcings for the given feature.
|
|
200
|
+
- `-r`, `--realization`: Create a realization for the given feature.
|
|
201
|
+
- `--start_date START_DATE`, `--start START_DATE`: Start date for forcings/realization (format YYYY-MM-DD).
|
|
202
|
+
- `--end_date END_DATE`, `--end END_DATE`: End date for forcings/realization (format YYYY-MM-DD).
|
|
203
|
+
- `-o OUTPUT_NAME`, `--output_name OUTPUT_NAME`: Name of the output folder.
|
|
204
|
+
- `--source` : The datasource you want to use, either `nwm` for retrospective v3 or `aorc`. Default is `nwm`
|
|
205
|
+
- `-D`, `--debug`: Enable debug logging.
|
|
206
|
+
- `--run`: Automatically run [NGIAB's docker distribution](https://github.com/CIROH-UA/NGIAB-CloudInfra) against the output folder.
|
|
207
|
+
- `--validate`: Run every missing step required to run NGIAB.
|
|
208
|
+
- `-a`, `--all`: Run all operations. Equivalent to `-sfr` and `--run`.
|
|
209
|
+
|
|
210
|
+
## Usage notes
|
|
211
|
+
- If your input has a prefix of `gage-`, you do not need to pass `-g`.
|
|
212
|
+
- The `-l`, `-g`, `-s`, `-f`, `-r` flags can be combined like normal CLI flags. For example, to subset, generate forcings, and create a realization, you can use `-sfr` or `-s -f -r`.
|
|
213
|
+
- When using the `--all` flag, it automatically sets `subset`, `forcings`, `realization`, and `run` to `True`.
|
|
214
|
+
- Using the `--run` flag automatically sets the `--validate` flag.
|
|
215
|
+
|
|
216
|
+
## Examples
|
|
217
|
+
|
|
218
|
+
1. Prepare everything for an NGIAB run at a given gage:
|
|
219
|
+
```bash
|
|
220
|
+
uvx ngiab-prep -i gage-10154200 -sfr --start 2022-01-01 --end 2022-02-28
|
|
221
|
+
# add --run or replace -sfr with --all to run NGIAB, too
|
|
222
|
+
# to name the folder, add -o folder_name
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
2. Subset the hydrofabric using a catchment ID or VPU:
|
|
226
|
+
```bash
|
|
227
|
+
uvx ngiab-prep -i cat-7080 -s
|
|
228
|
+
uvx ngiab-prep --vpu 01 -s
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
3. Generate forcings using a single catchment ID:
|
|
232
|
+
```bash
|
|
233
|
+
uvx ngiab-prep -i cat-5173 -f --start 2022-01-01 --end 2022-02-28
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
4. Create realization using a latitude/longitude pair and output to a named folder:
|
|
237
|
+
```bash
|
|
238
|
+
uvx ngiab-prep -i 33.22,-87.54 -l -r --start 2022-01-01 --end 2022-02-28 -o custom_output
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
5. Perform all operations using a latitude/longitude pair:
|
|
242
|
+
```bash
|
|
243
|
+
uvx ngiab-prep -i 33.22,-87.54 -l -s -f -r --start 2022-01-01 --end 2022-02-28
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
6. Subset the hydrofabric using a gage ID:
|
|
247
|
+
```bash
|
|
248
|
+
uvx ngiab-prep -i 10154200 -g -s
|
|
249
|
+
# or
|
|
250
|
+
uvx ngiab-prep -i gage-10154200 -s
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
7. Generate forcings using a single gage ID:
|
|
254
|
+
```bash
|
|
255
|
+
uvx ngiab-prep -i 01646500 -g -f --start 2022-01-01 --end 2022-02-28
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
# Realization information
|
|
259
|
+
|
|
260
|
+
This tool currently offers one default realization.
|
|
261
|
+
|
|
262
|
+
## NOAH + CFE
|
|
263
|
+
|
|
264
|
+
[This realization](https://github.com/CIROH-UA/NGIAB_data_preprocess/blob/main/modules/data_sources/cfe-nowpm-realization-template.json) is intended to be roughly comparable to earlier versions of the National Water Model.
|
|
265
|
+
- [NOAH-OWP-Modular](https://github.com/NOAA-OWP/NOAH-OWP-Modular): A refactoring of Noah-MP, a land-surface model. Used to model groundwater properties.
|
|
266
|
+
- [Conceptual Functional Equivalent (CFE)](https://github.com/NOAA-OWP/CFE): A simplified conceptual approximation of versions 1.2, 2.0, and 2.1 of the National Water Model. Used to model precipitation and evaporation.
|
|
267
|
+
- [SLoTH](https://github.com/NOAA-OWP/SLoTH): A module used to feed through unchanged values. In this default configuration, it simply forces certain soil moisture and ice fraction properties to zero.
|
|
Binary file
|