ngiab-data-preprocess 4.2.2__py3-none-any.whl → 4.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_processing/create_realization.py +15 -21
- data_processing/dask_utils.py +92 -0
- data_processing/dataset_utils.py +127 -44
- data_processing/datasets.py +18 -29
- data_processing/file_paths.py +7 -7
- data_processing/forcings.py +40 -38
- data_processing/gpkg_utils.py +13 -13
- data_processing/graph_utils.py +4 -4
- data_processing/s3fs_utils.py +1 -1
- data_processing/subset.py +1 -1
- data_sources/source_validation.py +57 -32
- map_app/__main__.py +3 -2
- map_app/static/css/main.css +14 -3
- map_app/static/js/main.js +225 -106
- map_app/templates/index.html +10 -1
- map_app/views.py +1 -1
- ngiab_data_cli/__main__.py +31 -28
- ngiab_data_cli/arguments.py +0 -1
- ngiab_data_cli/forcing_cli.py +10 -19
- {ngiab_data_preprocess-4.2.2.dist-info → ngiab_data_preprocess-4.3.0.dist-info}/METADATA +15 -11
- ngiab_data_preprocess-4.3.0.dist-info/RECORD +43 -0
- {ngiab_data_preprocess-4.2.2.dist-info → ngiab_data_preprocess-4.3.0.dist-info}/WHEEL +1 -1
- ngiab_data_preprocess-4.2.2.dist-info/RECORD +0 -42
- {ngiab_data_preprocess-4.2.2.dist-info → ngiab_data_preprocess-4.3.0.dist-info}/entry_points.txt +0 -0
- {ngiab_data_preprocess-4.2.2.dist-info → ngiab_data_preprocess-4.3.0.dist-info}/licenses/LICENSE +0 -0
- {ngiab_data_preprocess-4.2.2.dist-info → ngiab_data_preprocess-4.3.0.dist-info}/top_level.txt +0 -0
map_app/static/js/main.js
CHANGED
|
@@ -1,34 +1,34 @@
|
|
|
1
1
|
var colorDict = {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
2
|
+
selectedCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--selected-cat-outline'),
|
|
3
|
+
selectedCatFill: getComputedStyle(document.documentElement).getPropertyValue('--selected-cat-fill'),
|
|
4
|
+
upstreamCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--upstream-cat-outline'),
|
|
5
|
+
upstreamCatFill: getComputedStyle(document.documentElement).getPropertyValue('--upstream-cat-fill'),
|
|
6
|
+
flowlineToCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--flowline-to-cat-outline'),
|
|
7
|
+
flowlineToNexusOutline: getComputedStyle(document.documentElement).getPropertyValue('--flowline-to-nexus-outline'),
|
|
8
|
+
nexusOutline: getComputedStyle(document.documentElement).getPropertyValue('--nexus-outline'),
|
|
9
|
+
nexusFill: getComputedStyle(document.documentElement).getPropertyValue('--nexus-fill'),
|
|
10
|
+
clearFill: getComputedStyle(document.documentElement).getPropertyValue('--clear-fill')
|
|
11
11
|
};
|
|
12
12
|
|
|
13
13
|
// A function that creates a cli command from the interface
|
|
14
14
|
function create_cli_command() {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
15
|
+
const cliPrefix = document.getElementById("cli-prefix");
|
|
16
|
+
cliPrefix.style.opacity = 1;
|
|
17
|
+
var selected_basins = $("#selected-basins").text();
|
|
18
|
+
var start_time = document.getElementById("start-time").value.split("T")[0];
|
|
19
|
+
var end_time = document.getElementById("end-time").value.split("T")[0];
|
|
20
|
+
var command = `-i ${selected_basins} --subset --start ${start_time} --end ${end_time} --forcings --realization --run`;
|
|
21
|
+
var command_all = `-i ${selected_basins} --start ${start_time} --end ${end_time} --all`;
|
|
22
|
+
if (selected_basins != "None - get clicking!") {
|
|
23
|
+
$("#cli-command").text(command);
|
|
24
|
+
}
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
function updateCommandPrefix(){
|
|
28
|
-
const toggleInput = document.getElementById(
|
|
29
|
-
const cliPrefix = document.getElementById(
|
|
30
|
-
const uvxText = "uvx --from ngiab_data_preprocess cli"
|
|
31
|
-
const pythonText = "python -m ngiab_data_cli"
|
|
27
|
+
function updateCommandPrefix() {
|
|
28
|
+
const toggleInput = document.getElementById("runcmd-toggle");
|
|
29
|
+
const cliPrefix = document.getElementById("cli-prefix");
|
|
30
|
+
const uvxText = "uvx --from ngiab_data_preprocess cli";
|
|
31
|
+
const pythonText = "python -m ngiab_data_cli";
|
|
32
32
|
// Set initial handle text based on the default state using data attribute
|
|
33
33
|
cliPrefix.textContent = toggleInput.checked ? pythonText : uvxText;
|
|
34
34
|
}
|
|
@@ -41,121 +41,240 @@ document.getElementById('end-time').addEventListener('change', create_cli_comman
|
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
// add the PMTiles plugin to the maplibregl global.
|
|
44
|
-
let protocol = new pmtiles.Protocol({metadata: true});
|
|
44
|
+
let protocol = new pmtiles.Protocol({ metadata: true });
|
|
45
45
|
maplibregl.addProtocol("pmtiles", protocol.tile);
|
|
46
46
|
|
|
47
47
|
// select light-style if the browser is in light mode
|
|
48
48
|
// select dark-style if the browser is in dark mode
|
|
49
49
|
var style = 'https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/styles/light-style.json';
|
|
50
|
+
var colorScheme = "light";
|
|
50
51
|
if (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) {
|
|
51
52
|
style = 'https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/styles/dark-style.json';
|
|
53
|
+
colorScheme = "dark";
|
|
52
54
|
}
|
|
53
55
|
var map = new maplibregl.Map({
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
container: "map", // container id
|
|
57
|
+
style: style, // style URL
|
|
58
|
+
center: [-96, 40], // starting position [lng, lat]
|
|
59
|
+
zoom: 4, // starting zoom
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
map.on("load", () => {
|
|
63
|
+
map.addSource("camels_basins", {
|
|
64
|
+
type: "vector",
|
|
65
|
+
url: "pmtiles://https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/camels.pmtiles",
|
|
66
|
+
});
|
|
67
|
+
map.addLayer({
|
|
68
|
+
id: "camels",
|
|
69
|
+
type: "line",
|
|
70
|
+
source: "camels_basins",
|
|
71
|
+
"source-layer": "camels_basins",
|
|
72
|
+
layout: {},
|
|
73
|
+
filter: ["any", ["==", "hru_id", ""]],
|
|
74
|
+
paint: {
|
|
75
|
+
"line-width": 1.5,
|
|
76
|
+
"line-color": ["rgba", 134, 30, 232, 1],
|
|
77
|
+
},
|
|
78
|
+
});
|
|
58
79
|
});
|
|
80
|
+
|
|
81
|
+
if (colorScheme == "light") {
|
|
82
|
+
nwm_paint = {
|
|
83
|
+
"line-width": 1,
|
|
84
|
+
"line-color": ["rgba", 0, 0, 0, 1],
|
|
85
|
+
};
|
|
86
|
+
aorc_paint = {
|
|
87
|
+
"line-width": 1,
|
|
88
|
+
"line-color": ["rgba", 71, 58, 222, 1],
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (colorScheme == "dark") {
|
|
92
|
+
nwm_paint = {
|
|
93
|
+
"line-width": 1,
|
|
94
|
+
"line-color": ["rgba", 255, 255, 255, 1],
|
|
95
|
+
};
|
|
96
|
+
aorc_paint = {
|
|
97
|
+
"line-width": 1,
|
|
98
|
+
"line-color": ["rgba", 242, 252, 126, 1],
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
map.on("load", () => {
|
|
104
|
+
map.addSource("nwm_zarr_chunks", {
|
|
105
|
+
type: "vector",
|
|
106
|
+
url: "pmtiles://https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/forcing_chunks/nwm_retro_v3_zarr_chunks.pmtiles",
|
|
107
|
+
});
|
|
108
|
+
map.addSource("aorc_zarr_chunks", {
|
|
109
|
+
type: "vector",
|
|
110
|
+
url: "pmtiles://https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/forcing_chunks/aorc_zarr_chunks.pmtiles",
|
|
111
|
+
});
|
|
112
|
+
map.addLayer({
|
|
113
|
+
id: "nwm_zarr_chunks",
|
|
114
|
+
type: "line",
|
|
115
|
+
source: "nwm_zarr_chunks",
|
|
116
|
+
"source-layer": "nwm_zarr_chunks",
|
|
117
|
+
layout: {},
|
|
118
|
+
filter: ["any"],
|
|
119
|
+
paint: nwm_paint,
|
|
120
|
+
});
|
|
121
|
+
map.addLayer({
|
|
122
|
+
id: "aorc_zarr_chunks",
|
|
123
|
+
type: "line",
|
|
124
|
+
source: "aorc_zarr_chunks",
|
|
125
|
+
"source-layer": "aorc_zarr_chunks",
|
|
126
|
+
layout: {},
|
|
127
|
+
filter: ["any"],
|
|
128
|
+
paint: aorc_paint,
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
59
132
|
function update_map(cat_id, e) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
133
|
+
$('#selected-basins').text(cat_id)
|
|
134
|
+
map.setFilter('selected-catchments', ['any', ['in', 'divide_id', cat_id]]);
|
|
135
|
+
map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ""]])
|
|
136
|
+
|
|
137
|
+
fetch('/get_upstream_catids', {
|
|
138
|
+
method: 'POST',
|
|
139
|
+
headers: { 'Content-Type': 'application/json' },
|
|
140
|
+
body: JSON.stringify(cat_id),
|
|
141
|
+
})
|
|
142
|
+
.then(response => response.json())
|
|
143
|
+
.then(data => {
|
|
144
|
+
map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ...data]]);
|
|
145
|
+
if (data.length === 0) {
|
|
146
|
+
new maplibregl.Popup()
|
|
147
|
+
.setLngLat(e.lngLat)
|
|
148
|
+
.setHTML('No upstreams')
|
|
149
|
+
.addTo(map);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
79
152
|
}
|
|
80
153
|
map.on('click', 'catchments', (e) => {
|
|
81
|
-
|
|
82
|
-
|
|
154
|
+
cat_id = e.features[0].properties.divide_id;
|
|
155
|
+
update_map(cat_id, e);
|
|
83
156
|
});
|
|
84
157
|
|
|
85
158
|
// Create a popup, but don't add it to the map yet.
|
|
86
159
|
const popup = new maplibregl.Popup({
|
|
87
|
-
|
|
88
|
-
|
|
160
|
+
closeButton: false,
|
|
161
|
+
closeOnClick: false
|
|
89
162
|
});
|
|
90
163
|
|
|
91
164
|
map.on('mouseenter', 'conus_gages', (e) => {
|
|
92
|
-
|
|
93
|
-
|
|
165
|
+
// Change the cursor style as a UI indicator.
|
|
166
|
+
map.getCanvas().style.cursor = 'pointer';
|
|
94
167
|
|
|
95
|
-
|
|
96
|
-
|
|
168
|
+
const coordinates = e.features[0].geometry.coordinates.slice();
|
|
169
|
+
const description = e.features[0].properties.hl_uri + "<br> click for more info";
|
|
97
170
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
171
|
+
// Ensure that if the map is zoomed out such that multiple
|
|
172
|
+
// copies of the feature are visible, the popup appears
|
|
173
|
+
// over the copy being pointed to.
|
|
174
|
+
while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
|
|
175
|
+
coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
|
|
176
|
+
}
|
|
104
177
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
178
|
+
// Populate the popup and set its coordinates
|
|
179
|
+
// based on the feature found.
|
|
180
|
+
popup.setLngLat(coordinates).setHTML(description).addTo(map);
|
|
108
181
|
});
|
|
109
182
|
|
|
110
|
-
map.on(
|
|
111
|
-
|
|
112
|
-
|
|
183
|
+
map.on("mouseleave", "conus_gages", () => {
|
|
184
|
+
map.getCanvas().style.cursor = "";
|
|
185
|
+
popup.remove();
|
|
113
186
|
});
|
|
114
187
|
|
|
115
|
-
map.on(
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
188
|
+
map.on("click", "conus_gages", (e) => {
|
|
189
|
+
// https://waterdata.usgs.gov/monitoring-location/02465000
|
|
190
|
+
window.open(
|
|
191
|
+
"https://waterdata.usgs.gov/monitoring-location/" +
|
|
192
|
+
e.features[0].properties.hl_link,
|
|
193
|
+
"_blank",
|
|
194
|
+
);
|
|
195
|
+
});
|
|
120
196
|
show = false;
|
|
121
197
|
|
|
122
198
|
// TOGGLE BUTTON LOGIC
|
|
123
199
|
function initializeToggleSwitches() {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
}
|
|
200
|
+
// Find all toggle switches
|
|
201
|
+
const toggleSwitches = document.querySelectorAll(".toggle-switch");
|
|
202
|
+
// Process each toggle switch
|
|
203
|
+
toggleSwitches.forEach((toggleSwitch) => {
|
|
204
|
+
const toggleInput = toggleSwitch.querySelector(".toggle-input");
|
|
205
|
+
const toggleHandle = toggleSwitch.querySelector(".toggle-handle");
|
|
206
|
+
const leftText =
|
|
207
|
+
toggleSwitch.querySelector(".toggle-text-left").textContent;
|
|
208
|
+
const rightText =
|
|
209
|
+
toggleSwitch.querySelector(".toggle-text-right").textContent;
|
|
210
|
+
// Set initial handle text based on the default state using data attribute
|
|
211
|
+
toggleHandle.textContent = toggleInput.checked ? rightText : leftText;
|
|
212
|
+
// Add event listener
|
|
213
|
+
toggleInput.addEventListener("change", function () {
|
|
214
|
+
setTimeout(() => {
|
|
215
|
+
if (this.checked) {
|
|
216
|
+
toggleHandle.textContent = rightText;
|
|
217
|
+
} else {
|
|
218
|
+
toggleHandle.textContent = leftText;
|
|
219
|
+
}
|
|
220
|
+
}, 180);
|
|
144
221
|
});
|
|
222
|
+
});
|
|
145
223
|
}
|
|
146
|
-
document.addEventListener(
|
|
224
|
+
document.addEventListener("DOMContentLoaded", initializeToggleSwitches);
|
|
225
|
+
|
|
226
|
+
showGages = false;
|
|
227
|
+
const toggleButtonGages = document.querySelector("#toggle-button-gages");
|
|
228
|
+
toggleButtonGages.addEventListener("click", () => {
|
|
229
|
+
if (showGages) {
|
|
230
|
+
map.setFilter("conus_gages", ["any", ["==", "hl_uri", ""]]);
|
|
231
|
+
toggleButtonGages.innerText = "Show gages";
|
|
232
|
+
showGages = false;
|
|
233
|
+
} else {
|
|
234
|
+
map.setFilter("conus_gages", null);
|
|
235
|
+
toggleButtonGages.innerText = "Hide gages";
|
|
236
|
+
showGages = true;
|
|
237
|
+
}
|
|
238
|
+
});
|
|
147
239
|
|
|
240
|
+
showCamels = false;
|
|
241
|
+
const toggleButtonCamels = document.querySelector("#toggle-button-camels");
|
|
242
|
+
toggleButtonCamels.addEventListener("click", () => {
|
|
243
|
+
if (showCamels) {
|
|
244
|
+
map.setFilter("camels", ["any", ["==", "hru_id", ""]]);
|
|
245
|
+
toggleButtonCamels.innerText = "Show CAMELS basins";
|
|
246
|
+
showCamels = false;
|
|
247
|
+
} else {
|
|
248
|
+
map.setFilter("camels", null);
|
|
249
|
+
toggleButtonCamels.innerText = "Hide CAMELS basins";
|
|
250
|
+
showCamels = true;
|
|
251
|
+
}
|
|
252
|
+
});
|
|
148
253
|
|
|
149
|
-
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
254
|
+
showNwm = false;
|
|
255
|
+
const toggleButtonNwm = document.querySelector("#toggle-button-nwm");
|
|
256
|
+
toggleButtonNwm.addEventListener("click", () => {
|
|
257
|
+
if (showNwm) {
|
|
258
|
+
map.setFilter("nwm_zarr_chunks", ["any"]);
|
|
259
|
+
toggleButtonNwm.innerText = "Overlay NWM chunks";
|
|
260
|
+
showNwm = false;
|
|
261
|
+
} else {
|
|
262
|
+
map.setFilter("nwm_zarr_chunks", null);
|
|
263
|
+
toggleButtonNwm.innerText = "Hide NWM chunks";
|
|
264
|
+
showNwm = true;
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
showAorc = false;
|
|
269
|
+
const toggleButtonAorc = document.querySelector("#toggle-button-aorc");
|
|
270
|
+
toggleButtonAorc.addEventListener("click", () => {
|
|
271
|
+
if (showAorc) {
|
|
272
|
+
map.setFilter("aorc_zarr_chunks", ["any"]);
|
|
273
|
+
toggleButtonAorc.innerText = "Overlay AORC chunks";
|
|
274
|
+
showAorc = false;
|
|
275
|
+
} else {
|
|
276
|
+
map.setFilter("aorc_zarr_chunks", null);
|
|
277
|
+
toggleButtonAorc.innerText = "Hide AORC chunks";
|
|
278
|
+
showAorc = true;
|
|
279
|
+
}
|
|
161
280
|
});
|
map_app/templates/index.html
CHANGED
|
@@ -22,7 +22,12 @@
|
|
|
22
22
|
|
|
23
23
|
<main>
|
|
24
24
|
<section id="map-container">
|
|
25
|
-
<div id="map"
|
|
25
|
+
<div id="map">
|
|
26
|
+
<button id="toggle-button-gages">Show gages</button>
|
|
27
|
+
<button id="toggle-button-camels">Show CAMELS basins</button>
|
|
28
|
+
<button id="toggle-button-nwm">Overlay NWM chunks</button>
|
|
29
|
+
<button id="toggle-button-aorc">Overlay AORC chunks</button>
|
|
30
|
+
</div>
|
|
26
31
|
|
|
27
32
|
<div class="command-container">
|
|
28
33
|
<div class="command-header">
|
|
@@ -52,6 +57,10 @@
|
|
|
52
57
|
document.getElementById('copy-to-clip').addEventListener('click', async function() {
|
|
53
58
|
const commandPrefix = document.getElementById('cli-prefix').textContent;
|
|
54
59
|
const commandText = document.getElementById('cli-command').textContent;
|
|
60
|
+
if (commandPrefix && !commandPrefix.endsWith(' ')) {
|
|
61
|
+
// Ensure there's a space after the prefix
|
|
62
|
+
commandPrefix += ' ';
|
|
63
|
+
}
|
|
55
64
|
const full_command = commandPrefix + commandText;
|
|
56
65
|
const button = this;
|
|
57
66
|
|
map_app/views.py
CHANGED
|
@@ -96,7 +96,7 @@ def get_forcings():
|
|
|
96
96
|
gdf = gpd.read_file(paths.geopackage_path, layer="divides")
|
|
97
97
|
cached_data = save_and_clip_dataset(data, gdf, start_time, end_time, paths.cached_nc_file)
|
|
98
98
|
|
|
99
|
-
create_forcings(cached_data, paths.output_dir.stem)
|
|
99
|
+
create_forcings(cached_data, paths.output_dir.stem) # type: ignore
|
|
100
100
|
except Exception as e:
|
|
101
101
|
logger.info(f"get_forcings() failed with error: {str(e)}")
|
|
102
102
|
return jsonify({"error": str(e)}), 500
|
ngiab_data_cli/__main__.py
CHANGED
|
@@ -1,34 +1,36 @@
|
|
|
1
|
+
from typing import Tuple
|
|
1
2
|
import rich.status
|
|
2
3
|
|
|
3
4
|
# add a status bar for these imports so the cli feels more responsive
|
|
4
5
|
with rich.status.Status("Initializing...") as status:
|
|
5
|
-
from data_sources.source_validation import validate_all
|
|
6
|
-
from ngiab_data_cli.custom_logging import setup_logging, set_logging_to_critical_only
|
|
7
|
-
from ngiab_data_cli.arguments import parse_arguments
|
|
8
|
-
from data_processing.file_paths import file_paths
|
|
9
6
|
import argparse
|
|
10
7
|
import logging
|
|
11
|
-
import time
|
|
12
|
-
from typing import List
|
|
13
8
|
import subprocess
|
|
14
9
|
import time
|
|
15
|
-
from
|
|
16
|
-
|
|
10
|
+
from typing import List
|
|
11
|
+
|
|
12
|
+
import geopandas as gpd
|
|
13
|
+
from data_processing.create_realization import create_em_realization, create_realization
|
|
14
|
+
from data_processing.dask_utils import shutdown_cluster
|
|
15
|
+
from data_processing.dataset_utils import save_and_clip_dataset
|
|
16
|
+
from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
|
|
17
|
+
from data_processing.file_paths import file_paths
|
|
18
|
+
from data_processing.forcings import create_forcings
|
|
19
|
+
from data_processing.gpkg_utils import get_cat_from_gage_id, get_catid_from_point
|
|
17
20
|
from data_processing.graph_utils import get_upstream_cats
|
|
18
21
|
from data_processing.subset import subset, subset_vpu
|
|
19
|
-
from
|
|
20
|
-
from
|
|
21
|
-
from
|
|
22
|
-
from data_processing.dataset_utils import save_and_clip_dataset
|
|
23
|
-
import geopandas as gpd
|
|
22
|
+
from data_sources.source_validation import validate_output_dir, validate_hydrofabric
|
|
23
|
+
from ngiab_data_cli.arguments import parse_arguments
|
|
24
|
+
from ngiab_data_cli.custom_logging import set_logging_to_critical_only, setup_logging
|
|
24
25
|
|
|
25
26
|
|
|
26
|
-
def validate_input(args: argparse.Namespace) ->
|
|
27
|
+
def validate_input(args: argparse.Namespace) -> Tuple[str, str]:
|
|
27
28
|
"""Validate input arguments."""
|
|
28
29
|
|
|
29
30
|
if args.vpu:
|
|
30
31
|
if not args.output_name:
|
|
31
32
|
args.output_name = f"vpu-{args.vpu}"
|
|
33
|
+
validate_output_dir()
|
|
32
34
|
return args.vpu, args.output_name
|
|
33
35
|
|
|
34
36
|
input_feature = args.input_feature.replace("_", "-")
|
|
@@ -51,9 +53,11 @@ def validate_input(args: argparse.Namespace) -> None:
|
|
|
51
53
|
raise ValueError("Cannot use both --latlon and --gage options at the same time.")
|
|
52
54
|
|
|
53
55
|
if args.latlon:
|
|
56
|
+
validate_hydrofabric()
|
|
54
57
|
feature_name = get_cat_id_from_lat_lon(input_feature)
|
|
55
58
|
logging.info(f"Found {feature_name} from {input_feature}")
|
|
56
59
|
elif args.gage:
|
|
60
|
+
validate_hydrofabric()
|
|
57
61
|
feature_name = get_cat_from_gage_id(input_feature)
|
|
58
62
|
logging.info(f"Found {feature_name} from {input_feature}")
|
|
59
63
|
else:
|
|
@@ -61,15 +65,18 @@ def validate_input(args: argparse.Namespace) -> None:
|
|
|
61
65
|
|
|
62
66
|
if args.output_name:
|
|
63
67
|
output_folder = args.output_name
|
|
68
|
+
validate_output_dir()
|
|
64
69
|
elif args.gage:
|
|
65
70
|
output_folder = input_feature
|
|
71
|
+
validate_output_dir()
|
|
66
72
|
else:
|
|
67
73
|
output_folder = feature_name
|
|
74
|
+
validate_output_dir()
|
|
68
75
|
|
|
69
76
|
return feature_name, output_folder
|
|
70
77
|
|
|
71
78
|
|
|
72
|
-
def get_cat_id_from_lat_lon(input_feature: str) ->
|
|
79
|
+
def get_cat_id_from_lat_lon(input_feature: str) -> str:
|
|
73
80
|
"""Read catchment IDs from input file or return single ID."""
|
|
74
81
|
if "," in input_feature:
|
|
75
82
|
coords = input_feature.split(",")
|
|
@@ -79,7 +86,6 @@ def get_cat_id_from_lat_lon(input_feature: str) -> List[str]:
|
|
|
79
86
|
|
|
80
87
|
|
|
81
88
|
def set_dependent_flags(args, paths: file_paths):
|
|
82
|
-
|
|
83
89
|
# if validate is set, run everything that is missing
|
|
84
90
|
if args.validate:
|
|
85
91
|
logging.info("Running all missing steps required to run ngiab.")
|
|
@@ -122,7 +128,6 @@ def validate_run_directory(args, paths: file_paths):
|
|
|
122
128
|
|
|
123
129
|
def main() -> None:
|
|
124
130
|
setup_logging()
|
|
125
|
-
validate_all()
|
|
126
131
|
try:
|
|
127
132
|
args = parse_arguments()
|
|
128
133
|
if args.debug:
|
|
@@ -146,11 +151,15 @@ def main() -> None:
|
|
|
146
151
|
subset_vpu(args.vpu, output_gpkg_path=paths.geopackage_path)
|
|
147
152
|
logging.info("Subsetting complete.")
|
|
148
153
|
else:
|
|
149
|
-
logging.info(
|
|
154
|
+
logging.info("Subsetting hydrofabric")
|
|
150
155
|
include_outlet = True
|
|
151
156
|
if args.gage:
|
|
152
157
|
include_outlet = False
|
|
153
|
-
subset(
|
|
158
|
+
subset(
|
|
159
|
+
feature_to_subset,
|
|
160
|
+
output_gpkg_path=paths.geopackage_path,
|
|
161
|
+
include_outlet=include_outlet,
|
|
162
|
+
)
|
|
154
163
|
logging.info("Subsetting complete.")
|
|
155
164
|
|
|
156
165
|
if args.forcings:
|
|
@@ -189,14 +198,6 @@ def main() -> None:
|
|
|
189
198
|
)
|
|
190
199
|
logging.info("Realization creation complete.")
|
|
191
200
|
|
|
192
|
-
# check if the dask client is still running and close it
|
|
193
|
-
try:
|
|
194
|
-
client = Client.current()
|
|
195
|
-
client.shutdown()
|
|
196
|
-
except ValueError:
|
|
197
|
-
# value error is raised if no client is running
|
|
198
|
-
pass
|
|
199
|
-
|
|
200
201
|
if args.run:
|
|
201
202
|
logging.info("Running Next Gen using NGIAB...")
|
|
202
203
|
# open the partitions.json file and get the number of partitions
|
|
@@ -218,7 +219,8 @@ def main() -> None:
|
|
|
218
219
|
if args.eval:
|
|
219
220
|
plot = False
|
|
220
221
|
try:
|
|
221
|
-
import
|
|
222
|
+
import matplotlib
|
|
223
|
+
import seaborn
|
|
222
224
|
|
|
223
225
|
plot = True
|
|
224
226
|
except ImportError:
|
|
@@ -254,6 +256,7 @@ def main() -> None:
|
|
|
254
256
|
except Exception as e:
|
|
255
257
|
logging.error(f"An error occurred: {str(e)}")
|
|
256
258
|
raise
|
|
259
|
+
shutdown_cluster()
|
|
257
260
|
|
|
258
261
|
|
|
259
262
|
if __name__ == "__main__":
|
ngiab_data_cli/arguments.py
CHANGED
ngiab_data_cli/forcing_cli.py
CHANGED
|
@@ -1,19 +1,17 @@
|
|
|
1
|
-
from data_sources.source_validation import validate_all
|
|
2
|
-
from ngiab_data_cli.custom_logging import setup_logging
|
|
3
|
-
from data_processing.forcings import compute_zonal_stats
|
|
4
|
-
from data_processing.dataset_utils import check_local_cache, save_to_cache, clip_dataset_to_bounds
|
|
5
|
-
from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
|
|
6
|
-
from data_processing.file_paths import file_paths
|
|
7
1
|
import argparse
|
|
8
2
|
import logging
|
|
3
|
+
import shutil
|
|
9
4
|
import time
|
|
10
|
-
import xarray as xr
|
|
11
|
-
import geopandas as gpd
|
|
12
5
|
from datetime import datetime
|
|
13
6
|
from pathlib import Path
|
|
14
|
-
import shutil
|
|
15
7
|
|
|
16
|
-
|
|
8
|
+
import geopandas as gpd
|
|
9
|
+
from data_processing.dask_utils import shutdown_cluster
|
|
10
|
+
from data_processing.dataset_utils import check_local_cache, clip_dataset_to_bounds, save_to_cache
|
|
11
|
+
from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
|
|
12
|
+
from data_processing.forcings import compute_zonal_stats
|
|
13
|
+
from data_sources.source_validation import validate_all
|
|
14
|
+
from ngiab_data_cli.custom_logging import setup_logging
|
|
17
15
|
|
|
18
16
|
# Constants
|
|
19
17
|
DATE_FORMAT = "%Y-%m-%d" # used for datetime parsing
|
|
@@ -69,6 +67,7 @@ def parse_arguments() -> argparse.Namespace:
|
|
|
69
67
|
|
|
70
68
|
return parser.parse_args()
|
|
71
69
|
|
|
70
|
+
|
|
72
71
|
def main() -> None:
|
|
73
72
|
time.sleep(0.01)
|
|
74
73
|
setup_logging()
|
|
@@ -111,15 +110,7 @@ def main() -> None:
|
|
|
111
110
|
# remove the working directory
|
|
112
111
|
shutil.rmtree(forcing_working_dir)
|
|
113
112
|
|
|
114
|
-
|
|
115
|
-
client = Client.current()
|
|
116
|
-
except ValueError:
|
|
117
|
-
cluster = LocalCluster()
|
|
118
|
-
client = Client(cluster)
|
|
119
|
-
cluster.close()
|
|
120
|
-
|
|
121
|
-
# shut down the client and cluster
|
|
122
|
-
client.close()
|
|
113
|
+
shutdown_cluster()
|
|
123
114
|
|
|
124
115
|
|
|
125
116
|
if __name__ == "__main__":
|