ngiab-data-preprocess 4.2.2__py3-none-any.whl → 4.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
map_app/static/js/main.js CHANGED
@@ -1,34 +1,34 @@
1
1
  var colorDict = {
2
- selectedCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--selected-cat-outline'),
3
- selectedCatFill: getComputedStyle(document.documentElement).getPropertyValue('--selected-cat-fill'),
4
- upstreamCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--upstream-cat-outline'),
5
- upstreamCatFill: getComputedStyle(document.documentElement).getPropertyValue('--upstream-cat-fill'),
6
- flowlineToCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--flowline-to-cat-outline'),
7
- flowlineToNexusOutline: getComputedStyle(document.documentElement).getPropertyValue('--flowline-to-nexus-outline'),
8
- nexusOutline: getComputedStyle(document.documentElement).getPropertyValue('--nexus-outline'),
9
- nexusFill: getComputedStyle(document.documentElement).getPropertyValue('--nexus-fill'),
10
- clearFill: getComputedStyle(document.documentElement).getPropertyValue('--clear-fill')
2
+ selectedCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--selected-cat-outline'),
3
+ selectedCatFill: getComputedStyle(document.documentElement).getPropertyValue('--selected-cat-fill'),
4
+ upstreamCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--upstream-cat-outline'),
5
+ upstreamCatFill: getComputedStyle(document.documentElement).getPropertyValue('--upstream-cat-fill'),
6
+ flowlineToCatOutline: getComputedStyle(document.documentElement).getPropertyValue('--flowline-to-cat-outline'),
7
+ flowlineToNexusOutline: getComputedStyle(document.documentElement).getPropertyValue('--flowline-to-nexus-outline'),
8
+ nexusOutline: getComputedStyle(document.documentElement).getPropertyValue('--nexus-outline'),
9
+ nexusFill: getComputedStyle(document.documentElement).getPropertyValue('--nexus-fill'),
10
+ clearFill: getComputedStyle(document.documentElement).getPropertyValue('--clear-fill')
11
11
  };
12
12
 
13
13
  // A function that creates a cli command from the interface
14
14
  function create_cli_command() {
15
- const cliPrefix = document.getElementById('cli-prefix');
16
- cliPrefix.style.opacity = 1;
17
- var selected_basins = $('#selected-basins').text();
18
- var start_time = document.getElementById('start-time').value.split('T')[0];
19
- var end_time = document.getElementById('end-time').value.split('T')[0];
20
- var command = `-i ${selected_basins} --subset --start ${start_time} --end ${end_time} --forcings --realization --run`;
21
- var command_all = `-i ${selected_basins} --start ${start_time} --end ${end_time} --all`;
22
- if (selected_basins != "None - get clicking!") {
23
- $('#cli-command').text(command);
24
- }
15
+ const cliPrefix = document.getElementById("cli-prefix");
16
+ cliPrefix.style.opacity = 1;
17
+ var selected_basins = $("#selected-basins").text();
18
+ var start_time = document.getElementById("start-time").value.split("T")[0];
19
+ var end_time = document.getElementById("end-time").value.split("T")[0];
20
+ var command = `-i ${selected_basins} --subset --start ${start_time} --end ${end_time} --forcings --realization --run`;
21
+ var command_all = `-i ${selected_basins} --start ${start_time} --end ${end_time} --all`;
22
+ if (selected_basins != "None - get clicking!") {
23
+ $("#cli-command").text(command);
24
+ }
25
25
  }
26
26
 
27
- function updateCommandPrefix(){
28
- const toggleInput = document.getElementById('runcmd-toggle');
29
- const cliPrefix = document.getElementById('cli-prefix');
30
- const uvxText = "uvx --from ngiab_data_preprocess cli"
31
- const pythonText = "python -m ngiab_data_cli"
27
+ function updateCommandPrefix() {
28
+ const toggleInput = document.getElementById("runcmd-toggle");
29
+ const cliPrefix = document.getElementById("cli-prefix");
30
+ const uvxText = "uvx --from ngiab_data_preprocess cli";
31
+ const pythonText = "python -m ngiab_data_cli";
32
32
  // Set initial handle text based on the default state using data attribute
33
33
  cliPrefix.textContent = toggleInput.checked ? pythonText : uvxText;
34
34
  }
@@ -41,121 +41,240 @@ document.getElementById('end-time').addEventListener('change', create_cli_comman
41
41
 
42
42
 
43
43
  // add the PMTiles plugin to the maplibregl global.
44
- let protocol = new pmtiles.Protocol({metadata: true});
44
+ let protocol = new pmtiles.Protocol({ metadata: true });
45
45
  maplibregl.addProtocol("pmtiles", protocol.tile);
46
46
 
47
47
  // select light-style if the browser is in light mode
48
48
  // select dark-style if the browser is in dark mode
49
49
  var style = 'https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/styles/light-style.json';
50
+ var colorScheme = "light";
50
51
  if (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) {
51
52
  style = 'https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/styles/dark-style.json';
53
+ colorScheme = "dark";
52
54
  }
53
55
  var map = new maplibregl.Map({
54
- container: 'map', // container id
55
- style: style, // style URL
56
- center: [-96, 40], // starting position [lng, lat]
57
- zoom: 4 // starting zoom
56
+ container: "map", // container id
57
+ style: style, // style URL
58
+ center: [-96, 40], // starting position [lng, lat]
59
+ zoom: 4, // starting zoom
60
+ });
61
+
62
+ map.on("load", () => {
63
+ map.addSource("camels_basins", {
64
+ type: "vector",
65
+ url: "pmtiles://https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/camels.pmtiles",
66
+ });
67
+ map.addLayer({
68
+ id: "camels",
69
+ type: "line",
70
+ source: "camels_basins",
71
+ "source-layer": "camels_basins",
72
+ layout: {},
73
+ filter: ["any", ["==", "hru_id", ""]],
74
+ paint: {
75
+ "line-width": 1.5,
76
+ "line-color": ["rgba", 134, 30, 232, 1],
77
+ },
78
+ });
58
79
  });
80
+
81
+ if (colorScheme == "light") {
82
+ nwm_paint = {
83
+ "line-width": 1,
84
+ "line-color": ["rgba", 0, 0, 0, 1],
85
+ };
86
+ aorc_paint = {
87
+ "line-width": 1,
88
+ "line-color": ["rgba", 71, 58, 222, 1],
89
+ };
90
+ }
91
+ if (colorScheme == "dark") {
92
+ nwm_paint = {
93
+ "line-width": 1,
94
+ "line-color": ["rgba", 255, 255, 255, 1],
95
+ };
96
+ aorc_paint = {
97
+ "line-width": 1,
98
+ "line-color": ["rgba", 242, 252, 126, 1],
99
+ };
100
+ }
101
+
102
+
103
+ map.on("load", () => {
104
+ map.addSource("nwm_zarr_chunks", {
105
+ type: "vector",
106
+ url: "pmtiles://https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/forcing_chunks/nwm_retro_v3_zarr_chunks.pmtiles",
107
+ });
108
+ map.addSource("aorc_zarr_chunks", {
109
+ type: "vector",
110
+ url: "pmtiles://https://communityhydrofabric.s3.us-east-1.amazonaws.com/map/forcing_chunks/aorc_zarr_chunks.pmtiles",
111
+ });
112
+ map.addLayer({
113
+ id: "nwm_zarr_chunks",
114
+ type: "line",
115
+ source: "nwm_zarr_chunks",
116
+ "source-layer": "nwm_zarr_chunks",
117
+ layout: {},
118
+ filter: ["any"],
119
+ paint: nwm_paint,
120
+ });
121
+ map.addLayer({
122
+ id: "aorc_zarr_chunks",
123
+ type: "line",
124
+ source: "aorc_zarr_chunks",
125
+ "source-layer": "aorc_zarr_chunks",
126
+ layout: {},
127
+ filter: ["any"],
128
+ paint: aorc_paint,
129
+ });
130
+ });
131
+
59
132
  function update_map(cat_id, e) {
60
- $('#selected-basins').text(cat_id)
61
- map.setFilter('selected-catchments', ['any', ['in', 'divide_id', cat_id]]);
62
- map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ""]])
63
-
64
- fetch('/get_upstream_catids', {
65
- method: 'POST',
66
- headers: { 'Content-Type': 'application/json' },
67
- body: JSON.stringify(cat_id),
68
- })
69
- .then(response => response.json())
70
- .then(data => {
71
- map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ...data]]);
72
- if (data.length === 0) {
73
- new maplibregl.Popup()
74
- .setLngLat(e.lngLat)
75
- .setHTML('No upstreams')
76
- .addTo(map);
77
- }
78
- });
133
+ $('#selected-basins').text(cat_id)
134
+ map.setFilter('selected-catchments', ['any', ['in', 'divide_id', cat_id]]);
135
+ map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ""]])
136
+
137
+ fetch('/get_upstream_catids', {
138
+ method: 'POST',
139
+ headers: { 'Content-Type': 'application/json' },
140
+ body: JSON.stringify(cat_id),
141
+ })
142
+ .then(response => response.json())
143
+ .then(data => {
144
+ map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ...data]]);
145
+ if (data.length === 0) {
146
+ new maplibregl.Popup()
147
+ .setLngLat(e.lngLat)
148
+ .setHTML('No upstreams')
149
+ .addTo(map);
150
+ }
151
+ });
79
152
  }
80
153
  map.on('click', 'catchments', (e) => {
81
- cat_id = e.features[0].properties.divide_id;
82
- update_map(cat_id, e);
154
+ cat_id = e.features[0].properties.divide_id;
155
+ update_map(cat_id, e);
83
156
  });
84
157
 
85
158
  // Create a popup, but don't add it to the map yet.
86
159
  const popup = new maplibregl.Popup({
87
- closeButton: false,
88
- closeOnClick: false
160
+ closeButton: false,
161
+ closeOnClick: false
89
162
  });
90
163
 
91
164
  map.on('mouseenter', 'conus_gages', (e) => {
92
- // Change the cursor style as a UI indicator.
93
- map.getCanvas().style.cursor = 'pointer';
165
+ // Change the cursor style as a UI indicator.
166
+ map.getCanvas().style.cursor = 'pointer';
94
167
 
95
- const coordinates = e.features[0].geometry.coordinates.slice();
96
- const description = e.features[0].properties.hl_uri + "<br> click for more info";
168
+ const coordinates = e.features[0].geometry.coordinates.slice();
169
+ const description = e.features[0].properties.hl_uri + "<br> click for more info";
97
170
 
98
- // Ensure that if the map is zoomed out such that multiple
99
- // copies of the feature are visible, the popup appears
100
- // over the copy being pointed to.
101
- while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
102
- coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
103
- }
171
+ // Ensure that if the map is zoomed out such that multiple
172
+ // copies of the feature are visible, the popup appears
173
+ // over the copy being pointed to.
174
+ while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
175
+ coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
176
+ }
104
177
 
105
- // Populate the popup and set its coordinates
106
- // based on the feature found.
107
- popup.setLngLat(coordinates).setHTML(description).addTo(map);
178
+ // Populate the popup and set its coordinates
179
+ // based on the feature found.
180
+ popup.setLngLat(coordinates).setHTML(description).addTo(map);
108
181
  });
109
182
 
110
- map.on('mouseleave', 'conus_gages', () => {
111
- map.getCanvas().style.cursor = '';
112
- popup.remove();
183
+ map.on("mouseleave", "conus_gages", () => {
184
+ map.getCanvas().style.cursor = "";
185
+ popup.remove();
113
186
  });
114
187
 
115
- map.on('click', 'conus_gages', (e) => {
116
- // https://waterdata.usgs.gov/monitoring-location/02465000
117
- window.open("https://waterdata.usgs.gov/monitoring-location/" + e.features[0].properties.hl_link, '_blank');
118
- }
119
- );
188
+ map.on("click", "conus_gages", (e) => {
189
+ // https://waterdata.usgs.gov/monitoring-location/02465000
190
+ window.open(
191
+ "https://waterdata.usgs.gov/monitoring-location/" +
192
+ e.features[0].properties.hl_link,
193
+ "_blank",
194
+ );
195
+ });
120
196
  show = false;
121
197
 
122
198
  // TOGGLE BUTTON LOGIC
123
199
  function initializeToggleSwitches() {
124
- // Find all toggle switches
125
- const toggleSwitches = document.querySelectorAll('.toggle-switch');
126
- // Process each toggle switch
127
- toggleSwitches.forEach(toggleSwitch => {
128
- const toggleInput = toggleSwitch.querySelector('.toggle-input');
129
- const toggleHandle = toggleSwitch.querySelector('.toggle-handle');
130
- const leftText = toggleSwitch.querySelector('.toggle-text-left').textContent;
131
- const rightText = toggleSwitch.querySelector('.toggle-text-right').textContent;
132
- // Set initial handle text based on the default state using data attribute
133
- toggleHandle.textContent = toggleInput.checked ? rightText : leftText;
134
- // Add event listener
135
- toggleInput.addEventListener('change', function() {
136
- setTimeout(() => {
137
- if (this.checked) {
138
- toggleHandle.textContent = rightText;
139
- } else {
140
- toggleHandle.textContent = leftText;
141
- }
142
- }, 180);
143
- });
200
+ // Find all toggle switches
201
+ const toggleSwitches = document.querySelectorAll(".toggle-switch");
202
+ // Process each toggle switch
203
+ toggleSwitches.forEach((toggleSwitch) => {
204
+ const toggleInput = toggleSwitch.querySelector(".toggle-input");
205
+ const toggleHandle = toggleSwitch.querySelector(".toggle-handle");
206
+ const leftText =
207
+ toggleSwitch.querySelector(".toggle-text-left").textContent;
208
+ const rightText =
209
+ toggleSwitch.querySelector(".toggle-text-right").textContent;
210
+ // Set initial handle text based on the default state using data attribute
211
+ toggleHandle.textContent = toggleInput.checked ? rightText : leftText;
212
+ // Add event listener
213
+ toggleInput.addEventListener("change", function () {
214
+ setTimeout(() => {
215
+ if (this.checked) {
216
+ toggleHandle.textContent = rightText;
217
+ } else {
218
+ toggleHandle.textContent = leftText;
219
+ }
220
+ }, 180);
144
221
  });
222
+ });
145
223
  }
146
- document.addEventListener('DOMContentLoaded', initializeToggleSwitches);
224
+ document.addEventListener("DOMContentLoaded", initializeToggleSwitches);
225
+
226
+ showGages = false;
227
+ const toggleButtonGages = document.querySelector("#toggle-button-gages");
228
+ toggleButtonGages.addEventListener("click", () => {
229
+ if (showGages) {
230
+ map.setFilter("conus_gages", ["any", ["==", "hl_uri", ""]]);
231
+ toggleButtonGages.innerText = "Show gages";
232
+ showGages = false;
233
+ } else {
234
+ map.setFilter("conus_gages", null);
235
+ toggleButtonGages.innerText = "Hide gages";
236
+ showGages = true;
237
+ }
238
+ });
147
239
 
240
+ showCamels = false;
241
+ const toggleButtonCamels = document.querySelector("#toggle-button-camels");
242
+ toggleButtonCamels.addEventListener("click", () => {
243
+ if (showCamels) {
244
+ map.setFilter("camels", ["any", ["==", "hru_id", ""]]);
245
+ toggleButtonCamels.innerText = "Show CAMELS basins";
246
+ showCamels = false;
247
+ } else {
248
+ map.setFilter("camels", null);
249
+ toggleButtonCamels.innerText = "Hide CAMELS basins";
250
+ showCamels = true;
251
+ }
252
+ });
148
253
 
149
- show = false;
150
- const toggleButton = document.querySelector('#toggle-button');
151
- toggleButton.addEventListener('click', () => {
152
- if (show) {
153
- map.setFilter('conus_gages', ['any', ['==', 'hl_uri', ""]])
154
- toggleButton.innerText = 'Show gages';
155
- show = false;
156
- } else {
157
- map.setFilter('conus_gages', null)
158
- toggleButton.innerText = 'Hide gages';
159
- show = true;
160
- }
254
+ showNwm = false;
255
+ const toggleButtonNwm = document.querySelector("#toggle-button-nwm");
256
+ toggleButtonNwm.addEventListener("click", () => {
257
+ if (showNwm) {
258
+ map.setFilter("nwm_zarr_chunks", ["any"]);
259
+ toggleButtonNwm.innerText = "Overlay NWM chunks";
260
+ showNwm = false;
261
+ } else {
262
+ map.setFilter("nwm_zarr_chunks", null);
263
+ toggleButtonNwm.innerText = "Hide NWM chunks";
264
+ showNwm = true;
265
+ }
266
+ });
267
+
268
+ showAorc = false;
269
+ const toggleButtonAorc = document.querySelector("#toggle-button-aorc");
270
+ toggleButtonAorc.addEventListener("click", () => {
271
+ if (showAorc) {
272
+ map.setFilter("aorc_zarr_chunks", ["any"]);
273
+ toggleButtonAorc.innerText = "Overlay AORC chunks";
274
+ showAorc = false;
275
+ } else {
276
+ map.setFilter("aorc_zarr_chunks", null);
277
+ toggleButtonAorc.innerText = "Hide AORC chunks";
278
+ showAorc = true;
279
+ }
161
280
  });
@@ -22,7 +22,12 @@
22
22
 
23
23
  <main>
24
24
  <section id="map-container">
25
- <div id="map"><button id="toggle-button">Show gages</button></div>
25
+ <div id="map">
26
+ <button id="toggle-button-gages">Show gages</button>
27
+ <button id="toggle-button-camels">Show CAMELS basins</button>
28
+ <button id="toggle-button-nwm">Overlay NWM chunks</button>
29
+ <button id="toggle-button-aorc">Overlay AORC chunks</button>
30
+ </div>
26
31
 
27
32
  <div class="command-container">
28
33
  <div class="command-header">
@@ -52,6 +57,10 @@
52
57
  document.getElementById('copy-to-clip').addEventListener('click', async function() {
53
58
  const commandPrefix = document.getElementById('cli-prefix').textContent;
54
59
  const commandText = document.getElementById('cli-command').textContent;
60
+ if (commandPrefix && !commandPrefix.endsWith(' ')) {
61
+ // Ensure there's a space after the prefix
62
+ commandPrefix += ' ';
63
+ }
55
64
  const full_command = commandPrefix + commandText;
56
65
  const button = this;
57
66
 
map_app/views.py CHANGED
@@ -96,7 +96,7 @@ def get_forcings():
96
96
  gdf = gpd.read_file(paths.geopackage_path, layer="divides")
97
97
  cached_data = save_and_clip_dataset(data, gdf, start_time, end_time, paths.cached_nc_file)
98
98
 
99
- create_forcings(cached_data, paths.output_dir.stem)
99
+ create_forcings(cached_data, paths.output_dir.stem) # type: ignore
100
100
  except Exception as e:
101
101
  logger.info(f"get_forcings() failed with error: {str(e)}")
102
102
  return jsonify({"error": str(e)}), 500
@@ -1,34 +1,36 @@
1
+ from typing import Tuple
1
2
  import rich.status
2
3
 
3
4
  # add a status bar for these imports so the cli feels more responsive
4
5
  with rich.status.Status("Initializing...") as status:
5
- from data_sources.source_validation import validate_all
6
- from ngiab_data_cli.custom_logging import setup_logging, set_logging_to_critical_only
7
- from ngiab_data_cli.arguments import parse_arguments
8
- from data_processing.file_paths import file_paths
9
6
  import argparse
10
7
  import logging
11
- import time
12
- from typing import List
13
8
  import subprocess
14
9
  import time
15
- from dask.distributed import Client
16
- from data_processing.gpkg_utils import get_catid_from_point, get_cat_from_gage_id
10
+ from typing import List
11
+
12
+ import geopandas as gpd
13
+ from data_processing.create_realization import create_em_realization, create_realization
14
+ from data_processing.dask_utils import shutdown_cluster
15
+ from data_processing.dataset_utils import save_and_clip_dataset
16
+ from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
17
+ from data_processing.file_paths import file_paths
18
+ from data_processing.forcings import create_forcings
19
+ from data_processing.gpkg_utils import get_cat_from_gage_id, get_catid_from_point
17
20
  from data_processing.graph_utils import get_upstream_cats
18
21
  from data_processing.subset import subset, subset_vpu
19
- from data_processing.forcings import create_forcings
20
- from data_processing.create_realization import create_realization, create_em_realization
21
- from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
22
- from data_processing.dataset_utils import save_and_clip_dataset
23
- import geopandas as gpd
22
+ from data_sources.source_validation import validate_output_dir, validate_hydrofabric
23
+ from ngiab_data_cli.arguments import parse_arguments
24
+ from ngiab_data_cli.custom_logging import set_logging_to_critical_only, setup_logging
24
25
 
25
26
 
26
- def validate_input(args: argparse.Namespace) -> None:
27
+ def validate_input(args: argparse.Namespace) -> Tuple[str, str]:
27
28
  """Validate input arguments."""
28
29
 
29
30
  if args.vpu:
30
31
  if not args.output_name:
31
32
  args.output_name = f"vpu-{args.vpu}"
33
+ validate_output_dir()
32
34
  return args.vpu, args.output_name
33
35
 
34
36
  input_feature = args.input_feature.replace("_", "-")
@@ -51,9 +53,11 @@ def validate_input(args: argparse.Namespace) -> None:
51
53
  raise ValueError("Cannot use both --latlon and --gage options at the same time.")
52
54
 
53
55
  if args.latlon:
56
+ validate_hydrofabric()
54
57
  feature_name = get_cat_id_from_lat_lon(input_feature)
55
58
  logging.info(f"Found {feature_name} from {input_feature}")
56
59
  elif args.gage:
60
+ validate_hydrofabric()
57
61
  feature_name = get_cat_from_gage_id(input_feature)
58
62
  logging.info(f"Found {feature_name} from {input_feature}")
59
63
  else:
@@ -61,15 +65,18 @@ def validate_input(args: argparse.Namespace) -> None:
61
65
 
62
66
  if args.output_name:
63
67
  output_folder = args.output_name
68
+ validate_output_dir()
64
69
  elif args.gage:
65
70
  output_folder = input_feature
71
+ validate_output_dir()
66
72
  else:
67
73
  output_folder = feature_name
74
+ validate_output_dir()
68
75
 
69
76
  return feature_name, output_folder
70
77
 
71
78
 
72
- def get_cat_id_from_lat_lon(input_feature: str) -> List[str]:
79
+ def get_cat_id_from_lat_lon(input_feature: str) -> str:
73
80
  """Read catchment IDs from input file or return single ID."""
74
81
  if "," in input_feature:
75
82
  coords = input_feature.split(",")
@@ -79,7 +86,6 @@ def get_cat_id_from_lat_lon(input_feature: str) -> List[str]:
79
86
 
80
87
 
81
88
  def set_dependent_flags(args, paths: file_paths):
82
-
83
89
  # if validate is set, run everything that is missing
84
90
  if args.validate:
85
91
  logging.info("Running all missing steps required to run ngiab.")
@@ -122,7 +128,6 @@ def validate_run_directory(args, paths: file_paths):
122
128
 
123
129
  def main() -> None:
124
130
  setup_logging()
125
- validate_all()
126
131
  try:
127
132
  args = parse_arguments()
128
133
  if args.debug:
@@ -146,11 +151,15 @@ def main() -> None:
146
151
  subset_vpu(args.vpu, output_gpkg_path=paths.geopackage_path)
147
152
  logging.info("Subsetting complete.")
148
153
  else:
149
- logging.info(f"Subsetting hydrofabric")
154
+ logging.info("Subsetting hydrofabric")
150
155
  include_outlet = True
151
156
  if args.gage:
152
157
  include_outlet = False
153
- subset(feature_to_subset, output_gpkg_path=paths.geopackage_path, include_outlet=include_outlet)
158
+ subset(
159
+ feature_to_subset,
160
+ output_gpkg_path=paths.geopackage_path,
161
+ include_outlet=include_outlet,
162
+ )
154
163
  logging.info("Subsetting complete.")
155
164
 
156
165
  if args.forcings:
@@ -189,14 +198,6 @@ def main() -> None:
189
198
  )
190
199
  logging.info("Realization creation complete.")
191
200
 
192
- # check if the dask client is still running and close it
193
- try:
194
- client = Client.current()
195
- client.shutdown()
196
- except ValueError:
197
- # value error is raised if no client is running
198
- pass
199
-
200
201
  if args.run:
201
202
  logging.info("Running Next Gen using NGIAB...")
202
203
  # open the partitions.json file and get the number of partitions
@@ -218,7 +219,8 @@ def main() -> None:
218
219
  if args.eval:
219
220
  plot = False
220
221
  try:
221
- import seaborn, matplotlib
222
+ import matplotlib
223
+ import seaborn
222
224
 
223
225
  plot = True
224
226
  except ImportError:
@@ -254,6 +256,7 @@ def main() -> None:
254
256
  except Exception as e:
255
257
  logging.error(f"An error occurred: {str(e)}")
256
258
  raise
259
+ shutdown_cluster()
257
260
 
258
261
 
259
262
  if __name__ == "__main__":
@@ -1,6 +1,5 @@
1
1
  import argparse
2
2
  from datetime import datetime
3
- from data_processing.file_paths import file_paths
4
3
 
5
4
  # Constants
6
5
  DATE_FORMAT = "%Y-%m-%d" # used for datetime parsing
@@ -1,19 +1,17 @@
1
- from data_sources.source_validation import validate_all
2
- from ngiab_data_cli.custom_logging import setup_logging
3
- from data_processing.forcings import compute_zonal_stats
4
- from data_processing.dataset_utils import check_local_cache, save_to_cache, clip_dataset_to_bounds
5
- from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
6
- from data_processing.file_paths import file_paths
7
1
  import argparse
8
2
  import logging
3
+ import shutil
9
4
  import time
10
- import xarray as xr
11
- import geopandas as gpd
12
5
  from datetime import datetime
13
6
  from pathlib import Path
14
- import shutil
15
7
 
16
- from dask.distributed import Client, LocalCluster
8
+ import geopandas as gpd
9
+ from data_processing.dask_utils import shutdown_cluster
10
+ from data_processing.dataset_utils import check_local_cache, clip_dataset_to_bounds, save_to_cache
11
+ from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
12
+ from data_processing.forcings import compute_zonal_stats
13
+ from data_sources.source_validation import validate_all
14
+ from ngiab_data_cli.custom_logging import setup_logging
17
15
 
18
16
  # Constants
19
17
  DATE_FORMAT = "%Y-%m-%d" # used for datetime parsing
@@ -69,6 +67,7 @@ def parse_arguments() -> argparse.Namespace:
69
67
 
70
68
  return parser.parse_args()
71
69
 
70
+
72
71
  def main() -> None:
73
72
  time.sleep(0.01)
74
73
  setup_logging()
@@ -111,15 +110,7 @@ def main() -> None:
111
110
  # remove the working directory
112
111
  shutil.rmtree(forcing_working_dir)
113
112
 
114
- try:
115
- client = Client.current()
116
- except ValueError:
117
- cluster = LocalCluster()
118
- client = Client(cluster)
119
- cluster.close()
120
-
121
- # shut down the client and cluster
122
- client.close()
113
+ shutdown_cluster()
123
114
 
124
115
 
125
116
  if __name__ == "__main__":