pyPRMS 0.9.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. pyPRMS/Exceptions_custom.py +31 -0
  2. pyPRMS/__init__.py +52 -0
  3. pyPRMS/cbh/Cbh.py +431 -0
  4. pyPRMS/cbh/CbhAscii.py +458 -0
  5. pyPRMS/cbh/CbhNetcdf.py +199 -0
  6. pyPRMS/cbh/__init__.py +3 -0
  7. pyPRMS/constants.py +131 -0
  8. pyPRMS/control/Control.py +362 -0
  9. pyPRMS/control/ControlFile.py +161 -0
  10. pyPRMS/control/ControlVariable.py +208 -0
  11. pyPRMS/control/__init__.py +3 -0
  12. pyPRMS/dimensions/Dimension.py +154 -0
  13. pyPRMS/dimensions/Dimensions.py +256 -0
  14. pyPRMS/dimensions/__init__.py +2 -0
  15. pyPRMS/input/DataFile.py +354 -0
  16. pyPRMS/input/InputVariable.py +61 -0
  17. pyPRMS/input/__init__.py +0 -0
  18. pyPRMS/metadata/__init__.py +1 -0
  19. pyPRMS/metadata/metadata.py +430 -0
  20. pyPRMS/parameters/ParamDb.py +73 -0
  21. pyPRMS/parameters/Parameter.py +624 -0
  22. pyPRMS/parameters/ParameterFile.py +190 -0
  23. pyPRMS/parameters/ParameterNetCDF.py +74 -0
  24. pyPRMS/parameters/ParameterSet.py +96 -0
  25. pyPRMS/parameters/Parameters.py +1506 -0
  26. pyPRMS/parameters/__init__.py +5 -0
  27. pyPRMS/plot_helpers.py +305 -0
  28. pyPRMS/prms_helpers.py +235 -0
  29. pyPRMS/py.typed +0 -0
  30. pyPRMS/summary/OutputCSV.py +64 -0
  31. pyPRMS/summary/OutputVariable.py +163 -0
  32. pyPRMS/summary/OutputVariables.py +227 -0
  33. pyPRMS/summary/__init__.py +2 -0
  34. pyPRMS/utilities/__init__.py +0 -0
  35. pyPRMS/utilities/convert_cbh.py +107 -0
  36. pyPRMS/utilities/convert_model_output.py +91 -0
  37. pyPRMS/utilities/convert_params.py +60 -0
  38. pyPRMS/version.py +13 -0
  39. pyPRMS/xml/cbh.xml +163 -0
  40. pyPRMS/xml/control.xml +1447 -0
  41. pyPRMS/xml/dimensions.xml +311 -0
  42. pyPRMS/xml/modules.xml +251 -0
  43. pyPRMS/xml/parameters.xml +6932 -0
  44. pyPRMS/xml/time_series_input.xml +198 -0
  45. pyPRMS/xml/variables.xml +8173 -0
  46. pyprms-0.9.7.dist-info/LICENSE.md +21 -0
  47. pyprms-0.9.7.dist-info/METADATA +67 -0
  48. pyprms-0.9.7.dist-info/RECORD +51 -0
  49. pyprms-0.9.7.dist-info/WHEEL +5 -0
  50. pyprms-0.9.7.dist-info/entry_points.txt +3 -0
  51. pyprms-0.9.7.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1506 @@
1
+
2
+ import cartopy.crs as ccrs # type: ignore
3
+ import gc
4
+ import matplotlib as mpl # type: ignore
5
+ import matplotlib.pyplot as plt # type: ignore
6
+ import netCDF4 as nc # type: ignore
7
+ import networkx as nx # type: ignore
8
+ import numpy as np
9
+ import numpy.typing as npt
10
+ import pandas as pd # type: ignore
11
+ import sys
12
+ import xml.dom.minidom as minidom
13
+ import xml.etree.ElementTree as xmlET
14
+
15
+ from collections import defaultdict
16
+ from collections.abc import KeysView
17
+ from functools import cached_property
18
+ from packaging.version import Version
19
+ from typing import Any, Literal, Optional, Sequence, Union, Dict, List, Set, Tuple
20
+ from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER # type: ignore
21
+
22
+ from ..control.Control import Control
23
+ from ..dimensions.Dimensions import Dimensions
24
+ from ..Exceptions_custom import ParameterError, ParameterExistsError, ParameterNotValidError
25
+ from .Parameter import Parameter, ParamDataRawType
26
+ from ..plot_helpers import set_colormap, get_projection, plot_line_collection, plot_polygon_collection, get_figsize
27
+ from ..prms_helpers import cond_check, flex_type, get_streamnet_subset
28
+ from ..constants import (CATEGORY_DELIM, DIMENSIONS_XML, external_module_map, MetaDataType, NETCDF_DATATYPES,
29
+ NEW_PTYPE_TO_DTYPE, PRMS_VERSION, PRMS6_DEV_VERSION, PTYPE_TO_PRMS_TYPE, NHM_DATATYPES,
30
+ PARAMETERS_XML, VAR_DELIM)
31
+
32
+ from rich.console import Console
33
+ from rich import pretty
34
+
35
+ import os
36
+ os.environ['USE_PYGEOS'] = '0'
37
+ import geopandas # type: ignore
38
+
39
+ pretty.install()
40
+ con = Console(force_jupyter=False)
41
+
42
+ LimitOptions = Literal['valid', 'centered', 'absolute']
43
+
44
+
45
+ class Parameters(object):
46
+ """Container of multiple pyPRMS.Parameter objects.
47
+ """
48
+
49
+ # Author: Parker Norton (pnorton@usgs.gov)
50
+ # Create date: 2017-05-01
51
+
52
+ def __init__(self, metadata: MetaDataType,
53
+ verbose: Optional[bool] = False):
54
+ """Initialize the Parameters object.
55
+
56
+ Create an ordered dictionary to contain pyPRMS.Parameter objects
57
+
58
+ :param metadata: Metadata for the parameters
59
+ :param verbose: Output debugging information
60
+ """
61
+
62
+ self.__dimensions = Dimensions(metadata=metadata, verbose=verbose)
63
+ self.__parameters: Dict[str, Parameter] = dict()
64
+
65
+ self.verbose = verbose
66
+ self.__control: Optional[Control] = None
67
+ self.__hru_poly = None
68
+ self.__hru_shape_key: Optional[str] = None
69
+ self.__seg_poly = None
70
+ self.__seg_shape_key: Optional[str] = None
71
+ self.__seg_to_hru: Dict[int, List[int]] = dict()
72
+ self.__hru_to_seg: Dict[int, int] = dict()
73
+ self.metadata = metadata['parameters']
74
+ self.prms_version = Version(metadata['info']['version'])
75
+
76
+ def __getattr__(self, name: str):
77
+ """Not sure what to write yet.
78
+
79
+ :param name: Name of the attribute
80
+ """
81
+
82
+ # Undefined attributes will look up the given parameter
83
+ # return self.get(item)
84
+ # https://nedbatchelder.com/blog/201010/surprising_getattr_recursion.html
85
+ if name == "__setstate__":
86
+ raise AttributeError(name)
87
+ return getattr(self.__parameters, name)
88
+
89
+ def __getitem__(self, item):
90
+ """Not sure what to write yet.
91
+ """
92
+
93
+ return self.get(item)
94
+
95
+ def __str__(self) -> str:
96
+ """Pretty-print string representation of the Parameters object.
97
+
98
+ :return: Pretty-print string of Parameters
99
+ """
100
+
101
+ outstr = '----- Dimensions -----\n'
102
+ for vv in self.__dimensions.values():
103
+ outstr += f'{vv.name}: size={vv.size}\n'
104
+
105
+ outstr += '----- Parameters -----\n'
106
+ for vv in self.__parameters.values():
107
+ outstr += f'{vv.name} [{", ".join(vv.meta["dimensions"])}]\n'
108
+
109
+ return outstr
110
+
111
+ @property
112
+ def control(self) -> Optional[Control]:
113
+ """Get Control object.
114
+
115
+ :returns: Control object
116
+ """
117
+
118
+ return self.__control
119
+
120
+ @control.setter
121
+ def control(self, ctl_obj: Control):
122
+ """Sets the Control object for the ParameterSet.
123
+
124
+ :param ctl_obj: Control object
125
+ """
126
+
127
+ self.__control = ctl_obj
128
+
129
+ @property
130
+ def dimensions(self) -> Dimensions:
131
+ """Get dimensions object.
132
+
133
+ :returns: Dimensions object
134
+ """
135
+
136
+ return self.__dimensions
137
+
138
+ @cached_property
139
+ def hru_to_seg(self) -> Dict[int, int]:
140
+ """Returns an ordered dictionary mapping NHM HRU IDs to HRU NHM segment IDs.
141
+
142
+ :returns: dictionary mapping nhm_id to hru_segment_nhm
143
+ """
144
+
145
+ # Only supported with python >= 3.9
146
+ hru_segment = self.get('hru_segment_nhm').data_raw
147
+ nhm_id = self.get('nhm_id').data_raw
148
+
149
+ self.__hru_to_seg = dict(np.rec.fromarrays([nhm_id, hru_segment]).tolist()) # type: ignore
150
+ return self.__hru_to_seg
151
+
152
+ @property
153
+ def missing_params(self) -> Set:
154
+ """Get set of parameters that are required for the modules selected
155
+ in the control file but are missing in the current set of parameters.
156
+ """
157
+
158
+ # TODO: 20240726 PAN - raise warning/error when certain parameters
159
+ # like hru_segment_nhm, etc are missing
160
+ pset = self._required_parameters()
161
+ return pset.difference(set(self.parameters.keys()))
162
+
163
+ @property
164
+ def parameters(self) -> Dict[str, Parameter]:
165
+ """Returns an ordered dictionary of parameter objects.
166
+
167
+ :returns: dictionary of Parameter objects
168
+ """
169
+
170
+ return self.__parameters
171
+
172
+ @property
173
+ def poi_to_seg(self) -> Dict[str, int]:
174
+ """Returns a dictionary mapping poi_id to local poi_seg.
175
+
176
+ :returns: dictionary mapping poi_id to local poi_seg"""
177
+
178
+ gage_ids = self.get('poi_gage_id').data_raw
179
+ gage_segments = self.get('poi_gage_segment').data_raw
180
+ return dict(np.rec.fromarrays([gage_ids, gage_segments]).tolist()) # type: ignore
181
+
182
+ @property
183
+ def poi_to_seg0(self) -> Dict[str, int]:
184
+ """Returns a dictionary mapping poi_id to local, zero-based poi_seg.
185
+
186
+ :returns: dictionary mapping poi_id to local, zero-based poi_seg"""
187
+ gage_ids = self.get('poi_gage_id').data_raw # .tolist()
188
+ gage_segments = np.subtract(self.get('poi_gage_segment').data_raw, 1)
189
+ return dict(np.rec.fromarrays([gage_ids, gage_segments]).tolist()) # type: ignore
190
+
191
+ @cached_property
192
+ def seg_to_hru(self) -> Dict[int, List[int]]:
193
+ """Returns a dictionary mapping HRU global segment IDs to global HRU IDs.
194
+
195
+ Segment keys equal to zero are for non-routed HRUs. Segment keys greater than
196
+ zero are global segment IDs.
197
+
198
+ :returns: dictionary mapping hru_segment_nhm to nhm_id
199
+ """
200
+
201
+ hru_segment = self.get('hru_segment_nhm').data_raw.tolist()
202
+ nhm_id = self.get('nhm_id').data_raw
203
+ assert type(nhm_id) is np.ndarray and nhm_id.dtype == np.int32
204
+
205
+ for chru_idx, cseg in enumerate(hru_segment): # type: ignore
206
+ # keys are 1-based, values in arrays are 1-based
207
+ # Non-routed HRUs have a seg key = zero
208
+ self.__seg_to_hru.setdefault(cseg, []).append(nhm_id[chru_idx].item()) # type: ignore
209
+ return self.__seg_to_hru
210
+
211
+ @property
212
+ def unneeded_parameters(self) -> Set:
213
+ """Get set of parameters that are defined but not needed by any of the
214
+ modules selected in the control file.
215
+
216
+ :returns: set of unneeded parameter names
217
+ """
218
+
219
+ if self.verbose: # pragma: no cover
220
+ con.print('-'*20, 'unneeded_parameters', '-'*20)
221
+
222
+ pset = self._required_parameters()
223
+ return set(self.parameters.keys()).difference(pset)
224
+
225
+ @property
226
+ def xml_global_dimensions(self) -> xmlET.Element:
227
+ """Get XML element tree of the dimensions used by all parameters.
228
+
229
+ :returns: element tree of dimensions
230
+ """
231
+
232
+ dims_xml = xmlET.Element('dimensions')
233
+
234
+ for cdim in self.dimensions.dimensions.values():
235
+ dim_sub = xmlET.SubElement(dims_xml, 'dimension')
236
+ dim_sub.set('name', cdim.name)
237
+
238
+ for kk, vv in cdim.meta.items():
239
+ if kk in ['is_fixed', 'requires_control']:
240
+ pass
241
+ else:
242
+ xmlET.SubElement(dim_sub, kk).text = flex_type(vv)
243
+ return dims_xml
244
+
245
+ @property
246
+ def xml_global_parameters(self) -> xmlET.Element:
247
+ """Get XML element tree of the parameters.
248
+
249
+ :returns: element tree of parameters
250
+ """
251
+
252
+ # Map datatypes (e.g. np.int32) to the parameter type strings (e.g. 'I')
253
+ inv_nhm_datatypes = {vv: kk for kk, vv in NHM_DATATYPES.items()}
254
+ datatype_to_prms_type = {kk: inv_nhm_datatypes[vv] for kk, vv in PTYPE_TO_PRMS_TYPE.items()}
255
+
256
+ params_xml = xmlET.Element('parameters')
257
+
258
+ for pk in sorted(list(self.parameters.keys())):
259
+ vv = self.get(pk)
260
+
261
+ param_sub = xmlET.SubElement(params_xml, 'parameter')
262
+ param_sub.set('name', vv.name)
263
+
264
+ xmlET.SubElement(param_sub, 'type').text = datatype_to_prms_type[vv.meta['datatype']]
265
+
266
+ for mm, md in vv.meta.items():
267
+ if mm == 'modules':
268
+ modules_sub = xmlET.SubElement(param_sub, 'modules')
269
+
270
+ for cmod in md:
271
+ xmlET.SubElement(modules_sub, 'module').text = cmod
272
+ elif mm in ['datatype', 'requires_control', 'requires_dimension', 'version']:
273
+ pass
274
+ elif mm == 'dimensions':
275
+ param_sub.append(vv.dimensions.xml)
276
+ else:
277
+ xmlET.SubElement(param_sub, mm).text = flex_type(md)
278
+
279
+ return params_xml
280
+
281
+ # =========================================================================
282
+ # Methods
283
+ def add(self, name: str):
284
+ """Add a new parameter by name.
285
+
286
+ :param name: A valid PRMS parameter name
287
+
288
+ :raises ParameterError: if parameter already exists or name is None
289
+ """
290
+
291
+ # Add a new parameter
292
+ if self.exists(name):
293
+ raise ParameterExistsError(f'{name}: Parameter already exists')
294
+
295
+ if name not in self.metadata:
296
+ raise ParameterNotValidError(f'{name}: Parameter is not a valid PRMS parameter')
297
+
298
+ for cdim in self.metadata[name]['dimensions']:
299
+ if not self.__dimensions.exists(cdim):
300
+ raise KeyError(f'Global dimension, {cdim}, does not exist')
301
+
302
+ self.__parameters[name] = Parameter(name=name, meta=self.metadata, global_dims=self.__dimensions)
303
+
304
+ def add_metadata(self, name: str,
305
+ metadata: Dict):
306
+ """Add a new parameter entry to the parameter metadata. This is useful for adding ad-hoc parameters.
307
+
308
+ :param name: Name of the parameter
309
+ :param metadata: Dictionary of metadata for the parameter
310
+ """
311
+
312
+ # entry_items = dict(datatype='float32', description='something new today', help='get your own help',
313
+ # units='dontmatter', default=0.0, minimum=0.0, maximum=100.0, dimensions=['nhru'])
314
+ new_entry = defaultdict(list)
315
+
316
+ for kk, vv in metadata.items():
317
+ new_entry[kk] = vv
318
+
319
+ self.metadata[name] = new_entry
320
+
321
+ def add_missing_parameters(self):
322
+ """Add missing parameters that are required by the selected modules.
323
+ """
324
+ if self.verbose:
325
+ con.print('-'*20, 'add_missing_parameters', '-'*20) # pragma: no cover
326
+
327
+ for cparam in self.missing_params:
328
+ if cparam in ['nhm_deplcrv']:
329
+ if self.verbose: # pragma: no cover
330
+ con.print(f'[bold]{cparam}[/] is missing but lacks the information to be added')
331
+ continue
332
+
333
+ self.add(cparam)
334
+ self.get(cparam).data = self.metadata[cparam].get('default')
335
+
336
+ if self.verbose: # pragma: no cover
337
+ con.print(f'[bold]{cparam}[/] [gold3] parameter added with default value[/]')
338
+
339
+ def adjust_bounded_parameters(self):
340
+ """Adjust the valid upper and lower values for bounded parameters.
341
+ """
342
+
343
+ for cparam in self.parameters.values():
344
+ cmeta = cparam.meta
345
+
346
+ if cmeta.get('maximum') in list(self.dimensions.keys()):
347
+ # if isinstance(cmeta.get('maximum'), str):
348
+ try:
349
+ cmeta['maximum'] = self.dimensions.get(cmeta.get('maximum')).size
350
+
351
+ if self.verbose: # pragma: no cover
352
+ con.print(f'[bold]{cparam.name}[/]: valid upper bound adjusted to {cmeta["maximum"]}')
353
+ except ValueError:
354
+ print(f'{cparam.name} has bad valid uppper bound value')
355
+ raise
356
+
357
+ def check(self): # pragma: no cover
358
+ """Check all parameter variables for proper array size.
359
+ """
360
+
361
+ # for pp in self.__parameters.values():
362
+ for pk in sorted(list(self.parameters.keys())):
363
+ pp = self.get(pk)
364
+
365
+ # print(pp.check())
366
+ if pp.has_correct_size():
367
+ con.print(f'[bold]{pk}[/]: Size [green4]OK[/green4]')
368
+ else:
369
+ con.print(f'[bold]{pk}[/]: [red]Incorrect number of values for dimensions[/red]')
370
+
371
+ if not pp.check_values():
372
+ pp_stats = pp.stats()
373
+ pp_outliers = pp.outliers()
374
+ valid_min = pp.meta['minimum']
375
+ valid_max = pp.meta['maximum']
376
+ default_val = pp.meta['default']
377
+
378
+ if not (isinstance(valid_min, str) or isinstance(valid_max, str)):
379
+ con.print(f' [dark_orange3]WARNING[/]: Value(s) (range: {pp_stats.min}, {pp_stats.max}) outside '
380
+ + f'the valid range of ({valid_min}, {valid_max}); '
381
+ + f'under/over=({pp_outliers.under}, {pp_outliers.over})')
382
+ # print(f' WARNING: Value(s) (range: {pp.data.min()}, {pp.data.max()}) outside ' +
383
+ # f'the valid range of ({pp.minimum}, {pp.maximum})')
384
+ elif valid_min == 'bounded':
385
+ # TODO: Handling bounded parameters needs improvement
386
+ con.print(f' [dark_orange3]WARNING[/]: Bounded parameter value(s) '
387
+ + f'(range: {pp_stats.min}, {pp_stats.max}) outside '
388
+ + f'the valid range of ({default_val}, {valid_max})')
389
+
390
+ if pp.all_equal():
391
+ dims = list(pp.dimensions.keys())
392
+
393
+ if pp.is_scalar:
394
+ con.print(f' INFO: Scalar; value = {pp.data}')
395
+ elif pp.data.ndim == 2:
396
+ con.print(f' INFO: dimensioned {dims}; all values by {dims[0]} are equal to {pp.data[0]}')
397
+ elif pp.data.ndim == 1:
398
+ con.print(f' INFO: dimensioned {dims}; all values are equal to {pp.data[0]}')
399
+
400
+ if pp.name == 'snarea_curve':
401
+ if pp.as_dataframe.values.reshape((-1, 11)).shape[0] != self.get('hru_deplcrv').unique().size:
402
+ con.print(' [yellow3]WARNING[/]: snarea_curve has more entries than needed by hru_deplcrv')
403
+
404
+ def exists(self, name: str) -> bool:
405
+ """Checks if a parameter name exists.
406
+
407
+ :param name: Name of the parameter
408
+ :returns: True if parameter exists, otherwise False
409
+ """
410
+
411
+ return name in self.parameters.keys()
412
+
413
+ def get(self, name: str) -> Parameter:
414
+ """Returns a parameter object.
415
+
416
+ :param name: The name of the parameter
417
+ :returns: Parameter object
418
+ """
419
+
420
+ # Return the given parameter
421
+ if self.exists(name):
422
+ return self.__parameters[name]
423
+
424
+ raise ParameterError(f'Parameter, {name}, does not exist.')
425
+
426
+ def get_dataframe(self, name: str) -> pd.DataFrame:
427
+ """Returns a pandas DataFrame for a parameter.
428
+
429
+ If the parameter dimensions include either nhru or nsegment then the
430
+ respective national ids (nhm_seg, nhm_id) are included, if they exist, as the index in the
431
+ returned dataframe.
432
+
433
+ :param name: The name of the parameter
434
+ :returns: Pandas DataFrame of the parameter data
435
+ """
436
+
437
+ if not self.exists(name):
438
+ raise KeyError(f'Parameter, {name}, does not exist')
439
+
440
+ cparam = self.get(name)
441
+ param_data = cparam.as_dataframe
442
+
443
+ if cparam.is_hru_param():
444
+ if name != 'nhm_id':
445
+ if self.exists('nhm_id'):
446
+ param_id = self.get('nhm_id').as_dataframe
447
+
448
+ # Create a DataFrame of the parameter
449
+ param_data = param_data.merge(param_id, left_index=True, right_index=True)
450
+ param_data.set_index('nhm_id', inplace=True)
451
+ param_data.index.name = 'nhm_id'
452
+ else:
453
+ param_data = self.get('nhm_id').as_dataframe
454
+ elif cparam.is_seg_param():
455
+ if name != 'nhm_seg':
456
+ if self.exists('nhm_seg'):
457
+ param_id = self.get('nhm_seg').as_dataframe
458
+
459
+ # Create a DataFrame of the parameter
460
+ param_data = param_data.merge(param_id, left_index=True, right_index=True)
461
+ param_data.set_index('nhm_seg', inplace=True)
462
+ param_data.index.name = 'nhm_seg'
463
+ else:
464
+ param_data = self.get('nhm_seg').as_dataframe
465
+ elif name == 'snarea_curve':
466
+ # Special handling for snarea_curve parameter
467
+ param_data = pd.DataFrame(cparam.as_dataframe.values.reshape((-1, 11)))
468
+ param_data.rename(columns={k: k+1 for k in param_data.columns}, # type: ignore
469
+ index={k: k+1 for k in param_data.index},
470
+ inplace=True)
471
+ param_data.index.name = 'curve_index'
472
+ return param_data
473
+
474
+ def get_subset(self, name: str,
475
+ global_ids: List[int]) -> ParamDataRawType:
476
+ """Returns a subset for a parameter based on the global_ids (e.g. nhm_id, nhm_seg).
477
+
478
+ :param name: Name of the parameter
479
+ :param global_ids: List of global IDs to extract
480
+ :returns: Array of extracted values
481
+ """
482
+
483
+ param = self.get(name)
484
+ dim_set = set(param.dimensions.keys()).intersection({'nhru', 'nssr', 'ngw', 'nsegment', 'ndeplval'})
485
+ id_index_map: Union[Dict[Any, int], None] = {}
486
+ cdim = dim_set.pop()
487
+
488
+ if cdim in ['nhru', 'nssr', 'ngw', 'ndeplval']:
489
+ # Global IDs should be in the range of nhm_id
490
+ id_index_map = self.get('nhm_id').index_map
491
+ elif cdim in ['nsegment']:
492
+ # Global IDs should be in the range of nhm_seg
493
+ id_index_map = self.get('nhm_seg').index_map
494
+
495
+ # Zero-based indices in order of global_ids
496
+ assert id_index_map is not None
497
+ nhm_idx0 = [id_index_map[kk] for kk in global_ids]
498
+
499
+ if param.dimensions.ndim == 2:
500
+ return np.take(param.data_raw, nhm_idx0, axis=0) # axis: 0 rows, 1 columns
501
+ # return param.data_raw[tuple(nhm_idx0), :]
502
+ else:
503
+ if name in ['hru_deplcrv', 'snarea_curve']:
504
+ init_data = np.take(self.get('hru_deplcrv').data_raw, nhm_idx0, axis=0)
505
+ # init_data = self.get('hru_deplcrv').data_raw[tuple(nhm_idx0), ]
506
+ uniq_deplcrv = np.unique(init_data).tolist()
507
+
508
+ match name:
509
+ case 'hru_deplcrv':
510
+ # Renumber the hru_deplcrv indices for the subset
511
+ uniq_dict = {xx: ii+1 for ii, xx in enumerate(uniq_deplcrv)}
512
+
513
+ # Create new hru_deplcrv and renumber
514
+ res = np.array([uniq_dict[xx] for xx in init_data])
515
+ # return np.array([uniq_dict[xx] for xx in init_data])
516
+ case 'snarea_curve':
517
+ uniq_deplcrv0 = [xx - 1 for xx in uniq_deplcrv]
518
+ res = param.data_raw.reshape((-1, 11))[tuple(uniq_deplcrv0), :].reshape((-1))
519
+ return res
520
+
521
+ # if name == 'hru_deplcrv':
522
+ # # Renumber the hru_deplcrv indices for the subset
523
+ # uniq_dict = {xx: ii+1 for ii, xx in enumerate(uniq_deplcrv)}
524
+ #
525
+ # # Create new hru_deplcrv and renumber
526
+ # return np.array([uniq_dict[xx] for xx in init_data])
527
+ # elif name == 'snarea_curve':
528
+ # uniq_deplcrv0 = [xx - 1 for xx in uniq_deplcrv]
529
+ # return param.data_raw.reshape((-1, 11))[tuple(uniq_deplcrv0), :].reshape((-1))
530
+ else:
531
+ # All other 1D arrays
532
+ return np.take(param.data_raw, nhm_idx0, axis=0) # axis: 0 rows, 1 columns
533
+ # return param.data_raw[tuple(nhm_idx0), ]
534
+
535
+ def outlier_ids(self, name: str) -> List[int]:
536
+ """Returns list of HRU or segment IDs of invalid parameter values
537
+
538
+ :param name: Name of the parameter
539
+ :returns: List of HRU or segment IDs
540
+ """
541
+
542
+ cparam = self.get(name)
543
+
544
+ param_data = self.get_dataframe(name)
545
+ bad_value_ids = param_data[(param_data[name] < cparam.meta['minimum']) |
546
+ (param_data[name] > cparam.meta['maximum'])].index.tolist()
547
+
548
+ return bad_value_ids
549
+
550
+ def poi_upstream_hrus(self, poi: Union[str, List[str], KeysView]) -> Dict[str, List[int]]:
551
+ """Returns a dictionary of POI to upstream global HRU IDs.
552
+
553
+ :param poi: POI ID or list of POI IDs
554
+
555
+ :returns: Dictionary of POI to upstream global HRU IDs
556
+ """
557
+
558
+ if isinstance(poi, str):
559
+ poi = [poi]
560
+ elif isinstance(poi, KeysView):
561
+ poi = list(poi)
562
+
563
+ poi_hrus = {}
564
+ nhm_seg = self.get('nhm_seg').data_raw
565
+ assert type(nhm_seg) is np.ndarray and nhm_seg.dtype == np.int32
566
+ poi_seg0_dict = self.poi_to_seg0
567
+
568
+ # Generate stream network for the model
569
+ dag_streamnet = self.stream_network()
570
+
571
+ for cpoi in poi:
572
+ # Lookup global segment id for the current POI
573
+ cseg = poi_seg0_dict[cpoi]
574
+ dsmost_seg = [nhm_seg[cseg].item()]
575
+
576
+ poi_hrus[cpoi] = self._upstream_hrus(dag_streamnet, dsmost_seg)
577
+
578
+ return poi_hrus
579
+
580
+ def poi_upstream_segments(self, poi: Union[str, List[str], KeysView]) -> Dict[str, List[int]]:
581
+ """Returns a dictionary of POI to upstream global segment IDs.
582
+
583
+ :param poi: POI ID or list of POI IDs
584
+
585
+ :returns: Dictionary of POI to upstream global segment IDs
586
+ """
587
+
588
+ if isinstance(poi, str):
589
+ poi = [poi]
590
+ elif isinstance(poi, KeysView):
591
+ poi = list(poi)
592
+
593
+ poi_segs = {}
594
+ nhm_seg = self.get('nhm_seg').data_raw
595
+ assert type(nhm_seg) is np.ndarray
596
+ pois_dict = self.poi_to_seg0
597
+
598
+ # Generate stream network for the model
599
+ dag_streamnet = self.stream_network()
600
+
601
+ for cpoi in poi:
602
+ # Lookup global segment id for the current POI
603
+ dsmost_seg = [nhm_seg[pois_dict[cpoi]].item()]
604
+
605
+ poi_segs[cpoi] = self._upstream_segments(dag_streamnet, dsmost_seg)
606
+
607
+ return poi_segs
608
+
609
+ def plot(self, name: str,
610
+ output_dir: Optional[str] = None,
611
+ limits: Optional[Union[LimitOptions, List[float], Tuple[float, float]]] = 'absolute',
612
+ mask_defaults: Optional[str] = None,
613
+ **kwargs): # pragma: no cover
614
+ """Plot a parameter.
615
+
616
+ Plots either to the screen or an output directory.
617
+
618
+ :param name: Name of parameter to plot
619
+ :param output_dir: Directory to write plot to (None for write to screen only)
620
+ :param limits: Limits to use for colorbar. One of 'valid', 'centered', 'absolute', or list of floats.
621
+ :param mask_defaults: Color for defaults values
622
+ """
623
+
624
+ # is_monthly = False
625
+ time_index = None
626
+
627
+ if self.exists(name):
628
+ cparam = self.get(name)
629
+
630
+ if set(cparam.dimensions.keys()).intersection({'nmonths'}):
631
+ # Need 12 monthly plots of parameter
632
+ # is_monthly = True
633
+ time_index = 0 # starting time index
634
+ param_data = self.get_dataframe(name).iloc[:, time_index].to_frame(name=name)
635
+ else:
636
+ param_data = self.get_dataframe(name).iloc[:]
637
+
638
+ if mask_defaults is not None:
639
+ param_data = param_data.mask(param_data == cparam.meta['default'])
640
+
641
+ if isinstance(limits, str):
642
+ if limits == 'valid':
643
+ valid_min = cparam.meta['minimum']
644
+ valid_max = cparam.meta['maximum']
645
+
646
+ # Use the defined valid range of possible values
647
+ if valid_min == 'bounded':
648
+ # Parameters with bounded values need to always use the actual range of values
649
+ drange = [cparam.data_raw.min().min(), cparam.data_raw.max().max()]
650
+ elif name == 'jh_coef':
651
+ drange = [-0.05, 0.05]
652
+ else:
653
+ drange = [valid_min, valid_max]
654
+ elif limits == 'centered':
655
+ # Use the maximum range of the actual data values
656
+ lim = max(abs(cparam.data_raw.min().min()), abs(cparam.data_raw.max().max()))
657
+ drange = [-lim, lim]
658
+ elif limits == 'absolute':
659
+ # Use the min and max of the data values
660
+ drange = [cparam.data_raw.min().min(), cparam.data_raw.max().max()]
661
+ else:
662
+ raise ValueError('String argument for limits must be "valid", "centered", or "absolute"')
663
+ elif isinstance(limits, (list, tuple)):
664
+ if len(limits) != 2:
665
+ raise ValueError('When a list is used for plotting limits it should have 2 values (min, max)')
666
+
667
+ drange = [min(limits), max(limits)]
668
+ else:
669
+ raise TypeError('Argument, limits, must be string or a list[min,max]')
670
+
671
+ cmap, norm = set_colormap(name, param_data, min_val=drange[0],
672
+ max_val=drange[1], **kwargs)
673
+ kwargs.pop('cmap', None)
674
+
675
+ if mask_defaults is not None:
676
+ cmap.set_bad(mask_defaults, 0.7)
677
+
678
+ if set(cparam.dimensions.keys()).intersection({'nhru', 'ngw', 'nssr'}):
679
+ # Get extent information
680
+ if self.__hru_poly is not None:
681
+ minx, miny, maxx, maxy = self.__hru_poly.geometry.total_bounds
682
+
683
+ crs_proj = get_projection(self.__hru_poly)
684
+
685
+ # Takes care of multipolygons that are in the NHM geodatabase/shapefile
686
+ geoms_exploded = self.__hru_poly.explode(index_parts=True).reset_index(level=1, drop=True)
687
+
688
+ # print('Writing first plot')
689
+ df_mrg = geoms_exploded.merge(param_data, left_on=self.__hru_shape_key,
690
+ right_index=True, how='left')
691
+
692
+ fig_width, fig_height = get_figsize([minx, maxx, miny, maxy], **dict(kwargs))
693
+ kwargs.pop('init_size', None)
694
+
695
+ fig = plt.figure(figsize=(fig_width, fig_height))
696
+
697
+ ax = plt.axes(projection=crs_proj)
698
+
699
+ try:
700
+ ax.coastlines()
701
+ except AttributeError:
702
+ pass
703
+
704
+ gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True)
705
+ gl.top_labels = None
706
+ gl.right_labels = None
707
+ gl.xformatter = LONGITUDE_FORMATTER
708
+ gl.yformatter = LATITUDE_FORMATTER
709
+
710
+ ax.set_extent([minx, maxx, miny, maxy], crs=crs_proj)
711
+
712
+ if time_index is not None:
713
+ plt.title(f'Variable: {name}, Month: {time_index+1}')
714
+ else:
715
+ plt.title(f'Variable: {name}')
716
+
717
+ if mask_defaults is not None:
718
+ plt.annotate(f'NOTE: Values = {cparam.meta["default"]} are masked', xy=(0.5, 0.01),
719
+ xycoords='axes fraction', va='center', ha='center',
720
+ fontsize=10, fontweight='bold',
721
+ bbox=dict(boxstyle="round", facecolor=mask_defaults, alpha=1.0))
722
+
723
+ # Setup the color bar
724
+ mapper = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)
725
+ mapper.set_array(df_mrg[name])
726
+ cax = fig.add_axes((ax.get_position().x1 + 0.01,
727
+ ax.get_position().y0,
728
+ 0.02,
729
+ ax.get_position().height))
730
+
731
+ # TODO: 2022-06-17 PAN - Categorical variables require entry in two places.
732
+ # The first place is here for labelling and the second place is in
733
+ # plot_helpers.py for adjusting the color boundaries. Need to figure out
734
+ # a more consistent/better way to do this.
735
+ if name == 'hru_deplcrv':
736
+ # tck_arr = np.arange(param_data.min().min(), param_data.max().max()+1)
737
+ tck_arr = np.arange(drange[0], drange[1]+1)
738
+ cb = plt.colorbar(mapper, cax=cax, ticks=tck_arr, label='Curve index')
739
+ cb.ax.tick_params(length=0)
740
+ elif name == 'soil_type':
741
+ tck_arr = np.arange(drange[0], drange[1]+1)
742
+ cb = plt.colorbar(mapper, cax=cax, ticks=tck_arr, label=name)
743
+ cb.ax.tick_params(length=0)
744
+ elif name == 'calibration_status':
745
+ tck_arr = np.arange(drange[0], drange[1]+1)
746
+ cb = plt.colorbar(mapper, cax=cax, ticks=tck_arr, label=name)
747
+ cb.ax.tick_params(length=0)
748
+ else:
749
+ plt.colorbar(mapper, cax=cax, label=cparam.meta['units'])
750
+
751
+ col = plot_polygon_collection(ax, df_mrg.geometry, values=df_mrg[name],
752
+ cmap=cmap, norm=norm, **dict(kwargs))
753
+ # col = plot_polygon_collection(ax, df_mrg.geometry, values=df_mrg[name],
754
+ # **dict(kwargs, cmap=cmap, norm=norm))
755
+
756
+ if output_dir is not None:
757
+ if time_index is not None:
758
+ # First month
759
+ plt.savefig(f'{output_dir}/{name}_{time_index+1:02}.png', dpi=150, bbox_inches='tight')
760
+
761
+ for tt in range(1, 12):
762
+ # Months 2 through 12
763
+ # print(f' Index: {tt}')
764
+ param_data = self.get_dataframe(name).iloc[:, tt].to_frame(name=name)
765
+
766
+ if mask_defaults is not None:
767
+ param_data = param_data.mask(param_data == cparam.meta['default'])
768
+
769
+ df_mrg = geoms_exploded.merge(param_data, left_on=self.__hru_shape_key,
770
+ right_index=True, how='left')
771
+
772
+ ax.set_title(f'Variable: {name}, Month: {tt+1}')
773
+ col.set_array(df_mrg[name])
774
+ plt.savefig(f'{output_dir}/{name}_{tt+1:02}.png', dpi=150, bbox_inches='tight')
775
+ else:
776
+ plt.savefig(f'{output_dir}/{name}.png', dpi=150, bbox_inches='tight')
777
+
778
+ # Close the figure so we don't chew up memory
779
+ fig.clf()
780
+ plt.close()
781
+ gc.collect()
782
+ elif set(cparam.dimensions.keys()).intersection({'nsegment'}):
783
+ # Plot segment-related parameters
784
+ if self.__seg_poly is not None:
785
+ if self.__hru_poly is not None:
786
+ minx, miny, maxx, maxy = self.__hru_poly.geometry.total_bounds
787
+ else:
788
+ minx, miny, maxx, maxy = self.__seg_poly.geometry.total_bounds
789
+
790
+ seg_geoms_exploded = self.__seg_poly.explode(index_parts=True).reset_index(level=1, drop=True)
791
+
792
+ crs_proj = get_projection(self.__seg_poly)
793
+
794
+ df_mrg = seg_geoms_exploded.merge(param_data, left_on=self.__seg_shape_key,
795
+ right_index=True, how='left')
796
+
797
+ fig_width, fig_height = get_figsize([minx, maxx, miny, maxy], **dict(kwargs))
798
+ kwargs.pop('init_size', None)
799
+
800
+ fig = plt.figure(figsize=(fig_width, fig_height))
801
+
802
+ ax = plt.axes(projection=crs_proj)
803
+
804
+ try:
805
+ ax.coastlines()
806
+ except AttributeError:
807
+ pass
808
+
809
+ gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True)
810
+ gl.top_labels = None
811
+ gl.right_labels = None
812
+ gl.xformatter = LONGITUDE_FORMATTER
813
+ gl.yformatter = LATITUDE_FORMATTER
814
+
815
+ ax.set_extent([minx, maxx, miny, maxy], crs=crs_proj)
816
+
817
+ plt.title('Variable: {}'.format(name))
818
+
819
+ if kwargs.get('vary_color', True):
820
+ mapper = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)
821
+ mapper.set_array(df_mrg[name])
822
+ cax = fig.add_axes((ax.get_position().x1 + 0.01,
823
+ ax.get_position().y0, 0.02,
824
+ ax.get_position().height))
825
+ plt.colorbar(mapper, cax=cax, label=cparam.meta['units']) # , shrink=0.6
826
+
827
+ if self.__hru_poly is not None:
828
+ hru_geoms_exploded = self.__hru_poly.explode(index_parts=True).reset_index(level=1, drop=True)
829
+ hru_poly = plot_polygon_collection(ax, hru_geoms_exploded.geometry,
830
+ cmap=cmap, norm=norm,
831
+ **dict(kwargs, linewidth=0.5, alpha=0.7))
832
+
833
+ col = plot_line_collection(ax, df_mrg.geometry, values=df_mrg[name], cmap=cmap,
834
+ norm=norm, **dict(kwargs))
835
+
836
+ if mask_defaults is not None:
837
+ plt.annotate(f'NOTE: Values = {cparam.meta["default"]} are masked', xy=(0.5, 0.01),
838
+ xycoords='axes fraction', fontsize=12, fontweight='bold',
839
+ bbox=dict(facecolor=mask_defaults, alpha=1.0))
840
+
841
+ if output_dir is not None:
842
+ plt.savefig(f'{output_dir}/{name}.png', dpi=150, bbox_inches='tight')
843
+
844
+ # Close the figure so we don't chew up memory
845
+ fig.clf()
846
+ plt.close()
847
+ gc.collect()
848
+ else:
849
+ print('No segment shapefile is loaded; skipping')
850
+ else:
851
+ print('Non-plottable parameter')
852
+
853
+ def remove(self, name: Union[str, Sequence[str], Set[str]]):
854
+ """Delete one or more parameters if they exist.
855
+
856
+ :param name: parameter or list of parameters to remove
857
+ """
858
+
859
+ if isinstance(name, str):
860
+ name = [name]
861
+
862
+ for cparam in name:
863
+ if self.exists(cparam):
864
+ del self.__parameters[cparam]
865
+
866
+ if self.verbose: # pragma: no cover
867
+ con.print(f'[bold]{cparam}[/] [gold3]parameter removed[/]')
868
+
869
+ def remove_poi(self, poi: Union[str, List[str]]):
870
+ """Remove POIs by gage_id.
871
+
872
+ :param poi: POI id to remove
873
+ """
874
+
875
+ if isinstance(poi, str):
876
+ poi = [poi]
877
+
878
+ # First get array of poi_gage_id indices matching the specified POI IDs
879
+ poi_ids = self.get('poi_gage_id').data_raw.tolist()
880
+ assert type(poi_ids) is list
881
+ poi_del_indices = []
882
+ for xx in poi:
883
+ # We silently ignore missing POIs
884
+ if xx in poi_ids:
885
+ poi_del_indices.append(poi_ids.index(xx)) # type: ignore
886
+
887
+ # poi_ids = self.get('poi_gage_id').data
888
+ # sorter = np.argsort(poi_ids)
889
+ # poi_del_indices = sorter[np.searchsorted(poi_ids, poi, sorter=sorter)]
890
+
891
+ poi_parameters = ['poi_gage_id', 'poi_gage_segment', 'poi_type']
892
+
893
+ # print(f'POIs to delete: {poi}')
894
+ # print(f'Current POIs: {poi_ids}')
895
+ # print(f'Size of poi_del_indices: {poi_del_indices.size}')
896
+ if len(poi_del_indices) > 0:
897
+ if self.get('poi_gage_id').dimensions.get('npoigages').size == len(poi_del_indices):
898
+ # We're trying to remove all the POIs
899
+ for pp in poi_parameters:
900
+ self.remove(pp)
901
+
902
+ self.dimensions.get('npoigages').size = 0
903
+ else:
904
+ # Remove the matching poi gage entries from each of the poi-related parameters
905
+ for pp in poi_parameters:
906
+ self.get(pp).remove_by_index('npoigages', poi_del_indices)
907
+
908
+ # Update the global npoigages dimension
909
+ self.dimensions.get('npoigages').size -= len(poi_del_indices)
910
+ self.dimensions.get('nobs').size -= len(poi_del_indices)
911
+
912
+ def segment_upstream_hrus(self, segs: Union[int, List[int], KeysView, npt.NDArray]) -> Dict[int, List[int]]:
913
+ """Returns a dictionary of segment to upstream global HRU IDs.
914
+
915
+ :param segs: Global segment ID or list of global segment IDs
916
+
917
+ :returns: Dictionary of global segment ID to upstream global HRU IDs
918
+ """
919
+
920
+ if isinstance(segs, int):
921
+ segs = [segs]
922
+ elif isinstance(segs, KeysView):
923
+ segs = list(segs)
924
+ elif isinstance(segs, np.ndarray):
925
+ segs = segs.tolist() # type: ignore
926
+
927
+ assert type(segs) is list
928
+
929
+ seg_hrus = {}
930
+
931
+ # Generate stream network for the model
932
+ dag_streamnet = self.stream_network()
933
+
934
+ for cseg in segs:
935
+ # Lookup segment for the current POI
936
+ dsmost_seg = [cseg]
937
+
938
+ seg_hrus[cseg] = self._upstream_hrus(dag_streamnet, dsmost_seg)
939
+
940
+ return seg_hrus
941
+
942
+ def segment_upstream_segments(self, segs: Union[int, List[int], KeysView, npt.NDArray]) -> Dict[int, List[int]]:
943
+ """Returns a dictionary of global segment IDs to upstream global segment IDs.
944
+
945
+ :param segs: global segment IDs or list of global segment IDs
946
+
947
+ :returns: Dictionary of global segment ID to upstream global segment IDs
948
+ """
949
+
950
+ if isinstance(segs, int):
951
+ segs = [segs]
952
+ elif isinstance(segs, KeysView):
953
+ segs = list(segs)
954
+ elif isinstance(segs, np.ndarray):
955
+ segs = segs.tolist()
956
+
957
+ assert type(segs) is list
958
+
959
+ us_segs = {}
960
+
961
+ # Generate stream network for the model
962
+ dag_streamnet = self.stream_network()
963
+
964
+ for cseg in segs:
965
+ # Lookup segment for the current segment
966
+ dsmost_seg = [cseg]
967
+ us_segs[cseg] = self._upstream_segments(dag_streamnet, dsmost_seg)
968
+
969
+ return us_segs
970
+
971
+ def shapefile_hrus(self, filename: str,
972
+ layer_name: Optional[str] = None,
973
+ shape_key: Optional[str] = None): # pragma: no cover
974
+ """Read a shapefile or geodatabase that corresponds to HRUs.
975
+
976
+ :param filename: name of shapefile or geodatabase
977
+ :param layer_name: name of layer in geodatabase
978
+ :param shape_key: name of attribute for key
979
+ """
980
+
981
+ self.__hru_poly = geopandas.read_file(filename, layer=layer_name)
982
+
983
+ if self.__hru_poly.crs.name == 'USA_Contiguous_Albers_Equal_Area_Conic_USGS_version': # type: ignore
984
+ print('Overriding USGS aea crs with EPSG:5070')
985
+ self.__hru_poly.crs = 'EPSG:5070' # type: ignore
986
+ elif self.__hru_poly.crs.name[0:5] == 'NAD83': # type: ignore
987
+ self.__hru_poly.to_crs('epsg:4326', inplace=True)
988
+
989
+ self.__hru_shape_key = shape_key
990
+
991
+ def shapefile_segments(self, filename: str,
992
+ layer_name: Optional[str] = None,
993
+ shape_key: Optional[str] = None): # pragma: no cover
994
+ """Read a shapefile or geodatabase that corresponds to stream segments.
995
+
996
+ :param filename: name of shapefile or geodatabase
997
+ :param layer_name: name of layer in geodatabase
998
+ :param shape_key: name of attribute for key
999
+ """
1000
+
1001
+ self.__seg_poly = geopandas.read_file(filename, layer=layer_name)
1002
+
1003
+ if self.__seg_poly.crs.name == 'USA_Contiguous_Albers_Equal_Area_Conic_USGS_version': # type: ignore
1004
+ print('Overriding USGS aea crs with EPSG:5070')
1005
+ self.__seg_poly.crs = 'EPSG:5070' # type: ignore
1006
+ elif self.__seg_poly.crs.name[0:5] == 'NAD83': # type: ignore
1007
+ self.__seg_poly.to_crs('epsg:4326', inplace=True)
1008
+
1009
+ self.__seg_shape_key = shape_key
1010
+
1011
+ def stream_network(self, tosegment: str = 'tosegment_nhm',
1012
+ seg_id: str = 'nhm_seg') -> Union[nx.DiGraph, None]:
1013
+ """Create Directed, Acyclic Graph (DAG) of stream network.
1014
+
1015
+ :param tosegment: name of parameter to use for HRU tosegment
1016
+ :param seg_id: name of parameter to use for the segment IDs
1017
+ :returns: directed-acyclic-graph (DAG)
1018
+ """
1019
+
1020
+ seg = self.get(seg_id).tolist()
1021
+ toseg = self.get(tosegment).tolist()
1022
+ assert type(seg) is list and type(toseg) is list
1023
+
1024
+ dag_ds = nx.DiGraph()
1025
+ for ii, vv in enumerate(toseg):
1026
+ # dag_ds.add_edge(ii+1, vv)
1027
+ if vv == 0:
1028
+ dag_ds.add_edge(seg[ii], 'Out_{}'.format(seg[ii]))
1029
+ else:
1030
+ dag_ds.add_edge(seg[ii], vv)
1031
+
1032
+ return dag_ds
1033
+
1034
+ def update_element(self, name: str,
1035
+ id1: int,
1036
+ value: Union[int, float, List[int], List[float]]):
1037
+ """Update single value or row of values (e.g. nhru by nmonths) for a
1038
+ given nhm_id, nhm_seg, or 0 (for scalars).
1039
+
1040
+ :param name: name of parameter to update
1041
+ :param id1: scalar nhm_id or nhm_seg
1042
+ :param value: new value(s)
1043
+ """
1044
+
1045
+ # NOTE: id1 is either an nhm_id or nhm_seg (both are 1-based)
1046
+ cparam = self.get(name)
1047
+ idx0 = 0
1048
+
1049
+ try:
1050
+ if cparam.is_hru_param():
1051
+ # Lookup index for nhm_id
1052
+ idx0 = self.get('nhm_id')._value_index_1d(id1).item()
1053
+ elif cparam.is_seg_param():
1054
+ # Lookup index for nhm_seg
1055
+ idx0 = self.get('nhm_seg')._value_index_1d(id1).item()
1056
+ elif cparam.is_scalar:
1057
+ idx0 = 0
1058
+ except ValueError:
1059
+ raise ValueError(f'Exactly one index should be found for {id1}')
1060
+
1061
+ # This will raise a ValueError if more than one index is found
1062
+ cparam.update_element(idx0, value)
1063
+
1064
+ def write_dimensions_xml(self, output_dir: str):
1065
+ """Write global dimensions.xml file.
1066
+
1067
+ :param output_dir: output path for dimensions.xml file
1068
+ """
1069
+
1070
+ # Write the global dimensions xml file
1071
+ xmlstr = minidom.parseString(xmlET.tostring(self.xml_global_dimensions)).toprettyxml(indent=' ')
1072
+ with open(f'{output_dir}/{DIMENSIONS_XML}', 'w') as ff:
1073
+ ff.write(xmlstr)
1074
+
1075
+ def write_paramdb(self, output_dir: str):
1076
+ """Write all parameters using the paramDb output format.
1077
+
1078
+ :param output_dir: output path for paramDb files
1079
+ """
1080
+
1081
+ # Check for / create output directory
1082
+ try:
1083
+ if self.verbose: # pragma: no cover
1084
+ print(f'Creating output directory: {output_dir}')
1085
+ os.makedirs(output_dir)
1086
+ except OSError:
1087
+ if self.verbose: # pragma: no cover
1088
+ print("\tUsing existing directory")
1089
+
1090
+ # Write the global dimensions xml file
1091
+ self.write_dimensions_xml(output_dir)
1092
+
1093
+ # Write the global parameters xml file
1094
+ self.write_parameters_xml(output_dir)
1095
+
1096
+ for xx in self.parameters.values():
1097
+ # Write out each parameter in the paramDb csv format
1098
+ if self.verbose: # pragma: no cover
1099
+ print(xx.name)
1100
+
1101
+ with open(f'{output_dir}/{xx.name}.csv', 'w') as ff:
1102
+ ff.write(xx.toparamdb())
1103
+
1104
+ def write_parameter_file(self, filename: str,
1105
+ header: Optional[List[str]] = None):
1106
+ """Write a PRMS parameter file.
1107
+
1108
+ :param filename: name of parameter file
1109
+ :param header: list of header lines
1110
+ """
1111
+
1112
+ # Write the parameters out to a file
1113
+ outfile = open(filename, 'w')
1114
+
1115
+ if header is not None:
1116
+ if len(header) > 2:
1117
+ # TODO: 2023-04-19 - this check should happen before
1118
+ # opening the output file.
1119
+ raise ValueError('Header should be a list of two items')
1120
+ if len(header) == 1:
1121
+ # Must have two header lines
1122
+ outfile.write('Written by pyPRMS\n')
1123
+
1124
+ for hh in header:
1125
+ # Write out any header stuff
1126
+ outfile.write(f'{hh}\n')
1127
+ else:
1128
+ # Write a default header
1129
+ outfile.write('Written by pyPRMS\n')
1130
+ outfile.write('Comment: It is all downhill from here\n')
1131
+
1132
+ # Dimension section must be written first
1133
+ outfile.write(f'{CATEGORY_DELIM} Dimensions {CATEGORY_DELIM}\n')
1134
+
1135
+ for (kk, vv) in self.dimensions.items():
1136
+ # Write each dimension name and size separated by VAR_DELIM
1137
+ outfile.write(f'{VAR_DELIM}\n')
1138
+ outfile.write(f'{kk}\n')
1139
+ outfile.write(f'{vv.size:d}\n')
1140
+
1141
+ if self.prms_version < PRMS6_DEV_VERSION and {'ngw', 'nssr'}.isdisjoint(set(self.dimensions.keys())):
1142
+ # Add the ngw and nssr dimensions. These are always equal to nhru.
1143
+ for kk in ['ngw', 'nssr']:
1144
+ outfile.write(f'{VAR_DELIM}\n')
1145
+ outfile.write(f'{kk}\n')
1146
+ outfile.write(f'{self.dimensions["nhru"].size:d}\n')
1147
+
1148
+ # Now write out the Parameter category
1149
+ order = ['name', 'dimensions', 'datatype', 'data']
1150
+
1151
+ outfile.write(f'{CATEGORY_DELIM} Parameters {CATEGORY_DELIM}\n')
1152
+
1153
+ for vv in self.parameters.values():
1154
+ datatype = PTYPE_TO_PRMS_TYPE[vv.meta.get('datatype')]
1155
+
1156
+ for item in order:
1157
+ # Write each variable out separated by self.__rowdelim
1158
+ if item == 'dimensions':
1159
+ # Write number of dimensions first
1160
+ outfile.write(f'{vv.dimensions.ndim}\n')
1161
+
1162
+ for dd in vv.dimensions.values():
1163
+ # Write dimension names
1164
+ if self.prms_version < PRMS6_DEV_VERSION:
1165
+ # On-the-fly change of dimension names for certain parameters
1166
+ # when the prms version is not an experimental development version.
1167
+ if dd.name == 'nhru':
1168
+ if vv.name in ['gwflow_coef', 'gwsink_coef', 'gwstor_init',
1169
+ 'gwstor_min', 'gw_seep_coef']:
1170
+ outfile.write('ngw\n')
1171
+ elif vv.name in ['ssr2gw_exp', 'ssr2gw_rate', 'ssstor_init',
1172
+ 'ssstor_init_frac']:
1173
+ outfile.write('nssr\n')
1174
+ else:
1175
+ outfile.write(f'{dd.name}\n')
1176
+ else:
1177
+ outfile.write(f'{dd.name}\n')
1178
+ else:
1179
+ outfile.write(f'{dd.name}\n')
1180
+ elif item == 'datatype':
1181
+ # dimsize (which is computed) must be written before datatype
1182
+ outfile.write(f'{vv.data_raw.size}\n')
1183
+ outfile.write(f'{datatype}\n')
1184
+ elif item == 'data':
1185
+ # Write one value per line
1186
+ # WARNING: 2019-10-10: had to change next line from order='A' to order='F'
1187
+ # because flatten with 'A' was only honoring the Fortran memory layout
1188
+ # if the array was contiguous which isn't always the
1189
+ # case if the arrays have been altered in size.
1190
+ for xx in vv.data_raw.ravel(order='F'):
1191
+ if vv.meta.get('datatype', 'null') in ['float32', 'float64']:
1192
+ # Float and double types have to be formatted specially so
1193
+ # they aren't written in exponential notation or with
1194
+ # extraneous zeroes
1195
+ tmp = f'{xx:<20.7f}'.rstrip('0 ')
1196
+ if tmp[-1] == '.':
1197
+ tmp += '0'
1198
+
1199
+ outfile.write(f'{tmp}\n')
1200
+ else:
1201
+ outfile.write(f'{xx}\n')
1202
+ elif item == 'name':
1203
+ # Write the self.__rowdelim before the variable name
1204
+ outfile.write(f'{VAR_DELIM}\n')
1205
+ outfile.write(f'{vv.name}\n')
1206
+
1207
+ outfile.close()
1208
+
1209
+ def write_parameter_netcdf(self, filename: str):
1210
+ """Write parameters to a netcdf format file.
1211
+
1212
+ :param filename: full path for output file
1213
+ """
1214
+
1215
+ # Create the netcdf file
1216
+ nc_hdl = nc.Dataset(filename, 'w', clobber=True)
1217
+
1218
+ # Create dimensions
1219
+ for (kk, vv) in self.dimensions.items():
1220
+ if kk != 'one':
1221
+ # Dimension 'one' is only used for scalars in PRMS
1222
+ nc_hdl.createDimension(kk, vv.size)
1223
+
1224
+ # Create the variables
1225
+ # hruo = nco.createVariable('hru', 'i4', ('hru'))
1226
+ for vv in self.parameters.values():
1227
+ curr_datatype = NETCDF_DATATYPES[PTYPE_TO_PRMS_TYPE[vv.meta.get('datatype')]]
1228
+
1229
+ if curr_datatype != 'S1':
1230
+ if list(vv.dimensions.keys())[0] == 'one':
1231
+ # Scalar values
1232
+ curr_param = nc_hdl.createVariable(vv.name, curr_datatype,
1233
+ fill_value=nc.default_fillvals[curr_datatype], zlib=True)
1234
+ else:
1235
+ curr_param = nc_hdl.createVariable(vv.name, curr_datatype,
1236
+ tuple(list(vv.dimensions.keys())[::-1]),
1237
+ fill_value=nc.default_fillvals[curr_datatype], zlib=True)
1238
+
1239
+ # Add the attributes
1240
+ for ck, cmeta in vv.meta.items():
1241
+ if ck in ['minimum', 'maximum']:
1242
+ curr_param.setncattr(f'valid_{ck}', cmeta)
1243
+ elif ck in ['help', 'description', 'units']:
1244
+ curr_param.setncattr(ck, cmeta)
1245
+
1246
+ # Write the data
1247
+ if len(vv.dimensions.keys()) == 1:
1248
+ curr_param[:] = vv.data
1249
+ elif len(vv.dimensions.keys()) == 2:
1250
+ curr_param[:, :] = vv.data.transpose()
1251
+ else:
1252
+ # String parameter
1253
+ # Get the maximum string length in the array of data
1254
+ # print('String parameter: {}'.format(vv.name))
1255
+ str_size = len(max(vv.data, key=len))
1256
+ # print('size: {}'.format(str_size))
1257
+
1258
+ # Create a dimension for the string length
1259
+ nc_hdl.createDimension(vv.name + '_nchars', str_size)
1260
+
1261
+ # Temporary to add extra dimension for number of characters
1262
+ tmp_dims = list(vv.dimensions.keys())
1263
+ tmp_dims.extend([vv.name + '_nchars'])
1264
+ curr_param = nc_hdl.createVariable(vv.name, curr_datatype, tuple(tmp_dims),
1265
+ fill_value=nc.default_fillvals[curr_datatype], zlib=True)
1266
+
1267
+ # Add the attributes
1268
+ for ck, cmeta in vv.meta.items():
1269
+ if ck in ['help', 'description', 'units']:
1270
+ curr_param.setncattr(ck, cmeta)
1271
+
1272
+ # Write the data
1273
+ if len(tmp_dims) == 1:
1274
+ curr_param[:] = nc.stringtochar(vv.data)
1275
+ elif len(tmp_dims) == 2:
1276
+ # curr_param._Encoding = 'ascii'
1277
+
1278
+ # The stringtochar() routine won't handle the unicode numpy
1279
+ # datatype properly so we force it to dtype='S'
1280
+ curr_param[:, :] = nc.stringtochar(vv.data.astype('S'))
1281
+ sys.stdout.flush()
1282
+ # Close the netcdf file
1283
+ nc_hdl.close()
1284
+
1285
+ def write_parameters_metadata_csv(self, filename: str,
1286
+ sep: str = '\t'):
1287
+ """Writes the parameter metadata to a CSV file.
1288
+
1289
+ :param filename: output filename
1290
+ :param sep: separator character
1291
+ """
1292
+
1293
+ out_list = []
1294
+
1295
+ if self.__control is not None:
1296
+ modules_used = set(self.__control.modules.values()).union(set(self.__control.additional_modules))
1297
+
1298
+ for pk in sorted(list(self.parameters.keys())):
1299
+ pp = self.get(pk)
1300
+ md = pp.meta
1301
+
1302
+ param_modules = md.get('modules')
1303
+ assert type(param_modules) is list
1304
+
1305
+ if self.__control is not None:
1306
+ module_list = list(modules_used.intersection(set(param_modules)))
1307
+
1308
+ if len(module_list) == 0:
1309
+ # We have a parameter that is not needed by the declared modules
1310
+ if self.verbose: # pragma: no cover
1311
+ print(f'{pp.name} not used with selected modules')
1312
+ print(f' {md.get("modules")}')
1313
+ continue
1314
+ else:
1315
+ module_list = param_modules
1316
+
1317
+ for idx, mm in enumerate(module_list):
1318
+ if mm in external_module_map:
1319
+ module_list[idx] = external_module_map[mm]
1320
+
1321
+ modules = ', '.join(module_list)
1322
+
1323
+ dim_list = list(pp.dimensions.keys())
1324
+
1325
+ if pk in ['gwflow_coef', 'gwsink_coef', 'gwstor_init', 'gwstor_min', 'gw_seep_coef']:
1326
+ dim_list[0] = 'ngw'
1327
+ elif pk in ['ssr2gw_exp', 'ssr2gw_rate', 'ssstor_init', 'ssstor_init_frac']:
1328
+ dim_list[0] = 'nssr'
1329
+
1330
+ dims = ', '.join(dim_list)
1331
+
1332
+ try:
1333
+ act_min = pp.data_raw.min()
1334
+ act_max = pp.data_raw.max()
1335
+ except np.core._exceptions.UFuncTypeError: # type: ignore
1336
+ act_min = ''
1337
+ act_max = ''
1338
+
1339
+ out_list.append([pp.name, md.get('datatype', ''),
1340
+ md.get('units', ''),
1341
+ md.get('description', ''),
1342
+ md.get('minimum'),
1343
+ md.get('maximum'),
1344
+ act_min,
1345
+ act_max,
1346
+ md.get('default'),
1347
+ dims,
1348
+ modules])
1349
+
1350
+ col_names = ['parameter_name', 'datatype', 'units', 'description',
1351
+ 'valid_minimum', 'valid_maximum', 'actual_minimum',
1352
+ 'actual_maximum', 'default', 'dimension', 'modules']
1353
+
1354
+ df = pd.DataFrame.from_records(out_list, columns=col_names)
1355
+ if sep == ',':
1356
+ df.to_csv(filename, sep=sep, quotechar='"', index=False)
1357
+ else:
1358
+ df.to_csv(filename, sep=sep, index=False)
1359
+
1360
+ def write_parameters_xml(self, output_dir: str):
1361
+ """Write global parameters.xml file.
1362
+
1363
+ :param output_dir: output path for parameters.xml file
1364
+ """
1365
+
1366
+ # Write the global parameters xml file
1367
+ xmlstr = minidom.parseString(xmlET.tostring(self.xml_global_parameters)).toprettyxml(indent=' ')
1368
+ with open(f'{output_dir}/{PARAMETERS_XML}', 'w') as ff:
1369
+ ff.write(xmlstr)
1370
+
1371
+ def _condition_check_ctl(self, cstr: str) -> bool:
1372
+ """Takes a string of the form '<control_var> <op> <value>' and checks
1373
+ if the condition is True.
1374
+
1375
+ :param cstr: string of the form '<control_var> <op> <value>'
1376
+ :returns: True if the condition is met, False otherwise
1377
+ """
1378
+
1379
+ assert self.control is not None
1380
+ var, op, value = cstr.split(' ')
1381
+
1382
+ cdtype = NEW_PTYPE_TO_DTYPE[self.control.get(var).meta['datatype']]
1383
+ ctl_val = self.control.get(var).values
1384
+ assert ctl_val is not None
1385
+ value = cdtype(value)
1386
+
1387
+ return cond_check[op](ctl_val, value)
1388
+
1389
+ def _condition_check_dim(self, cstr: str) -> bool:
1390
+ """Takes a string of the form '<dimension> <op> <value>' and checks
1391
+ if the condition is True.
1392
+
1393
+ :param cstr: string of the form '<dimension> <op> <value>'
1394
+ :returns: True if the condition is met, False otherwise
1395
+ """
1396
+
1397
+ var, op, value = cstr.split(' ')
1398
+ value = int(value) # type: ignore
1399
+
1400
+ if self.dimensions.exists(var):
1401
+ return cond_check[op](self.dimensions.get(var).size, value)
1402
+ return False
1403
+
1404
+ def _read(self):
1405
+ """Abstract function for reading parameters into Parameters object.
1406
+ """
1407
+
1408
+ assert False, 'Parameters._read() must be defined by child class'
1409
+
1410
+ def _required_parameters(self) -> Set:
1411
+ """Return set of parameters required by modules selected in control file.
1412
+
1413
+ :returns: set of required parameter names
1414
+ """
1415
+ if self.__control is None:
1416
+ # TODO: 20230727 PAN - this should raise an exception
1417
+ return set()
1418
+
1419
+ modules_used = set(self.__control.modules.values()).union(set(self.__control.additional_modules))
1420
+
1421
+ # -------------------------------
1422
+ # Get set of parameters required by the modules used
1423
+ pset = set()
1424
+ for kk, vv in self.metadata.items():
1425
+ for mm in vv['modules']:
1426
+ if mm in modules_used:
1427
+ pset.add(kk)
1428
+
1429
+ # Remove parameters that do not meet secondary requirements defined
1430
+ # in the metadata
1431
+ return self._trim_req_params(pset)
1432
+
1433
+ def _trim_req_params(self, param_set: Set) -> Set:
1434
+ """Remove parameters from a set of parameters that do not meet secondary requirements.
1435
+
1436
+ :param param_set: set of parameter names
1437
+ :returns: set of parameter names that meet secondary requirements
1438
+ """
1439
+
1440
+ remove_set = set()
1441
+
1442
+ for cparam in param_set:
1443
+ for xx in self.metadata[cparam].get('requires_control', []):
1444
+ if not self._condition_check_ctl(xx):
1445
+ remove_set.add(cparam)
1446
+ if self.verbose: # pragma: no cover
1447
+ con.print(f'[bold]{cparam}[/]: Control condition ({xx}) not met')
1448
+
1449
+ for xx in self.metadata[cparam].get('requires_dimension', []):
1450
+ if not self._condition_check_dim(xx):
1451
+ remove_set.add(cparam)
1452
+ if self.verbose: # pragma: no cover
1453
+ con.print(f'[bold]{cparam}[/]: Dimension condition ({xx}) not met')
1454
+
1455
+ for vv in remove_set:
1456
+ param_set.remove(vv)
1457
+
1458
+ return param_set
1459
+
1460
+ def _upstream_hrus(self, streamnet: nx.DiGraph,
1461
+ dsmost_seg: List[int]) -> List[int]:
1462
+ """Get list of HRUs that contribute to the given stream segments.
1463
+
1464
+ :param streamnet: Directed, Acyclic Graph (DAG) of stream network
1465
+ :param dsmost_seg: list of downstream-most segment IDs
1466
+
1467
+ :returns: list of HRUs that contribute to the stream segments
1468
+ """
1469
+
1470
+ # Get subset of stream network for given segment
1471
+ dag_ds_subset = get_streamnet_subset(streamnet, [], dsmost_seg)
1472
+
1473
+ # Create list of segments in the subset
1474
+ toseg_idx = list(set(xx[0] for xx in dag_ds_subset.edges))
1475
+
1476
+ # Build list of HRUs that contribute to the POI
1477
+ final_hru_list = []
1478
+
1479
+ for xx in toseg_idx:
1480
+ try:
1481
+ for yy in self.seg_to_hru[xx]:
1482
+ final_hru_list.append(yy)
1483
+ except KeyError:
1484
+ # Not all segments have HRUs connected to them
1485
+ print(f'Segment {xx} has no HRUs connected to it')
1486
+
1487
+ final_hru_list.sort()
1488
+ return final_hru_list
1489
+
1490
+ def _upstream_segments(self, streamnet: nx.DiGraph,
1491
+ dsmost_seg: List[int]) -> List[int]:
1492
+ """Get list of segments that contribute to the given stream segments.
1493
+
1494
+ :param streamnet: Directed, Acyclic Graph (DAG) of stream network
1495
+ :param dsmost_seg: list of downstream-most segment IDs
1496
+
1497
+ :returns: list of segments that contribute to the stream segments
1498
+ """
1499
+
1500
+ # Get subset of stream network for given POI
1501
+ dag_ds_subset = get_streamnet_subset(streamnet, [], dsmost_seg)
1502
+
1503
+ # Create list of segments in the subset
1504
+ toseg_idx = list(set(xx[0] for xx in dag_ds_subset.edges))
1505
+
1506
+ return toseg_idx