pyPRMS 0.9.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyPRMS/Exceptions_custom.py +31 -0
- pyPRMS/__init__.py +52 -0
- pyPRMS/cbh/Cbh.py +431 -0
- pyPRMS/cbh/CbhAscii.py +458 -0
- pyPRMS/cbh/CbhNetcdf.py +199 -0
- pyPRMS/cbh/__init__.py +3 -0
- pyPRMS/constants.py +131 -0
- pyPRMS/control/Control.py +362 -0
- pyPRMS/control/ControlFile.py +161 -0
- pyPRMS/control/ControlVariable.py +208 -0
- pyPRMS/control/__init__.py +3 -0
- pyPRMS/dimensions/Dimension.py +154 -0
- pyPRMS/dimensions/Dimensions.py +256 -0
- pyPRMS/dimensions/__init__.py +2 -0
- pyPRMS/input/DataFile.py +354 -0
- pyPRMS/input/InputVariable.py +61 -0
- pyPRMS/input/__init__.py +0 -0
- pyPRMS/metadata/__init__.py +1 -0
- pyPRMS/metadata/metadata.py +430 -0
- pyPRMS/parameters/ParamDb.py +73 -0
- pyPRMS/parameters/Parameter.py +624 -0
- pyPRMS/parameters/ParameterFile.py +190 -0
- pyPRMS/parameters/ParameterNetCDF.py +74 -0
- pyPRMS/parameters/ParameterSet.py +96 -0
- pyPRMS/parameters/Parameters.py +1506 -0
- pyPRMS/parameters/__init__.py +5 -0
- pyPRMS/plot_helpers.py +305 -0
- pyPRMS/prms_helpers.py +235 -0
- pyPRMS/py.typed +0 -0
- pyPRMS/summary/OutputCSV.py +64 -0
- pyPRMS/summary/OutputVariable.py +163 -0
- pyPRMS/summary/OutputVariables.py +227 -0
- pyPRMS/summary/__init__.py +2 -0
- pyPRMS/utilities/__init__.py +0 -0
- pyPRMS/utilities/convert_cbh.py +107 -0
- pyPRMS/utilities/convert_model_output.py +91 -0
- pyPRMS/utilities/convert_params.py +60 -0
- pyPRMS/version.py +13 -0
- pyPRMS/xml/cbh.xml +163 -0
- pyPRMS/xml/control.xml +1447 -0
- pyPRMS/xml/dimensions.xml +311 -0
- pyPRMS/xml/modules.xml +251 -0
- pyPRMS/xml/parameters.xml +6932 -0
- pyPRMS/xml/time_series_input.xml +198 -0
- pyPRMS/xml/variables.xml +8173 -0
- pyprms-0.9.7.dist-info/LICENSE.md +21 -0
- pyprms-0.9.7.dist-info/METADATA +67 -0
- pyprms-0.9.7.dist-info/RECORD +51 -0
- pyprms-0.9.7.dist-info/WHEEL +5 -0
- pyprms-0.9.7.dist-info/entry_points.txt +3 -0
- pyprms-0.9.7.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1506 @@
|
|
|
1
|
+
|
|
2
|
+
import cartopy.crs as ccrs # type: ignore
|
|
3
|
+
import gc
|
|
4
|
+
import matplotlib as mpl # type: ignore
|
|
5
|
+
import matplotlib.pyplot as plt # type: ignore
|
|
6
|
+
import netCDF4 as nc # type: ignore
|
|
7
|
+
import networkx as nx # type: ignore
|
|
8
|
+
import numpy as np
|
|
9
|
+
import numpy.typing as npt
|
|
10
|
+
import pandas as pd # type: ignore
|
|
11
|
+
import sys
|
|
12
|
+
import xml.dom.minidom as minidom
|
|
13
|
+
import xml.etree.ElementTree as xmlET
|
|
14
|
+
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from collections.abc import KeysView
|
|
17
|
+
from functools import cached_property
|
|
18
|
+
from packaging.version import Version
|
|
19
|
+
from typing import Any, Literal, Optional, Sequence, Union, Dict, List, Set, Tuple
|
|
20
|
+
from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER # type: ignore
|
|
21
|
+
|
|
22
|
+
from ..control.Control import Control
|
|
23
|
+
from ..dimensions.Dimensions import Dimensions
|
|
24
|
+
from ..Exceptions_custom import ParameterError, ParameterExistsError, ParameterNotValidError
|
|
25
|
+
from .Parameter import Parameter, ParamDataRawType
|
|
26
|
+
from ..plot_helpers import set_colormap, get_projection, plot_line_collection, plot_polygon_collection, get_figsize
|
|
27
|
+
from ..prms_helpers import cond_check, flex_type, get_streamnet_subset
|
|
28
|
+
from ..constants import (CATEGORY_DELIM, DIMENSIONS_XML, external_module_map, MetaDataType, NETCDF_DATATYPES,
|
|
29
|
+
NEW_PTYPE_TO_DTYPE, PRMS_VERSION, PRMS6_DEV_VERSION, PTYPE_TO_PRMS_TYPE, NHM_DATATYPES,
|
|
30
|
+
PARAMETERS_XML, VAR_DELIM)
|
|
31
|
+
|
|
32
|
+
from rich.console import Console
|
|
33
|
+
from rich import pretty
|
|
34
|
+
|
|
35
|
+
import os
|
|
36
|
+
os.environ['USE_PYGEOS'] = '0'
|
|
37
|
+
import geopandas # type: ignore
|
|
38
|
+
|
|
39
|
+
pretty.install()
|
|
40
|
+
con = Console(force_jupyter=False)
|
|
41
|
+
|
|
42
|
+
LimitOptions = Literal['valid', 'centered', 'absolute']
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Parameters(object):
|
|
46
|
+
"""Container of multiple pyPRMS.Parameter objects.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
# Author: Parker Norton (pnorton@usgs.gov)
|
|
50
|
+
# Create date: 2017-05-01
|
|
51
|
+
|
|
52
|
+
def __init__(self, metadata: MetaDataType,
|
|
53
|
+
verbose: Optional[bool] = False):
|
|
54
|
+
"""Initialize the Parameters object.
|
|
55
|
+
|
|
56
|
+
Create an ordered dictionary to contain pyPRMS.Parameter objects
|
|
57
|
+
|
|
58
|
+
:param metadata: Metadata for the parameters
|
|
59
|
+
:param verbose: Output debugging information
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
self.__dimensions = Dimensions(metadata=metadata, verbose=verbose)
|
|
63
|
+
self.__parameters: Dict[str, Parameter] = dict()
|
|
64
|
+
|
|
65
|
+
self.verbose = verbose
|
|
66
|
+
self.__control: Optional[Control] = None
|
|
67
|
+
self.__hru_poly = None
|
|
68
|
+
self.__hru_shape_key: Optional[str] = None
|
|
69
|
+
self.__seg_poly = None
|
|
70
|
+
self.__seg_shape_key: Optional[str] = None
|
|
71
|
+
self.__seg_to_hru: Dict[int, List[int]] = dict()
|
|
72
|
+
self.__hru_to_seg: Dict[int, int] = dict()
|
|
73
|
+
self.metadata = metadata['parameters']
|
|
74
|
+
self.prms_version = Version(metadata['info']['version'])
|
|
75
|
+
|
|
76
|
+
def __getattr__(self, name: str):
|
|
77
|
+
"""Not sure what to write yet.
|
|
78
|
+
|
|
79
|
+
:param name: Name of the attribute
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
# Undefined attributes will look up the given parameter
|
|
83
|
+
# return self.get(item)
|
|
84
|
+
# https://nedbatchelder.com/blog/201010/surprising_getattr_recursion.html
|
|
85
|
+
if name == "__setstate__":
|
|
86
|
+
raise AttributeError(name)
|
|
87
|
+
return getattr(self.__parameters, name)
|
|
88
|
+
|
|
89
|
+
def __getitem__(self, item):
|
|
90
|
+
"""Not sure what to write yet.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
return self.get(item)
|
|
94
|
+
|
|
95
|
+
def __str__(self) -> str:
|
|
96
|
+
"""Pretty-print string representation of the Parameters object.
|
|
97
|
+
|
|
98
|
+
:return: Pretty-print string of Parameters
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
outstr = '----- Dimensions -----\n'
|
|
102
|
+
for vv in self.__dimensions.values():
|
|
103
|
+
outstr += f'{vv.name}: size={vv.size}\n'
|
|
104
|
+
|
|
105
|
+
outstr += '----- Parameters -----\n'
|
|
106
|
+
for vv in self.__parameters.values():
|
|
107
|
+
outstr += f'{vv.name} [{", ".join(vv.meta["dimensions"])}]\n'
|
|
108
|
+
|
|
109
|
+
return outstr
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def control(self) -> Optional[Control]:
|
|
113
|
+
"""Get Control object.
|
|
114
|
+
|
|
115
|
+
:returns: Control object
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
return self.__control
|
|
119
|
+
|
|
120
|
+
@control.setter
|
|
121
|
+
def control(self, ctl_obj: Control):
|
|
122
|
+
"""Sets the Control object for the ParameterSet.
|
|
123
|
+
|
|
124
|
+
:param ctl_obj: Control object
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
self.__control = ctl_obj
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def dimensions(self) -> Dimensions:
|
|
131
|
+
"""Get dimensions object.
|
|
132
|
+
|
|
133
|
+
:returns: Dimensions object
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
return self.__dimensions
|
|
137
|
+
|
|
138
|
+
@cached_property
|
|
139
|
+
def hru_to_seg(self) -> Dict[int, int]:
|
|
140
|
+
"""Returns an ordered dictionary mapping NHM HRU IDs to HRU NHM segment IDs.
|
|
141
|
+
|
|
142
|
+
:returns: dictionary mapping nhm_id to hru_segment_nhm
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
# Only supported with python >= 3.9
|
|
146
|
+
hru_segment = self.get('hru_segment_nhm').data_raw
|
|
147
|
+
nhm_id = self.get('nhm_id').data_raw
|
|
148
|
+
|
|
149
|
+
self.__hru_to_seg = dict(np.rec.fromarrays([nhm_id, hru_segment]).tolist()) # type: ignore
|
|
150
|
+
return self.__hru_to_seg
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def missing_params(self) -> Set:
|
|
154
|
+
"""Get set of parameters that are required for the modules selected
|
|
155
|
+
in the control file but are missing in the current set of parameters.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
# TODO: 20240726 PAN - raise warning/error when certain parameters
|
|
159
|
+
# like hru_segment_nhm, etc are missing
|
|
160
|
+
pset = self._required_parameters()
|
|
161
|
+
return pset.difference(set(self.parameters.keys()))
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def parameters(self) -> Dict[str, Parameter]:
|
|
165
|
+
"""Returns an ordered dictionary of parameter objects.
|
|
166
|
+
|
|
167
|
+
:returns: dictionary of Parameter objects
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
return self.__parameters
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def poi_to_seg(self) -> Dict[str, int]:
|
|
174
|
+
"""Returns a dictionary mapping poi_id to local poi_seg.
|
|
175
|
+
|
|
176
|
+
:returns: dictionary mapping poi_id to local poi_seg"""
|
|
177
|
+
|
|
178
|
+
gage_ids = self.get('poi_gage_id').data_raw
|
|
179
|
+
gage_segments = self.get('poi_gage_segment').data_raw
|
|
180
|
+
return dict(np.rec.fromarrays([gage_ids, gage_segments]).tolist()) # type: ignore
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def poi_to_seg0(self) -> Dict[str, int]:
|
|
184
|
+
"""Returns a dictionary mapping poi_id to local, zero-based poi_seg.
|
|
185
|
+
|
|
186
|
+
:returns: dictionary mapping poi_id to local, zero-based poi_seg"""
|
|
187
|
+
gage_ids = self.get('poi_gage_id').data_raw # .tolist()
|
|
188
|
+
gage_segments = np.subtract(self.get('poi_gage_segment').data_raw, 1)
|
|
189
|
+
return dict(np.rec.fromarrays([gage_ids, gage_segments]).tolist()) # type: ignore
|
|
190
|
+
|
|
191
|
+
@cached_property
|
|
192
|
+
def seg_to_hru(self) -> Dict[int, List[int]]:
|
|
193
|
+
"""Returns a dictionary mapping HRU global segment IDs to global HRU IDs.
|
|
194
|
+
|
|
195
|
+
Segment keys equal to zero are for non-routed HRUs. Segment keys greater than
|
|
196
|
+
zero are global segment IDs.
|
|
197
|
+
|
|
198
|
+
:returns: dictionary mapping hru_segment_nhm to nhm_id
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
hru_segment = self.get('hru_segment_nhm').data_raw.tolist()
|
|
202
|
+
nhm_id = self.get('nhm_id').data_raw
|
|
203
|
+
assert type(nhm_id) is np.ndarray and nhm_id.dtype == np.int32
|
|
204
|
+
|
|
205
|
+
for chru_idx, cseg in enumerate(hru_segment): # type: ignore
|
|
206
|
+
# keys are 1-based, values in arrays are 1-based
|
|
207
|
+
# Non-routed HRUs have a seg key = zero
|
|
208
|
+
self.__seg_to_hru.setdefault(cseg, []).append(nhm_id[chru_idx].item()) # type: ignore
|
|
209
|
+
return self.__seg_to_hru
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def unneeded_parameters(self) -> Set:
|
|
213
|
+
"""Get set of parameters that are defined but not needed by any of the
|
|
214
|
+
modules selected in the control file.
|
|
215
|
+
|
|
216
|
+
:returns: set of unneeded parameter names
|
|
217
|
+
"""
|
|
218
|
+
|
|
219
|
+
if self.verbose: # pragma: no cover
|
|
220
|
+
con.print('-'*20, 'unneeded_parameters', '-'*20)
|
|
221
|
+
|
|
222
|
+
pset = self._required_parameters()
|
|
223
|
+
return set(self.parameters.keys()).difference(pset)
|
|
224
|
+
|
|
225
|
+
@property
|
|
226
|
+
def xml_global_dimensions(self) -> xmlET.Element:
|
|
227
|
+
"""Get XML element tree of the dimensions used by all parameters.
|
|
228
|
+
|
|
229
|
+
:returns: element tree of dimensions
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
dims_xml = xmlET.Element('dimensions')
|
|
233
|
+
|
|
234
|
+
for cdim in self.dimensions.dimensions.values():
|
|
235
|
+
dim_sub = xmlET.SubElement(dims_xml, 'dimension')
|
|
236
|
+
dim_sub.set('name', cdim.name)
|
|
237
|
+
|
|
238
|
+
for kk, vv in cdim.meta.items():
|
|
239
|
+
if kk in ['is_fixed', 'requires_control']:
|
|
240
|
+
pass
|
|
241
|
+
else:
|
|
242
|
+
xmlET.SubElement(dim_sub, kk).text = flex_type(vv)
|
|
243
|
+
return dims_xml
|
|
244
|
+
|
|
245
|
+
@property
|
|
246
|
+
def xml_global_parameters(self) -> xmlET.Element:
|
|
247
|
+
"""Get XML element tree of the parameters.
|
|
248
|
+
|
|
249
|
+
:returns: element tree of parameters
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
# Map datatypes (e.g. np.int32) to the parameter type strings (e.g. 'I')
|
|
253
|
+
inv_nhm_datatypes = {vv: kk for kk, vv in NHM_DATATYPES.items()}
|
|
254
|
+
datatype_to_prms_type = {kk: inv_nhm_datatypes[vv] for kk, vv in PTYPE_TO_PRMS_TYPE.items()}
|
|
255
|
+
|
|
256
|
+
params_xml = xmlET.Element('parameters')
|
|
257
|
+
|
|
258
|
+
for pk in sorted(list(self.parameters.keys())):
|
|
259
|
+
vv = self.get(pk)
|
|
260
|
+
|
|
261
|
+
param_sub = xmlET.SubElement(params_xml, 'parameter')
|
|
262
|
+
param_sub.set('name', vv.name)
|
|
263
|
+
|
|
264
|
+
xmlET.SubElement(param_sub, 'type').text = datatype_to_prms_type[vv.meta['datatype']]
|
|
265
|
+
|
|
266
|
+
for mm, md in vv.meta.items():
|
|
267
|
+
if mm == 'modules':
|
|
268
|
+
modules_sub = xmlET.SubElement(param_sub, 'modules')
|
|
269
|
+
|
|
270
|
+
for cmod in md:
|
|
271
|
+
xmlET.SubElement(modules_sub, 'module').text = cmod
|
|
272
|
+
elif mm in ['datatype', 'requires_control', 'requires_dimension', 'version']:
|
|
273
|
+
pass
|
|
274
|
+
elif mm == 'dimensions':
|
|
275
|
+
param_sub.append(vv.dimensions.xml)
|
|
276
|
+
else:
|
|
277
|
+
xmlET.SubElement(param_sub, mm).text = flex_type(md)
|
|
278
|
+
|
|
279
|
+
return params_xml
|
|
280
|
+
|
|
281
|
+
# =========================================================================
|
|
282
|
+
# Methods
|
|
283
|
+
def add(self, name: str):
|
|
284
|
+
"""Add a new parameter by name.
|
|
285
|
+
|
|
286
|
+
:param name: A valid PRMS parameter name
|
|
287
|
+
|
|
288
|
+
:raises ParameterError: if parameter already exists or name is None
|
|
289
|
+
"""
|
|
290
|
+
|
|
291
|
+
# Add a new parameter
|
|
292
|
+
if self.exists(name):
|
|
293
|
+
raise ParameterExistsError(f'{name}: Parameter already exists')
|
|
294
|
+
|
|
295
|
+
if name not in self.metadata:
|
|
296
|
+
raise ParameterNotValidError(f'{name}: Parameter is not a valid PRMS parameter')
|
|
297
|
+
|
|
298
|
+
for cdim in self.metadata[name]['dimensions']:
|
|
299
|
+
if not self.__dimensions.exists(cdim):
|
|
300
|
+
raise KeyError(f'Global dimension, {cdim}, does not exist')
|
|
301
|
+
|
|
302
|
+
self.__parameters[name] = Parameter(name=name, meta=self.metadata, global_dims=self.__dimensions)
|
|
303
|
+
|
|
304
|
+
def add_metadata(self, name: str,
|
|
305
|
+
metadata: Dict):
|
|
306
|
+
"""Add a new parameter entry to the parameter metadata. This is useful for adding ad-hoc parameters.
|
|
307
|
+
|
|
308
|
+
:param name: Name of the parameter
|
|
309
|
+
:param metadata: Dictionary of metadata for the parameter
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
# entry_items = dict(datatype='float32', description='something new today', help='get your own help',
|
|
313
|
+
# units='dontmatter', default=0.0, minimum=0.0, maximum=100.0, dimensions=['nhru'])
|
|
314
|
+
new_entry = defaultdict(list)
|
|
315
|
+
|
|
316
|
+
for kk, vv in metadata.items():
|
|
317
|
+
new_entry[kk] = vv
|
|
318
|
+
|
|
319
|
+
self.metadata[name] = new_entry
|
|
320
|
+
|
|
321
|
+
def add_missing_parameters(self):
|
|
322
|
+
"""Add missing parameters that are required by the selected modules.
|
|
323
|
+
"""
|
|
324
|
+
if self.verbose:
|
|
325
|
+
con.print('-'*20, 'add_missing_parameters', '-'*20) # pragma: no cover
|
|
326
|
+
|
|
327
|
+
for cparam in self.missing_params:
|
|
328
|
+
if cparam in ['nhm_deplcrv']:
|
|
329
|
+
if self.verbose: # pragma: no cover
|
|
330
|
+
con.print(f'[bold]{cparam}[/] is missing but lacks the information to be added')
|
|
331
|
+
continue
|
|
332
|
+
|
|
333
|
+
self.add(cparam)
|
|
334
|
+
self.get(cparam).data = self.metadata[cparam].get('default')
|
|
335
|
+
|
|
336
|
+
if self.verbose: # pragma: no cover
|
|
337
|
+
con.print(f'[bold]{cparam}[/] [gold3] parameter added with default value[/]')
|
|
338
|
+
|
|
339
|
+
def adjust_bounded_parameters(self):
|
|
340
|
+
"""Adjust the valid upper and lower values for bounded parameters.
|
|
341
|
+
"""
|
|
342
|
+
|
|
343
|
+
for cparam in self.parameters.values():
|
|
344
|
+
cmeta = cparam.meta
|
|
345
|
+
|
|
346
|
+
if cmeta.get('maximum') in list(self.dimensions.keys()):
|
|
347
|
+
# if isinstance(cmeta.get('maximum'), str):
|
|
348
|
+
try:
|
|
349
|
+
cmeta['maximum'] = self.dimensions.get(cmeta.get('maximum')).size
|
|
350
|
+
|
|
351
|
+
if self.verbose: # pragma: no cover
|
|
352
|
+
con.print(f'[bold]{cparam.name}[/]: valid upper bound adjusted to {cmeta["maximum"]}')
|
|
353
|
+
except ValueError:
|
|
354
|
+
print(f'{cparam.name} has bad valid uppper bound value')
|
|
355
|
+
raise
|
|
356
|
+
|
|
357
|
+
def check(self): # pragma: no cover
|
|
358
|
+
"""Check all parameter variables for proper array size.
|
|
359
|
+
"""
|
|
360
|
+
|
|
361
|
+
# for pp in self.__parameters.values():
|
|
362
|
+
for pk in sorted(list(self.parameters.keys())):
|
|
363
|
+
pp = self.get(pk)
|
|
364
|
+
|
|
365
|
+
# print(pp.check())
|
|
366
|
+
if pp.has_correct_size():
|
|
367
|
+
con.print(f'[bold]{pk}[/]: Size [green4]OK[/green4]')
|
|
368
|
+
else:
|
|
369
|
+
con.print(f'[bold]{pk}[/]: [red]Incorrect number of values for dimensions[/red]')
|
|
370
|
+
|
|
371
|
+
if not pp.check_values():
|
|
372
|
+
pp_stats = pp.stats()
|
|
373
|
+
pp_outliers = pp.outliers()
|
|
374
|
+
valid_min = pp.meta['minimum']
|
|
375
|
+
valid_max = pp.meta['maximum']
|
|
376
|
+
default_val = pp.meta['default']
|
|
377
|
+
|
|
378
|
+
if not (isinstance(valid_min, str) or isinstance(valid_max, str)):
|
|
379
|
+
con.print(f' [dark_orange3]WARNING[/]: Value(s) (range: {pp_stats.min}, {pp_stats.max}) outside '
|
|
380
|
+
+ f'the valid range of ({valid_min}, {valid_max}); '
|
|
381
|
+
+ f'under/over=({pp_outliers.under}, {pp_outliers.over})')
|
|
382
|
+
# print(f' WARNING: Value(s) (range: {pp.data.min()}, {pp.data.max()}) outside ' +
|
|
383
|
+
# f'the valid range of ({pp.minimum}, {pp.maximum})')
|
|
384
|
+
elif valid_min == 'bounded':
|
|
385
|
+
# TODO: Handling bounded parameters needs improvement
|
|
386
|
+
con.print(f' [dark_orange3]WARNING[/]: Bounded parameter value(s) '
|
|
387
|
+
+ f'(range: {pp_stats.min}, {pp_stats.max}) outside '
|
|
388
|
+
+ f'the valid range of ({default_val}, {valid_max})')
|
|
389
|
+
|
|
390
|
+
if pp.all_equal():
|
|
391
|
+
dims = list(pp.dimensions.keys())
|
|
392
|
+
|
|
393
|
+
if pp.is_scalar:
|
|
394
|
+
con.print(f' INFO: Scalar; value = {pp.data}')
|
|
395
|
+
elif pp.data.ndim == 2:
|
|
396
|
+
con.print(f' INFO: dimensioned {dims}; all values by {dims[0]} are equal to {pp.data[0]}')
|
|
397
|
+
elif pp.data.ndim == 1:
|
|
398
|
+
con.print(f' INFO: dimensioned {dims}; all values are equal to {pp.data[0]}')
|
|
399
|
+
|
|
400
|
+
if pp.name == 'snarea_curve':
|
|
401
|
+
if pp.as_dataframe.values.reshape((-1, 11)).shape[0] != self.get('hru_deplcrv').unique().size:
|
|
402
|
+
con.print(' [yellow3]WARNING[/]: snarea_curve has more entries than needed by hru_deplcrv')
|
|
403
|
+
|
|
404
|
+
def exists(self, name: str) -> bool:
|
|
405
|
+
"""Checks if a parameter name exists.
|
|
406
|
+
|
|
407
|
+
:param name: Name of the parameter
|
|
408
|
+
:returns: True if parameter exists, otherwise False
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
return name in self.parameters.keys()
|
|
412
|
+
|
|
413
|
+
def get(self, name: str) -> Parameter:
|
|
414
|
+
"""Returns a parameter object.
|
|
415
|
+
|
|
416
|
+
:param name: The name of the parameter
|
|
417
|
+
:returns: Parameter object
|
|
418
|
+
"""
|
|
419
|
+
|
|
420
|
+
# Return the given parameter
|
|
421
|
+
if self.exists(name):
|
|
422
|
+
return self.__parameters[name]
|
|
423
|
+
|
|
424
|
+
raise ParameterError(f'Parameter, {name}, does not exist.')
|
|
425
|
+
|
|
426
|
+
def get_dataframe(self, name: str) -> pd.DataFrame:
|
|
427
|
+
"""Returns a pandas DataFrame for a parameter.
|
|
428
|
+
|
|
429
|
+
If the parameter dimensions include either nhru or nsegment then the
|
|
430
|
+
respective national ids (nhm_seg, nhm_id) are included, if they exist, as the index in the
|
|
431
|
+
returned dataframe.
|
|
432
|
+
|
|
433
|
+
:param name: The name of the parameter
|
|
434
|
+
:returns: Pandas DataFrame of the parameter data
|
|
435
|
+
"""
|
|
436
|
+
|
|
437
|
+
if not self.exists(name):
|
|
438
|
+
raise KeyError(f'Parameter, {name}, does not exist')
|
|
439
|
+
|
|
440
|
+
cparam = self.get(name)
|
|
441
|
+
param_data = cparam.as_dataframe
|
|
442
|
+
|
|
443
|
+
if cparam.is_hru_param():
|
|
444
|
+
if name != 'nhm_id':
|
|
445
|
+
if self.exists('nhm_id'):
|
|
446
|
+
param_id = self.get('nhm_id').as_dataframe
|
|
447
|
+
|
|
448
|
+
# Create a DataFrame of the parameter
|
|
449
|
+
param_data = param_data.merge(param_id, left_index=True, right_index=True)
|
|
450
|
+
param_data.set_index('nhm_id', inplace=True)
|
|
451
|
+
param_data.index.name = 'nhm_id'
|
|
452
|
+
else:
|
|
453
|
+
param_data = self.get('nhm_id').as_dataframe
|
|
454
|
+
elif cparam.is_seg_param():
|
|
455
|
+
if name != 'nhm_seg':
|
|
456
|
+
if self.exists('nhm_seg'):
|
|
457
|
+
param_id = self.get('nhm_seg').as_dataframe
|
|
458
|
+
|
|
459
|
+
# Create a DataFrame of the parameter
|
|
460
|
+
param_data = param_data.merge(param_id, left_index=True, right_index=True)
|
|
461
|
+
param_data.set_index('nhm_seg', inplace=True)
|
|
462
|
+
param_data.index.name = 'nhm_seg'
|
|
463
|
+
else:
|
|
464
|
+
param_data = self.get('nhm_seg').as_dataframe
|
|
465
|
+
elif name == 'snarea_curve':
|
|
466
|
+
# Special handling for snarea_curve parameter
|
|
467
|
+
param_data = pd.DataFrame(cparam.as_dataframe.values.reshape((-1, 11)))
|
|
468
|
+
param_data.rename(columns={k: k+1 for k in param_data.columns}, # type: ignore
|
|
469
|
+
index={k: k+1 for k in param_data.index},
|
|
470
|
+
inplace=True)
|
|
471
|
+
param_data.index.name = 'curve_index'
|
|
472
|
+
return param_data
|
|
473
|
+
|
|
474
|
+
def get_subset(self, name: str,
|
|
475
|
+
global_ids: List[int]) -> ParamDataRawType:
|
|
476
|
+
"""Returns a subset for a parameter based on the global_ids (e.g. nhm_id, nhm_seg).
|
|
477
|
+
|
|
478
|
+
:param name: Name of the parameter
|
|
479
|
+
:param global_ids: List of global IDs to extract
|
|
480
|
+
:returns: Array of extracted values
|
|
481
|
+
"""
|
|
482
|
+
|
|
483
|
+
param = self.get(name)
|
|
484
|
+
dim_set = set(param.dimensions.keys()).intersection({'nhru', 'nssr', 'ngw', 'nsegment', 'ndeplval'})
|
|
485
|
+
id_index_map: Union[Dict[Any, int], None] = {}
|
|
486
|
+
cdim = dim_set.pop()
|
|
487
|
+
|
|
488
|
+
if cdim in ['nhru', 'nssr', 'ngw', 'ndeplval']:
|
|
489
|
+
# Global IDs should be in the range of nhm_id
|
|
490
|
+
id_index_map = self.get('nhm_id').index_map
|
|
491
|
+
elif cdim in ['nsegment']:
|
|
492
|
+
# Global IDs should be in the range of nhm_seg
|
|
493
|
+
id_index_map = self.get('nhm_seg').index_map
|
|
494
|
+
|
|
495
|
+
# Zero-based indices in order of global_ids
|
|
496
|
+
assert id_index_map is not None
|
|
497
|
+
nhm_idx0 = [id_index_map[kk] for kk in global_ids]
|
|
498
|
+
|
|
499
|
+
if param.dimensions.ndim == 2:
|
|
500
|
+
return np.take(param.data_raw, nhm_idx0, axis=0) # axis: 0 rows, 1 columns
|
|
501
|
+
# return param.data_raw[tuple(nhm_idx0), :]
|
|
502
|
+
else:
|
|
503
|
+
if name in ['hru_deplcrv', 'snarea_curve']:
|
|
504
|
+
init_data = np.take(self.get('hru_deplcrv').data_raw, nhm_idx0, axis=0)
|
|
505
|
+
# init_data = self.get('hru_deplcrv').data_raw[tuple(nhm_idx0), ]
|
|
506
|
+
uniq_deplcrv = np.unique(init_data).tolist()
|
|
507
|
+
|
|
508
|
+
match name:
|
|
509
|
+
case 'hru_deplcrv':
|
|
510
|
+
# Renumber the hru_deplcrv indices for the subset
|
|
511
|
+
uniq_dict = {xx: ii+1 for ii, xx in enumerate(uniq_deplcrv)}
|
|
512
|
+
|
|
513
|
+
# Create new hru_deplcrv and renumber
|
|
514
|
+
res = np.array([uniq_dict[xx] for xx in init_data])
|
|
515
|
+
# return np.array([uniq_dict[xx] for xx in init_data])
|
|
516
|
+
case 'snarea_curve':
|
|
517
|
+
uniq_deplcrv0 = [xx - 1 for xx in uniq_deplcrv]
|
|
518
|
+
res = param.data_raw.reshape((-1, 11))[tuple(uniq_deplcrv0), :].reshape((-1))
|
|
519
|
+
return res
|
|
520
|
+
|
|
521
|
+
# if name == 'hru_deplcrv':
|
|
522
|
+
# # Renumber the hru_deplcrv indices for the subset
|
|
523
|
+
# uniq_dict = {xx: ii+1 for ii, xx in enumerate(uniq_deplcrv)}
|
|
524
|
+
#
|
|
525
|
+
# # Create new hru_deplcrv and renumber
|
|
526
|
+
# return np.array([uniq_dict[xx] for xx in init_data])
|
|
527
|
+
# elif name == 'snarea_curve':
|
|
528
|
+
# uniq_deplcrv0 = [xx - 1 for xx in uniq_deplcrv]
|
|
529
|
+
# return param.data_raw.reshape((-1, 11))[tuple(uniq_deplcrv0), :].reshape((-1))
|
|
530
|
+
else:
|
|
531
|
+
# All other 1D arrays
|
|
532
|
+
return np.take(param.data_raw, nhm_idx0, axis=0) # axis: 0 rows, 1 columns
|
|
533
|
+
# return param.data_raw[tuple(nhm_idx0), ]
|
|
534
|
+
|
|
535
|
+
def outlier_ids(self, name: str) -> List[int]:
|
|
536
|
+
"""Returns list of HRU or segment IDs of invalid parameter values
|
|
537
|
+
|
|
538
|
+
:param name: Name of the parameter
|
|
539
|
+
:returns: List of HRU or segment IDs
|
|
540
|
+
"""
|
|
541
|
+
|
|
542
|
+
cparam = self.get(name)
|
|
543
|
+
|
|
544
|
+
param_data = self.get_dataframe(name)
|
|
545
|
+
bad_value_ids = param_data[(param_data[name] < cparam.meta['minimum']) |
|
|
546
|
+
(param_data[name] > cparam.meta['maximum'])].index.tolist()
|
|
547
|
+
|
|
548
|
+
return bad_value_ids
|
|
549
|
+
|
|
550
|
+
def poi_upstream_hrus(self, poi: Union[str, List[str], KeysView]) -> Dict[str, List[int]]:
|
|
551
|
+
"""Returns a dictionary of POI to upstream global HRU IDs.
|
|
552
|
+
|
|
553
|
+
:param poi: POI ID or list of POI IDs
|
|
554
|
+
|
|
555
|
+
:returns: Dictionary of POI to upstream global HRU IDs
|
|
556
|
+
"""
|
|
557
|
+
|
|
558
|
+
if isinstance(poi, str):
|
|
559
|
+
poi = [poi]
|
|
560
|
+
elif isinstance(poi, KeysView):
|
|
561
|
+
poi = list(poi)
|
|
562
|
+
|
|
563
|
+
poi_hrus = {}
|
|
564
|
+
nhm_seg = self.get('nhm_seg').data_raw
|
|
565
|
+
assert type(nhm_seg) is np.ndarray and nhm_seg.dtype == np.int32
|
|
566
|
+
poi_seg0_dict = self.poi_to_seg0
|
|
567
|
+
|
|
568
|
+
# Generate stream network for the model
|
|
569
|
+
dag_streamnet = self.stream_network()
|
|
570
|
+
|
|
571
|
+
for cpoi in poi:
|
|
572
|
+
# Lookup global segment id for the current POI
|
|
573
|
+
cseg = poi_seg0_dict[cpoi]
|
|
574
|
+
dsmost_seg = [nhm_seg[cseg].item()]
|
|
575
|
+
|
|
576
|
+
poi_hrus[cpoi] = self._upstream_hrus(dag_streamnet, dsmost_seg)
|
|
577
|
+
|
|
578
|
+
return poi_hrus
|
|
579
|
+
|
|
580
|
+
def poi_upstream_segments(self, poi: Union[str, List[str], KeysView]) -> Dict[str, List[int]]:
|
|
581
|
+
"""Returns a dictionary of POI to upstream global segment IDs.
|
|
582
|
+
|
|
583
|
+
:param poi: POI ID or list of POI IDs
|
|
584
|
+
|
|
585
|
+
:returns: Dictionary of POI to upstream global segment IDs
|
|
586
|
+
"""
|
|
587
|
+
|
|
588
|
+
if isinstance(poi, str):
|
|
589
|
+
poi = [poi]
|
|
590
|
+
elif isinstance(poi, KeysView):
|
|
591
|
+
poi = list(poi)
|
|
592
|
+
|
|
593
|
+
poi_segs = {}
|
|
594
|
+
nhm_seg = self.get('nhm_seg').data_raw
|
|
595
|
+
assert type(nhm_seg) is np.ndarray
|
|
596
|
+
pois_dict = self.poi_to_seg0
|
|
597
|
+
|
|
598
|
+
# Generate stream network for the model
|
|
599
|
+
dag_streamnet = self.stream_network()
|
|
600
|
+
|
|
601
|
+
for cpoi in poi:
|
|
602
|
+
# Lookup global segment id for the current POI
|
|
603
|
+
dsmost_seg = [nhm_seg[pois_dict[cpoi]].item()]
|
|
604
|
+
|
|
605
|
+
poi_segs[cpoi] = self._upstream_segments(dag_streamnet, dsmost_seg)
|
|
606
|
+
|
|
607
|
+
return poi_segs
|
|
608
|
+
|
|
609
|
+
def plot(self, name: str,
|
|
610
|
+
output_dir: Optional[str] = None,
|
|
611
|
+
limits: Optional[Union[LimitOptions, List[float], Tuple[float, float]]] = 'absolute',
|
|
612
|
+
mask_defaults: Optional[str] = None,
|
|
613
|
+
**kwargs): # pragma: no cover
|
|
614
|
+
"""Plot a parameter.
|
|
615
|
+
|
|
616
|
+
Plots either to the screen or an output directory.
|
|
617
|
+
|
|
618
|
+
:param name: Name of parameter to plot
|
|
619
|
+
:param output_dir: Directory to write plot to (None for write to screen only)
|
|
620
|
+
:param limits: Limits to use for colorbar. One of 'valid', 'centered', 'absolute', or list of floats.
|
|
621
|
+
:param mask_defaults: Color for defaults values
|
|
622
|
+
"""
|
|
623
|
+
|
|
624
|
+
# is_monthly = False
|
|
625
|
+
time_index = None
|
|
626
|
+
|
|
627
|
+
if self.exists(name):
|
|
628
|
+
cparam = self.get(name)
|
|
629
|
+
|
|
630
|
+
if set(cparam.dimensions.keys()).intersection({'nmonths'}):
|
|
631
|
+
# Need 12 monthly plots of parameter
|
|
632
|
+
# is_monthly = True
|
|
633
|
+
time_index = 0 # starting time index
|
|
634
|
+
param_data = self.get_dataframe(name).iloc[:, time_index].to_frame(name=name)
|
|
635
|
+
else:
|
|
636
|
+
param_data = self.get_dataframe(name).iloc[:]
|
|
637
|
+
|
|
638
|
+
if mask_defaults is not None:
|
|
639
|
+
param_data = param_data.mask(param_data == cparam.meta['default'])
|
|
640
|
+
|
|
641
|
+
if isinstance(limits, str):
|
|
642
|
+
if limits == 'valid':
|
|
643
|
+
valid_min = cparam.meta['minimum']
|
|
644
|
+
valid_max = cparam.meta['maximum']
|
|
645
|
+
|
|
646
|
+
# Use the defined valid range of possible values
|
|
647
|
+
if valid_min == 'bounded':
|
|
648
|
+
# Parameters with bounded values need to always use the actual range of values
|
|
649
|
+
drange = [cparam.data_raw.min().min(), cparam.data_raw.max().max()]
|
|
650
|
+
elif name == 'jh_coef':
|
|
651
|
+
drange = [-0.05, 0.05]
|
|
652
|
+
else:
|
|
653
|
+
drange = [valid_min, valid_max]
|
|
654
|
+
elif limits == 'centered':
|
|
655
|
+
# Use the maximum range of the actual data values
|
|
656
|
+
lim = max(abs(cparam.data_raw.min().min()), abs(cparam.data_raw.max().max()))
|
|
657
|
+
drange = [-lim, lim]
|
|
658
|
+
elif limits == 'absolute':
|
|
659
|
+
# Use the min and max of the data values
|
|
660
|
+
drange = [cparam.data_raw.min().min(), cparam.data_raw.max().max()]
|
|
661
|
+
else:
|
|
662
|
+
raise ValueError('String argument for limits must be "valid", "centered", or "absolute"')
|
|
663
|
+
elif isinstance(limits, (list, tuple)):
|
|
664
|
+
if len(limits) != 2:
|
|
665
|
+
raise ValueError('When a list is used for plotting limits it should have 2 values (min, max)')
|
|
666
|
+
|
|
667
|
+
drange = [min(limits), max(limits)]
|
|
668
|
+
else:
|
|
669
|
+
raise TypeError('Argument, limits, must be string or a list[min,max]')
|
|
670
|
+
|
|
671
|
+
cmap, norm = set_colormap(name, param_data, min_val=drange[0],
|
|
672
|
+
max_val=drange[1], **kwargs)
|
|
673
|
+
kwargs.pop('cmap', None)
|
|
674
|
+
|
|
675
|
+
if mask_defaults is not None:
|
|
676
|
+
cmap.set_bad(mask_defaults, 0.7)
|
|
677
|
+
|
|
678
|
+
if set(cparam.dimensions.keys()).intersection({'nhru', 'ngw', 'nssr'}):
|
|
679
|
+
# Get extent information
|
|
680
|
+
if self.__hru_poly is not None:
|
|
681
|
+
minx, miny, maxx, maxy = self.__hru_poly.geometry.total_bounds
|
|
682
|
+
|
|
683
|
+
crs_proj = get_projection(self.__hru_poly)
|
|
684
|
+
|
|
685
|
+
# Takes care of multipolygons that are in the NHM geodatabase/shapefile
|
|
686
|
+
geoms_exploded = self.__hru_poly.explode(index_parts=True).reset_index(level=1, drop=True)
|
|
687
|
+
|
|
688
|
+
# print('Writing first plot')
|
|
689
|
+
df_mrg = geoms_exploded.merge(param_data, left_on=self.__hru_shape_key,
|
|
690
|
+
right_index=True, how='left')
|
|
691
|
+
|
|
692
|
+
fig_width, fig_height = get_figsize([minx, maxx, miny, maxy], **dict(kwargs))
|
|
693
|
+
kwargs.pop('init_size', None)
|
|
694
|
+
|
|
695
|
+
fig = plt.figure(figsize=(fig_width, fig_height))
|
|
696
|
+
|
|
697
|
+
ax = plt.axes(projection=crs_proj)
|
|
698
|
+
|
|
699
|
+
try:
|
|
700
|
+
ax.coastlines()
|
|
701
|
+
except AttributeError:
|
|
702
|
+
pass
|
|
703
|
+
|
|
704
|
+
gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True)
|
|
705
|
+
gl.top_labels = None
|
|
706
|
+
gl.right_labels = None
|
|
707
|
+
gl.xformatter = LONGITUDE_FORMATTER
|
|
708
|
+
gl.yformatter = LATITUDE_FORMATTER
|
|
709
|
+
|
|
710
|
+
ax.set_extent([minx, maxx, miny, maxy], crs=crs_proj)
|
|
711
|
+
|
|
712
|
+
if time_index is not None:
|
|
713
|
+
plt.title(f'Variable: {name}, Month: {time_index+1}')
|
|
714
|
+
else:
|
|
715
|
+
plt.title(f'Variable: {name}')
|
|
716
|
+
|
|
717
|
+
if mask_defaults is not None:
|
|
718
|
+
plt.annotate(f'NOTE: Values = {cparam.meta["default"]} are masked', xy=(0.5, 0.01),
|
|
719
|
+
xycoords='axes fraction', va='center', ha='center',
|
|
720
|
+
fontsize=10, fontweight='bold',
|
|
721
|
+
bbox=dict(boxstyle="round", facecolor=mask_defaults, alpha=1.0))
|
|
722
|
+
|
|
723
|
+
# Setup the color bar
|
|
724
|
+
mapper = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)
|
|
725
|
+
mapper.set_array(df_mrg[name])
|
|
726
|
+
cax = fig.add_axes((ax.get_position().x1 + 0.01,
|
|
727
|
+
ax.get_position().y0,
|
|
728
|
+
0.02,
|
|
729
|
+
ax.get_position().height))
|
|
730
|
+
|
|
731
|
+
# TODO: 2022-06-17 PAN - Categorical variables require entry in two places.
|
|
732
|
+
# The first place is here for labelling and the second place is in
|
|
733
|
+
# plot_helpers.py for adjusting the color boundaries. Need to figure out
|
|
734
|
+
# a more consistent/better way to do this.
|
|
735
|
+
if name == 'hru_deplcrv':
|
|
736
|
+
# tck_arr = np.arange(param_data.min().min(), param_data.max().max()+1)
|
|
737
|
+
tck_arr = np.arange(drange[0], drange[1]+1)
|
|
738
|
+
cb = plt.colorbar(mapper, cax=cax, ticks=tck_arr, label='Curve index')
|
|
739
|
+
cb.ax.tick_params(length=0)
|
|
740
|
+
elif name == 'soil_type':
|
|
741
|
+
tck_arr = np.arange(drange[0], drange[1]+1)
|
|
742
|
+
cb = plt.colorbar(mapper, cax=cax, ticks=tck_arr, label=name)
|
|
743
|
+
cb.ax.tick_params(length=0)
|
|
744
|
+
elif name == 'calibration_status':
|
|
745
|
+
tck_arr = np.arange(drange[0], drange[1]+1)
|
|
746
|
+
cb = plt.colorbar(mapper, cax=cax, ticks=tck_arr, label=name)
|
|
747
|
+
cb.ax.tick_params(length=0)
|
|
748
|
+
else:
|
|
749
|
+
plt.colorbar(mapper, cax=cax, label=cparam.meta['units'])
|
|
750
|
+
|
|
751
|
+
col = plot_polygon_collection(ax, df_mrg.geometry, values=df_mrg[name],
|
|
752
|
+
cmap=cmap, norm=norm, **dict(kwargs))
|
|
753
|
+
# col = plot_polygon_collection(ax, df_mrg.geometry, values=df_mrg[name],
|
|
754
|
+
# **dict(kwargs, cmap=cmap, norm=norm))
|
|
755
|
+
|
|
756
|
+
if output_dir is not None:
|
|
757
|
+
if time_index is not None:
|
|
758
|
+
# First month
|
|
759
|
+
plt.savefig(f'{output_dir}/{name}_{time_index+1:02}.png', dpi=150, bbox_inches='tight')
|
|
760
|
+
|
|
761
|
+
for tt in range(1, 12):
|
|
762
|
+
# Months 2 through 12
|
|
763
|
+
# print(f' Index: {tt}')
|
|
764
|
+
param_data = self.get_dataframe(name).iloc[:, tt].to_frame(name=name)
|
|
765
|
+
|
|
766
|
+
if mask_defaults is not None:
|
|
767
|
+
param_data = param_data.mask(param_data == cparam.meta['default'])
|
|
768
|
+
|
|
769
|
+
df_mrg = geoms_exploded.merge(param_data, left_on=self.__hru_shape_key,
|
|
770
|
+
right_index=True, how='left')
|
|
771
|
+
|
|
772
|
+
ax.set_title(f'Variable: {name}, Month: {tt+1}')
|
|
773
|
+
col.set_array(df_mrg[name])
|
|
774
|
+
plt.savefig(f'{output_dir}/{name}_{tt+1:02}.png', dpi=150, bbox_inches='tight')
|
|
775
|
+
else:
|
|
776
|
+
plt.savefig(f'{output_dir}/{name}.png', dpi=150, bbox_inches='tight')
|
|
777
|
+
|
|
778
|
+
# Close the figure so we don't chew up memory
|
|
779
|
+
fig.clf()
|
|
780
|
+
plt.close()
|
|
781
|
+
gc.collect()
|
|
782
|
+
elif set(cparam.dimensions.keys()).intersection({'nsegment'}):
|
|
783
|
+
# Plot segment-related parameters
|
|
784
|
+
if self.__seg_poly is not None:
|
|
785
|
+
if self.__hru_poly is not None:
|
|
786
|
+
minx, miny, maxx, maxy = self.__hru_poly.geometry.total_bounds
|
|
787
|
+
else:
|
|
788
|
+
minx, miny, maxx, maxy = self.__seg_poly.geometry.total_bounds
|
|
789
|
+
|
|
790
|
+
seg_geoms_exploded = self.__seg_poly.explode(index_parts=True).reset_index(level=1, drop=True)
|
|
791
|
+
|
|
792
|
+
crs_proj = get_projection(self.__seg_poly)
|
|
793
|
+
|
|
794
|
+
df_mrg = seg_geoms_exploded.merge(param_data, left_on=self.__seg_shape_key,
|
|
795
|
+
right_index=True, how='left')
|
|
796
|
+
|
|
797
|
+
fig_width, fig_height = get_figsize([minx, maxx, miny, maxy], **dict(kwargs))
|
|
798
|
+
kwargs.pop('init_size', None)
|
|
799
|
+
|
|
800
|
+
fig = plt.figure(figsize=(fig_width, fig_height))
|
|
801
|
+
|
|
802
|
+
ax = plt.axes(projection=crs_proj)
|
|
803
|
+
|
|
804
|
+
try:
|
|
805
|
+
ax.coastlines()
|
|
806
|
+
except AttributeError:
|
|
807
|
+
pass
|
|
808
|
+
|
|
809
|
+
gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True)
|
|
810
|
+
gl.top_labels = None
|
|
811
|
+
gl.right_labels = None
|
|
812
|
+
gl.xformatter = LONGITUDE_FORMATTER
|
|
813
|
+
gl.yformatter = LATITUDE_FORMATTER
|
|
814
|
+
|
|
815
|
+
ax.set_extent([minx, maxx, miny, maxy], crs=crs_proj)
|
|
816
|
+
|
|
817
|
+
plt.title('Variable: {}'.format(name))
|
|
818
|
+
|
|
819
|
+
if kwargs.get('vary_color', True):
|
|
820
|
+
mapper = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)
|
|
821
|
+
mapper.set_array(df_mrg[name])
|
|
822
|
+
cax = fig.add_axes((ax.get_position().x1 + 0.01,
|
|
823
|
+
ax.get_position().y0, 0.02,
|
|
824
|
+
ax.get_position().height))
|
|
825
|
+
plt.colorbar(mapper, cax=cax, label=cparam.meta['units']) # , shrink=0.6
|
|
826
|
+
|
|
827
|
+
if self.__hru_poly is not None:
|
|
828
|
+
hru_geoms_exploded = self.__hru_poly.explode(index_parts=True).reset_index(level=1, drop=True)
|
|
829
|
+
hru_poly = plot_polygon_collection(ax, hru_geoms_exploded.geometry,
|
|
830
|
+
cmap=cmap, norm=norm,
|
|
831
|
+
**dict(kwargs, linewidth=0.5, alpha=0.7))
|
|
832
|
+
|
|
833
|
+
col = plot_line_collection(ax, df_mrg.geometry, values=df_mrg[name], cmap=cmap,
|
|
834
|
+
norm=norm, **dict(kwargs))
|
|
835
|
+
|
|
836
|
+
if mask_defaults is not None:
|
|
837
|
+
plt.annotate(f'NOTE: Values = {cparam.meta["default"]} are masked', xy=(0.5, 0.01),
|
|
838
|
+
xycoords='axes fraction', fontsize=12, fontweight='bold',
|
|
839
|
+
bbox=dict(facecolor=mask_defaults, alpha=1.0))
|
|
840
|
+
|
|
841
|
+
if output_dir is not None:
|
|
842
|
+
plt.savefig(f'{output_dir}/{name}.png', dpi=150, bbox_inches='tight')
|
|
843
|
+
|
|
844
|
+
# Close the figure so we don't chew up memory
|
|
845
|
+
fig.clf()
|
|
846
|
+
plt.close()
|
|
847
|
+
gc.collect()
|
|
848
|
+
else:
|
|
849
|
+
print('No segment shapefile is loaded; skipping')
|
|
850
|
+
else:
|
|
851
|
+
print('Non-plottable parameter')
|
|
852
|
+
|
|
853
|
+
def remove(self, name: Union[str, Sequence[str], Set[str]]):
|
|
854
|
+
"""Delete one or more parameters if they exist.
|
|
855
|
+
|
|
856
|
+
:param name: parameter or list of parameters to remove
|
|
857
|
+
"""
|
|
858
|
+
|
|
859
|
+
if isinstance(name, str):
|
|
860
|
+
name = [name]
|
|
861
|
+
|
|
862
|
+
for cparam in name:
|
|
863
|
+
if self.exists(cparam):
|
|
864
|
+
del self.__parameters[cparam]
|
|
865
|
+
|
|
866
|
+
if self.verbose: # pragma: no cover
|
|
867
|
+
con.print(f'[bold]{cparam}[/] [gold3]parameter removed[/]')
|
|
868
|
+
|
|
869
|
+
def remove_poi(self, poi: Union[str, List[str]]):
|
|
870
|
+
"""Remove POIs by gage_id.
|
|
871
|
+
|
|
872
|
+
:param poi: POI id to remove
|
|
873
|
+
"""
|
|
874
|
+
|
|
875
|
+
if isinstance(poi, str):
|
|
876
|
+
poi = [poi]
|
|
877
|
+
|
|
878
|
+
# First get array of poi_gage_id indices matching the specified POI IDs
|
|
879
|
+
poi_ids = self.get('poi_gage_id').data_raw.tolist()
|
|
880
|
+
assert type(poi_ids) is list
|
|
881
|
+
poi_del_indices = []
|
|
882
|
+
for xx in poi:
|
|
883
|
+
# We silently ignore missing POIs
|
|
884
|
+
if xx in poi_ids:
|
|
885
|
+
poi_del_indices.append(poi_ids.index(xx)) # type: ignore
|
|
886
|
+
|
|
887
|
+
# poi_ids = self.get('poi_gage_id').data
|
|
888
|
+
# sorter = np.argsort(poi_ids)
|
|
889
|
+
# poi_del_indices = sorter[np.searchsorted(poi_ids, poi, sorter=sorter)]
|
|
890
|
+
|
|
891
|
+
poi_parameters = ['poi_gage_id', 'poi_gage_segment', 'poi_type']
|
|
892
|
+
|
|
893
|
+
# print(f'POIs to delete: {poi}')
|
|
894
|
+
# print(f'Current POIs: {poi_ids}')
|
|
895
|
+
# print(f'Size of poi_del_indices: {poi_del_indices.size}')
|
|
896
|
+
if len(poi_del_indices) > 0:
|
|
897
|
+
if self.get('poi_gage_id').dimensions.get('npoigages').size == len(poi_del_indices):
|
|
898
|
+
# We're trying to remove all the POIs
|
|
899
|
+
for pp in poi_parameters:
|
|
900
|
+
self.remove(pp)
|
|
901
|
+
|
|
902
|
+
self.dimensions.get('npoigages').size = 0
|
|
903
|
+
else:
|
|
904
|
+
# Remove the matching poi gage entries from each of the poi-related parameters
|
|
905
|
+
for pp in poi_parameters:
|
|
906
|
+
self.get(pp).remove_by_index('npoigages', poi_del_indices)
|
|
907
|
+
|
|
908
|
+
# Update the global npoigages dimension
|
|
909
|
+
self.dimensions.get('npoigages').size -= len(poi_del_indices)
|
|
910
|
+
self.dimensions.get('nobs').size -= len(poi_del_indices)
|
|
911
|
+
|
|
912
|
+
def segment_upstream_hrus(self, segs: Union[int, List[int], KeysView, npt.NDArray]) -> Dict[int, List[int]]:
|
|
913
|
+
"""Returns a dictionary of segment to upstream global HRU IDs.
|
|
914
|
+
|
|
915
|
+
:param segs: Global segment ID or list of global segment IDs
|
|
916
|
+
|
|
917
|
+
:returns: Dictionary of global segment ID to upstream global HRU IDs
|
|
918
|
+
"""
|
|
919
|
+
|
|
920
|
+
if isinstance(segs, int):
|
|
921
|
+
segs = [segs]
|
|
922
|
+
elif isinstance(segs, KeysView):
|
|
923
|
+
segs = list(segs)
|
|
924
|
+
elif isinstance(segs, np.ndarray):
|
|
925
|
+
segs = segs.tolist() # type: ignore
|
|
926
|
+
|
|
927
|
+
assert type(segs) is list
|
|
928
|
+
|
|
929
|
+
seg_hrus = {}
|
|
930
|
+
|
|
931
|
+
# Generate stream network for the model
|
|
932
|
+
dag_streamnet = self.stream_network()
|
|
933
|
+
|
|
934
|
+
for cseg in segs:
|
|
935
|
+
# Lookup segment for the current POI
|
|
936
|
+
dsmost_seg = [cseg]
|
|
937
|
+
|
|
938
|
+
seg_hrus[cseg] = self._upstream_hrus(dag_streamnet, dsmost_seg)
|
|
939
|
+
|
|
940
|
+
return seg_hrus
|
|
941
|
+
|
|
942
|
+
def segment_upstream_segments(self, segs: Union[int, List[int], KeysView, npt.NDArray]) -> Dict[int, List[int]]:
|
|
943
|
+
"""Returns a dictionary of global segment IDs to upstream global segment IDs.
|
|
944
|
+
|
|
945
|
+
:param segs: global segment IDs or list of global segment IDs
|
|
946
|
+
|
|
947
|
+
:returns: Dictionary of global segment ID to upstream global segment IDs
|
|
948
|
+
"""
|
|
949
|
+
|
|
950
|
+
if isinstance(segs, int):
|
|
951
|
+
segs = [segs]
|
|
952
|
+
elif isinstance(segs, KeysView):
|
|
953
|
+
segs = list(segs)
|
|
954
|
+
elif isinstance(segs, np.ndarray):
|
|
955
|
+
segs = segs.tolist()
|
|
956
|
+
|
|
957
|
+
assert type(segs) is list
|
|
958
|
+
|
|
959
|
+
us_segs = {}
|
|
960
|
+
|
|
961
|
+
# Generate stream network for the model
|
|
962
|
+
dag_streamnet = self.stream_network()
|
|
963
|
+
|
|
964
|
+
for cseg in segs:
|
|
965
|
+
# Lookup segment for the current segment
|
|
966
|
+
dsmost_seg = [cseg]
|
|
967
|
+
us_segs[cseg] = self._upstream_segments(dag_streamnet, dsmost_seg)
|
|
968
|
+
|
|
969
|
+
return us_segs
|
|
970
|
+
|
|
971
|
+
def shapefile_hrus(self, filename: str,
|
|
972
|
+
layer_name: Optional[str] = None,
|
|
973
|
+
shape_key: Optional[str] = None): # pragma: no cover
|
|
974
|
+
"""Read a shapefile or geodatabase that corresponds to HRUs.
|
|
975
|
+
|
|
976
|
+
:param filename: name of shapefile or geodatabase
|
|
977
|
+
:param layer_name: name of layer in geodatabase
|
|
978
|
+
:param shape_key: name of attribute for key
|
|
979
|
+
"""
|
|
980
|
+
|
|
981
|
+
self.__hru_poly = geopandas.read_file(filename, layer=layer_name)
|
|
982
|
+
|
|
983
|
+
if self.__hru_poly.crs.name == 'USA_Contiguous_Albers_Equal_Area_Conic_USGS_version': # type: ignore
|
|
984
|
+
print('Overriding USGS aea crs with EPSG:5070')
|
|
985
|
+
self.__hru_poly.crs = 'EPSG:5070' # type: ignore
|
|
986
|
+
elif self.__hru_poly.crs.name[0:5] == 'NAD83': # type: ignore
|
|
987
|
+
self.__hru_poly.to_crs('epsg:4326', inplace=True)
|
|
988
|
+
|
|
989
|
+
self.__hru_shape_key = shape_key
|
|
990
|
+
|
|
991
|
+
def shapefile_segments(self, filename: str,
|
|
992
|
+
layer_name: Optional[str] = None,
|
|
993
|
+
shape_key: Optional[str] = None): # pragma: no cover
|
|
994
|
+
"""Read a shapefile or geodatabase that corresponds to stream segments.
|
|
995
|
+
|
|
996
|
+
:param filename: name of shapefile or geodatabase
|
|
997
|
+
:param layer_name: name of layer in geodatabase
|
|
998
|
+
:param shape_key: name of attribute for key
|
|
999
|
+
"""
|
|
1000
|
+
|
|
1001
|
+
self.__seg_poly = geopandas.read_file(filename, layer=layer_name)
|
|
1002
|
+
|
|
1003
|
+
if self.__seg_poly.crs.name == 'USA_Contiguous_Albers_Equal_Area_Conic_USGS_version': # type: ignore
|
|
1004
|
+
print('Overriding USGS aea crs with EPSG:5070')
|
|
1005
|
+
self.__seg_poly.crs = 'EPSG:5070' # type: ignore
|
|
1006
|
+
elif self.__seg_poly.crs.name[0:5] == 'NAD83': # type: ignore
|
|
1007
|
+
self.__seg_poly.to_crs('epsg:4326', inplace=True)
|
|
1008
|
+
|
|
1009
|
+
self.__seg_shape_key = shape_key
|
|
1010
|
+
|
|
1011
|
+
def stream_network(self, tosegment: str = 'tosegment_nhm',
|
|
1012
|
+
seg_id: str = 'nhm_seg') -> Union[nx.DiGraph, None]:
|
|
1013
|
+
"""Create Directed, Acyclic Graph (DAG) of stream network.
|
|
1014
|
+
|
|
1015
|
+
:param tosegment: name of parameter to use for HRU tosegment
|
|
1016
|
+
:param seg_id: name of parameter to use for the segment IDs
|
|
1017
|
+
:returns: directed-acyclic-graph (DAG)
|
|
1018
|
+
"""
|
|
1019
|
+
|
|
1020
|
+
seg = self.get(seg_id).tolist()
|
|
1021
|
+
toseg = self.get(tosegment).tolist()
|
|
1022
|
+
assert type(seg) is list and type(toseg) is list
|
|
1023
|
+
|
|
1024
|
+
dag_ds = nx.DiGraph()
|
|
1025
|
+
for ii, vv in enumerate(toseg):
|
|
1026
|
+
# dag_ds.add_edge(ii+1, vv)
|
|
1027
|
+
if vv == 0:
|
|
1028
|
+
dag_ds.add_edge(seg[ii], 'Out_{}'.format(seg[ii]))
|
|
1029
|
+
else:
|
|
1030
|
+
dag_ds.add_edge(seg[ii], vv)
|
|
1031
|
+
|
|
1032
|
+
return dag_ds
|
|
1033
|
+
|
|
1034
|
+
def update_element(self, name: str,
|
|
1035
|
+
id1: int,
|
|
1036
|
+
value: Union[int, float, List[int], List[float]]):
|
|
1037
|
+
"""Update single value or row of values (e.g. nhru by nmonths) for a
|
|
1038
|
+
given nhm_id, nhm_seg, or 0 (for scalars).
|
|
1039
|
+
|
|
1040
|
+
:param name: name of parameter to update
|
|
1041
|
+
:param id1: scalar nhm_id or nhm_seg
|
|
1042
|
+
:param value: new value(s)
|
|
1043
|
+
"""
|
|
1044
|
+
|
|
1045
|
+
# NOTE: id1 is either an nhm_id or nhm_seg (both are 1-based)
|
|
1046
|
+
cparam = self.get(name)
|
|
1047
|
+
idx0 = 0
|
|
1048
|
+
|
|
1049
|
+
try:
|
|
1050
|
+
if cparam.is_hru_param():
|
|
1051
|
+
# Lookup index for nhm_id
|
|
1052
|
+
idx0 = self.get('nhm_id')._value_index_1d(id1).item()
|
|
1053
|
+
elif cparam.is_seg_param():
|
|
1054
|
+
# Lookup index for nhm_seg
|
|
1055
|
+
idx0 = self.get('nhm_seg')._value_index_1d(id1).item()
|
|
1056
|
+
elif cparam.is_scalar:
|
|
1057
|
+
idx0 = 0
|
|
1058
|
+
except ValueError:
|
|
1059
|
+
raise ValueError(f'Exactly one index should be found for {id1}')
|
|
1060
|
+
|
|
1061
|
+
# This will raise a ValueError if more than one index is found
|
|
1062
|
+
cparam.update_element(idx0, value)
|
|
1063
|
+
|
|
1064
|
+
def write_dimensions_xml(self, output_dir: str):
|
|
1065
|
+
"""Write global dimensions.xml file.
|
|
1066
|
+
|
|
1067
|
+
:param output_dir: output path for dimensions.xml file
|
|
1068
|
+
"""
|
|
1069
|
+
|
|
1070
|
+
# Write the global dimensions xml file
|
|
1071
|
+
xmlstr = minidom.parseString(xmlET.tostring(self.xml_global_dimensions)).toprettyxml(indent=' ')
|
|
1072
|
+
with open(f'{output_dir}/{DIMENSIONS_XML}', 'w') as ff:
|
|
1073
|
+
ff.write(xmlstr)
|
|
1074
|
+
|
|
1075
|
+
def write_paramdb(self, output_dir: str):
|
|
1076
|
+
"""Write all parameters using the paramDb output format.
|
|
1077
|
+
|
|
1078
|
+
:param output_dir: output path for paramDb files
|
|
1079
|
+
"""
|
|
1080
|
+
|
|
1081
|
+
# Check for / create output directory
|
|
1082
|
+
try:
|
|
1083
|
+
if self.verbose: # pragma: no cover
|
|
1084
|
+
print(f'Creating output directory: {output_dir}')
|
|
1085
|
+
os.makedirs(output_dir)
|
|
1086
|
+
except OSError:
|
|
1087
|
+
if self.verbose: # pragma: no cover
|
|
1088
|
+
print("\tUsing existing directory")
|
|
1089
|
+
|
|
1090
|
+
# Write the global dimensions xml file
|
|
1091
|
+
self.write_dimensions_xml(output_dir)
|
|
1092
|
+
|
|
1093
|
+
# Write the global parameters xml file
|
|
1094
|
+
self.write_parameters_xml(output_dir)
|
|
1095
|
+
|
|
1096
|
+
for xx in self.parameters.values():
|
|
1097
|
+
# Write out each parameter in the paramDb csv format
|
|
1098
|
+
if self.verbose: # pragma: no cover
|
|
1099
|
+
print(xx.name)
|
|
1100
|
+
|
|
1101
|
+
with open(f'{output_dir}/{xx.name}.csv', 'w') as ff:
|
|
1102
|
+
ff.write(xx.toparamdb())
|
|
1103
|
+
|
|
1104
|
+
def write_parameter_file(self, filename: str,
|
|
1105
|
+
header: Optional[List[str]] = None):
|
|
1106
|
+
"""Write a PRMS parameter file.
|
|
1107
|
+
|
|
1108
|
+
:param filename: name of parameter file
|
|
1109
|
+
:param header: list of header lines
|
|
1110
|
+
"""
|
|
1111
|
+
|
|
1112
|
+
# Write the parameters out to a file
|
|
1113
|
+
outfile = open(filename, 'w')
|
|
1114
|
+
|
|
1115
|
+
if header is not None:
|
|
1116
|
+
if len(header) > 2:
|
|
1117
|
+
# TODO: 2023-04-19 - this check should happen before
|
|
1118
|
+
# opening the output file.
|
|
1119
|
+
raise ValueError('Header should be a list of two items')
|
|
1120
|
+
if len(header) == 1:
|
|
1121
|
+
# Must have two header lines
|
|
1122
|
+
outfile.write('Written by pyPRMS\n')
|
|
1123
|
+
|
|
1124
|
+
for hh in header:
|
|
1125
|
+
# Write out any header stuff
|
|
1126
|
+
outfile.write(f'{hh}\n')
|
|
1127
|
+
else:
|
|
1128
|
+
# Write a default header
|
|
1129
|
+
outfile.write('Written by pyPRMS\n')
|
|
1130
|
+
outfile.write('Comment: It is all downhill from here\n')
|
|
1131
|
+
|
|
1132
|
+
# Dimension section must be written first
|
|
1133
|
+
outfile.write(f'{CATEGORY_DELIM} Dimensions {CATEGORY_DELIM}\n')
|
|
1134
|
+
|
|
1135
|
+
for (kk, vv) in self.dimensions.items():
|
|
1136
|
+
# Write each dimension name and size separated by VAR_DELIM
|
|
1137
|
+
outfile.write(f'{VAR_DELIM}\n')
|
|
1138
|
+
outfile.write(f'{kk}\n')
|
|
1139
|
+
outfile.write(f'{vv.size:d}\n')
|
|
1140
|
+
|
|
1141
|
+
if self.prms_version < PRMS6_DEV_VERSION and {'ngw', 'nssr'}.isdisjoint(set(self.dimensions.keys())):
|
|
1142
|
+
# Add the ngw and nssr dimensions. These are always equal to nhru.
|
|
1143
|
+
for kk in ['ngw', 'nssr']:
|
|
1144
|
+
outfile.write(f'{VAR_DELIM}\n')
|
|
1145
|
+
outfile.write(f'{kk}\n')
|
|
1146
|
+
outfile.write(f'{self.dimensions["nhru"].size:d}\n')
|
|
1147
|
+
|
|
1148
|
+
# Now write out the Parameter category
|
|
1149
|
+
order = ['name', 'dimensions', 'datatype', 'data']
|
|
1150
|
+
|
|
1151
|
+
outfile.write(f'{CATEGORY_DELIM} Parameters {CATEGORY_DELIM}\n')
|
|
1152
|
+
|
|
1153
|
+
for vv in self.parameters.values():
|
|
1154
|
+
datatype = PTYPE_TO_PRMS_TYPE[vv.meta.get('datatype')]
|
|
1155
|
+
|
|
1156
|
+
for item in order:
|
|
1157
|
+
# Write each variable out separated by self.__rowdelim
|
|
1158
|
+
if item == 'dimensions':
|
|
1159
|
+
# Write number of dimensions first
|
|
1160
|
+
outfile.write(f'{vv.dimensions.ndim}\n')
|
|
1161
|
+
|
|
1162
|
+
for dd in vv.dimensions.values():
|
|
1163
|
+
# Write dimension names
|
|
1164
|
+
if self.prms_version < PRMS6_DEV_VERSION:
|
|
1165
|
+
# On-the-fly change of dimension names for certain parameters
|
|
1166
|
+
# when the prms version is not an experimental development version.
|
|
1167
|
+
if dd.name == 'nhru':
|
|
1168
|
+
if vv.name in ['gwflow_coef', 'gwsink_coef', 'gwstor_init',
|
|
1169
|
+
'gwstor_min', 'gw_seep_coef']:
|
|
1170
|
+
outfile.write('ngw\n')
|
|
1171
|
+
elif vv.name in ['ssr2gw_exp', 'ssr2gw_rate', 'ssstor_init',
|
|
1172
|
+
'ssstor_init_frac']:
|
|
1173
|
+
outfile.write('nssr\n')
|
|
1174
|
+
else:
|
|
1175
|
+
outfile.write(f'{dd.name}\n')
|
|
1176
|
+
else:
|
|
1177
|
+
outfile.write(f'{dd.name}\n')
|
|
1178
|
+
else:
|
|
1179
|
+
outfile.write(f'{dd.name}\n')
|
|
1180
|
+
elif item == 'datatype':
|
|
1181
|
+
# dimsize (which is computed) must be written before datatype
|
|
1182
|
+
outfile.write(f'{vv.data_raw.size}\n')
|
|
1183
|
+
outfile.write(f'{datatype}\n')
|
|
1184
|
+
elif item == 'data':
|
|
1185
|
+
# Write one value per line
|
|
1186
|
+
# WARNING: 2019-10-10: had to change next line from order='A' to order='F'
|
|
1187
|
+
# because flatten with 'A' was only honoring the Fortran memory layout
|
|
1188
|
+
# if the array was contiguous which isn't always the
|
|
1189
|
+
# case if the arrays have been altered in size.
|
|
1190
|
+
for xx in vv.data_raw.ravel(order='F'):
|
|
1191
|
+
if vv.meta.get('datatype', 'null') in ['float32', 'float64']:
|
|
1192
|
+
# Float and double types have to be formatted specially so
|
|
1193
|
+
# they aren't written in exponential notation or with
|
|
1194
|
+
# extraneous zeroes
|
|
1195
|
+
tmp = f'{xx:<20.7f}'.rstrip('0 ')
|
|
1196
|
+
if tmp[-1] == '.':
|
|
1197
|
+
tmp += '0'
|
|
1198
|
+
|
|
1199
|
+
outfile.write(f'{tmp}\n')
|
|
1200
|
+
else:
|
|
1201
|
+
outfile.write(f'{xx}\n')
|
|
1202
|
+
elif item == 'name':
|
|
1203
|
+
# Write the self.__rowdelim before the variable name
|
|
1204
|
+
outfile.write(f'{VAR_DELIM}\n')
|
|
1205
|
+
outfile.write(f'{vv.name}\n')
|
|
1206
|
+
|
|
1207
|
+
outfile.close()
|
|
1208
|
+
|
|
1209
|
+
def write_parameter_netcdf(self, filename: str):
|
|
1210
|
+
"""Write parameters to a netcdf format file.
|
|
1211
|
+
|
|
1212
|
+
:param filename: full path for output file
|
|
1213
|
+
"""
|
|
1214
|
+
|
|
1215
|
+
# Create the netcdf file
|
|
1216
|
+
nc_hdl = nc.Dataset(filename, 'w', clobber=True)
|
|
1217
|
+
|
|
1218
|
+
# Create dimensions
|
|
1219
|
+
for (kk, vv) in self.dimensions.items():
|
|
1220
|
+
if kk != 'one':
|
|
1221
|
+
# Dimension 'one' is only used for scalars in PRMS
|
|
1222
|
+
nc_hdl.createDimension(kk, vv.size)
|
|
1223
|
+
|
|
1224
|
+
# Create the variables
|
|
1225
|
+
# hruo = nco.createVariable('hru', 'i4', ('hru'))
|
|
1226
|
+
for vv in self.parameters.values():
|
|
1227
|
+
curr_datatype = NETCDF_DATATYPES[PTYPE_TO_PRMS_TYPE[vv.meta.get('datatype')]]
|
|
1228
|
+
|
|
1229
|
+
if curr_datatype != 'S1':
|
|
1230
|
+
if list(vv.dimensions.keys())[0] == 'one':
|
|
1231
|
+
# Scalar values
|
|
1232
|
+
curr_param = nc_hdl.createVariable(vv.name, curr_datatype,
|
|
1233
|
+
fill_value=nc.default_fillvals[curr_datatype], zlib=True)
|
|
1234
|
+
else:
|
|
1235
|
+
curr_param = nc_hdl.createVariable(vv.name, curr_datatype,
|
|
1236
|
+
tuple(list(vv.dimensions.keys())[::-1]),
|
|
1237
|
+
fill_value=nc.default_fillvals[curr_datatype], zlib=True)
|
|
1238
|
+
|
|
1239
|
+
# Add the attributes
|
|
1240
|
+
for ck, cmeta in vv.meta.items():
|
|
1241
|
+
if ck in ['minimum', 'maximum']:
|
|
1242
|
+
curr_param.setncattr(f'valid_{ck}', cmeta)
|
|
1243
|
+
elif ck in ['help', 'description', 'units']:
|
|
1244
|
+
curr_param.setncattr(ck, cmeta)
|
|
1245
|
+
|
|
1246
|
+
# Write the data
|
|
1247
|
+
if len(vv.dimensions.keys()) == 1:
|
|
1248
|
+
curr_param[:] = vv.data
|
|
1249
|
+
elif len(vv.dimensions.keys()) == 2:
|
|
1250
|
+
curr_param[:, :] = vv.data.transpose()
|
|
1251
|
+
else:
|
|
1252
|
+
# String parameter
|
|
1253
|
+
# Get the maximum string length in the array of data
|
|
1254
|
+
# print('String parameter: {}'.format(vv.name))
|
|
1255
|
+
str_size = len(max(vv.data, key=len))
|
|
1256
|
+
# print('size: {}'.format(str_size))
|
|
1257
|
+
|
|
1258
|
+
# Create a dimension for the string length
|
|
1259
|
+
nc_hdl.createDimension(vv.name + '_nchars', str_size)
|
|
1260
|
+
|
|
1261
|
+
# Temporary to add extra dimension for number of characters
|
|
1262
|
+
tmp_dims = list(vv.dimensions.keys())
|
|
1263
|
+
tmp_dims.extend([vv.name + '_nchars'])
|
|
1264
|
+
curr_param = nc_hdl.createVariable(vv.name, curr_datatype, tuple(tmp_dims),
|
|
1265
|
+
fill_value=nc.default_fillvals[curr_datatype], zlib=True)
|
|
1266
|
+
|
|
1267
|
+
# Add the attributes
|
|
1268
|
+
for ck, cmeta in vv.meta.items():
|
|
1269
|
+
if ck in ['help', 'description', 'units']:
|
|
1270
|
+
curr_param.setncattr(ck, cmeta)
|
|
1271
|
+
|
|
1272
|
+
# Write the data
|
|
1273
|
+
if len(tmp_dims) == 1:
|
|
1274
|
+
curr_param[:] = nc.stringtochar(vv.data)
|
|
1275
|
+
elif len(tmp_dims) == 2:
|
|
1276
|
+
# curr_param._Encoding = 'ascii'
|
|
1277
|
+
|
|
1278
|
+
# The stringtochar() routine won't handle the unicode numpy
|
|
1279
|
+
# datatype properly so we force it to dtype='S'
|
|
1280
|
+
curr_param[:, :] = nc.stringtochar(vv.data.astype('S'))
|
|
1281
|
+
sys.stdout.flush()
|
|
1282
|
+
# Close the netcdf file
|
|
1283
|
+
nc_hdl.close()
|
|
1284
|
+
|
|
1285
|
+
def write_parameters_metadata_csv(self, filename: str,
|
|
1286
|
+
sep: str = '\t'):
|
|
1287
|
+
"""Writes the parameter metadata to a CSV file.
|
|
1288
|
+
|
|
1289
|
+
:param filename: output filename
|
|
1290
|
+
:param sep: separator character
|
|
1291
|
+
"""
|
|
1292
|
+
|
|
1293
|
+
out_list = []
|
|
1294
|
+
|
|
1295
|
+
if self.__control is not None:
|
|
1296
|
+
modules_used = set(self.__control.modules.values()).union(set(self.__control.additional_modules))
|
|
1297
|
+
|
|
1298
|
+
for pk in sorted(list(self.parameters.keys())):
|
|
1299
|
+
pp = self.get(pk)
|
|
1300
|
+
md = pp.meta
|
|
1301
|
+
|
|
1302
|
+
param_modules = md.get('modules')
|
|
1303
|
+
assert type(param_modules) is list
|
|
1304
|
+
|
|
1305
|
+
if self.__control is not None:
|
|
1306
|
+
module_list = list(modules_used.intersection(set(param_modules)))
|
|
1307
|
+
|
|
1308
|
+
if len(module_list) == 0:
|
|
1309
|
+
# We have a parameter that is not needed by the declared modules
|
|
1310
|
+
if self.verbose: # pragma: no cover
|
|
1311
|
+
print(f'{pp.name} not used with selected modules')
|
|
1312
|
+
print(f' {md.get("modules")}')
|
|
1313
|
+
continue
|
|
1314
|
+
else:
|
|
1315
|
+
module_list = param_modules
|
|
1316
|
+
|
|
1317
|
+
for idx, mm in enumerate(module_list):
|
|
1318
|
+
if mm in external_module_map:
|
|
1319
|
+
module_list[idx] = external_module_map[mm]
|
|
1320
|
+
|
|
1321
|
+
modules = ', '.join(module_list)
|
|
1322
|
+
|
|
1323
|
+
dim_list = list(pp.dimensions.keys())
|
|
1324
|
+
|
|
1325
|
+
if pk in ['gwflow_coef', 'gwsink_coef', 'gwstor_init', 'gwstor_min', 'gw_seep_coef']:
|
|
1326
|
+
dim_list[0] = 'ngw'
|
|
1327
|
+
elif pk in ['ssr2gw_exp', 'ssr2gw_rate', 'ssstor_init', 'ssstor_init_frac']:
|
|
1328
|
+
dim_list[0] = 'nssr'
|
|
1329
|
+
|
|
1330
|
+
dims = ', '.join(dim_list)
|
|
1331
|
+
|
|
1332
|
+
try:
|
|
1333
|
+
act_min = pp.data_raw.min()
|
|
1334
|
+
act_max = pp.data_raw.max()
|
|
1335
|
+
except np.core._exceptions.UFuncTypeError: # type: ignore
|
|
1336
|
+
act_min = ''
|
|
1337
|
+
act_max = ''
|
|
1338
|
+
|
|
1339
|
+
out_list.append([pp.name, md.get('datatype', ''),
|
|
1340
|
+
md.get('units', ''),
|
|
1341
|
+
md.get('description', ''),
|
|
1342
|
+
md.get('minimum'),
|
|
1343
|
+
md.get('maximum'),
|
|
1344
|
+
act_min,
|
|
1345
|
+
act_max,
|
|
1346
|
+
md.get('default'),
|
|
1347
|
+
dims,
|
|
1348
|
+
modules])
|
|
1349
|
+
|
|
1350
|
+
col_names = ['parameter_name', 'datatype', 'units', 'description',
|
|
1351
|
+
'valid_minimum', 'valid_maximum', 'actual_minimum',
|
|
1352
|
+
'actual_maximum', 'default', 'dimension', 'modules']
|
|
1353
|
+
|
|
1354
|
+
df = pd.DataFrame.from_records(out_list, columns=col_names)
|
|
1355
|
+
if sep == ',':
|
|
1356
|
+
df.to_csv(filename, sep=sep, quotechar='"', index=False)
|
|
1357
|
+
else:
|
|
1358
|
+
df.to_csv(filename, sep=sep, index=False)
|
|
1359
|
+
|
|
1360
|
+
def write_parameters_xml(self, output_dir: str):
|
|
1361
|
+
"""Write global parameters.xml file.
|
|
1362
|
+
|
|
1363
|
+
:param output_dir: output path for parameters.xml file
|
|
1364
|
+
"""
|
|
1365
|
+
|
|
1366
|
+
# Write the global parameters xml file
|
|
1367
|
+
xmlstr = minidom.parseString(xmlET.tostring(self.xml_global_parameters)).toprettyxml(indent=' ')
|
|
1368
|
+
with open(f'{output_dir}/{PARAMETERS_XML}', 'w') as ff:
|
|
1369
|
+
ff.write(xmlstr)
|
|
1370
|
+
|
|
1371
|
+
def _condition_check_ctl(self, cstr: str) -> bool:
|
|
1372
|
+
"""Takes a string of the form '<control_var> <op> <value>' and checks
|
|
1373
|
+
if the condition is True.
|
|
1374
|
+
|
|
1375
|
+
:param cstr: string of the form '<control_var> <op> <value>'
|
|
1376
|
+
:returns: True if the condition is met, False otherwise
|
|
1377
|
+
"""
|
|
1378
|
+
|
|
1379
|
+
assert self.control is not None
|
|
1380
|
+
var, op, value = cstr.split(' ')
|
|
1381
|
+
|
|
1382
|
+
cdtype = NEW_PTYPE_TO_DTYPE[self.control.get(var).meta['datatype']]
|
|
1383
|
+
ctl_val = self.control.get(var).values
|
|
1384
|
+
assert ctl_val is not None
|
|
1385
|
+
value = cdtype(value)
|
|
1386
|
+
|
|
1387
|
+
return cond_check[op](ctl_val, value)
|
|
1388
|
+
|
|
1389
|
+
def _condition_check_dim(self, cstr: str) -> bool:
|
|
1390
|
+
"""Takes a string of the form '<dimension> <op> <value>' and checks
|
|
1391
|
+
if the condition is True.
|
|
1392
|
+
|
|
1393
|
+
:param cstr: string of the form '<dimension> <op> <value>'
|
|
1394
|
+
:returns: True if the condition is met, False otherwise
|
|
1395
|
+
"""
|
|
1396
|
+
|
|
1397
|
+
var, op, value = cstr.split(' ')
|
|
1398
|
+
value = int(value) # type: ignore
|
|
1399
|
+
|
|
1400
|
+
if self.dimensions.exists(var):
|
|
1401
|
+
return cond_check[op](self.dimensions.get(var).size, value)
|
|
1402
|
+
return False
|
|
1403
|
+
|
|
1404
|
+
def _read(self):
|
|
1405
|
+
"""Abstract function for reading parameters into Parameters object.
|
|
1406
|
+
"""
|
|
1407
|
+
|
|
1408
|
+
assert False, 'Parameters._read() must be defined by child class'
|
|
1409
|
+
|
|
1410
|
+
def _required_parameters(self) -> Set:
|
|
1411
|
+
"""Return set of parameters required by modules selected in control file.
|
|
1412
|
+
|
|
1413
|
+
:returns: set of required parameter names
|
|
1414
|
+
"""
|
|
1415
|
+
if self.__control is None:
|
|
1416
|
+
# TODO: 20230727 PAN - this should raise an exception
|
|
1417
|
+
return set()
|
|
1418
|
+
|
|
1419
|
+
modules_used = set(self.__control.modules.values()).union(set(self.__control.additional_modules))
|
|
1420
|
+
|
|
1421
|
+
# -------------------------------
|
|
1422
|
+
# Get set of parameters required by the modules used
|
|
1423
|
+
pset = set()
|
|
1424
|
+
for kk, vv in self.metadata.items():
|
|
1425
|
+
for mm in vv['modules']:
|
|
1426
|
+
if mm in modules_used:
|
|
1427
|
+
pset.add(kk)
|
|
1428
|
+
|
|
1429
|
+
# Remove parameters that do not meet secondary requirements defined
|
|
1430
|
+
# in the metadata
|
|
1431
|
+
return self._trim_req_params(pset)
|
|
1432
|
+
|
|
1433
|
+
def _trim_req_params(self, param_set: Set) -> Set:
|
|
1434
|
+
"""Remove parameters from a set of parameters that do not meet secondary requirements.
|
|
1435
|
+
|
|
1436
|
+
:param param_set: set of parameter names
|
|
1437
|
+
:returns: set of parameter names that meet secondary requirements
|
|
1438
|
+
"""
|
|
1439
|
+
|
|
1440
|
+
remove_set = set()
|
|
1441
|
+
|
|
1442
|
+
for cparam in param_set:
|
|
1443
|
+
for xx in self.metadata[cparam].get('requires_control', []):
|
|
1444
|
+
if not self._condition_check_ctl(xx):
|
|
1445
|
+
remove_set.add(cparam)
|
|
1446
|
+
if self.verbose: # pragma: no cover
|
|
1447
|
+
con.print(f'[bold]{cparam}[/]: Control condition ({xx}) not met')
|
|
1448
|
+
|
|
1449
|
+
for xx in self.metadata[cparam].get('requires_dimension', []):
|
|
1450
|
+
if not self._condition_check_dim(xx):
|
|
1451
|
+
remove_set.add(cparam)
|
|
1452
|
+
if self.verbose: # pragma: no cover
|
|
1453
|
+
con.print(f'[bold]{cparam}[/]: Dimension condition ({xx}) not met')
|
|
1454
|
+
|
|
1455
|
+
for vv in remove_set:
|
|
1456
|
+
param_set.remove(vv)
|
|
1457
|
+
|
|
1458
|
+
return param_set
|
|
1459
|
+
|
|
1460
|
+
def _upstream_hrus(self, streamnet: nx.DiGraph,
|
|
1461
|
+
dsmost_seg: List[int]) -> List[int]:
|
|
1462
|
+
"""Get list of HRUs that contribute to the given stream segments.
|
|
1463
|
+
|
|
1464
|
+
:param streamnet: Directed, Acyclic Graph (DAG) of stream network
|
|
1465
|
+
:param dsmost_seg: list of downstream-most segment IDs
|
|
1466
|
+
|
|
1467
|
+
:returns: list of HRUs that contribute to the stream segments
|
|
1468
|
+
"""
|
|
1469
|
+
|
|
1470
|
+
# Get subset of stream network for given segment
|
|
1471
|
+
dag_ds_subset = get_streamnet_subset(streamnet, [], dsmost_seg)
|
|
1472
|
+
|
|
1473
|
+
# Create list of segments in the subset
|
|
1474
|
+
toseg_idx = list(set(xx[0] for xx in dag_ds_subset.edges))
|
|
1475
|
+
|
|
1476
|
+
# Build list of HRUs that contribute to the POI
|
|
1477
|
+
final_hru_list = []
|
|
1478
|
+
|
|
1479
|
+
for xx in toseg_idx:
|
|
1480
|
+
try:
|
|
1481
|
+
for yy in self.seg_to_hru[xx]:
|
|
1482
|
+
final_hru_list.append(yy)
|
|
1483
|
+
except KeyError:
|
|
1484
|
+
# Not all segments have HRUs connected to them
|
|
1485
|
+
print(f'Segment {xx} has no HRUs connected to it')
|
|
1486
|
+
|
|
1487
|
+
final_hru_list.sort()
|
|
1488
|
+
return final_hru_list
|
|
1489
|
+
|
|
1490
|
+
def _upstream_segments(self, streamnet: nx.DiGraph,
|
|
1491
|
+
dsmost_seg: List[int]) -> List[int]:
|
|
1492
|
+
"""Get list of segments that contribute to the given stream segments.
|
|
1493
|
+
|
|
1494
|
+
:param streamnet: Directed, Acyclic Graph (DAG) of stream network
|
|
1495
|
+
:param dsmost_seg: list of downstream-most segment IDs
|
|
1496
|
+
|
|
1497
|
+
:returns: list of segments that contribute to the stream segments
|
|
1498
|
+
"""
|
|
1499
|
+
|
|
1500
|
+
# Get subset of stream network for given POI
|
|
1501
|
+
dag_ds_subset = get_streamnet_subset(streamnet, [], dsmost_seg)
|
|
1502
|
+
|
|
1503
|
+
# Create list of segments in the subset
|
|
1504
|
+
toseg_idx = list(set(xx[0] for xx in dag_ds_subset.edges))
|
|
1505
|
+
|
|
1506
|
+
return toseg_idx
|