NREL-reV 0.8.7__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/METADATA +13 -10
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/RECORD +43 -43
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/WHEEL +1 -1
- reV/SAM/SAM.py +217 -133
- reV/SAM/econ.py +18 -14
- reV/SAM/generation.py +611 -422
- reV/SAM/windbos.py +93 -79
- reV/bespoke/bespoke.py +681 -377
- reV/bespoke/cli_bespoke.py +2 -0
- reV/bespoke/place_turbines.py +187 -43
- reV/config/output_request.py +2 -1
- reV/config/project_points.py +218 -140
- reV/econ/econ.py +166 -114
- reV/econ/economies_of_scale.py +91 -45
- reV/generation/base.py +331 -184
- reV/generation/generation.py +326 -200
- reV/generation/output_attributes/lcoe_fcr_inputs.json +38 -3
- reV/handlers/__init__.py +0 -1
- reV/handlers/exclusions.py +16 -15
- reV/handlers/multi_year.py +57 -26
- reV/handlers/outputs.py +6 -5
- reV/handlers/transmission.py +44 -27
- reV/hybrids/hybrid_methods.py +30 -30
- reV/hybrids/hybrids.py +305 -189
- reV/nrwal/nrwal.py +262 -168
- reV/qa_qc/cli_qa_qc.py +14 -10
- reV/qa_qc/qa_qc.py +217 -119
- reV/qa_qc/summary.py +228 -146
- reV/rep_profiles/rep_profiles.py +349 -230
- reV/supply_curve/aggregation.py +349 -188
- reV/supply_curve/competitive_wind_farms.py +90 -48
- reV/supply_curve/exclusions.py +138 -85
- reV/supply_curve/extent.py +75 -50
- reV/supply_curve/points.py +735 -390
- reV/supply_curve/sc_aggregation.py +357 -248
- reV/supply_curve/supply_curve.py +604 -347
- reV/supply_curve/tech_mapping.py +144 -82
- reV/utilities/__init__.py +274 -16
- reV/utilities/pytest_utils.py +8 -4
- reV/version.py +1 -1
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/LICENSE +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/entry_points.txt +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/top_level.txt +0 -0
reV/nrwal/nrwal.py
CHANGED
@@ -11,18 +11,20 @@ Everything in this module operates on the spatiotemporal resolution of the reV
|
|
11
11
|
generation output file. This is usually the wind or solar resource resolution
|
12
12
|
but could be the supply curve resolution after representative profiles is run.
|
13
13
|
"""
|
14
|
-
import numpy as np
|
15
|
-
import pandas as pd
|
16
14
|
import logging
|
17
15
|
from warnings import warn
|
18
16
|
|
17
|
+
import numpy as np
|
18
|
+
import pandas as pd
|
19
|
+
|
19
20
|
from reV.generation.generation import Gen
|
20
21
|
from reV.handlers.outputs import Outputs
|
21
|
-
from reV.utilities
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
22
|
+
from reV.utilities import SiteDataField, ResourceMetaField, log_versions
|
23
|
+
from reV.utilities.exceptions import (
|
24
|
+
DataShapeError,
|
25
|
+
OffshoreWindInputError,
|
26
|
+
OffshoreWindInputWarning,
|
27
|
+
)
|
26
28
|
|
27
29
|
logger = logging.getLogger(__name__)
|
28
30
|
|
@@ -30,11 +32,12 @@ logger = logging.getLogger(__name__)
|
|
30
32
|
class RevNrwal:
|
31
33
|
"""RevNrwal"""
|
32
34
|
|
33
|
-
DEFAULT_META_COLS = (
|
35
|
+
DEFAULT_META_COLS = (SiteDataField.CONFIG, )
|
34
36
|
"""Columns from the `site_data` table to join to the output meta data"""
|
35
37
|
|
36
38
|
def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs,
|
37
|
-
output_request, save_raw=True,
|
39
|
+
output_request, save_raw=True,
|
40
|
+
meta_gid_col=str(ResourceMetaField.GID), # str() to fix docs
|
38
41
|
site_meta_cols=None):
|
39
42
|
"""Framework to handle reV-NRWAL analysis.
|
40
43
|
|
@@ -163,8 +166,9 @@ class RevNrwal:
|
|
163
166
|
self._save_raw = save_raw
|
164
167
|
self._nrwal_inputs = self._out = None
|
165
168
|
|
166
|
-
self._nrwal_configs = {
|
167
|
-
|
169
|
+
self._nrwal_configs = {
|
170
|
+
k: NrwalConfig(v) for k, v in nrwal_configs.items()
|
171
|
+
}
|
168
172
|
|
169
173
|
self._site_meta_cols = site_meta_cols
|
170
174
|
if self._site_meta_cols is None:
|
@@ -178,20 +182,27 @@ class RevNrwal:
|
|
178
182
|
self._meta_source = self._parse_gen_data()
|
179
183
|
self._analysis_gids, self._site_data = self._parse_analysis_gids()
|
180
184
|
|
181
|
-
pc = Gen.get_pc(
|
182
|
-
|
185
|
+
pc = Gen.get_pc(
|
186
|
+
self._site_data[[SiteDataField.GID, SiteDataField.CONFIG]],
|
187
|
+
points_range=None, sam_configs=sam_files, tech='windpower')
|
183
188
|
self._project_points = pc.project_points
|
184
189
|
|
185
190
|
self._sam_sys_inputs = self._parse_sam_sys_inputs()
|
186
191
|
meta_gids = self.meta_source[self._meta_gid_col].values
|
187
|
-
logger.info(
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
192
|
+
logger.info(
|
193
|
+
'Finished initializing NRWAL analysis module for "{}" '
|
194
|
+
"{} through {} with {} total generation points and "
|
195
|
+
"{} NRWAL analysis points.".format(
|
196
|
+
self._meta_gid_col,
|
197
|
+
meta_gids.min(),
|
198
|
+
meta_gids.max(),
|
199
|
+
len(self.meta_source),
|
200
|
+
len(self.analysis_gids),
|
201
|
+
)
|
202
|
+
)
|
203
|
+
|
204
|
+
def _parse_site_data(self, required_columns=(SiteDataField.GID,
|
205
|
+
SiteDataField.CONFIG)):
|
195
206
|
"""Parse the site-specific spatial input data file
|
196
207
|
|
197
208
|
Parameters
|
@@ -210,24 +221,27 @@ class RevNrwal:
|
|
210
221
|
if isinstance(self._site_data, str):
|
211
222
|
self._site_data = pd.read_csv(self._site_data)
|
212
223
|
|
213
|
-
if
|
214
|
-
if self._site_data[
|
215
|
-
w = (
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
224
|
+
if "dist_l_to_ts" in self._site_data:
|
225
|
+
if self._site_data["dist_l_to_ts"].sum() > 0:
|
226
|
+
w = (
|
227
|
+
'Possible incorrect Offshore data input! "dist_l_to_ts" '
|
228
|
+
"(distance land to transmission) input is non-zero. "
|
229
|
+
"Most reV runs set this to zero and input the cost "
|
230
|
+
"of transmission from landfall tie-in to "
|
231
|
+
"transmission feature in the supply curve module."
|
232
|
+
)
|
220
233
|
logger.warning(w)
|
221
234
|
warn(w, OffshoreWindInputWarning)
|
222
235
|
|
223
236
|
for c in required_columns:
|
224
237
|
if c not in self._site_data:
|
225
|
-
msg =
|
226
|
-
|
238
|
+
msg = 'Did not find required "{}" column in site_data!'.format(
|
239
|
+
c
|
240
|
+
)
|
227
241
|
logger.error(msg)
|
228
242
|
raise KeyError(msg)
|
229
243
|
|
230
|
-
self._site_data = self._site_data.sort_values(
|
244
|
+
self._site_data = self._site_data.sort_values(SiteDataField.GID)
|
231
245
|
|
232
246
|
return self._site_data
|
233
247
|
|
@@ -240,16 +254,19 @@ class RevNrwal:
|
|
240
254
|
Full meta data from gen_fpath.
|
241
255
|
"""
|
242
256
|
|
243
|
-
with Outputs(self._gen_fpath, mode=
|
257
|
+
with Outputs(self._gen_fpath, mode="r") as out:
|
244
258
|
meta = out.meta
|
245
259
|
|
246
|
-
msg = (
|
247
|
-
|
248
|
-
|
260
|
+
msg = (
|
261
|
+
'Could not find "{}" column in source generation h5 file '
|
262
|
+
"meta data! Available cols: {}".format(
|
263
|
+
self._meta_gid_col, meta.columns.values.tolist()
|
264
|
+
)
|
265
|
+
)
|
249
266
|
assert self._meta_gid_col in meta, msg
|
250
267
|
|
251
268
|
# currently an assumption of sorted gids in the reV gen output
|
252
|
-
msg =
|
269
|
+
msg = "Source capacity factor meta data is not ordered!"
|
253
270
|
meta_gids = list(meta[self._meta_gid_col])
|
254
271
|
assert meta_gids == sorted(meta_gids), msg
|
255
272
|
|
@@ -272,27 +289,31 @@ class RevNrwal:
|
|
272
289
|
|
273
290
|
meta_gids = self.meta_source[self._meta_gid_col].values
|
274
291
|
|
275
|
-
missing = ~np.isin(meta_gids, self._site_data[
|
292
|
+
missing = ~np.isin(meta_gids, self._site_data[SiteDataField.GID])
|
276
293
|
if any(missing):
|
277
|
-
msg = (
|
278
|
-
|
279
|
-
|
280
|
-
|
294
|
+
msg = (
|
295
|
+
"{} sites from the generation meta data input were "
|
296
|
+
'missing from the "site_data" input and will not be '
|
297
|
+
"run through NRWAL: {}".format(
|
298
|
+
missing.sum(), meta_gids[missing]
|
299
|
+
)
|
300
|
+
)
|
281
301
|
logger.info(msg)
|
282
302
|
|
283
|
-
missing = ~np.isin(self._site_data[
|
303
|
+
missing = ~np.isin(self._site_data[SiteDataField.GID], meta_gids)
|
284
304
|
if any(missing):
|
285
|
-
missing = self._site_data[
|
305
|
+
missing = self._site_data[SiteDataField.GID].values[missing]
|
286
306
|
msg = ('{} sites from the "site_data" input were missing from the '
|
287
307
|
'generation meta data and will not be run through NRWAL: {}'
|
288
308
|
.format(len(missing), missing))
|
289
309
|
logger.info(msg)
|
290
310
|
|
291
|
-
analysis_gids = set(meta_gids)
|
311
|
+
analysis_gids = (set(meta_gids)
|
312
|
+
& set(self._site_data[SiteDataField.GID]))
|
292
313
|
analysis_gids = np.array(sorted(list(analysis_gids)))
|
293
314
|
|
294
315
|
# reduce the site data table to only those sites being analyzed
|
295
|
-
mask = np.isin(self._site_data[
|
316
|
+
mask = np.isin(self._site_data[SiteDataField.GID], meta_gids)
|
296
317
|
self._site_data = self._site_data[mask]
|
297
318
|
|
298
319
|
return analysis_gids, self._site_data
|
@@ -315,9 +336,9 @@ class RevNrwal:
|
|
315
336
|
|
316
337
|
system_inputs = pd.DataFrame(system_inputs).T
|
317
338
|
system_inputs = system_inputs.sort_index()
|
318
|
-
system_inputs[
|
319
|
-
system_inputs.index.name =
|
320
|
-
mask = system_inputs[
|
339
|
+
system_inputs[SiteDataField.GID] = system_inputs.index.values
|
340
|
+
system_inputs.index.name = SiteDataField.GID
|
341
|
+
mask = system_inputs[SiteDataField.GID].isin(self.analysis_gids)
|
321
342
|
system_inputs = system_inputs[mask]
|
322
343
|
|
323
344
|
return system_inputs
|
@@ -337,25 +358,30 @@ class RevNrwal:
|
|
337
358
|
out = {}
|
338
359
|
|
339
360
|
for key in self._output_request:
|
340
|
-
out[key] = np.full(
|
341
|
-
|
361
|
+
out[key] = np.full(
|
362
|
+
len(self.analysis_gids), np.nan, dtype=np.float32
|
363
|
+
)
|
342
364
|
|
343
365
|
if key in self.gen_dsets and not self._save_raw:
|
344
|
-
msg = (
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
366
|
+
msg = (
|
367
|
+
'Output request "{0}" was also found in '
|
368
|
+
"the source gen file but save_raw=False! If "
|
369
|
+
"you are manipulating this "
|
370
|
+
"dset, make sure you set save_raw=False "
|
371
|
+
'and reference "{0}_raw" as the '
|
372
|
+
'input in the NRWAL equations and then define "{0}" '
|
373
|
+
"as the final manipulated dataset.".format(key)
|
374
|
+
)
|
351
375
|
logger.warning(msg)
|
352
376
|
warn(msg)
|
353
377
|
elif key in self.gen_dsets:
|
354
|
-
msg = (
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
378
|
+
msg = (
|
379
|
+
'Output request "{0}" was also found in '
|
380
|
+
"the source gen file. If you are manipulating this "
|
381
|
+
'dset, make sure you reference "{0}_raw" as the '
|
382
|
+
'input in the NRWAL equations and then define "{0}" '
|
383
|
+
"as the final manipulated dataset.".format(key)
|
384
|
+
)
|
359
385
|
logger.info(msg)
|
360
386
|
|
361
387
|
if key in self._nrwal_inputs:
|
@@ -365,40 +391,48 @@ class RevNrwal:
|
|
365
391
|
|
366
392
|
def _preflight_checks(self):
|
367
393
|
"""Run some preflight checks on the offshore inputs"""
|
368
|
-
sam_files = {
|
369
|
-
|
370
|
-
|
394
|
+
sam_files = {
|
395
|
+
k: v
|
396
|
+
for k, v in self._project_points.sam_inputs.items()
|
397
|
+
if k in self._nrwal_configs
|
398
|
+
}
|
371
399
|
|
372
400
|
for cid, sys_in in sam_files.items():
|
373
|
-
loss1 = sys_in.get(
|
374
|
-
loss2 = sys_in.get(
|
401
|
+
loss1 = sys_in.get("wind_farm_losses_percent", 0)
|
402
|
+
loss2 = sys_in.get("turb_generic_loss", 0)
|
375
403
|
if loss1 != 0 or loss2 != 0:
|
376
|
-
msg = (
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
404
|
+
msg = (
|
405
|
+
'Wind farm loss for config "{}" is not 0. When using '
|
406
|
+
"NRWAL for offshore analysis, consider using gross "
|
407
|
+
"capacity factors from reV generation and applying "
|
408
|
+
"spatially dependent losses from the NRWAL equations"
|
409
|
+
.format(cid)
|
410
|
+
)
|
381
411
|
logger.info(msg)
|
382
412
|
|
383
413
|
available_ids = list(self._nrwal_configs.keys())
|
384
|
-
requested_ids = list(self._site_data[
|
414
|
+
requested_ids = list(self._site_data[SiteDataField.CONFIG].values)
|
385
415
|
missing = set(requested_ids) - set(available_ids)
|
386
416
|
if any(missing):
|
387
|
-
msg = (
|
388
|
-
|
389
|
-
|
417
|
+
msg = (
|
418
|
+
"The following config ids were requested in the offshore "
|
419
|
+
"data input but were not available in the NRWAL config "
|
420
|
+
"input dict: {}".format(missing)
|
421
|
+
)
|
390
422
|
logger.error(msg)
|
391
423
|
raise OffshoreWindInputError(msg)
|
392
424
|
|
393
|
-
check_gid_order = (self._site_data[
|
394
|
-
== self._sam_sys_inputs[
|
425
|
+
check_gid_order = (self._site_data[SiteDataField.GID].values
|
426
|
+
== self._sam_sys_inputs[SiteDataField.GID].values)
|
395
427
|
msg = 'NRWAL site_data and system input dataframe had bad order'
|
396
428
|
assert (check_gid_order).all(), msg
|
397
429
|
|
398
430
|
missing = [c for c in self._site_meta_cols if c not in self._site_data]
|
399
431
|
if any(missing):
|
400
|
-
msg = (
|
401
|
-
|
432
|
+
msg = (
|
433
|
+
"Could not find requested NRWAL site data pass through "
|
434
|
+
"columns in offshore input data: {}".format(missing)
|
435
|
+
)
|
402
436
|
logger.error(msg)
|
403
437
|
raise OffshoreWindInputError(msg)
|
404
438
|
|
@@ -413,7 +447,7 @@ class RevNrwal:
|
|
413
447
|
or 2D arrays of inputs for all the analysis_gids
|
414
448
|
"""
|
415
449
|
|
416
|
-
logger.info(
|
450
|
+
logger.info("Setting up input data for NRWAL...")
|
417
451
|
|
418
452
|
# preconditions for this to work properly
|
419
453
|
assert len(self._site_data) == len(self.analysis_gids)
|
@@ -424,50 +458,70 @@ class RevNrwal:
|
|
424
458
|
all_required += list(nrwal_config.required_inputs)
|
425
459
|
all_required = list(set(all_required))
|
426
460
|
|
427
|
-
missing_vars = [
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
461
|
+
missing_vars = [
|
462
|
+
var
|
463
|
+
for var in nrwal_config.required_inputs
|
464
|
+
if var not in self._site_data
|
465
|
+
and var not in self.meta_source
|
466
|
+
and var not in self._sam_sys_inputs
|
467
|
+
and var not in self.gen_dsets
|
468
|
+
]
|
432
469
|
|
433
470
|
if any(missing_vars):
|
434
|
-
msg = (
|
435
|
-
|
436
|
-
|
437
|
-
|
471
|
+
msg = (
|
472
|
+
"Could not find required input variables {} "
|
473
|
+
'for NRWAL config "{}" in either the offshore '
|
474
|
+
"data or the SAM system data!".format(
|
475
|
+
missing_vars, config_id
|
476
|
+
)
|
477
|
+
)
|
438
478
|
logger.error(msg)
|
439
479
|
raise OffshoreWindInputError(msg)
|
440
480
|
|
441
|
-
meta_data_vars = [
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
481
|
+
meta_data_vars = [
|
482
|
+
var for var in all_required if var in self.meta_source
|
483
|
+
]
|
484
|
+
logger.info(
|
485
|
+
"Pulling the following inputs from the gen meta data: {}".format(
|
486
|
+
meta_data_vars
|
487
|
+
)
|
488
|
+
)
|
489
|
+
nrwal_inputs = {
|
490
|
+
var: self.meta_source[var].values[self.analysis_mask]
|
491
|
+
for var in meta_data_vars
|
492
|
+
}
|
447
493
|
|
448
494
|
site_data_vars = [var for var in all_required
|
449
495
|
if var in self._site_data
|
450
496
|
and var not in nrwal_inputs]
|
451
|
-
site_data_vars.append(
|
497
|
+
site_data_vars.append(SiteDataField.CONFIG)
|
452
498
|
logger.info('Pulling the following inputs from the site_data input: {}'
|
453
499
|
.format(site_data_vars))
|
454
500
|
for var in site_data_vars:
|
455
501
|
nrwal_inputs[var] = self._site_data[var].values
|
456
502
|
|
457
|
-
sam_sys_vars = [
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
503
|
+
sam_sys_vars = [
|
504
|
+
var
|
505
|
+
for var in all_required
|
506
|
+
if var in self._sam_sys_inputs and var not in nrwal_inputs
|
507
|
+
]
|
508
|
+
logger.info(
|
509
|
+
"Pulling the following inputs from the SAM system "
|
510
|
+
"configs: {}".format(sam_sys_vars)
|
511
|
+
)
|
462
512
|
for var in sam_sys_vars:
|
463
513
|
nrwal_inputs[var] = self._sam_sys_inputs[var].values
|
464
514
|
|
465
|
-
gen_vars = [
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
515
|
+
gen_vars = [
|
516
|
+
var
|
517
|
+
for var in all_required
|
518
|
+
if var in self.gen_dsets and var not in nrwal_inputs
|
519
|
+
]
|
520
|
+
logger.info(
|
521
|
+
"Pulling the following inputs from the generation "
|
522
|
+
"h5 file: {}".format(gen_vars)
|
523
|
+
)
|
524
|
+
with Outputs(self._gen_fpath, mode="r") as f:
|
471
525
|
source_gids = self.meta_source[self._meta_gid_col]
|
472
526
|
gen_gids = np.where(source_gids.isin(self.analysis_gids))[0]
|
473
527
|
for var in gen_vars:
|
@@ -477,12 +531,14 @@ class RevNrwal:
|
|
477
531
|
elif len(shape) == 2:
|
478
532
|
nrwal_inputs[var] = f[var, :, gen_gids]
|
479
533
|
else:
|
480
|
-
msg = (
|
481
|
-
|
534
|
+
msg = (
|
535
|
+
'Data shape for "{}" must be 1 or 2D but '
|
536
|
+
"received: {}".format(var, shape)
|
537
|
+
)
|
482
538
|
logger.error(msg)
|
483
539
|
raise DataShapeError(msg)
|
484
540
|
|
485
|
-
logger.info(
|
541
|
+
logger.info("Finished setting up input data for NRWAL!")
|
486
542
|
|
487
543
|
return nrwal_inputs
|
488
544
|
|
@@ -490,7 +546,7 @@ class RevNrwal:
|
|
490
546
|
def time_index(self):
|
491
547
|
"""Get the source time index."""
|
492
548
|
if self._time_index is None:
|
493
|
-
with Outputs(self._gen_fpath, mode=
|
549
|
+
with Outputs(self._gen_fpath, mode="r") as out:
|
494
550
|
self._time_index = out.time_index
|
495
551
|
|
496
552
|
return self._time_index
|
@@ -498,7 +554,7 @@ class RevNrwal:
|
|
498
554
|
@property
|
499
555
|
def gen_dsets(self):
|
500
556
|
"""Get the available datasets from the gen source file"""
|
501
|
-
with Outputs(self._gen_fpath, mode=
|
557
|
+
with Outputs(self._gen_fpath, mode="r") as out:
|
502
558
|
dsets = out.dsets
|
503
559
|
|
504
560
|
return dsets
|
@@ -528,8 +584,9 @@ class RevNrwal:
|
|
528
584
|
-------
|
529
585
|
np.ndarray
|
530
586
|
"""
|
531
|
-
mask = np.isin(
|
532
|
-
|
587
|
+
mask = np.isin(
|
588
|
+
self.meta_source[self._meta_gid_col], self.analysis_gids
|
589
|
+
)
|
533
590
|
return mask
|
534
591
|
|
535
592
|
@property
|
@@ -576,9 +633,12 @@ class RevNrwal:
|
|
576
633
|
elif len(value.shape) == 2:
|
577
634
|
if len(self._out[name].shape) == 1:
|
578
635
|
if not all(np.isnan(self._out[name])):
|
579
|
-
msg = (
|
580
|
-
|
581
|
-
|
636
|
+
msg = (
|
637
|
+
'Output dataset "{}" was initialized as 1D but was '
|
638
|
+
"later found to be 2D but was not all NaN!".format(
|
639
|
+
name
|
640
|
+
)
|
641
|
+
)
|
582
642
|
logger.error(msg)
|
583
643
|
raise DataShapeError(msg)
|
584
644
|
|
@@ -590,8 +650,10 @@ class RevNrwal:
|
|
590
650
|
self._out[name][:, output_mask] = value[:, output_mask]
|
591
651
|
|
592
652
|
else:
|
593
|
-
msg = (
|
594
|
-
|
653
|
+
msg = (
|
654
|
+
'Could not make sense of NRWAL output "{}" '
|
655
|
+
"with shape {}".format(name, value.shape)
|
656
|
+
)
|
595
657
|
logger.error(msg)
|
596
658
|
raise DataShapeError(msg)
|
597
659
|
|
@@ -614,11 +676,14 @@ class RevNrwal:
|
|
614
676
|
"""
|
615
677
|
|
616
678
|
from NRWAL import Equation
|
679
|
+
|
617
680
|
value = nrwal_config[name]
|
618
681
|
|
619
682
|
if isinstance(value, Equation):
|
620
|
-
msg = (
|
621
|
-
|
683
|
+
msg = (
|
684
|
+
'Cannot retrieve Equation "{}" from NRWAL. '
|
685
|
+
"Must be a number!".format(name)
|
686
|
+
)
|
622
687
|
assert not any(value.variables), msg
|
623
688
|
value = value.eval()
|
624
689
|
|
@@ -631,8 +696,10 @@ class RevNrwal:
|
|
631
696
|
value *= np.ones(len(self.analysis_gids), dtype=np.float32)
|
632
697
|
|
633
698
|
if not isinstance(value, np.ndarray):
|
634
|
-
msg = (
|
635
|
-
|
699
|
+
msg = (
|
700
|
+
'NRWAL key "{}" returned bad type of "{}", needs to be '
|
701
|
+
"numeric or an output array.".format(name, type(value))
|
702
|
+
)
|
636
703
|
logger.error(msg)
|
637
704
|
raise TypeError(msg)
|
638
705
|
return value
|
@@ -646,7 +713,7 @@ class RevNrwal:
|
|
646
713
|
self._out = self._init_outputs()
|
647
714
|
|
648
715
|
for i, (cid, nrwal_config) in enumerate(self._nrwal_configs.items()):
|
649
|
-
output_mask = self._site_data[
|
716
|
+
output_mask = self._site_data[SiteDataField.CONFIG].values == cid
|
650
717
|
logger.info('Running NRWAL config {} of {}: "{}" and applying '
|
651
718
|
'to {} out of {} total sites'
|
652
719
|
.format(i + 1, len(self._nrwal_configs), cid,
|
@@ -663,8 +730,10 @@ class RevNrwal:
|
|
663
730
|
self._save_nrwal_misc(name, nrwal_config, output_mask)
|
664
731
|
|
665
732
|
elif name not in self._nrwal_inputs:
|
666
|
-
msg = (
|
667
|
-
|
733
|
+
msg = (
|
734
|
+
'Could not find "{}" in the output dict of NRWAL '
|
735
|
+
"config {}".format(name, cid)
|
736
|
+
)
|
668
737
|
logger.warning(msg)
|
669
738
|
warn(msg)
|
670
739
|
|
@@ -672,36 +741,50 @@ class RevNrwal:
|
|
672
741
|
"""Check the nrwal outputs for nan values and raise errors if found."""
|
673
742
|
for name, arr in self._out.items():
|
674
743
|
if np.isnan(arr).all():
|
675
|
-
msg = (
|
676
|
-
|
744
|
+
msg = (
|
745
|
+
'Output array "{}" is all NaN! Probably was not '
|
746
|
+
"found in the available NRWAL keys.".format(name)
|
747
|
+
)
|
677
748
|
logger.warning(msg)
|
678
749
|
warn(msg)
|
679
750
|
elif np.isnan(arr).any():
|
680
751
|
mask = np.isnan(arr)
|
681
752
|
nan_meta = self.meta_source[self.analysis_mask][mask]
|
682
753
|
nan_gids = nan_meta[self._meta_gid_col].values
|
683
|
-
msg = (
|
684
|
-
|
685
|
-
|
754
|
+
msg = (
|
755
|
+
"NaN values ({} out of {}) persist in NRWAL "
|
756
|
+
'output "{}"!'.format(np.isnan(arr).sum(), len(arr), name)
|
757
|
+
)
|
686
758
|
logger.warning(msg)
|
687
|
-
logger.warning(
|
688
|
-
|
689
|
-
|
690
|
-
|
759
|
+
logger.warning(
|
760
|
+
"This is the NRWAL meta that is causing NaN "
|
761
|
+
"outputs: {}".format(nan_meta)
|
762
|
+
)
|
763
|
+
logger.warning(
|
764
|
+
"These are the resource gids causing NaN "
|
765
|
+
"outputs: {}".format(nan_gids)
|
766
|
+
)
|
691
767
|
warn(msg)
|
692
768
|
|
693
769
|
def save_raw_dsets(self):
|
694
770
|
"""If requested by save_raw=True, archive raw datasets that exist in
|
695
771
|
the gen_fpath file and are also requested in the output_request"""
|
696
772
|
if self._save_raw:
|
697
|
-
with Outputs(self._gen_fpath,
|
773
|
+
with Outputs(self._gen_fpath, "a") as f:
|
698
774
|
for dset in self._output_request:
|
699
|
-
dset_raw =
|
775
|
+
dset_raw = "{}_raw".format(dset)
|
700
776
|
if dset in f and dset_raw not in f:
|
701
|
-
logger.info(
|
702
|
-
|
703
|
-
|
704
|
-
|
777
|
+
logger.info(
|
778
|
+
'Saving raw data from "{}" to "{}"'.format(
|
779
|
+
dset, dset_raw
|
780
|
+
)
|
781
|
+
)
|
782
|
+
f._add_dset(
|
783
|
+
dset_raw,
|
784
|
+
f[dset],
|
785
|
+
f.dtypes[dset],
|
786
|
+
attrs=f.attrs[dset],
|
787
|
+
)
|
705
788
|
|
706
789
|
def write_to_gen_fpath(self):
|
707
790
|
"""Save NRWAL outputs to input generation fpath file.
|
@@ -712,30 +795,34 @@ class RevNrwal:
|
|
712
795
|
Path to output file.
|
713
796
|
"""
|
714
797
|
|
715
|
-
logger.info(
|
798
|
+
logger.info("Writing NRWAL outputs to: {}".format(self._gen_fpath))
|
716
799
|
write_all = self.analysis_mask.all()
|
717
800
|
|
718
|
-
with Outputs(self._gen_fpath,
|
719
|
-
meta_attrs = f.attrs[
|
720
|
-
del f._h5[
|
721
|
-
f._set_meta(
|
801
|
+
with Outputs(self._gen_fpath, "a") as f:
|
802
|
+
meta_attrs = f.attrs["meta"]
|
803
|
+
del f._h5["meta"]
|
804
|
+
f._set_meta("meta", self.meta_out, attrs=meta_attrs)
|
722
805
|
|
723
806
|
for dset, arr in self._out.items():
|
724
807
|
if len(arr.shape) == 1:
|
725
|
-
data = np.full(
|
726
|
-
|
808
|
+
data = np.full(
|
809
|
+
len(self.meta_source), np.nan, dtype=np.float32
|
810
|
+
)
|
727
811
|
else:
|
728
|
-
full_shape = (len(self.time_index),
|
729
|
-
len(self.meta_source))
|
812
|
+
full_shape = (len(self.time_index), len(self.meta_source))
|
730
813
|
data = np.full(full_shape, np.nan, dtype=np.float32)
|
731
814
|
|
732
|
-
dset_attrs = {
|
815
|
+
dset_attrs = {"scale_factor": 1}
|
733
816
|
dset_dtype = np.float32
|
734
817
|
if dset in f.dsets:
|
735
|
-
logger.info(
|
736
|
-
|
737
|
-
|
738
|
-
|
818
|
+
logger.info(
|
819
|
+
'Found "{}" in file, loading data and '
|
820
|
+
"overwriting data for {} out of {} sites.".format(
|
821
|
+
dset,
|
822
|
+
self.analysis_mask.sum(),
|
823
|
+
len(self.analysis_mask),
|
824
|
+
)
|
825
|
+
)
|
739
826
|
dset_attrs = f.attrs[dset]
|
740
827
|
dset_dtype = f.dtypes[dset]
|
741
828
|
if not write_all:
|
@@ -746,12 +833,14 @@ class RevNrwal:
|
|
746
833
|
else:
|
747
834
|
data[:, self.analysis_mask] = arr
|
748
835
|
|
749
|
-
logger.info(
|
750
|
-
|
836
|
+
logger.info(
|
837
|
+
'Writing final "{}" to: {}'.format(dset, self._gen_fpath)
|
838
|
+
)
|
751
839
|
f._add_dset(dset, data, dset_dtype, attrs=dset_attrs)
|
752
840
|
|
753
|
-
logger.info(
|
754
|
-
|
841
|
+
logger.info(
|
842
|
+
"Finished writing NRWAL outputs to: {}".format(self._gen_fpath)
|
843
|
+
)
|
755
844
|
return self._gen_fpath
|
756
845
|
|
757
846
|
def write_meta_to_csv(self, out_fpath=None):
|
@@ -776,21 +865,24 @@ class RevNrwal:
|
|
776
865
|
elif not out_fpath.endswith(".csv"):
|
777
866
|
out_fpath = "{}.csv".format(out_fpath)
|
778
867
|
|
779
|
-
logger.info(
|
868
|
+
logger.info("Writing NRWAL outputs to: {}".format(out_fpath))
|
780
869
|
meta_out = self.meta_out[self.analysis_mask].copy()
|
781
870
|
|
782
871
|
for dset, arr in self._out.items():
|
783
872
|
if len(arr.shape) != 1 or arr.shape[0] != meta_out.shape[0]:
|
784
|
-
msg = (
|
785
|
-
|
786
|
-
|
873
|
+
msg = (
|
874
|
+
"Skipping output {!r}: shape {} cannot be combined "
|
875
|
+
"with meta of shape {}!".format(
|
876
|
+
dset, arr.shape, meta_out.shape
|
877
|
+
)
|
878
|
+
)
|
787
879
|
logger.warning(msg)
|
788
880
|
warn(msg)
|
789
881
|
continue
|
790
882
|
meta_out[dset] = arr
|
791
883
|
|
792
884
|
meta_out.to_csv(out_fpath, index=False)
|
793
|
-
logger.info(
|
885
|
+
logger.info("Finished writing NRWAL outputs to: {}".format(out_fpath))
|
794
886
|
return out_fpath
|
795
887
|
|
796
888
|
def run(self, csv_output=False, out_fpath=None):
|
@@ -831,8 +923,10 @@ class RevNrwal:
|
|
831
923
|
Path to output file.
|
832
924
|
"""
|
833
925
|
if csv_output and self._save_raw:
|
834
|
-
msg = (
|
835
|
-
|
926
|
+
msg = (
|
927
|
+
"`save_raw` option not allowed with `csv_output`. Setting"
|
928
|
+
"`save_raw=False`"
|
929
|
+
)
|
836
930
|
logger.warning(msg)
|
837
931
|
warn(msg)
|
838
932
|
self._save_raw = False
|
@@ -845,6 +939,6 @@ class RevNrwal:
|
|
845
939
|
else:
|
846
940
|
out_fp = self.write_to_gen_fpath()
|
847
941
|
|
848
|
-
logger.info(
|
942
|
+
logger.info("NRWAL module complete!")
|
849
943
|
|
850
944
|
return out_fp
|