NREL-reV 0.8.7__py3-none-any.whl → 0.8.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/METADATA +12 -10
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/RECORD +38 -38
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/WHEEL +1 -1
- reV/SAM/SAM.py +182 -133
- reV/SAM/econ.py +18 -14
- reV/SAM/generation.py +608 -419
- reV/SAM/windbos.py +93 -79
- reV/bespoke/bespoke.py +690 -445
- reV/bespoke/place_turbines.py +6 -6
- reV/config/project_points.py +220 -140
- reV/econ/econ.py +165 -113
- reV/econ/economies_of_scale.py +57 -34
- reV/generation/base.py +310 -183
- reV/generation/generation.py +298 -190
- reV/handlers/exclusions.py +16 -15
- reV/handlers/multi_year.py +12 -9
- reV/handlers/outputs.py +6 -5
- reV/hybrids/hybrid_methods.py +28 -30
- reV/hybrids/hybrids.py +304 -188
- reV/nrwal/nrwal.py +262 -168
- reV/qa_qc/cli_qa_qc.py +14 -10
- reV/qa_qc/qa_qc.py +217 -119
- reV/qa_qc/summary.py +228 -146
- reV/rep_profiles/rep_profiles.py +349 -230
- reV/supply_curve/aggregation.py +349 -188
- reV/supply_curve/competitive_wind_farms.py +90 -48
- reV/supply_curve/exclusions.py +138 -85
- reV/supply_curve/extent.py +75 -50
- reV/supply_curve/points.py +536 -309
- reV/supply_curve/sc_aggregation.py +366 -225
- reV/supply_curve/supply_curve.py +505 -308
- reV/supply_curve/tech_mapping.py +144 -82
- reV/utilities/__init__.py +199 -16
- reV/utilities/pytest_utils.py +8 -4
- reV/version.py +1 -1
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/LICENSE +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/entry_points.txt +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/top_level.txt +0 -0
reV/generation/base.py
CHANGED
@@ -2,44 +2,47 @@
|
|
2
2
|
"""
|
3
3
|
reV base gen and econ module.
|
4
4
|
"""
|
5
|
-
from abc import ABC, abstractmethod
|
6
5
|
import copy
|
7
|
-
|
6
|
+
import json
|
8
7
|
import logging
|
9
|
-
import pandas as pd
|
10
|
-
import numpy as np
|
11
8
|
import os
|
12
|
-
import psutil
|
13
|
-
import json
|
14
9
|
import sys
|
10
|
+
from abc import ABC, abstractmethod
|
11
|
+
from concurrent.futures import TimeoutError
|
15
12
|
from warnings import warn
|
16
13
|
|
14
|
+
import numpy as np
|
15
|
+
import pandas as pd
|
16
|
+
import psutil
|
17
|
+
from rex.resource import Resource
|
18
|
+
from rex.utilities.execution import SpawnProcessPool
|
19
|
+
|
17
20
|
from reV.config.output_request import SAMOutputRequest
|
18
|
-
from reV.config.project_points import
|
21
|
+
from reV.config.project_points import PointsControl, ProjectPoints
|
19
22
|
from reV.handlers.outputs import Outputs
|
20
23
|
from reV.SAM.version_checker import PySamVersionChecker
|
21
|
-
from reV.utilities
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
from reV.utilities import ModuleName, ResourceMetaField, log_versions
|
25
|
+
from reV.utilities.exceptions import (
|
26
|
+
ExecutionError,
|
27
|
+
OffshoreWindInputWarning,
|
28
|
+
OutputWarning,
|
29
|
+
ParallelExecutionWarning,
|
30
|
+
)
|
28
31
|
|
29
32
|
logger = logging.getLogger(__name__)
|
30
33
|
|
31
34
|
|
32
35
|
ATTR_DIR = os.path.dirname(os.path.realpath(__file__))
|
33
36
|
ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes')
|
34
|
-
with open(os.path.join(ATTR_DIR, 'other.json')
|
37
|
+
with open(os.path.join(ATTR_DIR, 'other.json')) as f:
|
35
38
|
OTHER_ATTRS = json.load(f)
|
36
|
-
with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json')
|
39
|
+
with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json')) as f:
|
37
40
|
LCOE_ATTRS = json.load(f)
|
38
|
-
with open(os.path.join(ATTR_DIR, 'single_owner.json')
|
41
|
+
with open(os.path.join(ATTR_DIR, 'single_owner.json')) as f:
|
39
42
|
SO_ATTRS = json.load(f)
|
40
|
-
with open(os.path.join(ATTR_DIR, 'windbos.json')
|
43
|
+
with open(os.path.join(ATTR_DIR, 'windbos.json')) as f:
|
41
44
|
BOS_ATTRS = json.load(f)
|
42
|
-
with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json')
|
45
|
+
with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json')) as f:
|
43
46
|
LCOE_IN_ATTRS = json.load(f)
|
44
47
|
|
45
48
|
|
@@ -65,12 +68,19 @@ class BaseGen(ABC):
|
|
65
68
|
# SAM argument names used to calculate LCOE
|
66
69
|
# Note that system_capacity is not included here because it is never used
|
67
70
|
# downstream and could be confused with the supply_curve point capacity
|
68
|
-
LCOE_ARGS = ('fixed_charge_rate', 'capital_cost',
|
71
|
+
LCOE_ARGS = ('fixed_charge_rate', 'capital_cost',
|
72
|
+
'fixed_operating_cost',
|
69
73
|
'variable_operating_cost')
|
70
74
|
|
71
|
-
def __init__(
|
72
|
-
|
73
|
-
|
75
|
+
def __init__(
|
76
|
+
self,
|
77
|
+
points_control,
|
78
|
+
output_request,
|
79
|
+
site_data=None,
|
80
|
+
drop_leap=False,
|
81
|
+
memory_utilization_limit=0.4,
|
82
|
+
scale_outputs=True,
|
83
|
+
):
|
74
84
|
"""
|
75
85
|
Parameters
|
76
86
|
----------
|
@@ -106,11 +116,13 @@ class BaseGen(ABC):
|
|
106
116
|
self.mem_util_lim = memory_utilization_limit
|
107
117
|
self.scale_outputs = scale_outputs
|
108
118
|
|
109
|
-
self._run_attrs = {
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
119
|
+
self._run_attrs = {
|
120
|
+
"points_control": str(points_control),
|
121
|
+
"output_request": output_request,
|
122
|
+
"site_data": str(site_data),
|
123
|
+
"drop_leap": str(drop_leap),
|
124
|
+
"memory_utilization_limit": self.mem_util_lim,
|
125
|
+
}
|
114
126
|
|
115
127
|
self._site_data = self._parse_site_data(site_data)
|
116
128
|
self.add_site_data_to_pp(self._site_data)
|
@@ -174,11 +186,16 @@ class BaseGen(ABC):
|
|
174
186
|
tot_mem = psutil.virtual_memory().total / 1e6
|
175
187
|
avail_mem = self.mem_util_lim * tot_mem
|
176
188
|
self._site_limit = int(np.floor(avail_mem / self.site_mem))
|
177
|
-
logger.info(
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
189
|
+
logger.info(
|
190
|
+
"Limited to storing {0} sites in memory "
|
191
|
+
"({1:.1f} GB total hardware, {2:.1f} GB available "
|
192
|
+
"with {3:.1f}% utilization).".format(
|
193
|
+
self._site_limit,
|
194
|
+
tot_mem / 1e3,
|
195
|
+
avail_mem / 1e3,
|
196
|
+
self.mem_util_lim * 100,
|
197
|
+
)
|
198
|
+
)
|
182
199
|
|
183
200
|
return self._site_limit
|
184
201
|
|
@@ -199,17 +216,18 @@ class BaseGen(ABC):
|
|
199
216
|
n = 100
|
200
217
|
self._site_mem = 0
|
201
218
|
for request in self.output_request:
|
202
|
-
dtype =
|
219
|
+
dtype = "float32"
|
203
220
|
if request in self.OUT_ATTRS:
|
204
|
-
dtype = self.OUT_ATTRS[request].get(
|
221
|
+
dtype = self.OUT_ATTRS[request].get("dtype", "float32")
|
205
222
|
|
206
223
|
shape = self._get_data_shape(request, n)
|
207
224
|
self._site_mem += sys.getsizeof(np.ones(shape, dtype=dtype))
|
208
225
|
|
209
226
|
self._site_mem = self._site_mem / 1e6 / n
|
210
|
-
logger.info(
|
211
|
-
|
212
|
-
|
227
|
+
logger.info(
|
228
|
+
"Output results from a single site are calculated to "
|
229
|
+
"use {0:.1f} KB of memory.".format(self._site_mem / 1000)
|
230
|
+
)
|
213
231
|
|
214
232
|
return self._site_mem
|
215
233
|
|
@@ -259,7 +277,7 @@ class BaseGen(ABC):
|
|
259
277
|
"""
|
260
278
|
sam_metas = self.sam_configs.copy()
|
261
279
|
for v in sam_metas.values():
|
262
|
-
v.update({
|
280
|
+
v.update({"module": self._sam_module.MODULE})
|
263
281
|
|
264
282
|
return sam_metas
|
265
283
|
|
@@ -284,7 +302,8 @@ class BaseGen(ABC):
|
|
284
302
|
Meta data df for sites in project points. Column names are meta
|
285
303
|
data variables, rows are different sites. The row index
|
286
304
|
does not indicate the site number if the project points are
|
287
|
-
non-sequential or do not start from 0, so a
|
305
|
+
non-sequential or do not start from 0, so a `SupplyCurveField.GID`
|
306
|
+
column is added.
|
288
307
|
"""
|
289
308
|
return self._meta
|
290
309
|
|
@@ -351,12 +370,12 @@ class BaseGen(ABC):
|
|
351
370
|
out = {}
|
352
371
|
for k, v in self._out.items():
|
353
372
|
if k in self.OUT_ATTRS:
|
354
|
-
scale_factor = self.OUT_ATTRS[k].get(
|
373
|
+
scale_factor = self.OUT_ATTRS[k].get("scale_factor", 1)
|
355
374
|
else:
|
356
375
|
scale_factor = 1
|
357
376
|
|
358
377
|
if scale_factor != 1 and self.scale_outputs:
|
359
|
-
v = v.astype(
|
378
|
+
v = v.astype("float32")
|
360
379
|
v /= scale_factor
|
361
380
|
|
362
381
|
out[k] = v
|
@@ -381,16 +400,14 @@ class BaseGen(ABC):
|
|
381
400
|
result = self.unpack_futures(result)
|
382
401
|
|
383
402
|
if isinstance(result, dict):
|
384
|
-
|
385
403
|
# iterate through dict where sites are keys and values are
|
386
404
|
# corresponding results
|
387
405
|
for site_gid, site_output in result.items():
|
388
|
-
|
389
406
|
# check that the sites are stored sequentially then add to
|
390
407
|
# the finished site list
|
391
408
|
if self._finished_sites:
|
392
409
|
if int(site_gid) < np.max(self._finished_sites):
|
393
|
-
raise Exception(
|
410
|
+
raise Exception("Site results are non sequential!")
|
394
411
|
|
395
412
|
# unpack site output object
|
396
413
|
self.unpack_output(site_gid, site_output)
|
@@ -402,9 +419,11 @@ class BaseGen(ABC):
|
|
402
419
|
self._out.clear()
|
403
420
|
self._finished_sites.clear()
|
404
421
|
else:
|
405
|
-
raise TypeError(
|
406
|
-
|
407
|
-
|
422
|
+
raise TypeError(
|
423
|
+
"Did not recognize the type of output. "
|
424
|
+
'Tried to set output type "{}", but requires '
|
425
|
+
"list, dict or None.".format(type(result))
|
426
|
+
)
|
408
427
|
|
409
428
|
@staticmethod
|
410
429
|
def _output_request_type_check(req):
|
@@ -428,8 +447,10 @@ class BaseGen(ABC):
|
|
428
447
|
elif isinstance(req, str):
|
429
448
|
output_request = [req]
|
430
449
|
else:
|
431
|
-
raise TypeError(
|
432
|
-
|
450
|
+
raise TypeError(
|
451
|
+
"Output request must be str, list, or tuple but "
|
452
|
+
"received: {}".format(type(req))
|
453
|
+
)
|
433
454
|
|
434
455
|
return output_request
|
435
456
|
|
@@ -452,7 +473,7 @@ class BaseGen(ABC):
|
|
452
473
|
"""
|
453
474
|
|
454
475
|
# Drop leap day or last day
|
455
|
-
leap_day = (
|
476
|
+
leap_day = (ti.month == 2) & (ti.day == 29)
|
456
477
|
leap_year = ti.year % 4 == 0
|
457
478
|
last_day = ((ti.month == 12) & (ti.day == 31)) * leap_year
|
458
479
|
if drop_leap:
|
@@ -463,14 +484,23 @@ class BaseGen(ABC):
|
|
463
484
|
ti = ti.drop(ti[last_day])
|
464
485
|
|
465
486
|
if len(ti) % 365 != 0:
|
466
|
-
raise ValueError(
|
467
|
-
|
487
|
+
raise ValueError(
|
488
|
+
"Bad time index with length not a multiple of "
|
489
|
+
"365: {}".format(ti)
|
490
|
+
)
|
468
491
|
|
469
492
|
return ti
|
470
493
|
|
471
494
|
@staticmethod
|
472
|
-
def _pp_to_pc(
|
473
|
-
|
495
|
+
def _pp_to_pc(
|
496
|
+
points,
|
497
|
+
points_range,
|
498
|
+
sam_configs,
|
499
|
+
tech,
|
500
|
+
sites_per_worker=None,
|
501
|
+
res_file=None,
|
502
|
+
curtailment=None,
|
503
|
+
):
|
474
504
|
"""
|
475
505
|
Create ProjectControl from ProjectPoints
|
476
506
|
|
@@ -519,16 +549,25 @@ class BaseGen(ABC):
|
|
519
549
|
if hasattr(points, "df"):
|
520
550
|
points = points.df
|
521
551
|
|
522
|
-
pp = ProjectPoints(
|
523
|
-
|
552
|
+
pp = ProjectPoints(
|
553
|
+
points,
|
554
|
+
sam_configs,
|
555
|
+
tech=tech,
|
556
|
+
res_file=res_file,
|
557
|
+
curtailment=curtailment,
|
558
|
+
)
|
524
559
|
|
525
560
|
# make Points Control instance
|
526
561
|
if points_range is not None:
|
527
562
|
# PointsControl is for just a subset of the project points...
|
528
563
|
# this is the case if generation is being initialized on one
|
529
564
|
# of many HPC nodes in a large project
|
530
|
-
pc = PointsControl.split(
|
531
|
-
|
565
|
+
pc = PointsControl.split(
|
566
|
+
points_range[0],
|
567
|
+
points_range[1],
|
568
|
+
pp,
|
569
|
+
sites_per_split=sites_per_worker,
|
570
|
+
)
|
532
571
|
else:
|
533
572
|
# PointsControl is for all of the project points
|
534
573
|
pc = PointsControl(pp, sites_per_split=sites_per_worker)
|
@@ -536,8 +575,16 @@ class BaseGen(ABC):
|
|
536
575
|
return pc
|
537
576
|
|
538
577
|
@classmethod
|
539
|
-
def get_pc(
|
540
|
-
|
578
|
+
def get_pc(
|
579
|
+
cls,
|
580
|
+
points,
|
581
|
+
points_range,
|
582
|
+
sam_configs,
|
583
|
+
tech,
|
584
|
+
sites_per_worker=None,
|
585
|
+
res_file=None,
|
586
|
+
curtailment=None,
|
587
|
+
):
|
541
588
|
"""Get a PointsControl instance.
|
542
589
|
|
543
590
|
Parameters
|
@@ -585,9 +632,12 @@ class BaseGen(ABC):
|
|
585
632
|
"""
|
586
633
|
|
587
634
|
if tech not in cls.OPTIONS and tech.lower() != ModuleName.ECON:
|
588
|
-
msg = (
|
589
|
-
|
590
|
-
|
635
|
+
msg = (
|
636
|
+
'Did not recognize reV-SAM technology string "{}". '
|
637
|
+
"Technology string options are: {}".format(
|
638
|
+
tech, list(cls.OPTIONS.keys())
|
639
|
+
)
|
640
|
+
)
|
591
641
|
logger.error(msg)
|
592
642
|
raise KeyError(msg)
|
593
643
|
|
@@ -595,16 +645,25 @@ class BaseGen(ABC):
|
|
595
645
|
# get the optimal sites per split based on res file chunk size
|
596
646
|
sites_per_worker = cls.get_sites_per_worker(res_file)
|
597
647
|
|
598
|
-
logger.debug(
|
599
|
-
|
648
|
+
logger.debug(
|
649
|
+
"Sites per worker being set to {} for " "PointsControl.".format(
|
650
|
+
sites_per_worker
|
651
|
+
)
|
652
|
+
)
|
600
653
|
|
601
654
|
if isinstance(points, PointsControl):
|
602
655
|
# received a pre-intialized instance of pointscontrol
|
603
656
|
pc = points
|
604
657
|
else:
|
605
|
-
pc = cls._pp_to_pc(
|
606
|
-
|
607
|
-
|
658
|
+
pc = cls._pp_to_pc(
|
659
|
+
points,
|
660
|
+
points_range,
|
661
|
+
sam_configs,
|
662
|
+
tech,
|
663
|
+
sites_per_worker=sites_per_worker,
|
664
|
+
res_file=res_file,
|
665
|
+
curtailment=curtailment,
|
666
|
+
)
|
608
667
|
|
609
668
|
return pc
|
610
669
|
|
@@ -637,31 +696,37 @@ class BaseGen(ABC):
|
|
637
696
|
return default
|
638
697
|
|
639
698
|
with Resource(res_file) as res:
|
640
|
-
if
|
699
|
+
if "wtk" in res_file.lower():
|
641
700
|
for dset in res.datasets:
|
642
|
-
if
|
701
|
+
if "speed" in dset:
|
643
702
|
# take nominal WTK chunks from windspeed
|
644
703
|
_, _, chunks = res.get_dset_properties(dset)
|
645
704
|
break
|
646
|
-
elif
|
705
|
+
elif "nsrdb" in res_file.lower():
|
647
706
|
# take nominal NSRDB chunks from dni
|
648
|
-
_, _, chunks = res.get_dset_properties(
|
707
|
+
_, _, chunks = res.get_dset_properties("dni")
|
649
708
|
else:
|
650
|
-
warn(
|
651
|
-
|
652
|
-
|
709
|
+
warn(
|
710
|
+
"Could not infer dataset chunk size as the resource type "
|
711
|
+
"could not be determined from the filename: {}".format(
|
712
|
+
res_file
|
713
|
+
)
|
714
|
+
)
|
653
715
|
chunks = None
|
654
716
|
|
655
717
|
if chunks is None:
|
656
718
|
# if chunks not set, go to default
|
657
719
|
sites_per_worker = default
|
658
|
-
logger.debug(
|
659
|
-
|
660
|
-
|
720
|
+
logger.debug(
|
721
|
+
"Sites per worker being set to {} (default) based on "
|
722
|
+
"no set chunk size in {}.".format(sites_per_worker, res_file)
|
723
|
+
)
|
661
724
|
else:
|
662
725
|
sites_per_worker = chunks[1]
|
663
|
-
logger.debug(
|
664
|
-
|
726
|
+
logger.debug(
|
727
|
+
"Sites per worker being set to {} based on chunk "
|
728
|
+
"size of {}.".format(sites_per_worker, res_file)
|
729
|
+
)
|
665
730
|
|
666
731
|
return sites_per_worker
|
667
732
|
|
@@ -688,8 +753,13 @@ class BaseGen(ABC):
|
|
688
753
|
|
689
754
|
@staticmethod
|
690
755
|
@abstractmethod
|
691
|
-
def _run_single_worker(
|
692
|
-
|
756
|
+
def _run_single_worker(
|
757
|
+
points_control,
|
758
|
+
tech=None,
|
759
|
+
res_file=None,
|
760
|
+
output_request=None,
|
761
|
+
scale_outputs=True,
|
762
|
+
):
|
693
763
|
"""Run a reV-SAM analysis based on the points_control iterator.
|
694
764
|
|
695
765
|
Parameters
|
@@ -735,31 +805,37 @@ class BaseGen(ABC):
|
|
735
805
|
if inp is None or inp is False:
|
736
806
|
# no input, just initialize dataframe with site gids as index
|
737
807
|
site_data = pd.DataFrame(index=self.project_points.sites)
|
738
|
-
site_data.index.name =
|
808
|
+
site_data.index.name = ResourceMetaField.GID
|
739
809
|
else:
|
740
810
|
# explicit input, initialize df
|
741
811
|
if isinstance(inp, str):
|
742
|
-
if inp.endswith(
|
812
|
+
if inp.endswith(".csv"):
|
743
813
|
site_data = pd.read_csv(inp)
|
744
814
|
elif isinstance(inp, pd.DataFrame):
|
745
815
|
site_data = inp
|
746
816
|
else:
|
747
817
|
# site data was not able to be set. Raise error.
|
748
|
-
raise Exception(
|
749
|
-
|
750
|
-
|
751
|
-
|
818
|
+
raise Exception(
|
819
|
+
"Site data input must be .csv or "
|
820
|
+
"dataframe, but received: {}".format(inp)
|
821
|
+
)
|
822
|
+
|
823
|
+
gid_not_in_site_data = ResourceMetaField.GID not in site_data
|
824
|
+
index_name_not_gid = site_data.index.name != ResourceMetaField.GID
|
825
|
+
if gid_not_in_site_data and index_name_not_gid:
|
752
826
|
# require gid as column label or index
|
753
|
-
raise KeyError('Site data input must have
|
754
|
-
'to match
|
827
|
+
raise KeyError('Site data input must have '
|
828
|
+
f'{ResourceMetaField.GID} column to match '
|
829
|
+
'reV site gid.')
|
755
830
|
|
756
831
|
# pylint: disable=no-member
|
757
|
-
if site_data.index.name !=
|
832
|
+
if site_data.index.name != ResourceMetaField.GID:
|
758
833
|
# make gid the dataframe index if not already
|
759
|
-
site_data = site_data.set_index(
|
834
|
+
site_data = site_data.set_index(ResourceMetaField.GID,
|
835
|
+
drop=True)
|
760
836
|
|
761
|
-
if
|
762
|
-
if site_data[
|
837
|
+
if "offshore" in site_data:
|
838
|
+
if site_data["offshore"].sum() > 1:
|
763
839
|
w = ('Found offshore sites in econ site data input. '
|
764
840
|
'This functionality has been deprecated. '
|
765
841
|
'Please run the reV offshore module to '
|
@@ -824,23 +900,25 @@ class BaseGen(ABC):
|
|
824
900
|
|
825
901
|
def _get_data_shape_from_out_attrs(self, dset, n_sites):
|
826
902
|
"""Get data shape from ``OUT_ATTRS`` variable"""
|
827
|
-
if self.OUT_ATTRS[dset][
|
903
|
+
if self.OUT_ATTRS[dset]["type"] == "array":
|
828
904
|
return (len(self.time_index), n_sites)
|
829
905
|
return (n_sites,)
|
830
906
|
|
831
907
|
def _get_data_shape_from_sam_config(self, dset, n_sites):
|
832
|
-
"""Get data shape from SAM input config
|
908
|
+
"""Get data shape from SAM input config"""
|
833
909
|
data = list(self.project_points.sam_inputs.values())[0][dset]
|
834
910
|
if isinstance(data, (list, tuple, np.ndarray)):
|
835
911
|
return (*np.array(data).shape, n_sites)
|
836
912
|
|
837
913
|
if isinstance(data, str):
|
838
|
-
msg = (
|
839
|
-
|
914
|
+
msg = (
|
915
|
+
'Cannot pass through non-scalar SAM input key "{}" '
|
916
|
+
"as an output_request!".format(dset)
|
917
|
+
)
|
840
918
|
logger.error(msg)
|
841
919
|
raise ExecutionError(msg)
|
842
920
|
|
843
|
-
return (n_sites,
|
921
|
+
return (n_sites,)
|
844
922
|
|
845
923
|
def _get_data_shape_from_pysam(self, dset, n_sites):
|
846
924
|
"""Get data shape from PySAM output object"""
|
@@ -850,10 +928,13 @@ class BaseGen(ABC):
|
|
850
928
|
try:
|
851
929
|
out_data = getattr(self._sam_obj_default.Outputs, dset)
|
852
930
|
except AttributeError as e:
|
853
|
-
msg = (
|
854
|
-
|
855
|
-
|
856
|
-
|
931
|
+
msg = (
|
932
|
+
'Could not get data shape for dset "{}" '
|
933
|
+
'from object "{}". '
|
934
|
+
'Received the following error: "{}"'.format(
|
935
|
+
dset, self._sam_obj_default, e
|
936
|
+
)
|
937
|
+
)
|
857
938
|
logger.error(msg)
|
858
939
|
raise ExecutionError(msg) from e
|
859
940
|
|
@@ -873,26 +954,27 @@ class BaseGen(ABC):
|
|
873
954
|
project_dir, out_fn = os.path.split(out_fpath)
|
874
955
|
|
875
956
|
# ensure output file is an h5
|
876
|
-
if not out_fn.endswith(
|
877
|
-
out_fn +=
|
957
|
+
if not out_fn.endswith(".h5"):
|
958
|
+
out_fn += ".h5"
|
878
959
|
|
879
960
|
if module not in out_fn:
|
880
961
|
extension_with_module = "_{}.h5".format(module)
|
881
962
|
out_fn = out_fn.replace(".h5", extension_with_module)
|
882
963
|
|
883
964
|
# ensure year is in out_fpath
|
884
|
-
if self.year is not None
|
965
|
+
if self.year is not None:
|
885
966
|
extension_with_year = "_{}.h5".format(self.year)
|
886
|
-
|
967
|
+
if extension_with_year not in out_fn:
|
968
|
+
out_fn = out_fn.replace(".h5", extension_with_year)
|
887
969
|
|
888
970
|
# create and use optional output dir
|
889
971
|
if project_dir and not os.path.exists(project_dir):
|
890
972
|
os.makedirs(project_dir, exist_ok=True)
|
891
973
|
|
892
974
|
self._out_fpath = os.path.join(project_dir, out_fn)
|
893
|
-
self._run_attrs[
|
975
|
+
self._run_attrs["out_fpath"] = out_fpath
|
894
976
|
|
895
|
-
def _init_h5(self, mode=
|
977
|
+
def _init_h5(self, mode="w"):
|
896
978
|
"""Initialize the single h5 output file with all output requests.
|
897
979
|
|
898
980
|
Parameters
|
@@ -904,12 +986,18 @@ class BaseGen(ABC):
|
|
904
986
|
if self._out_fpath is None:
|
905
987
|
return
|
906
988
|
|
907
|
-
if
|
908
|
-
logger.info(
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
989
|
+
if "w" in mode:
|
990
|
+
logger.info(
|
991
|
+
'Initializing full output file: "{}" with mode: {}'.format(
|
992
|
+
self._out_fpath, mode
|
993
|
+
)
|
994
|
+
)
|
995
|
+
elif "a" in mode:
|
996
|
+
logger.info(
|
997
|
+
'Appending data to output file: "{}" with mode: {}'.format(
|
998
|
+
self._out_fpath, mode
|
999
|
+
)
|
1000
|
+
)
|
913
1001
|
|
914
1002
|
attrs = {d: {} for d in self.output_request}
|
915
1003
|
chunks = {}
|
@@ -920,17 +1008,16 @@ class BaseGen(ABC):
|
|
920
1008
|
write_ti = False
|
921
1009
|
|
922
1010
|
for dset in self.output_request:
|
923
|
-
|
924
|
-
tmp = 'other'
|
1011
|
+
tmp = "other"
|
925
1012
|
if dset in self.OUT_ATTRS:
|
926
1013
|
tmp = dset
|
927
1014
|
|
928
|
-
attrs[dset][
|
929
|
-
|
930
|
-
|
931
|
-
|
932
|
-
chunks[dset] = self.OUT_ATTRS[tmp].get(
|
933
|
-
dtypes[dset] = self.OUT_ATTRS[tmp].get(
|
1015
|
+
attrs[dset]["units"] = self.OUT_ATTRS[tmp].get("units", "unknown")
|
1016
|
+
attrs[dset]["scale_factor"] = self.OUT_ATTRS[tmp].get(
|
1017
|
+
"scale_factor", 1
|
1018
|
+
)
|
1019
|
+
chunks[dset] = self.OUT_ATTRS[tmp].get("chunks", None)
|
1020
|
+
dtypes[dset] = self.OUT_ATTRS[tmp].get("dtype", "float32")
|
934
1021
|
shapes[dset] = self._get_data_shape(dset, len(self.meta))
|
935
1022
|
if len(shapes[dset]) > 1:
|
936
1023
|
write_ti = True
|
@@ -941,10 +1028,19 @@ class BaseGen(ABC):
|
|
941
1028
|
else:
|
942
1029
|
ti = None
|
943
1030
|
|
944
|
-
Outputs.init_h5(
|
945
|
-
|
946
|
-
|
947
|
-
|
1031
|
+
Outputs.init_h5(
|
1032
|
+
self._out_fpath,
|
1033
|
+
self.output_request,
|
1034
|
+
shapes,
|
1035
|
+
attrs,
|
1036
|
+
chunks,
|
1037
|
+
dtypes,
|
1038
|
+
self.meta,
|
1039
|
+
time_index=ti,
|
1040
|
+
configs=self.sam_metas,
|
1041
|
+
run_attrs=self.run_attrs,
|
1042
|
+
mode=mode,
|
1043
|
+
)
|
948
1044
|
|
949
1045
|
def _init_out_arrays(self, index_0=0):
|
950
1046
|
"""Initialize output arrays based on the number of sites that can be
|
@@ -962,21 +1058,27 @@ class BaseGen(ABC):
|
|
962
1058
|
self._finished_sites = []
|
963
1059
|
|
964
1060
|
# Output chunk is the index range (inclusive) of this set of site outs
|
965
|
-
self._out_chunk = (
|
966
|
-
|
1061
|
+
self._out_chunk = (
|
1062
|
+
index_0,
|
1063
|
+
np.min((index_0 + self.site_limit, len(self.project_points) - 1)),
|
1064
|
+
)
|
967
1065
|
self._out_n_sites = int(self.out_chunk[1] - self.out_chunk[0]) + 1
|
968
1066
|
|
969
|
-
logger.info(
|
970
|
-
|
971
|
-
|
972
|
-
|
973
|
-
|
974
|
-
|
1067
|
+
logger.info(
|
1068
|
+
"Initializing in-memory outputs for {} sites with gids "
|
1069
|
+
"{} through {} inclusive (site list index {} through {})".format(
|
1070
|
+
self._out_n_sites,
|
1071
|
+
self.project_points.sites[self.out_chunk[0]],
|
1072
|
+
self.project_points.sites[self.out_chunk[1]],
|
1073
|
+
self.out_chunk[0],
|
1074
|
+
self.out_chunk[1],
|
1075
|
+
)
|
1076
|
+
)
|
975
1077
|
|
976
1078
|
for request in self.output_request:
|
977
|
-
dtype =
|
1079
|
+
dtype = "float32"
|
978
1080
|
if request in self.OUT_ATTRS and self.scale_outputs:
|
979
|
-
dtype = self.OUT_ATTRS[request].get(
|
1081
|
+
dtype = self.OUT_ATTRS[request].get("dtype", "float32")
|
980
1082
|
|
981
1083
|
shape = self._get_data_shape(request, self._out_n_sites)
|
982
1084
|
|
@@ -1004,9 +1106,11 @@ class BaseGen(ABC):
|
|
1004
1106
|
# iterate through the site results
|
1005
1107
|
for var, value in site_output.items():
|
1006
1108
|
if var not in self._out:
|
1007
|
-
raise KeyError(
|
1008
|
-
|
1009
|
-
|
1109
|
+
raise KeyError(
|
1110
|
+
'Tried to collect output variable "{}", but it '
|
1111
|
+
"was not yet initialized in the output "
|
1112
|
+
"dictionary."
|
1113
|
+
)
|
1010
1114
|
|
1011
1115
|
# get the index in the output array for the current site
|
1012
1116
|
i = self.site_index(site_gid, out_index=True)
|
@@ -1055,12 +1159,14 @@ class BaseGen(ABC):
|
|
1055
1159
|
else:
|
1056
1160
|
output_index = global_site_index - self.out_chunk[0]
|
1057
1161
|
if output_index < 0:
|
1058
|
-
raise ValueError(
|
1059
|
-
|
1060
|
-
|
1061
|
-
|
1062
|
-
|
1063
|
-
|
1162
|
+
raise ValueError(
|
1163
|
+
"Attempting to set output data for site with "
|
1164
|
+
"gid {} to global site index {}, which was "
|
1165
|
+
"already set based on the current output "
|
1166
|
+
"index chunk of {}".format(
|
1167
|
+
site_gid, global_site_index, self.out_chunk
|
1168
|
+
)
|
1169
|
+
)
|
1064
1170
|
|
1065
1171
|
return output_index
|
1066
1172
|
|
@@ -1075,15 +1181,17 @@ class BaseGen(ABC):
|
|
1075
1181
|
|
1076
1182
|
# handle output file request if file is specified and .out is not empty
|
1077
1183
|
if isinstance(self._out_fpath, str) and self._out:
|
1078
|
-
logger.info(
|
1079
|
-
|
1184
|
+
logger.info(
|
1185
|
+
'Flushing outputs to disk, target file: "{}"'.format(
|
1186
|
+
self._out_fpath
|
1187
|
+
)
|
1188
|
+
)
|
1080
1189
|
|
1081
1190
|
# get the slice of indices to write outputs to
|
1082
1191
|
islice = slice(self.out_chunk[0], self.out_chunk[1] + 1)
|
1083
1192
|
|
1084
1193
|
# open output file in append mode to add output results to
|
1085
|
-
with Outputs(self._out_fpath, mode=
|
1086
|
-
|
1194
|
+
with Outputs(self._out_fpath, mode="a") as f:
|
1087
1195
|
# iterate through all output requests writing each as a dataset
|
1088
1196
|
for dset, arr in self._out.items():
|
1089
1197
|
if len(arr.shape) == 1:
|
@@ -1093,7 +1201,7 @@ class BaseGen(ABC):
|
|
1093
1201
|
# write 2D array of profiles
|
1094
1202
|
f[dset, :, islice] = arr
|
1095
1203
|
|
1096
|
-
logger.debug(
|
1204
|
+
logger.debug("Flushed output successfully to disk.")
|
1097
1205
|
|
1098
1206
|
def _pre_split_pc(self, pool_size=None):
|
1099
1207
|
"""Pre-split project control iterator into sub chunks to further
|
@@ -1129,11 +1237,15 @@ class BaseGen(ABC):
|
|
1129
1237
|
if i_chunk:
|
1130
1238
|
pc_chunks.append(i_chunk)
|
1131
1239
|
|
1132
|
-
logger.debug(
|
1133
|
-
|
1134
|
-
|
1240
|
+
logger.debug(
|
1241
|
+
"Pre-splitting points control into {} chunks with the "
|
1242
|
+
"following chunk sizes: {}".format(
|
1243
|
+
len(pc_chunks), [len(x) for x in pc_chunks]
|
1244
|
+
)
|
1245
|
+
)
|
1135
1246
|
return N, pc_chunks
|
1136
1247
|
|
1248
|
+
# pylint: disable=unused-argument
|
1137
1249
|
def _reduce_kwargs(self, pc, **kwargs):
|
1138
1250
|
"""Placeholder for functions that need to reduce the global kwargs that
|
1139
1251
|
they send to workers to reduce memory footprint
|
@@ -1153,8 +1265,9 @@ class BaseGen(ABC):
|
|
1153
1265
|
"""
|
1154
1266
|
return kwargs
|
1155
1267
|
|
1156
|
-
def _parallel_run(
|
1157
|
-
|
1268
|
+
def _parallel_run(
|
1269
|
+
self, max_workers=None, pool_size=None, timeout=1800, **kwargs
|
1270
|
+
):
|
1158
1271
|
"""Execute parallel compute.
|
1159
1272
|
|
1160
1273
|
Parameters
|
@@ -1176,25 +1289,31 @@ class BaseGen(ABC):
|
|
1176
1289
|
pool_size = os.cpu_count() * 2
|
1177
1290
|
if max_workers is None:
|
1178
1291
|
max_workers = os.cpu_count()
|
1179
|
-
logger.info(
|
1180
|
-
|
1292
|
+
logger.info(
|
1293
|
+
"Running parallel execution with max_workers={}".format(
|
1294
|
+
max_workers
|
1295
|
+
)
|
1296
|
+
)
|
1181
1297
|
i = 0
|
1182
1298
|
N, pc_chunks = self._pre_split_pc(pool_size=pool_size)
|
1183
1299
|
for j, pc_chunk in enumerate(pc_chunks):
|
1184
|
-
logger.debug(
|
1185
|
-
|
1186
|
-
|
1300
|
+
logger.debug(
|
1301
|
+
"Starting process pool for points control "
|
1302
|
+
"iteration {} out of {}".format(j + 1, len(pc_chunks))
|
1303
|
+
)
|
1187
1304
|
|
1188
1305
|
failed_futures = False
|
1189
1306
|
chunks = {}
|
1190
1307
|
futures = []
|
1191
|
-
loggers = [__name__,
|
1192
|
-
with SpawnProcessPool(
|
1193
|
-
|
1308
|
+
loggers = [__name__, "reV.gen", "reV.econ", "reV"]
|
1309
|
+
with SpawnProcessPool(
|
1310
|
+
max_workers=max_workers, loggers=loggers
|
1311
|
+
) as exe:
|
1194
1312
|
for pc in pc_chunk:
|
1195
1313
|
pc_kwargs = self._reduce_kwargs(pc, **kwargs)
|
1196
|
-
future = exe.submit(
|
1197
|
-
|
1314
|
+
future = exe.submit(
|
1315
|
+
self._run_single_worker, pc, **pc_kwargs
|
1316
|
+
)
|
1198
1317
|
futures.append(future)
|
1199
1318
|
chunks[future] = pc
|
1200
1319
|
|
@@ -1205,24 +1324,32 @@ class BaseGen(ABC):
|
|
1205
1324
|
except TimeoutError:
|
1206
1325
|
failed_futures = True
|
1207
1326
|
sites = chunks[future].project_points.sites
|
1208
|
-
result = self._handle_failed_future(
|
1209
|
-
|
1327
|
+
result = self._handle_failed_future(
|
1328
|
+
future, i, sites, timeout
|
1329
|
+
)
|
1210
1330
|
|
1211
1331
|
self.out = result
|
1212
1332
|
|
1213
1333
|
mem = psutil.virtual_memory()
|
1214
|
-
m = (
|
1215
|
-
|
1216
|
-
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1334
|
+
m = (
|
1335
|
+
"Parallel run at iteration {0} out of {1}. "
|
1336
|
+
"Memory utilization is {2:.3f} GB out of {3:.3f} GB "
|
1337
|
+
"total ({4:.1f}% used, intended limit of {5:.1f}%)"
|
1338
|
+
.format(
|
1339
|
+
i,
|
1340
|
+
N,
|
1341
|
+
mem.used / 1e9,
|
1342
|
+
mem.total / 1e9,
|
1343
|
+
100 * mem.used / mem.total,
|
1344
|
+
100 * self.mem_util_lim,
|
1345
|
+
)
|
1346
|
+
)
|
1220
1347
|
logger.info(m)
|
1221
1348
|
|
1222
1349
|
if failed_futures:
|
1223
|
-
logger.info(
|
1350
|
+
logger.info("Forcing pool shutdown after failed futures.")
|
1224
1351
|
exe.shutdown(wait=False)
|
1225
|
-
logger.info(
|
1352
|
+
logger.info("Forced pool shutdown complete.")
|
1226
1353
|
|
1227
1354
|
self.flush()
|
1228
1355
|
|
@@ -1242,23 +1369,23 @@ class BaseGen(ABC):
|
|
1242
1369
|
before returning zeros.
|
1243
1370
|
"""
|
1244
1371
|
|
1245
|
-
w = (
|
1246
|
-
.format(i, timeout))
|
1372
|
+
w = ("Iteration {} hit the timeout limit of {} seconds! "
|
1373
|
+
"Passing zeros.".format(i, timeout))
|
1247
1374
|
logger.warning(w)
|
1248
1375
|
warn(w, OutputWarning)
|
1249
1376
|
|
1250
|
-
site_out =
|
1251
|
-
result =
|
1377
|
+
site_out = dict.fromkeys(self.output_request, 0)
|
1378
|
+
result = dict.fromkeys(sites, site_out)
|
1252
1379
|
|
1253
1380
|
try:
|
1254
1381
|
cancelled = future.cancel()
|
1255
1382
|
except Exception as e:
|
1256
|
-
w =
|
1383
|
+
w = "Could not cancel future! Received exception: {}".format(e)
|
1257
1384
|
logger.warning(w)
|
1258
1385
|
warn(w, ParallelExecutionWarning)
|
1259
1386
|
|
1260
1387
|
if not cancelled:
|
1261
|
-
w =
|
1388
|
+
w = "Could not cancel future!"
|
1262
1389
|
logger.warning(w)
|
1263
1390
|
warn(w, ParallelExecutionWarning)
|
1264
1391
|
|