NREL-reV 0.8.7__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/METADATA +13 -10
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/RECORD +43 -43
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/WHEEL +1 -1
- reV/SAM/SAM.py +217 -133
- reV/SAM/econ.py +18 -14
- reV/SAM/generation.py +611 -422
- reV/SAM/windbos.py +93 -79
- reV/bespoke/bespoke.py +681 -377
- reV/bespoke/cli_bespoke.py +2 -0
- reV/bespoke/place_turbines.py +187 -43
- reV/config/output_request.py +2 -1
- reV/config/project_points.py +218 -140
- reV/econ/econ.py +166 -114
- reV/econ/economies_of_scale.py +91 -45
- reV/generation/base.py +331 -184
- reV/generation/generation.py +326 -200
- reV/generation/output_attributes/lcoe_fcr_inputs.json +38 -3
- reV/handlers/__init__.py +0 -1
- reV/handlers/exclusions.py +16 -15
- reV/handlers/multi_year.py +57 -26
- reV/handlers/outputs.py +6 -5
- reV/handlers/transmission.py +44 -27
- reV/hybrids/hybrid_methods.py +30 -30
- reV/hybrids/hybrids.py +305 -189
- reV/nrwal/nrwal.py +262 -168
- reV/qa_qc/cli_qa_qc.py +14 -10
- reV/qa_qc/qa_qc.py +217 -119
- reV/qa_qc/summary.py +228 -146
- reV/rep_profiles/rep_profiles.py +349 -230
- reV/supply_curve/aggregation.py +349 -188
- reV/supply_curve/competitive_wind_farms.py +90 -48
- reV/supply_curve/exclusions.py +138 -85
- reV/supply_curve/extent.py +75 -50
- reV/supply_curve/points.py +735 -390
- reV/supply_curve/sc_aggregation.py +357 -248
- reV/supply_curve/supply_curve.py +604 -347
- reV/supply_curve/tech_mapping.py +144 -82
- reV/utilities/__init__.py +274 -16
- reV/utilities/pytest_utils.py +8 -4
- reV/version.py +1 -1
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/LICENSE +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/entry_points.txt +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/top_level.txt +0 -0
@@ -6,29 +6,35 @@ Created on Fri Jun 21 13:24:31 2019
|
|
6
6
|
|
7
7
|
@author: gbuster
|
8
8
|
"""
|
9
|
-
from concurrent.futures import as_completed
|
10
9
|
import logging
|
11
|
-
import numpy as np
|
12
|
-
import psutil
|
13
10
|
import os
|
14
|
-
|
11
|
+
from concurrent.futures import as_completed
|
15
12
|
from warnings import warn
|
16
13
|
|
14
|
+
import numpy as np
|
15
|
+
import pandas as pd
|
16
|
+
import psutil
|
17
|
+
from rex.multi_file_resource import MultiFileResource
|
18
|
+
from rex.resource import Resource
|
19
|
+
from rex.utilities.execution import SpawnProcessPool
|
20
|
+
|
17
21
|
from reV.generation.base import BaseGen
|
18
22
|
from reV.handlers.exclusions import ExclusionLayers
|
19
|
-
from reV.supply_curve.aggregation import (
|
20
|
-
|
23
|
+
from reV.supply_curve.aggregation import (
|
24
|
+
AbstractAggFileHandler,
|
25
|
+
Aggregation,
|
26
|
+
BaseAggregation,
|
27
|
+
)
|
21
28
|
from reV.supply_curve.exclusions import FrictionMask
|
22
29
|
from reV.supply_curve.extent import SupplyCurveExtent
|
23
30
|
from reV.supply_curve.points import GenerationSupplyCurvePoint
|
24
|
-
from reV.utilities
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
from rex.utilities.execution import SpawnProcessPool
|
31
|
+
from reV.utilities import ResourceMetaField, SupplyCurveField, log_versions
|
32
|
+
from reV.utilities.exceptions import (
|
33
|
+
EmptySupplyCurvePointError,
|
34
|
+
FileInputError,
|
35
|
+
InputWarning,
|
36
|
+
OutputWarning,
|
37
|
+
)
|
32
38
|
|
33
39
|
logger = logging.getLogger(__name__)
|
34
40
|
|
@@ -43,10 +49,19 @@ class SupplyCurveAggFileHandler(AbstractAggFileHandler):
|
|
43
49
|
- variable power density .csv (optional)
|
44
50
|
"""
|
45
51
|
|
46
|
-
def __init__(
|
47
|
-
|
48
|
-
|
49
|
-
|
52
|
+
def __init__(
|
53
|
+
self,
|
54
|
+
excl_fpath,
|
55
|
+
gen_fpath,
|
56
|
+
econ_fpath=None,
|
57
|
+
data_layers=None,
|
58
|
+
power_density=None,
|
59
|
+
excl_dict=None,
|
60
|
+
friction_fpath=None,
|
61
|
+
friction_dset=None,
|
62
|
+
area_filter_kernel="queen",
|
63
|
+
min_area=None,
|
64
|
+
):
|
50
65
|
"""
|
51
66
|
Parameters
|
52
67
|
----------
|
@@ -89,9 +104,12 @@ class SupplyCurveAggFileHandler(AbstractAggFileHandler):
|
|
89
104
|
min_area : float | None
|
90
105
|
Minimum required contiguous area filter in sq-km
|
91
106
|
"""
|
92
|
-
super().__init__(
|
93
|
-
|
94
|
-
|
107
|
+
super().__init__(
|
108
|
+
excl_fpath,
|
109
|
+
excl_dict=excl_dict,
|
110
|
+
area_filter_kernel=area_filter_kernel,
|
111
|
+
min_area=min_area,
|
112
|
+
)
|
95
113
|
|
96
114
|
self._gen = self._open_gen_econ_resource(gen_fpath, econ_fpath)
|
97
115
|
# pre-initialize the resource meta data
|
@@ -106,7 +124,7 @@ class SupplyCurveAggFileHandler(AbstractAggFileHandler):
|
|
106
124
|
self._friction_layer = FrictionMask(friction_fpath, friction_dset)
|
107
125
|
|
108
126
|
if not np.all(self._friction_layer.shape == self._excl.shape):
|
109
|
-
e = (
|
127
|
+
e = ("Friction layer shape {} must match exclusions shape {}!"
|
110
128
|
.format(self._friction_layer.shape, self._excl.shape))
|
111
129
|
logger.error(e)
|
112
130
|
raise FileInputError(e)
|
@@ -132,14 +150,15 @@ class SupplyCurveAggFileHandler(AbstractAggFileHandler):
|
|
132
150
|
"""
|
133
151
|
|
134
152
|
handler = None
|
135
|
-
is_gen_h5 = isinstance(gen_fpath, str) and gen_fpath.endswith(
|
136
|
-
is_econ_h5 = isinstance(econ_fpath, str) and econ_fpath.endswith(
|
153
|
+
is_gen_h5 = isinstance(gen_fpath, str) and gen_fpath.endswith(".h5")
|
154
|
+
is_econ_h5 = isinstance(econ_fpath, str) and econ_fpath.endswith(".h5")
|
137
155
|
|
138
156
|
if is_gen_h5 and not is_econ_h5:
|
139
157
|
handler = Resource(gen_fpath)
|
140
158
|
elif is_gen_h5 and is_econ_h5:
|
141
|
-
handler = MultiFileResource(
|
142
|
-
|
159
|
+
handler = MultiFileResource(
|
160
|
+
[gen_fpath, econ_fpath], check_files=True
|
161
|
+
)
|
143
162
|
|
144
163
|
return handler
|
145
164
|
|
@@ -149,20 +168,24 @@ class SupplyCurveAggFileHandler(AbstractAggFileHandler):
|
|
149
168
|
if isinstance(self._power_density, str):
|
150
169
|
self._pdf = self._power_density
|
151
170
|
|
152
|
-
if self._pdf.endswith(
|
171
|
+
if self._pdf.endswith(".csv"):
|
153
172
|
self._power_density = pd.read_csv(self._pdf)
|
154
|
-
if (
|
173
|
+
if (ResourceMetaField.GID in self._power_density
|
155
174
|
and 'power_density' in self._power_density):
|
156
|
-
self._power_density =
|
175
|
+
self._power_density = \
|
176
|
+
self._power_density.set_index(ResourceMetaField.GID)
|
157
177
|
else:
|
158
|
-
msg = ('Variable power density file must include "
|
178
|
+
msg = ('Variable power density file must include "{}" '
|
159
179
|
'and "power_density" columns, but received: {}'
|
160
|
-
.format(
|
180
|
+
.format(ResourceMetaField.GID,
|
181
|
+
self._power_density.columns.values))
|
161
182
|
logger.error(msg)
|
162
183
|
raise FileInputError(msg)
|
163
184
|
else:
|
164
|
-
msg = (
|
165
|
-
|
185
|
+
msg = (
|
186
|
+
"Variable power density file must be csv but received: "
|
187
|
+
"{}".format(self._pdf)
|
188
|
+
)
|
166
189
|
logger.error(msg)
|
167
190
|
raise FileInputError(msg)
|
168
191
|
|
@@ -231,7 +254,7 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
231
254
|
lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None,
|
232
255
|
power_density=None, friction_fpath=None, friction_dset=None,
|
233
256
|
cap_cost_scale=None, recalc_lcoe=True):
|
234
|
-
"""
|
257
|
+
r"""ReV supply curve points aggregation framework.
|
235
258
|
|
236
259
|
``reV`` supply curve aggregation combines a high-resolution
|
237
260
|
(e.g. 90m) exclusion dataset with a (typically) lower resolution
|
@@ -327,6 +350,13 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
327
350
|
"more_developable_land": {
|
328
351
|
"force_include_range": [5, 10]
|
329
352
|
},
|
353
|
+
"viewsheds": {
|
354
|
+
"exclude_values": 1,
|
355
|
+
"extent": {
|
356
|
+
"layer": "federal_parks",
|
357
|
+
"include_range": [1, 5]
|
358
|
+
}
|
359
|
+
}
|
330
360
|
...
|
331
361
|
}
|
332
362
|
|
@@ -650,15 +680,22 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
650
680
|
associated with all pixels with that unique value.
|
651
681
|
"""
|
652
682
|
log_versions(logger)
|
653
|
-
logger.info(
|
654
|
-
logger.debug(
|
655
|
-
logger.debug(
|
656
|
-
|
657
|
-
super().__init__(
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
683
|
+
logger.info("Initializing SupplyCurveAggregation...")
|
684
|
+
logger.debug("Exclusion filepath: {}".format(excl_fpath))
|
685
|
+
logger.debug("Exclusion dict: {}".format(excl_dict))
|
686
|
+
|
687
|
+
super().__init__(
|
688
|
+
excl_fpath,
|
689
|
+
tm_dset,
|
690
|
+
excl_dict=excl_dict,
|
691
|
+
area_filter_kernel=area_filter_kernel,
|
692
|
+
min_area=min_area,
|
693
|
+
resolution=resolution,
|
694
|
+
excl_area=excl_area,
|
695
|
+
res_fpath=res_fpath,
|
696
|
+
gids=gids,
|
697
|
+
pre_extract_inclusions=pre_extract_inclusions,
|
698
|
+
)
|
662
699
|
|
663
700
|
self._econ_fpath = econ_fpath
|
664
701
|
self._res_class_dset = res_class_dset
|
@@ -673,26 +710,23 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
673
710
|
self._data_layers = data_layers
|
674
711
|
self._recalc_lcoe = recalc_lcoe
|
675
712
|
|
676
|
-
logger.debug(
|
677
|
-
|
678
|
-
if self._cap_cost_scale is not None:
|
679
|
-
if self._h5_dsets is None:
|
680
|
-
self._h5_dsets = []
|
681
|
-
|
682
|
-
self._h5_dsets += list(BaseGen.LCOE_ARGS)
|
683
|
-
self._h5_dsets = list(set(self._h5_dsets))
|
713
|
+
logger.debug("Resource class bins: {}".format(self._res_class_bins))
|
684
714
|
|
685
715
|
if self._power_density is None:
|
686
|
-
msg = (
|
687
|
-
|
688
|
-
|
716
|
+
msg = (
|
717
|
+
"Supply curve aggregation power density not specified. "
|
718
|
+
"Will try to infer based on lookup table: {}".format(
|
719
|
+
GenerationSupplyCurvePoint.POWER_DENSITY
|
720
|
+
)
|
721
|
+
)
|
689
722
|
logger.warning(msg)
|
690
723
|
warn(msg, InputWarning)
|
691
724
|
|
692
725
|
self._check_data_layers()
|
693
726
|
|
694
|
-
def _check_data_layers(
|
695
|
-
|
727
|
+
def _check_data_layers(
|
728
|
+
self, methods=("mean", "max", "min", "mode", "sum", "category")
|
729
|
+
):
|
696
730
|
"""Run pre-flight checks on requested aggregation data layers.
|
697
731
|
|
698
732
|
Parameters
|
@@ -702,40 +736,49 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
702
736
|
"""
|
703
737
|
|
704
738
|
if self._data_layers is not None:
|
705
|
-
logger.debug(
|
739
|
+
logger.debug("Checking data layers...")
|
706
740
|
|
707
741
|
with ExclusionLayers(self._excl_fpath) as f:
|
708
742
|
shape_base = f.shape
|
709
743
|
|
710
744
|
for k, v in self._data_layers.items():
|
711
|
-
if
|
712
|
-
raise KeyError(
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
if
|
722
|
-
|
745
|
+
if "dset" not in v:
|
746
|
+
raise KeyError(
|
747
|
+
'Data aggregation "dset" data layer "{}" '
|
748
|
+
"must be specified.".format(k)
|
749
|
+
)
|
750
|
+
if "method" not in v:
|
751
|
+
raise KeyError(
|
752
|
+
'Data aggregation "method" data layer "{}" '
|
753
|
+
"must be specified.".format(k)
|
754
|
+
)
|
755
|
+
if v["method"].lower() not in methods:
|
756
|
+
raise ValueError(
|
757
|
+
"Cannot recognize data layer agg method: "
|
758
|
+
'"{}". Can only do: {}.'.format(v["method"], methods)
|
759
|
+
)
|
760
|
+
if "fpath" in v:
|
761
|
+
with ExclusionLayers(v["fpath"]) as f:
|
723
762
|
try:
|
724
763
|
mismatched_shapes = any(f.shape != shape_base)
|
725
764
|
except TypeError:
|
726
765
|
mismatched_shapes = f.shape != shape_base
|
727
766
|
if mismatched_shapes:
|
728
|
-
msg = (
|
729
|
-
|
730
|
-
|
731
|
-
|
767
|
+
msg = (
|
768
|
+
'Data shape of data layer "{}" is {}, '
|
769
|
+
"which does not match the baseline "
|
770
|
+
"exclusions shape {}.".format(
|
771
|
+
k, f.shape, shape_base
|
772
|
+
)
|
773
|
+
)
|
732
774
|
raise FileInputError(msg)
|
733
775
|
|
734
|
-
logger.debug(
|
776
|
+
logger.debug("Finished checking data layers.")
|
735
777
|
|
736
778
|
@staticmethod
|
737
|
-
def _get_res_gen_lcoe_data(
|
738
|
-
|
779
|
+
def _get_res_gen_lcoe_data(
|
780
|
+
gen, res_class_dset, res_class_bins, lcoe_dset
|
781
|
+
):
|
739
782
|
"""Extract the basic resource / generation / lcoe data to be used in
|
740
783
|
the aggregation process.
|
741
784
|
|
@@ -750,8 +793,6 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
750
793
|
res_class_bins : list | None
|
751
794
|
List of two-entry lists dictating the resource class bins.
|
752
795
|
None if no resource classes.
|
753
|
-
cf_dset : str
|
754
|
-
Dataset name from f_gen containing capacity factor mean values.
|
755
796
|
lcoe_dset : str
|
756
797
|
Dataset name from f_gen containing LCOE mean values.
|
757
798
|
|
@@ -761,24 +802,23 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
761
802
|
Extracted resource data from res_class_dset
|
762
803
|
res_class_bins : list
|
763
804
|
List of resouce class bin ranges.
|
764
|
-
cf_data : np.ndarray | None
|
765
|
-
Capacity factor data extracted from cf_dset in gen
|
766
805
|
lcoe_data : np.ndarray | None
|
767
806
|
LCOE data extracted from lcoe_dset in gen
|
768
807
|
"""
|
769
808
|
|
770
|
-
dset_list = (res_class_dset,
|
809
|
+
dset_list = (res_class_dset, lcoe_dset)
|
771
810
|
gen_dsets = [] if gen is None else gen.datasets
|
772
|
-
labels = (
|
773
|
-
temp = [None, None
|
811
|
+
labels = ("res_class_dset", "lcoe_dset")
|
812
|
+
temp = [None, None]
|
774
813
|
|
775
814
|
if isinstance(gen, Resource):
|
776
815
|
source_fps = [gen.h5_file]
|
777
816
|
elif isinstance(gen, MultiFileResource):
|
778
817
|
source_fps = gen._h5_files
|
779
818
|
else:
|
780
|
-
msg =
|
781
|
-
|
819
|
+
msg = 'Did not recognize gen object input of type "{}": {}'.format(
|
820
|
+
type(gen), gen
|
821
|
+
)
|
782
822
|
logger.error(msg)
|
783
823
|
raise TypeError(msg)
|
784
824
|
|
@@ -787,18 +827,21 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
787
827
|
_warn_about_large_datasets(gen, dset)
|
788
828
|
temp[i] = gen[dset]
|
789
829
|
elif dset not in gen_dsets and dset is not None:
|
790
|
-
w = (
|
791
|
-
|
792
|
-
|
830
|
+
w = (
|
831
|
+
'Could not find "{}" input as "{}" in source files: {}. '
|
832
|
+
"Available datasets: {}".format(
|
833
|
+
labels[i], dset, source_fps, gen_dsets
|
834
|
+
)
|
835
|
+
)
|
793
836
|
logger.warning(w)
|
794
837
|
warn(w, OutputWarning)
|
795
838
|
|
796
|
-
res_data,
|
839
|
+
res_data, lcoe_data = temp
|
797
840
|
|
798
841
|
if res_class_dset is None or res_class_bins is None:
|
799
842
|
res_class_bins = [None]
|
800
843
|
|
801
|
-
return res_data, res_class_bins,
|
844
|
+
return res_data, res_class_bins, lcoe_data
|
802
845
|
|
803
846
|
@staticmethod
|
804
847
|
def _get_extra_dsets(gen, h5_dsets):
|
@@ -825,73 +868,90 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
825
868
|
# look for the datasets required by the LCOE re-calculation and make
|
826
869
|
# lists of the missing datasets
|
827
870
|
gen_dsets = [] if gen is None else gen.datasets
|
828
|
-
lcoe_recalc_req = ('fixed_charge_rate',
|
829
|
-
'
|
871
|
+
lcoe_recalc_req = ('fixed_charge_rate',
|
872
|
+
'capital_cost',
|
873
|
+
'fixed_operating_cost',
|
874
|
+
'variable_operating_cost',
|
830
875
|
'system_capacity')
|
831
876
|
missing_lcoe_source = [k for k in lcoe_recalc_req
|
832
877
|
if k not in gen_dsets]
|
833
|
-
missing_lcoe_request = []
|
834
878
|
|
835
879
|
if isinstance(gen, Resource):
|
836
880
|
source_fps = [gen.h5_file]
|
837
881
|
elif isinstance(gen, MultiFileResource):
|
838
882
|
source_fps = gen._h5_files
|
839
883
|
else:
|
840
|
-
msg =
|
841
|
-
|
884
|
+
msg = 'Did not recognize gen object input of type "{}": {}'.format(
|
885
|
+
type(gen), gen
|
886
|
+
)
|
842
887
|
logger.error(msg)
|
843
888
|
raise TypeError(msg)
|
844
889
|
|
845
890
|
h5_dsets_data = None
|
846
891
|
if h5_dsets is not None:
|
847
|
-
missing_lcoe_request = [k for k in lcoe_recalc_req
|
848
|
-
if k not in h5_dsets]
|
849
892
|
|
850
893
|
if not isinstance(h5_dsets, (list, tuple)):
|
851
|
-
e = (
|
852
|
-
|
894
|
+
e = (
|
895
|
+
"Additional h5_dsets argument must be a list or tuple "
|
896
|
+
"but received: {} {}".format(type(h5_dsets), h5_dsets)
|
897
|
+
)
|
853
898
|
logger.error(e)
|
854
899
|
raise TypeError(e)
|
855
900
|
|
856
901
|
missing_h5_dsets = [k for k in h5_dsets if k not in gen_dsets]
|
857
902
|
if any(missing_h5_dsets):
|
858
|
-
msg = (
|
859
|
-
|
860
|
-
|
903
|
+
msg = (
|
904
|
+
'Could not find requested h5_dsets "{}" in '
|
905
|
+
"source files: {}. Available datasets: {}".format(
|
906
|
+
missing_h5_dsets, source_fps, gen_dsets
|
907
|
+
)
|
908
|
+
)
|
861
909
|
logger.error(msg)
|
862
910
|
raise FileInputError(msg)
|
863
911
|
|
864
912
|
h5_dsets_data = {dset: gen[dset] for dset in h5_dsets}
|
865
913
|
|
866
914
|
if any(missing_lcoe_source):
|
867
|
-
msg = (
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
warn(msg, InputWarning)
|
875
|
-
|
876
|
-
if any(missing_lcoe_request):
|
877
|
-
msg = ('It is strongly advised that you include the following '
|
878
|
-
'datasets in the h5_dsets request in order to re-calculate '
|
879
|
-
'the LCOE from the multi-year mean CF and AEP: {}'
|
880
|
-
.format(missing_lcoe_request))
|
915
|
+
msg = (
|
916
|
+
"Could not find the datasets in the gen source file that "
|
917
|
+
"are required to re-calculate the multi-year LCOE. If you "
|
918
|
+
"are running a multi-year job, it is strongly suggested "
|
919
|
+
"you pass through these datasets to re-calculate the LCOE "
|
920
|
+
"from the multi-year mean CF: {}".format(missing_lcoe_source)
|
921
|
+
)
|
881
922
|
logger.warning(msg)
|
882
923
|
warn(msg, InputWarning)
|
883
924
|
|
884
925
|
return h5_dsets_data
|
885
926
|
|
886
927
|
@classmethod
|
887
|
-
def run_serial(
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
|
928
|
+
def run_serial(
|
929
|
+
cls,
|
930
|
+
excl_fpath,
|
931
|
+
gen_fpath,
|
932
|
+
tm_dset,
|
933
|
+
gen_index,
|
934
|
+
econ_fpath=None,
|
935
|
+
excl_dict=None,
|
936
|
+
inclusion_mask=None,
|
937
|
+
area_filter_kernel="queen",
|
938
|
+
min_area=None,
|
939
|
+
resolution=64,
|
940
|
+
gids=None,
|
941
|
+
args=None,
|
942
|
+
res_class_dset=None,
|
943
|
+
res_class_bins=None,
|
944
|
+
cf_dset="cf_mean-means",
|
945
|
+
lcoe_dset="lcoe_fcr-means",
|
946
|
+
h5_dsets=None,
|
947
|
+
data_layers=None,
|
948
|
+
power_density=None,
|
949
|
+
friction_fpath=None,
|
950
|
+
friction_dset=None,
|
951
|
+
excl_area=None,
|
952
|
+
cap_cost_scale=None,
|
953
|
+
recalc_lcoe=True,
|
954
|
+
):
|
895
955
|
"""Standalone method to create agg summary - can be parallelized.
|
896
956
|
|
897
957
|
Parameters
|
@@ -993,7 +1053,6 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
993
1053
|
summary = []
|
994
1054
|
|
995
1055
|
with SupplyCurveExtent(excl_fpath, resolution=resolution) as sc:
|
996
|
-
points = sc.points
|
997
1056
|
exclusion_shape = sc.exclusions.shape
|
998
1057
|
if gids is None:
|
999
1058
|
gids = sc.valid_sc_points(tm_dset)
|
@@ -1002,34 +1061,38 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1002
1061
|
|
1003
1062
|
slice_lookup = sc.get_slice_lookup(gids)
|
1004
1063
|
|
1005
|
-
logger.debug(
|
1006
|
-
|
1064
|
+
logger.debug(
|
1065
|
+
"Starting SupplyCurveAggregation serial with "
|
1066
|
+
"supply curve {} gids".format(len(gids))
|
1067
|
+
)
|
1007
1068
|
|
1008
1069
|
cls._check_inclusion_mask(inclusion_mask, gids, exclusion_shape)
|
1009
1070
|
|
1010
1071
|
# pre-extract handlers so they are not repeatedly initialized
|
1011
|
-
file_kwargs = {
|
1012
|
-
|
1013
|
-
|
1014
|
-
|
1015
|
-
|
1016
|
-
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1022
|
-
|
1023
|
-
|
1024
|
-
|
1025
|
-
|
1072
|
+
file_kwargs = {
|
1073
|
+
"econ_fpath": econ_fpath,
|
1074
|
+
"data_layers": data_layers,
|
1075
|
+
"power_density": power_density,
|
1076
|
+
"excl_dict": excl_dict,
|
1077
|
+
"area_filter_kernel": area_filter_kernel,
|
1078
|
+
"min_area": min_area,
|
1079
|
+
"friction_fpath": friction_fpath,
|
1080
|
+
"friction_dset": friction_dset,
|
1081
|
+
}
|
1082
|
+
with SupplyCurveAggFileHandler(
|
1083
|
+
excl_fpath, gen_fpath, **file_kwargs
|
1084
|
+
) as fh:
|
1085
|
+
temp = cls._get_res_gen_lcoe_data(
|
1086
|
+
fh.gen, res_class_dset, res_class_bins, lcoe_dset
|
1087
|
+
)
|
1088
|
+
res_data, res_class_bins, lcoe_data = temp
|
1026
1089
|
h5_dsets_data = cls._get_extra_dsets(fh.gen, h5_dsets)
|
1027
1090
|
|
1028
1091
|
n_finished = 0
|
1029
1092
|
for gid in gids:
|
1030
1093
|
gid_inclusions = cls._get_gid_inclusion_mask(
|
1031
|
-
inclusion_mask, gid, slice_lookup,
|
1032
|
-
|
1094
|
+
inclusion_mask, gid, slice_lookup, resolution=resolution
|
1095
|
+
)
|
1033
1096
|
|
1034
1097
|
for ri, res_bin in enumerate(res_class_bins):
|
1035
1098
|
try:
|
@@ -1041,7 +1104,7 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1041
1104
|
gen_index,
|
1042
1105
|
res_class_dset=res_data,
|
1043
1106
|
res_class_bin=res_bin,
|
1044
|
-
cf_dset=
|
1107
|
+
cf_dset=cf_dset,
|
1045
1108
|
lcoe_dset=lcoe_data,
|
1046
1109
|
h5_dsets=h5_dsets_data,
|
1047
1110
|
data_layers=fh.data_layers,
|
@@ -1055,27 +1118,29 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1055
1118
|
close=False,
|
1056
1119
|
friction_layer=fh.friction_layer,
|
1057
1120
|
cap_cost_scale=cap_cost_scale,
|
1058
|
-
recalc_lcoe=recalc_lcoe
|
1121
|
+
recalc_lcoe=recalc_lcoe,
|
1122
|
+
)
|
1059
1123
|
|
1060
1124
|
except EmptySupplyCurvePointError:
|
1061
|
-
logger.debug(
|
1125
|
+
logger.debug("SC point {} is empty".format(gid))
|
1062
1126
|
else:
|
1063
|
-
pointsum['sc_point_gid'] = gid
|
1064
|
-
pointsum['sc_row_ind'] = points.loc[gid, 'row_ind']
|
1065
|
-
pointsum['sc_col_ind'] = points.loc[gid, 'col_ind']
|
1066
1127
|
pointsum['res_class'] = ri
|
1067
1128
|
|
1068
1129
|
summary.append(pointsum)
|
1069
|
-
logger.debug(
|
1070
|
-
|
1071
|
-
|
1130
|
+
logger.debug(
|
1131
|
+
"Serial aggregation completed gid {}: "
|
1132
|
+
"{} out of {} points complete".format(
|
1133
|
+
gid, n_finished, len(gids)
|
1134
|
+
)
|
1135
|
+
)
|
1072
1136
|
|
1073
1137
|
n_finished += 1
|
1074
1138
|
|
1075
1139
|
return summary
|
1076
1140
|
|
1077
|
-
def run_parallel(
|
1078
|
-
|
1141
|
+
def run_parallel(
|
1142
|
+
self, gen_fpath, args=None, max_workers=None, sites_per_worker=100
|
1143
|
+
):
|
1079
1144
|
"""Get the supply curve points aggregation summary using futures.
|
1080
1145
|
|
1081
1146
|
Parameters
|
@@ -1101,25 +1166,31 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1101
1166
|
chunks = int(np.ceil(len(self.gids) / sites_per_worker))
|
1102
1167
|
chunks = np.array_split(self.gids, chunks)
|
1103
1168
|
|
1104
|
-
logger.info(
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1169
|
+
logger.info(
|
1170
|
+
"Running supply curve point aggregation for "
|
1171
|
+
"points {} through {} at a resolution of {} "
|
1172
|
+
"on {} cores in {} chunks.".format(
|
1173
|
+
self.gids[0],
|
1174
|
+
self.gids[-1],
|
1175
|
+
self._resolution,
|
1176
|
+
max_workers,
|
1177
|
+
len(chunks),
|
1178
|
+
)
|
1179
|
+
)
|
1109
1180
|
|
1110
1181
|
slice_lookup = None
|
1111
1182
|
if self._inclusion_mask is not None:
|
1112
|
-
with SupplyCurveExtent(
|
1113
|
-
|
1183
|
+
with SupplyCurveExtent(
|
1184
|
+
self._excl_fpath, resolution=self._resolution
|
1185
|
+
) as sc:
|
1114
1186
|
assert sc.exclusions.shape == self._inclusion_mask.shape
|
1115
1187
|
slice_lookup = sc.get_slice_lookup(self.gids)
|
1116
1188
|
|
1117
1189
|
futures = []
|
1118
1190
|
summary = []
|
1119
1191
|
n_finished = 0
|
1120
|
-
loggers = [__name__,
|
1192
|
+
loggers = [__name__, "reV.supply_curve.point_summary", "reV"]
|
1121
1193
|
with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe:
|
1122
|
-
|
1123
1194
|
# iterate through split executions, submitting each to worker
|
1124
1195
|
for gid_set in chunks:
|
1125
1196
|
# submit executions and append to futures list
|
@@ -1130,30 +1201,35 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1130
1201
|
rs, cs = slice_lookup[gid]
|
1131
1202
|
chunk_incl_masks[gid] = self._inclusion_mask[rs, cs]
|
1132
1203
|
|
1133
|
-
futures.append(
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1204
|
+
futures.append(
|
1205
|
+
exe.submit(
|
1206
|
+
self.run_serial,
|
1207
|
+
self._excl_fpath,
|
1208
|
+
gen_fpath,
|
1209
|
+
self._tm_dset,
|
1210
|
+
gen_index,
|
1211
|
+
econ_fpath=self._econ_fpath,
|
1212
|
+
excl_dict=self._excl_dict,
|
1213
|
+
inclusion_mask=chunk_incl_masks,
|
1214
|
+
res_class_dset=self._res_class_dset,
|
1215
|
+
res_class_bins=self._res_class_bins,
|
1216
|
+
cf_dset=self._cf_dset,
|
1217
|
+
lcoe_dset=self._lcoe_dset,
|
1218
|
+
h5_dsets=self._h5_dsets,
|
1219
|
+
data_layers=self._data_layers,
|
1220
|
+
resolution=self._resolution,
|
1221
|
+
power_density=self._power_density,
|
1222
|
+
friction_fpath=self._friction_fpath,
|
1223
|
+
friction_dset=self._friction_dset,
|
1224
|
+
area_filter_kernel=self._area_filter_kernel,
|
1225
|
+
min_area=self._min_area,
|
1226
|
+
gids=gid_set,
|
1227
|
+
args=args,
|
1228
|
+
excl_area=self._excl_area,
|
1229
|
+
cap_cost_scale=self._cap_cost_scale,
|
1230
|
+
recalc_lcoe=self._recalc_lcoe,
|
1231
|
+
)
|
1232
|
+
)
|
1157
1233
|
|
1158
1234
|
# gather results
|
1159
1235
|
for future in as_completed(futures):
|
@@ -1161,12 +1237,17 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1161
1237
|
summary += future.result()
|
1162
1238
|
if n_finished % 10 == 0:
|
1163
1239
|
mem = psutil.virtual_memory()
|
1164
|
-
logger.info(
|
1165
|
-
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1240
|
+
logger.info(
|
1241
|
+
"Parallel aggregation futures collected: "
|
1242
|
+
"{} out of {}. Memory usage is {:.3f} GB out "
|
1243
|
+
"of {:.3f} GB ({:.2f}% utilized).".format(
|
1244
|
+
n_finished,
|
1245
|
+
len(chunks),
|
1246
|
+
mem.used / 1e9,
|
1247
|
+
mem.total / 1e9,
|
1248
|
+
100 * mem.used / mem.total,
|
1249
|
+
)
|
1250
|
+
)
|
1170
1251
|
|
1171
1252
|
return summary
|
1172
1253
|
|
@@ -1194,17 +1275,18 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1194
1275
|
if all(type_check):
|
1195
1276
|
return bins
|
1196
1277
|
|
1197
|
-
|
1198
|
-
raise TypeError(
|
1199
|
-
|
1278
|
+
if any(type_check):
|
1279
|
+
raise TypeError(
|
1280
|
+
"Resource class bins has inconsistent "
|
1281
|
+
"entry type: {}".format(bins)
|
1282
|
+
)
|
1200
1283
|
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
bbins.append([b, bins[i + 1]])
|
1284
|
+
bbins = []
|
1285
|
+
for i, b in enumerate(sorted(bins)):
|
1286
|
+
if i < len(bins) - 1:
|
1287
|
+
bbins.append([b, bins[i + 1]])
|
1206
1288
|
|
1207
|
-
|
1289
|
+
return bbins
|
1208
1290
|
|
1209
1291
|
@staticmethod
|
1210
1292
|
def _summary_to_df(summary):
|
@@ -1221,15 +1303,17 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1221
1303
|
Summary of the SC points.
|
1222
1304
|
"""
|
1223
1305
|
summary = pd.DataFrame(summary)
|
1224
|
-
sort_by = [x for x in (
|
1306
|
+
sort_by = [x for x in (SupplyCurveField.SC_POINT_GID, 'res_class')
|
1307
|
+
if x in summary]
|
1225
1308
|
summary = summary.sort_values(sort_by)
|
1226
1309
|
summary = summary.reset_index(drop=True)
|
1227
|
-
summary.index.name =
|
1310
|
+
summary.index.name = SupplyCurveField.SC_GID
|
1228
1311
|
|
1229
1312
|
return summary
|
1230
1313
|
|
1231
|
-
def summarize(
|
1232
|
-
|
1314
|
+
def summarize(
|
1315
|
+
self, gen_fpath, args=None, max_workers=None, sites_per_worker=100
|
1316
|
+
):
|
1233
1317
|
"""
|
1234
1318
|
Get the supply curve points aggregation summary
|
1235
1319
|
|
@@ -1257,35 +1341,45 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1257
1341
|
if max_workers == 1:
|
1258
1342
|
gen_index = self._parse_gen_index(gen_fpath)
|
1259
1343
|
afk = self._area_filter_kernel
|
1260
|
-
summary = self.run_serial(
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1265
|
-
|
1266
|
-
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1270
|
-
|
1271
|
-
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1275
|
-
|
1276
|
-
|
1277
|
-
|
1278
|
-
|
1279
|
-
|
1280
|
-
|
1344
|
+
summary = self.run_serial(
|
1345
|
+
self._excl_fpath,
|
1346
|
+
gen_fpath,
|
1347
|
+
self._tm_dset,
|
1348
|
+
gen_index,
|
1349
|
+
econ_fpath=self._econ_fpath,
|
1350
|
+
excl_dict=self._excl_dict,
|
1351
|
+
inclusion_mask=self._inclusion_mask,
|
1352
|
+
res_class_dset=self._res_class_dset,
|
1353
|
+
res_class_bins=self._res_class_bins,
|
1354
|
+
cf_dset=self._cf_dset,
|
1355
|
+
lcoe_dset=self._lcoe_dset,
|
1356
|
+
h5_dsets=self._h5_dsets,
|
1357
|
+
data_layers=self._data_layers,
|
1358
|
+
resolution=self._resolution,
|
1359
|
+
power_density=self._power_density,
|
1360
|
+
friction_fpath=self._friction_fpath,
|
1361
|
+
friction_dset=self._friction_dset,
|
1362
|
+
area_filter_kernel=afk,
|
1363
|
+
min_area=self._min_area,
|
1364
|
+
gids=self.gids,
|
1365
|
+
args=args,
|
1366
|
+
excl_area=self._excl_area,
|
1367
|
+
cap_cost_scale=self._cap_cost_scale,
|
1368
|
+
recalc_lcoe=self._recalc_lcoe,
|
1369
|
+
)
|
1281
1370
|
else:
|
1282
|
-
summary = self.run_parallel(
|
1283
|
-
|
1284
|
-
|
1371
|
+
summary = self.run_parallel(
|
1372
|
+
gen_fpath=gen_fpath,
|
1373
|
+
args=args,
|
1374
|
+
max_workers=max_workers,
|
1375
|
+
sites_per_worker=sites_per_worker,
|
1376
|
+
)
|
1285
1377
|
|
1286
1378
|
if not any(summary):
|
1287
|
-
e = (
|
1288
|
-
|
1379
|
+
e = (
|
1380
|
+
"Supply curve aggregation found no non-excluded SC points. "
|
1381
|
+
"Please check your exclusions or subset SC GID selection."
|
1382
|
+
)
|
1289
1383
|
logger.error(e)
|
1290
1384
|
raise EmptySupplyCurvePointError(e)
|
1291
1385
|
|
@@ -1293,8 +1387,14 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1293
1387
|
|
1294
1388
|
return summary
|
1295
1389
|
|
1296
|
-
def run(
|
1297
|
-
|
1390
|
+
def run(
|
1391
|
+
self,
|
1392
|
+
out_fpath,
|
1393
|
+
gen_fpath=None,
|
1394
|
+
args=None,
|
1395
|
+
max_workers=None,
|
1396
|
+
sites_per_worker=100,
|
1397
|
+
):
|
1298
1398
|
"""Run a supply curve aggregation.
|
1299
1399
|
|
1300
1400
|
Parameters
|
@@ -1333,7 +1433,9 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1333
1433
|
|
1334
1434
|
if gen_fpath is None:
|
1335
1435
|
out = Aggregation.run(
|
1336
|
-
self._excl_fpath,
|
1436
|
+
self._excl_fpath,
|
1437
|
+
self._res_fpath,
|
1438
|
+
self._tm_dset,
|
1337
1439
|
excl_dict=self._excl_dict,
|
1338
1440
|
resolution=self._resolution,
|
1339
1441
|
excl_area=self._excl_area,
|
@@ -1341,12 +1443,16 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1341
1443
|
min_area=self._min_area,
|
1342
1444
|
pre_extract_inclusions=self._pre_extract_inclusions,
|
1343
1445
|
max_workers=max_workers,
|
1344
|
-
sites_per_worker=sites_per_worker
|
1345
|
-
|
1446
|
+
sites_per_worker=sites_per_worker,
|
1447
|
+
)
|
1448
|
+
summary = out["meta"]
|
1346
1449
|
else:
|
1347
|
-
summary = self.summarize(
|
1348
|
-
|
1349
|
-
|
1450
|
+
summary = self.summarize(
|
1451
|
+
gen_fpath=gen_fpath,
|
1452
|
+
args=args,
|
1453
|
+
max_workers=max_workers,
|
1454
|
+
sites_per_worker=sites_per_worker,
|
1455
|
+
)
|
1350
1456
|
|
1351
1457
|
out_fpath = _format_sc_agg_out_fpath(out_fpath)
|
1352
1458
|
summary.to_csv(out_fpath)
|
@@ -1357,11 +1463,12 @@ class SupplyCurveAggregation(BaseAggregation):
|
|
1357
1463
|
def _format_sc_agg_out_fpath(out_fpath):
|
1358
1464
|
"""Add CSV file ending and replace underscore, if necessary."""
|
1359
1465
|
if not out_fpath.endswith(".csv"):
|
1360
|
-
out_fpath =
|
1466
|
+
out_fpath = "{}.csv".format(out_fpath)
|
1361
1467
|
|
1362
1468
|
project_dir, out_fn = os.path.split(out_fpath)
|
1363
|
-
out_fn = out_fn.replace(
|
1364
|
-
|
1469
|
+
out_fn = out_fn.replace(
|
1470
|
+
"supply_curve_aggregation", "supply-curve-aggregation"
|
1471
|
+
)
|
1365
1472
|
return os.path.join(project_dir, out_fn)
|
1366
1473
|
|
1367
1474
|
|
@@ -1369,9 +1476,11 @@ def _warn_about_large_datasets(gen, dset):
|
|
1369
1476
|
"""Warn user about multi-dimensional datasets in passthrough datasets"""
|
1370
1477
|
dset_shape = gen.shapes.get(dset, (1,))
|
1371
1478
|
if len(dset_shape) > 1:
|
1372
|
-
msg = (
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1479
|
+
msg = (
|
1480
|
+
"Generation dataset {!r} is not 1-dimensional (shape: {})."
|
1481
|
+
"You may run into memory errors during aggregation - use "
|
1482
|
+
"rep-profiles for aggregating higher-order datasets instead!"
|
1483
|
+
.format(dset, dset_shape)
|
1484
|
+
)
|
1376
1485
|
logger.warning(msg)
|
1377
1486
|
warn(msg, UserWarning)
|