NREL-reV 0.8.7__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/METADATA +13 -10
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/RECORD +43 -43
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/WHEEL +1 -1
- reV/SAM/SAM.py +217 -133
- reV/SAM/econ.py +18 -14
- reV/SAM/generation.py +611 -422
- reV/SAM/windbos.py +93 -79
- reV/bespoke/bespoke.py +681 -377
- reV/bespoke/cli_bespoke.py +2 -0
- reV/bespoke/place_turbines.py +187 -43
- reV/config/output_request.py +2 -1
- reV/config/project_points.py +218 -140
- reV/econ/econ.py +166 -114
- reV/econ/economies_of_scale.py +91 -45
- reV/generation/base.py +331 -184
- reV/generation/generation.py +326 -200
- reV/generation/output_attributes/lcoe_fcr_inputs.json +38 -3
- reV/handlers/__init__.py +0 -1
- reV/handlers/exclusions.py +16 -15
- reV/handlers/multi_year.py +57 -26
- reV/handlers/outputs.py +6 -5
- reV/handlers/transmission.py +44 -27
- reV/hybrids/hybrid_methods.py +30 -30
- reV/hybrids/hybrids.py +305 -189
- reV/nrwal/nrwal.py +262 -168
- reV/qa_qc/cli_qa_qc.py +14 -10
- reV/qa_qc/qa_qc.py +217 -119
- reV/qa_qc/summary.py +228 -146
- reV/rep_profiles/rep_profiles.py +349 -230
- reV/supply_curve/aggregation.py +349 -188
- reV/supply_curve/competitive_wind_farms.py +90 -48
- reV/supply_curve/exclusions.py +138 -85
- reV/supply_curve/extent.py +75 -50
- reV/supply_curve/points.py +735 -390
- reV/supply_curve/sc_aggregation.py +357 -248
- reV/supply_curve/supply_curve.py +604 -347
- reV/supply_curve/tech_mapping.py +144 -82
- reV/utilities/__init__.py +274 -16
- reV/utilities/pytest_utils.py +8 -4
- reV/version.py +1 -1
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/LICENSE +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/entry_points.txt +0 -0
- {NREL_reV-0.8.7.dist-info → NREL_reV-0.9.0.dist-info}/top_level.txt +0 -0
reV/rep_profiles/rep_profiles.py
CHANGED
@@ -5,26 +5,26 @@ Created on Thu Oct 31 12:49:23 2019
|
|
5
5
|
|
6
6
|
@author: gbuster
|
7
7
|
"""
|
8
|
-
|
9
|
-
from concurrent.futures import as_completed
|
10
|
-
from copy import deepcopy
|
8
|
+
import contextlib
|
11
9
|
import json
|
12
10
|
import logging
|
13
|
-
import numpy as np
|
14
11
|
import os
|
15
|
-
import
|
16
|
-
from
|
12
|
+
from abc import ABC, abstractmethod
|
13
|
+
from concurrent.futures import as_completed
|
14
|
+
from copy import deepcopy
|
17
15
|
from warnings import warn
|
18
16
|
|
19
|
-
|
20
|
-
|
21
|
-
from reV.utilities.exceptions import FileInputError, DataShapeError
|
22
|
-
from reV.utilities import log_versions
|
23
|
-
|
17
|
+
import numpy as np
|
18
|
+
import pandas as pd
|
24
19
|
from rex.resource import Resource
|
25
20
|
from rex.utilities.execution import SpawnProcessPool
|
26
21
|
from rex.utilities.loggers import log_mem
|
27
22
|
from rex.utilities.utilities import parse_year, to_records_array
|
23
|
+
from scipy import stats
|
24
|
+
|
25
|
+
from reV.handlers.outputs import Outputs
|
26
|
+
from reV.utilities import ResourceMetaField, SupplyCurveField, log_versions
|
27
|
+
from reV.utilities.exceptions import DataShapeError, FileInputError
|
28
28
|
|
29
29
|
logger = logging.getLogger(__name__)
|
30
30
|
|
@@ -32,8 +32,9 @@ logger = logging.getLogger(__name__)
|
|
32
32
|
class RepresentativeMethods:
|
33
33
|
"""Class for organizing the methods to determine representative-ness"""
|
34
34
|
|
35
|
-
def __init__(
|
36
|
-
|
35
|
+
def __init__(
|
36
|
+
self, profiles, weights=None, rep_method="meanoid", err_method="rmse"
|
37
|
+
):
|
37
38
|
"""
|
38
39
|
Parameters
|
39
40
|
----------
|
@@ -61,30 +62,35 @@ class RepresentativeMethods:
|
|
61
62
|
self._weights = np.array(self._weights)
|
62
63
|
|
63
64
|
if self._weights is not None:
|
64
|
-
emsg = (
|
65
|
-
|
66
|
-
|
65
|
+
emsg = (
|
66
|
+
"Weighting factors array of length {} does not match "
|
67
|
+
"profiles of shape {}".format(
|
68
|
+
len(self._weights), self._profiles.shape[1]
|
69
|
+
)
|
70
|
+
)
|
67
71
|
assert len(self._weights) == self._profiles.shape[1], emsg
|
68
72
|
|
69
73
|
@property
|
70
74
|
def rep_methods(self):
|
71
75
|
"""Lookup table of representative methods"""
|
72
|
-
methods = {
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
76
|
+
methods = {
|
77
|
+
"mean": self.meanoid,
|
78
|
+
"meanoid": self.meanoid,
|
79
|
+
"median": self.medianoid,
|
80
|
+
"medianoid": self.medianoid,
|
81
|
+
}
|
77
82
|
|
78
83
|
return methods
|
79
84
|
|
80
85
|
@property
|
81
86
|
def err_methods(self):
|
82
87
|
"""Lookup table of error methods"""
|
83
|
-
methods = {
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
+
methods = {
|
89
|
+
"mbe": self.mbe,
|
90
|
+
"mae": self.mae,
|
91
|
+
"rmse": self.rmse,
|
92
|
+
None: None,
|
93
|
+
}
|
88
94
|
|
89
95
|
return methods
|
90
96
|
|
@@ -105,7 +111,7 @@ class RepresentativeMethods:
|
|
105
111
|
i : int
|
106
112
|
Location of the Nth min value in arr.
|
107
113
|
"""
|
108
|
-
return arr.argsort()[:(n + 1)][-1]
|
114
|
+
return arr.argsort()[: (n + 1)][-1]
|
109
115
|
|
110
116
|
@staticmethod
|
111
117
|
def meanoid(profiles, weights=None):
|
@@ -241,8 +247,14 @@ class RepresentativeMethods:
|
|
241
247
|
return profiles[:, i_rep], i_rep
|
242
248
|
|
243
249
|
@classmethod
|
244
|
-
def run(
|
245
|
-
|
250
|
+
def run(
|
251
|
+
cls,
|
252
|
+
profiles,
|
253
|
+
weights=None,
|
254
|
+
rep_method="meanoid",
|
255
|
+
err_method="rmse",
|
256
|
+
n_profiles=1,
|
257
|
+
):
|
246
258
|
"""Run representative profile methods.
|
247
259
|
|
248
260
|
Parameters
|
@@ -270,8 +282,12 @@ class RepresentativeMethods:
|
|
270
282
|
representative profile(s). If err_method is None, this value is
|
271
283
|
also set to None.
|
272
284
|
"""
|
273
|
-
inst = cls(
|
274
|
-
|
285
|
+
inst = cls(
|
286
|
+
profiles,
|
287
|
+
weights=weights,
|
288
|
+
rep_method=rep_method,
|
289
|
+
err_method=err_method,
|
290
|
+
)
|
275
291
|
|
276
292
|
if inst._weights is not None:
|
277
293
|
baseline = inst._rep_method(inst._profiles, weights=inst._weights)
|
@@ -299,11 +315,12 @@ class RepresentativeMethods:
|
|
299
315
|
class RegionRepProfile:
|
300
316
|
"""Framework to handle rep profile for one resource region"""
|
301
317
|
|
302
|
-
RES_GID_COL =
|
303
|
-
GEN_GID_COL =
|
318
|
+
RES_GID_COL = SupplyCurveField.RES_GIDS
|
319
|
+
GEN_GID_COL = SupplyCurveField.GEN_GIDS
|
304
320
|
|
305
321
|
def __init__(self, gen_fpath, rev_summary, cf_dset='cf_profile',
|
306
|
-
rep_method='meanoid', err_method='rmse',
|
322
|
+
rep_method='meanoid', err_method='rmse',
|
323
|
+
weight=SupplyCurveField.GID_COUNTS,
|
307
324
|
n_profiles=1):
|
308
325
|
"""
|
309
326
|
Parameters
|
@@ -355,38 +372,46 @@ class RegionRepProfile:
|
|
355
372
|
|
356
373
|
self._weights = np.ones(len(res_gids))
|
357
374
|
if self._weight is not None:
|
358
|
-
self._weights = self._get_region_attr(
|
359
|
-
|
375
|
+
self._weights = self._get_region_attr(
|
376
|
+
self._rev_summary, self._weight
|
377
|
+
)
|
360
378
|
|
361
|
-
df = pd.DataFrame(
|
362
|
-
|
363
|
-
|
379
|
+
df = pd.DataFrame(
|
380
|
+
{
|
381
|
+
self.GEN_GID_COL: gen_gids,
|
382
|
+
self.RES_GID_COL: res_gids,
|
383
|
+
"weights": self._weights,
|
384
|
+
}
|
385
|
+
)
|
364
386
|
df = df.sort_values(self.RES_GID_COL)
|
365
387
|
self._gen_gids = df[self.GEN_GID_COL].values
|
366
388
|
self._res_gids = df[self.RES_GID_COL].values
|
367
389
|
if self._weight is not None:
|
368
|
-
self._weights = df[
|
390
|
+
self._weights = df["weights"].values
|
369
391
|
else:
|
370
392
|
self._weights = None
|
371
393
|
|
372
394
|
with Resource(self._gen_fpath) as res:
|
373
395
|
meta = res.meta
|
374
396
|
|
375
|
-
assert
|
376
|
-
source_res_gids = meta[
|
397
|
+
assert ResourceMetaField.GID in meta
|
398
|
+
source_res_gids = meta[ResourceMetaField.GID].values
|
377
399
|
msg = ('Resource gids from "gid" column in meta data from "{}" '
|
378
400
|
'must be sorted! reV generation should always be run with '
|
379
401
|
'sequential project points.'.format(self._gen_fpath))
|
380
402
|
assert np.all(source_res_gids[:-1] <= source_res_gids[1:]), msg
|
381
403
|
|
382
404
|
missing = set(self._res_gids) - set(source_res_gids)
|
383
|
-
msg = (
|
384
|
-
|
385
|
-
|
405
|
+
msg = (
|
406
|
+
"The following resource gids were found in the rev summary "
|
407
|
+
"supply curve file but not in the source generation meta "
|
408
|
+
"data: {}".format(missing)
|
409
|
+
)
|
386
410
|
assert not any(missing), msg
|
387
411
|
|
388
|
-
unique_res_gids, u_idxs = np.unique(
|
389
|
-
|
412
|
+
unique_res_gids, u_idxs = np.unique(
|
413
|
+
self._res_gids, return_inverse=True
|
414
|
+
)
|
390
415
|
iloc = np.where(np.isin(source_res_gids, unique_res_gids))[0]
|
391
416
|
self._source_profiles = res[self._cf_dset, :, iloc[u_idxs]]
|
392
417
|
|
@@ -441,7 +466,7 @@ class RegionRepProfile:
|
|
441
466
|
if any(data):
|
442
467
|
if isinstance(data[0], str):
|
443
468
|
# pylint: disable=simplifiable-condition
|
444
|
-
if (
|
469
|
+
if ("[" and "]" in data[0]) or ("(" and ")" in data[0]):
|
445
470
|
data = [json.loads(s) for s in data]
|
446
471
|
|
447
472
|
if isinstance(data[0], (list, tuple)):
|
@@ -453,27 +478,32 @@ class RegionRepProfile:
|
|
453
478
|
"""Run the representative profile methods to find the meanoid/medianoid
|
454
479
|
profile and find the profiles most similar."""
|
455
480
|
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
481
|
+
num_profiles = self.source_profiles.shape[1]
|
482
|
+
bad_data_shape = (self.weights is not None
|
483
|
+
and len(self.weights) != num_profiles)
|
484
|
+
if bad_data_shape:
|
485
|
+
e = ('Weights column "{}" resulted in {} weight scalars '
|
486
|
+
'which doesnt match gid column which yields '
|
487
|
+
'profiles with shape {}.'
|
488
|
+
.format(self._weight, len(self.weights),
|
489
|
+
self.source_profiles.shape))
|
490
|
+
logger.debug('Gids from column "res_gids" with len {}: {}'
|
491
|
+
.format(len(self._res_gids), self._res_gids))
|
492
|
+
logger.debug('Weights from column "{}" with len {}: {}'
|
493
|
+
.format(self._weight, len(self.weights),
|
494
|
+
self.weights))
|
495
|
+
logger.error(e)
|
496
|
+
raise DataShapeError(e)
|
470
497
|
|
471
498
|
self._profiles, self._i_reps = RepresentativeMethods.run(
|
472
|
-
self.source_profiles,
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
499
|
+
self.source_profiles,
|
500
|
+
weights=self.weights,
|
501
|
+
rep_method=self._rep_method,
|
502
|
+
err_method=self._err_method,
|
503
|
+
n_profiles=self._n_profiles,
|
504
|
+
)
|
505
|
+
|
506
|
+
@ property
|
477
507
|
def rep_profiles(self):
|
478
508
|
"""Get the representative profiles of this region."""
|
479
509
|
if self._profiles is None:
|
@@ -481,7 +511,7 @@ class RegionRepProfile:
|
|
481
511
|
|
482
512
|
return self._profiles
|
483
513
|
|
484
|
-
@property
|
514
|
+
@ property
|
485
515
|
def i_reps(self):
|
486
516
|
"""Get the representative profile index(es) of this region."""
|
487
517
|
if self._i_reps is None:
|
@@ -489,7 +519,7 @@ class RegionRepProfile:
|
|
489
519
|
|
490
520
|
return self._i_reps
|
491
521
|
|
492
|
-
@property
|
522
|
+
@ property
|
493
523
|
def rep_gen_gids(self):
|
494
524
|
"""Get the representative profile gen gids of this region."""
|
495
525
|
gids = self._gen_gids
|
@@ -500,7 +530,7 @@ class RegionRepProfile:
|
|
500
530
|
|
501
531
|
return rep_gids
|
502
532
|
|
503
|
-
@property
|
533
|
+
@ property
|
504
534
|
def rep_res_gids(self):
|
505
535
|
"""Get the representative profile resource gids of this region."""
|
506
536
|
gids = self._res_gids
|
@@ -511,10 +541,12 @@ class RegionRepProfile:
|
|
511
541
|
|
512
542
|
return rep_gids
|
513
543
|
|
514
|
-
@classmethod
|
544
|
+
@ classmethod
|
515
545
|
def get_region_rep_profile(cls, gen_fpath, rev_summary,
|
516
|
-
cf_dset='cf_profile',
|
517
|
-
|
546
|
+
cf_dset='cf_profile',
|
547
|
+
rep_method='meanoid',
|
548
|
+
err_method='rmse',
|
549
|
+
weight=SupplyCurveField.GID_COUNTS,
|
518
550
|
n_profiles=1):
|
519
551
|
"""Class method for parallelization of rep profile calc.
|
520
552
|
|
@@ -554,9 +586,15 @@ class RegionRepProfile:
|
|
554
586
|
res_gid_reps : list
|
555
587
|
Resource gid(s) of the representative profile(s).
|
556
588
|
"""
|
557
|
-
r = cls(
|
558
|
-
|
559
|
-
|
589
|
+
r = cls(
|
590
|
+
gen_fpath,
|
591
|
+
rev_summary,
|
592
|
+
cf_dset=cf_dset,
|
593
|
+
rep_method=rep_method,
|
594
|
+
err_method=err_method,
|
595
|
+
weight=weight,
|
596
|
+
n_profiles=n_profiles,
|
597
|
+
)
|
560
598
|
|
561
599
|
return r.rep_profiles, r.i_reps, r.rep_gen_gids, r.rep_res_gids
|
562
600
|
|
@@ -565,8 +603,9 @@ class RepProfilesBase(ABC):
|
|
565
603
|
"""Abstract utility framework for representative profile run classes."""
|
566
604
|
|
567
605
|
def __init__(self, gen_fpath, rev_summary, reg_cols=None,
|
568
|
-
cf_dset='cf_profile', rep_method='meanoid',
|
569
|
-
|
606
|
+
cf_dset='cf_profile', rep_method='meanoid',
|
607
|
+
err_method='rmse', weight=SupplyCurveField.GID_COUNTS,
|
608
|
+
n_profiles=1):
|
570
609
|
"""
|
571
610
|
Parameters
|
572
611
|
----------
|
@@ -597,18 +636,26 @@ class RepProfilesBase(ABC):
|
|
597
636
|
Number of representative profiles to save to fout.
|
598
637
|
"""
|
599
638
|
|
600
|
-
logger.info(
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
logger.info(
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
639
|
+
logger.info(
|
640
|
+
'Running rep profiles with gen_fpath: "{}"'.format(gen_fpath)
|
641
|
+
)
|
642
|
+
logger.info(
|
643
|
+
'Running rep profiles with rev_summary: "{}"'.format(rev_summary)
|
644
|
+
)
|
645
|
+
logger.info(
|
646
|
+
'Running rep profiles with region columns: "{}"'.format(reg_cols)
|
647
|
+
)
|
648
|
+
logger.info(
|
649
|
+
'Running rep profiles with representative method: "{}"'.format(
|
650
|
+
rep_method
|
651
|
+
)
|
652
|
+
)
|
653
|
+
logger.info(
|
654
|
+
'Running rep profiles with error method: "{}"'.format(err_method)
|
655
|
+
)
|
656
|
+
logger.info(
|
657
|
+
'Running rep profiles with weight factor: "{}"'.format(weight)
|
658
|
+
)
|
612
659
|
|
613
660
|
self._weight = weight
|
614
661
|
self._n_profiles = n_profiles
|
@@ -630,7 +677,7 @@ class RepProfilesBase(ABC):
|
|
630
677
|
self._rep_method = rep_method
|
631
678
|
self._err_method = err_method
|
632
679
|
|
633
|
-
@staticmethod
|
680
|
+
@ staticmethod
|
634
681
|
def _parse_rev_summary(rev_summary):
|
635
682
|
"""Extract, parse, and check the rev summary table.
|
636
683
|
|
@@ -650,23 +697,24 @@ class RepProfilesBase(ABC):
|
|
650
697
|
"""
|
651
698
|
|
652
699
|
if isinstance(rev_summary, str):
|
653
|
-
if os.path.exists(rev_summary) and rev_summary.endswith(
|
700
|
+
if os.path.exists(rev_summary) and rev_summary.endswith(".csv"):
|
654
701
|
rev_summary = pd.read_csv(rev_summary)
|
655
|
-
elif os.path.exists(rev_summary) and rev_summary.endswith(
|
702
|
+
elif os.path.exists(rev_summary) and rev_summary.endswith(".json"):
|
656
703
|
rev_summary = pd.read_json(rev_summary)
|
657
704
|
else:
|
658
|
-
e =
|
705
|
+
e = "Could not parse reV summary file: {}".format(rev_summary)
|
659
706
|
logger.error(e)
|
660
707
|
raise FileInputError(e)
|
661
708
|
elif not isinstance(rev_summary, pd.DataFrame):
|
662
|
-
e =
|
663
|
-
|
709
|
+
e = "Bad input dtype for rev_summary input: {}".format(
|
710
|
+
type(rev_summary)
|
711
|
+
)
|
664
712
|
logger.error(e)
|
665
713
|
raise TypeError(e)
|
666
714
|
|
667
715
|
return rev_summary
|
668
716
|
|
669
|
-
@staticmethod
|
717
|
+
@ staticmethod
|
670
718
|
def _check_req_cols(df, cols):
|
671
719
|
"""Check a dataframe for required columns.
|
672
720
|
|
@@ -681,18 +729,16 @@ class RepProfilesBase(ABC):
|
|
681
729
|
if isinstance(cols, str):
|
682
730
|
cols = [cols]
|
683
731
|
|
684
|
-
missing = []
|
685
|
-
for c in cols:
|
686
|
-
if c not in df:
|
687
|
-
missing.append(c)
|
732
|
+
missing = [c for c in cols if c not in df]
|
688
733
|
|
689
734
|
if any(missing):
|
690
|
-
e =
|
691
|
-
|
735
|
+
e = "Column labels not found in rev_summary table: {}".format(
|
736
|
+
missing
|
737
|
+
)
|
692
738
|
logger.error(e)
|
693
739
|
raise KeyError(e)
|
694
740
|
|
695
|
-
@staticmethod
|
741
|
+
@ staticmethod
|
696
742
|
def _check_rev_gen(gen_fpath, cf_dset, rev_summary):
|
697
743
|
"""Check rev gen file for requisite datasets.
|
698
744
|
|
@@ -710,32 +756,42 @@ class RepProfilesBase(ABC):
|
|
710
756
|
with Resource(gen_fpath) as res:
|
711
757
|
dsets = res.datasets
|
712
758
|
if cf_dset not in dsets:
|
713
|
-
raise KeyError(
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
759
|
+
raise KeyError(
|
760
|
+
'reV gen file needs to have "{}" '
|
761
|
+
"dataset to calculate representative profiles!".format(
|
762
|
+
cf_dset
|
763
|
+
)
|
764
|
+
)
|
765
|
+
|
766
|
+
if "time_index" not in str(dsets):
|
767
|
+
raise KeyError(
|
768
|
+
'reV gen file needs to have "time_index" '
|
769
|
+
"dataset to calculate representative profiles!"
|
770
|
+
)
|
720
771
|
|
721
772
|
shape = res.get_dset_properties(cf_dset)[0]
|
722
773
|
|
723
774
|
if len(rev_summary) > shape[1]:
|
724
|
-
msg = (
|
725
|
-
|
726
|
-
|
727
|
-
|
775
|
+
msg = (
|
776
|
+
"WARNING: reV SC summary table has {} sc points and CF "
|
777
|
+
'dataset "{}" has {} profiles. There should never be more '
|
778
|
+
"SC points than CF profiles.".format(
|
779
|
+
len(rev_summary), cf_dset, shape[1]
|
780
|
+
)
|
781
|
+
)
|
728
782
|
logger.warning(msg)
|
729
783
|
warn(msg)
|
730
784
|
|
731
785
|
def _init_profiles(self):
|
732
786
|
"""Initialize the output rep profiles attribute."""
|
733
|
-
self._profiles = {
|
734
|
-
|
735
|
-
|
736
|
-
|
787
|
+
self._profiles = {
|
788
|
+
k: np.zeros(
|
789
|
+
(len(self.time_index), len(self.meta)), dtype=np.float32
|
790
|
+
)
|
791
|
+
for k in range(self._n_profiles)
|
792
|
+
}
|
737
793
|
|
738
|
-
@property
|
794
|
+
@ property
|
739
795
|
def time_index(self):
|
740
796
|
"""Get the time index for the rep profiles.
|
741
797
|
|
@@ -746,16 +802,16 @@ class RepProfilesBase(ABC):
|
|
746
802
|
"""
|
747
803
|
if self._time_index is None:
|
748
804
|
with Resource(self._gen_fpath) as res:
|
749
|
-
ds =
|
750
|
-
if parse_year(self._cf_dset, option=
|
751
|
-
year = parse_year(self._cf_dset, option=
|
752
|
-
ds +=
|
805
|
+
ds = "time_index"
|
806
|
+
if parse_year(self._cf_dset, option="bool"):
|
807
|
+
year = parse_year(self._cf_dset, option="raise")
|
808
|
+
ds += "-{}".format(year)
|
753
809
|
|
754
810
|
self._time_index = res._get_time_index(ds, slice(None))
|
755
811
|
|
756
812
|
return self._time_index
|
757
813
|
|
758
|
-
@property
|
814
|
+
@ property
|
759
815
|
def meta(self):
|
760
816
|
"""Meta data for the representative profiles.
|
761
817
|
|
@@ -767,7 +823,7 @@ class RepProfilesBase(ABC):
|
|
767
823
|
"""
|
768
824
|
return self._meta
|
769
825
|
|
770
|
-
@property
|
826
|
+
@ property
|
771
827
|
def profiles(self):
|
772
828
|
"""Get the arrays of representative CF profiles corresponding to meta.
|
773
829
|
|
@@ -779,8 +835,9 @@ class RepProfilesBase(ABC):
|
|
779
835
|
"""
|
780
836
|
return self._profiles
|
781
837
|
|
782
|
-
def _init_h5_out(
|
783
|
-
|
838
|
+
def _init_h5_out(
|
839
|
+
self, fout, save_rev_summary=True, scaled_precision=False
|
840
|
+
):
|
784
841
|
"""Initialize an output h5 file for n_profiles
|
785
842
|
|
786
843
|
Parameters
|
@@ -799,13 +856,13 @@ class RepProfilesBase(ABC):
|
|
799
856
|
dtypes = {}
|
800
857
|
|
801
858
|
for i in range(self._n_profiles):
|
802
|
-
dset =
|
859
|
+
dset = "rep_profiles_{}".format(i)
|
803
860
|
dsets.append(dset)
|
804
861
|
shapes[dset] = self.profiles[0].shape
|
805
862
|
chunks[dset] = None
|
806
863
|
|
807
864
|
if scaled_precision:
|
808
|
-
attrs[dset] = {
|
865
|
+
attrs[dset] = {"scale_factor": 1000}
|
809
866
|
dtypes[dset] = np.uint16
|
810
867
|
else:
|
811
868
|
attrs[dset] = None
|
@@ -813,19 +870,26 @@ class RepProfilesBase(ABC):
|
|
813
870
|
|
814
871
|
meta = self.meta.copy()
|
815
872
|
for c in meta.columns:
|
816
|
-
|
873
|
+
with contextlib.suppress(ValueError):
|
817
874
|
meta[c] = pd.to_numeric(meta[c])
|
818
|
-
except ValueError:
|
819
|
-
pass
|
820
875
|
|
821
|
-
Outputs.init_h5(
|
822
|
-
|
876
|
+
Outputs.init_h5(
|
877
|
+
fout,
|
878
|
+
dsets,
|
879
|
+
shapes,
|
880
|
+
attrs,
|
881
|
+
chunks,
|
882
|
+
dtypes,
|
883
|
+
meta,
|
884
|
+
time_index=self.time_index,
|
885
|
+
)
|
823
886
|
|
824
887
|
if save_rev_summary:
|
825
|
-
with Outputs(fout, mode=
|
888
|
+
with Outputs(fout, mode="a") as out:
|
826
889
|
rev_sum = to_records_array(self._rev_summary)
|
827
|
-
out._create_dset(
|
828
|
-
|
890
|
+
out._create_dset(
|
891
|
+
"rev_summary", rev_sum.shape, rev_sum.dtype, data=rev_sum
|
892
|
+
)
|
829
893
|
|
830
894
|
def _write_h5_out(self, fout, save_rev_summary=True):
|
831
895
|
"""Write profiles and meta to an output file.
|
@@ -839,18 +903,18 @@ class RepProfilesBase(ABC):
|
|
839
903
|
scaled_precision : bool
|
840
904
|
Flag to scale cf_profiles by 1000 and save as uint16.
|
841
905
|
"""
|
842
|
-
with Outputs(fout, mode=
|
843
|
-
|
844
|
-
if 'rev_summary' in out.datasets and save_rev_summary:
|
906
|
+
with Outputs(fout, mode="a") as out:
|
907
|
+
if "rev_summary" in out.datasets and save_rev_summary:
|
845
908
|
rev_sum = to_records_array(self._rev_summary)
|
846
|
-
out[
|
909
|
+
out["rev_summary"] = rev_sum
|
847
910
|
|
848
911
|
for i in range(self._n_profiles):
|
849
|
-
dset =
|
912
|
+
dset = "rep_profiles_{}".format(i)
|
850
913
|
out[dset] = self.profiles[i]
|
851
914
|
|
852
|
-
def save_profiles(
|
853
|
-
|
915
|
+
def save_profiles(
|
916
|
+
self, fout, save_rev_summary=True, scaled_precision=False
|
917
|
+
):
|
854
918
|
"""Initialize fout and save profiles.
|
855
919
|
|
856
920
|
Parameters
|
@@ -863,19 +927,22 @@ class RepProfilesBase(ABC):
|
|
863
927
|
Flag to scale cf_profiles by 1000 and save as uint16.
|
864
928
|
"""
|
865
929
|
|
866
|
-
self._init_h5_out(
|
867
|
-
|
930
|
+
self._init_h5_out(
|
931
|
+
fout,
|
932
|
+
save_rev_summary=save_rev_summary,
|
933
|
+
scaled_precision=scaled_precision,
|
934
|
+
)
|
868
935
|
self._write_h5_out(fout, save_rev_summary=save_rev_summary)
|
869
936
|
|
870
|
-
@abstractmethod
|
937
|
+
@ abstractmethod
|
871
938
|
def _run_serial(self):
|
872
939
|
"""Abstract method for serial run method."""
|
873
940
|
|
874
|
-
@abstractmethod
|
941
|
+
@ abstractmethod
|
875
942
|
def _run_parallel(self):
|
876
943
|
"""Abstract method for parallel run method."""
|
877
944
|
|
878
|
-
@abstractmethod
|
945
|
+
@ abstractmethod
|
879
946
|
def run(self):
|
880
947
|
"""Abstract method for generic run method."""
|
881
948
|
|
@@ -883,10 +950,12 @@ class RepProfilesBase(ABC):
|
|
883
950
|
class RepProfiles(RepProfilesBase):
|
884
951
|
"""RepProfiles"""
|
885
952
|
|
886
|
-
def __init__(self, gen_fpath, rev_summary, reg_cols,
|
887
|
-
|
953
|
+
def __init__(self, gen_fpath, rev_summary, reg_cols,
|
954
|
+
cf_dset='cf_profile',
|
955
|
+
rep_method='meanoid', err_method='rmse',
|
956
|
+
weight=str(SupplyCurveField.GID_COUNTS), # str() to fix docs
|
888
957
|
n_profiles=1, aggregate_profiles=False):
|
889
|
-
"""
|
958
|
+
"""ReV rep profiles class.
|
890
959
|
|
891
960
|
``reV`` rep profiles compute representative generation profiles
|
892
961
|
for each supply curve point output by ``reV`` supply curve
|
@@ -977,7 +1046,7 @@ class RepProfiles(RepProfilesBase):
|
|
977
1046
|
*equally* to the meanoid profile unless these weights are
|
978
1047
|
specified.
|
979
1048
|
|
980
|
-
By default,
|
1049
|
+
By default, :obj:`SupplyCurveField.GID_COUNTS`.
|
981
1050
|
n_profiles : int, optional
|
982
1051
|
Number of representative profiles to save to the output
|
983
1052
|
file. By default, ``1``.
|
@@ -991,33 +1060,45 @@ class RepProfiles(RepProfilesBase):
|
|
991
1060
|
"""
|
992
1061
|
|
993
1062
|
log_versions(logger)
|
994
|
-
logger.info(
|
995
|
-
|
1063
|
+
logger.info(
|
1064
|
+
"Finding representative profiles that are most similar "
|
1065
|
+
"to the weighted meanoid for each supply curve region."
|
1066
|
+
)
|
996
1067
|
|
997
1068
|
if reg_cols is None:
|
998
|
-
e = (
|
999
|
-
|
1000
|
-
|
1069
|
+
e = (
|
1070
|
+
'Need to define "reg_cols"! If you want a profile for each '
|
1071
|
+
'supply curve point, try setting "reg_cols" to a primary '
|
1072
|
+
'key such as "sc_gid".'
|
1073
|
+
)
|
1001
1074
|
logger.error(e)
|
1002
1075
|
raise ValueError(e)
|
1003
|
-
|
1076
|
+
if isinstance(reg_cols, str):
|
1004
1077
|
reg_cols = [reg_cols]
|
1005
1078
|
elif not isinstance(reg_cols, list):
|
1006
1079
|
reg_cols = list(reg_cols)
|
1007
1080
|
|
1008
1081
|
self._aggregate_profiles = aggregate_profiles
|
1009
1082
|
if self._aggregate_profiles:
|
1010
|
-
logger.info(
|
1011
|
-
|
1012
|
-
|
1013
|
-
|
1083
|
+
logger.info(
|
1084
|
+
"Aggregate profiles input set to `True`. Setting "
|
1085
|
+
"'rep_method' to `'meanoid'`, 'err_method' to `None`, "
|
1086
|
+
"and 'n_profiles' to `1`"
|
1087
|
+
)
|
1088
|
+
rep_method = "meanoid"
|
1014
1089
|
err_method = None
|
1015
1090
|
n_profiles = 1
|
1016
1091
|
|
1017
|
-
super().__init__(
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1092
|
+
super().__init__(
|
1093
|
+
gen_fpath,
|
1094
|
+
rev_summary,
|
1095
|
+
reg_cols=reg_cols,
|
1096
|
+
cf_dset=cf_dset,
|
1097
|
+
rep_method=rep_method,
|
1098
|
+
err_method=err_method,
|
1099
|
+
weight=weight,
|
1100
|
+
n_profiles=n_profiles,
|
1101
|
+
)
|
1021
1102
|
|
1022
1103
|
self._set_meta()
|
1023
1104
|
self._init_profiles()
|
@@ -1030,13 +1111,13 @@ class RepProfiles(RepProfilesBase):
|
|
1030
1111
|
else:
|
1031
1112
|
self._meta = self._rev_summary.groupby(self._reg_cols)
|
1032
1113
|
self._meta = (
|
1033
|
-
self._meta[
|
1114
|
+
self._meta[SupplyCurveField.TIMEZONE]
|
1034
1115
|
.apply(lambda x: stats.mode(x, keepdims=True).mode[0])
|
1035
1116
|
)
|
1036
1117
|
self._meta = self._meta.reset_index()
|
1037
1118
|
|
1038
|
-
self._meta[
|
1039
|
-
self._meta[
|
1119
|
+
self._meta["rep_gen_gid"] = None
|
1120
|
+
self._meta["rep_res_gid"] = None
|
1040
1121
|
|
1041
1122
|
def _get_mask(self, region_dict):
|
1042
1123
|
"""Get the mask for a given region and res class.
|
@@ -1054,54 +1135,70 @@ class RepProfiles(RepProfilesBase):
|
|
1054
1135
|
"""
|
1055
1136
|
mask = None
|
1056
1137
|
for k, v in region_dict.items():
|
1057
|
-
temp =
|
1138
|
+
temp = self._rev_summary[k] == v
|
1058
1139
|
if mask is None:
|
1059
1140
|
mask = temp
|
1060
1141
|
else:
|
1061
|
-
mask =
|
1142
|
+
mask = mask & temp
|
1062
1143
|
|
1063
1144
|
return mask
|
1064
1145
|
|
1065
1146
|
def _run_serial(self):
|
1066
1147
|
"""Compute all representative profiles in serial."""
|
1067
1148
|
|
1068
|
-
logger.info(
|
1069
|
-
|
1149
|
+
logger.info(
|
1150
|
+
"Running {} rep profile calculations in serial.".format(
|
1151
|
+
len(self.meta)
|
1152
|
+
)
|
1153
|
+
)
|
1070
1154
|
meta_static = deepcopy(self.meta)
|
1071
1155
|
for i, row in meta_static.iterrows():
|
1072
|
-
region_dict = {
|
1073
|
-
|
1156
|
+
region_dict = {
|
1157
|
+
k: v for (k, v) in row.to_dict().items() if k in self._reg_cols
|
1158
|
+
}
|
1074
1159
|
mask = self._get_mask(region_dict)
|
1075
1160
|
|
1076
1161
|
if not any(mask):
|
1077
|
-
logger.warning(
|
1078
|
-
|
1079
|
-
|
1162
|
+
logger.warning(
|
1163
|
+
"Skipping profile {} out of {} "
|
1164
|
+
"for region: {} with no valid mask.".format(
|
1165
|
+
i + 1, len(meta_static), region_dict
|
1166
|
+
)
|
1167
|
+
)
|
1080
1168
|
else:
|
1081
|
-
logger.debug(
|
1082
|
-
|
1169
|
+
logger.debug(
|
1170
|
+
"Working on profile {} out of {} for region: {}".format(
|
1171
|
+
i + 1, len(meta_static), region_dict
|
1172
|
+
)
|
1173
|
+
)
|
1083
1174
|
out = RegionRepProfile.get_region_rep_profile(
|
1084
|
-
self._gen_fpath,
|
1085
|
-
|
1086
|
-
|
1087
|
-
|
1175
|
+
self._gen_fpath,
|
1176
|
+
self._rev_summary[mask],
|
1177
|
+
cf_dset=self._cf_dset,
|
1178
|
+
rep_method=self._rep_method,
|
1179
|
+
err_method=self._err_method,
|
1180
|
+
weight=self._weight,
|
1181
|
+
n_profiles=self._n_profiles,
|
1182
|
+
)
|
1088
1183
|
profiles, _, ggids, rgids = out
|
1089
|
-
logger.info(
|
1090
|
-
|
1091
|
-
|
1184
|
+
logger.info(
|
1185
|
+
"Profile {} out of {} complete " "for region: {}".format(
|
1186
|
+
i + 1, len(meta_static), region_dict
|
1187
|
+
)
|
1188
|
+
)
|
1092
1189
|
|
1093
1190
|
for n in range(profiles.shape[1]):
|
1094
1191
|
self._profiles[n][:, i] = profiles[:, n]
|
1095
1192
|
|
1096
1193
|
if ggids is None:
|
1097
|
-
self._meta.at[i,
|
1098
|
-
self._meta.at[i,
|
1194
|
+
self._meta.at[i, "rep_gen_gid"] = None
|
1195
|
+
self._meta.at[i, "rep_res_gid"] = None
|
1099
1196
|
elif len(ggids) == 1:
|
1100
|
-
self._meta.at[i,
|
1101
|
-
self._meta.at[i,
|
1197
|
+
self._meta.at[i, "rep_gen_gid"] = ggids[0]
|
1198
|
+
self._meta.at[i, "rep_res_gid"] = rgids[0]
|
1102
1199
|
else:
|
1103
|
-
self._meta.at[i,
|
1104
|
-
self._meta.at[i,
|
1200
|
+
self._meta.at[i, "rep_gen_gid"] = str(ggids)
|
1201
|
+
self._meta.at[i, "rep_res_gid"] = str(rgids)
|
1105
1202
|
|
1106
1203
|
def _run_parallel(self, max_workers=None, pool_size=72):
|
1107
1204
|
"""Compute all representative profiles in parallel.
|
@@ -1116,39 +1213,49 @@ class RepProfiles(RepProfilesBase):
|
|
1116
1213
|
parallel futures.
|
1117
1214
|
"""
|
1118
1215
|
|
1119
|
-
logger.info(
|
1120
|
-
|
1216
|
+
logger.info(
|
1217
|
+
"Kicking off {} rep profile futures.".format(len(self.meta))
|
1218
|
+
)
|
1121
1219
|
|
1122
|
-
iter_chunks = np.array_split(
|
1123
|
-
|
1220
|
+
iter_chunks = np.array_split(
|
1221
|
+
self.meta.index.values, np.ceil(len(self.meta) / pool_size)
|
1222
|
+
)
|
1124
1223
|
n_complete = 0
|
1125
1224
|
for iter_chunk in iter_chunks:
|
1126
|
-
logger.debug(
|
1225
|
+
logger.debug("Starting process pool...")
|
1127
1226
|
futures = {}
|
1128
|
-
loggers = [__name__,
|
1129
|
-
with SpawnProcessPool(
|
1130
|
-
|
1227
|
+
loggers = [__name__, "reV"]
|
1228
|
+
with SpawnProcessPool(
|
1229
|
+
max_workers=max_workers, loggers=loggers
|
1230
|
+
) as exe:
|
1131
1231
|
for i in iter_chunk:
|
1132
1232
|
row = self.meta.loc[i, :]
|
1133
|
-
region_dict = {
|
1134
|
-
|
1233
|
+
region_dict = {
|
1234
|
+
k: v
|
1235
|
+
for (k, v) in row.to_dict().items()
|
1236
|
+
if k in self._reg_cols
|
1237
|
+
}
|
1135
1238
|
|
1136
1239
|
mask = self._get_mask(region_dict)
|
1137
1240
|
|
1138
1241
|
if not any(mask):
|
1139
|
-
logger.info(
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1242
|
+
logger.info(
|
1243
|
+
"Skipping profile {} out of {} "
|
1244
|
+
"for region: {} with no valid mask.".format(
|
1245
|
+
i + 1, len(self.meta), region_dict
|
1246
|
+
)
|
1247
|
+
)
|
1143
1248
|
else:
|
1144
1249
|
future = exe.submit(
|
1145
1250
|
RegionRepProfile.get_region_rep_profile,
|
1146
|
-
self._gen_fpath,
|
1251
|
+
self._gen_fpath,
|
1252
|
+
self._rev_summary[mask],
|
1147
1253
|
cf_dset=self._cf_dset,
|
1148
1254
|
rep_method=self._rep_method,
|
1149
1255
|
err_method=self._err_method,
|
1150
1256
|
weight=self._weight,
|
1151
|
-
n_profiles=self._n_profiles
|
1257
|
+
n_profiles=self._n_profiles,
|
1258
|
+
)
|
1152
1259
|
|
1153
1260
|
futures[future] = [i, region_dict]
|
1154
1261
|
|
@@ -1156,27 +1263,34 @@ class RepProfiles(RepProfilesBase):
|
|
1156
1263
|
i, region_dict = futures[future]
|
1157
1264
|
profiles, _, ggids, rgids = future.result()
|
1158
1265
|
n_complete += 1
|
1159
|
-
logger.info(
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1163
|
-
|
1266
|
+
logger.info(
|
1267
|
+
"Future {} out of {} complete "
|
1268
|
+
"for region: {}".format(
|
1269
|
+
n_complete, len(self.meta), region_dict
|
1270
|
+
)
|
1271
|
+
)
|
1272
|
+
log_mem(logger, log_level="DEBUG")
|
1164
1273
|
|
1165
1274
|
for n in range(profiles.shape[1]):
|
1166
1275
|
self._profiles[n][:, i] = profiles[:, n]
|
1167
1276
|
|
1168
1277
|
if ggids is None:
|
1169
|
-
self._meta.at[i,
|
1170
|
-
self._meta.at[i,
|
1278
|
+
self._meta.at[i, "rep_gen_gid"] = None
|
1279
|
+
self._meta.at[i, "rep_res_gid"] = None
|
1171
1280
|
elif len(ggids) == 1:
|
1172
|
-
self._meta.at[i,
|
1173
|
-
self._meta.at[i,
|
1281
|
+
self._meta.at[i, "rep_gen_gid"] = ggids[0]
|
1282
|
+
self._meta.at[i, "rep_res_gid"] = rgids[0]
|
1174
1283
|
else:
|
1175
|
-
self._meta.at[i,
|
1176
|
-
self._meta.at[i,
|
1177
|
-
|
1178
|
-
def run(
|
1179
|
-
|
1284
|
+
self._meta.at[i, "rep_gen_gid"] = str(ggids)
|
1285
|
+
self._meta.at[i, "rep_res_gid"] = str(rgids)
|
1286
|
+
|
1287
|
+
def run(
|
1288
|
+
self,
|
1289
|
+
fout=None,
|
1290
|
+
save_rev_summary=True,
|
1291
|
+
scaled_precision=False,
|
1292
|
+
max_workers=None,
|
1293
|
+
):
|
1180
1294
|
"""
|
1181
1295
|
Run representative profiles in serial or parallel and save to disc
|
1182
1296
|
|
@@ -1204,12 +1318,17 @@ class RepProfiles(RepProfilesBase):
|
|
1204
1318
|
|
1205
1319
|
if fout is not None:
|
1206
1320
|
if self._aggregate_profiles:
|
1207
|
-
logger.info(
|
1208
|
-
|
1321
|
+
logger.info(
|
1322
|
+
"Aggregate profiles input set to `True`. Setting "
|
1323
|
+
"'save_rev_summary' input to `False`"
|
1324
|
+
)
|
1209
1325
|
save_rev_summary = False
|
1210
|
-
self.save_profiles(
|
1211
|
-
|
1326
|
+
self.save_profiles(
|
1327
|
+
fout,
|
1328
|
+
save_rev_summary=save_rev_summary,
|
1329
|
+
scaled_precision=scaled_precision,
|
1330
|
+
)
|
1212
1331
|
|
1213
|
-
logger.info(
|
1332
|
+
logger.info("Representative profiles complete!")
|
1214
1333
|
|
1215
1334
|
return fout
|