NREL-reV 0.8.7__py3-none-any.whl → 0.8.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/METADATA +12 -10
  2. {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/RECORD +38 -38
  3. {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/WHEEL +1 -1
  4. reV/SAM/SAM.py +182 -133
  5. reV/SAM/econ.py +18 -14
  6. reV/SAM/generation.py +608 -419
  7. reV/SAM/windbos.py +93 -79
  8. reV/bespoke/bespoke.py +690 -445
  9. reV/bespoke/place_turbines.py +6 -6
  10. reV/config/project_points.py +220 -140
  11. reV/econ/econ.py +165 -113
  12. reV/econ/economies_of_scale.py +57 -34
  13. reV/generation/base.py +310 -183
  14. reV/generation/generation.py +298 -190
  15. reV/handlers/exclusions.py +16 -15
  16. reV/handlers/multi_year.py +12 -9
  17. reV/handlers/outputs.py +6 -5
  18. reV/hybrids/hybrid_methods.py +28 -30
  19. reV/hybrids/hybrids.py +304 -188
  20. reV/nrwal/nrwal.py +262 -168
  21. reV/qa_qc/cli_qa_qc.py +14 -10
  22. reV/qa_qc/qa_qc.py +217 -119
  23. reV/qa_qc/summary.py +228 -146
  24. reV/rep_profiles/rep_profiles.py +349 -230
  25. reV/supply_curve/aggregation.py +349 -188
  26. reV/supply_curve/competitive_wind_farms.py +90 -48
  27. reV/supply_curve/exclusions.py +138 -85
  28. reV/supply_curve/extent.py +75 -50
  29. reV/supply_curve/points.py +536 -309
  30. reV/supply_curve/sc_aggregation.py +366 -225
  31. reV/supply_curve/supply_curve.py +505 -308
  32. reV/supply_curve/tech_mapping.py +144 -82
  33. reV/utilities/__init__.py +199 -16
  34. reV/utilities/pytest_utils.py +8 -4
  35. reV/version.py +1 -1
  36. {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/LICENSE +0 -0
  37. {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/entry_points.txt +0 -0
  38. {NREL_reV-0.8.7.dist-info → NREL_reV-0.8.9.dist-info}/top_level.txt +0 -0
reV/generation/base.py CHANGED
@@ -2,44 +2,47 @@
2
2
  """
3
3
  reV base gen and econ module.
4
4
  """
5
- from abc import ABC, abstractmethod
6
5
  import copy
7
- from concurrent.futures import TimeoutError
6
+ import json
8
7
  import logging
9
- import pandas as pd
10
- import numpy as np
11
8
  import os
12
- import psutil
13
- import json
14
9
  import sys
10
+ from abc import ABC, abstractmethod
11
+ from concurrent.futures import TimeoutError
15
12
  from warnings import warn
16
13
 
14
+ import numpy as np
15
+ import pandas as pd
16
+ import psutil
17
+ from rex.resource import Resource
18
+ from rex.utilities.execution import SpawnProcessPool
19
+
17
20
  from reV.config.output_request import SAMOutputRequest
18
- from reV.config.project_points import ProjectPoints, PointsControl
21
+ from reV.config.project_points import PointsControl, ProjectPoints
19
22
  from reV.handlers.outputs import Outputs
20
23
  from reV.SAM.version_checker import PySamVersionChecker
21
- from reV.utilities.exceptions import (OutputWarning, ExecutionError,
22
- ParallelExecutionWarning,
23
- OffshoreWindInputWarning)
24
- from reV.utilities import log_versions, ModuleName
25
-
26
- from rex.resource import Resource
27
- from rex.utilities.execution import SpawnProcessPool
24
+ from reV.utilities import ModuleName, ResourceMetaField, log_versions
25
+ from reV.utilities.exceptions import (
26
+ ExecutionError,
27
+ OffshoreWindInputWarning,
28
+ OutputWarning,
29
+ ParallelExecutionWarning,
30
+ )
28
31
 
29
32
  logger = logging.getLogger(__name__)
30
33
 
31
34
 
32
35
  ATTR_DIR = os.path.dirname(os.path.realpath(__file__))
33
36
  ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes')
34
- with open(os.path.join(ATTR_DIR, 'other.json'), 'r') as f:
37
+ with open(os.path.join(ATTR_DIR, 'other.json')) as f:
35
38
  OTHER_ATTRS = json.load(f)
36
- with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json'), 'r') as f:
39
+ with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json')) as f:
37
40
  LCOE_ATTRS = json.load(f)
38
- with open(os.path.join(ATTR_DIR, 'single_owner.json'), 'r') as f:
41
+ with open(os.path.join(ATTR_DIR, 'single_owner.json')) as f:
39
42
  SO_ATTRS = json.load(f)
40
- with open(os.path.join(ATTR_DIR, 'windbos.json'), 'r') as f:
43
+ with open(os.path.join(ATTR_DIR, 'windbos.json')) as f:
41
44
  BOS_ATTRS = json.load(f)
42
- with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json'), 'r') as f:
45
+ with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json')) as f:
43
46
  LCOE_IN_ATTRS = json.load(f)
44
47
 
45
48
 
@@ -65,12 +68,19 @@ class BaseGen(ABC):
65
68
  # SAM argument names used to calculate LCOE
66
69
  # Note that system_capacity is not included here because it is never used
67
70
  # downstream and could be confused with the supply_curve point capacity
68
- LCOE_ARGS = ('fixed_charge_rate', 'capital_cost', 'fixed_operating_cost',
71
+ LCOE_ARGS = ('fixed_charge_rate', 'capital_cost',
72
+ 'fixed_operating_cost',
69
73
  'variable_operating_cost')
70
74
 
71
- def __init__(self, points_control, output_request, site_data=None,
72
- drop_leap=False, memory_utilization_limit=0.4,
73
- scale_outputs=True):
75
+ def __init__(
76
+ self,
77
+ points_control,
78
+ output_request,
79
+ site_data=None,
80
+ drop_leap=False,
81
+ memory_utilization_limit=0.4,
82
+ scale_outputs=True,
83
+ ):
74
84
  """
75
85
  Parameters
76
86
  ----------
@@ -106,11 +116,13 @@ class BaseGen(ABC):
106
116
  self.mem_util_lim = memory_utilization_limit
107
117
  self.scale_outputs = scale_outputs
108
118
 
109
- self._run_attrs = {'points_control': str(points_control),
110
- 'output_request': output_request,
111
- 'site_data': str(site_data),
112
- 'drop_leap': str(drop_leap),
113
- 'memory_utilization_limit': self.mem_util_lim}
119
+ self._run_attrs = {
120
+ "points_control": str(points_control),
121
+ "output_request": output_request,
122
+ "site_data": str(site_data),
123
+ "drop_leap": str(drop_leap),
124
+ "memory_utilization_limit": self.mem_util_lim,
125
+ }
114
126
 
115
127
  self._site_data = self._parse_site_data(site_data)
116
128
  self.add_site_data_to_pp(self._site_data)
@@ -174,11 +186,16 @@ class BaseGen(ABC):
174
186
  tot_mem = psutil.virtual_memory().total / 1e6
175
187
  avail_mem = self.mem_util_lim * tot_mem
176
188
  self._site_limit = int(np.floor(avail_mem / self.site_mem))
177
- logger.info('Limited to storing {0} sites in memory '
178
- '({1:.1f} GB total hardware, {2:.1f} GB available '
179
- 'with {3:.1f}% utilization).'
180
- .format(self._site_limit, tot_mem / 1e3,
181
- avail_mem / 1e3, self.mem_util_lim * 100))
189
+ logger.info(
190
+ "Limited to storing {0} sites in memory "
191
+ "({1:.1f} GB total hardware, {2:.1f} GB available "
192
+ "with {3:.1f}% utilization).".format(
193
+ self._site_limit,
194
+ tot_mem / 1e3,
195
+ avail_mem / 1e3,
196
+ self.mem_util_lim * 100,
197
+ )
198
+ )
182
199
 
183
200
  return self._site_limit
184
201
 
@@ -199,17 +216,18 @@ class BaseGen(ABC):
199
216
  n = 100
200
217
  self._site_mem = 0
201
218
  for request in self.output_request:
202
- dtype = 'float32'
219
+ dtype = "float32"
203
220
  if request in self.OUT_ATTRS:
204
- dtype = self.OUT_ATTRS[request].get('dtype', 'float32')
221
+ dtype = self.OUT_ATTRS[request].get("dtype", "float32")
205
222
 
206
223
  shape = self._get_data_shape(request, n)
207
224
  self._site_mem += sys.getsizeof(np.ones(shape, dtype=dtype))
208
225
 
209
226
  self._site_mem = self._site_mem / 1e6 / n
210
- logger.info('Output results from a single site are calculated to '
211
- 'use {0:.1f} KB of memory.'
212
- .format(self._site_mem / 1000))
227
+ logger.info(
228
+ "Output results from a single site are calculated to "
229
+ "use {0:.1f} KB of memory.".format(self._site_mem / 1000)
230
+ )
213
231
 
214
232
  return self._site_mem
215
233
 
@@ -259,7 +277,7 @@ class BaseGen(ABC):
259
277
  """
260
278
  sam_metas = self.sam_configs.copy()
261
279
  for v in sam_metas.values():
262
- v.update({'module': self._sam_module.MODULE})
280
+ v.update({"module": self._sam_module.MODULE})
263
281
 
264
282
  return sam_metas
265
283
 
@@ -284,7 +302,8 @@ class BaseGen(ABC):
284
302
  Meta data df for sites in project points. Column names are meta
285
303
  data variables, rows are different sites. The row index
286
304
  does not indicate the site number if the project points are
287
- non-sequential or do not start from 0, so a 'gid' column is added.
305
+ non-sequential or do not start from 0, so a `SupplyCurveField.GID`
306
+ column is added.
288
307
  """
289
308
  return self._meta
290
309
 
@@ -351,12 +370,12 @@ class BaseGen(ABC):
351
370
  out = {}
352
371
  for k, v in self._out.items():
353
372
  if k in self.OUT_ATTRS:
354
- scale_factor = self.OUT_ATTRS[k].get('scale_factor', 1)
373
+ scale_factor = self.OUT_ATTRS[k].get("scale_factor", 1)
355
374
  else:
356
375
  scale_factor = 1
357
376
 
358
377
  if scale_factor != 1 and self.scale_outputs:
359
- v = v.astype('float32')
378
+ v = v.astype("float32")
360
379
  v /= scale_factor
361
380
 
362
381
  out[k] = v
@@ -381,16 +400,14 @@ class BaseGen(ABC):
381
400
  result = self.unpack_futures(result)
382
401
 
383
402
  if isinstance(result, dict):
384
-
385
403
  # iterate through dict where sites are keys and values are
386
404
  # corresponding results
387
405
  for site_gid, site_output in result.items():
388
-
389
406
  # check that the sites are stored sequentially then add to
390
407
  # the finished site list
391
408
  if self._finished_sites:
392
409
  if int(site_gid) < np.max(self._finished_sites):
393
- raise Exception('Site results are non sequential!')
410
+ raise Exception("Site results are non sequential!")
394
411
 
395
412
  # unpack site output object
396
413
  self.unpack_output(site_gid, site_output)
@@ -402,9 +419,11 @@ class BaseGen(ABC):
402
419
  self._out.clear()
403
420
  self._finished_sites.clear()
404
421
  else:
405
- raise TypeError('Did not recognize the type of output. '
406
- 'Tried to set output type "{}", but requires '
407
- 'list, dict or None.'.format(type(result)))
422
+ raise TypeError(
423
+ "Did not recognize the type of output. "
424
+ 'Tried to set output type "{}", but requires '
425
+ "list, dict or None.".format(type(result))
426
+ )
408
427
 
409
428
  @staticmethod
410
429
  def _output_request_type_check(req):
@@ -428,8 +447,10 @@ class BaseGen(ABC):
428
447
  elif isinstance(req, str):
429
448
  output_request = [req]
430
449
  else:
431
- raise TypeError('Output request must be str, list, or tuple but '
432
- 'received: {}'.format(type(req)))
450
+ raise TypeError(
451
+ "Output request must be str, list, or tuple but "
452
+ "received: {}".format(type(req))
453
+ )
433
454
 
434
455
  return output_request
435
456
 
@@ -452,7 +473,7 @@ class BaseGen(ABC):
452
473
  """
453
474
 
454
475
  # Drop leap day or last day
455
- leap_day = ((ti.month == 2) & (ti.day == 29))
476
+ leap_day = (ti.month == 2) & (ti.day == 29)
456
477
  leap_year = ti.year % 4 == 0
457
478
  last_day = ((ti.month == 12) & (ti.day == 31)) * leap_year
458
479
  if drop_leap:
@@ -463,14 +484,23 @@ class BaseGen(ABC):
463
484
  ti = ti.drop(ti[last_day])
464
485
 
465
486
  if len(ti) % 365 != 0:
466
- raise ValueError('Bad time index with length not a multiple of '
467
- '365: {}'.format(ti))
487
+ raise ValueError(
488
+ "Bad time index with length not a multiple of "
489
+ "365: {}".format(ti)
490
+ )
468
491
 
469
492
  return ti
470
493
 
471
494
  @staticmethod
472
- def _pp_to_pc(points, points_range, sam_configs, tech,
473
- sites_per_worker=None, res_file=None, curtailment=None):
495
+ def _pp_to_pc(
496
+ points,
497
+ points_range,
498
+ sam_configs,
499
+ tech,
500
+ sites_per_worker=None,
501
+ res_file=None,
502
+ curtailment=None,
503
+ ):
474
504
  """
475
505
  Create ProjectControl from ProjectPoints
476
506
 
@@ -519,16 +549,25 @@ class BaseGen(ABC):
519
549
  if hasattr(points, "df"):
520
550
  points = points.df
521
551
 
522
- pp = ProjectPoints(points, sam_configs, tech=tech, res_file=res_file,
523
- curtailment=curtailment)
552
+ pp = ProjectPoints(
553
+ points,
554
+ sam_configs,
555
+ tech=tech,
556
+ res_file=res_file,
557
+ curtailment=curtailment,
558
+ )
524
559
 
525
560
  # make Points Control instance
526
561
  if points_range is not None:
527
562
  # PointsControl is for just a subset of the project points...
528
563
  # this is the case if generation is being initialized on one
529
564
  # of many HPC nodes in a large project
530
- pc = PointsControl.split(points_range[0], points_range[1], pp,
531
- sites_per_split=sites_per_worker)
565
+ pc = PointsControl.split(
566
+ points_range[0],
567
+ points_range[1],
568
+ pp,
569
+ sites_per_split=sites_per_worker,
570
+ )
532
571
  else:
533
572
  # PointsControl is for all of the project points
534
573
  pc = PointsControl(pp, sites_per_split=sites_per_worker)
@@ -536,8 +575,16 @@ class BaseGen(ABC):
536
575
  return pc
537
576
 
538
577
  @classmethod
539
- def get_pc(cls, points, points_range, sam_configs, tech,
540
- sites_per_worker=None, res_file=None, curtailment=None):
578
+ def get_pc(
579
+ cls,
580
+ points,
581
+ points_range,
582
+ sam_configs,
583
+ tech,
584
+ sites_per_worker=None,
585
+ res_file=None,
586
+ curtailment=None,
587
+ ):
541
588
  """Get a PointsControl instance.
542
589
 
543
590
  Parameters
@@ -585,9 +632,12 @@ class BaseGen(ABC):
585
632
  """
586
633
 
587
634
  if tech not in cls.OPTIONS and tech.lower() != ModuleName.ECON:
588
- msg = ('Did not recognize reV-SAM technology string "{}". '
589
- 'Technology string options are: {}'
590
- .format(tech, list(cls.OPTIONS.keys())))
635
+ msg = (
636
+ 'Did not recognize reV-SAM technology string "{}". '
637
+ "Technology string options are: {}".format(
638
+ tech, list(cls.OPTIONS.keys())
639
+ )
640
+ )
591
641
  logger.error(msg)
592
642
  raise KeyError(msg)
593
643
 
@@ -595,16 +645,25 @@ class BaseGen(ABC):
595
645
  # get the optimal sites per split based on res file chunk size
596
646
  sites_per_worker = cls.get_sites_per_worker(res_file)
597
647
 
598
- logger.debug('Sites per worker being set to {} for '
599
- 'PointsControl.'.format(sites_per_worker))
648
+ logger.debug(
649
+ "Sites per worker being set to {} for " "PointsControl.".format(
650
+ sites_per_worker
651
+ )
652
+ )
600
653
 
601
654
  if isinstance(points, PointsControl):
602
655
  # received a pre-intialized instance of pointscontrol
603
656
  pc = points
604
657
  else:
605
- pc = cls._pp_to_pc(points, points_range, sam_configs, tech,
606
- sites_per_worker=sites_per_worker,
607
- res_file=res_file, curtailment=curtailment)
658
+ pc = cls._pp_to_pc(
659
+ points,
660
+ points_range,
661
+ sam_configs,
662
+ tech,
663
+ sites_per_worker=sites_per_worker,
664
+ res_file=res_file,
665
+ curtailment=curtailment,
666
+ )
608
667
 
609
668
  return pc
610
669
 
@@ -637,31 +696,37 @@ class BaseGen(ABC):
637
696
  return default
638
697
 
639
698
  with Resource(res_file) as res:
640
- if 'wtk' in res_file.lower():
699
+ if "wtk" in res_file.lower():
641
700
  for dset in res.datasets:
642
- if 'speed' in dset:
701
+ if "speed" in dset:
643
702
  # take nominal WTK chunks from windspeed
644
703
  _, _, chunks = res.get_dset_properties(dset)
645
704
  break
646
- elif 'nsrdb' in res_file.lower():
705
+ elif "nsrdb" in res_file.lower():
647
706
  # take nominal NSRDB chunks from dni
648
- _, _, chunks = res.get_dset_properties('dni')
707
+ _, _, chunks = res.get_dset_properties("dni")
649
708
  else:
650
- warn('Could not infer dataset chunk size as the resource type '
651
- 'could not be determined from the filename: {}'
652
- .format(res_file))
709
+ warn(
710
+ "Could not infer dataset chunk size as the resource type "
711
+ "could not be determined from the filename: {}".format(
712
+ res_file
713
+ )
714
+ )
653
715
  chunks = None
654
716
 
655
717
  if chunks is None:
656
718
  # if chunks not set, go to default
657
719
  sites_per_worker = default
658
- logger.debug('Sites per worker being set to {} (default) based on '
659
- 'no set chunk size in {}.'
660
- .format(sites_per_worker, res_file))
720
+ logger.debug(
721
+ "Sites per worker being set to {} (default) based on "
722
+ "no set chunk size in {}.".format(sites_per_worker, res_file)
723
+ )
661
724
  else:
662
725
  sites_per_worker = chunks[1]
663
- logger.debug('Sites per worker being set to {} based on chunk '
664
- 'size of {}.'.format(sites_per_worker, res_file))
726
+ logger.debug(
727
+ "Sites per worker being set to {} based on chunk "
728
+ "size of {}.".format(sites_per_worker, res_file)
729
+ )
665
730
 
666
731
  return sites_per_worker
667
732
 
@@ -688,8 +753,13 @@ class BaseGen(ABC):
688
753
 
689
754
  @staticmethod
690
755
  @abstractmethod
691
- def _run_single_worker(points_control, tech=None, res_file=None,
692
- output_request=None, scale_outputs=True):
756
+ def _run_single_worker(
757
+ points_control,
758
+ tech=None,
759
+ res_file=None,
760
+ output_request=None,
761
+ scale_outputs=True,
762
+ ):
693
763
  """Run a reV-SAM analysis based on the points_control iterator.
694
764
 
695
765
  Parameters
@@ -735,31 +805,37 @@ class BaseGen(ABC):
735
805
  if inp is None or inp is False:
736
806
  # no input, just initialize dataframe with site gids as index
737
807
  site_data = pd.DataFrame(index=self.project_points.sites)
738
- site_data.index.name = 'gid'
808
+ site_data.index.name = ResourceMetaField.GID
739
809
  else:
740
810
  # explicit input, initialize df
741
811
  if isinstance(inp, str):
742
- if inp.endswith('.csv'):
812
+ if inp.endswith(".csv"):
743
813
  site_data = pd.read_csv(inp)
744
814
  elif isinstance(inp, pd.DataFrame):
745
815
  site_data = inp
746
816
  else:
747
817
  # site data was not able to be set. Raise error.
748
- raise Exception('Site data input must be .csv or '
749
- 'dataframe, but received: {}'.format(inp))
750
-
751
- if 'gid' not in site_data and site_data.index.name != 'gid':
818
+ raise Exception(
819
+ "Site data input must be .csv or "
820
+ "dataframe, but received: {}".format(inp)
821
+ )
822
+
823
+ gid_not_in_site_data = ResourceMetaField.GID not in site_data
824
+ index_name_not_gid = site_data.index.name != ResourceMetaField.GID
825
+ if gid_not_in_site_data and index_name_not_gid:
752
826
  # require gid as column label or index
753
- raise KeyError('Site data input must have "gid" column '
754
- 'to match reV site gid.')
827
+ raise KeyError('Site data input must have '
828
+ f'{ResourceMetaField.GID} column to match '
829
+ 'reV site gid.')
755
830
 
756
831
  # pylint: disable=no-member
757
- if site_data.index.name != 'gid':
832
+ if site_data.index.name != ResourceMetaField.GID:
758
833
  # make gid the dataframe index if not already
759
- site_data = site_data.set_index('gid', drop=True)
834
+ site_data = site_data.set_index(ResourceMetaField.GID,
835
+ drop=True)
760
836
 
761
- if 'offshore' in site_data:
762
- if site_data['offshore'].sum() > 1:
837
+ if "offshore" in site_data:
838
+ if site_data["offshore"].sum() > 1:
763
839
  w = ('Found offshore sites in econ site data input. '
764
840
  'This functionality has been deprecated. '
765
841
  'Please run the reV offshore module to '
@@ -824,23 +900,25 @@ class BaseGen(ABC):
824
900
 
825
901
  def _get_data_shape_from_out_attrs(self, dset, n_sites):
826
902
  """Get data shape from ``OUT_ATTRS`` variable"""
827
- if self.OUT_ATTRS[dset]['type'] == 'array':
903
+ if self.OUT_ATTRS[dset]["type"] == "array":
828
904
  return (len(self.time_index), n_sites)
829
905
  return (n_sites,)
830
906
 
831
907
  def _get_data_shape_from_sam_config(self, dset, n_sites):
832
- """Get data shape from SAM input config """
908
+ """Get data shape from SAM input config"""
833
909
  data = list(self.project_points.sam_inputs.values())[0][dset]
834
910
  if isinstance(data, (list, tuple, np.ndarray)):
835
911
  return (*np.array(data).shape, n_sites)
836
912
 
837
913
  if isinstance(data, str):
838
- msg = ('Cannot pass through non-scalar SAM input key "{}" '
839
- 'as an output_request!'.format(dset))
914
+ msg = (
915
+ 'Cannot pass through non-scalar SAM input key "{}" '
916
+ "as an output_request!".format(dset)
917
+ )
840
918
  logger.error(msg)
841
919
  raise ExecutionError(msg)
842
920
 
843
- return (n_sites, )
921
+ return (n_sites,)
844
922
 
845
923
  def _get_data_shape_from_pysam(self, dset, n_sites):
846
924
  """Get data shape from PySAM output object"""
@@ -850,10 +928,13 @@ class BaseGen(ABC):
850
928
  try:
851
929
  out_data = getattr(self._sam_obj_default.Outputs, dset)
852
930
  except AttributeError as e:
853
- msg = ('Could not get data shape for dset "{}" '
854
- 'from object "{}". '
855
- 'Received the following error: "{}"'
856
- .format(dset, self._sam_obj_default, e))
931
+ msg = (
932
+ 'Could not get data shape for dset "{}" '
933
+ 'from object "{}". '
934
+ 'Received the following error: "{}"'.format(
935
+ dset, self._sam_obj_default, e
936
+ )
937
+ )
857
938
  logger.error(msg)
858
939
  raise ExecutionError(msg) from e
859
940
 
@@ -873,26 +954,27 @@ class BaseGen(ABC):
873
954
  project_dir, out_fn = os.path.split(out_fpath)
874
955
 
875
956
  # ensure output file is an h5
876
- if not out_fn.endswith('.h5'):
877
- out_fn += '.h5'
957
+ if not out_fn.endswith(".h5"):
958
+ out_fn += ".h5"
878
959
 
879
960
  if module not in out_fn:
880
961
  extension_with_module = "_{}.h5".format(module)
881
962
  out_fn = out_fn.replace(".h5", extension_with_module)
882
963
 
883
964
  # ensure year is in out_fpath
884
- if self.year is not None and str(self.year) not in out_fn:
965
+ if self.year is not None:
885
966
  extension_with_year = "_{}.h5".format(self.year)
886
- out_fn = out_fn.replace(".h5", extension_with_year)
967
+ if extension_with_year not in out_fn:
968
+ out_fn = out_fn.replace(".h5", extension_with_year)
887
969
 
888
970
  # create and use optional output dir
889
971
  if project_dir and not os.path.exists(project_dir):
890
972
  os.makedirs(project_dir, exist_ok=True)
891
973
 
892
974
  self._out_fpath = os.path.join(project_dir, out_fn)
893
- self._run_attrs['out_fpath'] = out_fpath
975
+ self._run_attrs["out_fpath"] = out_fpath
894
976
 
895
- def _init_h5(self, mode='w'):
977
+ def _init_h5(self, mode="w"):
896
978
  """Initialize the single h5 output file with all output requests.
897
979
 
898
980
  Parameters
@@ -904,12 +986,18 @@ class BaseGen(ABC):
904
986
  if self._out_fpath is None:
905
987
  return
906
988
 
907
- if 'w' in mode:
908
- logger.info('Initializing full output file: "{}" with mode: {}'
909
- .format(self._out_fpath, mode))
910
- elif 'a' in mode:
911
- logger.info('Appending data to output file: "{}" with mode: {}'
912
- .format(self._out_fpath, mode))
989
+ if "w" in mode:
990
+ logger.info(
991
+ 'Initializing full output file: "{}" with mode: {}'.format(
992
+ self._out_fpath, mode
993
+ )
994
+ )
995
+ elif "a" in mode:
996
+ logger.info(
997
+ 'Appending data to output file: "{}" with mode: {}'.format(
998
+ self._out_fpath, mode
999
+ )
1000
+ )
913
1001
 
914
1002
  attrs = {d: {} for d in self.output_request}
915
1003
  chunks = {}
@@ -920,17 +1008,16 @@ class BaseGen(ABC):
920
1008
  write_ti = False
921
1009
 
922
1010
  for dset in self.output_request:
923
-
924
- tmp = 'other'
1011
+ tmp = "other"
925
1012
  if dset in self.OUT_ATTRS:
926
1013
  tmp = dset
927
1014
 
928
- attrs[dset]['units'] = self.OUT_ATTRS[tmp].get('units',
929
- 'unknown')
930
- attrs[dset]['scale_factor'] = \
931
- self.OUT_ATTRS[tmp].get('scale_factor', 1)
932
- chunks[dset] = self.OUT_ATTRS[tmp].get('chunks', None)
933
- dtypes[dset] = self.OUT_ATTRS[tmp].get('dtype', 'float32')
1015
+ attrs[dset]["units"] = self.OUT_ATTRS[tmp].get("units", "unknown")
1016
+ attrs[dset]["scale_factor"] = self.OUT_ATTRS[tmp].get(
1017
+ "scale_factor", 1
1018
+ )
1019
+ chunks[dset] = self.OUT_ATTRS[tmp].get("chunks", None)
1020
+ dtypes[dset] = self.OUT_ATTRS[tmp].get("dtype", "float32")
934
1021
  shapes[dset] = self._get_data_shape(dset, len(self.meta))
935
1022
  if len(shapes[dset]) > 1:
936
1023
  write_ti = True
@@ -941,10 +1028,19 @@ class BaseGen(ABC):
941
1028
  else:
942
1029
  ti = None
943
1030
 
944
- Outputs.init_h5(self._out_fpath, self.output_request, shapes,
945
- attrs, chunks, dtypes, self.meta, time_index=ti,
946
- configs=self.sam_metas, run_attrs=self.run_attrs,
947
- mode=mode)
1031
+ Outputs.init_h5(
1032
+ self._out_fpath,
1033
+ self.output_request,
1034
+ shapes,
1035
+ attrs,
1036
+ chunks,
1037
+ dtypes,
1038
+ self.meta,
1039
+ time_index=ti,
1040
+ configs=self.sam_metas,
1041
+ run_attrs=self.run_attrs,
1042
+ mode=mode,
1043
+ )
948
1044
 
949
1045
  def _init_out_arrays(self, index_0=0):
950
1046
  """Initialize output arrays based on the number of sites that can be
@@ -962,21 +1058,27 @@ class BaseGen(ABC):
962
1058
  self._finished_sites = []
963
1059
 
964
1060
  # Output chunk is the index range (inclusive) of this set of site outs
965
- self._out_chunk = (index_0, np.min((index_0 + self.site_limit,
966
- len(self.project_points) - 1)))
1061
+ self._out_chunk = (
1062
+ index_0,
1063
+ np.min((index_0 + self.site_limit, len(self.project_points) - 1)),
1064
+ )
967
1065
  self._out_n_sites = int(self.out_chunk[1] - self.out_chunk[0]) + 1
968
1066
 
969
- logger.info('Initializing in-memory outputs for {} sites with gids '
970
- '{} through {} inclusive (site list index {} through {})'
971
- .format(self._out_n_sites,
972
- self.project_points.sites[self.out_chunk[0]],
973
- self.project_points.sites[self.out_chunk[1]],
974
- self.out_chunk[0], self.out_chunk[1]))
1067
+ logger.info(
1068
+ "Initializing in-memory outputs for {} sites with gids "
1069
+ "{} through {} inclusive (site list index {} through {})".format(
1070
+ self._out_n_sites,
1071
+ self.project_points.sites[self.out_chunk[0]],
1072
+ self.project_points.sites[self.out_chunk[1]],
1073
+ self.out_chunk[0],
1074
+ self.out_chunk[1],
1075
+ )
1076
+ )
975
1077
 
976
1078
  for request in self.output_request:
977
- dtype = 'float32'
1079
+ dtype = "float32"
978
1080
  if request in self.OUT_ATTRS and self.scale_outputs:
979
- dtype = self.OUT_ATTRS[request].get('dtype', 'float32')
1081
+ dtype = self.OUT_ATTRS[request].get("dtype", "float32")
980
1082
 
981
1083
  shape = self._get_data_shape(request, self._out_n_sites)
982
1084
 
@@ -1004,9 +1106,11 @@ class BaseGen(ABC):
1004
1106
  # iterate through the site results
1005
1107
  for var, value in site_output.items():
1006
1108
  if var not in self._out:
1007
- raise KeyError('Tried to collect output variable "{}", but it '
1008
- 'was not yet initialized in the output '
1009
- 'dictionary.')
1109
+ raise KeyError(
1110
+ 'Tried to collect output variable "{}", but it '
1111
+ "was not yet initialized in the output "
1112
+ "dictionary."
1113
+ )
1010
1114
 
1011
1115
  # get the index in the output array for the current site
1012
1116
  i = self.site_index(site_gid, out_index=True)
@@ -1055,12 +1159,14 @@ class BaseGen(ABC):
1055
1159
  else:
1056
1160
  output_index = global_site_index - self.out_chunk[0]
1057
1161
  if output_index < 0:
1058
- raise ValueError('Attempting to set output data for site with '
1059
- 'gid {} to global site index {}, which was '
1060
- 'already set based on the current output '
1061
- 'index chunk of {}'
1062
- .format(site_gid, global_site_index,
1063
- self.out_chunk))
1162
+ raise ValueError(
1163
+ "Attempting to set output data for site with "
1164
+ "gid {} to global site index {}, which was "
1165
+ "already set based on the current output "
1166
+ "index chunk of {}".format(
1167
+ site_gid, global_site_index, self.out_chunk
1168
+ )
1169
+ )
1064
1170
 
1065
1171
  return output_index
1066
1172
 
@@ -1075,15 +1181,17 @@ class BaseGen(ABC):
1075
1181
 
1076
1182
  # handle output file request if file is specified and .out is not empty
1077
1183
  if isinstance(self._out_fpath, str) and self._out:
1078
- logger.info('Flushing outputs to disk, target file: "{}"'
1079
- .format(self._out_fpath))
1184
+ logger.info(
1185
+ 'Flushing outputs to disk, target file: "{}"'.format(
1186
+ self._out_fpath
1187
+ )
1188
+ )
1080
1189
 
1081
1190
  # get the slice of indices to write outputs to
1082
1191
  islice = slice(self.out_chunk[0], self.out_chunk[1] + 1)
1083
1192
 
1084
1193
  # open output file in append mode to add output results to
1085
- with Outputs(self._out_fpath, mode='a') as f:
1086
-
1194
+ with Outputs(self._out_fpath, mode="a") as f:
1087
1195
  # iterate through all output requests writing each as a dataset
1088
1196
  for dset, arr in self._out.items():
1089
1197
  if len(arr.shape) == 1:
@@ -1093,7 +1201,7 @@ class BaseGen(ABC):
1093
1201
  # write 2D array of profiles
1094
1202
  f[dset, :, islice] = arr
1095
1203
 
1096
- logger.debug('Flushed output successfully to disk.')
1204
+ logger.debug("Flushed output successfully to disk.")
1097
1205
 
1098
1206
  def _pre_split_pc(self, pool_size=None):
1099
1207
  """Pre-split project control iterator into sub chunks to further
@@ -1129,11 +1237,15 @@ class BaseGen(ABC):
1129
1237
  if i_chunk:
1130
1238
  pc_chunks.append(i_chunk)
1131
1239
 
1132
- logger.debug('Pre-splitting points control into {} chunks with the '
1133
- 'following chunk sizes: {}'
1134
- .format(len(pc_chunks), [len(x) for x in pc_chunks]))
1240
+ logger.debug(
1241
+ "Pre-splitting points control into {} chunks with the "
1242
+ "following chunk sizes: {}".format(
1243
+ len(pc_chunks), [len(x) for x in pc_chunks]
1244
+ )
1245
+ )
1135
1246
  return N, pc_chunks
1136
1247
 
1248
+ # pylint: disable=unused-argument
1137
1249
  def _reduce_kwargs(self, pc, **kwargs):
1138
1250
  """Placeholder for functions that need to reduce the global kwargs that
1139
1251
  they send to workers to reduce memory footprint
@@ -1153,8 +1265,9 @@ class BaseGen(ABC):
1153
1265
  """
1154
1266
  return kwargs
1155
1267
 
1156
- def _parallel_run(self, max_workers=None, pool_size=None, timeout=1800,
1157
- **kwargs):
1268
+ def _parallel_run(
1269
+ self, max_workers=None, pool_size=None, timeout=1800, **kwargs
1270
+ ):
1158
1271
  """Execute parallel compute.
1159
1272
 
1160
1273
  Parameters
@@ -1176,25 +1289,31 @@ class BaseGen(ABC):
1176
1289
  pool_size = os.cpu_count() * 2
1177
1290
  if max_workers is None:
1178
1291
  max_workers = os.cpu_count()
1179
- logger.info('Running parallel execution with max_workers={}'
1180
- .format(max_workers))
1292
+ logger.info(
1293
+ "Running parallel execution with max_workers={}".format(
1294
+ max_workers
1295
+ )
1296
+ )
1181
1297
  i = 0
1182
1298
  N, pc_chunks = self._pre_split_pc(pool_size=pool_size)
1183
1299
  for j, pc_chunk in enumerate(pc_chunks):
1184
- logger.debug('Starting process pool for points control '
1185
- 'iteration {} out of {}'
1186
- .format(j + 1, len(pc_chunks)))
1300
+ logger.debug(
1301
+ "Starting process pool for points control "
1302
+ "iteration {} out of {}".format(j + 1, len(pc_chunks))
1303
+ )
1187
1304
 
1188
1305
  failed_futures = False
1189
1306
  chunks = {}
1190
1307
  futures = []
1191
- loggers = [__name__, 'reV.gen', 'reV.econ', 'reV']
1192
- with SpawnProcessPool(max_workers=max_workers,
1193
- loggers=loggers) as exe:
1308
+ loggers = [__name__, "reV.gen", "reV.econ", "reV"]
1309
+ with SpawnProcessPool(
1310
+ max_workers=max_workers, loggers=loggers
1311
+ ) as exe:
1194
1312
  for pc in pc_chunk:
1195
1313
  pc_kwargs = self._reduce_kwargs(pc, **kwargs)
1196
- future = exe.submit(self._run_single_worker, pc,
1197
- **pc_kwargs)
1314
+ future = exe.submit(
1315
+ self._run_single_worker, pc, **pc_kwargs
1316
+ )
1198
1317
  futures.append(future)
1199
1318
  chunks[future] = pc
1200
1319
 
@@ -1205,24 +1324,32 @@ class BaseGen(ABC):
1205
1324
  except TimeoutError:
1206
1325
  failed_futures = True
1207
1326
  sites = chunks[future].project_points.sites
1208
- result = self._handle_failed_future(future, i, sites,
1209
- timeout)
1327
+ result = self._handle_failed_future(
1328
+ future, i, sites, timeout
1329
+ )
1210
1330
 
1211
1331
  self.out = result
1212
1332
 
1213
1333
  mem = psutil.virtual_memory()
1214
- m = ('Parallel run at iteration {0} out of {1}. '
1215
- 'Memory utilization is {2:.3f} GB out of {3:.3f} GB '
1216
- 'total ({4:.1f}% used, intended limit of {5:.1f}%)'
1217
- .format(i, N, mem.used / 1e9, mem.total / 1e9,
1218
- 100 * mem.used / mem.total,
1219
- 100 * self.mem_util_lim))
1334
+ m = (
1335
+ "Parallel run at iteration {0} out of {1}. "
1336
+ "Memory utilization is {2:.3f} GB out of {3:.3f} GB "
1337
+ "total ({4:.1f}% used, intended limit of {5:.1f}%)"
1338
+ .format(
1339
+ i,
1340
+ N,
1341
+ mem.used / 1e9,
1342
+ mem.total / 1e9,
1343
+ 100 * mem.used / mem.total,
1344
+ 100 * self.mem_util_lim,
1345
+ )
1346
+ )
1220
1347
  logger.info(m)
1221
1348
 
1222
1349
  if failed_futures:
1223
- logger.info('Forcing pool shutdown after failed futures.')
1350
+ logger.info("Forcing pool shutdown after failed futures.")
1224
1351
  exe.shutdown(wait=False)
1225
- logger.info('Forced pool shutdown complete.')
1352
+ logger.info("Forced pool shutdown complete.")
1226
1353
 
1227
1354
  self.flush()
1228
1355
 
@@ -1242,23 +1369,23 @@ class BaseGen(ABC):
1242
1369
  before returning zeros.
1243
1370
  """
1244
1371
 
1245
- w = ('Iteration {} hit the timeout limit of {} seconds! Passing zeros.'
1246
- .format(i, timeout))
1372
+ w = ("Iteration {} hit the timeout limit of {} seconds! "
1373
+ "Passing zeros.".format(i, timeout))
1247
1374
  logger.warning(w)
1248
1375
  warn(w, OutputWarning)
1249
1376
 
1250
- site_out = {k: 0 for k in self.output_request}
1251
- result = {site: site_out for site in sites}
1377
+ site_out = dict.fromkeys(self.output_request, 0)
1378
+ result = dict.fromkeys(sites, site_out)
1252
1379
 
1253
1380
  try:
1254
1381
  cancelled = future.cancel()
1255
1382
  except Exception as e:
1256
- w = 'Could not cancel future! Received exception: {}'.format(e)
1383
+ w = "Could not cancel future! Received exception: {}".format(e)
1257
1384
  logger.warning(w)
1258
1385
  warn(w, ParallelExecutionWarning)
1259
1386
 
1260
1387
  if not cancelled:
1261
- w = 'Could not cancel future!'
1388
+ w = "Could not cancel future!"
1262
1389
  logger.warning(w)
1263
1390
  warn(w, ParallelExecutionWarning)
1264
1391