NREL-reV 0.13.1__py3-none-any.whl → 0.14.1__py3-none-any.whl

reV/supply_curve/tech_mapping.py

@@ -16,10 +16,10 @@ from warnings import warn
 
 import h5py
 import numpy as np
+from scipy.spatial import cKDTree
 from rex.resource import Resource
 from rex.utilities.execution import SpawnProcessPool
-from rex.utilities.utilities import res_dist_threshold
-from scipy.spatial import cKDTree
+from rex.utilities.utilities import check_res_file, res_dist_threshold
 
 from reV.supply_curve.extent import SupplyCurveExtent, LATITUDE, LONGITUDE
 from reV.utilities.exceptions import FileInputError, FileInputWarning
@@ -30,36 +30,31 @@ logger = logging.getLogger(__name__)
 class TechMapping:
     """Framework to create map between tech layer (exclusions), res, and gen"""
 
-    def __init__(
-        self, excl_fpath, res_fpath, sc_resolution=2560, dist_margin=1.05
-    ):
+    def __init__(self, excl_fpath, sc_resolution=1200):
         """
         Parameters
         ----------
         excl_fpath : str
             Filepath to exclusions h5 file, must contain latitude and longitude
             arrays to allow for mapping to resource points
-        res_fpath : str
-            Filepath to .h5 resource file that we're mapping to.
         sc_resolution : int | None, optional
-            Supply curve resolution, does not affect the exclusion to resource
-            (tech) mapping, but defines how many exclusion pixels are mapped
-            at a time, by default 2560
-        dist_margin : float, optional
-            Extra margin to multiply times the computed distance between
-            neighboring resource points, by default 1.05
+            Defines how many exclusion pixels are mapped at a time. Units
+            indicate the length of one dimension, in pixels, of each square
+            chunk to be mapped. By default, this value is 1200, which will
+            map the exclusion pixels in 1200x1200 pixel chunks.
+
+            .. Note:: This parameter does not affect the exclusion to resource
+            (tech) mapping, which deviates from how the effect of the
+            ``sc_resolution`` parameter works in other functionality within
+            ``reV``.
+
         """
         self._excl_fpath = excl_fpath
         self._check_fout()
 
-        self._tree, self._dist_thresh = self._build_tree(
-            res_fpath, dist_margin=dist_margin
-        )
-
         with SupplyCurveExtent(
             self._excl_fpath, resolution=sc_resolution
         ) as sc:
-            self._sc_resolution = sc.resolution
             self._gids = np.array(list(range(len(sc))), dtype=np.uint32)
             self._excl_shape = sc.exclusions.shape
             self._n_excl = np.prod(self._excl_shape)
@@ -74,19 +69,6 @@ class TechMapping:
                 )
             )
 
-    @property
-    def distance_threshold(self):
-        """Get the upper bound on NN distance between excl and res points.
-
-        Returns
-        -------
-        float
-            Estimate the distance between resource points. Calculated as half
-            of the diagonal between closest resource points, with desired
-            extra margin
-        """
-        return self._dist_thresh
-
     @staticmethod
     def _build_tree(res_fpath, dist_margin=1.05):
         """
@@ -110,7 +92,8 @@ class TechMapping:
             of the diagonal between closest resource points, with desired
             extra margin
         """
-        with Resource(res_fpath) as f:
+        __, hsds = check_res_file(res_fpath)
+        with Resource(res_fpath, hsds=hsds) as f:
             lat_lons = f.lat_lon
 
         # pylint: disable=not-callable
@@ -181,16 +164,17 @@ class TechMapping:
         return row_slice, col_slice
 
     @classmethod
-    def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices,
+    def _get_excl_coords(cls, excl_fpath, gid, sc_row_indices, sc_col_indices,
                          excl_row_slices, excl_col_slices,
                          coord_labels=(LATITUDE, LONGITUDE)):
         """
-        Extract the exclusion coordinates for the desired gids for TechMapping.
+        Extract the exclusion coordinates for the desired supply curve point
+        gid for TechMapping.
 
         Parameters
         ----------
-        gids : np.ndarray
-            Supply curve gids with tech exclusion points to map to the
+        gid : int
+            Supply curve gid with tech exclusion points to map to the
             resource meta points.
         excl_fpath : str
             Filepath to exclusions h5 file, must contain latitude and longitude
@@ -213,72 +197,50 @@ class TechMapping:
 
         Returns
         -------
-        coords_out : list
-            List of arrays of the un-projected latitude, longitude array of
-            tech exclusion points. List entries correspond to input gids.
+        coords_out : ndarray
+            2D array (Nx2) of the un-projected latitude, longitude of
+            tech exclusion pixels within the specified gid point. Rows
+            correspond to exclusion pixels of the specified gid, columns
+            correspond to latitude and longitude, respectively.
         """
-        coords_out = []
         with h5py.File(excl_fpath, "r") as f:
-            for gid in gids:
-                row_slice, col_slice = cls._get_excl_slices(
-                    gid,
-                    sc_row_indices,
-                    sc_col_indices,
-                    excl_row_slices,
-                    excl_col_slices,
+            row_slice, col_slice = cls._get_excl_slices(
+                gid,
+                sc_row_indices,
+                sc_col_indices,
+                excl_row_slices,
+                excl_col_slices,
+            )
+            try:
+                lats = f[coord_labels[0]][row_slice, col_slice]
+                lons = f[coord_labels[1]][row_slice, col_slice]
+                coords_out = np.vstack((lats.flatten(), lons.flatten())).T
+            except Exception as e:
+                m = (
+                    "Could not unpack coordinates for gid {} with "
+                    "row/col slice {}/{}. Received the following "
+                    "error:\n{}".format(gid, row_slice, col_slice, e)
                 )
-                try:
-                    lats = f[coord_labels[0]][row_slice, col_slice]
-                    lons = f[coord_labels[1]][row_slice, col_slice]
-                    emeta = np.vstack((lats.flatten(), lons.flatten())).T
-                except Exception as e:
-                    m = (
-                        "Could not unpack coordinates for gid {} with "
-                        "row/col slice {}/{}. Received the following "
-                        "error:\n{}".format(gid, row_slice, col_slice, e)
-                    )
-                    logger.error(m)
-                    raise e
-
-                coords_out.append(emeta)
+                logger.error(m)
+                raise e
 
         return coords_out
 
     @classmethod
     def map_resource_gids(
         cls,
-        gids,
-        excl_fpath,
-        sc_row_indices,
-        sc_col_indices,
-        excl_row_slices,
-        excl_col_slices,
+        excl_coords,
         tree,
         dist_thresh,
     ):
-        """Map exclusion gids to the resource meta.
+        """Map exclusion pixels to the resource meta.
 
         Parameters
         ----------
-        gids : np.ndarray
-            Supply curve gids with tech exclusion points to map to the
-            resource meta points.
-        excl_fpath : str
-            Filepath to exclusions h5 file, must contain latitude and longitude
-            arrays to allow for mapping to resource points
-        sc_row_indices : list
-            List of row indices in exclusion array for for every sc_point gid
-        sc_col_indices : list
-            List of column indices in exclusion array for for every sc_point
-            gid
-        excl_row_slices : list
-            List representing the supply curve points rows. Each list entry
-            contains the exclusion row slice that are included in the sc
-            point.
-        excl_col_slices : list
-            List representing the supply curve points columns. Each list entry
-            contains the exclusion columns slice that are included in the sc
-            point.
+        excl_coords : ndarray
+            2D Array of the un-projected latitude, longitude array of
+            tech exclusion pixels.  Rows correspond to exclusion pixels,
+            columns correspond to latitude and longitude, respectively.
         tree : cKDTree
             cKDTree built from resource lat, lon coordinates
         dist_tresh : float
@@ -288,100 +250,51 @@ class TechMapping:
 
         Returns
         -------
-        ind : list
-            List of arrays of index values from the NN. List entries correspond
-            to input gids.
+        ind : ndarray
+            1D arrays of index values from the NN. Entries correspond
+            to input exclusion pixels.
         """
-        logger.debug(
-            "Getting tech map coordinates for chunks {} through {}".format(
-                gids[0], gids[-1]
-            )
-        )
-        ind_out = []
-        coords_out = cls._get_excl_coords(
-            excl_fpath,
-            gids,
-            sc_row_indices,
-            sc_col_indices,
-            excl_row_slices,
-            excl_col_slices,
-        )
+        dist, ind = tree.query(excl_coords)
+        ind[(dist >= dist_thresh)] = -1
 
-        logger.debug(
-            "Running tech mapping for chunks {} through {}".format(
-                gids[0], gids[-1]
-            )
-        )
-        for i, _ in enumerate(gids):
-            dist, ind = tree.query(coords_out[i])
-            ind[(dist >= dist_thresh)] = -1
-            ind_out.append(ind)
+        return ind
 
-        return ind_out
-
-    @staticmethod
-    def save_tech_map(
-        excl_fpath,
-        dset,
-        indices,
-        distance_threshold=None,
-        res_fpath=None,
-        chunks=(128, 128),
-    ):
-        """Save tech mapping indices and coordinates to an h5 output file.
+    def initialize_dataset(self, dset, chunks=(128, 128)):
+        """
+        Initialize output dataset in exclusions h5 file. If dataset already
+        exists, a warning will be issued.
 
         Parameters
         ----------
-        excl_fpath : str
-            Filepath to exclusions h5 file to add techmap to as 'dset'
         dset : str
-            Dataset name in fpath_out to save mapping results to.
-        indices : np.ndarray
-            Index values of the NN resource point. -1 if no res point found.
-            2D integer array with shape equal to the exclusions extent shape.
-        distance_threshold : float
-            Distance upper bound to save as attr.
-        res_fpath : str, optional
-            Filepath to .h5 resource file that we're mapping to,
-            by default None
-        chunks : tuple
-            Chunk shape of the 2D output datasets.
+            Name of the dataset in the exclusions H5 file to create.
+        chunks : tuple, optional
+            Chunk size for the dataset, by default (128, 128).
         """
-        logger.info('Writing tech map "{}" to {}'.format(dset, excl_fpath))
 
-        shape = indices.shape
-        chunks = (np.min((shape[0], chunks[0])), np.min((shape[1], chunks[1])))
-
-        with h5py.File(excl_fpath, "a") as f:
+        with h5py.File(self._excl_fpath, "a") as f:
             if dset in list(f):
                 wmsg = (
-                    'TechMap results dataset "{}" is being replaced '
+                    'TechMap results dataset "{}" already exists '
                     'in pre-existing Exclusions TechMapping file "{}"'.format(
-                        dset, excl_fpath
+                        dset, self._excl_fpath
                     )
                 )
                 logger.warning(wmsg)
                 warn(wmsg, FileInputWarning)
-                f[dset][...] = indices
             else:
+                logger.info(
+                    f"Initializing tech map dataset {dset} in "
+                    f"{self._excl_fpath}"
+                )
                 f.create_dataset(
                     dset,
-                    shape=shape,
-                    dtype=indices.dtype,
-                    data=indices,
+                    shape=self._excl_shape,
+                    dtype=np.int32,
                     chunks=chunks,
+                    fillvalue=-1
                 )
 
-            if distance_threshold:
-                f[dset].attrs["distance_threshold"] = distance_threshold
-
-            if res_fpath:
-                f[dset].attrs["src_res_fpath"] = res_fpath
-
-        logger.info(
-            'Successfully saved tech map "{}" to {}'.format(dset, excl_fpath)
-        )
-
     def _check_fout(self):
         """Check the TechMapping output file for cached data."""
         with h5py.File(self._excl_fpath, 'r') as f:
@@ -393,134 +306,166 @@ class TechMapping:
                 logger.exception(emsg)
                 raise FileInputError(emsg)
 
-    def map_resource(self, max_workers=None, points_per_worker=10):
+    def map_resource(self, dset, res_fpath, dist_margin=1.05,
+                     max_workers=None, batch_size=100):
         """
-        Map all resource gids to exclusion gids
+        Map all resource gids to exclusion gids. Save results to dset in
+        exclusions h5 file.
 
         Parameters
         ----------
+        dset : str, optional
+            Name of the output dataset in the exclusions H5 file to which the
+            tech map will be saved.
+        res_fpath : str
+            Filepath to .h5 resource file that we're mapping to.
+        dist_margin : float, optional
+            Extra margin to multiply times the computed distance between
+            neighboring resource points, by default 1.05
         max_workers : int, optional
             Number of cores to run mapping on. None uses all available cpus,
             by default None
-        points_per_worker : int, optional
-            Number of supply curve points to map to resource gids on each
-            worker, by default 10
-
-        Returns
-        -------
-        indices : np.ndarray
-            Index values of the NN resource point. -1 if no res point found.
-            2D integer array with shape equal to the exclusions extent shape.
+        batch_size : int, optional
+            Number of tasks to be submitted to parallel worker pool at one
+            time, by default 1000. As a rule of thumb, this number should be
+            set to ~10x the number of max_workers. Higher values are not
+            necessarily better, and may slow down processing and/or result in
+            out-of-memory errors. Values less than the number of workers can
+            also lead to slower processing, due to poor load balancing.
         """
-        gid_chunks = ceil(len(self._gids) / points_per_worker)
-        gid_chunks = np.array_split(self._gids, gid_chunks)
+        loggers = [__name__, "reV"]
 
-        # init full output arrays
-        indices = -1 * np.ones((self._n_excl,), dtype=np.int32)
-        iarr = self._make_excl_iarr(self._excl_shape)
+        logger.info(
+            f"Computing cKDtree for tech mapping using {res_fpath=!r} "
+            f"and {dist_margin=!r}"
+        )
+        tree, dist_thresh = self._build_tree(
+            res_fpath, dist_margin=dist_margin
+        )
+        with h5py.File(self._excl_fpath, "a") as f:
+            f[dset].attrs["distance_threshold"] = dist_thresh
+            f[dset].attrs["src_res_fpath"] = res_fpath
 
-        futures = {}
-        loggers = [__name__, "reV"]
+        n_jobs = len(self._gids)
+        n_batches = ceil(n_jobs / batch_size)
+        gid_batches = np.array_split(self._gids, n_batches)
+
+        logger.info(
+            f"Kicking off {n_jobs} resource mapping jobs in {n_batches} "
+            "batches."
+        )
+        n_finished = 0
         with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe:
-            # iterate through split executions, submitting each to worker
-            for i, gid_set in enumerate(gid_chunks):
-                # submit executions and append to futures list
-                futures[
-                    exe.submit(
-                        self.map_resource_gids,
-                        gid_set,
+            for gid_batch in gid_batches:
+                futures = {}
+                # iterate through split executions, submitting each to worker
+                for i, gid in enumerate(gid_batch):
+                    # submit executions and append to futures list
+                    excl_coords = self._get_excl_coords(
                         self._excl_fpath,
+                        gid,
                         self._sc_row_indices,
                         self._sc_col_indices,
                         self._excl_row_slices,
                         self._excl_col_slices,
-                        self._tree,
-                        self.distance_threshold,
                     )
-                ] = i
+                    futures[
+                        exe.submit(
+                            self.map_resource_gids,
+                            excl_coords,
+                            tree,
+                            dist_thresh,
+                        )
+                    ] = i
+
+                with h5py.File(self._excl_fpath, "a") as f:
+                    indices = f[dset]
+                    for future in as_completed(futures):
+                        i = futures[future]
+                        result = future.result()
+
+                        gid = gid_batch[i]
+                        row_slice, col_slice = self._get_excl_slices(
+                            gid,
+                            self._sc_row_indices,
+                            self._sc_col_indices,
+                            self._excl_row_slices,
+                            self._excl_col_slices,
+                        )
+                        n_rows = row_slice.stop - row_slice.start
+                        n_cols = col_slice.stop - col_slice.start
+                        result_shape = (n_rows, n_cols)
+                        indices[row_slice, col_slice] = result.reshape(
+                            result_shape
+                        )
 
-            n_finished = 0
-            for future in as_completed(futures):
                 n_finished += 1
                 logger.info(
-                    "Parallel TechMapping futures collected: "
-                    "{} out of {}".format(n_finished, len(futures))
+                    "Parallel TechMapping batches completed: "
+                    f"{n_finished} out of {n_batches}"
                 )
 
-                i = futures[future]
-                result = future.result()
-
-                for j, gid in enumerate(gid_chunks[i]):
-                    row_slice, col_slice = self._get_excl_slices(
-                        gid,
-                        self._sc_row_indices,
-                        self._sc_col_indices,
-                        self._excl_row_slices,
-                        self._excl_col_slices,
-                    )
-                    ind_slice = iarr[row_slice, col_slice].flatten()
-                    indices[ind_slice] = result[j]
-
-        indices = indices.reshape(self._excl_shape)
-
-        return indices
-
     @classmethod
     def run(
         cls,
         excl_fpath,
         res_fpath,
-        dset=None,
-        sc_resolution=2560,
+        dset,
+        sc_resolution=1200,
         dist_margin=1.05,
         max_workers=None,
-        points_per_worker=10,
+        batch_size=1000,
     ):
         """Run parallel mapping and save to h5 file.
 
         Parameters
         ----------
         excl_fpath : str
-            Filepath to exclusions h5 (tech layer). dset will be
-            created in excl_fpath.
+            Filepath to exclusions data HDF5 file. This file must must contain
+            latitude and longitude datasets.
         res_fpath : str
-            Filepath to .h5 resource file that we're mapping to.
-        dset : str, optional
-            Dataset name in excl_fpath to save mapping results to, if None
-            do not save tech_map to excl_fpath, by default None
+            Filepath to HDF5 resource file (e.g. WTK or NSRDB) to which
+            the exclusions will be mapped. Can refer to a single file (e.g.,
+            "/path/to/nsrdb_2024.h5" or a wild-card e.g.,
+            "/path/to/nsrdb_{}.h5")
+        dset : str
+            Dataset name in the `excl_fpath` file to which the the
+            techmap (exclusions-to-resource mapping data) will be saved.
+
+            .. Important:: If this dataset already exists in the h5 file,
+              it will be overwritten.
+
         sc_resolution : int | None, optional
-            Supply curve resolution, does not affect the exclusion to resource
-            (tech) mapping, but defines how many exclusion pixels are mapped
-            at a time, by default 2560
+            Defines how many exclusion pixels are mapped at a time. Units
+            indicate the length of one dimension, in pixels, of each square
+            chunk to be mapped. By default, this value is 1200, which will
+            map the exclusion pixels in 1200x1200 pixel chunks.
+
+            .. Note:: This parameter does not affect the exclusion to resource
+            (tech) mapping, which deviates from how the effect of the
+            ``sc_resolution`` parameter works in other functionality within
+            ``reV``.
+
         dist_margin : float, optional
             Extra margin to multiply times the computed distance between
             neighboring resource points, by default 1.05
         max_workers : int, optional
             Number of cores to run mapping on. None uses all available cpus,
             by default None
-        points_per_worker : int, optional
-            Number of supply curve points to map to resource gids on each
-            worker, by default 10
-
-        Returns
-        -------
-        indices : np.ndarray
-            Index values of the NN resource point. -1 if no res point found.
-            2D integer array with shape equal to the exclusions extent shape.
+        batch_size : int, optional
+            Number of tasks to be submitted to parallel worker pool at one
+            time, by default 1000. As a rule of thumb, this number should be
+            set to ~10x the number of max_workers. Higher values are not
+            necessarily better, and may slow down processing and/or result in
+            out-of-memory errors. Values less than the number of workers can
+            also lead to slower processing, due to poor load balancing.
         """
-        kwargs = {"dist_margin": dist_margin, "sc_resolution": sc_resolution}
-        mapper = cls(excl_fpath, res_fpath, **kwargs)
-        indices = mapper.map_resource(
-            max_workers=max_workers, points_per_worker=points_per_worker
+        mapper = cls(excl_fpath, sc_resolution=sc_resolution)
+        mapper.initialize_dataset(dset)
+        mapper.map_resource(
+            dset=dset,
+            res_fpath=res_fpath,
+            dist_margin=dist_margin,
+            max_workers=max_workers,
+            batch_size=batch_size,
         )
-
-        if dset:
-            mapper.save_tech_map(
-                excl_fpath,
-                dset,
-                indices,
-                distance_threshold=mapper.distance_threshold,
-                res_fpath=res_fpath,
-            )
-
-        return indices

reV/utilities/__init__.py

@@ -149,14 +149,16 @@ class SupplyCurveField(FieldEnum):
     MEAN_LCOE_FRICTION = "lcoe_friction_usd_per_mwh"
     RAW_LCOE = "lcoe_raw_usd_per_mwh"
     EOS_MULT = "multiplier_cc_eos"
+    FIXED_EOS_MULT = "multiplier_foc_eos"
+    VAR_EOS_MULT = "multiplier_voc_eos"
     REG_MULT = "multiplier_cc_regional"
     SC_POINT_ANNUAL_ENERGY_MWH = "annual_energy_site_mwh"
-    COST_BASE_OCC_USD_PER_AC_MW = "cost_base_occ_usd_per_ac_mw"
-    COST_SITE_OCC_USD_PER_AC_MW = "cost_site_occ_usd_per_ac_mw"
+    COST_BASE_CC_USD_PER_AC_MW = "cost_base_cc_usd_per_ac_mw"
+    COST_SITE_CC_USD_PER_AC_MW = "cost_site_cc_usd_per_ac_mw"
     COST_BASE_FOC_USD_PER_AC_MW = "cost_base_foc_usd_per_ac_mw"
     COST_SITE_FOC_USD_PER_AC_MW = "cost_site_foc_usd_per_ac_mw"
-    COST_BASE_VOC_USD_PER_AC_MW = "cost_base_voc_usd_per_ac_mw"
-    COST_SITE_VOC_USD_PER_AC_MW = "cost_site_voc_usd_per_ac_mw"
+    COST_BASE_VOC_USD_PER_AC_MWH = "cost_base_voc_usd_per_ac_mwh"
+    COST_SITE_VOC_USD_PER_AC_MWH = "cost_site_voc_usd_per_ac_mwh"
     FIXED_CHARGE_RATE = "fixed_charge_rate"
 
     # Bespoke outputs
@@ -266,6 +268,8 @@ class _LegacySCAliases(Enum):
     CONVEX_HULL_AREA = "convex_hull_area"
     CONVEX_HULL_CAPACITY_DENSITY = "convex_hull_capacity_density"
     FULL_CELL_CAPACITY_DENSITY = "full_cell_capacity_density"
+    COST_BASE_CC_USD_PER_AC_MW = "cost_base_occ_usd_per_ac_mw"
+    COST_SITE_CC_USD_PER_AC_MW = "cost_site_occ_usd_per_ac_mw"
 
 
 class ModuleName(str, Enum):
@@ -293,6 +297,7 @@ class ModuleName(str, Enum):
     REP_PROFILES = "rep-profiles"
     SUPPLY_CURVE = "supply-curve"
     SUPPLY_CURVE_AGGREGATION = "supply-curve-aggregation"
+    TECH_MAPPING = "tech-mapping"
 
     def __str__(self):
         return self.value

reV/version.py

@@ -2,4 +2,4 @@
 reV Version number
 """
 
-__version__ = "0.13.1"
+__version__ = "0.14.1"