NREL-erad 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

erad/scenarios/utilities.py

@@ -0,0 +1,76 @@
+from shapely.geometry import MultiPolygon, Point, LineString
+from shapely.ops import nearest_points
+import matplotlib.pyplot as plt
+import scipy.stats as stats
+import geopy.distance
+import numpy as np
+import stateplane
+
+
+class GeoUtilities:
+    
+    @property
+    def identify_stateplane_projection(self) -> str:
+        """ Automatically identifies stateplane projection ID  """ 
+        x = self.centroid.x
+        y = self.centroid.y
+        return stateplane.identify(x, y)
+    
+    def in_polygon(self, point : Point) -> bool:
+        return self.multipolygon.contains(point)
+
+    def distance_from_boundary(self, point : Point) -> float:
+        """ Calculates distance of a point to polygon boundary. Correct calculations require conversion to cartesian coordinates""" 
+        if self.multipolygon.contains(point):
+            p1, p2 = nearest_points(self.boundary, point)
+        else:
+            p1, p2 = nearest_points(self.multipolygon, point)    
+        coords_1 = (p1.y, p1.x)
+        coords_2 = (p2.y, p2.x)
+        return geopy.distance.geodesic(coords_1, coords_2).km    
+
+    def distance_from_centroid(self, point : Point):
+        """ Calculates distance of a point to polygon centroid. Correct calculations require conversion to cartesian coordinates """ 
+        coords_1 = (self.centroid.y, self.centroid.x)
+        coords_2 = (point.y, point.x)
+        return geopy.distance.geodesic(coords_1, coords_2).km
+    
+
+
+class ProbabilityFunctionBuilder:
+    """Class containing utility fuctions for sceario definations."""
+    
+    
+    def __init__(self, dist, params):
+        """Constructor for BaseScenario class.
+
+        Args:
+            dist (str): Name of teh distribution. Should follow Scipy naming convention
+            params (list): A list of parameters for the chosen distribution function. See Scipy.stats documentation
+        """
+        
+        self.dist = getattr(stats, dist)
+        self.params = params
+        return 
+
+    def sample(self):
+        """Sample the distribution """
+        return self.dist.rvs(*self.params, size=1)[0]
+
+    def plot_cdf(self, x:np.linspace, ax =None, label="") -> None:
+        """Plot the cumalative distribution fuction"""
+        cdf = self.dist.cdf
+        if ax is None:
+            plt.plot(x,cdf(x, *self.params), label=label)
+        else:
+            ax.plot(x,cdf(x, *self.params), label=label)
+    
+
+    def probability(self, value: float) -> float:
+        """Calculates survival probability of a given asset.
+
+        Args:
+            value (float): value for vetor of interest. Will change with scenarions
+        """
+        cdf = self.dist.cdf
+        return cdf(value, *self.params)

erad/scenarios/wind_scenario.py

@@ -0,0 +1,89 @@
+from erad.constants import FIRE_HISTORIC_GEODATAFRAME_PATH, DATA_FOLDER
+from shapely.geometry import MultiPolygon, Point, LineString
+from erad.scenarios.utilities import ProbabilityFunctionBuilder
+from erad.scenarios.abstract_scenario import BaseScenario
+from erad.exceptions import FeatureNotImplementedError
+from erad.scenarios.utilities import GeoUtilities
+import matplotlib.pyplot as plt
+from datetime import datetime
+import geopandas as gpd
+import numpy as np
+import random
+import pyproj
+import os
+
+from erad.scenarios.common import AssetTypes
+from erad.scenarios.utilities import ProbabilityFunctionBuilder
+
+
+class WindScenario(BaseScenario, GeoUtilities): 
+    """Base class for FireScenario. Extends BaseScenario and GeoUtilities
+
+    Attributes:
+        multipolygon (MultiPolygon): MultiPolygon enclosing wild fire regions 
+        probability_model (dict): Dictionary mapping asset types to probability funcitons
+        timestamp (datetime): Scenario occurance time 
+    """
+    
+    fragility_curves = {
+        #Extending energy system modelling to include extreme weather risks and application to hurricane events in Puerto Rico
+        AssetTypes.substation.name : ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+        AssetTypes.solar_panels.name :  ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+        AssetTypes.buried_lines.name :  ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+        AssetTypes.wind_turbines.name :  ProbabilityFunctionBuilder("norm", [0.8, 10, 5]),
+        #AssetTypes.battery_storage.name : ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+        #AssetTypes.transmission_poles.name  :   ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+        AssetTypes.distribution_poles.name  :  ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+       # AssetTypes.transmission_overhead_lines.name  : ProbabilityFunctionBuilder("lognorm", [0.8, 10, 5]),
+        AssetTypes.distribution_overhead_lines.name  :  ProbabilityFunctionBuilder("beta", [0.8, 10, 5]),
+    }
+    
+    def __init__(self,  multipolygon : MultiPolygon , probability_model : dict, timestamp : datetime) -> None:
+        """Constructor for FireScenario.
+
+        Args:
+            multipolygon (MultiPolygon):  MultiPolygon enclosing wild fire regions 
+            probability_model (dict): Dictionary mapping asset types to probability funcitons
+            timestamp (datetime): Scenario occurance time 
+        """
+
+        super(WindScenario, self).__init__(multipolygon, probability_model, timestamp)     
+        return      
+    
+    @property
+    def area(self) -> float:
+        """Method to calculate area of affected region."""
+        geod = pyproj.Geod(ellps="WGS84")
+        area = abs(geod.geometry_area_perimeter(self.polygon)[0])
+        return area
+    
+    @property
+    def polygon(self) -> MultiPolygon:
+        """Method to return polygon for the affected region."""
+        return self.multipolygon
+        
+    @property
+    def boundary(self) -> LineString:
+        """Method to return boundary for the affected region."""
+        return self.multipolygon.boundary
+        
+    @property
+    def centroid(self) -> Point:
+        """Method to return the centroid of the affected region."""
+        return self.polygon.centroid
+            
+    def increment_time(self):
+        """Method to increment simulation time for time evolviong scenarios."""
+        raise FeatureNotImplementedError()
+
+    def calculate_survival_probability(self, assets : dict, timestamp : datetime, plot: bool) -> dict:
+        """Method to calculate survival probaility of asset types.
+
+        Args:
+            assets (dict): The dictionary of all assets and their corresponding asset types
+            plot (bool): Set to true to plot the fire survival model
+        """     
+        return assets
+                
+
+

erad/utils/ditto_utils.py

@@ -0,0 +1,252 @@
+""" Utility functions for dealing with SMART DS dataset. 
+
+Examples:
+
+    >>> from erad import ditto_utils
+    >>> ditto_utils.download_smartds_data('P4R',  '.')
+
+"""
+
+# standard libraries
+from pathlib import Path
+import shutil
+import logging
+from typing import List
+
+# third-party libraries
+import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
+from ditto.store import Store
+from ditto.readers.opendss.read import Reader
+from ditto.network.network import Network
+from ditto.models.power_source import PowerSource
+import networkx as nx
+from networkx.readwrite import json_graph
+
+
+# internal libraries
+from erad.constants import SMARTDS_VALID_AREAS, SMARTDS_VALID_YEARS
+from erad.exceptions import SMARTDSInvalidInput, DittoException
+from erad.utils.util import timeit, write_file, path_validation
+from erad.utils.util import read_file, setup_logging
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def download_aws_dir(
+    bucket: str, path: str, target: str, unsigned=True, **kwargs
+) -> None:
+    """Utility function download data from AWS S3 directory.
+
+    Args:
+        bucket (str): Name of the bucket.
+        path (str): S3 bucket prefix
+        target (str): Path for downloading the data
+        unsigned (bool): Indicate whether to use credential or not
+        kwargs (dict): Keyword arguments accepted by `boto3.client`
+    """
+
+    target = Path(target)
+    if unsigned:
+        client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+    else:
+        if kwargs:
+            client = boto3.client("s3", **kwargs)
+        else:
+            client = boto3.client("s3")
+
+    # Handle missing / at end of prefix
+    if not path.endswith("/"):
+        path += "/"
+
+    paginator = client.get_paginator("list_objects_v2")
+    for result in paginator.paginate(Bucket=bucket, Prefix=path):
+
+        # Download each file individually
+        for key in result["Contents"]:
+
+            # Calculate relative path
+            rel_path = key["Key"][len(path) :]
+
+            # Skip paths ending in /
+            if not key["Key"].endswith("/"):
+                local_file_path = target / rel_path
+                local_file_path.parent.mkdir(parents=True, exist_ok=True)
+                client.download_file(bucket, key["Key"], str(local_file_path))
+
+
+@timeit
+def download_smartds_data(
+    smartds_region: str,
+    output_path: str = "./smart_ds_downloads",
+    year: int = 2018,
+    area: str = "SFO",
+    s3_bucket_name: str = "oedi-data-lake",
+    folder_name: str = "opendss_no_loadshapes",
+    cache_folder: str = "cache",
+) -> str:
+    """Utility function to download SMARTDS data from AWS S3 bucket.
+
+    Args:
+        smartds_region (str): SMARTDS region name
+        output_path (str): Path for downloaded data
+        year (int): Valid year input for downloading the data
+        area (str): Valid SMARTDS area
+        s3_bucket_name (str): S3 bucket name storing the SMARTDS data
+        folder_name (str): S3 bucket folder to download
+        cache_folder (str): Folder path for caching the results
+
+    Raises:
+        SMARTDSInvalidInput: Raises this error if year and/or area
+            provided is not valid.
+
+    Returns:
+        str: Folder path containing downloaded data.
+    """
+    if year not in SMARTDS_VALID_YEARS or area not in SMARTDS_VALID_AREAS:
+        raise SMARTDSInvalidInput(
+            f"Not valid input! year= {year} area={area}, \
+            valid_years={SMARTDS_VALID_YEARS}, valid_areas={SMARTDS_VALID_AREAS}"
+        )
+
+    output_path = Path(output_path)
+    cache_folder = Path(cache_folder)
+
+    output_path.mkdir(exist_ok=True)
+    cache_folder.mkdir(exist_ok=True)
+
+    cache_key = (
+        f"{smartds_region}__{year}__{area}__{s3_bucket_name}_{folder_name}"
+    )
+    cache_data_folder = cache_folder / cache_key
+    output_folder = output_path / cache_key
+
+    if cache_data_folder.exists():
+        logger.info(f"Cache hit for {cache_data_folder}")
+        shutil.copytree(cache_data_folder, output_folder, dirs_exist_ok=True)
+
+    else:
+        logger.info(
+            f"Cache missed reaching to AWS for downloading the data ..."
+        )
+        output_folder.mkdir(exist_ok=True)
+        prefix = f"SMART-DS/v1.0/{year}/{area}/{smartds_region}/scenarios/base_timeseries/{folder_name}/"
+        download_aws_dir(s3_bucket_name, prefix, output_folder)
+        shutil.copytree(output_folder, cache_data_folder, dirs_exist_ok=False)
+
+    logger.info(f"Check the folder {output_folder} for downloaded data")
+    return output_folder
+
+
+@timeit
+def _create_networkx_from_ditto(
+    output_path: str, file_name: str, **kwargs
+) -> List:
+    """Creates networkx graph from OpenDSS model using Ditto.
+
+    Args:
+        output_path (str): Path to store the networkx
+            data in json file format
+        file_name (str): JSON file name used to export
+            the network
+        kwargs (dict): Keyword arguments accepted
+            by Ditto
+
+    Raises:
+        DittoException: Raises if multiple sources are found.
+
+
+    Returns:
+        List: Pair of networkx graph and
+            path containing JSON file
+    """
+    file_name = Path(file_name).stem
+    logger.debug(
+        "Attempting to create NetworkX representation from OpenDSS \
+        files using DiTTo"
+    )
+
+    path_validation(output_path)
+
+    store = Store()
+    reader = Reader(
+        master_file=kwargs["master_file"],
+        buscoordinates_file=kwargs["buscoordinates_file"],
+        coordinates_delimiter=kwargs["coordinates_delimiter"],
+    )
+    reader.parse(store)
+
+    all_sources = []
+    for i in store.models:
+        if isinstance(i, PowerSource) and i.connecting_element is not None:
+            all_sources.append(i)
+        elif isinstance(i, PowerSource):
+            print(
+                "Warning - a PowerSource element has a None connecting element"
+            )
+
+    if len(all_sources) > 1:
+        raise DittoException(
+            f"This feeder has lots of sources {len(all_sources)}"
+        )
+
+    ditto_graph = Network()
+    ditto_graph.build(store, all_sources[0].connecting_element)
+    ditto_graph.set_attributes(store)
+
+    data = dict(ditto_graph.graph.nodes.data())
+    data_new = {}
+    for node, node_data in data.items():
+        try:
+            data_new[node] = node_data["positions"][0]._trait_values
+        except Exception as e:
+            connecting_node = node_data["connecting_element"]
+            data_new[node] = data[connecting_node]["positions"][0]._trait_values
+
+    adj_file = file_name + ".adjlist"
+    nx.write_adjlist(ditto_graph.graph, output_path / adj_file)
+    g = nx.read_adjlist(output_path / adj_file)
+    nx.set_node_attributes(g, data_new)
+
+    data = json_graph.adjacency_data(g)
+    json_file = file_name + ".json"
+    output_file = output_path / json_file
+    write_file(data, output_file)
+
+    logger.debug(
+        f"Successfully created json file representing the network \
+        check the file {output_file}"
+    )
+
+    return (g, output_file)
+
+
+def create_networkx_from_ditto(
+    output_path: str, file_name: str, **kwargs
+) -> None:
+    """Creates networkx graph from OpenDSS model using Ditto.
+
+    Args:
+        output_path (str): Path to store the networkx
+            data in json file format
+        file_name (str): JSON file name used to export
+            the network
+        kwargs (dict): Keyword arguments accepted
+            by Ditto
+    """
+    try:
+        output_path = Path(output_path)
+        return _create_networkx_from_ditto(output_path, file_name, **kwargs)
+    finally:
+        for file_path in output_path.iterdir():
+            if file_path.suffix == ".adjlist":
+                file_path.unlink(missing_ok=True)
+
+
+def create_networkx_from_json(json_file_path: str):
+    """Returns networkx graph from JSON file."""
+    content = read_file(json_file_path)
+    return json_graph.adjacency_graph(content)

erad/utils/hifld_utils.py

@@ -0,0 +1,147 @@
+""" Module for parsing Homeland infrastructure foundation level-data.
+
+Idea is to take the bounding box and find the subset of 
+infrastructure in that region.
+"""
+# standard imports
+from pathlib import Path
+import math
+from typing import Union, List
+
+# third-party imports
+import pandas as pd
+import stateplane
+
+# internal imports
+from erad.utils.util import path_validation
+
+
+def get_subset_of_hifld_data(
+    csv_file: str,
+    bounds: List,
+    output_folder: str,
+    logitude_column_name: str = "X",
+    latitude_column_name: str = "Y",
+    columns_to_keep: List[str] = ["X", "Y"],
+    name_of_csv_file: Union[str, None] = None,
+) -> None:
+    """Extracts a subset of HIFLD data set.
+
+    Args:
+        csv_file (str): Path to HIFLD data csv file
+        bounds (List): Bounding box coordinates
+        output_folder (str): Path to output folder
+        logitude_column_name (str): Expects column with name 'X'
+        latitude_column_name (str): Expects column with name 'Y'
+        columns_to_keep (List): List of column names to keep
+            by default keeps all of them
+       name_of_csv_file (Union[str, None]): Name of csv file to export
+            filtered set
+    """
+
+    # Unpacking the bounds data
+    longitude_min, latitude_min, longitude_max, latitude_max = bounds
+
+    # Do a path validation
+    csv_file = Path(csv_file)
+    output_folder = Path(output_folder)
+    path_validation(csv_file, check_for_file=True, check_for_file_type=".csv")
+    path_validation(output_folder)
+
+    # Reading the hifld csv data
+    df = pd.read_csv(csv_file)
+
+    # filtering for bounds
+    df_filtered = df[
+        (df[logitude_column_name] >= longitude_min)
+        & (df[logitude_column_name] <= longitude_max)
+        & (df[latitude_column_name] >= latitude_min)
+        & (df[latitude_column_name] <= latitude_max)
+    ]
+
+    # Keep only the limited columns
+    df_subset = df_filtered[columns_to_keep]
+
+    # export the subset
+    file_name = name_of_csv_file if name_of_csv_file else csv_file.name
+    df_subset.to_csv(output_folder / file_name)
+
+
+def get_relationship_between_hifld_infrastructures(
+    hifld_data_csv: str,
+    unique_id_column: str,
+    load_csv: str,
+    bus_csv: str,
+    output_csv_path: str,
+    distance_threshold: float = 2000.0,
+):
+    """Creates a relationship between consumers and HIFLD infrastructures.
+
+    Args:
+        hifld_data_csv (str): Path to filtered HIFLD data csv file
+        unique_id_column (List): Column name used as identifier
+            for critical infrastructures
+        load_csv (str): Path to load csv file
+        bus_csv (str): Path to bus csv file
+        output_csv_path (str): output csv path for storing relationship csv
+        distance_threshold (float): Distance threshold used for mapping
+            customer to critical infrastructure
+    """
+    hifld_data_csv = Path(hifld_data_csv)
+    bus_csv = Path(bus_csv)
+    load_csv = Path(load_csv)
+    output_csv_path = Path(output_csv_path)
+
+    path_validation(
+        hifld_data_csv, check_for_file=True, check_for_file_type=".csv"
+    )
+    path_validation(bus_csv, check_for_file=True, check_for_file_type=".csv")
+    path_validation(load_csv, check_for_file=True, check_for_file_type=".csv")
+    path_validation(output_csv_path.parents[0])
+
+    hifld_data_df = pd.read_csv(hifld_data_csv)
+    load_df = pd.read_csv(load_csv)
+    bus_df = pd.read_csv(bus_csv)
+
+    merged_data = pd.merge(
+        load_df, bus_df, how="left", left_on="source", right_on="name"
+    ).to_dict(orient="records")
+
+    # Container for storing shelter relationships
+    _relationship = []
+    for _record in hifld_data_df.to_dict(orient="records"):
+        _lon, _lat = _record["LONGITUDE"], _record["LATITUDE"]
+
+        # convert into state plane coordinates
+        _lon_translated, _lat_translated = stateplane.from_lonlat(_lon, _lat)
+
+        # Loop through all the loads
+        for load_record in merged_data:
+
+            load_lon, load_lat = (
+                load_record["longitude"],
+                load_record["latitude"],
+            )
+
+            # convert into state plane coordinates
+            load_lon_translated, load_lat_translated = stateplane.from_lonlat(
+                load_lon, load_lat
+            )
+
+            # computes distance
+            distance = math.sqrt(
+                (_lat_translated - load_lat_translated) ** 2
+                + (_lon_translated - load_lon_translated) ** 2
+            )
+
+            if distance < distance_threshold:
+                _relationship.append(
+                    {
+                        unique_id_column: _record[unique_id_column],
+                        "load_name": load_record["name_x"],
+                        "distance": distance,
+                    }
+                )
+
+    df = pd.DataFrame(_relationship)
+    df.to_csv(output_csv_path)