NREL-erad 0.0.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. NREL_erad-0.0.0a0.dist-info/LICENSE.txt +29 -0
  2. NREL_erad-0.0.0a0.dist-info/METADATA +61 -0
  3. NREL_erad-0.0.0a0.dist-info/RECORD +42 -0
  4. NREL_erad-0.0.0a0.dist-info/WHEEL +5 -0
  5. NREL_erad-0.0.0a0.dist-info/top_level.txt +1 -0
  6. erad/__init__.py +0 -0
  7. erad/constants.py +20 -0
  8. erad/cypher_queries/load_data_v1.cypher +212 -0
  9. erad/data/World_Earthquakes_1960_2016.csv +23410 -0
  10. erad/db/__init__.py +0 -0
  11. erad/db/assets/__init__.py +0 -0
  12. erad/db/assets/critical_infras.py +171 -0
  13. erad/db/assets/distribution_lines.py +101 -0
  14. erad/db/credential_model.py +20 -0
  15. erad/db/disaster_input_model.py +23 -0
  16. erad/db/inject_earthquake.py +52 -0
  17. erad/db/inject_flooding.py +53 -0
  18. erad/db/neo4j_.py +162 -0
  19. erad/db/utils.py +14 -0
  20. erad/exceptions.py +68 -0
  21. erad/metrics/__init__.py +0 -0
  22. erad/metrics/check_microgrid.py +208 -0
  23. erad/metrics/metric.py +178 -0
  24. erad/programs/__init__.py +0 -0
  25. erad/programs/backup.py +62 -0
  26. erad/programs/microgrid.py +45 -0
  27. erad/scenarios/__init__.py +0 -0
  28. erad/scenarios/abstract_scenario.py +103 -0
  29. erad/scenarios/common.py +93 -0
  30. erad/scenarios/earthquake_scenario.py +161 -0
  31. erad/scenarios/fire_scenario.py +160 -0
  32. erad/scenarios/flood_scenario.py +494 -0
  33. erad/scenarios/utilities.py +76 -0
  34. erad/scenarios/wind_scenario.py +89 -0
  35. erad/utils/__init__.py +0 -0
  36. erad/utils/ditto_utils.py +252 -0
  37. erad/utils/hifld_utils.py +147 -0
  38. erad/utils/opendss_utils.py +357 -0
  39. erad/utils/overpass.py +76 -0
  40. erad/utils/util.py +178 -0
  41. erad/visualization/__init__.py +0 -0
  42. erad/visualization/plot_graph.py +218 -0
@@ -0,0 +1,252 @@
1
+ """ Utility functions for dealing with SMART DS dataset.
2
+
3
+ Examples:
4
+
5
+ >>> from erad import ditto_utils
6
+ >>> ditto_utils.download_smartds_data('P4R', '.')
7
+
8
+ """
9
+
10
+ # standard libraries
11
+ from pathlib import Path
12
+ import shutil
13
+ import logging
14
+ from typing import List
15
+
16
+ # third-party libraries
17
+ import boto3
18
+ from botocore import UNSIGNED
19
+ from botocore.config import Config
20
+ from ditto.store import Store
21
+ from ditto.readers.opendss.read import Reader
22
+ from ditto.network.network import Network
23
+ from ditto.models.power_source import PowerSource
24
+ import networkx as nx
25
+ from networkx.readwrite import json_graph
26
+
27
+
28
+ # internal libraries
29
+ from erad.constants import SMARTDS_VALID_AREAS, SMARTDS_VALID_YEARS
30
+ from erad.exceptions import SMARTDSInvalidInput, DittoException
31
+ from erad.utils.util import timeit, write_file, path_validation
32
+ from erad.utils.util import read_file, setup_logging
33
+
34
+
35
+ logger = logging.getLogger(__name__)
36
+
37
+
38
+ @timeit
39
+ def download_aws_dir(
40
+ bucket: str, path: str, target: str, unsigned=True, **kwargs
41
+ ) -> None:
42
+ """Utility function download data from AWS S3 directory.
43
+
44
+ Args:
45
+ bucket (str): Name of the bucket.
46
+ path (str): S3 bucket prefix
47
+ target (str): Path for downloading the data
48
+ unsigned (bool): Indicate whether to use credential or not
49
+ kwargs (dict): Keyword arguments accepted by `boto3.client`
50
+ """
51
+
52
+ target = Path(target)
53
+ if unsigned:
54
+ client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
55
+ else:
56
+ if kwargs:
57
+ client = boto3.client("s3", **kwargs)
58
+ else:
59
+ client = boto3.client("s3")
60
+
61
+ # Handle missing / at end of prefix
62
+ if not path.endswith("/"):
63
+ path += "/"
64
+
65
+ paginator = client.get_paginator("list_objects_v2")
66
+ for result in paginator.paginate(Bucket=bucket, Prefix=path):
67
+
68
+ # Download each file individually
69
+ for key in result["Contents"]:
70
+
71
+ # Calculate relative path
72
+ rel_path = key["Key"][len(path) :]
73
+
74
+ # Skip paths ending in /
75
+ if not key["Key"].endswith("/"):
76
+ local_file_path = target / rel_path
77
+ local_file_path.parent.mkdir(parents=True, exist_ok=True)
78
+ client.download_file(bucket, key["Key"], str(local_file_path))
79
+
80
+
81
+ @timeit
82
+ def download_smartds_data(
83
+ smartds_region: str,
84
+ output_path: str = "./smart_ds_downloads",
85
+ year: int = 2018,
86
+ area: str = "SFO",
87
+ s3_bucket_name: str = "oedi-data-lake",
88
+ folder_name: str = "opendss_no_loadshapes",
89
+ cache_folder: str = "cache",
90
+ ) -> str:
91
+ """Utility function to download SMARTDS data from AWS S3 bucket.
92
+
93
+ Args:
94
+ smartds_region (str): SMARTDS region name
95
+ output_path (str): Path for downloaded data
96
+ year (int): Valid year input for downloading the data
97
+ area (str): Valid SMARTDS area
98
+ s3_bucket_name (str): S3 bucket name storing the SMARTDS data
99
+ folder_name (str): S3 bucket folder to download
100
+ cache_folder (str): Folder path for caching the results
101
+
102
+ Raises:
103
+ SMARTDSInvalidInput: Raises this error if year and/or area
104
+ provided is not valid.
105
+
106
+ Returns:
107
+ str: Folder path containing downloaded data.
108
+ """
109
+ if year not in SMARTDS_VALID_YEARS or area not in SMARTDS_VALID_AREAS:
110
+ raise SMARTDSInvalidInput(
111
+ f"Not valid input! year= {year} area={area}, \
112
+ valid_years={SMARTDS_VALID_YEARS}, valid_areas={SMARTDS_VALID_AREAS}"
113
+ )
114
+
115
+ output_path = Path(output_path)
116
+ cache_folder = Path(cache_folder)
117
+
118
+ output_path.mkdir(exist_ok=True)
119
+ cache_folder.mkdir(exist_ok=True)
120
+
121
+ cache_key = (
122
+ f"{smartds_region}__{year}__{area}__{s3_bucket_name}_{folder_name}"
123
+ )
124
+ cache_data_folder = cache_folder / cache_key
125
+ output_folder = output_path / cache_key
126
+
127
+ if cache_data_folder.exists():
128
+ logger.info(f"Cache hit for {cache_data_folder}")
129
+ shutil.copytree(cache_data_folder, output_folder, dirs_exist_ok=True)
130
+
131
+ else:
132
+ logger.info(
133
+ f"Cache missed reaching to AWS for downloading the data ..."
134
+ )
135
+ output_folder.mkdir(exist_ok=True)
136
+ prefix = f"SMART-DS/v1.0/{year}/{area}/{smartds_region}/scenarios/base_timeseries/{folder_name}/"
137
+ download_aws_dir(s3_bucket_name, prefix, output_folder)
138
+ shutil.copytree(output_folder, cache_data_folder, dirs_exist_ok=False)
139
+
140
+ logger.info(f"Check the folder {output_folder} for downloaded data")
141
+ return output_folder
142
+
143
+
144
+ @timeit
145
+ def _create_networkx_from_ditto(
146
+ output_path: str, file_name: str, **kwargs
147
+ ) -> List:
148
+ """Creates networkx graph from OpenDSS model using Ditto.
149
+
150
+ Args:
151
+ output_path (str): Path to store the networkx
152
+ data in json file format
153
+ file_name (str): JSON file name used to export
154
+ the network
155
+ kwargs (dict): Keyword arguments accepted
156
+ by Ditto
157
+
158
+ Raises:
159
+ DittoException: Raises if multiple sources are found.
160
+
161
+
162
+ Returns:
163
+ List: Pair of networkx graph and
164
+ path containing JSON file
165
+ """
166
+ file_name = Path(file_name).stem
167
+ logger.debug(
168
+ "Attempting to create NetworkX representation from OpenDSS \
169
+ files using DiTTo"
170
+ )
171
+
172
+ path_validation(output_path)
173
+
174
+ store = Store()
175
+ reader = Reader(
176
+ master_file=kwargs["master_file"],
177
+ buscoordinates_file=kwargs["buscoordinates_file"],
178
+ coordinates_delimiter=kwargs["coordinates_delimiter"],
179
+ )
180
+ reader.parse(store)
181
+
182
+ all_sources = []
183
+ for i in store.models:
184
+ if isinstance(i, PowerSource) and i.connecting_element is not None:
185
+ all_sources.append(i)
186
+ elif isinstance(i, PowerSource):
187
+ print(
188
+ "Warning - a PowerSource element has a None connecting element"
189
+ )
190
+
191
+ if len(all_sources) > 1:
192
+ raise DittoException(
193
+ f"This feeder has lots of sources {len(all_sources)}"
194
+ )
195
+
196
+ ditto_graph = Network()
197
+ ditto_graph.build(store, all_sources[0].connecting_element)
198
+ ditto_graph.set_attributes(store)
199
+
200
+ data = dict(ditto_graph.graph.nodes.data())
201
+ data_new = {}
202
+ for node, node_data in data.items():
203
+ try:
204
+ data_new[node] = node_data["positions"][0]._trait_values
205
+ except Exception as e:
206
+ connecting_node = node_data["connecting_element"]
207
+ data_new[node] = data[connecting_node]["positions"][0]._trait_values
208
+
209
+ adj_file = file_name + ".adjlist"
210
+ nx.write_adjlist(ditto_graph.graph, output_path / adj_file)
211
+ g = nx.read_adjlist(output_path / adj_file)
212
+ nx.set_node_attributes(g, data_new)
213
+
214
+ data = json_graph.adjacency_data(g)
215
+ json_file = file_name + ".json"
216
+ output_file = output_path / json_file
217
+ write_file(data, output_file)
218
+
219
+ logger.debug(
220
+ f"Successfully created json file representing the network \
221
+ check the file {output_file}"
222
+ )
223
+
224
+ return (g, output_file)
225
+
226
+
227
+ def create_networkx_from_ditto(
228
+ output_path: str, file_name: str, **kwargs
229
+ ) -> None:
230
+ """Creates networkx graph from OpenDSS model using Ditto.
231
+
232
+ Args:
233
+ output_path (str): Path to store the networkx
234
+ data in json file format
235
+ file_name (str): JSON file name used to export
236
+ the network
237
+ kwargs (dict): Keyword arguments accepted
238
+ by Ditto
239
+ """
240
+ try:
241
+ output_path = Path(output_path)
242
+ return _create_networkx_from_ditto(output_path, file_name, **kwargs)
243
+ finally:
244
+ for file_path in output_path.iterdir():
245
+ if file_path.suffix == ".adjlist":
246
+ file_path.unlink(missing_ok=True)
247
+
248
+
249
+ def create_networkx_from_json(json_file_path: str):
250
+ """Returns networkx graph from JSON file."""
251
+ content = read_file(json_file_path)
252
+ return json_graph.adjacency_graph(content)
@@ -0,0 +1,147 @@
1
+ """ Module for parsing Homeland infrastructure foundation level-data.
2
+
3
+ Idea is to take the bounding box and find the subset of
4
+ infrastructure in that region.
5
+ """
6
+ # standard imports
7
+ from pathlib import Path
8
+ import math
9
+ from typing import Union, List
10
+
11
+ # third-party imports
12
+ import pandas as pd
13
+ import stateplane
14
+
15
+ # internal imports
16
+ from erad.utils.util import path_validation
17
+
18
+
19
+ def get_subset_of_hifld_data(
20
+ csv_file: str,
21
+ bounds: List,
22
+ output_folder: str,
23
+ logitude_column_name: str = "X",
24
+ latitude_column_name: str = "Y",
25
+ columns_to_keep: List[str] = ["X", "Y"],
26
+ name_of_csv_file: Union[str, None] = None,
27
+ ) -> None:
28
+ """Extracts a subset of HIFLD data set.
29
+
30
+ Args:
31
+ csv_file (str): Path to HIFLD data csv file
32
+ bounds (List): Bounding box coordinates
33
+ output_folder (str): Path to output folder
34
+ logitude_column_name (str): Expects column with name 'X'
35
+ latitude_column_name (str): Expects column with name 'Y'
36
+ columns_to_keep (List): List of column names to keep
37
+ by default keeps all of them
38
+ name_of_csv_file (Union[str, None]): Name of csv file to export
39
+ filtered set
40
+ """
41
+
42
+ # Unpacking the bounds data
43
+ longitude_min, latitude_min, longitude_max, latitude_max = bounds
44
+
45
+ # Do a path validation
46
+ csv_file = Path(csv_file)
47
+ output_folder = Path(output_folder)
48
+ path_validation(csv_file, check_for_file=True, check_for_file_type=".csv")
49
+ path_validation(output_folder)
50
+
51
+ # Reading the hifld csv data
52
+ df = pd.read_csv(csv_file)
53
+
54
+ # filtering for bounds
55
+ df_filtered = df[
56
+ (df[logitude_column_name] >= longitude_min)
57
+ & (df[logitude_column_name] <= longitude_max)
58
+ & (df[latitude_column_name] >= latitude_min)
59
+ & (df[latitude_column_name] <= latitude_max)
60
+ ]
61
+
62
+ # Keep only the limited columns
63
+ df_subset = df_filtered[columns_to_keep]
64
+
65
+ # export the subset
66
+ file_name = name_of_csv_file if name_of_csv_file else csv_file.name
67
+ df_subset.to_csv(output_folder / file_name)
68
+
69
+
70
+ def get_relationship_between_hifld_infrastructures(
71
+ hifld_data_csv: str,
72
+ unique_id_column: str,
73
+ load_csv: str,
74
+ bus_csv: str,
75
+ output_csv_path: str,
76
+ distance_threshold: float = 2000.0,
77
+ ):
78
+ """Creates a relationship between consumers and HIFLD infrastructures.
79
+
80
+ Args:
81
+ hifld_data_csv (str): Path to filtered HIFLD data csv file
82
+ unique_id_column (List): Column name used as identifier
83
+ for critical infrastructures
84
+ load_csv (str): Path to load csv file
85
+ bus_csv (str): Path to bus csv file
86
+ output_csv_path (str): output csv path for storing relationship csv
87
+ distance_threshold (float): Distance threshold used for mapping
88
+ customer to critical infrastructure
89
+ """
90
+ hifld_data_csv = Path(hifld_data_csv)
91
+ bus_csv = Path(bus_csv)
92
+ load_csv = Path(load_csv)
93
+ output_csv_path = Path(output_csv_path)
94
+
95
+ path_validation(
96
+ hifld_data_csv, check_for_file=True, check_for_file_type=".csv"
97
+ )
98
+ path_validation(bus_csv, check_for_file=True, check_for_file_type=".csv")
99
+ path_validation(load_csv, check_for_file=True, check_for_file_type=".csv")
100
+ path_validation(output_csv_path.parents[0])
101
+
102
+ hifld_data_df = pd.read_csv(hifld_data_csv)
103
+ load_df = pd.read_csv(load_csv)
104
+ bus_df = pd.read_csv(bus_csv)
105
+
106
+ merged_data = pd.merge(
107
+ load_df, bus_df, how="left", left_on="source", right_on="name"
108
+ ).to_dict(orient="records")
109
+
110
+ # Container for storing shelter relationships
111
+ _relationship = []
112
+ for _record in hifld_data_df.to_dict(orient="records"):
113
+ _lon, _lat = _record["LONGITUDE"], _record["LATITUDE"]
114
+
115
+ # convert into state plane coordinates
116
+ _lon_translated, _lat_translated = stateplane.from_lonlat(_lon, _lat)
117
+
118
+ # Loop through all the loads
119
+ for load_record in merged_data:
120
+
121
+ load_lon, load_lat = (
122
+ load_record["longitude"],
123
+ load_record["latitude"],
124
+ )
125
+
126
+ # convert into state plane coordinates
127
+ load_lon_translated, load_lat_translated = stateplane.from_lonlat(
128
+ load_lon, load_lat
129
+ )
130
+
131
+ # computes distance
132
+ distance = math.sqrt(
133
+ (_lat_translated - load_lat_translated) ** 2
134
+ + (_lon_translated - load_lon_translated) ** 2
135
+ )
136
+
137
+ if distance < distance_threshold:
138
+ _relationship.append(
139
+ {
140
+ unique_id_column: _record[unique_id_column],
141
+ "load_name": load_record["name_x"],
142
+ "distance": distance,
143
+ }
144
+ )
145
+
146
+ df = pd.DataFrame(_relationship)
147
+ df.to_csv(output_csv_path)