NREL-erad 0.0.0a0__py3-none-any.whl → 0.1.1__py3-none-any.whl

erad/utils/ditto_utils.py

@@ -1,252 +0,0 @@
-""" Utility functions for dealing with SMART DS dataset. 
-
-Examples:
-
-    >>> from erad import ditto_utils
-    >>> ditto_utils.download_smartds_data('P4R',  '.')
-
-"""
-
-# standard libraries
-from pathlib import Path
-import shutil
-import logging
-from typing import List
-
-# third-party libraries
-import boto3
-from botocore import UNSIGNED
-from botocore.config import Config
-from ditto.store import Store
-from ditto.readers.opendss.read import Reader
-from ditto.network.network import Network
-from ditto.models.power_source import PowerSource
-import networkx as nx
-from networkx.readwrite import json_graph
-
-
-# internal libraries
-from erad.constants import SMARTDS_VALID_AREAS, SMARTDS_VALID_YEARS
-from erad.exceptions import SMARTDSInvalidInput, DittoException
-from erad.utils.util import timeit, write_file, path_validation
-from erad.utils.util import read_file, setup_logging
-
-
-logger = logging.getLogger(__name__)
-
-
-@timeit
-def download_aws_dir(
-    bucket: str, path: str, target: str, unsigned=True, **kwargs
-) -> None:
-    """Utility function download data from AWS S3 directory.
-
-    Args:
-        bucket (str): Name of the bucket.
-        path (str): S3 bucket prefix
-        target (str): Path for downloading the data
-        unsigned (bool): Indicate whether to use credential or not
-        kwargs (dict): Keyword arguments accepted by `boto3.client`
-    """
-
-    target = Path(target)
-    if unsigned:
-        client = boto3.client("s3", config=Config(signature_version=UNSIGNED))
-    else:
-        if kwargs:
-            client = boto3.client("s3", **kwargs)
-        else:
-            client = boto3.client("s3")
-
-    # Handle missing / at end of prefix
-    if not path.endswith("/"):
-        path += "/"
-
-    paginator = client.get_paginator("list_objects_v2")
-    for result in paginator.paginate(Bucket=bucket, Prefix=path):
-
-        # Download each file individually
-        for key in result["Contents"]:
-
-            # Calculate relative path
-            rel_path = key["Key"][len(path) :]
-
-            # Skip paths ending in /
-            if not key["Key"].endswith("/"):
-                local_file_path = target / rel_path
-                local_file_path.parent.mkdir(parents=True, exist_ok=True)
-                client.download_file(bucket, key["Key"], str(local_file_path))
-
-
-@timeit
-def download_smartds_data(
-    smartds_region: str,
-    output_path: str = "./smart_ds_downloads",
-    year: int = 2018,
-    area: str = "SFO",
-    s3_bucket_name: str = "oedi-data-lake",
-    folder_name: str = "opendss_no_loadshapes",
-    cache_folder: str = "cache",
-) -> str:
-    """Utility function to download SMARTDS data from AWS S3 bucket.
-
-    Args:
-        smartds_region (str): SMARTDS region name
-        output_path (str): Path for downloaded data
-        year (int): Valid year input for downloading the data
-        area (str): Valid SMARTDS area
-        s3_bucket_name (str): S3 bucket name storing the SMARTDS data
-        folder_name (str): S3 bucket folder to download
-        cache_folder (str): Folder path for caching the results
-
-    Raises:
-        SMARTDSInvalidInput: Raises this error if year and/or area
-            provided is not valid.
-
-    Returns:
-        str: Folder path containing downloaded data.
-    """
-    if year not in SMARTDS_VALID_YEARS or area not in SMARTDS_VALID_AREAS:
-        raise SMARTDSInvalidInput(
-            f"Not valid input! year= {year} area={area}, \
-            valid_years={SMARTDS_VALID_YEARS}, valid_areas={SMARTDS_VALID_AREAS}"
-        )
-
-    output_path = Path(output_path)
-    cache_folder = Path(cache_folder)
-
-    output_path.mkdir(exist_ok=True)
-    cache_folder.mkdir(exist_ok=True)
-
-    cache_key = (
-        f"{smartds_region}__{year}__{area}__{s3_bucket_name}_{folder_name}"
-    )
-    cache_data_folder = cache_folder / cache_key
-    output_folder = output_path / cache_key
-
-    if cache_data_folder.exists():
-        logger.info(f"Cache hit for {cache_data_folder}")
-        shutil.copytree(cache_data_folder, output_folder, dirs_exist_ok=True)
-
-    else:
-        logger.info(
-            f"Cache missed reaching to AWS for downloading the data ..."
-        )
-        output_folder.mkdir(exist_ok=True)
-        prefix = f"SMART-DS/v1.0/{year}/{area}/{smartds_region}/scenarios/base_timeseries/{folder_name}/"
-        download_aws_dir(s3_bucket_name, prefix, output_folder)
-        shutil.copytree(output_folder, cache_data_folder, dirs_exist_ok=False)
-
-    logger.info(f"Check the folder {output_folder} for downloaded data")
-    return output_folder
-
-
-@timeit
-def _create_networkx_from_ditto(
-    output_path: str, file_name: str, **kwargs
-) -> List:
-    """Creates networkx graph from OpenDSS model using Ditto.
-
-    Args:
-        output_path (str): Path to store the networkx
-            data in json file format
-        file_name (str): JSON file name used to export
-            the network
-        kwargs (dict): Keyword arguments accepted
-            by Ditto
-
-    Raises:
-        DittoException: Raises if multiple sources are found.
-
-
-    Returns:
-        List: Pair of networkx graph and
-            path containing JSON file
-    """
-    file_name = Path(file_name).stem
-    logger.debug(
-        "Attempting to create NetworkX representation from OpenDSS \
-        files using DiTTo"
-    )
-
-    path_validation(output_path)
-
-    store = Store()
-    reader = Reader(
-        master_file=kwargs["master_file"],
-        buscoordinates_file=kwargs["buscoordinates_file"],
-        coordinates_delimiter=kwargs["coordinates_delimiter"],
-    )
-    reader.parse(store)
-
-    all_sources = []
-    for i in store.models:
-        if isinstance(i, PowerSource) and i.connecting_element is not None:
-            all_sources.append(i)
-        elif isinstance(i, PowerSource):
-            print(
-                "Warning - a PowerSource element has a None connecting element"
-            )
-
-    if len(all_sources) > 1:
-        raise DittoException(
-            f"This feeder has lots of sources {len(all_sources)}"
-        )
-
-    ditto_graph = Network()
-    ditto_graph.build(store, all_sources[0].connecting_element)
-    ditto_graph.set_attributes(store)
-
-    data = dict(ditto_graph.graph.nodes.data())
-    data_new = {}
-    for node, node_data in data.items():
-        try:
-            data_new[node] = node_data["positions"][0]._trait_values
-        except Exception as e:
-            connecting_node = node_data["connecting_element"]
-            data_new[node] = data[connecting_node]["positions"][0]._trait_values
-
-    adj_file = file_name + ".adjlist"
-    nx.write_adjlist(ditto_graph.graph, output_path / adj_file)
-    g = nx.read_adjlist(output_path / adj_file)
-    nx.set_node_attributes(g, data_new)
-
-    data = json_graph.adjacency_data(g)
-    json_file = file_name + ".json"
-    output_file = output_path / json_file
-    write_file(data, output_file)
-
-    logger.debug(
-        f"Successfully created json file representing the network \
-        check the file {output_file}"
-    )
-
-    return (g, output_file)
-
-
-def create_networkx_from_ditto(
-    output_path: str, file_name: str, **kwargs
-) -> None:
-    """Creates networkx graph from OpenDSS model using Ditto.
-
-    Args:
-        output_path (str): Path to store the networkx
-            data in json file format
-        file_name (str): JSON file name used to export
-            the network
-        kwargs (dict): Keyword arguments accepted
-            by Ditto
-    """
-    try:
-        output_path = Path(output_path)
-        return _create_networkx_from_ditto(output_path, file_name, **kwargs)
-    finally:
-        for file_path in output_path.iterdir():
-            if file_path.suffix == ".adjlist":
-                file_path.unlink(missing_ok=True)
-
-
-def create_networkx_from_json(json_file_path: str):
-    """Returns networkx graph from JSON file."""
-    content = read_file(json_file_path)
-    return json_graph.adjacency_graph(content)

erad/utils/hifld_utils.py

@@ -1,147 +0,0 @@
-""" Module for parsing Homeland infrastructure foundation level-data.
-
-Idea is to take the bounding box and find the subset of 
-infrastructure in that region.
-"""
-# standard imports
-from pathlib import Path
-import math
-from typing import Union, List
-
-# third-party imports
-import pandas as pd
-import stateplane
-
-# internal imports
-from erad.utils.util import path_validation
-
-
-def get_subset_of_hifld_data(
-    csv_file: str,
-    bounds: List,
-    output_folder: str,
-    logitude_column_name: str = "X",
-    latitude_column_name: str = "Y",
-    columns_to_keep: List[str] = ["X", "Y"],
-    name_of_csv_file: Union[str, None] = None,
-) -> None:
-    """Extracts a subset of HIFLD data set.
-
-    Args:
-        csv_file (str): Path to HIFLD data csv file
-        bounds (List): Bounding box coordinates
-        output_folder (str): Path to output folder
-        logitude_column_name (str): Expects column with name 'X'
-        latitude_column_name (str): Expects column with name 'Y'
-        columns_to_keep (List): List of column names to keep
-            by default keeps all of them
-       name_of_csv_file (Union[str, None]): Name of csv file to export
-            filtered set
-    """
-
-    # Unpacking the bounds data
-    longitude_min, latitude_min, longitude_max, latitude_max = bounds
-
-    # Do a path validation
-    csv_file = Path(csv_file)
-    output_folder = Path(output_folder)
-    path_validation(csv_file, check_for_file=True, check_for_file_type=".csv")
-    path_validation(output_folder)
-
-    # Reading the hifld csv data
-    df = pd.read_csv(csv_file)
-
-    # filtering for bounds
-    df_filtered = df[
-        (df[logitude_column_name] >= longitude_min)
-        & (df[logitude_column_name] <= longitude_max)
-        & (df[latitude_column_name] >= latitude_min)
-        & (df[latitude_column_name] <= latitude_max)
-    ]
-
-    # Keep only the limited columns
-    df_subset = df_filtered[columns_to_keep]
-
-    # export the subset
-    file_name = name_of_csv_file if name_of_csv_file else csv_file.name
-    df_subset.to_csv(output_folder / file_name)
-
-
-def get_relationship_between_hifld_infrastructures(
-    hifld_data_csv: str,
-    unique_id_column: str,
-    load_csv: str,
-    bus_csv: str,
-    output_csv_path: str,
-    distance_threshold: float = 2000.0,
-):
-    """Creates a relationship between consumers and HIFLD infrastructures.
-
-    Args:
-        hifld_data_csv (str): Path to filtered HIFLD data csv file
-        unique_id_column (List): Column name used as identifier
-            for critical infrastructures
-        load_csv (str): Path to load csv file
-        bus_csv (str): Path to bus csv file
-        output_csv_path (str): output csv path for storing relationship csv
-        distance_threshold (float): Distance threshold used for mapping
-            customer to critical infrastructure
-    """
-    hifld_data_csv = Path(hifld_data_csv)
-    bus_csv = Path(bus_csv)
-    load_csv = Path(load_csv)
-    output_csv_path = Path(output_csv_path)
-
-    path_validation(
-        hifld_data_csv, check_for_file=True, check_for_file_type=".csv"
-    )
-    path_validation(bus_csv, check_for_file=True, check_for_file_type=".csv")
-    path_validation(load_csv, check_for_file=True, check_for_file_type=".csv")
-    path_validation(output_csv_path.parents[0])
-
-    hifld_data_df = pd.read_csv(hifld_data_csv)
-    load_df = pd.read_csv(load_csv)
-    bus_df = pd.read_csv(bus_csv)
-
-    merged_data = pd.merge(
-        load_df, bus_df, how="left", left_on="source", right_on="name"
-    ).to_dict(orient="records")
-
-    # Container for storing shelter relationships
-    _relationship = []
-    for _record in hifld_data_df.to_dict(orient="records"):
-        _lon, _lat = _record["LONGITUDE"], _record["LATITUDE"]
-
-        # convert into state plane coordinates
-        _lon_translated, _lat_translated = stateplane.from_lonlat(_lon, _lat)
-
-        # Loop through all the loads
-        for load_record in merged_data:
-
-            load_lon, load_lat = (
-                load_record["longitude"],
-                load_record["latitude"],
-            )
-
-            # convert into state plane coordinates
-            load_lon_translated, load_lat_translated = stateplane.from_lonlat(
-                load_lon, load_lat
-            )
-
-            # computes distance
-            distance = math.sqrt(
-                (_lat_translated - load_lat_translated) ** 2
-                + (_lon_translated - load_lon_translated) ** 2
-            )
-
-            if distance < distance_threshold:
-                _relationship.append(
-                    {
-                        unique_id_column: _record[unique_id_column],
-                        "load_name": load_record["name_x"],
-                        "distance": distance,
-                    }
-                )
-
-    df = pd.DataFrame(_relationship)
-    df.to_csv(output_csv_path)