dwind 0.3__py3-none-any.whl → 0.3.2__py3-none-any.whl

dwind/scenarios.py

@@ -1,48 +1,80 @@
+"""Provides the scenario-specific mapping for varying financial and model configuration data."""
+
 import json
 from pathlib import Path
 
 import pandas as pd
 
+from dwind.config import Year, Scenario
 
-def config_nem(scenario, year):
-    # NEM_SCENARIO_CSV
-    nem_opt_scens = ["highrecost", "lowrecost", "re100"]
-    # nem_opt_scens = ['der_value_HighREcost', 'der_value_LowREcost', 're_100']
-    if scenario in nem_opt_scens:
-        nem_scenario_csv = "nem_optimistic_der_value_2035.csv"
-    elif scenario == "baseline" and year in (2022, 2025, 2035):
-        nem_scenario_csv = f"nem_baseline_{year}.csv"
-    else:
-        nem_scenario_csv = "nem_baseline_2035.csv"
 
-    return nem_scenario_csv
+def config_nem(scenario: Scenario, year: Year) -> str:
+    """Provides NEM configuration based on :py:attr:`scenario` and :py:attr:`year`.
 
+    Args:
+        scenario (:py:class:`dwind.config.Scenario`): Valid :py:class:`dwind.config.Scenario`.
+        year (:py:class:`dwind.config.Year`): Valid :py:class:`dwind.config.Year`.
 
-def config_cambium(scenario):
-    # CAMBIUM_SCENARIO
-    if scenario == "highrecost" or scenario == "re100":
-        cambium_scenario = "StdScen20_HighRECost"
-    elif scenario == "lowrecost":
-        cambium_scenario = "StdScen20_LowRECost"
-    else:
-        # cambium_scenario = "StdScen20_MidCase"
-        cambium_scenario = "Cambium23_MidCase"
+    Returns:
+        str: Name of the NEM scenario file to use.
+    """
+    if scenario in (Scenario.HIGHRECOST, Scenario.LOWRECOST, Scenario.RE100):
+        return "nem_optimistic_der_value_2035.csv"
+
+    if scenario is Scenario.BASELINE and year in (Year._2022, Year._2025, Year._2035):
+        return f"nem_baseline_{year.value}.csv"
+
+    return "nem_baseline_2035.csv"
+
+
+def config_cambium(scenario: Scenario) -> str:
+    """Loads the cambium configuration name based on :py:attr:`scenario`.
+
+    Args:
+        scenario (:py:class:`dwind.config.Scenario`): Valid :py:class:`dwind.config.Scenario`.
+
+    Returns:
+        str: Name of the Cambium scenario to use.
+    """
+    if scenario in (Scenario.HIGHRECOST, Scenario.RE100):
+        return "StdScen20_HighRECost"
 
-    return cambium_scenario
+    if scenario is Scenario.LOWRECOST:
+        return "StdScen20_LowRECost"
 
+    return "Cambium23_MidCase"
 
-def config_costs(scenario, year):
-    # COST_INPUTS
-    f = Path(f"/projects/dwind/configs/costs/atb24/ATB24_costs_{scenario}_{year}.json").resolve()
+
+def config_costs(scenario: Scenario, year: Year) -> dict:
+    """Loads the cost configuration based on the ATB analysis.
+
+    Args:
+        scenario (:py:class:`dwind.config.Scenario`): Valid :py:class:`dwind.config.Scenario`.
+        year (:py:class:`dwind.config.Year`): Valid :py:class:`dwind.config.Year`.
+
+    Returns:
+        dict: Dictionary of ATB assumptions to be used for PySAM's cost inputs.
+    """
+    f = Path(
+        f"/projects/dwind/configs/costs/atb24/ATB24_costs_{scenario.value}_{year.value}.json"
+    ).resolve()
     with f.open("r") as f_in:
         cost_inputs = json.load(f_in)
 
     return cost_inputs
 
 
-def config_performance(scenario, year):
-    # PERFORMANCE_INPUTS
-    if scenario == "baseline" and year == 2022:
+def config_performance(scenario: Scenario, year: Year) -> pd.DataFrame:
+    """Loads the technology performance configurations.
+
+    Args:
+        scenario (:py:class:`dwind.config.Scenario`): Valid :py:class:`dwind.config.Scenario`.
+        year (:py:class:`dwind.config.Year`): Valid :py:class:`dwind.config.Year`.
+
+    Returns:
+        pd.DataFrame: Performance data based on the scale of each technology.
+    """
+    if scenario is Scenario.BASELINE and year is Year._2022:
         performance_inputs = {
             "solar": pd.DataFrame(
                 [
@@ -108,15 +140,21 @@ def config_performance(scenario, year):
     return performance_inputs
 
 
-def config_financial(scenario, year):
-    # FINANCIAL_INPUTS
-    scenarios = ("baseline", "metering", "billing")
-    if scenario in scenarios and year == 2025:
+def config_financial(scenario: Scenario, year: Year) -> dict:
+    """Loads the financial configuration based on the ATB analysis.
+
+    Args:
+        scenario (:py:class:`dwind.config.Scenario`): Valid :py:class:`dwind.config.Scenario`.
+        year (:py:class:`dwind.config.Year`): Valid :py:class:`dwind.config.Year`.
+
+    Returns:
+        dict: Dictionary of ATB assumptions to be used for configuration PySAM.
+    """
+    if year is Year._2025:
         f = f"/projects/dwind/configs/costs/atb24/ATB24_financing_baseline_{year}.json"
-        i = Path("/projects/dwind/data/incentives/2025_incentives.json").resolve()
-        with i.open("r") as i_in:
-            incentives = json.load(i_in)
-    elif scenario in scenarios and year in (2035, 2040):
+        i = Path("/projects/dwind/data/incentives/2025_incentives.pqt").resolve()
+        incentives = pd.read_parquet(i, dtype_backend="pyarrow")
+    elif year in (Year._2035, Year._2040):
         f = "/projects/dwind/configs/costs/atb24/ATB24_financing_baseline_2035.json"
     else:
         # use old assumptions
@@ -125,6 +163,8 @@ def config_financial(scenario, year):
 
     with f.open("r") as f_in:
         financials = json.load(f_in)
+
+    # TODO: determine if shared settings is applicable going forward, or separate should be reserved
     if year == 2025:
         financials["BTM"]["itc_fraction_of_capex"] = incentives
         financials["FOM"]["itc_fraction_of_capex"] = incentives

dwind/utils/array.py

@@ -0,0 +1,99 @@
+"""Provides a series of generic NumPy and Pandas utility functions."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def memory_downcaster(df: pd.DataFrame | pd.Series) -> pd.DataFrame | pd.Series:
+    """Downcasts ``int`` and ``float`` columns to the lowest memory alternative possible. For
+    integers this means converting to either signed or unsigned 8-, 16-, 32-, or 64-bit integers,
+    and for floats, converting to ``np.float32``.
+
+    Args:
+        df (pd.DataFrame | pd.Series): DataFrame or Series to have its memory footprint reduced.
+
+    Returns:
+        pd.DataFrame | pd.Series: Reduced footprint version of the passed :py:attr:`df`.
+    """
+    # if not isinstance(df, pd.DataFrame | pd.Series):
+    if not isinstance(df, (pd.DataFrame, pd.Series)):  # noqa
+        raise TypeError("Input value must be a Pandas DataFrame or Series.")
+
+    NAlist = []
+    for col in df.select_dtypes(include=[np.number]).columns:
+        IsInt = False
+        mx = df[col].max()
+        mn = df[col].min()
+
+        # integer does not support na; fill na
+        if not np.isfinite(df[col]).all():
+            NAlist.append(col)
+            df[col].fillna(mn - 1, inplace=True)
+
+        # test if column can be converted to an integer
+        asint = df[col].fillna(0).astype(np.int64)
+        result = df[col] - asint
+        result = result.sum()
+        if result > -0.01 and result < 0.01:
+            IsInt = True
+
+        # make integer/unsigned integer datatypes
+        if IsInt:
+            try:
+                if mn >= 0:
+                    if mx < 255:
+                        df[col] = df[col].astype(np.uint8)
+                    elif mx < 65535:
+                        df[col] = df[col].astype(np.uint16)
+                    elif mx < 4294967295:
+                        df[col] = df[col].astype(np.uint32)
+                    else:
+                        df[col] = df[col].astype(np.uint64)
+                else:
+                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:
+                        df[col] = df[col].astype(np.int8)
+                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:
+                        df[col] = df[col].astype(np.int16)
+                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:
+                        df[col] = df[col].astype(np.int32)
+                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:
+                        df[col] = df[col].astype(np.int64)
+            except:  # noqa: E722
+                df[col] = df[col].astype(np.float32)
+
+        # make float datatypes 32 bit
+        else:
+            df[col] = df[col].astype(np.float32)
+
+    return df
+
+
+def split_by_index(
+    arr: pd.DataFrame | np.ndarray | pd.Series, n_splits: int
+) -> tuple[np.ndarray, np.ndarray]:
+    """Split a DataFrame, Series, or array like with np.array_split, but only return the start and
+    stop indices, rather than chunks. For Pandas objects, this are equivalent to
+    ``arr.iloc[start: end]`` and for NumPy: ``arr[start: end]``. Splits are done according
+    to the 0th dimension.
+
+    Args:
+        arr(pd.DataFrame | pd.Series | np.ndarray): The array, data frame, or series to split.
+        n_splits(:obj:`int`): The number of near equal or equal splits.
+
+    Returns:
+        tuple[np.ndarray, np.ndarray]
+    """
+    size = arr.shape[0]
+    base = np.arange(n_splits)
+    split_size = size // n_splits
+    extra = size % n_splits
+
+    starts = base * split_size
+    ends = starts + split_size
+
+    for i in range(extra):
+        ends[i:] += 1
+        starts[i + 1 :] += 1
+    return starts, ends

dwind/utils/hpc.py

@@ -0,0 +1,138 @@
+"""Provides the live timing table functionalities for the Kestrel :py:class:`MultiProcess` class."""
+
+from __future__ import annotations
+
+import io
+import re
+import time
+import subprocess
+from copy import deepcopy
+
+import pandas as pd
+from rich.table import Table
+from rex.utilities.hpc import SLURM
+
+
+def convert_seconds_for_print(time: float) -> str:
+    """Convert number of seconds to number of hours, minutes, and seconds."""
+    div = ((60, "seconds"), (60, "minutes"), (24, "hours"))
+
+    result = []
+    value = time
+    for divisor, label in div:
+        if not divisor:
+            remainder = value
+            if not remainder:
+                break
+        else:
+            value, remainder = divmod(value, divisor)
+            if not value and not remainder:
+                break
+        if remainder == 1:
+            label = label[:-1]
+
+        # 0.2 second precision for seconds, and no decimals otherwise
+        if result:
+            result.append(f"{remainder:,.0f} {label}")
+        else:
+            result.append(f"{remainder:.1f} {label}")
+    if result:
+        return ", ".join(reversed(result))
+    return "0"
+
+
+def update_status(job_status: dict) -> dict:
+    """Get an updated status and timing statistics for all running jobs on the HPC.
+
+    Args:
+        job_status (dict): Dictionary of job id (primary key) with sub keys of "status",
+            "start_time" (initial or start of run status), "wait", and "run".
+
+    Returns:
+        dict: Dictionary of updated statuses and timing statistics for all current queued and
+            running jobs.
+    """
+    slurm = SLURM()
+    update = {}
+    for job, vals in job_status.items():
+        original_status = vals["status"]
+        if original_status in ("CG", "CF", "None", None):
+            continue
+        new_status = slurm.check_status(job_id=job)
+        if new_status == "PD":
+            update[job] = vals | {"status": new_status, "wait": time.perf_counter() - vals["start"]}
+        elif new_status == "R":
+            if original_status != "R":
+                update[job] = vals | {
+                    "status": new_status,
+                    "wait": time.perf_counter() - vals["start"],
+                    "start": time.perf_counter(),
+                }
+            else:
+                update[job] = vals | {"run": time.perf_counter() - vals["start"]}
+        elif new_status in ("CG", "CF", "None", None):
+            update[job] = vals | {"status": new_status, "run": time.perf_counter() - vals["start"]}
+        else:
+            raise ValueError(f"Unaccounted for status code: {new_status}")
+    return update
+
+
+def generate_run_status_table(job_status: dict) -> tuple[Table, bool]:
+    """Generate the job status run time statistics table.
+
+    Args:
+        job_status (dict): Dictionary of job id (primary key) with sub keys of "status",
+            "start_time" (initial or start of run status), "wait", and "run".
+
+    Returns:
+        Table: ``rich.Table`` of human readable statistics.
+        bool: True if all jobs are complete, otherwise False.
+    """
+    table = Table()
+    table.add_column("Job ID")
+    table.add_column("Status")
+    table.add_column("Wait time")
+    table.add_column("Run time")
+
+    for job, vals in job_status.items():
+        status = vals["status"]
+        _wait = vals["wait"]
+        _run = vals["run"]
+        table.add_row(
+            job, status, convert_seconds_for_print(_wait), convert_seconds_for_print(_run)
+        )
+    done = all(el["status"] in ("CG", "CF", None) for el in job_status.values())
+    return table, done
+
+
+def get_finished_run_status(jobs: int | str | list[int | str]) -> dict[str, str]:
+    """Extracts a dictionary of job_id and status from the ``sacct`` output for a single
+    job or series of jobs.
+
+    Args:
+        jobs (int | str | list[int  |  str]): Single job ID or list of job IDs that have finished
+            running.
+
+    Returns:
+        dict[str, str]: Dictionary of {job_id_1: status_1, ..., job_id_N: status_N}.
+    """
+    if isinstance(jobs, (int, str)):  # noqa
+        jobs = [jobs]
+    jobs = [str(j) for j in jobs]
+
+    # Format the command to be in the form of [sacct, -j, job_id_1, ..., -j job_id_N]
+    command = deepcopy(jobs)
+    for i in range(len(command) - 1, -1, -1):
+        command.insert(i, "-j")
+    command.insert(0, "sacct")
+    results = subprocess.check_output(command)
+
+    # Convert the sacct string output to be table-like
+    buffer = io.StringIO(results.decode("utf8", "ignore"))
+    lines = [re.split(" +", line) for line in buffer.readlines() if not line.startswith("-")]
+
+    # Create a dataframe, and export a dictionary of the form job_id: job_status
+    df = pd.DataFrame(lines[1:], columns=lines[0])
+    df = df.loc[df.JobID.isin(jobs), ["JobID", "State"]]
+    df.JobID = df.JobID.astype(int)
+    return dict(df.values.tolist())

dwind/utils/loader.py

@@ -0,0 +1,63 @@
+"""Provides the core data loading methods for importing scenario data from flat files or SQL."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+from sqlalchemy import create_engine
+
+from dwind.config import Year
+
+
+def load_df(file_or_table: str | Path, year: Year | None, sql_constructor: str | None = None):
+    """Loads data from either a SQL table or file to a pandas ``DataFrame``.
+
+    Args:
+        file_or_table (str | Path): File name or path object, or SQL table where the data are
+            located.
+        year (:py:class:`dwind.config.Year`, optional): If used, only extracts the single year from
+        a column called "year". Defaults to None.
+        sql_constructor (str | None, optional): The SQL engine constructor string. Required if
+            extracting from SQL. Defaults to None.
+    """
+    valid_extenstions = (".csv", ".pqt", ".parquet", ".pkl", ".pickle")
+    if str(file_or_table).endswith(valid_extenstions):
+        return _load_from_file(filename=file_or_table, year=year)
+
+    return _load_from_sql(table=file_or_table, sql_constructor=sql_constructor, year=year)
+
+
+def _load_from_file(filename: str | Path, year: Year | None) -> pd.DataFrame:
+    """Loads tabular data from a file to a ``pandas.DataFrame``."""
+    if isinstance(filename, str):
+        filename = Path(filename).resolve()
+    if not isinstance(filename, Path):
+        raise TypeError(f"`filename` must be a valid path, not {filename=}")
+
+    if filename.suffix == ".csv":
+        df = pd.read_csv(filename, dtype_backend="pyarrow")
+    elif filename.suffix in (".parquet", ".pqt"):
+        df = pd.read_parquet(filename, dtype_backend="pyarrow")
+    elif filename.suffix in (".pickle", ".pkl"):
+        df = pd.read_pickle(filename, dtype_backend="pyarrow")
+    else:
+        raise ValueError(f"Only CSV, Parquet, and Pickle files allowed, not {filename=}")
+
+    if year is not None:
+        df = df.loc[df.year == year]
+
+    return df
+
+
+def _load_from_sql(table: str, sql_constructor: str, year: Year | None) -> pd.DataFrame:
+    """Load tabular data from SQL."""
+    where = f"where year = {year}" if year is not None else ""
+    sql = f"""select * from diffusion_shared."{table}" {where};"""
+    atlas_engine = create_engine(sql_constructor)
+
+    with atlas_engine.connect() as conn:
+        df = pd.read_sql(sql, con=conn.connection, dtype_backend="pyarrow")
+
+    atlas_engine.dispose()
+    return df

dwind/utils/progress.py

@@ -0,0 +1,60 @@
+import numpy as np
+from joblib import Parallel, delayed
+from threading import Thread
+from rich.progress import Progress, BarColumn, TimeRemainingColumn, TextColumn
+from rich.console import Console
+from rich.live import Live
+import time
+
+# Define the number of tasks and create a shared memory numpy array to hold their progress
+num_tasks = 4
+progress_array = np.memmap("progress.mmap2", dtype=np.float32, mode="w+", shape=N)
+
+# Define a function that performs a task and updates the progress array
+def perform_task(task_idx, progress_array):
+    for i in range(100):
+        # Do some work here
+        # ...
+
+        # Update the progress array
+        time.sleep(0.1)
+        progress_array[task_idx] = i / 100
+
+    # Update the progress array to 100% on completion
+    progress_array[task_idx] = 1
+
+# Define a function to continuously update the Rich progress bar
+def update_progress_bar(
+    progress_array=progress_array,
+    num_tasks=num_tasks,
+):
+    with Progress(
+        TextColumn("[bold blue]{task.fields[name]}"),
+        BarColumn(),
+        TextColumn("[bold green]{task.fields[status]}"),
+        TimeRemainingColumn(),
+        # console=console,
+    ) as progress:
+        tasks = [
+            progress.add_task(
+                description=f"Task {i}",
+                name=f"Task {i}",
+                status="pending",
+                total=100,
+            )
+            for i in range(num_tasks)
+        ]
+
+        while not all(progress_array == 1):
+            for i, task in enumerate(tasks):
+                progress.update(task, completed=int(progress_array[i] * 100))
+            time.sleep(0.1 * 2 ** abs(*np.random.randn(1)))
+
+
+# Launch the progress bar update function in a separate thread
+Thread(target=update_progress_bar, args=[progress_array, num_tasks]).start()
+
+# Launch the tasks in parallel using joblib and the perform_task function
+Parallel(n_jobs=-2, backend="loky")(
+    delayed(perform_task)(i, progress_array) for i in range(num_tasks)
+)