dwind 0.3__tar.gz → 0.3.1__tar.gz

{dwind-0.3 → dwind-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dwind
-Version: 0.3
+Version: 0.3.1
 Summary: Distributed Wind Generation Model
 Author-email: Jane Lockshin <jane.lockshin@nrel.gov>, Paritosh Das <paritosh.das@nrel.gov>, Rob Hammond <rob.hammond@nrel.gov>
 Project-URL: source, https://github.com/NREL/dwind

{dwind-0.3 → dwind-0.3.1}/dwind/__init__.py

@@ -1,3 +1,3 @@
 from .config import Configuration
 
-__version__ = "0.3"
+__version__ = "0.3.1"

{dwind-0.3 → dwind-0.3.1}/dwind/btm_sizing.py

@@ -5,7 +5,8 @@ import logging
 import numpy as np
 import pandas as pd
 
-from dwind import Configuration, helper
+from dwind import Configuration
+from dwind.utils import array
 
 
 log = logging.getLogger("dwfs")
@@ -117,7 +118,7 @@ def sizer(agents: pd.DataFrame, config: Configuration):
             agents.drop_duplicates(subset=["gid"], inplace=True)
 
         # make small
-        agents = helper.memory_downcaster(agents)
+        agents = array.memory_downcaster(agents)
 
         return agents
 

{dwind-0.3 → dwind-0.3.1}/dwind/config.py

@@ -2,11 +2,21 @@
 attributes.
 """
 
+from __future__ import annotations
+
 import re
-import tomllib
+import sys
 from pathlib import Path
 
 
+# fmt: off
+if sys.version_info >= (3, 11):  # noqa
+    import tomllib
+else:
+    import tomli as tomllib
+# fmt: on
+
+
 class Mapping(dict):
     """Dict-like class that allows for the use of dictionary style attribute calls on class
     attributes.
@@ -80,7 +90,7 @@ class Configuration(Mapping):
                 to read and convert. If passing a filename, it must be a TOML file.
             initial (bool, optional): Option to disable post-processing of configuration data.
         """
-        if isinstance(config, str | Path):
+        if isinstance(config, (str, Path)):  # noqa: UP038
             config = Path(config).resolve()
             with config.open("rb") as f:
                 config = tomllib.load(f)

{dwind-0.3 → dwind-0.3.1}/dwind/loader.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from pathlib import Path
 
 import pandas as pd
@@ -54,6 +56,7 @@ def _load_from_sql(table: str, sql_constructor: str, year: str | int | None) ->
     atlas_engine = create_engine(sql_constructor)
 
     with atlas_engine.connect() as conn:
-        pd.read_sql(sql, con=conn.connection)
+        df = pd.read_sql(sql, con=conn.connection)
 
     atlas_engine.dispose()
+    return df

{dwind-0.3 → dwind-0.3.1}/dwind/model.py

@@ -7,7 +7,8 @@ from pathlib import Path
 import numpy as np
 import pandas as pd
 
-from dwind import Configuration, helper, resource, scenarios, valuation, btm_sizing
+from dwind import Configuration, resource, scenarios, valuation, btm_sizing
+from dwind.utils import array
 
 
 # POTENTIALLY DANGEROUS!
@@ -34,11 +35,61 @@ class Agents:
         either a pickle file (.pkl or .pickle) or a parquet file (.pqt or .parquet).
     """
 
-    def __init__(self, agent_file: str | Path):
+    def __init__(
+        self,
+        agent_file: str | Path,
+        sector: str | None = None,
+        model_config: str | Path | None = None,
+        *,
+        resource_year: int = 2018,
+    ):
         self.agent_file = Path(agent_file).resolve()
-        self.load_agents()
-
-    def load_agents(self):
+        self.sector = sector
+        self.config = model_config
+        self.resource_year = resource_year
+        self._load_agents()
+
+    @classmethod
+    def load_and_prepare_agents(
+        cls,
+        agent_file: str | Path,
+        sector: str,
+        model_config: str | Path,
+        *,
+        save_results: bool = False,
+        file_name: str | Path | None = None,
+    ) -> pd.DataFrame:
+        """Load and prepare the agent files to run through ``Model``.
+
+        Args:
+            agent_file (str | Path): The full file path of the agent parquet, CSV, or pickle data.
+            save_results (bool, optional): True to save any updates to the data. Defaults to False.
+            file_name (str | Path | None, optional): The file path and name for where to save the
+                prepared data, if not overwriting the existing agent data. Defaults to None.
+
+        Returns:
+            pd.DataFrame: The prepared agent data.
+        """
+        agents = cls(agent_file, sector, model_config)
+        agents.prepare()
+        if save_results:
+            agents.save_agents(file_name=file_name)
+        return agents.agents
+
+    @classmethod
+    def load_agents(cls, agent_file: str | Path) -> pd.DataFrame:
+        """Load the agent data without making any additional modifications.
+
+        Args:
+            agent_file (str | Path): The full file path of the agent parquet, pickle, or CSV data.
+
+        Returns:
+            pd.DataFrame: The agent data.
+        """
+        agents = cls(agent_file)
+        return agents.agents
+
+    def _load_agents(self):
         """Loads in the agent file and drops any indices."""
         suffix = self.agent_file.suffix
         if suffix in (".pqt", ".parquet"):
@@ -56,20 +107,101 @@ class Agents:
         if suffix == ".csv":
             self.agents = self.agents.reset_index(drop=True)
 
+    def prepare(self):
+        """Prepares the agent data so that it has the necessary columns required for modeling.
+
+        Steps:
+        1. Extract `state_fips` from the `fips_code` column.
+        2. If `census_tract_id` is missing, load and merge the 2020 census tracts
+          based on the `pgid` column.
+        3. Convert the 2012 rev ID to the 2018 rev id in `rev_index_wind`.
+        4. Attach the universal resource generation data.
+        """
+        self.config = Configuration(self.config)
         if "state_fips" not in self.agents.columns:
-            self.agents["state_fips"] = self.agents["fips_code"].str[:2]
+            self.agents["state_fips"] = [el[:2] for el in self.agents["fips_code"]]
 
         if "census_tract_id" not in self.agents.columns:
-            census_tracts = pd.read_csv(
-                "/projects/dwind/configs/sizing/wind/lkup_block_to_pgid_2020.csv",
-                dtype={"fips_block": str, "pgid": str},
+            self.merge_census_data()
+
+        self.update_rev_id()
+        self.merge_generation()
+
+    def save_agents(self, file_name: str | Path | None = None):
+        if file_name is None:
+            file_name = self.agent_file
+
+        suffix = file_name.suffix
+        if suffix in (".pqt", ".parquet"):
+            file_saver = self.agents.to_parquet
+        elif suffix in (".pkl", ".pickle"):
+            file_saver = self.agents.to_pickle
+        elif suffix == ".csv":
+            file_saver = self.agents.to_csv
+        else:
+            raise ValueError(
+                f"File types ending in {suffix} can't be read as pickle, parquet, or CSV"
             )
-            census_tracts["census_tract_id"] = census_tracts["fips_block"].str[:11]
-            census_tracts = census_tracts[["pgid", "census_tract_id"]]
-            census_tracts = census_tracts.drop_duplicates()
-            self.agents = self.agents.merge(census_tracts, how="left", on="pgid")
-            self.agents = self.agents.drop_duplicates(subset=["gid"])
-            self.agents = self.agents.reset_index(drop=True)
+
+        file_saver(file_name)
+
+    def merge_census_data(self):
+        census_tracts = pd.read_csv(
+            "/projects/dwind/configs/sizing/wind/lkup_block_to_pgid_2020.csv",
+            usecols=["pgid", "fips_block"],
+            dtype=str,
+        ).drop_duplicates()
+        census_tracts["census_tract_id"] = [el[:11] for el in census_tracts["fips_block"]]
+        self.agents = (
+            self.agents.merge(census_tracts, how="left", on="pgid")
+            .drop_duplicates(subset=["gid"])
+            .reset_index(drop=True)
+        )
+
+    def update_rev_id(self, resource_year="2018"):
+        """Update 2012 rev index to 2018 index."""
+        if resource_year != "2018":
+            return
+
+        index_file = "/projects/dwind/configs/rev/wind/lkup_rev_index_2012_to_2018.csv"
+        rev_index_map = (
+            pd.read_csv(index_file, usecols=["rev_index_wind_2012", "rev_index_wind_2018"])
+            .rename(columns={"rev_index_wind_2012": "rev_index_wind"})
+            .set_index("rev_index_wind")
+        )
+
+        ix_original = self.agents.index.name
+        if ix_original is None:
+            self.agents = (
+                self.agents.set_index("rev_index_wind", drop=True)
+                .join(rev_index_map, how="left")
+                .reset_index(drop=True)
+                .rename(columns={"rev_index_wind_2018": "rev_index_wind"})
+                .dropna(subset="rev_index_wind")
+            )
+        else:
+            self.agents = (
+                self.agents.reset_index(drop=False)
+                .set_index("rev_index_wind")
+                .join(rev_index_map, how="left")
+                .set_index(ix_original, drop=True)
+                .rename(columns={"rev_index_wind_2018": "rev_index_wind"})
+                .dropna(subset="rev_index_wind")
+            )
+
+    def merge_generation(self):
+        if self.resource_year != "2018":
+            return
+
+        # update 2012 rev cf/naep/aep to 2018 values
+        # self.agents = self.agents.drop(columns=["wind_naep", "wind_cf", "wind_aep"])
+        resource_potential = resource.ResourcePotential(
+            parcels=self.agents,
+            application=self.sector,
+            year=self.resource_year,
+            model_config=self.model_config,
+        )
+        self.agents = resource_potential.match_rev_summary_to_agents()
 
 
 class Model:
@@ -126,33 +258,6 @@ class Model:
 
         self.log = logging.getLogger("dwfs")
 
-    def get_gen(self, resource_year="2018"):
-        if resource_year != "2018":
-            return
-
-        # update 2012 rev index to 2018 index
-        f = "/projects/dwind/configs/rev/wind/lkup_rev_index_2012_to_2018.csv"
-        lkup = pd.read_csv(f)[["rev_index_wind_2012", "rev_index_wind_2018"]]
-
-        self.agents = (
-            self.agents.merge(
-                lkup, left_on="rev_index_wind", right_on="rev_index_wind_2012", how="left"
-            )
-            .drop(columns=["rev_index_wind", "rev_index_wind_2012"])
-            .rename(columns={"rev_index_wind_2018": "rev_index_wind"})
-            .dropna(subset="rev_index_wind")
-        )
-
-        # update 2012 rev cf/naep/aep to 2018 values
-        # self.agents = self.agents.drop(columns=["wind_naep", "wind_cf", "wind_aep"])
-        resource_potential = resource.ResourcePotential(
-            parcels=self.agents,
-            application=self.sector,
-            year=resource_year,
-            model_config=self.config,
-        )
-        self.agents = resource_potential.match_rev_summary_to_agents()
-
     def get_rates(self):
         self.agents = self.agents[~self.agents["rate_id_alias"].isna()]
         self.agents["rate_id_alias"] = self.agents["rate_id_alias"].astype(int)
@@ -173,7 +278,7 @@ class Model:
         consumption_hourly = pd.read_parquet("/projects/dwind/data/crb_consumption_hourly.pqt")
 
         consumption_hourly["scale_offset"] = 1e8
-        consumption_hourly = helper.scale_array_precision(
+        consumption_hourly = array.scale_array_precision(
             consumption_hourly, "consumption_hourly", "scale_offset"
         )
 
@@ -218,7 +323,7 @@ class Model:
             self.agents["max_demand_kw"] *= self.agents["load_multiplier"]
             self.agents = self.agents.drop(columns=["load_multiplier", "nerc_region_abbr"])
 
-        self.agents = helper.scale_array_sum(self.agents, "consumption_hourly", "load_kwh")
+        self.agents = array.scale_array_sum(self.agents, "consumption_hourly", "load_kwh")
 
     def get_nem(self):
         if self.scenario == "metering":
@@ -251,10 +356,6 @@ class Model:
                 ] = "net billing"
 
     def prepare_agents(self):
-        # get generation data
-        self.log.info("....fetching resource information")
-        self.get_gen()
-
         if self.sector == "btm":
             # map tariffs
             self.log.info("....running with pre-processed tariffs")
@@ -349,7 +450,7 @@ class Model:
                 self.log.info("\n")
                 self.log.info(f"starting valuation for {len(self.agents)} FOM agents")
 
-                self.agents = valuer.run_multiprocessing(self.agents, configuration="fom")
+                self.agents = valuer.run_multiprocessing(self.agents, "fom")
 
                 self.log.info("null counts:")
                 self.log.info(self.agents.isnull().sum().sort_values())

{dwind-0.3 → dwind-0.3.1}/dwind/mp.py

@@ -4,9 +4,11 @@ import time
 from pathlib import Path
 
 import pandas as pd
+from rich.live import Live
 from rex.utilities.hpc import SLURM
 
-from dwind.helper import split_by_index
+from dwind.utils import hpc
+from dwind.utils.array import split_by_index
 
 
 class MultiProcess:
@@ -127,44 +129,45 @@ class MultiProcess:
                 log_dir.mkdir()
             self.stdout_path = log_dir
 
-    def check_status(self, job_ids: list[int]):
+    def check_status(self, job_ids: list[int], start_time: float):
         """Prints the status of all :py:attr:`jobs` submitted.
 
         Parameters
         ----------
         job_ids : list[int]
             The list of HPC ``job_id``s to check on.
+        start_time : float
+            The results of initial ``time.perf_counter()``.
         """
-        hpc = SLURM()
-        print(f"{len(job_ids)} job(s) started")
-
-        jobs_status = {j: hpc.check_status(job_id=j) for j in job_ids}
-        n_remaining = len([s for s in jobs_status.values() if s in ("PD", "R")])
-        print(f"{n_remaining} job(s) remaining: {jobs_status}")
-        time.sleep(30)
-
-        while n_remaining > 0:
-            hpc = SLURM()
-            for job, status in jobs_status.items():
-                if status in ("GC", "None"):
-                    continue
-                jobs_status.update({job: hpc.check_status(job_id=job)})
-
-            n_remaining = len([s for s in jobs_status.values() if s in ("PD", "R")])
-            print(f"{n_remaining} job(s) remaining: {jobs_status}")
-            if n_remaining > 0:
-                time.sleep(30)
+        slurm = SLURM()
+        job_status = {
+            j: {
+                "status": slurm.check_status(job_id=j),
+                "start": start_time,
+                "wait": time.perf_counter() - start_time,
+                "run": 0,
+            }
+            for j in job_ids
+        }
+        table, complete = hpc.generate_table(job_status)
+        with Live(table, refresh_per_second=1) as live:
+            while not complete:
+                time.sleep(5)
+                job_status |= hpc.update_status(job_status)
+                table, complete = hpc.generate_table(job_status)
+                live.update(table)
 
     def aggregate_outputs(self):
         """Collect the chunked results files, combine them into a single output parquet file, and
         delete the chunked results files.
         """
-        result_files = [f for f in self.out_path.iterdir() if f.suffix in (".pickle", ".pkl")]
+        result_files = [f for f in self.out_path.iterdir() if f.suffix == (".pqt")]
 
         if len(result_files) > 0:
-            result_agents = pd.concat([pd.read_pickle(f) for f in result_files])
+            result_agents = pd.concat([pd.read_parquet(f) for f in result_files])
             f_out = self.dir_out / f"run_{self.run_name}.pqt"
             result_agents.to_parquet(f_out)
+            print(f"Aggregated results saved to: {f_out}")
 
         for f in result_files:
             f.unlink()
@@ -185,20 +188,22 @@ class MultiProcess:
         base_cmd_str = f"module load conda; conda activate {self.env}; "
         base_cmd_str += "dwind run-chunk "
 
-        base_args = f"--location {self.location} "
-        base_args += f"--sector {self.sector} "
-        base_args += f"--scenario {self.scenario} "
-        base_args += f"--year {self.year} "
-        base_args += f"--repository {self.repository} "
-        base_args += f"--model-config {self.model_config} "
-        base_args += f"--out-path {self.out_path}"
-
-        for i, (start, end) in enumerate(zip(starts, ends, strict=True)):
+        base_args = f" {self.location} "
+        base_args += f" {self.sector} "
+        base_args += f" {self.scenario} "
+        base_args += f" {self.year} "
+        base_args += f" {self.out_path}"
+        base_args += f" {self.repository} "
+        base_args += f" {self.model_config} "
+
+        start_time = time.perf_counter()
+        # for i, (start, end) in enumerate(zip(starts, ends, strict=True)):
+        for i, (start, end) in enumerate(zip(starts, ends)):  # noqa: B905
             fn = self.out_path / f"agents_{i}.pqt"
             agent_df.iloc[start:end].to_parquet(fn)
 
             job_name = f"{self.run_name}_{i}"
-            cmd_str = f"{base_cmd_str} --chunk-ix {i} {base_args}"
+            cmd_str = f"{base_cmd_str} {i} {base_args}"
             print("cmd:", cmd_str)
 
             slurm_manager = SLURM()
@@ -221,5 +226,5 @@ class MultiProcess:
                 )
 
         # Check on the job statuses until they're complete, then aggregate the results
-        self.check_status(jobs)
+        self.check_status(jobs, start_time)
         self.aggregate_outputs()

{dwind-0.3 → dwind-0.3.1}/dwind/resource.py

@@ -62,7 +62,7 @@ class ResourcePotential:
                 self.df[config_col].astype(str) + "_" + self.df[f"tilt_{self.tech}"].astype(str)
             )
         elif self.tech == "wind":
-            configs = self.rev.settings.wind
+            configs = self.config.rev.settings.wind
             config_col = "turbine_class"
             col_list = [
                 "gid",
@@ -75,6 +75,9 @@ class ResourcePotential:
 
         out_cols = [*col_list, f"rev_index_{self.tech}", f"{self.tech}_naep", f"{self.tech}_cf"]
 
+        drop_cols = [f"rev_gid_{self.tech}", f"{self.tech}_naep", f"{self.tech}_cf"]
+        self.df = self.df.drop(columns=[c for c in drop_cols if c in self.df])
+
         f_gen = (
             self.config.rev.generation[f"{self.tech}_DIR"]
             / f"lkup_rev_gid_to_summary_{self.tech}_{self.year}.csv"

{dwind-0.3 → dwind-0.3.1}/dwind/run.py

@@ -3,19 +3,26 @@
 from __future__ import annotations
 
 import sys
-import tomllib
 from enum import Enum
-from typing import Annotated
+from typing import Optional, Annotated
 from pathlib import Path
 
 import typer
 import pandas as pd
+from rich.style import Style
+from rich.pretty import pprint
+from rich.console import Console
 
 
-# from memory_profiler import profile
-
+# fmt: off
+if sys.version_info >= (3, 11):  # noqa
+    import tomllib
+else:
+    import tomli as tomllib
+# fmt: on
 
 app = typer.Typer()
+console = Console()
 
 DWIND = Path("/projects/dwind/agents")
 
@@ -71,20 +78,25 @@ def year_callback(ctx: typer.Context, param: typer.CallbackParam, value: int):
 
 
 def load_agents(
-    file_name: str | Path | None = None,
+    file_name: Path | None = None,
     location: str | None = None,
     sector: str | None = None,
+    model_config: str | Path | None = None,
+    *,
+    prepare: bool = False,
 ) -> pd.DataFrame:
     """Load the agent file based on a filename or the location and sector to a Pandas DataFrame,
     and return the data frame.
 
     Args:
-        file_name (str | Path | None, optional): Name of the agent file, if not auto-generating from
+        file_name (Path | None, optional): Name of the agent file, if not auto-generating from
             the :py:attr:`location` and :py:attr:`sector` inputs. Defaults to None.
         location (str | None, optional): The name of the location or grouping, such as
             "colorado_larimer" or "priority1". Defaults to None.
         sector (str | None, optional): The name of the section. Must be one of "btm" or "fom".
             Defaults to None.
+        prepare (bool, optional): True if loading pre-chunked and prepared agent data, which should
+            bypass the standard column checking for additional joins, by default False.
 
     Returns:
         pd.DataFrame: The agent DataFrame.
@@ -97,10 +109,28 @@ def load_agents(
     f_agents = (
         file_name if file_name is not None else DWIND / f"{location}/agents_dwind_{sector}.parquet"
     )
+    if not isinstance(f_agents, Path):
+        f_agents = Path(f_agents).resolve()
+
+    alternative_suffix = (".pqt", ".parquet", ".pkl", ".pickle", ".csv")
+    base_style = Style.parse("cyan")
     if not f_agents.exists():
-        f_agents = f_agents.with_suffix(".pickle")
-    agents = Agents(agent_file=f_agents).agents
-    return agents
+        for suffix in alternative_suffix:
+            if (new_fn := f_agents.with_suffix(suffix)).exists():
+                if new_fn != f_agents:
+                    msg = (
+                        f"Using alternative agent file: {new_fn}\n\t"
+                        f"Requested agent file: {f_agents}"
+                    )
+                    console.print(msg, style=base_style)
+                    f_agents = new_fn
+                    break
+
+    if prepare:
+        return Agents.load_and_prepare_agents(
+            agent_file=f_agents, sector=sector, model_config=model_config
+        )
+    return Agents.load_agents(agent_file=f_agents)
 
 
 @app.command()
@@ -155,7 +185,9 @@ def run_hpc(
     dir_out: Annotated[
         str, typer.Option(help="Path to where the chunked outputs should be saved.")
     ],
-    stdout_path: Annotated[str | None, typer.Option(help="The path to write stdout logs.")] = None,
+    stdout_path: Annotated[
+        Optional[str], typer.Option(help="The path to write stdout logs.")  # noqa
+    ] = None,
 ):
     """Run dwind via the HPC multiprocessing interface."""
     sys.path.append(repository)
@@ -180,7 +212,9 @@ def run_hpc(
         stdout_path=stdout_path,
     )
 
-    agent_df = load_agents(location=location, sector=sector)
+    agent_df = load_agents(
+        location=location, sector=sector, model_config=model_config, prepare=True
+    )
     mp.run_jobs(agent_df)
 
 
@@ -194,15 +228,15 @@ def run_hpc_from_config(
     config_path = Path(config_path).resolve()
     with config_path.open("rb") as f:
         config = tomllib.load(f)
-    print(config)
+    print("Running the following configuration:")
+    pprint(config)
 
     run_hpc(**config)
 
 
 @app.command()
 def run_chunk(
-    # start: Annotated[int, typer.Option(help="chunk start index")],
-    # end: Annotated[int, typer.Option(help="chunk end index")],
+    chunk_ix: Annotated[int, typer.Option(help="Chunk number/index. Used for logging.")],
     location: Annotated[
         str, typer.Option(help="The state, state_county, or priority region to run.")
     ],
@@ -213,7 +247,6 @@ def run_chunk(
     year: Annotated[
         int, typer.Option(callback=year_callback, help="The year basis of the scenario.")
     ],
-    chunk_ix: Annotated[int, typer.Option(help="Chunk number/index. Used for logging.")],
     out_path: Annotated[str, typer.Option(help="save path")],
     repository: Annotated[
         str, typer.Option(help="Path to the dwind repository to use when running the model.")
@@ -230,7 +263,7 @@ def run_chunk(
     from dwind.model import Model
 
     agent_file = Path(out_path).resolve() / f"agents_{chunk_ix}.pqt"
-    agents = load_agents(file_name=agent_file)
+    agents = load_agents(file_name=agent_file, prepare=False)
     agent_file.unlink()
 
     model = Model(
@@ -271,7 +304,7 @@ def run(
     sys.path.append(repository)
     from dwind.model import Model
 
-    agents = load_agents(location=location, sector=sector)
+    agents = load_agents(location=location, sector=sector, model_config=model_config, prepare=True)
     model = Model(
         agents=agents,
         location=location,

{dwind-0.3 → dwind-0.3.1}/dwind/scenarios.py

@@ -115,7 +115,8 @@ def config_financial(scenario, year):
         f = f"/projects/dwind/configs/costs/atb24/ATB24_financing_baseline_{year}.json"
         i = Path("/projects/dwind/data/incentives/2025_incentives.json").resolve()
         with i.open("r") as i_in:
-            incentives = json.load(i_in)
+            incentives = pd.DataFrame.from_dict(json.load(i_in)).T
+        incentives.index.name = "census_tract_id"
     elif scenario in scenarios and year in (2035, 2040):
         f = "/projects/dwind/configs/costs/atb24/ATB24_financing_baseline_2035.json"
     else:
@@ -125,6 +126,8 @@ def config_financial(scenario, year):
 
     with f.open("r") as f_in:
         financials = json.load(f_in)
+
+    # TODO: determine if shared settings is applicable going forward, or separate should be reserved
     if year == 2025:
         financials["BTM"]["itc_fraction_of_capex"] = incentives
         financials["FOM"]["itc_fraction_of_capex"] = incentives

dwind-0.3.1/dwind/utils/array.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+
+def memory_downcaster(df):
+    assert isinstance(df, pd.DataFrame) | isinstance(df, pd.Series)
+
+    NAlist = []
+    for col in df.select_dtypes(include=[np.number]).columns:
+        IsInt = False
+        mx = df[col].max()
+        mn = df[col].min()
+
+        # integer does not support na; fill na
+        if not np.isfinite(df[col]).all():
+            NAlist.append(col)
+            df[col].fillna(mn - 1, inplace=True)
+
+        # test if column can be converted to an integer
+        asint = df[col].fillna(0).astype(np.int64)
+        result = df[col] - asint
+        result = result.sum()
+        if result > -0.01 and result < 0.01:
+            IsInt = True
+
+        # make integer/unsigned integer datatypes
+        if IsInt:
+            try:
+                if mn >= 0:
+                    if mx < 255:
+                        df[col] = df[col].astype(np.uint8)
+                    elif mx < 65535:
+                        df[col] = df[col].astype(np.uint16)
+                    elif mx < 4294967295:
+                        df[col] = df[col].astype(np.uint32)
+                    else:
+                        df[col] = df[col].astype(np.uint64)
+                else:
+                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:
+                        df[col] = df[col].astype(np.int8)
+                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:
+                        df[col] = df[col].astype(np.int16)
+                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:
+                        df[col] = df[col].astype(np.int32)
+                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:
+                        df[col] = df[col].astype(np.int64)
+            except:  # noqa: E722
+                df[col] = df[col].astype(np.float32)
+
+        # make float datatypes 32 bit
+        else:
+            df[col] = df[col].astype(np.float32)
+
+    return df
+
+
+def interpolate_array(row, col_1, col_2, col_in, col_out):
+    if row[col_in] != 0:
+        interpolated = row[col_in] * (row[col_2] - row[col_1]) + row[col_1]
+    else:
+        interpolated = row[col_1]
+
+    row[col_out] = interpolated
+
+    return row
+
+
+def scale_array_precision(df: pd.DataFrame, hourly_col: str, prec_offset_col: str):
+    """Scales the precision of :py:attr:`hourly_col` by the :py:attr:`prec_offset_col`.
+
+    Args:
+        df (pd.DataFrame): A Pandas DataFrame containing :py:att:`hourly_col` and
+            :py:att:`prec_offset_col`.
+        hourly_col (str) The column to adjust the precision.
+        prec_offset_col (str): The column for scaling the precison of :py:attr:`hourly_col`.
+
+    Returns:
+        pd.DataFrame: The input :py:attr:`df` with the precision of :py:attr:`hourly_col` scaled.
+    """
+    df[hourly_col] = (
+        np.array(df[hourly_col].values.tolist(), dtype="float64")
+        / df[prec_offset_col].values.reshape(-1, 1)
+    ).tolist()
+    return df
+
+
+def scale_array_deprecision(df: pd.DataFrame, col: str | list[str]) -> pd.DataFrame:
+    """Rounds the column(s) :py:attr:`col` to the nearest 2nd decimal and converts to NumPy's
+    float32.
+
+    Args:
+        df (pd.DataFrame): A Pandas DataFrame containing :py:att:`col`.
+        col (str | list[str]): The column(s) to have reduced precision.
+
+    Returns:
+        pd.DataFrame: The input :py:attr:`df` with the precision of :py:attr:`col` lowered.
+    """
+    df[col] = np.round(np.round(df[col], 2).astype(np.float32), 2)
+    return df
+
+
+def scale_array_sum(df: pd.DataFrame, hourly_col: str, scale_col: str) -> pd.DataFrame:
+    """Scales the :py:attr:`hourly_col` by its sum and multiples by the :py:attr:`scale_col`.
+
+    Args:
+        df (pd.DataFrame): Pandas DataFrame containing the :py:attr:`hourly_col` and
+            :py:attr:`scale_col`.
+        hourly_col (str): The name of the column to be scaled whose values are lists.
+        scale_col (str): The column to scale the :py:attr:`hourly_col`.
+
+    Returns:
+        pandas.DataFrame: The input dataframe, but with the values of the :py:attr:`hourly_col`
+            scaled appropriately.
+    """
+    hourly_array = np.array(df[hourly_col].values.tolist())
+    df[hourly_col] = (
+        hourly_array / hourly_array.sum(axis=1).reshape(-1, 1) * df[scale_col].values.reshape(-1, 1)
+    ).tolist()
+    return df
+
+
+def scale_array_multiplier(
+    df: pd.DataFrame, hourly_col: str, multiplier_col: str, col_out: str
+) -> pd.DataFrame:
+    """Scales the :py:attr:hourly_col` values by the :py:attr:`multiplier_col`, and places it in
+    the :py:attr:`col_out`.
+
+    Args:
+        df (pd.DataFrame): The Pandas DataFrame containing the :py:attr:`hourly_col` and
+            :py:attr:`multiplier_col`.
+        hourly_col (str): A column of hourly values as a list of floats in each cell.
+        multiplier_col (str): The column used to scale the :py:attr:`hourly_col`.
+        col_out (str): A new column that will contain the scaled data.
+
+    Returns:
+        pd.DataFrame: A new copy of the original data (:py:attr:`df`) containing the
+            :py:attr:`col_out` column.
+    """
+    hourly_array = np.array(df[hourly_col].values.tolist())
+    df[col_out] = (hourly_array * df[multiplier_col].values.reshape(-1, 1)).tolist()
+    return df
+
+
+def split_by_index(
+    arr: pd.DataFrame | np.ndarray | pd.Series, n_splits: int
+) -> tuple[np.ndarray, np.ndarray]:
+    """Split a DataFrame, Series, or array like with np.array_split, but only return the start and
+    stop indices, rather than chunks. For Pandas objects, this are equivalent to
+    ``arr.iloc[start: end]`` and for NumPy: ``arr[start: end]``. Splits are done according
+    to the 0th dimension.
+
+    Args:
+        arr(pd.DataFrame | pd.Series | np.ndarray): The array, data frame, or series to split.
+        n_splits(:obj:`int`): The number of near equal or equal splits.
+
+    Returns:
+        tuple[np.ndarray, np.ndarray]
+    """
+    size = arr.shape[0]
+    base = np.arange(n_splits)
+    split_size = size // n_splits
+    extra = size % n_splits
+
+    starts = base * split_size
+    ends = starts + split_size
+
+    for i in range(extra):
+        ends[i:] += 1
+        starts[i + 1 :] += 1
+    return starts, ends

dwind-0.3.1/dwind/utils/hpc.py

@@ -0,0 +1,96 @@
+import time
+
+from rich.table import Table
+from rex.utilities.hpc import SLURM
+
+
+def convert_seconds_for_print(time: float) -> str:
+    """Convert number of seconds to number of hours, minutes, and seconds."""
+    div = ((60, "seconds"), (60, "minutes"), (24, "hours"))
+
+    result = []
+    value = time
+    for divisor, label in div:
+        if not divisor:
+            remainder = value
+            if not remainder:
+                break
+        else:
+            value, remainder = divmod(value, divisor)
+            if not value and not remainder:
+                break
+        if remainder == 1:
+            label = label[:-1]
+
+        # 0.2 second precision for seconds, and no decimals otherwise
+        if result:
+            result.append(f"{remainder:,.0f} {label}")
+        else:
+            result.append(f"{remainder:.1f} {label}")
+    if result:
+        return ", ".join(reversed(result))
+    return "0"
+
+
+def update_status(job_status: dict) -> dict:
+    """Get an updated status and timing statistics for all running jobs on the HPC.
+
+    Args:
+        job_status (dict): Dictionary of job id (primary key) with sub keys of "status",
+            "start_time" (initial or start of run status), "wait", and "run".
+
+    Returns:
+        dict: Dictionary of updated statuses and timing statistics for all current queued and
+            running jobs.
+    """
+    slurm = SLURM()
+    update = {}
+    for job, vals in job_status.items():
+        original_status = vals["status"]
+        if original_status in ("CG", "CF", "None", None):
+            continue
+        new_status = slurm.check_status(job_id=job)
+        if new_status == "PD":
+            update[job] = vals | {"status": new_status, "wait": time.perf_counter() - vals["start"]}
+        elif new_status == "R":
+            if original_status != "R":
+                update[job] = vals | {
+                    "status": new_status,
+                    "wait": time.perf_counter() - vals["start"],
+                    "start": time.perf_counter(),
+                }
+            else:
+                update[job] = vals | {"run": time.perf_counter() - vals["start"]}
+        elif new_status in ("CG", "CF", "None", None):
+            update[job] = vals | {"status": new_status, "run": time.perf_counter() - vals["start"]}
+        else:
+            raise ValueError(f"Unaccounted for status code: {new_status}")
+    return update
+
+
+def generate_table(job_status: dict) -> tuple[Table, bool]:
+    """Generate the job status run time statistics table.
+
+    Args:
+        job_status (dict): Dictionary of job id (primary key) with sub keys of "status",
+            "start_time" (initial or start of run status), "wait", and "run".
+
+    Returns:
+        Table: ``rich.Table`` of human readable statistics.
+        bool: True if all jobs are complete, otherwise False.
+    """
+    table = Table()
+    table.add_column("Job ID")
+    table.add_column("Status")
+    table.add_column("Wait time")
+    table.add_column("Run time")
+
+    for job, vals in job_status.items():
+        status = vals["status"]
+        _wait = vals["wait"]
+        _run = vals["run"]
+        table.add_row(
+            job, status, convert_seconds_for_print(_wait), convert_seconds_for_print(_run)
+        )
+    done = all(el["status"] in ("CG", None) for el in job_status.values())
+    return table, done

{dwind-0.3 → dwind-0.3.1}/dwind/valuation.py

@@ -1,3 +1,4 @@
+import os
 import time
 import logging
 import functools
@@ -54,15 +55,20 @@ class ValueFunctions:
         _load_sql = functools.partial(
             loader.load_df,
             year=self.year,
-            sql_constructur=self.confg.sql.ATLAS_PG_CON_STR,
+            sql_constructor=self.config.sql.ATLAS_PG_CON_STR,
         )
+        cost_dir = self.config.cost.DIR
 
-        self.retail_rate_inputs = _load_csv(self.config.cost.RETAIL_RATE_INPUT_TABLE)
-        self.wholesale_rate_inputs = _load_csv(self.config.cost.WHOLESALE_RATE_INPUT_TABLE)
-        self.depreciation_schedule_inputs = _load_csv(self.config.cost.DEPREC_INPUTS_TABLE)
+        self.retail_rate_inputs = _load_csv(cost_dir / self.config.cost.RETAIL_RATE_INPUT_TABLE)
+        self.wholesale_rate_inputs = _load_csv(
+            cost_dir / self.config.cost.WHOLESALE_RATE_INPUT_TABLE
+        )
+        self.depreciation_schedule_inputs = _load_csv(
+            cost_dir / self.config.cost.DEPREC_INPUTS_TABLE
+        )
 
         if "wind" in self.config.project.settings.TECHS:
-            self.wind_price_inputs = _load_sql(self.config.cost.WIND_PRICE_INPUT_TABLE)
+            self.wind_price_inputs = _load_csv(cost_dir / self.config.cost.WIND_PRICE_INPUT_TABLE)
             self.wind_tech_inputs = _load_sql(self.config.cost.WIND_TECH_INPUT_TABLE)
             self.wind_derate_inputs = _load_sql(self.config.cost.WIND_DERATE_INPUT_TABLE)
 
@@ -177,10 +183,7 @@ class ValueFunctions:
         # field, and need to be removed, then rejoined with the appropriate column names
         financial = self.FINANCIAL_INPUTS["BTM"].copy()
         if self.year == 2025:
-            incentives = pd.DataFrame.from_dict(
-                self.FINANCIAL_INPUTS["BTM"].pop("itc_fraction_of_capex")
-            ).T
-            incentives.index.name = "census_tract_id"
+            incentives = self.FINANCIAL_INPUTS["BTM"].pop("itc_fraction_of_capex")
 
         deprec_sch = pd.DataFrame()
         deprec_sch["sector_abbr"] = financial["deprec_sch"].keys()
@@ -237,52 +240,30 @@ class ValueFunctions:
         return df
 
     def _preprocess_fom(self, df, tech="wind"):
-        columns = [
-            "yr",
-            "cambium_scenario",
-            "analysis_period",
-            "debt_option",
-            "debt_percent",
-            "inflation_rate",
-            "dscr",
-            "real_discount_rate",
-            "term_int_rate",
-            "term_tenor",
-            f"ptc_fed_amt_{tech}",
-            "itc_fed_pct",
-            "deg",
-            "system_capex_per_kw",
-            "system_om_per_kw",
-        ]
-
         itc_fraction_of_capex = self.FINANCIAL_INPUTS["FOM"]["itc_fraction_of_capex"]
-        values = [
-            self.year,
-            self.CAMBIUM_SCENARIO,
-            self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
-            self.FINANCIAL_INPUTS["FOM"]["debt_option"],
-            self.FINANCIAL_INPUTS["FOM"]["debt_percent"] * 100,
-            self.FINANCIAL_INPUTS["FOM"]["inflation"] * 100,
-            self.FINANCIAL_INPUTS["FOM"]["dscr"],
-            self.FINANCIAL_INPUTS["FOM"]["discount_rate"] * 100,
-            self.FINANCIAL_INPUTS["FOM"]["interest_rate"] * 100,
-            self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
-            self.FINANCIAL_INPUTS["FOM"]["ptc_fed_dlrs_per_kwh"][tech],
-            itc_fraction_of_capex if self.year != 2025 else 0.3,
-            self.FINANCIAL_INPUTS["FOM"]["degradation"],
-            self.COST_INPUTS["FOM"]["system_capex_per_kw"][tech],
-            self.COST_INPUTS["FOM"]["system_om_per_kw"][tech],
-        ]
-        df[columns] = values
+        df = df.assign(
+            yr=self.year,
+            cambium_scenario=self.CAMBIUM_SCENARIO,
+            analysis_period=self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
+            debt_option=self.FINANCIAL_INPUTS["FOM"]["debt_option"],
+            debt_percent=self.FINANCIAL_INPUTS["FOM"]["debt_percent"] * 100,
+            inflation_rate=self.FINANCIAL_INPUTS["FOM"]["inflation"] * 100,
+            dscr=self.FINANCIAL_INPUTS["FOM"]["dscr"],
+            real_discount_rate=self.FINANCIAL_INPUTS["FOM"]["discount_rate"] * 100,
+            term_int_rate=self.FINANCIAL_INPUTS["FOM"]["interest_rate"] * 100,
+            term_tenor=self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
+            itc_fed_pct=itc_fraction_of_capex if self.year != 2025 else 0.3,
+            deg=self.FINANCIAL_INPUTS["FOM"]["degradation"],
+            system_capex_per_kw=self.COST_INPUTS["FOM"]["system_capex_per_kw"][tech],
+            system_om_per_kw=self.COST_INPUTS["FOM"]["system_om_per_kw"][tech],
+            **{f"ptc_fed_amt_{tech}": self.FINANCIAL_INPUTS["FOM"]["ptc_fed_dlrs_per_kwh"][tech]},
+        )
         # 2025 uses census-tract based applicable credit for the itc_fed_pct, so update accordingly
         if self.year == 2025:
-            incentives = pd.DataFrame.from_dict(
-                self.FINANCIAL_INPUTS["FOM"]["itc_fraction_of_capex"]
-            ).T
-            incentives.index.name = "census_tract_id"
+            incentives = self.FINANCIAL_INPUTS["FOM"]["itc_fraction_of_capex"]
+
             df = df.set_index("census_tract_id", drop=False).join(incentives).reset_index(drop=True)
-            df.itc_fed_pct = df.applicable_credit
-            df.itc_fed_pct = df.itc_fed_pct.fillna(0.3)
+            df.itc_fed_pct = df.applicable_credit.fillna(0.3)
 
         return df
 
@@ -353,6 +334,9 @@ class ValueFunctions:
         verb = self.config.project.settings.VERBOSITY
 
         if max_w > 1:
+            # Override project-level setting to ensure memory intensive calculations don't
+            # cause jobs to silently fail
+            max_w = min(int(os.cpu_count() * 0.8), max_w)
             results_list = []
 
             with cf.ProcessPoolExecutor(max_workers=max_w) as executor:
@@ -479,7 +463,7 @@ def fetch_cambium_values(row, generation_hourly, cambium_dir, cambium_value, low
     rev["cleared"] = rev["cleared"].apply(np.floor)
 
     rev = rev[["cleared", "value"]]
-    tup = tuple(map(tuple, rev.values))
+    tup = tuple(map(tuple, rev.values.tolist()))
 
     return tup
 
@@ -642,10 +626,8 @@ def find_breakeven(
                 full_output=full_output,
                 disp=disp,
             )
-
-            return breakeven_cost_usd_p_kw, {
-                k: loan.Outputs.export().get(k, None) for k in pysam_outputs
-            }
+            results = loan.Outputs.export()
+            return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
 
         except Exception as e:
             raise ValueError("Root finding failed.") from e
@@ -678,9 +660,8 @@ def find_breakeven(
                 disp=disp,
             )
 
-            return breakeven_cost_usd_p_kw, {
-                k: loan.Outputs.export().get(k, None) for k in pysam_outputs
-            }
+            results = loan.Outputs.export()
+            return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
 
         except Exception as e:
             raise ValueError("Root finding failed.") from e
@@ -719,9 +700,8 @@ def find_breakeven(
                 disp=disp,
             )
 
-            return breakeven_cost_usd_p_kw, {
-                k: loan.Outputs.export().get(k, None) for k in pysam_outputs
-            }
+            results = loan.Outputs.export()
+            return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
 
         except Exception as e:
             raise ValueError("Root finding failed.") from e
@@ -825,9 +805,8 @@ def find_breakeven_fom(
                 disp=disp,
             )
 
-            return breakeven_cost_usd_p_kw, {
-                k: financial.Outputs.export().get(k, None) for k in pysam_outputs
-            }
+            results = financial.Outputs.export()
+            return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
 
         except Exception as e:
             raise ValueError("Root finding failed.") from e
@@ -860,9 +839,8 @@ def find_breakeven_fom(
                 disp=disp,
             )
 
-            return breakeven_cost_usd_p_kw, {
-                k: financial.Outputs.export().get(k, None) for k in pysam_outputs
-            }
+            results = financial.Outputs.export()
+            return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
 
         except Exception as e:
             raise ValueError("Root finding failed.") from e
@@ -900,9 +878,8 @@ def find_breakeven_fom(
                 disp=disp,
             )
 
-            return breakeven_cost_usd_p_kw, {
-                k: financial.Outputs.export().get(k, None) for k in pysam_outputs
-            }
+            results = financial.Outputs.export()
+            return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
 
         except Exception as e:
             raise ValueError("Root finding failed") from e
@@ -1312,7 +1289,8 @@ def process_btm(
 
     _ = calc_financial_performance(row["system_capex_per_kw"], row, loan, batt_costs)
 
-    row["additional_pysam_outputs"] = {k: loan.Outputs.export().get(k, None) for k in pysam_outputs}
+    results = loan.Outputs.export()
+    row["additional_pysam_outputs"] = {k: results.get(k) for k in pysam_outputs}
 
     # run root finding algorithm to find breakeven cost based on calculated NPV
     out, _ = find_breakeven(
@@ -1469,9 +1447,8 @@ def process_fom(
 
         log.info(f"row {row.loc['gid']} calculating financial performance")
         _ = calc_financial_performance_fom(system_capex_per_kw, row, financial)
-        row["additional_pysam_outputs"] = {
-            k: financial.Outputs.export().get(k, None) for k in pysam_outputs
-        }
+        results = financial.Outputs.export()
+        row["additional_pysam_outputs"] = {k: results.get(k) for k in pysam_outputs}
 
         # run root finding algorithm to find breakeven cost based on calculated NPV
         log.info(f"row {row.loc['gid']} breakeven")
@@ -1533,7 +1510,7 @@ def worker(row: pd.Series, sector: str, config: Configuration):
                     row,
                     generation_hourly,
                     config.project.settings.CAMBIUM_DATA_DIR,
-                    config.CAMBIUM_VALUE,
+                    config.project.settings.CAMBIUM_VALUE,
                 )
 
                 row = process_fom(
@@ -1548,7 +1525,7 @@ def worker(row: pd.Series, sector: str, config: Configuration):
 
             # store results in dictionary
             results[f"{tech}_breakeven_cost_{sector}"] = row["breakeven_cost_usd_p_kw"]
-            results[f"{tech}_pysam_outputs_{sector}"] = row["additional_pysam_outputs"]
+            results |= row["additional_pysam_outputs"]
 
         return (row["gid"], results)
 

{dwind-0.3 → dwind-0.3.1}/dwind.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dwind
-Version: 0.3
+Version: 0.3.1
 Summary: Distributed Wind Generation Model
 Author-email: Jane Lockshin <jane.lockshin@nrel.gov>, Paritosh Das <paritosh.das@nrel.gov>, Rob Hammond <rob.hammond@nrel.gov>
 Project-URL: source, https://github.com/NREL/dwind

{dwind-0.3 → dwind-0.3.1}/dwind.egg-info/SOURCES.txt

@@ -24,5 +24,8 @@ dwind.egg-info/dependency_links.txt
 dwind.egg-info/entry_points.txt
 dwind.egg-info/requires.txt
 dwind.egg-info/top_level.txt
+dwind/utils/__init__.py
+dwind/utils/array.py
+dwind/utils/hpc.py
 examples/larimer_county_btm_baseline_2025.toml
 examples/model_config.toml