dwind 0.3__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dwind/__init__.py +1 -1
- dwind/btm_sizing.py +3 -2
- dwind/config.py +12 -2
- dwind/loader.py +4 -1
- dwind/model.py +150 -49
- dwind/mp.py +39 -34
- dwind/resource.py +4 -1
- dwind/run.py +50 -17
- dwind/scenarios.py +4 -1
- dwind/utils/__init__.py +0 -0
- dwind/utils/array.py +172 -0
- dwind/utils/hpc.py +96 -0
- dwind/valuation.py +54 -77
- {dwind-0.3.dist-info → dwind-0.3.1.dist-info}/METADATA +1 -1
- dwind-0.3.1.dist-info/RECORD +20 -0
- dwind-0.3.dist-info/RECORD +0 -17
- {dwind-0.3.dist-info → dwind-0.3.1.dist-info}/WHEEL +0 -0
- {dwind-0.3.dist-info → dwind-0.3.1.dist-info}/entry_points.txt +0 -0
- {dwind-0.3.dist-info → dwind-0.3.1.dist-info}/licenses/LICENSE.txt +0 -0
- {dwind-0.3.dist-info → dwind-0.3.1.dist-info}/top_level.txt +0 -0
dwind/__init__.py
CHANGED
dwind/btm_sizing.py
CHANGED
@@ -5,7 +5,8 @@ import logging
|
|
5
5
|
import numpy as np
|
6
6
|
import pandas as pd
|
7
7
|
|
8
|
-
from dwind import Configuration
|
8
|
+
from dwind import Configuration
|
9
|
+
from dwind.utils import array
|
9
10
|
|
10
11
|
|
11
12
|
log = logging.getLogger("dwfs")
|
@@ -117,7 +118,7 @@ def sizer(agents: pd.DataFrame, config: Configuration):
|
|
117
118
|
agents.drop_duplicates(subset=["gid"], inplace=True)
|
118
119
|
|
119
120
|
# make small
|
120
|
-
agents =
|
121
|
+
agents = array.memory_downcaster(agents)
|
121
122
|
|
122
123
|
return agents
|
123
124
|
|
dwind/config.py
CHANGED
@@ -2,11 +2,21 @@
|
|
2
2
|
attributes.
|
3
3
|
"""
|
4
4
|
|
5
|
+
from __future__ import annotations
|
6
|
+
|
5
7
|
import re
|
6
|
-
import
|
8
|
+
import sys
|
7
9
|
from pathlib import Path
|
8
10
|
|
9
11
|
|
12
|
+
# fmt: off
|
13
|
+
if sys.version_info >= (3, 11): # noqa
|
14
|
+
import tomllib
|
15
|
+
else:
|
16
|
+
import tomli as tomllib
|
17
|
+
# fmt: on
|
18
|
+
|
19
|
+
|
10
20
|
class Mapping(dict):
|
11
21
|
"""Dict-like class that allows for the use of dictionary style attribute calls on class
|
12
22
|
attributes.
|
@@ -80,7 +90,7 @@ class Configuration(Mapping):
|
|
80
90
|
to read and convert. If passing a filename, it must be a TOML file.
|
81
91
|
initial (bool, optional): Option to disable post-processing of configuration data.
|
82
92
|
"""
|
83
|
-
if isinstance(config, str
|
93
|
+
if isinstance(config, (str, Path)): # noqa: UP038
|
84
94
|
config = Path(config).resolve()
|
85
95
|
with config.open("rb") as f:
|
86
96
|
config = tomllib.load(f)
|
dwind/loader.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
1
3
|
from pathlib import Path
|
2
4
|
|
3
5
|
import pandas as pd
|
@@ -54,6 +56,7 @@ def _load_from_sql(table: str, sql_constructor: str, year: str | int | None) ->
|
|
54
56
|
atlas_engine = create_engine(sql_constructor)
|
55
57
|
|
56
58
|
with atlas_engine.connect() as conn:
|
57
|
-
pd.read_sql(sql, con=conn.connection)
|
59
|
+
df = pd.read_sql(sql, con=conn.connection)
|
58
60
|
|
59
61
|
atlas_engine.dispose()
|
62
|
+
return df
|
dwind/model.py
CHANGED
@@ -7,7 +7,8 @@ from pathlib import Path
|
|
7
7
|
import numpy as np
|
8
8
|
import pandas as pd
|
9
9
|
|
10
|
-
from dwind import Configuration,
|
10
|
+
from dwind import Configuration, resource, scenarios, valuation, btm_sizing
|
11
|
+
from dwind.utils import array
|
11
12
|
|
12
13
|
|
13
14
|
# POTENTIALLY DANGEROUS!
|
@@ -34,11 +35,61 @@ class Agents:
|
|
34
35
|
either a pickle file (.pkl or .pickle) or a parquet file (.pqt or .parquet).
|
35
36
|
"""
|
36
37
|
|
37
|
-
def __init__(
|
38
|
+
def __init__(
|
39
|
+
self,
|
40
|
+
agent_file: str | Path,
|
41
|
+
sector: str | None = None,
|
42
|
+
model_config: str | Path | None = None,
|
43
|
+
*,
|
44
|
+
resource_year: int = 2018,
|
45
|
+
):
|
38
46
|
self.agent_file = Path(agent_file).resolve()
|
39
|
-
self.
|
40
|
-
|
41
|
-
|
47
|
+
self.sector = sector
|
48
|
+
self.config = model_config
|
49
|
+
self.resource_year = resource_year
|
50
|
+
self._load_agents()
|
51
|
+
|
52
|
+
@classmethod
|
53
|
+
def load_and_prepare_agents(
|
54
|
+
cls,
|
55
|
+
agent_file: str | Path,
|
56
|
+
sector: str,
|
57
|
+
model_config: str | Path,
|
58
|
+
*,
|
59
|
+
save_results: bool = False,
|
60
|
+
file_name: str | Path | None = None,
|
61
|
+
) -> pd.DataFrame:
|
62
|
+
"""Load and prepare the agent files to run through ``Model``.
|
63
|
+
|
64
|
+
Args:
|
65
|
+
agent_file (str | Path): The full file path of the agent parquet, CSV, or pickle data.
|
66
|
+
save_results (bool, optional): True to save any updates to the data. Defaults to False.
|
67
|
+
file_name (str | Path | None, optional): The file path and name for where to save the
|
68
|
+
prepared data, if not overwriting the existing agent data. Defaults to None.
|
69
|
+
|
70
|
+
Returns:
|
71
|
+
pd.DataFrame: The prepared agent data.
|
72
|
+
"""
|
73
|
+
agents = cls(agent_file, sector, model_config)
|
74
|
+
agents.prepare()
|
75
|
+
if save_results:
|
76
|
+
agents.save_agents(file_name=file_name)
|
77
|
+
return agents.agents
|
78
|
+
|
79
|
+
@classmethod
|
80
|
+
def load_agents(cls, agent_file: str | Path) -> pd.DataFrame:
|
81
|
+
"""Load the agent data without making any additional modifications.
|
82
|
+
|
83
|
+
Args:
|
84
|
+
agent_file (str | Path): The full file path of the agent parquet, pickle, or CSV data.
|
85
|
+
|
86
|
+
Returns:
|
87
|
+
pd.DataFrame: The agent data.
|
88
|
+
"""
|
89
|
+
agents = cls(agent_file)
|
90
|
+
return agents.agents
|
91
|
+
|
92
|
+
def _load_agents(self):
|
42
93
|
"""Loads in the agent file and drops any indices."""
|
43
94
|
suffix = self.agent_file.suffix
|
44
95
|
if suffix in (".pqt", ".parquet"):
|
@@ -56,20 +107,101 @@ class Agents:
|
|
56
107
|
if suffix == ".csv":
|
57
108
|
self.agents = self.agents.reset_index(drop=True)
|
58
109
|
|
110
|
+
def prepare(self):
|
111
|
+
"""Prepares the agent data so that it has the necessary columns required for modeling.
|
112
|
+
|
113
|
+
Steps:
|
114
|
+
1. Extract `state_fips` from the `fips_code` column.
|
115
|
+
2. If `census_tract_id` is missing, load and merge the 2020 census tracts
|
116
|
+
based on the `pgid` column.
|
117
|
+
3. Convert the 2012 rev ID to the 2018 rev id in `rev_index_wind`.
|
118
|
+
4. Attach the universal resource generation data.
|
119
|
+
"""
|
120
|
+
self.config = Configuration(self.config)
|
59
121
|
if "state_fips" not in self.agents.columns:
|
60
|
-
self.agents["state_fips"] = self.agents["fips_code"]
|
122
|
+
self.agents["state_fips"] = [el[:2] for el in self.agents["fips_code"]]
|
61
123
|
|
62
124
|
if "census_tract_id" not in self.agents.columns:
|
63
|
-
|
64
|
-
|
65
|
-
|
125
|
+
self.merge_census_data()
|
126
|
+
|
127
|
+
self.update_rev_id()
|
128
|
+
self.merge_generation()
|
129
|
+
|
130
|
+
def save_agents(self, file_name: str | Path | None = None):
|
131
|
+
if file_name is None:
|
132
|
+
file_name = self.agent_file
|
133
|
+
|
134
|
+
suffix = file_name.suffix
|
135
|
+
if suffix in (".pqt", ".parquet"):
|
136
|
+
file_saver = self.agents.to_parquet
|
137
|
+
elif suffix in (".pkl", ".pickle"):
|
138
|
+
file_saver = self.agents.to_pickle
|
139
|
+
elif suffix == ".csv":
|
140
|
+
file_saver = self.agents.to_csv
|
141
|
+
else:
|
142
|
+
raise ValueError(
|
143
|
+
f"File types ending in {suffix} can't be read as pickle, parquet, or CSV"
|
66
144
|
)
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
145
|
+
|
146
|
+
file_saver(file_name)
|
147
|
+
|
148
|
+
def merge_census_data(self):
|
149
|
+
census_tracts = pd.read_csv(
|
150
|
+
"/projects/dwind/configs/sizing/wind/lkup_block_to_pgid_2020.csv",
|
151
|
+
usecols=["pgid", "fips_block"],
|
152
|
+
dtype=str,
|
153
|
+
).drop_duplicates()
|
154
|
+
census_tracts["census_tract_id"] = [el[:11] for el in census_tracts["fips_block"]]
|
155
|
+
self.agents = (
|
156
|
+
self.agents.merge(census_tracts, how="left", on="pgid")
|
157
|
+
.drop_duplicates(subset=["gid"])
|
158
|
+
.reset_index(drop=True)
|
159
|
+
)
|
160
|
+
|
161
|
+
def update_rev_id(self, resource_year="2018"):
|
162
|
+
"""Update 2012 rev index to 2018 index."""
|
163
|
+
if resource_year != "2018":
|
164
|
+
return
|
165
|
+
|
166
|
+
index_file = "/projects/dwind/configs/rev/wind/lkup_rev_index_2012_to_2018.csv"
|
167
|
+
rev_index_map = (
|
168
|
+
pd.read_csv(index_file, usecols=["rev_index_wind_2012", "rev_index_wind_2018"])
|
169
|
+
.rename(columns={"rev_index_wind_2012": "rev_index_wind"})
|
170
|
+
.set_index("rev_index_wind")
|
171
|
+
)
|
172
|
+
|
173
|
+
ix_original = self.agents.index.name
|
174
|
+
if ix_original is None:
|
175
|
+
self.agents = (
|
176
|
+
self.agents.set_index("rev_index_wind", drop=True)
|
177
|
+
.join(rev_index_map, how="left")
|
178
|
+
.reset_index(drop=True)
|
179
|
+
.rename(columns={"rev_index_wind_2018": "rev_index_wind"})
|
180
|
+
.dropna(subset="rev_index_wind")
|
181
|
+
)
|
182
|
+
else:
|
183
|
+
self.agents = (
|
184
|
+
self.agents.reset_index(drop=False)
|
185
|
+
.set_index("rev_index_wind")
|
186
|
+
.join(rev_index_map, how="left")
|
187
|
+
.set_index(ix_original, drop=True)
|
188
|
+
.rename(columns={"rev_index_wind_2018": "rev_index_wind"})
|
189
|
+
.dropna(subset="rev_index_wind")
|
190
|
+
)
|
191
|
+
|
192
|
+
def merge_generation(self):
|
193
|
+
if self.resource_year != "2018":
|
194
|
+
return
|
195
|
+
|
196
|
+
# update 2012 rev cf/naep/aep to 2018 values
|
197
|
+
# self.agents = self.agents.drop(columns=["wind_naep", "wind_cf", "wind_aep"])
|
198
|
+
resource_potential = resource.ResourcePotential(
|
199
|
+
parcels=self.agents,
|
200
|
+
application=self.sector,
|
201
|
+
year=self.resource_year,
|
202
|
+
model_config=self.model_config,
|
203
|
+
)
|
204
|
+
self.agents = resource_potential.match_rev_summary_to_agents()
|
73
205
|
|
74
206
|
|
75
207
|
class Model:
|
@@ -126,33 +258,6 @@ class Model:
|
|
126
258
|
|
127
259
|
self.log = logging.getLogger("dwfs")
|
128
260
|
|
129
|
-
def get_gen(self, resource_year="2018"):
|
130
|
-
if resource_year != "2018":
|
131
|
-
return
|
132
|
-
|
133
|
-
# update 2012 rev index to 2018 index
|
134
|
-
f = "/projects/dwind/configs/rev/wind/lkup_rev_index_2012_to_2018.csv"
|
135
|
-
lkup = pd.read_csv(f)[["rev_index_wind_2012", "rev_index_wind_2018"]]
|
136
|
-
|
137
|
-
self.agents = (
|
138
|
-
self.agents.merge(
|
139
|
-
lkup, left_on="rev_index_wind", right_on="rev_index_wind_2012", how="left"
|
140
|
-
)
|
141
|
-
.drop(columns=["rev_index_wind", "rev_index_wind_2012"])
|
142
|
-
.rename(columns={"rev_index_wind_2018": "rev_index_wind"})
|
143
|
-
.dropna(subset="rev_index_wind")
|
144
|
-
)
|
145
|
-
|
146
|
-
# update 2012 rev cf/naep/aep to 2018 values
|
147
|
-
# self.agents = self.agents.drop(columns=["wind_naep", "wind_cf", "wind_aep"])
|
148
|
-
resource_potential = resource.ResourcePotential(
|
149
|
-
parcels=self.agents,
|
150
|
-
application=self.sector,
|
151
|
-
year=resource_year,
|
152
|
-
model_config=self.config,
|
153
|
-
)
|
154
|
-
self.agents = resource_potential.match_rev_summary_to_agents()
|
155
|
-
|
156
261
|
def get_rates(self):
|
157
262
|
self.agents = self.agents[~self.agents["rate_id_alias"].isna()]
|
158
263
|
self.agents["rate_id_alias"] = self.agents["rate_id_alias"].astype(int)
|
@@ -173,7 +278,7 @@ class Model:
|
|
173
278
|
consumption_hourly = pd.read_parquet("/projects/dwind/data/crb_consumption_hourly.pqt")
|
174
279
|
|
175
280
|
consumption_hourly["scale_offset"] = 1e8
|
176
|
-
consumption_hourly =
|
281
|
+
consumption_hourly = array.scale_array_precision(
|
177
282
|
consumption_hourly, "consumption_hourly", "scale_offset"
|
178
283
|
)
|
179
284
|
|
@@ -218,7 +323,7 @@ class Model:
|
|
218
323
|
self.agents["max_demand_kw"] *= self.agents["load_multiplier"]
|
219
324
|
self.agents = self.agents.drop(columns=["load_multiplier", "nerc_region_abbr"])
|
220
325
|
|
221
|
-
self.agents =
|
326
|
+
self.agents = array.scale_array_sum(self.agents, "consumption_hourly", "load_kwh")
|
222
327
|
|
223
328
|
def get_nem(self):
|
224
329
|
if self.scenario == "metering":
|
@@ -251,10 +356,6 @@ class Model:
|
|
251
356
|
] = "net billing"
|
252
357
|
|
253
358
|
def prepare_agents(self):
|
254
|
-
# get generation data
|
255
|
-
self.log.info("....fetching resource information")
|
256
|
-
self.get_gen()
|
257
|
-
|
258
359
|
if self.sector == "btm":
|
259
360
|
# map tariffs
|
260
361
|
self.log.info("....running with pre-processed tariffs")
|
@@ -349,7 +450,7 @@ class Model:
|
|
349
450
|
self.log.info("\n")
|
350
451
|
self.log.info(f"starting valuation for {len(self.agents)} FOM agents")
|
351
452
|
|
352
|
-
self.agents = valuer.run_multiprocessing(self.agents,
|
453
|
+
self.agents = valuer.run_multiprocessing(self.agents, "fom")
|
353
454
|
|
354
455
|
self.log.info("null counts:")
|
355
456
|
self.log.info(self.agents.isnull().sum().sort_values())
|
dwind/mp.py
CHANGED
@@ -4,9 +4,11 @@ import time
|
|
4
4
|
from pathlib import Path
|
5
5
|
|
6
6
|
import pandas as pd
|
7
|
+
from rich.live import Live
|
7
8
|
from rex.utilities.hpc import SLURM
|
8
9
|
|
9
|
-
from dwind.
|
10
|
+
from dwind.utils import hpc
|
11
|
+
from dwind.utils.array import split_by_index
|
10
12
|
|
11
13
|
|
12
14
|
class MultiProcess:
|
@@ -127,44 +129,45 @@ class MultiProcess:
|
|
127
129
|
log_dir.mkdir()
|
128
130
|
self.stdout_path = log_dir
|
129
131
|
|
130
|
-
def check_status(self, job_ids: list[int]):
|
132
|
+
def check_status(self, job_ids: list[int], start_time: float):
|
131
133
|
"""Prints the status of all :py:attr:`jobs` submitted.
|
132
134
|
|
133
135
|
Parameters
|
134
136
|
----------
|
135
137
|
job_ids : list[int]
|
136
138
|
The list of HPC ``job_id``s to check on.
|
139
|
+
start_time : float
|
140
|
+
The results of initial ``time.perf_counter()``.
|
137
141
|
"""
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
if n_remaining > 0:
|
156
|
-
time.sleep(30)
|
142
|
+
slurm = SLURM()
|
143
|
+
job_status = {
|
144
|
+
j: {
|
145
|
+
"status": slurm.check_status(job_id=j),
|
146
|
+
"start": start_time,
|
147
|
+
"wait": time.perf_counter() - start_time,
|
148
|
+
"run": 0,
|
149
|
+
}
|
150
|
+
for j in job_ids
|
151
|
+
}
|
152
|
+
table, complete = hpc.generate_table(job_status)
|
153
|
+
with Live(table, refresh_per_second=1) as live:
|
154
|
+
while not complete:
|
155
|
+
time.sleep(5)
|
156
|
+
job_status |= hpc.update_status(job_status)
|
157
|
+
table, complete = hpc.generate_table(job_status)
|
158
|
+
live.update(table)
|
157
159
|
|
158
160
|
def aggregate_outputs(self):
|
159
161
|
"""Collect the chunked results files, combine them into a single output parquet file, and
|
160
162
|
delete the chunked results files.
|
161
163
|
"""
|
162
|
-
result_files = [f for f in self.out_path.iterdir() if f.suffix
|
164
|
+
result_files = [f for f in self.out_path.iterdir() if f.suffix == (".pqt")]
|
163
165
|
|
164
166
|
if len(result_files) > 0:
|
165
|
-
result_agents = pd.concat([pd.
|
167
|
+
result_agents = pd.concat([pd.read_parquet(f) for f in result_files])
|
166
168
|
f_out = self.dir_out / f"run_{self.run_name}.pqt"
|
167
169
|
result_agents.to_parquet(f_out)
|
170
|
+
print(f"Aggregated results saved to: {f_out}")
|
168
171
|
|
169
172
|
for f in result_files:
|
170
173
|
f.unlink()
|
@@ -185,20 +188,22 @@ class MultiProcess:
|
|
185
188
|
base_cmd_str = f"module load conda; conda activate {self.env}; "
|
186
189
|
base_cmd_str += "dwind run-chunk "
|
187
190
|
|
188
|
-
base_args = f"
|
189
|
-
base_args += f"
|
190
|
-
base_args += f"
|
191
|
-
base_args += f"
|
192
|
-
base_args += f"
|
193
|
-
base_args += f"
|
194
|
-
base_args += f"
|
195
|
-
|
196
|
-
|
191
|
+
base_args = f" {self.location} "
|
192
|
+
base_args += f" {self.sector} "
|
193
|
+
base_args += f" {self.scenario} "
|
194
|
+
base_args += f" {self.year} "
|
195
|
+
base_args += f" {self.out_path}"
|
196
|
+
base_args += f" {self.repository} "
|
197
|
+
base_args += f" {self.model_config} "
|
198
|
+
|
199
|
+
start_time = time.perf_counter()
|
200
|
+
# for i, (start, end) in enumerate(zip(starts, ends, strict=True)):
|
201
|
+
for i, (start, end) in enumerate(zip(starts, ends)): # noqa: B905
|
197
202
|
fn = self.out_path / f"agents_{i}.pqt"
|
198
203
|
agent_df.iloc[start:end].to_parquet(fn)
|
199
204
|
|
200
205
|
job_name = f"{self.run_name}_{i}"
|
201
|
-
cmd_str = f"{base_cmd_str}
|
206
|
+
cmd_str = f"{base_cmd_str} {i} {base_args}"
|
202
207
|
print("cmd:", cmd_str)
|
203
208
|
|
204
209
|
slurm_manager = SLURM()
|
@@ -221,5 +226,5 @@ class MultiProcess:
|
|
221
226
|
)
|
222
227
|
|
223
228
|
# Check on the job statuses until they're complete, then aggregate the results
|
224
|
-
self.check_status(jobs)
|
229
|
+
self.check_status(jobs, start_time)
|
225
230
|
self.aggregate_outputs()
|
dwind/resource.py
CHANGED
@@ -62,7 +62,7 @@ class ResourcePotential:
|
|
62
62
|
self.df[config_col].astype(str) + "_" + self.df[f"tilt_{self.tech}"].astype(str)
|
63
63
|
)
|
64
64
|
elif self.tech == "wind":
|
65
|
-
configs = self.rev.settings.wind
|
65
|
+
configs = self.config.rev.settings.wind
|
66
66
|
config_col = "turbine_class"
|
67
67
|
col_list = [
|
68
68
|
"gid",
|
@@ -75,6 +75,9 @@ class ResourcePotential:
|
|
75
75
|
|
76
76
|
out_cols = [*col_list, f"rev_index_{self.tech}", f"{self.tech}_naep", f"{self.tech}_cf"]
|
77
77
|
|
78
|
+
drop_cols = [f"rev_gid_{self.tech}", f"{self.tech}_naep", f"{self.tech}_cf"]
|
79
|
+
self.df = self.df.drop(columns=[c for c in drop_cols if c in self.df])
|
80
|
+
|
78
81
|
f_gen = (
|
79
82
|
self.config.rev.generation[f"{self.tech}_DIR"]
|
80
83
|
/ f"lkup_rev_gid_to_summary_{self.tech}_{self.year}.csv"
|
dwind/run.py
CHANGED
@@ -3,19 +3,26 @@
|
|
3
3
|
from __future__ import annotations
|
4
4
|
|
5
5
|
import sys
|
6
|
-
import tomllib
|
7
6
|
from enum import Enum
|
8
|
-
from typing import Annotated
|
7
|
+
from typing import Optional, Annotated
|
9
8
|
from pathlib import Path
|
10
9
|
|
11
10
|
import typer
|
12
11
|
import pandas as pd
|
12
|
+
from rich.style import Style
|
13
|
+
from rich.pretty import pprint
|
14
|
+
from rich.console import Console
|
13
15
|
|
14
16
|
|
15
|
-
#
|
16
|
-
|
17
|
+
# fmt: off
|
18
|
+
if sys.version_info >= (3, 11): # noqa
|
19
|
+
import tomllib
|
20
|
+
else:
|
21
|
+
import tomli as tomllib
|
22
|
+
# fmt: on
|
17
23
|
|
18
24
|
app = typer.Typer()
|
25
|
+
console = Console()
|
19
26
|
|
20
27
|
DWIND = Path("/projects/dwind/agents")
|
21
28
|
|
@@ -71,20 +78,25 @@ def year_callback(ctx: typer.Context, param: typer.CallbackParam, value: int):
|
|
71
78
|
|
72
79
|
|
73
80
|
def load_agents(
|
74
|
-
file_name:
|
81
|
+
file_name: Path | None = None,
|
75
82
|
location: str | None = None,
|
76
83
|
sector: str | None = None,
|
84
|
+
model_config: str | Path | None = None,
|
85
|
+
*,
|
86
|
+
prepare: bool = False,
|
77
87
|
) -> pd.DataFrame:
|
78
88
|
"""Load the agent file based on a filename or the location and sector to a Pandas DataFrame,
|
79
89
|
and return the data frame.
|
80
90
|
|
81
91
|
Args:
|
82
|
-
file_name (
|
92
|
+
file_name (Path | None, optional): Name of the agent file, if not auto-generating from
|
83
93
|
the :py:attr:`location` and :py:attr:`sector` inputs. Defaults to None.
|
84
94
|
location (str | None, optional): The name of the location or grouping, such as
|
85
95
|
"colorado_larimer" or "priority1". Defaults to None.
|
86
96
|
sector (str | None, optional): The name of the section. Must be one of "btm" or "fom".
|
87
97
|
Defaults to None.
|
98
|
+
prepare (bool, optional): True if loading pre-chunked and prepared agent data, which should
|
99
|
+
bypass the standard column checking for additional joins, by default False.
|
88
100
|
|
89
101
|
Returns:
|
90
102
|
pd.DataFrame: The agent DataFrame.
|
@@ -97,10 +109,28 @@ def load_agents(
|
|
97
109
|
f_agents = (
|
98
110
|
file_name if file_name is not None else DWIND / f"{location}/agents_dwind_{sector}.parquet"
|
99
111
|
)
|
112
|
+
if not isinstance(f_agents, Path):
|
113
|
+
f_agents = Path(f_agents).resolve()
|
114
|
+
|
115
|
+
alternative_suffix = (".pqt", ".parquet", ".pkl", ".pickle", ".csv")
|
116
|
+
base_style = Style.parse("cyan")
|
100
117
|
if not f_agents.exists():
|
101
|
-
|
102
|
-
|
103
|
-
|
118
|
+
for suffix in alternative_suffix:
|
119
|
+
if (new_fn := f_agents.with_suffix(suffix)).exists():
|
120
|
+
if new_fn != f_agents:
|
121
|
+
msg = (
|
122
|
+
f"Using alternative agent file: {new_fn}\n\t"
|
123
|
+
f"Requested agent file: {f_agents}"
|
124
|
+
)
|
125
|
+
console.print(msg, style=base_style)
|
126
|
+
f_agents = new_fn
|
127
|
+
break
|
128
|
+
|
129
|
+
if prepare:
|
130
|
+
return Agents.load_and_prepare_agents(
|
131
|
+
agent_file=f_agents, sector=sector, model_config=model_config
|
132
|
+
)
|
133
|
+
return Agents.load_agents(agent_file=f_agents)
|
104
134
|
|
105
135
|
|
106
136
|
@app.command()
|
@@ -155,7 +185,9 @@ def run_hpc(
|
|
155
185
|
dir_out: Annotated[
|
156
186
|
str, typer.Option(help="Path to where the chunked outputs should be saved.")
|
157
187
|
],
|
158
|
-
stdout_path: Annotated[
|
188
|
+
stdout_path: Annotated[
|
189
|
+
Optional[str], typer.Option(help="The path to write stdout logs.") # noqa
|
190
|
+
] = None,
|
159
191
|
):
|
160
192
|
"""Run dwind via the HPC multiprocessing interface."""
|
161
193
|
sys.path.append(repository)
|
@@ -180,7 +212,9 @@ def run_hpc(
|
|
180
212
|
stdout_path=stdout_path,
|
181
213
|
)
|
182
214
|
|
183
|
-
agent_df = load_agents(
|
215
|
+
agent_df = load_agents(
|
216
|
+
location=location, sector=sector, model_config=model_config, prepare=True
|
217
|
+
)
|
184
218
|
mp.run_jobs(agent_df)
|
185
219
|
|
186
220
|
|
@@ -194,15 +228,15 @@ def run_hpc_from_config(
|
|
194
228
|
config_path = Path(config_path).resolve()
|
195
229
|
with config_path.open("rb") as f:
|
196
230
|
config = tomllib.load(f)
|
197
|
-
print(
|
231
|
+
print("Running the following configuration:")
|
232
|
+
pprint(config)
|
198
233
|
|
199
234
|
run_hpc(**config)
|
200
235
|
|
201
236
|
|
202
237
|
@app.command()
|
203
238
|
def run_chunk(
|
204
|
-
|
205
|
-
# end: Annotated[int, typer.Option(help="chunk end index")],
|
239
|
+
chunk_ix: Annotated[int, typer.Option(help="Chunk number/index. Used for logging.")],
|
206
240
|
location: Annotated[
|
207
241
|
str, typer.Option(help="The state, state_county, or priority region to run.")
|
208
242
|
],
|
@@ -213,7 +247,6 @@ def run_chunk(
|
|
213
247
|
year: Annotated[
|
214
248
|
int, typer.Option(callback=year_callback, help="The year basis of the scenario.")
|
215
249
|
],
|
216
|
-
chunk_ix: Annotated[int, typer.Option(help="Chunk number/index. Used for logging.")],
|
217
250
|
out_path: Annotated[str, typer.Option(help="save path")],
|
218
251
|
repository: Annotated[
|
219
252
|
str, typer.Option(help="Path to the dwind repository to use when running the model.")
|
@@ -230,7 +263,7 @@ def run_chunk(
|
|
230
263
|
from dwind.model import Model
|
231
264
|
|
232
265
|
agent_file = Path(out_path).resolve() / f"agents_{chunk_ix}.pqt"
|
233
|
-
agents = load_agents(file_name=agent_file)
|
266
|
+
agents = load_agents(file_name=agent_file, prepare=False)
|
234
267
|
agent_file.unlink()
|
235
268
|
|
236
269
|
model = Model(
|
@@ -271,7 +304,7 @@ def run(
|
|
271
304
|
sys.path.append(repository)
|
272
305
|
from dwind.model import Model
|
273
306
|
|
274
|
-
agents = load_agents(location=location, sector=sector)
|
307
|
+
agents = load_agents(location=location, sector=sector, model_config=model_config, prepare=True)
|
275
308
|
model = Model(
|
276
309
|
agents=agents,
|
277
310
|
location=location,
|
dwind/scenarios.py
CHANGED
@@ -115,7 +115,8 @@ def config_financial(scenario, year):
|
|
115
115
|
f = f"/projects/dwind/configs/costs/atb24/ATB24_financing_baseline_{year}.json"
|
116
116
|
i = Path("/projects/dwind/data/incentives/2025_incentives.json").resolve()
|
117
117
|
with i.open("r") as i_in:
|
118
|
-
incentives = json.load(i_in)
|
118
|
+
incentives = pd.DataFrame.from_dict(json.load(i_in)).T
|
119
|
+
incentives.index.name = "census_tract_id"
|
119
120
|
elif scenario in scenarios and year in (2035, 2040):
|
120
121
|
f = "/projects/dwind/configs/costs/atb24/ATB24_financing_baseline_2035.json"
|
121
122
|
else:
|
@@ -125,6 +126,8 @@ def config_financial(scenario, year):
|
|
125
126
|
|
126
127
|
with f.open("r") as f_in:
|
127
128
|
financials = json.load(f_in)
|
129
|
+
|
130
|
+
# TODO: determine if shared settings is applicable going forward, or separate should be reserved
|
128
131
|
if year == 2025:
|
129
132
|
financials["BTM"]["itc_fraction_of_capex"] = incentives
|
130
133
|
financials["FOM"]["itc_fraction_of_capex"] = incentives
|
dwind/utils/__init__.py
ADDED
File without changes
|
dwind/utils/array.py
ADDED
@@ -0,0 +1,172 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import numpy as np
|
4
|
+
import pandas as pd
|
5
|
+
|
6
|
+
|
7
|
+
def memory_downcaster(df):
|
8
|
+
assert isinstance(df, pd.DataFrame) | isinstance(df, pd.Series)
|
9
|
+
|
10
|
+
NAlist = []
|
11
|
+
for col in df.select_dtypes(include=[np.number]).columns:
|
12
|
+
IsInt = False
|
13
|
+
mx = df[col].max()
|
14
|
+
mn = df[col].min()
|
15
|
+
|
16
|
+
# integer does not support na; fill na
|
17
|
+
if not np.isfinite(df[col]).all():
|
18
|
+
NAlist.append(col)
|
19
|
+
df[col].fillna(mn - 1, inplace=True)
|
20
|
+
|
21
|
+
# test if column can be converted to an integer
|
22
|
+
asint = df[col].fillna(0).astype(np.int64)
|
23
|
+
result = df[col] - asint
|
24
|
+
result = result.sum()
|
25
|
+
if result > -0.01 and result < 0.01:
|
26
|
+
IsInt = True
|
27
|
+
|
28
|
+
# make integer/unsigned integer datatypes
|
29
|
+
if IsInt:
|
30
|
+
try:
|
31
|
+
if mn >= 0:
|
32
|
+
if mx < 255:
|
33
|
+
df[col] = df[col].astype(np.uint8)
|
34
|
+
elif mx < 65535:
|
35
|
+
df[col] = df[col].astype(np.uint16)
|
36
|
+
elif mx < 4294967295:
|
37
|
+
df[col] = df[col].astype(np.uint32)
|
38
|
+
else:
|
39
|
+
df[col] = df[col].astype(np.uint64)
|
40
|
+
else:
|
41
|
+
if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:
|
42
|
+
df[col] = df[col].astype(np.int8)
|
43
|
+
elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:
|
44
|
+
df[col] = df[col].astype(np.int16)
|
45
|
+
elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:
|
46
|
+
df[col] = df[col].astype(np.int32)
|
47
|
+
elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:
|
48
|
+
df[col] = df[col].astype(np.int64)
|
49
|
+
except: # noqa: E722
|
50
|
+
df[col] = df[col].astype(np.float32)
|
51
|
+
|
52
|
+
# make float datatypes 32 bit
|
53
|
+
else:
|
54
|
+
df[col] = df[col].astype(np.float32)
|
55
|
+
|
56
|
+
return df
|
57
|
+
|
58
|
+
|
59
|
+
def interpolate_array(row, col_1, col_2, col_in, col_out):
|
60
|
+
if row[col_in] != 0:
|
61
|
+
interpolated = row[col_in] * (row[col_2] - row[col_1]) + row[col_1]
|
62
|
+
else:
|
63
|
+
interpolated = row[col_1]
|
64
|
+
|
65
|
+
row[col_out] = interpolated
|
66
|
+
|
67
|
+
return row
|
68
|
+
|
69
|
+
|
70
|
+
def scale_array_precision(df: pd.DataFrame, hourly_col: str, prec_offset_col: str):
|
71
|
+
"""Scales the precision of :py:attr:`hourly_col` by the :py:attr:`prec_offset_col`.
|
72
|
+
|
73
|
+
Args:
|
74
|
+
df (pd.DataFrame): A Pandas DataFrame containing :py:att:`hourly_col` and
|
75
|
+
:py:att:`prec_offset_col`.
|
76
|
+
hourly_col (str) The column to adjust the precision.
|
77
|
+
prec_offset_col (str): The column for scaling the precison of :py:attr:`hourly_col`.
|
78
|
+
|
79
|
+
Returns:
|
80
|
+
pd.DataFrame: The input :py:attr:`df` with the precision of :py:attr:`hourly_col` scaled.
|
81
|
+
"""
|
82
|
+
df[hourly_col] = (
|
83
|
+
np.array(df[hourly_col].values.tolist(), dtype="float64")
|
84
|
+
/ df[prec_offset_col].values.reshape(-1, 1)
|
85
|
+
).tolist()
|
86
|
+
return df
|
87
|
+
|
88
|
+
|
89
|
+
def scale_array_deprecision(df: pd.DataFrame, col: str | list[str]) -> pd.DataFrame:
|
90
|
+
"""Rounds the column(s) :py:attr:`col` to the nearest 2nd decimal and converts to NumPy's
|
91
|
+
float32.
|
92
|
+
|
93
|
+
Args:
|
94
|
+
df (pd.DataFrame): A Pandas DataFrame containing :py:att:`col`.
|
95
|
+
col (str | list[str]): The column(s) to have reduced precision.
|
96
|
+
|
97
|
+
Returns:
|
98
|
+
pd.DataFrame: The input :py:attr:`df` with the precision of :py:attr:`col` lowered.
|
99
|
+
"""
|
100
|
+
df[col] = np.round(np.round(df[col], 2).astype(np.float32), 2)
|
101
|
+
return df
|
102
|
+
|
103
|
+
|
104
|
+
def scale_array_sum(df: pd.DataFrame, hourly_col: str, scale_col: str) -> pd.DataFrame:
|
105
|
+
"""Scales the :py:attr:`hourly_col` by its sum and multiples by the :py:attr:`scale_col`.
|
106
|
+
|
107
|
+
Args:
|
108
|
+
df (pd.DataFrame): Pandas DataFrame containing the :py:attr:`hourly_col` and
|
109
|
+
:py:attr:`scale_col`.
|
110
|
+
hourly_col (str): The name of the column to be scaled whose values are lists.
|
111
|
+
scale_col (str): The column to scale the :py:attr:`hourly_col`.
|
112
|
+
|
113
|
+
Returns:
|
114
|
+
pandas.DataFrame: The input dataframe, but with the values of the :py:attr:`hourly_col`
|
115
|
+
scaled appropriately.
|
116
|
+
"""
|
117
|
+
hourly_array = np.array(df[hourly_col].values.tolist())
|
118
|
+
df[hourly_col] = (
|
119
|
+
hourly_array / hourly_array.sum(axis=1).reshape(-1, 1) * df[scale_col].values.reshape(-1, 1)
|
120
|
+
).tolist()
|
121
|
+
return df
|
122
|
+
|
123
|
+
|
124
|
+
def scale_array_multiplier(
|
125
|
+
df: pd.DataFrame, hourly_col: str, multiplier_col: str, col_out: str
|
126
|
+
) -> pd.DataFrame:
|
127
|
+
"""Scales the :py:attr:hourly_col` values by the :py:attr:`multiplier_col`, and places it in
|
128
|
+
the :py:attr:`col_out`.
|
129
|
+
|
130
|
+
Args:
|
131
|
+
df (pd.DataFrame): The Pandas DataFrame containing the :py:attr:`hourly_col` and
|
132
|
+
:py:attr:`multiplier_col`.
|
133
|
+
hourly_col (str): A column of hourly values as a list of floats in each cell.
|
134
|
+
multiplier_col (str): The column used to scale the :py:attr:`hourly_col`.
|
135
|
+
col_out (str): A new column that will contain the scaled data.
|
136
|
+
|
137
|
+
Returns:
|
138
|
+
pd.DataFrame: A new copy of the original data (:py:attr:`df`) containing the
|
139
|
+
:py:attr:`col_out` column.
|
140
|
+
"""
|
141
|
+
hourly_array = np.array(df[hourly_col].values.tolist())
|
142
|
+
df[col_out] = (hourly_array * df[multiplier_col].values.reshape(-1, 1)).tolist()
|
143
|
+
return df
|
144
|
+
|
145
|
+
|
146
|
+
def split_by_index(
|
147
|
+
arr: pd.DataFrame | np.ndarray | pd.Series, n_splits: int
|
148
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
149
|
+
"""Split a DataFrame, Series, or array like with np.array_split, but only return the start and
|
150
|
+
stop indices, rather than chunks. For Pandas objects, this are equivalent to
|
151
|
+
``arr.iloc[start: end]`` and for NumPy: ``arr[start: end]``. Splits are done according
|
152
|
+
to the 0th dimension.
|
153
|
+
|
154
|
+
Args:
|
155
|
+
arr(pd.DataFrame | pd.Series | np.ndarray): The array, data frame, or series to split.
|
156
|
+
n_splits(:obj:`int`): The number of near equal or equal splits.
|
157
|
+
|
158
|
+
Returns:
|
159
|
+
tuple[np.ndarray, np.ndarray]
|
160
|
+
"""
|
161
|
+
size = arr.shape[0]
|
162
|
+
base = np.arange(n_splits)
|
163
|
+
split_size = size // n_splits
|
164
|
+
extra = size % n_splits
|
165
|
+
|
166
|
+
starts = base * split_size
|
167
|
+
ends = starts + split_size
|
168
|
+
|
169
|
+
for i in range(extra):
|
170
|
+
ends[i:] += 1
|
171
|
+
starts[i + 1 :] += 1
|
172
|
+
return starts, ends
|
dwind/utils/hpc.py
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
import time
|
2
|
+
|
3
|
+
from rich.table import Table
|
4
|
+
from rex.utilities.hpc import SLURM
|
5
|
+
|
6
|
+
|
7
|
+
def convert_seconds_for_print(time: float) -> str:
|
8
|
+
"""Convert number of seconds to number of hours, minutes, and seconds."""
|
9
|
+
div = ((60, "seconds"), (60, "minutes"), (24, "hours"))
|
10
|
+
|
11
|
+
result = []
|
12
|
+
value = time
|
13
|
+
for divisor, label in div:
|
14
|
+
if not divisor:
|
15
|
+
remainder = value
|
16
|
+
if not remainder:
|
17
|
+
break
|
18
|
+
else:
|
19
|
+
value, remainder = divmod(value, divisor)
|
20
|
+
if not value and not remainder:
|
21
|
+
break
|
22
|
+
if remainder == 1:
|
23
|
+
label = label[:-1]
|
24
|
+
|
25
|
+
# 0.2 second precision for seconds, and no decimals otherwise
|
26
|
+
if result:
|
27
|
+
result.append(f"{remainder:,.0f} {label}")
|
28
|
+
else:
|
29
|
+
result.append(f"{remainder:.1f} {label}")
|
30
|
+
if result:
|
31
|
+
return ", ".join(reversed(result))
|
32
|
+
return "0"
|
33
|
+
|
34
|
+
|
35
|
+
def update_status(job_status: dict) -> dict:
|
36
|
+
"""Get an updated status and timing statistics for all running jobs on the HPC.
|
37
|
+
|
38
|
+
Args:
|
39
|
+
job_status (dict): Dictionary of job id (primary key) with sub keys of "status",
|
40
|
+
"start_time" (initial or start of run status), "wait", and "run".
|
41
|
+
|
42
|
+
Returns:
|
43
|
+
dict: Dictionary of updated statuses and timing statistics for all current queued and
|
44
|
+
running jobs.
|
45
|
+
"""
|
46
|
+
slurm = SLURM()
|
47
|
+
update = {}
|
48
|
+
for job, vals in job_status.items():
|
49
|
+
original_status = vals["status"]
|
50
|
+
if original_status in ("CG", "CF", "None", None):
|
51
|
+
continue
|
52
|
+
new_status = slurm.check_status(job_id=job)
|
53
|
+
if new_status == "PD":
|
54
|
+
update[job] = vals | {"status": new_status, "wait": time.perf_counter() - vals["start"]}
|
55
|
+
elif new_status == "R":
|
56
|
+
if original_status != "R":
|
57
|
+
update[job] = vals | {
|
58
|
+
"status": new_status,
|
59
|
+
"wait": time.perf_counter() - vals["start"],
|
60
|
+
"start": time.perf_counter(),
|
61
|
+
}
|
62
|
+
else:
|
63
|
+
update[job] = vals | {"run": time.perf_counter() - vals["start"]}
|
64
|
+
elif new_status in ("CG", "CF", "None", None):
|
65
|
+
update[job] = vals | {"status": new_status, "run": time.perf_counter() - vals["start"]}
|
66
|
+
else:
|
67
|
+
raise ValueError(f"Unaccounted for status code: {new_status}")
|
68
|
+
return update
|
69
|
+
|
70
|
+
|
71
|
+
def generate_table(job_status: dict) -> tuple[Table, bool]:
|
72
|
+
"""Generate the job status run time statistics table.
|
73
|
+
|
74
|
+
Args:
|
75
|
+
job_status (dict): Dictionary of job id (primary key) with sub keys of "status",
|
76
|
+
"start_time" (initial or start of run status), "wait", and "run".
|
77
|
+
|
78
|
+
Returns:
|
79
|
+
Table: ``rich.Table`` of human readable statistics.
|
80
|
+
bool: True if all jobs are complete, otherwise False.
|
81
|
+
"""
|
82
|
+
table = Table()
|
83
|
+
table.add_column("Job ID")
|
84
|
+
table.add_column("Status")
|
85
|
+
table.add_column("Wait time")
|
86
|
+
table.add_column("Run time")
|
87
|
+
|
88
|
+
for job, vals in job_status.items():
|
89
|
+
status = vals["status"]
|
90
|
+
_wait = vals["wait"]
|
91
|
+
_run = vals["run"]
|
92
|
+
table.add_row(
|
93
|
+
job, status, convert_seconds_for_print(_wait), convert_seconds_for_print(_run)
|
94
|
+
)
|
95
|
+
done = all(el["status"] in ("CG", None) for el in job_status.values())
|
96
|
+
return table, done
|
dwind/valuation.py
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import os
|
1
2
|
import time
|
2
3
|
import logging
|
3
4
|
import functools
|
@@ -54,15 +55,20 @@ class ValueFunctions:
|
|
54
55
|
_load_sql = functools.partial(
|
55
56
|
loader.load_df,
|
56
57
|
year=self.year,
|
57
|
-
|
58
|
+
sql_constructor=self.config.sql.ATLAS_PG_CON_STR,
|
58
59
|
)
|
60
|
+
cost_dir = self.config.cost.DIR
|
59
61
|
|
60
|
-
self.retail_rate_inputs = _load_csv(self.config.cost.RETAIL_RATE_INPUT_TABLE)
|
61
|
-
self.wholesale_rate_inputs = _load_csv(
|
62
|
-
|
62
|
+
self.retail_rate_inputs = _load_csv(cost_dir / self.config.cost.RETAIL_RATE_INPUT_TABLE)
|
63
|
+
self.wholesale_rate_inputs = _load_csv(
|
64
|
+
cost_dir / self.config.cost.WHOLESALE_RATE_INPUT_TABLE
|
65
|
+
)
|
66
|
+
self.depreciation_schedule_inputs = _load_csv(
|
67
|
+
cost_dir / self.config.cost.DEPREC_INPUTS_TABLE
|
68
|
+
)
|
63
69
|
|
64
70
|
if "wind" in self.config.project.settings.TECHS:
|
65
|
-
self.wind_price_inputs =
|
71
|
+
self.wind_price_inputs = _load_csv(cost_dir / self.config.cost.WIND_PRICE_INPUT_TABLE)
|
66
72
|
self.wind_tech_inputs = _load_sql(self.config.cost.WIND_TECH_INPUT_TABLE)
|
67
73
|
self.wind_derate_inputs = _load_sql(self.config.cost.WIND_DERATE_INPUT_TABLE)
|
68
74
|
|
@@ -177,10 +183,7 @@ class ValueFunctions:
|
|
177
183
|
# field, and need to be removed, then rejoined with the appropriate column names
|
178
184
|
financial = self.FINANCIAL_INPUTS["BTM"].copy()
|
179
185
|
if self.year == 2025:
|
180
|
-
incentives =
|
181
|
-
self.FINANCIAL_INPUTS["BTM"].pop("itc_fraction_of_capex")
|
182
|
-
).T
|
183
|
-
incentives.index.name = "census_tract_id"
|
186
|
+
incentives = self.FINANCIAL_INPUTS["BTM"].pop("itc_fraction_of_capex")
|
184
187
|
|
185
188
|
deprec_sch = pd.DataFrame()
|
186
189
|
deprec_sch["sector_abbr"] = financial["deprec_sch"].keys()
|
@@ -237,52 +240,30 @@ class ValueFunctions:
|
|
237
240
|
return df
|
238
241
|
|
239
242
|
def _preprocess_fom(self, df, tech="wind"):
|
240
|
-
columns = [
|
241
|
-
"yr",
|
242
|
-
"cambium_scenario",
|
243
|
-
"analysis_period",
|
244
|
-
"debt_option",
|
245
|
-
"debt_percent",
|
246
|
-
"inflation_rate",
|
247
|
-
"dscr",
|
248
|
-
"real_discount_rate",
|
249
|
-
"term_int_rate",
|
250
|
-
"term_tenor",
|
251
|
-
f"ptc_fed_amt_{tech}",
|
252
|
-
"itc_fed_pct",
|
253
|
-
"deg",
|
254
|
-
"system_capex_per_kw",
|
255
|
-
"system_om_per_kw",
|
256
|
-
]
|
257
|
-
|
258
243
|
itc_fraction_of_capex = self.FINANCIAL_INPUTS["FOM"]["itc_fraction_of_capex"]
|
259
|
-
|
260
|
-
self.year,
|
261
|
-
self.CAMBIUM_SCENARIO,
|
262
|
-
self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
|
263
|
-
self.FINANCIAL_INPUTS["FOM"]["debt_option"],
|
264
|
-
self.FINANCIAL_INPUTS["FOM"]["debt_percent"] * 100,
|
265
|
-
self.FINANCIAL_INPUTS["FOM"]["inflation"] * 100,
|
266
|
-
self.FINANCIAL_INPUTS["FOM"]["dscr"],
|
267
|
-
self.FINANCIAL_INPUTS["FOM"]["discount_rate"] * 100,
|
268
|
-
self.FINANCIAL_INPUTS["FOM"]["interest_rate"] * 100,
|
269
|
-
self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
|
270
|
-
self.
|
271
|
-
|
272
|
-
self.
|
273
|
-
self.COST_INPUTS["FOM"]["
|
274
|
-
self.
|
275
|
-
|
276
|
-
df[columns] = values
|
244
|
+
df = df.assign(
|
245
|
+
yr=self.year,
|
246
|
+
cambium_scenario=self.CAMBIUM_SCENARIO,
|
247
|
+
analysis_period=self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
|
248
|
+
debt_option=self.FINANCIAL_INPUTS["FOM"]["debt_option"],
|
249
|
+
debt_percent=self.FINANCIAL_INPUTS["FOM"]["debt_percent"] * 100,
|
250
|
+
inflation_rate=self.FINANCIAL_INPUTS["FOM"]["inflation"] * 100,
|
251
|
+
dscr=self.FINANCIAL_INPUTS["FOM"]["dscr"],
|
252
|
+
real_discount_rate=self.FINANCIAL_INPUTS["FOM"]["discount_rate"] * 100,
|
253
|
+
term_int_rate=self.FINANCIAL_INPUTS["FOM"]["interest_rate"] * 100,
|
254
|
+
term_tenor=self.FINANCIAL_INPUTS["FOM"]["system_lifetime"],
|
255
|
+
itc_fed_pct=itc_fraction_of_capex if self.year != 2025 else 0.3,
|
256
|
+
deg=self.FINANCIAL_INPUTS["FOM"]["degradation"],
|
257
|
+
system_capex_per_kw=self.COST_INPUTS["FOM"]["system_capex_per_kw"][tech],
|
258
|
+
system_om_per_kw=self.COST_INPUTS["FOM"]["system_om_per_kw"][tech],
|
259
|
+
**{f"ptc_fed_amt_{tech}": self.FINANCIAL_INPUTS["FOM"]["ptc_fed_dlrs_per_kwh"][tech]},
|
260
|
+
)
|
277
261
|
# 2025 uses census-tract based applicable credit for the itc_fed_pct, so update accordingly
|
278
262
|
if self.year == 2025:
|
279
|
-
incentives =
|
280
|
-
|
281
|
-
).T
|
282
|
-
incentives.index.name = "census_tract_id"
|
263
|
+
incentives = self.FINANCIAL_INPUTS["FOM"]["itc_fraction_of_capex"]
|
264
|
+
|
283
265
|
df = df.set_index("census_tract_id", drop=False).join(incentives).reset_index(drop=True)
|
284
|
-
df.itc_fed_pct = df.applicable_credit
|
285
|
-
df.itc_fed_pct = df.itc_fed_pct.fillna(0.3)
|
266
|
+
df.itc_fed_pct = df.applicable_credit.fillna(0.3)
|
286
267
|
|
287
268
|
return df
|
288
269
|
|
@@ -353,6 +334,9 @@ class ValueFunctions:
|
|
353
334
|
verb = self.config.project.settings.VERBOSITY
|
354
335
|
|
355
336
|
if max_w > 1:
|
337
|
+
# Override project-level setting to ensure memory intensive calculations don't
|
338
|
+
# cause jobs to silently fail
|
339
|
+
max_w = min(int(os.cpu_count() * 0.8), max_w)
|
356
340
|
results_list = []
|
357
341
|
|
358
342
|
with cf.ProcessPoolExecutor(max_workers=max_w) as executor:
|
@@ -479,7 +463,7 @@ def fetch_cambium_values(row, generation_hourly, cambium_dir, cambium_value, low
|
|
479
463
|
rev["cleared"] = rev["cleared"].apply(np.floor)
|
480
464
|
|
481
465
|
rev = rev[["cleared", "value"]]
|
482
|
-
tup = tuple(map(tuple, rev.values))
|
466
|
+
tup = tuple(map(tuple, rev.values.tolist()))
|
483
467
|
|
484
468
|
return tup
|
485
469
|
|
@@ -642,10 +626,8 @@ def find_breakeven(
|
|
642
626
|
full_output=full_output,
|
643
627
|
disp=disp,
|
644
628
|
)
|
645
|
-
|
646
|
-
return breakeven_cost_usd_p_kw, {
|
647
|
-
k: loan.Outputs.export().get(k, None) for k in pysam_outputs
|
648
|
-
}
|
629
|
+
results = loan.Outputs.export()
|
630
|
+
return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
|
649
631
|
|
650
632
|
except Exception as e:
|
651
633
|
raise ValueError("Root finding failed.") from e
|
@@ -678,9 +660,8 @@ def find_breakeven(
|
|
678
660
|
disp=disp,
|
679
661
|
)
|
680
662
|
|
681
|
-
|
682
|
-
|
683
|
-
}
|
663
|
+
results = loan.Outputs.export()
|
664
|
+
return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
|
684
665
|
|
685
666
|
except Exception as e:
|
686
667
|
raise ValueError("Root finding failed.") from e
|
@@ -719,9 +700,8 @@ def find_breakeven(
|
|
719
700
|
disp=disp,
|
720
701
|
)
|
721
702
|
|
722
|
-
|
723
|
-
|
724
|
-
}
|
703
|
+
results = loan.Outputs.export()
|
704
|
+
return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
|
725
705
|
|
726
706
|
except Exception as e:
|
727
707
|
raise ValueError("Root finding failed.") from e
|
@@ -825,9 +805,8 @@ def find_breakeven_fom(
|
|
825
805
|
disp=disp,
|
826
806
|
)
|
827
807
|
|
828
|
-
|
829
|
-
|
830
|
-
}
|
808
|
+
results = financial.Outputs.export()
|
809
|
+
return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
|
831
810
|
|
832
811
|
except Exception as e:
|
833
812
|
raise ValueError("Root finding failed.") from e
|
@@ -860,9 +839,8 @@ def find_breakeven_fom(
|
|
860
839
|
disp=disp,
|
861
840
|
)
|
862
841
|
|
863
|
-
|
864
|
-
|
865
|
-
}
|
842
|
+
results = financial.Outputs.export()
|
843
|
+
return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
|
866
844
|
|
867
845
|
except Exception as e:
|
868
846
|
raise ValueError("Root finding failed.") from e
|
@@ -900,9 +878,8 @@ def find_breakeven_fom(
|
|
900
878
|
disp=disp,
|
901
879
|
)
|
902
880
|
|
903
|
-
|
904
|
-
|
905
|
-
}
|
881
|
+
results = financial.Outputs.export()
|
882
|
+
return breakeven_cost_usd_p_kw, {k: results.get(k) for k in pysam_outputs}
|
906
883
|
|
907
884
|
except Exception as e:
|
908
885
|
raise ValueError("Root finding failed") from e
|
@@ -1312,7 +1289,8 @@ def process_btm(
|
|
1312
1289
|
|
1313
1290
|
_ = calc_financial_performance(row["system_capex_per_kw"], row, loan, batt_costs)
|
1314
1291
|
|
1315
|
-
|
1292
|
+
results = loan.Outputs.export()
|
1293
|
+
row["additional_pysam_outputs"] = {k: results.get(k) for k in pysam_outputs}
|
1316
1294
|
|
1317
1295
|
# run root finding algorithm to find breakeven cost based on calculated NPV
|
1318
1296
|
out, _ = find_breakeven(
|
@@ -1469,9 +1447,8 @@ def process_fom(
|
|
1469
1447
|
|
1470
1448
|
log.info(f"row {row.loc['gid']} calculating financial performance")
|
1471
1449
|
_ = calc_financial_performance_fom(system_capex_per_kw, row, financial)
|
1472
|
-
|
1473
|
-
|
1474
|
-
}
|
1450
|
+
results = financial.Outputs.export()
|
1451
|
+
row["additional_pysam_outputs"] = {k: results.get(k) for k in pysam_outputs}
|
1475
1452
|
|
1476
1453
|
# run root finding algorithm to find breakeven cost based on calculated NPV
|
1477
1454
|
log.info(f"row {row.loc['gid']} breakeven")
|
@@ -1533,7 +1510,7 @@ def worker(row: pd.Series, sector: str, config: Configuration):
|
|
1533
1510
|
row,
|
1534
1511
|
generation_hourly,
|
1535
1512
|
config.project.settings.CAMBIUM_DATA_DIR,
|
1536
|
-
config.CAMBIUM_VALUE,
|
1513
|
+
config.project.settings.CAMBIUM_VALUE,
|
1537
1514
|
)
|
1538
1515
|
|
1539
1516
|
row = process_fom(
|
@@ -1548,7 +1525,7 @@ def worker(row: pd.Series, sector: str, config: Configuration):
|
|
1548
1525
|
|
1549
1526
|
# store results in dictionary
|
1550
1527
|
results[f"{tech}_breakeven_cost_{sector}"] = row["breakeven_cost_usd_p_kw"]
|
1551
|
-
results
|
1528
|
+
results |= row["additional_pysam_outputs"]
|
1552
1529
|
|
1553
1530
|
return (row["gid"], results)
|
1554
1531
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: dwind
|
3
|
-
Version: 0.3
|
3
|
+
Version: 0.3.1
|
4
4
|
Summary: Distributed Wind Generation Model
|
5
5
|
Author-email: Jane Lockshin <jane.lockshin@nrel.gov>, Paritosh Das <paritosh.das@nrel.gov>, Rob Hammond <rob.hammond@nrel.gov>
|
6
6
|
Project-URL: source, https://github.com/NREL/dwind
|
@@ -0,0 +1,20 @@
|
|
1
|
+
dwind/__init__.py,sha256=hm8U4Yhz8PoEl1YZ0kyeQoX41q5S4MGWH0iUa7EjjT4,57
|
2
|
+
dwind/btm_sizing.py,sha256=SJJWV2ZVT5UapnRWDwUc6owjDwVOu8QxiMLR8BCPdeI,5037
|
3
|
+
dwind/config.py,sha256=k-O4qNjj1YDb8ViqEZ9DKmANs-JLEVSskJ7WCLBpsZ4,4165
|
4
|
+
dwind/helper.py,sha256=BCYnZYq1aKUZZ5FBEw69Et4nEspG7l7p3X4Qb0VOkV8,6285
|
5
|
+
dwind/loader.py,sha256=-E2LYkAkXx30BwDdGmQ_pMtAUVLYaoe8zQ6BxkdCBMk,2294
|
6
|
+
dwind/model.py,sha256=oS78wG9WedOAEQvt0WawHliZcgJTjWnGeHSeElhpyKY,18541
|
7
|
+
dwind/mp.py,sha256=HzHgO8tVd5we03DtwAPv43K_c-HdppVThkvdtqNS_PE,8579
|
8
|
+
dwind/resource.py,sha256=jo5FakoN4HI569RxU1JG3eH08lXvLmfuyx4IT2Qpcz4,6834
|
9
|
+
dwind/run.py,sha256=fdxBBKEHEh9wXgJ3EtXCx1VI-wOSsfHiHQCz1I3DJu0,10015
|
10
|
+
dwind/scenarios.py,sha256=-gKfGxaeCsknBADr_uRbfpiHg-46aHGrXtDSD4jeYo8,5104
|
11
|
+
dwind/valuation.py,sha256=71q2WV77XkfBI25U6_PR32pHVwB-_OUvDzT9wniF0O8,60909
|
12
|
+
dwind/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
13
|
+
dwind/utils/array.py,sha256=BCYnZYq1aKUZZ5FBEw69Et4nEspG7l7p3X4Qb0VOkV8,6285
|
14
|
+
dwind/utils/hpc.py,sha256=90ZdBFYXGQE_b4KwvW9D0VXGycLGLpuAf2EvKI9o9uc,3340
|
15
|
+
dwind-0.3.1.dist-info/licenses/LICENSE.txt,sha256=a6siYqaIG9I77f9adAqiWPBgyL-MGjAorTMa5c1YtUY,1536
|
16
|
+
dwind-0.3.1.dist-info/METADATA,sha256=2XVOAwj10_ThOuvlAnB7RA47-SA_SbdPGCxN_6-G14w,6933
|
17
|
+
dwind-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
18
|
+
dwind-0.3.1.dist-info/entry_points.txt,sha256=i0jOR14bQHIF8Ayw9QTQ2F_lWM9spZWApao58FpgFL4,40
|
19
|
+
dwind-0.3.1.dist-info/top_level.txt,sha256=fj_esaS-KV1YSmx7yCUoXFiRJ0dHx34B876svoE7NGE,6
|
20
|
+
dwind-0.3.1.dist-info/RECORD,,
|
dwind-0.3.dist-info/RECORD
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
dwind/__init__.py,sha256=1SS8cDsZlyc8ptI9EiFCN7kRnI9sqd0sz8D78uMYL8A,55
|
2
|
-
dwind/btm_sizing.py,sha256=cDOHWgS0Mzqyqx9mgct5M6ve33TqQFn5dU3JkFRy2gY,5016
|
3
|
-
dwind/config.py,sha256=KdDj0FjaOLI4XqBE2Q3-SKslRGeFvo49JKFiKlj6kI4,4001
|
4
|
-
dwind/helper.py,sha256=BCYnZYq1aKUZZ5FBEw69Et4nEspG7l7p3X4Qb0VOkV8,6285
|
5
|
-
dwind/loader.py,sha256=DDmaI6scXK8xC15KB7I4F1yF77_vx-bvad65TIV0eik,2239
|
6
|
-
dwind/model.py,sha256=g-MYvIGW0n_uxCQtt5Egt3MRdzuFV7DSlDpZCsaMsZg,15142
|
7
|
-
dwind/mp.py,sha256=TSwv8ql2UtVyZfPmRgbixYbVepQUVKptHgLBgfrnT3Q,8469
|
8
|
-
dwind/resource.py,sha256=l7Oz-yFEmRSbK1CwMUGB16URcxkr1onkKE0nJLIJgWM,6662
|
9
|
-
dwind/run.py,sha256=3AN084jOg0XU9ArTn7LySRs6v0p64-6bQfXqVhJY3sM,8801
|
10
|
-
dwind/scenarios.py,sha256=UOACORe_Sf0a2MialEKAHNAwXHu1bD--0B94az6VNJA,4926
|
11
|
-
dwind/valuation.py,sha256=1QLPm-X_QsxrTHOBSoVHjWWLtafhaqNPCTa2u9sFaqI,61145
|
12
|
-
dwind-0.3.dist-info/licenses/LICENSE.txt,sha256=a6siYqaIG9I77f9adAqiWPBgyL-MGjAorTMa5c1YtUY,1536
|
13
|
-
dwind-0.3.dist-info/METADATA,sha256=GL6UpyyGY-kj6uy3Sf1zB7Y5rYxyFWSYUEs0nd-u_vk,6931
|
14
|
-
dwind-0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
15
|
-
dwind-0.3.dist-info/entry_points.txt,sha256=i0jOR14bQHIF8Ayw9QTQ2F_lWM9spZWApao58FpgFL4,40
|
16
|
-
dwind-0.3.dist-info/top_level.txt,sha256=fj_esaS-KV1YSmx7yCUoXFiRJ0dHx34B876svoE7NGE,6
|
17
|
-
dwind-0.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|