buildstock-fetch 1.2.0__tar.gz → 1.3.0__tar.gz

{buildstock_fetch-1.2.0/buildstock_fetch.egg-info → buildstock_fetch-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildstock-fetch
-Version: 1.2.0
+Version: 1.3.0
 Summary: This library simplifies downloading building characteristics and load curve data from NREL's ResStock and ComStock projects.
 Author-email: Switchbox <hello@switch.box>
 Project-URL: Homepage, https://switchbox-data.github.io/buildstock-fetch/

{buildstock_fetch-1.2.0 → buildstock_fetch-1.3.0}/buildstock_fetch/main.py

@@ -3,6 +3,7 @@ import json
 import tempfile
 import zipfile
 from dataclasses import asdict, dataclass
+from datetime import timedelta
 from importlib.resources import files
 from pathlib import Path
 from typing import Optional, Union
@@ -81,14 +82,7 @@ METADATA_DIR = Path(
     str(files("buildstock_fetch").joinpath("data").joinpath("building_data").joinpath("combined_metadata.parquet"))
 )
 RELEASE_JSON_FILE = Path(str(files("buildstock_fetch").joinpath("data").joinpath("buildstock_releases.json")))
-LOAD_CURVE_COLUMN_AGGREGATION = Path(
-    str(
-        files("buildstock_fetch")
-        .joinpath("data")
-        .joinpath("load_curve_column_map")
-        .joinpath("2024_resstock_load_curve_columns.csv")
-    )
-)
+LOAD_CURVE_COLUMN_AGGREGATION = Path(str(files("buildstock_fetch").joinpath("data").joinpath("load_curve_column_map")))
 WEATHER_FILE_DIR = Path(str(files("buildstock_fetch").joinpath("data").joinpath("weather_station_map")))
 
 
@@ -752,10 +746,19 @@ def _create_aggregation_expressions(load_curve: pl.DataFrame, column_aggregation
     return agg_exprs
 
 
-def _aggregate_load_curve_aggregate(load_curve: pl.DataFrame, aggregate_time_step: str) -> pl.DataFrame:
-    """Aggregate the 15-minute load curve to specified time step based on aggregation rules."""
+def _aggregate_load_curve_aggregate(
+    load_curve: pl.DataFrame, aggregate_time_step: str, release_year: str
+) -> pl.DataFrame:
+    """Aggregate the 15-minute load curve to specified time step based on aggregation rules.
+
+    Removes the last row to ensure complete aggregation periods.
+    """
     # Read the aggregation rules from CSV
-    aggregation_rules = pl.read_csv(LOAD_CURVE_COLUMN_AGGREGATION)
+    if release_year == "2024":
+        load_curve_map = LOAD_CURVE_COLUMN_AGGREGATION.joinpath("2024_resstock_load_curve_columns.csv")
+    elif release_year == "2022":
+        load_curve_map = LOAD_CURVE_COLUMN_AGGREGATION.joinpath("2022_resstock_load_curve_columns.csv")
+    aggregation_rules = pl.read_csv(load_curve_map)
 
     # Create a dictionary mapping column names to their aggregation functions
     column_aggregations = dict(zip(aggregation_rules["name"], aggregation_rules["Aggregate_function"]))
@@ -768,6 +771,13 @@ def _aggregate_load_curve_aggregate(load_curve: pl.DataFrame, aggregate_time_ste
     # Convert timestamp to datetime if it's not already
     load_curve = load_curve.with_columns(pl.col("timestamp").cast(pl.Datetime))
 
+    # We want to subtract 15 minutes because the original load curve provides information
+    # for the previous 15 minutes for each timestamp. For example, the first timestamp is 00:00:15,
+    # and the columns correspond to consumption from 00:00:00 to 00:00:15. When aggregating,
+    # we want the 00:00:00 timestamp to correspond to the consumption from 00:00:00 to whenever the
+    # next timestamp is.
+    load_curve = load_curve.with_columns((pl.col("timestamp") - timedelta(minutes=15)).alias("timestamp"))
+
     # Get the grouping key configuration
     grouping_key, format_string = _get_time_step_grouping_key(aggregate_time_step)
 
@@ -787,7 +797,7 @@ def _aggregate_load_curve_aggregate(load_curve: pl.DataFrame, aggregate_time_ste
 
 
 def _download_and_process_aggregate(
-    url: str, output_file: Path, progress: Progress, task_id: TaskID, aggregate_time_step: str
+    url: str, output_file: Path, progress: Progress, task_id: TaskID, aggregate_time_step: str, release_year: str
 ) -> int:
     """Download aggregate time step load curve to temporary file, process with Polars, and save result."""
     # Get file size first for progress tracking
@@ -822,7 +832,7 @@ def _download_and_process_aggregate(
 
             # Process with Polars
             load_curve_15min = pl.read_parquet(temp_path)
-            load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step)
+            load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step, release_year)
 
             # Save processed file to final destination
             load_curve_aggregate.write_parquet(output_file)
@@ -1040,7 +1050,9 @@ def download_aggregate_time_step_load_curve_with_progress(
 
     # Download with progress tracking if progress object is provided
     if progress and task_id is not None:
-        _download_and_process_aggregate(download_url, output_file, progress, task_id, aggregate_time_step)
+        _download_and_process_aggregate(
+            download_url, output_file, progress, task_id, aggregate_time_step, bldg_id.release_year
+        )
     else:
         # For non-progress downloads, still use temp file approach for consistency
         with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as temp_file:
@@ -1052,7 +1064,9 @@ def download_aggregate_time_step_load_curve_with_progress(
 
                 # Process with Polars
                 load_curve_15min = pl.read_parquet(temp_path)
-                load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step)
+                load_curve_aggregate = _aggregate_load_curve_aggregate(
+                    load_curve_15min, aggregate_time_step, bldg_id.release_year
+                )
 
                 # Save processed file to final destination
                 load_curve_aggregate.write_parquet(output_file)
@@ -1715,8 +1729,10 @@ def fetch_bldg_data(
         total_files += len(unique_metadata_urls)  # Add metadata file
     if file_type_obj.load_curve_15min:
         total_files += len(bldg_ids)  # Add 15-minute load curve files
+    if file_type_obj.load_curve_hourly:
+        total_files += len(bldg_ids)  # Add hourly load curve files
     if file_type_obj.load_curve_monthly:
-        total_files += len(bldg_ids)  # Add 15-minute load curve files
+        total_files += len(bldg_ids)  # Add monthly load curve files
     if file_type_obj.load_curve_annual:
         total_files += len(bldg_ids)  # Add annual load curve files
     if file_type_obj.weather:
@@ -1788,6 +1804,19 @@ def _execute_downloads(
             bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
         )
 
+    if file_type_obj.load_curve_hourly:
+        aggregate_time_step = "hourly"
+        _download_aggregate_load_curves_parallel(
+            bldg_ids,
+            output_dir,
+            aggregate_time_step,
+            max_workers,
+            progress,
+            downloaded_paths,
+            failed_downloads,
+            console,
+        )
+
     if file_type_obj.load_curve_monthly:
         aggregate_time_step = "monthly"
         _download_aggregate_load_curves_parallel(

{buildstock_fetch-1.2.0 → buildstock_fetch-1.3.0}/buildstock_fetch/main_cli.py

@@ -29,7 +29,7 @@ app = typer.Typer(
 BUILDSTOCK_RELEASES_FILE = str(files("buildstock_fetch").joinpath("data").joinpath("buildstock_releases.json"))
 
 # File types that haven't been implemented yet
-UNAVAILABLE_FILE_TYPES = ["load_curve_hourly", "load_curve_daily"]
+UNAVAILABLE_FILE_TYPES = ["load_curve_daily"]
 
 
 class InvalidProductError(Exception):
@@ -125,6 +125,14 @@ def _get_release_versions_options(
     )
     available_release_versions = list({parsed_releases[release]["release_version"] for release in available_releases})
 
+    if (
+        product_type == "resstock"
+        and weather_file == "tmy3"
+        and release_year == "2024"
+        and "1" in available_release_versions
+    ):
+        available_release_versions.remove("1")
+
     # Define the desired order: "2", "1.1", "1"
     display_order = ["2", "1.1", "1"]
 
@@ -785,9 +793,6 @@ def _process_data_download(inputs: dict[str, Union[str, list[str]]]) -> None:
             if isinstance(output_dir, list):
                 output_dir = output_dir[0] if output_dir else "."
 
-            print(selected_bldg_ids)
-            print(available_weather_states)
-            print(file_type_tuple)
             fetch_bldg_data(
                 selected_bldg_ids, file_type_tuple, Path(output_dir), weather_states=available_weather_states
             )

{buildstock_fetch-1.2.0 → buildstock_fetch-1.3.0/buildstock_fetch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildstock-fetch
-Version: 1.2.0
+Version: 1.3.0
 Summary: This library simplifies downloading building characteristics and load curve data from NREL's ResStock and ComStock projects.
 Author-email: Switchbox <hello@switch.box>
 Project-URL: Homepage, https://switchbox-data.github.io/buildstock-fetch/

{buildstock_fetch-1.2.0 → buildstock_fetch-1.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "buildstock-fetch"
-version = "v1.2.0"
+version = "v1.3.0"
 description = "This library simplifies downloading building characteristics and load curve data from NREL's ResStock and ComStock projects."
 authors = [{ name = "Switchbox", email = "hello@switch.box" }]
 readme = "README.md"

{buildstock_fetch-1.2.0 → buildstock_fetch-1.3.0}/tests/test_main.py

@@ -1,6 +1,7 @@
 import json
 import shutil
 import sys
+from datetime import timedelta
 from pathlib import Path
 
 import polars as pl
@@ -724,6 +725,325 @@ def test_fetch_monthly_load_curve(cleanup_downloads):
     ).exists()
 
 
+def test_fetch_hourly_load_curve(cleanup_downloads):
+    # 2022 release
+    bldg_ids = [
+        BuildingID(
+            bldg_id=386459,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2012",
+            upgrade_id="0",
+            release_number="1",
+            state="ME",
+        ),
+        BuildingID(
+            bldg_id=247072,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2012",
+            upgrade_id="0",
+            release_number="1",
+            state="ME",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8784
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+    bldg_ids = [
+        BuildingID(
+            bldg_id=43469,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2012",
+            upgrade_id="0",
+            release_number="1.1",
+            state="FL",
+        ),
+        BuildingID(
+            bldg_id=25395,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2012",
+            upgrade_id="0",
+            release_number="1.1",
+            state="FL",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8784
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+    bldg_ids = [
+        BuildingID(
+            bldg_id=349561,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="1.1",
+            state="CO",
+        ),
+        BuildingID(
+            bldg_id=547230,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="1.1",
+            state="CO",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8760
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+    bldg_ids = [
+        BuildingID(
+            bldg_id=37283,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="1",
+            state="GA",
+        ),
+        BuildingID(
+            bldg_id=436794,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="1",
+            state="GA",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8760
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+    bldg_ids = [
+        BuildingID(
+            bldg_id=276508,
+            release_year="2022",
+            res_com="resstock",
+            weather="tmy3",
+            upgrade_id="0",
+            release_number="1",
+            state="DE",
+        ),
+        BuildingID(
+            bldg_id=330393,
+            release_year="2022",
+            res_com="resstock",
+            weather="tmy3",
+            upgrade_id="0",
+            release_number="1",
+            state="DE",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    bldg_ids = [
+        BuildingID(
+            bldg_id=394846,
+            release_year="2022",
+            res_com="resstock",
+            weather="tmy3",
+            upgrade_id="0",
+            release_number="1.1",
+            state="IN",
+        ),
+        BuildingID(
+            bldg_id=159923,
+            release_year="2022",
+            res_com="resstock",
+            weather="tmy3",
+            upgrade_id="0",
+            release_number="1.1",
+            state="IN",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8760
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+    bldg_ids = [
+        BuildingID(
+            bldg_id=37283,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="1",
+            state="GA",
+        ),
+        BuildingID(
+            bldg_id=436794,
+            release_year="2022",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="1",
+            state="GA",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8760
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+    # 2024 release - should work fine
+    bldg_ids = [
+        BuildingID(
+            bldg_id=173038,
+            release_year="2024",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="2",
+            state="RI",
+        ),
+        BuildingID(
+            bldg_id=119411,
+            release_year="2024",
+            res_com="resstock",
+            weather="amy2018",
+            upgrade_id="0",
+            release_number="2",
+            state="RI",
+        ),
+    ]
+    file_type = ("load_curve_hourly",)
+    output_dir = Path("data")
+
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir)
+    assert len(downloaded_paths) == 2
+    assert len(failed_downloads) == 0
+    bldg_id = bldg_ids[0]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+    bldg_id = bldg_ids[1]
+    assert Path(
+        f"data/{bldg_id.get_release_name()}/load_curve_hourly/state={bldg_id.state}/upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}/bldg{bldg_id.bldg_id:07d}_load_curve_hourly.parquet"
+    ).exists()
+
+    hourly_load_curve = pl.read_parquet(downloaded_paths[0])
+    timestamps = hourly_load_curve["timestamp"].to_list()
+    assert len(timestamps) == 8760
+    assert timestamps[1] - timestamps[0] == timedelta(hours=1)
+
+
 def test_fetch_weather_station_name(cleanup_downloads):
     bldg_ids = [
         BuildingID(