buildstock-fetch 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

buildstock_fetch/main.py

@@ -1,5 +1,6 @@
 import concurrent.futures
 import json
+import tempfile
 import zipfile
 from dataclasses import asdict, dataclass
 from importlib.resources import files
@@ -52,6 +53,38 @@ class No15minLoadCurveError(ValueError):
     pass
 
 
+class NoAnnualLoadCurveError(ValueError):
+    """Raised when annual load curve is not available for a release."""
+
+    pass
+
+
+class NoAggregateLoadCurveError(ValueError):
+    """Raised when no monthly load curve is available for a given release."""
+
+    pass
+
+
+class UnknownAggregationFunctionError(ValueError):
+    """Raised when an unknown aggregation function is provided."""
+
+    pass
+
+
+METADATA_DIR = Path(
+    str(files("buildstock_fetch").joinpath("data").joinpath("building_data").joinpath("combined_metadata.parquet"))
+)
+RELEASE_JSON_FILE = Path(str(files("buildstock_fetch").joinpath("data").joinpath("buildstock_releases.json")))
+LOAD_CURVE_COLUMN_AGGREGATION = Path(
+    str(
+        files("buildstock_fetch")
+        .joinpath("data")
+        .joinpath("load_curve_column_map")
+        .joinpath("2024_resstock_load_curve_columns.csv")
+    )
+)
+
+
 @dataclass
 class RequestedFileTypes:
     hpxml: bool = False
@@ -76,16 +109,45 @@ class BuildingID:
 
     @property
     def base_url(self) -> str:
-        return (
-            f"https://oedi-data-lake.s3.amazonaws.com/"
-            "nrel-pds-building-stock/"
-            "end-use-load-profiles-for-us-building-stock/"
-            f"{self.release_year}/"
-            f"{self.res_com}_{self.weather}_release_{self.release_number}/"
-        )
+        if (
+            self.release_year == "2024"
+            and self.res_com == "resstock"
+            and self.weather == "tmy3"
+            and self.release_number == "1"
+        ):
+            return (
+                f"https://oedi-data-lake.s3.amazonaws.com/"
+                "nrel-pds-building-stock/"
+                "end-use-load-profiles-for-us-building-stock/"
+                f"{self.release_year}/"
+                f"{self.res_com}_dataset_{self.release_year}.{self.release_number}/"
+                f"{self.res_com}_{self.weather}/"
+            )
+        else:
+            return (
+                f"https://oedi-data-lake.s3.amazonaws.com/"
+                "nrel-pds-building-stock/"
+                "end-use-load-profiles-for-us-building-stock/"
+                f"{self.release_year}/"
+                f"{self.res_com}_{self.weather}_release_{self.release_number}/"
+            )
+
+    def _validate_requested_file_type_availability(self, file_type: str) -> bool:
+        """Validate the requested file type is available for this release."""
+        with open(RELEASE_JSON_FILE) as f:
+            releases_data = json.load(f)
+        release_name = self.get_release_name()
+        if release_name not in releases_data:
+            return False
+        release_data = releases_data[release_name]
+        return file_type in release_data["available_data"]
 
     def get_building_data_url(self) -> str:
         """Generate the S3 download URL for this building."""
+        if not self._validate_requested_file_type_availability(
+            "hpxml"
+        ) or not self._validate_requested_file_type_availability("schedule"):
+            return ""
         if self.release_year == "2021":
             return ""
         elif self.release_year == "2022":
@@ -118,7 +180,8 @@ class BuildingID:
 
     def get_metadata_url(self) -> str:
         """Generate the S3 download URL for this building."""
-
+        if not self._validate_requested_file_type_availability("metadata"):
+            return ""
         if self.release_year == "2021":
             return f"{self.base_url}metadata/metadata.parquet"
         elif self.release_year == "2022" or self.release_year == "2023":
@@ -152,6 +215,8 @@ class BuildingID:
 
     def get_15min_load_curve_url(self) -> str:
         """Generate the S3 download URL for this building."""
+        if not self._validate_requested_file_type_availability("load_curve_15min"):
+            return ""
         if self.release_year == "2021":
             if self.upgrade_id != "0":
                 return ""  # This release only has baseline timeseries
@@ -190,6 +255,153 @@ class BuildingID:
         else:
             return ""
 
+    def get_aggregate_load_curve_url(self) -> str:
+        """Generate the S3 download URL for this building. The url is the same as the 15-minute load curve url."""
+        return self.get_15min_load_curve_url()
+
+    def get_annual_load_curve_url(self) -> str:
+        """Generate the S3 download URL for this building."""
+        if not self._validate_requested_file_type_availability("load_curve_annual"):
+            return ""
+        if self.release_year == "2021":
+            return ""
+        elif self.release_year == "2022" or self.release_year == "2023":
+            return self._build_annual_load_state_url()
+        elif self.release_year == "2024":
+            return self._handle_2024_release_annual_load()
+        elif self.release_year == "2025":
+            return self._handle_2025_release_annual_load()
+        else:
+            return ""
+
+    def get_annual_load_curve_filename(self) -> str:
+        """Generate the filename for the annual load curve."""
+        if self.release_year == "2021":
+            return ""
+        elif self.release_year == "2022" or self.release_year == "2023":
+            return f"{self.state}_upgrade{str(int(self.upgrade_id)).zfill(2)}_metadata_and_annual_results.parquet"
+        elif self.release_year == "2024":
+            if self.res_com == "comstock" and self.weather == "amy2018" and self.release_number == "2":
+                county = self._get_county_name()
+                if county == "":
+                    return ""
+                else:
+                    return f"{self.state}_{county}_upgrade{str(int(self.upgrade_id)).zfill(2)}_metadata_and_annual_results.parquet"
+            elif self.res_com == "resstock" and self.weather == "tmy3" and self.release_number == "1":
+                return ""
+            else:
+                return f"{self.state}_upgrade{str(int(self.upgrade_id)).zfill(2)}_metadata_and_annual_results.parquet"
+        elif self.release_year == "2025":
+            if self.res_com == "comstock" and self.weather == "amy2018" and self.release_number == "1":
+                county = self._get_county_name()
+                if county == "":
+                    return ""
+                else:
+                    return f"{self.state}_{county}_upgrade{str(int(self.upgrade_id)).zfill(2)}_metadata_and_annual_results.parquet"
+            else:
+                return ""
+        else:
+            return ""
+
+    def _build_annual_load_state_url(self) -> str:
+        """Build the state-level URL for annual load curve data.
+
+        Returns:
+            The constructed URL for the state-level data.
+        """
+        if self.upgrade_id == "0":
+            return (
+                f"{self.base_url}metadata_and_annual_results/"
+                f"by_state/state={self.state}/parquet/"
+                f"{self.state}_baseline_metadata_and_annual_results.parquet"
+            )
+        else:
+            return (
+                f"{self.base_url}metadata_and_annual_results/"
+                f"by_state/state={self.state}/parquet/"
+                f"{self.state}_upgrade{str(int(self.upgrade_id)).zfill(2)}_metadata_and_annual_results.parquet"
+            )
+
+    def _handle_2024_release_annual_load(self) -> str:
+        """Handle the 2024 release logic for annual load curve URLs.
+
+        Returns:
+            The constructed URL or empty string if not applicable.
+        """
+        if self.res_com == "comstock" and self.weather == "amy2018" and self.release_number == "2":
+            county = self._get_county_name()
+            if county == "":
+                return ""
+            if self.upgrade_id == "0":
+                return (
+                    f"{self.base_url}metadata_and_annual_results/"
+                    f"by_state_and_county/full/parquet/"
+                    f"state={self.state}/county={county}/"
+                    f"{self.state}_{county}_baseline.parquet"
+                )
+            else:
+                return (
+                    f"{self.base_url}metadata_and_annual_results/"
+                    f"by_state_and_county/full/parquet/"
+                    f"state={self.state}/county={county}/"
+                    f"{self.state}_{county}_upgrade{str(int(self.upgrade_id)).zfill(2)}.parquet"
+                )
+        elif self.res_com == "resstock" and self.weather == "tmy3" and self.release_number == "1":
+            return ""  # This release has a different structure. Need further development
+        else:
+            return self._build_annual_load_state_url()
+
+    def _handle_2025_release_annual_load(self) -> str:
+        """Handle the 2025 release logic for annual load curve URLs.
+
+        Returns:
+            The constructed URL or empty string if not applicable.
+        """
+        if self.res_com == "comstock" and self.weather == "amy2018" and self.release_number == "1":
+            county = self._get_county_name()
+            if county == "":
+                return ""
+            else:
+                return (
+                    f"{self.base_url}metadata_and_annual_results/"
+                    "by_state_and_county/full/parquet/"
+                    f"state={self.state}/county={county}/"
+                    f"{self.state}_{county}_upgrade{int(self.upgrade_id)!s}.parquet"
+                )
+        else:
+            return ""
+
+    def _get_county_name(self) -> str:
+        """Get the county-based URL by reading from metadata partition.
+
+        Returns:
+            The constructed URL or empty string if not found.
+        """
+        # Read the specific partition that matches our criteria
+        partition_path = (
+            METADATA_DIR
+            / f"product={self.res_com}"
+            / f"release_year={self.release_year}"
+            / f"weather_file={self.weather}"
+            / f"release_version={self.release_number}"
+            / f"state={self.state}"
+        )
+
+        # Check if the partition exists
+        if not partition_path.exists():
+            return ""
+
+        # Read the parquet files in the specific partition
+        df = pl.read_parquet(str(partition_path))
+        building_row = df.filter(pl.col("bldg_id") == self.bldg_id)
+
+        if building_row.height == 0:
+            return ""
+
+        # Return the county value from the matching row
+        county = building_row[0].select("county").item()
+        return str(county)
+
     def get_release_name(self) -> str:
         """Generate the release name for this building."""
         res_com_str = "res" if self.res_com == "resstock" else "com"
@@ -235,10 +447,6 @@ def fetch_bldg_ids(
     Returns:
         A list of building ID's for the given state.
     """
-    # Construct the absolute path to the parquet directory
-    parquet_dir = Path(
-        str(files("buildstock_fetch").joinpath("data").joinpath("building_data").joinpath("combined_metadata.parquet"))
-    )
 
     if product == "resstock":
         product_str = "res"
@@ -253,7 +461,7 @@ def fetch_bldg_ids(
 
     # Read the specific partition that matches our criteria
     partition_path = (
-        parquet_dir
+        METADATA_DIR
         / f"product={product}"
         / f"release_year={release_year}"
         / f"weather_file={weather_file}"
@@ -314,6 +522,146 @@ def _download_with_progress(url: str, output_file: Path, progress: Progress, tas
     return downloaded_size
 
 
+def _get_time_step_grouping_key(aggregate_time_step: str) -> tuple[str, str]:
+    """Get the grouping key and format string for a given time step.
+
+    Args:
+        aggregate_time_step: The time step to aggregate to ("monthly", "hourly", "daily")
+
+    Returns:
+        A tuple of (grouping_key_name, format_string)
+    """
+    time_step_configs = {
+        "monthly": ("year_month", "%Y-%m"),
+        "hourly": ("year_month_day_hour", "%Y-%m-%d-%H"),
+        "daily": ("year_month_day", "%Y-%m-%d"),
+    }
+
+    if aggregate_time_step not in time_step_configs:
+        msg = f"Unknown aggregate time step: {aggregate_time_step}"
+        raise ValueError(msg)
+
+    return time_step_configs[aggregate_time_step]
+
+
+def _create_aggregation_expressions(load_curve: pl.DataFrame, column_aggregations: dict[str, str]) -> list[pl.Expr]:
+    """Create aggregation expressions for each column based on the aggregation rules.
+
+    Args:
+        load_curve: The DataFrame to create expressions for
+        column_aggregations: Dictionary mapping column names to aggregation functions
+
+    Returns:
+        List of Polars expressions for aggregation
+    """
+    agg_exprs = []
+
+    for col in load_curve.columns:
+        if col in ["timestamp", "year_month", "year_month_day_hour", "year_month_day"]:
+            continue
+
+        if col in column_aggregations:
+            agg_func = column_aggregations[col]
+            if agg_func == "sum":
+                agg_exprs.append(pl.col(col).sum().alias(col))
+            elif agg_func == "mean":
+                agg_exprs.append(pl.col(col).mean().alias(col))
+            elif agg_func == "first":
+                agg_exprs.append(pl.col(col).first().alias(col))
+            else:
+                raise UnknownAggregationFunctionError()
+        else:
+            raise UnknownAggregationFunctionError()
+
+    # Add timestamp aggregation (take the first timestamp of each group)
+    agg_exprs.append(pl.col("timestamp").first().alias("timestamp"))
+
+    return agg_exprs
+
+
+def _aggregate_load_curve_aggregate(load_curve: pl.DataFrame, aggregate_time_step: str) -> pl.DataFrame:
+    """Aggregate the 15-minute load curve to specified time step based on aggregation rules."""
+    # Read the aggregation rules from CSV
+    aggregation_rules = pl.read_csv(LOAD_CURVE_COLUMN_AGGREGATION)
+
+    # Create a dictionary mapping column names to their aggregation functions
+    column_aggregations = dict(zip(aggregation_rules["name"], aggregation_rules["Aggregate_function"]))
+
+    # Ensure timestamp column exists and convert to datetime if needed
+    if "timestamp" not in load_curve.columns:
+        msg = "DataFrame must contain a 'timestamp' column"
+        raise ValueError(msg)
+
+    # Convert timestamp to datetime if it's not already
+    load_curve = load_curve.with_columns(pl.col("timestamp").cast(pl.Datetime))
+
+    # Get the grouping key configuration
+    grouping_key, format_string = _get_time_step_grouping_key(aggregate_time_step)
+
+    # Create grouping key
+    load_curve = load_curve.with_columns(pl.col("timestamp").dt.strftime(format_string).alias(grouping_key))
+
+    # Create aggregation expressions
+    agg_exprs = _create_aggregation_expressions(load_curve, column_aggregations)
+
+    # Perform the aggregation
+    aggregate_data = load_curve.group_by(grouping_key).agg(agg_exprs)
+
+    # Sort by timestamp and drop the grouping column
+    aggregate_data = aggregate_data.sort("timestamp").drop(grouping_key)
+
+    return aggregate_data
+
+
+def _download_and_process_aggregate(
+    url: str, output_file: Path, progress: Progress, task_id: TaskID, aggregate_time_step: str
+) -> int:
+    """Download aggregate time step load curve to temporary file, process with Polars, and save result."""
+    # Get file size first for progress tracking
+    response = requests.head(url, timeout=30)
+    response.raise_for_status()
+    total_size = int(response.headers.get("content-length", 0))
+    progress.update(task_id, total=total_size)
+
+    # Download to temporary file
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as temp_file:
+        temp_path = Path(temp_file.name)
+
+        try:
+            # Create session with retry logic
+            session = requests.Session()
+            retry_strategy = requests.adapters.HTTPAdapter(max_retries=15)
+            session.mount("http://", retry_strategy)
+            session.mount("https://", retry_strategy)
+
+            # Download with streaming to temp file
+            response = session.get(url, stream=True, timeout=60)
+            response.raise_for_status()
+
+            downloaded_size = 0
+            with open(temp_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        file.write(chunk)
+                        downloaded_size += len(chunk)
+                        if total_size > 0:
+                            progress.update(task_id, completed=downloaded_size)
+
+            # Process with Polars
+            load_curve_15min = pl.read_parquet(temp_path)
+            load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step)
+
+            # Save processed file to final destination
+            load_curve_aggregate.write_parquet(output_file)
+
+            return downloaded_size
+
+        finally:
+            # Clean up temporary file
+            if temp_path.exists():
+                temp_path.unlink()
+
+
 def download_bldg_data(
     bldg_id: BuildingID,
     file_type: RequestedFileTypes,
@@ -375,7 +723,14 @@ def download_bldg_data(
                     # Rename to the specified convention
                     old_path = temp_dir / xml_file
                     new_name = f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{bldg_id.upgrade_id.zfill(2)}.xml"
-                    new_path = output_dir / bldg_id.get_release_name() / "hpxml" / bldg_id.state / new_name
+                    new_path = (
+                        output_dir
+                        / bldg_id.get_release_name()
+                        / "hpxml"
+                        / bldg_id.state
+                        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                        / new_name
+                    )
                     new_path.parent.mkdir(parents=True, exist_ok=True)
                     old_path.rename(new_path)
                     downloaded_paths["hpxml"] = new_path
@@ -389,7 +744,14 @@ def download_bldg_data(
                     # Rename to the specified convention
                     old_path = temp_dir / schedule_file
                     new_name = f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{bldg_id.upgrade_id.zfill(2)}_schedule.csv"
-                    new_path = output_dir / bldg_id.get_release_name() / "schedule" / bldg_id.state / new_name
+                    new_path = (
+                        output_dir
+                        / bldg_id.get_release_name()
+                        / "schedule"
+                        / bldg_id.state
+                        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                        / new_name
+                    )
                     new_path.parent.mkdir(parents=True, exist_ok=True)
                     old_path.rename(new_path)
                     downloaded_paths["schedule"] = new_path
@@ -415,7 +777,14 @@ def download_metadata(bldg_id: BuildingID, output_dir: Path) -> Path:
         raise NoMetadataError(message)
     response = requests.get(download_url, timeout=30)
     response.raise_for_status()
-    output_file = output_dir / bldg_id.get_release_name() / "metadata" / bldg_id.state / "metadata.parquet"
+    output_file = (
+        output_dir
+        / bldg_id.get_release_name()
+        / "metadata"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / "metadata.parquet"
+    )
     output_file.parent.mkdir(parents=True, exist_ok=True)
     output_file.write_bytes(response.content)
     return output_file
@@ -439,8 +808,9 @@ def download_15min_load_curve(bldg_id: BuildingID, output_dir: Path) -> Path:
         output_dir
         / bldg_id.get_release_name()
         / "load_curve_15min"
-        / bldg_id.state
-        / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
     )
     output_file.parent.mkdir(parents=True, exist_ok=True)
     output_file.write_bytes(response.content)
@@ -470,8 +840,9 @@ def download_15min_load_curve_with_progress(
         output_dir
         / bldg_id.get_release_name()
         / "load_curve_15min"
-        / bldg_id.state
-        / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
     )
     output_file.parent.mkdir(parents=True, exist_ok=True)
 
@@ -486,6 +857,66 @@ def download_15min_load_curve_with_progress(
     return output_file
 
 
+def download_aggregate_time_step_load_curve_with_progress(
+    bldg_id: BuildingID,
+    output_dir: Path,
+    progress: Optional[Progress],
+    task_id: Optional[TaskID],
+    aggregate_time_step: str,
+) -> Path:
+    """Download the aggregate time step load profile timeseries for a given building with progress tracking."""
+
+    download_url = bldg_id.get_aggregate_load_curve_url()
+    if download_url == "":
+        message = f"Aggregate load profile timeseries is not available for {bldg_id.get_release_name()}"
+        raise NoAggregateLoadCurveError(message)
+
+    if aggregate_time_step == "monthly":
+        load_curve_dir = "load_curve_monthly"
+    elif aggregate_time_step == "hourly":
+        load_curve_dir = "load_curve_hourly"
+    elif aggregate_time_step == "daily":
+        load_curve_dir = "load_curve_daily"
+    else:
+        message = f"Unknown aggregate time step: {aggregate_time_step}"
+        raise ValueError(message)
+
+    output_file = (
+        output_dir
+        / bldg_id.get_release_name()
+        / load_curve_dir
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_{aggregate_time_step}.parquet"
+    )
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Download with progress tracking if progress object is provided
+    if progress and task_id is not None:
+        _download_and_process_aggregate(download_url, output_file, progress, task_id, aggregate_time_step)
+    else:
+        # For non-progress downloads, still use temp file approach for consistency
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as temp_file:
+            temp_path = Path(temp_file.name)
+            try:
+                response = requests.get(download_url, timeout=30)
+                response.raise_for_status()
+                temp_path.write_bytes(response.content)
+
+                # Process with Polars
+                load_curve_15min = pl.read_parquet(temp_path)
+                load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step)
+
+                # Save processed file to final destination
+                load_curve_aggregate.write_parquet(output_file)
+            finally:
+                if temp_path.exists():
+                    temp_path.unlink()
+
+    return output_file
+
+
 def _parse_requested_file_type(file_type: tuple[str, ...]) -> RequestedFileTypes:
     """Parse the file type string into a RequestedFileTypes object."""
     file_type_obj = RequestedFileTypes()
@@ -554,7 +985,14 @@ def _download_metadata_with_progress(bldg: BuildingID, output_dir: Path, progres
     progress.update(metadata_task, total=total_size)
 
     # Download with progress
-    output_file = output_dir / bldg.get_release_name() / "metadata" / bldg.state / "metadata.parquet"
+    output_file = (
+        output_dir
+        / bldg.get_release_name()
+        / "metadata"
+        / f"state={bldg.state}"
+        / f"upgrade={str(int(bldg.upgrade_id)).zfill(2)}"
+        / "metadata.parquet"
+    )
     output_file.parent.mkdir(parents=True, exist_ok=True)
     _download_with_progress(download_url, output_file, progress, metadata_task)
 
@@ -638,8 +1076,9 @@ def _download_15min_load_curves_parallel(
                     output_dir
                     / bldg_id.get_release_name()
                     / "load_curve_15min"
-                    / bldg_id.state
-                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
                 )
                 failed_downloads.append(str(output_file))
                 console.print(f"[red]15 min load curve not available for {bldg_id.get_release_name()}[/red]")
@@ -649,13 +1088,162 @@ def _download_15min_load_curves_parallel(
                     output_dir
                     / bldg_id.get_release_name()
                     / "load_curve_15min"
-                    / bldg_id.state
-                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
                 )
                 failed_downloads.append(str(output_file))
                 console.print(f"[red]Download failed for 15 min load curve {bldg_id.bldg_id}: {e}[/red]")
 
 
+def _create_batch_progress_tasks(
+    bldg_ids: list[BuildingID], aggregate_time_step: str, progress: Progress, console: Console
+) -> dict[int, TaskID]:
+    """Create progress tasks for batch processing."""
+    batch_size = 100
+    num_batches = (len(bldg_ids) + batch_size - 1) // batch_size
+    console.print(f"[blue]Using batch processing: {len(bldg_ids)} buildings split into {num_batches} batches[/blue]")
+
+    load_curve_tasks = {}
+    for i in range(num_batches):
+        # Calculate how many buildings are in this batch
+        start_idx = i * batch_size
+        end_idx = min(start_idx + batch_size, len(bldg_ids))
+        batch_count = end_idx - start_idx
+
+        console.print(f"[blue]Batch {i + 1}/{num_batches}: {batch_count} buildings[/blue]")
+
+        task_id = progress.add_task(
+            f"[magenta]Batch {i + 1}/{num_batches} ({aggregate_time_step})",
+            total=batch_count,  # Set total to the number of buildings in this batch
+        )
+        load_curve_tasks[i] = task_id
+
+    return load_curve_tasks
+
+
+def _create_individual_progress_tasks(bldg_ids: list[BuildingID], progress: Progress) -> dict[int, TaskID]:
+    """Create progress tasks for individual building processing."""
+    load_curve_tasks = {}
+    for i, bldg_id in enumerate(bldg_ids):
+        task_id = progress.add_task(
+            f"[magenta]Load curve {bldg_id.bldg_id} (upgrade {bldg_id.upgrade_id})",
+            total=0,  # Will be updated when we get the file size
+        )
+        load_curve_tasks[i] = task_id
+    return load_curve_tasks
+
+
+def _download_aggregate_with_batch_progress(
+    bldg_id: BuildingID, output_dir: Path, task_id: TaskID, aggregate_time_step: str, progress: Progress
+) -> Path:
+    """Download with batch progress tracking."""
+    # Download the file without individual progress tracking
+    result = download_aggregate_time_step_load_curve_with_progress(bldg_id, output_dir, None, None, aggregate_time_step)
+    # Update batch progress by 1
+    progress.update(task_id, advance=1)
+    return result
+
+
+def _process_download_future(
+    future: concurrent.futures.Future,
+    bldg_id: BuildingID,
+    output_dir: Path,
+    aggregate_time_step: str,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+) -> None:
+    """Process a completed download future."""
+    try:
+        output_file = future.result()
+        downloaded_paths.append(output_file)
+    except NoAggregateLoadCurveError:
+        output_file = (
+            output_dir
+            / bldg_id.get_release_name()
+            / "load_curve_monthly"
+            / f"state={bldg_id.state}"
+            / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_monthly.parquet"
+        )
+        failed_downloads.append(str(output_file))
+        console.print(f"[red]Monthly load curve not available for {bldg_id.get_release_name()}[/red]")
+        raise
+    except Exception as e:
+        output_file = (
+            output_dir
+            / bldg_id.get_release_name()
+            / "load_curve_monthly"
+            / f"state={bldg_id.state}"
+            / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_monthly.parquet"
+        )
+        failed_downloads.append(str(output_file))
+        console.print(f"[red]Download failed for monthly load curve {bldg_id.bldg_id}: {e}[/red]")
+
+
+def _download_aggregate_load_curves_parallel(
+    bldg_ids: list[BuildingID],
+    output_dir: Path,
+    aggregate_time_step: str,
+    max_workers: int,
+    progress: Progress,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+) -> None:
+    """Download monthly load curves in parallel with progress tracking."""
+
+    # Create progress tasks based on dataset size
+    if len(bldg_ids) > 500:
+        load_curve_tasks = _create_batch_progress_tasks(bldg_ids, aggregate_time_step, progress, console)
+    else:
+        load_curve_tasks = _create_individual_progress_tasks(bldg_ids, progress)
+
+    # Create download functions
+    def download_aggregate_with_task_id(
+        bldg_id: BuildingID, output_dir: Path, task_id: TaskID, aggregate_time_step: str
+    ) -> Path:
+        return download_aggregate_time_step_load_curve_with_progress(
+            bldg_id, output_dir, progress, task_id, aggregate_time_step
+        )
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        if len(bldg_ids) > 500:
+            # Process in batches for large datasets
+            batch_size = 100
+            future_to_bldg = {}
+
+            for batch_idx in range(0, len(bldg_ids), batch_size):
+                batch = bldg_ids[batch_idx : batch_idx + batch_size]
+                task_id = load_curve_tasks[batch_idx // batch_size]
+
+                for bldg_id in batch:
+                    future = executor.submit(
+                        _download_aggregate_with_batch_progress,
+                        bldg_id,
+                        output_dir,
+                        task_id,
+                        aggregate_time_step,
+                        progress,
+                    )
+                    future_to_bldg[future] = bldg_id
+        else:
+            # Original behavior for smaller datasets
+            future_to_bldg = {
+                executor.submit(
+                    download_aggregate_with_task_id, bldg_id, output_dir, load_curve_tasks[i], aggregate_time_step
+                ): bldg_id
+                for i, bldg_id in enumerate(bldg_ids)
+            }
+
+        # Process completed futures
+        for future in concurrent.futures.as_completed(future_to_bldg):
+            bldg_id = future_to_bldg[future]
+            _process_download_future(
+                future, bldg_id, output_dir, aggregate_time_step, downloaded_paths, failed_downloads, console
+            )
+
+
 def _download_metadata_single(
     bldg_ids: list[BuildingID],
     output_dir: Path,
@@ -671,6 +1259,116 @@ def _download_metadata_single(
     downloaded_paths.append(metadata_file)
 
 
+def download_annual_load_curve_with_progress(
+    bldg_id: BuildingID, output_dir: Path, progress: Optional[Progress] = None, task_id: Optional[TaskID] = None
+) -> Path:
+    """Download the annual load curve for a given building with progress tracking.
+
+    Args:
+        bldg_id: A BuildingID object to download annual load curve for.
+        output_dir: Directory to save the downloaded annual load curve.
+        progress: Optional Rich progress object for tracking download progress.
+        task_id: Optional task ID for progress tracking.
+
+    Returns:
+        Path to the downloaded file.
+    """
+    download_url = bldg_id.get_annual_load_curve_url()
+    if download_url == "":
+        message = f"Annual load curve is not available for {bldg_id.get_release_name()}"
+        raise NoAnnualLoadCurveError(message)
+
+    output_filename = bldg_id.get_annual_load_curve_filename()
+    if output_filename == "":
+        message = f"Annual load curve is not available for {bldg_id.get_release_name()}"
+        raise NoAnnualLoadCurveError(message)
+
+    output_file = (
+        output_dir
+        / bldg_id.get_release_name()
+        / "load_curve_annual"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / output_filename
+    )
+
+    # If the file already exists, return it. We only need to download the file for each unique annual load curve.
+    if output_file.exists():
+        return output_file
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Download with progress tracking if progress object is provided
+    if progress and task_id is not None:
+        _download_with_progress(download_url, output_file, progress, task_id)
+    else:
+        response = requests.get(download_url, timeout=30)
+        response.raise_for_status()
+        with open(output_file, "wb") as file:
+            file.write(response.content)
+
+    return output_file
+
+
+def _download_annual_load_curves_parallel(
+    bldg_ids: list[BuildingID],
+    output_dir: Path,
+    max_workers: int,
+    progress: Progress,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+) -> None:
+    """Download annual load curves in parallel with progress tracking."""
+    # Create progress tasks for annual load curve downloads
+    annual_load_curve_tasks = {}
+    for i, bldg_id in enumerate(bldg_ids):
+        task_id = progress.add_task(
+            f"[magenta]Annual load curve {bldg_id.bldg_id} (upgrade {bldg_id.upgrade_id})",
+            total=0,  # Will be updated when we get the file size
+        )
+        annual_load_curve_tasks[i] = task_id
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Create a modified version of the download function that uses the specific task IDs
+        def download_annual_with_task_id(bldg_id: BuildingID, output_dir: Path, task_id: TaskID) -> Path:
+            return download_annual_load_curve_with_progress(bldg_id, output_dir, progress, task_id)
+
+        future_to_bldg = {
+            executor.submit(download_annual_with_task_id, bldg_id, output_dir, annual_load_curve_tasks[i]): bldg_id
+            for i, bldg_id in enumerate(bldg_ids)
+        }
+
+        for future in concurrent.futures.as_completed(future_to_bldg):
+            bldg_id = future_to_bldg[future]
+            try:
+                output_file = future.result()
+                downloaded_paths.append(output_file)
+            except NoAnnualLoadCurveError:
+                output_file = (
+                    output_dir
+                    / bldg_id.get_release_name()
+                    / "load_curve_annual"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_annual.parquet"
+                )
+                failed_downloads.append(str(output_file))
+                console.print(f"[red]Annual load curve not available for {bldg_id.get_release_name()}[/red]")
+                raise
+            except Exception as e:
+                output_file = (
+                    output_dir
+                    / bldg_id.get_release_name()
+                    / "load_curve_annual"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_annual.parquet"
+                )
+                failed_downloads.append(str(output_file))
+                console.print(f"[red]Download failed for annual load curve {bldg_id.bldg_id}: {e}[/red]")
+
+
 def _print_download_summary(downloaded_paths: list[Path], failed_downloads: list[str], console: Console) -> None:
     """Print a summary of the download results."""
     console.print("\n[bold green]Download complete![/bold green]")
@@ -701,11 +1399,15 @@ def fetch_bldg_data(
     failed_downloads: list[str] = []
 
     # Calculate total files to download
-    total_files = len(bldg_ids)
+    total_files = 0
     if file_type_obj.metadata:
         total_files += 1  # Add metadata file
     if file_type_obj.load_curve_15min:
         total_files += len(bldg_ids)  # Add 15-minute load curve files
+    if file_type_obj.load_curve_monthly:
+        total_files += len(bldg_ids)  # Add 15-minute load curve files
+    if file_type_obj.load_curve_annual:
+        total_files += len(bldg_ids)  # Add annual load curve files
 
     console.print(f"\n[bold blue]Starting download of {total_files} files...[/bold blue]")
 
@@ -739,13 +1441,29 @@ def fetch_bldg_data(
                 bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
             )
 
+        if file_type_obj.load_curve_monthly:
+            aggregate_time_step = "monthly"
+            _download_aggregate_load_curves_parallel(
+                bldg_ids,
+                output_dir,
+                aggregate_time_step,
+                max_workers,
+                progress,
+                downloaded_paths,
+                failed_downloads,
+                console,
+            )
+
+        # Get annual load curve if requested.
+        if file_type_obj.load_curve_annual:
+            _download_annual_load_curves_parallel(
+                bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
+            )
+
     _print_download_summary(downloaded_paths, failed_downloads, console)
 
     return downloaded_paths, failed_downloads
 
 
 if __name__ == "__main__":  # pragma: no cover
-    bldg_ids = fetch_bldg_ids(
-        product="resstock", release_year="2024", weather_file="amy2018", release_version="2", upgrade_id="0", state="NY"
-    )
-    print(bldg_ids[:10])
+    print(fetch_bldg_ids("comstock", "2021", "tmy3", "1", "MA", "0")[:3])