buildstock-fetch 1.0.4__py3-none-any.whl → 1.2.0__py3-none-any.whl

buildstock_fetch/main.py

@@ -1,5 +1,6 @@
 import concurrent.futures
 import json
+import tempfile
 import zipfile
 from dataclasses import asdict, dataclass
 from importlib.resources import files
@@ -58,10 +59,37 @@ class NoAnnualLoadCurveError(ValueError):
     pass
 
 
+class NoAggregateLoadCurveError(ValueError):
+    """Raised when no monthly load curve is available for a given release."""
+
+    pass
+
+
+class UnknownAggregationFunctionError(ValueError):
+    """Raised when an unknown aggregation function is provided."""
+
+    pass
+
+
+class NoWeatherFileError(ValueError):
+    """Raised when weather file is not available for a release."""
+
+    pass
+
+
 METADATA_DIR = Path(
     str(files("buildstock_fetch").joinpath("data").joinpath("building_data").joinpath("combined_metadata.parquet"))
 )
 RELEASE_JSON_FILE = Path(str(files("buildstock_fetch").joinpath("data").joinpath("buildstock_releases.json")))
+LOAD_CURVE_COLUMN_AGGREGATION = Path(
+    str(
+        files("buildstock_fetch")
+        .joinpath("data")
+        .joinpath("load_curve_column_map")
+        .joinpath("2024_resstock_load_curve_columns.csv")
+    )
+)
+WEATHER_FILE_DIR = Path(str(files("buildstock_fetch").joinpath("data").joinpath("weather_station_map")))
 
 
 @dataclass
@@ -74,6 +102,7 @@ class RequestedFileTypes:
     load_curve_daily: bool = False
     load_curve_monthly: bool = False
     load_curve_annual: bool = False
+    weather: bool = False
 
 
 @dataclass
@@ -234,6 +263,10 @@ class BuildingID:
         else:
             return ""
 
+    def get_aggregate_load_curve_url(self) -> str:
+        """Generate the S3 download URL for this building. The url is the same as the 15-minute load curve url."""
+        return self.get_15min_load_curve_url()
+
     def get_annual_load_curve_url(self) -> str:
         """Generate the S3 download URL for this building."""
         if not self._validate_requested_file_type_availability("load_curve_annual"):
@@ -249,6 +282,85 @@ class BuildingID:
         else:
             return ""
 
+    def get_weather_file_url(self) -> str:
+        """Generate the S3 download URL for this building."""
+        if self.get_weather_station_name() == "":
+            return ""
+        return self._build_weather_url()
+
+    def _build_weather_url(self) -> str:
+        """Build the weather file URL based on release year and weather type."""
+        if self.release_year == "2021":
+            return self._build_2021_weather_url()
+        elif self.release_year == "2022":
+            return self._build_2022_weather_url()
+        elif self.release_year == "2023":
+            return self._build_2023_weather_url()
+        elif self.release_year == "2024":
+            return self._build_2024_weather_url()
+        elif self.release_year == "2025":
+            return self._build_2025_weather_url()
+        else:
+            return ""
+
+    def _build_2021_weather_url(self) -> str:
+        """Build weather URL for 2021 release."""
+        if self.weather == "tmy3":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_tmy3.csv"
+        elif self.weather == "amy2018":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2018.csv"
+        elif self.weather == "amy2012":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2012.csv"
+        else:
+            return ""
+
+    def _build_2022_weather_url(self) -> str:
+        """Build weather URL for 2022 release."""
+        if self.weather == "tmy3":
+            return f"{self.base_url}weather/state={self.state}/{self.get_weather_station_name()}_TMY3.csv"
+        elif self.weather == "amy2018":
+            return f"{self.base_url}weather/state={self.state}/{self.get_weather_station_name()}_2018.csv"
+        elif self.weather == "amy2012":
+            return f"{self.base_url}weather/state={self.state}/{self.get_weather_station_name()}_2012.csv"
+        else:
+            return ""
+
+    def _build_2023_weather_url(self) -> str:
+        """Build weather URL for 2023 release."""
+        if self.weather == "tmy3":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_TMY3.csv"
+        elif self.weather == "amy2018":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2018.csv"
+        elif self.weather == "amy2012":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2012.csv"
+        else:
+            return ""
+
+    def _build_2024_weather_url(self) -> str:
+        """Build weather URL for 2024 release."""
+        if self.res_com == "comstock" and self.weather == "amy2018":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2018.csv"
+        else:
+            if self.weather == "tmy3":
+                return f"{self.base_url}weather/state={self.state}/{self.get_weather_station_name()}_TMY3.csv"
+            elif self.weather == "amy2018":
+                return f"{self.base_url}weather/state={self.state}/{self.get_weather_station_name()}_2018.csv"
+            elif self.weather == "amy2012":
+                return f"{self.base_url}weather/state={self.state}/{self.get_weather_station_name()}_2012.csv"
+            else:
+                return ""
+
+    def _build_2025_weather_url(self) -> str:
+        """Build weather URL for 2025 release."""
+        if self.weather == "tmy3":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_TMY3.csv"
+        elif self.weather == "amy2018":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2018.csv"
+        elif self.weather == "amy2012":
+            return f"{self.base_url}weather/{self.weather}/{self.get_weather_station_name()}_2012.csv"
+        else:
+            return ""
+
     def get_annual_load_curve_filename(self) -> str:
         """Generate the filename for the annual load curve."""
         if self.release_year == "2021":
@@ -278,6 +390,28 @@ class BuildingID:
         else:
             return ""
 
+    def get_weather_station_name(self) -> str:
+        """Get the weather station name for this building."""
+        weather_map_df = pl.read_parquet(WEATHER_FILE_DIR)
+
+        # Filter by multiple fields for a more specific match
+        weather_station_map = weather_map_df.filter(
+            (pl.col("product") == self.res_com)
+            & (pl.col("release_year") == self.release_year)
+            & (pl.col("weather_file") == self.weather)
+            & (pl.col("release_version") == self.release_number)
+            & (pl.col("bldg_id") == self.bldg_id)
+        )
+
+        # Check if we found a match
+        if weather_station_map.height > 0:
+            # Return the weather station name from the first (and should be only) match
+            weather_station_name = weather_station_map.select("weather_station_name").item()
+            return str(weather_station_name) if weather_station_name is not None else ""
+        else:
+            # No match found, return empty string
+            return ""
+
     def _build_annual_load_state_url(self) -> str:
         """Build the state-level URL for annual load curve data.
 
@@ -405,6 +539,11 @@ def _validate_release_name(release_name: str) -> bool:
     return release_name in valid_release_names
 
 
+def _resolve_unique_metadata_urls(bldg_ids: list[BuildingID]) -> list[str]:
+    """Resolve the unique metadata URLs for a list of building IDs."""
+    return list({bldg_id.get_metadata_url() for bldg_id in bldg_ids})
+
+
 def fetch_bldg_ids(
     product: str, release_year: str, weather_file: str, release_version: str, state: str, upgrade_id: str
 ) -> list[BuildingID]:
@@ -474,13 +613,13 @@ def fetch_bldg_ids(
 def _download_with_progress(url: str, output_file: Path, progress: Progress, task_id: TaskID) -> int:
     """Download a file with progress tracking."""
     # Get file size first
-    response = requests.head(url, timeout=30)
+    response = requests.head(url, timeout=30, verify=True)
     response.raise_for_status()
     total_size = int(response.headers.get("content-length", 0))
     progress.update(task_id, total=total_size)
 
     # Download with streaming
-    response = requests.get(url, stream=True, timeout=30)
+    response = requests.get(url, stream=True, timeout=30, verify=True)
     response.raise_for_status()
 
     downloaded_size = 0
@@ -497,6 +636,205 @@ def _download_with_progress(url: str, output_file: Path, progress: Progress, tas
     return downloaded_size
 
 
+def _download_with_progress_metadata(url: str, output_file: Path, progress: Progress, task_id: TaskID) -> int:
+    """Download a metadata file with progress tracking and append to existing file if it exists."""
+    # Get file size first
+    response = requests.head(url, timeout=30, verify=True)
+    response.raise_for_status()
+    total_size = int(response.headers.get("content-length", 0))
+    progress.update(task_id, total=total_size)
+
+    # Download with streaming
+    response = requests.get(url, stream=True, timeout=30, verify=True)
+    response.raise_for_status()
+
+    downloaded_size = 0
+
+    # Check if output file already exists
+    if output_file.exists():
+        # Read existing parquet file
+        existing_df = pl.read_parquet(output_file)
+
+        # Download new data to temporary file
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as temp_file:
+            temp_path = Path(temp_file.name)
+
+            try:
+                # Download to temp file
+                with open(temp_path, "wb") as file:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        if chunk:
+                            file.write(chunk)
+                            downloaded_size += len(chunk)
+                            if total_size > 0:
+                                progress.update(task_id, completed=downloaded_size)
+
+                # Read new data
+                new_df = pl.read_parquet(temp_path)
+
+                # Concatenate existing and new data, removing duplicates
+                combined_df = pl.concat([existing_df, new_df]).unique()
+
+                # Write combined data back to original file
+                combined_df.write_parquet(output_file)
+
+            finally:
+                # Clean up temp file
+                if temp_path.exists():
+                    temp_path.unlink()
+    else:
+        # File doesn't exist, download normally
+        with open(str(output_file), "wb") as file:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    file.write(chunk)
+                    downloaded_size += len(chunk)
+                    if total_size > 0:
+                        progress.update(task_id, completed=downloaded_size)
+
+    return downloaded_size
+
+
+def _get_time_step_grouping_key(aggregate_time_step: str) -> tuple[str, str]:
+    """Get the grouping key and format string for a given time step.
+
+    Args:
+        aggregate_time_step: The time step to aggregate to ("monthly", "hourly", "daily")
+
+    Returns:
+        A tuple of (grouping_key_name, format_string)
+    """
+    time_step_configs = {
+        "monthly": ("year_month", "%Y-%m"),
+        "hourly": ("year_month_day_hour", "%Y-%m-%d-%H"),
+        "daily": ("year_month_day", "%Y-%m-%d"),
+    }
+
+    if aggregate_time_step not in time_step_configs:
+        msg = f"Unknown aggregate time step: {aggregate_time_step}"
+        raise ValueError(msg)
+
+    return time_step_configs[aggregate_time_step]
+
+
+def _create_aggregation_expressions(load_curve: pl.DataFrame, column_aggregations: dict[str, str]) -> list[pl.Expr]:
+    """Create aggregation expressions for each column based on the aggregation rules.
+
+    Args:
+        load_curve: The DataFrame to create expressions for
+        column_aggregations: Dictionary mapping column names to aggregation functions
+
+    Returns:
+        List of Polars expressions for aggregation
+    """
+    agg_exprs = []
+
+    for col in load_curve.columns:
+        if col in ["timestamp", "year_month", "year_month_day_hour", "year_month_day"]:
+            continue
+
+        if col in column_aggregations:
+            agg_func = column_aggregations[col]
+            if agg_func == "sum":
+                agg_exprs.append(pl.col(col).sum().alias(col))
+            elif agg_func == "mean":
+                agg_exprs.append(pl.col(col).mean().alias(col))
+            elif agg_func == "first":
+                agg_exprs.append(pl.col(col).first().alias(col))
+            else:
+                raise UnknownAggregationFunctionError()
+        else:
+            raise UnknownAggregationFunctionError()
+
+    # Add timestamp aggregation (take the first timestamp of each group)
+    agg_exprs.append(pl.col("timestamp").first().alias("timestamp"))
+
+    return agg_exprs
+
+
+def _aggregate_load_curve_aggregate(load_curve: pl.DataFrame, aggregate_time_step: str) -> pl.DataFrame:
+    """Aggregate the 15-minute load curve to specified time step based on aggregation rules."""
+    # Read the aggregation rules from CSV
+    aggregation_rules = pl.read_csv(LOAD_CURVE_COLUMN_AGGREGATION)
+
+    # Create a dictionary mapping column names to their aggregation functions
+    column_aggregations = dict(zip(aggregation_rules["name"], aggregation_rules["Aggregate_function"]))
+
+    # Ensure timestamp column exists and convert to datetime if needed
+    if "timestamp" not in load_curve.columns:
+        msg = "DataFrame must contain a 'timestamp' column"
+        raise ValueError(msg)
+
+    # Convert timestamp to datetime if it's not already
+    load_curve = load_curve.with_columns(pl.col("timestamp").cast(pl.Datetime))
+
+    # Get the grouping key configuration
+    grouping_key, format_string = _get_time_step_grouping_key(aggregate_time_step)
+
+    # Create grouping key
+    load_curve = load_curve.with_columns(pl.col("timestamp").dt.strftime(format_string).alias(grouping_key))
+
+    # Create aggregation expressions
+    agg_exprs = _create_aggregation_expressions(load_curve, column_aggregations)
+
+    # Perform the aggregation
+    aggregate_data = load_curve.group_by(grouping_key).agg(agg_exprs)
+
+    # Sort by timestamp and drop the grouping column
+    aggregate_data = aggregate_data.sort("timestamp").drop(grouping_key)
+
+    return aggregate_data
+
+
+def _download_and_process_aggregate(
+    url: str, output_file: Path, progress: Progress, task_id: TaskID, aggregate_time_step: str
+) -> int:
+    """Download aggregate time step load curve to temporary file, process with Polars, and save result."""
+    # Get file size first for progress tracking
+    response = requests.head(url, timeout=30, verify=True)
+    response.raise_for_status()
+    total_size = int(response.headers.get("content-length", 0))
+    progress.update(task_id, total=total_size)
+
+    # Download to temporary file
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as temp_file:
+        temp_path = Path(temp_file.name)
+
+        try:
+            # Create session with retry logic
+            session = requests.Session()
+            retry_strategy = requests.adapters.HTTPAdapter(max_retries=15)
+            session.mount("http://", retry_strategy)
+            session.mount("https://", retry_strategy)
+
+            # Download with streaming to temp file
+            response = session.get(url, stream=True, timeout=60, verify=True)
+            response.raise_for_status()
+
+            downloaded_size = 0
+            with open(temp_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        file.write(chunk)
+                        downloaded_size += len(chunk)
+                        if total_size > 0:
+                            progress.update(task_id, completed=downloaded_size)
+
+            # Process with Polars
+            load_curve_15min = pl.read_parquet(temp_path)
+            load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step)
+
+            # Save processed file to final destination
+            load_curve_aggregate.write_parquet(output_file)
+
+            return downloaded_size
+
+        finally:
+            # Clean up temporary file
+            if temp_path.exists():
+                temp_path.unlink()
+
+
 def download_bldg_data(
     bldg_id: BuildingID,
     file_type: RequestedFileTypes,
@@ -541,7 +879,7 @@ def download_bldg_data(
         if progress and task_id is not None:
             _download_with_progress(download_url, output_file, progress, task_id)
         else:
-            response = requests.get(download_url, timeout=30)
+            response = requests.get(download_url, timeout=30, verify=True)
             response.raise_for_status()
             output_file.write_bytes(response.content)
 
@@ -563,7 +901,7 @@ def download_bldg_data(
                         / bldg_id.get_release_name()
                         / "hpxml"
                         / bldg_id.state
-                        / f"up{str(int(bldg_id.upgrade_id)).zfill(2)}"
+                        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
                         / new_name
                     )
                     new_path.parent.mkdir(parents=True, exist_ok=True)
@@ -584,7 +922,7 @@ def download_bldg_data(
                         / bldg_id.get_release_name()
                         / "schedule"
                         / bldg_id.state
-                        / f"up{str(int(bldg_id.upgrade_id)).zfill(2)}"
+                        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
                         / new_name
                     )
                     new_path.parent.mkdir(parents=True, exist_ok=True)
@@ -598,33 +936,6 @@ def download_bldg_data(
     return downloaded_paths
 
 
-def download_metadata(bldg_id: BuildingID, output_dir: Path) -> Path:
-    """Download the metadata for a given building.
-
-    Args:
-        bldg_id: A BuildingID object to download metadata for.
-        output_dir: Directory to save the downloaded metadata.
-    """
-
-    download_url = bldg_id.get_metadata_url()
-    if download_url == "":
-        message = f"Metadata is not available for {bldg_id.get_release_name()}"
-        raise NoMetadataError(message)
-    response = requests.get(download_url, timeout=30)
-    response.raise_for_status()
-    output_file = (
-        output_dir
-        / bldg_id.get_release_name()
-        / "metadata"
-        / bldg_id.state
-        / f"up{str(int(bldg_id.upgrade_id)).zfill(2)}"
-        / "metadata.parquet"
-    )
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    output_file.write_bytes(response.content)
-    return output_file
-
-
 def download_15min_load_curve(bldg_id: BuildingID, output_dir: Path) -> Path:
     """Download the 15 min load profile timeseries for a given building.
 
@@ -637,15 +948,15 @@ def download_15min_load_curve(bldg_id: BuildingID, output_dir: Path) -> Path:
     if download_url == "":
         message = f"15 min load profile timeseries is not available for {bldg_id.get_release_name()}"
         raise No15minLoadCurveError(message)
-    response = requests.get(download_url, timeout=30)
+    response = requests.get(download_url, timeout=30, verify=True)
     response.raise_for_status()
     output_file = (
         output_dir
         / bldg_id.get_release_name()
         / "load_curve_15min"
-        / bldg_id.state
-        / f"up{str(int(bldg_id.upgrade_id)).zfill(2)}"
-        / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
     )
     output_file.parent.mkdir(parents=True, exist_ok=True)
     output_file.write_bytes(response.content)
@@ -675,9 +986,9 @@ def download_15min_load_curve_with_progress(
         output_dir
         / bldg_id.get_release_name()
         / "load_curve_15min"
-        / bldg_id.state
-        / f"up{str(int(bldg_id.upgrade_id)).zfill(2)}"
-        / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
     )
     output_file.parent.mkdir(parents=True, exist_ok=True)
 
@@ -685,13 +996,73 @@ def download_15min_load_curve_with_progress(
     if progress and task_id is not None:
         _download_with_progress(download_url, output_file, progress, task_id)
     else:
-        response = requests.get(download_url, timeout=30)
+        response = requests.get(download_url, timeout=30, verify=True)
         response.raise_for_status()
         output_file.write_bytes(response.content)
 
     return output_file
 
 
+def download_aggregate_time_step_load_curve_with_progress(
+    bldg_id: BuildingID,
+    output_dir: Path,
+    progress: Optional[Progress],
+    task_id: Optional[TaskID],
+    aggregate_time_step: str,
+) -> Path:
+    """Download the aggregate time step load profile timeseries for a given building with progress tracking."""
+
+    download_url = bldg_id.get_aggregate_load_curve_url()
+    if download_url == "":
+        message = f"Aggregate load profile timeseries is not available for {bldg_id.get_release_name()}"
+        raise NoAggregateLoadCurveError(message)
+
+    if aggregate_time_step == "monthly":
+        load_curve_dir = "load_curve_monthly"
+    elif aggregate_time_step == "hourly":
+        load_curve_dir = "load_curve_hourly"
+    elif aggregate_time_step == "daily":
+        load_curve_dir = "load_curve_daily"
+    else:
+        message = f"Unknown aggregate time step: {aggregate_time_step}"
+        raise ValueError(message)
+
+    output_file = (
+        output_dir
+        / bldg_id.get_release_name()
+        / load_curve_dir
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_{aggregate_time_step}.parquet"
+    )
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Download with progress tracking if progress object is provided
+    if progress and task_id is not None:
+        _download_and_process_aggregate(download_url, output_file, progress, task_id, aggregate_time_step)
+    else:
+        # For non-progress downloads, still use temp file approach for consistency
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as temp_file:
+            temp_path = Path(temp_file.name)
+            try:
+                response = requests.get(download_url, timeout=30, verify=True)
+                response.raise_for_status()
+                temp_path.write_bytes(response.content)
+
+                # Process with Polars
+                load_curve_15min = pl.read_parquet(temp_path)
+                load_curve_aggregate = _aggregate_load_curve_aggregate(load_curve_15min, aggregate_time_step)
+
+                # Save processed file to final destination
+                load_curve_aggregate.write_parquet(output_file)
+            finally:
+                if temp_path.exists():
+                    temp_path.unlink()
+
+    return output_file
+
+
 def _parse_requested_file_type(file_type: tuple[str, ...]) -> RequestedFileTypes:
     """Parse the file type string into a RequestedFileTypes object."""
     file_type_obj = RequestedFileTypes()
@@ -711,9 +1082,38 @@ def _parse_requested_file_type(file_type: tuple[str, ...]) -> RequestedFileTypes
         file_type_obj.load_curve_monthly = True
     if "load_curve_annual" in file_type:
         file_type_obj.load_curve_annual = True
+    if "weather" in file_type:
+        file_type_obj.weather = True
     return file_type_obj
 
 
+def _process_metadata_results(bldg_ids: list[BuildingID], output_dir: Path, downloaded_paths: list[Path]) -> None:
+    """Process the results of a completed metadata download."""
+    metadata_to_bldg_id_mapping: dict[Path, list[int]] = {}
+    for bldg_id in bldg_ids:
+        output_file = (
+            output_dir
+            / bldg_id.get_release_name()
+            / "metadata"
+            / f"state={bldg_id.state}"
+            / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+            / "metadata.parquet"
+        )
+        if output_file in downloaded_paths:
+            if output_file in metadata_to_bldg_id_mapping:
+                metadata_to_bldg_id_mapping[output_file].append(bldg_id.bldg_id)
+            else:
+                metadata_to_bldg_id_mapping[output_file] = [bldg_id.bldg_id]
+
+    for metadata_file, bldg_id_list in metadata_to_bldg_id_mapping.items():
+        # Use scan_parquet for lazy evaluation and better memory efficiency
+        metadata_df_filtered = pl.scan_parquet(metadata_file).filter(pl.col("bldg_id").is_in(bldg_id_list)).collect()
+        # Write the filtered dataframe back to the same file
+        metadata_df_filtered.write_parquet(metadata_file)
+
+    return
+
+
 def _process_download_results(
     future: concurrent.futures.Future,
     bldg_id: BuildingID,
@@ -740,37 +1140,73 @@ def _process_download_results(
         console.print(f"[red]Download failed for bldg_id {bldg_id}: {e}[/red]")
 
 
-def _download_metadata_with_progress(bldg: BuildingID, output_dir: Path, progress: Progress) -> Path:
+def _download_metadata_with_progress(
+    bldg_ids: list[BuildingID],
+    output_dir: Path,
+    progress: Progress,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+) -> tuple[list[Path], list[str]]:
     """Download metadata file with progress tracking."""
-    download_url = bldg.get_metadata_url()
-    if download_url == "":
-        message = f"Metadata is not available for {bldg.get_release_name()}"
-        raise NoMetadataError(message)
+    metadata_urls = _resolve_unique_metadata_urls(bldg_ids)
+    downloaded_urls: list[str] = []
+    for bldg_id in bldg_ids:
+        output_file = (
+            output_dir
+            / bldg_id.get_release_name()
+            / "metadata"
+            / f"state={bldg_id.state}"
+            / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+            / "metadata.parquet"
+        )
+        download_url = bldg_id.get_metadata_url()
+        if download_url == "":
+            failed_downloads.append(str(output_file))
+            continue
+        if download_url in downloaded_urls:
+            continue
+        downloaded_urls.append(download_url)
+        if download_url in metadata_urls:
+            metadata_urls.remove(download_url)
+        metadata_task = progress.add_task(
+            f"[yellow]Downloading metadata: {download_url}",
+            total=0,  # Will be updated when we get the file size
+        )
+        # Get file size first
+        response = requests.head(download_url, timeout=30)
+        response.raise_for_status()
+        total_size = int(response.headers.get("content-length", 0))
+        progress.update(metadata_task, total=total_size)
 
-    # Create metadata task with progress tracking
-    metadata_task = progress.add_task(
-        "[yellow]Downloading metadata",
-        total=0,  # Will be updated when we get the file size
-    )
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            _download_with_progress_metadata(download_url, output_file, progress, metadata_task)
+            downloaded_paths.append(output_file)
+        except Exception as e:
+            failed_downloads.append(str(output_file))
+            console.print(f"[red]Download failed for metadata {bldg_id.bldg_id}: {e}[/red]")
+
+    return downloaded_paths, failed_downloads
 
-    # Get file size first
-    response = requests.head(download_url, timeout=30)
-    response.raise_for_status()
-    total_size = int(response.headers.get("content-length", 0))
-    progress.update(metadata_task, total=total_size)
 
-    # Download with progress
+def download_weather_file_with_progress(
+    bldg_id: BuildingID, output_dir: Path, progress: Progress, task_id: TaskID
+) -> Path:
+    """Download weather file with progress tracking."""
+    download_url = bldg_id.get_weather_file_url()
+    if download_url == "":
+        raise NoWeatherFileError()
     output_file = (
         output_dir
-        / bldg.get_release_name()
-        / "metadata"
-        / bldg.state
-        / f"up{str(int(bldg.upgrade_id)).zfill(2)}"
-        / "metadata.parquet"
+        / bldg_id.get_release_name()
+        / "weather"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"{bldg_id.get_weather_station_name()}.csv"
     )
     output_file.parent.mkdir(parents=True, exist_ok=True)
-    _download_with_progress(download_url, output_file, progress, metadata_task)
-
+    _download_with_progress(download_url, output_file, progress, task_id)
     return output_file
 
 
@@ -851,8 +1287,9 @@ def _download_15min_load_curves_parallel(
                     output_dir
                     / bldg_id.get_release_name()
                     / "load_curve_15min"
-                    / bldg_id.state
-                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
                 )
                 failed_downloads.append(str(output_file))
                 console.print(f"[red]15 min load curve not available for {bldg_id.get_release_name()}[/red]")
@@ -862,26 +1299,175 @@ def _download_15min_load_curves_parallel(
                     output_dir
                     / bldg_id.get_release_name()
                     / "load_curve_15min"
-                    / bldg_id.state
-                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_15min.parquet"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_15min.parquet"
                 )
                 failed_downloads.append(str(output_file))
                 console.print(f"[red]Download failed for 15 min load curve {bldg_id.bldg_id}: {e}[/red]")
 
 
-def _download_metadata_single(
+def _create_batch_progress_tasks(
+    bldg_ids: list[BuildingID], aggregate_time_step: str, progress: Progress, console: Console
+) -> dict[int, TaskID]:
+    """Create progress tasks for batch processing."""
+    batch_size = 100
+    num_batches = (len(bldg_ids) + batch_size - 1) // batch_size
+    console.print(f"[blue]Using batch processing: {len(bldg_ids)} buildings split into {num_batches} batches[/blue]")
+
+    load_curve_tasks = {}
+    for i in range(num_batches):
+        # Calculate how many buildings are in this batch
+        start_idx = i * batch_size
+        end_idx = min(start_idx + batch_size, len(bldg_ids))
+        batch_count = end_idx - start_idx
+
+        console.print(f"[blue]Batch {i + 1}/{num_batches}: {batch_count} buildings[/blue]")
+
+        task_id = progress.add_task(
+            f"[magenta]Batch {i + 1}/{num_batches} ({aggregate_time_step})",
+            total=batch_count,  # Set total to the number of buildings in this batch
+        )
+        load_curve_tasks[i] = task_id
+
+    return load_curve_tasks
+
+
+def _create_individual_progress_tasks(bldg_ids: list[BuildingID], progress: Progress) -> dict[int, TaskID]:
+    """Create progress tasks for individual building processing."""
+    load_curve_tasks = {}
+    for i, bldg_id in enumerate(bldg_ids):
+        task_id = progress.add_task(
+            f"[magenta]Load curve {bldg_id.bldg_id} (upgrade {bldg_id.upgrade_id})",
+            total=0,  # Will be updated when we get the file size
+        )
+        load_curve_tasks[i] = task_id
+    return load_curve_tasks
+
+
+def _download_aggregate_with_batch_progress(
+    bldg_id: BuildingID, output_dir: Path, task_id: TaskID, aggregate_time_step: str, progress: Progress
+) -> Path:
+    """Download with batch progress tracking."""
+    # Download the file without individual progress tracking
+    result = download_aggregate_time_step_load_curve_with_progress(bldg_id, output_dir, None, None, aggregate_time_step)
+    # Update batch progress by 1
+    progress.update(task_id, advance=1)
+    return result
+
+
+def _process_download_future(
+    future: concurrent.futures.Future,
+    bldg_id: BuildingID,
+    output_dir: Path,
+    aggregate_time_step: str,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+) -> None:
+    """Process a completed download future."""
+    try:
+        output_file = future.result()
+        downloaded_paths.append(output_file)
+    except NoAggregateLoadCurveError:
+        output_file = (
+            output_dir
+            / bldg_id.get_release_name()
+            / "load_curve_monthly"
+            / f"state={bldg_id.state}"
+            / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_monthly.parquet"
+        )
+        failed_downloads.append(str(output_file))
+        console.print(f"[red]Monthly load curve not available for {bldg_id.get_release_name()}[/red]")
+        raise
+    except Exception as e:
+        output_file = (
+            output_dir
+            / bldg_id.get_release_name()
+            / "load_curve_monthly"
+            / f"state={bldg_id.state}"
+            / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_monthly.parquet"
+        )
+        failed_downloads.append(str(output_file))
+        console.print(f"[red]Download failed for monthly load curve {bldg_id.bldg_id}: {e}[/red]")
+
+
+def _download_aggregate_load_curves_parallel(
     bldg_ids: list[BuildingID],
     output_dir: Path,
+    aggregate_time_step: str,
+    max_workers: int,
     progress: Progress,
     downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+) -> None:
+    """Download monthly load curves in parallel with progress tracking."""
+
+    # Create progress tasks based on dataset size
+    if len(bldg_ids) > 500:
+        load_curve_tasks = _create_batch_progress_tasks(bldg_ids, aggregate_time_step, progress, console)
+    else:
+        load_curve_tasks = _create_individual_progress_tasks(bldg_ids, progress)
+
+    # Create download functions
+    def download_aggregate_with_task_id(
+        bldg_id: BuildingID, output_dir: Path, task_id: TaskID, aggregate_time_step: str
+    ) -> Path:
+        return download_aggregate_time_step_load_curve_with_progress(
+            bldg_id, output_dir, progress, task_id, aggregate_time_step
+        )
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        if len(bldg_ids) > 500:
+            # Process in batches for large datasets
+            batch_size = 100
+            future_to_bldg = {}
+
+            for batch_idx in range(0, len(bldg_ids), batch_size):
+                batch = bldg_ids[batch_idx : batch_idx + batch_size]
+                task_id = load_curve_tasks[batch_idx // batch_size]
+
+                for bldg_id in batch:
+                    future = executor.submit(
+                        _download_aggregate_with_batch_progress,
+                        bldg_id,
+                        output_dir,
+                        task_id,
+                        aggregate_time_step,
+                        progress,
+                    )
+                    future_to_bldg[future] = bldg_id
+        else:
+            # Original behavior for smaller datasets
+            future_to_bldg = {
+                executor.submit(
+                    download_aggregate_with_task_id, bldg_id, output_dir, load_curve_tasks[i], aggregate_time_step
+                ): bldg_id
+                for i, bldg_id in enumerate(bldg_ids)
+            }
+
+        # Process completed futures
+        for future in concurrent.futures.as_completed(future_to_bldg):
+            bldg_id = future_to_bldg[future]
+            _process_download_future(
+                future, bldg_id, output_dir, aggregate_time_step, downloaded_paths, failed_downloads, console
+            )
+
+
+def _download_metadata(
+    bldg_ids: list[BuildingID],
+    output_dir: Path,
+    progress: Progress,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
 ) -> None:
     """Download metadata file (only one needed per release)."""
     if not bldg_ids:
         return
-
-    bldg = bldg_ids[0]
-    metadata_file = _download_metadata_with_progress(bldg, output_dir, progress)
-    downloaded_paths.append(metadata_file)
+    _download_metadata_with_progress(bldg_ids, output_dir, progress, downloaded_paths, failed_downloads, console)
+    _process_metadata_results(bldg_ids, output_dir, downloaded_paths)
 
 
 def download_annual_load_curve_with_progress(
@@ -912,8 +1498,8 @@ def download_annual_load_curve_with_progress(
         output_dir
         / bldg_id.get_release_name()
         / "load_curve_annual"
-        / bldg_id.state
-        / f"up{str(int(bldg_id.upgrade_id)).zfill(2)}"
+        / f"state={bldg_id.state}"
+        / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
         / output_filename
     )
 
@@ -927,7 +1513,7 @@ def download_annual_load_curve_with_progress(
     if progress and task_id is not None:
         _download_with_progress(download_url, output_file, progress, task_id)
     else:
-        response = requests.get(download_url, timeout=30)
+        response = requests.get(download_url, timeout=30, verify=True)
         response.raise_for_status()
         with open(output_file, "wb") as file:
             file.write(response.content)
@@ -974,8 +1560,9 @@ def _download_annual_load_curves_parallel(
                     output_dir
                     / bldg_id.get_release_name()
                     / "load_curve_annual"
-                    / bldg_id.state
-                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_annual.parquet"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_annual.parquet"
                 )
                 failed_downloads.append(str(output_file))
                 console.print(f"[red]Annual load curve not available for {bldg_id.get_release_name()}[/red]")
@@ -985,13 +1572,105 @@ def _download_annual_load_curves_parallel(
                     output_dir
                     / bldg_id.get_release_name()
                     / "load_curve_annual"
-                    / bldg_id.state
-                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}-up{str(int(bldg_id.upgrade_id)).zfill(2)}_load_curve_annual.parquet"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"bldg{str(bldg_id.bldg_id).zfill(7)}_load_curve_annual.parquet"
                 )
                 failed_downloads.append(str(output_file))
                 console.print(f"[red]Download failed for annual load curve {bldg_id.bldg_id}: {e}[/red]")
 
 
+def _download_weather_files_parallel(
+    bldg_ids: list[BuildingID],
+    output_dir: Path,
+    max_workers: int,
+    progress: Progress,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+    weather_states: Union[list[str], None] = None,
+) -> None:
+    """Download weather files in parallel with progress tracking."""
+    # Initialize weather_states to empty list if None
+    if weather_states is None:
+        weather_states = []
+
+    # Break if weather_states is empty
+    if len(weather_states) == 0:
+        for bldg_id in bldg_ids:
+            output_file = (
+                output_dir
+                / bldg_id.get_release_name()
+                / "weather"
+                / f"state={bldg_id.state}"
+                / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                / f"{bldg_id.get_weather_station_name()}.csv"
+            )
+            failed_downloads.append(str(output_file))
+            console.print(f"[red]Weather file not available for {bldg_id.get_release_name()}[/red]")
+        return
+    # Create progress tasks for weather file downloads
+    weather_file_tasks = {}
+    for i, bldg_id in enumerate(bldg_ids):
+        if bldg_id.state in weather_states:
+            task_id = progress.add_task(
+                f"[magenta]Weather file {bldg_id.bldg_id} (upgrade {bldg_id.upgrade_id})",
+                total=0,  # Will be updated when we get the file size
+            )
+            weather_file_tasks[i] = task_id
+        else:
+            output_file = (
+                output_dir
+                / bldg_id.get_release_name()
+                / "weather"
+                / f"state={bldg_id.state}"
+                / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                / f"{bldg_id.get_weather_station_name()}.csv"
+            )
+            failed_downloads.append(str(output_file))
+            console.print(f"[red]Weather file not available for {bldg_id.get_release_name()}[/red]")
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Create a modified version of the download function that uses the specific task IDs
+        def download_weather_file_with_task_id(bldg_id: BuildingID, output_dir: Path, task_id: TaskID) -> Path:
+            return download_weather_file_with_progress(bldg_id, output_dir, progress, task_id)
+
+        future_to_bldg = {
+            executor.submit(download_weather_file_with_task_id, bldg_id, output_dir, weather_file_tasks[i]): bldg_id
+            for i, bldg_id in enumerate(bldg_ids)
+        }
+
+        for future in concurrent.futures.as_completed(future_to_bldg):
+            bldg_id = future_to_bldg[future]
+            try:
+                output_file = future.result()
+                downloaded_paths.append(output_file)
+            except NoWeatherFileError:
+                output_file = (
+                    output_dir
+                    / bldg_id.get_release_name()
+                    / "weather"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"{bldg_id.get_weather_station_name()}.csv"
+                )
+                failed_downloads.append(str(output_file))
+                console.print(f"[red]Weather file not available for {bldg_id.get_release_name()}[/red]")
+                raise
+            except Exception as e:
+                output_file = (
+                    output_dir
+                    / bldg_id.get_release_name()
+                    / "weather"
+                    / f"state={bldg_id.state}"
+                    / f"upgrade={str(int(bldg_id.upgrade_id)).zfill(2)}"
+                    / f"{bldg_id.get_weather_station_name()}.csv"
+                )
+                failed_downloads.append(str(output_file))
+                console.print(f"[red]Download failed for weather file {bldg_id.bldg_id}: {e}[/red]")
+                raise
+
+
 def _print_download_summary(downloaded_paths: list[Path], failed_downloads: list[str], console: Console) -> None:
     """Print a summary of the download results."""
     console.print("\n[bold green]Download complete![/bold green]")
@@ -1003,7 +1682,11 @@ def _print_download_summary(downloaded_paths: list[Path], failed_downloads: list
 
 
 def fetch_bldg_data(
-    bldg_ids: list[BuildingID], file_type: tuple[str, ...], output_dir: Path, max_workers: int = 5
+    bldg_ids: list[BuildingID],
+    file_type: tuple[str, ...],
+    output_dir: Path,
+    max_workers: int = 5,
+    weather_states: Union[list[str], None] = None,
 ) -> tuple[list[Path], list[str]]:
     """Download building data for a given list of building ids
 
@@ -1018,17 +1701,27 @@ def fetch_bldg_data(
     file_type_obj = _parse_requested_file_type(file_type)
     console = Console()
 
+    # Initialize weather_states to empty list if None
+    if weather_states is None:
+        weather_states = []
+
     downloaded_paths: list[Path] = []
     failed_downloads: list[str] = []
 
     # Calculate total files to download
-    total_files = len(bldg_ids)
+    total_files = 0
     if file_type_obj.metadata:
-        total_files += 1  # Add metadata file
+        unique_metadata_urls = _resolve_unique_metadata_urls(bldg_ids)
+        total_files += len(unique_metadata_urls)  # Add metadata file
     if file_type_obj.load_curve_15min:
         total_files += len(bldg_ids)  # Add 15-minute load curve files
+    if file_type_obj.load_curve_monthly:
+        total_files += len(bldg_ids)  # Add 15-minute load curve files
     if file_type_obj.load_curve_annual:
         total_files += len(bldg_ids)  # Add annual load curve files
+    if file_type_obj.weather:
+        available_bldg_ids = [bldg_id for bldg_id in bldg_ids if bldg_id.state in weather_states]
+        total_files += len(available_bldg_ids) * len(weather_states)  # Add weather map files
 
     console.print(f"\n[bold blue]Starting download of {total_files} files...[/bold blue]")
 
@@ -1046,53 +1739,90 @@ def fetch_bldg_data(
         console=console,
         transient=False,
     ) as progress:
-        # Download building data if requested.
-        if file_type_obj.hpxml or file_type_obj.schedule:
-            _download_building_data_parallel(
-                bldg_ids, file_type_obj, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
-            )
+        _execute_downloads(
+            file_type_obj,
+            bldg_ids,
+            output_dir,
+            max_workers,
+            progress,
+            downloaded_paths,
+            failed_downloads,
+            console,
+            weather_states,
+        )
 
-        # Get metadata if requested. Only one building is needed to get the metadata.
-        if file_type_obj.metadata:
-            _download_metadata_single(bldg_ids, output_dir, progress, downloaded_paths)
+    _print_download_summary(downloaded_paths, failed_downloads, console)
 
-        # Get 15 min load profile timeseries if requested.
-        if file_type_obj.load_curve_15min:
-            _download_15min_load_curves_parallel(
-                bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
-            )
+    return downloaded_paths, failed_downloads
 
-        # Get annual load curve if requested.
-        if file_type_obj.load_curve_annual:
-            _download_annual_load_curves_parallel(
-                bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
-            )
 
-    _print_download_summary(downloaded_paths, failed_downloads, console)
+def _execute_downloads(
+    file_type_obj: RequestedFileTypes,
+    bldg_ids: list[BuildingID],
+    output_dir: Path,
+    max_workers: int,
+    progress: Progress,
+    downloaded_paths: list[Path],
+    failed_downloads: list[str],
+    console: Console,
+    weather_states: Union[list[str], None] = None,
+) -> None:
+    """Execute all requested downloads based on file type configuration."""
+    # Initialize weather_states to empty list if None
+    if weather_states is None:
+        weather_states = []
+
+    # Download building data if requested.
+    if file_type_obj.hpxml or file_type_obj.schedule:
+        _download_building_data_parallel(
+            bldg_ids, file_type_obj, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
+        )
 
-    return downloaded_paths, failed_downloads
+    # Get metadata if requested. Only one building is needed to get the metadata.
+    if file_type_obj.metadata:
+        _download_metadata(bldg_ids, output_dir, progress, downloaded_paths, failed_downloads, console)
+
+    # Get 15 min load profile timeseries if requested.
+    if file_type_obj.load_curve_15min:
+        _download_15min_load_curves_parallel(
+            bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
+        )
+
+    if file_type_obj.load_curve_monthly:
+        aggregate_time_step = "monthly"
+        _download_aggregate_load_curves_parallel(
+            bldg_ids,
+            output_dir,
+            aggregate_time_step,
+            max_workers,
+            progress,
+            downloaded_paths,
+            failed_downloads,
+            console,
+        )
+
+    # Get annual load curve if requested.
+    if file_type_obj.load_curve_annual:
+        _download_annual_load_curves_parallel(
+            bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console
+        )
+
+    # Get weather files if requested.
+    if file_type_obj.weather:
+        _download_weather_files_parallel(
+            bldg_ids, output_dir, max_workers, progress, downloaded_paths, failed_downloads, console, weather_states
+        )
 
 
 if __name__ == "__main__":  # pragma: no cover
     bldg_ids = [
         BuildingID(
-            bldg_id=5634,
-            release_year="2024",
-            res_com="comstock",
-            weather="amy2018",
-            upgrade_id="1",
-            release_number="2",
-            state="AL",
-        ),
-        BuildingID(
-            bldg_id=78270,
-            release_year="2024",
-            res_com="comstock",
-            weather="amy2018",
-            upgrade_id="1",
-            release_number="2",
-            state="DE",
+            bldg_id=67, release_year="2024", res_com="comstock", weather="tmy3", upgrade_id="0", release_number="2"
         ),
     ]
-    for bldg_id in bldg_ids:
-        print(bldg_id._get_county_name())
+    file_type = ("weather",)
+    output_dir = Path("data")
+    weather_states: list[str] = []
+    downloaded_paths, failed_downloads = fetch_bldg_data(bldg_ids, file_type, output_dir, weather_states=weather_states)
+    print(downloaded_paths)
+    print(failed_downloads)