buildstock-fetch 0.1.0__py3-none-any.whl → 1.0a0__py3-none-any.whl

utils/build_metadata_tables.py

@@ -0,0 +1,522 @@
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Union
+
+import boto3
+import pandas as pd
+import pyarrow.fs as fs
+import pyarrow.parquet as pq
+from botocore import UNSIGNED
+from botocore.config import Config
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler("build_metadata_tables.log")],
+)
+logger = logging.getLogger(__name__)
+
+# Global column definitions - organized by category with priority
+PUMA_COLUMNS_TO_KEEP = [
+    "in.puma",
+    "in.nhgis_puma_gisjoin",
+    "in.resstock_puma_id",
+]
+
+STATE_COLUMNS_TO_KEEP = [
+    "in.state",
+    "in.state_name",
+]
+
+COUNTY_COLUMNS_TO_KEEP = [
+    "in.resstock_county_id",
+    "in.nhgis_county_gisjoin",
+    "in.county_name",
+    "in.county",
+]
+
+BUILDING_COLUMNS_TO_KEEP = [
+    "bldg_id",
+]
+
+
+def select_priority_columns(available_columns: list) -> list:
+    """
+    Select one column from each category based on priority (first available).
+
+    Args:
+        available_columns (list): List of available columns in the parquet file
+
+    Returns:
+        list: Selected columns with one from each category
+    """
+    selected_columns = []
+
+    # Select one column from each category
+    for col_list in [PUMA_COLUMNS_TO_KEEP, STATE_COLUMNS_TO_KEEP, COUNTY_COLUMNS_TO_KEEP, BUILDING_COLUMNS_TO_KEEP]:
+        for col in col_list:
+            if col in available_columns:
+                selected_columns.append(col)
+                break  # Only take the first available column from this category
+
+    return selected_columns
+
+
+def rename_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Rename columns to standardized names based on their category.
+
+    Args:
+        df (pd.DataFrame): DataFrame with columns to rename
+
+    Returns:
+        pd.DataFrame: DataFrame with renamed columns
+    """
+    column_mapping = {}
+    for col in df.columns:
+        if col in PUMA_COLUMNS_TO_KEEP:
+            column_mapping[col] = "puma"
+        elif col in STATE_COLUMNS_TO_KEEP:
+            column_mapping[col] = "state"
+        elif col in COUNTY_COLUMNS_TO_KEEP:
+            column_mapping[col] = "county"
+        elif col in BUILDING_COLUMNS_TO_KEEP:
+            column_mapping[col] = "bldg_id"
+    logger.info(f"Column mapping: {column_mapping}")
+
+    return df.rename(columns=column_mapping)
+
+
+def _read_parquet_with_columns(file_key: str, s3: fs.S3FileSystem, bucket_name: str) -> tuple[pd.DataFrame, list]:
+    """
+    Read parquet file and return DataFrame with selected columns.
+
+    Args:
+        file_key (str): The S3 key of the parquet file
+        s3 (fs.S3FileSystem): S3 filesystem instance
+        bucket_name (str): Name of the S3 bucket
+
+    Returns:
+        tuple: (DataFrame, list of available columns)
+    """
+    # Check available columns
+    parquet_file = pq.ParquetFile(f"{bucket_name}/{file_key}", filesystem=s3)
+    available_columns = parquet_file.schema_arrow.names
+
+    # Select columns with priority (first available in each category)
+    selected_columns = select_priority_columns(available_columns)
+
+    # Read with selected columns (will read all if selected_columns is empty)
+    table = pq.read_table(f"{bucket_name}/{file_key}", columns=selected_columns or None, filesystem=s3)
+
+    # Convert to pandas, preserving index if bldg_id is the index
+    df = table.to_pandas(use_threads=False)
+
+    return df, available_columns
+
+
+def _handle_bldg_id_column(df: pd.DataFrame, file_key: str, s3: fs.S3FileSystem, bucket_name: str) -> pd.DataFrame:
+    """
+    Handle bldg_id column extraction and validation.
+
+    Args:
+        df (pd.DataFrame): DataFrame to process
+        file_key (str): The S3 key of the parquet file
+        s3 (fs.S3FileSystem): S3 filesystem instance
+        bucket_name (str): Name of the S3 bucket
+
+    Returns:
+        pd.DataFrame: DataFrame with bldg_id properly handled
+    """
+    # Check if bldg_id is missing from columns but present as index
+    if "bldg_id" not in df.columns and df.index.name == "bldg_id":
+        # Add bldg_id from index to columns
+        df["bldg_id"] = df.index
+        df = df.reset_index(drop=True)
+        logger.info("Added bldg_id from index to columns")
+
+    # If bldg_id is still missing, try reading the parquet file with index
+    if "bldg_id" not in df.columns:
+        # Read the parquet file directly to check for index
+        parquet_file = pq.ParquetFile(f"{bucket_name}/{file_key}", filesystem=s3)
+        df_with_index = parquet_file.read().to_pandas()
+
+        if df_with_index.index.name == "bldg_id":
+            # Re-read with index preserved
+            df = parquet_file.read().to_pandas()
+            df["bldg_id"] = df.index
+            df = df.reset_index(drop=True)
+            logger.info("Added bldg_id from index to columns (second attempt)")
+        else:
+            logger.warning("Warning: bldg_id column not found in data or index")
+
+    return df
+
+
+def _validate_bldg_id(df: pd.DataFrame, release_name: Union[str, None]) -> bool:
+    """
+    Validate bldg_id column and return True if valid.
+
+    Args:
+        df (pd.DataFrame): DataFrame to validate
+        release_name (Union[str, None]): Name of the release for error reporting
+
+    Returns:
+        bool: True if bldg_id is valid, False otherwise
+    """
+    if "bldg_id" not in df.columns:
+        logger.exception(f"ERROR: bldg_id column not found in DataFrame after processing in {release_name}")
+        logger.exception("This indicates the column was not properly extracted from the parquet file")
+        return False
+
+    # Ensure bldg_id is integer if it exists
+    df["bldg_id"] = df["bldg_id"].astype("Int64")
+    logger.info("Converted bldg_id to integer type")
+
+    # Verify that bldg_id contains proper integer values
+    nan_count = df["bldg_id"].isna().sum()
+    total_rows = len(df)
+
+    if nan_count == total_rows:
+        logger.exception(f"ERROR: bldg_id column contains only NaN values after conversion in {release_name}")
+        logger.exception("This indicates the column was not properly extracted from the parquet file")
+        return False
+    elif nan_count > total_rows * 0.5:  # More than 50% NaN
+        logger.exception(
+            f"ERROR: bldg_id column has {nan_count} NaN values out of {total_rows} total rows in {release_name}"
+        )
+        logger.exception("This indicates the column was not properly extracted from the parquet file")
+        return False
+    else:
+        # Remove rows where bldg_id is NaN
+        df.dropna(subset=["bldg_id"], inplace=True)
+        logger.info(f"Removed {nan_count} rows with NaN bldg_id values in {release_name}")
+        logger.info(f"bldg_id column verified: {total_rows - nan_count} valid integer values in {release_name}")
+        return True
+
+
+def _add_metadata_columns(
+    df: pd.DataFrame,
+    res_com: Union[str, None],
+    weather: Union[str, None],
+    release_version: Union[str, None],
+    release_year: Union[str, None],
+) -> pd.DataFrame:
+    """
+    Add metadata columns to DataFrame.
+
+    Args:
+        df (pd.DataFrame): DataFrame to add columns to
+        res_com (Union[str, None]): The res_com value
+        weather (Union[str, None]): The weather value
+        release_version (Union[str, None]): The release number
+        release_year (Union[str, None]): The release year
+
+    Returns:
+        pd.DataFrame: DataFrame with metadata columns added
+    """
+    if res_com is not None:
+        df["product"] = res_com
+    if weather is not None:
+        df["weather_file"] = weather
+    if release_version is not None:
+        df["release_version"] = release_version
+    if release_year is not None:
+        df["release_year"] = release_year
+
+    return df
+
+
+def process_parquet_file(
+    file_key: str,
+    s3: fs.S3FileSystem,
+    bucket_name: str,
+    res_com: Union[str, None] = None,
+    weather: Union[str, None] = None,
+    release_version: Union[str, None] = None,
+    release_year: Union[str, None] = None,
+    release_name: Union[str, None] = None,
+) -> Union[pd.DataFrame, None]:
+    """
+    Process a parquet file from S3 and return a pandas DataFrame.
+
+    Args:
+        file_key (str): The S3 key of the parquet file
+        s3 (fs.S3FileSystem): S3 filesystem instance
+        bucket_name (str): Name of the S3 bucket
+        res_com (str): The res_com value (resstock or comstock)
+        weather (str): The weather value (tmy3, amy2012, or amy2018)
+        release_version (str): The release number
+        release_year (str): The release year
+        release_name (str): The release name
+
+    Returns:
+        pd.DataFrame: Processed DataFrame
+    """
+    try:
+        # Read parquet file with selected columns
+        df, available_columns = _read_parquet_with_columns(file_key, s3, bucket_name)
+
+        # Handle bldg_id column extraction
+        df = _handle_bldg_id_column(df, file_key, s3, bucket_name)
+
+        # Rename columns to standardized names
+        df = rename_columns(df)
+        logger.info(f"Columns after renaming: {df.columns}")
+
+        # Validate bldg_id column
+        if not _validate_bldg_id(df, release_name):
+            return None
+
+        # Add metadata columns
+        df = _add_metadata_columns(df, res_com, weather, release_version, release_year)
+
+        logger.info(f"Successfully loaded {len(df)} rows with {len(df.columns)} columns in {release_name}")
+
+    except Exception:
+        logger.exception("Error processing file")
+        return None
+    else:
+        return df
+
+
+def find_metadata_files(base_file_key: str, file_key_suffix: str, s3_client, bucket_name: str) -> list:
+    """
+    Find all metadata files in the S3 bucket.
+    """
+    download_files = []
+    paginator = s3_client.get_paginator("list_objects_v2")
+    pages = paginator.paginate(Bucket=bucket_name, Prefix=base_file_key)
+    for page in pages:
+        if "Contents" in page:
+            for obj in page["Contents"]:
+                key = obj["Key"]
+
+                # Check if file ends with 'baseline.parquet'
+                if key.endswith(file_key_suffix):
+                    download_files.append(key)
+    return download_files
+
+
+def _extract_upgrade_number(filename: str) -> Union[str, None]:
+    """
+    Extract upgrade number from filename like 'AK_G0200130_upgrade01.parquet'.
+
+    Args:
+        filename (str): The filename to parse
+
+    Returns:
+        Union[str, None]: The upgrade number (e.g., '01') or None if not found
+    """
+    if "upgrade" not in filename:
+        return None
+
+    # Find the upgrade number after "upgrade" in the filename
+    upgrade_index = filename.find("upgrade")
+    if upgrade_index == -1:
+        return None
+
+    # Extract the number after "upgrade"
+    upgrade_part = filename[upgrade_index + 7 :]  # 7 is length of "upgrade"
+    # Find where the number ends (before .parquet)
+    dot_index = upgrade_part.find(".parquet")
+    if dot_index == -1:
+        return None
+
+    return upgrade_part[:dot_index]
+
+
+def find_all_parquet_files(base_file_key: str, s3_client, bucket_name: str) -> dict:
+    """
+    Find all parquet files in the S3 bucket and organize them by upgrade type.
+
+    Returns:
+        dict: Dictionary with keys 'baseline' and 'upgrade_XX' containing lists of file paths
+    """
+    all_files = {}
+    paginator = s3_client.get_paginator("list_objects_v2")
+    pages = paginator.paginate(Bucket=bucket_name, Prefix=base_file_key)
+
+    for page in pages:
+        if "Contents" in page:
+            for obj in page["Contents"]:
+                key = obj["Key"]
+
+                # Check if file ends with .parquet
+                if not key.endswith(".parquet"):
+                    continue
+
+                if key.endswith("baseline.parquet"):
+                    if "baseline" not in all_files:
+                        all_files["baseline"] = []
+                    all_files["baseline"].append(key)
+                elif "upgrade" in key:
+                    # Extract upgrade number from filename
+                    parts = key.split("/")
+                    filename = parts[-1]
+                    upgrade_num = _extract_upgrade_number(filename)
+
+                    if upgrade_num is not None:
+                        upgrade_key = f"upgrade_{upgrade_num}"
+                        if upgrade_key not in all_files:
+                            all_files[upgrade_key] = []
+                        all_files[upgrade_key].append(key)
+
+    return all_files
+
+
+if __name__ == "__main__":
+    # Load the release data
+    with open("/workspaces/buildstock-fetcher/utils/buildstock_releases.json") as file:
+        data = json.load(file)
+
+    # Directory to save the data
+    data_dir = Path(__file__).parent.parent / "data"
+    downloaded_paths = []
+    os.makedirs(data_dir, exist_ok=True)
+
+    # List to collect all DataFrames
+    all_dataframes = []
+
+    # S3 bucket and filesystem
+    bucket_name = "oedi-data-lake"
+    s3 = fs.S3FileSystem(anonymous=True, region="us-west-2")
+    s3_client = boto3.client("s3", region_name="us-west-2", config=Config(signature_version=UNSIGNED))
+
+    for release_name, release in data.items():
+        release_year = release["release_year"]
+        res_com = release["res_com"]
+        weather = release["weather"]
+        release_version = release["release_number"]
+        upgrade_ids = release.get("upgrade_ids", [])
+
+        if release_year == "2021":
+            file_key = (
+                f"nrel-pds-building-stock/end-use-load-profiles-for-us-building-stock/"
+                f"{release_year}/{res_com}_{weather}_release_{release_version}/metadata/metadata.parquet"
+            )
+            df = process_parquet_file(
+                file_key, s3, bucket_name, res_com, weather, release_version, release_year, release_name
+            )
+            if df is not None and len(df) > 0:
+                all_dataframes.append(df)
+            else:
+                logger.exception(
+                    f"ERROR: Failed to process {release_name}. bldg_id column is either missing or contains non-integer values"
+                )
+                logger.exception("Exiting script...")
+                sys.exit(1)
+
+        elif (
+            release_year == "2022"
+            or release_year == "2023"
+            or (release_year == "2024" and release_name != "com_2024_amy2018_2")
+        ):
+            # Process only the first upgrade_id
+            if upgrade_ids:
+                upgrade_id = int(upgrade_ids[0])
+                if upgrade_id == 0:
+                    file_key = (
+                        f"nrel-pds-building-stock/end-use-load-profiles-for-us-building-stock/"
+                        f"{release_year}/{res_com}_{weather}_release_{release_version}/metadata/baseline.parquet"
+                    )
+                else:
+                    file_key = (
+                        f"nrel-pds-building-stock/end-use-load-profiles-for-us-building-stock/"
+                        f"{release_year}/{res_com}_{weather}_release_{release_version}/metadata/"
+                        f"upgrade{upgrade_id:02d}.parquet"
+                    )
+
+                df = process_parquet_file(
+                    file_key, s3, bucket_name, res_com, weather, release_version, release_year, release_name
+                )
+                if df is not None and len(df) > 0:
+                    all_dataframes.append(df)
+                else:
+                    logger.exception(
+                        f"ERROR: Failed to process {release_name}. bldg_id column is either missing or contains non-integer values"
+                    )
+                    logger.exception("Exiting script...")
+                    sys.exit(1)
+
+        elif release_name == "com_2024_amy2018_2" or release_name == "com_2025_amy2018_1":
+            base_file_key = (
+                f"nrel-pds-building-stock/end-use-load-profiles-for-us-building-stock/"
+                f"{release_year}/{res_com}_{weather}_release_{release_version}/"
+                f"metadata_and_annual_results/by_state_and_county/full/parquet/"
+            )
+
+            # Find all available parquet files at once
+            all_parquet_files = find_all_parquet_files(base_file_key, s3_client, bucket_name)
+            logger.info(f"Found parquet files: {list(all_parquet_files.keys())}")
+
+            # Process only the first upgrade_id
+            if upgrade_ids:
+                upgrade_id = int(upgrade_ids[0])
+                if upgrade_id == 0 and release_name == "com_2024_amy2018_2":
+                    file_key_suffix = "baseline.parquet"
+                    available_files = all_parquet_files.get("baseline", [])
+                elif release_name == "com_2025_amy2018_1":
+                    file_key_suffix = f"upgrade{upgrade_id}.parquet"
+                    available_files = all_parquet_files.get(f"upgrade_{upgrade_id}", [])
+                else:
+                    file_key_suffix = f"upgrade{upgrade_id:02d}.parquet"
+                    available_files = all_parquet_files.get(f"upgrade_{upgrade_id:02d}", [])
+
+                logger.info(f"Found {len(available_files)} {file_key_suffix} files:")
+
+                if available_files:
+                    # Combine all parquet files for this upgrade
+                    combined_dfs = []
+                    files_to_process = available_files  # Process all available files
+
+                    for i, file_key in enumerate(files_to_process, 1):
+                        logger.info(f"Processing file {i} out of {len(files_to_process)}")
+                        df = process_parquet_file(
+                            file_key, s3, bucket_name, res_com, weather, release_version, release_year, release_name
+                        )
+                        if df is not None and len(df) > 0:
+                            combined_dfs.append(df)
+                        else:
+                            logger.exception(
+                                f"ERROR: Failed to process {release_name}. bldg_id column is either missing or contains non-integer values"
+                            )
+                            logger.exception("Exiting script...")
+                            sys.exit(1)
+
+                    if combined_dfs:
+                        # Concatenate all DataFrames for this release
+                        combined_df = pd.concat(combined_dfs, ignore_index=True)
+                        logger.info(
+                            f"Combined {len(combined_dfs)} files into {len(combined_df)} total rows in {release_name}"
+                        )
+                        all_dataframes.append(combined_df)
+                    else:
+                        logger.warning(f"No valid data found for {file_key_suffix} in {release_name}")
+                else:
+                    logger.warning(f"No files found for {file_key_suffix} in {release_name}")
+
+    # Combine all DataFrames and save as partitioned parquet
+    if all_dataframes:
+        logger.info(f"Combining {len(all_dataframes)} DataFrames...")
+        final_df = pd.concat(all_dataframes, ignore_index=True)
+        logger.info(f"Final DataFrame has {len(final_df)} rows and {len(final_df.columns)} columns")
+
+        # Sort by county
+        if "county" in final_df.columns:
+            final_df = final_df.sort_values("county")
+
+        # Save as partitioned parquet file
+        output_file = f"{data_dir}/building_data/combined_metadata.parquet"
+        final_df.to_parquet(
+            output_file,
+            partition_cols=["product", "release_year", "weather_file", "release_version", "state"],
+            index=False,
+        )
+        logger.info(f"Successfully saved combined DataFrame to {output_file}")
+    else:
+        logger.warning("No data to save")

utils/ev_data/inputs/download_2022_nhts_data.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+Utility script to download NHTS (National Household Travel Survey) data.
+
+Downloads the zip file from https://nhts.ornl.gov/media/2022/download/csv.zip,
+extracts the tripv2pub.csv file, and saves it as NHTS_v2_1_trip_surveys.csv.
+Cleans up temporary files after extraction.
+"""
+
+import argparse
+import logging
+import zipfile
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+
+class NHTSDownloadError(Exception):
+    """Custom exception for NHTS download errors."""
+
+    pass
+
+
+class NHTSFileNotFoundError(NHTSDownloadError):
+    """Raised when required file is not found in downloaded zip."""
+
+    pass
+
+
+class NHTSFileExtractionError(NHTSDownloadError):
+    """Raised when file extraction fails."""
+
+    pass
+
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def _handle_file_not_found(all_files: list[str]) -> None:
+    """Handle case when required file is not found in zip."""
+    logger.error(f"tripv2pub.csv not found in zip. Available files: {all_files}")
+    raise NHTSFileNotFoundError()
+
+
+def _handle_extraction_failure() -> None:
+    """Handle case when file extraction fails."""
+    raise NHTSFileExtractionError()
+
+
+def download_nhts_data(output_dir: Optional[str] = None) -> str:
+    """
+    Download NHTS data from the official website and extract the trip survey file.
+
+    Args:
+        output_dir: Directory to save the extracted file. If None, uses current directory.
+
+    Returns:
+        Path to the extracted NHTS_v2_1_trip_surveys.csv file
+    """
+    # NHTS data URL
+    url = "https://nhts.ornl.gov/media/2022/download/csv.zip"
+
+    # Set output directory
+    output_dir = Path(__file__).parent if output_dir is None else Path(output_dir)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # File paths
+    zip_path = output_dir / "nhts_temp.zip"
+    target_file = output_dir / "NHTS_v2_1_trip_surveys.csv"
+
+    try:
+        logger.info(f"Downloading NHTS data from {url}")
+
+        # Download the zip file
+        response = requests.get(url, stream=True, timeout=30)
+        response.raise_for_status()
+
+        with open(zip_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+        logger.info(f"Downloaded {zip_path}")
+
+        # Extract the specific file
+        logger.info("Extracting tripv2pub.csv from zip file")
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            # Look for tripv2pub.csv in the zip
+            csv_files = [f for f in zip_ref.namelist() if f.endswith("tripv2pub.csv")]
+
+            if not csv_files:
+                # List all files in zip for debugging
+                all_files = zip_ref.namelist()
+                _handle_file_not_found(all_files)
+
+            csv_file = csv_files[0]
+            logger.info(f"Found {csv_file} in zip file")
+
+            # Extract the file
+            zip_ref.extract(csv_file, output_dir)
+
+            # Rename to desired filename
+            extracted_path = output_dir / csv_file
+            if extracted_path.exists():
+                extracted_path.rename(target_file)
+                logger.info(f"Successfully extracted and renamed to {target_file}")
+            else:
+                _handle_extraction_failure()
+
+        # Clean up temporary zip file
+        if zip_path.exists():
+            zip_path.unlink()
+            logger.info("Cleaned up temporary zip file")
+
+        return str(target_file)
+
+    except requests.RequestException:
+        logger.exception("Failed to download file")
+        raise
+    except zipfile.BadZipFile:
+        logger.exception("Invalid zip file")
+        raise
+    except Exception:
+        logger.exception("Unexpected error")
+        raise
+    finally:
+        # Clean up temporary files in case of error
+        if zip_path.exists():
+            zip_path.unlink()
+            logger.info("Cleaned up temporary zip file")
+
+
+def main():
+    """Main function to run the download process."""
+    parser = argparse.ArgumentParser(description="Download NHTS (National Household Travel Survey) data")
+    parser.add_argument(
+        "--output-dir", "-o", type=str, help="Output directory for the downloaded file (default: script directory)"
+    )
+
+    args = parser.parse_args()
+
+    try:
+        output_file = download_nhts_data(output_dir=args.output_dir)
+        print(f"Successfully downloaded NHTS data to: {output_file}")
+    except Exception as e:
+        print(f"Error downloading NHTS data: {e}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())