buildingdata 0.1.0__py3-none-any.whl

buildingdata/simulation/era5.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+from pathlib import Path
+
+from ..cache import ensure_subdir
+from ..exceptions import BuildingDataError, ConfigurationError
+
+# ERA5 variables needed to build an EPW file
+_ERA5_VARIABLES = [
+    "2m_temperature",
+    "2m_dewpoint_temperature",
+    "10m_u_component_of_wind",
+    "10m_v_component_of_wind",
+    "surface_solar_radiation_downwards",
+    "surface_thermal_radiation_downwards",
+    "total_cloud_cover",
+    "mean_sea_level_pressure",
+]
+
+_TIMEOUT = 600  # seconds — ERA5 downloads can be slow
+
+
+def _epw_path(lat, lon, year):
+    subdir = ensure_subdir("era5")
+    lat_str = f"{lat:.4f}".replace("-", "m")
+    lon_str = f"{lon:.4f}".replace("-", "m")
+    return subdir / f"{year}_{lat_str}_{lon_str}.epw"
+
+
+def _nc_path(lat, lon, year):
+    subdir = ensure_subdir("era5")
+    lat_str = f"{lat:.4f}".replace("-", "m")
+    lon_str = f"{lon:.4f}".replace("-", "m")
+    return subdir / f"{year}_{lat_str}_{lon_str}.nc"
+
+
+def _check_cdsapi():
+    try:
+        import cdsapi  # noqa: F401
+    except ImportError as e:
+        raise ConfigurationError(
+            "cdsapi is not installed. Install buildingdata with the era5 extra: "
+            "pip install buildingdata[era5]"
+        ) from e
+
+
+def _download_era5(lat, lon, year, nc_path):
+    """Download ERA5 hourly data for a year using ARCO timeseries format.
+
+    Uses the ERA5 timeseries dataset (ARCO Zarr format) optimized for single-point requests.
+
+    Args:
+        lat (float): latitude (WGS84).
+        lon (float): longitude (WGS84).
+        year (int): calendar year.
+        nc_path (pathlib.Path): destination file path (data is in Zarr format).
+    """
+    import cdsapi
+
+    client = cdsapi.Client()
+    print(f"Fetching ERA5 climate data (lat={lat}, lon={lon}, year={year}) ...")
+
+    client.retrieve(
+        "reanalysis-era5-single-levels-timeseries",
+        {
+            "variable": _ERA5_VARIABLES,
+            "location": {"longitude": lon, "latitude": lat},
+            "date": [f"{year}-01-01/{year}-12-31"],
+        },
+        str(nc_path),
+    )
+
+
+def _nc_to_epw(nc_path, lat, lon, year, epw_path):
+    """Convert ERA5 NetCDF to EPW format using pvlib.
+
+    Handles NetCDF files, ZIP-wrapped NetCDF, and Zarr format.
+
+    Args:
+        nc_path (pathlib.Path): input NetCDF file (may be ZIP-wrapped).
+        lat (float): latitude (WGS84).
+        lon (float): longitude (WGS84).
+        year (int): calendar year.
+        epw_path (pathlib.Path): output EPW file.
+    """
+    try:
+        import pvlib
+        import xarray as xr
+    except ImportError as e:
+        raise ConfigurationError(
+            "pvlib or xarray is not installed. Install buildingdata with the era5 extra: "
+            "pip install buildingdata[era5]"
+        ) from e
+
+    import numpy as np
+    import pandas as pd
+    import zipfile
+
+    # CDS API may return NetCDF wrapped in a ZIP; extract if needed
+    actual_nc_path = nc_path
+    if str(nc_path).endswith(".nc"):
+        try:
+            with zipfile.ZipFile(nc_path) as zf:
+                if len(zf.namelist()) == 1:
+                    inner_file = zf.namelist()[0]
+                    actual_nc_path = nc_path.parent / inner_file
+                    if not actual_nc_path.exists():
+                        zf.extractall(nc_path.parent)
+        except zipfile.BadZipFile:
+            actual_nc_path = nc_path
+
+    try:
+        ds = xr.open_dataset(actual_nc_path).squeeze()
+    except ValueError:
+        # ARCO format is Zarr, not NetCDF; try zarr engine
+        ds = xr.open_dataset(actual_nc_path, engine="zarr").squeeze()
+
+    times = pd.to_datetime(ds["valid_time"].values)
+
+    # ERA5 variable mapping to EPW columns
+    # Temperature: K → °C
+    temp_air = ds["t2m"].values - 273.15
+    # Dew point: K → °C
+    temp_dew = ds["d2m"].values - 273.15
+    # Wind speed: u and v components → magnitude
+    u10 = ds["u10"].values
+    v10 = ds["v10"].values
+    wind_speed = np.sqrt(u10**2 + v10**2)
+    wind_dir = (np.degrees(np.arctan2(u10, v10)) + 180) % 360
+    # Solar radiation: J/m² accumulated per hour → W/m² (divide by 3600)
+    ghi = np.maximum(ds["ssrd"].values / 3600, 0)
+    # Thermal (longwave) radiation downwelling
+    dhi_lw = np.maximum(ds["strd"].values / 3600, 0)
+    # Cloud cover (0–1 fraction → oktas 0–8 for EPW opaque_sky_cover)
+    tcc = ds["tcc"].values
+    opaque_sky_cover = np.clip(np.round(tcc * 8), 0, 8).astype(int)
+    # Pressure: Pa → Pa (EPW expects Pa)
+    pressure = ds["msl"].values
+
+    # Decompose GHI into DNI and DHI using pvlib's DISC model
+    loc = pvlib.location.Location(latitude=lat, longitude=lon)
+    solar_pos = loc.get_solarposition(times)
+    disc = pvlib.irradiance.disc(ghi, solar_pos["zenith"], times)
+    dni = np.maximum(disc["dni"].values, 0)
+    dhi = np.maximum(ghi - dni * np.cos(np.radians(solar_pos["zenith"].values)), 0)
+
+    epw_data = pd.DataFrame(
+        {
+            "year": times.year,
+            "month": times.month,
+            "day": times.day,
+            "hour": times.hour + 1,  # EPW hours are 1-indexed
+            "minute": 0,
+            "data_source_and_uncertainty_flags": "?9?9?9?9?9?9?9?9?9?9?9?9?9?9",
+            "dry_bulb_temperature": temp_air,
+            "dew_point_temperature": temp_dew,
+            "relative_humidity": _dew_to_rh(temp_air, temp_dew),
+            "atmospheric_station_pressure": pressure,
+            "extraterrestrial_horizontal_radiation": 9999,
+            "extraterrestrial_direct_normal_radiation": 9999,
+            "horizontal_infrared_radiation_intensity": dhi_lw,
+            "global_horizontal_radiation": ghi,
+            "direct_normal_radiation": dni,
+            "diffuse_horizontal_radiation": dhi,
+            "global_horizontal_illuminance": 999999,
+            "direct_normal_illuminance": 999999,
+            "diffuse_horizontal_illuminance": 999999,
+            "zenith_luminance": 9999,
+            "wind_direction": wind_dir,
+            "wind_speed": wind_speed,
+            "total_sky_cover": opaque_sky_cover,
+            "opaque_sky_cover": opaque_sky_cover,
+            "visibility": 9999,
+            "ceiling_height": 99999,
+            "present_weather_observation": 9,
+            "present_weather_codes": 999999999,
+            "precipitable_water": 999,
+            "aerosol_optical_depth": 0.999,
+            "snow_depth": 999,
+            "days_since_last_snowfall": 99,
+            "albedo": 999,
+            "liquid_precipitation_depth": 999,
+            "liquid_precipitation_quantity": 99,
+        }
+    )
+
+    header = _build_epw_header(lat, lon, year)
+    with open(epw_path, "w") as f:
+        f.write(header)
+        epw_data.to_csv(f, index=False, header=False)
+
+    ds.close()
+
+
+def _dew_to_rh(temp_c, dew_c):
+    """Convert dry-bulb and dew-point temperatures to relative humidity (%).
+
+    Uses the Magnus formula approximation.
+    """
+    import numpy as np
+
+    a, b = 17.625, 243.04
+    gamma_t = a * temp_c / (b + temp_c)
+    gamma_d = a * dew_c / (b + dew_c)
+    rh = 100 * np.exp(gamma_d - gamma_t)
+    return np.clip(rh, 0, 100)
+
+
+def _build_epw_header(lat, lon, year):
+    """Build a minimal EPW file header string."""
+    return (
+        f"LOCATION,ERA5,ERA5,ERA5,ERA5-{year},{lat:.2f}_{lon:.2f},{lat:.2f},{lon:.2f},0,0\n"
+        "DESIGN CONDITIONS,0\n"
+        "TYPICAL/EXTREME PERIODS,0\n"
+        "GROUND TEMPERATURES,0\n"
+        "HOLIDAYS/DAYLIGHT SAVINGS,No,0,0,0\n"
+        "COMMENTS 1,Generated from ERA5 reanalysis by buildingdata\n"
+        f"COMMENTS 2,Year {year} | Lat {lat:.4f} Lon {lon:.4f}\n"
+        "DATA PERIODS,1,1,Data,Sunday, 1/ 1,12/31\n"
+    )
+
+
+def get_era5_climate(lat, lon, year, refresh=False):
+    """Download ERA5 reanalysis and return the path to a synthetic EPW file.
+
+    Queries the Copernicus CDS API for hourly ERA5 reanalysis at the grid point
+    nearest to (lat, lon) for the full given year. Converts the NetCDF output to
+    EPW format using pvlib's irradiance decomposition. The EPW file is cached
+    locally; subsequent calls with the same arguments return the cached path.
+
+    Requires a valid ~/.cdsapirc file or the CDS_API_KEY environment variable.
+    Install the era5 optional dependencies: pip install buildingdata[era5].
+
+    Args:
+        lat (float): latitude in decimal degrees (WGS84).
+        lon (float): longitude in decimal degrees (WGS84).
+        year (int): calendar year, e.g. 2022.
+        refresh (bool): force re-download even if cached. Defaults to False.
+
+    Returns:
+        pathlib.Path: absolute path to the synthetic .epw file.
+
+    Raises:
+        ConfigurationError: if cdsapi or pvlib is not installed, or if the
+            CDS API key is not configured.
+        BuildingDataError: if the ERA5 download or conversion fails.
+    """
+    _check_cdsapi()
+
+    epw_path = _epw_path(lat, lon, year)
+    nc_path = _nc_path(lat, lon, year)
+
+    if epw_path.exists() and not refresh:
+        return epw_path
+
+    try:
+        if not nc_path.exists() or refresh:
+            _download_era5(lat, lon, year, nc_path)
+        _nc_to_epw(nc_path, lat, lon, year, epw_path)
+    except Exception as e:
+        raise BuildingDataError(
+            f"Failed to produce ERA5 EPW for lat={lat}, lon={lon}, year={year}: {e}"
+        ) from e
+
+    return epw_path

buildingdata/tests/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

buildingdata/tests/conftest.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""Shared fixtures for the buildingdata test suite.
+
+Every test runs against an isolated cache directory and config file so the
+suite never touches the user's real ~/.config/buildingdata/config.ini or
+on-disk cache, and never reaches GCS or any external API.
+"""
+import pytest
+
+from buildingdata import cache, config
+
+
+class FakeBlob:
+    """Stand-in for a google.cloud.storage.Blob used to mock GCS.
+
+    Only the attributes the library actually reads are implemented:
+    ``name``, ``generation``, ``size`` and ``exists()``/``reload()``.
+    """
+
+    def __init__(self, name, generation=1, size=0, exists=True):
+        self.name = name
+        self.generation = generation
+        self.size = size
+        self._exists = exists
+
+    def exists(self):
+        return self._exists
+
+    def reload(self):
+        pass
+
+
+@pytest.fixture(autouse=True)
+def isolate_environment(tmp_path, monkeypatch):
+    """Redirect config file and cache dir into a tmp dir, clear env vars.
+
+    Autouse so no test can accidentally read or write the real user config
+    or cache. The config file does not exist by default, so every setting
+    resolves to its built-in default unless a test writes one.
+    """
+    for var in (
+        "BUILDINGDATA_BUCKET",
+        "BUILDINGDATA_CACHE_DIR",
+        "GOOGLE_APPLICATION_CREDENTIALS",
+    ):
+        monkeypatch.delenv(var, raising=False)
+
+    cfg_file = tmp_path / "config" / "config.ini"
+    monkeypatch.setattr(config, "_CONFIG_FILE", cfg_file)
+
+    cache_dir = tmp_path / "cache"
+    monkeypatch.setattr(config, "_default_cache_dir", lambda: cache_dir)
+
+    return tmp_path
+
+
+@pytest.fixture
+def cfg_file(isolate_environment):
+    """Path to the (initially absent) isolated config.ini."""
+    return config._CONFIG_FILE

buildingdata/tests/test_cache.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""Local cache bookkeeping: paths, freshness, sidecars."""
+import json
+
+import pytest
+
+from buildingdata import cache
+from buildingdata.exceptions import CacheError
+
+from .conftest import FakeBlob
+
+
+def test_cache_path_lives_under_cache_dir():
+    from buildingdata.config import get_cache_dir
+
+    path = cache.cache_path("foo.parquet")
+    assert path.parent == get_cache_dir()
+    assert path.name == "foo.parquet"
+
+
+def test_is_cached_requires_both_file_and_sidecar():
+    name = "census_latest.parquet"
+    assert not cache.is_cached(name)
+
+    cache.cache_path(name).write_text("data")
+    assert not cache.is_cached(name)  # sidecar still missing
+
+    cache.write_sidecar(name, FakeBlob(name, generation=5))
+    assert cache.is_cached(name)
+
+
+def test_write_sidecar_records_blob_metadata():
+    name = "districts_latest.parquet"
+    cache.write_sidecar(name, FakeBlob(name, generation=42))
+
+    meta = json.loads(cache._sidecar_path(name).read_text())
+    assert meta["blob_name"] == name
+    assert meta["generation"] == 42
+    assert "downloaded_at" in meta
+
+
+def test_write_sidecar_wraps_oserror(monkeypatch):
+    def boom(*args, **kwargs):
+        raise OSError("disk full")
+
+    monkeypatch.setattr("builtins.open", boom)
+    with pytest.raises(CacheError):
+        cache.write_sidecar("x.parquet", FakeBlob("x.parquet"))
+
+
+def test_needs_refresh_true_when_no_sidecar():
+    assert cache.needs_refresh("missing.parquet", FakeBlob("missing.parquet", generation=1))
+
+
+def test_needs_refresh_false_when_generation_matches():
+    name = "a.parquet"
+    cache.write_sidecar(name, FakeBlob(name, generation=7))
+    assert not cache.needs_refresh(name, FakeBlob(name, generation=7))
+
+
+def test_needs_refresh_true_when_blob_generation_advances():
+    name = "a.parquet"
+    cache.write_sidecar(name, FakeBlob(name, generation=7))
+    assert cache.needs_refresh(name, FakeBlob(name, generation=8))
+
+
+def test_needs_refresh_true_on_corrupt_sidecar():
+    name = "a.parquet"
+    cache._sidecar_path(name).write_text("{ not valid json")
+    assert cache.needs_refresh(name, FakeBlob(name, generation=1))
+
+
+def test_ensure_subdir_creates_and_returns_path():
+    from buildingdata.config import get_cache_dir
+
+    sub = cache.ensure_subdir("bdtopo")
+    assert sub == get_cache_dir() / "bdtopo"
+    assert sub.is_dir()

buildingdata/tests/test_config.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""Configuration resolution: env var -> config.ini -> default."""
+from pathlib import Path
+
+from buildingdata import config
+
+
+# --- bucket -----------------------------------------------------------------
+
+def test_bucket_defaults_when_unset():
+    assert config.get_bucket() == config._DEFAULT_BUCKET
+
+
+def test_bucket_env_var_wins_over_config_file(monkeypatch):
+    config.write_config(bucket="from-file")
+    monkeypatch.setenv("BUILDINGDATA_BUCKET", "from-env")
+    assert config.get_bucket() == "from-env"
+
+
+def test_bucket_falls_back_to_config_file_without_env(monkeypatch):
+    monkeypatch.delenv("BUILDINGDATA_BUCKET", raising=False)
+    config.write_config(bucket="from-file")
+    assert config.get_bucket() == "from-file"
+
+
+# --- cache dir --------------------------------------------------------------
+
+def test_cache_dir_defaults_and_is_created():
+    path = config.get_cache_dir()
+    assert path == config._default_cache_dir()
+    assert path.is_dir()
+
+
+def test_cache_dir_env_var_wins(monkeypatch, tmp_path):
+    target = tmp_path / "envcache"
+    monkeypatch.setenv("BUILDINGDATA_CACHE_DIR", str(target))
+    assert config.get_cache_dir() == target
+    assert target.is_dir()
+
+
+def test_cache_dir_from_config_file(tmp_path):
+    target = tmp_path / "filecache"
+    config.write_config(cache_dir=target)
+    assert config.get_cache_dir() == target
+
+
+# --- credentials ------------------------------------------------------------
+
+def test_credentials_none_by_default():
+    assert config.get_credentials_file() is None
+
+
+def test_credentials_env_var_wins(monkeypatch):
+    monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/keys/env.json")
+    assert config.get_credentials_file() == Path("/keys/env.json")
+
+
+def test_credentials_from_config_file():
+    config.write_config(credentials="/keys/file.json")
+    assert config.get_credentials_file() == Path("/keys/file.json")
+
+
+# --- write_config -----------------------------------------------------------
+
+def test_write_config_creates_parent_and_merges(cfg_file, tmp_path):
+    assert not cfg_file.parent.exists()
+
+    cache_dir = tmp_path / "merged_cache"
+    config.write_config(bucket="b1")
+    config.write_config(cache_dir=cache_dir)  # second call must not wipe the first
+
+    assert config.get_bucket() == "b1"
+    assert config.get_cache_dir() == cache_dir
+
+
+def test_install_id_is_stable_and_short():
+    first = config._install_id()
+    assert first == config._install_id()
+    assert len(first) == 8
+    assert all(c in "0123456789abcdef" for c in first)

buildingdata/tests/test_public_api.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""The public surface of the package: importable names and configure()."""
+import buildingdata
+from buildingdata import config
+
+
+def test_public_callables_are_exposed():
+    expected = [
+        "configure",
+        "get_census",
+        "get_diagnosis",
+        "get_districts",
+        "get_gas_network",
+        "get_bdtopo",
+        "get_era5_climate",
+    ]
+    for name in expected:
+        assert hasattr(buildingdata, name), f"buildingdata.{name} is missing"
+        assert callable(getattr(buildingdata, name))
+
+
+def test_configure_delegates_to_write_config(monkeypatch):
+    captured = {}
+
+    def fake_write_config(bucket=None, cache_dir=None, credentials=None):
+        captured.update(bucket=bucket, cache_dir=cache_dir, credentials=credentials)
+
+    # configure() imports write_config into the package namespace at import time.
+    monkeypatch.setattr(buildingdata, "write_config", fake_write_config)
+
+    buildingdata.configure(bucket="b", cache_dir="/tmp/c", credentials="/tmp/k.json")
+
+    assert captured == {
+        "bucket": "b",
+        "cache_dir": "/tmp/c",
+        "credentials": "/tmp/k.json",
+    }
+
+
+def test_configure_writes_a_usable_config_file(cfg_file, tmp_path):
+    assert not cfg_file.exists()
+
+    cache_dir = tmp_path / "configured_cache"
+    buildingdata.configure(bucket="my-bucket", cache_dir=cache_dir)
+
+    assert cfg_file.exists()
+    # Values round-trip through the resolution helpers.
+    assert config.get_bucket() == "my-bucket"
+    assert config.get_cache_dir() == cache_dir