gla-data 0.3.0__tar.gz → 0.4.0__tar.gz

{gla_data-0.3.0 → gla_data-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gla-data
-Version: 0.3.0
+Version: 0.4.0
 Summary: Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
 Author-email: Conor Dempsey <conor.dempsey@london.gov.uk>
 License: MIT
@@ -140,14 +140,14 @@ A misspelt `GLA_DATA_` setting in a settings file raises an error rather than be
 | Setting | Needed for |
 |---|---|
 | `GLA_DATA_GIS_DB_HOST`, `_PORT`, `_NAME`, `_USER`, `_PASSWORD` | Anything from the GIS team's Postgres database (`source="postgres"`, `load_gis_layer`, `load_front_gardens`, `load_mastermap_topography`, `gla-data db ...`). Use a read-only login. |
-| `GLA_DATA_GIS_DB_SCHEMA` | Optional: the AddressBase schema, if not the default search path. |
-| `GLA_DATA_ADDRESSBASE_BLPU_TABLE`, `_CLASSIFICATION_TABLE` | `load_addressbase(source="postgres")`, if the tables are not named `blpu` / `classification`. |
 | `GLA_DATA_GISAPDATA_<LAYER>_TABLE` | Each `gisapdata` layer used by `load_boundaries(source="postgres")`, `load_gis_layer` and `load_front_gardens`. |
 | `GLA_DATA_GISAPDATA_MASTERMAP_TOPO_2024_TABLE` | `load_mastermap_topography` (2024 snapshot). |
 | `GLA_DATA_BLUESKY_IMAGERY_BUCKET` | Bluesky aerial imagery (`build_imagery_tile_index`, `download_imagery_tiles`). |
 | `GLA_DATA_BLUESKY_IMAGERY_DIR` | Optional: folder for the imagery tile index and downloads (default `data/bluesky_imagery`). Tiles are 256MB each. |
 | `GLA_DATA_AWS_PROFILE`, `GLA_DATA_AWS_REGION` | Optional: pin the AWS profile or region used for S3. |
 | `GLA_DATA_TREE_CANOPY_DIR` | Optional: folder for the tree canopy download and the GeoPackage built from it (default `data/tree_canopy`). Point every project at the same folder so the data is downloaded once. |
+| `GLA_DATA_EPC_API_TOKEN` | `download_epc`. Token from the "my account" page of the MHCLG EPC service (get-energy-performance-data.communities.gov.uk, free sign-in with GOV.UK One Login). Keep it in the user-level file. |
+| `GLA_DATA_EPC_DIR` | Optional: folder for the London EPC file built by `download_epc` (default `data/epc`). Point every project at the same folder. |
 
 AWS credentials are never set here. gla-data uses the standard AWS sources: an AWS profile (`~/.aws`), `AWS_*` environment variables, or the machine's own role on GLA cloud machines. Keep database passwords in your user-level file, never in a project's repository.
 

{gla_data-0.3.0 → gla_data-0.4.0}/README.md

@@ -114,14 +114,14 @@ A misspelt `GLA_DATA_` setting in a settings file raises an error rather than be
 | Setting | Needed for |
 |---|---|
 | `GLA_DATA_GIS_DB_HOST`, `_PORT`, `_NAME`, `_USER`, `_PASSWORD` | Anything from the GIS team's Postgres database (`source="postgres"`, `load_gis_layer`, `load_front_gardens`, `load_mastermap_topography`, `gla-data db ...`). Use a read-only login. |
-| `GLA_DATA_GIS_DB_SCHEMA` | Optional: the AddressBase schema, if not the default search path. |
-| `GLA_DATA_ADDRESSBASE_BLPU_TABLE`, `_CLASSIFICATION_TABLE` | `load_addressbase(source="postgres")`, if the tables are not named `blpu` / `classification`. |
 | `GLA_DATA_GISAPDATA_<LAYER>_TABLE` | Each `gisapdata` layer used by `load_boundaries(source="postgres")`, `load_gis_layer` and `load_front_gardens`. |
 | `GLA_DATA_GISAPDATA_MASTERMAP_TOPO_2024_TABLE` | `load_mastermap_topography` (2024 snapshot). |
 | `GLA_DATA_BLUESKY_IMAGERY_BUCKET` | Bluesky aerial imagery (`build_imagery_tile_index`, `download_imagery_tiles`). |
 | `GLA_DATA_BLUESKY_IMAGERY_DIR` | Optional: folder for the imagery tile index and downloads (default `data/bluesky_imagery`). Tiles are 256MB each. |
 | `GLA_DATA_AWS_PROFILE`, `GLA_DATA_AWS_REGION` | Optional: pin the AWS profile or region used for S3. |
 | `GLA_DATA_TREE_CANOPY_DIR` | Optional: folder for the tree canopy download and the GeoPackage built from it (default `data/tree_canopy`). Point every project at the same folder so the data is downloaded once. |
+| `GLA_DATA_EPC_API_TOKEN` | `download_epc`. Token from the "my account" page of the MHCLG EPC service (get-energy-performance-data.communities.gov.uk, free sign-in with GOV.UK One Login). Keep it in the user-level file. |
+| `GLA_DATA_EPC_DIR` | Optional: folder for the London EPC file built by `download_epc` (default `data/epc`). Point every project at the same folder. |
 
 AWS credentials are never set here. gla-data uses the standard AWS sources: an AWS profile (`~/.aws`), `AWS_*` environment variables, or the machine's own role on GLA cloud machines. Keep database passwords in your user-level file, never in a project's repository.
 

{gla_data-0.3.0 → gla_data-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gla-data"
-version = "0.3.0"
+version = "0.4.0"
 description = "Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis."
 readme = "README.md"
 license = { text = "MIT" }
@@ -27,8 +27,8 @@ dependencies = [
     # Postgres driver (psycopg2) is a conda-only dep here (see pixi.toml) to
     # avoid pip/conda installing it twice in the pixi env, matching how
     # openpyxl (needed by census.py) is handled — a plain `pip install
-    # gla-data` user wanting load_addressbase(source="postgres") needs to
-    # separately install a psycopg2 driver.
+    # gla-data` user wanting the Postgres-backed loaders needs to separately
+    # install a psycopg2 driver.
 ]
 
 [project.scripts]
@@ -89,7 +89,7 @@ disallow_untyped_defs = false
 disallow_incomplete_defs = false
 
 [tool.bumpversion]
-current_version = "0.3.0"
+current_version = "0.4.0"
 commit = true
 tag    = true
 

{gla_data-0.3.0 → gla_data-0.4.0}/src/gla_data/__init__.py

@@ -22,6 +22,14 @@ from gla_data.aerial_imagery import (
 )
 from gla_data.catalogue import catalogue
 from gla_data.census import load_census_dwellings
+from gla_data.epc import (
+    clean_epc,
+    download_epc,
+    get_epc_feature_spec,
+    get_latest_epc_per_uprn,
+    load_epc,
+    prepare_epc_features,
+)
 from gla_data.explore import describe_table, list_schemas, list_tables
 from gla_data.front_gardens import load_front_gardens
 from gla_data.geography import load_boundaries, load_geography_lookup, load_gis_layer
@@ -57,4 +65,10 @@ __all__ = [
     "load_mastermap_topography",
     "load_tree_canopy",
     "download_tree_canopy",
+    "download_epc",
+    "load_epc",
+    "get_latest_epc_per_uprn",
+    "clean_epc",
+    "prepare_epc_features",
+    "get_epc_feature_spec",
 ]

gla_data-0.4.0/src/gla_data/_epc.py

@@ -0,0 +1,560 @@
+"""Domestic EPCs: MHCLG EPC API bulk download, London parquet build and reads.
+
+The API's bulk endpoint (``BULK_PATH``) has no filters: it answers with a
+redirect to a signed S3 link for one zip of every domestic EPC in England and
+Wales, rebuilt on the 1st of each month. As of the file of 1 October 2026 the
+zip is 8.3GB and holds one ``certificates-<year>.csv`` per lodgement year
+(2008-2026) plus recommendation CSVs, which are skipped. Column names are
+not documented on the API pages; ``COLUMN_TYPES`` lists the 93 found in that
+file, and a file with any other set of columns is refused rather than
+guessed at.
+
+``build`` reads each certificates CSV in chunks, keeps London rows (local
+authority code ``E09...``), cleans them (``clean_chunk``) and appends them to
+one parquet file. Like ``_tree_canopy.build`` it runs under a file lock and
+writes under a temporary name that is renamed into place at the end, with a
+JSON record of the snapshot alongside.
+
+The address fields are Ordnance Survey and Royal Mail data (see the licence
+note in ``epc.py``): nothing here logs or prints row values.
+"""
+
+import hashlib
+import json
+import re
+import zipfile
+from dataclasses import dataclass
+from datetime import UTC, date, datetime
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+import requests
+from filelock import FileLock
+
+from gla_data._schemas import epc_schema as _epc_schema
+
+API_BASE = "https://api.get-energy-performance-data.communities.gov.uk"
+INFO_PATH = "/api/files/domestic/csv/info"
+BULK_PATH = "/api/files/domestic/csv"
+_REDIRECT_CODES = (301, 302, 303, 307, 308)
+_DOWNLOAD_CHUNK_BYTES = 8 * 1024 * 1024
+_READ_CHUNK_ROWS = 200_000
+_LONDON_PREFIX = "E09"
+
+STEM = "epc_domestic_london"
+ZIP_NAME = "epc_domestic_bulk.zip"
+
+# Column types, from a scan of every London row of the file of 1 October
+# 2026 (4,631,787 certificates). "string" columns are kept as text; numeric
+# columns whose values were all whole numbers are nullable integers.
+_STRING = "string"
+_INT = "Int64"
+_FLOAT = "float64"
+_BOOL = "boolean"
+_DATE = "date"
+_DATETIME = "datetime"
+COLUMN_TYPES: dict[str, str] = {
+    "certificate_number": _STRING,
+    "address1": _STRING,
+    "address2": _STRING,
+    "address3": _STRING,
+    "postcode": _STRING,
+    "posttown": _STRING,
+    "address": _STRING,
+    "constituency": _STRING,
+    "constituency_label": _STRING,
+    "local_authority": _STRING,
+    "local_authority_label": _STRING,
+    "built_form": _STRING,
+    "co2_emiss_curr_per_floor_area": _FLOAT,
+    "co2_emissions_current": _FLOAT,
+    "co2_emissions_potential": _FLOAT,
+    "construction_age_band": _STRING,
+    "current_energy_efficiency": _INT,
+    "current_energy_rating": _STRING,
+    "energy_consumption_current": _FLOAT,
+    "energy_consumption_potential": _FLOAT,
+    "energy_tariff": _STRING,
+    "environment_impact_current": _INT,
+    "environment_impact_potential": _INT,
+    "extension_count": _INT,
+    "fixed_lighting_outlets_count": _INT,
+    "flat_storey_count": _INT,
+    "flat_top_storey": _BOOL,
+    "floor_description": _STRING,
+    "floor_energy_eff": _STRING,
+    "floor_height": _FLOAT,
+    "floor_level": _STRING,
+    "glazed_area": _STRING,
+    "glazed_type": _STRING,
+    "heat_loss_corridor": _STRING,
+    "heating_cost_current": _FLOAT,
+    "heating_cost_potential": _FLOAT,
+    "hot_water_cost_current": _FLOAT,
+    "hot_water_cost_potential": _FLOAT,
+    "hot_water_energy_eff": _STRING,
+    "hot_water_env_eff": _STRING,
+    "hotwater_description": _STRING,
+    "inspection_date": _DATE,
+    "lighting_cost_current": _FLOAT,
+    "lighting_cost_potential": _FLOAT,
+    "lighting_description": _STRING,
+    "lighting_energy_eff": _STRING,
+    "lighting_env_eff": _STRING,
+    "lodgement_date": _DATE,
+    "lodgement_datetime": _DATETIME,
+    "low_energy_lighting": _FLOAT,
+    "low_energy_fixed_lighting_outlets_count": _INT,
+    "main_fuel": _STRING,
+    "mainheat_description": _STRING,
+    "mainheat_energy_eff": _STRING,
+    "mainheat_env_eff": _STRING,
+    "mainheatc_energy_eff": _STRING,
+    "mainheatc_env_eff": _STRING,
+    "mainheatcont_description": _STRING,
+    "main_heating_controls": _STRING,
+    "mains_gas_flag": _BOOL,
+    "mechanical_ventilation": _STRING,
+    "multi_glaze_proportion": _INT,
+    "number_habitable_rooms": _INT,
+    "number_heated_rooms": _INT,
+    "number_open_fireplaces": _INT,
+    "photo_supply": _INT,
+    "potential_energy_efficiency": _INT,
+    "potential_energy_rating": _STRING,
+    "property_type": _STRING,
+    "report_type": _INT,
+    "roof_description": _STRING,
+    "roof_energy_eff": _STRING,
+    "roof_env_eff": _STRING,
+    "secondheat_description": _STRING,
+    "sheating_energy_eff": _STRING,
+    "sheating_env_eff": _STRING,
+    "solar_water_heating_flag": _BOOL,
+    "tenure": _STRING,
+    "total_floor_area": _FLOAT,
+    "transaction_type": _STRING,
+    "unheated_corridor_length": _FLOAT,
+    "walls_description": _STRING,
+    "walls_energy_eff": _STRING,
+    "walls_env_eff": _STRING,
+    "wind_turbine_count": _INT,
+    "windows_description": _STRING,
+    "windows_energy_eff": _STRING,
+    "windows_env_eff": _STRING,
+    "floor_env_eff": _STRING,
+    "region": _STRING,
+    "country": _STRING,
+    "uprn": _INT,
+    "uprn_source": _STRING,
+}
+
+# Address fields: Ordnance Survey and Royal Mail data under their own notice.
+ADDRESS_COLUMNS = (
+    "address1",
+    "address2",
+    "address3",
+    "address",
+    "postcode",
+    "posttown",
+)
+
+# Values that mean "no value" in the source.
+_BLANK_MARKERS = {"", "NO DATA!", "N/A", "NULL"}
+
+# Category values that differ only in case: each maps (by casefold) to the
+# spelling most common in the London rows of the file of 1 October 2026.
+# Values not listed here are kept as published (whitespace trimmed).
+CASE_NORMALISED: dict[str, dict[str, str]] = {
+    "tenure": {
+        "owner-occupied": "owner-occupied",
+        "rented (private)": "rented (private)",
+        "rented (social)": "rented (social)",
+        "unknown": "unknown",
+    },
+    "transaction_type": {
+        "marketed sale": "Marketed sale",
+        "new dwelling": "New dwelling",
+        "rental": "Rental",
+    },
+    "floor_level": {"basement": "Basement"},
+}
+# The same age band appears with and without this prefix.
+_AGE_BAND_PREFIX = "England and Wales: "
+
+_FLAG_VALUES = {"Y": True, "N": False}
+
+
+@dataclass(frozen=True)
+class LocalPaths:
+    """Files kept in the destination folder."""
+
+    zip: Path
+    parquet: Path
+    record: Path
+    lock: Path
+    building: Path
+
+
+def local_paths(destination: Path) -> LocalPaths:
+    """Return the local file paths in ``destination``."""
+    return LocalPaths(
+        zip=destination / ZIP_NAME,
+        parquet=destination / f"{STEM}.parquet",
+        record=destination / f"{STEM}.json",
+        lock=destination / f"{STEM}.lock",
+        building=destination / f"{STEM}.building.parquet",
+    )
+
+
+# ---------------------------------------------------------------------------
+# API
+# ---------------------------------------------------------------------------
+
+
+def _auth_headers(token: str) -> dict[str, str]:
+    return {"Authorization": f"Bearer {token}", "Accept": "application/json"}
+
+
+def fetch_file_info(token: str) -> dict[str, Any]:
+    """Return the current bulk file's size and date (``fileSize``, ``lastUpdated``)."""
+    response = requests.get(
+        API_BASE + INFO_PATH, headers=_auth_headers(token), timeout=60
+    )
+    response.raise_for_status()
+    data: dict[str, Any] = response.json()["data"]
+    return data
+
+
+def download_bulk_zip(token: str, partial: Path) -> None:
+    """Download the bulk zip to ``partial``.
+
+    The API answers with a redirect to a signed S3 link. The redirect is
+    followed by hand so the bearer token is sent only to the API, never to S3.
+    """
+    response = requests.get(
+        API_BASE + BULK_PATH,
+        headers=_auth_headers(token),
+        allow_redirects=False,
+        timeout=60,
+    )
+    if response.status_code not in _REDIRECT_CODES:
+        response.raise_for_status()
+        raise RuntimeError(
+            f"Expected a redirect from {BULK_PATH}, got HTTP {response.status_code}."
+        )
+    signed_url = response.headers["location"]
+    with requests.get(signed_url, stream=True, timeout=600) as download:
+        download.raise_for_status()
+        with open(partial, "wb") as f:
+            for block in download.iter_content(chunk_size=_DOWNLOAD_CHUNK_BYTES):
+                f.write(block)
+
+
+def snapshot_date(last_updated: str) -> date:
+    """Return the date part of the API's ``lastUpdated`` timestamp."""
+    return date.fromisoformat(last_updated[:10])
+
+
+# ---------------------------------------------------------------------------
+# Cleaning
+# ---------------------------------------------------------------------------
+
+
+def normalise_column_name(name: str) -> str:
+    """'LODGEMENT_DATE' / 'lodgementDate' / 'Lodgement Date' -> 'lodgement_date'."""
+    snake = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", name.strip())
+    return re.sub(r"[^a-z0-9]+", "_", snake.lower()).strip("_")
+
+
+def certificate_members(names: list[str]) -> list[str]:
+    """Return the certificate CSVs in the bulk zip, without the recommendations."""
+    members = sorted(
+        n
+        for n in names
+        if n.lower().endswith(".csv") and "recommendation" not in n.lower()
+    )
+    if not members:
+        raise ValueError("No certificate CSV found in the EPC zip.")
+    return members
+
+
+def check_columns(columns: list[str]) -> None:
+    """Refuse a file whose columns differ from ``COLUMN_TYPES``."""
+    missing = sorted(set(COLUMN_TYPES) - set(columns))
+    extra = sorted(set(columns) - set(COLUMN_TYPES))
+    if missing or extra:
+        raise ValueError(
+            "The EPC file's columns have changed since gla-data's column list "
+            f"was written. Missing: {missing}. New: {extra}. Update "
+            "_epc.COLUMN_TYPES and the schema before building from this file."
+        )
+
+
+def _normalise_categories(chunk: pd.DataFrame) -> None:
+    for col, spellings in CASE_NORMALISED.items():
+        values = chunk[col]
+        mapped = values.str.casefold().map(spellings)
+        chunk[col] = mapped.fillna(values)
+    band = chunk["construction_age_band"]
+    chunk["construction_age_band"] = band.str.removeprefix(_AGE_BAND_PREFIX)
+
+
+def clean_chunk(chunk: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, int]]:
+    """Keep London rows of one chunk of strings and give columns their types.
+
+    Returns
+    -------
+    tuple[pd.DataFrame, dict[str, int]]
+        The cleaned rows, and per column the number of non-blank values that
+        couldn't be converted to the column's type (they become null).
+    """
+    london = chunk.loc[
+        chunk["local_authority"].str.startswith(_LONDON_PREFIX, na=False)
+    ]
+    out = london.apply(lambda s: s.str.strip()).astype("string")
+    out = out.mask(out.isin(_BLANK_MARKERS))
+    _normalise_categories(out)
+
+    unparsed: dict[str, int] = {}
+    for col, kind in COLUMN_TYPES.items():
+        values = out[col]
+        if kind == _STRING:
+            continue
+        if kind in (_INT, _FLOAT):
+            converted = pd.to_numeric(values, errors="coerce")
+            if kind == _INT:
+                whole = converted.notna() & (converted == converted.round())
+                converted = converted.where(whole).astype("Int64")
+            else:
+                converted = converted.astype("float64")
+        elif kind == _BOOL:
+            converted = values.map(_FLAG_VALUES).astype("boolean")
+        else:
+            converted = pd.to_datetime(values, errors="coerce", format="ISO8601")
+            if kind == _DATE:
+                converted = converted.dt.normalize()
+            converted = converted.astype("datetime64[ms]")
+        failed = int((values.notna() & converted.isna()).sum())
+        if failed:
+            unparsed[col] = failed
+        out[col] = converted
+    return out[list(COLUMN_TYPES)].reset_index(drop=True), unparsed
+
+
+def arrow_schema() -> pa.Schema:
+    """Return the parquet file's schema, from ``COLUMN_TYPES``."""
+    types = {
+        _STRING: pa.string(),
+        _INT: pa.int64(),
+        _FLOAT: pa.float64(),
+        _BOOL: pa.bool_(),
+        _DATE: pa.timestamp("ms"),
+        _DATETIME: pa.timestamp("ms"),
+    }
+    return pa.schema([(col, types[kind]) for col, kind in COLUMN_TYPES.items()])
+
+
+# ---------------------------------------------------------------------------
+# Build
+# ---------------------------------------------------------------------------
+
+
+def _write_parquet(zip_path: Path, out: Path) -> dict[str, Any]:
+    """Write the London rows of every certificates CSV in ``zip_path`` to ``out``.
+
+    Returns
+    -------
+    dict[str, Any]
+        Counts for the record: rows read and kept, unparsed values per column,
+        local authority code to name.
+    """
+    rows_read = 0
+    rows_kept = 0
+    unparsed: dict[str, int] = {}
+    authorities: dict[str, str] = {}
+    schema = arrow_schema()
+    # Checked chunk by chunk, so the whole file is never in memory at once.
+    validator = _epc_schema(COLUMN_TYPES)
+    with zipfile.ZipFile(zip_path) as zf, pq.ParquetWriter(out, schema) as writer:  # type: ignore[no-untyped-call]
+        for member in certificate_members(zf.namelist()):
+            with zf.open(member) as fh:
+                reader = pd.read_csv(
+                    fh,
+                    dtype=str,
+                    keep_default_na=False,
+                    chunksize=_READ_CHUNK_ROWS,
+                    encoding="utf-8",
+                    encoding_errors="replace",
+                )
+                for chunk in reader:
+                    chunk.columns = [normalise_column_name(c) for c in chunk.columns]
+                    check_columns(list(chunk.columns))
+                    rows_read += len(chunk)
+                    london, failed = clean_chunk(chunk)
+                    for col, n in failed.items():
+                        unparsed[col] = unparsed.get(col, 0) + n
+                    pairs = london[
+                        ["local_authority", "local_authority_label"]
+                    ].dropna()
+                    authorities.update(
+                        zip(pairs.iloc[:, 0], pairs.iloc[:, 1], strict=True)
+                    )
+                    if len(london):
+                        validator.validate(london)
+                        rows_kept += len(london)
+                        table = pa.Table.from_pandas(
+                            london, schema=schema, preserve_index=False
+                        )
+                        writer.write_table(table)
+    if rows_kept == 0:
+        raise ValueError(f"No London certificates found in {zip_path}.")
+    numbers = pq.read_table(out, columns=["certificate_number"]).column(0)  # type: ignore[no-untyped-call]
+    return {
+        "rows_read": rows_read,
+        "rows_kept_london": rows_kept,
+        "duplicate_certificate_numbers": rows_kept - len(numbers.unique()),
+        "unparsed_values": unparsed,
+        "local_authorities": dict(sorted(authorities.items())),
+    }
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _iso(timestamp: float | None = None) -> str:
+    moment = (
+        datetime.now(UTC)
+        if timestamp is None
+        else datetime.fromtimestamp(timestamp, UTC)
+    )
+    return moment.isoformat(timespec="seconds")
+
+
+def build(
+    destination: Path,
+    *,
+    token: str | None,
+    from_zip: Path | None,
+    snapshot: date | None,
+    keep_download: bool,
+) -> Path:
+    """Build the London parquet file and its record in ``destination``.
+
+    Either downloads the current bulk zip with ``token`` (its date comes from
+    the API), or builds from ``from_zip`` with its ``snapshot`` date. The
+    finished file replaces any earlier one only once complete and valid.
+
+    Returns
+    -------
+    Path
+        The parquet file.
+    """
+    paths = local_paths(destination)
+    file_info: dict[str, Any] = {}
+    if from_zip is None:
+        if token is None:
+            raise ValueError("token is required to download the EPC bulk file.")
+        file_info = fetch_file_info(token)
+        snapshot = snapshot_date(file_info["lastUpdated"])
+    if snapshot is None:
+        raise ValueError("snapshot is required when building from a zip on disk.")
+
+    destination.mkdir(parents=True, exist_ok=True)
+    with FileLock(paths.lock):
+        zip_path = from_zip
+        if zip_path is None:
+            partial = paths.zip.with_name(f"{paths.zip.name}.part")
+            download_bulk_zip(token or "", partial)
+            partial.replace(paths.zip)
+            zip_path = paths.zip
+
+        paths.building.unlink(missing_ok=True)
+        try:
+            counts = _write_parquet(zip_path, paths.building)
+            record = {
+                "dataset": "Domestic Energy Performance Certificates, London",
+                "source": API_BASE + BULK_PATH,
+                "snapshot_date": snapshot.isoformat(),
+                "file_info": file_info,
+                "zip_name": zip_path.name,
+                "zip_sha256": _sha256(zip_path),
+                "zip_size_bytes": zip_path.stat().st_size,
+                **counts,
+                "built": _iso(),
+                "gla_data_version": version("gla-data"),
+            }
+            record_partial = paths.record.with_name(f"{paths.record.name}.part")
+            record_partial.write_text(json.dumps(record, indent=2), encoding="utf-8")
+            paths.building.replace(paths.parquet)
+            record_partial.replace(paths.record)
+        finally:
+            paths.building.unlink(missing_ok=True)
+
+        if from_zip is None and not keep_download:
+            paths.zip.unlink(missing_ok=True)
+    return paths.parquet
+
+
+# ---------------------------------------------------------------------------
+# Reads
+# ---------------------------------------------------------------------------
+
+
+def read_record(destination: Path) -> dict[str, Any]:
+    """Return the JSON record written alongside the parquet file."""
+    paths = local_paths(destination)
+    if not paths.parquet.exists():
+        raise FileNotFoundError(
+            f"No EPC data in {destination}. Run gla_data.download_epc() first."
+        )
+    if not paths.record.exists():
+        raise FileNotFoundError(
+            f"{paths.record} is missing. Rebuild with gla_data.download_epc()."
+        )
+    record: dict[str, Any] = json.loads(paths.record.read_text(encoding="utf-8"))
+    return record
+
+
+def is_out_of_date(snapshot: date, today: date) -> bool:
+    """Whether a newer monthly bulk file should exist.
+
+    The bulk file is rebuilt on the 1st of each month, so a snapshot from
+    before the start of the current month has been superseded.
+    """
+    return snapshot < today.replace(day=1)
+
+
+def read(
+    parquet: Path, columns: list[str] | None, local_authorities: list[str] | None
+) -> pd.DataFrame:
+    """Read the London file, optionally only some columns and authorities.
+
+    Both filters are applied by pyarrow while reading, so only what is asked
+    for is loaded.
+    """
+    filters = (
+        [("local_authority", "in", local_authorities)] if local_authorities else None
+    )
+    table = pq.read_table(parquet, columns=columns, filters=filters)  # type: ignore[no-untyped-call]
+    return table.to_pandas(types_mapper=_pandas_type).reset_index(drop=True)
+
+
+def _pandas_type(arrow_type: pa.DataType) -> Any:
+    """Nullable pandas types for the parquet file's columns."""
+    if pa.types.is_integer(arrow_type):
+        return pd.Int64Dtype()
+    if pa.types.is_boolean(arrow_type):
+        return pd.BooleanDtype()
+    if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type):
+        return pd.StringDtype()
+    return None