gla-data 0.1.0__py3-none-any.whl

gla_data/__init__.py

@@ -0,0 +1,16 @@
+from importlib.metadata import version
+
+from gla_data.catalogue import catalogue
+from gla_data.census import load_census_dwellings
+from gla_data.geography import load_boundaries, load_geography_lookup
+from gla_data.transforms import aggregate, crosswalk
+
+__version__ = version("gla-data")
+__all__ = [
+    "load_boundaries",
+    "crosswalk",
+    "aggregate",
+    "load_geography_lookup",
+    "load_census_dwellings",
+    "catalogue",
+]

gla_data/_arcgis.py

@@ -0,0 +1,112 @@
+"""Private module for querying ArcGIS FeatureServer REST APIs.
+
+Implements paginated attribute and GeoJSON queries against the ArcGIS
+FeatureServer /query endpoint. The WHERE clause uses the ArcGIS SQL dialect:
+equality, LIKE with % wildcard, IN lists, and 1=1 for no filter. Subqueries
+and joins are not supported.
+
+Documentation:
+https://developers.arcgis.com/rest/services-reference/enterprise/query-feature-service-layer/
+"""
+
+from typing import Any
+
+import requests
+
+PAGE_SIZE = 1000
+
+
+def query_attributes(
+    base_url: str,
+    service_name: str,
+    where: str = "1=1",
+    out_fields: str = "*",
+) -> list[dict[str, Any]]:
+    """Query an ArcGIS FeatureServer layer, returning attributes only (no geometry).
+
+    Results are paginated automatically using ``resultOffset``.
+    """
+    url = f"{base_url}/{service_name}/FeatureServer/0/query"
+    attributes: list[dict[str, Any]] = []
+    offset = 0
+
+    while True:
+        response = requests.post(
+            url,
+            data={
+                "where": where,
+                "outFields": out_fields,
+                "f": "json",
+                "returnGeometry": "false",
+                "resultOffset": offset,
+                "resultRecordCount": PAGE_SIZE,
+            },
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        if "error" in result:
+            raise RuntimeError(
+                f"ArcGIS API error querying {service_name!r}: {result['error']}"
+            )
+
+        batch = result.get("features", [])
+        attributes.extend(f["attributes"] for f in batch)
+
+        if len(batch) < PAGE_SIZE:
+            break
+        offset += PAGE_SIZE
+
+    return attributes
+
+
+def query_geojson(
+    base_url: str,
+    service_name: str,
+    where: str = "1=1",
+    out_fields: str = "*",
+    out_sr: int = 27700,
+) -> list[dict[str, Any]]:
+    """Query an ArcGIS FeatureServer layer, returning GeoJSON features with geometry.
+
+    Returns a list of GeoJSON feature dicts (each with ``geometry`` and
+    ``properties`` keys). Uses POST to avoid URL length limits for large IN clauses.
+
+    Note: GeoJSON is formally defined to use WGS84, but ArcGIS honours ``outSR``
+    and returns coordinates in the requested CRS. The caller is responsible for
+    passing the same EPSG code to ``GeoDataFrame.from_features``.
+
+    Results are paginated automatically using ``resultOffset``.
+    """
+    url = f"{base_url}/{service_name}/FeatureServer/0/query"
+    features = []
+    offset = 0
+
+    while True:
+        response = requests.post(
+            url,
+            data={
+                "where": where,
+                "outFields": out_fields,
+                "outSR": out_sr,
+                "f": "geojson",
+                "resultOffset": offset,
+                "resultRecordCount": PAGE_SIZE,
+            },
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        if "error" in result:
+            raise RuntimeError(
+                f"ArcGIS API error querying {service_name!r}: {result['error']}"
+            )
+
+        batch = result.get("features", [])
+        features.extend(batch)
+
+        if len(batch) < PAGE_SIZE:
+            break
+        offset += PAGE_SIZE
+
+    return features

gla_data/_cache.py

@@ -0,0 +1,48 @@
+"""Shared disk cache for slow external data fetches."""
+
+import warnings
+from pathlib import Path
+
+import geopandas as gpd
+import pandas as pd
+
+CACHE_DIR = Path.home() / ".cache" / "gla_data"
+
+
+def cache_path(key: str) -> Path:
+    return CACHE_DIR / f"{key}.parquet"
+
+
+def read_cache(key: str) -> pd.DataFrame | gpd.GeoDataFrame | None:
+    path = cache_path(key)
+    if not path.exists():
+        return None
+    # GeoDataFrame parquet files contain geometry metadata that gpd.read_parquet
+    # understands; plain DataFrames can also be read by gpd but return without
+    # a geometry column. We try gpd first and fall back to pd if it errors.
+    try:
+        return gpd.read_parquet(path)
+    except Exception:
+        return pd.read_parquet(path)
+
+
+def write_cache(key: str, df: pd.DataFrame | gpd.GeoDataFrame) -> None:
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    df.to_parquet(cache_path(key), index=False)
+
+
+def warn_slow_fetch(function_name: str, key: str, cache: bool) -> None:
+    path = cache_path(key)
+    if cache:
+        detail = (
+            f"The result will be cached to {path} for future calls. "
+            f"Pass cache=False to disable caching."
+        )
+    else:
+        detail = "Pass cache=True to cache the result and avoid this on future calls."
+    warnings.warn(
+        f"{function_name} fetches data from an external source and may be slow. "
+        f"{detail}",
+        UserWarning,
+        stacklevel=3,
+    )

gla_data/_ons.py

@@ -0,0 +1,207 @@
+"""Private module for fetching data from the ONS Open Geography Portal.
+
+The ONS Open Geography Portal exposes boundaries and lookup tables as ArcGIS
+FeatureServer layers queried via _arcgis. Service names and field mappings are
+defined here; HTTP mechanics live in _arcgis.
+
+Browse available datasets: https://geoportal.statistics.gov.uk
+"""
+
+from typing import Any
+
+import geopandas as gpd
+import pandas as pd
+
+from gla_data._arcgis import query_attributes, query_geojson
+
+ONS_ARCGIS_BASE = "https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services"
+BATCH_SIZE = 500
+
+# London borough LAD codes all begin with E09 — a permanent ONS GSS prefix.
+# E = England, 09 = London Borough at the LAD level.
+LONDON_LAD_PREFIX = "E09"
+
+# (geography, year) -> (service_name, code_col, name_col)
+# BFC = Full Resolution Clipped, BFE = Full Resolution Extent,
+# BSC = Super Generalised Clipped.
+BOUNDARY_SERVICES: dict[tuple[str, int | None], tuple[str, str, str]] = {
+    ("lsoa", 2021): (
+        "Lower_layer_Super_Output_Areas_December_2021_Boundaries_EW_BFC_V10",
+        "LSOA21CD",
+        "LSOA21NM",
+    ),
+    ("lsoa", 2011): (
+        "LSOA_2011_Boundaries_Super_Generalised_Clipped_BSC_EW_V4",
+        "LSOA11CD",
+        "LSOA11NM",
+    ),
+    ("msoa", 2021): (
+        "Middle_layer_Super_Output_Areas_December_2021_Boundaries_EW_BFE_V8",
+        "MSOA21CD",
+        "MSOA21NM",
+    ),
+    ("msoa", 2011): (
+        "Middle_layer_Super_Output_Areas_Dec_2011_Boundaries_Full_Clipped_BFC_EW_V3_2022",
+        "MSOA11CD",
+        "MSOA11NM",
+    ),
+    ("borough", 2021): (
+        "LAD_Dec_2021_GB_BFC_2022",
+        "LAD21CD",
+        "LAD21NM",
+    ),
+    ("borough", 2011): (
+        "Local_Authority_Districts_December_2011_FCB_EW_2022",
+        "LAD11CD",
+        "LAD11NM",
+    ),
+}
+
+# (geography, year) -> (service_name, code_col, lad_col)
+# Lookup tables used to find London codes before fetching boundaries.
+LOOKUP_SERVICES: dict[tuple[str, int], tuple[str, str, str]] = {
+    ("lsoa", 2021): ("LSOA21_LAD23_EW_LU", "LSOA21CD", "LAD23CD"),
+    ("lsoa", 2011): ("LSOA11_LSOA21_LAD22_EW_LU_v5", "LSOA11CD", "LAD22CD"),
+    ("msoa", 2021): ("MSOA11_MSOA21_LAD22_EW_LU_v2", "MSOA21CD", "LAD22CD"),
+    ("msoa", 2011): ("MSOA11_MSOA21_LAD22_EW_LU_v2", "MSOA11CD", "LAD22CD"),
+}
+
+# year -> (service_name, lad_col, oa_col, out_fields)
+# OA-grain lookup tables covering OA, LSOA, MSOA, and LAD for England and Wales.
+# The 2011 table uses 2020 LAD boundaries — this is what ONS provides at this grain.
+GEOGRAPHY_LOOKUP_SERVICES: dict[int, tuple[str, str, str, str]] = {
+    2021: (
+        "OA_LSOA_MSOA_EW_DEC_2021_LU_v3",
+        "LAD22CD",
+        "OA21CD",
+        "OA21CD,LSOA21CD,LSOA21NM,MSOA21CD,MSOA21NM,LAD22CD,LAD22NM",
+    ),
+    2011: (
+        "OA11_LSOA11_MSOA11_LAD20_RGN20_EW_LU_a1cf695c9b074c708921b2a7555f808a",
+        "LAD20CD",
+        "OA11CD",
+        "OA11CD,LSOA11CD,LSOA11NM,MSOA11CD,MSOA11NM,LAD20CD,LAD20NM",
+    ),
+}
+
+
+def london_codes(geography: str, year: int) -> list[str]:
+    """Return geography codes for all London areas using the ONS lookup table."""
+    key = (geography, year)
+    if key not in LOOKUP_SERVICES:
+        raise ValueError(
+            f"No London lookup available for geography={geography!r}, year={year}"
+        )
+    service_name, code_col, lad_col = LOOKUP_SERVICES[key]
+    rows = query_attributes(
+        ONS_ARCGIS_BASE,
+        service_name,
+        where=f"{lad_col} LIKE '{LONDON_LAD_PREFIX}%'",
+        out_fields=code_col,
+    )
+    return list({row[code_col] for row in rows})
+
+
+def fetch_boundaries_for_codes(
+    service_name: str,
+    code_col: str,
+    name_col: str,
+    codes: list[str],
+) -> list[dict[str, Any]]:
+    """Fetch boundary GeoJSON features for a list of geography codes in batches."""
+    features = []
+    for i in range(0, len(codes), BATCH_SIZE):
+        batch = codes[i : i + BATCH_SIZE]
+        codes_str = ",".join(f"'{c}'" for c in batch)
+        features.extend(
+            query_geojson(
+                ONS_ARCGIS_BASE,
+                service_name,
+                where=f"{code_col} IN ({codes_str})",
+                out_fields=f"{code_col},{name_col}",
+            )
+        )
+    return features
+
+
+def fetch_boundaries(geography: str, year: int | None) -> gpd.GeoDataFrame:
+    """Fetch London boundary polygons from ONS for the given geography and year.
+
+    Returns a GeoDataFrame in EPSG:27700 (British National Grid).
+    """
+    key = (geography, year)
+    if key not in BOUNDARY_SERVICES:
+        available = sorted(f"geography={g!r}, year={y}" for g, y in BOUNDARY_SERVICES)
+        raise ValueError(
+            f"No boundary data available for geography={geography!r}, year={year}. "
+            f"Available combinations: {available}"
+        )
+
+    service_name, code_col, name_col = BOUNDARY_SERVICES[key]
+
+    if geography == "borough":
+        features = query_geojson(
+            ONS_ARCGIS_BASE,
+            service_name,
+            where=f"{code_col} LIKE '{LONDON_LAD_PREFIX}%'",
+            out_fields=f"{code_col},{name_col}",
+        )
+    else:
+        codes = london_codes(geography, year)  # type: ignore[arg-type]
+        features = fetch_boundaries_for_codes(service_name, code_col, name_col, codes)
+
+    if not features:
+        raise RuntimeError(
+            f"No features returned from ONS for geography={geography!r}, year={year}"
+        )
+
+    gdf = gpd.GeoDataFrame.from_features(features, crs=27700)
+    gdf.columns = [c.upper() if c != "geometry" else c for c in gdf.columns]
+    return gdf.reset_index(drop=True)
+
+
+def fetch_lsoa_crosswalk() -> pd.DataFrame:
+    """Fetch the ONS LSOA 2011→2021 exact-fit crosswalk for London.
+
+    Returns one row per 2011→2021 LSOA pair. CHGIND: U=unchanged, X=split, M=merged.
+    """
+    rows = query_attributes(
+        ONS_ARCGIS_BASE,
+        "LSOA11_LSOA21_LAD22_EW_LU_v5",
+        where=f"LAD22CD LIKE '{LONDON_LAD_PREFIX}%'",
+        out_fields="LSOA11CD,LSOA21CD,CHGIND",
+    )
+    return pd.DataFrame(rows)
+
+
+def fetch_geography_lookup(year: int, smallest_geography: str) -> pd.DataFrame:
+    """Fetch the ONS geography lookup table for London.
+
+    Source tables are at OA grain. For smallest_geography="lsoa", deduplicates
+    to LSOA grain by dropping the OA column.
+    """
+    if year not in GEOGRAPHY_LOOKUP_SERVICES:
+        raise ValueError(
+            f"year={year!r} is not supported. "
+            f"Available years: {sorted(GEOGRAPHY_LOOKUP_SERVICES)}"
+        )
+
+    service_name, lad_col, oa_col, out_fields = GEOGRAPHY_LOOKUP_SERVICES[year]
+    rows = query_attributes(
+        ONS_ARCGIS_BASE,
+        service_name,
+        where=f"{lad_col} LIKE '{LONDON_LAD_PREFIX}%'",
+        out_fields=out_fields,
+    )
+
+    df = pd.DataFrame(rows)
+    yr = str(year)[2:]
+
+    if smallest_geography == "lsoa":
+        df = df.drop(columns=[oa_col]).drop_duplicates(subset=f"LSOA{yr}CD")
+    elif smallest_geography == "msoa":
+        df = df.drop(columns=[oa_col, f"LSOA{yr}CD", f"LSOA{yr}NM"]).drop_duplicates(
+            subset=f"MSOA{yr}CD"
+        )
+
+    return df.reset_index(drop=True)

gla_data/_schemas.py

@@ -0,0 +1,68 @@
+"""Output schemas for gla_data public functions.
+
+Each public load function validates its output against a schema defined here.
+This catches upstream data format changes (e.g. ONS renaming a column) at the
+point of load rather than surfacing as a confusing error in user code.
+
+Schemas are constructed as factory functions rather than class-level constants
+because the expected columns depend on function parameters (year, geography).
+"""
+
+import pandera as pa
+
+# LAD column year suffix varies by lookup vintage — this is what ONS provides.
+LAD_YEAR_BY_LOOKUP_YEAR = {2021: "22", 2011: "20"}
+
+
+def boundary_schema(geography: str, year: int | None) -> pa.DataFrameSchema:
+    """Schema for the output of load_boundaries()."""
+    yr = str(year)[2:]
+    geo = geography.upper()
+
+    if geography == "borough":
+        code_col = f"LAD{yr}CD"
+        name_col = f"LAD{yr}NM"
+    else:
+        code_col = f"{geo}{yr}CD"
+        name_col = f"{geo}{yr}NM"
+
+    return pa.DataFrameSchema(  # type: ignore[no-untyped-call]
+        {
+            code_col: pa.Column(str, nullable=False),
+            name_col: pa.Column(str, nullable=False),
+        },
+        coerce=False,
+    )
+
+
+def geography_lookup_schema(year: int, smallest_geography: str) -> pa.DataFrameSchema:
+    """Schema for the output of load_geography_lookup()."""
+    yr = str(year)[2:]
+    lad_yr = LAD_YEAR_BY_LOOKUP_YEAR[year]
+    cols: dict[str, pa.Column] = {}
+
+    if smallest_geography == "oa":
+        cols[f"OA{yr}CD"] = pa.Column(str, nullable=False)
+
+    if smallest_geography in ("oa", "lsoa"):
+        cols[f"LSOA{yr}CD"] = pa.Column(str, nullable=False)
+        cols[f"LSOA{yr}NM"] = pa.Column(str, nullable=False)
+
+    cols[f"MSOA{yr}CD"] = pa.Column(str, nullable=False)
+    cols[f"MSOA{yr}NM"] = pa.Column(str, nullable=False)
+    cols[f"LAD{lad_yr}CD"] = pa.Column(str, nullable=False)
+    cols[f"LAD{lad_yr}NM"] = pa.Column(str, nullable=False)
+
+    return pa.DataFrameSchema(cols, coerce=False)  # type: ignore[no-untyped-call]
+
+
+def census_dwellings_schema(year: int) -> pa.DataFrameSchema:
+    """Schema for the output of load_census_dwellings()."""
+    yr = str(year)[2:]
+    return pa.DataFrameSchema(  # type: ignore[no-untyped-call]
+        {
+            f"LSOA{yr}CD": pa.Column(str, nullable=False),
+            "dwellings_total": pa.Column(int, pa.Check.ge(0), nullable=False),
+        },
+        coerce=False,
+    )

gla_data/_ukds.py

@@ -0,0 +1,55 @@
+"""Private module for downloading datasets from the UK Data Service (UKDS).
+
+UK Data Service is the principal repository for economic, population, and
+social research data in the UK, funded by UKRI. Datasets are available at:
+https://ukdataservice.ac.uk
+
+UKDS hosts data as static files. There is no query API — each entry below
+is a direct download URL. Files are downloaded in full and filtered
+client-side where needed.
+
+To browse available datasets: https://beta.ukdataservice.ac.uk/datacatalogue/
+"""
+
+import io
+from typing import Any
+
+import pandas as pd
+import requests
+
+# Registry of all UKDS datasets used in this package.
+# Each entry: dataset_key -> (url, format, description)
+# format is "csv" or "excel".
+DATASETS: dict[str, tuple[str, str, str]] = {
+    "census_dwellings_2011": (
+        "https://ukds-ckan.s3.eu-west-1.amazonaws.com/DWLTYP/DWLTYP_LSOADZ_England_Northern_Ireland_Scotland_Wales_Descriptions.csv",
+        "csv",
+        "QS418EW - Dwellings, 2011 Census, LSOA level, England and Wales",
+    ),
+    "census_dwellings_2021": (
+        "https://ukds-ckan.s3.eu-west-1.amazonaws.com/2021/ONS/number-of-dwellings/RM204-Number-Of-Dwellings-2021-lsoa-ONS.xlsx",
+        "excel",
+        "RM204 - Number of Dwellings, 2021 Census, LSOA level, England and Wales",
+    ),
+}
+
+
+def fetch(dataset_key: str, **kwargs: Any) -> pd.DataFrame:
+    """Download a UKDS dataset and return it as a DataFrame.
+
+    Extra keyword arguments are forwarded to ``pd.read_csv`` or
+    ``pd.read_excel`` as appropriate.
+    """
+    if dataset_key not in DATASETS:
+        raise KeyError(
+            f"Unknown UKDS dataset {dataset_key!r}. Available: {sorted(DATASETS)}"
+        )
+
+    url, fmt, _ = DATASETS[dataset_key]
+    response = requests.get(url)
+    response.raise_for_status()
+
+    if fmt == "csv":
+        return pd.read_csv(io.BytesIO(response.content), **kwargs)
+    else:
+        return pd.read_excel(io.BytesIO(response.content), **kwargs)

gla_data/catalogue.py

@@ -0,0 +1,62 @@
+"""Discovery catalogue listing all datasets available in gla_data."""
+
+import pandas as pd
+
+_CATALOGUE = [
+    {
+        "dataset": "ONS Boundaries",
+        "function": "load_boundaries",
+        "description": "London boundary polygons (OA, LSOA, MSOA, borough) in BNG (EPSG:27700)",  # noqa: E501
+        "access_tier": "public",
+        "status": "implemented",
+        "sources": "public",
+    },
+    {
+        "dataset": "ONS Geography Lookup",
+        "function": "load_geography_lookup",
+        "description": "ONS lookup table mapping London LSOAs (or OAs) to MSOA and borough",  # noqa: E501
+        "access_tier": "public",
+        "status": "implemented",
+        "sources": "public",
+    },
+    {
+        "dataset": "Census Dwellings",
+        "function": "load_census_dwellings",
+        "description": "Census dwelling counts for London LSOAs (2011 and 2021)",
+        "access_tier": "public",
+        "status": "implemented",
+        "sources": "public",
+    },
+    {
+        "dataset": "OS AddressBase",
+        "function": "load_addressbase",
+        "description": "Ordnance Survey AddressBase Premium — full address and UPRN data for London",  # noqa: E501
+        "access_tier": "licensed",
+        "status": "planned",
+        "sources": "s3",
+    },
+    {
+        "dataset": "EPC",
+        "function": "load_epc",
+        "description": "Energy Performance Certificate ratings for London dwellings",
+        "access_tier": "registered",
+        "status": "planned",
+        "sources": "public",
+    },
+]
+
+
+def catalogue() -> pd.DataFrame:
+    """Return a DataFrame listing all datasets available in gla_data.
+
+    Includes both implemented and planned datasets. For datasets with
+    ``access_tier`` other than ``"public"``, credentials are required —
+    see the README for setup instructions.
+
+    Returns
+    -------
+    pd.DataFrame
+        Columns: ``dataset``, ``function``, ``description``,
+        ``access_tier``, ``status``, ``sources``.
+    """
+    return pd.DataFrame(_CATALOGUE)

gla_data/census.py

@@ -0,0 +1,101 @@
+"""Load ONS census data for London."""
+
+from typing import Literal
+
+import pandas as pd
+
+from gla_data import _ukds
+from gla_data._cache import read_cache, warn_slow_fetch, write_cache
+from gla_data._schemas import census_dwellings_schema
+from gla_data.geography import load_geography_lookup
+
+_VALID_YEARS = (2011, 2021)
+
+_RAW_COLUMNS = {
+    2011: {
+        "code": "GEO_CODE",
+        "total": "Dwellings : Total\\ Dwellings - Unit : Dwellings",
+    },
+    2021: {"code": "Lower layer Super Output Areas Code", "total": "Observation"},
+}
+
+_EXCEL_KWARGS = {
+    2021: {"sheet_name": "Dataset"},
+}
+
+
+def _fetch_census_dwellings(year: int) -> pd.DataFrame:
+    """Download and normalise the raw UKDS census dwellings file for the given year."""
+    kwargs = _EXCEL_KWARGS.get(year, {})
+    raw = _ukds.fetch(f"census_dwellings_{year}", **kwargs)
+
+    cols = _RAW_COLUMNS[year]
+    code_col = f"LSOA{str(year)[2:]}CD"
+
+    return raw[[cols["code"], cols["total"]]].rename(
+        columns={
+            cols["code"]: code_col,
+            cols["total"]: "dwellings_total",
+        }
+    )
+
+
+def load_census_dwellings(
+    year: int = 2021,
+    geography: str = "lsoa",
+    source: Literal["public", "s3"] = "public",
+    cache: bool = True,
+) -> pd.DataFrame:
+    """Load census dwelling counts for London LSOAs.
+
+    Parameters
+    ----------
+    year:
+        Census year. One of: 2011, 2021.
+    geography:
+        Geographic unit. Only ``"lsoa"`` is currently supported.
+    source:
+        Data source. Only ``"public"`` is currently supported.
+    cache:
+        If True (default), save the result to ``~/.cache/gla_data/`` after the
+        first fetch and read from there on subsequent calls.
+
+    Returns
+    -------
+    pd.DataFrame
+        Columns: geography code (e.g. ``LSOA21CD``), ``dwellings_total``.
+
+    Examples
+    --------
+    >>> df = load_census_dwellings(year=2021)
+    """
+    if source == "s3":
+        raise NotImplementedError(
+            "S3 source is not yet supported for census dwellings."
+        )
+
+    if year not in _VALID_YEARS:
+        raise ValueError(f"year={year!r} is not valid. Choose from: {_VALID_YEARS}")
+
+    if geography != "lsoa":
+        raise NotImplementedError(
+            f"geography={geography!r} is not yet supported. "
+            f"Only 'lsoa' is currently implemented."
+        )
+
+    cache_key = f"census_dwellings_{year}_{geography}"
+
+    if cache and (cached := read_cache(cache_key)) is not None:
+        df = cached
+    else:
+        warn_slow_fetch("load_census_dwellings", cache_key, cache)
+        df = _fetch_census_dwellings(year)
+        code_col = f"LSOA{str(year)[2:]}CD"
+        lookup = load_geography_lookup(year=year, smallest_geography="lsoa")
+        london_codes = lookup[code_col]
+        df = df[df[code_col].isin(london_codes)].reset_index(drop=True)
+        if cache:
+            write_cache(cache_key, df)
+
+    census_dwellings_schema(year).validate(df)
+    return df

gla_data/cli.py

@@ -0,0 +1,35 @@
+"""Command-line interface for gla_data."""
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+import gla_data
+
+app = typer.Typer(
+    help="GLA City Intelligence Data Science package.", no_args_is_help=True
+)
+
+
+@app.command()
+def catalogue() -> None:
+    """List all datasets available in gla_data."""
+    df = gla_data.catalogue()
+
+    display_cols = ["dataset", "function", "description", "access_tier", "status"]
+
+    table = Table(show_header=True, header_style="bold", expand=True)
+    table.add_column("dataset", no_wrap=True, ratio=2)
+    table.add_column("function", no_wrap=True, ratio=2)
+    table.add_column("description", ratio=4)
+    table.add_column("access_tier", no_wrap=True, ratio=2)
+    table.add_column("status", no_wrap=True, ratio=2)
+
+    for row in df[display_cols].itertuples(index=False):
+        table.add_row(*[str(v) for v in row])
+
+    Console(width=120).print(table)
+
+
+def main() -> None:
+    app()

gla_data/config.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    """Runtime configuration.
+
+    Values are loaded from .env (gitignored).
+    All settings can be overridden with environment variables prefixed
+    GLA_DATA_.
+    """
+
+    data_dir: Path = Path("data")
+    output_dir: Path = Path("outputs")
+
+    model_config = {"env_prefix": "GLA_DATA_", "env_file": ".env"}
+
+
+settings = Settings()

gla_data/geography.py

@@ -0,0 +1,156 @@
+"""Load ONS geographic reference data for London."""
+
+from typing import Literal
+
+import geopandas as gpd
+import pandas as pd
+
+from gla_data._cache import read_cache, warn_slow_fetch, write_cache
+from gla_data._ons import fetch_boundaries as _fetch_boundaries
+from gla_data._ons import fetch_geography_lookup as _fetch_geography_lookup
+from gla_data._schemas import boundary_schema, geography_lookup_schema
+
+_VALID_BOUNDARY_GEOGRAPHIES = ("oa", "lsoa", "msoa", "ward", "borough", "london")
+_BOUNDARY_GEOGRAPHIES_WITH_YEAR = ("oa", "lsoa", "msoa", "borough")
+_VALID_LOOKUP_YEARS = (2011, 2021)
+_VALID_LOOKUP_GEOGRAPHIES = ("oa", "lsoa", "msoa")
+
+
+def load_boundaries(
+    geography: str = "lsoa",
+    year: int | None = 2021,
+    source: Literal["public", "s3"] = "public",
+    cache: bool = True,
+) -> gpd.GeoDataFrame:
+    """Load London boundary polygons from the ONS Open Geography Portal.
+
+    Parameters
+    ----------
+    geography:
+        Geographic unit. One of: ``"oa"``, ``"lsoa"``, ``"msoa"``,
+        ``"ward"``, ``"borough"``, ``"london"``. Defaults to ``"lsoa"``.
+    year:
+        Boundary vintage year. Required for ``"oa"``, ``"lsoa"``, ``"msoa"``,
+        and ``"borough"``; ignored for ``"london"``.
+        Valid values depend on geography — LSOAs: 2011, 2021; boroughs: 2011, 2021.
+    source:
+        Data source. Only ``"public"`` is currently supported.
+    cache:
+        If True (default), save the result to ``~/.cache/gla_data/`` after the
+        first fetch and read from there on subsequent calls. Pass ``cache=False``
+        to always fetch fresh from the ONS API.
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Boundary polygons for London in British National Grid (EPSG:27700).
+        Columns: geography code, geography name, geometry.
+
+    Examples
+    --------
+    >>> gdf = load_boundaries(geography="lsoa", year=2021)
+    >>> gdf = load_boundaries(geography="borough", year=2021)
+    """
+    if source == "s3":
+        raise NotImplementedError(
+            "S3 source is not yet supported for boundaries. "
+            "See the README for setup instructions when S3 support is added."
+        )
+
+    if geography not in _VALID_BOUNDARY_GEOGRAPHIES:
+        raise ValueError(
+            f"geography={geography!r} is not valid. "
+            f"Choose from: {_VALID_BOUNDARY_GEOGRAPHIES}"
+        )
+
+    if geography in _BOUNDARY_GEOGRAPHIES_WITH_YEAR and year is None:
+        raise ValueError(f"year is required for geography={geography!r}.")
+
+    resolved_year = None if geography not in _BOUNDARY_GEOGRAPHIES_WITH_YEAR else year
+    cache_key = f"boundaries_{geography}_{resolved_year}"
+
+    if cache and (cached := read_cache(cache_key)) is not None:
+        gdf = cached
+    else:
+        warn_slow_fetch("load_boundaries", cache_key, cache)
+        gdf = _fetch_boundaries(geography, resolved_year)
+        if cache:
+            write_cache(cache_key, gdf)
+
+    boundary_schema(geography, resolved_year).validate(gdf)
+
+    return gdf
+
+
+def load_geography_lookup(
+    year: int = 2021,
+    smallest_geography: str = "lsoa",
+    source: Literal["public", "s3"] = "public",
+    cache: bool = True,
+) -> pd.DataFrame:
+    """Load the ONS geography lookup table for London.
+
+    Maps each London area to its parent geographies. Useful for aggregating
+    data to higher geographies via a merge.
+
+    Parameters
+    ----------
+    year:
+        Boundary vintage year. One of: 2011, 2021.
+    smallest_geography:
+        The finest geographic unit to include. ``"lsoa"`` returns one row per
+        LSOA (default); ``"oa"`` returns one row per output area and includes
+        the OA code column.
+    source:
+        Data source. Only ``"public"`` is currently supported.
+    cache:
+        If True (default), save the result to ``~/.cache/gla_data/`` after the
+        first fetch and read from there on subsequent calls. Pass ``cache=False``
+        to always fetch fresh from the ONS API.
+
+    Returns
+    -------
+    pd.DataFrame
+        Columns for 2021 (lsoa grain):
+        ``LSOA21CD``, ``LSOA21NM``, ``MSOA21CD``, ``MSOA21NM``,
+        ``LAD22CD``, ``LAD22NM``.
+        For 2011: ``LSOA11CD``, ``LSOA11NM``, ``MSOA11CD``, ``MSOA11NM``,
+        ``LAD20CD``, ``LAD20NM``.
+        Note: the 2011 table uses 2020 LAD boundaries, which is what ONS
+        provides at this grain.
+
+    Examples
+    --------
+    >>> df = load_geography_lookup(year=2021)
+    >>> df_msoa = data.merge(df[["LSOA21CD", "MSOA21CD"]], on="LSOA21CD")
+    """
+    if source == "s3":
+        raise NotImplementedError(
+            "S3 source is not yet supported for geography lookups. "
+            "See the README for setup instructions when S3 support is added."
+        )
+
+    if year not in _VALID_LOOKUP_YEARS:
+        raise ValueError(
+            f"year={year!r} is not valid. Choose from: {_VALID_LOOKUP_YEARS}"
+        )
+
+    if smallest_geography not in _VALID_LOOKUP_GEOGRAPHIES:
+        raise ValueError(
+            f"smallest_geography={smallest_geography!r} is not valid. "
+            f"Choose from: {_VALID_LOOKUP_GEOGRAPHIES}"
+        )
+
+    cache_key = f"geography_lookup_{year}_{smallest_geography}"
+
+    if cache and (cached := read_cache(cache_key)) is not None:
+        df = cached
+    else:
+        warn_slow_fetch("load_geography_lookup", cache_key, cache)
+        df = _fetch_geography_lookup(year, smallest_geography)
+        if cache:
+            write_cache(cache_key, df)
+
+    geography_lookup_schema(year, smallest_geography).validate(df)
+
+    return df

gla_data/transforms.py

@@ -0,0 +1,244 @@
+"""Geographic transformation functions: crosswalk and aggregate."""
+
+import pandas as pd
+
+from gla_data._cache import read_cache, warn_slow_fetch, write_cache
+from gla_data._ons import fetch_lsoa_crosswalk
+from gla_data.geography import load_geography_lookup
+
+_SUPPORTED_CROSSWALKS = {(2011, 2021)}
+_SUPPORTED_METHODS = {"uniform"}
+_SUPPORTED_GEOGRAPHIES = {"lsoa"}
+
+# Ordered from smallest to largest — used to validate that
+# to_geography > from_geography.
+_GEOGRAPHY_ORDER = ["oa", "lsoa", "msoa", "borough"]
+
+# Maps (from_geography, to_geography) → the smallest_geography grain to load from
+# load_geography_lookup. Only combinations where ONS publishes a lookup at the
+# from_geography grain are listed; aggregations not in this dict are unsupported.
+#
+# OA grain lookup contains:  OA, LSOA, MSOA, LAD  → oa→{lsoa,msoa,borough}
+# LSOA grain lookup contains: LSOA, MSOA, LAD      → lsoa→{msoa,borough}
+# No MSOA grain lookup exists, so msoa→borough is not supported.
+_SUPPORTED_AGGREGATIONS: dict[tuple[str, str], str] = {
+    ("oa", "lsoa"): "oa",
+    ("oa", "msoa"): "oa",
+    ("oa", "borough"): "oa",
+    ("lsoa", "msoa"): "lsoa",
+    ("lsoa", "borough"): "lsoa",
+    ("msoa", "borough"): "msoa",
+}
+
+# Borough (LAD) column suffix varies by lookup year — ONS provides 2022 boundaries
+# for 2021 vintage and 2020 boundaries for 2011 vintage.
+_LAD_COL_SUFFIX_BY_YEAR: dict[int, str] = {2021: "LAD22CD", 2011: "LAD20CD"}
+
+_SUPPORTED_AGGREGATE_METHODS = {"sum"}
+
+
+def _lookup_col(geography: str, year: int) -> str:
+    """Return the column name for a geography in the ONS lookup table."""
+    if geography == "borough":
+        return _LAD_COL_SUFFIX_BY_YEAR[year]
+    yr = str(year)[2:]
+    return f"{geography.upper()}{yr}CD"
+
+
+def crosswalk(
+    df: pd.DataFrame,
+    from_year: int,
+    to_year: int,
+    value_cols: list[str],
+    geography: str = "lsoa",
+    method: str = "uniform",
+    cache: bool = True,
+) -> pd.DataFrame:
+    """Convert LSOA-level data from one boundary vintage to another.
+
+    Uses the ONS exact-fit lookup to map values from 2011 LSOA boundaries to
+    2021 LSOA boundaries. Where a 2011 LSOA was split into multiple 2021 LSOAs,
+    the value is divided equally among them (method="uniform"). Where multiple
+    2011 LSOAs were merged into one 2021 LSOA, their values are summed.
+
+    Parameters
+    ----------
+    df:
+        DataFrame containing a column named ``LSOA{from_year_short}CD``
+        (e.g. ``LSOA11CD``) and the columns listed in ``value_cols``.
+    from_year:
+        Boundary vintage of the input data. Currently only 2011 is supported.
+    to_year:
+        Target boundary vintage. Currently only 2021 is supported.
+    value_cols:
+        Columns to crosswalk. Must all be present in ``df``.
+    geography:
+        Geographic unit to crosswalk. Currently only ``"lsoa"`` is supported.
+    method:
+        Split method for LSOAs that changed boundaries. Currently only
+        ``"uniform"`` is supported: values are divided equally among target LSOAs.
+    cache:
+        If True (default), cache the crosswalk lookup table to
+        ``~/.cache/gla_data/`` after the first fetch.
+
+    Returns
+    -------
+    pd.DataFrame
+        One row per target LSOA, with columns ``LSOA{to_year_short}CD`` and
+        the crosswalked value columns.
+
+    Examples
+    --------
+    >>> df_2021 = crosswalk(df_2011, from_year=2011, to_year=2021, value_cols=["count"])
+    """
+    if geography not in _SUPPORTED_GEOGRAPHIES:
+        raise NotImplementedError(
+            f"geography={geography!r} is not yet supported. "
+            f"Currently supported: {sorted(_SUPPORTED_GEOGRAPHIES)}"
+        )
+
+    if (from_year, to_year) not in _SUPPORTED_CROSSWALKS:
+        raise ValueError(
+            f"from_year={from_year!r}, to_year={to_year!r} is not supported. "
+            f"Supported combinations: {sorted(_SUPPORTED_CROSSWALKS)}"
+        )
+
+    if method not in _SUPPORTED_METHODS:
+        raise ValueError(
+            f"method={method!r} is not supported. "
+            f"Choose from: {sorted(_SUPPORTED_METHODS)}"
+        )
+
+    from_col = f"LSOA{str(from_year)[2:]}CD"
+    to_col = f"LSOA{str(to_year)[2:]}CD"
+
+    if from_col not in df.columns:
+        raise ValueError(
+            f"{from_col!r} column not found in df. "
+            f"Expected a column named {from_col!r} for from_year={from_year!r}."
+        )
+
+    missing = [c for c in value_cols if c not in df.columns]
+    if missing:
+        raise ValueError(f"value_cols {missing} not found in df.")
+
+    cache_key = f"crosswalk_{geography}_{from_year}_{to_year}"
+    if cache and (cached := read_cache(cache_key)) is not None:
+        lookup = cached[[from_col, to_col]]
+    else:
+        warn_slow_fetch("crosswalk", cache_key, cache)
+        lookup = fetch_lsoa_crosswalk()[[from_col, to_col]]
+        if cache:
+            write_cache(cache_key, lookup)
+
+    merged = df[[from_col] + value_cols].merge(lookup, on=from_col, how="left")
+
+    # Count how many target LSOAs each source LSOA maps to, then divide
+    # values by that count so the total is preserved across splits.
+    counts = merged.groupby(from_col)[to_col].transform("count")
+    for col in value_cols:
+        merged[col] = merged[col] / counts
+
+    return merged.groupby(to_col)[value_cols].sum().reset_index()
+
+
+def aggregate(
+    df: pd.DataFrame,
+    from_geography: str,
+    to_geography: str,
+    value_cols: list[str],
+    method: str = "sum",
+    year: int = 2021,
+) -> pd.DataFrame:
+    """Aggregate data from a smaller to a larger geography using the ONS lookup.
+
+    Parameters
+    ----------
+    df:
+        DataFrame containing a source geography code column (e.g. ``OA21CD``)
+        and the columns listed in ``value_cols``. Any additional columns not
+        in ``value_cols`` are treated as extra grouping keys and preserved in
+        the output (e.g. a ``financial_year`` column).
+    from_geography:
+        Source geographic unit. One of: ``"oa"``, ``"lsoa"``, ``"msoa"``.
+    to_geography:
+        Target geographic unit. Must be larger than ``from_geography``.
+        One of: ``"lsoa"``, ``"msoa"``, ``"borough"``.
+    value_cols:
+        Columns to aggregate. Must all be present in ``df``.
+    method:
+        Aggregation method. Currently only ``"sum"`` is supported.
+    year:
+        Lookup vintage year. Determines which ONS geography lookup is used
+        and the year suffix on geography code columns (e.g. 2021 → ``OA21CD``).
+
+    Returns
+    -------
+    pd.DataFrame
+        One row per unique combination of target geography code and any extra
+        grouping columns, with ``value_cols`` aggregated by ``method``.
+
+    Examples
+    --------
+    >>> df_lsoa = aggregate(df_oa, "oa", "lsoa", value_cols=["count"])
+    >>> df_borough = aggregate(df_lsoa, "lsoa", "borough", value_cols=["count"])
+    """
+    if from_geography not in _GEOGRAPHY_ORDER:
+        raise ValueError(
+            f"from_geography={from_geography!r} is not a valid geography. "
+            f"Choose from: {_GEOGRAPHY_ORDER}"
+        )
+
+    if to_geography not in _GEOGRAPHY_ORDER:
+        raise ValueError(
+            f"to_geography={to_geography!r} is not a valid geography. "
+            f"Choose from: {_GEOGRAPHY_ORDER}"
+        )
+
+    from_idx = _GEOGRAPHY_ORDER.index(from_geography)
+    to_idx = _GEOGRAPHY_ORDER.index(to_geography)
+    if to_idx <= from_idx:
+        raise ValueError(
+            f"to_geography={to_geography!r} must be larger than "
+            f"from_geography={from_geography!r}."
+        )
+
+    if (from_geography, to_geography) not in _SUPPORTED_AGGREGATIONS:
+        supported_for = sorted(
+            t for f, t in _SUPPORTED_AGGREGATIONS if f == from_geography
+        )
+        raise NotImplementedError(
+            f"Aggregation from {from_geography!r} to {to_geography!r} is not supported "
+            f"because no ONS lookup table is published at {from_geography!r} grain. "
+            f"Supported targets for from_geography={from_geography!r}: {supported_for}"
+        )
+
+    if method not in _SUPPORTED_AGGREGATE_METHODS:
+        raise ValueError(
+            f"method={method!r} is not supported. "
+            f"Choose from: {sorted(_SUPPORTED_AGGREGATE_METHODS)}"
+        )
+
+    from_col = _lookup_col(from_geography, year)
+    to_col = _lookup_col(to_geography, year)
+
+    if from_col not in df.columns:
+        raise ValueError(
+            f"{from_col!r} column not found in df. "
+            f"Expected a column named {from_col!r} for "
+            f"from_geography={from_geography!r}, year={year}."
+        )
+
+    missing = [c for c in value_cols if c not in df.columns]
+    if missing:
+        raise ValueError(f"value_cols {missing} not found in df.")
+
+    grain = _SUPPORTED_AGGREGATIONS[(from_geography, to_geography)]
+    lookup = load_geography_lookup(year=year, smallest_geography=grain)
+    lookup = lookup[[from_col, to_col]].drop_duplicates()
+
+    merged = df.merge(lookup, on=from_col, how="left").drop(columns=[from_col])
+
+    group_cols = [c for c in merged.columns if c not in value_cols]
+
+    return merged.groupby(group_cols)[value_cols].sum().reset_index()

gla_data-0.1.0.dist-info/METADATA

@@ -0,0 +1,43 @@
+Metadata-Version: 2.4
+Name: gla-data
+Version: 0.1.0
+Summary: Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
+Author-email: Conor Dempsey <conor.dempsey@london.gov.uk>
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: geopandas>=1.0
+Requires-Dist: pandas>=2.2
+Requires-Dist: pandera>=0.32.1
+Requires-Dist: pyarrow>=24.0.0
+Requires-Dist: pydantic-settings
+Requires-Dist: requests>=2.32
+Requires-Dist: typer>=0.26.8
+Description-Content-Type: text/markdown
+
+# gla-data
+
+Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
+
+## Installation
+
+Add `gla-data` as a PyPI dependency in your `pixi.toml`:
+
+```toml
+[pypi-dependencies]
+gla-data = ">=0.1"
+```
+
+Then run `pixi install`. Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
+
+## Development
+
+```bash
+git clone <repo-url>
+cd gla-data
+pixi install
+
+pixi run test       # run tests
+pixi run lint       # check code style
+pixi run format     # auto-format
+pixi run typecheck  # type checking
+```

gla_data-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+gla_data/__init__.py,sha256=dNcSoqJS5fLxC6TpFPieC-pe5iYFYY0zX4I-H11o_VQ,435
+gla_data/_arcgis.py,sha256=57MjMqusVXl5qqkwEZohP32hYQNAr132XCPTyia5DgE,3236
+gla_data/_cache.py,sha256=zFrjmJTxfcellN0qGVMABccDxWh0QbLlvC8uh8SYaSI,1458
+gla_data/_ons.py,sha256=vncHPMzPneFJxZgLB17Uv-u-aVgih0WdzpoJtmasL2Y,7105
+gla_data/_schemas.py,sha256=EqaHm79_6B8v-vtguzfrGTziWjWjQcnyQj5y4qcxyUo,2405
+gla_data/_ukds.py,sha256=GqM-H2HKpvqK-k-iHR6EMg73I4_OVkGf5T8A5xO0JSE,1959
+gla_data/catalogue.py,sha256=xoBIfv4BO6-7PX18TqCm6RKZ4SzQ8EKY2EyzP4Wpg3Y,2020
+gla_data/census.py,sha256=BO0FTgLHbs95BrVfmXnqrUhl8xeyTYDupd3yDjKnRWI,2952
+gla_data/cli.py,sha256=ugzCWhhuhuoz2zng5Bm3lFNuzbk2iFbvkm0gxWJjaFM,955
+gla_data/config.py,sha256=hhIcMyjnGkrjww_9UWH6HVZsq62j52U6pMcTfc1WcKg,438
+gla_data/geography.py,sha256=DYHEtoGpRd5arj8E-4Y0JtplIYNbE3KK-u8aEhyU4yM,5513
+gla_data/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gla_data/transforms.py,sha256=0c5gCdOOP3alScIjfU6049tvc9pMsdiPC-DnOJdFcNw,9130
+gla_data-0.1.0.dist-info/METADATA,sha256=JeowQGuDekHSoSyGopdJiNMxwXiXpWRDvvQ0v1NRMCI,1156
+gla_data-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+gla_data-0.1.0.dist-info/entry_points.txt,sha256=H6tQ2DyXVUT3fksx2EnX7qSirx2rRaMTq8VVewet4WQ,47
+gla_data-0.1.0.dist-info/RECORD,,

gla_data-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

gla_data-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gla-data = gla_data.cli:main