gla-data 0.1.0__tar.gz

gla_data-0.1.0/.copier-answers.yml

@@ -0,0 +1,13 @@
+# Changes here will be overwritten by Copier
+_commit: v0.5.0
+_src_path: gh:Greater-London-Authority/ds-project-template
+author_email: conor.dempsey@london.gov.uk
+author_name: Conor Dempsey
+description: Shared tools for loading, transforming, and visualising the GLA datasets
+    used in City Data analysis.
+has_cli: false
+import_name: gla_data
+package_name: gla-data
+platforms: '["osx-arm64", "linux-64", "win-64"]'
+project_type: package
+python_version: '3.12'

gla_data-0.1.0/.env.example

@@ -0,0 +1,11 @@
+# Copy this file to .env and fill in values. Never commit .env.
+# Settings are loaded from this file automatically by config.py.
+
+# AWS (if used)
+# AWS_DEFAULT_REGION=eu-west-2
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+
+# Application settings — override here for local development if needed.
+# {{ import_name | upper }}_DATA_DIR=data
+# {{ import_name | upper }}_OUTPUT_DIR=outputs

gla_data-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.8.3
+        with:
+          pixi-version: latest
+      - run: pixi run test
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.8.3
+        with:
+          pixi-version: latest
+      - run: pixi run lint
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.8.3
+        with:
+          pixi-version: latest
+      - run: pixi run typecheck

gla_data-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/setup-pixi@v0.8.3
+        with:
+          pixi-version: latest
+      - run: pixi run build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

gla_data-0.1.0/.gitignore

@@ -0,0 +1,34 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Pixi
+.pixi/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Type checking
+.mypy_cache/
+
+# Editors
+.vscode/
+.idea/
+*.swp
+
+# Secrets
+.env
+
+# Data — immutable source data and generated outputs are not committed
+data/raw/
+outputs/
+
+# OS
+.DS_Store
+Thumbs.db

gla_data-0.1.0/DESIGN.md

@@ -0,0 +1,283 @@
+# gla-data design
+
+## Purpose
+
+`gla-data` is a shared Python package for the GLA City Intelligence Data Science team. It provides clean, consistent access to the GLA-relevant datasets used across City Data analysis projects.
+
+The primary goal is to reduce the cost of getting started with a new dataset. A user who knows a dataset exists but has not yet used it — because the API is unfamiliar, credentials are unclear, or the raw data requires significant cleaning — should be able to load analysis-ready data in a few lines of code.
+
+## Who this is for
+
+Data scientists on the City Intelligence team working on GLA analysis projects. Users are assumed to have some familiarity with the datasets, but `gla-data` should make it easy to discover what is available and how to use it, including:
+
+- What credentials or sign-ups are required to access a dataset
+- What the default filters are and why
+- What parameters are available to adjust those defaults
+
+## What this package does
+
+- Loads GLA-relevant datasets from public sources and (in future) from the GLA S3 data lake
+- Returns data filtered to sensible defaults (typically London, latest available year)
+- Provides standard geography transformations: aggregation, disaggregation, and year crosswalks
+- Documents required credentials and setup steps for each data source in the README; functions raise clear errors with a pointer to the README if credentials are missing
+
+## What this package does not do
+
+- Project-specific cleaning, transformation, or modelling — these belong in project code
+- Cross-dataset merging — users join datasets themselves; `gla-data` provides the building blocks
+- Exploratory analysis or visualisation — may be revisited once clear patterns emerge across projects
+
+---
+
+## Design principles
+
+**Defaults should be sensible and explicit.** Every parameter with a default should have a reason for that default, and the docstring should state it. A user should be able to call any function with no arguments and get something useful.
+
+**Return types are consistent.** All load functions return a `pd.DataFrame`. Functions that return geographic data return a `gpd.GeoDataFrame`. Column naming follows ONS conventions where they exist (e.g. `LSOA21CD`, `LAD22NM`).
+
+**Functions are stateless.** Call a function, get data back. No objects to manage, no state to track between calls.
+
+**Private functions do the work, public functions are the interface.** HTTP calls, API authentication, raw data parsing, and column renaming all happen in private functions prefixed with `_`. Public functions are thin and readable.
+
+**Source switching is per-function.** Each function accepts a `source` parameter defaulting to `"public"`. As datasets become available on the GLA S3 data lake, `source="s3"` is added to that function. Functions that do not yet support S3 raise a clear error if `source="s3"` is passed.
+
+---
+
+## Data access tiers
+
+Datasets differ in what setup is required. Each function's docstring and the catalogue state which tier it belongs to. Setup instructions for datasets requiring credentials are in the README. Functions that require credentials raise a clear, descriptive error if the relevant environment variable is missing — for example:
+
+```
+EnvironmentError: EPC_API_KEY not set. This dataset requires a free DLUHC account.
+See the README for setup instructions.
+```
+
+| Tier | Description | Examples |
+|---|---|---|
+| `public` | No credentials required. URLs are embedded in the package. | ONS boundaries, census data via Nomis |
+| `registered` | Requires a free account or API key. Set via environment variable. | EPC (DLUHC portal), Nomis bulk API key |
+| `licensed` | Requires a government or commercial licence. | OS AddressBase |
+| `s3` | Requires access to the GLA internal data lake (provisioned by GLA engineering). | TBD |
+
+---
+
+## API conventions
+
+### Naming
+
+- Data loading functions: `load_<dataset>` or `load_boundaries(geography=...)`
+- Geography transformation functions: `aggregate(...)`, `crosswalk(...)`
+- Discovery functions: `catalogue()`
+
+Parameters describe what the user wants, not how the code fetches it.
+
+### Standard parameters
+
+| Parameter | Type | Notes |
+|---|---|---|
+| `geography` | `str` | Geographic unit: `"oa"`, `"lsoa"`, `"msoa"`, `"ward"`, `"borough"`, `"london"` |
+| `year` | `int` | Reference year. Valid values are documented per function |
+| `source` | `Literal["public", "s3"]` | Data source. Defaults to `"public"` |
+
+### Return types
+
+| Function type | Return type | Notes |
+|---|---|---|
+| Tabular data | `pd.DataFrame` | Index is always a plain integer range index |
+| Geographic data | `gpd.GeoDataFrame` | CRS is always British National Grid (EPSG:27700) |
+
+### Column naming
+
+- Geography codes follow ONS conventions: `LSOA21CD`, `LAD22NM`, `OA21CD` etc.
+- The year in a geography code column name reflects the geography vintage, not the data year
+- All column names are uppercase with underscores
+
+---
+
+## Public API
+
+### Discovery
+
+```python
+gla_data.catalogue()
+```
+
+Returns a `pd.DataFrame` listing all available datasets with columns: `dataset`, `description`, `access_tier`, `status`, `source_supported`. For datasets with `access_tier` other than `public`, the catalogue notes that setup is required — see the README for instructions.
+
+---
+
+### Boundaries
+
+```python
+gdf = gla_data.load_boundaries(geography="lsoa", year=2021)
+```
+
+Returns boundary polygons for London at the specified geography and year.
+
+**Parameters:**
+- `geography`: `"oa"`, `"lsoa"`, `"msoa"`, `"ward"`, `"borough"`, `"london"`
+- `year`: valid values depend on geography (e.g. LSOAs: `2011`, `2021`; borough boundaries do not have a year)
+- `source`: `"public"` only for now
+
+**Returns:** `gpd.GeoDataFrame` with columns:
+- Geography code and name (e.g. `LSOA21CD`, `LSOA21NM`)
+- `geometry` — polygon in EPSG:27700
+
+**Access tier:** `public`
+
+---
+
+### Geography lookup
+
+```python
+df = gla_data.load_geography_lookup(year=2021, smallest_geography="lsoa")
+```
+
+Returns the ONS lookup table mapping London LSOAs (or OAs) to higher geographies (MSOA, borough).
+
+**Parameters:**
+- `year`: `2011` or `2021`
+- `smallest_geography`: `"lsoa"` (default, one row per LSOA) or `"oa"` (one row per OA, includes OA code column)
+- `source`: `"public"` only for now
+- `cache`: if `True` (default), result is cached to `~/.cache/gla_data/`
+
+**Returns:** `pd.DataFrame`. Columns for 2021 (lsoa grain):
+`LSOA21CD`, `LSOA21NM`, `MSOA21CD`, `MSOA21NM`, `LAD22CD`, `LAD22NM`.
+For 2011: `LSOA11CD`, `LSOA11NM`, `MSOA11CD`, `MSOA11NM`, `LAD20CD`, `LAD20NM`.
+Note: the 2011 table uses 2020 LAD boundaries — this is what ONS provides at this grain.
+
+**Access tier:** `public`
+
+**Notes:**
+- Source tables are at OA grain. For `smallest_geography="lsoa"`, deduplication is applied.
+- Fetching is slow (~17s). A `UserWarning` is raised on first call; result is cached to disk by default.
+
+---
+
+### Census dwellings
+
+```python
+df = gla_data.load_census_dwellings(year=2021, geography="lsoa")
+```
+
+Returns dwelling counts for London at the specified geography and census year.
+
+**Parameters:**
+- `year`: `2011` or `2021`
+- `geography`: `"oa"`, `"lsoa"`, `"msoa"`, `"borough"`, `"london"`
+- `source`: `"public"` only for now
+
+**Returns:** `pd.DataFrame` with a geography code column (e.g. `LSOA21CD`) and:
+- `dwellings_total`
+- Dwelling type breakdown: `dwellings_detached`, `dwellings_semi_detached`, `dwellings_terraced`, `dwellings_flat`, `dwellings_other`
+- Household space type breakdown: `spaces_total`, `spaces_occupied`, `spaces_vacant`
+
+**Access tier:** `public`
+
+**Notes:**
+- 2011 data uses 2011 LSOA boundaries; 2021 data uses 2021 LSOA boundaries. Use `crosswalk()` if you need both years on consistent boundaries.
+- When `geography` is above LSOA, data is aggregated via the standard ONS lookup.
+
+---
+
+### EPC
+
+```python
+df = gla_data.load_epc(geography="lsoa", year=2023)
+```
+
+Returns Energy Performance Certificate data for London.
+
+**Access tier:** `registered` — requires a free account at the DLUHC EPC portal. Run `gla_data.setup_instructions(dataset="epc")` for details.
+
+*Further design TBD once initial datasets are implemented.*
+
+---
+
+### Geography transformation
+
+#### Aggregation
+
+```python
+df_msoa = gla_data.aggregate(
+    df,
+    from_geography="lsoa",
+    to_geography="msoa",
+    value_cols=["dwellings_total", "dwellings_vacant"],
+    method="sum",
+    year=2021,
+)
+```
+
+Aggregates a DataFrame from a lower to a higher geography using the standard ONS lookup. The `year` parameter specifies which vintage of lookup to use.
+
+**Parameters:**
+- `df`: input DataFrame containing a geography code column (auto-detected by name)
+- `from_geography`: source geography
+- `to_geography`: target geography (must be higher than `from_geography`)
+- `value_cols`: columns to aggregate
+- `method`: `"sum"` or `"mean"`
+- `year`: lookup vintage
+
+#### Crosswalk (same geography, different year)
+
+```python
+df_2021 = gla_data.crosswalk(
+    df,
+    geography="lsoa",
+    from_year=2011,
+    to_year=2021,
+    value_cols=["dwellings_total"],
+    method="proportional",
+)
+```
+
+Converts a DataFrame from one year's geography boundaries to another using the ONS exact-fit or best-fit lookup.
+
+**Parameters:**
+- `df`: input DataFrame
+- `geography`: the geographic unit
+- `from_year`, `to_year`: boundary vintages
+- `value_cols`: numeric columns to convert
+- `method`: `"proportional"` (split values by area overlap) or `"exact_fit"` (for cases where boundaries nest cleanly)
+
+**Notes:**
+- Disaggregation (higher to lower geography) is not supported in the first version. The `aggregate` function is one-directional.
+- Disaggregation requires assumptions about how values are distributed within a higher geography. If this is needed, it should be designed explicitly with a stated method (e.g. uniform distribution, population-weighted).
+
+---
+
+## Package structure
+
+```
+src/gla_data/
+├── __init__.py      ← public API: re-exports all public functions
+├── py.typed
+├── config.py        ← environment variable handling (S3 config etc.)
+├── _arcgis.py       ← private: generic ArcGIS FeatureServer HTTP/pagination
+├── _ons.py          ← private: ONS service constants and fetch functions
+├── _ukds.py         ← private: UKDS dataset registry and static file downloads
+├── _cache.py        ← private: shared disk cache (~/.cache/gla_data/)
+├── geography.py     ← load_boundaries(), load_geography_lookup()
+├── transforms.py    ← crosswalk() [aggregate() to be added]
+├── catalogue.py     ← catalogue() [not yet implemented]
+├── census.py        ← load_census_dwellings()
+├── epc.py           ← load_epc() [not yet implemented]
+```
+
+### Notes on the private layer
+
+`_arcgis.py` contains generic ArcGIS FeatureServer pagination logic (no ONS-specific
+knowledge). `_ons.py` holds ONS service constants and the fetch functions that use them.
+When new data sources are added (e.g. Nomis for census, DLUHC for EPC), they get their
+own private module and introduce new HTTP helpers only if needed.
+
+---
+
+## Open questions
+
+- `aggregate` and `crosswalk` are generic functions (not per-geography-pair). They auto-detect the geography code column by name — open question: what happens if detection is ambiguous?
+- What is the right disaggregation method for numeric quantities (MSOA → LSOA)? Uniform? Population-weighted? Defer until there is a concrete use case.
+- Should `load_boundaries` accept `year=None` for geographies that do not have vintages (e.g. borough boundaries)?
+- Where does Land Registry data fit — access tier, likely `public` (price paid) or `registered` (title register)?
+- OS AddressBase: confirm whether the GLA licence covers use in this package or whether each team member needs their own licence.

gla_data-0.1.0/PKG-INFO

@@ -0,0 +1,43 @@
+Metadata-Version: 2.4
+Name: gla-data
+Version: 0.1.0
+Summary: Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
+Author-email: Conor Dempsey <conor.dempsey@london.gov.uk>
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: geopandas>=1.0
+Requires-Dist: pandas>=2.2
+Requires-Dist: pandera>=0.32.1
+Requires-Dist: pyarrow>=24.0.0
+Requires-Dist: pydantic-settings
+Requires-Dist: requests>=2.32
+Requires-Dist: typer>=0.26.8
+Description-Content-Type: text/markdown
+
+# gla-data
+
+Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
+
+## Installation
+
+Add `gla-data` as a PyPI dependency in your `pixi.toml`:
+
+```toml
+[pypi-dependencies]
+gla-data = ">=0.1"
+```
+
+Then run `pixi install`. Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
+
+## Development
+
+```bash
+git clone <repo-url>
+cd gla-data
+pixi install
+
+pixi run test       # run tests
+pixi run lint       # check code style
+pixi run format     # auto-format
+pixi run typecheck  # type checking
+```

gla_data-0.1.0/README.md

@@ -0,0 +1,27 @@
+# gla-data
+
+Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
+
+## Installation
+
+Add `gla-data` as a PyPI dependency in your `pixi.toml`:
+
+```toml
+[pypi-dependencies]
+gla-data = ">=0.1"
+```
+
+Then run `pixi install`. Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
+
+## Development
+
+```bash
+git clone <repo-url>
+cd gla-data
+pixi install
+
+pixi run test       # run tests
+pixi run lint       # check code style
+pixi run format     # auto-format
+pixi run typecheck  # type checking
+```