gla-data 0.1.0__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {gla_data-0.1.0 → gla_data-0.1.2}/.github/workflows/publish.yml +11 -1
  2. {gla_data-0.1.0 → gla_data-0.1.2}/.gitignore +6 -0
  3. gla_data-0.1.2/PKG-INFO +109 -0
  4. gla_data-0.1.2/README.md +93 -0
  5. {gla_data-0.1.0 → gla_data-0.1.2}/pyproject.toml +8 -3
  6. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/_schemas.py +4 -4
  7. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/census.py +10 -1
  8. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/cli.py +19 -0
  9. {gla_data-0.1.0 → gla_data-0.1.2}/tests/test_census_dwellings.py +37 -0
  10. gla_data-0.1.2/tests/test_cli.py +19 -0
  11. gla_data-0.1.0/DESIGN.md +0 -283
  12. gla_data-0.1.0/PKG-INFO +0 -43
  13. gla_data-0.1.0/README.md +0 -27
  14. gla_data-0.1.0/tests/test_placeholder.py +0 -6
  15. {gla_data-0.1.0 → gla_data-0.1.2}/.copier-answers.yml +0 -0
  16. {gla_data-0.1.0 → gla_data-0.1.2}/.env.example +0 -0
  17. {gla_data-0.1.0 → gla_data-0.1.2}/.github/workflows/ci.yml +0 -0
  18. {gla_data-0.1.0 → gla_data-0.1.2}/pixi.lock +0 -0
  19. {gla_data-0.1.0 → gla_data-0.1.2}/pixi.toml +0 -0
  20. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/__init__.py +2 -2
  21. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/_arcgis.py +0 -0
  22. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/_cache.py +0 -0
  23. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/_ons.py +0 -0
  24. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/_ukds.py +0 -0
  25. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/catalogue.py +0 -0
  26. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/config.py +0 -0
  27. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/geography.py +0 -0
  28. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/py.typed +0 -0
  29. {gla_data-0.1.0 → gla_data-0.1.2}/src/gla_data/transforms.py +0 -0
  30. {gla_data-0.1.0 → gla_data-0.1.2}/tests/__init__.py +0 -0
  31. {gla_data-0.1.0 → gla_data-0.1.2}/tests/conftest.py +0 -0
  32. {gla_data-0.1.0 → gla_data-0.1.2}/tests/test_aggregate.py +0 -0
  33. {gla_data-0.1.0 → gla_data-0.1.2}/tests/test_boundaries.py +0 -0
  34. {gla_data-0.1.0 → gla_data-0.1.2}/tests/test_catalogue.py +0 -0
  35. {gla_data-0.1.0 → gla_data-0.1.2}/tests/test_crosswalk.py +0 -0
  36. {gla_data-0.1.0 → gla_data-0.1.2}/tests/test_geography_lookup.py +0 -0
  37. {gla_data-0.1.0 → gla_data-0.1.2}/uv.lock +0 -0
@@ -19,9 +19,19 @@ jobs:
19
19
  name: dist
20
20
  path: dist/
21
21
 
22
- publish:
22
+ integration-test:
23
23
  needs: build
24
24
  runs-on: ubuntu-latest
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+ - uses: prefix-dev/setup-pixi@v0.8.3
28
+ with:
29
+ pixi-version: latest
30
+ - run: pixi run test-integration
31
+
32
+ publish:
33
+ needs: [build, integration-test]
34
+ runs-on: ubuntu-latest
25
35
  environment: pypi
26
36
  permissions:
27
37
  id-token: write
@@ -32,3 +32,9 @@ outputs/
32
32
  # OS
33
33
  .DS_Store
34
34
  Thumbs.db
35
+
36
+ # Plan docs and Claude instructions — internal, not part of the shipped project
37
+ PLAN*.md
38
+ WORKPLAN.md
39
+ DESIGN.md
40
+ CLAUDE.md
@@ -0,0 +1,109 @@
1
+ Metadata-Version: 2.4
2
+ Name: gla-data
3
+ Version: 0.1.2
4
+ Summary: Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
5
+ Author-email: Conor Dempsey <conor.dempsey@london.gov.uk>
6
+ License: MIT
7
+ Requires-Python: >=3.12
8
+ Requires-Dist: geopandas>=1.0
9
+ Requires-Dist: pandas>=2.2
10
+ Requires-Dist: pandera>=0.32.1
11
+ Requires-Dist: pyarrow>=24.0.0
12
+ Requires-Dist: pydantic-settings
13
+ Requires-Dist: requests>=2.32
14
+ Requires-Dist: typer>=0.26.8
15
+ Description-Content-Type: text/markdown
16
+
17
+ # gla-data
18
+
19
+ Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
20
+
21
+ ## Installation
22
+
23
+ ### Global install (CLI)
24
+
25
+ To install `gla-data` globally so the `gla-data` CLI is available anywhere in your terminal, use `uv tool`:
26
+
27
+ ```bash
28
+ uv tool install gla-data
29
+ ```
30
+
31
+ Normally `pixi global install` would be the preferred approach, but it does not yet support PyPI packages — so `uv tool` is the right choice for now.
32
+
33
+ If you don't have `uv` installed, you can install it via pixi:
34
+
35
+ ```bash
36
+ pixi global install uv
37
+ ```
38
+
39
+ ### pip
40
+
41
+ ```bash
42
+ pip install gla-data
43
+ ```
44
+
45
+ ### pixi
46
+
47
+ ```bash
48
+ pixi add gla-data
49
+ ```
50
+
51
+ Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
52
+
53
+ ## Usage
54
+
55
+ ### Python API
56
+
57
+ ```python
58
+ import gla_data
59
+
60
+ # Load 2021 LSOA boundaries for London
61
+ lsoa_gdf = gla_data.load_boundaries(geography='lsoa', year=2021)
62
+
63
+ # Load census dwelling counts
64
+ dwellings_2021 = gla_data.load_census_dwellings(year=2021)
65
+ dwellings_2011 = gla_data.load_census_dwellings(year=2011)
66
+
67
+ # Crosswalk 2011 dwelling counts onto 2021 LSOA boundaries
68
+ dwellings_2011_xw = gla_data.crosswalk(
69
+ dwellings_2011,
70
+ from_year=2011,
71
+ to_year=2021,
72
+ value_cols=['dwellings_total'],
73
+ )
74
+
75
+ # Load an OA → LSOA geography lookup
76
+ lookup = gla_data.load_geography_lookup(year=2021, smallest_geography='oa')
77
+
78
+ # Aggregate OA-level data up to LSOA level
79
+ lsoa_totals = gla_data.aggregate(
80
+ df,
81
+ from_geography='oa',
82
+ to_geography='lsoa',
83
+ value_cols=['count'],
84
+ year=2021,
85
+ )
86
+ ```
87
+
88
+ ### CLI
89
+
90
+ The package installs a `gla-data` command. Run `catalogue` to list all available datasets:
91
+
92
+ ```bash
93
+ gla-data catalogue
94
+ ```
95
+
96
+ This prints a table of every dataset, the function used to load it, a description, and its access tier (public/S3).
97
+
98
+ ## Development
99
+
100
+ ```bash
101
+ git clone <repo-url>
102
+ cd gla-data
103
+ pixi install
104
+
105
+ pixi run test # run tests
106
+ pixi run lint # check code style
107
+ pixi run format # auto-format
108
+ pixi run typecheck # type checking
109
+ ```
@@ -0,0 +1,93 @@
1
+ # gla-data
2
+
3
+ Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
4
+
5
+ ## Installation
6
+
7
+ ### Global install (CLI)
8
+
9
+ To install `gla-data` globally so the `gla-data` CLI is available anywhere in your terminal, use `uv tool`:
10
+
11
+ ```bash
12
+ uv tool install gla-data
13
+ ```
14
+
15
+ Normally `pixi global install` would be the preferred approach, but it does not yet support PyPI packages — so `uv tool` is the right choice for now.
16
+
17
+ If you don't have `uv` installed, you can install it via pixi:
18
+
19
+ ```bash
20
+ pixi global install uv
21
+ ```
22
+
23
+ ### pip
24
+
25
+ ```bash
26
+ pip install gla-data
27
+ ```
28
+
29
+ ### pixi
30
+
31
+ ```bash
32
+ pixi add gla-data
33
+ ```
34
+
35
+ Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
36
+
37
+ ## Usage
38
+
39
+ ### Python API
40
+
41
+ ```python
42
+ import gla_data
43
+
44
+ # Load 2021 LSOA boundaries for London
45
+ lsoa_gdf = gla_data.load_boundaries(geography='lsoa', year=2021)
46
+
47
+ # Load census dwelling counts
48
+ dwellings_2021 = gla_data.load_census_dwellings(year=2021)
49
+ dwellings_2011 = gla_data.load_census_dwellings(year=2011)
50
+
51
+ # Crosswalk 2011 dwelling counts onto 2021 LSOA boundaries
52
+ dwellings_2011_xw = gla_data.crosswalk(
53
+ dwellings_2011,
54
+ from_year=2011,
55
+ to_year=2021,
56
+ value_cols=['dwellings_total'],
57
+ )
58
+
59
+ # Load an OA → LSOA geography lookup
60
+ lookup = gla_data.load_geography_lookup(year=2021, smallest_geography='oa')
61
+
62
+ # Aggregate OA-level data up to LSOA level
63
+ lsoa_totals = gla_data.aggregate(
64
+ df,
65
+ from_geography='oa',
66
+ to_geography='lsoa',
67
+ value_cols=['count'],
68
+ year=2021,
69
+ )
70
+ ```
71
+
72
+ ### CLI
73
+
74
+ The package installs a `gla-data` command. Run `catalogue` to list all available datasets:
75
+
76
+ ```bash
77
+ gla-data catalogue
78
+ ```
79
+
80
+ This prints a table of every dataset, the function used to load it, a description, and its access tier (public/S3).
81
+
82
+ ## Development
83
+
84
+ ```bash
85
+ git clone <repo-url>
86
+ cd gla-data
87
+ pixi install
88
+
89
+ pixi run test # run tests
90
+ pixi run lint # check code style
91
+ pixi run format # auto-format
92
+ pixi run typecheck # type checking
93
+ ```
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "gla-data"
3
- version = "0.1.0"
3
+ version = "0.1.2"
4
4
  description = "Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis."
5
5
  readme = "README.md"
6
6
  license = { text = "MIT" }
@@ -40,11 +40,16 @@ select = ["E", "F", "I", "UP", "B"]
40
40
 
41
41
  [tool.mypy]
42
42
  strict = true
43
- packages = ["gla_data"]
43
+ packages = ["gla_data", "tests"]
44
44
  ignore_missing_imports = true
45
45
 
46
+ [[tool.mypy.overrides]]
47
+ module = "tests.*"
48
+ disallow_untyped_defs = false
49
+ disallow_incomplete_defs = false
50
+
46
51
  [tool.bumpversion]
47
- current_version = "0.1.0"
52
+ current_version = "0.1.2"
48
53
  commit = true
49
54
  tag = true
50
55
 
@@ -8,7 +8,7 @@ Schemas are constructed as factory functions rather than class-level constants
8
8
  because the expected columns depend on function parameters (year, geography).
9
9
  """
10
10
 
11
- import pandera as pa
11
+ import pandera.pandas as pa
12
12
 
13
13
  # LAD column year suffix varies by lookup vintage — this is what ONS provides.
14
14
  LAD_YEAR_BY_LOOKUP_YEAR = {2021: "22", 2011: "20"}
@@ -26,7 +26,7 @@ def boundary_schema(geography: str, year: int | None) -> pa.DataFrameSchema:
26
26
  code_col = f"{geo}{yr}CD"
27
27
  name_col = f"{geo}{yr}NM"
28
28
 
29
- return pa.DataFrameSchema( # type: ignore[no-untyped-call]
29
+ return pa.DataFrameSchema(
30
30
  {
31
31
  code_col: pa.Column(str, nullable=False),
32
32
  name_col: pa.Column(str, nullable=False),
@@ -53,13 +53,13 @@ def geography_lookup_schema(year: int, smallest_geography: str) -> pa.DataFrameS
53
53
  cols[f"LAD{lad_yr}CD"] = pa.Column(str, nullable=False)
54
54
  cols[f"LAD{lad_yr}NM"] = pa.Column(str, nullable=False)
55
55
 
56
- return pa.DataFrameSchema(cols, coerce=False) # type: ignore[no-untyped-call]
56
+ return pa.DataFrameSchema(cols, coerce=False)
57
57
 
58
58
 
59
59
  def census_dwellings_schema(year: int) -> pa.DataFrameSchema:
60
60
  """Schema for the output of load_census_dwellings()."""
61
61
  yr = str(year)[2:]
62
- return pa.DataFrameSchema( # type: ignore[no-untyped-call]
62
+ return pa.DataFrameSchema(
63
63
  {
64
64
  f"LSOA{yr}CD": pa.Column(str, nullable=False),
65
65
  "dwellings_total": pa.Column(int, pa.Check.ge(0), nullable=False),
@@ -25,10 +25,19 @@ _EXCEL_KWARGS = {
25
25
 
26
26
 
27
27
  def _fetch_census_dwellings(year: int) -> pd.DataFrame:
28
- """Download and normalise the raw UKDS census dwellings file for the given year."""
28
+ """Download the raw UKDS census dwellings
29
+ file for the given year and normalise it."""
29
30
  kwargs = _EXCEL_KWARGS.get(year, {})
30
31
  raw = _ukds.fetch(f"census_dwellings_{year}", **kwargs)
32
+ return _clean_census_dwellings(raw, year)
31
33
 
34
+
35
+ def _clean_census_dwellings(raw: pd.DataFrame, year: int) -> pd.DataFrame:
36
+ """Select and rename the raw UKDS columns into the public schema for a given year.
37
+
38
+ Pure and deterministic — no I/O — so it can be unit-tested directly with an
39
+ in-memory fixture, without mocking ``_ukds.fetch``.
40
+ """
32
41
  cols = _RAW_COLUMNS[year]
33
42
  code_col = f"LSOA{str(year)[2:]}CD"
34
43
 
@@ -1,5 +1,7 @@
1
1
  """Command-line interface for gla_data."""
2
2
 
3
+ from typing import Annotated
4
+
3
5
  import typer
4
6
  from rich.console import Console
5
7
  from rich.table import Table
@@ -11,6 +13,23 @@ app = typer.Typer(
11
13
  )
12
14
 
13
15
 
16
+ def _version_callback(value: bool) -> None:
17
+ if value:
18
+ typer.echo(gla_data.__version__)
19
+ raise typer.Exit()
20
+
21
+
22
+ @app.callback()
23
+ def main_callback(
24
+ version: Annotated[
25
+ bool,
26
+ typer.Option("--version", "-V", callback=_version_callback, is_eager=True,
27
+ help="Show version and exit."),
28
+ ] = False,
29
+ ) -> None:
30
+ pass
31
+
32
+
14
33
  @app.command()
15
34
  def catalogue() -> None:
16
35
  """List all datasets available in gla_data."""
@@ -5,6 +5,7 @@ import pytest
5
5
 
6
6
  import gla_data
7
7
  import gla_data._cache as cache_module
8
+ from gla_data.census import _clean_census_dwellings
8
9
 
9
10
  # ---------------------------------------------------------------------------
10
11
  # Fixtures
@@ -31,6 +32,42 @@ def fake_dwellings_2011():
31
32
  )
32
33
 
33
34
 
35
+ # ---------------------------------------------------------------------------
36
+ # Pure transform (no I/O, no mocking needed)
37
+ # ---------------------------------------------------------------------------
38
+
39
+
40
+ def test_clean_census_dwellings_2021_selects_and_renames():
41
+ raw = pd.DataFrame(
42
+ {
43
+ "Lower layer Super Output Areas Code": ["E01000001", "E01000002"],
44
+ "Observation": [1079, 1178],
45
+ "Some Other Column": ["x", "y"],
46
+ }
47
+ )
48
+
49
+ result = _clean_census_dwellings(raw, year=2021)
50
+
51
+ assert list(result.columns) == ["LSOA21CD", "dwellings_total"]
52
+ assert result["LSOA21CD"].tolist() == ["E01000001", "E01000002"]
53
+ assert result["dwellings_total"].tolist() == [1079, 1178]
54
+
55
+
56
+ def test_clean_census_dwellings_2011_selects_and_renames():
57
+ raw = pd.DataFrame(
58
+ {
59
+ "GEO_CODE": ["E01000001", "E01000002"],
60
+ "Dwellings : Total\\ Dwellings - Unit : Dwellings": [950, 1050],
61
+ }
62
+ )
63
+
64
+ result = _clean_census_dwellings(raw, year=2011)
65
+
66
+ assert list(result.columns) == ["LSOA11CD", "dwellings_total"]
67
+ assert result["LSOA11CD"].tolist() == ["E01000001", "E01000002"]
68
+ assert result["dwellings_total"].tolist() == [950, 1050]
69
+
70
+
34
71
  # ---------------------------------------------------------------------------
35
72
  # Validation (no network calls)
36
73
  # ---------------------------------------------------------------------------
@@ -0,0 +1,19 @@
1
+ """Tests for the gla_data command-line interface."""
2
+
3
+ from typer.testing import CliRunner
4
+
5
+ from gla_data.cli import app
6
+
7
+ runner = CliRunner()
8
+
9
+
10
+ def test_catalogue_command_exits_cleanly():
11
+ result = runner.invoke(app, ["catalogue"])
12
+ assert result.exit_code == 0
13
+
14
+
15
+ def test_catalogue_command_lists_implemented_datasets():
16
+ result = runner.invoke(app, ["catalogue"])
17
+ assert "Census Dwellings" in result.output
18
+ assert "load_census_dwel" in result.output
19
+ assert "ONS Boundaries" in result.output
gla_data-0.1.0/DESIGN.md DELETED
@@ -1,283 +0,0 @@
1
- # gla-data design
2
-
3
- ## Purpose
4
-
5
- `gla-data` is a shared Python package for the GLA City Intelligence Data Science team. It provides clean, consistent access to the GLA-relevant datasets used across City Data analysis projects.
6
-
7
- The primary goal is to reduce the cost of getting started with a new dataset. A user who knows a dataset exists but has not yet used it — because the API is unfamiliar, credentials are unclear, or the raw data requires significant cleaning — should be able to load analysis-ready data in a few lines of code.
8
-
9
- ## Who this is for
10
-
11
- Data scientists on the City Intelligence team working on GLA analysis projects. Users are assumed to have some familiarity with the datasets, but `gla-data` should make it easy to discover what is available and how to use it, including:
12
-
13
- - What credentials or sign-ups are required to access a dataset
14
- - What the default filters are and why
15
- - What parameters are available to adjust those defaults
16
-
17
- ## What this package does
18
-
19
- - Loads GLA-relevant datasets from public sources and (in future) from the GLA S3 data lake
20
- - Returns data filtered to sensible defaults (typically London, latest available year)
21
- - Provides standard geography transformations: aggregation, disaggregation, and year crosswalks
22
- - Documents required credentials and setup steps for each data source in the README; functions raise clear errors with a pointer to the README if credentials are missing
23
-
24
- ## What this package does not do
25
-
26
- - Project-specific cleaning, transformation, or modelling — these belong in project code
27
- - Cross-dataset merging — users join datasets themselves; `gla-data` provides the building blocks
28
- - Exploratory analysis or visualisation — may be revisited once clear patterns emerge across projects
29
-
30
- ---
31
-
32
- ## Design principles
33
-
34
- **Defaults should be sensible and explicit.** Every parameter with a default should have a reason for that default, and the docstring should state it. A user should be able to call any function with no arguments and get something useful.
35
-
36
- **Return types are consistent.** All load functions return a `pd.DataFrame`. Functions that return geographic data return a `gpd.GeoDataFrame`. Column naming follows ONS conventions where they exist (e.g. `LSOA21CD`, `LAD22NM`).
37
-
38
- **Functions are stateless.** Call a function, get data back. No objects to manage, no state to track between calls.
39
-
40
- **Private functions do the work, public functions are the interface.** HTTP calls, API authentication, raw data parsing, and column renaming all happen in private functions prefixed with `_`. Public functions are thin and readable.
41
-
42
- **Source switching is per-function.** Each function accepts a `source` parameter defaulting to `"public"`. As datasets become available on the GLA S3 data lake, `source="s3"` is added to that function. Functions that do not yet support S3 raise a clear error if `source="s3"` is passed.
43
-
44
- ---
45
-
46
- ## Data access tiers
47
-
48
- Datasets differ in what setup is required. Each function's docstring and the catalogue state which tier it belongs to. Setup instructions for datasets requiring credentials are in the README. Functions that require credentials raise a clear, descriptive error if the relevant environment variable is missing — for example:
49
-
50
- ```
51
- EnvironmentError: EPC_API_KEY not set. This dataset requires a free DLUHC account.
52
- See the README for setup instructions.
53
- ```
54
-
55
- | Tier | Description | Examples |
56
- |---|---|---|
57
- | `public` | No credentials required. URLs are embedded in the package. | ONS boundaries, census data via Nomis |
58
- | `registered` | Requires a free account or API key. Set via environment variable. | EPC (DLUHC portal), Nomis bulk API key |
59
- | `licensed` | Requires a government or commercial licence. | OS AddressBase |
60
- | `s3` | Requires access to the GLA internal data lake (provisioned by GLA engineering). | TBD |
61
-
62
- ---
63
-
64
- ## API conventions
65
-
66
- ### Naming
67
-
68
- - Data loading functions: `load_<dataset>` or `load_boundaries(geography=...)`
69
- - Geography transformation functions: `aggregate(...)`, `crosswalk(...)`
70
- - Discovery functions: `catalogue()`
71
-
72
- Parameters describe what the user wants, not how the code fetches it.
73
-
74
- ### Standard parameters
75
-
76
- | Parameter | Type | Notes |
77
- |---|---|---|
78
- | `geography` | `str` | Geographic unit: `"oa"`, `"lsoa"`, `"msoa"`, `"ward"`, `"borough"`, `"london"` |
79
- | `year` | `int` | Reference year. Valid values are documented per function |
80
- | `source` | `Literal["public", "s3"]` | Data source. Defaults to `"public"` |
81
-
82
- ### Return types
83
-
84
- | Function type | Return type | Notes |
85
- |---|---|---|
86
- | Tabular data | `pd.DataFrame` | Index is always a plain integer range index |
87
- | Geographic data | `gpd.GeoDataFrame` | CRS is always British National Grid (EPSG:27700) |
88
-
89
- ### Column naming
90
-
91
- - Geography codes follow ONS conventions: `LSOA21CD`, `LAD22NM`, `OA21CD` etc.
92
- - The year in a geography code column name reflects the geography vintage, not the data year
93
- - All column names are uppercase with underscores
94
-
95
- ---
96
-
97
- ## Public API
98
-
99
- ### Discovery
100
-
101
- ```python
102
- gla_data.catalogue()
103
- ```
104
-
105
- Returns a `pd.DataFrame` listing all available datasets with columns: `dataset`, `description`, `access_tier`, `status`, `source_supported`. For datasets with `access_tier` other than `public`, the catalogue notes that setup is required — see the README for instructions.
106
-
107
- ---
108
-
109
- ### Boundaries
110
-
111
- ```python
112
- gdf = gla_data.load_boundaries(geography="lsoa", year=2021)
113
- ```
114
-
115
- Returns boundary polygons for London at the specified geography and year.
116
-
117
- **Parameters:**
118
- - `geography`: `"oa"`, `"lsoa"`, `"msoa"`, `"ward"`, `"borough"`, `"london"`
119
- - `year`: valid values depend on geography (e.g. LSOAs: `2011`, `2021`; borough boundaries do not have a year)
120
- - `source`: `"public"` only for now
121
-
122
- **Returns:** `gpd.GeoDataFrame` with columns:
123
- - Geography code and name (e.g. `LSOA21CD`, `LSOA21NM`)
124
- - `geometry` — polygon in EPSG:27700
125
-
126
- **Access tier:** `public`
127
-
128
- ---
129
-
130
- ### Geography lookup
131
-
132
- ```python
133
- df = gla_data.load_geography_lookup(year=2021, smallest_geography="lsoa")
134
- ```
135
-
136
- Returns the ONS lookup table mapping London LSOAs (or OAs) to higher geographies (MSOA, borough).
137
-
138
- **Parameters:**
139
- - `year`: `2011` or `2021`
140
- - `smallest_geography`: `"lsoa"` (default, one row per LSOA) or `"oa"` (one row per OA, includes OA code column)
141
- - `source`: `"public"` only for now
142
- - `cache`: if `True` (default), result is cached to `~/.cache/gla_data/`
143
-
144
- **Returns:** `pd.DataFrame`. Columns for 2021 (lsoa grain):
145
- `LSOA21CD`, `LSOA21NM`, `MSOA21CD`, `MSOA21NM`, `LAD22CD`, `LAD22NM`.
146
- For 2011: `LSOA11CD`, `LSOA11NM`, `MSOA11CD`, `MSOA11NM`, `LAD20CD`, `LAD20NM`.
147
- Note: the 2011 table uses 2020 LAD boundaries — this is what ONS provides at this grain.
148
-
149
- **Access tier:** `public`
150
-
151
- **Notes:**
152
- - Source tables are at OA grain. For `smallest_geography="lsoa"`, deduplication is applied.
153
- - Fetching is slow (~17s). A `UserWarning` is raised on first call; result is cached to disk by default.
154
-
155
- ---
156
-
157
- ### Census dwellings
158
-
159
- ```python
160
- df = gla_data.load_census_dwellings(year=2021, geography="lsoa")
161
- ```
162
-
163
- Returns dwelling counts for London at the specified geography and census year.
164
-
165
- **Parameters:**
166
- - `year`: `2011` or `2021`
167
- - `geography`: `"oa"`, `"lsoa"`, `"msoa"`, `"borough"`, `"london"`
168
- - `source`: `"public"` only for now
169
-
170
- **Returns:** `pd.DataFrame` with a geography code column (e.g. `LSOA21CD`) and:
171
- - `dwellings_total`
172
- - Dwelling type breakdown: `dwellings_detached`, `dwellings_semi_detached`, `dwellings_terraced`, `dwellings_flat`, `dwellings_other`
173
- - Household space type breakdown: `spaces_total`, `spaces_occupied`, `spaces_vacant`
174
-
175
- **Access tier:** `public`
176
-
177
- **Notes:**
178
- - 2011 data uses 2011 LSOA boundaries; 2021 data uses 2021 LSOA boundaries. Use `crosswalk()` if you need both years on consistent boundaries.
179
- - When `geography` is above LSOA, data is aggregated via the standard ONS lookup.
180
-
181
- ---
182
-
183
- ### EPC
184
-
185
- ```python
186
- df = gla_data.load_epc(geography="lsoa", year=2023)
187
- ```
188
-
189
- Returns Energy Performance Certificate data for London.
190
-
191
- **Access tier:** `registered` — requires a free account at the DLUHC EPC portal. Run `gla_data.setup_instructions(dataset="epc")` for details.
192
-
193
- *Further design TBD once initial datasets are implemented.*
194
-
195
- ---
196
-
197
- ### Geography transformation
198
-
199
- #### Aggregation
200
-
201
- ```python
202
- df_msoa = gla_data.aggregate(
203
- df,
204
- from_geography="lsoa",
205
- to_geography="msoa",
206
- value_cols=["dwellings_total", "dwellings_vacant"],
207
- method="sum",
208
- year=2021,
209
- )
210
- ```
211
-
212
- Aggregates a DataFrame from a lower to a higher geography using the standard ONS lookup. The `year` parameter specifies which vintage of lookup to use.
213
-
214
- **Parameters:**
215
- - `df`: input DataFrame containing a geography code column (auto-detected by name)
216
- - `from_geography`: source geography
217
- - `to_geography`: target geography (must be higher than `from_geography`)
218
- - `value_cols`: columns to aggregate
219
- - `method`: `"sum"` or `"mean"`
220
- - `year`: lookup vintage
221
-
222
- #### Crosswalk (same geography, different year)
223
-
224
- ```python
225
- df_2021 = gla_data.crosswalk(
226
- df,
227
- geography="lsoa",
228
- from_year=2011,
229
- to_year=2021,
230
- value_cols=["dwellings_total"],
231
- method="proportional",
232
- )
233
- ```
234
-
235
- Converts a DataFrame from one year's geography boundaries to another using the ONS exact-fit or best-fit lookup.
236
-
237
- **Parameters:**
238
- - `df`: input DataFrame
239
- - `geography`: the geographic unit
240
- - `from_year`, `to_year`: boundary vintages
241
- - `value_cols`: numeric columns to convert
242
- - `method`: `"proportional"` (split values by area overlap) or `"exact_fit"` (for cases where boundaries nest cleanly)
243
-
244
- **Notes:**
245
- - Disaggregation (higher to lower geography) is not supported in the first version. The `aggregate` function is one-directional.
246
- - Disaggregation requires assumptions about how values are distributed within a higher geography. If this is needed, it should be designed explicitly with a stated method (e.g. uniform distribution, population-weighted).
247
-
248
- ---
249
-
250
- ## Package structure
251
-
252
- ```
253
- src/gla_data/
254
- ├── __init__.py ← public API: re-exports all public functions
255
- ├── py.typed
256
- ├── config.py ← environment variable handling (S3 config etc.)
257
- ├── _arcgis.py ← private: generic ArcGIS FeatureServer HTTP/pagination
258
- ├── _ons.py ← private: ONS service constants and fetch functions
259
- ├── _ukds.py ← private: UKDS dataset registry and static file downloads
260
- ├── _cache.py ← private: shared disk cache (~/.cache/gla_data/)
261
- ├── geography.py ← load_boundaries(), load_geography_lookup()
262
- ├── transforms.py ← crosswalk() [aggregate() to be added]
263
- ├── catalogue.py ← catalogue() [not yet implemented]
264
- ├── census.py ← load_census_dwellings()
265
- ├── epc.py ← load_epc() [not yet implemented]
266
- ```
267
-
268
- ### Notes on the private layer
269
-
270
- `_arcgis.py` contains generic ArcGIS FeatureServer pagination logic (no ONS-specific
271
- knowledge). `_ons.py` holds ONS service constants and the fetch functions that use them.
272
- When new data sources are added (e.g. Nomis for census, DLUHC for EPC), they get their
273
- own private module and introduce new HTTP helpers only if needed.
274
-
275
- ---
276
-
277
- ## Open questions
278
-
279
- - `aggregate` and `crosswalk` are generic functions (not per-geography-pair). They auto-detect the geography code column by name — open question: what happens if detection is ambiguous?
280
- - What is the right disaggregation method for numeric quantities (MSOA → LSOA)? Uniform? Population-weighted? Defer until there is a concrete use case.
281
- - Should `load_boundaries` accept `year=None` for geographies that do not have vintages (e.g. borough boundaries)?
282
- - Where does Land Registry data fit — access tier, likely `public` (price paid) or `registered` (title register)?
283
- - OS AddressBase: confirm whether the GLA licence covers use in this package or whether each team member needs their own licence.
gla_data-0.1.0/PKG-INFO DELETED
@@ -1,43 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: gla-data
3
- Version: 0.1.0
4
- Summary: Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
5
- Author-email: Conor Dempsey <conor.dempsey@london.gov.uk>
6
- License: MIT
7
- Requires-Python: >=3.12
8
- Requires-Dist: geopandas>=1.0
9
- Requires-Dist: pandas>=2.2
10
- Requires-Dist: pandera>=0.32.1
11
- Requires-Dist: pyarrow>=24.0.0
12
- Requires-Dist: pydantic-settings
13
- Requires-Dist: requests>=2.32
14
- Requires-Dist: typer>=0.26.8
15
- Description-Content-Type: text/markdown
16
-
17
- # gla-data
18
-
19
- Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
20
-
21
- ## Installation
22
-
23
- Add `gla-data` as a PyPI dependency in your `pixi.toml`:
24
-
25
- ```toml
26
- [pypi-dependencies]
27
- gla-data = ">=0.1"
28
- ```
29
-
30
- Then run `pixi install`. Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
31
-
32
- ## Development
33
-
34
- ```bash
35
- git clone <repo-url>
36
- cd gla-data
37
- pixi install
38
-
39
- pixi run test # run tests
40
- pixi run lint # check code style
41
- pixi run format # auto-format
42
- pixi run typecheck # type checking
43
- ```
gla_data-0.1.0/README.md DELETED
@@ -1,27 +0,0 @@
1
- # gla-data
2
-
3
- Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
4
-
5
- ## Installation
6
-
7
- Add `gla-data` as a PyPI dependency in your `pixi.toml`:
8
-
9
- ```toml
10
- [pypi-dependencies]
11
- gla-data = ">=0.1"
12
- ```
13
-
14
- Then run `pixi install`. Pixi pulls geospatial dependencies (geopandas, pyproj, etc.) from conda-forge, which provides reliable pre-built binaries across all platforms and Python versions.
15
-
16
- ## Development
17
-
18
- ```bash
19
- git clone <repo-url>
20
- cd gla-data
21
- pixi install
22
-
23
- pixi run test # run tests
24
- pixi run lint # check code style
25
- pixi run format # auto-format
26
- pixi run typecheck # type checking
27
- ```
@@ -1,6 +0,0 @@
1
- # Replace this file with real tests as you add functions to the package.
2
- import gla_data
3
-
4
-
5
- def test_package_importable():
6
- assert gla_data.__version__ is not None
File without changes
File without changes
File without changes
File without changes
@@ -8,9 +8,9 @@ from gla_data.transforms import aggregate, crosswalk
8
8
  __version__ = version("gla-data")
9
9
  __all__ = [
10
10
  "load_boundaries",
11
- "crosswalk",
12
- "aggregate",
13
11
  "load_geography_lookup",
14
12
  "load_census_dwellings",
13
+ "crosswalk",
14
+ "aggregate",
15
15
  "catalogue",
16
16
  ]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes