gla-data 0.1.3__tar.gz → 0.2.2__tar.gz

gla_data-0.2.2/.env.example

@@ -0,0 +1,43 @@
+# Copy this file to .env and fill in values. Never commit .env.
+# Settings are loaded from this file automatically by config.py.
+
+# AWS (if used)
+# AWS_DEFAULT_REGION=eu-west-2
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+
+# Application settings — override here for local development if needed.
+# {{ import_name | upper }}_DATA_DIR=data
+# {{ import_name | upper }}_OUTPUT_DIR=outputs
+
+# --- GLA GIS team Postgres/PostGIS database (shared across many datasets) ---
+# One shared connection is used for every dataset the GIS team hosts in Postgres.
+GLA_DATA_GIS_DB_HOST=
+GLA_DATA_GIS_DB_PORT=5432
+GLA_DATA_GIS_DB_NAME=
+GLA_DATA_GIS_DB_USER=
+GLA_DATA_GIS_DB_PASSWORD=
+# Optional: postgres schema (namespace) the tables live in. Leave blank for "public".
+GLA_DATA_GIS_DB_SCHEMA=
+# Table names for the BLPU and classification tables (defaults shown match
+# AddressBase Premium's own GeoPackage layer names — override if different).
+GLA_DATA_ADDRESSBASE_BLPU_TABLE=blpu
+GLA_DATA_ADDRESSBASE_CLASSIFICATION_TABLE=classification
+
+# GIS team "gisapdata" schema (within the same shared GIS DB above) — Highstreets,
+# Town Centres, BIDs, CAZ, Boroughs, TfL Hexes, LSOA/MSOA 2011+2021, GLA boundary,
+# London parks. Table names are GLA-internal — deliberately left blank here
+# (not committed to git). Fill in your own .env with the real table names.
+GLA_DATA_GISAPDATA_SCHEMA=gisapdata
+GLA_DATA_GISAPDATA_BOROUGH_TABLE=
+GLA_DATA_GISAPDATA_LSOA_2011_TABLE=
+GLA_DATA_GISAPDATA_MSOA_2011_TABLE=
+GLA_DATA_GISAPDATA_LSOA_2021_TABLE=
+GLA_DATA_GISAPDATA_MSOA_2021_TABLE=
+GLA_DATA_GISAPDATA_GLA_BOUNDARY_TABLE=
+GLA_DATA_GISAPDATA_HIGH_STREETS_TABLE=
+GLA_DATA_GISAPDATA_TOWN_CENTRES_TABLE=
+GLA_DATA_GISAPDATA_BIDS_TABLE=
+GLA_DATA_GISAPDATA_CAZ_TABLE=
+GLA_DATA_GISAPDATA_TFL_HEX_TABLE=
+GLA_DATA_GISAPDATA_PARKS_TABLE=

{gla_data-0.1.3 → gla_data-0.2.2}/.gitignore

@@ -22,6 +22,9 @@ htmlcov/
 .idea/
 *.swp
 
+# Jupyter
+.ipynb_checkpoints/
+
 # Secrets
 .env
 

{gla_data-0.1.3 → gla_data-0.2.2}/PKG-INFO

@@ -1,16 +1,18 @@
 Metadata-Version: 2.4
 Name: gla-data
-Version: 0.1.3
+Version: 0.2.2
 Summary: Shared tools for loading, transforming, and visualising the GLA datasets used in City Data analysis.
 Author-email: Conor Dempsey <conor.dempsey@london.gov.uk>
 License: MIT
 Requires-Python: >=3.12
+Requires-Dist: beautifulsoup4>=4.12
 Requires-Dist: geopandas>=1.0
 Requires-Dist: pandas>=2.2
 Requires-Dist: pandera>=0.32.1
 Requires-Dist: pyarrow>=24.0.0
 Requires-Dist: pydantic-settings
 Requires-Dist: requests>=2.32
+Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: typer>=0.26.8
 Description-Content-Type: text/markdown
 
@@ -20,21 +22,15 @@ Shared tools for loading, transforming, and visualising the GLA datasets used in
 
 ## Installation
 
-### Global install (CLI)
+### CLI
 
-To install `gla-data` globally so the `gla-data` CLI is available anywhere in your terminal, use `uv tool`:
+To install `gla-data` globally so the `gla-data` CLI is available anywhere in your terminal:
 
 ```bash
 uv tool install gla-data
 ```
 
-Normally `pixi global install` would be the preferred approach, but it does not yet support PyPI packages — so `uv tool` is the right choice for now.
-
-If you don't have `uv` installed, you can install it via pixi:
-
-```bash
-pixi global install uv
-```
+This requires [uv](https://docs.astral.sh/uv/) to be installed.
 
 ### pip
 

{gla_data-0.1.3 → gla_data-0.2.2}/README.md

@@ -4,21 +4,15 @@ Shared tools for loading, transforming, and visualising the GLA datasets used in
 
 ## Installation
 
-### Global install (CLI)
+### CLI
 
-To install `gla-data` globally so the `gla-data` CLI is available anywhere in your terminal, use `uv tool`:
+To install `gla-data` globally so the `gla-data` CLI is available anywhere in your terminal:
 
 ```bash
 uv tool install gla-data
 ```
 
-Normally `pixi global install` would be the preferred approach, but it does not yet support PyPI packages — so `uv tool` is the right choice for now.
-
-If you don't have `uv` installed, you can install it via pixi:
-
-```bash
-pixi global install uv
-```
+This requires [uv](https://docs.astral.sh/uv/) to be installed.
 
 ### pip
 

gla_data-0.2.2/notebooks/addressbase_postgres_setup.ipynb

@@ -0,0 +1,182 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cell-00",
+   "metadata": {},
+   "source": [
+    "# AddressBase Premium \u2014 Postgres connection setup & schema reflection\n",
+    "\n",
+    "Run this once you've populated `.env` with the `GLA_DATA_GIS_DB_*` variables (see `.env.example` in the repo root).\n",
+    "\n",
+    "This notebook does three things, in order:\n",
+    "\n",
+    "1. **Connect and reflect** the real table/column schema directly from the database \u2014 a one-off discovery step, since the exact table/column names in this database haven't been confirmed yet.\n",
+    "2. **Export** the reflected schema to a file, so it can be read in a follow-up task to write a proper pandera schema in code (rather than guessing column names).\n",
+    "3. **Sanity-check** `gla_data.load_addressbase(source='postgres')` against the real data.\n",
+    "\n",
+    "Nothing here is destructive \u2014 every step is a read-only query."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-01",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gla_data\n",
+    "from gla_data import _postgres\n",
+    "from gla_data.config import settings\n",
+    "\n",
+    "print(\"Configured settings (password hidden):\")\n",
+    "print(f\"  host:   {settings.gis_db_host}\")\n",
+    "print(f\"  port:   {settings.gis_db_port}\")\n",
+    "print(f\"  dbname: {settings.gis_db_name}\")\n",
+    "print(f\"  user:   {settings.gis_db_user}\")\n",
+    "print(f\"  schema: {settings.gis_db_schema or '(default search path)'}\")\n",
+    "\n",
+    "if not settings.gis_db_host:\n",
+    "    raise RuntimeError(\n",
+    "        \"GLA_DATA_GIS_DB_* settings not found \u2014 check your \"\n",
+    "        \".env file is in the working directory this notebook runs from, \"\n",
+    "        \"and matches .env.example's variable names.\"\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell-02",
+   "metadata": {},
+   "source": [
+    "## 1. Connect and reflect the schema\n",
+    "\n",
+    "Lists every table and column gla_data's configured database user can see."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-03",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "engine = _postgres.get_engine()\n",
+    "schema_df = _postgres.reflect_schema(engine, schema=settings.gis_db_schema)\n",
+    "\n",
+    "print(f\"{schema_df['table_name'].nunique()} tables, {len(schema_df)} columns total\")\n",
+    "schema_df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell-04",
+   "metadata": {},
+   "source": [
+    "### Tables of interest\n",
+    "\n",
+    "Filtered to the two tables `compute_uprn_net_change` expects (BLPU-equivalent + classification-equivalent), using the configured table names as a starting guess. **Look at the full table list above first** \u2014 if the real tables are named differently, update `GLA_DATA_ADDRESSBASE_BLPU_TABLE` / `GLA_DATA_ADDRESSBASE_CLASSIFICATION_TABLE` in `.env` (or just change `tables_of_interest` below for this exploratory pass)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tables_of_interest = [\n",
+    "    settings.addressbase_blpu_table,\n",
+    "    settings.addressbase_classification_table,\n",
+    "]\n",
+    "schema_df[schema_df[\"table_name\"].isin(tables_of_interest)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell-06",
+   "metadata": {},
+   "source": [
+    "## 2. Export the reflected schema\n",
+    "\n",
+    "Saved next to this notebook. This file is what gets read in the next task to write `compute_uprn_net_change`'s real pandera input schema \u2014 don't rename it without updating that task."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "export_path = Path(\"addressbase_schema_reflection.json\")\n",
+    "schema_df.to_json(export_path, orient=\"records\", indent=2)\n",
+    "print(f\"Exported {len(schema_df)} rows to {export_path.resolve()}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell-08",
+   "metadata": {},
+   "source": [
+    "## 3. Sanity-check `load_addressbase`\n",
+    "\n",
+    "Reads the configured BLPU table as-is (no column mapping yet \u2014 see the function's docstring). Confirms the connection, driver, and geometry handling all work end-to-end against the real database, independent of the schema-reflection step above."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-09",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raw = gla_data.load_addressbase(source=\"postgres\")\n",
+    "print(type(raw))\n",
+    "print(f\"{len(raw)} rows\")\n",
+    "print(f\"columns: {list(raw.columns)}\")\n",
+    "raw.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cell-10",
+   "metadata": {},
+   "source": [
+    "### Optional: the classification table\n",
+    "\n",
+    "Same check against the second table `compute_uprn_net_change` will eventually need (the residential-classification filter)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cell-11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "classification_raw = gla_data.load_addressbase(\n",
+    "    source=\"postgres\", table=settings.addressbase_classification_table\n",
+    ")\n",
+    "print(f\"{len(classification_raw)} rows\")\n",
+    "print(f\"columns: {list(classification_raw.columns)}\")\n",
+    "classification_raw.head()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (gla-data)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}