crossroads-uk 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. crossroads_uk-1.0.0/.gitattributes +2 -0
  2. crossroads_uk-1.0.0/.github/workflows/publish.yml +78 -0
  3. crossroads_uk-1.0.0/.github/workflows/tests.yml +55 -0
  4. crossroads_uk-1.0.0/.gitignore +36 -0
  5. crossroads_uk-1.0.0/CHANGELOG.md +59 -0
  6. crossroads_uk-1.0.0/CITATION.cff +23 -0
  7. crossroads_uk-1.0.0/LICENSE +21 -0
  8. crossroads_uk-1.0.0/PKG-INFO +279 -0
  9. crossroads_uk-1.0.0/README.md +229 -0
  10. crossroads_uk-1.0.0/docs/ai-disclosure.md +16 -0
  11. crossroads_uk-1.0.0/docs/assets/logo.png +0 -0
  12. crossroads_uk-1.0.0/docs/data-sources.md +104 -0
  13. crossroads_uk-1.0.0/docs/maintenance/pypi-release.md +33 -0
  14. crossroads_uk-1.0.0/docs/maintenance/updating-ons-boundaries.md +148 -0
  15. crossroads_uk-1.0.0/docs/methodology.md +104 -0
  16. crossroads_uk-1.0.0/docs/plans/001_project_foundation/00-overview.md +106 -0
  17. crossroads_uk-1.0.0/docs/plans/001_project_foundation/01-package-skeleton.md +186 -0
  18. crossroads_uk-1.0.0/docs/plans/001_project_foundation/02-transformer-registry.md +305 -0
  19. crossroads_uk-1.0.0/docs/plans/001_project_foundation/03-orchestrator-harness.md +173 -0
  20. crossroads_uk-1.0.0/docs/plans/002_quality_engine/00-overview.md +206 -0
  21. crossroads_uk-1.0.0/docs/plans/002_quality_engine/01-layered-schema-helpers.md +278 -0
  22. crossroads_uk-1.0.0/docs/plans/002_quality_engine/02-ledger-quarantine.md +298 -0
  23. crossroads_uk-1.0.0/docs/plans/002_quality_engine/03-invariants-build-integration.md +812 -0
  24. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/00-overview.md +235 -0
  25. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/01-spatial-extension.md +107 -0
  26. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/02-boundary-ingestion.md +480 -0
  27. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/03-temporal-slicing.md +343 -0
  28. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/04-spatial-indices.md +174 -0
  29. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/ONS_boundary_vintages/00-overview.md +215 -0
  30. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/ONS_boundary_vintages/01-manifest-refactor.md +213 -0
  31. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/ONS_boundary_vintages/02-populate-vintages.md +223 -0
  32. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/ONS_boundary_vintages/03-maintenance-script.md +395 -0
  33. crossroads_uk-1.0.0/docs/plans/003_spatial_infrastructure/ONS_boundary_vintages/04-update-runbook.md +179 -0
  34. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/00-overview.md +477 -0
  35. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/01-download-bronze.md +577 -0
  36. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/02-collision-silver.md +254 -0
  37. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/03-vehicle-casualty-linkage.md +189 -0
  38. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/04-spatial-join.md +351 -0
  39. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/05-reference-data.md +539 -0
  40. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/06-keep-in-place-silver.md +415 -0
  41. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/07-core-severity-audit.md +286 -0
  42. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/08-completeness-report.md +190 -0
  43. crossroads_uk-1.0.0/docs/plans/004_stats19_ingestion/09-labelled-views.md +275 -0
  44. crossroads_uk-1.0.0/docs/plans/005_console_architecture/00-overview.md +166 -0
  45. crossroads_uk-1.0.0/docs/plans/005_console_architecture/01-wizard-prompts.md +293 -0
  46. crossroads_uk-1.0.0/docs/plans/005_console_architecture/02-build-wiring-entrypoint.md +331 -0
  47. crossroads_uk-1.0.0/docs/plans/005_console_architecture/03-dataset-selection.md +512 -0
  48. crossroads_uk-1.0.0/docs/plans/006_weather_integration/00-overview.md +130 -0
  49. crossroads_uk-1.0.0/docs/plans/006_weather_integration/01-dependency-resolution.md +266 -0
  50. crossroads_uk-1.0.0/docs/plans/006_weather_integration/02-weather-grid-source.md +421 -0
  51. crossroads_uk-1.0.0/docs/plans/006_weather_integration/03-stats19-weather-stamping.md +312 -0
  52. crossroads_uk-1.0.0/docs/plans/007_release_prep/00-overview.md +124 -0
  53. crossroads_uk-1.0.0/docs/plans/007_release_prep/01-versioning.md +238 -0
  54. crossroads_uk-1.0.0/docs/plans/007_release_prep/02-data-licensing.md +172 -0
  55. crossroads_uk-1.0.0/docs/plans/007_release_prep/03-user-facing-docs.md +293 -0
  56. crossroads_uk-1.0.0/docs/plans/007_release_prep/04-reference-fixes-hygiene.md +298 -0
  57. crossroads_uk-1.0.0/docs/plans/007_release_prep/05-source-comment-cleanup.md +135 -0
  58. crossroads_uk-1.0.0/docs/plans/007_release_prep/06-ci-workflow.md +135 -0
  59. crossroads_uk-1.0.0/docs/plans/007_release_prep/07-db-provenance.md +191 -0
  60. crossroads_uk-1.0.0/docs/plans/007_release_prep/08-schema-dictionary.md +266 -0
  61. crossroads_uk-1.0.0/docs/plans/008_weather_chunking/00-weather-chunking.md +647 -0
  62. crossroads_uk-1.0.0/docs/plans/009_solar_geometry/00-solar-geometry.md +582 -0
  63. crossroads_uk-1.0.0/docs/plans/010_weather_credentials/00-cds-key-prompt.md +535 -0
  64. crossroads_uk-1.0.0/docs/plans/011_bank_holidays/00-overview.md +189 -0
  65. crossroads_uk-1.0.0/docs/plans/011_bank_holidays/01-bank-holidays-source.md +364 -0
  66. crossroads_uk-1.0.0/docs/plans/011_bank_holidays/02-collision-stamp.md +308 -0
  67. crossroads_uk-1.0.0/docs/plans/012_aadf_traffic/00-overview.md +140 -0
  68. crossroads_uk-1.0.0/docs/plans/012_aadf_traffic/01-aadf-source.md +393 -0
  69. crossroads_uk-1.0.0/docs/plans/012_aadf_traffic/02-risk-showcase-test.md +172 -0
  70. crossroads_uk-1.0.0/docs/plans/012_aadf_traffic/03-wizard-temporal-warning.md +197 -0
  71. crossroads_uk-1.0.0/docs/plans/012_aadf_traffic/04-docs-readme.md +160 -0
  72. crossroads_uk-1.0.0/docs/plans/012_aadf_traffic/showcase-output.txt +68 -0
  73. crossroads_uk-1.0.0/docs/plans/013_pypi_publishing/00-overview.md +130 -0
  74. crossroads_uk-1.0.0/docs/plans/013_pypi_publishing/01-version-bump.md +154 -0
  75. crossroads_uk-1.0.0/docs/plans/013_pypi_publishing/02-packaging-validation.md +184 -0
  76. crossroads_uk-1.0.0/docs/plans/013_pypi_publishing/03-publish-workflow.md +222 -0
  77. crossroads_uk-1.0.0/docs/plans/013_pypi_publishing/04-release-runbook.md +162 -0
  78. crossroads_uk-1.0.0/docs/schema.md +477 -0
  79. crossroads_uk-1.0.0/docs/spec.md +276 -0
  80. crossroads_uk-1.0.0/pyproject.toml +77 -0
  81. crossroads_uk-1.0.0/scripts/build_stats19_codebook.py +133 -0
  82. crossroads_uk-1.0.0/scripts/build_stats19_column_manifest.py +149 -0
  83. crossroads_uk-1.0.0/scripts/build_weather_fixture.py +171 -0
  84. crossroads_uk-1.0.0/scripts/update_ons_boundaries.py +227 -0
  85. crossroads_uk-1.0.0/src/crossroads/__init__.py +25 -0
  86. crossroads_uk-1.0.0/src/crossroads/client.py +81 -0
  87. crossroads_uk-1.0.0/src/crossroads/console.py +448 -0
  88. crossroads_uk-1.0.0/src/crossroads/net.py +50 -0
  89. crossroads_uk-1.0.0/src/crossroads/quality.py +626 -0
  90. crossroads_uk-1.0.0/src/crossroads/reference/README.md +119 -0
  91. crossroads_uk-1.0.0/src/crossroads/reference/ons_boundaries.json +32 -0
  92. crossroads_uk-1.0.0/src/crossroads/reference/stats19_codebook.csv +1131 -0
  93. crossroads_uk-1.0.0/src/crossroads/reference/stats19_columns.csv +100 -0
  94. crossroads_uk-1.0.0/src/crossroads/registry.py +142 -0
  95. crossroads_uk-1.0.0/src/crossroads/sql.py +25 -0
  96. crossroads_uk-1.0.0/src/crossroads/transformers/__init__.py +0 -0
  97. crossroads_uk-1.0.0/src/crossroads/transformers/aadf.py +314 -0
  98. crossroads_uk-1.0.0/src/crossroads/transformers/bank_holidays.py +123 -0
  99. crossroads_uk-1.0.0/src/crossroads/transformers/base.py +92 -0
  100. crossroads_uk-1.0.0/src/crossroads/transformers/spatial.py +341 -0
  101. crossroads_uk-1.0.0/src/crossroads/transformers/stats19.py +1086 -0
  102. crossroads_uk-1.0.0/src/crossroads/transformers/weather.py +385 -0
  103. crossroads_uk-1.0.0/tests/conftest.py +20 -0
  104. crossroads_uk-1.0.0/tests/fixtures/aadf/README.md +71 -0
  105. crossroads_uk-1.0.0/tests/fixtures/aadf/dft_traffic_counts_aadf_sample.csv +15 -0
  106. crossroads_uk-1.0.0/tests/fixtures/bank_holidays/README.md +21 -0
  107. crossroads_uk-1.0.0/tests/fixtures/bank_holidays/bank-holidays-sample.json +31 -0
  108. crossroads_uk-1.0.0/tests/fixtures/ons/README.md +30 -0
  109. crossroads_uk-1.0.0/tests/fixtures/ons/ctyua_2024/ctyua_sample.dbf +0 -0
  110. crossroads_uk-1.0.0/tests/fixtures/ons/ctyua_2024/ctyua_sample.geojson +1 -0
  111. crossroads_uk-1.0.0/tests/fixtures/ons/ctyua_2024/ctyua_sample.prj +1 -0
  112. crossroads_uk-1.0.0/tests/fixtures/ons/ctyua_2024/ctyua_sample.shp +0 -0
  113. crossroads_uk-1.0.0/tests/fixtures/ons/ctyua_2024/ctyua_sample.shx +0 -0
  114. crossroads_uk-1.0.0/tests/fixtures/ons/ctyua_2025/ctyua_sample.geojson +1 -0
  115. crossroads_uk-1.0.0/tests/fixtures/ons/lad_2024/lad_sample.dbf +0 -0
  116. crossroads_uk-1.0.0/tests/fixtures/ons/lad_2024/lad_sample.geojson +1 -0
  117. crossroads_uk-1.0.0/tests/fixtures/ons/lad_2024/lad_sample.prj +1 -0
  118. crossroads_uk-1.0.0/tests/fixtures/ons/lad_2024/lad_sample.shp +0 -0
  119. crossroads_uk-1.0.0/tests/fixtures/ons/lad_2024/lad_sample.shx +0 -0
  120. crossroads_uk-1.0.0/tests/fixtures/ons/lad_2025/lad_sample.geojson +1 -0
  121. crossroads_uk-1.0.0/tests/fixtures/stats19/README.md +79 -0
  122. crossroads_uk-1.0.0/tests/fixtures/stats19/dft-road-casualty-statistics-casualty-2023.csv +13 -0
  123. crossroads_uk-1.0.0/tests/fixtures/stats19/dft-road-casualty-statistics-collision-2023.csv +9 -0
  124. crossroads_uk-1.0.0/tests/fixtures/stats19/dft-road-casualty-statistics-vehicle-2023.csv +18 -0
  125. crossroads_uk-1.0.0/tests/fixtures/weather/README.md +74 -0
  126. crossroads_uk-1.0.0/tests/fixtures/weather/era5_land_sample.nc +0 -0
  127. crossroads_uk-1.0.0/tests/test_aadf.py +435 -0
  128. crossroads_uk-1.0.0/tests/test_bank_holidays.py +61 -0
  129. crossroads_uk-1.0.0/tests/test_client.py +48 -0
  130. crossroads_uk-1.0.0/tests/test_console.py +553 -0
  131. crossroads_uk-1.0.0/tests/test_docs_links.py +100 -0
  132. crossroads_uk-1.0.0/tests/test_net.py +74 -0
  133. crossroads_uk-1.0.0/tests/test_package.py +10 -0
  134. crossroads_uk-1.0.0/tests/test_provenance.py +34 -0
  135. crossroads_uk-1.0.0/tests/test_quality.py +799 -0
  136. crossroads_uk-1.0.0/tests/test_readme_examples.py +160 -0
  137. crossroads_uk-1.0.0/tests/test_registry.py +240 -0
  138. crossroads_uk-1.0.0/tests/test_release.py +57 -0
  139. crossroads_uk-1.0.0/tests/test_schema_doc.py +103 -0
  140. crossroads_uk-1.0.0/tests/test_spatial.py +530 -0
  141. crossroads_uk-1.0.0/tests/test_sql_escaping.py +67 -0
  142. crossroads_uk-1.0.0/tests/test_stats19.py +1308 -0
  143. crossroads_uk-1.0.0/tests/test_update_script.py +70 -0
  144. crossroads_uk-1.0.0/tests/test_weather.py +300 -0
@@ -0,0 +1,2 @@
1
+ # Normalize all text files to LF inside the repository.
2
+ * text=auto eol=lf
@@ -0,0 +1,78 @@
1
+ name: publish
2
+
3
+ # Two ways to run:
4
+ # - Manually (workflow_dispatch): builds and uploads to TestPyPI. This is the dry run.
5
+ # - When a GitHub Release is published: builds and uploads to real PyPI.
6
+ # Uploads use PyPI Trusted Publishing (OIDC): no API tokens or passwords are stored.
7
+ on:
8
+ workflow_dispatch:
9
+ release:
10
+ types: [published]
11
+
12
+ jobs:
13
+ build:
14
+ name: Build sdist + wheel
15
+ runs-on: ubuntu-latest
16
+ steps:
17
+ - uses: actions/checkout@v5
18
+ with:
19
+ # hatch-vcs derives the version from git tags, so we need full history
20
+ # AND tags. Without fetch-depth: 0 the version silently falls back.
21
+ fetch-depth: 0
22
+
23
+ - name: Set up Python
24
+ uses: actions/setup-python@v6
25
+ with:
26
+ python-version: "3.12"
27
+
28
+ - name: Build and metadata-check
29
+ run: |
30
+ python -m pip install --upgrade pip build twine
31
+ python -m build
32
+ python -m twine check dist/*
33
+
34
+ - name: Upload built distribution as an artifact
35
+ uses: actions/upload-artifact@v4
36
+ with:
37
+ name: dist
38
+ path: dist/
39
+
40
+ publish-testpypi:
41
+ name: Publish to TestPyPI (dry run)
42
+ # Only on a manual run — this is the pre-release validation upload.
43
+ if: github.event_name == 'workflow_dispatch'
44
+ needs: build
45
+ runs-on: ubuntu-latest
46
+ environment: testpypi
47
+ permissions:
48
+ id-token: write # required for OIDC Trusted Publishing
49
+ steps:
50
+ - name: Download the built distribution
51
+ uses: actions/download-artifact@v4
52
+ with:
53
+ name: dist
54
+ path: dist/
55
+
56
+ - name: Publish to TestPyPI
57
+ uses: pypa/gh-action-pypi-publish@release/v1
58
+ with:
59
+ repository-url: https://test.pypi.org/legacy/
60
+
61
+ publish-pypi:
62
+ name: Publish to PyPI
63
+ # Only when a GitHub Release is published — the real release.
64
+ if: github.event_name == 'release'
65
+ needs: build
66
+ runs-on: ubuntu-latest
67
+ environment: pypi
68
+ permissions:
69
+ id-token: write # required for OIDC Trusted Publishing
70
+ steps:
71
+ - name: Download the built distribution
72
+ uses: actions/download-artifact@v4
73
+ with:
74
+ name: dist
75
+ path: dist/
76
+
77
+ - name: Publish to PyPI
78
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,55 @@
1
+ name: tests
2
+
3
+ # Run on pushes to main and on every pull request.
4
+ on:
5
+ push:
6
+ branches: [main]
7
+ pull_request:
8
+
9
+ jobs:
10
+ test:
11
+ # Run the offline suite on all three OSes so cross-platform regressions
12
+ # (path / home-directory differences) surface in CI, not just on a user's machine.
13
+ runs-on: ${{ matrix.os }}
14
+ strategy:
15
+ fail-fast: false
16
+ matrix:
17
+ os: [ubuntu-latest, windows-latest, macos-latest]
18
+ python-version: ["3.11", "3.12"]
19
+
20
+ steps:
21
+ - uses: actions/checkout@v5
22
+
23
+ - name: Set up Python ${{ matrix.python-version }}
24
+ uses: actions/setup-python@v6
25
+ with:
26
+ python-version: ${{ matrix.python-version }}
27
+
28
+ - name: Install (with dev extra)
29
+ run: |
30
+ python -m pip install --upgrade pip
31
+ pip install -e ".[dev]"
32
+
33
+ - name: Run the offline test suite
34
+ run: python -m pytest -q
35
+
36
+ integration:
37
+ # The integration suite is deselected by default (see pyproject addopts). Run it here
38
+ # in an environment that installs the [weather] extra, so the pre-release drift guards
39
+ # (e.g. tests/test_schema_doc.py) actually execute instead of silently skipping.
40
+ runs-on: ubuntu-latest
41
+ steps:
42
+ - uses: actions/checkout@v5
43
+
44
+ - name: Set up Python
45
+ uses: actions/setup-python@v6
46
+ with:
47
+ python-version: "3.12"
48
+
49
+ - name: Install (with dev + weather extras)
50
+ run: |
51
+ python -m pip install --upgrade pip
52
+ pip install -e ".[dev,weather]"
53
+
54
+ - name: Run the integration test suite
55
+ run: python -m pytest -m integration
@@ -0,0 +1,36 @@
1
+ /CLAUDE.md
2
+ .claude/
3
+ .idea/
4
+
5
+ # Plan implementer notes (AI/developer reference, not for commit)
6
+ docs/plans/**/IMPLEMENTER_NOTES.md
7
+
8
+ # Python
9
+ __pycache__/
10
+ *.py[cod]
11
+ *.egg-info/
12
+ .pytest_cache/
13
+ build/
14
+ dist/
15
+
16
+ # Test coverage artifacts
17
+ .coverage
18
+ .coverage.*
19
+
20
+ # Virtual environments
21
+ .venv/
22
+ venv/
23
+
24
+ # macOS metadata
25
+ .DS_Store
26
+
27
+ # Crossroads build artifacts
28
+ *.db
29
+ *.duckdb
30
+ # DuckDB write-ahead log (e.g. crossroads.db.wal), transient
31
+ *.wal
32
+ .crossroads_cache/
33
+ /htmlcov/
34
+
35
+ # Extensionless database copies produced during ad-hoc testing
36
+ weather_test
@@ -0,0 +1,59 @@
1
+ # Changelog
2
+
3
+ All notable changes to Crossroads-UK are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
5
+ [Semantic Versioning](https://semver.org/spec/v2.0.0.html), read for this project as:
6
+
7
+ - **MAJOR** — a breaking change to the stable contract (a removed/renamed database column or
8
+ table, or a CLI/`init_engine`/`build` API break). Stays `1` while schema changes are additive.
9
+ Bumped by hand, by cutting a new GitHub Release (a git tag such as `v2.0.0`).
10
+ - **MINOR** — additive schema/feature changes: a new datasource, column, or table. Bumped by hand,
11
+ by cutting a GitHub Release such as `v1.1.0`.
12
+ - **Build identity** (automatic, dev builds only) — set by `hatch-vcs` from git. A release (git tag)
13
+ is a clean number such as `1.1.0`; any commit after it reads e.g. `1.1.1.dev3+g1a2b3c4` — the
14
+ commit distance plus the exact git hash, so a dev build always pins its exact source. Never
15
+ hand-edited.
16
+
17
+ The physical database shape also carries its own monotonic `schema_version` integer in the
18
+ `crossroads_meta` table. Because reproducibility depends on the runtime stack, **any change to
19
+ declared dependencies or ingestion behaviour is a release** and is recorded here; the versions each
20
+ release was tested against are listed in [docs/methodology.md](docs/methodology.md).
21
+
22
+ ## [1.0.0] - 2026-07-15
23
+
24
+ First stable release, and the first published to PyPI (`pip install crossroads-uk`). The
25
+ DuckDB schema and public API (`crossroads` CLI, `init_engine`, `build`) are now the stable
26
+ contract described above; subsequent changes follow the MAJOR/MINOR policy at the top of
27
+ this file.
28
+
29
+ ### Added
30
+ - `solar_elevation_deg` and `solar_azimuth_deg` on the `collisions` table — the sun's apparent
31
+ elevation and azimuth at each collision's place and time, computed mathematically (NOAA solar
32
+ position algorithm, in SQL) from `geom` and `datetime_local`, with no external data or new
33
+ dependency. Enables isolating low-angle solar glare as a casualty factor.
34
+ - DfT AADF traffic-count source (`aadf` dataset): national count-point volumes 2000-onward,
35
+ LAD/CTYUA-stamped honouring `boundary_mode`, with an `aadf_clean` gold view and an R-Tree
36
+ index on the count-point geometry. Adds a README risk showcase (collisions per million
37
+ vehicle-km) and a wizard temporal-mode warning when traffic counts are built in temporal mode.
38
+
39
+ ### Changed
40
+ - Database `schema_version` 1 → 2 (additive: the two `collisions` columns above).
41
+ - Database `schema_version` 2 → 3 (additive: the `aadf`/`aadf_clean`/`aadf_raw` tables above).
42
+
43
+ ## [0.9.0] - 2026-07-10
44
+
45
+ First public release. **Pre-1.0 (Beta):** the pipeline is usable and reproducible, but the
46
+ DuckDB schema and public API are not yet frozen — they may still change before `1.0.0`. The
47
+ stable-contract guarantees described above take effect at `1.0.0`.
48
+
49
+ ### Added
50
+ - Reproducible build pipeline unifying DfT STATS19 road-safety data, Copernicus
51
+ ERA5-Land weather, and ONS boundaries into a single local DuckDB database.
52
+ - Keep-in-place data-quality model (bronze/silver/gold) with a queryable
53
+ `data_quality_log` exclusion ledger and build-time conservation invariants.
54
+ - Interactive data-compilation wizard (`crossroads`).
55
+ - Spatial standardisation to EPSG:27700 with R-Tree indices; snapshot and temporal
56
+ boundary modes.
57
+
58
+ [1.0.0]: https://github.com/wpatke/crossroads-uk/releases/tag/v1.0.0
59
+ [0.9.0]: https://github.com/wpatke/crossroads-uk/releases/tag/v0.9.0
@@ -0,0 +1,23 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use Crossroads-UK in your research, please cite it as below."
3
+ title: "Crossroads-UK: A Reproducible Pipeline for Unifying UK Road-Safety, Weather, and Boundary Data"
4
+ abstract: >-
5
+ Crossroads-UK is an open-source Python pipeline that downloads, cleanses, and unifies
6
+ UK road-safety (DfT STATS19), meteorological (Copernicus ERA5-Land), and ONS boundary
7
+ data into a single reproducible local DuckDB database.
8
+ type: software
9
+ authors:
10
+ - given-names: Will
11
+ family-names: Patke
12
+ # orcid: "https://orcid.org/0000-0000-0000-0000" # add if available
13
+ version: "1.0.0"
14
+ date-released: "2026-07-15"
15
+ license: MIT
16
+ repository-code: "https://github.com/wpatke/crossroads-uk"
17
+ url: "https://github.com/wpatke/crossroads-uk"
18
+ keywords:
19
+ - UK road safety
20
+ - STATS19
21
+ - DuckDB
22
+ - reproducible research
23
+ - ETL
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Will Patke
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,279 @@
1
+ Metadata-Version: 2.4
2
+ Name: crossroads-uk
3
+ Version: 1.0.0
4
+ Summary: A reproducible pipeline for unifying UK road-safety, weather, and boundary data into a single local DuckDB database.
5
+ Project-URL: Homepage, https://github.com/wpatke/crossroads-uk
6
+ Project-URL: Repository, https://github.com/wpatke/crossroads-uk
7
+ Project-URL: Issues, https://github.com/wpatke/crossroads-uk/issues
8
+ Project-URL: Changelog, https://github.com/wpatke/crossroads-uk/blob/main/CHANGELOG.md
9
+ Author: wpatke
10
+ License: MIT License
11
+
12
+ Copyright (c) 2026 Will Patke
13
+
14
+ Permission is hereby granted, free of charge, to any person obtaining a copy
15
+ of this software and associated documentation files (the "Software"), to deal
16
+ in the Software without restriction, including without limitation the rights
17
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
18
+ copies of the Software, and to permit persons to whom the Software is
19
+ furnished to do so, subject to the following conditions:
20
+
21
+ The above copyright notice and this permission notice shall be included in all
22
+ copies or substantial portions of the Software.
23
+
24
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
29
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30
+ SOFTWARE.
31
+ License-File: LICENSE
32
+ Keywords: duckdb,etl,reproducible-research,road-safety,stats19,uk
33
+ Classifier: Development Status :: 5 - Production/Stable
34
+ Classifier: Intended Audience :: Science/Research
35
+ Classifier: Operating System :: OS Independent
36
+ Classifier: Programming Language :: Python :: 3
37
+ Classifier: Programming Language :: Python :: 3.11
38
+ Classifier: Programming Language :: Python :: 3.12
39
+ Classifier: Topic :: Scientific/Engineering
40
+ Requires-Python: >=3.11
41
+ Requires-Dist: duckdb>=1.5
42
+ Provides-Extra: dev
43
+ Requires-Dist: pytest>=7.0; extra == 'dev'
44
+ Requires-Dist: pyyaml>=6.0; extra == 'dev'
45
+ Provides-Extra: weather
46
+ Requires-Dist: cdsapi>=0.7; extra == 'weather'
47
+ Requires-Dist: netcdf4>=1.6; extra == 'weather'
48
+ Requires-Dist: xarray>=2023.1; extra == 'weather'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # Crossroads-UK <img src="https://raw.githubusercontent.com/wpatke/crossroads-uk/main/docs/assets/logo.png" alt="Crossroads-UK" align="right" width="260">
52
+
53
+ [![tests](https://github.com/wpatke/crossroads-uk/actions/workflows/tests.yml/badge.svg)](https://github.com/wpatke/crossroads-uk/actions/workflows/tests.yml)
54
+
55
+ A reproducible Python pipeline that downloads, cleanses, and unifies UK road-safety
56
+ (DfT STATS19), meteorological (Copernicus ERA5-Land), and ONS boundary data into a
57
+ single local DuckDB database, built on the fly from version-controlled code.
58
+
59
+ Crossroads-UK does not ship a pre-baked database. You choose what to build; the pipeline
60
+ fetches the raw public sources and compiles a DuckDB file on your machine, so the result
61
+ is fresh, reproducible, and exactly scoped to your query.
62
+
63
+ ## Why Crossroads-UK? What can it do?
64
+
65
+ **More than road safety.**
66
+ Crossroads-UK is a reproducible engine for unifying UK public datasets
67
+ into one local, analysis-ready DuckDB database. Road safety is where it starts, not where it stops:
68
+ the engine is dataset-agnostic by design, so any UK public source can be added as a new transformer
69
+ without touching the core (see [Modular Data Architecture](docs/spec.md#4-modular-data-architecture) in
70
+ the spec). The rest of this section shows what that
71
+ unification buys a researcher today.
72
+
73
+ **Join anything to anything.**
74
+ Every source lands in one local DuckDB database on standardised keys (ONS
75
+ local-authority codes, road numbers, dates, British National Grid coordinates), so you can combine them
76
+ in ways no one built a feature for, using ordinary SQL. Does heavy rain shift the severity mix on the
77
+ M1? Do collisions spike on bank holidays in one local authority? Which roads see the most crashes per
78
+ vehicle-km? Each is just a join you write, and DuckDB runs it locally. See the
79
+ [real-world examples](#real-world-examples) below.
80
+
81
+ **A schema built for fast reads.**
82
+ A single collision row carries its weather, its
83
+ local authority, its bank-holiday status, and the position of the sun overhead. Those are datasets that
84
+ normally take a lot of work to align: gridded weather matched to each crash point, coordinates
85
+ geocoded to ONS boundaries. Crossroads does that alignment once, at build time, so what would be a
86
+ multi-source join is a single `SELECT`:
87
+
88
+ ```sql
89
+ SELECT accident_index,
90
+ collision_severity, -- 1=Fatal 2=Serious 3=Slight
91
+ temperature_c, -- ERA5-Land 2 m air temp at the collision hour
92
+ precipitation_mm, -- ERA5-Land hourly precipitation at that cell
93
+ solar_elevation_deg, -- sun's elevation (NOAA); negative = below horizon
94
+ lad_code, -- ONS local authority (point-in-polygon)
95
+ is_bank_holiday -- gov.uk bank-holiday calendar for that nation
96
+ FROM collisions
97
+ WHERE datetime_valid AND geom_valid
98
+ ORDER BY datetime_local
99
+ LIMIT 1
100
+ ```
101
+
102
+ **Measure risk, not raw counts.**
103
+ Traffic-exposure denominators turn collision counts into an
104
+ exposure-adjusted rate. See [Example 1: real risk, not raw counts](#example-1-real-risk-not-raw-counts)
105
+ below.
106
+
107
+ ### Crossroads-UK is built to be trusted
108
+
109
+ It doesn't just download data and hand it over. It builds the database on your machine, records
110
+ anything it couldn't use instead of silently dropping it, and stamps each build so a colleague can
111
+ reproduce it exactly:
112
+
113
+ - A **keep-in-place** data model (bronze/silver/gold): raw rows are never deleted; records
114
+ that fail validation are flagged with a reason in a queryable `data_quality_log`.
115
+ - Spatial standardisation to the British National Grid (EPSG:27700) with R-Tree indices.
116
+ - Snapshot or temporally-sliced boundary joins.
117
+ - Build-time conservation invariants that halt the build if any row goes unaccounted for.
118
+ - DfT AADF traffic volumes, LAD-stamped, enabling per-vehicle-km risk denominators (turn raw
119
+ collision counts into an exposure-adjusted rate).
120
+ - A **provenance stamp** on every database (version, schema, build parameters, and build time)
121
+ so a re-run reproduces the result and you can cite it: `SELECT * FROM crossroads_meta;`
122
+
123
+ The full table/column data dictionary is in **[docs/schema.md](docs/schema.md)**.
124
+
125
+ See **[docs/methodology.md](docs/methodology.md)** for how the data is joined, converted, and
126
+ quality-flagged, and **[docs/spec.md](docs/spec.md)** for the full product definition.
127
+
128
+ ## Install
129
+
130
+ ```bash
131
+ python3 -m venv .venv # Python 3.11 or newer
132
+ source .venv/bin/activate
133
+ pip install -e . # add ".[weather]" for the ERA5-Land weather source
134
+ ```
135
+
136
+ ## Usage
137
+
138
+ Run the interactive wizard:
139
+
140
+ ```bash
141
+ crossroads
142
+ ```
143
+
144
+ It asks for an output database path, which datasets to build, which years to ingest, and
145
+ the boundary mode, then compiles the database. Or drive it from Python:
146
+
147
+ ```python
148
+ import crossroads as cr
149
+
150
+ client = cr.init_engine(database_path="local_analytics.db")
151
+ client.build(
152
+ datasets=["stats19", "aadf"], # aadf = DfT traffic counts
153
+ years=[2022, 2023, 2024], boundary_mode="snapshot",
154
+ )
155
+ client.close()
156
+ ```
157
+
158
+ The **weather** source additionally needs the `weather` extra installed and a free
159
+ Copernicus CDS API key. The build prints setup steps if it is missing.
160
+
161
+ ## Real-world examples
162
+
163
+ ### Example 1: real risk, not raw counts
164
+
165
+ Raw collision counts mislead: a busy motorway *looks* dangerous simply because more traffic
166
+ means more crashes. Dividing collisions by traffic volume gives **collisions per million
167
+ vehicle-km** (an exposure-adjusted rate), and because collisions and AADF count points already
168
+ share road names and ONS LAD codes in your database, the join is plain SQL, with no point-to-line
169
+ snapping.
170
+
171
+ ```sql
172
+ WITH traffic AS (
173
+ SELECT road_name, lad_code,
174
+ SUM(all_motor_vehicles * link_length_km) AS daily_vehicle_km,
175
+ COUNT(DISTINCT count_point_id) AS count_points -- how many AADF counters back this road
176
+ FROM aadf_clean
177
+ WHERE year = 2023 AND lad_code IS NOT NULL
178
+ GROUP BY road_name, lad_code
179
+ ),
180
+ crashes AS (
181
+ SELECT 'M' || first_road_number AS road_name, lad_code,
182
+ COUNT(*) AS collisions
183
+ FROM collisions_spatial
184
+ WHERE first_road_class = 1 AND first_road_number > 0
185
+ AND lad_code IS NOT NULL
186
+ GROUP BY 1, 2
187
+ )
188
+ SELECT t.road_name, t.lad_code, c.collisions, t.daily_vehicle_km, t.count_points,
189
+ c.collisions / (t.daily_vehicle_km * 365 / 1e6) -- * 365: annualise the daily flow to match a full year of collisions
190
+ AS collisions_per_million_vehicle_km
191
+ FROM traffic t
192
+ JOIN crashes c USING (road_name, lad_code)
193
+ ORDER BY collisions_per_million_vehicle_km DESC
194
+ ```
195
+
196
+ M1 by local authority from a real 2023 build (`datasets=["stats19", "aadf"]`).
197
+
198
+ | road | lad_code | collisions | daily_vehicle_km | count_points | collisions per M vehicle-km |
199
+ |------|-----------|-----------:|-----------------:|-------------:|----------------------------:|
200
+ | M1 † | E07000096 | 15 | 5,895.2 | 1 | 6.9711 |
201
+ | M1 | E06000056 | 80 | 3,965,864.3 | 17 | 0.0553 |
202
+ | M1 | E07000240 | 55 | 3,047,956.0 | 6 | 0.0494 |
203
+ | M1 | E06000062 | 43 | 4,380,953.0 | 6 | 0.0269 |
204
+ | M1 | E08000018 | 21 | 2,169,706.9 | 5 | 0.0265 |
205
+ | M1 | E09000003 | 18 | 680,304.0 | 4 | 0.0725 |
206
+ | M1 | E08000036 | 17 | 1,357,071.6 | 3 | 0.0343 |
207
+ | M1 | E07000033 | 16 | 2,739,534.6 | 4 | 0.0160 |
208
+ | M1 | E08000035 | 15 | 2,524,444.8 | 11 | 0.0163 |
209
+
210
+ † **Why does E07000096 top the list? Is it really dangerous?** What looks like a deadly road is just
211
+ a counting mismatch: we counted all 15 crashes along this stretch of the M1, but measured traffic on
212
+ only a tiny section of it (AADF has just one count point there). We do not have data for the true
213
+ road traffic. A single count point can't tell you a road is dangerous *or* safe.
214
+
215
+ ### Example 2: questions the raw data can't answer
216
+
217
+ Crossroads can calculate facts beyond what is in the data. For example, the raw STATS19 record tells you
218
+ which way each vehicle was pointing but not where the sun was. Crossroads computes the sun's exact
219
+ elevation and azimuth for every collision from the crash's own place and time, in-database and with
220
+ nothing downloaded (the standard NOAA solar-position math). Put the two together, and you can isolate
221
+ collisions where a low sun sat directly ahead of the driver.
222
+
223
+ ```sql
224
+ -- Low sun close to the driver's line of travel: classic sun-glare geometry.
225
+ WITH v AS (
226
+ SELECT accident_index,
227
+ CASE vehicle_direction_to -- DfT 8-point code -> compass bearing (deg)
228
+ WHEN 1 THEN 0 WHEN 2 THEN 45 WHEN 3 THEN 90 WHEN 4 THEN 135
229
+ WHEN 5 THEN 180 WHEN 6 THEN 225 WHEN 7 THEN 270 WHEN 8 THEN 315
230
+ END AS bearing_deg
231
+ FROM vehicles
232
+ WHERE vehicle_direction_to BETWEEN 1 AND 8 -- 0 = parked, -1/9 = unknown
233
+ )
234
+ SELECT c.accident_index,
235
+ c.collision_severity, -- 1=Fatal 2=Serious 3=Slight
236
+ round(c.solar_elevation_deg, 1) AS sun_elevation_deg, -- low = near the horizon
237
+ round(180 - abs(abs(c.solar_azimuth_deg - v.bearing_deg) - 180), 1) AS sun_offset_deg
238
+ FROM collisions c
239
+ JOIN v USING (accident_index)
240
+ WHERE c.solar_elevation_deg BETWEEN 0 AND 10 -- sun above the horizon, but low
241
+ AND 180 - abs(abs(c.solar_azimuth_deg - v.bearing_deg) - 180) <= 30 -- within 30 deg ahead
242
+ ORDER BY sun_offset_deg
243
+ LIMIT 1
244
+ ```
245
+
246
+ `sun_offset_deg` is the angle between the sun and the direction of travel. `0` means the sun was
247
+ dead ahead. The single most head-on case in a real 2023 national build: a low morning sun 6° above
248
+ the horizon, dead in the driver's line of travel.
249
+
250
+ | accident_index | collision_severity | sun_elevation_deg | sun_offset_deg |
251
+ |----------------|-------------------:|------------------:|---------------:|
252
+ | 2023311351084 | 3 | 6.0 | 0.0 |
253
+
254
+ ## Data & licences
255
+
256
+ Crossroads-UK downloads data directly from DfT, ONS, Copernicus, and gov.uk. You are responsible
257
+ for honouring each source's licence when you publish. See **[docs/data-sources.md](docs/data-sources.md)**
258
+ for each source, its licence, and the exact attribution to reproduce.
259
+
260
+ ## Citing
261
+
262
+ If you use Crossroads-UK in research, please cite it. See **[CITATION.cff](CITATION.cff)**
263
+ (GitHub's "Cite this repository" button) or the latest release entry in
264
+ **[CHANGELOG.md](CHANGELOG.md)**.
265
+
266
+ ## Development
267
+
268
+ ```bash
269
+ pip install -e ".[dev]"
270
+ python -m pytest # fast, offline suite
271
+ python -m pytest -m integration # slow / networked tests (run deliberately)
272
+ ```
273
+
274
+ ## Licence & AI disclosure
275
+
276
+ Crossroads-UK is released under the [MIT Licence](LICENSE). This project is **not
277
+ affiliated with or endorsed by** the Department for Transport, the Office for National
278
+ Statistics, Ordnance Survey, or Copernicus/ECMWF. AI usage in development is documented in
279
+ **[docs/ai-disclosure.md](docs/ai-disclosure.md)**.