spatial-data-foundation 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/PKG-INFO +7 -9
- spatial_data_foundation-0.2.1/README.md +165 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/pyproject.toml +8 -7
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_data_foundation.egg-info/PKG-INFO +7 -9
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_data_foundation.egg-info/SOURCES.txt +9 -1
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_data_foundation.egg-info/requires.txt +6 -10
- spatial_data_foundation-0.2.1/src/spatial_foundation/_runtime.py +49 -0
- spatial_data_foundation-0.2.1/src/spatial_foundation/geography/_gadm_io.py +169 -0
- spatial_data_foundation-0.2.1/src/spatial_foundation/geography/_validation.py +82 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/geography/gadm.py +15 -5
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/geography/materialize.py +62 -65
- spatial_data_foundation-0.2.1/src/spatial_foundation/geography/membership.py +122 -0
- spatial_data_foundation-0.2.1/src/spatial_foundation/geography/overlap.py +193 -0
- spatial_data_foundation-0.2.1/tests/test_benchmarks.py +63 -0
- spatial_data_foundation-0.2.1/tests/test_dependency_boundary.py +25 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_distribution.py +52 -0
- spatial_data_foundation-0.2.1/tests/test_gadm_io.py +159 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_gadm_materialization.py +41 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_membership_kernel.py +74 -1
- spatial_data_foundation-0.2.1/tests/test_relation_kernel_parity.py +93 -0
- spatial_data_foundation-0.2.1/tests/test_spatial_invariants.py +72 -0
- spatial_data_foundation-0.1.0/README.md +0 -56
- spatial_data_foundation-0.1.0/src/spatial_foundation/geography/membership.py +0 -74
- spatial_data_foundation-0.1.0/src/spatial_foundation/geography/overlap.py +0 -178
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/setup.cfg +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_data_foundation.egg-info/dependency_links.txt +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_data_foundation.egg-info/top_level.txt +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/__init__.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/catalog.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/geography/__init__.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/geography/models.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/periods.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/presentation/__init__.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/presentation/basemaps.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/src/spatial_foundation/presentation/plotting.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_adversarial_spatial.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_areal_overlap.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_catalog.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_external_consumer.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_geography.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_periods.py +0 -0
- {spatial_data_foundation-0.1.0 → spatial_data_foundation-0.2.1}/tests/test_presentation.py +0 -0
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spatial-data-foundation
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Reusable geography, time, provenance and spatial-membership infrastructure for empirical research
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: empirical-data-contracts<0.2,>=0.1
|
|
7
|
-
Requires-Dist:
|
|
8
|
-
Requires-Dist:
|
|
9
|
-
Requires-Dist:
|
|
7
|
+
Requires-Dist: numpy>=1.24
|
|
8
|
+
Requires-Dist: pandas>=2.2
|
|
9
|
+
Requires-Dist: geopandas>=1.1
|
|
10
|
+
Requires-Dist: shapely>=2.1
|
|
11
|
+
Requires-Dist: pyproj>=3.7
|
|
12
|
+
Requires-Dist: pyogrio>=0.8
|
|
10
13
|
Requires-Dist: pydantic<3,>=2.7
|
|
11
14
|
Provides-Extra: io
|
|
12
15
|
Requires-Dist: pyarrow>=15; extra == "io"
|
|
13
|
-
Requires-Dist: duckdb>=1.0; extra == "io"
|
|
14
|
-
Provides-Extra: raster
|
|
15
|
-
Requires-Dist: rasterio>=1.3; extra == "raster"
|
|
16
|
-
Provides-Extra: cli
|
|
17
|
-
Requires-Dist: typer>=0.12; extra == "cli"
|
|
18
16
|
Provides-Extra: presentation
|
|
19
17
|
Requires-Dist: matplotlib>=3.8; extra == "presentation"
|
|
20
18
|
Requires-Dist: contextily<2,>=1.7.1; extra == "presentation"
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# spatial-data-foundation
|
|
2
|
+
|
|
3
|
+
Reusable spatial/time infrastructure for research systems.
|
|
4
|
+
|
|
5
|
+
This repository owns geography authority, period indexing, source registration, auditable spatial relations, and materialization provenance. It does **not** own FCV treatments, outcomes, matching, survey harmonization, regressions, domain assignment policy, or source acquisition.
|
|
6
|
+
|
|
7
|
+
The current implementation is the completed engineering candidate produced by Build Pack B: the public spatial semantics remain small, while the internals use the modern GeoPandas/Shapely/PyProj/Pyogrio stack deliberately. See [`docs/RELEASE_READINESS_0_2.md`](docs/RELEASE_READINESS_0_2.md) for the evidence and release gates. The package is not published as `0.2.0` until a separate human-approved release step.
|
|
8
|
+
|
|
9
|
+
## Public spatial contracts
|
|
10
|
+
|
|
11
|
+
### Point membership
|
|
12
|
+
|
|
13
|
+
`assign_points(...)` returns candidate memberships without silently resolving ambiguity.
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from spatial_foundation import assign_points
|
|
17
|
+
|
|
18
|
+
result, audit = assign_points(
|
|
19
|
+
points,
|
|
20
|
+
analytical_polygons,
|
|
21
|
+
point_id_col="event_id",
|
|
22
|
+
)
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The supported source geometry family is Point. Interior one-candidate points are `matched_unique`; points with no candidate are `unmatched_outside`; exact-boundary/multiple candidates remain `ambiguous_multiple`; and missing, empty, or invalid Point geometry is `invalid_point`. Unsupported source geometry families fail closed.
|
|
26
|
+
|
|
27
|
+
Target geography must have a CRS, unique/non-missing IDs, valid non-empty Polygon/MultiPolygon geometry, and analytical geometry role when that role is present.
|
|
28
|
+
|
|
29
|
+
### Areal relations
|
|
30
|
+
|
|
31
|
+
`relate_areal_objects(...)` returns positive-area geometric relation facts without assigning ownership.
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from spatial_foundation import relate_areal_objects
|
|
35
|
+
|
|
36
|
+
relations, audit = relate_areal_objects(
|
|
37
|
+
source_polygons,
|
|
38
|
+
analytical_polygons,
|
|
39
|
+
object_id_col="source_id",
|
|
40
|
+
area_crs="EPSG:6933",
|
|
41
|
+
)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
One source object may legitimately relate to multiple target units. The foundation emits `matched_single`, `matched_multiple`, `unmatched_outside`, or `invalid_geometry`, plus `overlap_area_m2` and `overlap_share_of_object`. Boundary-only touches do not become positive-area relations. Winner/tie rules, allocation, exposure, population, or treatment interpretation remain downstream policy.
|
|
45
|
+
|
|
46
|
+
Any CRS used to produce `_m2` / `_km2` quantities must be projected with metre horizontal axes. This guarantees unit correctness, not universal distortion-free area measurement.
|
|
47
|
+
|
|
48
|
+
## Relation engine
|
|
49
|
+
|
|
50
|
+
The implementation intentionally delegates spatial mechanics to mature libraries rather than creating a custom GIS layer:
|
|
51
|
+
|
|
52
|
+
```text
|
|
53
|
+
GeoPandas CRS-aware geometry arrays + cached spatial index
|
|
54
|
+
|
|
|
55
|
+
v
|
|
56
|
+
sindex.query positional pairs
|
|
57
|
+
|
|
|
58
|
+
v
|
|
59
|
+
Shapely vectorized GEOS operations
|
|
60
|
+
|
|
|
61
|
+
v
|
|
62
|
+
NumPy counts / masks
|
|
63
|
+
|
|
|
64
|
+
v
|
|
65
|
+
compact pandas results
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Candidate row order is not a public contract. The benchmark suite protects semantic parity and records candidate amplification and phase timings without turning absolute wall time into a brittle CI threshold.
|
|
69
|
+
|
|
70
|
+
## Operational GADM materialization
|
|
71
|
+
|
|
72
|
+
Materialization works from registered immutable **local** source snapshots; it does not download GADM or other source data.
|
|
73
|
+
|
|
74
|
+
Install the I/O extra and materialize the native levels required by downstream work:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install "spatial-data-foundation[io]"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from spatial_foundation import materialize_gadm
|
|
82
|
+
|
|
83
|
+
materialize_gadm(
|
|
84
|
+
snapshot=gadm_4_1_snapshot,
|
|
85
|
+
levels=[0, 1, 2, 3],
|
|
86
|
+
output_root=data_root,
|
|
87
|
+
)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Before loading complete geometry, vector sources are inspected through Pyogrio/GDAL to identify requested ADM layers and required identity fields. Reads are column-projected and the PyArrow-backed materialization workflow uses GDAL Arrow transfer where applicable. Separate-level vector files, multi-layer GeoPackages, and GeoParquet sources are supported.
|
|
91
|
+
|
|
92
|
+
The workflow publishes full analytical geometry as GeoParquet under `silver/geography/gadm/<version>/` and writes `run_manifest.json` plus `geography_qa.json` under the run directory. Registered source hashes are rechecked before interpretation; source drift or malformed requested levels produce a RED run rather than partial silver publication.
|
|
93
|
+
|
|
94
|
+
Installed wheels are first-class. Git metadata is optional: when no checkout SHA exists, `code_commit` is `null` and the manifest still records installed package version, runtime/native stack versions, source snapshot hashes, output hashes, parameters, and QA.
|
|
95
|
+
|
|
96
|
+
## Dependency boundary
|
|
97
|
+
|
|
98
|
+
Core implementation dependencies are declared directly rather than relying on GeoPandas transitively:
|
|
99
|
+
|
|
100
|
+
```text
|
|
101
|
+
Python >= 3.10
|
|
102
|
+
empirical-data-contracts >= 0.1,<0.2
|
|
103
|
+
NumPy >= 1.24
|
|
104
|
+
pandas >= 2.2
|
|
105
|
+
GeoPandas >= 1.1
|
|
106
|
+
Shapely >= 2.1
|
|
107
|
+
PyProj >= 3.7
|
|
108
|
+
Pyogrio >= 0.8
|
|
109
|
+
Pydantic >= 2.7,<3
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Optional extras are deliberately limited to capabilities the repository actually owns:
|
|
113
|
+
|
|
114
|
+
- `io` — PyArrow / GeoParquet materialization support;
|
|
115
|
+
- `presentation` — contextual map presentation;
|
|
116
|
+
- `dev` — tests, lint, build, and development dependencies.
|
|
117
|
+
|
|
118
|
+
Raster, DuckDB query, and CLI surfaces are not advertised until owned implementations exist.
|
|
119
|
+
|
|
120
|
+
## Optional contextual presentation
|
|
121
|
+
|
|
122
|
+
Presentation helpers are an optional, non-analytical surface. Core imports do not require plotting or tile dependencies:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install "spatial-data-foundation[presentation]"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Add contextual imagery beneath an existing analytical plot:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from spatial_foundation.presentation import add_basemap
|
|
132
|
+
|
|
133
|
+
ax = gdf.plot(column="value", alpha=0.65)
|
|
134
|
+
add_basemap(ax, crs=gdf.crs, kind="imagery", alpha=0.55)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Or use the compact GeoDataFrame helper:
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from spatial_foundation.presentation import plot_context
|
|
141
|
+
|
|
142
|
+
ax = plot_context(
|
|
143
|
+
gdf,
|
|
144
|
+
basemap="neutral",
|
|
145
|
+
column="value",
|
|
146
|
+
)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
The governed convenience aliases are `neutral`, `imagery`, and `terrain`. A concrete tile source or local raster path can also be supplied. Local rasters allow reproducible/offline presentation; web providers may perform network requests only when explicitly selected by the caller.
|
|
150
|
+
|
|
151
|
+
Presentation never changes analytical geometry, membership, identifiers, or domain interpretation. Provider attribution remains enabled, and no provider credentials are bundled with this package.
|
|
152
|
+
|
|
153
|
+
## Scope discipline
|
|
154
|
+
|
|
155
|
+
The foundation deliberately does not contain:
|
|
156
|
+
|
|
157
|
+
- treatment/control/exposure semantics;
|
|
158
|
+
- census/electoral winner or crosswalk policy;
|
|
159
|
+
- automatic provider downloads;
|
|
160
|
+
- generic geometry repair or precision policy;
|
|
161
|
+
- public STRtree/cache classes;
|
|
162
|
+
- distributed orchestration;
|
|
163
|
+
- speculative raster, query, or CLI frameworks.
|
|
164
|
+
|
|
165
|
+
New primitives should enter this package only when they are reusable geometric/spatiotemporal mechanics with a concrete consumer, not merely because a GIS library makes them possible.
|
|
@@ -4,21 +4,22 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "spatial-data-foundation"
|
|
7
|
-
version = "0.1
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "Reusable geography, time, provenance and spatial-membership infrastructure for empirical research"
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"empirical-data-contracts>=0.1,<0.2",
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
"
|
|
12
|
+
"numpy>=1.24",
|
|
13
|
+
"pandas>=2.2",
|
|
14
|
+
"geopandas>=1.1",
|
|
15
|
+
"shapely>=2.1",
|
|
16
|
+
"pyproj>=3.7",
|
|
17
|
+
"pyogrio>=0.8",
|
|
15
18
|
"pydantic>=2.7,<3",
|
|
16
19
|
]
|
|
17
20
|
|
|
18
21
|
[project.optional-dependencies]
|
|
19
|
-
io = ["pyarrow>=15"
|
|
20
|
-
raster = ["rasterio>=1.3"]
|
|
21
|
-
cli = ["typer>=0.12"]
|
|
22
|
+
io = ["pyarrow>=15"]
|
|
22
23
|
presentation = [
|
|
23
24
|
"matplotlib>=3.8",
|
|
24
25
|
"contextily>=1.7.1,<2",
|
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spatial-data-foundation
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Reusable geography, time, provenance and spatial-membership infrastructure for empirical research
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: empirical-data-contracts<0.2,>=0.1
|
|
7
|
-
Requires-Dist:
|
|
8
|
-
Requires-Dist:
|
|
9
|
-
Requires-Dist:
|
|
7
|
+
Requires-Dist: numpy>=1.24
|
|
8
|
+
Requires-Dist: pandas>=2.2
|
|
9
|
+
Requires-Dist: geopandas>=1.1
|
|
10
|
+
Requires-Dist: shapely>=2.1
|
|
11
|
+
Requires-Dist: pyproj>=3.7
|
|
12
|
+
Requires-Dist: pyogrio>=0.8
|
|
10
13
|
Requires-Dist: pydantic<3,>=2.7
|
|
11
14
|
Provides-Extra: io
|
|
12
15
|
Requires-Dist: pyarrow>=15; extra == "io"
|
|
13
|
-
Requires-Dist: duckdb>=1.0; extra == "io"
|
|
14
|
-
Provides-Extra: raster
|
|
15
|
-
Requires-Dist: rasterio>=1.3; extra == "raster"
|
|
16
|
-
Provides-Extra: cli
|
|
17
|
-
Requires-Dist: typer>=0.12; extra == "cli"
|
|
18
16
|
Provides-Extra: presentation
|
|
19
17
|
Requires-Dist: matplotlib>=3.8; extra == "presentation"
|
|
20
18
|
Requires-Dist: contextily<2,>=1.7.1; extra == "presentation"
|
|
@@ -6,9 +6,12 @@ src/spatial_data_foundation.egg-info/dependency_links.txt
|
|
|
6
6
|
src/spatial_data_foundation.egg-info/requires.txt
|
|
7
7
|
src/spatial_data_foundation.egg-info/top_level.txt
|
|
8
8
|
src/spatial_foundation/__init__.py
|
|
9
|
+
src/spatial_foundation/_runtime.py
|
|
9
10
|
src/spatial_foundation/catalog.py
|
|
10
11
|
src/spatial_foundation/periods.py
|
|
11
12
|
src/spatial_foundation/geography/__init__.py
|
|
13
|
+
src/spatial_foundation/geography/_gadm_io.py
|
|
14
|
+
src/spatial_foundation/geography/_validation.py
|
|
12
15
|
src/spatial_foundation/geography/gadm.py
|
|
13
16
|
src/spatial_foundation/geography/materialize.py
|
|
14
17
|
src/spatial_foundation/geography/membership.py
|
|
@@ -19,11 +22,16 @@ src/spatial_foundation/presentation/basemaps.py
|
|
|
19
22
|
src/spatial_foundation/presentation/plotting.py
|
|
20
23
|
tests/test_adversarial_spatial.py
|
|
21
24
|
tests/test_areal_overlap.py
|
|
25
|
+
tests/test_benchmarks.py
|
|
22
26
|
tests/test_catalog.py
|
|
27
|
+
tests/test_dependency_boundary.py
|
|
23
28
|
tests/test_distribution.py
|
|
24
29
|
tests/test_external_consumer.py
|
|
30
|
+
tests/test_gadm_io.py
|
|
25
31
|
tests/test_gadm_materialization.py
|
|
26
32
|
tests/test_geography.py
|
|
27
33
|
tests/test_membership_kernel.py
|
|
28
34
|
tests/test_periods.py
|
|
29
|
-
tests/test_presentation.py
|
|
35
|
+
tests/test_presentation.py
|
|
36
|
+
tests/test_relation_kernel_parity.py
|
|
37
|
+
tests/test_spatial_invariants.py
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
empirical-data-contracts<0.2,>=0.1
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
numpy>=1.24
|
|
3
|
+
pandas>=2.2
|
|
4
|
+
geopandas>=1.1
|
|
5
|
+
shapely>=2.1
|
|
6
|
+
pyproj>=3.7
|
|
7
|
+
pyogrio>=0.8
|
|
5
8
|
pydantic<3,>=2.7
|
|
6
9
|
|
|
7
|
-
[cli]
|
|
8
|
-
typer>=0.12
|
|
9
|
-
|
|
10
10
|
[dev]
|
|
11
11
|
pytest>=8
|
|
12
12
|
ruff>=0.6
|
|
@@ -18,12 +18,8 @@ xyzservices<2027,>=2026.3
|
|
|
18
18
|
|
|
19
19
|
[io]
|
|
20
20
|
pyarrow>=15
|
|
21
|
-
duckdb>=1.0
|
|
22
21
|
|
|
23
22
|
[presentation]
|
|
24
23
|
matplotlib>=3.8
|
|
25
24
|
contextily<2,>=1.7.1
|
|
26
25
|
xyzservices<2027,>=2026.3
|
|
27
|
-
|
|
28
|
-
[raster]
|
|
29
|
-
rasterio>=1.3
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import platform
|
|
4
|
+
from importlib import metadata
|
|
5
|
+
|
|
6
|
+
_PACKAGE_NAMES = (
|
|
7
|
+
"spatial-data-foundation",
|
|
8
|
+
"empirical-data-contracts",
|
|
9
|
+
"geopandas",
|
|
10
|
+
"pandas",
|
|
11
|
+
"numpy",
|
|
12
|
+
"shapely",
|
|
13
|
+
"pyproj",
|
|
14
|
+
"pyogrio",
|
|
15
|
+
"pyarrow",
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def runtime_versions() -> dict[str, str]:
|
|
20
|
+
"""Return installed Python/geospatial runtime versions for provenance."""
|
|
21
|
+
versions = {"python": platform.python_version()}
|
|
22
|
+
for name in _PACKAGE_NAMES:
|
|
23
|
+
try:
|
|
24
|
+
versions[name] = metadata.version(name)
|
|
25
|
+
except metadata.PackageNotFoundError:
|
|
26
|
+
versions[name] = "not-installed"
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import shapely
|
|
30
|
+
|
|
31
|
+
versions["GEOS"] = shapely.geos_version_string
|
|
32
|
+
except (ImportError, AttributeError):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
import pyproj
|
|
37
|
+
|
|
38
|
+
versions["PROJ"] = pyproj.proj_version_str
|
|
39
|
+
except (ImportError, AttributeError):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
import pyogrio
|
|
44
|
+
|
|
45
|
+
versions["GDAL"] = ".".join(str(part) for part in pyogrio.__gdal_version__)
|
|
46
|
+
except (ImportError, AttributeError):
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
return versions
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import geopandas as gpd
|
|
9
|
+
import pyogrio
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class GADMSourceSelection:
|
|
14
|
+
path: Path
|
|
15
|
+
layer: str | None
|
|
16
|
+
columns: tuple[str, ...]
|
|
17
|
+
geometry_column: str | None = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _native_level_from_fields(fields: Iterable[str]) -> int | None:
|
|
21
|
+
levels = []
|
|
22
|
+
for field in fields:
|
|
23
|
+
if not field.startswith("GID_"):
|
|
24
|
+
continue
|
|
25
|
+
suffix = field[4:]
|
|
26
|
+
if suffix.isdigit():
|
|
27
|
+
levels.append(int(suffix))
|
|
28
|
+
return max(levels) if levels else None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _required_identity_columns(fields: Iterable[str], level: int) -> tuple[str, ...]:
|
|
32
|
+
available = set(fields)
|
|
33
|
+
required = ["GID_0", f"GID_{level}"]
|
|
34
|
+
missing = sorted(set(required) - available)
|
|
35
|
+
if missing:
|
|
36
|
+
raise ValueError(
|
|
37
|
+
f"GADM ADM{level} source is missing required identity fields: {missing}"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
columns = ["GID_0"]
|
|
41
|
+
if level > 0:
|
|
42
|
+
parent = f"GID_{level - 1}"
|
|
43
|
+
if parent in available and parent not in columns:
|
|
44
|
+
columns.append(parent)
|
|
45
|
+
current = f"GID_{level}"
|
|
46
|
+
if current not in columns:
|
|
47
|
+
columns.append(current)
|
|
48
|
+
return tuple(columns)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _inspect_vector_source(path: Path, level: int) -> GADMSourceSelection | None:
|
|
52
|
+
try:
|
|
53
|
+
layers = pyogrio.list_layers(path)
|
|
54
|
+
except Exception as exc:
|
|
55
|
+
raise ValueError(f"cannot inspect registered GADM source file {path}: {exc}") from exc
|
|
56
|
+
|
|
57
|
+
matches: list[GADMSourceSelection] = []
|
|
58
|
+
for row in layers:
|
|
59
|
+
layer = str(row[0])
|
|
60
|
+
try:
|
|
61
|
+
info = pyogrio.read_info(path, layer=layer)
|
|
62
|
+
except Exception as exc:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"cannot inspect registered GADM layer {layer!r} in {path}: {exc}"
|
|
65
|
+
) from exc
|
|
66
|
+
fields = tuple(str(field) for field in info.get("fields", ()))
|
|
67
|
+
if _native_level_from_fields(fields) != level:
|
|
68
|
+
continue
|
|
69
|
+
columns = _required_identity_columns(fields, level)
|
|
70
|
+
matches.append(GADMSourceSelection(path=path, layer=layer, columns=columns))
|
|
71
|
+
|
|
72
|
+
if len(matches) > 1:
|
|
73
|
+
layer_names = [selection.layer for selection in matches]
|
|
74
|
+
raise ValueError(
|
|
75
|
+
f"registered GADM source {path} contains multiple ADM{level} layers: {layer_names}"
|
|
76
|
+
)
|
|
77
|
+
return matches[0] if matches else None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _inspect_geoparquet_source(path: Path, level: int) -> GADMSourceSelection | None:
|
|
81
|
+
try:
|
|
82
|
+
import pyarrow.parquet as pq
|
|
83
|
+
except ImportError as exc: # pragma: no cover - materialization checks this first
|
|
84
|
+
raise RuntimeError(
|
|
85
|
+
"GeoParquet GADM source inspection requires pyarrow; "
|
|
86
|
+
"install spatial-data-foundation[io]"
|
|
87
|
+
) from exc
|
|
88
|
+
|
|
89
|
+
try:
|
|
90
|
+
schema = pq.read_schema(path)
|
|
91
|
+
except Exception as exc:
|
|
92
|
+
raise ValueError(f"cannot inspect registered GADM GeoParquet {path}: {exc}") from exc
|
|
93
|
+
|
|
94
|
+
fields = tuple(schema.names)
|
|
95
|
+
if _native_level_from_fields(fields) != level:
|
|
96
|
+
return None
|
|
97
|
+
columns = _required_identity_columns(fields, level)
|
|
98
|
+
|
|
99
|
+
metadata = schema.metadata or {}
|
|
100
|
+
geo_metadata = metadata.get(b"geo")
|
|
101
|
+
if not geo_metadata:
|
|
102
|
+
raise ValueError(f"registered GADM parquet source is missing GeoParquet metadata: {path}")
|
|
103
|
+
try:
|
|
104
|
+
primary_geometry = json.loads(geo_metadata.decode("utf-8"))["primary_column"]
|
|
105
|
+
except (KeyError, TypeError, ValueError, UnicodeDecodeError) as exc:
|
|
106
|
+
raise ValueError(f"cannot resolve GeoParquet primary geometry column for {path}") from exc
|
|
107
|
+
if primary_geometry not in fields:
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"GeoParquet primary geometry column {primary_geometry!r} is missing from {path}"
|
|
110
|
+
)
|
|
111
|
+
return GADMSourceSelection(
|
|
112
|
+
path=path,
|
|
113
|
+
layer=None,
|
|
114
|
+
columns=columns,
|
|
115
|
+
geometry_column=primary_geometry,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def inspect_gadm_source(path: Path, level: int) -> GADMSourceSelection | None:
|
|
120
|
+
"""Select one requested native GADM level without materializing source geometry."""
|
|
121
|
+
suffix = path.suffix.lower()
|
|
122
|
+
if suffix in {".parquet", ".geoparquet"}:
|
|
123
|
+
return _inspect_geoparquet_source(path, level)
|
|
124
|
+
return _inspect_vector_source(path, level)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def read_gadm_selection(
|
|
128
|
+
selection: GADMSourceSelection,
|
|
129
|
+
*,
|
|
130
|
+
use_arrow: bool,
|
|
131
|
+
) -> gpd.GeoDataFrame:
|
|
132
|
+
"""Read only the selected GADM identity columns plus the authoritative geometry."""
|
|
133
|
+
if selection.geometry_column is not None:
|
|
134
|
+
columns = [*selection.columns, selection.geometry_column]
|
|
135
|
+
try:
|
|
136
|
+
return gpd.read_parquet(selection.path, columns=columns)
|
|
137
|
+
except Exception as exc:
|
|
138
|
+
raise ValueError(
|
|
139
|
+
f"cannot read registered GADM GeoParquet {selection.path}: {exc}"
|
|
140
|
+
) from exc
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
return pyogrio.read_dataframe(
|
|
144
|
+
selection.path,
|
|
145
|
+
layer=selection.layer,
|
|
146
|
+
columns=list(selection.columns),
|
|
147
|
+
use_arrow=use_arrow,
|
|
148
|
+
)
|
|
149
|
+
except Exception as exc:
|
|
150
|
+
layer_text = "" if selection.layer is None else f" layer {selection.layer!r}"
|
|
151
|
+
raise ValueError(
|
|
152
|
+
f"cannot read registered GADM source file {selection.path}{layer_text}: {exc}"
|
|
153
|
+
) from exc
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def read_gadm_level_parts(
|
|
157
|
+
paths: Iterable[Path],
|
|
158
|
+
level: int,
|
|
159
|
+
*,
|
|
160
|
+
use_arrow: bool,
|
|
161
|
+
) -> list[gpd.GeoDataFrame]:
|
|
162
|
+
"""Inspect registered sources first, then load only parts matching ``level``."""
|
|
163
|
+
parts = []
|
|
164
|
+
for path in paths:
|
|
165
|
+
selection = inspect_gadm_source(path, level)
|
|
166
|
+
if selection is None:
|
|
167
|
+
continue
|
|
168
|
+
parts.append(read_gadm_selection(selection, use_arrow=use_arrow))
|
|
169
|
+
return parts
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
import geopandas as gpd
|
|
6
|
+
from pyproj import CRS
|
|
7
|
+
|
|
8
|
+
_AREAL_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def require_unique_nonmissing_ids(
|
|
12
|
+
frame: gpd.GeoDataFrame,
|
|
13
|
+
column: str,
|
|
14
|
+
*,
|
|
15
|
+
label: str,
|
|
16
|
+
) -> None:
|
|
17
|
+
if column not in frame.columns:
|
|
18
|
+
raise ValueError(f"missing {label} id column: {column}")
|
|
19
|
+
values = frame[column]
|
|
20
|
+
if values.isna().any():
|
|
21
|
+
raise ValueError(f"{label} IDs must be non-missing")
|
|
22
|
+
if values.duplicated().any():
|
|
23
|
+
raise ValueError(f"{label} IDs must be unique")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def require_analytical_polygons(
|
|
27
|
+
polygons: gpd.GeoDataFrame,
|
|
28
|
+
*,
|
|
29
|
+
polygon_id_col: str,
|
|
30
|
+
operation: str,
|
|
31
|
+
) -> None:
|
|
32
|
+
if polygons.crs is None:
|
|
33
|
+
raise ValueError(f"{operation} analytical geography inputs require a CRS")
|
|
34
|
+
require_unique_nonmissing_ids(polygons, polygon_id_col, label="polygon")
|
|
35
|
+
|
|
36
|
+
if "geometry_role" in polygons.columns:
|
|
37
|
+
roles = polygons["geometry_role"].astype("string")
|
|
38
|
+
if roles.isna().any() or roles.ne("analytical").any():
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"{operation} requires analytical geometry; analytical geography only"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
geometry = polygons.geometry
|
|
44
|
+
if geometry.isna().any() or geometry.is_empty.any() or (~geometry.is_valid).any():
|
|
45
|
+
raise ValueError("analytical geography contains missing, empty, or invalid geometry")
|
|
46
|
+
non_areal = ~geometry.geom_type.isin(_AREAL_GEOMETRY_TYPES)
|
|
47
|
+
if non_areal.any():
|
|
48
|
+
found = sorted(geometry.loc[non_areal].geom_type.unique().tolist())
|
|
49
|
+
raise ValueError(
|
|
50
|
+
f"{operation} analytical geography requires Polygon/MultiPolygon geometry; "
|
|
51
|
+
f"found {found}"
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def require_projected_metre_crs(value: object, *, label: str = "area_crs") -> CRS:
|
|
56
|
+
"""Return a CRS that is projected and whose horizontal axes use metres.
|
|
57
|
+
|
|
58
|
+
This establishes unit correctness for quantities labelled in square metres or
|
|
59
|
+
square kilometres. It deliberately does not claim that every projected metre
|
|
60
|
+
CRS is equal-area or otherwise suitable for every scientific measurement.
|
|
61
|
+
"""
|
|
62
|
+
try:
|
|
63
|
+
crs = CRS.from_user_input(value)
|
|
64
|
+
except Exception as exc: # pyproj accepts several user-input exception types
|
|
65
|
+
raise ValueError(f"{label} is not a resolvable CRS: {value!r}") from exc
|
|
66
|
+
|
|
67
|
+
if not crs.is_projected:
|
|
68
|
+
raise ValueError(f"{label} must be a projected CRS with metre horizontal units")
|
|
69
|
+
|
|
70
|
+
axes = crs.axis_info[:2]
|
|
71
|
+
if len(axes) < 2:
|
|
72
|
+
raise ValueError(f"{label} must expose two horizontal axes with metre units")
|
|
73
|
+
if any(
|
|
74
|
+
axis.unit_conversion_factor is None
|
|
75
|
+
or not math.isclose(float(axis.unit_conversion_factor), 1.0, rel_tol=0.0, abs_tol=1e-12)
|
|
76
|
+
for axis in axes
|
|
77
|
+
):
|
|
78
|
+
units = [axis.unit_name for axis in axes]
|
|
79
|
+
raise ValueError(
|
|
80
|
+
f"{label} must use metre horizontal units; resolved horizontal units are {units}"
|
|
81
|
+
)
|
|
82
|
+
return crs
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import geopandas as gpd
|
|
4
4
|
|
|
5
|
+
from ._validation import require_projected_metre_crs
|
|
5
6
|
from .models import GeometryRole, geography_uid
|
|
6
7
|
|
|
7
8
|
REQUIRED_BASE = {"GID_0"}
|
|
@@ -25,8 +26,10 @@ def normalize_gadm_frame(
|
|
|
25
26
|
|
|
26
27
|
Expected source fields are GID_0 .. GID_{level}. Parent identity is retained when
|
|
27
28
|
available. Geometry is stored in EPSG:4326; area is calculated in the declared
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
projected metre CRS. The unit check ensures ``area_km2`` is dimensionally valid,
|
|
30
|
+
but does not by itself claim equal-area scientific accuracy. The returned frame
|
|
31
|
+
records the area CRS and geometry validity profile in ``GeoDataFrame.attrs`` for
|
|
32
|
+
run-manifest/provenance capture.
|
|
30
33
|
"""
|
|
31
34
|
if level < 0:
|
|
32
35
|
raise ValueError("GADM admin level must be >= 0")
|
|
@@ -41,6 +44,7 @@ def normalize_gadm_frame(
|
|
|
41
44
|
raise ValueError(f"missing required GADM columns: {sorted(missing)}")
|
|
42
45
|
if frame.crs is None:
|
|
43
46
|
raise ValueError("GADM source must have a declared CRS")
|
|
47
|
+
require_projected_metre_crs(area_crs)
|
|
44
48
|
|
|
45
49
|
source_ids = _require_identity(frame[gid_col], label="source geography identity")
|
|
46
50
|
country_ids = _require_identity(frame["GID_0"], label="country identity")
|
|
@@ -53,9 +57,13 @@ def normalize_gadm_frame(
|
|
|
53
57
|
data["source_geo_id"] = source_ids.to_numpy()
|
|
54
58
|
data["country_iso3"] = country_ids.str.slice(0, 3).to_numpy()
|
|
55
59
|
data["native_admin_level"] = level
|
|
56
|
-
data["geo_uid"] = data["source_geo_id"].map(
|
|
60
|
+
data["geo_uid"] = data["source_geo_id"].map(
|
|
61
|
+
lambda x: geography_uid("gadm", version, level, x)
|
|
62
|
+
)
|
|
57
63
|
if parent_col and parent_col in data.columns:
|
|
58
|
-
parent_present =
|
|
64
|
+
parent_present = (
|
|
65
|
+
data[parent_col].notna() & data[parent_col].astype("string").str.strip().ne("")
|
|
66
|
+
)
|
|
59
67
|
data["parent_geo_uid"] = [
|
|
60
68
|
geography_uid("gadm", version, level - 1, str(value)) if present else None
|
|
61
69
|
for value, present in zip(data[parent_col], parent_present)
|
|
@@ -78,7 +86,9 @@ def normalize_gadm_frame(
|
|
|
78
86
|
]
|
|
79
87
|
out = data[keep].copy()
|
|
80
88
|
if out["geo_uid"].duplicated().any():
|
|
81
|
-
dupes = out.loc[
|
|
89
|
+
dupes = out.loc[
|
|
90
|
+
out["geo_uid"].duplicated(keep=False), "geo_uid"
|
|
91
|
+
].unique().tolist()[:10]
|
|
82
92
|
raise ValueError(f"duplicate geography IDs after normalization: {dupes}")
|
|
83
93
|
|
|
84
94
|
null_geometry = out.geometry.isna()
|