spatial-data-foundation 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spatial_data_foundation-0.1.0.dist-info/METADATA +29 -0
- spatial_data_foundation-0.1.0.dist-info/RECORD +16 -0
- spatial_data_foundation-0.1.0.dist-info/WHEEL +5 -0
- spatial_data_foundation-0.1.0.dist-info/top_level.txt +1 -0
- spatial_foundation/__init__.py +20 -0
- spatial_foundation/catalog.py +52 -0
- spatial_foundation/geography/__init__.py +19 -0
- spatial_foundation/geography/gadm.py +92 -0
- spatial_foundation/geography/materialize.py +450 -0
- spatial_foundation/geography/membership.py +74 -0
- spatial_foundation/geography/models.py +37 -0
- spatial_foundation/geography/overlap.py +178 -0
- spatial_foundation/periods.py +48 -0
- spatial_foundation/presentation/__init__.py +10 -0
- spatial_foundation/presentation/basemaps.py +85 -0
- spatial_foundation/presentation/plotting.py +32 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spatial-data-foundation
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reusable geography, time, provenance and spatial-membership infrastructure for empirical research
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: empirical-data-contracts<0.2,>=0.1
|
|
7
|
+
Requires-Dist: geopandas>=0.14
|
|
8
|
+
Requires-Dist: shapely>=2
|
|
9
|
+
Requires-Dist: pyproj>=3.5
|
|
10
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
11
|
+
Provides-Extra: io
|
|
12
|
+
Requires-Dist: pyarrow>=15; extra == "io"
|
|
13
|
+
Requires-Dist: duckdb>=1.0; extra == "io"
|
|
14
|
+
Provides-Extra: raster
|
|
15
|
+
Requires-Dist: rasterio>=1.3; extra == "raster"
|
|
16
|
+
Provides-Extra: cli
|
|
17
|
+
Requires-Dist: typer>=0.12; extra == "cli"
|
|
18
|
+
Provides-Extra: presentation
|
|
19
|
+
Requires-Dist: matplotlib>=3.8; extra == "presentation"
|
|
20
|
+
Requires-Dist: contextily<2,>=1.7.1; extra == "presentation"
|
|
21
|
+
Requires-Dist: xyzservices<2027,>=2026.3; extra == "presentation"
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
24
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
25
|
+
Requires-Dist: pyarrow>=15; extra == "dev"
|
|
26
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
27
|
+
Requires-Dist: matplotlib>=3.8; extra == "dev"
|
|
28
|
+
Requires-Dist: contextily<2,>=1.7.1; extra == "dev"
|
|
29
|
+
Requires-Dist: xyzservices<2027,>=2026.3; extra == "dev"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
spatial_foundation/__init__.py,sha256=GRmq8A_bqovaHHE21ATVDhiDXsII8VctSClcS3B5Rh4,456
|
|
2
|
+
spatial_foundation/catalog.py,sha256=53WUvgza-bOuPMcK0qg8riCJW_HhuSeM1HTeOp_QHhM,1794
|
|
3
|
+
spatial_foundation/periods.py,sha256=dYqwonfGY9mFjcRaIYDZ-OPy_hOZlmr2hiPoO5T5DKs,1568
|
|
4
|
+
spatial_foundation/geography/__init__.py,sha256=npTzZR_375PyzYnw_tAS5AXst0wN0wshy6UgEpzQBto,576
|
|
5
|
+
spatial_foundation/geography/gadm.py,sha256=ppmEjdcjLTXFZ8Uh0jU9j-zyhok6VTgQdVZhdZHIuIg,3388
|
|
6
|
+
spatial_foundation/geography/materialize.py,sha256=BgFWJZmYUgkbX-Sos4bzFxLw706MsuuPexGZOziNJNA,15447
|
|
7
|
+
spatial_foundation/geography/membership.py,sha256=UddRKWwesOrxCoepsocw3jkZIsaQ2kAtKRobSpYhzKI,2954
|
|
8
|
+
spatial_foundation/geography/models.py,sha256=qdDWCygh3bk8zH4GYPas06Wao8J5BeEu2sb2V-JJkBw,991
|
|
9
|
+
spatial_foundation/geography/overlap.py,sha256=6aUmSc4hzg-KJmUND5nMpptojPkOoU3yFIMnorcYe50,7360
|
|
10
|
+
spatial_foundation/presentation/__init__.py,sha256=bRN9F32Xyi20HYH0uLNLfJq36M0CPEHljRQ4i6BLlLs,239
|
|
11
|
+
spatial_foundation/presentation/basemaps.py,sha256=8jP7NcUrtfT_LnUig3Onz1GetC6QxrvLbDa35rCqcvE,2785
|
|
12
|
+
spatial_foundation/presentation/plotting.py,sha256=3_moU3OPdKf_E2dxFdLOYnb6LFaxqC8Ix2l22allhuw,786
|
|
13
|
+
spatial_data_foundation-0.1.0.dist-info/METADATA,sha256=uDVIezdAnH5pO6NgFP-3hGFnKK3b_a8BUVE6FzP4j_0,1156
|
|
14
|
+
spatial_data_foundation-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
15
|
+
spatial_data_foundation-0.1.0.dist-info/top_level.txt,sha256=96_8oHgbT_BKvhDHctnGy-57j29d1FMC5uOvEaJ_j8s,19
|
|
16
|
+
spatial_data_foundation-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
spatial_foundation
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from .catalog import DataRoot, register_external_snapshot, sha256_file
|
|
2
|
+
from .geography import (
|
|
3
|
+
ArealOverlapAudit,
|
|
4
|
+
GADMMaterialization,
|
|
5
|
+
materialize_gadm,
|
|
6
|
+
relate_areal_objects,
|
|
7
|
+
)
|
|
8
|
+
from .periods import Period, PeriodIndex
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"ArealOverlapAudit",
|
|
12
|
+
"DataRoot",
|
|
13
|
+
"GADMMaterialization",
|
|
14
|
+
"Period",
|
|
15
|
+
"PeriodIndex",
|
|
16
|
+
"materialize_gadm",
|
|
17
|
+
"register_external_snapshot",
|
|
18
|
+
"relate_areal_objects",
|
|
19
|
+
"sha256_file",
|
|
20
|
+
]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from hashlib import sha256
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from empirical_contracts import SourceFileRef, SourceSnapshotRef
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class DataRoot:
|
|
12
|
+
root: Path
|
|
13
|
+
|
|
14
|
+
@classmethod
|
|
15
|
+
def from_path(cls, value: str | Path) -> DataRoot:
|
|
16
|
+
return cls(Path(value).expanduser().resolve())
|
|
17
|
+
|
|
18
|
+
def bronze(self, source: str, release: str, snapshot_id: str) -> Path:
|
|
19
|
+
return self.root / "bronze" / source / release / snapshot_id
|
|
20
|
+
|
|
21
|
+
def silver(self, domain: str, dataset: str, version: str) -> Path:
|
|
22
|
+
return self.root / "silver" / domain / dataset / version
|
|
23
|
+
|
|
24
|
+
def gold(self, domain: str, dataset: str, version: str) -> Path:
|
|
25
|
+
return self.root / "gold" / domain / dataset / version
|
|
26
|
+
|
|
27
|
+
def run(self, package: str, run_id: str) -> Path:
|
|
28
|
+
return self.root / "runs" / package / run_id
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sha256_file(path: str | Path, chunk_size: int = 1024 * 1024) -> str:
|
|
32
|
+
digest = sha256()
|
|
33
|
+
with Path(path).open("rb") as handle:
|
|
34
|
+
while chunk := handle.read(chunk_size):
|
|
35
|
+
digest.update(chunk)
|
|
36
|
+
return digest.hexdigest()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def register_external_snapshot(source: str, release: str, paths: list[str | Path]) -> SourceSnapshotRef:
|
|
40
|
+
resolved_paths = sorted(Path(raw).expanduser().resolve() for raw in paths)
|
|
41
|
+
refs = []
|
|
42
|
+
for path in resolved_paths:
|
|
43
|
+
stat = path.stat()
|
|
44
|
+
refs.append(SourceFileRef(path=str(path), sha256=sha256_file(path), size_bytes=stat.st_size))
|
|
45
|
+
short = sha256("".join(ref.sha256 for ref in refs).encode()).hexdigest()[:12]
|
|
46
|
+
return SourceSnapshotRef(
|
|
47
|
+
source=source,
|
|
48
|
+
release=release,
|
|
49
|
+
snapshot_id=f"{source}-{release}-{short}",
|
|
50
|
+
storage_mode="external_immutable",
|
|
51
|
+
files=tuple(refs),
|
|
52
|
+
)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from .gadm import normalize_gadm_frame
|
|
2
|
+
from .materialize import GADMMaterialization, materialize_gadm
|
|
3
|
+
from .membership import MembershipAudit, assign_points
|
|
4
|
+
from .models import GeographyUnit, GeometryRole, MembershipStatus, geography_uid
|
|
5
|
+
from .overlap import ArealOverlapAudit, relate_areal_objects
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"ArealOverlapAudit",
|
|
9
|
+
"GADMMaterialization",
|
|
10
|
+
"GeographyUnit",
|
|
11
|
+
"GeometryRole",
|
|
12
|
+
"MembershipAudit",
|
|
13
|
+
"MembershipStatus",
|
|
14
|
+
"assign_points",
|
|
15
|
+
"geography_uid",
|
|
16
|
+
"materialize_gadm",
|
|
17
|
+
"normalize_gadm_frame",
|
|
18
|
+
"relate_areal_objects",
|
|
19
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import geopandas as gpd
|
|
4
|
+
|
|
5
|
+
from .models import GeometryRole, geography_uid
|
|
6
|
+
|
|
7
|
+
REQUIRED_BASE = {"GID_0"}
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _require_identity(series, *, label: str):
|
|
11
|
+
missing = series.isna() | series.astype("string").str.strip().eq("")
|
|
12
|
+
if missing.any():
|
|
13
|
+
raise ValueError(f"GADM {label} is not resolvable for {int(missing.sum())} row(s)")
|
|
14
|
+
return series.astype(str)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def normalize_gadm_frame(
|
|
18
|
+
frame: gpd.GeoDataFrame,
|
|
19
|
+
*,
|
|
20
|
+
version: str,
|
|
21
|
+
level: int,
|
|
22
|
+
area_crs: str = "EPSG:6933",
|
|
23
|
+
) -> gpd.GeoDataFrame:
|
|
24
|
+
"""Normalize one native GADM level without simplifying analytical geometry.
|
|
25
|
+
|
|
26
|
+
Expected source fields are GID_0 .. GID_{level}. Parent identity is retained when
|
|
27
|
+
available. Geometry is stored in EPSG:4326; area is calculated in the declared
|
|
28
|
+
equal-area CRS. The returned frame records the area CRS and geometry validity
|
|
29
|
+
profile in ``GeoDataFrame.attrs`` for run-manifest/provenance capture.
|
|
30
|
+
"""
|
|
31
|
+
if level < 0:
|
|
32
|
+
raise ValueError("GADM admin level must be >= 0")
|
|
33
|
+
if not version.strip():
|
|
34
|
+
raise ValueError("GADM version must be non-empty")
|
|
35
|
+
|
|
36
|
+
gid_col = f"GID_{level}"
|
|
37
|
+
parent_col = f"GID_{level - 1}" if level > 0 else None
|
|
38
|
+
required = REQUIRED_BASE | {gid_col}
|
|
39
|
+
missing = required - set(frame.columns)
|
|
40
|
+
if missing:
|
|
41
|
+
raise ValueError(f"missing required GADM columns: {sorted(missing)}")
|
|
42
|
+
if frame.crs is None:
|
|
43
|
+
raise ValueError("GADM source must have a declared CRS")
|
|
44
|
+
|
|
45
|
+
source_ids = _require_identity(frame[gid_col], label="source geography identity")
|
|
46
|
+
country_ids = _require_identity(frame["GID_0"], label="country identity")
|
|
47
|
+
|
|
48
|
+
data = frame.to_crs("EPSG:4326").copy()
|
|
49
|
+
metric = data.to_crs(area_crs)
|
|
50
|
+
data["area_km2"] = metric.geometry.area / 1_000_000.0
|
|
51
|
+
data["provider"] = "gadm"
|
|
52
|
+
data["version"] = version
|
|
53
|
+
data["source_geo_id"] = source_ids.to_numpy()
|
|
54
|
+
data["country_iso3"] = country_ids.str.slice(0, 3).to_numpy()
|
|
55
|
+
data["native_admin_level"] = level
|
|
56
|
+
data["geo_uid"] = data["source_geo_id"].map(lambda x: geography_uid("gadm", version, level, x))
|
|
57
|
+
if parent_col and parent_col in data.columns:
|
|
58
|
+
parent_present = data[parent_col].notna() & data[parent_col].astype("string").str.strip().ne("")
|
|
59
|
+
data["parent_geo_uid"] = [
|
|
60
|
+
geography_uid("gadm", version, level - 1, str(value)) if present else None
|
|
61
|
+
for value, present in zip(data[parent_col], parent_present)
|
|
62
|
+
]
|
|
63
|
+
else:
|
|
64
|
+
data["parent_geo_uid"] = None
|
|
65
|
+
data["geometry_role"] = GeometryRole.ANALYTICAL.value
|
|
66
|
+
|
|
67
|
+
keep = [
|
|
68
|
+
"geo_uid",
|
|
69
|
+
"provider",
|
|
70
|
+
"version",
|
|
71
|
+
"source_geo_id",
|
|
72
|
+
"country_iso3",
|
|
73
|
+
"native_admin_level",
|
|
74
|
+
"parent_geo_uid",
|
|
75
|
+
"area_km2",
|
|
76
|
+
"geometry_role",
|
|
77
|
+
"geometry",
|
|
78
|
+
]
|
|
79
|
+
out = data[keep].copy()
|
|
80
|
+
if out["geo_uid"].duplicated().any():
|
|
81
|
+
dupes = out.loc[out["geo_uid"].duplicated(keep=False), "geo_uid"].unique().tolist()[:10]
|
|
82
|
+
raise ValueError(f"duplicate geography IDs after normalization: {dupes}")
|
|
83
|
+
|
|
84
|
+
null_geometry = out.geometry.isna()
|
|
85
|
+
invalid_geometry = ~null_geometry & ~out.geometry.is_valid
|
|
86
|
+
out.attrs["area_crs"] = area_crs
|
|
87
|
+
out.attrs["geometry_profile"] = {
|
|
88
|
+
"rows": len(out),
|
|
89
|
+
"null": int(null_geometry.sum()),
|
|
90
|
+
"invalid": int(invalid_geometry.sum()),
|
|
91
|
+
}
|
|
92
|
+
return out
|
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import subprocess
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from hashlib import sha256
|
|
10
|
+
from importlib.metadata import PackageNotFoundError
|
|
11
|
+
from importlib.metadata import version as package_version
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import geopandas as gpd
|
|
15
|
+
import pandas as pd
|
|
16
|
+
from empirical_contracts import (
|
|
17
|
+
AuthorityLevel,
|
|
18
|
+
DataLayer,
|
|
19
|
+
DatasetRef,
|
|
20
|
+
GeographySpec,
|
|
21
|
+
GrainSpec,
|
|
22
|
+
QAResult,
|
|
23
|
+
RunManifest,
|
|
24
|
+
SourceSnapshotRef,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
from ..catalog import DataRoot, sha256_file
|
|
28
|
+
from .gadm import normalize_gadm_frame
|
|
29
|
+
|
|
30
|
+
PACKAGE_NAME = "spatial-data-foundation"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class GADMMaterialization:
|
|
35
|
+
run_id: str
|
|
36
|
+
dataset_root: Path
|
|
37
|
+
run_root: Path
|
|
38
|
+
outputs: dict[int, Path]
|
|
39
|
+
manifest_path: Path
|
|
40
|
+
qa_path: Path
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _utc_now() -> datetime:
|
|
44
|
+
return datetime.now(timezone.utc)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _package_version() -> str:
|
|
48
|
+
try:
|
|
49
|
+
return package_version(PACKAGE_NAME)
|
|
50
|
+
except PackageNotFoundError:
|
|
51
|
+
return "0.1.0"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _resolve_code_commit(explicit: str | None) -> str:
|
|
55
|
+
if explicit and explicit.strip():
|
|
56
|
+
return explicit.strip()
|
|
57
|
+
env_value = os.environ.get("SPATIAL_DATA_FOUNDATION_CODE_COMMIT")
|
|
58
|
+
if env_value and env_value.strip():
|
|
59
|
+
return env_value.strip()
|
|
60
|
+
|
|
61
|
+
repo_root = Path(__file__).resolve().parents[3]
|
|
62
|
+
try:
|
|
63
|
+
result = subprocess.run(
|
|
64
|
+
["git", "rev-parse", "HEAD"],
|
|
65
|
+
cwd=repo_root,
|
|
66
|
+
check=True,
|
|
67
|
+
capture_output=True,
|
|
68
|
+
text=True,
|
|
69
|
+
timeout=2,
|
|
70
|
+
)
|
|
71
|
+
except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
72
|
+
result = None
|
|
73
|
+
if result and result.stdout.strip():
|
|
74
|
+
return result.stdout.strip()
|
|
75
|
+
raise ValueError(
|
|
76
|
+
"code_commit is required when Git metadata is unavailable; pass it explicitly "
|
|
77
|
+
"or set SPATIAL_DATA_FOUNDATION_CODE_COMMIT"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _validate_levels(levels: Iterable[int]) -> tuple[int, ...]:
|
|
82
|
+
normalized = tuple(sorted(set(levels)))
|
|
83
|
+
if not normalized:
|
|
84
|
+
raise ValueError("at least one GADM level is required")
|
|
85
|
+
if any(not isinstance(level, int) or isinstance(level, bool) or level < 0 for level in normalized):
|
|
86
|
+
raise ValueError("GADM levels must be non-negative integers")
|
|
87
|
+
return normalized
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _validate_run_id(run_id: str) -> str:
|
|
91
|
+
if not run_id or run_id in {".", ".."} or "/" in run_id or "\\" in run_id:
|
|
92
|
+
raise ValueError("run_id must be a non-empty path-safe identifier")
|
|
93
|
+
return run_id
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _default_run_id(
|
|
97
|
+
*,
|
|
98
|
+
snapshot: SourceSnapshotRef,
|
|
99
|
+
levels: tuple[int, ...],
|
|
100
|
+
area_crs: str,
|
|
101
|
+
code_commit: str,
|
|
102
|
+
started_at: datetime,
|
|
103
|
+
) -> str:
|
|
104
|
+
payload = "|".join(
|
|
105
|
+
[snapshot.snapshot_id, ",".join(map(str, levels)), area_crs, code_commit]
|
|
106
|
+
).encode()
|
|
107
|
+
short = sha256(payload).hexdigest()[:8]
|
|
108
|
+
stamp = started_at.strftime("%Y%m%dT%H%M%S%fZ")
|
|
109
|
+
release = snapshot.release.replace(".", "-").replace("/", "-")
|
|
110
|
+
return f"gadm-{release}-{stamp}-{short}"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _write_json(path: Path, payload: dict) -> None:
|
|
114
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
115
|
+
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _verify_snapshot(snapshot: SourceSnapshotRef) -> None:
|
|
119
|
+
if snapshot.source.lower() != "gadm":
|
|
120
|
+
raise ValueError(f"materialize_gadm requires a GADM snapshot, got {snapshot.source!r}")
|
|
121
|
+
for ref in snapshot.files:
|
|
122
|
+
path = Path(ref.path)
|
|
123
|
+
if not path.exists():
|
|
124
|
+
raise ValueError(f"source snapshot file is missing: {path}")
|
|
125
|
+
actual_hash = sha256_file(path)
|
|
126
|
+
if actual_hash != ref.sha256:
|
|
127
|
+
raise ValueError(
|
|
128
|
+
f"source snapshot hash mismatch for {path}: expected {ref.sha256}, got {actual_hash}"
|
|
129
|
+
)
|
|
130
|
+
actual_size = path.stat().st_size
|
|
131
|
+
if actual_size != ref.size_bytes:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"source snapshot size mismatch for {path}: expected {ref.size_bytes}, got {actual_size}"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _native_level(frame: gpd.GeoDataFrame) -> int | None:
|
|
138
|
+
levels = []
|
|
139
|
+
for column in frame.columns:
|
|
140
|
+
if not column.startswith("GID_"):
|
|
141
|
+
continue
|
|
142
|
+
suffix = column[4:]
|
|
143
|
+
if suffix.isdigit():
|
|
144
|
+
levels.append(int(suffix))
|
|
145
|
+
return max(levels) if levels else None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _read_single_level(path: Path, level: int) -> gpd.GeoDataFrame | None:
|
|
149
|
+
suffix = path.suffix.lower()
|
|
150
|
+
if suffix in {".parquet", ".geoparquet"}:
|
|
151
|
+
frame = gpd.read_parquet(path)
|
|
152
|
+
return frame if _native_level(frame) == level else None
|
|
153
|
+
|
|
154
|
+
if suffix == ".gpkg":
|
|
155
|
+
layer = f"ADM_ADM_{level}"
|
|
156
|
+
try:
|
|
157
|
+
frame = gpd.read_file(path, layer=layer)
|
|
158
|
+
except (ValueError, RuntimeError, OSError):
|
|
159
|
+
try:
|
|
160
|
+
frame = gpd.read_file(path)
|
|
161
|
+
except (ValueError, RuntimeError, OSError):
|
|
162
|
+
return None
|
|
163
|
+
return frame if _native_level(frame) == level else None
|
|
164
|
+
|
|
165
|
+
try:
|
|
166
|
+
frame = gpd.read_file(path)
|
|
167
|
+
except (ValueError, RuntimeError, OSError) as exc:
|
|
168
|
+
raise ValueError(f"cannot read registered GADM source file {path}: {exc}") from exc
|
|
169
|
+
return frame if _native_level(frame) == level else None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _read_level(
|
|
173
|
+
snapshot: SourceSnapshotRef,
|
|
174
|
+
level: int,
|
|
175
|
+
*,
|
|
176
|
+
area_crs: str,
|
|
177
|
+
) -> gpd.GeoDataFrame:
|
|
178
|
+
normalized_parts = []
|
|
179
|
+
for ref in snapshot.files:
|
|
180
|
+
source = _read_single_level(Path(ref.path), level)
|
|
181
|
+
if source is None:
|
|
182
|
+
continue
|
|
183
|
+
normalized_parts.append(
|
|
184
|
+
normalize_gadm_frame(
|
|
185
|
+
source,
|
|
186
|
+
version=snapshot.release,
|
|
187
|
+
level=level,
|
|
188
|
+
area_crs=area_crs,
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
if not normalized_parts:
|
|
193
|
+
raise ValueError(
|
|
194
|
+
f"registered GADM snapshot {snapshot.snapshot_id!r} contains no readable ADM{level} layer"
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
combined = gpd.GeoDataFrame(
|
|
198
|
+
pd.concat(normalized_parts, ignore_index=True),
|
|
199
|
+
geometry="geometry",
|
|
200
|
+
crs="EPSG:4326",
|
|
201
|
+
)
|
|
202
|
+
if combined["geo_uid"].duplicated().any():
|
|
203
|
+
duplicates = int(combined["geo_uid"].duplicated(keep=False).sum())
|
|
204
|
+
raise ValueError(f"duplicate geography IDs across registered GADM files: {duplicates} row(s)")
|
|
205
|
+
return combined
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _level_metrics(frame: gpd.GeoDataFrame) -> dict[str, int]:
|
|
209
|
+
null_geometry = frame.geometry.isna()
|
|
210
|
+
invalid_geometry = ~null_geometry & ~frame.geometry.is_valid
|
|
211
|
+
return {
|
|
212
|
+
"row_count": len(frame),
|
|
213
|
+
"country_count": int(frame["country_iso3"].nunique(dropna=True)),
|
|
214
|
+
"null_geometry_count": int(null_geometry.sum()),
|
|
215
|
+
"invalid_geometry_count": int(invalid_geometry.sum()),
|
|
216
|
+
"duplicate_id_count": int(frame["geo_uid"].duplicated(keep=False).sum()),
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _dataset_ref(level: int, release: str, content_hash: str) -> DatasetRef:
|
|
221
|
+
return DatasetRef(
|
|
222
|
+
dataset_id=f"gadm_native_adm{level}",
|
|
223
|
+
version=release,
|
|
224
|
+
schema_version="1",
|
|
225
|
+
layer=DataLayer.SILVER,
|
|
226
|
+
authority=AuthorityLevel.L1_NORMALIZED,
|
|
227
|
+
grain=GrainSpec(keys=("geo_uid",)),
|
|
228
|
+
geography=GeographySpec(
|
|
229
|
+
provider="gadm",
|
|
230
|
+
version=release,
|
|
231
|
+
scheme="native",
|
|
232
|
+
level=f"adm{level}",
|
|
233
|
+
),
|
|
234
|
+
content_sha256=content_hash,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def materialize_gadm(
|
|
239
|
+
*,
|
|
240
|
+
snapshot: SourceSnapshotRef,
|
|
241
|
+
levels: Iterable[int],
|
|
242
|
+
output_root: str | Path,
|
|
243
|
+
area_crs: str = "EPSG:6933",
|
|
244
|
+
code_commit: str | None = None,
|
|
245
|
+
run_id: str | None = None,
|
|
246
|
+
overwrite: bool = False,
|
|
247
|
+
) -> GADMMaterialization:
|
|
248
|
+
"""Materialize registered GADM sources as auditable silver GeoParquet assets.
|
|
249
|
+
|
|
250
|
+
Source bytes are never downloaded or copied by this function. Registered hashes are
|
|
251
|
+
revalidated before reading. All requested levels are normalized and staged before
|
|
252
|
+
final silver paths are published. Run provenance and geography QA are persisted as
|
|
253
|
+
JSON companions under ``runs/spatial-data-foundation/<run_id>/``.
|
|
254
|
+
|
|
255
|
+
The registered snapshot may contain separate level files or standard GADM 4.x
|
|
256
|
+
GeoPackages with ``ADM_ADM_<level>`` layers. GeoParquet materialization requires
|
|
257
|
+
the package ``io`` extra (pyarrow).
|
|
258
|
+
"""
|
|
259
|
+
try:
|
|
260
|
+
import pyarrow # noqa: F401
|
|
261
|
+
except ImportError as exc:
|
|
262
|
+
raise RuntimeError(
|
|
263
|
+
"materialize_gadm requires pyarrow; install spatial-data-foundation[io]"
|
|
264
|
+
) from exc
|
|
265
|
+
|
|
266
|
+
requested_levels = _validate_levels(levels)
|
|
267
|
+
if not area_crs or not area_crs.strip():
|
|
268
|
+
raise ValueError("area_crs must be non-empty")
|
|
269
|
+
commit = _resolve_code_commit(code_commit)
|
|
270
|
+
started_at = _utc_now()
|
|
271
|
+
resolved_run_id = _validate_run_id(
|
|
272
|
+
run_id
|
|
273
|
+
or _default_run_id(
|
|
274
|
+
snapshot=snapshot,
|
|
275
|
+
levels=requested_levels,
|
|
276
|
+
area_crs=area_crs,
|
|
277
|
+
code_commit=commit,
|
|
278
|
+
started_at=started_at,
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
data_root = DataRoot.from_path(output_root)
|
|
283
|
+
dataset_root = data_root.silver("geography", "gadm", snapshot.release)
|
|
284
|
+
run_root = data_root.run(PACKAGE_NAME, resolved_run_id)
|
|
285
|
+
manifest_path = run_root / "run_manifest.json"
|
|
286
|
+
qa_path = run_root / "geography_qa.json"
|
|
287
|
+
run_root.mkdir(parents=True, exist_ok=True)
|
|
288
|
+
|
|
289
|
+
outputs = {level: dataset_root / f"adm{level}.geoparquet" for level in requested_levels}
|
|
290
|
+
temp_paths = {level: dataset_root / f".adm{level}.geoparquet.tmp" for level in requested_levels}
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
_verify_snapshot(snapshot)
|
|
294
|
+
normalized = {
|
|
295
|
+
level: _read_level(snapshot, level, area_crs=area_crs)
|
|
296
|
+
for level in requested_levels
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
for path in outputs.values():
|
|
300
|
+
if path.exists() and not overwrite:
|
|
301
|
+
raise FileExistsError(
|
|
302
|
+
f"refusing to overwrite existing GADM asset {path}; pass overwrite=True explicitly"
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
dataset_root.mkdir(parents=True, exist_ok=True)
|
|
306
|
+
output_hashes: dict[int, str] = {}
|
|
307
|
+
level_qa: dict[str, dict[str, int | str]] = {}
|
|
308
|
+
for level, frame in normalized.items():
|
|
309
|
+
temp = temp_paths[level]
|
|
310
|
+
if temp.exists():
|
|
311
|
+
temp.unlink()
|
|
312
|
+
frame.to_parquet(temp, index=False)
|
|
313
|
+
output_hash = sha256_file(temp)
|
|
314
|
+
output_hashes[level] = output_hash
|
|
315
|
+
level_qa[str(level)] = {
|
|
316
|
+
**_level_metrics(frame),
|
|
317
|
+
"output_path": str(outputs[level]),
|
|
318
|
+
"output_sha256": output_hash,
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
dataset_refs = tuple(
|
|
322
|
+
_dataset_ref(level, snapshot.release, output_hashes[level])
|
|
323
|
+
for level in requested_levels
|
|
324
|
+
)
|
|
325
|
+
qa_results = [
|
|
326
|
+
QAResult(
|
|
327
|
+
check_id="gadm_source_snapshot_integrity",
|
|
328
|
+
state="GREEN",
|
|
329
|
+
message="registered GADM source files match their snapshot hashes",
|
|
330
|
+
metrics={"file_count": len(snapshot.files)},
|
|
331
|
+
)
|
|
332
|
+
]
|
|
333
|
+
for level in requested_levels:
|
|
334
|
+
metrics = level_qa[str(level)]
|
|
335
|
+
has_geometry_issue = bool(
|
|
336
|
+
metrics["null_geometry_count"] or metrics["invalid_geometry_count"]
|
|
337
|
+
)
|
|
338
|
+
qa_results.append(
|
|
339
|
+
QAResult(
|
|
340
|
+
check_id=f"gadm_adm{level}_geometry_profile",
|
|
341
|
+
state="YELLOW" if has_geometry_issue else "GREEN",
|
|
342
|
+
message=(
|
|
343
|
+
"geometry profile contains null or invalid geometries"
|
|
344
|
+
if has_geometry_issue
|
|
345
|
+
else "geometry profile is clean"
|
|
346
|
+
),
|
|
347
|
+
metrics={
|
|
348
|
+
"row_count": metrics["row_count"],
|
|
349
|
+
"country_count": metrics["country_count"],
|
|
350
|
+
"null_geometry_count": metrics["null_geometry_count"],
|
|
351
|
+
"invalid_geometry_count": metrics["invalid_geometry_count"],
|
|
352
|
+
"duplicate_id_count": metrics["duplicate_id_count"],
|
|
353
|
+
},
|
|
354
|
+
)
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
finished_at = _utc_now()
|
|
358
|
+
manifest = RunManifest(
|
|
359
|
+
run_id=resolved_run_id,
|
|
360
|
+
package=PACKAGE_NAME,
|
|
361
|
+
package_version=_package_version(),
|
|
362
|
+
code_commit=commit,
|
|
363
|
+
started_at=started_at,
|
|
364
|
+
finished_at=finished_at,
|
|
365
|
+
inputs=(snapshot,),
|
|
366
|
+
parameters={
|
|
367
|
+
"provider": "gadm",
|
|
368
|
+
"version": snapshot.release,
|
|
369
|
+
"area_crs": area_crs,
|
|
370
|
+
"requested_levels": list(requested_levels),
|
|
371
|
+
"available_levels": list(requested_levels),
|
|
372
|
+
"output_files": {
|
|
373
|
+
str(level): {
|
|
374
|
+
"path": str(outputs[level]),
|
|
375
|
+
"sha256": output_hashes[level],
|
|
376
|
+
}
|
|
377
|
+
for level in requested_levels
|
|
378
|
+
},
|
|
379
|
+
},
|
|
380
|
+
outputs=dataset_refs,
|
|
381
|
+
qa=tuple(qa_results),
|
|
382
|
+
)
|
|
383
|
+
qa_payload = {
|
|
384
|
+
"state": "YELLOW"
|
|
385
|
+
if any(result.state == "YELLOW" for result in qa_results)
|
|
386
|
+
else "GREEN",
|
|
387
|
+
"provider": "gadm",
|
|
388
|
+
"version": snapshot.release,
|
|
389
|
+
"area_crs": area_crs,
|
|
390
|
+
"available_levels": list(requested_levels),
|
|
391
|
+
"source_snapshot": snapshot.model_dump(mode="json"),
|
|
392
|
+
"levels": level_qa,
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
for level in requested_levels:
|
|
396
|
+
temp_paths[level].replace(outputs[level])
|
|
397
|
+
_write_json(manifest_path, manifest.model_dump(mode="json"))
|
|
398
|
+
_write_json(qa_path, qa_payload)
|
|
399
|
+
|
|
400
|
+
return GADMMaterialization(
|
|
401
|
+
run_id=resolved_run_id,
|
|
402
|
+
dataset_root=dataset_root,
|
|
403
|
+
run_root=run_root,
|
|
404
|
+
outputs=outputs,
|
|
405
|
+
manifest_path=manifest_path,
|
|
406
|
+
qa_path=qa_path,
|
|
407
|
+
)
|
|
408
|
+
except Exception as exc:
|
|
409
|
+
for temp in temp_paths.values():
|
|
410
|
+
if temp.exists():
|
|
411
|
+
temp.unlink()
|
|
412
|
+
failure = QAResult(
|
|
413
|
+
check_id="gadm_materialization",
|
|
414
|
+
state="RED",
|
|
415
|
+
message=str(exc),
|
|
416
|
+
metrics={"requested_level_count": len(requested_levels)},
|
|
417
|
+
)
|
|
418
|
+
failure_manifest = RunManifest(
|
|
419
|
+
run_id=resolved_run_id,
|
|
420
|
+
package=PACKAGE_NAME,
|
|
421
|
+
package_version=_package_version(),
|
|
422
|
+
code_commit=commit,
|
|
423
|
+
started_at=started_at,
|
|
424
|
+
finished_at=_utc_now(),
|
|
425
|
+
inputs=(snapshot,),
|
|
426
|
+
parameters={
|
|
427
|
+
"provider": "gadm",
|
|
428
|
+
"version": snapshot.release,
|
|
429
|
+
"area_crs": area_crs,
|
|
430
|
+
"requested_levels": list(requested_levels),
|
|
431
|
+
"available_levels": [],
|
|
432
|
+
},
|
|
433
|
+
outputs=(),
|
|
434
|
+
qa=(failure,),
|
|
435
|
+
)
|
|
436
|
+
_write_json(manifest_path, failure_manifest.model_dump(mode="json"))
|
|
437
|
+
_write_json(
|
|
438
|
+
qa_path,
|
|
439
|
+
{
|
|
440
|
+
"state": "RED",
|
|
441
|
+
"provider": "gadm",
|
|
442
|
+
"version": snapshot.release,
|
|
443
|
+
"area_crs": area_crs,
|
|
444
|
+
"available_levels": [],
|
|
445
|
+
"source_snapshot": snapshot.model_dump(mode="json"),
|
|
446
|
+
"error": str(exc),
|
|
447
|
+
"levels": {},
|
|
448
|
+
},
|
|
449
|
+
)
|
|
450
|
+
raise
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import geopandas as gpd
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from .models import MembershipStatus
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True)
|
|
12
|
+
class MembershipAudit:
|
|
13
|
+
input_points: int
|
|
14
|
+
matched_unique: int
|
|
15
|
+
unmatched_outside: int
|
|
16
|
+
ambiguous_multiple: int
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def assign_points(
|
|
20
|
+
points: gpd.GeoDataFrame,
|
|
21
|
+
polygons: gpd.GeoDataFrame,
|
|
22
|
+
*,
|
|
23
|
+
point_id_col: str,
|
|
24
|
+
polygon_id_col: str = "geo_uid",
|
|
25
|
+
) -> tuple[pd.DataFrame, MembershipAudit]:
|
|
26
|
+
"""Return candidate memberships and explicit point-level assignment status.
|
|
27
|
+
|
|
28
|
+
Boundary/multiple-polygon cases are *not* silently tie-broken. A downstream
|
|
29
|
+
source-specific policy may decide what to do with them.
|
|
30
|
+
"""
|
|
31
|
+
if points.crs is None or polygons.crs is None:
|
|
32
|
+
raise ValueError("both points and polygons require a CRS")
|
|
33
|
+
if point_id_col not in points.columns:
|
|
34
|
+
raise ValueError(f"missing point id column: {point_id_col}")
|
|
35
|
+
if polygon_id_col not in polygons.columns:
|
|
36
|
+
raise ValueError(f"missing polygon id column: {polygon_id_col}")
|
|
37
|
+
if points[point_id_col].duplicated().any():
|
|
38
|
+
raise ValueError("point IDs must be unique")
|
|
39
|
+
if polygons[polygon_id_col].duplicated().any():
|
|
40
|
+
raise ValueError("polygon IDs must be unique")
|
|
41
|
+
if "geometry_role" in polygons.columns:
|
|
42
|
+
roles = polygons["geometry_role"].astype("string")
|
|
43
|
+
if roles.isna().any() or roles.ne("analytical").any():
|
|
44
|
+
raise ValueError("point assignment requires analytical geometry")
|
|
45
|
+
|
|
46
|
+
left = points[[point_id_col, "geometry"]].to_crs(polygons.crs)
|
|
47
|
+
right = polygons[[polygon_id_col, "geometry"]]
|
|
48
|
+
|
|
49
|
+
# intersects intentionally preserves exact-boundary candidates instead of
|
|
50
|
+
# dropping them as `within` could do.
|
|
51
|
+
joined = gpd.sjoin(left, right, how="left", predicate="intersects")
|
|
52
|
+
candidates = joined[[point_id_col, polygon_id_col]].copy()
|
|
53
|
+
|
|
54
|
+
counts = candidates.groupby(point_id_col)[polygon_id_col].count()
|
|
55
|
+
status_rows = []
|
|
56
|
+
for point_id in left[point_id_col]:
|
|
57
|
+
n = int(counts.get(point_id, 0))
|
|
58
|
+
if n == 0:
|
|
59
|
+
status = MembershipStatus.UNMATCHED_OUTSIDE.value
|
|
60
|
+
elif n == 1:
|
|
61
|
+
status = MembershipStatus.MATCHED_UNIQUE.value
|
|
62
|
+
else:
|
|
63
|
+
status = MembershipStatus.AMBIGUOUS_MULTIPLE.value
|
|
64
|
+
status_rows.append({point_id_col: point_id, "candidate_count": n, "assignment_status": status})
|
|
65
|
+
status_df = pd.DataFrame(status_rows)
|
|
66
|
+
result = candidates.merge(status_df, on=point_id_col, how="left")
|
|
67
|
+
|
|
68
|
+
audit = MembershipAudit(
|
|
69
|
+
input_points=len(left),
|
|
70
|
+
matched_unique=int((status_df.assignment_status == MembershipStatus.MATCHED_UNIQUE.value).sum()),
|
|
71
|
+
unmatched_outside=int((status_df.assignment_status == MembershipStatus.UNMATCHED_OUTSIDE.value).sum()),
|
|
72
|
+
ambiguous_multiple=int((status_df.assignment_status == MembershipStatus.AMBIGUOUS_MULTIPLE.value).sum()),
|
|
73
|
+
)
|
|
74
|
+
return result, audit
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from enum import Enum
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class FrozenModel(BaseModel):
|
|
9
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class GeometryRole(str, Enum):
|
|
13
|
+
ANALYTICAL = "analytical"
|
|
14
|
+
DISPLAY = "display"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class MembershipStatus(str, Enum):
|
|
18
|
+
MATCHED_UNIQUE = "matched_unique"
|
|
19
|
+
UNMATCHED_OUTSIDE = "unmatched_outside"
|
|
20
|
+
AMBIGUOUS_MULTIPLE = "ambiguous_multiple"
|
|
21
|
+
INVALID_POINT = "invalid_point"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class GeographyUnit(FrozenModel):
|
|
25
|
+
geo_uid: str
|
|
26
|
+
provider: str
|
|
27
|
+
version: str
|
|
28
|
+
source_geo_id: str
|
|
29
|
+
country_iso3: str
|
|
30
|
+
native_admin_level: int = Field(ge=0)
|
|
31
|
+
parent_geo_uid: str | None = None
|
|
32
|
+
area_km2: float | None = Field(default=None, ge=0)
|
|
33
|
+
geometry_role: GeometryRole = GeometryRole.ANALYTICAL
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def geography_uid(provider: str, version: str, native_admin_level: int, source_geo_id: str) -> str:
|
|
37
|
+
return f"{provider}:{version}:adm{native_admin_level}:{source_geo_id}"
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import geopandas as gpd
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class ArealOverlapAudit:
|
|
11
|
+
input_objects: int
|
|
12
|
+
matched_single: int
|
|
13
|
+
matched_multiple: int
|
|
14
|
+
unmatched_outside: int
|
|
15
|
+
invalid_geometry: int
|
|
16
|
+
relation_rows: int
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
_AREAL_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _require_analytical_geography(polygons: gpd.GeoDataFrame) -> None:
|
|
23
|
+
if "geometry_role" not in polygons.columns:
|
|
24
|
+
return
|
|
25
|
+
roles = polygons["geometry_role"].astype("string")
|
|
26
|
+
if roles.isna().any() or roles.ne("analytical").any():
|
|
27
|
+
raise ValueError("areal overlap requires analytical geography geometry")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _positive_overlap_pairs(
|
|
31
|
+
objects: gpd.GeoDataFrame,
|
|
32
|
+
polygons: gpd.GeoDataFrame,
|
|
33
|
+
*,
|
|
34
|
+
object_id_col: str,
|
|
35
|
+
polygon_id_col: str,
|
|
36
|
+
area_crs: str,
|
|
37
|
+
min_overlap_area_m2: float,
|
|
38
|
+
) -> pd.DataFrame:
|
|
39
|
+
left = objects[[object_id_col, "geometry"]].to_crs(polygons.crs)
|
|
40
|
+
right = polygons[[polygon_id_col, "geometry"]]
|
|
41
|
+
candidates = gpd.sjoin(left, right, how="inner", predicate="intersects")
|
|
42
|
+
pairs = candidates[[object_id_col, polygon_id_col]].drop_duplicates().copy()
|
|
43
|
+
if pairs.empty:
|
|
44
|
+
return pd.DataFrame(
|
|
45
|
+
columns=[object_id_col, polygon_id_col, "overlap_area_m2", "overlap_share_of_object"]
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
object_area = objects[[object_id_col, "geometry"]].to_crs(area_crs)
|
|
49
|
+
polygon_area = polygons[[polygon_id_col, "geometry"]].to_crs(area_crs)
|
|
50
|
+
object_geometries = object_area.set_index(object_id_col)["geometry"]
|
|
51
|
+
polygon_geometries = polygon_area.set_index(polygon_id_col)["geometry"]
|
|
52
|
+
|
|
53
|
+
overlap_areas: list[float] = []
|
|
54
|
+
object_areas: list[float] = []
|
|
55
|
+
for object_id, polygon_id in pairs[[object_id_col, polygon_id_col]].itertuples(
|
|
56
|
+
index=False, name=None
|
|
57
|
+
):
|
|
58
|
+
object_geometry = object_geometries.loc[object_id]
|
|
59
|
+
polygon_geometry = polygon_geometries.loc[polygon_id]
|
|
60
|
+
overlap_areas.append(float(object_geometry.intersection(polygon_geometry).area))
|
|
61
|
+
object_areas.append(float(object_geometry.area))
|
|
62
|
+
|
|
63
|
+
pairs["overlap_area_m2"] = overlap_areas
|
|
64
|
+
pairs["_object_area_m2"] = object_areas
|
|
65
|
+
pairs = pairs.loc[pairs["overlap_area_m2"] > min_overlap_area_m2].copy()
|
|
66
|
+
pairs["overlap_share_of_object"] = pairs["overlap_area_m2"] / pairs["_object_area_m2"]
|
|
67
|
+
return pairs.drop(columns=["_object_area_m2"]).reset_index(drop=True)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def relate_areal_objects(
|
|
71
|
+
objects: gpd.GeoDataFrame,
|
|
72
|
+
polygons: gpd.GeoDataFrame,
|
|
73
|
+
*,
|
|
74
|
+
object_id_col: str,
|
|
75
|
+
polygon_id_col: str = "geo_uid",
|
|
76
|
+
area_crs: str = "EPSG:6933",
|
|
77
|
+
min_overlap_area_m2: float = 0.0,
|
|
78
|
+
) -> tuple[pd.DataFrame, ArealOverlapAudit]:
|
|
79
|
+
"""Relate areal source objects to analytical geography without assigning ownership.
|
|
80
|
+
|
|
81
|
+
One source geometry may legitimately overlap several geography units. Those rows are
|
|
82
|
+
retained as a many-to-many relation rather than treated as assignment ambiguity.
|
|
83
|
+
Intersection area and the share of the source object's area are geometric facts only;
|
|
84
|
+
they do not allocate money, exposure, population, or any other substantive quantity.
|
|
85
|
+
|
|
86
|
+
Boundary-only touches have zero intersection area and are excluded by default.
|
|
87
|
+
"""
|
|
88
|
+
if min_overlap_area_m2 < 0:
|
|
89
|
+
raise ValueError("min_overlap_area_m2 must be non-negative")
|
|
90
|
+
if objects.crs is None or polygons.crs is None:
|
|
91
|
+
raise ValueError("both objects and polygons require a CRS")
|
|
92
|
+
if object_id_col not in objects.columns:
|
|
93
|
+
raise ValueError(f"missing object id column: {object_id_col}")
|
|
94
|
+
if polygon_id_col not in polygons.columns:
|
|
95
|
+
raise ValueError(f"missing polygon id column: {polygon_id_col}")
|
|
96
|
+
if objects[object_id_col].isna().any() or objects[object_id_col].duplicated().any():
|
|
97
|
+
raise ValueError("object IDs must be non-missing and unique")
|
|
98
|
+
if polygons[polygon_id_col].isna().any() or polygons[polygon_id_col].duplicated().any():
|
|
99
|
+
raise ValueError("polygon IDs must be non-missing and unique")
|
|
100
|
+
_require_analytical_geography(polygons)
|
|
101
|
+
|
|
102
|
+
polygon_geometry = polygons.geometry
|
|
103
|
+
if polygon_geometry.isna().any() or polygon_geometry.is_empty.any() or (~polygon_geometry.is_valid).any():
|
|
104
|
+
raise ValueError("analytical geography contains missing, empty, or invalid geometry")
|
|
105
|
+
|
|
106
|
+
object_geometry = objects.geometry
|
|
107
|
+
nonempty = object_geometry.notna() & ~object_geometry.is_empty
|
|
108
|
+
non_areal = nonempty & ~object_geometry.geom_type.isin(_AREAL_GEOMETRY_TYPES)
|
|
109
|
+
if non_areal.any():
|
|
110
|
+
found = sorted(object_geometry.loc[non_areal].geom_type.unique().tolist())
|
|
111
|
+
raise ValueError(
|
|
112
|
+
"relate_areal_objects accepts Polygon/MultiPolygon source geometry only; "
|
|
113
|
+
f"found {found}. Use point membership for point sources."
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
valid = nonempty & object_geometry.is_valid
|
|
117
|
+
valid_objects = objects.loc[valid, [object_id_col, "geometry"]].copy()
|
|
118
|
+
pairs = _positive_overlap_pairs(
|
|
119
|
+
valid_objects,
|
|
120
|
+
polygons,
|
|
121
|
+
object_id_col=object_id_col,
|
|
122
|
+
polygon_id_col=polygon_id_col,
|
|
123
|
+
area_crs=area_crs,
|
|
124
|
+
min_overlap_area_m2=min_overlap_area_m2,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
overlap_counts = pairs.groupby(object_id_col)[polygon_id_col].count() if len(pairs) else pd.Series(dtype="int64")
|
|
128
|
+
rows: list[dict] = []
|
|
129
|
+
for object_id in objects[object_id_col]:
|
|
130
|
+
if not bool(valid.loc[objects[object_id_col].eq(object_id)].iloc[0]):
|
|
131
|
+
rows.append(
|
|
132
|
+
{
|
|
133
|
+
object_id_col: object_id,
|
|
134
|
+
polygon_id_col: pd.NA,
|
|
135
|
+
"overlap_area_m2": pd.NA,
|
|
136
|
+
"overlap_share_of_object": pd.NA,
|
|
137
|
+
"overlap_count": 0,
|
|
138
|
+
"relation_status": "invalid_geometry",
|
|
139
|
+
}
|
|
140
|
+
)
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
count = int(overlap_counts.get(object_id, 0))
|
|
144
|
+
if count == 0:
|
|
145
|
+
rows.append(
|
|
146
|
+
{
|
|
147
|
+
object_id_col: object_id,
|
|
148
|
+
polygon_id_col: pd.NA,
|
|
149
|
+
"overlap_area_m2": pd.NA,
|
|
150
|
+
"overlap_share_of_object": pd.NA,
|
|
151
|
+
"overlap_count": 0,
|
|
152
|
+
"relation_status": "unmatched_outside",
|
|
153
|
+
}
|
|
154
|
+
)
|
|
155
|
+
continue
|
|
156
|
+
|
|
157
|
+
status = "matched_single" if count == 1 else "matched_multiple"
|
|
158
|
+
object_pairs = pairs.loc[pairs[object_id_col].eq(object_id)]
|
|
159
|
+
for pair in object_pairs.to_dict(orient="records"):
|
|
160
|
+
rows.append(
|
|
161
|
+
{
|
|
162
|
+
**pair,
|
|
163
|
+
"overlap_count": count,
|
|
164
|
+
"relation_status": status,
|
|
165
|
+
}
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
result = pd.DataFrame(rows)
|
|
169
|
+
status_by_object = result[[object_id_col, "relation_status"]].drop_duplicates(object_id_col)
|
|
170
|
+
audit = ArealOverlapAudit(
|
|
171
|
+
input_objects=len(objects),
|
|
172
|
+
matched_single=int(status_by_object["relation_status"].eq("matched_single").sum()),
|
|
173
|
+
matched_multiple=int(status_by_object["relation_status"].eq("matched_multiple").sum()),
|
|
174
|
+
unmatched_outside=int(status_by_object["relation_status"].eq("unmatched_outside").sum()),
|
|
175
|
+
invalid_geometry=int(status_by_object["relation_status"].eq("invalid_geometry").sum()),
|
|
176
|
+
relation_rows=int(result[polygon_id_col].notna().sum()),
|
|
177
|
+
)
|
|
178
|
+
return result, audit
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from datetime import date, datetime
|
|
6
|
+
|
|
7
|
+
from empirical_contracts import PeriodScheme
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Period:
|
|
12
|
+
period_id: str
|
|
13
|
+
start_year: int
|
|
14
|
+
end_year: int
|
|
15
|
+
ordinal: int
|
|
16
|
+
start_date: date
|
|
17
|
+
end_date_exclusive: date
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PeriodIndex:
|
|
21
|
+
def __init__(self, scheme: PeriodScheme):
|
|
22
|
+
self.scheme = scheme
|
|
23
|
+
|
|
24
|
+
def start_year_for(self, value: date | datetime | int) -> int:
|
|
25
|
+
year = value if isinstance(value, int) else value.year
|
|
26
|
+
width = self.scheme.width_years
|
|
27
|
+
return self.scheme.anchor_year + math.floor((year - self.scheme.anchor_year) / width) * width
|
|
28
|
+
|
|
29
|
+
def period_for(self, value: date | datetime | int) -> Period:
|
|
30
|
+
start = self.start_year_for(value)
|
|
31
|
+
width = self.scheme.width_years
|
|
32
|
+
end = start + width - 1
|
|
33
|
+
ordinal = (start - self.scheme.anchor_year) // width
|
|
34
|
+
return Period(
|
|
35
|
+
period_id=f"{start}-{end}",
|
|
36
|
+
start_year=start,
|
|
37
|
+
end_year=end,
|
|
38
|
+
ordinal=ordinal,
|
|
39
|
+
start_date=date(start, 1, 1),
|
|
40
|
+
end_date_exclusive=date(start + width, 1, 1),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def range(self, start_year: int, end_year: int) -> tuple[Period, ...]:
|
|
44
|
+
if end_year < start_year:
|
|
45
|
+
raise ValueError("end_year must be >= start_year")
|
|
46
|
+
first = self.start_year_for(start_year)
|
|
47
|
+
last = self.start_year_for(end_year)
|
|
48
|
+
return tuple(self.period_for(year) for year in range(first, last + 1, self.scheme.width_years))
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
_PRESENTATION_INSTALL_HINT = (
|
|
7
|
+
"spatial_foundation.presentation requires the optional presentation dependencies; "
|
|
8
|
+
"install with `pip install 'spatial-data-foundation[presentation]'`"
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
import contextily as cx
|
|
13
|
+
from xyzservices import TileProvider, providers
|
|
14
|
+
except ModuleNotFoundError as exc: # pragma: no cover - exercised in an isolated subprocess
|
|
15
|
+
raise ModuleNotFoundError(_PRESENTATION_INSTALL_HINT) from exc
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
BASEMAP_ALIASES: dict[str, TileProvider] = {
|
|
19
|
+
"neutral": providers.CartoDB.Positron,
|
|
20
|
+
"imagery": providers.Esri.WorldImagery,
|
|
21
|
+
"terrain": providers.OpenTopoMap,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
BasemapSource = str | Path | TileProvider
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def resolve_basemap(source: BasemapSource = "neutral") -> str | TileProvider:
|
|
28
|
+
"""Resolve a small governed alias vocabulary or pass through a concrete source.
|
|
29
|
+
|
|
30
|
+
String sources outside the alias vocabulary are intentionally passed through to
|
|
31
|
+
contextily, which can interpret a tile URL/provider name or a local raster path.
|
|
32
|
+
`Path` values are normalized to strings for the same reason.
|
|
33
|
+
"""
|
|
34
|
+
if isinstance(source, Path):
|
|
35
|
+
return str(source)
|
|
36
|
+
if isinstance(source, str) and source in BASEMAP_ALIASES:
|
|
37
|
+
return BASEMAP_ALIASES[source]
|
|
38
|
+
return source
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def add_basemap(
|
|
42
|
+
ax: Any,
|
|
43
|
+
*,
|
|
44
|
+
crs: Any,
|
|
45
|
+
kind: str = "neutral",
|
|
46
|
+
source: BasemapSource | None = None,
|
|
47
|
+
alpha: float = 1.0,
|
|
48
|
+
attribution: str | None = None,
|
|
49
|
+
zoom: int | str = "auto",
|
|
50
|
+
interpolation: str = "bilinear",
|
|
51
|
+
) -> Any:
|
|
52
|
+
"""Add contextual cartography beneath an existing analytical plot.
|
|
53
|
+
|
|
54
|
+
This is presentation-only. It never changes source geometry or spatial
|
|
55
|
+
membership. The caller must provide the CRS of the plotted analytical data.
|
|
56
|
+
Provider attribution is preserved by default and cannot be disabled here.
|
|
57
|
+
"""
|
|
58
|
+
if crs is None:
|
|
59
|
+
raise ValueError("a CRS is required to align a contextual basemap")
|
|
60
|
+
if attribution is False:
|
|
61
|
+
raise ValueError("basemap attribution cannot be disabled")
|
|
62
|
+
if not 0 <= alpha <= 1:
|
|
63
|
+
raise ValueError("alpha must be between 0 and 1")
|
|
64
|
+
|
|
65
|
+
selected: BasemapSource = source if source is not None else kind
|
|
66
|
+
resolved_source = resolve_basemap(selected)
|
|
67
|
+
xlim = ax.get_xlim()
|
|
68
|
+
ylim = ax.get_ylim()
|
|
69
|
+
|
|
70
|
+
cx.add_basemap(
|
|
71
|
+
ax,
|
|
72
|
+
crs=crs,
|
|
73
|
+
source=resolved_source,
|
|
74
|
+
alpha=alpha,
|
|
75
|
+
attribution=attribution,
|
|
76
|
+
reset_extent=True,
|
|
77
|
+
zoom=zoom,
|
|
78
|
+
interpolation=interpolation,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# contextily already promises reset_extent=True, but preserve the public
|
|
82
|
+
# presentation contract even if a provider/local raster behaves unusually.
|
|
83
|
+
ax.set_xlim(xlim)
|
|
84
|
+
ax.set_ylim(ylim)
|
|
85
|
+
return ax
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import geopandas as gpd
|
|
6
|
+
|
|
7
|
+
from .basemaps import BasemapSource, add_basemap
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def plot_context(
|
|
11
|
+
gdf: gpd.GeoDataFrame,
|
|
12
|
+
*,
|
|
13
|
+
basemap: BasemapSource = "neutral",
|
|
14
|
+
basemap_alpha: float = 1.0,
|
|
15
|
+
basemap_attribution: str | None = None,
|
|
16
|
+
basemap_zoom: int | str = "auto",
|
|
17
|
+
**plot_kwargs: Any,
|
|
18
|
+
) -> Any:
|
|
19
|
+
"""Plot a GeoDataFrame and add an optional contextual basemap beneath it."""
|
|
20
|
+
if gdf.crs is None:
|
|
21
|
+
raise ValueError("GeoDataFrame requires a CRS for contextual plotting")
|
|
22
|
+
|
|
23
|
+
ax = gdf.plot(**plot_kwargs)
|
|
24
|
+
add_basemap(
|
|
25
|
+
ax,
|
|
26
|
+
crs=gdf.crs,
|
|
27
|
+
source=basemap,
|
|
28
|
+
alpha=basemap_alpha,
|
|
29
|
+
attribution=basemap_attribution,
|
|
30
|
+
zoom=basemap_zoom,
|
|
31
|
+
)
|
|
32
|
+
return ax
|