gridforge-spatial 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gridforge/__init__.py +10 -0
- gridforge/align/__init__.py +1 -0
- gridforge/align/points.py +172 -0
- gridforge/align/raster.py +129 -0
- gridforge/align/vector.py +204 -0
- gridforge/cli.py +345 -0
- gridforge/demo.py +173 -0
- gridforge/errors.py +13 -0
- gridforge/grid/__init__.py +6 -0
- gridforge/grid/build.py +57 -0
- gridforge/grid/spec.py +209 -0
- gridforge/io/__init__.py +1 -0
- gridforge/io/datasets.py +317 -0
- gridforge/io/grid.py +110 -0
- gridforge/join.py +75 -0
- gridforge/provenance.py +77 -0
- gridforge/validation/__init__.py +11 -0
- gridforge/validation/dataset.py +376 -0
- gridforge/validation/report.py +41 -0
- gridforge_spatial-0.1.0.dist-info/METADATA +220 -0
- gridforge_spatial-0.1.0.dist-info/RECORD +24 -0
- gridforge_spatial-0.1.0.dist-info/WHEEL +4 -0
- gridforge_spatial-0.1.0.dist-info/entry_points.txt +2 -0
- gridforge_spatial-0.1.0.dist-info/licenses/LICENSE +202 -0
gridforge/grid/spec.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Validated, deterministic canonical grid specifications."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from decimal import ROUND_CEILING, ROUND_FLOOR, ROUND_HALF_UP, Decimal, InvalidOperation
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
from pyproj import CRS
|
|
14
|
+
|
|
15
|
+
from gridforge.errors import GridSpecError
|
|
16
|
+
|
|
17
|
+
_SPEC_KEYS = {"crs", "cell_size", "bounds", "origin", "boundary_tolerance"}
|
|
18
|
+
_BOUND_KEYS = {"min_x", "min_y", "max_x", "max_y"}
|
|
19
|
+
_ORIGIN_KEYS = {"x", "y"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _decimal(value: Any, label: str) -> Decimal:
|
|
23
|
+
if isinstance(value, bool):
|
|
24
|
+
raise GridSpecError(f"{label} must be a finite number")
|
|
25
|
+
try:
|
|
26
|
+
result = Decimal(str(value))
|
|
27
|
+
except (InvalidOperation, ValueError):
|
|
28
|
+
raise GridSpecError(f"{label} must be a finite number") from None
|
|
29
|
+
if not result.is_finite():
|
|
30
|
+
raise GridSpecError(f"{label} must be a finite number")
|
|
31
|
+
return result
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _text(value: Decimal) -> str:
|
|
35
|
+
normalized = value.normalize()
|
|
36
|
+
if normalized == normalized.to_integral_value():
|
|
37
|
+
return str(normalized.quantize(Decimal(1)))
|
|
38
|
+
return format(normalized, "f")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _floor(value: Decimal) -> int:
|
|
42
|
+
return int(value.to_integral_value(rounding=ROUND_FLOOR))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _ceil(value: Decimal) -> int:
|
|
46
|
+
return int(value.to_integral_value(rounding=ROUND_CEILING))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _require_mapping(value: Any, label: str, allowed: set[str]) -> dict[str, Any]:
|
|
50
|
+
if not isinstance(value, dict):
|
|
51
|
+
raise GridSpecError(f"{label} must be a mapping")
|
|
52
|
+
unknown = sorted(set(value) - allowed)
|
|
53
|
+
if unknown:
|
|
54
|
+
raise GridSpecError(f"unknown {label} key(s): {', '.join(unknown)}")
|
|
55
|
+
return value
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class Bounds:
|
|
60
|
+
min_x: Decimal
|
|
61
|
+
min_y: Decimal
|
|
62
|
+
max_x: Decimal
|
|
63
|
+
max_y: Decimal
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Origin:
|
|
68
|
+
x: Decimal
|
|
69
|
+
y: Decimal
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class GridSpec:
|
|
74
|
+
"""A square grid anchored to an explicit top-left cell-corner origin.
|
|
75
|
+
|
|
76
|
+
Coordinates and ``cell_size`` use the units of the projected CRS. Row
|
|
77
|
+
indices increase south and column indices increase east. Requested bounds
|
|
78
|
+
include every cell with positive-area overlap, expanding out to cell edges.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
crs: CRS
|
|
82
|
+
cell_size: Decimal
|
|
83
|
+
bounds: Bounds
|
|
84
|
+
origin: Origin
|
|
85
|
+
boundary_tolerance: Decimal = Decimal("1e-9")
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def from_yaml(cls, path: str | Path) -> GridSpec:
|
|
89
|
+
"""Load a grid specification from a YAML file."""
|
|
90
|
+
try:
|
|
91
|
+
value = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
|
|
92
|
+
except (OSError, yaml.YAMLError) as exc:
|
|
93
|
+
raise GridSpecError(f"cannot read grid specification {path}: {exc}") from exc
|
|
94
|
+
return cls.from_mapping(value)
|
|
95
|
+
|
|
96
|
+
@classmethod
|
|
97
|
+
def from_mapping(cls, value: Any) -> GridSpec:
|
|
98
|
+
"""Parse and validate a mapping with explicit CRS, bounds, and origin."""
|
|
99
|
+
data = _require_mapping(value, "grid specification", _SPEC_KEYS)
|
|
100
|
+
missing = sorted(_SPEC_KEYS - {"boundary_tolerance"} - set(data))
|
|
101
|
+
if missing:
|
|
102
|
+
raise GridSpecError(f"missing grid specification key(s): {', '.join(missing)}")
|
|
103
|
+
|
|
104
|
+
try:
|
|
105
|
+
crs = CRS.from_user_input(data["crs"])
|
|
106
|
+
except Exception as exc:
|
|
107
|
+
raise GridSpecError(f"invalid CRS: {data['crs']!r}") from exc
|
|
108
|
+
if not crs.is_projected:
|
|
109
|
+
raise GridSpecError("canonical grid CRS must be projected")
|
|
110
|
+
|
|
111
|
+
cell_size = _decimal(data["cell_size"], "cell_size")
|
|
112
|
+
if cell_size <= 0:
|
|
113
|
+
raise GridSpecError("cell_size must be greater than zero")
|
|
114
|
+
|
|
115
|
+
bounds_data = _require_mapping(data["bounds"], "bounds", _BOUND_KEYS)
|
|
116
|
+
origin_data = _require_mapping(data["origin"], "origin", _ORIGIN_KEYS)
|
|
117
|
+
if set(bounds_data) != _BOUND_KEYS:
|
|
118
|
+
raise GridSpecError("bounds must define min_x, min_y, max_x, and max_y")
|
|
119
|
+
if set(origin_data) != _ORIGIN_KEYS:
|
|
120
|
+
raise GridSpecError("origin must define x and y")
|
|
121
|
+
|
|
122
|
+
bounds = Bounds(**{key: _decimal(bounds_data[key], f"bounds.{key}") for key in _BOUND_KEYS})
|
|
123
|
+
origin = Origin(
|
|
124
|
+
x=_decimal(origin_data["x"], "origin.x"),
|
|
125
|
+
y=_decimal(origin_data["y"], "origin.y"),
|
|
126
|
+
)
|
|
127
|
+
if bounds.min_x >= bounds.max_x or bounds.min_y >= bounds.max_y:
|
|
128
|
+
raise GridSpecError("bounds must have positive width and height")
|
|
129
|
+
|
|
130
|
+
tolerance = _decimal(data.get("boundary_tolerance", "1e-9"), "boundary_tolerance")
|
|
131
|
+
if tolerance < 0 or tolerance * 2 >= cell_size:
|
|
132
|
+
raise GridSpecError("boundary_tolerance must be nonnegative and less than half a cell")
|
|
133
|
+
|
|
134
|
+
return cls(crs, cell_size, bounds, origin, tolerance)
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def crs_id(self) -> str:
|
|
138
|
+
"""Return a stable authority identifier or normalized WKT for the CRS."""
|
|
139
|
+
authority = self.crs.to_authority()
|
|
140
|
+
if authority:
|
|
141
|
+
return f"{authority[0]}:{authority[1]}"
|
|
142
|
+
return self.crs.to_wkt(version="WKT2_2019", pretty=False)
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def index_extent(self) -> tuple[int, int, int, int]:
|
|
146
|
+
"""Return inclusive ``(row_start, row_end, col_start, col_end)``."""
|
|
147
|
+
row_start = _floor((self.origin.y - self.bounds.max_y) / self.cell_size)
|
|
148
|
+
row_end = _ceil((self.origin.y - self.bounds.min_y) / self.cell_size) - 1
|
|
149
|
+
col_start = _floor((self.bounds.min_x - self.origin.x) / self.cell_size)
|
|
150
|
+
col_end = _ceil((self.bounds.max_x - self.origin.x) / self.cell_size) - 1
|
|
151
|
+
return row_start, row_end, col_start, col_end
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def fingerprint(self) -> str:
|
|
155
|
+
"""Hash the normalized CRS, grid spacing, origin, and indexed extent."""
|
|
156
|
+
row_start, row_end, col_start, col_end = self.index_extent
|
|
157
|
+
identity = {
|
|
158
|
+
"cell_size": _text(self.cell_size),
|
|
159
|
+
"col_extent": [col_start, col_end],
|
|
160
|
+
"crs": self.crs_id,
|
|
161
|
+
"origin": [_text(self.origin.x), _text(self.origin.y)],
|
|
162
|
+
"row_extent": [row_start, row_end],
|
|
163
|
+
"schema": "gridforge-grid-v1",
|
|
164
|
+
}
|
|
165
|
+
canonical = json.dumps(identity, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
166
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
167
|
+
|
|
168
|
+
def cell_index(self, x: float, y: float) -> tuple[int, int] | None:
|
|
169
|
+
"""Return the containing ``(row, column)`` or ``None`` outside the grid.
|
|
170
|
+
|
|
171
|
+
Cells use ``[left, right) × (bottom, top]``. Coordinates within the
|
|
172
|
+
configured tolerance of an anchored grid line are snapped to that line.
|
|
173
|
+
"""
|
|
174
|
+
try:
|
|
175
|
+
x_value = _decimal(x, "x")
|
|
176
|
+
y_value = _decimal(y, "y")
|
|
177
|
+
except GridSpecError:
|
|
178
|
+
return None
|
|
179
|
+
col_position = (x_value - self.origin.x) / self.cell_size
|
|
180
|
+
row_position = (self.origin.y - y_value) / self.cell_size
|
|
181
|
+
col_position = self._snap_index(col_position)
|
|
182
|
+
row_position = self._snap_index(row_position)
|
|
183
|
+
column = int(col_position.to_integral_value(rounding=ROUND_FLOOR))
|
|
184
|
+
row = int(row_position.to_integral_value(rounding=ROUND_FLOOR))
|
|
185
|
+
row_start, row_end, col_start, col_end = self.index_extent
|
|
186
|
+
if row_start <= row <= row_end and col_start <= column <= col_end:
|
|
187
|
+
return row, column
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
def _snap_index(self, value: Decimal) -> Decimal:
|
|
191
|
+
nearest = value.to_integral_value(rounding=ROUND_HALF_UP)
|
|
192
|
+
if abs(value - nearest) * self.cell_size <= self.boundary_tolerance:
|
|
193
|
+
return nearest
|
|
194
|
+
return value
|
|
195
|
+
|
|
196
|
+
def to_dict(self) -> dict[str, Any]:
|
|
197
|
+
"""Return JSON-compatible normalized grid specification metadata."""
|
|
198
|
+
return {
|
|
199
|
+
"crs": self.crs_id,
|
|
200
|
+
"cell_size": _text(self.cell_size),
|
|
201
|
+
"bounds": {
|
|
202
|
+
"min_x": _text(self.bounds.min_x),
|
|
203
|
+
"min_y": _text(self.bounds.min_y),
|
|
204
|
+
"max_x": _text(self.bounds.max_x),
|
|
205
|
+
"max_y": _text(self.bounds.max_y),
|
|
206
|
+
},
|
|
207
|
+
"origin": {"x": _text(self.origin.x), "y": _text(self.origin.y)},
|
|
208
|
+
"boundary_tolerance": _text(self.boundary_tolerance),
|
|
209
|
+
}
|
gridforge/io/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Spatial dataset input and output helpers."""
|
gridforge/io/datasets.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""Explicit-CRS inspection and loading for spatial source datasets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import asdict, dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
import geopandas as gpd
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import pyarrow.parquet as pq
|
|
14
|
+
import rasterio
|
|
15
|
+
from pyproj import CRS
|
|
16
|
+
from shapely.geometry import Point
|
|
17
|
+
|
|
18
|
+
from gridforge.errors import DatasetError
|
|
19
|
+
from gridforge.grid.spec import GridSpec
|
|
20
|
+
|
|
21
|
+
_RASTER_SUFFIXES = {".tif", ".tiff", ".vrt", ".img", ".asc", ".bil", ".grd"}
|
|
22
|
+
_VECTOR_SUFFIXES = {".geojson", ".json", ".gpkg", ".shp", ".gml", ".fgb"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class DatasetInfo:
|
|
27
|
+
"""Spatial metadata reported by :func:`inspect_dataset`."""
|
|
28
|
+
|
|
29
|
+
path: str
|
|
30
|
+
kind: Literal["vector", "table", "raster"]
|
|
31
|
+
geometry_type: str | None
|
|
32
|
+
crs: str
|
|
33
|
+
feature_count: int | None
|
|
34
|
+
band_count: int | None
|
|
35
|
+
bounds: tuple[float, float, float, float] | None
|
|
36
|
+
|
|
37
|
+
def to_dict(self) -> dict[str, object]:
|
|
38
|
+
return asdict(self)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _crs_label(crs: CRS) -> str:
|
|
42
|
+
authority = crs.to_authority()
|
|
43
|
+
return f"{authority[0]}:{authority[1]}" if authority else crs.to_wkt("WKT2_2019", pretty=False)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _resolved_crs(embedded: object | None, source_crs: str | CRS | None) -> CRS:
|
|
47
|
+
parsed_embedded: CRS | None = None
|
|
48
|
+
if embedded is not None:
|
|
49
|
+
try:
|
|
50
|
+
parsed_embedded = CRS.from_user_input(embedded)
|
|
51
|
+
except Exception as exc:
|
|
52
|
+
raise DatasetError(f"embedded source CRS is invalid: {embedded!r}") from exc
|
|
53
|
+
if parsed_embedded is None:
|
|
54
|
+
if source_crs is None:
|
|
55
|
+
raise DatasetError("source CRS is unknown; provide --source-crs explicitly")
|
|
56
|
+
try:
|
|
57
|
+
return CRS.from_user_input(source_crs)
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
raise DatasetError(f"invalid explicit source CRS: {source_crs!r}") from exc
|
|
60
|
+
if source_crs is not None:
|
|
61
|
+
try:
|
|
62
|
+
requested = CRS.from_user_input(source_crs)
|
|
63
|
+
except Exception as exc:
|
|
64
|
+
raise DatasetError(f"invalid explicit source CRS: {source_crs!r}") from exc
|
|
65
|
+
if not parsed_embedded.equals(requested):
|
|
66
|
+
raise DatasetError("explicit source CRS conflicts with embedded CRS")
|
|
67
|
+
return parsed_embedded
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _read_table(path: Path) -> pd.DataFrame:
|
|
71
|
+
try:
|
|
72
|
+
if path.suffix.lower() == ".csv":
|
|
73
|
+
return pd.read_csv(path)
|
|
74
|
+
if path.suffix.lower() == ".parquet":
|
|
75
|
+
return pd.read_parquet(path)
|
|
76
|
+
except Exception as exc:
|
|
77
|
+
raise DatasetError(f"cannot read tabular dataset {path}: {exc}") from exc
|
|
78
|
+
raise DatasetError(f"unsupported tabular input format: {path.suffix or '<none>'}")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _table_coordinates(
|
|
82
|
+
table: pd.DataFrame,
|
|
83
|
+
*,
|
|
84
|
+
x_column: str,
|
|
85
|
+
y_column: str,
|
|
86
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
87
|
+
missing = [name for name in (x_column, y_column) if name not in table.columns]
|
|
88
|
+
if missing:
|
|
89
|
+
raise DatasetError(f"point table is missing coordinate column(s): {', '.join(missing)}")
|
|
90
|
+
try:
|
|
91
|
+
x_values = pd.to_numeric(table[x_column], errors="raise").to_numpy(dtype=float)
|
|
92
|
+
y_values = pd.to_numeric(table[y_column], errors="raise").to_numpy(dtype=float)
|
|
93
|
+
except (TypeError, ValueError) as exc:
|
|
94
|
+
raise DatasetError("point coordinates must be numeric") from exc
|
|
95
|
+
if not np.isfinite(x_values).all() or not np.isfinite(y_values).all():
|
|
96
|
+
raise DatasetError("point coordinates must be finite and non-null")
|
|
97
|
+
return x_values, y_values
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _is_geoparquet(path: Path) -> bool:
|
|
101
|
+
if path.suffix.lower() != ".parquet":
|
|
102
|
+
return False
|
|
103
|
+
try:
|
|
104
|
+
metadata = pq.read_schema(path).metadata or {}
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
raise DatasetError(f"cannot inspect Parquet dataset {path}: {exc}") from exc
|
|
107
|
+
return b"geo" in metadata
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _read_geodataframe(path: Path) -> gpd.GeoDataFrame:
|
|
111
|
+
try:
|
|
112
|
+
if path.suffix.lower() == ".parquet" and _is_geoparquet(path):
|
|
113
|
+
return gpd.read_parquet(path)
|
|
114
|
+
if path.suffix.lower() in _VECTOR_SUFFIXES:
|
|
115
|
+
return gpd.read_file(path)
|
|
116
|
+
except Exception as exc:
|
|
117
|
+
raise DatasetError(f"cannot read vector dataset {path}: {exc}") from exc
|
|
118
|
+
raise DatasetError(f"unsupported vector input format: {path.suffix or '<none>'}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _bounds_tuple(values: object) -> tuple[float, float, float, float] | None:
|
|
122
|
+
array = np.asarray(values, dtype=float)
|
|
123
|
+
if array.size != 4 or not np.isfinite(array).all():
|
|
124
|
+
return None
|
|
125
|
+
return tuple(float(value) for value in array) # type: ignore[return-value]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def inspect_dataset(
|
|
129
|
+
path: str | Path,
|
|
130
|
+
*,
|
|
131
|
+
source_crs: str | CRS | None = None,
|
|
132
|
+
x_column: str = "x",
|
|
133
|
+
y_column: str = "y",
|
|
134
|
+
) -> DatasetInfo:
|
|
135
|
+
"""Inspect vector, coordinate-table, or raster input without guessing CRS."""
|
|
136
|
+
input_path = Path(path)
|
|
137
|
+
suffix = input_path.suffix.lower()
|
|
138
|
+
if suffix in _RASTER_SUFFIXES:
|
|
139
|
+
try:
|
|
140
|
+
with rasterio.open(input_path) as dataset:
|
|
141
|
+
crs = _resolved_crs(dataset.crs, source_crs)
|
|
142
|
+
return DatasetInfo(
|
|
143
|
+
str(input_path),
|
|
144
|
+
"raster",
|
|
145
|
+
"Raster",
|
|
146
|
+
_crs_label(crs),
|
|
147
|
+
None,
|
|
148
|
+
dataset.count,
|
|
149
|
+
_bounds_tuple(dataset.bounds),
|
|
150
|
+
)
|
|
151
|
+
except DatasetError:
|
|
152
|
+
raise
|
|
153
|
+
except Exception as exc:
|
|
154
|
+
raise DatasetError(f"cannot inspect raster dataset {input_path}: {exc}") from exc
|
|
155
|
+
|
|
156
|
+
if suffix == ".csv" or (suffix == ".parquet" and not _is_geoparquet(input_path)):
|
|
157
|
+
table = _read_table(input_path)
|
|
158
|
+
x_values, y_values = _table_coordinates(table, x_column=x_column, y_column=y_column)
|
|
159
|
+
crs = _resolved_crs(None, source_crs)
|
|
160
|
+
if not len(table):
|
|
161
|
+
bounds = None
|
|
162
|
+
else:
|
|
163
|
+
bounds = (
|
|
164
|
+
float(x_values.min()),
|
|
165
|
+
float(y_values.min()),
|
|
166
|
+
float(x_values.max()),
|
|
167
|
+
float(y_values.max()),
|
|
168
|
+
)
|
|
169
|
+
return DatasetInfo(
|
|
170
|
+
str(input_path), "table", "Point", _crs_label(crs), len(table), None, bounds
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
vector = _read_geodataframe(input_path)
|
|
174
|
+
crs = _resolved_crs(vector.crs, source_crs)
|
|
175
|
+
geometry_types = sorted({str(value) for value in vector.geometry.geom_type.dropna().unique()})
|
|
176
|
+
geometry_type = ", ".join(geometry_types) if geometry_types else None
|
|
177
|
+
bounds = _bounds_tuple(vector.total_bounds) if not vector.empty else None
|
|
178
|
+
return DatasetInfo(
|
|
179
|
+
str(input_path),
|
|
180
|
+
"vector",
|
|
181
|
+
geometry_type,
|
|
182
|
+
_crs_label(crs),
|
|
183
|
+
len(vector),
|
|
184
|
+
None,
|
|
185
|
+
bounds,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def load_vector(
|
|
190
|
+
path: str | Path,
|
|
191
|
+
*,
|
|
192
|
+
source_crs: str | CRS | None = None,
|
|
193
|
+
) -> gpd.GeoDataFrame:
|
|
194
|
+
"""Load a supported vector file and require an explicit, consistent CRS."""
|
|
195
|
+
input_path = Path(path)
|
|
196
|
+
vector = _read_geodataframe(input_path)
|
|
197
|
+
crs = _resolved_crs(vector.crs, source_crs)
|
|
198
|
+
if vector.crs is None:
|
|
199
|
+
vector = vector.set_crs(crs)
|
|
200
|
+
return vector
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def load_points(
|
|
204
|
+
path: str | Path,
|
|
205
|
+
*,
|
|
206
|
+
x_column: str = "x",
|
|
207
|
+
y_column: str = "y",
|
|
208
|
+
source_crs: str | CRS | None = None,
|
|
209
|
+
) -> gpd.GeoDataFrame:
|
|
210
|
+
"""Load point geometry or an x/y table with an explicit source CRS."""
|
|
211
|
+
input_path = Path(path)
|
|
212
|
+
if input_path.suffix.lower() in {".csv", ".parquet"} and not _is_geoparquet(input_path):
|
|
213
|
+
table = _read_table(input_path)
|
|
214
|
+
crs = _resolved_crs(None, source_crs)
|
|
215
|
+
x_values, y_values = _table_coordinates(table, x_column=x_column, y_column=y_column)
|
|
216
|
+
result = gpd.GeoDataFrame(
|
|
217
|
+
table.copy(),
|
|
218
|
+
geometry=[Point(x, y) for x, y in zip(x_values, y_values, strict=True)],
|
|
219
|
+
crs=crs,
|
|
220
|
+
)
|
|
221
|
+
else:
|
|
222
|
+
result = load_vector(input_path, source_crs=source_crs)
|
|
223
|
+
if result.geometry.isna().any() or result.geometry.is_empty.any():
|
|
224
|
+
raise DatasetError("point dataset contains null or empty geometries")
|
|
225
|
+
if not result.geometry.geom_type.eq("Point").all():
|
|
226
|
+
raise DatasetError("point alignment requires Point geometries")
|
|
227
|
+
if result.geometry.isna().any() or result.geometry.is_empty.any():
|
|
228
|
+
raise DatasetError("point dataset contains null or empty geometries")
|
|
229
|
+
if not result.geometry.geom_type.eq("Point").all():
|
|
230
|
+
raise DatasetError("point alignment requires Point geometries")
|
|
231
|
+
return result
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def write_dataset(dataset: gpd.GeoDataFrame, path: str | Path) -> Path:
|
|
235
|
+
"""Write one validated aligned dataset as GridForge GeoParquet.
|
|
236
|
+
|
|
237
|
+
Null feature values are retained as nodata and reported as warnings. A
|
|
238
|
+
dataset with structural, identity, geometry, or coverage errors is refused.
|
|
239
|
+
"""
|
|
240
|
+
from gridforge.validation import validate_dataset
|
|
241
|
+
|
|
242
|
+
report = validate_dataset(dataset)
|
|
243
|
+
errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
|
|
244
|
+
if errors:
|
|
245
|
+
raise DatasetError("aligned dataset is invalid: " + "; ".join(errors[:3]))
|
|
246
|
+
spec_value = dataset.attrs.get("gridforge_spec")
|
|
247
|
+
if not isinstance(spec_value, dict):
|
|
248
|
+
raise DatasetError("aligned dataset is missing GridForge grid metadata")
|
|
249
|
+
try:
|
|
250
|
+
spec = GridSpec.from_mapping(spec_value)
|
|
251
|
+
except ValueError as exc:
|
|
252
|
+
raise DatasetError(f"aligned dataset grid metadata is invalid: {exc}") from exc
|
|
253
|
+
fingerprint = str(dataset["grid_fingerprint"].iloc[0])
|
|
254
|
+
if fingerprint != spec.fingerprint:
|
|
255
|
+
raise DatasetError("aligned dataset grid fingerprint does not match its specification")
|
|
256
|
+
|
|
257
|
+
output = Path(path)
|
|
258
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
259
|
+
try:
|
|
260
|
+
dataset.to_parquet(output, index=False)
|
|
261
|
+
table = pq.read_table(output)
|
|
262
|
+
metadata = dict(table.schema.metadata or {})
|
|
263
|
+
metadata[b"gridforge:grid_fingerprint"] = fingerprint.encode("utf-8")
|
|
264
|
+
metadata[b"gridforge:grid_spec"] = json.dumps(
|
|
265
|
+
spec.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
|
266
|
+
).encode("utf-8")
|
|
267
|
+
operation = dataset.attrs.get("gridforge_operation")
|
|
268
|
+
if operation is not None:
|
|
269
|
+
metadata[b"gridforge:operation"] = json.dumps(
|
|
270
|
+
operation, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
|
271
|
+
).encode("utf-8")
|
|
272
|
+
pq.write_table(table.replace_schema_metadata(metadata), output, compression="zstd")
|
|
273
|
+
except Exception as exc:
|
|
274
|
+
raise DatasetError(f"cannot write aligned GeoParquet {output}: {exc}") from exc
|
|
275
|
+
return output
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def read_dataset(path: str | Path) -> gpd.GeoDataFrame:
|
|
279
|
+
"""Read GridForge GeoParquet and restore its canonical grid identity."""
|
|
280
|
+
from gridforge.validation import validate_dataset
|
|
281
|
+
|
|
282
|
+
input_path = Path(path)
|
|
283
|
+
try:
|
|
284
|
+
dataset = gpd.read_parquet(input_path)
|
|
285
|
+
schema_metadata = pq.read_schema(input_path).metadata or {}
|
|
286
|
+
except Exception as exc:
|
|
287
|
+
raise DatasetError(f"cannot read aligned GeoParquet {input_path}: {exc}") from exc
|
|
288
|
+
metadata_fingerprint = schema_metadata.get(b"gridforge:grid_fingerprint")
|
|
289
|
+
metadata_spec = schema_metadata.get(b"gridforge:grid_spec")
|
|
290
|
+
if metadata_fingerprint is None or metadata_spec is None:
|
|
291
|
+
raise DatasetError("GeoParquet is missing GridForge grid metadata")
|
|
292
|
+
try:
|
|
293
|
+
spec_value = json.loads(metadata_spec.decode("utf-8"))
|
|
294
|
+
spec = GridSpec.from_mapping(spec_value)
|
|
295
|
+
fingerprint = metadata_fingerprint.decode("utf-8")
|
|
296
|
+
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
297
|
+
raise DatasetError("GridForge grid metadata is invalid") from exc
|
|
298
|
+
if spec.fingerprint != fingerprint:
|
|
299
|
+
raise DatasetError("GridForge grid metadata fingerprint does not match grid specification")
|
|
300
|
+
dataset.attrs["gridforge_spec"] = spec.to_dict()
|
|
301
|
+
operation = schema_metadata.get(b"gridforge:operation")
|
|
302
|
+
if operation is not None:
|
|
303
|
+
try:
|
|
304
|
+
dataset.attrs["gridforge_operation"] = json.loads(operation.decode("utf-8"))
|
|
305
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
306
|
+
raise DatasetError("GridForge operation metadata is invalid") from exc
|
|
307
|
+
if dataset.empty or "grid_fingerprint" not in dataset:
|
|
308
|
+
raise DatasetError("aligned dataset has no grid rows")
|
|
309
|
+
if not dataset["grid_fingerprint"].astype(str).eq(fingerprint).all():
|
|
310
|
+
raise DatasetError("aligned dataset grid fingerprint does not match metadata")
|
|
311
|
+
if dataset.crs is None or not spec.crs.equals(dataset.crs):
|
|
312
|
+
raise DatasetError("aligned dataset CRS does not match GridForge grid metadata")
|
|
313
|
+
report = validate_dataset(dataset)
|
|
314
|
+
errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
|
|
315
|
+
if errors:
|
|
316
|
+
raise DatasetError("aligned dataset is invalid: " + "; ".join(errors[:3]))
|
|
317
|
+
return dataset
|
gridforge/io/grid.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""GeoParquet persistence for canonical grids."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import geopandas as gpd
|
|
9
|
+
import pyarrow.parquet as pq
|
|
10
|
+
|
|
11
|
+
from gridforge.errors import DatasetError
|
|
12
|
+
from gridforge.grid.spec import GridSpec
|
|
13
|
+
|
|
14
|
+
_GRID_COLUMNS = {
|
|
15
|
+
"grid_id",
|
|
16
|
+
"grid_fingerprint",
|
|
17
|
+
"row",
|
|
18
|
+
"column",
|
|
19
|
+
"left",
|
|
20
|
+
"bottom",
|
|
21
|
+
"right",
|
|
22
|
+
"top",
|
|
23
|
+
"geometry",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def write_grid(grid: gpd.GeoDataFrame, path: str | Path) -> Path:
|
|
28
|
+
"""Write a canonical grid as GeoParquet with GridForge metadata."""
|
|
29
|
+
_validate_grid_schema(grid)
|
|
30
|
+
spec_value = grid.attrs.get("gridforge_spec")
|
|
31
|
+
if not isinstance(spec_value, dict):
|
|
32
|
+
raise DatasetError("grid is missing GridForge specification metadata")
|
|
33
|
+
try:
|
|
34
|
+
spec = GridSpec.from_mapping(spec_value)
|
|
35
|
+
except ValueError as exc:
|
|
36
|
+
raise DatasetError(f"grid specification metadata is invalid: {exc}") from exc
|
|
37
|
+
if grid.crs is None or not spec.crs.equals(grid.crs):
|
|
38
|
+
raise DatasetError("grid CRS does not match GridForge specification")
|
|
39
|
+
if not grid["grid_fingerprint"].astype(str).eq(spec.fingerprint).all():
|
|
40
|
+
raise DatasetError("grid fingerprint does not match GridForge specification")
|
|
41
|
+
from gridforge.validation import validate_grid
|
|
42
|
+
|
|
43
|
+
report = validate_grid(grid)
|
|
44
|
+
errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
|
|
45
|
+
if errors:
|
|
46
|
+
raise DatasetError("grid is invalid: " + "; ".join(errors[:3]))
|
|
47
|
+
output = Path(path)
|
|
48
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
grid.to_parquet(output, index=False)
|
|
50
|
+
table = pq.read_table(output)
|
|
51
|
+
metadata = dict(table.schema.metadata or {})
|
|
52
|
+
metadata[b"gridforge:grid_fingerprint"] = str(grid["grid_fingerprint"].iloc[0]).encode()
|
|
53
|
+
metadata[b"gridforge:grid_spec"] = json.dumps(
|
|
54
|
+
spec.to_dict(), sort_keys=True, separators=(",", ":")
|
|
55
|
+
).encode()
|
|
56
|
+
pq.write_table(table.replace_schema_metadata(metadata), output, compression="zstd")
|
|
57
|
+
return output
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def read_grid(path: str | Path) -> gpd.GeoDataFrame:
|
|
61
|
+
"""Read a GeoParquet grid and reject incomplete or ambiguous identity."""
|
|
62
|
+
input_path = Path(path)
|
|
63
|
+
try:
|
|
64
|
+
grid = gpd.read_parquet(input_path)
|
|
65
|
+
table = pq.read_table(input_path)
|
|
66
|
+
except Exception as exc:
|
|
67
|
+
raise DatasetError(f"cannot read grid GeoParquet {input_path}: {exc}") from exc
|
|
68
|
+
_validate_grid_schema(grid)
|
|
69
|
+
metadata_fingerprint = (table.schema.metadata or {}).get(b"gridforge:grid_fingerprint")
|
|
70
|
+
metadata_spec = (table.schema.metadata or {}).get(b"gridforge:grid_spec")
|
|
71
|
+
if metadata_fingerprint is None or metadata_spec is None:
|
|
72
|
+
raise DatasetError("GeoParquet is missing GridForge grid metadata")
|
|
73
|
+
try:
|
|
74
|
+
decoded_fingerprint = metadata_fingerprint.decode("utf-8")
|
|
75
|
+
except UnicodeDecodeError as exc:
|
|
76
|
+
raise DatasetError("grid fingerprint metadata is invalid") from exc
|
|
77
|
+
if decoded_fingerprint != str(grid["grid_fingerprint"].iloc[0]):
|
|
78
|
+
raise DatasetError("grid fingerprint metadata does not match grid rows")
|
|
79
|
+
try:
|
|
80
|
+
grid.attrs["gridforge_spec"] = json.loads(metadata_spec.decode())
|
|
81
|
+
spec = GridSpec.from_mapping(grid.attrs["gridforge_spec"])
|
|
82
|
+
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
83
|
+
raise DatasetError("grid specification metadata is invalid") from exc
|
|
84
|
+
if spec.fingerprint != str(grid["grid_fingerprint"].iloc[0]):
|
|
85
|
+
raise DatasetError("grid specification does not match grid fingerprint")
|
|
86
|
+
from gridforge.validation import validate_grid
|
|
87
|
+
|
|
88
|
+
report = validate_grid(grid)
|
|
89
|
+
errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
|
|
90
|
+
if errors:
|
|
91
|
+
raise DatasetError("grid is invalid: " + "; ".join(errors[:3]))
|
|
92
|
+
return grid
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _validate_grid_schema(grid: gpd.GeoDataFrame) -> None:
|
|
96
|
+
missing = sorted(_GRID_COLUMNS - set(grid.columns))
|
|
97
|
+
if missing:
|
|
98
|
+
raise DatasetError(f"grid is missing required columns: {', '.join(missing)}")
|
|
99
|
+
if grid.empty:
|
|
100
|
+
raise DatasetError("grid must contain at least one cell")
|
|
101
|
+
if grid.crs is None:
|
|
102
|
+
raise DatasetError("grid CRS is unknown")
|
|
103
|
+
if grid["grid_id"].isna().any() or grid["grid_id"].duplicated().any():
|
|
104
|
+
raise DatasetError("grid IDs must be non-null and unique")
|
|
105
|
+
fingerprints = grid["grid_fingerprint"].dropna().astype(str).unique()
|
|
106
|
+
if len(fingerprints) != 1 or grid["grid_fingerprint"].isna().any():
|
|
107
|
+
raise DatasetError("grid rows must have one non-null grid fingerprint")
|
|
108
|
+
expected = grid.sort_values(["row", "column"], kind="stable").index
|
|
109
|
+
if not grid.index.equals(expected):
|
|
110
|
+
raise DatasetError("grid rows must be sorted by row and column")
|