gridforge-spatial 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gridforge/grid/spec.py ADDED
@@ -0,0 +1,209 @@
1
+ """Validated, deterministic canonical grid specifications."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from dataclasses import dataclass
8
+ from decimal import ROUND_CEILING, ROUND_FLOOR, ROUND_HALF_UP, Decimal, InvalidOperation
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import yaml
13
+ from pyproj import CRS
14
+
15
+ from gridforge.errors import GridSpecError
16
+
17
+ _SPEC_KEYS = {"crs", "cell_size", "bounds", "origin", "boundary_tolerance"}
18
+ _BOUND_KEYS = {"min_x", "min_y", "max_x", "max_y"}
19
+ _ORIGIN_KEYS = {"x", "y"}
20
+
21
+
22
+ def _decimal(value: Any, label: str) -> Decimal:
23
+ if isinstance(value, bool):
24
+ raise GridSpecError(f"{label} must be a finite number")
25
+ try:
26
+ result = Decimal(str(value))
27
+ except (InvalidOperation, ValueError):
28
+ raise GridSpecError(f"{label} must be a finite number") from None
29
+ if not result.is_finite():
30
+ raise GridSpecError(f"{label} must be a finite number")
31
+ return result
32
+
33
+
34
+ def _text(value: Decimal) -> str:
35
+ normalized = value.normalize()
36
+ if normalized == normalized.to_integral_value():
37
+ return str(normalized.quantize(Decimal(1)))
38
+ return format(normalized, "f")
39
+
40
+
41
+ def _floor(value: Decimal) -> int:
42
+ return int(value.to_integral_value(rounding=ROUND_FLOOR))
43
+
44
+
45
+ def _ceil(value: Decimal) -> int:
46
+ return int(value.to_integral_value(rounding=ROUND_CEILING))
47
+
48
+
49
+ def _require_mapping(value: Any, label: str, allowed: set[str]) -> dict[str, Any]:
50
+ if not isinstance(value, dict):
51
+ raise GridSpecError(f"{label} must be a mapping")
52
+ unknown = sorted(set(value) - allowed)
53
+ if unknown:
54
+ raise GridSpecError(f"unknown {label} key(s): {', '.join(unknown)}")
55
+ return value
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class Bounds:
60
+ min_x: Decimal
61
+ min_y: Decimal
62
+ max_x: Decimal
63
+ max_y: Decimal
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Origin:
68
+ x: Decimal
69
+ y: Decimal
70
+
71
+
72
+ @dataclass(frozen=True)
73
+ class GridSpec:
74
+ """A square grid anchored to an explicit top-left cell-corner origin.
75
+
76
+ Coordinates and ``cell_size`` use the units of the projected CRS. Row
77
+ indices increase south and column indices increase east. Requested bounds
78
+ include every cell with positive-area overlap, expanding out to cell edges.
79
+ """
80
+
81
+ crs: CRS
82
+ cell_size: Decimal
83
+ bounds: Bounds
84
+ origin: Origin
85
+ boundary_tolerance: Decimal = Decimal("1e-9")
86
+
87
+ @classmethod
88
+ def from_yaml(cls, path: str | Path) -> GridSpec:
89
+ """Load a grid specification from a YAML file."""
90
+ try:
91
+ value = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
92
+ except (OSError, yaml.YAMLError) as exc:
93
+ raise GridSpecError(f"cannot read grid specification {path}: {exc}") from exc
94
+ return cls.from_mapping(value)
95
+
96
+ @classmethod
97
+ def from_mapping(cls, value: Any) -> GridSpec:
98
+ """Parse and validate a mapping with explicit CRS, bounds, and origin."""
99
+ data = _require_mapping(value, "grid specification", _SPEC_KEYS)
100
+ missing = sorted(_SPEC_KEYS - {"boundary_tolerance"} - set(data))
101
+ if missing:
102
+ raise GridSpecError(f"missing grid specification key(s): {', '.join(missing)}")
103
+
104
+ try:
105
+ crs = CRS.from_user_input(data["crs"])
106
+ except Exception as exc:
107
+ raise GridSpecError(f"invalid CRS: {data['crs']!r}") from exc
108
+ if not crs.is_projected:
109
+ raise GridSpecError("canonical grid CRS must be projected")
110
+
111
+ cell_size = _decimal(data["cell_size"], "cell_size")
112
+ if cell_size <= 0:
113
+ raise GridSpecError("cell_size must be greater than zero")
114
+
115
+ bounds_data = _require_mapping(data["bounds"], "bounds", _BOUND_KEYS)
116
+ origin_data = _require_mapping(data["origin"], "origin", _ORIGIN_KEYS)
117
+ if set(bounds_data) != _BOUND_KEYS:
118
+ raise GridSpecError("bounds must define min_x, min_y, max_x, and max_y")
119
+ if set(origin_data) != _ORIGIN_KEYS:
120
+ raise GridSpecError("origin must define x and y")
121
+
122
+ bounds = Bounds(**{key: _decimal(bounds_data[key], f"bounds.{key}") for key in _BOUND_KEYS})
123
+ origin = Origin(
124
+ x=_decimal(origin_data["x"], "origin.x"),
125
+ y=_decimal(origin_data["y"], "origin.y"),
126
+ )
127
+ if bounds.min_x >= bounds.max_x or bounds.min_y >= bounds.max_y:
128
+ raise GridSpecError("bounds must have positive width and height")
129
+
130
+ tolerance = _decimal(data.get("boundary_tolerance", "1e-9"), "boundary_tolerance")
131
+ if tolerance < 0 or tolerance * 2 >= cell_size:
132
+ raise GridSpecError("boundary_tolerance must be nonnegative and less than half a cell")
133
+
134
+ return cls(crs, cell_size, bounds, origin, tolerance)
135
+
136
+ @property
137
+ def crs_id(self) -> str:
138
+ """Return a stable authority identifier or normalized WKT for the CRS."""
139
+ authority = self.crs.to_authority()
140
+ if authority:
141
+ return f"{authority[0]}:{authority[1]}"
142
+ return self.crs.to_wkt(version="WKT2_2019", pretty=False)
143
+
144
+ @property
145
+ def index_extent(self) -> tuple[int, int, int, int]:
146
+ """Return inclusive ``(row_start, row_end, col_start, col_end)``."""
147
+ row_start = _floor((self.origin.y - self.bounds.max_y) / self.cell_size)
148
+ row_end = _ceil((self.origin.y - self.bounds.min_y) / self.cell_size) - 1
149
+ col_start = _floor((self.bounds.min_x - self.origin.x) / self.cell_size)
150
+ col_end = _ceil((self.bounds.max_x - self.origin.x) / self.cell_size) - 1
151
+ return row_start, row_end, col_start, col_end
152
+
153
+ @property
154
+ def fingerprint(self) -> str:
155
+ """Hash the normalized CRS, grid spacing, origin, and indexed extent."""
156
+ row_start, row_end, col_start, col_end = self.index_extent
157
+ identity = {
158
+ "cell_size": _text(self.cell_size),
159
+ "col_extent": [col_start, col_end],
160
+ "crs": self.crs_id,
161
+ "origin": [_text(self.origin.x), _text(self.origin.y)],
162
+ "row_extent": [row_start, row_end],
163
+ "schema": "gridforge-grid-v1",
164
+ }
165
+ canonical = json.dumps(identity, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
166
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
167
+
168
+ def cell_index(self, x: float, y: float) -> tuple[int, int] | None:
169
+ """Return the containing ``(row, column)`` or ``None`` outside the grid.
170
+
171
+ Cells use ``[left, right) × (bottom, top]``. Coordinates within the
172
+ configured tolerance of an anchored grid line are snapped to that line.
173
+ """
174
+ try:
175
+ x_value = _decimal(x, "x")
176
+ y_value = _decimal(y, "y")
177
+ except GridSpecError:
178
+ return None
179
+ col_position = (x_value - self.origin.x) / self.cell_size
180
+ row_position = (self.origin.y - y_value) / self.cell_size
181
+ col_position = self._snap_index(col_position)
182
+ row_position = self._snap_index(row_position)
183
+ column = int(col_position.to_integral_value(rounding=ROUND_FLOOR))
184
+ row = int(row_position.to_integral_value(rounding=ROUND_FLOOR))
185
+ row_start, row_end, col_start, col_end = self.index_extent
186
+ if row_start <= row <= row_end and col_start <= column <= col_end:
187
+ return row, column
188
+ return None
189
+
190
+ def _snap_index(self, value: Decimal) -> Decimal:
191
+ nearest = value.to_integral_value(rounding=ROUND_HALF_UP)
192
+ if abs(value - nearest) * self.cell_size <= self.boundary_tolerance:
193
+ return nearest
194
+ return value
195
+
196
+ def to_dict(self) -> dict[str, Any]:
197
+ """Return JSON-compatible normalized grid specification metadata."""
198
+ return {
199
+ "crs": self.crs_id,
200
+ "cell_size": _text(self.cell_size),
201
+ "bounds": {
202
+ "min_x": _text(self.bounds.min_x),
203
+ "min_y": _text(self.bounds.min_y),
204
+ "max_x": _text(self.bounds.max_x),
205
+ "max_y": _text(self.bounds.max_y),
206
+ },
207
+ "origin": {"x": _text(self.origin.x), "y": _text(self.origin.y)},
208
+ "boundary_tolerance": _text(self.boundary_tolerance),
209
+ }
@@ -0,0 +1 @@
1
+ """Spatial dataset input and output helpers."""
@@ -0,0 +1,317 @@
1
+ """Explicit-CRS inspection and loading for spatial source datasets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from dataclasses import asdict, dataclass
7
+ from pathlib import Path
8
+ from typing import Literal
9
+
10
+ import geopandas as gpd
11
+ import numpy as np
12
+ import pandas as pd
13
+ import pyarrow.parquet as pq
14
+ import rasterio
15
+ from pyproj import CRS
16
+ from shapely.geometry import Point
17
+
18
+ from gridforge.errors import DatasetError
19
+ from gridforge.grid.spec import GridSpec
20
+
21
+ _RASTER_SUFFIXES = {".tif", ".tiff", ".vrt", ".img", ".asc", ".bil", ".grd"}
22
+ _VECTOR_SUFFIXES = {".geojson", ".json", ".gpkg", ".shp", ".gml", ".fgb"}
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class DatasetInfo:
27
+ """Spatial metadata reported by :func:`inspect_dataset`."""
28
+
29
+ path: str
30
+ kind: Literal["vector", "table", "raster"]
31
+ geometry_type: str | None
32
+ crs: str
33
+ feature_count: int | None
34
+ band_count: int | None
35
+ bounds: tuple[float, float, float, float] | None
36
+
37
+ def to_dict(self) -> dict[str, object]:
38
+ return asdict(self)
39
+
40
+
41
+ def _crs_label(crs: CRS) -> str:
42
+ authority = crs.to_authority()
43
+ return f"{authority[0]}:{authority[1]}" if authority else crs.to_wkt("WKT2_2019", pretty=False)
44
+
45
+
46
+ def _resolved_crs(embedded: object | None, source_crs: str | CRS | None) -> CRS:
47
+ parsed_embedded: CRS | None = None
48
+ if embedded is not None:
49
+ try:
50
+ parsed_embedded = CRS.from_user_input(embedded)
51
+ except Exception as exc:
52
+ raise DatasetError(f"embedded source CRS is invalid: {embedded!r}") from exc
53
+ if parsed_embedded is None:
54
+ if source_crs is None:
55
+ raise DatasetError("source CRS is unknown; provide --source-crs explicitly")
56
+ try:
57
+ return CRS.from_user_input(source_crs)
58
+ except Exception as exc:
59
+ raise DatasetError(f"invalid explicit source CRS: {source_crs!r}") from exc
60
+ if source_crs is not None:
61
+ try:
62
+ requested = CRS.from_user_input(source_crs)
63
+ except Exception as exc:
64
+ raise DatasetError(f"invalid explicit source CRS: {source_crs!r}") from exc
65
+ if not parsed_embedded.equals(requested):
66
+ raise DatasetError("explicit source CRS conflicts with embedded CRS")
67
+ return parsed_embedded
68
+
69
+
70
+ def _read_table(path: Path) -> pd.DataFrame:
71
+ try:
72
+ if path.suffix.lower() == ".csv":
73
+ return pd.read_csv(path)
74
+ if path.suffix.lower() == ".parquet":
75
+ return pd.read_parquet(path)
76
+ except Exception as exc:
77
+ raise DatasetError(f"cannot read tabular dataset {path}: {exc}") from exc
78
+ raise DatasetError(f"unsupported tabular input format: {path.suffix or '<none>'}")
79
+
80
+
81
+ def _table_coordinates(
82
+ table: pd.DataFrame,
83
+ *,
84
+ x_column: str,
85
+ y_column: str,
86
+ ) -> tuple[np.ndarray, np.ndarray]:
87
+ missing = [name for name in (x_column, y_column) if name not in table.columns]
88
+ if missing:
89
+ raise DatasetError(f"point table is missing coordinate column(s): {', '.join(missing)}")
90
+ try:
91
+ x_values = pd.to_numeric(table[x_column], errors="raise").to_numpy(dtype=float)
92
+ y_values = pd.to_numeric(table[y_column], errors="raise").to_numpy(dtype=float)
93
+ except (TypeError, ValueError) as exc:
94
+ raise DatasetError("point coordinates must be numeric") from exc
95
+ if not np.isfinite(x_values).all() or not np.isfinite(y_values).all():
96
+ raise DatasetError("point coordinates must be finite and non-null")
97
+ return x_values, y_values
98
+
99
+
100
+ def _is_geoparquet(path: Path) -> bool:
101
+ if path.suffix.lower() != ".parquet":
102
+ return False
103
+ try:
104
+ metadata = pq.read_schema(path).metadata or {}
105
+ except Exception as exc:
106
+ raise DatasetError(f"cannot inspect Parquet dataset {path}: {exc}") from exc
107
+ return b"geo" in metadata
108
+
109
+
110
+ def _read_geodataframe(path: Path) -> gpd.GeoDataFrame:
111
+ try:
112
+ if path.suffix.lower() == ".parquet" and _is_geoparquet(path):
113
+ return gpd.read_parquet(path)
114
+ if path.suffix.lower() in _VECTOR_SUFFIXES:
115
+ return gpd.read_file(path)
116
+ except Exception as exc:
117
+ raise DatasetError(f"cannot read vector dataset {path}: {exc}") from exc
118
+ raise DatasetError(f"unsupported vector input format: {path.suffix or '<none>'}")
119
+
120
+
121
+ def _bounds_tuple(values: object) -> tuple[float, float, float, float] | None:
122
+ array = np.asarray(values, dtype=float)
123
+ if array.size != 4 or not np.isfinite(array).all():
124
+ return None
125
+ return tuple(float(value) for value in array) # type: ignore[return-value]
126
+
127
+
128
+ def inspect_dataset(
129
+ path: str | Path,
130
+ *,
131
+ source_crs: str | CRS | None = None,
132
+ x_column: str = "x",
133
+ y_column: str = "y",
134
+ ) -> DatasetInfo:
135
+ """Inspect vector, coordinate-table, or raster input without guessing CRS."""
136
+ input_path = Path(path)
137
+ suffix = input_path.suffix.lower()
138
+ if suffix in _RASTER_SUFFIXES:
139
+ try:
140
+ with rasterio.open(input_path) as dataset:
141
+ crs = _resolved_crs(dataset.crs, source_crs)
142
+ return DatasetInfo(
143
+ str(input_path),
144
+ "raster",
145
+ "Raster",
146
+ _crs_label(crs),
147
+ None,
148
+ dataset.count,
149
+ _bounds_tuple(dataset.bounds),
150
+ )
151
+ except DatasetError:
152
+ raise
153
+ except Exception as exc:
154
+ raise DatasetError(f"cannot inspect raster dataset {input_path}: {exc}") from exc
155
+
156
+ if suffix == ".csv" or (suffix == ".parquet" and not _is_geoparquet(input_path)):
157
+ table = _read_table(input_path)
158
+ x_values, y_values = _table_coordinates(table, x_column=x_column, y_column=y_column)
159
+ crs = _resolved_crs(None, source_crs)
160
+ if not len(table):
161
+ bounds = None
162
+ else:
163
+ bounds = (
164
+ float(x_values.min()),
165
+ float(y_values.min()),
166
+ float(x_values.max()),
167
+ float(y_values.max()),
168
+ )
169
+ return DatasetInfo(
170
+ str(input_path), "table", "Point", _crs_label(crs), len(table), None, bounds
171
+ )
172
+
173
+ vector = _read_geodataframe(input_path)
174
+ crs = _resolved_crs(vector.crs, source_crs)
175
+ geometry_types = sorted({str(value) for value in vector.geometry.geom_type.dropna().unique()})
176
+ geometry_type = ", ".join(geometry_types) if geometry_types else None
177
+ bounds = _bounds_tuple(vector.total_bounds) if not vector.empty else None
178
+ return DatasetInfo(
179
+ str(input_path),
180
+ "vector",
181
+ geometry_type,
182
+ _crs_label(crs),
183
+ len(vector),
184
+ None,
185
+ bounds,
186
+ )
187
+
188
+
189
+ def load_vector(
190
+ path: str | Path,
191
+ *,
192
+ source_crs: str | CRS | None = None,
193
+ ) -> gpd.GeoDataFrame:
194
+ """Load a supported vector file and require an explicit, consistent CRS."""
195
+ input_path = Path(path)
196
+ vector = _read_geodataframe(input_path)
197
+ crs = _resolved_crs(vector.crs, source_crs)
198
+ if vector.crs is None:
199
+ vector = vector.set_crs(crs)
200
+ return vector
201
+
202
+
203
+ def load_points(
204
+ path: str | Path,
205
+ *,
206
+ x_column: str = "x",
207
+ y_column: str = "y",
208
+ source_crs: str | CRS | None = None,
209
+ ) -> gpd.GeoDataFrame:
210
+ """Load point geometry or an x/y table with an explicit source CRS."""
211
+ input_path = Path(path)
212
+ if input_path.suffix.lower() in {".csv", ".parquet"} and not _is_geoparquet(input_path):
213
+ table = _read_table(input_path)
214
+ crs = _resolved_crs(None, source_crs)
215
+ x_values, y_values = _table_coordinates(table, x_column=x_column, y_column=y_column)
216
+ result = gpd.GeoDataFrame(
217
+ table.copy(),
218
+ geometry=[Point(x, y) for x, y in zip(x_values, y_values, strict=True)],
219
+ crs=crs,
220
+ )
221
+ else:
222
+ result = load_vector(input_path, source_crs=source_crs)
223
+ if result.geometry.isna().any() or result.geometry.is_empty.any():
224
+ raise DatasetError("point dataset contains null or empty geometries")
225
+ if not result.geometry.geom_type.eq("Point").all():
226
+ raise DatasetError("point alignment requires Point geometries")
227
+ if result.geometry.isna().any() or result.geometry.is_empty.any():
228
+ raise DatasetError("point dataset contains null or empty geometries")
229
+ if not result.geometry.geom_type.eq("Point").all():
230
+ raise DatasetError("point alignment requires Point geometries")
231
+ return result
232
+
233
+
234
+ def write_dataset(dataset: gpd.GeoDataFrame, path: str | Path) -> Path:
235
+ """Write one validated aligned dataset as GridForge GeoParquet.
236
+
237
+ Null feature values are retained as nodata and reported as warnings. A
238
+ dataset with structural, identity, geometry, or coverage errors is refused.
239
+ """
240
+ from gridforge.validation import validate_dataset
241
+
242
+ report = validate_dataset(dataset)
243
+ errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
244
+ if errors:
245
+ raise DatasetError("aligned dataset is invalid: " + "; ".join(errors[:3]))
246
+ spec_value = dataset.attrs.get("gridforge_spec")
247
+ if not isinstance(spec_value, dict):
248
+ raise DatasetError("aligned dataset is missing GridForge grid metadata")
249
+ try:
250
+ spec = GridSpec.from_mapping(spec_value)
251
+ except ValueError as exc:
252
+ raise DatasetError(f"aligned dataset grid metadata is invalid: {exc}") from exc
253
+ fingerprint = str(dataset["grid_fingerprint"].iloc[0])
254
+ if fingerprint != spec.fingerprint:
255
+ raise DatasetError("aligned dataset grid fingerprint does not match its specification")
256
+
257
+ output = Path(path)
258
+ output.parent.mkdir(parents=True, exist_ok=True)
259
+ try:
260
+ dataset.to_parquet(output, index=False)
261
+ table = pq.read_table(output)
262
+ metadata = dict(table.schema.metadata or {})
263
+ metadata[b"gridforge:grid_fingerprint"] = fingerprint.encode("utf-8")
264
+ metadata[b"gridforge:grid_spec"] = json.dumps(
265
+ spec.to_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")
266
+ ).encode("utf-8")
267
+ operation = dataset.attrs.get("gridforge_operation")
268
+ if operation is not None:
269
+ metadata[b"gridforge:operation"] = json.dumps(
270
+ operation, ensure_ascii=False, sort_keys=True, separators=(",", ":")
271
+ ).encode("utf-8")
272
+ pq.write_table(table.replace_schema_metadata(metadata), output, compression="zstd")
273
+ except Exception as exc:
274
+ raise DatasetError(f"cannot write aligned GeoParquet {output}: {exc}") from exc
275
+ return output
276
+
277
+
278
+ def read_dataset(path: str | Path) -> gpd.GeoDataFrame:
279
+ """Read GridForge GeoParquet and restore its canonical grid identity."""
280
+ from gridforge.validation import validate_dataset
281
+
282
+ input_path = Path(path)
283
+ try:
284
+ dataset = gpd.read_parquet(input_path)
285
+ schema_metadata = pq.read_schema(input_path).metadata or {}
286
+ except Exception as exc:
287
+ raise DatasetError(f"cannot read aligned GeoParquet {input_path}: {exc}") from exc
288
+ metadata_fingerprint = schema_metadata.get(b"gridforge:grid_fingerprint")
289
+ metadata_spec = schema_metadata.get(b"gridforge:grid_spec")
290
+ if metadata_fingerprint is None or metadata_spec is None:
291
+ raise DatasetError("GeoParquet is missing GridForge grid metadata")
292
+ try:
293
+ spec_value = json.loads(metadata_spec.decode("utf-8"))
294
+ spec = GridSpec.from_mapping(spec_value)
295
+ fingerprint = metadata_fingerprint.decode("utf-8")
296
+ except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
297
+ raise DatasetError("GridForge grid metadata is invalid") from exc
298
+ if spec.fingerprint != fingerprint:
299
+ raise DatasetError("GridForge grid metadata fingerprint does not match grid specification")
300
+ dataset.attrs["gridforge_spec"] = spec.to_dict()
301
+ operation = schema_metadata.get(b"gridforge:operation")
302
+ if operation is not None:
303
+ try:
304
+ dataset.attrs["gridforge_operation"] = json.loads(operation.decode("utf-8"))
305
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
306
+ raise DatasetError("GridForge operation metadata is invalid") from exc
307
+ if dataset.empty or "grid_fingerprint" not in dataset:
308
+ raise DatasetError("aligned dataset has no grid rows")
309
+ if not dataset["grid_fingerprint"].astype(str).eq(fingerprint).all():
310
+ raise DatasetError("aligned dataset grid fingerprint does not match metadata")
311
+ if dataset.crs is None or not spec.crs.equals(dataset.crs):
312
+ raise DatasetError("aligned dataset CRS does not match GridForge grid metadata")
313
+ report = validate_dataset(dataset)
314
+ errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
315
+ if errors:
316
+ raise DatasetError("aligned dataset is invalid: " + "; ".join(errors[:3]))
317
+ return dataset
gridforge/io/grid.py ADDED
@@ -0,0 +1,110 @@
1
+ """GeoParquet persistence for canonical grids."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ import geopandas as gpd
9
+ import pyarrow.parquet as pq
10
+
11
+ from gridforge.errors import DatasetError
12
+ from gridforge.grid.spec import GridSpec
13
+
14
+ _GRID_COLUMNS = {
15
+ "grid_id",
16
+ "grid_fingerprint",
17
+ "row",
18
+ "column",
19
+ "left",
20
+ "bottom",
21
+ "right",
22
+ "top",
23
+ "geometry",
24
+ }
25
+
26
+
27
+ def write_grid(grid: gpd.GeoDataFrame, path: str | Path) -> Path:
28
+ """Write a canonical grid as GeoParquet with GridForge metadata."""
29
+ _validate_grid_schema(grid)
30
+ spec_value = grid.attrs.get("gridforge_spec")
31
+ if not isinstance(spec_value, dict):
32
+ raise DatasetError("grid is missing GridForge specification metadata")
33
+ try:
34
+ spec = GridSpec.from_mapping(spec_value)
35
+ except ValueError as exc:
36
+ raise DatasetError(f"grid specification metadata is invalid: {exc}") from exc
37
+ if grid.crs is None or not spec.crs.equals(grid.crs):
38
+ raise DatasetError("grid CRS does not match GridForge specification")
39
+ if not grid["grid_fingerprint"].astype(str).eq(spec.fingerprint).all():
40
+ raise DatasetError("grid fingerprint does not match GridForge specification")
41
+ from gridforge.validation import validate_grid
42
+
43
+ report = validate_grid(grid)
44
+ errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
45
+ if errors:
46
+ raise DatasetError("grid is invalid: " + "; ".join(errors[:3]))
47
+ output = Path(path)
48
+ output.parent.mkdir(parents=True, exist_ok=True)
49
+ grid.to_parquet(output, index=False)
50
+ table = pq.read_table(output)
51
+ metadata = dict(table.schema.metadata or {})
52
+ metadata[b"gridforge:grid_fingerprint"] = str(grid["grid_fingerprint"].iloc[0]).encode()
53
+ metadata[b"gridforge:grid_spec"] = json.dumps(
54
+ spec.to_dict(), sort_keys=True, separators=(",", ":")
55
+ ).encode()
56
+ pq.write_table(table.replace_schema_metadata(metadata), output, compression="zstd")
57
+ return output
58
+
59
+
60
+ def read_grid(path: str | Path) -> gpd.GeoDataFrame:
61
+ """Read a GeoParquet grid and reject incomplete or ambiguous identity."""
62
+ input_path = Path(path)
63
+ try:
64
+ grid = gpd.read_parquet(input_path)
65
+ table = pq.read_table(input_path)
66
+ except Exception as exc:
67
+ raise DatasetError(f"cannot read grid GeoParquet {input_path}: {exc}") from exc
68
+ _validate_grid_schema(grid)
69
+ metadata_fingerprint = (table.schema.metadata or {}).get(b"gridforge:grid_fingerprint")
70
+ metadata_spec = (table.schema.metadata or {}).get(b"gridforge:grid_spec")
71
+ if metadata_fingerprint is None or metadata_spec is None:
72
+ raise DatasetError("GeoParquet is missing GridForge grid metadata")
73
+ try:
74
+ decoded_fingerprint = metadata_fingerprint.decode("utf-8")
75
+ except UnicodeDecodeError as exc:
76
+ raise DatasetError("grid fingerprint metadata is invalid") from exc
77
+ if decoded_fingerprint != str(grid["grid_fingerprint"].iloc[0]):
78
+ raise DatasetError("grid fingerprint metadata does not match grid rows")
79
+ try:
80
+ grid.attrs["gridforge_spec"] = json.loads(metadata_spec.decode())
81
+ spec = GridSpec.from_mapping(grid.attrs["gridforge_spec"])
82
+ except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
83
+ raise DatasetError("grid specification metadata is invalid") from exc
84
+ if spec.fingerprint != str(grid["grid_fingerprint"].iloc[0]):
85
+ raise DatasetError("grid specification does not match grid fingerprint")
86
+ from gridforge.validation import validate_grid
87
+
88
+ report = validate_grid(grid)
89
+ errors = [finding.message for finding in report.findings if finding.severity == "ERROR"]
90
+ if errors:
91
+ raise DatasetError("grid is invalid: " + "; ".join(errors[:3]))
92
+ return grid
93
+
94
+
95
+ def _validate_grid_schema(grid: gpd.GeoDataFrame) -> None:
96
+ missing = sorted(_GRID_COLUMNS - set(grid.columns))
97
+ if missing:
98
+ raise DatasetError(f"grid is missing required columns: {', '.join(missing)}")
99
+ if grid.empty:
100
+ raise DatasetError("grid must contain at least one cell")
101
+ if grid.crs is None:
102
+ raise DatasetError("grid CRS is unknown")
103
+ if grid["grid_id"].isna().any() or grid["grid_id"].duplicated().any():
104
+ raise DatasetError("grid IDs must be non-null and unique")
105
+ fingerprints = grid["grid_fingerprint"].dropna().astype(str).unique()
106
+ if len(fingerprints) != 1 or grid["grid_fingerprint"].isna().any():
107
+ raise DatasetError("grid rows must have one non-null grid fingerprint")
108
+ expected = grid.sort_values(["row", "column"], kind="stable").index
109
+ if not grid.index.equals(expected):
110
+ raise DatasetError("grid rows must be sorted by row and column")