gisknife-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gisknife_engine/__init__.py +3 -0
- gisknife_engine/cli.py +125 -0
- gisknife_engine/duckdb_ops.py +191 -0
- gisknife_engine/raster_ops.py +41 -0
- gisknife_engine/vector_ops.py +284 -0
- gisknife_engine-0.1.0.dist-info/METADATA +24 -0
- gisknife_engine-0.1.0.dist-info/RECORD +10 -0
- gisknife_engine-0.1.0.dist-info/WHEEL +4 -0
- gisknife_engine-0.1.0.dist-info/entry_points.txt +2 -0
- gisknife_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
gisknife_engine/cli.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import importlib.util
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import sys
|
|
8
|
+
import traceback
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .duckdb_ops import fetch_overture_bbox, run_spatial_sql
|
|
12
|
+
from .raster_ops import has_rasterio, inspect_raster
|
|
13
|
+
from .vector_ops import analyze_vector, convert_vector, qa_vector, reproject_vector
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_capabilities() -> dict[str, Any]:
|
|
17
|
+
return {
|
|
18
|
+
"ok": True,
|
|
19
|
+
"engine": "python-uvx",
|
|
20
|
+
"pythonVersion": sys.version.split()[0],
|
|
21
|
+
"capabilities": {
|
|
22
|
+
"duckdb": importlib.util.find_spec("duckdb") is not None,
|
|
23
|
+
"geopandas": importlib.util.find_spec("geopandas") is not None,
|
|
24
|
+
"pyogrio": importlib.util.find_spec("pyogrio") is not None,
|
|
25
|
+
"pyproj": importlib.util.find_spec("pyproj") is not None,
|
|
26
|
+
"shapely": importlib.util.find_spec("shapely") is not None,
|
|
27
|
+
"rasterio": has_rasterio(),
|
|
28
|
+
"libpysal": importlib.util.find_spec("libpysal") is not None,
|
|
29
|
+
},
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def dispatch_rpc(req: dict[str, Any]) -> dict[str, Any]:
|
|
34
|
+
op = req.get("op")
|
|
35
|
+
params = req.get("params") or {}
|
|
36
|
+
if params.get('outputPath'):
|
|
37
|
+
target = Path(params['outputPath']).expanduser()
|
|
38
|
+
for key in ('inputPath', 'secondInputPath'):
|
|
39
|
+
if params.get(key) and target.exists() and target.samefile(Path(params[key]).expanduser()):
|
|
40
|
+
raise ValueError('Output would overwrite an input file')
|
|
41
|
+
|
|
42
|
+
if op == "capabilities":
|
|
43
|
+
return get_capabilities()
|
|
44
|
+
if op == "convert":
|
|
45
|
+
return convert_vector(
|
|
46
|
+
input_path=params["inputPath"],
|
|
47
|
+
output_path=params["outputPath"],
|
|
48
|
+
target_crs=params.get("targetCrs"),
|
|
49
|
+
)
|
|
50
|
+
if op == "reproject":
|
|
51
|
+
return reproject_vector(
|
|
52
|
+
input_path=params["inputPath"],
|
|
53
|
+
output_path=params["outputPath"],
|
|
54
|
+
to_crs=params.get("toCrs", "EPSG:4326"),
|
|
55
|
+
from_crs=params.get("fromCrs"),
|
|
56
|
+
)
|
|
57
|
+
if op == "qa":
|
|
58
|
+
return qa_vector(
|
|
59
|
+
input_path=params["inputPath"],
|
|
60
|
+
output_path=params.get("outputPath"),
|
|
61
|
+
fix=bool(params.get("fix", False)),
|
|
62
|
+
)
|
|
63
|
+
if op == "analyze":
|
|
64
|
+
return analyze_vector(
|
|
65
|
+
op=params["analysisOp"],
|
|
66
|
+
input_path=params["inputPath"],
|
|
67
|
+
output_path=params.get("outputPath"),
|
|
68
|
+
second_input_path=params.get("secondInputPath"),
|
|
69
|
+
distance_meters=float(params.get("distanceMeters", 500.0)),
|
|
70
|
+
tolerance=float(params.get("tolerance", 0.0005)),
|
|
71
|
+
dissolve_by=params.get("dissolveBy"),
|
|
72
|
+
)
|
|
73
|
+
if op == "sql":
|
|
74
|
+
return run_spatial_sql(
|
|
75
|
+
sql=params["sql"],
|
|
76
|
+
output_path=params.get("outputPath"),
|
|
77
|
+
result_crs=params.get('resultCrs'),
|
|
78
|
+
)
|
|
79
|
+
if op == "overture_fetch":
|
|
80
|
+
return fetch_overture_bbox(
|
|
81
|
+
bbox=params["bbox"],
|
|
82
|
+
layers=params.get("layers", ["roads", "buildings"]),
|
|
83
|
+
output_path=params["outputPath"],
|
|
84
|
+
release=params.get('release'),
|
|
85
|
+
)
|
|
86
|
+
if op == "raster_inspect":
|
|
87
|
+
return inspect_raster(input_path=params["inputPath"])
|
|
88
|
+
|
|
89
|
+
raise ValueError(f"Unknown gisknife-engine RPC op: {op}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def main(argv: list[str] | None = None) -> int:
|
|
93
|
+
parser = argparse.ArgumentParser(prog="gisknife-engine")
|
|
94
|
+
parser.add_argument("--capabilities", action="store_true", help="Print engine capabilities JSON")
|
|
95
|
+
parser.add_argument("--rpc", action="store_true", help="Read JSON RPC request from stdin")
|
|
96
|
+
args = parser.parse_args(argv)
|
|
97
|
+
|
|
98
|
+
if args.capabilities:
|
|
99
|
+
print(json.dumps(get_capabilities()))
|
|
100
|
+
return 0
|
|
101
|
+
|
|
102
|
+
if args.rpc:
|
|
103
|
+
try:
|
|
104
|
+
raw = sys.stdin.read()
|
|
105
|
+
req = json.loads(raw)
|
|
106
|
+
res = dispatch_rpc(req)
|
|
107
|
+
print(json.dumps(res, ensure_ascii=False))
|
|
108
|
+
return 0
|
|
109
|
+
except Exception as exc:
|
|
110
|
+
err_payload = {
|
|
111
|
+
"ok": False,
|
|
112
|
+
"status": "error",
|
|
113
|
+
"engine": "python-uvx",
|
|
114
|
+
"error": str(exc),
|
|
115
|
+
"traceback": traceback.format_exc(),
|
|
116
|
+
}
|
|
117
|
+
print(json.dumps(err_payload, ensure_ascii=False))
|
|
118
|
+
return 1
|
|
119
|
+
|
|
120
|
+
parser.print_help()
|
|
121
|
+
return 0
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
if __name__ == "__main__":
|
|
125
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import re
|
|
5
|
+
import tempfile
|
|
6
|
+
import urllib.request
|
|
7
|
+
import xml.etree.ElementTree as ET
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import duckdb
|
|
12
|
+
import geopandas as gpd
|
|
13
|
+
from shapely import wkb as shapely_wkb
|
|
14
|
+
|
|
15
|
+
from .vector_ops import summarize_gdf, write_vector_smart
|
|
16
|
+
|
|
17
|
+
def current_overture_release() -> str:
|
|
18
|
+
url = 'https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&delimiter=%2F&prefix=release%2F'
|
|
19
|
+
with urllib.request.urlopen(url, timeout=15) as response:
|
|
20
|
+
root = ET.fromstring(response.read(256_000))
|
|
21
|
+
versions = [prefix.text.split('/')[1] for prefix in root.findall('.//{*}CommonPrefixes/{*}Prefix')
|
|
22
|
+
if prefix.text and re.fullmatch(r'release/\d{4}-\d{2}-\d{2}\.\d+/', prefix.text)]
|
|
23
|
+
if not versions:
|
|
24
|
+
raise ValueError('Overture has no identifiable release')
|
|
25
|
+
return max(versions, key=lambda version: (version[:10], int(version.split('.')[1])))
|
|
26
|
+
|
|
27
|
+
OVERTURE_LAYER_MAP = {
|
|
28
|
+
"roads": ("transportation", "segment"),
|
|
29
|
+
"buildings": ("buildings", "building"),
|
|
30
|
+
"pois": ("places", "place"),
|
|
31
|
+
"water": ("base", "water"),
|
|
32
|
+
"landuse": ("base", "land_use"),
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _open_spatial_duckdb(load_httpfs: bool = False) -> duckdb.DuckDBPyConnection:
|
|
37
|
+
con = duckdb.connect(database=":memory:")
|
|
38
|
+
con.execute("INSTALL spatial; LOAD spatial;")
|
|
39
|
+
try:
|
|
40
|
+
con.execute("SET geometry_always_xy = true;")
|
|
41
|
+
except Exception:
|
|
42
|
+
pass
|
|
43
|
+
if load_httpfs:
|
|
44
|
+
con.execute("INSTALL httpfs; LOAD httpfs;")
|
|
45
|
+
con.execute("SET s3_region='us-west-2';")
|
|
46
|
+
return con
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def run_spatial_sql(sql: str, output_path: str | None = None, result_crs: str | None = None) -> dict[str, Any]:
|
|
50
|
+
needs_http = any(p in sql.lower() for p in ("http://", "https://", "s3://"))
|
|
51
|
+
con = _open_spatial_duckdb(load_httpfs=needs_http)
|
|
52
|
+
try:
|
|
53
|
+
rel = con.sql(sql)
|
|
54
|
+
col_names = rel.columns
|
|
55
|
+
col_types = [str(t).upper() for t in rel.types]
|
|
56
|
+
|
|
57
|
+
geom_cols = [
|
|
58
|
+
name for name, typ in zip(col_names, col_types) if "GEOMETRY" in typ or "WKB_BLOB" in typ
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
if geom_cols:
|
|
62
|
+
geom_col = geom_cols[0]
|
|
63
|
+
other_cols = [c for c in col_names if c != geom_col]
|
|
64
|
+
select_parts = [f'"{c}"' for c in other_cols] + [f'ST_AsWKB("{geom_col}") AS __gisknife_wkb']
|
|
65
|
+
wrapped = f"SELECT {', '.join(select_parts)} FROM ({sql.rstrip(';')}) AS __sub"
|
|
66
|
+
rows = con.execute(wrapped).fetchall()
|
|
67
|
+
out_cols = other_cols
|
|
68
|
+
records: list[dict[str, Any]] = []
|
|
69
|
+
geoms = []
|
|
70
|
+
for row in rows:
|
|
71
|
+
rec = {col: val for col, val in zip(out_cols, row[:-1])}
|
|
72
|
+
raw_wkb = row[-1]
|
|
73
|
+
geoms.append(shapely_wkb.loads(bytes(raw_wkb)) if raw_wkb is not None else None)
|
|
74
|
+
records.append(rec)
|
|
75
|
+
|
|
76
|
+
declared_type = str(rel.types[col_names.index(geom_col)])
|
|
77
|
+
tagged = re.fullmatch(r"GEOMETRY\('([^']+)'\)", declared_type, flags=re.I)
|
|
78
|
+
crs = tagged.group(1) if tagged else result_crs
|
|
79
|
+
if not crs and output_path:
|
|
80
|
+
raise ValueError('SQL geometry has no CRS. Supply --result-crs to label its existing coordinates.')
|
|
81
|
+
gdf = gpd.GeoDataFrame(records, geometry=geoms, crs=crs)
|
|
82
|
+
written = write_vector_smart(gdf, output_path) if output_path else None
|
|
83
|
+
return summarize_gdf(
|
|
84
|
+
gdf,
|
|
85
|
+
command="sql",
|
|
86
|
+
output_path=written,
|
|
87
|
+
extra={"rowCount": len(gdf)},
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
rows = rel.fetchall()
|
|
91
|
+
dict_rows = [{col: val for col, val in zip(col_names, row)} for row in rows]
|
|
92
|
+
written: str | None = None
|
|
93
|
+
if output_path:
|
|
94
|
+
out_p = Path(output_path).expanduser().resolve()
|
|
95
|
+
out_p.parent.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
with out_p.open("w", encoding="utf-8", newline="") as fp:
|
|
97
|
+
writer = csv.DictWriter(fp, fieldnames=col_names)
|
|
98
|
+
writer.writeheader()
|
|
99
|
+
writer.writerows(dict_rows)
|
|
100
|
+
written = str(out_p)
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
"ok": True,
|
|
104
|
+
"status": "completed",
|
|
105
|
+
"command": "sql",
|
|
106
|
+
"engine": "python-uvx",
|
|
107
|
+
"outputPath": written,
|
|
108
|
+
"rowCount": len(dict_rows),
|
|
109
|
+
"columns": col_names,
|
|
110
|
+
"sampleRows": dict_rows[:20],
|
|
111
|
+
"warnings": [],
|
|
112
|
+
}
|
|
113
|
+
finally:
|
|
114
|
+
con.close()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def fetch_overture_bbox(
|
|
118
|
+
bbox: list[float],
|
|
119
|
+
layers: list[str],
|
|
120
|
+
output_path: str,
|
|
121
|
+
release: str | None = None,
|
|
122
|
+
limit_per_layer: int = 50000,
|
|
123
|
+
) -> dict[str, Any]:
|
|
124
|
+
if len(bbox) != 4:
|
|
125
|
+
raise ValueError("bbox must be [minLon, minLat, maxLon, maxLat]")
|
|
126
|
+
min_lon, min_lat, max_lon, max_lat = bbox
|
|
127
|
+
if not (-180 <= min_lon < max_lon <= 180 and -90 <= min_lat < max_lat <= 90):
|
|
128
|
+
raise ValueError('Invalid WGS84 bounding box')
|
|
129
|
+
release = release or current_overture_release()
|
|
130
|
+
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}\.\d+', release):
|
|
131
|
+
raise ValueError('Invalid Overture release identifier')
|
|
132
|
+
target_layers = (
|
|
133
|
+
list(OVERTURE_LAYER_MAP) if not layers or "all" in layers else layers
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
con = _open_spatial_duckdb(load_httpfs=True)
|
|
137
|
+
frames: list[gpd.GeoDataFrame] = []
|
|
138
|
+
try:
|
|
139
|
+
for layer in target_layers:
|
|
140
|
+
mapping = OVERTURE_LAYER_MAP.get(layer)
|
|
141
|
+
if not mapping:
|
|
142
|
+
raise ValueError(f'Unknown Overture layer: {layer}')
|
|
143
|
+
theme, o_type = mapping
|
|
144
|
+
s3_url = f"s3://overturemaps-us-west-2/release/{release}/theme={theme}/type={o_type}/*"
|
|
145
|
+
sql = f"""
|
|
146
|
+
SELECT
|
|
147
|
+
id,
|
|
148
|
+
'{layer}' AS gisknife_layer,
|
|
149
|
+
COALESCE(names.primary, '') AS name,
|
|
150
|
+
ST_AsWKB(geometry) AS __gisknife_wkb
|
|
151
|
+
FROM read_parquet('{s3_url}', filename=true, hive_partitioning=1)
|
|
152
|
+
WHERE bbox.xmin <= {max_lon}
|
|
153
|
+
AND bbox.xmax >= {min_lon}
|
|
154
|
+
AND bbox.ymin <= {max_lat}
|
|
155
|
+
AND bbox.ymax >= {min_lat}
|
|
156
|
+
LIMIT {int(limit_per_layer) + 1}
|
|
157
|
+
"""
|
|
158
|
+
rows = con.execute(sql).fetchall()
|
|
159
|
+
if len(rows) > limit_per_layer:
|
|
160
|
+
raise ValueError(f'{layer} exceeds {limit_per_layer} features. Request a smaller bounding box.')
|
|
161
|
+
if not rows:
|
|
162
|
+
continue
|
|
163
|
+
records = [{"id": r[0], "layer": r[1], "name": r[2]} for r in rows]
|
|
164
|
+
geoms = [shapely_wkb.loads(bytes(r[3])) for r in rows]
|
|
165
|
+
frames.append(gpd.GeoDataFrame(records, geometry=geoms, crs="EPSG:4326"))
|
|
166
|
+
finally:
|
|
167
|
+
con.close()
|
|
168
|
+
|
|
169
|
+
combined = (
|
|
170
|
+
gpd.GeoDataFrame(
|
|
171
|
+
gpd.pd.concat(frames, ignore_index=True), # type: ignore[attr-defined]
|
|
172
|
+
crs="EPSG:4326",
|
|
173
|
+
)
|
|
174
|
+
if frames
|
|
175
|
+
else gpd.GeoDataFrame({"id": [], "layer": [], "name": []}, geometry=[], crs="EPSG:4326")
|
|
176
|
+
)
|
|
177
|
+
target = Path(output_path).expanduser().resolve()
|
|
178
|
+
if target.suffix.lower() not in ('.parquet', '.geoparquet', '.json', '.geojson'):
|
|
179
|
+
raise ValueError('Overture output must be GeoJSON or GeoParquet')
|
|
180
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
181
|
+
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
|
|
182
|
+
staged = Path(directory) / target.name
|
|
183
|
+
write_vector_smart(combined, str(staged))
|
|
184
|
+
staged.replace(target)
|
|
185
|
+
written = str(target)
|
|
186
|
+
return summarize_gdf(
|
|
187
|
+
combined,
|
|
188
|
+
command="fetch overture",
|
|
189
|
+
output_path=written,
|
|
190
|
+
extra={"source": f"overture-maps-s3:{release}"},
|
|
191
|
+
)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def has_rasterio() -> bool:
|
|
7
|
+
try:
|
|
8
|
+
import rasterio # noqa: F401
|
|
9
|
+
|
|
10
|
+
return True
|
|
11
|
+
except ImportError:
|
|
12
|
+
return False
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def inspect_raster(input_path: str) -> dict[str, Any]:
|
|
16
|
+
try:
|
|
17
|
+
import rasterio
|
|
18
|
+
except ImportError as exc:
|
|
19
|
+
raise RuntimeError(
|
|
20
|
+
"GeoTIFF raster support requires the optional raster extra: "
|
|
21
|
+
"uv tool install 'gisknife-engine[raster]' or uv sync --extra raster"
|
|
22
|
+
) from exc
|
|
23
|
+
|
|
24
|
+
with rasterio.open(input_path) as src:
|
|
25
|
+
bounds = [float(src.bounds.left), float(src.bounds.bottom), float(src.bounds.right), float(src.bounds.top)]
|
|
26
|
+
return {
|
|
27
|
+
"ok": True,
|
|
28
|
+
"status": "completed",
|
|
29
|
+
"command": "raster inspect",
|
|
30
|
+
"engine": "python-uvx",
|
|
31
|
+
"outputPath": input_path,
|
|
32
|
+
"crs": str(src.crs) if src.crs else "unknown",
|
|
33
|
+
"width": int(src.width),
|
|
34
|
+
"height": int(src.height),
|
|
35
|
+
"bandCount": int(src.count),
|
|
36
|
+
"dtypes": list(src.dtypes),
|
|
37
|
+
"nodata": src.nodata,
|
|
38
|
+
"bbox": bounds,
|
|
39
|
+
"driver": src.driver,
|
|
40
|
+
"warnings": [],
|
|
41
|
+
}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import math
|
|
5
|
+
import tempfile
|
|
6
|
+
import zipfile
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import geopandas as gpd
|
|
11
|
+
import pyogrio
|
|
12
|
+
import shapely
|
|
13
|
+
from shapely.validation import explain_validity, make_valid
|
|
14
|
+
|
|
15
|
+
DRIVER_MAP = {
|
|
16
|
+
".geojson": "GeoJSON",
|
|
17
|
+
".json": "GeoJSON",
|
|
18
|
+
".gpkg": "GPKG",
|
|
19
|
+
".fgb": "FlatGeobuf",
|
|
20
|
+
".kml": "KML",
|
|
21
|
+
".dxf": "DXF",
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_vector_smart(input_path: str) -> gpd.GeoDataFrame:
|
|
26
|
+
path = Path(input_path).expanduser().resolve()
|
|
27
|
+
if not path.exists():
|
|
28
|
+
raise FileNotFoundError(f"Input file not found: {path}")
|
|
29
|
+
|
|
30
|
+
ext = path.suffix.lower()
|
|
31
|
+
if ext in (".parquet", ".geoparquet"):
|
|
32
|
+
return gpd.read_parquet(path)
|
|
33
|
+
|
|
34
|
+
target = f"/vsizip/{path}" if ext == ".zip" else str(path)
|
|
35
|
+
|
|
36
|
+
if ext in (".shp", ".zip"):
|
|
37
|
+
for encoding in ("UTF-8", "GB18030", "GBK", "LATIN1"):
|
|
38
|
+
try:
|
|
39
|
+
gdf = gpd.read_file(target, engine="pyogrio", encoding=encoding)
|
|
40
|
+
return gdf
|
|
41
|
+
except Exception:
|
|
42
|
+
continue
|
|
43
|
+
return gpd.read_file(target, engine="pyogrio")
|
|
44
|
+
|
|
45
|
+
return gpd.read_file(target, engine="pyogrio")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def write_vector_smart(gdf: gpd.GeoDataFrame, output_path: str) -> str:
|
|
49
|
+
path = Path(output_path).expanduser().resolve()
|
|
50
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
ext = path.suffix.lower()
|
|
52
|
+
|
|
53
|
+
if ext in (".parquet", ".geoparquet"):
|
|
54
|
+
gdf.to_parquet(path, index=False)
|
|
55
|
+
return str(path)
|
|
56
|
+
|
|
57
|
+
if ext == ".zip":
|
|
58
|
+
with tempfile.TemporaryDirectory(prefix="gk-shp-") as tmpdir:
|
|
59
|
+
stem = path.stem or "export"
|
|
60
|
+
shp_path = Path(tmpdir) / f"{stem}.shp"
|
|
61
|
+
pyogrio.write_dataframe(gdf, shp_path, driver="ESRI Shapefile", encoding="UTF-8")
|
|
62
|
+
cpg_path = Path(tmpdir) / f"{stem}.cpg"
|
|
63
|
+
cpg_path.write_text("UTF-8", encoding="utf-8")
|
|
64
|
+
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
65
|
+
for member in Path(tmpdir).iterdir():
|
|
66
|
+
if member.is_file():
|
|
67
|
+
zf.write(member, arcname=member.name)
|
|
68
|
+
return str(path)
|
|
69
|
+
|
|
70
|
+
if ext == ".shp":
|
|
71
|
+
pyogrio.write_dataframe(gdf, path, driver="ESRI Shapefile", encoding="UTF-8")
|
|
72
|
+
path.with_suffix(".cpg").write_text("UTF-8", encoding="utf-8")
|
|
73
|
+
return str(path)
|
|
74
|
+
|
|
75
|
+
driver = DRIVER_MAP.get(ext)
|
|
76
|
+
if driver:
|
|
77
|
+
pyogrio.write_dataframe(gdf, path, driver=driver)
|
|
78
|
+
return str(path)
|
|
79
|
+
|
|
80
|
+
pyogrio.write_dataframe(gdf, path)
|
|
81
|
+
return str(path)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def summarize_gdf(
|
|
85
|
+
gdf: gpd.GeoDataFrame,
|
|
86
|
+
command: str,
|
|
87
|
+
output_path: str | None = None,
|
|
88
|
+
warnings: list[dict[str, Any]] | None = None,
|
|
89
|
+
extra: dict[str, Any] | None = None,
|
|
90
|
+
) -> dict[str, Any]:
|
|
91
|
+
crs_str = str(gdf.crs) if gdf.crs else "unknown"
|
|
92
|
+
geom_types = sorted({str(t) for t in gdf.geometry.geom_type.dropna().unique()}) if len(gdf) else []
|
|
93
|
+
non_empty = gdf[~gdf.geometry.is_empty & gdf.geometry.notna()] if len(gdf) else gdf
|
|
94
|
+
bbox = [round(float(v), 7) for v in non_empty.total_bounds.tolist()] if len(non_empty) else None
|
|
95
|
+
sample_props = [c for c in gdf.columns if c != gdf.geometry.name][:20]
|
|
96
|
+
|
|
97
|
+
envelope: dict[str, Any] = {
|
|
98
|
+
"ok": True,
|
|
99
|
+
"status": "completed",
|
|
100
|
+
"command": command,
|
|
101
|
+
"engine": "python-uvx",
|
|
102
|
+
"crs": crs_str,
|
|
103
|
+
"featureCount": int(len(gdf)),
|
|
104
|
+
"geometryTypes": geom_types,
|
|
105
|
+
"sampleProperties": sample_props,
|
|
106
|
+
"warnings": warnings or [],
|
|
107
|
+
}
|
|
108
|
+
if output_path:
|
|
109
|
+
envelope["outputPath"] = str(Path(output_path).resolve())
|
|
110
|
+
if bbox:
|
|
111
|
+
envelope["bbox"] = bbox
|
|
112
|
+
if extra:
|
|
113
|
+
envelope.update(extra)
|
|
114
|
+
return envelope
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def convert_vector(
|
|
118
|
+
input_path: str,
|
|
119
|
+
output_path: str,
|
|
120
|
+
target_crs: str | None = None,
|
|
121
|
+
) -> dict[str, Any]:
|
|
122
|
+
gdf = read_vector_smart(input_path)
|
|
123
|
+
if target_crs:
|
|
124
|
+
if gdf.crs is None:
|
|
125
|
+
raise ValueError("Input vector has no CRS defined; pass --from before reprojecting.")
|
|
126
|
+
gdf = gdf.to_crs(target_crs)
|
|
127
|
+
written = write_vector_smart(gdf, output_path)
|
|
128
|
+
return summarize_gdf(gdf, command="convert", output_path=written)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def reproject_vector(
|
|
132
|
+
input_path: str,
|
|
133
|
+
output_path: str,
|
|
134
|
+
to_crs: str = "EPSG:4326",
|
|
135
|
+
from_crs: str | None = None,
|
|
136
|
+
) -> dict[str, Any]:
|
|
137
|
+
gdf = read_vector_smart(input_path)
|
|
138
|
+
if from_crs:
|
|
139
|
+
gdf = gdf.set_crs(from_crs, allow_override=True)
|
|
140
|
+
if gdf.crs is None:
|
|
141
|
+
raise ValueError("Source CRS is undefined. Provide from_crs explicitly.")
|
|
142
|
+
gdf = gdf.to_crs(to_crs)
|
|
143
|
+
written = write_vector_smart(gdf, output_path)
|
|
144
|
+
return summarize_gdf(gdf, command="reproject", output_path=written)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def qa_vector(
|
|
148
|
+
input_path: str,
|
|
149
|
+
output_path: str | None = None,
|
|
150
|
+
fix: bool = False,
|
|
151
|
+
) -> dict[str, Any]:
|
|
152
|
+
gdf = read_vector_smart(input_path)
|
|
153
|
+
issues: list[dict[str, Any]] = []
|
|
154
|
+
geom_col = gdf.geometry.name
|
|
155
|
+
|
|
156
|
+
for idx, geom in enumerate(gdf[geom_col]):
|
|
157
|
+
if geom is None or geom.is_empty:
|
|
158
|
+
issues.append(
|
|
159
|
+
{"featureIndex": idx, "code": "NULL_OR_EMPTY", "message": f"Feature #{idx} is null or empty"}
|
|
160
|
+
)
|
|
161
|
+
elif not shapely.is_valid(geom):
|
|
162
|
+
issues.append(
|
|
163
|
+
{
|
|
164
|
+
"featureIndex": idx,
|
|
165
|
+
"code": "INVALID_GEOMETRY",
|
|
166
|
+
"message": f"Feature #{idx}: {explain_validity(geom)}",
|
|
167
|
+
}
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
written: str | None = None
|
|
171
|
+
if fix:
|
|
172
|
+
fixed_geoms = [
|
|
173
|
+
make_valid(g) if (g is not None and not g.is_empty and not shapely.is_valid(g)) else g
|
|
174
|
+
for g in gdf[geom_col]
|
|
175
|
+
]
|
|
176
|
+
gdf = gdf.copy()
|
|
177
|
+
gdf[geom_col] = fixed_geoms
|
|
178
|
+
gdf = gdf[gdf[geom_col].notna() & ~gdf[geom_col].is_empty].reset_index(drop=True)
|
|
179
|
+
if output_path:
|
|
180
|
+
written = write_vector_smart(gdf, output_path)
|
|
181
|
+
|
|
182
|
+
return summarize_gdf(
|
|
183
|
+
gdf,
|
|
184
|
+
command="qa",
|
|
185
|
+
output_path=written,
|
|
186
|
+
extra={
|
|
187
|
+
"qa": {
|
|
188
|
+
"featureCount": int(len(gdf)),
|
|
189
|
+
"issueCount": len(issues),
|
|
190
|
+
"issues": issues[:100],
|
|
191
|
+
"fixed": bool(fix),
|
|
192
|
+
}
|
|
193
|
+
},
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def analyze_vector(
|
|
198
|
+
op: str,
|
|
199
|
+
input_path: str,
|
|
200
|
+
output_path: str | None = None,
|
|
201
|
+
second_input_path: str | None = None,
|
|
202
|
+
distance_meters: float = 500.0,
|
|
203
|
+
tolerance: float = 0.0005,
|
|
204
|
+
dissolve_by: str | None = None,
|
|
205
|
+
) -> dict[str, Any]:
|
|
206
|
+
gdf = read_vector_smart(input_path)
|
|
207
|
+
warnings: list[dict[str, Any]] = []
|
|
208
|
+
orig_crs = gdf.crs
|
|
209
|
+
unit_metres = 1.0
|
|
210
|
+
if op in ('measure', 'buffer'):
|
|
211
|
+
if not orig_crs or not (orig_crs.is_projected or orig_crs.is_geographic):
|
|
212
|
+
raise ValueError('Metric operations require a known horizontal CRS')
|
|
213
|
+
if orig_crs.is_projected:
|
|
214
|
+
axes = orig_crs.axis_info[:2]
|
|
215
|
+
if len(axes) != 2 or any(not axis.unit_name or axis.unit_name.lower() == 'unknown' for axis in axes):
|
|
216
|
+
raise ValueError('Unknown projected axis units')
|
|
217
|
+
unit_metres = axes[0].unit_conversion_factor
|
|
218
|
+
if not math.isfinite(unit_metres) or unit_metres <= 0 or unit_metres != axes[1].unit_conversion_factor:
|
|
219
|
+
raise ValueError('Unsupported projected axis units')
|
|
220
|
+
|
|
221
|
+
if op == "buffer":
|
|
222
|
+
if orig_crs is None:
|
|
223
|
+
raise ValueError("Buffer requires a known CRS.")
|
|
224
|
+
if orig_crs.is_geographic:
|
|
225
|
+
utm = gdf.estimate_utm_crs()
|
|
226
|
+
warnings.append(
|
|
227
|
+
{
|
|
228
|
+
"code": "AUTO_PROJECTED_METRIC",
|
|
229
|
+
"message": f"Projected geographic layer to {utm} for accurate {distance_meters}m buffer, then projected back to {orig_crs}.",
|
|
230
|
+
}
|
|
231
|
+
)
|
|
232
|
+
projected = gdf.to_crs(utm)
|
|
233
|
+
projected[projected.geometry.name] = projected.geometry.buffer(distance_meters)
|
|
234
|
+
out_gdf = projected.to_crs(orig_crs)
|
|
235
|
+
else:
|
|
236
|
+
out_gdf = gdf.copy()
|
|
237
|
+
out_gdf[out_gdf.geometry.name] = out_gdf.geometry.buffer(distance_meters / unit_metres)
|
|
238
|
+
elif op == "simplify":
|
|
239
|
+
out_gdf = gdf.copy()
|
|
240
|
+
out_gdf[out_gdf.geometry.name] = out_gdf.geometry.simplify(tolerance, preserve_topology=True)
|
|
241
|
+
elif op == "dissolve":
|
|
242
|
+
out_gdf = gdf.dissolve(by=dissolve_by, as_index=False)
|
|
243
|
+
elif op == "measure":
|
|
244
|
+
if orig_crs is None or orig_crs.is_geographic:
|
|
245
|
+
utm = gdf.estimate_utm_crs() if orig_crs else "EPSG:3857"
|
|
246
|
+
metric = gdf.to_crs(utm) if orig_crs else gdf
|
|
247
|
+
else:
|
|
248
|
+
metric = gdf
|
|
249
|
+
areas = metric.geometry.area * unit_metres ** 2
|
|
250
|
+
lengths = metric.geometry.length * unit_metres
|
|
251
|
+
return summarize_gdf(
|
|
252
|
+
gdf,
|
|
253
|
+
command="analyze measure",
|
|
254
|
+
extra={
|
|
255
|
+
"measure": {
|
|
256
|
+
"featureCount": int(len(gdf)),
|
|
257
|
+
"totalAreaSqMeters": round(float(areas.sum()), 2),
|
|
258
|
+
"totalAreaSqKm": round(float(areas.sum() / 1_000_000.0), 4),
|
|
259
|
+
"totalLengthMeters": round(float(lengths.sum()), 2),
|
|
260
|
+
"totalLengthKm": round(float(lengths.sum() / 1_000.0), 4),
|
|
261
|
+
}
|
|
262
|
+
},
|
|
263
|
+
)
|
|
264
|
+
elif op in ("clip", "intersect", "union", "difference", "spatial-join"):
|
|
265
|
+
if not second_input_path:
|
|
266
|
+
raise ValueError(f"Operation '{op}' requires second_input_path.")
|
|
267
|
+
other = read_vector_smart(second_input_path)
|
|
268
|
+
if gdf.crs and other.crs and gdf.crs != other.crs:
|
|
269
|
+
other = other.to_crs(gdf.crs)
|
|
270
|
+
if op == "clip":
|
|
271
|
+
out_gdf = gpd.clip(gdf, other)
|
|
272
|
+
elif op == "intersect":
|
|
273
|
+
out_gdf = gpd.overlay(gdf, other, how="intersection")
|
|
274
|
+
elif op == "union":
|
|
275
|
+
out_gdf = gpd.overlay(gdf, other, how="union")
|
|
276
|
+
elif op == "difference":
|
|
277
|
+
out_gdf = gpd.overlay(gdf, other, how="difference")
|
|
278
|
+
else:
|
|
279
|
+
out_gdf = gpd.sjoin(gdf, other, how="inner", predicate="intersects")
|
|
280
|
+
else:
|
|
281
|
+
raise ValueError(f"Unsupported Python vector analysis op: {op}")
|
|
282
|
+
|
|
283
|
+
written = write_vector_smart(out_gdf, output_path) if output_path else None
|
|
284
|
+
return summarize_gdf(out_gdf, command=f"analyze {op}", output_path=written, warnings=warnings)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: gisknife-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python heavy compute engine for gisknife (DuckDB Spatial + httpfs, GeoPandas, Pyogrio, PyProj, Shapely, optional Rasterio/PySAL).
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: <3.14,>=3.11
|
|
8
|
+
Requires-Dist: duckdb>=1.2.0
|
|
9
|
+
Requires-Dist: geopandas>=1.0.0
|
|
10
|
+
Requires-Dist: pyarrow>=18.0.0
|
|
11
|
+
Requires-Dist: pyogrio>=0.10.0
|
|
12
|
+
Requires-Dist: pyproj>=3.7.0
|
|
13
|
+
Requires-Dist: shapely>=2.0.0
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
16
|
+
Provides-Extra: raster
|
|
17
|
+
Requires-Dist: rasterio>=1.4.0; extra == 'raster'
|
|
18
|
+
Provides-Extra: stats
|
|
19
|
+
Requires-Dist: libpysal>=4.12.0; extra == 'stats'
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# gisknife-engine
|
|
23
|
+
|
|
24
|
+
Python engine for the `gisknife` CLI: DuckDB Spatial, GeoPandas, Pyogrio, PyProj and Shapely. Supports vector conversion, topology checks, spatial SQL and bounding-box extraction from Overture. Optional `rasterio` provides GeoTIFF metadata inspection, not raster conversion. The CLI uses a pinned engine version through `uv`; direct users can run `gisknife-engine --capabilities`. Requires Python 3.11–3.13; candidate testing used 3.12.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
gisknife_engine/__init__.py,sha256=6dvtbm4aJemYYYaL4hVsnZQHOK7ofd1VH4Bu8TEsBaU,67
|
|
2
|
+
gisknife_engine/cli.py,sha256=hSfKEsGmLTYKDmWiqX3fbqTrzBTxB_G6CLPkb5TxdEI,4339
|
|
3
|
+
gisknife_engine/duckdb_ops.py,sha256=3maKeXTed5iJOlkzHOKqJenAaYECQoBTkHpajnj9-qo,7662
|
|
4
|
+
gisknife_engine/raster_ops.py,sha256=WdLh-0QF9ZArb2HcPLeHayrnmLxpLmA4rN3kZAGTt08,1227
|
|
5
|
+
gisknife_engine/vector_ops.py,sha256=sBCfonVpEGvBeYvNIMmp2TQed3EIksU8GTe-oF36JSU,10280
|
|
6
|
+
gisknife_engine-0.1.0.dist-info/METADATA,sha256=JMerhKzu_sfhwp6BIc9Hmx_adQRgcUDcOM032bNy4N0,1165
|
|
7
|
+
gisknife_engine-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
8
|
+
gisknife_engine-0.1.0.dist-info/entry_points.txt,sha256=xgrGc0NWhXpxmt6r4To6uBTzeIs4QEq64KU6iF7PWgc,61
|
|
9
|
+
gisknife_engine-0.1.0.dist-info/licenses/LICENSE,sha256=FsQ4Qke7DFuIkxpLUZPGd9s-kOT4x6ir5XbRRW5XslM,1066
|
|
10
|
+
gisknife_engine-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 reed-soul
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|