gisknife-engine 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ # Dependencies
2
+ node_modules/
3
+ .pnpm-store/
4
+
5
+ # Build outputs
6
+ dist/
7
+ *.tsbuildinfo
8
+
9
+ # Python virtualenvs & caches
10
+ python/**/.venv/
11
+ python/**/__pycache__/
12
+ python/**/.pytest_cache/
13
+ *.pyc
14
+ *.egg-info/
15
+
16
+ # Runtime / test / user outputs
17
+ .cache/
18
+ coverage/
19
+ tmp/
20
+ *.log
21
+ .DS_Store
22
+ .env
23
+ .env.local
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 reed-soul
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,24 @@
1
+ Metadata-Version: 2.5
2
+ Name: gisknife-engine
3
+ Version: 0.1.0
4
+ Summary: Python heavy compute engine for gisknife (DuckDB Spatial + httpfs, GeoPandas, Pyogrio, PyProj, Shapely, optional Rasterio/PySAL).
5
+ License-Expression: MIT
6
+ License-File: LICENSE
7
+ Requires-Python: <3.14,>=3.11
8
+ Requires-Dist: duckdb>=1.2.0
9
+ Requires-Dist: geopandas>=1.0.0
10
+ Requires-Dist: pyarrow>=18.0.0
11
+ Requires-Dist: pyogrio>=0.10.0
12
+ Requires-Dist: pyproj>=3.7.0
13
+ Requires-Dist: shapely>=2.0.0
14
+ Provides-Extra: dev
15
+ Requires-Dist: pytest>=8.0.0; extra == 'dev'
16
+ Provides-Extra: raster
17
+ Requires-Dist: rasterio>=1.4.0; extra == 'raster'
18
+ Provides-Extra: stats
19
+ Requires-Dist: libpysal>=4.12.0; extra == 'stats'
20
+ Description-Content-Type: text/markdown
21
+
22
+ # gisknife-engine
23
+
24
+ Python engine for the `gisknife` CLI: DuckDB Spatial, GeoPandas, Pyogrio, PyProj and Shapely. Supports vector conversion, topology checks, spatial SQL and bounding-box extraction from Overture. Optional `rasterio` provides GeoTIFF metadata inspection, not raster conversion. The CLI uses a pinned engine version through `uv`; direct users can run `gisknife-engine --capabilities`. Requires Python 3.11–3.13; candidate testing used 3.12.
@@ -0,0 +1,3 @@
1
+ # gisknife-engine
2
+
3
+ Python engine for the `gisknife` CLI: DuckDB Spatial, GeoPandas, Pyogrio, PyProj and Shapely. Supports vector conversion, topology checks, spatial SQL and bounding-box extraction from Overture. Optional `rasterio` provides GeoTIFF metadata inspection, not raster conversion. The CLI uses a pinned engine version through `uv`; direct users can run `gisknife-engine --capabilities`. Requires Python 3.11–3.13; candidate testing used 3.12.
@@ -0,0 +1,31 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "gisknife-engine"
7
+ version = "0.1.0"
8
+ description = "Python heavy compute engine for gisknife (DuckDB Spatial + httpfs, GeoPandas, Pyogrio, PyProj, Shapely, optional Rasterio/PySAL)."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.11,<3.14"
13
+ dependencies = [
14
+ "duckdb>=1.2.0",
15
+ "geopandas>=1.0.0",
16
+ "pyarrow>=18.0.0",
17
+ "pyogrio>=0.10.0",
18
+ "pyproj>=3.7.0",
19
+ "shapely>=2.0.0",
20
+ ]
21
+
22
+ [project.optional-dependencies]
23
+ raster = ["rasterio>=1.4.0"]
24
+ stats = ["libpysal>=4.12.0"]
25
+ dev = ["pytest>=8.0.0"]
26
+
27
+ [project.scripts]
28
+ gisknife-engine = "gisknife_engine.cli:main"
29
+
30
+ [tool.hatch.build.targets.wheel]
31
+ packages = ["src/gisknife_engine"]
@@ -0,0 +1,3 @@
1
+ """gisknife Python heavy compute engine."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,125 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import importlib.util
5
+ import json
6
+ from pathlib import Path
7
+ import sys
8
+ import traceback
9
+ from typing import Any
10
+
11
+ from .duckdb_ops import fetch_overture_bbox, run_spatial_sql
12
+ from .raster_ops import has_rasterio, inspect_raster
13
+ from .vector_ops import analyze_vector, convert_vector, qa_vector, reproject_vector
14
+
15
+
16
+ def get_capabilities() -> dict[str, Any]:
17
+ return {
18
+ "ok": True,
19
+ "engine": "python-uvx",
20
+ "pythonVersion": sys.version.split()[0],
21
+ "capabilities": {
22
+ "duckdb": importlib.util.find_spec("duckdb") is not None,
23
+ "geopandas": importlib.util.find_spec("geopandas") is not None,
24
+ "pyogrio": importlib.util.find_spec("pyogrio") is not None,
25
+ "pyproj": importlib.util.find_spec("pyproj") is not None,
26
+ "shapely": importlib.util.find_spec("shapely") is not None,
27
+ "rasterio": has_rasterio(),
28
+ "libpysal": importlib.util.find_spec("libpysal") is not None,
29
+ },
30
+ }
31
+
32
+
33
+ def dispatch_rpc(req: dict[str, Any]) -> dict[str, Any]:
34
+ op = req.get("op")
35
+ params = req.get("params") or {}
36
+ if params.get('outputPath'):
37
+ target = Path(params['outputPath']).expanduser()
38
+ for key in ('inputPath', 'secondInputPath'):
39
+ if params.get(key) and target.exists() and target.samefile(Path(params[key]).expanduser()):
40
+ raise ValueError('Output would overwrite an input file')
41
+
42
+ if op == "capabilities":
43
+ return get_capabilities()
44
+ if op == "convert":
45
+ return convert_vector(
46
+ input_path=params["inputPath"],
47
+ output_path=params["outputPath"],
48
+ target_crs=params.get("targetCrs"),
49
+ )
50
+ if op == "reproject":
51
+ return reproject_vector(
52
+ input_path=params["inputPath"],
53
+ output_path=params["outputPath"],
54
+ to_crs=params.get("toCrs", "EPSG:4326"),
55
+ from_crs=params.get("fromCrs"),
56
+ )
57
+ if op == "qa":
58
+ return qa_vector(
59
+ input_path=params["inputPath"],
60
+ output_path=params.get("outputPath"),
61
+ fix=bool(params.get("fix", False)),
62
+ )
63
+ if op == "analyze":
64
+ return analyze_vector(
65
+ op=params["analysisOp"],
66
+ input_path=params["inputPath"],
67
+ output_path=params.get("outputPath"),
68
+ second_input_path=params.get("secondInputPath"),
69
+ distance_meters=float(params.get("distanceMeters", 500.0)),
70
+ tolerance=float(params.get("tolerance", 0.0005)),
71
+ dissolve_by=params.get("dissolveBy"),
72
+ )
73
+ if op == "sql":
74
+ return run_spatial_sql(
75
+ sql=params["sql"],
76
+ output_path=params.get("outputPath"),
77
+ result_crs=params.get('resultCrs'),
78
+ )
79
+ if op == "overture_fetch":
80
+ return fetch_overture_bbox(
81
+ bbox=params["bbox"],
82
+ layers=params.get("layers", ["roads", "buildings"]),
83
+ output_path=params["outputPath"],
84
+ release=params.get('release'),
85
+ )
86
+ if op == "raster_inspect":
87
+ return inspect_raster(input_path=params["inputPath"])
88
+
89
+ raise ValueError(f"Unknown gisknife-engine RPC op: {op}")
90
+
91
+
92
+ def main(argv: list[str] | None = None) -> int:
93
+ parser = argparse.ArgumentParser(prog="gisknife-engine")
94
+ parser.add_argument("--capabilities", action="store_true", help="Print engine capabilities JSON")
95
+ parser.add_argument("--rpc", action="store_true", help="Read JSON RPC request from stdin")
96
+ args = parser.parse_args(argv)
97
+
98
+ if args.capabilities:
99
+ print(json.dumps(get_capabilities()))
100
+ return 0
101
+
102
+ if args.rpc:
103
+ try:
104
+ raw = sys.stdin.read()
105
+ req = json.loads(raw)
106
+ res = dispatch_rpc(req)
107
+ print(json.dumps(res, ensure_ascii=False))
108
+ return 0
109
+ except Exception as exc:
110
+ err_payload = {
111
+ "ok": False,
112
+ "status": "error",
113
+ "engine": "python-uvx",
114
+ "error": str(exc),
115
+ "traceback": traceback.format_exc(),
116
+ }
117
+ print(json.dumps(err_payload, ensure_ascii=False))
118
+ return 1
119
+
120
+ parser.print_help()
121
+ return 0
122
+
123
+
124
+ if __name__ == "__main__":
125
+ raise SystemExit(main())
@@ -0,0 +1,191 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import re
5
+ import tempfile
6
+ import urllib.request
7
+ import xml.etree.ElementTree as ET
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ import duckdb
12
+ import geopandas as gpd
13
+ from shapely import wkb as shapely_wkb
14
+
15
+ from .vector_ops import summarize_gdf, write_vector_smart
16
+
17
+ def current_overture_release() -> str:
18
+ url = 'https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/?list-type=2&delimiter=%2F&prefix=release%2F'
19
+ with urllib.request.urlopen(url, timeout=15) as response:
20
+ root = ET.fromstring(response.read(256_000))
21
+ versions = [prefix.text.split('/')[1] for prefix in root.findall('.//{*}CommonPrefixes/{*}Prefix')
22
+ if prefix.text and re.fullmatch(r'release/\d{4}-\d{2}-\d{2}\.\d+/', prefix.text)]
23
+ if not versions:
24
+ raise ValueError('Overture has no identifiable release')
25
+ return max(versions, key=lambda version: (version[:10], int(version.split('.')[1])))
26
+
27
+ OVERTURE_LAYER_MAP = {
28
+ "roads": ("transportation", "segment"),
29
+ "buildings": ("buildings", "building"),
30
+ "pois": ("places", "place"),
31
+ "water": ("base", "water"),
32
+ "landuse": ("base", "land_use"),
33
+ }
34
+
35
+
36
+ def _open_spatial_duckdb(load_httpfs: bool = False) -> duckdb.DuckDBPyConnection:
37
+ con = duckdb.connect(database=":memory:")
38
+ con.execute("INSTALL spatial; LOAD spatial;")
39
+ try:
40
+ con.execute("SET geometry_always_xy = true;")
41
+ except Exception:
42
+ pass
43
+ if load_httpfs:
44
+ con.execute("INSTALL httpfs; LOAD httpfs;")
45
+ con.execute("SET s3_region='us-west-2';")
46
+ return con
47
+
48
+
49
+ def run_spatial_sql(sql: str, output_path: str | None = None, result_crs: str | None = None) -> dict[str, Any]:
50
+ needs_http = any(p in sql.lower() for p in ("http://", "https://", "s3://"))
51
+ con = _open_spatial_duckdb(load_httpfs=needs_http)
52
+ try:
53
+ rel = con.sql(sql)
54
+ col_names = rel.columns
55
+ col_types = [str(t).upper() for t in rel.types]
56
+
57
+ geom_cols = [
58
+ name for name, typ in zip(col_names, col_types) if "GEOMETRY" in typ or "WKB_BLOB" in typ
59
+ ]
60
+
61
+ if geom_cols:
62
+ geom_col = geom_cols[0]
63
+ other_cols = [c for c in col_names if c != geom_col]
64
+ select_parts = [f'"{c}"' for c in other_cols] + [f'ST_AsWKB("{geom_col}") AS __gisknife_wkb']
65
+ wrapped = f"SELECT {', '.join(select_parts)} FROM ({sql.rstrip(';')}) AS __sub"
66
+ rows = con.execute(wrapped).fetchall()
67
+ out_cols = other_cols
68
+ records: list[dict[str, Any]] = []
69
+ geoms = []
70
+ for row in rows:
71
+ rec = {col: val for col, val in zip(out_cols, row[:-1])}
72
+ raw_wkb = row[-1]
73
+ geoms.append(shapely_wkb.loads(bytes(raw_wkb)) if raw_wkb is not None else None)
74
+ records.append(rec)
75
+
76
+ declared_type = str(rel.types[col_names.index(geom_col)])
77
+ tagged = re.fullmatch(r"GEOMETRY\('([^']+)'\)", declared_type, flags=re.I)
78
+ crs = tagged.group(1) if tagged else result_crs
79
+ if not crs and output_path:
80
+ raise ValueError('SQL geometry has no CRS. Supply --result-crs to label its existing coordinates.')
81
+ gdf = gpd.GeoDataFrame(records, geometry=geoms, crs=crs)
82
+ written = write_vector_smart(gdf, output_path) if output_path else None
83
+ return summarize_gdf(
84
+ gdf,
85
+ command="sql",
86
+ output_path=written,
87
+ extra={"rowCount": len(gdf)},
88
+ )
89
+
90
+ rows = rel.fetchall()
91
+ dict_rows = [{col: val for col, val in zip(col_names, row)} for row in rows]
92
+ written: str | None = None
93
+ if output_path:
94
+ out_p = Path(output_path).expanduser().resolve()
95
+ out_p.parent.mkdir(parents=True, exist_ok=True)
96
+ with out_p.open("w", encoding="utf-8", newline="") as fp:
97
+ writer = csv.DictWriter(fp, fieldnames=col_names)
98
+ writer.writeheader()
99
+ writer.writerows(dict_rows)
100
+ written = str(out_p)
101
+
102
+ return {
103
+ "ok": True,
104
+ "status": "completed",
105
+ "command": "sql",
106
+ "engine": "python-uvx",
107
+ "outputPath": written,
108
+ "rowCount": len(dict_rows),
109
+ "columns": col_names,
110
+ "sampleRows": dict_rows[:20],
111
+ "warnings": [],
112
+ }
113
+ finally:
114
+ con.close()
115
+
116
+
117
+ def fetch_overture_bbox(
118
+ bbox: list[float],
119
+ layers: list[str],
120
+ output_path: str,
121
+ release: str | None = None,
122
+ limit_per_layer: int = 50000,
123
+ ) -> dict[str, Any]:
124
+ if len(bbox) != 4:
125
+ raise ValueError("bbox must be [minLon, minLat, maxLon, maxLat]")
126
+ min_lon, min_lat, max_lon, max_lat = bbox
127
+ if not (-180 <= min_lon < max_lon <= 180 and -90 <= min_lat < max_lat <= 90):
128
+ raise ValueError('Invalid WGS84 bounding box')
129
+ release = release or current_overture_release()
130
+ if not re.fullmatch(r'\d{4}-\d{2}-\d{2}\.\d+', release):
131
+ raise ValueError('Invalid Overture release identifier')
132
+ target_layers = (
133
+ list(OVERTURE_LAYER_MAP) if not layers or "all" in layers else layers
134
+ )
135
+
136
+ con = _open_spatial_duckdb(load_httpfs=True)
137
+ frames: list[gpd.GeoDataFrame] = []
138
+ try:
139
+ for layer in target_layers:
140
+ mapping = OVERTURE_LAYER_MAP.get(layer)
141
+ if not mapping:
142
+ raise ValueError(f'Unknown Overture layer: {layer}')
143
+ theme, o_type = mapping
144
+ s3_url = f"s3://overturemaps-us-west-2/release/{release}/theme={theme}/type={o_type}/*"
145
+ sql = f"""
146
+ SELECT
147
+ id,
148
+ '{layer}' AS gisknife_layer,
149
+ COALESCE(names.primary, '') AS name,
150
+ ST_AsWKB(geometry) AS __gisknife_wkb
151
+ FROM read_parquet('{s3_url}', filename=true, hive_partitioning=1)
152
+ WHERE bbox.xmin <= {max_lon}
153
+ AND bbox.xmax >= {min_lon}
154
+ AND bbox.ymin <= {max_lat}
155
+ AND bbox.ymax >= {min_lat}
156
+ LIMIT {int(limit_per_layer) + 1}
157
+ """
158
+ rows = con.execute(sql).fetchall()
159
+ if len(rows) > limit_per_layer:
160
+ raise ValueError(f'{layer} exceeds {limit_per_layer} features. Request a smaller bounding box.')
161
+ if not rows:
162
+ continue
163
+ records = [{"id": r[0], "layer": r[1], "name": r[2]} for r in rows]
164
+ geoms = [shapely_wkb.loads(bytes(r[3])) for r in rows]
165
+ frames.append(gpd.GeoDataFrame(records, geometry=geoms, crs="EPSG:4326"))
166
+ finally:
167
+ con.close()
168
+
169
+ combined = (
170
+ gpd.GeoDataFrame(
171
+ gpd.pd.concat(frames, ignore_index=True), # type: ignore[attr-defined]
172
+ crs="EPSG:4326",
173
+ )
174
+ if frames
175
+ else gpd.GeoDataFrame({"id": [], "layer": [], "name": []}, geometry=[], crs="EPSG:4326")
176
+ )
177
+ target = Path(output_path).expanduser().resolve()
178
+ if target.suffix.lower() not in ('.parquet', '.geoparquet', '.json', '.geojson'):
179
+ raise ValueError('Overture output must be GeoJSON or GeoParquet')
180
+ target.parent.mkdir(parents=True, exist_ok=True)
181
+ with tempfile.TemporaryDirectory(dir=target.parent) as directory:
182
+ staged = Path(directory) / target.name
183
+ write_vector_smart(combined, str(staged))
184
+ staged.replace(target)
185
+ written = str(target)
186
+ return summarize_gdf(
187
+ combined,
188
+ command="fetch overture",
189
+ output_path=written,
190
+ extra={"source": f"overture-maps-s3:{release}"},
191
+ )
@@ -0,0 +1,41 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+
6
+ def has_rasterio() -> bool:
7
+ try:
8
+ import rasterio # noqa: F401
9
+
10
+ return True
11
+ except ImportError:
12
+ return False
13
+
14
+
15
+ def inspect_raster(input_path: str) -> dict[str, Any]:
16
+ try:
17
+ import rasterio
18
+ except ImportError as exc:
19
+ raise RuntimeError(
20
+ "GeoTIFF raster support requires the optional raster extra: "
21
+ "uv tool install 'gisknife-engine[raster]' or uv sync --extra raster"
22
+ ) from exc
23
+
24
+ with rasterio.open(input_path) as src:
25
+ bounds = [float(src.bounds.left), float(src.bounds.bottom), float(src.bounds.right), float(src.bounds.top)]
26
+ return {
27
+ "ok": True,
28
+ "status": "completed",
29
+ "command": "raster inspect",
30
+ "engine": "python-uvx",
31
+ "outputPath": input_path,
32
+ "crs": str(src.crs) if src.crs else "unknown",
33
+ "width": int(src.width),
34
+ "height": int(src.height),
35
+ "bandCount": int(src.count),
36
+ "dtypes": list(src.dtypes),
37
+ "nodata": src.nodata,
38
+ "bbox": bounds,
39
+ "driver": src.driver,
40
+ "warnings": [],
41
+ }