zero2geospatial3d-sdk 0.1.0__py3-none-any.whl

zero2geospatial3d/osm.py

@@ -0,0 +1,450 @@
+"""OpenStreetMap buildings, trees and roads from the Overpass API.
+
+Headless port of the plugin's OSM download: the same Overpass query, the
+same mirror fail-over with one retry on transient errors, a one-week on-disk
+cache, and the same Simple 3D Buildings interpretation (``building:part``
+replaces its outline; parts inherit name and type and share ``bldg_group``).
+Everything is returned as GeoJSON-like dicts in WGS84 longitude/latitude; no
+QGIS, GDAL or Shapely is needed.
+
+Every network call has a timeout.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import math
+import os
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Callable, Optional
+
+from . import osm_tags
+
+__all__ = [
+    "OsmError",
+    "ENDPOINTS",
+    "build_query",
+    "fetch",
+    "payload_to_features",
+    "ring_centroid",
+    "point_in_ring",
+]
+
+ENDPOINTS = (
+    "https://overpass-api.de/api/interpreter",
+    "https://overpass.kumi.systems/api/interpreter",
+    "https://overpass.private.coffee/api/interpreter",
+)
+TIMEOUT_S = 90
+MAX_RESPONSE_BYTES = 192 * 1024 * 1024
+CACHE_TTL_S = 7 * 24 * 3600
+MAX_AREA_KM2 = 36.0
+
+Ring = list  # list of (lon, lat)
+
+
+class OsmError(RuntimeError):
+    """Raised when every Overpass mirror failed or the area is too large."""
+
+
+def _user_agent() -> str:
+    from . import __version__
+
+    return f"zero2geospatial3d-sdk/{__version__} (Python; +https://github.com/YusufEminoglu/zero2geospatial3d_sdk)"
+
+
+def build_query(s: float, w: float, n: float, e: float) -> str:
+    """Overpass QL for buildings, building parts, trees and traffic roads in a bbox."""
+    bbox = f"{s:.7f},{w:.7f},{n:.7f},{e:.7f}"
+    return (
+        f"[out:json][timeout:{TIMEOUT_S}];\n(\n"
+        f'  way["building"]({bbox});\n'
+        f'  relation["building"]["type"="multipolygon"]({bbox});\n'
+        f'  way["building:part"]({bbox});\n'
+        f'  relation["building:part"]["type"="multipolygon"]({bbox});\n'
+        f'  node["natural"="tree"]({bbox});\n'
+        f'  way["highway"~"{osm_tags.HIGHWAY_PATTERN}"]({bbox});\n'
+        f");\nout body geom;"
+    )
+
+
+def _cache_path(query: str, cache_dir: Optional[str]) -> str:
+    folder = cache_dir or os.path.join(tempfile.gettempdir(), "zero2geospatial3d_osm_cache")
+    with contextlib.suppress(OSError):
+        os.makedirs(folder, exist_ok=True)
+    digest = hashlib.sha256(query.encode("utf-8")).hexdigest()[:40]
+    return os.path.join(folder, f"{digest}.json")
+
+
+def _read_cache(query: str, cache_dir: Optional[str]) -> Optional[dict[str, Any]]:
+    path = _cache_path(query, cache_dir)
+    if not os.path.isfile(path) or time.time() - os.path.getmtime(path) > CACHE_TTL_S:
+        return None
+    with contextlib.suppress(OSError, ValueError):
+        with open(path, encoding="utf-8") as handle:
+            return json.load(handle)
+    return None
+
+
+def _write_cache(query: str, payload: dict[str, Any], cache_dir: Optional[str]) -> None:
+    with contextlib.suppress(OSError, TypeError, ValueError):
+        with open(_cache_path(query, cache_dir), "w", encoding="utf-8") as handle:
+            json.dump(payload, handle)
+
+
+def _post(endpoint: str, query: str, timeout: float) -> tuple[Optional[dict[str, Any]], str, bool]:
+    """(payload, error_text, transient)"""
+    body = urllib.parse.urlencode({"data": query}).encode("ascii")
+    request = urllib.request.Request(
+        endpoint,
+        data=body,
+        headers={
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+            "User-Agent": _user_agent(),
+        },
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as reply:  # noqa: S310
+            content = reply.read(MAX_RESPONSE_BYTES + 1)
+    except urllib.error.HTTPError as exc:
+        return None, f"HTTP {exc.code}", exc.code == 429 or exc.code >= 500
+    except (OSError, ValueError) as exc:
+        return None, str(exc) or "network error", True
+    if len(content) > MAX_RESPONSE_BYTES:
+        return None, "response too large; pick a smaller area", False
+    try:
+        payload = json.loads(content.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError) as exc:
+        return None, f"not JSON ({exc})", False
+    if not isinstance(payload, dict) or not isinstance(payload.get("elements"), list):
+        return None, "unexpected response", False
+    if payload.get("remark"):
+        return None, f"incomplete response: {payload['remark']}", True
+    return payload, "", False
+
+
+def _area_km2(s: float, w: float, n: float, e: float) -> float:
+    mid = math.radians((s + n) / 2)
+    return abs(n - s) * 111.32 * abs(e - w) * 111.32 * math.cos(mid)
+
+
+def fetch(
+    s: float,
+    w: float,
+    n: float,
+    e: float,
+    *,
+    progress: Optional[Callable[[str], None]] = None,
+    timeout: float = TIMEOUT_S + 15,
+    use_cache: bool = True,
+    cache_dir: Optional[str] = None,
+    endpoints: tuple[str, ...] = ENDPOINTS,
+) -> dict[str, Any]:
+    """Raw Overpass JSON for the bbox (south, west, north, east) in degrees."""
+    if not (s < n and w < e):
+        raise OsmError(
+            "The bbox must be (south, west, north, east) with south < north, west < east."
+        )
+    if _area_km2(s, w, n, e) > MAX_AREA_KM2:
+        raise OsmError(f"The area is larger than {MAX_AREA_KM2:.0f} km2; pick a smaller one.")
+    report = progress or (lambda msg: None)
+    query = build_query(s, w, n, e)
+    if use_cache:
+        cached = _read_cache(query, cache_dir)
+        if cached is not None:
+            report("Using the cached OSM response.")
+            return cached
+    failures = []
+    for endpoint in endpoints:
+        host = urllib.parse.urlparse(endpoint).hostname or endpoint
+        for attempt in range(2):
+            report(f"Querying {host} ..." if not attempt else f"Retrying {host} ...")
+            if attempt:
+                time.sleep(1.5)
+            payload, error, transient = _post(endpoint, query, timeout)
+            if payload is not None:
+                if use_cache:
+                    _write_cache(query, payload, cache_dir)
+                return payload
+            failures.append(f"{host}: {error}")
+            if not transient:
+                break
+    raise OsmError(
+        "Every Overpass server failed (" + "; ".join(failures[-3:]) + "). "
+        "Wait a minute and try again, or pick a smaller area."
+    )
+
+
+# ---------------------------------------------------------------- geometry
+def _points(geometry: Any) -> Ring:
+    out: Ring = []
+    for item in geometry or []:
+        try:
+            lon, lat = float(item["lon"]), float(item["lat"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        if math.isfinite(lon) and math.isfinite(lat) and (not out or out[-1] != (lon, lat)):
+            out.append((lon, lat))
+    return out
+
+
+def _closed(points: Ring) -> Optional[Ring]:
+    if len(points) < 3:
+        return None
+    if points[0] != points[-1]:
+        points = points + [points[0]]
+    return points if len(points) >= 4 else None
+
+
+def _stitch(paths: list[Ring]) -> list[Ring]:
+    def key(p: tuple[float, float]) -> tuple[float, float]:
+        return (round(p[0], 9), round(p[1], 9))
+
+    pending = [list(p) for p in paths if len(p) >= 2]
+    rings = []
+    while pending:
+        chain = pending.pop(0)
+        progress = True
+        while key(chain[0]) != key(chain[-1]) and progress:
+            progress = False
+            for index, path in enumerate(pending):
+                if key(chain[-1]) == key(path[0]):
+                    chain = chain + path[1:]
+                elif key(chain[-1]) == key(path[-1]):
+                    chain = chain + list(reversed(path[:-1]))
+                elif key(chain[0]) == key(path[-1]):
+                    chain = path[:-1] + chain
+                elif key(chain[0]) == key(path[0]):
+                    chain = list(reversed(path[1:])) + chain
+                else:
+                    continue
+                pending.pop(index)
+                progress = True
+                break
+        if key(chain[0]) == key(chain[-1]) and len(chain) >= 4:
+            rings.append(chain)
+    return rings
+
+
+def point_in_ring(x: float, y: float, ring: Ring) -> bool:
+    """Even-odd point-in-polygon test."""
+    inside = False
+    n = len(ring)
+    j = n - 1
+    for i in range(n):
+        xi, yi = ring[i][0], ring[i][1]
+        xj, yj = ring[j][0], ring[j][1]
+        if (yi > y) != (yj > y) and x < (xj - xi) * (y - yi) / ((yj - yi) or 1e-300) + xi:
+            inside = not inside
+        j = i
+    return inside
+
+
+def ring_centroid(ring: Ring) -> tuple[float, float]:
+    """Area centroid of a ring; a vertex mean for degenerate rings."""
+    a = cx = cy = 0.0
+    n = len(ring)
+    for i in range(n):
+        x1, y1 = ring[i][0], ring[i][1]
+        x2, y2 = ring[(i + 1) % n][0], ring[(i + 1) % n][1]
+        cross = x1 * y2 - x2 * y1
+        a += cross
+        cx += (x1 + x2) * cross
+        cy += (y1 + y2) * cross
+    if abs(a) < 1e-18:
+        return (sum(p[0] for p in ring) / n, sum(p[1] for p in ring) / n)
+    return (cx / (3 * a), cy / (3 * a))
+
+
+def _interior_point(polygons: list[list[Ring]]) -> tuple[float, float]:
+    outer = max((p[0] for p in polygons), key=lambda r: abs(_signed_area(r)))
+    c = ring_centroid(outer)
+    if point_in_ring(c[0], c[1], outer):
+        return c
+    # Concave outline: probe midpoints between the centroid and each vertex.
+    for v in outer:
+        m = ((c[0] + v[0]) / 2, (c[1] + v[1]) / 2)
+        if point_in_ring(m[0], m[1], outer):
+            return m
+    return outer[0][0], outer[0][1]
+
+
+def _signed_area(ring: Ring) -> float:
+    return (
+        sum(
+            ring[i][0] * ring[(i + 1) % len(ring)][1] - ring[(i + 1) % len(ring)][0] * ring[i][1]
+            for i in range(len(ring))
+        )
+        / 2
+    )
+
+
+def _relation_polygons(element: dict[str, Any]) -> Optional[list[list[Ring]]]:
+    outers: list[Ring] = []
+    inners: list[Ring] = []
+    for member in element.get("members") or []:
+        if member.get("type") != "way":
+            continue
+        role = (member.get("role") or "").lower()
+        (inners if role == "inner" else outers).append(_points(member.get("geometry")))
+    polys: list[list[Ring]] = [[r] for r in _stitch(outers)]
+    if not polys:
+        return None
+    for hole in _stitch(inners):
+        hx, hy = ring_centroid(hole)
+        for poly in polys:
+            if point_in_ring(hx, hy, poly[0]):
+                poly.append(hole)
+                break
+    return polys
+
+
+def _geometry(polys: list[list[Ring]]) -> dict[str, Any]:
+    coords = [[[list(p) for p in ring] for ring in poly] for poly in polys]
+    if len(coords) == 1:
+        return {"type": "Polygon", "coordinates": coords[0]}
+    return {"type": "MultiPolygon", "coordinates": coords}
+
+
+def _in_bbox(x: float, y: float, bbox: Optional[tuple[float, float, float, float]]) -> bool:
+    if bbox is None:
+        return True
+    s, w, n, e = bbox
+    return s <= y <= n and w <= x <= e
+
+
+def payload_to_features(
+    payload: dict[str, Any], bbox: Optional[tuple[float, float, float, float]] = None
+) -> dict[str, list[dict[str, Any]]]:
+    """Overpass JSON -> ``{"buildings": [...], "trees": [...], "roads": [...]}``.
+
+    Each item is a GeoJSON Feature (WGS84). Building properties: ``osm_id``,
+    ``building``, ``name``, ``height_m`` (top above ground), ``min_height_m``,
+    ``levels``, ``height_src`` (``height``/``levels``/``default``), ``is_part``
+    and ``bldg_group``. Trees carry ``height_m``; roads carry ``highway``,
+    ``name`` and ``traffic_weight``. With ``bbox`` (south, west, north, east)
+    buildings and trees are kept when their interior point lies inside it, and
+    roads when any vertex does.
+    """
+    outlines: list[dict[str, Any]] = []
+    parts: list[dict[str, Any]] = []
+    trees: list[dict[str, Any]] = []
+    roads: list[dict[str, Any]] = []
+    for element in payload.get("elements", []):
+        tags = element.get("tags") or {}
+        etype = element.get("type")
+        if etype == "node" and tags.get("natural") == "tree":
+            try:
+                lon, lat = float(element["lon"]), float(element["lat"])
+            except (KeyError, TypeError, ValueError):
+                continue
+            if _in_bbox(lon, lat, bbox):
+                trees.append(
+                    {
+                        "type": "Feature",
+                        "geometry": {"type": "Point", "coordinates": [lon, lat]},
+                        "properties": {
+                            "osm_id": f"n{element.get('id')}",
+                            "height_m": osm_tags.tree_height(tags),
+                        },
+                    }
+                )
+            continue
+        if etype == "way":
+            weight = osm_tags.road_traffic(tags)
+            if weight is not None:
+                pts = _points(element.get("geometry"))
+                if len(pts) >= 2 and any(_in_bbox(x, y, bbox) for x, y in pts):
+                    roads.append(
+                        {
+                            "type": "Feature",
+                            "geometry": {
+                                "type": "LineString",
+                                "coordinates": [list(p) for p in pts],
+                            },
+                            "properties": {
+                                "osm_id": f"w{element.get('id')}",
+                                "highway": str(tags.get("highway"))[:40],
+                                "name": str(tags.get("name") or "")[:120],
+                                "traffic_weight": weight,
+                            },
+                        }
+                    )
+                continue
+        is_part = tags.get("building:part") not in (None, "", "no")
+        is_building = tags.get("building") not in (None, "", "no", "construction")
+        if not (is_part or is_building):
+            continue
+        if etype == "relation":
+            polys = _relation_polygons(element)
+        else:
+            ring = _closed(_points(element.get("geometry")))
+            polys = [[ring]] if ring else None
+        if not polys:
+            continue
+        record = {
+            "id": f"{str(etype)[0]}{element.get('id')}",
+            "polys": polys,
+            "tags": tags,
+            "centre": _interior_point(polys),
+        }
+        (parts if is_part else outlines).append(record)
+
+    replaced = set()
+    for part in parts:
+        part["outline"] = None
+        cx, cy = part["centre"]
+        for k, o in enumerate(outlines):
+            if any(point_in_ring(cx, cy, poly[0]) for poly in o["polys"]):
+                part["outline"] = o
+                replaced.add(k)
+                break
+
+    buildings: list[dict[str, Any]] = []
+
+    def add(record: dict[str, Any], tags: dict[str, Any], group: str, part: bool) -> None:
+        if not _in_bbox(record["centre"][0], record["centre"][1], bbox):
+            return
+        top, levels, source = osm_tags.building_height(tags)
+        bottom = osm_tags.min_height(tags)
+        if bottom >= top:
+            bottom = 0.0
+        buildings.append(
+            {
+                "type": "Feature",
+                "geometry": _geometry(record["polys"]),
+                "properties": {
+                    "osm_id": record["id"],
+                    "building": str(tags.get("building") or tags.get("building:part") or "yes")[
+                        :40
+                    ],
+                    "name": str(tags.get("name") or "")[:120],
+                    "height_m": top,
+                    "min_height_m": bottom,
+                    "levels": levels,
+                    "height_src": source,
+                    "is_part": 1 if part else 0,
+                    "bldg_group": group,
+                },
+            }
+        )
+
+    for k, o in enumerate(outlines):
+        if k not in replaced:
+            add(o, o["tags"], o["id"], False)
+    for part in parts:
+        outline = part["outline"]
+        tags = dict(part["tags"])
+        if outline is not None:
+            for key in ("name", "building"):
+                if not tags.get(key) and outline["tags"].get(key):
+                    tags[key] = outline["tags"][key]
+        add(part, tags, outline["id"] if outline is not None else part["id"], True)
+    return {"buildings": buildings, "trees": trees, "roads": roads}

zero2geospatial3d/osm_tags.py

@@ -0,0 +1,180 @@
+"""OSM tag interpretation (pure Python, no QGIS)."""
+
+from __future__ import annotations
+
+import re
+
+FLOOR_HEIGHT_M = 3.0
+
+# Floor counts by OSM ``building`` type, used only when OSM has neither
+# building:levels nor height. Unknown types fall back to 3.
+DEFAULT_FLOORS = {
+    "apartments": 4,
+    "residential": 4,
+    "dormitory": 5,
+    "house": 2,
+    "detached": 2,
+    "terrace": 2,
+    "semidetached_house": 2,
+    "bungalow": 1,
+    "commercial": 3,
+    "office": 5,
+    "retail": 2,
+    "supermarket": 1,
+    "kiosk": 1,
+    "industrial": 1,
+    "warehouse": 1,
+    "manufacture": 1,
+    "hangar": 1,
+    "school": 3,
+    "university": 4,
+    "college": 4,
+    "kindergarten": 2,
+    "hospital": 5,
+    "clinic": 3,
+    "church": 1,
+    "mosque": 1,
+    "temple": 1,
+    "synagogue": 1,
+    "cathedral": 2,
+    "chapel": 1,
+    "public": 3,
+    "civic": 3,
+    "government": 4,
+    "townhall": 3,
+    "garage": 1,
+    "garages": 1,
+    "shed": 1,
+    "hut": 1,
+    "roof": 1,
+    "carport": 1,
+}
+# Single-storey types read taller than 3 m per floor.
+TALL_SINGLE_STOREY = {
+    "industrial": 7.0,
+    "warehouse": 7.0,
+    "hangar": 9.0,
+    "manufacture": 6.0,
+    "supermarket": 5.0,
+    "church": 10.0,
+    "mosque": 10.0,
+    "cathedral": 18.0,
+    "temple": 8.0,
+    "synagogue": 8.0,
+    "chapel": 6.0,
+}
+
+_NUMBER = r"([-+]?(?:\d+(?:[.,]\d*)?|[.,]\d+))"
+
+
+def parse_number(value):
+    """Parse an OSM numeric tag ('12', '12.5 m', "40'", '3;4') -> metres/float or None."""
+    if value is None:
+        return None
+    text = str(value).split(";", 1)[0].strip().lstrip("~").strip()
+    feet = re.fullmatch(
+        _NUMBER + r"\s*(?:ft|feet|foot|['′])(?:\s*" + _NUMBER + r"\s*(?:in|inches?|[\"″]))?",
+        text,
+        re.IGNORECASE,
+    )
+    if feet:
+        return (
+            float(feet.group(1).replace(",", ".")) * 0.3048
+            + float((feet.group(2) or "0").replace(",", ".")) * 0.0254
+        )
+    match = re.fullmatch(
+        _NUMBER + r"\s*(m|met(?:er|re)s?|cm|km|ft|feet|foot|in|inches?)?", text, re.IGNORECASE
+    )
+    if not match:
+        return None
+    number = float(match.group(1).replace(",", "."))
+    unit = (match.group(2) or "").lower()
+    if unit == "cm":
+        return number / 100.0
+    if unit == "km":
+        return number * 1000.0
+    if unit in ("ft", "feet", "foot"):
+        return number * 0.3048
+    if unit.startswith("in"):
+        return number * 0.0254
+    return number
+
+
+def building_height(tags: dict) -> tuple[float, int, str]:
+    """(top_m, levels, source) for an OSM building or building:part.
+
+    top_m is the height of the top above the ground, as the OSM ``height``
+    key defines it (Simple 3D Buildings); the bottom of a raised part is
+    min_height(). source is 'height', 'levels' or 'default' so a user can tell
+    measured heights from guesses.
+    """
+    kind = (tags.get("building") or tags.get("building:part") or "").strip().lower()
+    levels = None
+    for key in ("building:levels", "levels"):
+        n = parse_number(tags.get(key))
+        if n is not None and n > 0:
+            levels = max(1, int(round(n)))
+            break
+    roof = parse_number(tags.get("roof:levels"))
+    roof_levels = max(0, int(round(roof))) if roof else 0
+    height = parse_number(tags.get("height"))
+    if height is not None and 0 < height < 1000:
+        total_levels = levels or max(1, int(round(height / FLOOR_HEIGHT_M)))
+        return round(max(1.0, height), 2), total_levels, "height"
+    if levels is not None:
+        total = levels + roof_levels
+        return round(total * FLOOR_HEIGHT_M, 2), total, "levels"
+    floors = DEFAULT_FLOORS.get(kind, 3)
+    if floors == 1 and kind in TALL_SINGLE_STOREY:
+        return TALL_SINGLE_STOREY[kind], 1, "default"
+    return round((floors + roof_levels) * FLOOR_HEIGHT_M, 2), floors + roof_levels, "default"
+
+
+def min_height(tags: dict) -> float:
+    """Bottom of a building part above the ground: min_height, else
+    building:min_level floors (levels the part skips, Simple 3D Buildings)."""
+    h = parse_number(tags.get("min_height"))
+    if h is not None and 0 < h < 1000:
+        return round(h, 2)
+    lv = parse_number(tags.get("building:min_level"))
+    if lv is not None and lv > 0:
+        return round(lv * FLOOR_HEIGHT_M, 2)
+    return 0.0
+
+
+def tree_height(tags: dict) -> float:
+    h = parse_number(tags.get("height"))
+    if h is not None and 1 <= h <= 80:
+        return round(h, 2)
+    return 8.0
+
+
+# Relative traffic by OSM ``highway`` class for the noise and air-quality
+# screening: 1 is the road the viewer's default sound power and emission stand
+# for (a busy two-lane urban road). Screening values, not traffic counts; a
+# field with real counts belongs in the layer's Traffic expression instead.
+ROAD_TRAFFIC = {
+    "motorway": 8.0,
+    "trunk": 5.0,
+    "primary": 3.0,
+    "secondary": 2.0,
+    "tertiary": 1.2,
+    "unclassified": 0.6,
+    "residential": 0.4,
+    "living_street": 0.15,
+    "service": 0.15,
+}
+HIGHWAY_PATTERN = "^(" + "|".join(ROAD_TRAFFIC) + ")(_link)?$"
+
+
+def road_traffic(tags: dict):
+    """Relative traffic of an OSM way, or None when it is no traffic source
+    (footways, tracks, roads under construction, tunnels)."""
+    highway = str(tags.get("highway") or "")
+    link = highway.endswith("_link")
+    weight = ROAD_TRAFFIC.get(highway[:-5] if link else highway)
+    if weight is None:
+        return None
+    if str(tags.get("tunnel") or "no").lower() not in ("no", "false", "0"):
+        return None
+    return round(weight * (0.5 if link else 1.0), 3)