geojson-diff 0.1.0__py3-none-any.whl

geojson_diff/io.py

@@ -0,0 +1,254 @@
+"""Streaming readers for GeoJSON FeatureCollections and GeoJSONSeq.
+
+The reader never needs the whole document in memory at once: the
+``features`` array of a FeatureCollection is decoded one feature at a time,
+and GeoJSONSeq / newline-delimited files are decoded record by record.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import os
+import sys
+from collections.abc import Iterable, Iterator
+from typing import IO, Any, Union
+
+__all__ = ["GeoJSONError", "Source", "iter_features", "load_features"]
+
+Feature = dict[str, Any]
+Source = Union[str, "os.PathLike[str]", dict[str, Any], list[Any], IO[str], IO[bytes]]
+
+_GEOMETRY_TYPES = {
+    "Point",
+    "MultiPoint",
+    "LineString",
+    "MultiLineString",
+    "Polygon",
+    "MultiPolygon",
+    "GeometryCollection",
+}
+_RS = "\x1e"
+_WS = " \t\r\n" + _RS
+_CHUNK = 1 << 20
+
+
+class GeoJSONError(ValueError):
+    """Raised when an input is not valid (or not supported) GeoJSON."""
+
+
+def _as_feature(obj: Any, where: str) -> Feature:
+    if not isinstance(obj, dict):
+        raise GeoJSONError(f"{where}: expected a GeoJSON object, got {type(obj).__name__}")
+    kind = obj.get("type")
+    if kind == "Feature":
+        return obj
+    if kind in _GEOMETRY_TYPES:
+        return {"type": "Feature", "geometry": obj, "properties": {}}
+    raise GeoJSONError(f"{where}: expected a Feature, got type {kind!r}")
+
+
+def _iter_object(obj: Any, where: str) -> Iterator[Feature]:
+    if isinstance(obj, list):
+        for i, item in enumerate(obj):
+            yield _as_feature(item, f"{where}[{i}]")
+        return
+    if isinstance(obj, dict) and obj.get("type") == "FeatureCollection":
+        features = obj.get("features")
+        if not isinstance(features, list):
+            raise GeoJSONError(f"{where}: FeatureCollection without a 'features' array")
+        for i, item in enumerate(features):
+            yield _as_feature(item, f"{where} feature {i}")
+        return
+    yield _as_feature(obj, where)
+
+
+class _Reader:
+    """Incremental JSON tokenizer over a text stream (top level only)."""
+
+    def __init__(self, stream: IO[str], name: str) -> None:
+        self.stream = stream
+        self.name = name
+        self.buf = ""
+        self.pos = 0
+        self.eof = False
+        self.decoder = json.JSONDecoder()
+
+    def fill(self) -> bool:
+        if self.eof:
+            return False
+        if self.pos > _CHUNK:
+            self.buf = self.buf[self.pos :]
+            self.pos = 0
+        # Read geometrically larger chunks so huge single values stay linear.
+        data = self.stream.read(max(_CHUNK, len(self.buf)))
+        if not data:
+            self.eof = True
+            return False
+        if not self.buf and data.startswith(""):
+            data = data[1:]
+        self.buf += data
+        return True
+
+    def skip_ws(self) -> bool:
+        """Skip whitespace (and RS separators); return False at end of input."""
+        while True:
+            while self.pos < len(self.buf) and self.buf[self.pos] in _WS:
+                self.pos += 1
+            if self.pos < len(self.buf):
+                return True
+            if not self.fill():
+                return False
+
+    def peek(self) -> str:
+        if not self.skip_ws():
+            return ""
+        return self.buf[self.pos]
+
+    def expect(self, char: str) -> None:
+        if self.peek() != char:
+            got = self.peek() or "end of input"
+            raise GeoJSONError(f"{self.name}: expected {char!r} but found {got!r}")
+        self.pos += 1
+
+    def value(self) -> Any:
+        if not self.skip_ws():
+            raise GeoJSONError(f"{self.name}: unexpected end of input")
+        while True:
+            try:
+                obj, end = self.decoder.raw_decode(self.buf, self.pos)
+            except json.JSONDecodeError as exc:
+                if self.fill():
+                    continue
+                raise GeoJSONError(
+                    f"{self.name}: invalid JSON ({exc.msg} at char {exc.pos})"
+                ) from None
+            if end >= len(self.buf) and self.fill():
+                continue  # a number or literal may continue in the next chunk
+            self.pos = end
+            return obj
+
+
+def _iter_stream(stream: IO[str], name: str) -> Iterator[Feature]:
+    reader = _Reader(stream, name)
+    first = reader.peek()
+    if first == "":
+        return
+    if first == "[":
+        reader.expect("[")
+        if reader.peek() == "]":
+            reader.pos += 1
+            return
+        index = 0
+        while True:
+            yield _as_feature(reader.value(), f"{name}[{index}]")
+            index += 1
+            sep = reader.peek()
+            reader.pos += 1
+            if sep == "]":
+                return
+            if sep != ",":
+                raise GeoJSONError(f"{name}: expected ',' or ']' in feature array")
+    if first != "{":
+        raise GeoJSONError(f"{name}: not a GeoJSON document")
+
+    record = 0
+    while True:
+        # Decode one top-level object key by key, streaming a 'features' array.
+        reader.expect("{")
+        members: dict[str, Any] = {}
+        streamed = False
+        if reader.peek() == "}":
+            reader.pos += 1
+        else:
+            while True:
+                key = reader.value()
+                if not isinstance(key, str):
+                    raise GeoJSONError(f"{name}: invalid object key")
+                reader.expect(":")
+                if key == "features" and reader.peek() == "[":
+                    streamed = True
+                    reader.expect("[")
+                    if reader.peek() == "]":
+                        reader.pos += 1
+                    else:
+                        index = 0
+                        while True:
+                            item = reader.value()
+                            yield _as_feature(item, f"{name} feature {index}")
+                            index += 1
+                            sep = reader.peek()
+                            reader.pos += 1
+                            if sep == "]":
+                                break
+                            if sep != ",":
+                                raise GeoJSONError(f"{name}: expected ',' or ']' in features")
+                else:
+                    members[key] = reader.value()
+                sep = reader.peek()
+                reader.pos += 1
+                if sep == "}":
+                    break
+                if sep != ",":
+                    raise GeoJSONError(f"{name}: expected ',' or '}}' in object")
+        if streamed:
+            if members.get("type", "FeatureCollection") != "FeatureCollection":
+                raise GeoJSONError(f"{name}: object with 'features' is not a FeatureCollection")
+        elif members.get("type") == "FeatureCollection":
+            raise GeoJSONError(f"{name}: FeatureCollection without a 'features' array")
+        else:
+            yield _as_feature(members, f"{name} record {record}")
+        record += 1
+        # GeoJSONSeq / newline-delimited GeoJSON: further top-level objects follow.
+        nxt = reader.peek()
+        if nxt == "":
+            return
+        if nxt != "{":
+            raise GeoJSONError(f"{name}: unexpected data after record {record}")
+
+
+def _open_text(path: str) -> IO[str]:
+    return open(path, encoding="utf-8-sig", newline="")
+
+
+def iter_features(source: Source, name: str | None = None) -> Iterator[Feature]:
+    """Yield the features of a GeoJSON source one at a time.
+
+    ``source`` may be a path, ``"-"`` for standard input, an open text or
+    binary file, an already-parsed GeoJSON object (FeatureCollection, Feature,
+    geometry or list of features). FeatureCollections, single Features, bare
+    geometries, JSON arrays of features, GeoJSONSeq (RFC 8142) and
+    newline-delimited GeoJSON are all accepted.
+    """
+    if isinstance(source, (dict, list)):
+        yield from _iter_object(source, name or "<object>")
+        return
+    if isinstance(source, (str, os.PathLike)):
+        path = os.fspath(source)
+        if path == "-":
+            stdin = sys.stdin.buffer if hasattr(sys.stdin, "buffer") else sys.stdin
+            yield from iter_features(stdin, name or "<stdin>")
+            return
+        with _open_text(path) as handle:
+            yield from _iter_stream(handle, name or path)
+        return
+    if hasattr(source, "read"):
+        stream: Any = source
+        probe = stream.read(0)
+        if isinstance(probe, bytes):
+            stream = io.TextIOWrapper(stream, encoding="utf-8-sig")
+        yield from _iter_stream(stream, str(name or getattr(source, "name", "<stream>")))
+        return
+    raise TypeError(f"unsupported GeoJSON source: {type(source).__name__}")
+
+
+def load_features(source: Source, name: str | None = None) -> list[Feature]:
+    """Return all features of ``source`` as a list (see :func:`iter_features`)."""
+    return list(iter_features(source, name))
+
+
+def source_name(source: Source | Iterable[Any]) -> str:
+    """A short display name for ``source``."""
+    if isinstance(source, (str, os.PathLike)):
+        return os.fspath(source)
+    return str(getattr(source, "name", "<object>"))

geojson_diff/matching.py

@@ -0,0 +1,218 @@
+"""Feature matching strategies: id, property key(s), geometry hash, spatial."""
+
+from __future__ import annotations
+
+import json
+import math
+from collections import defaultdict, deque
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+from .geometry import Metric, centroid, geometry_hash, is_geographic
+from .properties import similarity
+from .tolerance import Tolerance
+
+__all__ = [
+    "INDEXABLE",
+    "STRATEGIES",
+    "Record",
+    "key_function",
+    "match_by_key",
+    "match_spatial",
+    "property_value",
+]
+
+STRATEGIES = ("id", "key", "hash", "spatial")
+INDEXABLE = ("id", "key", "hash")
+"""Strategies that can match through a dictionary index (and therefore stream)."""
+
+_M_PER_DEG = 111_320.0
+_MISSING = object()
+
+
+@dataclass
+class Record:
+    """A feature with its position in the input."""
+
+    index: int
+    feature: dict[str, Any]
+
+
+KeyFunction = Callable[[dict[str, Any]], Optional[str]]
+
+
+def _norm(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, float) and value.is_integer():
+        value = int(value)
+    return json.dumps(value, sort_keys=True)
+
+
+def property_value(feature: dict[str, Any], name: str) -> Any:
+    """Value of property ``name`` (dotted for nested), or a sentinel when absent.
+
+    ``@id`` reads the feature's top-level ``id``; ``id`` falls back to it when
+    there is no ``id`` property.
+    """
+    if name == "@id":
+        return feature.get("id", _MISSING)
+    props = feature.get("properties") or {}
+    current: Any = props
+    for part in name.split("."):
+        if isinstance(current, dict) and part in current:
+            current = current[part]
+        else:
+            current = _MISSING
+            break
+    if current is _MISSING and name == "id" and "id" in feature:
+        return feature["id"]
+    return current
+
+
+def key_function(strategy: str, keys: Sequence[str] = (), precision: int = 7) -> KeyFunction:
+    """Return the function mapping a feature to its match key (``None`` if it has none)."""
+    if strategy == "id":
+
+        def by_id(feature: dict[str, Any]) -> str | None:
+            value = feature.get("id")
+            return None if value is None else _norm(value)
+
+        return by_id
+    if strategy == "key":
+        if not keys:
+            raise ValueError("the 'key' strategy needs at least one property name (--key)")
+        names = list(keys)
+
+        def by_key(feature: dict[str, Any]) -> str | None:
+            values = [property_value(feature, n) for n in names]
+            if any(v is _MISSING or v is None for v in values):
+                return None
+            return "\x1f".join(_norm(v) for v in values)
+
+        return by_key
+    if strategy == "hash":
+
+        def by_hash(feature: dict[str, Any]) -> str | None:
+            geom = feature.get("geometry")
+            return None if not geom else geometry_hash(geom, precision)
+
+        return by_hash
+    raise ValueError(f"strategy {strategy!r} cannot be used as an index key")
+
+
+def build_index(
+    records: Iterable[Record], keyfn: KeyFunction
+) -> tuple[dict[str, deque[Record]], int]:
+    """Index records by key; returns the index and the number of duplicate keys."""
+    index: dict[str, deque[Record]] = defaultdict(deque)
+    duplicates = 0
+    for rec in records:
+        key = keyfn(rec.feature)
+        if key is None:
+            continue
+        if key in index:
+            duplicates += 1
+        index[key].append(rec)
+    return index, duplicates
+
+
+def match_by_key(
+    olds: Sequence[Record], news: Sequence[Record], keyfn: KeyFunction
+) -> list[tuple[Record, Record]]:
+    """Pair records with equal keys; duplicates pair up in input order."""
+    index, _ = build_index(olds, keyfn)
+    pairs: list[tuple[Record, Record]] = []
+    for rec in news:
+        key = keyfn(rec.feature)
+        if key is None:
+            continue
+        bucket = index.get(key)
+        if bucket:
+            pairs.append((bucket.popleft(), rec))
+    return pairs
+
+
+def _family(geom: dict[str, Any] | None) -> str:
+    if not geom:
+        return "none"
+    kind = str(geom.get("type", ""))
+    return kind.replace("Multi", "").replace("LineString", "Line")
+
+
+def match_spatial(
+    olds: Sequence[Record],
+    news: Sequence[Record],
+    *,
+    max_distance: Tolerance,
+    attribute_weight: float = 0.5,
+    min_score: float = 0.5,
+    ignore: Iterable[str] = (),
+    geographic: bool | None = None,
+) -> list[tuple[Record, Record, float]]:
+    """Pair features by centroid proximity and attribute similarity.
+
+    Candidates must share a geometry family (point, line, polygon) and lie
+    within ``max_distance``. Each candidate scores
+    ``(1 - w) * (1 - distance / max_distance) + w * similarity`` with
+    ``w = attribute_weight``; pairs scoring below ``min_score`` are dropped.
+    The assignment is greedy on descending score, then ascending distance,
+    old index and new index, so it is fully deterministic.
+    """
+    if not 0.0 <= attribute_weight <= 1.0:
+        raise ValueError("attribute_weight must be between 0 and 1")
+    ignore = list(ignore)
+    old_c = [(r, centroid(r.feature.get("geometry"))) for r in olds]
+    new_c = [(r, centroid(r.feature.get("geometry"))) for r in news]
+    if geographic is None:
+        geographic = all(
+            is_geographic(r.feature.get("geometry")) for r, _ in old_c + new_c if r.feature
+        )
+    metres = max_distance.is_metric and geographic
+    metric = Metric(metres)
+    limit = max_distance.value
+    cell = (limit / _M_PER_DEG if metres else limit) or 1e-9
+
+    grid: dict[tuple[int, int], list[tuple[Record, tuple[float, float]]]] = defaultdict(list)
+    for rec, c in new_c:
+        if c is not None:
+            grid[(math.floor(c[0] / cell), math.floor(c[1] / cell))].append((rec, c))
+
+    candidates: list[tuple[float, float, int, int, Record, Record]] = []
+    for rec, c in old_c:
+        if c is None:
+            continue
+        family = _family(rec.feature.get("geometry"))
+        cx, cy = math.floor(c[0] / cell), math.floor(c[1] / cell)
+        span = 1
+        if metres:
+            span = math.ceil(1.0 / max(math.cos(math.radians(min(89.0, abs(c[1]) + cell))), 0.01))
+        for gx in range(cx - span, cx + span + 1):
+            for gy in range(cy - 1, cy + 2):
+                for other, oc in grid.get((gx, gy), ()):
+                    if _family(other.feature.get("geometry")) != family:
+                        continue
+                    d = metric.point(c, oc)
+                    if d > limit:
+                        continue
+                    closeness = 1.0 - d / limit if limit > 0 else 1.0
+                    sim = similarity(
+                        rec.feature.get("properties"),
+                        other.feature.get("properties"),
+                        ignore,
+                    )
+                    score = (1.0 - attribute_weight) * closeness + attribute_weight * sim
+                    if score >= min_score:
+                        candidates.append((-score, d, rec.index, other.index, rec, other))
+    candidates.sort(key=lambda t: t[:4])
+    used_old: set = set()
+    used_new: set = set()
+    pairs: list[tuple[Record, Record, float]] = []
+    for neg_score, _, oi, ni, rec, other in candidates:
+        if oi in used_old or ni in used_new:
+            continue
+        used_old.add(oi)
+        used_new.add(ni)
+        pairs.append((rec, other, -neg_score))
+    return pairs

geojson_diff/properties.py

@@ -0,0 +1,208 @@
+"""Typed, nested diff of feature properties (JSON Pointer paths)."""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from typing import Any, Union
+
+__all__ = ["PropertyChange", "diff_properties", "json_type", "pointer", "similarity"]
+
+PathPart = Union[str, int]
+
+
+def json_type(value: Any) -> str:
+    """JSON type name of a Python value: null, boolean, integer, number, string, array, object."""
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, float):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, (list, tuple)):
+        return "array"
+    if isinstance(value, dict):
+        return "object"
+    return type(value).__name__
+
+
+def _escape(part: PathPart) -> str:
+    return str(part).replace("~", "~0").replace("/", "~1")
+
+
+def pointer(parts: Sequence[PathPart]) -> str:
+    """RFC 6901 JSON Pointer for ``parts``."""
+    return "".join("/" + _escape(p) for p in parts)
+
+
+def _dotted(parts: Sequence[PathPart]) -> str:
+    out = ""
+    for part in parts:
+        if isinstance(part, int):
+            out += f"[{part}]"
+        else:
+            out += ("." if out else "") + part
+    return out
+
+
+@dataclass
+class PropertyChange:
+    """One change inside a feature.
+
+    ``op`` is ``"add"``, ``"remove"`` or ``"replace"`` (JSON Patch vocabulary);
+    ``parts`` is the path below the feature, e.g. ``["properties", "tags", 0]``.
+    """
+
+    op: str
+    parts: list[PathPart]
+    old: Any = None
+    new: Any = None
+
+    @property
+    def path(self) -> str:
+        """JSON Pointer of the change, e.g. ``/properties/tags/0``."""
+        return pointer(self.parts)
+
+    @property
+    def name(self) -> str:
+        """Human-readable path without the ``properties`` prefix, e.g. ``tags[0]``."""
+        parts = self.parts[1:] if self.parts and self.parts[0] == "properties" else self.parts
+        return _dotted(parts) or "(root)"
+
+    @property
+    def old_type(self) -> str | None:
+        return None if self.op == "add" else json_type(self.old)
+
+    @property
+    def new_type(self) -> str | None:
+        return None if self.op == "remove" else json_type(self.new)
+
+    @property
+    def type_changed(self) -> bool:
+        return self.op == "replace" and self.old_type != self.new_type
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {"op": self.op, "path": self.path}
+        if self.op != "add":
+            out["old"] = self.old
+            out["old_type"] = self.old_type
+        if self.op != "remove":
+            out["value"] = self.new
+            out["new_type"] = self.new_type
+        return out
+
+
+@dataclass
+class _Ctx:
+    strict_types: bool
+    ignore: tuple[tuple[PathPart, ...], ...]
+    out: list[PropertyChange] = field(default_factory=list)
+
+
+def _scalars_equal(a: Any, b: Any, strict_types: bool) -> bool:
+    ta, tb = json_type(a), json_type(b)
+    if ta != tb:
+        if strict_types or {ta, tb} != {"integer", "number"}:
+            return False
+    if ta == "number" and tb == "number" and math.isnan(a) and math.isnan(b):
+        return True
+    return bool(a == b)
+
+
+def _ignored(parts: Sequence[PathPart], ignore: Iterable[tuple[PathPart, ...]]) -> bool:
+    return any(tuple(parts[: len(prefix)]) == prefix for prefix in ignore)
+
+
+def _walk(old: Any, new: Any, parts: list[PathPart], ctx: _Ctx) -> None:
+    if _ignored(parts, ctx.ignore):
+        return
+    if isinstance(old, dict) and isinstance(new, dict):
+        for key in old:
+            if key not in new and not _ignored(parts + [key], ctx.ignore):
+                ctx.out.append(PropertyChange("remove", parts + [key], old=old[key]))
+        for key in old:
+            if key in new:
+                _walk(old[key], new[key], parts + [key], ctx)
+        for key in new:
+            if key not in old and not _ignored(parts + [key], ctx.ignore):
+                ctx.out.append(PropertyChange("add", parts + [key], new=new[key]))
+        return
+    if isinstance(old, list) and isinstance(new, list) and len(old) == len(new):
+        for i, (a, b) in enumerate(zip(old, new)):
+            _walk(a, b, parts + [i], ctx)
+        return
+    if isinstance(old, (dict, list)) or isinstance(new, (dict, list)):
+        if _canonical(old) != _canonical(new):
+            ctx.out.append(PropertyChange("replace", list(parts), old=old, new=new))
+        return
+    if not _scalars_equal(old, new, ctx.strict_types):
+        ctx.out.append(PropertyChange("replace", list(parts), old=old, new=new))
+
+
+def _canonical(value: Any) -> str:
+    return json.dumps(value, sort_keys=True, default=str)
+
+
+def _ignore_paths(ignore: Iterable[str]) -> tuple[tuple[PathPart, ...], ...]:
+    out = []
+    for item in ignore:
+        out.append(("properties", *item.split(".")))
+    return tuple(out)
+
+
+def diff_properties(
+    old: dict[str, Any] | None,
+    new: dict[str, Any] | None,
+    *,
+    strict_types: bool = False,
+    ignore: Iterable[str] = (),
+) -> list[PropertyChange]:
+    """Diff two property dictionaries recursively.
+
+    Nested objects are compared key by key and equal-length arrays element by
+    element; arrays whose length changed are reported as one ``replace``.
+    ``1`` and ``1.0`` are equal unless ``strict_types`` is true; ``True`` and
+    ``1`` are never equal. ``ignore`` lists property names (dotted for nested
+    keys, e.g. ``"meta.updated"``) to skip.
+    """
+    ctx = _Ctx(strict_types=strict_types, ignore=_ignore_paths(ignore))
+    _walk(old or {}, new or {}, ["properties"], ctx)
+    return ctx.out
+
+
+def similarity(
+    old: dict[str, Any] | None, new: dict[str, Any] | None, ignore: Iterable[str] = ()
+) -> float:
+    """Attribute similarity in ``[0, 1]``: share of equal top-level properties.
+
+    Unequal strings earn partial credit from :class:`difflib.SequenceMatcher`.
+    Two empty property sets are fully similar.
+    """
+    from difflib import SequenceMatcher
+
+    skip = set(ignore)
+    a = {k: v for k, v in (old or {}).items() if k not in skip}
+    b = {k: v for k, v in (new or {}).items() if k not in skip}
+    keys = set(a) | set(b)
+    if not keys:
+        return 1.0
+    score = 0.0
+    for key in keys:
+        if key not in a or key not in b:
+            continue
+        va, vb = a[key], b[key]
+        if _canonical(va) == _canonical(vb) or (
+            json_type(va) in ("integer", "number")
+            and json_type(vb) in ("integer", "number")
+            and va == vb
+        ):
+            score += 1.0
+        elif isinstance(va, str) and isinstance(vb, str):
+            score += SequenceMatcher(None, va, vb).ratio()
+    return score / len(keys)