geojson-diff 0.1.0__py3-none-any.whl

geojson_diff/core.py

@@ -0,0 +1,678 @@
+"""The diff engine: match features, compare them and classify the changes."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Sequence
+from dataclasses import asdict, dataclass, field, fields, replace
+from typing import Any, Optional
+
+from .geometry import (
+    BBox,
+    Metric,
+    bbox,
+    bbox_union,
+    centroid,
+    count_vertices,
+    geometry_area,
+    geometry_length,
+    hausdorff_distance,
+    is_geographic,
+    iter_parts,
+    same_structure,
+    vertex_changes,
+)
+from .io import Source, iter_features, source_name
+from .matching import (
+    INDEXABLE,
+    STRATEGIES,
+    Record,
+    build_index,
+    key_function,
+    match_by_key,
+    match_spatial,
+    property_value,
+)
+from .properties import PropertyChange, diff_properties
+from .tolerance import Tolerance, parse_tolerance
+
+__all__ = [
+    "STATUSES",
+    "DiffOptions",
+    "DiffResult",
+    "FeatureChange",
+    "GeometryDiff",
+    "compare_features",
+    "compare_geometry",
+    "diff",
+]
+
+STATUSES = ("added", "removed", "modified", "moved", "unchanged")
+"""Every classification, in display order."""
+
+_M_PER_DEG = math.pi * 6371008.8 / 180.0
+
+Feature = dict[str, Any]
+Geometry = Optional[dict[str, Any]]
+
+
+@dataclass
+class DiffOptions:
+    """Settings for :func:`diff`.
+
+    ``match`` is an ordered list of strategies; features left unmatched by
+    one strategy are offered to the next. ``"auto"`` expands to
+    ``["key"]`` when ``keys`` are given and to ``["id", "hash"]`` otherwise.
+    """
+
+    match: Sequence[str] = ("auto",)
+    keys: Sequence[str] = ()
+    tolerance: Tolerance = field(default_factory=Tolerance)
+    geographic: bool | None = None
+    precision: int = 7
+    strict_types: bool = False
+    ignore_properties: Sequence[str] = ()
+    max_distance: Tolerance = field(default_factory=lambda: Tolerance(50.0, "m"))
+    attribute_weight: float = 0.5
+    min_score: float = 0.5
+    keep_unchanged: bool = False
+    use_shapely: bool = False
+
+    def __post_init__(self) -> None:
+        if isinstance(self.match, str):
+            self.match = [s.strip() for s in self.match.split(",") if s.strip()]
+        if isinstance(self.keys, str):
+            self.keys = [k.strip() for k in self.keys.split(",") if k.strip()]
+        if isinstance(self.ignore_properties, str):
+            self.ignore_properties = [self.ignore_properties]
+        self.tolerance = parse_tolerance(self.tolerance)  # type: ignore[arg-type]
+        self.max_distance = parse_tolerance(self.max_distance)  # type: ignore[arg-type]
+        for name in self.match:
+            if name != "auto" and name not in STRATEGIES:
+                raise ValueError(
+                    f"unknown match strategy {name!r}; choose from auto, {', '.join(STRATEGIES)}"
+                )
+        if "key" in self.match and not self.keys:
+            raise ValueError("the 'key' strategy needs at least one property name (keys=...)")
+
+    def strategies(self) -> list[str]:
+        """The effective, de-duplicated strategy chain."""
+        out: list[str] = []
+        for name in self.match or ["auto"]:
+            expanded = (["key"] if self.keys else ["id", "hash"]) if name == "auto" else [name]
+            for item in expanded:
+                if item not in out:
+                    out.append(item)
+        return out
+
+    def to_dict(self) -> dict[str, Any]:
+        out: dict[str, Any] = {}
+        for f in fields(self):
+            value = getattr(self, f.name)
+            if isinstance(value, Tolerance):
+                value = str(value)
+            elif isinstance(value, (list, tuple)):
+                value = list(value)
+            out[f.name] = value
+        out["strategies"] = self.strategies()
+        return out
+
+
+@dataclass
+class GeometryDiff:
+    """Geometry comparison of one matched feature pair.
+
+    Distances, lengths and areas are in metres / square metres for
+    longitude/latitude data (``units == "m"``) and in CRS units otherwise
+    (``units == "crs"``).
+    """
+
+    changed: bool
+    old_type: str | None
+    new_type: str | None
+    units: str = "m"
+    max_vertex_shift: float | None = None
+    hausdorff: float | None = None
+    centroid_shift: float | None = None
+    vertices_old: int = 0
+    vertices_new: int = 0
+    vertices_added: int = 0
+    vertices_removed: int = 0
+    vertices_moved: int = 0
+    area_old: float = 0.0
+    area_new: float = 0.0
+    length_old: float = 0.0
+    length_new: float = 0.0
+    bbox: BBox | None = None
+    symmetric_difference_area: float | None = None
+
+    @property
+    def type_changed(self) -> bool:
+        return self.old_type != self.new_type
+
+    @property
+    def area_delta(self) -> float:
+        return self.area_new - self.area_old
+
+    @property
+    def length_delta(self) -> float:
+        return self.length_new - self.length_old
+
+    def to_dict(self) -> dict[str, Any]:
+        if not self.changed:
+            # Metrics are only computed for geometries that changed.
+            return {
+                "changed": False,
+                "old_type": self.old_type,
+                "new_type": self.new_type,
+                "units": self.units,
+                "vertices": self.vertices_new,
+            }
+        out = asdict(self)
+        out["type_changed"] = self.type_changed
+        out["area_delta"] = self.area_delta
+        out["length_delta"] = self.length_delta
+        out["bbox"] = list(self.bbox) if self.bbox else None
+        return {k: _finite(v) for k, v in out.items()}
+
+
+def _finite(value: Any) -> Any:
+    if isinstance(value, float) and not math.isfinite(value):
+        return None
+    return value
+
+
+@dataclass
+class FeatureChange:
+    """The classification of one feature (pair)."""
+
+    status: str
+    key: str
+    old_index: int | None = None
+    new_index: int | None = None
+    old: Feature | None = None
+    new: Feature | None = None
+    properties: list[PropertyChange] = field(default_factory=list)
+    geometry: GeometryDiff | None = None
+    matched_by: str | None = None
+    score: float | None = None
+    area_delta: float = 0.0
+    length_delta: float = 0.0
+
+    @property
+    def feature(self) -> Feature:
+        """The newest available version of the feature."""
+        return self.new if self.new is not None else self.old or {}
+
+    @property
+    def geometry_type(self) -> str | None:
+        geom = self.feature.get("geometry")
+        return geom.get("type") if geom else None
+
+    @property
+    def bbox(self) -> BBox | None:
+        boxes = []
+        if self.status in ("removed", "modified", "moved") and self.old is not None:
+            boxes.append(bbox(self.old.get("geometry")))
+        if self.status in ("added", "modified", "moved") and self.new is not None:
+            boxes.append(bbox(self.new.get("geometry")))
+        return bbox_union(boxes)
+
+    def to_dict(self, include_features: bool = True) -> dict[str, Any]:
+        """JSON-Patch-like representation of the change."""
+        op = {"added": "add", "removed": "remove"}.get(self.status, "modify")
+        if self.status == "unchanged":
+            op = "none"
+        out: dict[str, Any] = {"op": op, "status": self.status, "key": self.key}
+        if self.old_index is not None:
+            out["old_index"] = self.old_index
+        if self.new_index is not None:
+            out["new_index"] = self.new_index
+        if self.matched_by:
+            out["matched_by"] = self.matched_by
+        if self.score is not None:
+            out["score"] = round(self.score, 4)
+        if self.status == "added" and include_features:
+            out["feature"] = self.new
+        elif self.status == "removed" and include_features:
+            out["feature"] = self.old
+        if self.status in ("modified", "moved", "unchanged"):
+            patch = [c.to_dict() for c in self.properties]
+            if self.geometry is not None and self.geometry.changed and include_features:
+                patch.append(
+                    {
+                        "op": "replace",
+                        "path": "/geometry",
+                        "old": (self.old or {}).get("geometry"),
+                        "value": (self.new or {}).get("geometry"),
+                    }
+                )
+            out["patch"] = patch
+            if self.geometry is not None:
+                out["geometry"] = self.geometry.to_dict()
+        out["area_delta"] = _finite(self.area_delta)
+        out["length_delta"] = _finite(self.length_delta)
+        box = self.bbox
+        out["bbox"] = list(box) if box else None
+        return out
+
+
+@dataclass
+class DiffResult:
+    """Everything :func:`diff` found."""
+
+    changes: list[FeatureChange]
+    counts: dict[str, int]
+    old_count: int
+    new_count: int
+    options: DiffOptions
+    old_name: str = "old"
+    new_name: str = "new"
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def has_changes(self) -> bool:
+        return any(self.counts.get(s, 0) for s in STATUSES if s != "unchanged")
+
+    @property
+    def changed(self) -> list[FeatureChange]:
+        """Changes other than ``unchanged``."""
+        return [c for c in self.changes if c.status != "unchanged"]
+
+    def by_status(self, status: str) -> list[FeatureChange]:
+        return [c for c in self.changes if c.status == status]
+
+    @property
+    def bbox(self) -> BBox | None:
+        """Bounding box of every changed geometry (before and after)."""
+        return bbox_union([c.bbox for c in self.changed])
+
+    @property
+    def area_delta(self) -> float:
+        return sum(c.area_delta for c in self.changed)
+
+    @property
+    def length_delta(self) -> float:
+        return sum(c.length_delta for c in self.changed)
+
+    @property
+    def units(self) -> str:
+        for change in self.changed:
+            if change.geometry is not None:
+                return change.geometry.units
+            geom = change.feature.get("geometry")
+            if geom:
+                geographic = self.options.geographic
+                if geographic is None:
+                    geographic = is_geographic(geom)
+                return "m" if geographic else "crs"
+        return "m"
+
+    def summary(self) -> dict[str, Any]:
+        box = self.bbox
+        out: dict[str, Any] = {
+            "old_features": self.old_count,
+            "new_features": self.new_count,
+        }
+        out.update({s: self.counts.get(s, 0) for s in STATUSES})
+        out["has_changes"] = self.has_changes
+        out["units"] = self.units
+        out["bbox"] = list(box) if box else None
+        out["area_delta"] = _finite(self.area_delta)
+        out["length_delta"] = _finite(self.length_delta)
+        return out
+
+    def to_dict(self, include_features: bool = True) -> dict[str, Any]:
+        from . import __version__
+
+        return {
+            "format": "geojson-diff",
+            "version": 1,
+            "generator": f"geojson-diff {__version__}",
+            "old": self.old_name,
+            "new": self.new_name,
+            "options": self.options.to_dict(),
+            "summary": self.summary(),
+            "warnings": list(self.warnings),
+            "changes": [c.to_dict(include_features) for c in self.changes],
+        }
+
+
+# ------------------------------------------------------------------- comparison
+
+
+def _z_shift(a: Geometry, b: Geometry) -> float:
+    worst = 0.0
+    for (_, pa), (_, pb) in zip(iter_parts(a), iter_parts(b)):
+        for p, q in zip(pa, pb):
+            za = p[2] if len(p) > 2 else 0.0
+            zb = q[2] if len(q) > 2 else 0.0
+            worst = max(worst, abs(za - zb))
+    return worst
+
+
+def compare_geometry(
+    old: Geometry, new: Geometry, options: DiffOptions | None = None
+) -> GeometryDiff:
+    """Compare two geometries against ``options.tolerance``.
+
+    The geometry is *unchanged* when both versions have the same structure
+    and every vertex stayed within the tolerance, or - for different vertex
+    layouts (re-ordered rings, extra collinear vertices) - when their
+    Hausdorff distance is within the tolerance. Z values are compared against
+    the tolerance value (in metres for metric tolerances).
+    """
+    opts = options or DiffOptions()
+    old_type = old.get("type") if old else None
+    new_type = new.get("type") if new else None
+    geographic = opts.geographic
+    if old == new:
+        # Fast path for the common case; the first vertex decides the units.
+        if geographic is None:
+            first = next(iter(iter_parts(old)), None)
+            geographic = True
+            if first and first[1]:
+                geographic = is_geographic({"type": "Point", "coordinates": first[1][0]})
+        n = count_vertices(old)
+        return GeometryDiff(
+            False, old_type, new_type, "m" if geographic else "crs", vertices_old=n, vertices_new=n
+        )
+
+    if geographic is None:
+        geographic = is_geographic(old, new)
+    units = "m" if geographic else "crs"
+    tol = opts.tolerance
+    check = Metric(geographic if tol.is_metric else False)
+    report = Metric(geographic)
+    grid = tol.value / _M_PER_DEG if (tol.is_metric and geographic) else tol.value
+
+    changed: bool
+    max_shift: float | None = None
+    if not old or not new or old_type != new_type:
+        changed = True
+    elif same_structure(old, new):
+        pairs = [
+            (p, q)
+            for (_, pa), (_, pb) in zip(iter_parts(old), iter_parts(new))
+            for p, q in zip(pa, pb)
+        ]
+        worst = max((check.point(p, q) for p, q in pairs), default=0.0)
+        max_shift = max((report.point(p, q) for p, q in pairs), default=0.0)
+        if worst <= tol.value:
+            changed = _z_shift(old, new) > (tol.value if tol.is_metric else 0.0)
+        else:
+            # Same layout but a vertex left the tolerance: a rotated ring may
+            # still be the same shape, so confirm with the Hausdorff distance.
+            changed = hausdorff_distance(old, new, check.metres, limit=tol.value) > tol.value
+    else:
+        changed = hausdorff_distance(old, new, check.metres, limit=tol.value) > tol.value
+    if opts.use_shapely and tol.value == 0.0 and old and new and old_type == new_type:
+        from ._shapely import exact_equal
+
+        changed = not exact_equal(old, new)
+
+    result = GeometryDiff(
+        changed,
+        old_type,
+        new_type,
+        units,
+        max_vertex_shift=max_shift,
+        vertices_old=count_vertices(old),
+        vertices_new=count_vertices(new),
+    )
+    if not changed:
+        return result
+    result.hausdorff = hausdorff_distance(old, new, report.metres) if old and new else None
+    ca, cb = centroid(old), centroid(new)
+    result.centroid_shift = report.point(ca, cb) if ca and cb else None
+    counts = vertex_changes(old, new, tol.value, check, grid)
+    result.vertices_added = counts["added"]
+    result.vertices_removed = counts["removed"]
+    result.vertices_moved = counts["moved"]
+    result.area_old = geometry_area(old, geographic)
+    result.area_new = geometry_area(new, geographic)
+    result.length_old = geometry_length(old, geographic)
+    result.length_new = geometry_length(new, geographic)
+    result.bbox = bbox_union([bbox(old), bbox(new)])
+    if opts.use_shapely and old and new:
+        from ._shapely import symmetric_difference_area
+
+        result.symmetric_difference_area = symmetric_difference_area(old, new, geographic)
+    return result
+
+
+def _label(feature: Feature, index: int, keys: Sequence[str]) -> str:
+    if feature.get("id") is not None:
+        return str(feature["id"])
+    if keys:
+        values = [property_value(feature, k) for k in keys]
+        if all(isinstance(v, (str, int, float, bool)) for v in values):
+            if len(keys) == 1:
+                return str(values[0])
+            return ", ".join(f"{k}={v}" for k, v in zip(keys, values))
+    return f"#{index}"
+
+
+def compare_features(
+    old: Feature,
+    new: Feature,
+    options: DiffOptions | None = None,
+    *,
+    old_index: int = 0,
+    new_index: int = 0,
+    matched_by: str | None = None,
+    score: float | None = None,
+) -> FeatureChange:
+    """Compare two versions of one feature and classify the change.
+
+    The status is ``moved`` when the geometry changed beyond the tolerance
+    (property changes are still listed), ``modified`` when only properties
+    (or the feature ``id``) changed and ``unchanged`` otherwise.
+    """
+    opts = options or DiffOptions()
+    props = diff_properties(
+        old.get("properties"),
+        new.get("properties"),
+        strict_types=opts.strict_types,
+        ignore=opts.ignore_properties,
+    )
+    if old.get("id") != new.get("id"):
+        op = "add" if "id" not in old else "remove" if "id" not in new else "replace"
+        props.insert(0, PropertyChange(op, ["id"], old=old.get("id"), new=new.get("id")))
+    geom = compare_geometry(old.get("geometry"), new.get("geometry"), opts)
+    if geom.changed:
+        status = "moved"
+    elif props:
+        status = "modified"
+    else:
+        status = "unchanged"
+    return FeatureChange(
+        status=status,
+        key=_label(new, new_index, opts.keys),
+        old_index=old_index,
+        new_index=new_index,
+        old=old,
+        new=new,
+        properties=props,
+        geometry=geom,
+        matched_by=matched_by,
+        score=score,
+        area_delta=geom.area_delta if geom.changed else 0.0,
+        length_delta=geom.length_delta if geom.changed else 0.0,
+    )
+
+
+# ----------------------------------------------------------------------- engine
+
+
+def diff(
+    old: Source | Iterable[Feature],
+    new: Source | Iterable[Feature],
+    options: DiffOptions | None = None,
+    **overrides: Any,
+) -> DiffResult:
+    """Diff two GeoJSON datasets.
+
+    ``old`` and ``new`` may be paths, open files, parsed GeoJSON objects or
+    iterables of features. Keyword ``overrides`` set :class:`DiffOptions`
+    fields, e.g. ``diff(a, b, keys=["parcel_id"], tolerance="0.5m")``.
+
+    The old dataset is indexed; the new one is streamed. With an indexable
+    first strategy (id, key or hash) unchanged features are discarded as soon
+    as they are seen, so memory stays proportional to the old file plus the
+    changes.
+    """
+    opts = options or DiffOptions()
+    if overrides:
+        opts = replace(opts, **overrides)
+    strategies = opts.strategies()
+    warnings: list[str] = []
+
+    old_iter = _features(old)
+    new_iter = _features(new)
+
+    remaining: dict[int, Record] = {}
+    for i, feature in enumerate(old_iter):
+        remaining[i] = Record(i, feature)
+    old_count = len(remaining)
+
+    counts = dict.fromkeys(STATUSES, 0)
+    changes: list[FeatureChange] = []
+
+    def record(change: FeatureChange) -> None:
+        counts[change.status] += 1
+        if change.status != "unchanged" or opts.keep_unchanged:
+            changes.append(change)
+
+    def pair(o: Record, n: Record, by: str, score: float | None = None) -> None:
+        remaining.pop(o.index, None)
+        record(
+            compare_features(
+                o.feature,
+                n.feature,
+                opts,
+                old_index=o.index,
+                new_index=n.index,
+                matched_by=by,
+                score=score,
+            )
+        )
+
+    first = strategies[0]
+    index = None
+    keyfn = None
+    if first in INDEXABLE:
+        keyfn = key_function(first, opts.keys, opts.precision)
+        index, dups = build_index(remaining.values(), keyfn)
+        if dups:
+            warnings.append(
+                f"{dups} duplicate {first} key(s) in {source_name(old)}; paired in input order"
+            )
+    pending: list[Record] = []
+    seen_new: set = set()
+    new_dups = 0
+    new_count = 0
+    for j, feature in enumerate(new_iter):
+        new_count += 1
+        if index is not None and keyfn is not None:
+            key = keyfn(feature)
+            if key is not None:
+                if key in seen_new:
+                    new_dups += 1
+                seen_new.add(key)
+                bucket = index.get(key)
+                if bucket:
+                    pair(bucket.popleft(), Record(j, feature), first)
+                    continue
+        pending.append(Record(j, feature))
+    if new_dups:
+        warnings.append(
+            f"{new_dups} duplicate {first} key(s) in {source_name(new)}; paired in input order"
+        )
+    del seen_new
+
+    for strategy in strategies[1 if index is not None else 0 :]:
+        if not remaining or not pending:
+            break
+        olds = list(remaining.values())
+        if strategy == "spatial":
+            matched = match_spatial(
+                olds,
+                pending,
+                max_distance=opts.max_distance,
+                attribute_weight=opts.attribute_weight,
+                min_score=opts.min_score,
+                ignore=opts.ignore_properties,
+                geographic=opts.geographic,
+            )
+            triples: list[tuple[Record, Record, float | None]] = list(matched)  # type: ignore[arg-type]
+        else:
+            fn = key_function(strategy, opts.keys, opts.precision)
+            triples = [(o, n, None) for o, n in match_by_key(olds, pending, fn)]
+        used = set()
+        for o, n, score in triples:
+            pair(o, n, strategy, score)
+            used.add(n.index)
+        pending = [n for n in pending if n.index not in used]
+
+    for rec in remaining.values():
+        geom = rec.feature.get("geometry")
+        record(
+            FeatureChange(
+                "removed",
+                _label(rec.feature, rec.index, opts.keys),
+                old_index=rec.index,
+                old=rec.feature,
+                area_delta=-geometry_area(geom, opts.geographic),
+                length_delta=-geometry_length(geom, opts.geographic),
+            )
+        )
+    for rec in pending:
+        geom = rec.feature.get("geometry")
+        record(
+            FeatureChange(
+                "added",
+                _label(rec.feature, rec.index, opts.keys),
+                new_index=rec.index,
+                new=rec.feature,
+                area_delta=geometry_area(geom, opts.geographic),
+                length_delta=geometry_length(geom, opts.geographic),
+            )
+        )
+
+    order = {s: i for i, s in enumerate(STATUSES)}
+
+    def sort_key(c: FeatureChange) -> tuple[int, int, int]:
+        primary = c.new_index if c.new_index is not None else c.old_index
+        return (
+            order[c.status],
+            primary if primary is not None else -1,
+            c.old_index or 0,
+        )
+
+    changes.sort(key=sort_key)
+    return DiffResult(
+        changes=changes,
+        counts=counts,
+        old_count=old_count,
+        new_count=new_count,
+        options=opts,
+        old_name=source_name(old) if not _is_feature_iterable(old) else "old",
+        new_name=source_name(new) if not _is_feature_iterable(new) else "new",
+        warnings=warnings,
+    )
+
+
+def _features(source: Any) -> Iterable[Any]:
+    return source if _is_feature_iterable(source) else iter_features(source)
+
+
+def _is_feature_iterable(obj: Any) -> bool:
+    """True for iterables of features that are not GeoJSON sources themselves."""
+    if (
+        isinstance(obj, (str, bytes, dict, list))
+        or hasattr(obj, "read")
+        or hasattr(obj, "__fspath__")
+    ):
+        return False
+    return hasattr(obj, "__iter__")