geofix 0.1.0__py3-none-any.whl

geofix/__init__.py

@@ -0,0 +1,59 @@
+"""geofix: fast, dependency-light GeoJSON (RFC 7946) validator and repairer.
+
+Quick start::
+
+    import geofix
+
+    issues = geofix.check_file("parcels.geojson")
+    for issue in issues:
+        print(issue.code, issue.severity, issue.path, issue.message)
+
+    result = geofix.fix_file("parcels.geojson", "parcels.fixed.geojson")
+    for change in result.changes:
+        print(change.code, change.path, change.message)
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+__author__ = "Yusuf Eminoğlu"
+__email__ = "yusufeminoglu@gmail.com"
+
+from .api import FixResult, check, check_file, fix, fix_file
+from .config import ConfigError, load_config
+from .model import Change, Issue, Rule, Severity
+from .options import Options
+from .reader import GeoJSONSyntaxError, Reader, iter_features
+from .repair import Repairer, shapely_available
+from .report import FileResult
+from .rules import RULES, resolve_codes
+from .stats import Stats, stats_file, stats_object
+from .validate import Validator
+
+__all__ = [
+    "RULES",
+    "Change",
+    "ConfigError",
+    "FileResult",
+    "FixResult",
+    "GeoJSONSyntaxError",
+    "Issue",
+    "Options",
+    "Reader",
+    "Repairer",
+    "Rule",
+    "Severity",
+    "Stats",
+    "Validator",
+    "__version__",
+    "check",
+    "check_file",
+    "fix",
+    "fix_file",
+    "iter_features",
+    "load_config",
+    "resolve_codes",
+    "shapely_available",
+    "stats_file",
+    "stats_object",
+]

geofix/__main__.py

@@ -0,0 +1,7 @@
+"""``python -m geofix``."""
+
+import sys
+
+from .cli import main
+
+sys.exit(main())

geofix/api.py

@@ -0,0 +1,359 @@
+"""Public high-level API: ``check``, ``check_file``, ``fix``, ``fix_file``."""
+
+from __future__ import annotations
+
+import io
+import json
+import os
+import shutil
+import sys
+import tempfile
+from dataclasses import dataclass, field
+from typing import IO, Any, Dict, List, Optional, Set, Tuple, Union
+
+from .extent import merge, new_extent
+from .model import Change, Issue, Severity
+from .options import Options
+from .reader import GeoJSONSyntaxError, Reader
+from .repair import Repairer
+from .rules import RULES
+from .validate import Validator
+
+PathLike = Union[str, "os.PathLike[str]"]
+
+GEOJSON_SUFFIXES = (
+    ".geojson",
+    ".geojsonl",
+    ".geojsons",
+    ".geojsonseq",
+    ".ndgeojson",
+    ".geo.json",
+)
+
+
+@dataclass
+class FixResult:
+    """Outcome of a repair.
+
+    ``data`` is the repaired object (``None`` for :func:`fix_file`, which streams).
+    ``issues`` are the problems that remain after the repair.
+    """
+
+    data: Any
+    changes: List[Change] = field(default_factory=list)
+    issues: List[Issue] = field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.changes)
+
+
+def _options(options: Optional[Options], kwargs: Dict[str, Any]) -> Options:
+    if options is not None and kwargs:
+        raise TypeError("pass either an Options object or keyword options, not both")
+    return options if options is not None else Options(**kwargs)
+
+
+def _parse_text(data: Union[str, bytes]) -> Any:
+    if isinstance(data, bytes):
+        data = data.decode("utf-8-sig")
+    return data
+
+
+def check(data: Any, options: Optional[Options] = None, **kwargs: Any) -> List[Issue]:
+    """Validate GeoJSON given as a parsed object, a JSON string or bytes.
+
+    Keyword arguments are forwarded to :class:`~geofix.Options` (``select``, ``ignore``,
+    ``precision``...). Returns the list of issues (empty when the input is valid).
+    """
+    opts = _options(options, kwargs)
+    v = Validator(opts)
+    if isinstance(data, (str, bytes)):
+        _run(io.StringIO(_parse_text(data)), v)
+    else:
+        v.document(data)
+    return v.issues
+
+
+def fix(data: Any, options: Optional[Options] = None, **kwargs: Any) -> FixResult:
+    """Repair GeoJSON given as a parsed object, a JSON string or bytes.
+
+    The input is not modified. ``FixResult.data`` holds the repaired object (parsed).
+    """
+    opts = _options(options, kwargs)
+    if isinstance(data, (str, bytes)):
+        out = io.StringIO()
+        v = Validator(opts)
+        r = Repairer(opts)
+        w = _Writer(out)
+        _run(io.StringIO(_parse_text(data)), v, r, w)
+        if w.aborted:
+            return FixResult(data=None, changes=r.changes, issues=v.issues)
+        text = out.getvalue()
+        parsed = _load_written(text, w.kind)
+        return FixResult(data=parsed, changes=r.changes, issues=v.issues)
+    r = Repairer(opts)
+    repaired = r.document(data)
+    v = Validator(opts)
+    v.document(repaired)
+    return FixResult(data=repaired, changes=r.changes, issues=v.issues)
+
+
+def _load_written(text: str, kind: str) -> Any:
+    if kind == "sequence":
+        return [json.loads(line.lstrip("\x1e")) for line in text.splitlines() if line.strip()]
+    return json.loads(text)
+
+
+def _open_text(path: PathLike) -> IO[str]:
+    if str(path) == "-":
+        return io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8-sig")
+    return open(path, encoding="utf-8-sig", newline="")
+
+
+def check_file(path: PathLike, options: Optional[Options] = None, **kwargs: Any) -> List[Issue]:
+    """Validate a GeoJSON or newline-delimited GeoJSON file without loading it whole.
+
+    ``path`` may be ``"-"`` for standard input. Raises ``OSError`` if it cannot be read.
+    """
+    opts = _options(options, kwargs)
+    v = Validator(opts)
+    fp = _open_text(path)
+    try:
+        _run(fp, v)
+    finally:
+        if str(path) != "-":
+            fp.close()
+    return v.issues
+
+
+def fix_file(
+    src: PathLike,
+    dst: Optional[PathLike] = None,
+    options: Optional[Options] = None,
+    *,
+    indent: Optional[int] = None,
+    **kwargs: Any,
+) -> FixResult:
+    """Repair a file, streaming feature by feature.
+
+    ``dst`` may be another path, the same path (in place), ``"-"`` for standard output
+    or ``None`` for a dry run. Nothing is written when the input has a syntax error. When
+    no change is needed the destination receives the original bytes, so an in-place run
+    leaves a valid file untouched.
+    """
+    opts = _options(options, kwargs)
+    v = Validator(opts)
+    r = Repairer(opts)
+    src_s = str(src)
+    dst_s = None if dst is None else str(dst)
+    tmp_path: Optional[str] = None
+    sink: IO[str]
+    if dst_s is None:
+        sink = io.StringIO()
+    elif dst_s == "-":
+        sink = io.StringIO()
+    else:
+        directory = os.path.dirname(os.path.abspath(dst_s))
+        fd, tmp_path = tempfile.mkstemp(prefix=".geofix-", suffix=".tmp", dir=directory)
+        sink = os.fdopen(fd, "w", encoding="utf-8", newline="\n")
+    w = _Writer(sink, indent=indent) if dst_s is not None else _DryWriter(sink)
+    fp = _open_text(src)
+    try:
+        _run(fp, v, r, w)
+    finally:
+        if src_s != "-":
+            fp.close()
+        if tmp_path is not None:
+            sink.close()
+    try:
+        if dst_s is None or w.aborted:
+            return FixResult(data=None, changes=r.changes, issues=v.issues)
+        if dst_s == "-":
+            assert isinstance(sink, io.StringIO)
+            text = sink.getvalue() if r.changes else None
+            if text is None:
+                with open(src_s, encoding="utf-8-sig", newline="") as f:
+                    text = f.read()
+            _write_stdout(text)
+        elif r.changes:
+            assert tmp_path is not None
+            os.replace(tmp_path, dst_s)
+            tmp_path = None
+        elif os.path.abspath(dst_s) != os.path.abspath(src_s):
+            shutil.copyfile(src_s, dst_s)
+    finally:
+        if tmp_path is not None and os.path.exists(tmp_path):
+            os.remove(tmp_path)
+    return FixResult(data=None, changes=r.changes, issues=v.issues)
+
+
+def _write_stdout(text: str) -> None:
+    buf = getattr(sys.stdout, "buffer", None)
+    if buf is None:  # e.g. replaced by a StringIO in tests
+        sys.stdout.write(text)
+        sys.stdout.flush()
+        return
+    sys.stdout.flush()
+    buf.write(text.encode("utf-8"))
+    buf.flush()
+
+
+# ---------------------------------------------------------------------------------------
+# Streaming driver and writer
+# ---------------------------------------------------------------------------------------
+
+
+def _dumps(obj: Any, indent: Optional[int] = None) -> str:
+    return json.dumps(obj, ensure_ascii=False, indent=indent, allow_nan=True)
+
+
+class _Writer:
+    """Writes repaired events back out: one feature per line for collections."""
+
+    def __init__(self, fp: IO[str], indent: Optional[int] = None) -> None:
+        self.fp = fp
+        self.indent = indent
+        self.aborted = False
+        self.kind = "object"
+        self.rs = False
+        self._first = True
+        self._written: Set[str] = set()
+
+    def fc_begin(self, members: Dict[str, Any]) -> None:
+        self.kind = "collection"
+        parts = []
+        for k, val in members.items():
+            if k == "bbox":
+                continue
+            parts.append(f"{_dumps(k)}: {_dumps(val)}")
+            self._written.add(k)
+        head = ", ".join(parts)
+        self.fp.write("{" + head + (", " if head else "") + '"features": [\n')
+
+    def feature(self, f: Any) -> None:
+        if not self._first:
+            self.fp.write(",\n")
+        self._first = False
+        self.fp.write(_dumps(f))
+
+    def fc_end(self, members: Dict[str, Any]) -> None:
+        self.fp.write("\n]")
+        for k, val in members.items():
+            if k not in self._written:
+                self.fp.write(f", {_dumps(k)}: {_dumps(val)}")
+        self.fp.write("}\n")
+
+    def object(self, obj: Any) -> None:
+        self.fp.write(_dumps(obj, self.indent) + "\n")
+
+    def record(self, obj: Any) -> None:
+        self.kind = "sequence"
+        self.fp.write(("\x1e" if self.rs else "") + _dumps(obj) + "\n")
+
+
+class _DryWriter(_Writer):
+    """Discards output (dry runs) without serialising anything."""
+
+    def fc_begin(self, members: Dict[str, Any]) -> None:
+        self.kind = "collection"
+
+    def feature(self, f: Any) -> None:
+        pass
+
+    def fc_end(self, members: Dict[str, Any]) -> None:
+        pass
+
+    def object(self, obj: Any) -> None:
+        pass
+
+    def record(self, obj: Any) -> None:
+        self.kind = "sequence"
+
+
+def _run(
+    fp: IO[str],
+    v: Validator,
+    r: Optional[Repairer] = None,
+    w: Optional[_Writer] = None,
+) -> None:
+    reader = Reader(fp)
+    try:
+        if r is None or w is None:
+            for ev in reader.events():
+                v.event(ev)
+            return
+        fixable = frozenset(c for c in v.codes if RULES[c].fix)
+
+        def needs_repair(token: Tuple[int, int, Any]) -> bool:
+            return any(i.code in fixable for i in v.issues[token[0] :])
+
+        begin_keys: Set[str] = set()
+        crs_dropped = False
+        for ev in reader.events():
+            kind = ev[0]
+            if kind == "feature" or kind == "record":
+                idx, obj, line = ev[1], ev[2], ev[3]
+                check = v.feature if kind == "feature" else v.record
+                token = v.mark()
+                ext = check(idx, obj, line)
+                if needs_repair(token):
+                    # Repair, then validate the repaired object in place of the original.
+                    v.rollback(token)
+                    fixed = (
+                        r.feature(idx, obj, line) if kind == "feature" else r.record(idx, obj, line)
+                    )
+                    if fixed is None:
+                        continue
+                    check(idx, fixed, line)
+                    obj = fixed
+                elif kind == "feature":
+                    merge(r.collection_extent, ext)
+                if kind == "feature":
+                    w.feature(obj)
+                else:
+                    w.rs = reader.record_separator
+                    w.record(obj)
+            elif kind == "fc_begin":
+                members = ev[1]
+                begin_keys = set(members)
+                r.collection_extent = new_extent()
+                fixed_members = r.begin_members(members, "$", ev[2])
+                crs_dropped = "crs" in members and "crs" not in fixed_members
+                v.fc_begin(fixed_members, ev[2])
+                w.fc_begin(fixed_members)
+            elif kind == "fc_end":
+                members = ev[1]
+                if crs_dropped:
+                    members = {k: val for k, val in members.items() if k != "crs"}
+                fixed_members = r.end_members(
+                    members, "$", check_crs="crs" not in begin_keys, line=ev[2]
+                )
+                v.fc_end(fixed_members, ev[2])
+                w.fc_end(fixed_members)
+            elif kind == "object":
+                token = v.mark()
+                v.document(ev[1], ev[2])
+                out = ev[1]
+                if needs_repair(token):
+                    v.rollback(token)
+                    out = r.document(ev[1], ev[2])
+                    v.document(out, ev[2])
+                w.object(out)
+            elif kind == "error":
+                v.syntax_error(ev[1])
+                w.aborted = True
+    except GeoJSONSyntaxError as exc:
+        v.syntax_error(exc)
+        if w is not None:
+            w.aborted = True
+    except UnicodeDecodeError as exc:
+        v._line = None
+        v._emit("GF000", "$", f"input is not UTF-8 ({exc.reason} at byte {exc.start} of a chunk)")
+        if w is not None:
+            w.aborted = True
+
+
+def worst_severity(issues: List[Issue]) -> Optional[Severity]:
+    """Highest severity among ``issues`` (``None`` when empty)."""
+    return max((i.severity for i in issues), default=None)