geodata-sync 0.1.0__py3-none-any.whl

geodata_sync/__init__.py

@@ -0,0 +1,4 @@
+"""geodata-sync: open GIS data loaded into PostGIS, and kept up to date, from YAML files."""
+
+__version__ = "0.1.0"
+API_VERSION = "geodata/v1alpha1"

geodata_sync/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

geodata_sync/cli.py

@@ -0,0 +1,141 @@
+"""The geodata command."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from . import __version__
+from .discover import DiscoverError
+from .discover import run as discover
+from .status import run as status
+from .sync import SyncError
+from .sync import run as sync
+from .loader import load
+from .validate import validate
+
+def shown(path: Path) -> str:
+    """A path relative to the current directory when that's shorter."""
+    try:
+        rel = os.path.relpath(path)
+    except ValueError:
+        return str(path)
+    return rel if len(rel) < len(str(path)) else str(path)
+
+
+def cmd_validate(args) -> int:
+    root = Path(args.dir)
+    if not root.is_dir():
+        print(f"geodata: {root} is not a directory", file=sys.stderr)
+        return 2
+    project = load(root)
+    problems = validate(project)
+    errors = [p for p in problems if p.severity == "error"]
+
+    if args.format == "json":
+        print(json.dumps([
+            {"file": shown(p.file), "line": p.line, "severity": p.severity, "resource": p.resource, "message": p.message}
+            for p in problems
+        ], indent=2))
+    else:
+        for p in problems:
+            where = f"{shown(p.file)}:{p.line}" if p.line else shown(p.file)
+            resource = f"{p.resource}: " if p.resource else ""
+            print(f"{where}: {p.severity}: {resource}{p.message}")
+        warnings = len(problems) - len(errors)
+        summary = f"{len(project.resources)} resources in {len(project.files)} files"
+        if problems:
+            print(f"\n{len(errors)} error(s), {warnings} warning(s); {summary}")
+        else:
+            print(f"Valid: {summary}")
+    return 1 if errors else 0
+
+
+def cmd_discover(args) -> int:
+    try:
+        return discover(
+            args.source, args.product, Path(args.dir), fmt=args.format,
+            layers=not args.no_layers, use_cache=not args.refresh,
+            search=args.search, limit=args.limit, resource=args.resource, url=args.url,
+        )
+    except DiscoverError as e:
+        print(f"geodata discover: {e}", file=sys.stderr)
+        return e.code
+
+
+def cmd_status(args) -> int:
+    try:
+        return status(Path(args.dir), fmt=args.format)
+    except SyncError as e:
+        print(f"geodata status: {e}", file=sys.stderr)
+        return e.code
+
+
+def cmd_sync(args) -> int:
+    try:
+        return sync(Path(args.dir), args.datasets, force=args.force, loaded_file=args.loaded)
+    except SyncError as e:
+        print(f"geodata sync: {e}", file=sys.stderr)
+        return e.code
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="geodata", description="Open GIS data loaded into PostGIS, and kept up to date, from YAML files.")
+    parser.add_argument("--version", action="version", version=f"geodata {__version__}")
+    commands = parser.add_subparsers(dest="command", metavar="command")
+
+    v = commands.add_parser("validate", help="check the files against the spec (no network)")
+    v.add_argument("dir", nargs="?", default=".", help="project directory (default: current)")
+    v.add_argument("--format", choices=["text", "json"], default="text")
+    v.set_defaults(func=cmd_validate)
+
+    d = commands.add_parser(
+        "discover", help="list what a source offers, or the layers inside one product",
+        description="List the products a source offers, or the downloads and layers of one product. "
+        "SOURCE is a Source in the project (e.g. os) or a source type: os-downloads, ckan, wfs or ogc-api-features (with --url), "
+        "or url (the product is then a file's address).",
+    )
+    d.add_argument("source", help="a Source name from the project, or a source type")
+    d.add_argument("product", nargs="?", help="a product ID (a CKAN package, a file's URL), to list its downloads and layers")
+    d.add_argument("--search", metavar="WORDS", help="list only products matching these words")
+    d.add_argument("--limit", type=int, metavar="N", help="list at most N products (CKAN default: 20)")
+    d.add_argument("--resource", metavar="ID", help="read this download of the product (a CKAN resource ID or name)")
+    d.add_argument("--url", help="the source's address, e.g. a CKAN portal (overrides the Source's spec.url)")
+    d.add_argument("-C", "--dir", default=".", help="project directory, for Sources and secrets (default: current)")
+    d.add_argument("--format", choices=["text", "json"], default="text")
+    d.add_argument("--no-layers", action="store_true", help="list downloads only; don't read layers from the file")
+    d.add_argument("--refresh", action="store_true", help="read layers again instead of using the cache")
+    d.set_defaults(func=cmd_discover)
+
+    st = commands.add_parser("status", help="datasets: loaded, when, next check",
+                             description="Show the project's datasets from the ingest log. Changes nothing.")
+    st.add_argument("-C", "--dir", default=".", help="project directory (default: current)")
+    st.add_argument("--format", choices=["text", "json"], default="text")
+    st.set_defaults(func=cmd_status)
+
+    s = commands.add_parser(
+        "sync", help="load new versions of datasets into PostGIS",
+        description="Check each due Dataset for a new version and load it: stage, check, back up, swap in. "
+        "Named datasets are checked whatever their schedule. The database comes from the Project's connection secret.",
+    )
+    s.add_argument("datasets", nargs="*", help="only these Datasets, due or not")
+    s.add_argument("-C", "--dir", default=".", help="project directory (default: current)")
+    s.add_argument("--force", action="store_true", help="load even when the version hasn't changed")
+    s.add_argument("--loaded", type=Path, metavar="FILE",
+                   help="write what was loaded to FILE, one per line (schema.table, or a raster's path), for `configops refresh --from FILE`")
+    s.set_defaults(func=cmd_sync)
+
+    argv = sys.argv[1:] if argv is None else argv
+    if argv[:1] == ["discover"]:
+        # options may come between the source and the product
+        args = d.parse_intermixed_args(argv[1:])
+        args.command = "discover"
+    else:
+        args = parser.parse_args(argv)
+    if not args.command:
+        parser.print_help()
+        return 2
+    return args.func(args)

geodata_sync/crs.py

@@ -0,0 +1,21 @@
+"""Coordinate reference system names, in the short form the spec uses (EPSG:27700)."""
+
+from __future__ import annotations
+
+import re
+
+# urn:ogc:def:crs:EPSG::27700, urn:x-ogc:def:crs:EPSG:27700, EPSG:27700,
+# http://www.opengis.net/def/crs/EPSG/0/27700, http://www.opengis.net/gml/srs/epsg.xml#27700
+EPSG = re.compile(r"(?:EPSG(?::[\d.]*)?:|/EPSG/[\d.]+/|epsg\.xml#)(\d+)$", re.IGNORECASE)
+CRS84 = re.compile(r"(?:OGC/1\.3/|OGC:(?:1\.3:)?)CRS84$", re.IGNORECASE)
+
+
+def short(name: str | None) -> str | None:
+    """EPSG:<code> or CRS84 when recognised, else the name as given."""
+    if not name:
+        return None
+    name = name.strip()
+    if CRS84.search(name):
+        return "CRS84"
+    m = EPSG.search(name)
+    return f"EPSG:{m.group(1)}" if m else name

geodata_sync/discover.py

@@ -0,0 +1,315 @@
+"""`geodata discover`: what a source offers, and the layers inside one product."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shlex
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from . import remote, secrets
+from .loader import load
+from .sources import Download, NotFound, Product, SourceError, source_types
+from .sources.formats import SERVICE_TYPES, service_address
+
+# Key in a source's credentials secret that holds the API key
+API_KEY = "apiKey"
+
+
+class DiscoverError(Exception):
+    """Ends the command with a message; `code` is the exit code."""
+
+    def __init__(self, message: str, code: int = 1):
+        super().__init__(message)
+        self.code = code
+
+
+@dataclass
+class ResolvedSource:
+    label: str  # how to name it in messages: the Source's name, or the type
+    type: str
+    client: object
+    command: str  # how to run discover on it again, for hints
+
+
+def resolve(name: str, root: Path, url: str | None = None) -> ResolvedSource:
+    """
+    A Source named `name` in the project at `root`, or else a source type
+    used directly. `url` is the source's address (a CKAN portal); it
+    overrides the Source's own.
+    """
+    types = source_types()
+    source = load(root).named("Source").get(name) if root.is_dir() else None
+    if source is not None:
+        stype = source.spec.get("type")
+        if stype not in types:
+            raise DiscoverError(f"Source {name} has type '{stype}', which can't be discovered (yet). Types that can: {', '.join(types)}")
+        api_key = None
+        secret = (source.spec.get("credentials") or {}).get("secret")
+        if secret:
+            api_key = secrets.lookup(secret, API_KEY, root)
+            if api_key is None:
+                print(
+                    f"geodata: note: secret {secret} (key {API_KEY}) isn't set "
+                    f"(${secrets.env_name(secret, API_KEY)} or secrets.yaml); trying without a key",
+                    file=sys.stderr,
+                )
+        c = client(types[stype], api_key, url or source.spec.get("url"))
+        return ResolvedSource(name, stype, c, command(name, c, url))
+    if name in types:
+        c = client(types[name], None, url)
+        return ResolvedSource(name, name, c, command(name, c, url))
+    raise DiscoverError(
+        f"no Source named '{name}' in {root}, and '{name}' isn't a source type. "
+        f"Name a Source from the project, or a type: {', '.join(types)}",
+        code=2,
+    )
+
+
+def command(name: str, c, url: str | None) -> str:
+    """How to run discover on this source again: with its tidied address when one was given."""
+    if not url:
+        return f"geodata discover {name}"
+    address = getattr(c, "url", None) or getattr(c, "portal", None) or url
+    return f"geodata discover {name} --url {shlex.quote(address)}"
+
+
+def client(source_type: type, api_key: str | None, url: str | None):
+    try:
+        return source_type(api_key=api_key, url=url)
+    except SourceError as e:
+        raise DiscoverError(str(e), code=2) from None
+
+
+# --- layer cache: a product version never changes, so its layers don't either
+
+
+def cache_dir() -> Path:
+    base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
+    return Path(base) / "geodata" / "layers"
+
+
+def cache_path(source_type: str, product: Product) -> Path | None:
+    d = product.chosen
+    if not product.version or d is None:
+        return None
+    key = json.dumps([source_type, product.id, product.version, d.id, d.url if d.id else "", d.format, d.subformat, d.area, d.file_name, d.md5])
+    return cache_dir() / f"{hashlib.sha256(key.encode()).hexdigest()[:32]}.json"
+
+
+def cached_layers(path: Path | None) -> list[remote.Layer] | None:
+    if path is None:
+        return None
+    try:
+        return [remote.Layer.from_dict(d) for d in json.loads(path.read_text())]
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+
+
+def save_layers(path: Path | None, layers: list[remote.Layer]) -> None:
+    if path is None:
+        return
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps([layer.to_dict() for layer in layers]))
+    except OSError:
+        pass  # a cache that can't be written only costs time
+
+
+def product_layers(src: ResolvedSource, product: Product, use_cache: bool = True) -> list[remote.Layer]:
+    path = cache_path(src.type, product)
+    if use_cache:
+        hit = cached_layers(path)
+        if hit is not None:
+            return hit
+    d = product.chosen
+    if hasattr(src.client, "layers"):  # a service: asked directly, no file
+        layers = src.client.layers(product)
+    else:
+        print(
+            f"Reading layers from {d.file_name or d.url} ({size(d.size)}) without downloading it; "
+            "a large file can take a minute or two...",
+            file=sys.stderr,
+        )
+        layers = remote.url_layers(src.client.download_url(d), file_name=d.file_name)
+    save_layers(path, layers)
+    return layers
+
+
+# --- output
+
+
+def size(n: int | None) -> str:
+    if n is None:
+        return "size unknown"
+    if n < 1000:
+        return f"{n} bytes"
+    for unit in ("KB", "MB", "GB"):
+        n /= 1000
+        if n < 1000 or unit == "GB":
+            return f"{n:.1f} {unit}"
+
+
+def table(rows: list[list[str]]) -> str:
+    widths = [max(len(r[i]) for r in rows) for i in range(len(rows[0]))]
+    return "\n".join("  ".join(c.ljust(w) for c, w in zip(r, widths)).rstrip() for r in rows)
+
+
+def short(text: str, width: int = 70) -> str:
+    text = " ".join(text.split())
+    return text if len(text) <= width else text[: width - 1] + "…"
+
+
+def version(v: str | None) -> str:
+    """A date-time version (CKAN's last change) shown as its date."""
+    if not v:
+        return "-"
+    return v[:10] if len(v) > 10 and v[4:5] == "-" and v[10:11] == "T" else v
+
+
+def print_products(src: ResolvedSource, products: list[Product], total: int | None, search: str | None) -> None:
+    if not products:
+        print(f"No products from {src.label}" + (f" match '{search}'" if search else ""))
+        return
+    def via_service(p: Product) -> str | None:
+        return next((f for f in p.formats if f in SERVICE_TYPES), None)
+
+    rows = [["PRODUCT", "VERSION", "LOADS AS", "NAME"]]
+    for p in products:
+        loads = p.loads_as or (f"via {via_service(p)}" if via_service(p) else "-")
+        rows.append([p.id, version(p.version), loads, short(p.name)])
+    print(table(rows))
+    services = sum(1 for p in products if not p.loads_as and via_service(p))
+    skipped = sum(1 for p in products if not p.loads_as) - services
+    shown = f"{len(products)} of {total:,}" if total and total > len(products) else f"{len(products)}"
+    print(f"\n{shown} product{'s' if len(products) != 1 or total != 1 else ''} from {src.label}" + (f" matching '{search}'" if search else ""), end="")
+    print(f"; {skipped} have nothing geodata can load" if skipped else "")
+    if services:
+        print(f"{services} offer only a WFS or OGC API service: the product's details show how to discover it")
+    if total and total > len(products):
+        print("More with --limit, or narrow with --search")
+    print(f"Layers inside one: {src.command} <product>")
+
+
+def download_line(d: Download, used: bool) -> str:
+    label = f"{d.format} {d.subformat}".strip()
+    file_name = d.file_name if "." in d.file_name and d.file_name != d.name else ""  # not ".../download"
+    what = ", ".join(x for x in (d.name, file_name) if x) or d.area or d.url
+    line = f"  {'*' if used else ' '} {label}: {short(what, 60)}"
+    if d.loadable:
+        line += f", {size(d.size)}"
+    elif d.format in SERVICE_TYPES:
+        line += " (a service, not a file)"
+    else:
+        line += " (not a file geodata loads)"
+    line += f"  [{d.id}]" if d.id else ""
+    if not d.loadable and d.format in SERVICE_TYPES:
+        line += f"\n        discover it: geodata discover {SERVICE_TYPES[d.format]} --url {shlex.quote(service_address(d.format, d.url))}"
+    return line
+
+
+def print_product(src: ResolvedSource, p: Product, layers: list[remote.Layer] | None) -> None:
+    if src.type == "url":
+        print(f"{p.name}\n  {p.id}")
+    else:
+        print(f"{p.name} ({p.id})" + (f", version {version(p.version)}" if p.version else ""))
+    if p.chosen is not None and p.chosen.service:
+        print(f"  {f'{p.chosen.format} {p.chosen.subformat}'.strip()}: {p.chosen.url}")
+        if p.description:
+            print(f"  {short(p.description, 100)}")
+        print_layers(layers, footer=f"In a Dataset: `product: {p.id}`; the table defaults to <dataset>_<layer>, so `tables` can name it.")
+        return
+    groups: dict[str, list[Download]] = {}
+    for d in p.downloads:
+        groups.setdefault(f"{d.format} {d.subformat}".strip(), []).append(d)
+    for label, ds in groups.items():
+        used = p.chosen in ds
+        if len(ds) > 1 and all(d.area and not d.id for d in ds):
+            # one format cut into many areas (OS grid squares): one line
+            line = f"  {'*' if used else ' '} {label}: {len(ds)} files by area"
+            if used:
+                line += f"; using {p.chosen.file_name or p.chosen.area}, {size(p.chosen.size)}"
+            print(line)
+        else:
+            for d in ds:
+                print(download_line(d, d is p.chosen))
+    if not p.downloads:
+        print("\nNo downloads listed.")
+        return
+    if p.chosen is None:
+        print(f"\nNo download geodata can load (it reads {', '.join(src.client.formats)} as one file).")
+        return
+    print("  (* the download sync uses)")
+    if sum(1 for d in p.downloads if d.loadable) > 1 and p.chosen.id:
+        print(f"Another one: --resource <id or name>; in a Dataset, `resource: {p.chosen.id}`")
+    print_layers(layers)
+
+
+def print_layers(layers: list[remote.Layer] | None, footer: str = "") -> None:
+    if layers is None:
+        return
+    print()
+    rows = [["LAYER", "GEOMETRY", "CRS", "FEATURES"]]
+    for layer in layers:
+        rows.append([
+            layer.name,
+            layer.geometry or "none",
+            layer.crs or "-",
+            f"{layer.features:,}" if layer.features is not None else "-",
+        ])
+    print(table(rows))
+    files = sorted({layer.file for layer in layers if layer.file})
+    if len(files) > 1:
+        print(f"\nfrom {len(files)} files: {', '.join(files)}")
+    print("\n" + (footer or "A Dataset loads every layer unless `tables` names some; each becomes table <dataset>_<layer>."))
+
+
+def run(
+    source: str, product_id: str | None, root: Path, fmt: str = "text", layers: bool = True, use_cache: bool = True,
+    search: str | None = None, limit: int | None = None, resource: str | None = None, url: str | None = None,
+) -> int:
+    src = resolve(source, root, url)
+    try:
+        if product_id is None:
+            if resource:
+                raise DiscoverError("--resource needs a product", code=2)
+            listing = src.client.products(search, limit)
+            if fmt == "json":
+                print(json.dumps({"total": listing.total, "products": [p.to_dict() for p in listing.products]}, indent=2, ensure_ascii=False))
+            else:
+                print_products(src, listing.products, listing.total, search)
+            return 0
+
+        if search:
+            raise DiscoverError("--search lists products; leave out the product to search", code=2)
+        try:
+            product = src.client.product(product_id, resource)
+        except NotFound:
+            raise DiscoverError(f"{src.label} has no product '{product_id}'; find one with: {src.command} --search <words>") from None
+    except SourceError as e:
+        raise DiscoverError(str(e)) from None
+
+    # A file that can't be read still leaves the downloads worth showing
+    found, layer_error = None, None
+    if layers and product.chosen:
+        try:
+            found = product_layers(src, product, use_cache)
+        except (SourceError, remote.RemoteError) as e:
+            layer_error = f"couldn't read layers: {e}"
+
+    if fmt == "json":
+        out = product.to_dict()
+        out["downloads"] = [d.to_dict() for d in product.downloads]
+        out["download"] = product.chosen.to_dict() if product.chosen else None
+        out["layers"] = [layer.to_dict() for layer in found] if found is not None else None
+        out["layersError"] = layer_error
+        print(json.dumps(out, indent=2, ensure_ascii=False))
+    else:
+        print_product(src, product, found)
+    if layer_error:
+        print(f"geodata discover: {layer_error}", file=sys.stderr)
+        return 1
+    return 0