django-gisserver 1.5.0__py3-none-any.whl → 2.0__py3-none-any.whl

gisserver/geometries.py

@@ -6,15 +6,16 @@ The bounding box can be calculated within Python, or read from a database result
 
 from __future__ import annotations
 
+import logging
 import re
 from dataclasses import dataclass, field
 from decimal import Decimal
-from functools import lru_cache
+from functools import cached_property, lru_cache
 
-from django.contrib.gis.gdal import CoordTransform, SpatialReference
-from django.contrib.gis.geos import GEOSGeometry, Polygon
+from django.contrib.gis.gdal import AxisOrder, CoordTransform, SpatialReference
+from django.contrib.gis.geos import GEOSGeometry
 
-from gisserver.exceptions import ExternalParsingError, ExternalValueError
+from gisserver.exceptions import ExternalValueError
 
 CRS_URN_REGEX = re.compile(
     r"^urn:(?P<domain>[a-z]+)"
@@ -27,15 +28,27 @@ CRS_URN_REGEX = re.compile(
 
 __all__ = [
     "CRS",
+    "WEB_MERCATOR",
     "WGS84",
     "BoundingBox",
 ]
 
+logger = logging.getLogger(__name__)
 
-@lru_cache(maxsize=200)
-def _get_spatial_reference(srs_input, srs_type="user", axis_order=None):
+
+@lru_cache(maxsize=200)  # Using lru-cache to avoid repeated GDAL c-object construction
+def _get_spatial_reference(srs_input: str | int, srs_type="user", axis_order=None):
     """Construct an GDAL object reference"""
-    # Using lru-cache to avoid repeated GDAL c-object construction
+    if axis_order is None:
+        # WFS 1.0 used x/y coordinates, WFS 2.0 uses the ordering from the CRS authority.
+        axis_order = AxisOrder.AUTHORITY
+
+    logger.debug(
+        "Constructed GDAL SpatialReference(%r, srs_type=%r, axis_order=%s)",
+        srs_input,
+        srs_type,
+        axis_order,
+    )
     return SpatialReference(srs_input, srs_type=srs_type, axis_order=axis_order)
 
 
@@ -162,6 +175,7 @@ class CRS:
                     f"CRS URI [{urn}] should contain a numeric SRID value."
                 ) from None
         elif authority == "OGC":
+            # urn:ogc:def:crs:OGC::CRS84 has x/y ordering (longitude/latitude)
             crsid = urn_match.group("id").upper()
             if crsid != "CRS84":
                 raise ExternalValueError(f"OGC CRS URI from [{urn}] contains unknown id [{id}]")
@@ -216,7 +230,7 @@ class CRS:
         """Return a legacy string in the format "EPSG:<srid>"""
         return f"EPSG:{self.srid:d}"
 
-    @property
+    @cached_property
     def urn(self):
         """Return The OGC URN corresponding to this CRS."""
         return f"urn:{self.domain}:def:crs:{self.authority}:{self.version or ''}:{self.crsid}"
@@ -227,7 +241,7 @@ class CRS:
     def __eq__(self, other):
         if isinstance(other, CRS):
             # CRS84 is NOT equivalent to EPSG:4326.
-            # EPSG:4326 specifies coordinates in lat/long order and CRS:84 in long/lat order.
+            # EPSG:4326 specifies coordinates in lat/long order and CRS84 in long/lat order.
             return self.authority == other.authority and self.srid == other.srid
         else:
             return NotImplemented
@@ -267,12 +281,19 @@ class CRS:
             return geometry.transform(transform, clone=clone)
 
 
-WGS84 = CRS.from_srid(4326)  # aka EPSG:4326
+# Worldwide GPS, latitude/longitude (y/x). https://epsg.io/4326
+WGS84 = CRS.from_string("urn:ogc:def:crs:EPSG::4326")
+
+# GeoJSON default. This is like WGS84 but with longitude/latitude (x/y).
+CRS84 = CRS.from_string("urn:ogc:def:crs:OGC::CRS84")
+
+#: Spherical Mercator (Google Maps, Bing Maps, OpenStreetMap, ...), see https://epsg.io/3857
+WEB_MERCATOR = CRS.from_string("urn:ogc:def:crs:EPSG::3857")
 
 
 @dataclass
 class BoundingBox:
-    """A bounding box that describes the extent of a map layer"""
+    """A bounding box (or "envelope") that describes the extent of a map layer"""
 
     south: Decimal  # longitude
     west: Decimal  # latitude
@@ -280,25 +301,6 @@ class BoundingBox:
     east: Decimal  # latitude
     crs: CRS | None = None
 
-    @classmethod
-    def from_string(cls, bbox):
-        """Parse the bounding box from an input string.
-
-        It can either be 4 coordinates, or 4 coordinates with a special reference system.
-        """
-        bbox = bbox.split(",")
-        if not (4 <= len(bbox) <= 5):
-            raise ExternalParsingError(
-                f"Input does not contain bounding box, expected 4 or 5 values, not {bbox}."
-            )
-        return cls(
-            Decimal(bbox[0]),
-            Decimal(bbox[1]),
-            Decimal(bbox[2]),
-            Decimal(bbox[3]),
-            CRS.from_string(bbox[4]) if len(bbox) == 5 else None,
-        )
-
     @classmethod
     def from_geometry(cls, geometry: GEOSGeometry, crs: CRS | None = None):
         """Construct the bounding box for a geometry"""
@@ -349,10 +351,3 @@ class BoundingBox:
             )
         else:
             return NotImplemented
-
-    def as_polygon(self) -> Polygon:
-        """Convert the value into a GEOS polygon."""
-        polygon = Polygon.from_bbox((self.south, self.west, self.north, self.east))
-        if self.crs is not None:
-            polygon.srid = self.crs.srid
-        return polygon

gisserver/management/commands/loadgeojson.py

@@ -0,0 +1,291 @@
+"""Quick utility to import GeoJSON data in the Django project."""
+
+from __future__ import annotations
+
+import json
+import operator
+from functools import reduce
+from itertools import islice
+from urllib.request import urlopen
+
+from django.apps import apps
+from django.contrib.gis.db.models import GeometryField
+from django.contrib.gis.geos import GEOSGeometry
+from django.core.exceptions import FieldDoesNotExist
+from django.core.management import BaseCommand, CommandError, CommandParser
+from django.db import DEFAULT_DB_ALIAS, connections, models, transaction
+
+from gisserver.geometries import CRS, WGS84
+
+
+def _parse_model(value):
+    try:
+        return apps.get_model(value)
+    except LookupError as e:
+        raise ValueError(str(e)) from e
+
+
+def _parse_fields(value):
+    field_map = {}
+    for pair in value.split(","):
+        geojson_name, _, field_name = pair.strip().partition("=")
+        if not value:
+            raise ValueError("Expect property=field,property2=field2 format")
+        field_map[geojson_name] = field_name
+    return field_map
+
+
+class Command(BaseCommand):
+    """Quick command to import data in the WFS server."""
+
+    help = (
+        "Import GeoJSON data into a WFS feature collection. This can be done using:"
+        "  manage.py loadgeojson --model=places.Province"
+        " -f name=name,prop2=field2 --geometry-field geometry provinces.json"
+    )
+
+    def add_arguments(self, parser: CommandParser):
+        parser.add_argument(
+            "--database",
+            default=DEFAULT_DB_ALIAS,
+            choices=tuple(connections),
+            help=(
+                "Nominates a specific database to load fixtures into. Defaults to the "
+                '"default" database.'
+            ),
+        )
+        parser.add_argument(
+            "-m",
+            "--model",
+            required=True,
+            metavar="MODEL",
+            type=_parse_model,
+            help="Django model, in the format: app_label.ModelName.",
+        )
+        parser.add_argument(
+            "-g",
+            "--geometry-field",
+            required=False,
+            metavar="NAME",
+            help="Name of the model's geometry field, auto-detected if not specified.",
+        )
+        parser.add_argument(
+            "-f",
+            "--field",
+            nargs="*",
+            dest="map_fields",
+            type=_parse_fields,
+            metavar="NAME=FIELD",
+            help=(
+                "Map GeoJSON properties to Django fields, in the format: property1=field1,property2=field2,... "
+                "By default, this autodetects common field names only. "
+                "Explicitly ignore fields by using -f property=,property2=field2."
+            ),
+        )
+        parser.add_argument(
+            "geojson-file",
+            help="GeoJSON file or URL to import. For debugging, it's better to download the file first.",
+        )
+
+    def handle(self, *args, **options):
+        # Get arguments
+        self.using = options["database"]
+        self.connection = connections[self.using]
+        model: type[models.Model] = options["model"]
+        main_geometry_field = self._get_geometry_field(model, options["geometry_field"])
+        field_map = (
+            self._parse_field_map(model, options["map_fields"]) if options["map_fields"] else {}
+        )
+
+        # Read the file
+        geojson = self._load_geojson(options["geojson-file"])
+        if not geojson["features"]:
+            self.stdout.write(self.style.NOTICE("Empty GeoJSON data"))
+            return
+
+        # See if properties match the Django field names, use those too.
+        # (unless these are mapped already via the command line args).
+        field_map.update(self._get_auto_field_map(model, geojson["features"][0], field_map))
+
+        # See if a CRS is declared
+        self.crs = self._read_crs(geojson)
+
+        # Import in chunks
+        num_imported = 0
+        id_field = model._meta.pk.name
+        with transaction.atomic(using=self.using):
+            features = iter(self._read_geojson(geojson, model, main_geometry_field, field_map))
+            while batch := list(islice(features, 100)):
+                if id_field in batch[0]:
+                    # The ID field is provided, allow "on conflict update..."
+                    unique_fields = [id_field]
+                    update_fields = list(batch[0].keys())
+                    update_fields.remove(id_field)
+
+                    model.objects.using(self.using).bulk_create(
+                        [model(**values) for values in batch],
+                        update_conflicts=True,
+                        unique_fields=unique_fields,
+                        update_fields=update_fields,
+                    )
+                else:
+                    model.objects.using(self.using).bulk_create(
+                        [model(**values) for values in batch]
+                    )
+
+                num_imported += len(batch)
+
+        self.stdout.write(f"Installed {num_imported} feature(s)")
+
+    def _get_geometry_field(self, model: type[models.Model], field_name: str | None):
+        """Find the geometry field name, or validate it."""
+        if field_name:
+            # Field is given in CLI args, validate it.
+            try:
+                field = model._meta.get_field(field_name)
+            except FieldDoesNotExist as e:
+                raise CommandError(f"Invalid value for --geometry-field: {e}") from e
+
+            if not isinstance(field, GeometryField):
+                raise CommandError("Invalid value for --geometry-field: not a GeometryField")
+        else:
+            # Field is not given, auto-detect
+            try:
+                field_name = next(
+                    f.name for f in model._meta.fields if isinstance(f, GeometryField)
+                )
+            except StopIteration:
+                raise CommandError(
+                    f"Model {model._meta.label} does not have any GeometryField member."
+                ) from None
+
+        return field_name
+
+    def _load_geojson(self, filename) -> dict:
+        """Parse and validate the GeoJSON data into Python dict data."""
+        try:
+            if "://" in filename:
+                with urlopen(filename, timeout=60) as response:  # noqa: S310
+                    geojson = json.load(response)
+            else:
+                with open(filename) as fh:
+                    geojson = json.load(fh)
+        except OSError as e:  # FileNotFoundError or HTTP errors
+            raise CommandError(str(e)) from e
+        except (ValueError, TypeError) as e:
+            raise CommandError(f"Unable to parse GeoJSON: {e}") from e
+
+        if (
+            not isinstance(geojson, dict)
+            or geojson.get("type") != "FeatureCollection"
+            or "features" not in geojson
+        ):
+            raise CommandError("Invalid GeoJSON data, expected FeatureCollection element.")
+
+        return geojson
+
+    def _read_crs(self, geojson: dict) -> CRS:
+        """Find the CRS that should be used for all geometry data."""
+        crs = geojson.get("crs")
+        if not crs:
+            return WGS84
+
+        try:
+            type = crs["type"]
+            if type == "name":
+                return CRS.from_string(crs["properties"]["name"])
+            else:
+                # type 'link' with subtype 'proj4/ogcwkt/esriwkt' are not handled here.
+                raise CommandError(f"CRS type {type} is not supported.")
+        except KeyError as e:
+            raise CommandError(f"CRS is invalid, missing '{e}' field") from e
+
+    def _parse_field_map(self, model, map_fields: list[dict]) -> dict:
+        """Validate that the provided field mapping points to model fields."""
+        field_map: dict = reduce(operator.or_, map_fields)
+        for field_name in field_map.values():
+            try:
+                model._meta.get_field(field_name)
+            except FieldDoesNotExist as e:
+                raise CommandError(f"Field '{field_name}' does not exist in Django model.") from e
+
+        return field_map
+
+    def _get_auto_field_map(self, model, feature: dict, field_map: dict):
+        """Autodetect which properties match model field names."""
+        auto_field_map = {}
+        for geojson_name in feature.get("properties", ()):
+            if geojson_name in field_map:
+                continue
+
+            try:
+                model._meta.get_field(geojson_name)
+            except FieldDoesNotExist:
+                msg = (
+                    f"GeoJSON property '{geojson_name}' is not mapped nor exists in Django model, skipping."
+                    if field_map
+                    else f"GeoJSON property '{geojson_name}' does not exist in Django model, skipping."
+                )
+                self.stderr.write(self.style.WARNING(msg))
+            else:
+                self.stdout.write(
+                    f"GeoJSON property '{geojson_name}' also exists as Django model field, using."
+                )
+                auto_field_map[geojson_name] = geojson_name
+
+        return auto_field_map
+
+    def _read_geojson(
+        self, geojson: dict, model: type[models.Model], main_geometry_field: str, field_map: dict
+    ):
+        """Convert the GeoJSON data to model field names."""
+        pk_field = model._meta.pk.name
+
+        for feature in geojson["features"]:
+            # Validate basic layout
+            try:
+                feature_type = feature["type"]
+                geometry_data = feature["geometry"]
+            except KeyError as e:
+                raise CommandError(f"Feature does not have required '{e}' element.") from e
+            if feature_type != "Feature":
+                raise CommandError(f"Expected 'Feature', not {feature_type}")
+
+            # Collect all fields
+            properties = feature.get("properties", {})
+            field_values = {
+                field_name: self._parse_value(
+                    model._meta.get_field(field_name), properties.get(geojson_name)
+                )
+                for geojson_name, field_name in field_map.items()
+                if field_name
+            }
+
+            # Store the geometry, note GEOS only parses stringified JSON.
+            geometry = GEOSGeometry(json.dumps(geometry_data), srid=self.crs.srid)
+            geometry.srid = self.crs.srid  # override default
+            field_values[main_geometry_field] = geometry
+
+            # Try to decode the identifier if it's present
+            if pk_field not in field_values and (id_value := feature.get("id")):
+                try:
+                    field_values[pk_field] = self._parse_id(model._meta.pk, id_value)
+                except (ValueError, TypeError):
+                    self.stderr.write(
+                        self.style.WARNING(
+                            f"Feature id value '{id_value}' can't be stored in the primary key field. ignoring."
+                        )
+                    )
+
+            # Pass as Django model fields
+            yield field_values
+
+    def _parse_id(self, pk_field: models.Field, id_value):
+        # Allow TypeName.id format, see if it parses
+        return pk_field.get_db_prep_save(id_value.rpartition(".")[2], connection=self.connection)
+
+    def _parse_value(self, field: models.Field, value):
+        try:
+            return field.get_db_prep_save(value, connection=self.connection)
+        except (ValueError, TypeError) as e:
+            raise CommandError(f"Can't parse {value!r} in model field '{field.name}': {e}.") from e