django-gisserver 1.5.0__py3-none-any.whl → 2.1__py3-none-any.whl

gisserver/output/csv.py

@@ -9,20 +9,14 @@ from io import StringIO
 from django.db import models
 
 from gisserver import conf
-from gisserver.db import (
-    AsEWKT,
-    build_db_annotations,
-    get_db_annotation,
-    get_db_geometry_selects,
-)
-from gisserver.geometries import CRS
-from gisserver.queries import FeatureProjection
-from gisserver.types import XsdElement
+from gisserver.db import AsEWKT, get_db_rendered_geometry, replace_queryset_geometries
+from gisserver.projection import FeatureProjection, FeatureRelation
+from gisserver.types import GeometryXsdElement, XsdElement, XsdTypes
 
-from .base import OutputRenderer
+from .base import CollectionOutputRenderer
 
 
-class CSVRenderer(OutputRenderer):
+class CSVRenderer(CollectionOutputRenderer):
     """Fast CSV renderer, using a stream response.
 
     The complex encoding bits are handled by the "csv" library.
@@ -37,32 +31,22 @@ class CSVRenderer(OutputRenderer):
     #: or one of the registered names like: "unix", "excel", "excel-tab"
     dialect = "unix"
 
-    @classmethod
     def decorate_queryset(
-        cls,
-        projection: FeatureProjection,
-        queryset: models.QuerySet,
-        output_crs: CRS,
-        **params,
+        self, projection: FeatureProjection, queryset: models.QuerySet
     ) -> models.QuerySet:
         """Make sure relations are included with select-related to avoid N-queries.
         Using prefetch_related() isn't possible with .iterator().
         """
-        # Take all relations that are expanded to complex elements,
-        # and all relations that are fetched for flattened elements.
-        related = {
-            xsd_element.orm_path
-            for xsd_element in projection.complex_elements
-            if not xsd_element.is_many
-        } | {
-            xsd_element.orm_relation[0]
-            for xsd_element in projection.flattened_elements
-            if not xsd_element.is_many
-        }
-        if related:
-            queryset = queryset.select_related(*related)
-
-        return queryset
+        # First, make sure no array or m2m elements exist,
+        # as these are not possible to render in CSV.
+        projection.remove_fields(
+            lambda e: (e.is_many and not e.is_array)
+            or e.type == XsdTypes.gmlCodeType  # gml:name
+            or e.type == XsdTypes.gmlBoundingShapeType  # gml:boundedBy
+        )
+
+        # All database optimizations
+        return super().decorate_queryset(projection, queryset)
 
     def render_stream(self):
         self.output = output = StringIO()
@@ -78,18 +62,12 @@ class CSVRenderer(OutputRenderer):
                 output.write("\n\n")
 
             # Write the header
-            fields = [
-                f
-                for f in projection.xsd_root_elements
-                if not f.is_many and f.xml_name not in ("gml:name", "gml:boundedBy")
-            ]
-            writer.writerow(self.get_header(projection, fields))
-
-            # By using .iterator(), the results are streamed with as little memory as
-            # possible. Doing prefetch_related() is not possible now. That could only
-            # be implemented with cursor pagination for large sets for 1000+ results.
-            for instance in sub_collection.iterator():
-                writer.writerow(self.get_row(instance, projection, fields))
+            xsd_elements = projection.xsd_root_elements
+            writer.writerow(self.get_header(projection, xsd_elements))
+
+            # Write all rows
+            for instance in self.read_features(sub_collection):
+                writer.writerow(self.get_row(instance, projection, xsd_elements))
 
                 # Only perform a 'yield' every once in a while,
                 # as it goes back-and-forth for writing it to the client.
@@ -108,18 +86,22 @@ class CSVRenderer(OutputRenderer):
         return f"{buffer}\n\n{message}\n"
 
     def get_header(
-        self, projection: FeatureProjection, xsd_elements: list[XsdElement]
+        self, projection: FeatureProjection, xsd_elements: list[XsdElement], prefix=""
     ) -> list[str]:
         """Return all field names."""
         names = []
         append = names.append
         for xsd_element in xsd_elements:
             if xsd_element.type.is_complex_type:
-                # Expand complex types
-                for sub_field in projection.xsd_child_nodes[xsd_element]:
-                    append(f"{xsd_element.name}.{sub_field.name}")
+                names.extend(
+                    self.get_header(
+                        projection,
+                        xsd_elements=projection.xsd_child_nodes[xsd_element],
+                        prefix=f"{prefix}{xsd_element.name}.",
+                    )
+                )
             else:
-                append(xsd_element.name)
+                append(f"{prefix}{xsd_element.name}")
 
         return names
 
@@ -130,14 +112,19 @@ class CSVRenderer(OutputRenderer):
         values = []
         append = values.append
         for xsd_element in xsd_elements:
-            if xsd_element.is_geometry:
-                append(self.render_geometry(instance, xsd_element))
+            if xsd_element.type.is_geometry:
+                append(self.render_geometry(projection, instance, xsd_element))
                 continue
 
             value = xsd_element.get_value(instance)
             if xsd_element.type.is_complex_type:
-                for sub_field in projection.xsd_child_nodes[xsd_element]:
-                    append(sub_field.get_value(value))
+                values.extend(
+                    self.get_row(
+                        instance=value,
+                        projection=projection,
+                        xsd_elements=projection.xsd_child_nodes[xsd_element],
+                    )
+                )
             elif isinstance(value, list):
                 # Array field
                 append(",".join(map(str, value)))
@@ -147,9 +134,16 @@ class CSVRenderer(OutputRenderer):
                 append(value)
         return values
 
-    def render_geometry(self, instance: models.Model, field: XsdElement):
+    def render_geometry(
+        self,
+        projection: FeatureProjection,
+        instance: models.Model,
+        geo_element: GeometryXsdElement,
+    ) -> str:
         """Render the contents of a geometry value."""
-        return field.get_value(instance)
+        geometry = geo_element.get_value(instance)
+        projection.output_crs.apply_to(geometry)
+        return geometry.ewkt
 
 
 class DBCSVRenderer(CSVRenderer):
@@ -157,25 +151,34 @@ class DBCSVRenderer(CSVRenderer):
     This is about 40% faster than calling the GEOS C-API from python.
     """
 
-    @classmethod
     def decorate_queryset(
-        cls,
-        projection: FeatureProjection,
-        queryset: models.QuerySet,
-        output_crs: CRS,
-        **params,
+        self, projection: FeatureProjection, queryset: models.QuerySet
     ) -> models.QuerySet:
-        queryset = super().decorate_queryset(projection, queryset, output_crs, **params)
-
-        # Instead of reading the binary geometry data,
-        # ask the database to generate EWKT data directly.
-        geo_selects = get_db_geometry_selects(projection.geometry_elements, output_crs)
-        if geo_selects:
-            queryset = queryset.defer(*geo_selects.keys()).annotate(
-                **build_db_annotations(geo_selects, "_as_ewkt_{name}", AsEWKT)
-            )
-
-        return queryset
-
-    def render_geometry(self, instance: models.Model, field: XsdElement):
-        return get_db_annotation(instance, field.name, "_as_ewkt_{name}")
+        # Instead of reading the binary geometry data, let the database generate EWKT data.
+        # As annotations can't be done for select_related() objects, prefetches are used instead.
+        queryset = super().decorate_queryset(projection, queryset)
+        return replace_queryset_geometries(
+            queryset, projection.geometry_elements, projection.output_crs, AsEWKT
+        )
+
+    def get_prefetch_queryset(
+        self, projection: FeatureProjection, feature_relation: FeatureRelation
+    ) -> models.QuerySet | None:
+        """Perform DB annotations for prefetched relations too."""
+        queryset = super().get_prefetch_queryset(projection, feature_relation)
+        if queryset is None:
+            return None
+
+        # Find which fields are GML elements, annotate these too.
+        return replace_queryset_geometries(
+            queryset, feature_relation.geometry_elements, projection.output_crs, AsEWKT
+        )
+
+    def render_geometry(
+        self,
+        projection: FeatureProjection,
+        instance: models.Model,
+        geo_element: GeometryXsdElement,
+    ):
+        """Render the geometry using a database-rendered version."""
+        return get_db_rendered_geometry(instance, geo_element, AsEWKT)

gisserver/output/geojson.py

@@ -6,16 +6,17 @@ from io import BytesIO
 
 import orjson
 from django.contrib.gis.db.models.functions import AsGeoJSON
+from django.contrib.gis.gdal import AxisOrder
 from django.db import models
 from django.utils.functional import Promise
 
 from gisserver import conf
+from gisserver.crs import CRS84, WGS84
 from gisserver.db import get_db_geometry_target
-from gisserver.geometries import CRS
-from gisserver.queries import FeatureProjection
+from gisserver.projection import FeatureProjection
 from gisserver.types import XsdElement
 
-from .base import OutputRenderer
+from .base import CollectionOutputRenderer
 
 
 def _json_default(obj):
@@ -25,13 +26,13 @@ def _json_default(obj):
     raise TypeError(f"Unable to serialize {obj.__class__.__name__} to JSON")
 
 
-class GeoJsonRenderer(OutputRenderer):
+class GeoJsonRenderer(CollectionOutputRenderer):
     """Fast GeoJSON renderer, using a stream response.
 
     The complex encoding bits are handled by the C-library "orjson"
     and the geojson property of GEOSGeometry.
 
-    NOTE: While Django has a GeoJSON serializer
+    While Django has a GeoJSON serializer
     (see https://docs.djangoproject.com/en/3.0/ref/contrib/gis/serializers/),
     it does not offer streaming response handling.
     """
@@ -41,29 +42,26 @@ class GeoJsonRenderer(OutputRenderer):
     max_page_size = conf.GISSERVER_GEOJSON_MAX_PAGE_SIZE
     chunk_size = 40_000
 
-    @classmethod
     def decorate_queryset(
-        cls,
+        self,
         projection: FeatureProjection,
         queryset: models.QuerySet,
-        output_crs: CRS,
-        **params,
     ):
         """Redefine which fields to query, always include geometry, but remove all others"""
-        main_geo_element = projection.feature_type.main_geometry_element
-        projection.add_field(main_geo_element)  # make sure geometry is always queried
-        queryset = super().decorate_queryset(projection, queryset, output_crs, **params)
+        # make sure output CRS matches the coordinate ordering that GEOSGeometry.json returns
+        if projection.output_crs == WGS84:
+            projection.output_crs = CRS84
 
+        # Make sure geometry is always queried.
         # Other geometries can be excluded as these are not rendered by 'properties'
-        other_geometries = [
-            gml_element.orm_path
-            for gml_element in projection.geometry_elements
-            if gml_element is not main_geo_element
-        ]
-        if other_geometries:
-            queryset = queryset.defer(*other_geometries)
+        main_geo_element = projection.feature_type.main_geometry_element
+        projection.add_field(main_geo_element)
+        projection.remove_fields(
+            lambda element: element.type.is_geometry and element is not main_geo_element
+        )
 
-        return queryset
+        # Apply the normal optimizations with this altered projection
+        return super().decorate_queryset(projection, queryset)
 
     def render_stream(self):
         self.output = output = BytesIO()
@@ -87,7 +85,7 @@ class GeoJsonRenderer(OutputRenderer):
                 output.write(b",\n")
 
             is_first = True
-            for instance in sub_collection:
+            for instance in self.read_features(sub_collection):
                 if is_first:
                     is_first = False
                 else:
@@ -117,7 +115,7 @@ class GeoJsonRenderer(OutputRenderer):
     def render_exception(self, exception: Exception):
         """Render the exception in a format that fits with the output."""
         message = super().render_exception(exception)
-        buffer = self.output.getvalue()
+        buffer = self.output.getvalue().decode()
         return f"{buffer}/* {message} */\n"
 
     def render_feature(self, projection: FeatureProjection, instance: models.Model) -> bytes:
@@ -137,7 +135,7 @@ class GeoJsonRenderer(OutputRenderer):
         return b'    {"type":"Feature","id":%b,%b"geometry":%b,"properties":%b}' % (
             orjson.dumps(f"{feature_type.name}.{instance.pk}"),
             json_geometry_name,
-            self.render_geometry(feature_type, instance),
+            self.render_geometry(projection, instance),
             orjson.dumps(properties, default=_json_default),
         )
 
@@ -150,15 +148,17 @@ class GeoJsonRenderer(OutputRenderer):
         else:
             return value
 
-    def render_geometry(self, feature_type, instance: models.Model) -> bytes:
+    def render_geometry(self, projection: FeatureProjection, instance: models.Model) -> bytes:
         """Generate the proper GeoJSON notation for a geometry.
         This calls the GDAL C-API rendering found in 'GEOSGeometry.json'
         """
-        geometry = getattr(instance, feature_type.geometry_field.name)
+        geometry = projection.get_main_geometry_value(instance)
         if geometry is None:
             return b"null"
 
-        self.output_crs.apply_to(geometry)
+        # The GeoJSON spec requires coordinates to be x,y (longitude,latitude),
+        # so web-based clients don't have to ship projection tables.
+        projection.output_crs.apply_to(geometry, axis_order=AxisOrder.TRADITIONAL)
         return geometry.json.encode()
 
     def get_header(self) -> dict:
@@ -167,11 +167,12 @@ class GeoJsonRenderer(OutputRenderer):
         The format is based on the WFS 3.0 DRAFT. The count fields are moved
         to the footer allowing them to be calculated without performing queries.
         """
+        output_crs = self.collection.results[0].projection.output_crs
         return {
             "type": "FeatureCollection",
             "timeStamp": self._format_geojson_value(self.collection.timestamp),
             # "numberReturned": is written at the end for better query performance.
-            "crs": {"type": "name", "properties": {"name": str(self.output_crs)}},
+            "crs": {"type": "name", "properties": {"name": str(output_crs)}},
         }
 
     def get_footer(self) -> dict:
@@ -227,7 +228,7 @@ class GeoJsonRenderer(OutputRenderer):
         """
         props = {}
         for xsd_element in xsd_elements:
-            if not xsd_element.is_geometry:
+            if not xsd_element.type.is_geometry:
                 value = xsd_element.get_value(instance)
                 if xsd_element.type.is_complex_type:
                     # Nested object data
@@ -258,30 +259,29 @@ class DBGeoJsonRenderer(GeoJsonRenderer):
     This is even more efficient than calling the C-API for each feature.
     """
 
-    @classmethod
-    def decorate_queryset(self, projection: FeatureProjection, queryset, output_crs, **params):
+    def decorate_queryset(self, projection: FeatureProjection, queryset):
         """Update the queryset to let the database render the GML output.
         This is far more efficient than GeoDjango's logic, which performs a
         C-API call for every single coordinate of a geometry.
         """
-        queryset = super().decorate_queryset(projection, queryset, output_crs, **params)
+        queryset = super().decorate_queryset(projection, queryset)
         # If desired, the entire FeatureCollection could be rendered
         # in PostgreSQL as well: https://postgis.net/docs/ST_AsGeoJSON.html
-        main_geometry_field = projection.feature_type.main_geometry_element
-        if main_geometry_field is not None:
-            queryset = queryset.defer(main_geometry_field.orm_path).annotate(
+        main_geo_element = projection.feature_type.main_geometry_element
+        if main_geo_element is not None:
+            queryset = queryset.defer(main_geo_element.orm_path).annotate(
                 _as_db_geojson=AsGeoJSON(
-                    get_db_geometry_target(main_geometry_field, output_crs),
+                    get_db_geometry_target(main_geo_element, projection.output_crs),
                     precision=conf.GISSERVER_DB_PRECISION,
                 )
             )
 
         return queryset
 
-    def render_geometry(self, feature_type, instance: models.Model) -> bytes:
+    def render_geometry(self, projection: FeatureProjection, instance: models.Model) -> bytes:
         """Generate the proper GeoJSON notation for a geometry"""
         # Database server rendering
-        if not feature_type.geometry_fields:
+        if projection.main_geometry_element is None:
             return b"null"
 
         geojson = instance._as_db_geojson