django-gisserver 1.4.1__py3-none-any.whl → 2.0__py3-none-any.whl

gisserver/output/csv.py

@@ -6,24 +6,17 @@ import csv
 from datetime import datetime, timezone
 from io import StringIO
 
-from django.conf import settings
 from django.db import models
 
 from gisserver import conf
-from gisserver.db import (
-    AsEWKT,
-    build_db_annotations,
-    get_db_annotation,
-    get_db_geometry_selects,
-)
-from gisserver.features import FeatureType
-from gisserver.geometries import CRS
-from gisserver.types import XsdComplexType, XsdElement
+from gisserver.db import AsEWKT, get_db_rendered_geometry, replace_queryset_geometries
+from gisserver.projection import FeatureProjection, FeatureRelation
+from gisserver.types import GeometryXsdElement, XsdElement, XsdTypes
 
-from .base import OutputRenderer
+from .base import CollectionOutputRenderer
 
 
-class CSVRenderer(OutputRenderer):
+class CSVRenderer(CollectionOutputRenderer):
     """Fast CSV renderer, using a stream response.
 
     The complex encoding bits are handled by the "csv" library.
@@ -38,41 +31,30 @@ class CSVRenderer(OutputRenderer):
     #: or one of the registered names like: "unix", "excel", "excel-tab"
     dialect = "unix"
 
-    @classmethod
     def decorate_queryset(
-        cls,
-        feature_type: FeatureType,
-        queryset: models.QuerySet,
-        output_crs: CRS,
-        **params,
+        self, projection: FeatureProjection, queryset: models.QuerySet
     ) -> models.QuerySet:
         """Make sure relations are included with select-related to avoid N-queries.
         Using prefetch_related() isn't possible with .iterator().
         """
-        xsd_type: XsdComplexType = feature_type.xsd_type
-
-        # Take all relations that are expanded to complex elements,
-        # and all relations that are fetched for flattened elements.
-        related = {
-            xsd_element.orm_path
-            for xsd_element in xsd_type.complex_elements
-            if not xsd_element.is_many
-        } | {
-            xsd_element.orm_relation[0]
-            for xsd_element in xsd_type.flattened_elements
-            if not xsd_element.is_many
-        }
-        if related:
-            queryset = queryset.select_related(*related)
-
-        return queryset
+        # First, make sure no array or m2m elements exist,
+        # as these are not possible to render in CSV.
+        projection.remove_fields(
+            lambda e: e.is_many
+            or e.type == XsdTypes.gmlCodeType  # gml:name
+            or e.type == XsdTypes.gmlBoundingShapeType  # gml:boundedBy
+        )
+
+        # All database optimizations
+        return super().decorate_queryset(projection, queryset)
 
     def render_stream(self):
-        output = StringIO()
+        self.output = output = StringIO()
         writer = csv.writer(output, dialect=self.dialect)
 
         is_first_collection = True
         for sub_collection in self.collection.results:
+            projection = sub_collection.projection
             if is_first_collection:
                 is_first_collection = False
             else:
@@ -80,14 +62,12 @@ class CSVRenderer(OutputRenderer):
                 output.write("\n\n")
 
             # Write the header
-            fields = [f for f in sub_collection.feature_type.xsd_type.elements if not f.is_many]
-            writer.writerow(self.get_header(fields))
+            xsd_elements = projection.xsd_root_elements
+            writer.writerow(self.get_header(projection, xsd_elements))
 
-            # By using .iterator(), the results are streamed with as little memory as
-            # possible. Doing prefetch_related() is not possible now. That could only
-            # be implemented with cursor pagination for large sets for 1000+ results.
-            for instance in sub_collection.iterator():
-                writer.writerow(self.get_row(instance, fields))
+            # Write all rows
+            for instance in sub_collection:
+                writer.writerow(self.get_row(instance, projection, xsd_elements))
 
                 # Only perform a 'yield' every once in a while,
                 # as it goes back-and-forth for writing it to the client.
@@ -101,38 +81,50 @@ class CSVRenderer(OutputRenderer):
 
     def render_exception(self, exception: Exception):
         """Render the exception in a format that fits with the output."""
-        if settings.DEBUG:
-            return f"\n\n{exception.__class__.__name__}: {exception}\n"
-        else:
-            return f"\n\n{exception.__class__.__name__} during rendering!\n"
+        message = super().render_exception(exception)
+        buffer = self.output.getvalue()
+        return f"{buffer}\n\n{message}\n"
 
-    def get_header(self, fields: list[XsdElement]) -> list[str]:
+    def get_header(
+        self, projection: FeatureProjection, xsd_elements: list[XsdElement], prefix=""
+    ) -> list[str]:
         """Return all field names."""
         names = []
         append = names.append
-        for field in fields:
-            if field.type.is_complex_type:
-                # Expand complex types
-                for sub_field in field.type.elements:
-                    append(f"{field.name}.{sub_field.name}")
+        for xsd_element in xsd_elements:
+            if xsd_element.type.is_complex_type:
+                names.extend(
+                    self.get_header(
+                        projection,
+                        xsd_elements=projection.xsd_child_nodes[xsd_element],
+                        prefix=f"{prefix}{xsd_element.name}.",
+                    )
+                )
             else:
-                append(field.name)
+                append(f"{prefix}{xsd_element.name}")
 
         return names
 
-    def get_row(self, instance: models.Model, fields: list[XsdElement]):
+    def get_row(
+        self, instance: models.Model, projection: FeatureProjection, xsd_elements: list[XsdElement]
+    ):
         """Return all field values for a single row."""
         values = []
         append = values.append
-        for field in fields:
-            if field.is_geometry:
-                append(self.render_geometry(instance, field))
+        for xsd_element in xsd_elements:
+            if xsd_element.type.is_geometry:
+                append(self.render_geometry(instance, xsd_element))
                 continue
 
-            value = field.get_value(instance)
-            if field.type.is_complex_type:
-                for sub_field in field.type.elements:
-                    append(sub_field.get_value(value))
+            value = xsd_element.get_value(instance)
+            if xsd_element.type.is_complex_type:
+                values.extend(
+                    self.get_row(
+                        instance=value,
+                        projection=projection,
+                        xsd_elements=projection.xsd_child_nodes[xsd_element],
+                    )
+                )
             elif isinstance(value, list):
                 # Array field
                 append(",".join(map(str, value)))
@@ -142,35 +134,39 @@ class CSVRenderer(OutputRenderer):
                 append(value)
         return values
 
-    def render_geometry(self, instance: models.Model, field: XsdElement):
+    def render_geometry(self, instance: models.Model, geo_element: GeometryXsdElement):
         """Render the contents of a geometry value."""
-        return field.get_value(instance)
+        return geo_element.get_value(instance)
 
 
 class DBCSVRenderer(CSVRenderer):
     """Further optimized CSV renderer that uses the database to render EWKT.
-    This is about 40% faster then calling the GEOS C-API from python.
+    This is about 40% faster than calling the GEOS C-API from python.
     """
 
-    @classmethod
     def decorate_queryset(
-        cls,
-        feature_type: FeatureType,
-        queryset: models.QuerySet,
-        output_crs: CRS,
-        **params,
+        self, projection: FeatureProjection, queryset: models.QuerySet
     ) -> models.QuerySet:
-        queryset = super().decorate_queryset(feature_type, queryset, output_crs, **params)
-
-        # Instead of reading the binary geometry data,
-        # ask the database to generate EWKT data directly.
-        geo_selects = get_db_geometry_selects(feature_type.xsd_type.geometry_elements, output_crs)
-        if geo_selects:
-            queryset = queryset.defer(*geo_selects.keys()).annotate(
-                **build_db_annotations(geo_selects, "_as_ewkt_{name}", AsEWKT)
-            )
-
-        return queryset
-
-    def render_geometry(self, instance: models.Model, field: XsdElement):
-        return get_db_annotation(instance, field.name, "_as_ewkt_{name}")
+        # Instead of reading the binary geometry data, let the database generate EWKT data.
+        # As annotations can't be done for select_related() objects, prefetches are used instead.
+        queryset = super().decorate_queryset(projection, queryset)
+        return replace_queryset_geometries(
+            queryset, projection.geometry_elements, projection.output_crs, AsEWKT
+        )
+
+    def get_prefetch_queryset(
+        self, projection: FeatureProjection, feature_relation: FeatureRelation
+    ) -> models.QuerySet | None:
+        """Perform DB annotations for prefetched relations too."""
+        queryset = super().get_prefetch_queryset(projection, feature_relation)
+        if queryset is None:
+            return None
+
+        # Find which fields are GML elements, annotate these too.
+        return replace_queryset_geometries(
+            queryset, feature_relation.geometry_elements, projection.output_crs, AsEWKT
+        )
+
+    def render_geometry(self, instance: models.Model, geo_element: GeometryXsdElement):
+        """Render the geometry using a database-rendered version."""
+        return get_db_rendered_geometry(instance, geo_element, AsEWKT)

gisserver/output/geojson.py

@@ -3,21 +3,19 @@
 from datetime import datetime, timezone
 from decimal import Decimal
 from io import BytesIO
-from typing import cast
 
 import orjson
-from django.conf import settings
 from django.contrib.gis.db.models.functions import AsGeoJSON
 from django.db import models
 from django.utils.functional import Promise
 
 from gisserver import conf
 from gisserver.db import get_db_geometry_target
-from gisserver.features import FeatureType
-from gisserver.geometries import CRS
-from gisserver.types import XsdComplexType
+from gisserver.geometries import CRS84, WGS84
+from gisserver.projection import FeatureProjection
+from gisserver.types import XsdElement
 
-from .base import OutputRenderer
+from .base import CollectionOutputRenderer
 
 
 def _json_default(obj):
@@ -27,7 +25,7 @@ def _json_default(obj):
     raise TypeError(f"Unable to serialize {obj.__class__.__name__} to JSON")
 
 
-class GeoJsonRenderer(OutputRenderer):
+class GeoJsonRenderer(CollectionOutputRenderer):
     """Fast GeoJSON renderer, using a stream response.
 
     The complex encoding bits are handled by the C-library "orjson"
@@ -43,29 +41,29 @@ class GeoJsonRenderer(OutputRenderer):
     max_page_size = conf.GISSERVER_GEOJSON_MAX_PAGE_SIZE
     chunk_size = 40_000
 
-    @classmethod
     def decorate_queryset(
-        cls,
-        feature_type: FeatureType,
+        self,
+        projection: FeatureProjection,
         queryset: models.QuerySet,
-        output_crs: CRS,
-        **params,
     ):
-        queryset = super().decorate_queryset(feature_type, queryset, output_crs, **params)
+        """Redefine which fields to query, always include geometry, but remove all others"""
+        # make sure output CRS matches the coordinate ordering that GEOSGeometry.json returns
+        if projection.output_crs == WGS84:
+            projection.output_crs = CRS84
 
+        # Make sure geometry is always queried.
         # Other geometries can be excluded as these are not rendered by 'properties'
-        other_geometries = [
-            model_field.name
-            for model_field in feature_type.geometry_fields
-            if model_field is not feature_type.geometry_field
-        ]
-        if other_geometries:
-            queryset = queryset.defer(*other_geometries)
+        main_geo_element = projection.feature_type.main_geometry_element
+        projection.add_field(main_geo_element)
+        projection.remove_fields(
+            lambda element: element.type.is_geometry and element is not main_geo_element
+        )
 
-        return queryset
+        # Apply the normal optimizations with this altered projection
+        return super().decorate_queryset(projection, queryset)
 
     def render_stream(self):
-        output = BytesIO()
+        self.output = output = BytesIO()
 
         # Generate the header from a Python dict,
         # but replace the last "}" into a comma, to allow writing more
@@ -78,6 +76,8 @@ class GeoJsonRenderer(OutputRenderer):
         # Flatten the results, they are not grouped in a second FeatureCollection
         is_first_collection = True
         for sub_collection in self.collection.results:
+            projection = sub_collection.projection
+
             if is_first_collection:
                 is_first_collection = False
             else:
@@ -92,7 +92,7 @@ class GeoJsonRenderer(OutputRenderer):
 
                 # The "properties" object is generated by orjson.dumps(),
                 # while the "geometry" object uses the built-in 'GEOSGeometry.json' result.
-                output.write(self.render_feature(sub_collection.feature_type, instance))
+                output.write(self.render_feature(projection, instance))
 
                 # Only perform a 'yield' every once in a while,
                 # as it goes back-and-forth for writing it to the client.
@@ -113,35 +113,28 @@ class GeoJsonRenderer(OutputRenderer):
 
     def render_exception(self, exception: Exception):
         """Render the exception in a format that fits with the output."""
-        if settings.DEBUG:
-            return f"/* {exception.__class__.__name__}: {exception} */\n"
-        else:
-            return f"/* {exception.__class__.__name__} during rendering! */\n"
+        message = super().render_exception(exception)
+        buffer = self.output.getvalue().decode()
+        return f"{buffer}/* {message} */\n"
 
-    def render_feature(self, feature_type: FeatureType, instance: models.Model) -> bytes:
+    def render_feature(self, projection: FeatureProjection, instance: models.Model) -> bytes:
         """Render the output of a single feature"""
 
         # Get all instance attributes:
-        properties = self.get_properties(feature_type, feature_type.xsd_type, instance)
+        properties = self.get_properties(projection, projection.xsd_root_elements, instance)
+        feature_type = projection.feature_type
 
+        # Add the name field
         if feature_type.show_name_field:
             name = feature_type.get_display_value(instance)
-            geometry_name = b'"geometry_name":%b,' % orjson.dumps(name)
+            json_geometry_name = b'"geometry_name":%b,' % orjson.dumps(name)
         else:
-            geometry_name = b""
-
-        return (
-            b"    {"
-            b'"type":"Feature",'
-            b'"id":%b,'
-            b"%b"
-            b'"geometry":%b,'
-            b'"properties":%b'
-            b"}"
-        ) % (
+            json_geometry_name = b""
+
+        return b'    {"type":"Feature","id":%b,%b"geometry":%b,"properties":%b}' % (
             orjson.dumps(f"{feature_type.name}.{instance.pk}"),
-            geometry_name,
-            self.render_geometry(feature_type, instance),
+            json_geometry_name,
+            self.render_geometry(projection, instance),
             orjson.dumps(properties, default=_json_default),
         )
 
@@ -154,15 +147,17 @@ class GeoJsonRenderer(OutputRenderer):
         else:
             return value
 
-    def render_geometry(self, feature_type, instance: models.Model) -> bytes:
+    def render_geometry(self, projection: FeatureProjection, instance: models.Model) -> bytes:
         """Generate the proper GeoJSON notation for a geometry.
         This calls the GDAL C-API rendering found in 'GEOSGeometry.json'
         """
-        geometry = getattr(instance, feature_type.geometry_field.name)
+        geometry = projection.get_main_geometry_value(instance)
         if geometry is None:
             return b"null"
 
-        self.output_crs.apply_to(geometry)
+        # The .json property always outputs coordinates as x,y (longitude,latitude),
+        # which the GeoJSON spec requires.
+        projection.output_crs.apply_to(geometry)
         return geometry.json.encode()
 
     def get_header(self) -> dict:
@@ -171,11 +166,12 @@ class GeoJsonRenderer(OutputRenderer):
         The format is based on the WFS 3.0 DRAFT. The count fields are moved
         to the footer allowing them to be calculated without performing queries.
         """
+        output_crs = self.collection.results[0].projection.output_crs
         return {
             "type": "FeatureCollection",
             "timeStamp": self._format_geojson_value(self.collection.timestamp),
             # "numberReturned": is written at the end for better query performance.
-            "crs": {"type": "name", "properties": {"name": str(self.output_crs)}},
+            "crs": {"type": "name", "properties": {"name": str(output_crs)}},
         }
 
     def get_footer(self) -> dict:
@@ -218,40 +214,36 @@ class GeoJsonRenderer(OutputRenderer):
         return links
 
     def get_properties(
-        self,
-        feature_type: FeatureType,
-        xsd_type: XsdComplexType,
-        instance: models.Model,
+        self, projection: FeatureProjection, xsd_elements: list[XsdElement], instance: models.Model
     ) -> dict:
         """Collect the data for the 'properties' field.
 
         This is based on the original XSD definition,
         so the rendering is consistent with other output formats.
+
+        :param projection: Overview of all fields to render.
+        :param xsd_elements: The XSD elements to render.
+        :param instance: The object instance.
         """
         props = {}
-        for xsd_element in xsd_type.elements:
-            if not xsd_element.is_geometry:
+        for xsd_element in xsd_elements:
+            if not xsd_element.type.is_geometry:
                 value = xsd_element.get_value(instance)
                 if xsd_element.type.is_complex_type:
                     # Nested object data
                     if value is None:
                         props[xsd_element.name] = None
                     else:
-                        value_xsd_type = cast(XsdComplexType, xsd_element.type)
+                        sub_elements = projection.xsd_child_nodes[xsd_element]
                         if xsd_element.is_many:
-                            # If the retrieved QuerySet was not filtered yet, do so now.
-                            # This can't be done in get_value() because the FeatureType
-                            # is not known there.
-                            value = feature_type.filter_related_queryset(value)
-
                             # "..._to_many relation; reverse FK, M2M or array field.
                             props[xsd_element.name] = [
-                                self.get_properties(feature_type, value_xsd_type, item)
+                                self.get_properties(projection, sub_elements, item)
                                 for item in value
                             ]
                         else:
                             props[xsd_element.name] = self.get_properties(
-                                feature_type, value_xsd_type, value
+                                projection, sub_elements, value
                             )
                 else:
                     # Scalar value, or list (for ArrayField).
@@ -263,33 +255,32 @@ class GeoJsonRenderer(OutputRenderer):
 class DBGeoJsonRenderer(GeoJsonRenderer):
     """GeoJSON renderer that relays the geometry rendering to the database.
 
-    This is even more efficient then calling the C-API for each feature.
+    This is even more efficient than calling the C-API for each feature.
     """
 
-    @classmethod
-    def decorate_queryset(self, feature_type: FeatureType, queryset, output_crs, **params):
+    def decorate_queryset(self, projection: FeatureProjection, queryset):
         """Update the queryset to let the database render the GML output.
-        This is far more efficient then GeoDjango's logic, which performs a
+        This is far more efficient than GeoDjango's logic, which performs a
         C-API call for every single coordinate of a geometry.
         """
-        queryset = super().decorate_queryset(feature_type, queryset, output_crs, **params)
+        queryset = super().decorate_queryset(projection, queryset)
         # If desired, the entire FeatureCollection could be rendered
         # in PostgreSQL as well: https://postgis.net/docs/ST_AsGeoJSON.html
-        if feature_type.geometry_fields:
-            match = feature_type.resolve_element(feature_type.geometry_field.name)
-            queryset = queryset.defer(feature_type.geometry_field.name).annotate(
+        main_geo_element = projection.feature_type.main_geometry_element
+        if main_geo_element is not None:
+            queryset = queryset.defer(main_geo_element.orm_path).annotate(
                 _as_db_geojson=AsGeoJSON(
-                    get_db_geometry_target(match, output_crs),
+                    get_db_geometry_target(main_geo_element, projection.output_crs),
                     precision=conf.GISSERVER_DB_PRECISION,
                 )
             )
 
         return queryset
 
-    def render_geometry(self, feature_type, instance: models.Model) -> bytes:
+    def render_geometry(self, projection: FeatureProjection, instance: models.Model) -> bytes:
         """Generate the proper GeoJSON notation for a geometry"""
         # Database server rendering
-        if not feature_type.geometry_fields:
+        if projection.main_geometry_element is None:
             return b"null"
 
         geojson = instance._as_db_geojson