django-gisserver 2.0__py3-none-any.whl → 2.1.1__py3-none-any.whl

gisserver/management/commands/loadgeojson.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import operator
+from argparse import ArgumentTypeError
 from functools import reduce
 from itertools import islice
 from urllib.request import urlopen
@@ -15,22 +16,22 @@ from django.core.exceptions import FieldDoesNotExist
 from django.core.management import BaseCommand, CommandError, CommandParser
 from django.db import DEFAULT_DB_ALIAS, connections, models, transaction
 
-from gisserver.geometries import CRS, WGS84
+from gisserver.crs import CRS, CRS84
 
 
 def _parse_model(value):
     try:
         return apps.get_model(value)
     except LookupError as e:
-        raise ValueError(str(e)) from e
+        raise ArgumentTypeError(str(e)) from e
 
 
 def _parse_fields(value):
     field_map = {}
     for pair in value.split(","):
         geojson_name, _, field_name = pair.strip().partition("=")
-        if not value:
-            raise ValueError("Expect property=field,property2=field2 format")
+        if not field_name:
+            raise ArgumentTypeError("Expect property=field,property2=field2 format")
         field_map[geojson_name] = field_name
     return field_map
 
@@ -72,7 +73,7 @@ class Command(BaseCommand):
         parser.add_argument(
             "-f",
             "--field",
-            nargs="*",
+            action="append",
             dest="map_fields",
             type=_parse_fields,
             metavar="NAME=FIELD",
@@ -93,9 +94,7 @@ class Command(BaseCommand):
         self.connection = connections[self.using]
         model: type[models.Model] = options["model"]
         main_geometry_field = self._get_geometry_field(model, options["geometry_field"])
-        field_map = (
-            self._parse_field_map(model, options["map_fields"]) if options["map_fields"] else {}
-        )
+        field_map = self._parse_field_map(model, options["map_fields"])
 
         # Read the file
         geojson = self._load_geojson(options["geojson-file"])
@@ -188,7 +187,7 @@ class Command(BaseCommand):
         """Find the CRS that should be used for all geometry data."""
         crs = geojson.get("crs")
         if not crs:
-            return WGS84
+            return CRS84  # default for GeoJSON
 
         try:
             type = crs["type"]
@@ -202,6 +201,9 @@ class Command(BaseCommand):
 
     def _parse_field_map(self, model, map_fields: list[dict]) -> dict:
         """Validate that the provided field mapping points to model fields."""
+        if not map_fields:
+            return {}
+
         field_map: dict = reduce(operator.or_, map_fields)
         for field_name in field_map.values():
             try:
@@ -262,27 +264,45 @@ class Command(BaseCommand):
             }
 
             # Store the geometry, note GEOS only parses stringified JSON.
-            geometry = GEOSGeometry(json.dumps(geometry_data), srid=self.crs.srid)
-            geometry.srid = self.crs.srid  # override default
+            if geometry_data is None:
+                geometry = None
+            else:
+                try:
+                    # NOTE: this looses the axis ordering information,
+                    # and assumes x/y as the standard prescribes.
+                    geometry = GEOSGeometry(json.dumps(geometry_data), srid=self.crs.srid)
+                except ValueError as e:
+                    raise CommandError(
+                        f"Unable to parse geometry data: {geometry_data!r}: {e}"
+                    ) from e
+                geometry.srid = self.crs.srid  # override default
+
             field_values[main_geometry_field] = geometry
 
             # Try to decode the identifier if it's present
-            if pk_field not in field_values and (id_value := feature.get("id")):
-                try:
-                    field_values[pk_field] = self._parse_id(model._meta.pk, id_value)
-                except (ValueError, TypeError):
-                    self.stderr.write(
-                        self.style.WARNING(
-                            f"Feature id value '{id_value}' can't be stored in the primary key field. ignoring."
-                        )
-                    )
+            if (
+                pk_field not in field_values
+                and (raw_value := feature.get("id")) is not None
+                and (id_value := self._parse_id(model._meta.pk, raw_value)) is not None
+            ):
+                field_values[pk_field] = id_value
 
             # Pass as Django model fields
             yield field_values
 
     def _parse_id(self, pk_field: models.Field, id_value):
         # Allow TypeName.id format, see if it parses
-        return pk_field.get_db_prep_save(id_value.rpartition(".")[2], connection=self.connection)
+        try:
+            return pk_field.get_db_prep_save(
+                id_value.rpartition(".")[2], connection=self.connection
+            )
+        except (ValueError, TypeError):
+            self.stderr.write(
+                self.style.WARNING(
+                    f"Feature id value '{id_value}' can't be stored in the primary key field. ignoring."
+                )
+            )
+            return None
 
     def _parse_value(self, field: models.Field, value):
         try:

gisserver/operations/base.py

@@ -50,7 +50,9 @@ __all__ = (
 class Parameter:
     """The ``<ows:Parameter>`` tag to output in ``GetCapabilities``."""
 
+    #: The name of the parameter
     name: str
+    #: The values to report in ``<ows:AllowedValues><ows:Value>``.
     allowed_values: list[str]
 
 
@@ -58,10 +60,13 @@ class OutputFormat(typing.Generic[R]):
     """Declare an output format for the method.
 
     These formats are used in the :meth:`get_output_formats` for
-    any WFSMethod that implements :class:`OutputFormatMixin`.
+    any :class:`WFSOperation` that implements :class:`OutputFormatMixin`.
     This also connects the output format with a :attr:`renderer_class`.
     """
 
+    #: The class that performs the output rendering.
+    renderer_class: type[R] | None = None
+
     def __init__(
         self,
         content_type,
@@ -161,7 +166,7 @@ class WFSOperation:
     @cached_property
     def all_feature_types_by_name(self) -> dict[str, FeatureType]:
         """Create a lookup for feature types by name.
-        This can be cached as the WFSMethod is instantiated with each request
+        This can be cached as the :class:`WFSOperation` is instantiated with each request.
         """
         return {ft.xml_name: ft for ft in self.view.get_bound_feature_types()}
 
@@ -194,12 +199,14 @@ class XmlTemplateMixin:
     """Mixin to support methods that render using a template."""
 
     #: Default template to use for rendering
+    #: This is resolved as :samp:`gisserver/{service}/{version}/{xml_template_name}`.
     xml_template_name = None
 
-    #: Default content-type for render_xml()
+    #: The content-type to render.
     xml_content_type = "text/xml; charset=utf-8"
 
     def process_request(self, ows_request: ows.BaseOwsRequest):
+        """Process the request by rendering a Django template."""
         context = self.get_context_data()
         return self.render_xml(context, ows_request)
 
@@ -208,10 +215,7 @@ class XmlTemplateMixin:
         return {}
 
     def render_xml(self, context, ows_request: ows.BaseOwsRequest):
-        """Shortcut to render XML.
-
-        This is the default method when the OutputFormat class doesn't have a renderer_class
-        """
+        """Render the response using a template."""
         return HttpResponse(
             render_to_string(
                 self._get_xml_template_name(ows_request),

gisserver/operations/wfs20.py

@@ -1,11 +1,14 @@
-"""Operations that implement the WFS 2.0 spec.
-
-Useful docs:
-* http://portal.opengeospatial.org/files/?artifact_id=39967
-* https://www.opengeospatial.org/standards/wfs#downloads
-* http://schemas.opengis.net/wfs/2.0/
-* https://mapserver.org/development/rfc/ms-rfc-105.html
-* https://enonline.supermap.com/iExpress9D/API/WFS/WFS200/WFS_2.0.0_introduction.htm
+"""Operations that implement the WFS 2.0 specification.
+
+These operations are called by the :class:`~gisserver.views.WFSView`,
+and receive the request that was parsed using :mod:`gisserver.parsers.wfs20`.
+Thus, there is nearly no difference between GET/POST requests here.
+
+In a way this looks like an MVC (Model-View-Controller) design:
+
+* :mod:`gisserver.parsers.wfs20` parsed the request, and provides the "model".
+* :mod:`gisserver.operations.wfs20` orchestrates here what to do (the controller).
+* :mod:`gisserver.output` performs the rendering (the view).
 """
 
 from __future__ import annotations
@@ -16,18 +19,14 @@ import re
 import typing
 from urllib.parse import urlencode
 
-from django.core.exceptions import FieldError, ImproperlyConfigured, ValidationError
-from django.db import InternalError, ProgrammingError
-from django.db.models import QuerySet
+from django.core.exceptions import ImproperlyConfigured
 from django.utils.module_loading import import_string
 
 from gisserver import conf, output
 from gisserver.exceptions import (
-    ExternalParsingError,
-    ExternalValueError,
     InvalidParameterValue,
-    OperationParsingFailed,
     VersionNegotiationFailed,
+    wrap_filter_errors,
 )
 from gisserver.extensions.functions import function_registry
 from gisserver.extensions.queries import stored_query_registry
@@ -42,6 +41,16 @@ logger = logging.getLogger(__name__)
 SAFE_VERSION = re.compile(r"\A[0-9.]+\Z")
 RE_SAFE_FILENAME = re.compile(r"\A[A-Za-z0-9]+[A-Za-z0-9.]*")  # no dot at the start.
 
+__all__ = [
+    "GetCapabilities",
+    "DescribeFeatureType",
+    "BaseWFSGetDataOperation",
+    "GetFeature",
+    "GetPropertyValue",
+    "ListStoredQueries",
+    "DescribeStoredQueries",
+]
+
 
 class GetCapabilities(XmlTemplateMixin, OutputFormatMixin, WFSOperation):
     """This operation returns map features, and available operations this WFS server supports."""
@@ -256,6 +265,8 @@ class BaseWFSGetDataOperation(OutputFormatMixin, WFSOperation):
         else:
             raise NotImplementedError()
 
+        # assert False, str(collection.results[0].queryset.query)
+
         # Initialize the renderer.
         # This can also decorate the querysets with projection information,
         # such as converting geometries to the correct CRS or add prefetch_related logic.
@@ -275,7 +286,9 @@ class BaseWFSGetDataOperation(OutputFormatMixin, WFSOperation):
         start, count = self.get_pagination()
         results = []
         for query in self.ows_request.queries:
-            queryset = self._get_queryset(query)
+            with wrap_filter_errors(query):
+                queryset = query.get_queryset()
+
             results.append(
                 output.SimpleFeatureCollection(
                     source_query=query,
@@ -300,7 +313,9 @@ class BaseWFSGetDataOperation(OutputFormatMixin, WFSOperation):
         results = []
         for query in self.ows_request.queries:
             # The querysets are not executed yet, until the output is reading them.
-            queryset = self._get_queryset(query)
+            with wrap_filter_errors(query):
+                queryset = query.get_queryset()
+
             results.append(
                 output.SimpleFeatureCollection(
                     source_query=query,
@@ -375,83 +390,6 @@ class BaseWFSGetDataOperation(OutputFormatMixin, WFSOperation):
         new_params.update(updates)
         return f"{self.view.server_url}?{urlencode(new_params)}"
 
-    def _get_queryset(self, query: wfs20.QueryExpression) -> QuerySet:  # noqa:C901
-        """Generate the queryset, and trap many parser errors in the making of it."""
-        try:
-            return query.get_queryset()
-        except ExternalParsingError as e:
-            # Bad input data
-            self._log_filter_error(query, logging.ERROR, e)
-            raise OperationParsingFailed(str(e), locator=query.query_locator) from e
-        except ExternalValueError as e:
-            # Bad input data
-            self._log_filter_error(query, logging.ERROR, e)
-            raise InvalidParameterValue(str(e), locator=query.query_locator) from e
-        except ValidationError as e:
-            # Bad input data
-            self._log_filter_error(query, logging.ERROR, e)
-            raise OperationParsingFailed(
-                "\n".join(map(str, e.messages)),
-                locator=query.query_locator,
-            ) from e
-        except FieldError as e:
-            # e.g. doing a LIKE on a foreign key, or requesting an unknown field.
-            if not conf.GISSERVER_WRAP_FILTER_DB_ERRORS:
-                raise
-            self._log_filter_error(query, logging.ERROR, e)
-            raise InvalidParameterValue(
-                "Internal error when processing filter",
-                locator=query.query_locator,
-            ) from e
-        except (InternalError, ProgrammingError) as e:
-            # e.g. comparing datetime against integer
-            if not conf.GISSERVER_WRAP_FILTER_DB_ERRORS:
-                raise
-            logger.exception("WFS request failed: %s\nRequest: %r", str(e), self.ows_request)
-            msg = str(e)
-            locator = "srsName" if "Cannot find SRID" in msg else query.query_locator
-            raise InvalidParameterValue(f"Invalid request: {msg}", locator=locator) from e
-        except (TypeError, ValueError) as e:
-            # TypeError/ValueError could reference a datatype mismatch in an
-            # ORM query, but it could also be an internal bug. In most cases,
-            # this is already caught by XsdElement.validate_comparison().
-            if self._is_orm_error(e):
-                if query.query_locator == "STOREDQUERY_ID":
-                    # This is a fallback, ideally the stored query performs its own validation.
-                    raise InvalidParameterValue(
-                        f"Invalid stored query parameter: {e}", locator=query.query_locator
-                    ) from e
-                else:
-                    raise InvalidParameterValue(
-                        f"Invalid filter query: {e}", locator=query.query_locator
-                    ) from e
-            raise
-
-    def _is_orm_error(self, exception: Exception):
-        traceback = exception.__traceback__
-        while traceback.tb_next is not None:
-            traceback = traceback.tb_next
-            if "/django/db/models/query" in traceback.tb_frame.f_code.co_filename:
-                return True
-        return False
-
-    def _log_filter_error(self, query, level, exc):
-        """Report a filtering parsing error in the logging"""
-        filter = getattr(query, "filter", None)  # AdhocQuery only
-        fes_xml = filter.source if filter is not None else "(not provided)"
-        try:
-            sql = exc.__cause__.cursor.query.decode()
-        except AttributeError:
-            logger.log(level, "WFS query failed: %s\nFilter:\n%s", exc, fes_xml)
-        else:
-            logger.log(
-                level,
-                "WFS query failed: %s\nSQL Query: %s\n\nFilter:\n%s",
-                exc,
-                sql,
-                fes_xml,
-            )
-
 
 class GetFeature(BaseWFSGetDataOperation):
     """This returns all properties of the feature.

gisserver/output/__init__.py

@@ -1,8 +1,11 @@
-"""All supported output formats"""
+"""The output rendering classes for all response types.
+
+This also includes collection classes for iterating over the Django QuerySet results.
+"""
 
 from .results import FeatureCollection, SimpleFeatureCollection  # isort: skip (fixes import loops)
 
-from .base import CollectionOutputRenderer, OutputRenderer
+from .base import CollectionOutputRenderer, OutputRenderer, XmlOutputRenderer
 from .csv import CSVRenderer, DBCSVRenderer
 from .geojson import DBGeoJsonRenderer, GeoJsonRenderer
 from .gml32 import (
@@ -28,6 +31,7 @@ __all__ = [
     "GeoJsonRenderer",
     "GML32Renderer",
     "GML32ValueRenderer",
+    "XmlOutputRenderer",
     "XmlSchemaRenderer",
     "ListStoredQueriesRenderer",
     "DescribeStoredQueriesRenderer",

gisserver/output/base.py

@@ -3,13 +3,16 @@ from __future__ import annotations
 import logging
 import math
 import typing
+from collections.abc import Iterator
 from io import BytesIO, StringIO
+from itertools import chain
 
 from django.conf import settings
 from django.db import models
 from django.http import HttpResponse, StreamingHttpResponse
 from django.http.response import HttpResponseBase  # Django 3.2 import location
 
+from gisserver.exceptions import wrap_filter_errors
 from gisserver.features import FeatureType
 from gisserver.parsers.values import fix_type_name
 from gisserver.parsers.xml import split_ns
@@ -23,15 +26,15 @@ if typing.TYPE_CHECKING:
     from gisserver.operations.base import WFSOperation
     from gisserver.projection import FeatureProjection, FeatureRelation
 
-    from .results import FeatureCollection
+    from .results import FeatureCollection, SimpleFeatureCollection
 
 
 class OutputRenderer:
     """Base class for rendering content.
 
     Note most rendering logic will generally
-    need to use :class:`XmlOutputRenderer` or :class:`ConnectionOutputRenderer`
-    as their base class
+    need to use :class:`~gisserver.output.XmlOutputRenderer`
+    or :class:`~gisserver.output.CollectionOutputRenderer` as their base class
     """
 
     #: Default content type for the HTTP response
@@ -53,15 +56,23 @@ class OutputRenderer:
             )
         else:
             # An actual generator.
+            # Peek the generator so initial exceptions can still be handled,
+            # and get rendered as normal HTTP responses with the proper status.
+            try:
+                start = next(stream)  # peek, so any raised OWSException here is handled by OWSView
+                stream = chain([start], self._trap_exceptions(stream))
+            except StopIteration:
+                pass
+
             # Handover to WSGI server (starts streaming when reading the contents)
-            stream = self._trap_exceptions(stream)
             return StreamingHttpResponse(
                 streaming_content=stream,
                 content_type=self.content_type,
                 headers=self.get_headers(),
             )
 
-    def get_headers(self):
+    def get_headers(self) -> dict[str, str]:
+        """Override to define HTTP headers to add."""
         return {}
 
     def _trap_exceptions(self, stream):
@@ -136,10 +147,7 @@ class XmlOutputRenderer(OutputRenderer):
 
 
 class CollectionOutputRenderer(OutputRenderer):
-    """Base class to create streaming responses.
-
-    It receives the collected 'context' data of the WFSMethod.
-    """
+    """Base class to create streaming responses."""
 
     #: Allow to override the maximum page size.
     #: This value can be 'math.inf' to support endless pages by default.
@@ -152,7 +160,7 @@ class CollectionOutputRenderer(OutputRenderer):
         """
         Receive the collected data to render.
 
-        :param operation: The calling WFS Method (e.g. GetFeature class)
+        :param operation: The calling WFS Operation (e.g. GetFeature class)
         :param collection: The collected data for rendering.
         """
         super().__init__(operation)
@@ -160,7 +168,10 @@ class CollectionOutputRenderer(OutputRenderer):
         self.apply_projection()
 
     def apply_projection(self):
-        """Perform presentation-layer logic enhancements on the queryset."""
+        """Perform presentation-layer logic enhancements on all results.
+        This calls :meth:`decorate_queryset` for
+        each :class:`~gisserver.output.SimpleFeatureCollection`.
+        """
         for sub_collection in self.collection.results:
             queryset = self.decorate_queryset(
                 projection=sub_collection.projection,
@@ -180,9 +191,8 @@ class CollectionOutputRenderer(OutputRenderer):
 
         This allows fine-tuning the queryset for any special needs of the output rendering type.
 
-        :param feature_type: The feature that is being queried.
+        :param projection: The projection information, including feature that is being rendered.
         :param queryset: The constructed queryset so far.
-        :param output_crs: The projected output
         """
         if queryset._result_cache is not None:
             raise RuntimeError(
@@ -266,3 +276,8 @@ class CollectionOutputRenderer(OutputRenderer):
             "date": self.collection.date.strftime("%Y-%m-%d %H.%M.%S%z"),
             "timestamp": self.collection.timestamp,
         }
+
+    def read_features(self, sub_collection: SimpleFeatureCollection) -> Iterator[models.Model]:
+        """A wrapper to read features from a collection, while raising WFS exceptions on query errors."""
+        with wrap_filter_errors(sub_collection.source_query):
+            yield from sub_collection

gisserver/output/csv.py

@@ -40,7 +40,7 @@ class CSVRenderer(CollectionOutputRenderer):
         # First, make sure no array or m2m elements exist,
         # as these are not possible to render in CSV.
         projection.remove_fields(
-            lambda e: e.is_many
+            lambda e: (e.is_many and not e.is_array)
             or e.type == XsdTypes.gmlCodeType  # gml:name
             or e.type == XsdTypes.gmlBoundingShapeType  # gml:boundedBy
         )
@@ -66,7 +66,7 @@ class CSVRenderer(CollectionOutputRenderer):
             writer.writerow(self.get_header(projection, xsd_elements))
 
             # Write all rows
-            for instance in sub_collection:
+            for instance in self.read_features(sub_collection):
                 writer.writerow(self.get_row(instance, projection, xsd_elements))
 
                 # Only perform a 'yield' every once in a while,
@@ -113,7 +113,7 @@ class CSVRenderer(CollectionOutputRenderer):
         append = values.append
         for xsd_element in xsd_elements:
             if xsd_element.type.is_geometry:
-                append(self.render_geometry(instance, xsd_element))
+                append(self.render_geometry(projection, instance, xsd_element))
                 continue
 
             value = xsd_element.get_value(instance)
@@ -134,9 +134,16 @@ class CSVRenderer(CollectionOutputRenderer):
                 append(value)
         return values
 
-    def render_geometry(self, instance: models.Model, geo_element: GeometryXsdElement):
+    def render_geometry(
+        self,
+        projection: FeatureProjection,
+        instance: models.Model,
+        geo_element: GeometryXsdElement,
+    ) -> str:
         """Render the contents of a geometry value."""
-        return geo_element.get_value(instance)
+        geometry = geo_element.get_value(instance)
+        projection.output_crs.apply_to(geometry)
+        return geometry.ewkt
 
 
 class DBCSVRenderer(CSVRenderer):
@@ -167,6 +174,11 @@ class DBCSVRenderer(CSVRenderer):
             queryset, feature_relation.geometry_elements, projection.output_crs, AsEWKT
         )
 
-    def render_geometry(self, instance: models.Model, geo_element: GeometryXsdElement):
+    def render_geometry(
+        self,
+        projection: FeatureProjection,
+        instance: models.Model,
+        geo_element: GeometryXsdElement,
+    ):
         """Render the geometry using a database-rendered version."""
         return get_db_rendered_geometry(instance, geo_element, AsEWKT)

gisserver/output/geojson.py

@@ -6,12 +6,13 @@ from io import BytesIO
 
 import orjson
 from django.contrib.gis.db.models.functions import AsGeoJSON
+from django.contrib.gis.gdal import AxisOrder
 from django.db import models
 from django.utils.functional import Promise
 
 from gisserver import conf
+from gisserver.crs import CRS84, WGS84
 from gisserver.db import get_db_geometry_target
-from gisserver.geometries import CRS84, WGS84
 from gisserver.projection import FeatureProjection
 from gisserver.types import XsdElement
 
@@ -31,7 +32,7 @@ class GeoJsonRenderer(CollectionOutputRenderer):
     The complex encoding bits are handled by the C-library "orjson"
     and the geojson property of GEOSGeometry.
 
-    NOTE: While Django has a GeoJSON serializer
+    While Django has a GeoJSON serializer
     (see https://docs.djangoproject.com/en/3.0/ref/contrib/gis/serializers/),
     it does not offer streaming response handling.
     """
@@ -84,7 +85,7 @@ class GeoJsonRenderer(CollectionOutputRenderer):
                 output.write(b",\n")
 
             is_first = True
-            for instance in sub_collection:
+            for instance in self.read_features(sub_collection):
                 if is_first:
                     is_first = False
                 else:
@@ -155,9 +156,9 @@ class GeoJsonRenderer(CollectionOutputRenderer):
         if geometry is None:
             return b"null"
 
-        # The .json property always outputs coordinates as x,y (longitude,latitude),
-        # which the GeoJSON spec requires.
-        projection.output_crs.apply_to(geometry)
+        # The GeoJSON spec requires coordinates to be x,y (longitude,latitude),
+        # so web-based clients don't have to ship projection tables.
+        projection.output_crs.apply_to(geometry, axis_order=AxisOrder.TRADITIONAL)
         return geometry.json.encode()
 
     def get_header(self) -> dict: