django-gisserver 1.5.0__py3-none-any.whl → 2.0__py3-none-any.whl

gisserver/parsers/ows/requests.py

@@ -0,0 +1,158 @@
+"""Request parsing for the common Generic Open Web Services (OWS) protocol bits."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+
+from django.http import QueryDict
+
+from gisserver.exceptions import (
+    ExternalParsingError,
+    InvalidParameterValue,
+    OperationNotSupported,
+    OperationParsingFailed,
+)
+from gisserver.parsers.ast import BaseNode, tag_registry
+from gisserver.parsers.xml import NSElement, parse_xml_from_string, split_ns, xmlns
+
+from .kvp import KVPRequest
+
+__all__ = (
+    "BaseOwsRequest",
+    "resolve_kvp_parser_class",
+    "resolve_xml_parser_class",
+    "parse_get_request",
+    "parse_post_request",
+)
+
+
+@dataclass
+class BaseOwsRequest(BaseNode):
+    """Base request data for all request types of the OWS standards.
+    This mirrors the ``<wfs:BaseRequestType>`` element from the WFS spec.
+    """
+
+    # dataclass limitation: when defaults are added, subclasses can't have required parameters anymore.
+    service: str
+    version: str | None  # none for GetCapabilities only
+    handle: str | None
+
+    @classmethod
+    def from_xml(cls, element: NSElement):
+        """Initialize from an XML POST request."""
+        return cls(**cls.base_xml_init_parameters(element))
+
+    @classmethod
+    def from_kvp_request(cls, kvp: KVPRequest):
+        """Initialize from an KVP GET request."""
+        return cls(**cls.base_kvp_init_parameters(kvp))
+
+    @classmethod
+    def base_xml_init_parameters(cls, element: NSElement) -> dict:
+        """Parse the base attributes.
+        This parses the syntax such as::
+
+            <wfs:BaseRequest service="WFS" version="2.0.0" handle="...">
+
+        """
+        return dict(
+            service=element.get_str_attribute("service"),
+            version=element.get_str_attribute("version"),
+            handle=element.attrib.get("handle"),
+        )
+
+    @classmethod
+    def base_kvp_init_parameters(cls, kvp: KVPRequest) -> dict:
+        """Parse the common Key-Value-Pair format (GET request parameters).
+        This parses the syntax::
+
+            ?SERVICE=WFS&VERSION=2.0.0
+        """
+        return dict(
+            service=kvp.get_str("SERVICE"),
+            version=kvp.get_str("VERSION"),
+            handle=None,
+        )
+
+    def as_kvp(self) -> dict:
+        """Translate the POST request into KVP GET parameters. This is needed for pagination."""
+        return {
+            "SERVICE": self.service,
+            "VERSION": str(self.version),
+            "REQUEST": split_ns(self.xml_tags[0])[1],
+        }
+
+
+@lru_cache
+def _get_kvp_parsers() -> dict[str, dict[str, type[BaseOwsRequest]]]:
+    """Find which classes are registered."""
+    # This late initialization keeps the ability open in the future
+    # to handle request types that are registered in third party code.
+    ows_parsers = {}
+
+    for xml_name, parser_class in tag_registry.find_subclasses(BaseOwsRequest).items():
+        namespace, local_name = split_ns(xml_name)
+        ows_parsers.setdefault(namespace, {})[local_name.upper()] = parser_class
+
+    return ows_parsers
+
+
+def resolve_kvp_parser_class(kvp: KVPRequest) -> type[BaseOwsRequest]:
+    """Find the appropriate class to parse the KVP GET request data."""
+    service = kvp.get_str("service").upper()
+    request = kvp.get_str("request")
+
+    # Find the appropriate request object
+    service_types = _get_kvp_parsers()
+    try:
+        # Translating the GET parameter to a namespace makes connecting to the parser registration.
+        # This doesn't take the VERSION/ACCEPTVERSIONS into account yet.
+        namespace = xmlns[service.lower()].value
+        request_classes = service_types[namespace]
+    except KeyError:
+        raise InvalidParameterValue(
+            f"Unsupported service type: {service}.", locator="service"
+        ) from None
+
+    try:
+        request_cls = request_classes[request.upper()]
+    except KeyError:
+        allowed = ", ".join(node.xml_tags[0] for node in request_classes.values())
+        raise OperationNotSupported(
+            f"'{request}' is not implemented, supported are: {allowed}.",
+            locator="request",
+        ) from None
+
+    return request_cls
+
+
+def resolve_xml_parser_class(root: NSElement) -> type[BaseOwsRequest]:
+    """Find the correct class to parse the XML POST data with."""
+    return tag_registry.resolve_class(root, allowed_types=(BaseOwsRequest,))
+
+
+def parse_get_request(
+    query_string: str | dict[str, str], ns_aliases: dict | None = None
+) -> BaseOwsRequest:
+    """Parse the WFS KVP GET request format into the internal request objects.
+    Most code calls the resolver internally, but this variation is easier for unit testing.
+    """
+    if isinstance(query_string, str):
+        query_string = QueryDict(query_string.lstrip("?"))
+
+    kvp = KVPRequest(query_string, ns_aliases=ns_aliases)
+    request_cls = resolve_kvp_parser_class(kvp)
+    return request_cls.from_kvp_request(kvp)
+
+
+def parse_post_request(xml_string: str | bytes, ns_aliases: dict | None = None) -> BaseOwsRequest:
+    """Parse the XML POST request format into the internal request objects.
+    Most code calls the resolver internally, but this variation is easier for unit testing.
+    """
+    try:
+        root = parse_xml_from_string(xml_string, extra_ns_aliases=ns_aliases)
+    except ExternalParsingError as e:
+        raise OperationParsingFailed(f"Unable to parse XML: {e}") from e
+
+    return tag_registry.node_from_xml(root, allowed_types=(BaseOwsRequest,))

gisserver/parsers/query.py

@@ -0,0 +1,175 @@
+"""The intermediate query result used by the various ``build_...()`` methods."""
+
+from __future__ import annotations
+
+import logging
+import operator
+from datetime import date, datetime
+from functools import reduce
+from typing import Union
+
+from django.contrib.gis.geos import GEOSGeometry
+from django.core.exceptions import FieldError
+from django.db.models import Q, QuerySet
+from django.db.models.expressions import Combinable, Func
+
+from gisserver.features import FeatureType
+
+logger = logging.getLogger(__name__)
+RhsTypes = Union[Combinable, Func, Q, GEOSGeometry, bool, int, str, date, datetime, tuple]
+
+
+class CompiledQuery:
+    """Intermediate data for translating FES queries to Django.
+
+    This class effectively contains all data from the ``<fes:Filter>`` object,
+    but using a format that can be translated to a django QuerySet.
+
+    As the Abstract Syntax Tree of a FES-filter creates the ORM query,
+    it fills this object with all intermediate bits. This allows building
+    the final QuerySet object in a single round. Each ``build_...()`` method
+    in the tree may add extra lookups and annotations.
+    """
+
+    def __init__(
+        self,
+        feature_types: list[FeatureType],
+        lookups: list[Q] | None = None,
+        typed_lookups: dict[str, list[Q]] | None = None,
+        annotations: dict[str, Combinable | Q] | None = None,
+    ):
+        """
+        :param feature_types: The feature types this query uses.
+                              Typically, this is one feature unless a JOIN syntax is used.
+
+        The extra parameters of the init method ar typically used only in unit tests.
+        """
+        self.feature_types = feature_types
+        self.lookups = lookups or []
+        self.typed_lookups = typed_lookups or {}
+        self.annotations = annotations or {}
+        self.aliases = 0
+        self.extra_lookups: list[Q] = []
+        self.ordering: list[str] = []
+        self.is_empty = False
+        self.distinct = False
+
+    def add_annotation(self, value: Combinable | Q) -> str:
+        """Create a named-alias for a function/Q object.
+        This alias can be used in a comparison, where expressions are used as left-hand-side.
+        """
+        self.aliases += 1
+        name = f"a{self.aliases}"
+        self.annotations[name] = value
+        return name
+
+    def add_distinct(self):
+        self.distinct = True
+
+    def add_lookups(self, q_object: Q, type_name: str | None = None):
+        """Register an extra 'WHERE' clause of the query.
+        This is used for comparisons, ID selectors and other query types.
+        """
+        if not isinstance(q_object, Q):
+            raise TypeError()
+
+        if type_name is not None:
+            if type_name not in self.typed_lookups:
+                self.typed_lookups[type_name] = []
+            self.typed_lookups[type_name].append(q_object)
+        else:
+            self.lookups.append(q_object)
+
+    def add_extra_lookup(self, q_object: Q):
+        """Temporary stash an extra lookup that the expression can't return yet.
+        This is used for XPath selectors that also filter on attributes,
+        e.g. "element[@attr=..]/child". The attribute lookup is processed as another filter.
+        """
+        if not isinstance(q_object, Q):
+            raise TypeError()
+        self.extra_lookups.append(q_object)
+
+    def add_ordering(self, ordering: list[str]):
+        """Read the desired result ordering from a ``<fes:SortBy>`` element."""
+        self.ordering.extend(ordering)
+
+    def apply_extra_lookups(self, comparison: Q) -> Q:
+        """Combine stashed lookups with the provided Q object.
+
+        This is called for functions that compile a "Q" object.
+        In case a node added extra lookups (for attributes), these are combined here
+        with the actual comparison.
+        """
+        if not self.extra_lookups:
+            return comparison
+
+        # The extra lookups are used for XPath queries such as "/node[@attr=..]/foo".
+        # A <ValueReference> with such lookup also requires to limit the filtered results,
+        # in addition to the comparison operator code that is wrapped up here.
+        result = reduce(operator.and_, [comparison] + self.extra_lookups)
+        self.extra_lookups.clear()
+        return result
+
+    def mark_empty(self):
+        """Mark as returning no results."""
+        self.is_empty = True
+
+    def get_queryset(self) -> QuerySet:
+        """Apply the filters and lookups to the queryset."""
+        queryset = self.feature_types[0].get_queryset()
+        if self.is_empty:
+            return queryset.none()
+
+        if self.extra_lookups:
+            # Each time an expression node calls add_extra_lookup(),
+            # the parent should have used apply_extra_lookups()
+            raise RuntimeError("apply_extra_lookups() was not called")
+
+        # All are applied at once.
+        if self.annotations:
+            queryset = queryset.annotate(**self.annotations)
+
+        lookups = self.lookups
+        try:
+            lookups += self.typed_lookups.pop(self.feature_types[0].xml_name)
+        except KeyError:
+            pass
+        if self.typed_lookups:
+            raise RuntimeError(
+                "Types lookups defined for unknown feature types: %r", list(self.typed_lookups)
+            )
+
+        if lookups:
+            try:
+                queryset = queryset.filter(*lookups)
+            except FieldError as e:
+                logger.debug("Query failed: %s, constructed query: %r", e.args[0], lookups)
+                e.args = (f"{e.args[0]} Constructed query: {lookups!r}",) + e.args[1:]
+                raise
+
+        if self.ordering:
+            queryset = queryset.order_by(*self.ordering)
+
+        if self.distinct:
+            queryset = queryset.distinct()
+
+        return queryset
+
+    def __repr__(self):
+        return (
+            "<CompiledQuery"
+            f" annotations={self.annotations!r},"
+            f" lookups={self.lookups!r},"
+            f" typed_lookups={self.typed_lookups!r}>"
+        )
+
+    def __eq__(self, other):
+        """For pytest comparisons."""
+        if isinstance(other, CompiledQuery):
+            return (
+                other.lookups == self.lookups
+                and other.typed_lookups == self.typed_lookups
+                and other.annotations == self.annotations
+            )
+        else:
+            return NotImplemented

gisserver/parsers/values.py

@@ -1,3 +1,4 @@
+import logging
 import re
 from datetime import datetime
 from decimal import Decimal as D
@@ -6,6 +7,7 @@ from django.utils.dateparse import parse_datetime
 
 from gisserver.exceptions import ExternalParsingError
 
+logger = logging.getLogger(__name__)
 RE_FLOAT = re.compile(r"\A[0-9]+(\.[0-9]+)\Z")
 
 
@@ -38,3 +40,27 @@ def parse_bool(raw_value: str):
         return False
     else:
         raise ExternalParsingError(f"Can't cast '{raw_value}' to boolean")
+
+
+def fix_type_name(type_name: str, feature_namespace: str):
+    """Fix the XML namespace for a typename value.
+
+    When the default namespace points to the "wfs" or "gml" namespaces,
+    parsing the QName of the type value will resolve that element as existing there.
+    This will correct such error, and restore the feature-type namespace.
+    """
+    if feature_namespace[0] == "{":
+        raise ValueError("Incorrect namespace argument")
+
+    alt_type_name = type_name
+    if type_name.startswith("{http://www.opengis.net/"):
+        # When the XML POST request used xmlns="http://www.opengis.net/wfs/2.0",
+        # this will define a default namespace that all QName values resolve to.
+        # As this is not detectable at the moment of parsing, correct it here.
+        alt_type_name = f"{{{feature_namespace}}}{type_name[type_name.index('}')+1:]}"
+        logger.debug("Corrected namespaced '%s' to '%s'", type_name, alt_type_name)
+    elif type_name[0] != "{":
+        # Typically happens in GET requests, or when no namespace prefix is used in XML POST
+        alt_type_name = f"{{{feature_namespace}}}{type_name}"
+        logger.debug("Corrected unnamespaced '%s' to namespaced '%s'", type_name, alt_type_name)
+    return alt_type_name

gisserver/parsers/wfs20/__init__.py

@@ -0,0 +1,37 @@
+"""WFS 2.0 element parsing.
+
+The full spec can be found at: https://www.ogc.org/publications/standard/wfs/.
+Secondly, using https://www.mediamaps.ch/ogc/schemas-xsdoc/sld/1.2/wfs_xsd.html can be very
+helpful to see which options each object type should support.
+"""
+
+from .adhoc import AdhocQuery
+from .base import QueryExpression
+from .projection import PropertyName
+from .requests import (
+    DescribeFeatureType,
+    DescribeStoredQueries,
+    GetCapabilities,
+    GetFeature,
+    GetPropertyValue,
+    ListStoredQueries,
+    ResultType,
+)
+from .stored import StoredQuery
+
+__all__ = (
+    # Requests
+    "GetCapabilities",
+    "DescribeFeatureType",
+    "GetFeature",
+    "GetPropertyValue",
+    "ListStoredQueries",
+    "DescribeStoredQueries",
+    "ResultType",
+    # Queries
+    "QueryExpression",
+    "AdhocQuery",
+    "StoredQuery",
+    # Projection
+    "PropertyName",
+)

gisserver/parsers/wfs20/adhoc.py

@@ -0,0 +1,245 @@
+"""Handle adhoc-query objects.
+
+The adhoc query is based on incoming request parameters,
+such as the "FILTER", "BBOX" and "RESOURCEID" parameters.
+
+These definitions follow the WFS spec.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from functools import cached_property
+
+from django.db.models import Q
+
+from gisserver.exceptions import (
+    InvalidParameterValue,
+    MissingParameterValue,
+    OperationNotSupported,
+)
+from gisserver.geometries import CRS
+from gisserver.parsers import fes20
+from gisserver.parsers.ast import tag_registry
+from gisserver.parsers.ows import KVPRequest
+from gisserver.parsers.query import CompiledQuery
+from gisserver.parsers.xml import NSElement, xmlns
+from gisserver.projection import FeatureProjection
+
+from .base import QueryExpression
+from .projection import PropertyName
+
+logger = logging.getLogger(__name__)
+
+ADHOC_QUERY_ELEMENTS = (PropertyName, fes20.Filter, fes20.SortBy)
+
+
+@dataclass
+@tag_registry.register("Query", xmlns.wfs)
+class AdhocQuery(QueryExpression):
+    """The Ad hoc query expression parameters.
+
+    This parses and handles the syntax::
+
+        <wfs:Query typeNames="...">
+            <wfs:PropertyName>...</wfs:PropertyName>
+            <fes:Filter>...</fes:Filter>
+            <fes:SortBy>...</fes:SortBy>
+        </wfs:Query>
+
+    And supports the KVP syntax::
+
+        ?SERVICE=WFS&...&TYPENAMES=ns:myType&FILTER=...&SORTBY=...&SRSNAME=...&PROPERTYNAME=...
+        ?SERVICE=WFS&...&TYPENAMES=ns:myType&BBOX=...&SORTBY=...&SRSNAME=...&PROPERTYNAME=...
+        ?SERVICE=WFS&...&TYPENAMES=ns:myType&RESOURCEID=...
+
+    This represents all dynamic queries received as request (hence "adhoc"),
+    such as the "FILTER" and "BBOX" arguments from an HTTP GET.
+
+    The WFS Spec has 3 class levels for this:
+
+    - AdhocQueryExpression (types, projection, selection, sorting)
+     - Query (adds srsName, featureVersion)
+     - StoredQuery (adds storedQueryID)
+
+    For KVP requests, this class seems almost identifical to the provided parameters.
+    However, the KVP format allows to provide parameter lists,
+    to perform support multiple queries in a single request!
+
+    .. seealso::
+        https://www.mediamaps.ch/ogc/schemas-xsdoc/sld/1.2/query_xsd.html#AbstractAdhocQueryExpressionType
+    """
+
+    # Tag attributes (fes:AbstractAdhocQueryExpression)
+    # WFS allows multiple names to construct JOIN queries.
+    # See https://docs.ogc.org/is/09-025r2/09-025r2.html#107
+    # and https://docs.ogc.org/is/09-025r2/09-025r2.html#190
+    typeNames: list[str]  # typeNames in WFS/FES spec, multiple values means a JOIN is made.
+    aliases: list[str] | None = None  # aliases for typeNames for joining the same table twice.
+    handle: str = ""  # only for XML POST requests, is returned in ows:Exception
+    # Query tag attributes
+    srsName: CRS | None = None
+
+    # Projection clause (fes:AbstractProjectionClause)
+    property_names: list[PropertyName] | None = None
+
+    # Selection clause (fes:AbstractSelectionClause):
+    # - for XML POST this is encoded in a <fes:Query>
+    # - for HTTP GET, this is encoded as FILTER, FILTER_LANGUAGE, RESOURCEID, BBOX.
+    filter: fes20.Filter | None = None
+
+    # Sorting Clause (fes:AbstractSortingClause)
+    sortBy: fes20.SortBy | None = None
+
+    def __post_init__(self):
+        if len(self.typeNames) > 1:
+            raise OperationNotSupported("Join queries are not supported", locator="typeNames")
+        if self.aliases:
+            raise OperationNotSupported("Join queries are not supported", locator="aliases")
+
+    @classmethod
+    def from_xml(cls, element: NSElement) -> AdhocQuery:
+        """Parse the XML element of the Query tag."""
+        type_names = [
+            element.parse_qname(qname)
+            for qname in element.get_str_attribute("typeNames").split(" ")
+        ]
+        aliases = element.attrib.get("aliases", None)
+        srsName = element.attrib.get("srsName", None)
+        property_names = []
+        filter = None
+        sortBy = None
+
+        for child in element:
+            # The FES XSD dictates the element ordering, but this is ignored here.
+            node = tag_registry.node_from_xml(child, allowed_types=ADHOC_QUERY_ELEMENTS)
+            if isinstance(node, PropertyName):
+                property_names.append(node)
+            elif isinstance(node, fes20.Filter):
+                filter = node
+            elif isinstance(node, fes20.SortBy):
+                sortBy = node
+            else:
+                raise NotImplementedError(
+                    f"Parsing {node.__class__} not handled in AdhocQuery.from_xml()"
+                )
+
+        return AdhocQuery(
+            typeNames=type_names,
+            aliases=aliases.split(" ") if aliases is not None else None,
+            handle=element.attrib.get("handle", ""),
+            property_names=property_names or None,
+            filter=filter,
+            sortBy=sortBy,
+            srsName=CRS.from_string(srsName) if srsName else None,
+        )
+
+    @classmethod
+    def from_kvp_request(cls, kvp: KVPRequest):
+        """Build this object from an HTTP GET (key-value-pair) request.
+
+        Note the caller should have split the KVP request parameter lists.
+        This class only handles a single parameter pair.
+        """
+        # Parse attributes
+        typeNames = [
+            kvp.parse_qname(qname)
+            for qname in kvp.get_list("typeNames", alias="TYPENAME", default=[])
+        ]
+        aliases = kvp.get_list("aliases", default=None)
+        srsName = kvp.get_custom("srsName", default=None, parser=CRS.from_string)
+
+        # KVP requests may omit the typenames if RESOURCEID=... is given.
+        if not typeNames and "RESOURCEID" not in kvp:
+            raise MissingParameterValue("Empty TYPENAMES parameter", locator="typeNames")
+
+        # Parse elements
+        filter = fes20.Filter.from_kvp_request(kvp)
+        sort_by = fes20.SortBy.from_kvp_request(kvp)
+
+        # Parse projection
+        property_names = None
+        if "PROPERTYNAME" in kvp:
+            names = kvp.get_list("propertyName", default=[])
+            # Check for WFS 1.x syntax of ?PROPERTYNAME=*
+            if names != ["*"]:
+                property_names = [
+                    PropertyName(xpath=name, xpath_ns_aliases=kvp.ns_aliases) for name in names
+                ]
+
+        return AdhocQuery(
+            # Attributes
+            typeNames=typeNames,
+            aliases=aliases,
+            srsName=srsName,
+            # Elements
+            property_names=property_names,
+            filter=filter,
+            sortBy=sort_by,
+        )
+
+    @cached_property
+    def query_locator(self):
+        """Overrides the 'query_locator' attribute, so the 'locator' argument is correctly set."""
+        if self.filter is not None and self.filter.get_resource_id_types():
+            return "resourceId"
+        else:
+            return "filter"
+
+    def get_type_names(self) -> list[str]:
+        """Tell which type names this query uses."""
+        if not self.typeNames and self.filter is not None:
+            # Also make the behavior consistent, always supply the type name.
+            return self.filter.get_resource_id_types() or []
+        else:
+            return self.typeNames
+
+    def get_projection(self) -> FeatureProjection:
+        """Tell how the <wfs:Query> element should be displayed."""
+        return FeatureProjection(
+            self.feature_types,
+            self.property_names,
+            self.value_reference,
+            output_crs=self.srsName,
+        )
+
+    def bind(self, *args, **kwargs):
+        """Make sure the 'locator' points to the actual object that defined the type."""
+        try:
+            super().bind(*args, **kwargs)
+        except InvalidParameterValue as e:
+            if not self.typeNames:
+                e.locator = "resourceId"
+            raise
+
+        # Validate the srsName too
+        if self.srsName is not None:
+            self.srsName = self.feature_types[0].resolve_crs(self.srsName, locator="srsName")
+
+    def build_query(self, compiler: CompiledQuery) -> Q | None:
+        """Apply our collected filter data to the compiler."""
+        # Add the
+        if self.sortBy is not None:
+            self.sortBy.build_ordering(compiler)
+
+        if self.filter is not None:
+            # Generate the internal query object from the <fes:Filter>,
+            # this can return a Q object.
+            return self.filter.build_query(compiler)
+        else:
+            return None
+
+    def as_kvp(self):
+        """Translate the POST request into KVP GET parameters. This is needed for pagination."""
+        params = super().as_kvp()
+        params["TYPENAMES"] = ",".join(self.typeNames)
+        if self.srsName is not None:
+            params["SRSNAME"] = str(self.srsName)
+
+        if self.filter is not None:
+            raise NotImplementedError()  # not going to parse that, nor does mapserver
+        if self.sortBy is not None:
+            params["SORTBY"] = self.sortBy.as_kvp()
+
+        return params