django-gisserver 1.5.0__py3-none-any.whl → 2.1__py3-none-any.whl

gisserver/parsers/ows/kvp.py

@@ -0,0 +1,198 @@
+"""Parsing the Key-Value-Pair (KVP) request format."""
+
+from __future__ import annotations
+
+from copy import copy
+
+from gisserver.exceptions import (
+    InvalidParameterValue,
+    MissingParameterValue,
+    OperationParsingFailed,
+    wrap_parser_errors,
+)
+from gisserver.parsers.xml import parse_qname, xmlns
+
+REQUIRED = ...  # sentinel value
+
+
+class KVPRequest:
+    """The Key-Value-Pair (KVP) request format.
+
+    This handles parameters from the HTTP GET request.
+    It includes notation format support for certain parameters,
+    such as comma-separated lists, and parenthesis-grouping notations.
+
+    Some basic validation is performed, allowing to convert the data into Python types.
+    """
+
+    def __init__(self, query_string: dict[str, str], ns_aliases: dict[str, str] | None = None):
+        # The parameters are case-insensitive.
+        self.params = {name.upper(): value for name, value in query_string.items()}
+
+        # Make sure most common namespaces are known for resolving them.
+        self.ns_aliases = {
+            **xmlns.as_ns_aliases(),  # some defaults in case these are missing in the request
+            **(ns_aliases or {}),  # our local application namespaces
+            **parse_kvp_namespaces(self.params.get("NAMESPACES")),  # extra in the request
+        }
+
+    def __contains__(self, name: str) -> bool:
+        """Tell whether a parameter is present."""
+        return name.upper() in self.params
+
+    def get_custom(self, name: str, *, alias: str | None = None, default=REQUIRED, parser=None):
+        """Retrieve a value by name.
+        This performs basic validation, similar to what the XML parsing does.
+
+        Any parsing errors or validation checks are raised as WFS Exceptions,
+        meaning the client will get the appropriate response.
+
+        :param name: The name of the parameter, typically given in its XML notation format (camelCase).
+        :param alias: An older WFS 1 to try for compatibility (e.g. TYPENAMES/TYPENAME, COUNT/MAXFEATURES)
+        :param default: The default value to return. If not provided, the parameter is required.
+        :param parser: A custom Python function or type to convert the value with.
+        """
+        kvp_name = name.upper()
+        value = self.params.get(kvp_name)
+        if not value and alias:
+            value = self.params.get(alias.upper())
+
+        # Check required field settings, both empty and missing value are treated the same.
+        if not value:
+            if default is REQUIRED:
+                if value is None:
+                    raise MissingParameterValue(
+                        f"Missing required '{name}' parameter.", locator=name
+                    )
+                else:
+                    raise InvalidParameterValue(f"Empty '{kvp_name}' parameter", locator=name)
+            return default
+
+        # Allow conversion into a python object
+        if parser is not None:
+            with wrap_parser_errors(kvp_name, locator=name):
+                return parser(value)
+        else:
+            return value
+
+    def get_str(
+        self, name: str, *, alias: str | None = None, default: str | None = REQUIRED
+    ) -> str | None:
+        """Retrieve a string value from the request."""
+        return self.get_custom(name, alias=alias, default=default)
+
+    def get_int(
+        self, name: str, *, alias: str | None = None, default: int | None = REQUIRED
+    ) -> int | None:
+        """Retrieve an integer value from the request."""
+        return self.get_custom(name, alias=alias, default=default, parser=int)
+
+    def get_list(
+        self, name, *, alias: str | None = None, default: list | None = REQUIRED
+    ) -> list[str] | None:
+        """Retrieve a comma-separated list value from the request."""
+        return self.get_custom(
+            name,
+            alias=alias,
+            default=default,
+            parser=lambda x: x.split(","),
+        )
+
+    def parse_qname(self, value) -> str:
+        """Convert the value to an XML fully qualified name."""
+        return parse_qname(value, self.ns_aliases)
+
+    def split_parameter_lists(self) -> list[KVPRequest]:
+        """Split the parameter lists into individual requests.
+
+        This translates a request such as:
+
+        .. code-block:: urlencoded
+
+            TYPENAMES=(ns1:F1,ns2:F2)(ns1:F1,ns1:F1)
+            &ALIASES=(A,B)(C,D)
+            &FILTER=(<Filter>… for A,B …</Filter>)(<Filter>…for C,D…</Filter>)
+
+        into separate pairs:
+
+        .. code-block:: urlencoded
+
+            TYPENAMES=ns1:F1,ns2:F2&ALIASES=A,B&FILTER=<Filter>…for A,B…</Filter>
+            TYPENAMES=ns1:F1,ns1:F1&ALIASES=C,D&FILTER=<Filter>…for C,D…</Filter>
+
+        It's both possible have some query parameters split and some shared.
+        For example to have two different bounding boxes:
+
+        .. code-block:: urlencoded
+
+             TYPENAMES=(INWATER_1M)(BuiltUpA_1M)&BBOX=(40.9821,...)(40.5874,...)
+
+        or have a single bounding box for both queries:
+
+        .. code-block:: urlencoded
+
+             TYPENAMES=(INWATER_1M)(BuiltUpA_1M)&BBOX=40.9821,23.4948,41.0257,23.5525
+        """
+        pairs = {
+            name: value[1:-1].split(")(")
+            for name, value in self.params.items()
+            if value.startswith("(") and value.endswith(")")
+        }
+        if not pairs:
+            return [self]
+
+        pair_sizes = {len(value) for value in pairs.values()}
+        if len(pair_sizes) > 1:
+            keys = sorted(pairs)
+            raise OperationParsingFailed(
+                f"Inconsistent pairs between: {', '.join(keys)}", locator=keys[0]
+            )
+
+        # Produce variations of the same request object
+        pair_size = next(iter(pair_sizes))
+        variants = []
+        for i in range(pair_size):
+            updates = {key: value[i] for key, value in pairs.items()}
+            variant = copy(self)
+            variant.params = {**self.params, **updates}
+            variants.append(variant)
+        return variants
+
+
+def parse_kvp_namespaces(value) -> dict[str, str]:
+    """Parse the 'NAMESPACES' parameter format to lookups.
+
+    The NAMESPACES parameter defines which namespaces are used in the KVP request.
+    When this parameter is not given, the default namespaces are assumed.
+    """
+    if not value:
+        return {}
+
+    # example single value: xmlns(http://example.org)
+    # or: NAMESPACES=xmlns(xml,http://www.w3.org/...),xmlns(wfs,http://www.opengis.net/...)
+    tokens = value.split(",")
+
+    namespaces = {}
+    tokens = iter(tokens)
+    for prefix in tokens:
+        if not prefix.startswith("xmlns("):
+            raise InvalidParameterValue(
+                f"Expected xmlns(...) format: {value}", locator="namespaces"
+            )
+        if prefix.endswith(")"):
+            # xmlns(http://...)
+            uri = prefix[6:-1]
+            prefix = ""
+        else:
+            uri = next(tokens, "")
+            if not uri.endswith(")"):
+                raise InvalidParameterValue(
+                    f"Expected xmlns(prefix,uri) format: {value}",
+                    locator="namespaces",
+                )
+            prefix = prefix[6:]
+            uri = uri[:-1]
+
+        namespaces[prefix] = uri
+
+    return namespaces

gisserver/parsers/ows/requests.py

@@ -0,0 +1,160 @@
+"""Request parsing for the common Generic Open Web Services (OWS) protocol bits."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+
+from django.http import QueryDict
+
+from gisserver.exceptions import (
+    ExternalParsingError,
+    InvalidParameterValue,
+    OperationNotSupported,
+    OperationParsingFailed,
+)
+from gisserver.parsers.ast import AstNode, tag_registry
+from gisserver.parsers.xml import NSElement, parse_xml_from_string, split_ns, xmlns
+
+from .kvp import KVPRequest
+
+__all__ = (
+    "BaseOwsRequest",
+    "resolve_kvp_parser_class",
+    "resolve_xml_parser_class",
+    "parse_get_request",
+    "parse_post_request",
+)
+
+
+@dataclass
+class BaseOwsRequest(AstNode):
+    """Base request data for all request types of the OWS standards.
+    This mirrors the ``<wfs:BaseRequestType>`` element from the WFS spec.
+    """
+
+    # dataclass limitation: when defaults are added, subclasses can't have required parameters anymore.
+    service: str
+    version: str | None  # none for GetCapabilities only
+    handle: str | None
+
+    @classmethod
+    def from_xml(cls, element: NSElement):
+        """Initialize from an XML POST request."""
+        return cls(**cls.base_xml_init_parameters(element))
+
+    @classmethod
+    def from_kvp_request(cls, kvp: KVPRequest):
+        """Initialize from an KVP GET request."""
+        return cls(**cls.base_kvp_init_parameters(kvp))
+
+    @classmethod
+    def base_xml_init_parameters(cls, element: NSElement) -> dict:
+        """Parse the base attributes.
+        This parses the syntax such as::
+
+            <wfs:BaseRequest service="WFS" version="2.0.0" handle="...">
+
+        """
+        return dict(
+            service=element.get_str_attribute("service"),
+            version=element.get_str_attribute("version"),
+            handle=element.attrib.get("handle"),
+        )
+
+    @classmethod
+    def base_kvp_init_parameters(cls, kvp: KVPRequest) -> dict:
+        """Parse the common Key-Value-Pair format (GET request parameters).
+        This parses the syntax::
+
+            ?SERVICE=WFS&VERSION=2.0.0
+        """
+        return dict(
+            service=kvp.get_str("SERVICE"),
+            version=kvp.get_str("VERSION"),
+            handle=None,
+        )
+
+    def as_kvp(self) -> dict:
+        """Translate the POST request into KVP GET parameters. This is needed for pagination."""
+        return {
+            "SERVICE": self.service,
+            "VERSION": str(self.version),
+            "REQUEST": split_ns(self.xml_name)[1],
+        }
+
+
+@lru_cache
+def _get_kvp_parsers() -> dict[str, dict[str, type[BaseOwsRequest]]]:
+    """Find which classes are registered."""
+    # This late initialization keeps the ability open in the future
+    # to handle request types that are registered in third party code.
+    ows_parsers = {}
+
+    for xml_name, parser_class in tag_registry.find_subclasses(BaseOwsRequest).items():
+        namespace, local_name = split_ns(xml_name)
+        ows_parsers.setdefault(namespace, {})[local_name.upper()] = parser_class
+
+    return ows_parsers
+
+
+def resolve_kvp_parser_class(kvp: KVPRequest) -> type[BaseOwsRequest]:
+    """Find the appropriate class to parse the KVP GET request data."""
+    service = kvp.get_str("service").upper()
+    request = kvp.get_str("request")
+
+    # Find the appropriate request object
+    service_types = _get_kvp_parsers()
+    try:
+        # Translating the GET parameter to a namespace makes connecting to the parser registration.
+        # This doesn't take the VERSION/ACCEPTVERSIONS into account yet.
+        namespace = xmlns[service.lower()].value
+        request_classes = service_types[namespace]
+    except KeyError:
+        raise InvalidParameterValue(
+            f"Unsupported service type: {service}.", locator="service"
+        ) from None
+
+    try:
+        request_cls = request_classes[request.upper()]
+    except KeyError:
+        allowed = ", ".join(
+            xml_tag for node in request_classes.values() for xml_tag in node.get_tag_names()
+        )
+        raise OperationNotSupported(
+            f"'{request}' is not implemented, supported are: {allowed}.",
+            locator="request",
+        ) from None
+
+    return request_cls
+
+
+def resolve_xml_parser_class(root: NSElement) -> type[BaseOwsRequest]:
+    """Find the correct class to parse the XML POST data with."""
+    return tag_registry.resolve_class(root, allowed_types=(BaseOwsRequest,))
+
+
+def parse_get_request(
+    query_string: str | dict[str, str], ns_aliases: dict | None = None
+) -> BaseOwsRequest:
+    """Parse the WFS KVP GET request format into the internal request objects.
+    Most code calls the resolver internally, but this variation is easier for unit testing.
+    """
+    if isinstance(query_string, str):
+        query_string = QueryDict(query_string.lstrip("?"))
+
+    kvp = KVPRequest(query_string, ns_aliases=ns_aliases)
+    request_cls = resolve_kvp_parser_class(kvp)
+    return request_cls.from_kvp_request(kvp)
+
+
+def parse_post_request(xml_string: str | bytes, ns_aliases: dict | None = None) -> BaseOwsRequest:
+    """Parse the XML POST request format into the internal request objects.
+    Most code calls the resolver internally, but this variation is easier for unit testing.
+    """
+    try:
+        root = parse_xml_from_string(xml_string, extra_ns_aliases=ns_aliases)
+    except ExternalParsingError as e:
+        raise OperationParsingFailed(f"Unable to parse XML: {e}") from e
+
+    return tag_registry.node_from_xml(root, allowed_types=(BaseOwsRequest,))

gisserver/parsers/query.py

@@ -0,0 +1,179 @@
+"""The intermediate query result used by the various ``build_...()`` methods."""
+
+from __future__ import annotations
+
+import logging
+import operator
+from datetime import date, datetime
+from decimal import Decimal as D
+from functools import reduce
+from typing import Union
+
+from django.contrib.gis.geos import GEOSGeometry
+from django.core.exceptions import FieldError
+from django.db.models import Q, QuerySet
+from django.db.models.expressions import Combinable, Func
+
+from gisserver.features import FeatureType
+
+logger = logging.getLogger(__name__)
+ScalarTypes = (bool, int, str, D, date, datetime)  # Not Union for Python 3.9
+RhsTypes = Union[Combinable, Func, Q, GEOSGeometry, bool, int, D, str, date, datetime, tuple]
+
+
+class CompiledQuery:
+    """Intermediate data for translating FES queries to Django.
+
+    This class effectively contains all data from the ``<fes:Filter>`` object,
+    but using a format that can be translated to a Django QuerySet.
+
+    As the Abstract Syntax Tree of a FES-filter creates the ORM query,
+    it fills this object with all intermediate bits. This allows building
+    the final QuerySet object in a single round. Each ``build_...()`` method
+    in the tree may add extra lookups and annotations.
+    """
+
+    def __init__(
+        self,
+        feature_types: list[FeatureType],
+        lookups: list[Q] | None = None,
+        typed_lookups: dict[str, list[Q]] | None = None,
+        annotations: dict[str, Combinable | Q] | None = None,
+    ):
+        """
+        :param feature_types: The feature types this query uses.
+                              Typically, this is one feature unless a JOIN syntax is used.
+
+        The extra parameters of the init method ar typically used only in unit tests.
+        """
+        self.feature_types = feature_types
+        self.lookups = lookups or []
+        self.typed_lookups = typed_lookups or {}
+        self.annotations = annotations or {}
+        self.aliases = 0
+        self.extra_lookups: list[Q] = []
+        self.ordering: list[str] = []
+        self.is_empty = False
+        self.distinct = False
+
+    def add_annotation(self, value: Combinable | Q) -> str:
+        """Create a named-alias for a function/Q object.
+        This alias can be used in a comparison, where expressions are used as left-hand-side.
+        """
+        self.aliases += 1
+        name = f"a{self.aliases}"
+        self.annotations[name] = value
+        return name
+
+    def add_distinct(self):
+        """Enforce "SELECT DISTINCT" on the query, used when joining 1-N or N-M relationships."""
+        self.distinct = True
+
+    def add_lookups(self, q_object: Q, type_name: str | None = None):
+        """Register an extra 'WHERE' clause of the query.
+        This is used for comparisons, ID selectors and other query types.
+        """
+        if not isinstance(q_object, Q):
+            raise TypeError()
+
+        if type_name is not None:
+            if type_name not in self.typed_lookups:
+                self.typed_lookups[type_name] = []
+            self.typed_lookups[type_name].append(q_object)
+        else:
+            self.lookups.append(q_object)
+
+    def add_extra_lookup(self, q_object: Q):
+        """Temporary stash an extra lookup that the expression can't return yet.
+        This is used for XPath selectors that also filter on attributes,
+        e.g. "element[@attr=..]/child". The attribute lookup is processed as another filter.
+        """
+        if not isinstance(q_object, Q):
+            raise TypeError()
+        # Note the ORM also provides a FilteredRelation() option, that is not explored yet here.
+        self.extra_lookups.append(q_object)
+
+    def add_ordering(self, ordering: list[str]):
+        """Read the desired result ordering from a ``<fes:SortBy>`` element."""
+        self.ordering.extend(ordering)
+
+    def apply_extra_lookups(self, comparison: Q) -> Q:
+        """Combine stashed lookups with the provided Q object.
+
+        This is called for functions that compile a "Q" object.
+        In case a node added extra lookups (for attributes), these are combined here
+        with the actual comparison.
+        """
+        if not self.extra_lookups:
+            return comparison
+
+        # The extra lookups are used for XPath queries such as "/node[@attr=..]/foo".
+        # A <ValueReference> with such lookup also requires to limit the filtered results,
+        # in addition to the comparison operator code that is wrapped up here.
+        result = reduce(operator.and_, [comparison] + self.extra_lookups)
+        self.extra_lookups.clear()
+        return result
+
+    def mark_empty(self):
+        """Mark as returning no results."""
+        self.is_empty = True
+
+    def get_queryset(self) -> QuerySet:
+        """Apply the filters and lookups to the queryset."""
+        queryset = self.feature_types[0].get_queryset()
+        if self.is_empty:
+            return queryset.none()
+
+        if self.extra_lookups:
+            # Each time an expression node calls add_extra_lookup(),
+            # the parent should have used apply_extra_lookups()
+            raise RuntimeError("apply_extra_lookups() was not called")
+
+        # All are applied at once.
+        if self.annotations:
+            queryset = queryset.annotate(**self.annotations)
+
+        lookups = self.lookups
+        try:
+            lookups += self.typed_lookups.pop(self.feature_types[0].xml_name)
+        except KeyError:
+            pass
+        if self.typed_lookups:
+            raise RuntimeError(
+                "Types lookups defined for unknown feature types: %r", list(self.typed_lookups)
+            )
+
+        if lookups:
+            try:
+                queryset = queryset.filter(*lookups)
+            except FieldError as e:
+                logger.debug("Query failed: %s, constructed query: %r", e.args[0], lookups)
+                e.args = (f"{e.args[0]} Constructed query: {lookups!r}",) + e.args[1:]
+                raise
+
+        if self.ordering:
+            queryset = queryset.order_by(*self.ordering)
+
+        if self.distinct:
+            queryset = queryset.distinct()
+
+        return queryset
+
+    def __repr__(self):
+        return (
+            "<CompiledQuery"
+            f" annotations={self.annotations!r},"
+            f" lookups={self.lookups!r},"
+            f" typed_lookups={self.typed_lookups!r}>"
+        )
+
+    def __eq__(self, other):
+        """For pytest comparisons."""
+        if isinstance(other, CompiledQuery):
+            return (
+                other.lookups == self.lookups
+                and other.typed_lookups == self.typed_lookups
+                and other.annotations == self.annotations
+            )
+        else:
+            return NotImplemented

gisserver/parsers/values.py

@@ -1,16 +1,28 @@
+"""Parsing of scalar values in the request."""
+
+import logging
 import re
-from datetime import datetime
+from datetime import date, datetime, time
 from decimal import Decimal as D
 
-from django.utils.dateparse import parse_datetime
+from django.utils.dateparse import parse_date, parse_datetime, parse_duration, parse_time
 
 from gisserver.exceptions import ExternalParsingError
 
+logger = logging.getLogger(__name__)
 RE_FLOAT = re.compile(r"\A[0-9]+(\.[0-9]+)\Z")
 
 
 def auto_cast(value: str):
-    """Automatically cast a value to a scalar."""
+    """Automatically cast a value to a scalar.
+
+    This recognizes integers, floats and ISO datetimes.
+    Booleans are not handled, as that leads to unpredictable behavior
+    (as seen in Yaml for example).
+    """
+    if not isinstance(value, str):
+        return value
+
     if value.isdigit():
         return int(value)
     elif RE_FLOAT.match(value):
@@ -24,17 +36,88 @@ def auto_cast(value: str):
     return value
 
 
+def parse_iso_date(raw_value: str) -> date:
+    """Translate ISO date into a Python date value."""
+    try:
+        value = parse_date(raw_value)
+    except ValueError as e:
+        raise ExternalParsingError(str(e)) from e
+
+    if value is None:
+        raise ExternalParsingError("Date must be in YYYY-MM-DD format.")
+    return value
+
+
 def parse_iso_datetime(raw_value: str) -> datetime:
-    value = parse_datetime(raw_value)
+    """Translate ISO datetimes into a Python datetime value."""
+    try:
+        value = parse_datetime(raw_value)
+    except ValueError as e:
+        raise ExternalParsingError(str(e)) from e
+
     if value is None:
         raise ExternalParsingError("Date must be in YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ] format.")
     return value
 
 
+def parse_iso_time(raw_value: str) -> time:
+    """Translate ISO times into a Python time value."""
+    try:
+        value = parse_time(raw_value)
+    except ValueError as e:
+        raise ExternalParsingError(str(e)) from e
+
+    if value is None:
+        raise ExternalParsingError("Time must be in HH:MM[:ss[.uuuuuu]][TZ] format.")
+    return value
+
+
+def parse_iso_duration(raw_value: str) -> time:
+    """Translate ISO times into a Python time value."""
+    # The parse_duration() supports multiple formats, including ISO8601.
+    # Limit it to only one format.
+    if not raw_value.startswith(("P", "-P")):
+        raise ExternalParsingError("Duration must be in ISO8601 format (e.g. PT1H).")
+
+    try:
+        value = parse_duration(raw_value)
+    except ValueError as e:
+        raise ExternalParsingError(str(e)) from e
+
+    if value is None:
+        raise ExternalParsingError("Duration must be in ISO8601 format (e.g. PT1H).")
+    return value
+
+
 def parse_bool(raw_value: str):
+    """Translate XML notations of true/1 and false/0 into a boolean."""
     if raw_value in ("true", "1"):
         return True
     elif raw_value in ("false", "0"):
         return False
     else:
         raise ExternalParsingError(f"Can't cast '{raw_value}' to boolean")
+
+
+def fix_type_name(type_name: str, feature_namespace: str):
+    """Fix the XML namespace for a typename value.
+
+    When the default namespace points to the "wfs" or "gml" namespaces,
+    parsing the QName of the type value will resolve that element as existing there.
+    This will correct such error, and restore the feature-type namespace.
+    """
+    if feature_namespace[0] == "{":
+        raise ValueError("Incorrect namespace argument")
+
+    alt_type_name = type_name
+    if type_name.startswith("{http://www.opengis.net/"):
+        # When the XML POST request used xmlns="http://www.opengis.net/wfs/2.0",
+        # this will define a default namespace that all QName values resolve to.
+        # As this is not detectable at the moment of parsing, correct it here.
+        alt_type_name = f"{{{feature_namespace}}}{type_name[type_name.index('}')+1:]}"
+        logger.debug("Corrected namespaced '%s' to '%s'", type_name, alt_type_name)
+    elif type_name[0] != "{":
+        # Typically happens in GET requests, or when no namespace prefix is used in XML POST
+        alt_type_name = f"{{{feature_namespace}}}{type_name}"
+        logger.debug("Corrected unnamespaced '%s' to namespaced '%s'", type_name, alt_type_name)
+    return alt_type_name

gisserver/parsers/wfs20/__init__.py

@@ -0,0 +1,39 @@
+"""WFS 2.0 element parsing.
+
+These classes parse the XML request body.
+
+The full spec can be found at: https://www.ogc.org/publications/standard/wfs/.
+Secondly, using https://www.mediamaps.ch/ogc/schemas-xsdoc/sld/1.2/wfs_xsd.html can be very
+helpful to see which options each object type should support.
+"""
+
+from .adhoc import AdhocQuery
+from .base import QueryExpression
+from .projection import PropertyName
+from .requests import (
+    DescribeFeatureType,
+    DescribeStoredQueries,
+    GetCapabilities,
+    GetFeature,
+    GetPropertyValue,
+    ListStoredQueries,
+    ResultType,
+)
+from .stored import StoredQuery
+
+__all__ = (
+    # Requests
+    "GetCapabilities",
+    "DescribeFeatureType",
+    "GetFeature",
+    "GetPropertyValue",
+    "ListStoredQueries",
+    "DescribeStoredQueries",
+    "ResultType",
+    # Queries
+    "QueryExpression",
+    "AdhocQuery",
+    "StoredQuery",
+    # Projection
+    "PropertyName",
+)