django-gisserver 1.5.0__py3-none-any.whl → 2.0__py3-none-any.whl

gisserver/parsers/ast.py

@@ -0,0 +1,320 @@
+"""Utilities for building an Abstract Syntax Tree (AST) from an XML fragment.
+
+By transforming the XML Element nodes into Python objects, most logic naturally follows.
+For example, the FES filter syntax can be processed into objects that build an ORM query.
+
+Python classes can inherit :class:`BaseNode` and register themselves as the parser/handler
+for a given tag. Both normal Python classes and dataclass work,
+as long as it has an :meth:`BaseNode.from_xml` class method.
+The custom `from_xml()` method should copy the XML data into local attributes.
+
+Next, when :meth:`TagRegistry.node_from_xml` is called,
+it will detect which class the XML Element refers to and initialize it using the ``from_xml()`` call.
+As convenience, calling :meth:`SomeNode.child_from_xml()` will also
+initialize the right subclass and initialize it.
+
+Since clients may not follow the desired XML schema, and make mistakes, one should avoid
+creating an invalid Abstract Syntax Tree. When using :meth:`TagRegistry.node_from_xml`,
+the allowed child types can also be provided, preventing invalid child elements.
+Furthermore, to support the creation of ``from_xml()`` methods, the :func:`expect_tag`,
+:func:`expect_children` and :func:`expect_no_children` decorators validate
+whether the given tag has the expected elements. This combination should make it easy
+to validate whether a provided XML structure confirms to the supported schema.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from enum import Enum
+from functools import wraps
+from itertools import chain
+from typing import TypeVar
+from xml.etree.ElementTree import QName
+
+from gisserver.exceptions import ExternalParsingError
+
+__all__ = (
+    "TagNameEnum",
+    "BaseNode",
+    "TagRegistry",
+    "tag_registry",
+    "expect_children",
+    "expect_tag",
+    "expect_no_children",
+)
+
+from gisserver.parsers.xml import NSElement, xmlns
+
+
+class TagNameEnum(Enum):
+    """An enumeration of XML tag names.
+
+    All enumerations that represent tag names inherit from this.
+    Each member name should be exactly the XML tag that it refers to.
+    """
+
+    @classmethod
+    def from_xml(cls, element: NSElement):
+        """Cast the element tag name into the enum member"""
+        tag_name = element.tag
+        if tag_name.startswith("{"):
+            # Split the element tag into the namespace and local name.
+            # The stdlib etree doesn't have the properties for this (lxml does).
+            end = tag_name.index("}")
+            tag_name = tag_name[end + 1 :]
+
+        return cls[tag_name]
+
+    @classmethod
+    def _missing_(cls, value):
+        raise NotImplementedError(f"<{value}> is not registered as valid {cls.__name__}")
+
+    def __repr__(self):
+        # Make repr(filter) easier to copy-paste
+        return f"{self.__class__.__name__}.{self.name}"
+
+
+class BaseNode:
+    """The base node for all classes that represent an XML tag.
+
+    All subclasses of this class build an Abstract Syntax Tree (AST)
+    that describes the XML content in Python objects. Each object can handle
+    implement additional logic to
+
+    Each subclass should implement the :meth:`from_xml` to translate
+    an XML tag into a Python (data) class.
+    """
+
+    xml_ns: xmlns | str | None = None
+    xml_tags = []
+
+    def __init_subclass__(cls):
+        # Each class level has a fresh list of supported child tags.
+        cls.xml_tags = []
+
+    @classmethod
+    def from_xml(cls, element: NSElement):
+        """Initialize this Python class from the data of the corresponding XML tag.
+        Each subclass overrides this to implement the XMl parsing of that particular XML tag.
+        """
+        raise NotImplementedError(
+            f"{cls.__name__}.from_xml() is not implemented to parse <{element.tag}>"
+        )
+
+    @classmethod
+    def child_from_xml(cls, element: NSElement) -> BaseNode:
+        """Parse the element, returning the correct subclass of this tag.
+
+        When ``Expression.child_from_xml(some_node)`` is given, it may
+        return a ``Literal``, ``ValueReference``, ``Function`` or ``BinaryOperator`` node.
+        """
+        sub_class = tag_registry.resolve_class(element, allowed_types=(cls,))
+        return sub_class.from_xml(element)
+
+    @classmethod
+    def get_tag_names(cls) -> Iterable[str]:
+        """Provide all known XMl tags that this code can parse."""
+        return chain.from_iterable(sub_type.xml_tags for sub_type in cls.__subclasses__())
+
+
+BN = TypeVar("BN", bound=BaseNode)
+
+
+class TagRegistry:
+    """Registration of all classes that can parse XML nodes.
+
+    The same class can be registered multiple times for different tag names.
+    """
+
+    parsers: dict[str, type[BaseNode]]
+
+    def __init__(self):
+        self.parsers = {}
+
+    def register(
+        self,
+        tag: str | type[TagNameEnum] | None = None,
+        namespace: xmlns | str | None = None,
+        hidden: bool = False,
+    ):
+        """Decorator to register a class as XML element parser.
+
+        Usage:
+
+            @dataclass
+            @tag_registry.register()
+            class SomeXmlTag(BaseNode):
+                xml_ns = FES
+
+                @classmethod
+                def from_xml(cls, element: NSElement):
+                    return cls(
+                        ...
+                    )
+
+        Whenever an element of the registered XML name is found,
+        the given "SomeXmlTag" will be initialized.
+
+        It's also possible to register tag names using an enum;
+        each member name is assumed to be an XML tag name.
+        """
+
+        def _dec(node_class: type[BaseNode]) -> type[BaseNode]:
+            if tag is None or isinstance(tag, str):
+                # Single tag name for the class.
+                self._register_tag_parser(
+                    node_class, tag=tag or node_class.__name__, namespace=namespace, hidden=hidden
+                )
+            elif issubclass(tag, TagNameEnum):
+                # Allow tags to be an Enum listing possible tag names.
+                # Note using __members__, not _member_names_.
+                # The latter will skip aliased items (like BBOX/Within).
+                for member_name in tag.__members__:
+                    self._register_tag_parser(node_class, tag=member_name, namespace=namespace)
+            else:
+                raise TypeError("tag type incorrect")
+
+            return node_class
+
+        return _dec
+
+    def _register_tag_parser(
+        self,
+        node_class: type[BaseNode],
+        tag: str,
+        namespace: xmlns | str | None = None,
+        hidden: bool = False,
+    ):
+        """Register a Python (data) class as parser for an XML node."""
+        if not issubclass(node_class, BaseNode):
+            raise TypeError(f"{node_class} must be a subclass of BaseNode")
+
+        if namespace is None and node_class.xml_ns is None:
+            raise RuntimeError(
+                f"{node_class.__name__}.xml_ns should be set, or namespace should be given."
+            )
+
+        xml_name = QName((namespace or node_class.xml_ns), tag=tag).text
+        if xml_name in self.parsers:
+            raise RuntimeError(
+                f"Another class is already registered to parse the <{xml_name}> tag."
+            )
+
+        self.parsers[xml_name] = node_class  # Track this parser to resolve the tag.
+        if not hidden:
+            node_class.xml_tags.append(xml_name)  # Allow fetching all names later
+
+    def node_from_xml(
+        self, element: NSElement, allowed_types: tuple[type[BN]] | None = None
+    ) -> BN:
+        """Find the ``BaseNode`` subclass that corresponds to the given XML element,
+        and initialize it with the element. This is a convenience shortcut.
+        ``"""
+        node_class = self.resolve_class(element, allowed_types)
+        return node_class.from_xml(element)
+
+    def resolve_class(
+        self, element: NSElement, allowed_types: tuple[type[BN]] | None = None
+    ) -> type[BN]:
+        """Find the ``BaseNode`` subclass that corresponds to the given XML element."""
+        try:
+            node_class = self.parsers[element.tag]
+        except KeyError:
+            msg = f"Unsupported tag: <{element.tag}>"
+            if "{" not in element.tag:
+                msg = f"{msg} without an XML namespace"
+            if allowed_types:
+                # Show better exception message
+                types = ", ".join(chain.from_iterable(c.get_tag_names() for c in allowed_types))
+                msg = f"{msg}, expected one of: {types}"
+
+            raise ExternalParsingError(msg) from None
+
+        # Check whether the resolved class is indeed a valid option here.
+        if allowed_types is not None and not issubclass(node_class, allowed_types):
+            types = ", ".join(c.__name__ for c in allowed_types)
+            raise ExternalParsingError(
+                f"Unexpected {node_class.__name__} for <{element.tag}> node, "
+                f"expected one of: {types}"
+            )
+
+        return node_class
+
+    def get_parser_class(self, xml_qname) -> type[BaseNode]:
+        """Provide the parser class for a given XML Qualified name."""
+        return self.parsers[xml_qname]
+
+    def find_subclasses(self, node_type: type[BN]) -> list[type[BN]]:
+        """Find all registered parsers for a given node."""
+        return {
+            tag: node_class
+            for tag, node_class in self.parsers.items()
+            if issubclass(node_class, node_type)
+        }
+
+
+def expect_tag(namespace: xmlns | str, *tag_names: str):
+    """Validate whether a given tag is need."""
+    valid_tags = {QName(namespace, name).text for name in tag_names}
+    expect0 = QName(namespace, tag_names[0]).text
+
+    def _wrapper(func):
+        @wraps(func)
+        def _expect_tag_decorator(cls, element: NSElement, *args, **kwargs):
+            if element.tag not in valid_tags:
+                raise ExternalParsingError(
+                    f"{cls.__name__} parser expects an <{expect0}> node, got <{element.tag}>"
+                )
+            return func(cls, element, *args, **kwargs)
+
+        return _expect_tag_decorator
+
+    return _wrapper
+
+
+def expect_no_children(from_xml_func):
+    """Validate that the XML tag has no child nodes."""
+
+    @wraps(from_xml_func)
+    def _expect_no_children_decorator(cls, element: NSElement, *args, **kwargs):
+        if len(element):
+            raise ExternalParsingError(
+                f"Unsupported child element for {element.tag} element: {element[0].tag}."
+            )
+
+        return from_xml_func(cls, element, *args, **kwargs)
+
+    return _expect_no_children_decorator
+
+
+def expect_children(min_child_nodes, *expect_types: str | type[BaseNode]):
+    """Validate whether an element has enough children to continue parsing."""
+    known_tag_names = set()
+    for child_type in expect_types:
+        if isinstance(child_type, type) and issubclass(child_type, BaseNode):
+            known_tag_names.update(child_type.get_tag_names())
+        elif isinstance(child_type, str):
+            known_tag_names.add(child_type)
+        else:
+            raise TypeError(f"Unexpected {child_type!r}")
+    known_tag_names = sorted(known_tag_names)
+
+    def _wrapper(func):
+        @wraps(func)
+        def _expect_children_decorator(cls, element: NSElement, *args, **kwargs):
+            if len(element) < min_child_nodes:
+                type_names = ", ".join(known_tag_names)
+                suffix = f" (possible tags: {type_names})" if type_names else ""
+                raise ExternalParsingError(
+                    f"<{element.tag}> should have {min_child_nodes} child nodes, "
+                    f"got {len(element)}{suffix}"
+                )
+
+            return func(cls, element, *args, **kwargs)
+
+        return _expect_children_decorator
+
+    return _wrapper
+
+
+tag_registry = TagRegistry()

gisserver/parsers/fes20/__init__.py

@@ -1,42 +1,28 @@
-from __future__ import annotations
+"""Parsing and processing of the OGC Filter Encoding Standard 2.0 (FES).
+
+This parses the XML syntax, and translates that into a Django ORM query.
+
+It's also possible to register custom functions,
+which will include Django ORM function as part of the filter language.
 
-from gisserver.exceptions import OperationParsingFailed
+The full spec can be found at: https://www.ogc.org/publications/standard/filter/.
+Secondly, using https://www.mediamaps.ch/ogc/schemas-xsdoc/sld/1.2/filter_xsd.html
+can be very helpful to see which options each object type should support.
+"""
 
+from __future__ import annotations
+
+from . import lookups  # noqa: F401 (need to register ORM lookups)
 from .expressions import ValueReference
 from .filters import Filter
-from .functions import function_registry
 from .identifiers import ResourceId
 from .operators import IdOperator
-from .query import CompiledQuery
 from .sorting import SortBy
 
 __all__ = [
     "Filter",
-    "CompiledQuery",
     "ValueReference",
     "ResourceId",
     "IdOperator",
     "SortBy",
-    "function_registry",
 ]
-
-
-def parse_resource_id_kvp(value) -> IdOperator:
-    """Parse the RESOURCEID parameter.
-    This returns an IdOperator, as it needs to support multiple pairs.
-    """
-    return IdOperator([ResourceId(rid) for rid in value.split(",")])
-
-
-def parse_property_name(value) -> list[ValueReference] | None:
-    """Parse the PROPERTYNAME parameter"""
-    if not value or value == "*":
-        return None  # WFS 1 logic
-
-    if "(" in value:
-        raise OperationParsingFailed(
-            "Parameter lists to perform multiple queries are not supported yet.",
-            locator="propertyname",
-        )
-
-    return [ValueReference(x) for x in value.split(",")]

gisserver/parsers/fes20/expressions.py

@@ -5,21 +5,27 @@ The class names are identical to those in the FES spec.
 from __future__ import annotations
 
 import operator
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import date, datetime
 from decimal import Decimal as D
 from functools import cached_property
 from typing import Union
-from xml.etree.ElementTree import Element
 
-from django.contrib.gis.geos import GEOSGeometry
+from django.contrib.gis import geos
+from django.contrib.gis.db import models as gis_models
 from django.db import models
-from django.db.models import Func, Q, Value
+from django.db.models import Q, Value
 from django.db.models.expressions import Combinable
 
 from gisserver.exceptions import ExternalParsingError
-from gisserver.parsers.base import BaseNode, TagNameEnum, tag_registry
-from gisserver.parsers.fes20.functions import function_registry
+from gisserver.extensions.functions import function_registry
+from gisserver.parsers.ast import (
+    BaseNode,
+    TagNameEnum,
+    expect_no_children,
+    expect_tag,
+    tag_registry,
+)
 from gisserver.parsers.gml import (
     GM_Envelope,
     GM_Object,
@@ -27,12 +33,12 @@ from gisserver.parsers.gml import (
     is_gml_element,
     parse_gml_node,
 )
-from gisserver.parsers.tags import expect_tag, get_attribute
+from gisserver.parsers.query import RhsTypes
 from gisserver.parsers.values import auto_cast
-from gisserver.types import FES20, ORMPath, XsdTypes, split_xml_name
+from gisserver.parsers.xml import NSElement, parse_qname, xmlns
+from gisserver.types import ORMPath, XsdTypes
 
 NoneType = type(None)
-RhsTypes = Union[Combinable, Func, Q, GEOSGeometry, bool, int, str, date, datetime, tuple]
 ParsedValue = Union[int, str, date, D, datetime, GM_Object, GM_Envelope, TM_Object, NoneType]
 
 OUTPUT_FIELDS = {
@@ -43,6 +49,15 @@ OUTPUT_FIELDS = {
     datetime: models.DateTimeField(),
     float: models.FloatField(),
     D: models.DecimalField(),
+    geos.GEOSGeometry: gis_models.GeometryField(),
+    geos.Point: gis_models.PointField(),
+    geos.LineString: gis_models.LineStringField(),
+    geos.LinearRing: gis_models.LineStringField(),
+    geos.Polygon: gis_models.PolygonField(),
+    geos.MultiPoint: gis_models.MultiPointField(),
+    geos.MultiPolygon: gis_models.MultiPolygonField(),
+    geos.MultiLineString: gis_models.MultiLineStringField(),
+    geos.GeometryCollection: gis_models.GeometryCollectionField(),
 }
 
 
@@ -60,9 +75,18 @@ class BinaryOperatorType(TagNameEnum):
 
 
 class Expression(BaseNode):
-    """Abstract base class, as defined by FES spec."""
+    """Abstract base class, as defined by FES spec.
+
+    The FES spec defines the following subclasses:
+    * :class:`ValueReference` (pointing to a field name)
+    * :class:`Literal` (a scalar value)
+    * :class:`Function` (a transformation for a value/field)
+
+    When code uses ``Expression.child_from_xml(element)``, the AST logic will
+    initialize the correct subclass for those elements.
+    """
 
-    xml_ns = FES20
+    xml_ns = xmlns.fes20
 
     def build_lhs(self, compiler) -> str:
         """Get the expression as the left-hand-side of the equation.
@@ -86,7 +110,21 @@ class Expression(BaseNode):
 @dataclass(repr=False)
 @tag_registry.register("Literal")
 class Literal(Expression):
-    """The <fes:Literal> element that holds a literal value"""
+    """The <fes:Literal> element that holds a literal value.
+
+    This can be a string value, possibly annotated with a type::
+
+        <fes:Literal type="xs:boolean">true</fes:Literal>
+
+    Following the spec, the value may also contain a complete geometry:
+
+        <fes:Literal>
+            <gml:Envelope xmlns:gml="http://www.opengis.net/gml/3.2" srsName="urn:ogc:def:crs:EPSG::4326">
+                <gml:lowerCorner>5.7 53.1</gml:lowerCorner>
+                <gml:upperCorner>6.1 53.5</gml:upperCorner>
+            </gml:Envelope>
+        </fes:Literal>
+    """
 
     # The XSD definition even defines a sequence of xsd:any as possible member!
     raw_value: NoneType | str | GM_Object | GM_Envelope | TM_Object
@@ -112,19 +150,20 @@ class Literal(Expression):
 
     @cached_property
     def type(self) -> XsdTypes | None:
+        """Tell which datatype the literal holds.
+        This returns the type="..." value of the element.
+        """
         if not self.raw_type:
             return None
 
-        xmlns, localname = split_xml_name(self.raw_type)
-        if xmlns == "gml":
-            return XsdTypes(self.raw_type)
-        else:
-            # No idea what XMLSchema was prefixed as (could be ns0 instead of xs:)
-            return XsdTypes(localname)
+        # The raw type is already translated into a fully qualified XML name,
+        # which also allows defining types as "ns0:string", "xs:string" or "xsd:string".
+        # These are directly matched to the XsdTypes enum value.
+        return XsdTypes(self.raw_type)
 
     @classmethod
-    @expect_tag(FES20, "Literal")
-    def from_xml(cls, element: Element):
+    @expect_tag(xmlns.fes20, "Literal")
+    def from_xml(cls, element: NSElement):
         children = len(element)
         if not children:
             # Common case: value is raw text
@@ -137,7 +176,10 @@ class Literal(Expression):
                 f"Unsupported child element for <Literal> element: {element[0].tag}."
             )
 
-        return cls(raw_value=raw_value, raw_type=element.get("type"))
+        return cls(
+            raw_value=raw_value,
+            raw_type=parse_qname(element.attrib.get("type"), element.ns_aliases),
+        )
 
     def build_lhs(self, compiler) -> str:
         """Alias the value when it's used in the left-hand-side.
@@ -174,6 +216,7 @@ class ValueReference(Expression):
     """
 
     xpath: str
+    xpath_ns_aliases: dict[str, str] | None = field(compare=False, default=None)
 
     def __str__(self):
         return self.xpath
@@ -182,25 +225,26 @@ class ValueReference(Expression):
         return f"ValueReference({self.xpath!r})"
 
     @classmethod
-    @expect_tag(FES20, "ValueReference", "PropertyName", leaf=True)
-    def from_xml(cls, element: Element):
-        return cls(xpath=element.text)
+    @expect_tag(xmlns.fes20, "ValueReference", "PropertyName")
+    @expect_no_children
+    def from_xml(cls, element: NSElement):
+        return cls(xpath=element.text, xpath_ns_aliases=element.ns_aliases)
 
     def build_lhs(self, compiler) -> str:
         """Optimized LHS: there is no need to alias a field lookup through an annotation."""
-        match = self.parse_xpath(compiler.feature_type)
+        match = self.parse_xpath(compiler.feature_types)
         return match.build_lhs(compiler)
 
     def build_rhs(self, compiler) -> RhsTypes:
         """Return the value as F-expression"""
-        match = self.parse_xpath(compiler.feature_type)
+        match = self.parse_xpath(compiler.feature_types)
         return match.build_rhs(compiler)
 
-    def parse_xpath(self, feature_type=None) -> ORMPath:
+    def parse_xpath(self, feature_types: list) -> ORMPath:
         """Convert the XPath into the required ORM query elements."""
-        if feature_type is not None:
+        if feature_types:
             # Can resolve against XSD paths, find the correct DB field name
-            return feature_type.resolve_element(self.xpath)
+            return feature_types[0].resolve_element(self.xpath, self.xpath_ns_aliases)
         else:
             # Only used by unit testing (when feature_type is not given).
             parts = [word.strip() for word in self.xpath.split("/")]
@@ -221,14 +265,14 @@ class Function(Expression):
     arguments: list[Expression]  # xsd:element ref="fes20:expression"
 
     @classmethod
-    @expect_tag(FES20, "Function")
-    def from_xml(cls, element: Element):
+    @expect_tag(xmlns.fes20, "Function")
+    def from_xml(cls, element: NSElement):
         return cls(
-            name=get_attribute(element, "name"),
-            arguments=[Expression.from_child_xml(child) for child in element],
+            name=element.get_str_attribute("name"),
+            arguments=[Expression.child_from_xml(child) for child in element],
         )
 
-    def build_rhs(self, compiler) -> RhsTypes:
+    def build_rhs(self, compiler) -> models.Func:
         """Build the SQL function object"""
         db_function = function_registry.resolve_function(self.name)
         args = [arg.build_rhs(compiler) for arg in self.arguments]
@@ -236,7 +280,7 @@ class Function(Expression):
 
 
 @dataclass
-@tag_registry.register_names(BinaryOperatorType)
+@tag_registry.register(BinaryOperatorType)
 class BinaryOperator(Expression):
     """Support for FES 1.0 arithmetic operators.
 
@@ -248,12 +292,12 @@ class BinaryOperator(Expression):
     expression: tuple[Expression, Expression]
 
     @classmethod
-    def from_xml(cls, element: Element):
+    def from_xml(cls, element: NSElement):
         return cls(
             _operatorType=BinaryOperatorType.from_xml(element),
             expression=(
-                Expression.from_child_xml(element[0]),
-                Expression.from_child_xml(element[1]),
+                Expression.child_from_xml(element[0]),
+                Expression.child_from_xml(element[1]),
             ),
         )