django-gisserver 1.5.0__py3-none-any.whl → 2.0__py3-none-any.whl

gisserver/output/utils.py

@@ -1,179 +1,84 @@
-from __future__ import annotations
+"""General utilities for outputting XML content"""
 
-from collections import defaultdict
-from collections.abc import Iterable
-from itertools import islice
-from typing import TypeVar
+from datetime import date, datetime, time, timezone
+from decimal import Decimal as D
 
-from django.db import connections, models
-from lru import LRU
+from django.core.exceptions import ImproperlyConfigured
 
-M = TypeVar("M", bound=models.Model)
+from gisserver.parsers.xml import xmlns
 
-DEFAULT_SQL_CHUNK_SIZE = 2000  # allow unit tests to alter this.
+AUTO_STR = (int, float, D, date, time)
 
+COMMON_NAMESPACES = xmlns.as_namespaces()
 
-class CountingIterator(Iterable[M]):
-    """A simple iterator that counts how many results are given."""
 
-    def __init__(self, iterator: Iterable[M], max_results=0):
-        self._iterator = iterator
-        self._number_returned = 0
-        self._in_iterator = False
-        self._max_results = max_results
-        self._has_more = None
+__all__ = (
+    "attr_escape",
+    "tag_escape",
+    "to_qname",
+    "render_xmlns_attributes",
+    "value_to_text",
+    "value_to_xml_string",
+)
 
-    def __iter__(self):
-        # Count the number of returned items while reading them.
-        # Tried using map(itemgetter(0), zip(model_iter, count_iter)) but that's not faster.
-        self._in_iterator = True
-        try:
-            self._number_returned = 0
-            for instance in self._iterator:
-                if self._max_results and self._number_returned == self._max_results:
-                    self._has_more = True
-                    break
-                self._number_returned += 1
-                yield instance
-        finally:
-            if self._max_results and self._has_more is None:
-                self._has_more = False  # ignored the sentinel item
-            self._in_iterator = False
-
-    @property
-    def number_returned(self) -> int:
-        """Tell how many objects the iterator processed"""
-        if self._in_iterator:
-            raise RuntimeError("Can't read number of returned results during iteration")
-        return self._number_returned
-
-    @property
-    def has_more(self) -> bool | None:
-        return self._has_more
-
-
-class ChunkedQuerySetIterator(Iterable[M]):
-    """An optimal strategy to perform ``prefetch_related()`` on large datasets.
-
-    It fetches data from the queryset in chunks,
-    and performs ``prefetch_related()`` behavior on each chunk.
-
-    Django's ``QuerySet.prefetch_related()`` works by loading the whole queryset into memory,
-    and performing an analysis of the related objects to fetch. When working on large datasets,
-    this is very inefficient as more memory is consumed. Instead, ``QuerySet.iterator()``
-    is preferred here as it returns instances while reading them. Nothing is stored in memory.
-    Hence, both approaches are fundamentally incompatible. This class performs a
-    mixed strategy: load a chunk, and perform prefetches for that particular batch.
-
-    As extra performance benefit, a local cache avoids prefetching the same records
-    again when the next chunk is analysed. It has a "least recently used" cache to avoid
-    flooding the caches when foreign keys constantly point to different unique objects.
-    """
-
-    def __init__(self, queryset: models.QuerySet, chunk_size=None, sql_chunk_size=None):
-        """
-        :param queryset: The queryset to iterate over, that has ``prefetch_related()`` data.
-        :param chunk_size: The size of each segment to analyse in-memory for related objects.
-        :param sql_chunk_size: The size of each segment to fetch from the database,
-            used when server-side cursors are not available. The default follows Django behavior.
-        """
-        self.queryset = queryset
-        self.sql_chunk_size = sql_chunk_size or DEFAULT_SQL_CHUNK_SIZE
-        self.chunk_size = chunk_size or self.sql_chunk_size
-        self._fk_caches = defaultdict(lambda: LRU(self.chunk_size // 2))
-        self._number_returned = 0
-        self._in_iterator = False
-
-    def __iter__(self):
-        # Using iter() ensures the ModelIterable is resumed with the next chunk.
-        self._number_returned = 0
-        self._in_iterator = True
+
+def tag_escape(s: str):
+    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+def attr_escape(s: str):
+    # Slightly faster then html.escape() as it doesn't replace single quotes.
+    # Having tried all possible variants, this code still outperforms other forms of escaping.
+    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+
+
+def value_to_xml_string(value):
+    # Simple scalar value
+    if isinstance(value, str):  # most cases
+        return tag_escape(value)
+    elif isinstance(value, datetime):
+        return value.astimezone(timezone.utc).isoformat()
+    elif isinstance(value, bool):
+        return "true" if value else "false"
+    elif isinstance(value, AUTO_STR):
+        return value  # no need for _tag_escape(), and f"{value}" works faster.
+    else:
+        return tag_escape(str(value))
+
+
+def value_to_text(value):
+    # Simple scalar value, no XML escapes
+    if isinstance(value, str):  # most cases
+        return value
+    elif isinstance(value, datetime):
+        return value.astimezone(timezone.utc).isoformat()
+    elif isinstance(value, bool):
+        return "true" if value else "false"
+    else:
+        return value  # f"{value} works faster and produces the right format.
+
+
+def render_xmlns_attributes(xml_namespaces: dict[str, str]):
+    """Render XML Namespace declaration attributes"""
+    return " ".join(
+        f'xmlns:{prefix}="{xml_namespace}"' if prefix else f'xmlns="{xml_namespace}"'
+        for xml_namespace, prefix in xml_namespaces.items()
+    )
+
+
+def to_qname(namespace, localname, namespaces: dict[str, str]) -> str:
+    """Convert a fully qualified XML tag name to a prefixed short name."""
+    if namespace is None:
+        return localname
+
+    prefix = namespaces.get(namespace)  # allow ""
+    if prefix is None:
         try:
-            qs_iter = iter(self._get_queryset_iterator())
-
-            # Keep fetching chunks
-            while True:
-                instances = list(islice(qs_iter, self.chunk_size))
-                if not instances:
-                    break
-
-                # Perform prefetches on this chunk:
-                if self.queryset._prefetch_related_lookups:
-                    self._add_prefetches(instances)
-
-                # And return to parent loop
-                yield from instances
-                self._number_returned += len(instances)
-        finally:
-            self._in_iterator = False
-
-    def _get_queryset_iterator(self) -> Iterable:
-        """The body of queryset.iterator(), while circumventing prefetching."""
-        # The old code did return `self.queryset.iterator(chunk_size=self.sql_chunk_size)`
-        # However, Django 4 supports using prefetch_related() with iterator() in that scenario.
-        #
-        # This code is the core of Django's QuerySet.iterator() that only produces the
-        # old-style iteration, without any prefetches. Those are added by this class instead.
-        use_chunked_fetch = not connections[self.queryset.db].settings_dict.get(
-            "DISABLE_SERVER_SIDE_CURSORS"
-        )
-        iterable = self.queryset._iterable_class(
-            self.queryset, chunked_fetch=use_chunked_fetch, chunk_size=self.sql_chunk_size
-        )
-
-        yield from iterable
-
-    @property
-    def number_returned(self) -> int:
-        """Tell how many objects the iterator processed"""
-        if self._in_iterator:
-            raise RuntimeError("Can't read number of returned results during iteration")
-        return self._number_returned
-
-    def _add_prefetches(self, instances: list[M]):
-        """Merge the prefetched objects for this batch with the model instances."""
-        if self._fk_caches:
-            # Make sure prefetch_related_objects() doesn't have
-            # to fetch items again that infrequently changes.
-            all_restored = self._restore_caches(instances)
-            if all_restored:
-                return
-
-        # Reuse the Django machinery for retrieving missing sub objects.
-        # and analyse the ForeignKey caches to allow faster prefetches next time
-        models.prefetch_related_objects(instances, *self.queryset._prefetch_related_lookups)
-        self._persist_prefetch_cache(instances)
-
-    def _persist_prefetch_cache(self, instances):
-        """Store the prefetched data so it can be applied to the next batch"""
-        for instance in instances:
-            for lookup, obj in instance._state.fields_cache.items():
-                if obj is not None:
-                    cache = self._fk_caches[lookup]
-                    cache[obj.pk] = obj
-
-    def _restore_caches(self, instances) -> bool:
-        """Restore prefetched data to the new set of instances.
-        This avoids unneeded prefetching of the same ForeignKey relation.
-        """
-        if not instances:
-            return True
-        if not self._fk_caches:
-            return False
-
-        all_restored = True
-
-        for lookup, cache in self._fk_caches.items():
-            field = instances[0]._meta.get_field(lookup)
-            for instance in instances:
-                id_value = getattr(instance, field.attname)
-                if id_value is None:
-                    continue
-
-                obj = cache.get(id_value, None)
-                if obj is not None:
-                    instance._state.fields_cache[lookup] = obj
-                else:
-                    all_restored = False
-
-        return all_restored
+            prefix = COMMON_NAMESPACES[namespace]
+        except KeyError:
+            raise ImproperlyConfigured(
+                f"No XML namespace prefix defined for '{namespace}'.\n"
+                "This can be configured in 'WFSView.xml_namespace_aliases'."
+            ) from None
+
+    return f"{prefix}:{localname}" if prefix else localname

gisserver/output/xmlschema.py

@@ -1,47 +1,62 @@
 from __future__ import annotations
 
+import typing
 from collections import deque
 from collections.abc import Iterable
 from io import StringIO
 from typing import cast
 
 from gisserver.features import FeatureType
-from gisserver.operations.base import WFSMethod
-from gisserver.types import XsdComplexType
+from gisserver.parsers.xml import xmlns
+from gisserver.types import XsdComplexType, XsdElement
 
-from .base import OutputRenderer
+from .base import XmlOutputRenderer
+from .utils import to_qname
 
+if typing.TYPE_CHECKING:
+    from gisserver.operations.base import WFSOperation
 
-class XMLSchemaRenderer(OutputRenderer):
+
+class XmlSchemaRenderer(XmlOutputRenderer):
     """Output rendering for DescribeFeatureType.
 
     This renders a valid XSD schema that describes the data type of the feature.
     """
 
     content_type = "application/gml+xml; version=3.2"  # mandatory for WFS
+    xml_namespaces = {
+        # Default extra namespaces to include in the xmlns="..." attributes
+        "http://www.w3.org/2001/XMLSchema": "",  # no xs:string, but "string"
+        "http://www.opengis.net/gml/3.2": "gml",
+    }
 
-    def __init__(self, method: WFSMethod, feature_types: list[FeatureType]):
+    def __init__(self, operation: WFSOperation, feature_types: list[FeatureType]):
         """Overwritten method to handle the DescribeFeatureType context."""
-        self.server_url = method.view.server_url
-        self.app_xml_namespace = method.view.xml_namespace
+        super().__init__(operation)
         self.feature_types = feature_types
 
+        # For rendering type="..." fields, avoid xs: prefix.
+        self.type_namespaces = self.app_namespaces.copy()
+        self.type_namespaces[xmlns.xs.value] = ""  # no "xs:string" but "string"
+
+    def get_headers(self):
+        """Make wget output slightly nicer."""
+        typenames = "+".join(feature_type.name for feature_type in self.feature_types)
+        return {"Content-Disposition": f'inline; filename="{typenames}.xsd"'}
+
     def render_stream(self):
-        # For now, all features have the same XML namespace despite allowing
-        # the prefixes to be different.
-        xmlns_features = "\n   ".join(
-            f'xmlns:{p}="{self.app_xml_namespace}"'
-            for p in sorted({feature_type.xml_prefix for feature_type in self.feature_types})
-        )
+        # Render first with original app_namespaces
+        xmlns_attrib = self.render_xmlns_attributes()
+
+        # Allow targetNamespace to be unprefixed by self.node_to_qname().
+        target_namespace = self.feature_types[0].xml_namespace
+        self.app_namespaces[target_namespace] = ""  # no "app:element" but "element"
 
-        output = StringIO()
+        self.output = output = StringIO()
         output.write(
             f"""<?xml version='1.0' encoding="UTF-8" ?>
-<schema
-   xmlns="http://www.w3.org/2001/XMLSchema"
-   {xmlns_features}
-   xmlns:gml="http://www.opengis.net/gml/3.2"
-   targetNamespace="{self.app_xml_namespace}"
+<schema {xmlns_attrib}
+   targetNamespace="{target_namespace}"
    elementFormDefault="qualified" version="0.1">
 
 """
@@ -50,7 +65,7 @@ class XMLSchemaRenderer(OutputRenderer):
         output.write("\n")
 
         for feature_type in self.feature_types:
-            output.write(self.render_feature_type(feature_type))
+            self.write_feature_type(feature_type)
 
         output.write("</schema>\n")
         return output.getvalue()
@@ -61,45 +76,69 @@ class XMLSchemaRenderer(OutputRenderer):
             ' schemaLocation="http://schemas.opengis.net/gml/3.2.1/gml.xsd" />\n'
         )
 
-    def render_feature_type(self, feature_type: FeatureType):
-        output = StringIO()
+    def write_feature_type(self, feature_type: FeatureType):
         xsd_type: XsdComplexType = feature_type.xsd_type
 
         # This declares the that a top-level <app:featureName> is a class of a type.
-        output.write(
-            f'  <element name="{feature_type.name}"'
-            f' type="{xsd_type}" substitutionGroup="gml:AbstractFeature" />\n\n'
+        xsd_type_qname = self.to_qname(xsd_type, self.type_namespaces)
+        feature_qname = to_qname(
+            feature_type.xml_namespace, feature_type.name, self.app_namespaces
+        )
+        self.output.write(
+            f'  <element name="{feature_qname}"'
+            f' type="{xsd_type_qname}" substitutionGroup="gml:AbstractFeature" />\n\n'
         )
 
         # Next, the complexType is rendered that defines the element contents.
         # Next, the complexType(s) are rendered that defines the element contents.
         # In case any fields are expanded (hence become subtypes), these are also included.
-        output.write(self.render_complex_type(xsd_type))
+        self.write_complex_type(xsd_type)
         for complex_type in self._get_complex_types(xsd_type):
-            output.write(self.render_complex_type(complex_type))
-
-        return output.getvalue()
+            self.write_complex_type(complex_type)
 
-    def render_complex_type(self, complex_type: XsdComplexType):
+    def write_complex_type(self, complex_type: XsdComplexType):
         """Write the definition of a single class."""
-        output = StringIO()
-        output.write(
-            f'  <complexType name="{complex_type.name}">\n'
-            "    <complexContent>\n"
-            f'      <extension base="{complex_type.base}">\n'
-            "        <sequence>\n"
-        )
+        complex_qname = self.to_qname(complex_type)
+        self.output.write(f'  <complexType name="{complex_qname}">\n')
 
-        for xsd_element in complex_type.elements:
-            output.write(f"          {xsd_element}\n")
+        if complex_type.base is not None:
+            base_qname = self.to_qname(complex_type.base)
+            self.output.write(
+                f"    <complexContent>\n"  # extend base class
+                f'      <extension base="{base_qname}">\n'
+            )
+            indent = "   "
+        else:
+            indent = ""
 
-        output.write(
-            "        </sequence>\n"
-            "      </extension>\n"
-            "    </complexContent>\n"
-            "  </complexType>\n\n"
-        )
-        return output.getvalue()
+        self.output.write(f"    {indent}<sequence>\n")
+
+        for xsd_element in complex_type.elements:
+            self.output.write(f"      {indent}{self.render_element(xsd_element)}\n")
+
+        self.output.write(f"    {indent}</sequence>\n")
+        if complex_type.base is not None:
+            self.output.write(
+                "      </extension>\n"  # end extension
+                "    </complexContent>\n"
+            )
+        self.output.write("  </complexType>\n\n")
+
+    def render_element(self, xsd_element: XsdElement):
+        """Staticmethod for unit testing."""
+        qname = self.to_qname(xsd_element)
+        type_qname = self.to_qname(xsd_element.type, self.type_namespaces)
+
+        attributes = [f'name="{qname}" type="{type_qname}"']
+        if xsd_element.min_occurs is not None:
+            attributes.append(f'minOccurs="{xsd_element.min_occurs}"')
+        if xsd_element.max_occurs is not None:
+            attributes.append(f'maxOccurs="{xsd_element.max_occurs}"')
+        if xsd_element.nillable:
+            str_bool = "true" if xsd_element.nillable else "false"
+            attributes.append(f'nillable="{str_bool}"')
+
+        return "<element {} />".format(" ".join(attributes))
 
     def _get_complex_types(self, root: XsdComplexType) -> Iterable[XsdComplexType]:
         """Find all fields that reference to complex types, including nested elements."""

gisserver/parsers/__init__.py

@@ -1,13 +1,13 @@
-"""All parser logic to process <fes:...> and <gml:...> tags."""
+"""All parser logic to process incoming XML data.
 
-from .fes20 import Filter
-from .fes20 import function_registry as fes_function_registry
-from .gml import parse_gml
+This handles all tags, including:
 
-parse_fes = Filter.from_string
+* ``<wfs:...>``
+* ``<fes:...>``
+* ``<gml:...>``
 
-__all__ = [
-    "parse_fes",
-    "parse_gml",
-    "fes_function_registry",
-]
+Internally, the XML string is translated into an Abstract Syntax Tree (AST).
+These objects contain to logic to process each bit of the XML tree.
+The GET request parameters are translated into that same tree structure,
+to have a uniform processing of the request.
+"""