udata 10.9.1.dev37462__py2.py3-none-any.whl → 10.9.1.dev37604__py2.py3-none-any.whl

udata/core/topic/forms.py

@@ -2,9 +2,30 @@ from udata.core.spatial.forms import SpatialCoverageField
 from udata.forms import ModelForm, fields, validators
 from udata.i18n import lazy_gettext as _
 
-from .models import Topic
+from .models import Topic, TopicElement
 
-__all__ = ("TopicForm",)
+__all__ = ("TopicForm", "TopicElementForm")
+
+
+class TopicElementForm(ModelForm):
+    model_class = TopicElement
+
+    title = fields.StringField(_("Title"))
+    description = fields.StringField(_("Description"))
+    tags = fields.TagField(_("Tags"))
+    extras = fields.ExtrasField()
+    element = fields.ModelField(_("Element"))
+
+    def validate(self, extra_validators=None):
+        """
+        Make sure that either title or element is set.
+        (Empty nested element is a valid use case for "placeholder" elements)
+        """
+        validation = super().validate(extra_validators)
+        if not self.element.data and not self.title.data:
+            self.element.errors.append(_("A topic element must have a title or an element."))
+            return False
+        return validation
 
 
 class TopicForm(ModelForm):
@@ -16,8 +37,41 @@ class TopicForm(ModelForm):
     name = fields.StringField(_("Name"), [validators.DataRequired()])
     description = fields.MarkdownField(_("Description"), [validators.DataRequired()])
 
-    datasets = fields.DatasetListField(_("Associated datasets"))
-    reuses = fields.ReuseListField(_("Associated reuses"))
+    elements = fields.NestedModelList(TopicElementForm)
+
+    @property
+    def data(self):
+        """Override to exclude non-model fields from data"""
+        # Get the base data from WTForms
+        base_data = super().data
+        # Filter out non-model fields
+        return {name: value for name, value in base_data.items() if name != "elements"}
+
+    def populate_obj(self, obj):
+        """Override populate_obj to exclude non-model fields"""
+        # Only populate model fields, skip elements
+        for name, field in self._fields.items():
+            if name != "elements":
+                field.populate_obj(obj, name)
+
+    def save(self, commit=True, **kwargs):
+        """Custom save to handle TopicElement creation properly"""
+        # Store elements data before parent save
+        elements_data = self.elements.data
+
+        # Use parent save method (elements field is excluded via populate_obj)
+        topic = super().save(commit=commit, **kwargs)
+
+        # Create elements and associate them with the topic
+        for element_data in elements_data or []:
+            element_form = TopicElementForm(data=element_data)
+            if element_form.validate():
+                element = element_form.save(commit=False)
+                element.topic = topic
+                if commit:
+                    element.save()
+
+        return topic
 
     spatial = SpatialCoverageField(
         _("Spatial coverage"), description=_("The geographical area covered by the data.")

udata/core/topic/models.py

@@ -1,13 +1,47 @@
 from blinker import Signal
-from mongoengine.signals import post_save, pre_save
+from mongoengine.signals import post_delete, post_save
 
 from udata.api_fields import field
 from udata.core.activity.models import Auditable
+from udata.core.dataset.models import Dataset
 from udata.core.owned import Owned, OwnedQuerySet
+from udata.core.reuse.models import Reuse
 from udata.models import SpatialCoverage, db
 from udata.search import reindex
+from udata.tasks import as_task_param
 
-__all__ = ("Topic",)
+__all__ = ("Topic", "TopicElement")
+
+
+class TopicElement(db.Document):
+    title = field(db.StringField(required=False))
+    description = field(db.StringField(required=False))
+    tags = field(db.ListField(db.StringField()))
+    extras = field(db.ExtrasField())
+    element = field(db.GenericReferenceField(choices=[Dataset, Reuse]))
+    # Made optional to allow proper form handling with commit=False
+    topic = field(db.ReferenceField("Topic", required=False))
+
+    meta = {
+        "indexes": [
+            {
+                "fields": ["$title", "$description"],
+            }
+        ],
+        "auto_create_index_on_save": True,
+    }
+
+    @classmethod
+    def post_save(cls, sender, document, **kwargs):
+        """Trigger reindex when element is saved"""
+        if document.topic and document.element and hasattr(document.element, "id"):
+            reindex.delay(*as_task_param(document.element))
+
+    @classmethod
+    def post_delete(cls, sender, document, **kwargs):
+        """Trigger reindex when element is deleted"""
+        if document.topic and document.element and hasattr(document.element, "id"):
+            reindex.delay(*as_task_param(document.element))
 
 
 class Topic(db.Datetimed, Auditable, db.Document, Owned):
@@ -20,9 +54,6 @@ class Topic(db.Datetimed, Auditable, db.Document, Owned):
     tags = field(db.ListField(db.StringField()))
     color = field(db.IntField())
 
-    datasets = field(db.ListField(db.LazyReferenceField("Dataset", reverse_delete_rule=db.PULL)))
-    reuses = field(db.ListField(db.LazyReferenceField("Reuse", reverse_delete_rule=db.PULL)))
-
     featured = field(db.BooleanField(default=False), auditable=False)
     private = field(db.BooleanField())
     extras = field(db.ExtrasField(), auditable=False)
@@ -30,7 +61,14 @@ class Topic(db.Datetimed, Auditable, db.Document, Owned):
     spatial = field(db.EmbeddedDocumentField(SpatialCoverage))
 
     meta = {
-        "indexes": ["$name", "created_at", "slug"] + Owned.meta["indexes"],
+        "indexes": [
+            {
+                "fields": ["$name", "$description"],
+            },
+            "created_at",
+            "slug",
+        ]
+        + Owned.meta["indexes"],
         "ordering": ["-created_at"],
         "auto_create_index_on_save": True,
         "queryset_class": OwnedQuerySet,
@@ -43,27 +81,34 @@ class Topic(db.Datetimed, Auditable, db.Document, Owned):
     def __str__(self):
         return self.name
 
-    @classmethod
-    def pre_save(cls, sender, document, **kwargs):
-        # Try catch is to prevent the mechanism to crash at the
-        # creation of the Topic, where an original state does not exist.
-        try:
-            original_doc = sender.objects.get(id=document.id)
-            # Get the diff between the original and current datasets
-            datasets_list_dif = set(original_doc.datasets) ^ set(document.datasets)
-        except cls.DoesNotExist:
-            datasets_list_dif = document.datasets
-        for dataset in datasets_list_dif:
-            reindex.delay("Dataset", str(dataset.pk))
-
     def count_discussions(self):
         # There are no metrics on Topic to store discussions count
         pass
 
+    @property
+    def elements(self):
+        """Get elements associated with this topic"""
+        return TopicElement.objects(topic=self)
+
+    def get_nested_elements_ids(self, cls: str) -> set[str]:
+        """Optimized query to get objects ids from related elements, filtered by class."""
+        # Return empty set if topic doesn't have an ID yet
+        if not self.id:
+            return set()
+
+        return set(
+            str(elem["element"]["_ref"].id)
+            for elem in TopicElement.objects.filter(topic=self, __raw__={"element._cls": cls})
+            .fields(element=1)
+            .no_dereference()
+            .as_pymongo()
+        )
+
     def self_web_url(self, **kwargs):
         # Useful for Discussions to call self_web_url on their `subject`
         return None
 
 
-pre_save.connect(Topic.pre_save, sender=Topic)
 post_save.connect(Topic.post_save, sender=Topic)
+post_save.connect(TopicElement.post_save, sender=TopicElement)
+post_delete.connect(TopicElement.post_delete, sender=TopicElement)

udata/core/topic/parsers.py

@@ -3,6 +3,46 @@ from flask_restx.inputs import boolean
 
 from udata.api import api
 from udata.api.parsers import ModelApiParser
+from udata.core.topic import DEFAULT_PAGE_SIZE
+
+
+class TopicElementsParser(ModelApiParser):
+    def __init__(self):
+        super().__init__()
+        self.parser.add_argument(
+            "page", type=int, default=1, location="args", help="The page to fetch"
+        )
+        self.parser.add_argument(
+            "page_size",
+            type=int,
+            default=DEFAULT_PAGE_SIZE,
+            location="args",
+            help="The page size to fetch",
+        )
+        self.parser.add_argument(
+            "class",
+            type=str,
+            location="args",
+            help="The class of elements to fetch (eg. Dataset or Reuse)",
+        )
+        self.parser.add_argument(
+            "q", type=str, location="args", help="query string to search through elements"
+        )
+        self.parser.add_argument("tag", type=str, location="args", action="append")
+
+    @staticmethod
+    def parse_filters(elements, args):
+        if args.get("q"):
+            phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
+            elements = elements.search_text(phrase_query)
+        if args.get("tag"):
+            elements = elements.filter(tags__all=args["tag"])
+        if element_class := args.get("class"):
+            if element_class == "None":
+                elements = elements.filter(element=None)
+            else:
+                elements = elements.filter(__raw__={"element._cls": element_class})
+        return elements
 
 
 class TopicApiParser(ModelApiParser):

udata/core/topic/tasks.py

@@ -0,0 +1,11 @@
+from udata.core.topic.models import TopicElement
+from udata.tasks import job
+
+
+@job("purge-topics-elements")
+def purge_topics_elements(self):
+    """
+    Purge topic elements that have neither title nor element
+    This should run *after* the purge-reuses and purge-datasets jobs
+    """
+    TopicElement.objects().filter(element=None, title=None).delete()

udata/forms/fields.py

@@ -480,7 +480,14 @@ class ModelField(Field):
         if not valuelist or len(valuelist) != 1 or not valuelist[0]:
             return
         specs = valuelist[0]
-        model_field = getattr(self._form.model_class, self.name)
+
+        try:
+            model_field = getattr(self._form.model_class, self.name)
+        # Handle the case where the field it is not fetchable in the model via self.name
+        # This can happen in nested forms like NestedModelList, where self.name is {parent}-{index}-{short_name}
+        except AttributeError:
+            model_field = getattr(self._form.model_class, self.short_name)
+
         if isinstance(specs, str):
             specs = {"id": specs}
         elif not specs.get("id", None):

udata/harvest/backends/dcat.py

@@ -2,10 +2,10 @@ import logging
 from datetime import date
 from typing import ClassVar, Generator
 
-import lxml.etree as ET
 from flask import current_app
 from rdflib import Graph
 from rdflib.namespace import RDF
+from saxonche import PySaxonProcessor, PyXdmNode
 from typing_extensions import override
 
 from udata.core.dataservices.rdf import dataservice_from_rdf
@@ -47,7 +47,6 @@ KNOWN_PAGINATION = (
 )
 
 CSW_NAMESPACE = "http://www.opengis.net/cat/csw/2.0.2"
-OWS_NAMESPACE = "http://www.opengis.net/ows"
 
 # Useful to patch essential failing URIs
 URIS_TO_REPLACE = {
@@ -325,9 +324,23 @@ class CswDcatBackend(DcatBackend):
 
     CSW_OUTPUT_SCHEMA = "http://www.w3.org/ns/dcat#"
 
+    SAXON_SECURITY_FEATURES = {
+        "http://saxon.sf.net/feature/allow-external-functions": "false",
+        "http://saxon.sf.net/feature/parserFeature?uri=http://apache.org/xml/features/nonvalidating/load-external-dtd": "false",
+        "http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities": "false",
+        "http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-parameter-entities": "false",
+    }
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.xml_parser = ET.XMLParser(resolve_entities=False)
+        self.saxon_proc = PySaxonProcessor(license=False)
+        for feature, value in self.SAXON_SECURITY_FEATURES.items():
+            self.saxon_proc.set_configuration_property(feature, value)
+        self.saxon_proc.set_configuration_property(
+            "http://saxon.sf.net/feature/strip-whitespace", "all"
+        )
+        self.xpath_proc = self.saxon_proc.new_xpath_processor()
+        self.xpath_proc.declare_namespace("csw", CSW_NAMESPACE)
 
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
@@ -341,19 +354,23 @@ class CswDcatBackend(DcatBackend):
             response = self.post(url, data=data, headers={"Content-Type": "application/xml"})
             response.raise_for_status()
 
-            content = response.content
-            tree = ET.fromstring(content, parser=self.xml_parser)
-            if tree.tag == "{" + OWS_NAMESPACE + "}ExceptionReport":
-                raise ValueError(f"Failed to query CSW:\n{content}")
+            text = response.text
+            tree = self.saxon_proc.parse_xml(xml_text=text)
+            self.xpath_proc.set_context(xdm_item=tree)
 
-            search_results = tree.find("csw:SearchResults", {"csw": CSW_NAMESPACE})
-            if not search_results:
+            # Using * namespace so we don't have to enumerate ows versions
+            if self.xpath_proc.evaluate("/*:ExceptionReport"):
+                raise ValueError(f"Failed to query CSW:\n{text}")
+
+            if r := self.xpath_proc.evaluate("/csw:GetRecordsResponse/csw:SearchResults"):
+                search_results = r.head
+            else:
                 log.error(f"No search results found for {url} on page {page_number}")
                 return
 
-            for result in search_results:
+            for result in search_results.children:
                 subgraph = Graph(namespace_manager=namespace_manager)
-                doc = ET.tostring(self.as_dcat(result))
+                doc = self.as_dcat(result).to_string("utf-8")
                 subgraph.parse(data=doc, format=fmt)
 
                 if not subgraph.subjects(
@@ -371,7 +388,7 @@ class CswDcatBackend(DcatBackend):
             if not start:
                 return
 
-    def as_dcat(self, tree: ET._Element) -> ET._Element:
+    def as_dcat(self, tree: PyXdmNode) -> PyXdmNode:
         """
         Return the input tree as a DCAT tree.
         For CswDcatBackend, this method return the incoming tree as-is, since it's already DCAT.
@@ -379,10 +396,10 @@ class CswDcatBackend(DcatBackend):
         """
         return tree
 
-    def next_position(self, start: int, search_results: ET._Element) -> int | None:
-        next_record = int(search_results.attrib["nextRecord"])
-        matched_count = int(search_results.attrib["numberOfRecordsMatched"])
-        returned_count = int(search_results.attrib["numberOfRecordsReturned"])
+    def next_position(self, start: int, search_results: PyXdmNode) -> int | None:
+        next_record = int(search_results.get_attribute_value("nextRecord"))
+        matched_count = int(search_results.get_attribute_value("numberOfRecordsMatched"))
+        returned_count = int(search_results.get_attribute_value("numberOfRecordsReturned"))
 
         # Break conditions copied gratefully from
         # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
@@ -423,9 +440,13 @@ class CswIso19139DcatBackend(CswDcatBackend):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         xslt_url = current_app.config["HARVEST_ISO19139_XSLT_URL"]
-        xslt = ET.fromstring(self.get(xslt_url).content, parser=self.xml_parser)
-        self.transform = ET.XSLT(xslt)
+        xslt_text = self.get(xslt_url).text
+        xslt_proc = self.saxon_proc.new_xslt30_processor()
+        self.xslt_exec = xslt_proc.compile_stylesheet(stylesheet_text=xslt_text)
+        self.xslt_exec.set_parameter(
+            "CoupledResourceLookUp", self.saxon_proc.make_string_value("disabled")
+        )
 
     @override
-    def as_dcat(self, tree: ET._Element) -> ET._Element:
-        return self.transform(tree, CoupledResourceLookUp="'disabled'")
+    def as_dcat(self, tree: PyXdmNode) -> PyXdmNode:
+        return self.xslt_exec.transform_to_value(xdm_node=tree).head

udata/harvest/tests/test_dcat_backend.py

@@ -881,6 +881,95 @@ class CswDcatBackendTest:
         assert "User-Agent" in get_mock.last_request.headers
         assert get_mock.last_request.headers["User-Agent"] == "uData/0.1 csw-dcat"
 
+    def test_csw_error(self, rmock):
+        exception_report = """<?xml version="1.0" encoding="UTF-8"?>
+        <ows:ExceptionReport xmlns:ows="http://www.opengis.net/ows/1.1"
+                             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                             xsi:schemaLocation="http://www.opengis.net/ows/1.1 http://schemas.opengis.net/ows/1.1.0/owsExceptionReport.xsd">
+          <ows:Exception exceptionCode="MissingParameterValue" locator="request">
+            <ows:ExceptionText>Mandatory parameter &lt;request&gt; was not specified</ows:ExceptionText>
+          </ows:Exception>
+        </ows:ExceptionReport>
+        """
+        rmock.head(rmock.ANY, headers={"Content-Type": "application/xml"})
+        rmock.post(rmock.ANY, text=exception_report)
+        source = HarvestSourceFactory(backend="csw-dcat")
+
+        actions.run(source)
+
+        source.reload()
+        job = source.get_last_job()
+
+        assert len(job.errors) == 1
+        assert "Failed to query CSW" in job.errors[0].message
+        assert job.status == "failed"
+
+    def test_disallow_external_entities(self, rmock):
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE root [
+          <!ENTITY entity SYSTEM "data:text/plain,EXTERNAL">
+        ]>
+        <csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                                xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
+          <csw:SearchStatus timestamp="2023-03-03T16:09:50.697645Z" />
+          <csw:SearchResults numberOfRecordsMatched="1" numberOfRecordsReturned="1" elementSet="full" nextRecord="0">
+            <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+              <rdf:Description rdf:about="https://example.com/test/">
+                <dct:identifier>https://example.com/test/</dct:identifier>
+                <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
+                <dct:title>test&entity;</dct:title>
+              </rdf:Description>
+            </rdf:RDF>
+          </csw:SearchResults>
+        </csw:GetRecordsResponse>
+        """
+
+        rmock.head(rmock.ANY, headers={"Content-Type": "application/xml"})
+        rmock.post(rmock.ANY, text=xml)
+        source = HarvestSourceFactory(backend="csw-dcat")
+
+        actions.run(source)
+
+        source.reload()
+        job = source.get_last_job()
+
+        assert job.status == "done"
+        assert Dataset.objects.first().title == "test"
+
+    def test_disallow_external_dtd(self, rmock):
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE root SYSTEM "http://www.example.com/evil.dtd">
+        <csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                                xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
+          <csw:SearchStatus timestamp="2023-03-03T16:09:50.697645Z" />
+          <csw:SearchResults numberOfRecordsMatched="1" numberOfRecordsReturned="1" elementSet="full" nextRecord="0">
+            <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+              <rdf:Description rdf:about="https://example.com/test/">
+                <dct:identifier>https://example.com/test/</dct:identifier>
+                <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
+                <dct:title>test</dct:title>
+              </rdf:Description>
+            </rdf:RDF>
+          </csw:SearchResults>
+        </csw:GetRecordsResponse>
+        """
+
+        rmock.get("http://www.example.com/evil.dtd", status_code=404)
+        rmock.head(rmock.ANY, headers={"Content-Type": "application/xml"})
+        rmock.post(rmock.ANY, text=xml)
+        source = HarvestSourceFactory(backend="csw-dcat")
+
+        actions.run(source)
+
+        source.reload()
+        job = source.get_last_job()
+
+        assert not any(h.method == "GET" for h in rmock.request_history)
+        assert job.status == "done"
+        assert len(job.items) == 1
+
 
 @pytest.mark.usefixtures("clean_db")
 @pytest.mark.options(PLUGINS=["csw"])

udata/migrations/2025-05-26-migrate-topics-to-elements.py

@@ -0,0 +1,59 @@
+"""Migrate topic.datasets and topics.reuses to topic.elements with TopicElement.topic references"""
+
+import logging
+
+from bson import DBRef, ObjectId
+from mongoengine.connection import get_db
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Processing topics…")
+
+    topics = get_db().topic.find()
+
+    for topic in topics:
+        log.info(f"Processing topic {topic['_id']}…")
+        total_elements = 0
+
+        # Convert datasets to TopicElement documents
+        for dataset_id in topic.get("datasets", []):
+            element_doc = {
+                "_id": ObjectId(),
+                "tags": [],
+                "extras": {},
+                "element": {"_cls": "Dataset", "_ref": DBRef("dataset", dataset_id)},
+                "topic": topic["_id"],  # Reference to the topic
+            }
+
+            # Insert TopicElement document
+            get_db().topic_element.insert_one(element_doc)
+            total_elements += 1
+
+        # Convert reuses to TopicElement documents
+        for reuse_id in topic.get("reuses", []):
+            element_doc = {
+                "_id": ObjectId(),
+                "tags": [],
+                "extras": {},
+                "element": {"_cls": "Reuse", "_ref": DBRef("reuse", reuse_id)},
+                "topic": topic["_id"],  # Reference to the topic
+            }
+
+            # Insert TopicElement document
+            get_db().topic_element.insert_one(element_doc)
+            total_elements += 1
+
+        log.info(f"Topic: {topic.get('name', 'Unnamed')} (ID: {topic['_id']})")
+        log.info(f"  - Converting {len(topic.get('datasets', []))} datasets")
+        log.info(f"  - Converting {len(topic.get('reuses', []))} reuses")
+        log.info(f"  - Total elements: {total_elements}")
+
+        # Remove old fields from topic document
+        get_db().topic.update_one(
+            {"_id": topic["_id"]},
+            {
+                "$unset": {"datasets": 1, "reuses": 1},  # Remove old fields
+            },
+        )

udata/migrations/2025-06-02-delete-topic-name-index.py

@@ -0,0 +1,19 @@
+"""Delete Topic index 'name_text'"""
+
+import logging
+
+from mongoengine.connection import get_db
+from pymongo.errors import OperationFailure
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Deleting index…")
+
+    collection = get_db().topic
+
+    try:
+        collection.drop_index("name_text")
+    except OperationFailure:
+        log.info("Index does not exist?", exc_info=True)