udata 10.9.0__py2.py3-none-any.whl → 10.9.1.dev37499__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "10.9.0"
+__version__ = "10.9.1.dev"
 __description__ = "Open data portal"

udata/harvest/backends/dcat.py

@@ -2,10 +2,10 @@ import logging
 from datetime import date
 from typing import ClassVar, Generator
 
-import lxml.etree as ET
 from flask import current_app
 from rdflib import Graph
 from rdflib.namespace import RDF
+from saxonche import PySaxonProcessor, PyXdmNode
 from typing_extensions import override
 
 from udata.core.dataservices.rdf import dataservice_from_rdf
@@ -47,7 +47,6 @@ KNOWN_PAGINATION = (
 )
 
 CSW_NAMESPACE = "http://www.opengis.net/cat/csw/2.0.2"
-OWS_NAMESPACE = "http://www.opengis.net/ows"
 
 # Useful to patch essential failing URIs
 URIS_TO_REPLACE = {
@@ -325,9 +324,23 @@ class CswDcatBackend(DcatBackend):
 
     CSW_OUTPUT_SCHEMA = "http://www.w3.org/ns/dcat#"
 
+    SAXON_SECURITY_FEATURES = {
+        "http://saxon.sf.net/feature/allow-external-functions": "false",
+        "http://saxon.sf.net/feature/parserFeature?uri=http://apache.org/xml/features/nonvalidating/load-external-dtd": "false",
+        "http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities": "false",
+        "http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-parameter-entities": "false",
+    }
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.xml_parser = ET.XMLParser(resolve_entities=False)
+        self.saxon_proc = PySaxonProcessor(license=False)
+        for feature, value in self.SAXON_SECURITY_FEATURES.items():
+            self.saxon_proc.set_configuration_property(feature, value)
+        self.saxon_proc.set_configuration_property(
+            "http://saxon.sf.net/feature/strip-whitespace", "all"
+        )
+        self.xpath_proc = self.saxon_proc.new_xpath_processor()
+        self.xpath_proc.declare_namespace("csw", CSW_NAMESPACE)
 
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
@@ -341,19 +354,23 @@ class CswDcatBackend(DcatBackend):
             response = self.post(url, data=data, headers={"Content-Type": "application/xml"})
             response.raise_for_status()
 
-            content = response.content
-            tree = ET.fromstring(content, parser=self.xml_parser)
-            if tree.tag == "{" + OWS_NAMESPACE + "}ExceptionReport":
-                raise ValueError(f"Failed to query CSW:\n{content}")
+            text = response.text
+            tree = self.saxon_proc.parse_xml(xml_text=text)
+            self.xpath_proc.set_context(xdm_item=tree)
 
-            search_results = tree.find("csw:SearchResults", {"csw": CSW_NAMESPACE})
-            if not search_results:
+            # Using * namespace so we don't have to enumerate ows versions
+            if self.xpath_proc.evaluate("/*:ExceptionReport"):
+                raise ValueError(f"Failed to query CSW:\n{text}")
+
+            if r := self.xpath_proc.evaluate("/csw:GetRecordsResponse/csw:SearchResults"):
+                search_results = r.head
+            else:
                 log.error(f"No search results found for {url} on page {page_number}")
                 return
 
-            for result in search_results:
+            for result in search_results.children:
                 subgraph = Graph(namespace_manager=namespace_manager)
-                doc = ET.tostring(self.as_dcat(result))
+                doc = self.as_dcat(result).to_string("utf-8")
                 subgraph.parse(data=doc, format=fmt)
 
                 if not subgraph.subjects(
@@ -371,7 +388,7 @@ class CswDcatBackend(DcatBackend):
             if not start:
                 return
 
-    def as_dcat(self, tree: ET._Element) -> ET._Element:
+    def as_dcat(self, tree: PyXdmNode) -> PyXdmNode:
         """
         Return the input tree as a DCAT tree.
         For CswDcatBackend, this method return the incoming tree as-is, since it's already DCAT.
@@ -379,10 +396,10 @@ class CswDcatBackend(DcatBackend):
         """
         return tree
 
-    def next_position(self, start: int, search_results: ET._Element) -> int | None:
-        next_record = int(search_results.attrib["nextRecord"])
-        matched_count = int(search_results.attrib["numberOfRecordsMatched"])
-        returned_count = int(search_results.attrib["numberOfRecordsReturned"])
+    def next_position(self, start: int, search_results: PyXdmNode) -> int | None:
+        next_record = int(search_results.get_attribute_value("nextRecord"))
+        matched_count = int(search_results.get_attribute_value("numberOfRecordsMatched"))
+        returned_count = int(search_results.get_attribute_value("numberOfRecordsReturned"))
 
         # Break conditions copied gratefully from
         # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
@@ -423,9 +440,13 @@ class CswIso19139DcatBackend(CswDcatBackend):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         xslt_url = current_app.config["HARVEST_ISO19139_XSLT_URL"]
-        xslt = ET.fromstring(self.get(xslt_url).content, parser=self.xml_parser)
-        self.transform = ET.XSLT(xslt)
+        xslt_text = self.get(xslt_url).text
+        xslt_proc = self.saxon_proc.new_xslt30_processor()
+        self.xslt_exec = xslt_proc.compile_stylesheet(stylesheet_text=xslt_text)
+        self.xslt_exec.set_parameter(
+            "CoupledResourceLookUp", self.saxon_proc.make_string_value("disabled")
+        )
 
     @override
-    def as_dcat(self, tree: ET._Element) -> ET._Element:
-        return self.transform(tree, CoupledResourceLookUp="'disabled'")
+    def as_dcat(self, tree: PyXdmNode) -> PyXdmNode:
+        return self.xslt_exec.transform_to_value(xdm_node=tree).head

udata/harvest/tests/test_dcat_backend.py

@@ -881,6 +881,95 @@ class CswDcatBackendTest:
         assert "User-Agent" in get_mock.last_request.headers
         assert get_mock.last_request.headers["User-Agent"] == "uData/0.1 csw-dcat"
 
+    def test_csw_error(self, rmock):
+        exception_report = """<?xml version="1.0" encoding="UTF-8"?>
+        <ows:ExceptionReport xmlns:ows="http://www.opengis.net/ows/1.1"
+                             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                             xsi:schemaLocation="http://www.opengis.net/ows/1.1 http://schemas.opengis.net/ows/1.1.0/owsExceptionReport.xsd">
+          <ows:Exception exceptionCode="MissingParameterValue" locator="request">
+            <ows:ExceptionText>Mandatory parameter &lt;request&gt; was not specified</ows:ExceptionText>
+          </ows:Exception>
+        </ows:ExceptionReport>
+        """
+        rmock.head(rmock.ANY, headers={"Content-Type": "application/xml"})
+        rmock.post(rmock.ANY, text=exception_report)
+        source = HarvestSourceFactory(backend="csw-dcat")
+
+        actions.run(source)
+
+        source.reload()
+        job = source.get_last_job()
+
+        assert len(job.errors) == 1
+        assert "Failed to query CSW" in job.errors[0].message
+        assert job.status == "failed"
+
+    def test_disallow_external_entities(self, rmock):
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE root [
+          <!ENTITY entity SYSTEM "data:text/plain,EXTERNAL">
+        ]>
+        <csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                                xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
+          <csw:SearchStatus timestamp="2023-03-03T16:09:50.697645Z" />
+          <csw:SearchResults numberOfRecordsMatched="1" numberOfRecordsReturned="1" elementSet="full" nextRecord="0">
+            <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+              <rdf:Description rdf:about="https://example.com/test/">
+                <dct:identifier>https://example.com/test/</dct:identifier>
+                <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
+                <dct:title>test&entity;</dct:title>
+              </rdf:Description>
+            </rdf:RDF>
+          </csw:SearchResults>
+        </csw:GetRecordsResponse>
+        """
+
+        rmock.head(rmock.ANY, headers={"Content-Type": "application/xml"})
+        rmock.post(rmock.ANY, text=xml)
+        source = HarvestSourceFactory(backend="csw-dcat")
+
+        actions.run(source)
+
+        source.reload()
+        job = source.get_last_job()
+
+        assert job.status == "done"
+        assert Dataset.objects.first().title == "test"
+
+    def test_disallow_external_dtd(self, rmock):
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE root SYSTEM "http://www.example.com/evil.dtd">
+        <csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                                xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd">
+          <csw:SearchStatus timestamp="2023-03-03T16:09:50.697645Z" />
+          <csw:SearchResults numberOfRecordsMatched="1" numberOfRecordsReturned="1" elementSet="full" nextRecord="0">
+            <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+              <rdf:Description rdf:about="https://example.com/test/">
+                <dct:identifier>https://example.com/test/</dct:identifier>
+                <rdf:type rdf:resource="http://www.w3.org/ns/dcat#Dataset"/>
+                <dct:title>test</dct:title>
+              </rdf:Description>
+            </rdf:RDF>
+          </csw:SearchResults>
+        </csw:GetRecordsResponse>
+        """
+
+        rmock.get("http://www.example.com/evil.dtd", status_code=404)
+        rmock.head(rmock.ANY, headers={"Content-Type": "application/xml"})
+        rmock.post(rmock.ANY, text=xml)
+        source = HarvestSourceFactory(backend="csw-dcat")
+
+        actions.run(source)
+
+        source.reload()
+        job = source.get_last_job()
+
+        assert not any(h.method == "GET" for h in rmock.request_history)
+        assert job.status == "done"
+        assert len(job.items) == 1
+
 
 @pytest.mark.usefixtures("clean_db")
 @pytest.mark.options(PLUGINS=["csw"])