udata 10.8.3.dev37170__py2.py3-none-any.whl → 10.8.3.dev37191__py2.py3-none-any.whl

udata/core/dataset/api_fields.py

@@ -266,6 +266,7 @@ DEFAULT_MASK = ",".join(
         "acronym",
         "slug",
         "description",
+        "description_short",
         "created_at",
         "last_modified",
         "deleted",
@@ -327,6 +328,7 @@ dataset_fields = api.model(
         "description": fields.Markdown(
             description="The dataset description in markdown", required=True
         ),
+        "description_short": fields.String(description="The dataset short description"),
         "created_at": fields.ISODateTime(
             description="This date is computed between harvested creation date if any and site's internal creation date",
             required=True,

udata/core/dataset/apiv2.py

@@ -44,6 +44,7 @@ DEFAULT_MASK_APIV2 = ",".join(
         "acronym",
         "slug",
         "description",
+        "description_short",
         "created_at",
         "last_modified",
         "deleted",
@@ -105,6 +106,9 @@ dataset_fields = apiv2.model(
         "description": fields.Markdown(
             description="The dataset description in markdown", required=True
         ),
+        "description_short": fields.String(
+            description="The dataset short description", required=False
+        ),
         "created_at": fields.ISODateTime(
             description="The dataset creation date", required=True, readonly=True
         ),

udata/core/dataset/constants.py

@@ -89,5 +89,6 @@ SCHEMA_CACHE_DURATION = 60 * 5  # In seconds
 
 TITLE_SIZE_LIMIT = 350
 DESCRIPTION_SIZE_LIMIT = 100000
+DESCRIPTION_SHORT_SIZE_LIMIT = 200
 
 FULL_OBJECTS_HEADER = "X-Get-Datasets-Full-Objects"

udata/core/dataset/csv.py

@@ -26,6 +26,7 @@ class DatasetCsvAdapter(csv.Adapter):
         ("owner_id", "owner.id"),
         # 'contact_point', #  ?
         "description",
+        "description_short",
         "frequency",
         "license",
         "temporal_coverage.start",

udata/core/dataset/forms.py

@@ -7,6 +7,7 @@ from udata.mongo.errors import FieldValidationError
 from .constants import (
     CHECKSUM_TYPES,
     DEFAULT_FREQUENCY,
+    DESCRIPTION_SHORT_SIZE_LIMIT,
     DESCRIPTION_SIZE_LIMIT,
     LEGACY_FREQUENCIES,
     RESOURCE_FILETYPES,
@@ -151,6 +152,11 @@ class DatasetForm(ModelForm):
         [validators.DataRequired(), validators.Length(max=DESCRIPTION_SIZE_LIMIT)],
         description=_("The details about the dataset (collection process, specifics...)."),
     )
+    description_short = fields.StringField(
+        _("Short description"),
+        [validators.Length(max=DESCRIPTION_SHORT_SIZE_LIMIT)],
+        description=_("A short description of the dataset."),
+    )
     license = fields.ModelSelectField(_("License"), model=License, allow_blank=True)
     frequency = fields.SelectField(
         _("Update frequency"),

udata/core/dataset/models.py

@@ -35,6 +35,7 @@ from .constants import (
     CHECKSUM_TYPES,
     CLOSED_FORMATS,
     DEFAULT_LICENSE,
+    DESCRIPTION_SHORT_SIZE_LIMIT,
     LEGACY_FREQUENCIES,
     MAX_DISTANCE,
     PIVOTAL_DATA,
@@ -560,6 +561,7 @@ class Dataset(Auditable, WithMetrics, DatasetBadgeMixin, Owned, Linkable, db.Doc
         auditable=False,
     )
     description = field(db.StringField(required=True, default=""))
+    description_short = field(db.StringField(max_length=DESCRIPTION_SHORT_SIZE_LIMIT))
     license = field(db.ReferenceField("License"))
 
     tags = field(db.TagListField())

udata/harvest/backends/dcat.py

@@ -1,11 +1,12 @@
 import logging
 from datetime import date
-from typing import Generator
+from typing import ClassVar, Generator
 
 import lxml.etree as ET
 from flask import current_app
 from rdflib import Graph
 from rdflib.namespace import RDF
+from typing_extensions import override
 
 from udata.core.dataservices.rdf import dataservice_from_rdf
 from udata.core.dataset.rdf import dataset_from_rdf
@@ -55,9 +56,6 @@ URIS_TO_REPLACE = {
 }
 
 
-SAFE_PARSER = ET.XMLParser(resolve_entities=False)
-
-
 def extract_graph(source, target, node, specs):
     for p, o in source.predicate_objects(node):
         target.add((node, p, o))
@@ -240,104 +238,165 @@ class DcatBackend(BaseBackend):
                 return node
         raise ValueError(f"Unable to find dataset with DCT.identifier:{item.remote_id}")
 
-    def next_record_if_should_continue(self, start, search_results):
-        next_record = int(search_results.attrib["nextRecord"])
-        matched_count = int(search_results.attrib["numberOfRecordsMatched"])
-        returned_count = int(search_results.attrib["numberOfRecordsReturned"])
 
-        # Break conditions copied gratefully from
-        # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
-        break_conditions = (
-            # standard CSW: A value of 0 means all records have been returned.
-            next_record == 0,
-            # Misbehaving CSW server returning a next record > matched count
-            next_record > matched_count,
-            # No results returned already
-            returned_count == 0,
-            # Current next record is lower than previous one
-            next_record < start,
-            # Enough items have been harvested already
-            self.max_items and len(self.job.items) >= self.max_items,
-        )
+class CswDcatBackend(DcatBackend):
+    """
+    CSW harvester fetching records as DCAT.
+    The parsing of items is then the same as for the DcatBackend.
+    """
 
-        if any(break_conditions):
-            return None
-        else:
-            return next_record
+    display_name = "CSW-DCAT"
 
+    # CSW_REQUEST is based on:
+    # - Request syntax from spec [1] and example requests [1] [2].
+    # - Sort settings to ensure stable paging [3].
+    # - Filter settings to only retrieve record types currently mapped in udata.
+    #
+    # If you modify the request, make sure:
+    # - `typeNames` and `outputSchema` are consistent. You'll likely want to keep "gmd:MD_Metadata",
+    #   since "csw:Record" contains less information.
+    # - `typeNames` and namespaces in `csw:Query` (`Filter`, `SortBy`, ...) are consistent, although
+    #   they are ignored on some servers [4] [5].
+    # - It works on real catalogs! Not many servers implement the whole spec.
+    #
+    # References:
+    # [1] OpenGIS Catalogue Services Specification 2.0.2 – ISO Metadata Application Profile: Corrigendum
+    #     https://portal.ogc.org/files/80534
+    # [2] GeoNetwork - CSW test requests
+    #     https://github.com/geonetwork/core-geonetwork/tree/3.10.4/web/src/main/webapp/xml/csw/test
+    # [3] Udata - Support csw dcat harvest
+    #     https://github.com/opendatateam/udata/pull/2800#discussion_r1129053500
+    # [4] GeoNetwork - GetRecords ignores namespaces for Filter/SortBy fields
+    #     https://github.com/geonetwork/core-geonetwork/blob/3.10.4/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/FieldMapper.java#L92
+    # [5] GeoNetwork - GetRecords ignores `typeNames`
+    #     https://github.com/geonetwork/core-geonetwork/blob/3.10.4/csw-server/src/main/java/org/fao/geonet/kernel/csw/services/getrecords/CatalogSearcher.java#L194
+    CSW_REQUEST: ClassVar[str] = """
+    <csw:GetRecords xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0"
+                    xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                    xmlns:ogc="http://www.opengis.net/ogc"
+                    service="CSW" version="2.0.2" outputFormat="application/xml"
+                    resultType="results" startPosition="{start}" maxRecords="25"
+                    outputSchema="{output_schema}">
+      <csw:Query typeNames="gmd:MD_Metadata">
+        <csw:ElementSetName>full</csw:ElementSetName>
+        <csw:Constraint version="1.1.0">
+          <ogc:Filter>
+            <ogc:Or>
+              <ogc:PropertyIsEqualTo>
+                <ogc:PropertyName>apiso:type</ogc:PropertyName>
+                <ogc:Literal>dataset</ogc:Literal>
+              </ogc:PropertyIsEqualTo>
+              <ogc:PropertyIsEqualTo>
+                <ogc:PropertyName>apiso:type</ogc:PropertyName>
+                <ogc:Literal>nonGeographicDataset</ogc:Literal>
+              </ogc:PropertyIsEqualTo>
+              <ogc:PropertyIsEqualTo>
+                <ogc:PropertyName>apiso:type</ogc:PropertyName>
+                <ogc:Literal>series</ogc:Literal>
+              </ogc:PropertyIsEqualTo>
+              <ogc:PropertyIsEqualTo>
+                <ogc:PropertyName>apiso:type</ogc:PropertyName>
+                <ogc:Literal>service</ogc:Literal>
+              </ogc:PropertyIsEqualTo>
+            </ogc:Or>
+          </ogc:Filter>
+        </csw:Constraint>
+        <ogc:SortBy>
+          <ogc:SortProperty>
+            <ogc:PropertyName>apiso:identifier</ogc:PropertyName>
+            <ogc:SortOrder>ASC</ogc:SortOrder>
+          </ogc:SortProperty>
+        </ogc:SortBy>
+      </csw:Query>
+    </csw:GetRecords>
+    """
 
-class CswDcatBackend(DcatBackend):
-    display_name = "CSW-DCAT"
+    CSW_OUTPUT_SCHEMA = "http://www.w3.org/ns/dcat#"
 
-    DCAT_SCHEMA = "http://www.w3.org/ns/dcat#"
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.xml_parser = ET.XMLParser(resolve_entities=False)
 
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
         Yield all RDF pages as `Graph` from the source
         """
-        body = """<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
-                                  xmlns:gmd="http://www.isotc211.org/2005/gmd"
-                                  service="CSW" version="2.0.2" resultType="results"
-                                  startPosition="{start}" maxPosition="200"
-                                  outputSchema="{schema}">
-                    <csw:Query typeNames="gmd:MD_Metadata">
-                        <csw:ElementSetName>full</csw:ElementSetName>
-                        <ogc:SortBy xmlns:ogc="http://www.opengis.net/ogc">
-                            <ogc:SortProperty>
-                                <ogc:PropertyName>identifier</ogc:PropertyName>
-                            <ogc:SortOrder>ASC</ogc:SortOrder>
-                            </ogc:SortProperty>
-                        </ogc:SortBy>
-                    </csw:Query>
-                </csw:GetRecords>"""
-        headers = {"Content-Type": "application/xml"}
-
         page_number = 0
         start = 1
 
-        response = self.post(
-            url, data=body.format(start=start, schema=self.DCAT_SCHEMA), headers=headers
-        )
-        response.raise_for_status()
-        content = response.content
-        tree = ET.fromstring(content, parser=SAFE_PARSER)
-        if tree.tag == "{" + OWS_NAMESPACE + "}ExceptionReport":
-            raise ValueError(f"Failed to query CSW:\n{content}")
-        while tree is not None:
+        while True:
+            data = self.CSW_REQUEST.format(output_schema=self.CSW_OUTPUT_SCHEMA, start=start)
+            response = self.post(url, data=data, headers={"Content-Type": "application/xml"})
+            response.raise_for_status()
+
+            content = response.content
+            tree = ET.fromstring(content, parser=self.xml_parser)
+            if tree.tag == "{" + OWS_NAMESPACE + "}ExceptionReport":
+                raise ValueError(f"Failed to query CSW:\n{content}")
+
             search_results = tree.find("csw:SearchResults", {"csw": CSW_NAMESPACE})
-            if search_results is None:
+            if not search_results:
                 log.error(f"No search results found for {url} on page {page_number}")
-                break
-            for child in search_results:
+                return
+
+            for result in search_results:
                 subgraph = Graph(namespace_manager=namespace_manager)
-                subgraph.parse(data=ET.tostring(child), format=fmt)
+                doc = ET.tostring(self.as_dcat(result))
+                subgraph.parse(data=doc, format=fmt)
+
+                if not subgraph.subjects(
+                    RDF.type, [DCAT.Dataset, DCAT.DatasetSeries, DCAT.DataService]
+                ):
+                    raise ValueError("Failed to fetch CSW content")
 
                 yield page_number, subgraph
+
                 if self.has_reached_max_items():
                     return
 
-            next_record = self.next_record_if_should_continue(start, search_results)
-            if not next_record:
-                break
-
-            start = next_record
             page_number += 1
+            start = self.next_position(start, search_results)
+            if not start:
+                return
 
-            tree = ET.fromstring(
-                self.post(
-                    url, data=body.format(start=start, schema=self.DCAT_SCHEMA), headers=headers
-                ).content,
-                parser=SAFE_PARSER,
-            )
+    def as_dcat(self, tree: ET._Element) -> ET._Element:
+        """
+        Return the input tree as a DCAT tree.
+        For CswDcatBackend, this method return the incoming tree as-is, since it's already DCAT.
+        For subclasses of CswDcatBackend, this method should convert the incoming tree to DCAT.
+        """
+        return tree
+
+    def next_position(self, start: int, search_results: ET._Element) -> int | None:
+        next_record = int(search_results.attrib["nextRecord"])
+        matched_count = int(search_results.attrib["numberOfRecordsMatched"])
+        returned_count = int(search_results.attrib["numberOfRecordsReturned"])
+
+        # Break conditions copied gratefully from
+        # noqa https://github.com/geonetwork/core-geonetwork/blob/main/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java#L338-L369
+        should_break = (
+            # A value of 0 means all records have been returned (standard CSW)
+            (next_record == 0)
+            # Misbehaving CSW server returning a next record > matched count
+            or (next_record > matched_count)
+            # No results returned already
+            or (returned_count == 0)
+            # Current next record is lower than previous one
+            or (next_record < start)
+            # Enough items have been harvested already
+            or self.has_reached_max_items()
+        )
+        return None if should_break else next_record
 
 
-class CswIso19139DcatBackend(DcatBackend):
+class CswIso19139DcatBackend(CswDcatBackend):
     """
-    An harvester that takes CSW ISO 19139 as input and transforms it to DCAT using SEMIC GeoDCAT-AP XSLT.
+    CSW harvester fetching records as ISO-19139 and using XSLT to convert them to DCAT.
     The parsing of items is then the same as for the DcatBackend.
     """
 
     display_name = "CSW-ISO-19139"
+
     extra_configs = (
         HarvestExtraConfig(
             _("Remote URL prefix"),
@@ -347,94 +406,14 @@ class CswIso19139DcatBackend(DcatBackend):
         ),
     )
 
-    ISO_SCHEMA = "http://www.isotc211.org/2005/gmd"
-
-    def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
-        """
-        Yield all RDF pages as `Graph` from the source
-
-        Parse CSW graph querying ISO schema.
-        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
-        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
-        """
-        # Load XSLT
-        xsl_url = current_app.config["HARVEST_ISO19139_XSL_URL"]
-        xsl = ET.fromstring(self.get(xsl_url).content, parser=SAFE_PARSER)
-        transform = ET.XSLT(xsl)
-
-        # Start querying and parsing graph
-        # Filter on dataset or serie records
-        body = """<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
-                                  xmlns:gmd="http://www.isotc211.org/2005/gmd"
-                                  service="CSW" version="2.0.2" resultType="results"
-                                  startPosition="{start}" maxPosition="10"
-                                  outputSchema="{schema}">
-                      <csw:Query typeNames="csw:Record">
-                        <csw:ElementSetName>full</csw:ElementSetName>
-                        <csw:Constraint version="1.1.0">
-                            <ogc:Filter xmlns:ogc="http://www.opengis.net/ogc">
-                                <ogc:Or xmlns:ogc="http://www.opengis.net/ogc">
-                                    <ogc:PropertyIsEqualTo>
-                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                        <ogc:Literal>dataset</ogc:Literal>
-                                    </ogc:PropertyIsEqualTo>
-                                    <ogc:PropertyIsEqualTo>
-                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                        <ogc:Literal>service</ogc:Literal>
-                                    </ogc:PropertyIsEqualTo>
-                                    <ogc:PropertyIsEqualTo>
-                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
-                                        <ogc:Literal>series</ogc:Literal>
-                                    </ogc:PropertyIsEqualTo>
-                                </ogc:Or>
-                            </ogc:Filter>
-                        </csw:Constraint>
-                    </csw:Query>
-                </csw:GetRecords>"""
-        headers = {"Content-Type": "application/xml"}
-
-        page_number = 0
-        start = 1
-
-        response = self.post(
-            url, data=body.format(start=start, schema=self.ISO_SCHEMA), headers=headers
-        )
-        response.raise_for_status()
-
-        tree_before_transform = ET.fromstring(response.content, parser=SAFE_PARSER)
-        # Disabling CoupledResourceLookUp to prevent failure on xlink:href
-        # https://github.com/SEMICeu/iso-19139-to-dcat-ap/blob/master/documentation/HowTo.md#parameter-coupledresourcelookup
-        tree = transform(tree_before_transform, CoupledResourceLookUp="'disabled'")
-
-        while tree:
-            # We query the tree before the transformation because the XSLT remove the search results
-            # infos (useful for pagination)
-            search_results = tree_before_transform.find("csw:SearchResults", {"csw": CSW_NAMESPACE})
-            if search_results is None:
-                log.error(f"No search results found for {url} on page {page_number}")
-                break
-
-            subgraph = Graph(namespace_manager=namespace_manager)
-            subgraph.parse(ET.tostring(tree), format=fmt)
-
-            if not subgraph.subjects(RDF.type, DCAT.Dataset):
-                raise ValueError("Failed to fetch CSW content")
+    CSW_OUTPUT_SCHEMA = "http://www.isotc211.org/2005/gmd"
 
-            yield page_number, subgraph
-            if self.has_reached_max_items():
-                return
-
-            next_record = self.next_record_if_should_continue(start, search_results)
-            if not next_record:
-                break
-
-            start = next_record
-            page_number += 1
-
-            response = self.post(
-                url, data=body.format(start=start, schema=self.ISO_SCHEMA), headers=headers
-            )
-            response.raise_for_status()
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        xslt_url = current_app.config["HARVEST_ISO19139_XSLT_URL"]
+        xslt = ET.fromstring(self.get(xslt_url).content, parser=self.xml_parser)
+        self.transform = ET.XSLT(xslt)
 
-            tree_before_transform = ET.fromstring(response.content, parser=SAFE_PARSER)
-            tree = transform(tree_before_transform, CoupledResourceLookUp="'disabled'")
+    @override
+    def as_dcat(self, tree: ET._Element) -> ET._Element:
+        return self.transform(tree, CoupledResourceLookUp="'disabled'")

udata/harvest/tests/test_dcat_backend.py

@@ -899,7 +899,7 @@ class CswIso19139DcatBackendTest:
         with open(os.path.join(CSW_DCAT_FILES_DIR, "XSLT.xml"), "r") as f:
             xslt = f.read()
         url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetwork-iso-page-{}.xml")
-        rmock.get(current_app.config.get("HARVEST_ISO19139_XSL_URL"), text=xslt)
+        rmock.get(current_app.config.get("HARVEST_ISO19139_XSLT_URL"), text=xslt)
         org = OrganizationFactory()
         source = HarvestSourceFactory(
             backend="csw-iso-19139",

udata/settings.py

@@ -283,7 +283,7 @@ class Defaults(object):
     HARVEST_GRAPHS_S3_BUCKET = None  # If the catalog is bigger than `HARVEST_MAX_CATALOG_SIZE_IN_MONGO` store the graph inside S3 instead of MongoDB
     HARVEST_GRAPHS_S3_FILENAME_PREFIX = ""  # Useful to store the graphs inside a subfolder of the bucket. For example by setting `HARVEST_GRAPHS_S3_FILENAME_PREFIX = 'graphs/'`
 
-    HARVEST_ISO19139_XSL_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/refs/heads/geodcat-ap-2.0.0/iso-19139-to-dcat-ap.xsl"
+    HARVEST_ISO19139_XSLT_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/refs/heads/geodcat-ap-2.0.0/iso-19139-to-dcat-ap.xsl"
 
     # S3 connection details
     S3_URL = None