udata 8.0.2.dev29284__py2.py3-none-any.whl → 8.0.2.dev29339__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -22,9 +22,10 @@ from udata.frontend.markdown import parse_html
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.models import db, ContactPoint
 from udata.rdf import (
-    DCAT, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, VCARD, RDFS,
-    namespace_manager, schema_from_rdf, url_from_rdf
+    DCAT, DCATAP, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, VCARD, RDFS,
+    HVD_LEGISLATION, namespace_manager, schema_from_rdf, url_from_rdf
 )
+from udata.tags import slug as slugify_tag
 from udata.utils import get_by, safe_unicode
 from udata.uris import endpoint_for
 
@@ -85,6 +86,7 @@ EU_HVD_CATEGORIES = {
     "http://data.europa.eu/bna/c_dd313021": "Observation de la terre et environnement",
     "http://data.europa.eu/bna/c_e1da4e07": "Statistiques"
 }
+TAG_TO_EU_HVD_CATEGORIES = {slugify_tag(EU_HVD_CATEGORIES[uri]): uri for uri in EU_HVD_CATEGORIES}
 
 
 class HTMLDetector(HTMLParser):
@@ -141,7 +143,7 @@ def owner_to_rdf(dataset, graph=None):
     return
 
 
-def resource_to_rdf(resource, dataset=None, graph=None):
+def resource_to_rdf(resource, dataset=None, graph=None, is_hvd=False):
     '''
     Map a Resource domain model to a DCAT/RDF graph
     '''
@@ -180,6 +182,9 @@ def resource_to_rdf(resource, dataset=None, graph=None):
         checksum.add(SPDX.algorithm, getattr(SPDX, algorithm))
         checksum.add(SPDX.checksumValue, Literal(resource.checksum.value))
         r.add(SPDX.checksum, checksum)
+    if is_hvd:
+        # DCAT-AP HVD applicable legislation is also expected at the distribution level
+        r.add(DCATAP.applicableLegislation, URIRef(HVD_LEGISLATION))
     return r
 
 
@@ -214,11 +219,20 @@ def dataset_to_rdf(dataset, graph=None):
     if dataset.acronym:
         d.set(SKOS.altLabel, Literal(dataset.acronym))
 
+    # Add DCAT-AP HVD properties if the dataset is tagged hvd.
+    # See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/
+    is_hvd = current_app.config['HVD_SUPPORT'] and 'hvd' in dataset.tags
+    if is_hvd:
+        d.add(DCATAP.applicableLegislation, URIRef(HVD_LEGISLATION))
+
     for tag in dataset.tags:
         d.add(DCAT.keyword, Literal(tag))
+        # Add HVD category if this dataset is tagged HVD
+        if is_hvd and tag in TAG_TO_EU_HVD_CATEGORIES:
+            d.add(DCATAP.hvdCategory, URIRef(TAG_TO_EU_HVD_CATEGORIES[tag]))
 
     for resource in dataset.resources:
-        d.add(DCAT.distribution, resource_to_rdf(resource, dataset, graph))
+        d.add(DCAT.distribution, resource_to_rdf(resource, dataset, graph, is_hvd))
 
     if dataset.temporal_coverage:
         d.set(DCT.temporal, temporal_to_rdf(dataset.temporal_coverage, graph))
@@ -381,23 +395,51 @@ def spatial_from_rdf(graph):
                     else:
                         continue
 
-                    if geojson['type'] == 'Polygon':
-                        geojson['type'] = 'MultiPolygon'
-                        geojson['coordinates'] = [geojson['coordinates']]
-
                     geojsons.append(geojson)
         except Exception as e:
             log.exception(f"Exception during `spatial_from_rdf` for term {term}: {e}", stack_info=True)
 
+    if not geojsons:
+        return None
+
+    # We first try to build a big MultiPolygon with all the spatial coverages found in RDF.
+    # We deduplicate the coordinates because some backend provides the same coordinates multiple
+    # times in different format. We only support in this first pass Polygons and MultiPolygons. Not sure
+    # if there are other types of spatial coverage worth integrating (points? line strings?). But these other
+    # formats are not compatible to be merged in the unique stored representation in MongoDB, we'll deal with them in a second pass.
+    # The merging lose the properties and other information inside the GeoJSON…
+    # Note that having multiple `Polygon` is not really the DCAT way of doing things, the standard require that you use 
+    # a `MultiPolygon` in this case. We support this right now, and wait and see if it raises problems in the future for
+    # people following the standard. (see https://github.com/datagouv/data.gouv.fr/issues/1362#issuecomment-2112774115)
+    polygons = []
     for geojson in geojsons:
-        spatial_coverage = SpatialCoverage(geom=geojson)
-        try:
-            spatial_coverage.clean()
-            return spatial_coverage
-        except ValidationError:
+        if geojson['type'] == 'Polygon':
+            if geojson['coordinates'] not in polygons:
+                polygons.append(geojson['coordinates'])
+        elif geojson['type'] == 'MultiPolygon':
+            for coordinates in geojson['coordinates']:
+                if coordinates not in polygons:
+                    polygons.append(coordinates)
+        else:
+            log.warning(f"Unsupported GeoJSON type '{geojson['type']}'")
             continue
 
-    return None
+    if not polygons:
+        log.warning(f"No supported types found in the GeoJSON data.")
+        return None
+
+    spatial_coverage = SpatialCoverage(geom={
+        'type': 'MultiPolygon',
+        'coordinates': polygons,
+    })
+
+    try:
+        spatial_coverage.clean()
+        return spatial_coverage
+    except ValidationError as e:
+        log.warning(f"Cannot save the spatial coverage {coordinates} (error was {e})")
+        return None
+
 
 def frequency_from_rdf(term):
     if isinstance(term, str):

udata/core/site/api.py

@@ -105,7 +105,10 @@ class SiteRdfCatalogFormat(API):
         params = multi_to_dict(request.args)
         page = int(params.get('page', 1))
         page_size = int(params.get('page_size', 100))
-        datasets = Dataset.objects.visible().paginate(page, page_size)
+        datasets = Dataset.objects.visible()
+        if 'tag' in params:
+            datasets = datasets.filter(tags=params.get('tag', ''))
+        datasets = datasets.paginate(page, page_size)
         catalog = build_catalog(current_site, datasets, format=format)
         # bypass flask-restplus make_response, since graph_response
         # is handling the content negociation directly

udata/harvest/tests/dcat/bnodes.xml

@@ -7,6 +7,7 @@
   xmlns:dct="http://purl.org/dc/terms/"
   xmlns:ogc="http://www.opengis.net/ogc"
   xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#"
+  xmlns:locn="http://www.w3.org/ns/locn#"
   xmlns:dcterms="http://purl.org/dc/terms/"
   xmlns:vcard="http://www.w3.org/2006/vcard/ns#"
   xmlns:schema="http://schema.org/"
@@ -89,8 +90,16 @@
         <dcterms:title>Dataset 2</dcterms:title>
         <dct:spatial>
             <ogc:Polygon>
+                <locn:geometry rdf:datatype="https://www.iana.org/assignments/media-types/application/vnd.geo+json"><![CDATA[{"type":"Polygon","coordinates":[[[-6,51],[10,51],[10,40],[-6,40],[-6,51]]]}]]></locn:geometry>
                 <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
-                    Polygon((4.44641288 45.54214467, 4.44641288 46.01316963, 4.75655252 46.01316963, 4.75655252 45.54214467, 4.44641288 45.54214467))
+                    Polygon((159 -25, 159 -11, 212 -11, 212 -25, 159 -25))
+                </geo:asWKT>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    Polygon((4 45, 4 46, 4 46, 4 45, 4 45))
+                </geo:asWKT>
+                <locn:geometry rdf:datatype="https://www.iana.org/assignments/media-types/application/vnd.geo+json"><![CDATA[{"type":"Polygon","coordinates":[[[4, 45], [4, 46], [4, 46], [4, 45], [4, 45]]]}]]></locn:geometry>
+                <geo:asWKT rdf:datatype="http://www.opengis.net/rdf#wktLiteral">
+                    Polygon((159 -25, 159 -11, 212 -11, 212 -25, 159 -25))
                 </geo:asWKT>
             </ogc:Polygon>
         </dct:spatial>

udata/harvest/tests/test_dcat_backend.py

@@ -268,7 +268,7 @@ class DcatBackendTest:
         datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
 
         assert datasets['1'].spatial == None
-        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[4.44641288, 45.54214467], [4.44641288, 46.01316963], [4.75655252, 46.01316963], [4.75655252, 45.54214467], [4.44641288, 45.54214467]]]]}
+        assert datasets['2'].spatial.geom == {'type': 'MultiPolygon', 'coordinates': [[[[-6,51],[10,51],[10,40],[-6,40],[-6,51]]], [[[4, 45], [4, 46], [4, 46], [4, 45], [4, 45]]], [[[159, -25.], [159, -11], [212, -11], [212, -25.], [159, -25.]]]]}
         assert datasets['3'].spatial == None
 
     @pytest.mark.options(SCHEMA_CATALOG_URL='https://example.com/schemas')

udata/rdf.py

@@ -21,6 +21,7 @@ log = logging.getLogger(__name__)
 # Extra Namespaces
 ADMS = Namespace('http://www.w3.org/ns/adms#')
 DCAT = Namespace('http://www.w3.org/ns/dcat#')
+DCATAP = Namespace('http://data.europa.eu/r5r/')
 HYDRA = Namespace('http://www.w3.org/ns/hydra/core#')
 SCHEMA = Namespace('http://schema.org/')
 SCV = Namespace('http://purl.org/NET/scovo#')
@@ -35,6 +36,7 @@ VCARD = Namespace('http://www.w3.org/2006/vcard/ns#')
 
 namespace_manager = NamespaceManager(Graph())
 namespace_manager.bind('dcat', DCAT)
+namespace_manager.bind('dcatap', DCATAP)
 namespace_manager.bind('dct', DCT)
 namespace_manager.bind('foaf', FOAF)
 namespace_manager.bind('foaf', FOAF)
@@ -98,6 +100,8 @@ RDF_EXTENSIONS = {
 # Includes control characters, unicode surrogate characters and unicode end-of-plane non-characters
 ILLEGAL_XML_CHARS = '[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]'
 
+HVD_LEGISLATION = 'http://data.europa.eu/eli/reg_impl/2023/138/oj'
+
 
 def guess_format(string):
     '''Guess format given an extension or a mime-type'''