udata 9.1.2.dev30355__py2.py3-none-any.whl → 9.1.2.dev30454__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -1,47 +1,67 @@
-'''
+"""
 This module centralize dataset helpers for RDF/DCAT serialization and parsing
-'''
+"""
+
 import calendar
 import json
 import logging
-
 from datetime import date
+
 from dateutil.parser import parse as parse_dt
 from flask import current_app
 from geomet import wkt
-from rdflib import Graph, URIRef, Literal, BNode
-from rdflib.resource import Resource as RdfResource
-from rdflib.namespace import RDF
 from mongoengine.errors import ValidationError
+from rdflib import BNode, Graph, Literal, URIRef
+from rdflib.namespace import RDF
+from rdflib.resource import Resource as RdfResource
 
 from udata import i18n, uris
-from udata.core.spatial.models import SpatialCoverage
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
+from udata.core.spatial.models import SpatialCoverage
 from udata.harvest.exceptions import HarvestSkipException
 from udata.models import db
 from udata.rdf import (
-    DCAT, DCATAP, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, TAG_TO_EU_HVD_CATEGORIES, RDFS, 
-    namespace_manager, rdf_value, remote_url_from_rdf, sanitize_html, schema_from_rdf, themes_from_rdf, url_from_rdf, HVD_LEGISLATION,
+    DCAT,
+    DCATAP,
+    DCT,
+    EUFORMAT,
+    EUFREQ,
+    FREQ,
+    HVD_LEGISLATION,
+    IANAFORMAT,
+    RDFS,
+    SCHEMA,
+    SCV,
+    SKOS,
+    SPDX,
+    TAG_TO_EU_HVD_CATEGORIES,
     contact_point_from_rdf,
+    namespace_manager,
+    rdf_value,
+    remote_url_from_rdf,
+    sanitize_html,
+    schema_from_rdf,
+    themes_from_rdf,
+    url_from_rdf,
 )
-from udata.utils import get_by, safe_unicode
 from udata.uris import endpoint_for
+from udata.utils import get_by, safe_unicode
 
-from .models import Dataset, Resource, Checksum, License
 from .constants import UPDATE_FREQUENCIES
+from .models import Checksum, Dataset, License, Resource
 
 log = logging.getLogger(__name__)
 
 # Map extra frequencies (ie. not defined in Dublin Core) to closest equivalent
 RDF_FREQUENCIES = {
-    'punctual': None,
-    'hourly': FREQ.continuous,
-    'fourTimesADay': FREQ.daily,
-    'threeTimesADay': FREQ.daily,
-    'semidaily': FREQ.daily,
-    'fourTimesAWeek': FREQ.threeTimesAWeek,
-    'quinquennial': None,
-    'unknown': None,
+    "punctual": None,
+    "hourly": FREQ.continuous,
+    "fourTimesADay": FREQ.daily,
+    "threeTimesADay": FREQ.daily,
+    "semidaily": FREQ.daily,
+    "fourTimesAWeek": FREQ.threeTimesAWeek,
+    "quinquennial": None,
+    "unknown": None,
 }
 
 # Map european frequencies to their closest equivalent
@@ -50,31 +70,32 @@ RDF_FREQUENCIES = {
 #  - https://publications.europa.eu/en/web/eu-vocabularies/at-dataset/-/resource/dataset/frequency # noqa: E501
 EU_RDF_REQUENCIES = {
     # Match Dublin Core name
-    EUFREQ.ANNUAL: 'annual',
-    EUFREQ.BIENNIAL: 'biennial',
-    EUFREQ.TRIENNIAL: 'triennial',
-    EUFREQ.QUARTERLY: 'quarterly',
-    EUFREQ.MONTHLY: 'monthly',
-    EUFREQ.BIMONTHLY: 'bimonthly',
-    EUFREQ.WEEKLY: 'weekly',
-    EUFREQ.BIWEEKLY: 'biweekly',
-    EUFREQ.DAILY: 'daily',
+    EUFREQ.ANNUAL: "annual",
+    EUFREQ.BIENNIAL: "biennial",
+    EUFREQ.TRIENNIAL: "triennial",
+    EUFREQ.QUARTERLY: "quarterly",
+    EUFREQ.MONTHLY: "monthly",
+    EUFREQ.BIMONTHLY: "bimonthly",
+    EUFREQ.WEEKLY: "weekly",
+    EUFREQ.BIWEEKLY: "biweekly",
+    EUFREQ.DAILY: "daily",
     # Name differs from Dublin Core
-    EUFREQ.ANNUAL_2: 'semiannual',
-    EUFREQ.ANNUAL_3: 'threeTimesAYear',
-    EUFREQ.MONTHLY_2: 'semimonthly',
-    EUFREQ.MONTHLY_3: 'threeTimesAMonth',
-    EUFREQ.WEEKLY_2: 'semiweekly',
-    EUFREQ.WEEKLY_3: 'threeTimesAWeek',
-    EUFREQ.DAILY_2: 'semidaily',
-    EUFREQ.CONT: 'continuous',
-    EUFREQ.UPDATE_CONT: 'continuous',
-    EUFREQ.IRREG: 'irregular',
-    EUFREQ.UNKNOWN: 'unknown',
-    EUFREQ.OTHER: 'unknown',
-    EUFREQ.NEVER: 'punctual',
+    EUFREQ.ANNUAL_2: "semiannual",
+    EUFREQ.ANNUAL_3: "threeTimesAYear",
+    EUFREQ.MONTHLY_2: "semimonthly",
+    EUFREQ.MONTHLY_3: "threeTimesAMonth",
+    EUFREQ.WEEKLY_2: "semiweekly",
+    EUFREQ.WEEKLY_3: "threeTimesAWeek",
+    EUFREQ.DAILY_2: "semidaily",
+    EUFREQ.CONT: "continuous",
+    EUFREQ.UPDATE_CONT: "continuous",
+    EUFREQ.IRREG: "irregular",
+    EUFREQ.UNKNOWN: "unknown",
+    EUFREQ.OTHER: "unknown",
+    EUFREQ.NEVER: "punctual",
 }
 
+
 def temporal_to_rdf(daterange, graph=None):
     if not daterange:
         return
@@ -104,18 +125,25 @@ def owner_to_rdf(dataset, graph=None):
 
 
 def resource_to_rdf(resource, dataset=None, graph=None, is_hvd=False):
-    '''
+    """
     Map a Resource domain model to a DCAT/RDF graph
-    '''
+    """
     graph = graph or Graph(namespace_manager=namespace_manager)
     if dataset and dataset.id:
-        id = URIRef(endpoint_for('datasets.show_redirect', 'api.dataset', dataset=dataset.id,
-                                 _external=True,
-                                 _anchor='resource-{0}'.format(resource.id)))
+        id = URIRef(
+            endpoint_for(
+                "datasets.show_redirect",
+                "api.dataset",
+                dataset=dataset.id,
+                _external=True,
+                _anchor="resource-{0}".format(resource.id),
+            )
+        )
     else:
         id = BNode(resource.id)
-    permalink = endpoint_for('datasets.resource', 'api.resource_redirect', id=resource.id,
-                             _external=True)
+    permalink = endpoint_for(
+        "datasets.resource", "api.resource_redirect", id=resource.id, _external=True
+    )
     r = graph.resource(id)
     r.set(RDF.type, DCAT.Distribution)
     r.set(DCT.identifier, Literal(resource.id))
@@ -138,7 +166,7 @@ def resource_to_rdf(resource, dataset=None, graph=None, is_hvd=False):
     if resource.checksum:
         checksum = graph.resource(BNode())
         checksum.set(RDF.type, SPDX.Checksum)
-        algorithm = 'checksumAlgorithm_{0}'.format(resource.checksum.type)
+        algorithm = "checksumAlgorithm_{0}".format(resource.checksum.type)
         checksum.add(SPDX.algorithm, getattr(SPDX, algorithm))
         checksum.add(SPDX.checksumValue, Literal(resource.checksum.value))
         r.add(SPDX.checksum, checksum)
@@ -148,21 +176,25 @@ def resource_to_rdf(resource, dataset=None, graph=None, is_hvd=False):
     return r
 
 
-def dataset_to_graph_id(dataset: Dataset) -> URIRef | BNode: 
+def dataset_to_graph_id(dataset: Dataset) -> URIRef | BNode:
     if dataset.harvest and dataset.harvest.uri:
         return URIRef(dataset.harvest.uri)
     elif dataset.id:
-        return URIRef(endpoint_for('datasets.show_redirect', 'api.dataset',
-            dataset=dataset.id, _external=True))
+        return URIRef(
+            endpoint_for(
+                "datasets.show_redirect", "api.dataset", dataset=dataset.id, _external=True
+            )
+        )
     else:
         # Should not happen in production. Some test only
         # `build()` a dataset without saving it to the DB.
         return BNode()
 
+
 def dataset_to_rdf(dataset, graph=None):
-    '''
+    """
     Map a dataset domain model to a DCAT/RDF graph
-    '''
+    """
     # Use the unlocalized permalink to the dataset as URI when available
     # unless there is already an upstream URI
     id = dataset_to_graph_id(dataset)
@@ -187,7 +219,7 @@ def dataset_to_rdf(dataset, graph=None):
 
     # Add DCAT-AP HVD properties if the dataset is tagged hvd.
     # See https://semiceu.github.io/DCAT-AP/releases/2.2.0-hvd/
-    is_hvd = current_app.config['HVD_SUPPORT'] and 'hvd' in dataset.tags
+    is_hvd = current_app.config["HVD_SUPPORT"] and "hvd" in dataset.tags
     if is_hvd:
         d.add(DCATAP.applicableLegislation, URIRef(HVD_LEGISLATION))
 
@@ -215,52 +247,46 @@ def dataset_to_rdf(dataset, graph=None):
 
 
 CHECKSUM_ALGORITHMS = {
-    SPDX.checksumAlgorithm_md5: 'md5',
-    SPDX.checksumAlgorithm_sha1: 'sha1',
-    SPDX.checksumAlgorithm_sha256: 'sha256',
+    SPDX.checksumAlgorithm_md5: "md5",
+    SPDX.checksumAlgorithm_sha1: "sha1",
+    SPDX.checksumAlgorithm_sha256: "sha256",
 }
 
 
 def temporal_from_literal(text):
-    '''
+    """
     Parse a temporal coverage from a literal ie. either:
     - an ISO date range
     - a single ISO date period (month,year)
-    '''
-    if text.count('/') == 1:
+    """
+    if text.count("/") == 1:
         # This is an ISO date range as preconized by Gov.uk
         # http://guidance.data.gov.uk/dcat_fields.html
-        start, end = text.split('/')
-        return db.DateRange(
-            start=parse_dt(start).date(),
-            end=parse_dt(end).date()
-        )
+        start, end = text.split("/")
+        return db.DateRange(start=parse_dt(start).date(), end=parse_dt(end).date())
     else:
-        separators = text.count('-')
+        separators = text.count("-")
         if separators == 0:
             # this is a year
-            return db.DateRange(
-                start=date(int(text), 1, 1),
-                end=date(int(text), 12, 31)
-            )
+            return db.DateRange(start=date(int(text), 1, 1), end=date(int(text), 12, 31))
         elif separators == 1:
             # this is a month
             dt = parse_dt(text).date()
             return db.DateRange(
                 start=dt.replace(day=1),
-                end=dt.replace(day=calendar.monthrange(dt.year, dt.month)[1])
+                end=dt.replace(day=calendar.monthrange(dt.year, dt.month)[1]),
             )
 
 
 def temporal_from_resource(resource):
-    '''
+    """
     Parse a temporal coverage from a RDF class/resource ie. either:
     - a `dct:PeriodOfTime` with schema.org `startDate` and `endDate` properties
     - a `dct:PeriodOfTime` with DCAT `startDate` and `endDate` properties
     - an inline gov.uk Time Interval value
     - an URI reference to a gov.uk Time Interval ontology
       http://reference.data.gov.uk/
-    '''
+    """
     if isinstance(resource.identifier, URIRef):
         # Fetch remote ontology if necessary
         g = Graph().parse(str(resource.identifier))
@@ -268,22 +294,21 @@ def temporal_from_resource(resource):
     if resource.value(SCHEMA.startDate):
         return db.DateRange(
             start=resource.value(SCHEMA.startDate).toPython(),
-            end=resource.value(SCHEMA.endDate).toPython()
+            end=resource.value(SCHEMA.endDate).toPython(),
         )
     elif resource.value(DCAT.startDate):
         return db.DateRange(
             start=resource.value(DCAT.startDate).toPython(),
-            end=resource.value(DCAT.endDate).toPython()
+            end=resource.value(DCAT.endDate).toPython(),
         )
     elif resource.value(SCV.min):
         return db.DateRange(
-            start=resource.value(SCV.min).toPython(),
-            end=resource.value(SCV.max).toPython()
+            start=resource.value(SCV.min).toPython(), end=resource.value(SCV.max).toPython()
         )
 
 
 def temporal_from_rdf(period_of_time):
-    '''Failsafe parsing of a temporal coverage'''
+    """Failsafe parsing of a temporal coverage"""
     try:
         if isinstance(period_of_time, Literal):
             return temporal_from_literal(str(period_of_time))
@@ -293,30 +318,34 @@ def temporal_from_rdf(period_of_time):
         # There are a lot of cases where parsing could/should fail
         # but we never want to break the whole dataset parsing
         # so we log the error for future investigation and improvement
-        log.warning('Unable to parse temporal coverage', exc_info=True)
+        log.warning("Unable to parse temporal coverage", exc_info=True)
+
 
 def spatial_from_rdf(graph):
     geojsons = []
     for term in graph.objects(DCT.spatial):
         try:
-            # This may not be official in the norm but some ArcGis return 
+            # This may not be official in the norm but some ArcGis return
             # bbox as literal directly in DCT.spatial.
             if isinstance(term, Literal):
                 geojson = bbox_to_geojson_multipolygon(term.toPython())
                 if geojson is not None:
                     geojsons.append(geojson)
-                
+
                 continue
 
             for object in term.objects():
                 if isinstance(object, Literal):
-                    if object.datatype.__str__() == 'https://www.iana.org/assignments/media-types/application/vnd.geo+json':
+                    if (
+                        object.datatype.__str__()
+                        == "https://www.iana.org/assignments/media-types/application/vnd.geo+json"
+                    ):
                         try:
                             geojson = json.loads(object.toPython())
                         except ValueError as e:
                             log.warning(f"Invalid JSON in spatial GeoJSON {object.toPython()} {e}")
                             continue
-                    elif object.datatype.__str__() == 'http://www.opengis.net/rdf#wktLiteral':
+                    elif object.datatype.__str__() == "http://www.opengis.net/rdf#wktLiteral":
                         try:
                             # .upper() si here because geomet doesn't support Polygon but only POLYGON
                             geojson = wkt.loads(object.toPython().strip().upper())
@@ -328,7 +357,9 @@ def spatial_from_rdf(graph):
 
                     geojsons.append(geojson)
         except Exception as e:
-            log.exception(f"Exception during `spatial_from_rdf` for term {term}: {e}", stack_info=True)
+            log.exception(
+                f"Exception during `spatial_from_rdf` for term {term}: {e}", stack_info=True
+            )
 
     if not geojsons:
         return None
@@ -339,16 +370,16 @@ def spatial_from_rdf(graph):
     # if there are other types of spatial coverage worth integrating (points? line strings?). But these other
     # formats are not compatible to be merged in the unique stored representation in MongoDB, we'll deal with them in a second pass.
     # The merging lose the properties and other information inside the GeoJSON…
-    # Note that having multiple `Polygon` is not really the DCAT way of doing things, the standard require that you use 
+    # Note that having multiple `Polygon` is not really the DCAT way of doing things, the standard require that you use
     # a `MultiPolygon` in this case. We support this right now, and wait and see if it raises problems in the future for
     # people following the standard. (see https://github.com/datagouv/data.gouv.fr/issues/1362#issuecomment-2112774115)
     polygons = []
     for geojson in geojsons:
-        if geojson['type'] == 'Polygon':
-            if geojson['coordinates'] not in polygons:
-                polygons.append(geojson['coordinates'])
-        elif geojson['type'] == 'MultiPolygon':
-            for coordinates in geojson['coordinates']:
+        if geojson["type"] == "Polygon":
+            if geojson["coordinates"] not in polygons:
+                polygons.append(geojson["coordinates"])
+        elif geojson["type"] == "MultiPolygon":
+            for coordinates in geojson["coordinates"]:
                 if coordinates not in polygons:
                     polygons.append(coordinates)
         else:
@@ -359,10 +390,12 @@ def spatial_from_rdf(graph):
         log.warning(f"No supported types found in the GeoJSON data.")
         return None
 
-    spatial_coverage = SpatialCoverage(geom={
-        'type': 'MultiPolygon',
-        'coordinates': polygons,
-    })
+    spatial_coverage = SpatialCoverage(
+        geom={
+            "type": "MultiPolygon",
+            "coordinates": polygons,
+        }
+    )
 
     try:
         spatial_coverage.clean()
@@ -397,7 +430,7 @@ def mime_from_rdf(resource):
     if not mime:
         return
     if IANAFORMAT in mime:
-        return '/'.join(mime.split('/')[-2:])
+        return "/".join(mime.split("/")[-2:])
     if isinstance(mime, str):
         return mime
 
@@ -413,36 +446,36 @@ def format_from_rdf(resource):
 
 
 def title_from_rdf(rdf, url):
-    '''
+    """
     Try to extract a distribution title from a property.
     As it's not a mandatory property,
     it fallback on building a title from the URL
     then the format and in last ressort a generic resource name.
-    '''
+    """
     title = rdf_value(rdf, DCT.title)
     if title:
         return title
     if url:
-        last_part = url.split('/')[-1]
-        if '.' in last_part and '?' not in last_part:
+        last_part = url.split("/")[-1]
+        if "." in last_part and "?" not in last_part:
             return last_part
     fmt = rdf_value(rdf, DCT.format)
-    lang = current_app.config['DEFAULT_LANGUAGE']
+    lang = current_app.config["DEFAULT_LANGUAGE"]
     with i18n.language(lang):
         if fmt:
-            return i18n._('{format} resource').format(format=fmt.lower())
+            return i18n._("{format} resource").format(format=fmt.lower())
         else:
-            return i18n._('Nameless resource')
+            return i18n._("Nameless resource")
+
 
 def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
-    '''
+    """
     Map a Resource domain model to a DCAT/RDF graph
-    '''
+    """
     if isinstance(graph_or_distrib, RdfResource):
         distrib = graph_or_distrib
     else:
-        node = graph_or_distrib.value(predicate=RDF.type,
-                                      object=DCAT.Distribution)
+        node = graph_or_distrib.value(predicate=RDF.type, object=DCAT.Distribution)
         distrib = graph_or_distrib.resource(node)
 
     if not is_additionnal:
@@ -453,16 +486,16 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
         url = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
     # we shouldn't create resources without URLs
     if not url:
-        log.warning(f'Resource without url: {distrib}')
+        log.warning(f"Resource without url: {distrib}")
         return
 
     if dataset:
-        resource = get_by(dataset.resources, 'url', url)
+        resource = get_by(dataset.resources, "url", url)
     if not dataset or not resource:
         resource = Resource()
         if dataset:
             dataset.resources.append(resource)
-    resource.filetype = 'remote'
+    resource.filetype = "remote"
     resource.title = title_from_rdf(distrib, url)
     resource.url = url
     resource.description = sanitize_html(distrib.value(DCT.description))
@@ -482,7 +515,7 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
             resource.checksum.value = rdf_value(checksum, SPDX.checksumValue)
             resource.checksum.type = algorithm
     if is_additionnal:
-        resource.type = 'other'
+        resource.type = "other"
 
     identifier = rdf_value(distrib, DCT.identifier)
     uri = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
@@ -500,9 +533,9 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
 
 
 def dataset_from_rdf(graph: Graph, dataset=None, node=None):
-    '''
+    """
     Create or update a dataset from a RDF/DCAT graph
-    '''
+    """
     dataset = dataset or Dataset()
 
     if node is None:  # Assume first match is the only match
@@ -542,13 +575,13 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     if access_rights:
         dataset.extras["harvest"] = {
             "dct:accessRights": access_rights,
-            **dataset.extras.get("harvest", {})
+            **dataset.extras.get("harvest", {}),
         }
     provenance = [p.value(RDFS.label) for p in d.objects(DCT.provenance)]
     if provenance:
         dataset.extras["harvest"] = {
             "dct:provenance": provenance,
-            **dataset.extras.get("harvest", {})
+            **dataset.extras.get("harvest", {}),
         }
 
     licenses = set()
@@ -584,11 +617,12 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
 
     return dataset
 
-def bbox_to_geojson_multipolygon(bbox_as_str: str) -> dict | None: 
-    bbox = bbox_as_str.strip().split(',')
+
+def bbox_to_geojson_multipolygon(bbox_as_str: str) -> dict | None:
+    bbox = bbox_as_str.strip().split(",")
     if len(bbox) != 4:
         return None
-    
+
     west = float(bbox[0])
     south = float(bbox[1])
     east = float(bbox[2])
@@ -600,10 +634,10 @@ def bbox_to_geojson_multipolygon(bbox_as_str: str) -> dict | None:
     low_right = [east, south]
 
     return {
-        'type': 'MultiPolygon',
-        'coordinates': [
+        "type": "MultiPolygon",
+        "coordinates": [
             [
                 [low_left, low_right, top_right, top_left, low_left],
-            ], 
+            ],
         ],
-    }
+    }