udata 8.0.2.dev29304__py2.py3-none-any.whl → 9.1.0__py2.py3-none-any.whl

udata/harvest/api.py

@@ -1,9 +1,11 @@
+from bson import ObjectId
 from werkzeug.exceptions import BadRequest
 from flask import request
 
 from udata.api import api, API, fields
 from udata.auth import admin_permission
 
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_ref_fields, dataset_fields
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.organization.permissions import EditOrganizationPermission
@@ -31,12 +33,22 @@ error_fields = api.model('HarvestError', {
     'details': fields.String(description='Optional details (ie. stacktrace)'),
 })
 
+
+log_fields = api.model('HarvestError', {
+    'level': fields.String(required=True),
+    'message': fields.String(required=True),
+})
+
+
 item_fields = api.model('HarvestItem', {
     'remote_id': fields.String(description='The item remote ID to process',
                                required=True),
     'dataset': fields.Nested(dataset_ref_fields,
                              description='The processed dataset',
                              allow_null=True),
+    'dataservice': fields.Nested(Dataservice.__read_fields__,
+                             description='The processed dataservice',
+                             allow_null=True),
     'status': fields.String(description='The item status',
                             required=True,
                             enum=list(HARVEST_ITEM_STATUS)),
@@ -46,6 +58,8 @@ item_fields = api.model('HarvestItem', {
     'ended': fields.ISODateTime(description='The item end date'),
     'errors': fields.List(fields.Nested(error_fields),
                           description='The item errors'),
+    'logs': fields.List(fields.Nested(log_fields),
+                          description='The item logs'),
     'args': fields.List(fields.String,
                         description='The item positional arguments',
                         default=[]),
@@ -181,6 +195,10 @@ class SourcesAPI(API):
     def get(self):
         '''List all harvest sources'''
         args = source_parser.parse_args()
+
+        if args.get('owner') and not ObjectId.is_valid(args.get('owner')):
+            api.abort(400, '`owner` arg must be an identifier')
+
         return actions.paginate_sources(args.get('owner'),
                                         page=args['page'],
                                         page_size=args['page_size'],
@@ -256,7 +274,7 @@ class ScheduleSourceAPI(API):
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
-        except BadRequest as e:
+        except BadRequest:
             data = request.data.decode('utf-8')
         return actions.schedule(ident, data)
 

udata/harvest/backends/base.py

@@ -2,20 +2,21 @@ import logging
 import traceback
 
 from datetime import datetime, date, timedelta
-from typing import Optional
 from uuid import UUID
 
 import requests
 
 from flask import current_app
+from udata.core.dataservices.models import Dataservice
 from voluptuous import MultipleInvalid, RequiredFieldInvalid
 
 from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.models import Dataset
 from udata.utils import safe_unicode
 
 from ..exceptions import HarvestException, HarvestSkipException, HarvestValidationError
-from ..models import HarvestItem, HarvestJob, HarvestError, archive_harvested_dataset
+from ..models import HarvestItem, HarvestJob, HarvestError, HarvestLog, archive_harvested_dataset
 from ..signals import before_harvest_job, after_harvest_job
 
 log = logging.getLogger(__name__)
@@ -72,8 +73,6 @@ class BaseBackend(object):
     """
     Base class that wrap children methods to add error management and debug logs.
     Also provides a few helpers needed on all or some backends.
-
-
     """
 
     name = None
@@ -139,6 +138,9 @@ class BaseBackend(object):
     def inner_process_dataset(self, item: HarvestItem) -> Dataset:
         raise NotImplementedError
 
+    def inner_process_dataservice(self, item: HarvestItem) -> Dataservice:
+        raise NotImplementedError
+
     def harvest(self):
         log.debug(f'Starting harvesting {self.source.name} ({self.source.url})…')
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
@@ -185,14 +187,17 @@ class BaseBackend(object):
         self.job.items.append(item)
         self.save_job()
 
+        log_catcher = LogCatcher()
+
         try:
             if not remote_id:
                 raise HarvestSkipException("missing identifier")
 
+            current_app.logger.addHandler(log_catcher)
             dataset = self.inner_process_dataset(item, **kwargs)
 
             # Use `item.remote_id` because `inner_process_dataset` could have modified it.
-            dataset.harvest = self.update_harvest_info(dataset.harvest, item.remote_id)
+            dataset.harvest = self.update_dataset_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
             # TODO: Apply editable mappings
@@ -220,24 +225,94 @@ class BaseBackend(object):
             error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
             item.errors.append(error)
         finally:
+            current_app.logger.removeHandler(log_catcher)
             item.ended = datetime.utcnow()
+            item.logs = [HarvestLog(level=record.levelname, message=record.getMessage()) for record in log_catcher.records]
             self.save_job()
 
     def is_done(self) -> bool:
         '''Should be called after process_dataset to know if we reach the max items'''
         return self.max_items and len(self.job.items) >= self.max_items
 
-    def update_harvest_info(self, harvest: Optional[HarvestDatasetMetadata], remote_id: int):
+    def process_dataservice(self, remote_id: str, **kwargs) -> bool :
+        '''
+        Return `True` if the parent should stop iterating because we exceed the number
+        of items to process.
+        '''
+        log.debug(f'Processing dataservice {remote_id}…')
+
+        # TODO add `type` to `HarvestItem` to differentiate `Dataset` from `Dataservice`
+        item = HarvestItem(status='started', started=datetime.utcnow(), remote_id=remote_id)
+        self.job.items.append(item)
+        self.save_job()
+
+        try:
+            if not remote_id:
+                raise HarvestSkipException("missing identifier")
+
+            dataservice = self.inner_process_dataservice(item, **kwargs)
+
+            dataservice.harvest = self.update_dataservice_harvest_info(dataservice.harvest, remote_id)
+            dataservice.archived_at = None
+
+            # TODO: Apply editable mappings
+
+            if self.dryrun:
+                dataservice.validate()
+            else:
+                dataservice.save()
+            item.dataservice = dataservice
+            item.status = 'done'
+        except HarvestSkipException as e:
+            item.status = 'skipped'
+
+            log.info(f'Skipped item {item.remote_id} : {safe_unicode(e)}')
+            item.errors.append(HarvestError(message=safe_unicode(e)))
+        except HarvestValidationError as e:
+            item.status = 'failed'
+
+            log.info(f'Error validating item {item.remote_id} : {safe_unicode(e)}')
+            item.errors.append(HarvestError(message=safe_unicode(e)))
+        except Exception as e:
+            item.status = 'failed'
+            log.exception(f'Error while processing {item.remote_id} : {safe_unicode(e)}')
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            item.errors.append(error)
+        finally:
+            item.ended = datetime.utcnow()
+            self.save_job()
+
+    def update_dataset_harvest_info(self, harvest: HarvestDatasetMetadata | None, remote_id: int):
         if not harvest:
             harvest = HarvestDatasetMetadata()
-        harvest.domain = self.source.domain
-        harvest.remote_id = remote_id
+
+        harvest.backend = self.display_name
         harvest.source_id = str(self.source.id)
+        harvest.remote_id = remote_id
+        harvest.domain = self.source.domain
         harvest.last_update = datetime.utcnow()
+        harvest.archived_at = None
+        harvest.archived = None
+
+        # created_at, modified_at, remote_url, uri, dct_identifier are set in `dataset_from_rdf`
+
+        return harvest
+
+    def update_dataservice_harvest_info(self, harvest: HarvestDataserviceMetadata | None, remote_id: int):
+        if not harvest:
+            harvest = HarvestDataserviceMetadata()
+
         harvest.backend = self.display_name
+        harvest.domain = self.source.domain
+
+        harvest.source_id = str(self.source.id)
+        harvest.source_url = str(self.source.url)
+
+        harvest.remote_id = remote_id
+        harvest.last_update = datetime.utcnow()
 
         harvest.archived_at = None
-        harvest.archived = None
 
         return harvest
 
@@ -302,6 +377,28 @@ class BaseBackend(object):
             return Dataset(owner=self.source.owner)
 
         return Dataset()
+    
+    def get_dataservice(self, remote_id):
+        '''Get or create a dataservice given its remote ID (and its source)
+        We first try to match `source_id` to be source domain independent
+        '''
+        dataservice = Dataservice.objects(__raw__={
+            'harvest.remote_id': remote_id,
+            '$or': [
+                {'harvest.domain': self.source.domain},
+                {'harvest.source_id': str(self.source.id)},
+            ],
+        }).first()
+
+        if dataservice:
+            return dataservice
+
+        if self.source.organization:
+            return Dataservice(organization=self.source.organization)
+        elif self.source.owner:
+            return Dataservice(owner=self.source.owner)
+
+        return Dataservice()
 
     def validate(self, data, schema):
         '''Perform a data validation against a given schema.
@@ -342,4 +439,15 @@ class BaseBackend(object):
                     msg = str(error)
                 errors.append(msg)
             msg = '\n- '.join(['Validation error:'] + errors)
-            raise HarvestValidationError(msg)
+            raise HarvestValidationError(msg)
+
+
+class LogCatcher(logging.Handler):
+    records: list[logging.LogRecord]
+
+    def __init__(self):
+        self.records = []
+        super().__init__()
+
+    def emit(self, record):
+        self.records.append(record)

udata/harvest/backends/dcat.py

@@ -1,20 +1,18 @@
 import logging
 
-from rdflib import Graph, URIRef
+from rdflib import Graph
 from rdflib.namespace import RDF
 import lxml.etree as ET
-import boto3
 from flask import current_app
 from datetime import date
-import json
-from typing import Generator, List
+from typing import Generator
 
-from udata.core.dataset.models import Dataset
 from udata.rdf import (
     DCAT, DCT, HYDRA, SPDX, namespace_manager, guess_format, url_from_rdf
 )
 from udata.core.dataset.rdf import dataset_from_rdf
-from udata.storage.s3 import store_as_json, get_from_json
+from udata.core.dataservices.rdf import dataservice_from_rdf
+from udata.storage.s3 import store_as_json
 from udata.harvest.models import HarvestItem
 
 from .base import BaseBackend
@@ -71,7 +69,8 @@ class DcatBackend(BaseBackend):
             self.process_one_datasets_page(page_number, page)
             serialized_graphs.append(page.serialize(format=fmt, indent=None))
 
-        # TODO call `walk_graph` with `process_dataservices`
+        for page_number, page in self.walk_graph(self.source.url, fmt):
+            self.process_one_dataservices_page(page_number, page)
 
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
         max_harvest_graph_size_in_mongo = current_app.config.get('HARVEST_MAX_CATALOG_SIZE_IN_MONGO')
@@ -145,6 +144,14 @@ class DcatBackend(BaseBackend):
 
             if self.is_done():
                 return
+
+    def process_one_dataservices_page(self, page_number: int, page: Graph):
+        for node in page.subjects(RDF.type, DCAT.DataService):
+            remote_id = page.value(node, DCT.identifier)
+            self.process_dataservice(remote_id, page_number=page_number, page=page, node=node)
+
+            if self.is_done():
+                return
             
     def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
         item.kwargs['page_number'] = page_number
@@ -152,6 +159,12 @@ class DcatBackend(BaseBackend):
         dataset = self.get_dataset(item.remote_id)
         return dataset_from_rdf(page, dataset, node=node)
 
+    def inner_process_dataservice(self, item: HarvestItem, page_number: int, page: Graph, node):
+        item.kwargs['page_number'] = page_number
+
+        dataservice = self.get_dataservice(item.remote_id)
+        return dataservice_from_rdf(page, dataservice, node, [item.dataset for item in self.job.items])
+
     def get_node_from_item(self, graph, item):
         for node in graph.subjects(RDF.type, DCAT.Dataset):
             if str(graph.value(node, DCT.identifier)) == item.remote_id:
@@ -263,6 +276,10 @@ class CswIso19139DcatBackend(DcatBackend):
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
         Yield all RDF pages as `Graph` from the source
+
+        Parse CSW graph querying ISO schema.
+        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
+        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
         """
         # Load XSLT
         xsl = ET.fromstring(self.get(self.XSL_URL).content)
@@ -284,6 +301,10 @@ class CswIso19139DcatBackend(DcatBackend):
                                         <ogc:PropertyName>dc:type</ogc:PropertyName>
                                         <ogc:Literal>dataset</ogc:Literal>
                                     </ogc:PropertyIsEqualTo>
+                                    <ogc:PropertyIsEqualTo>
+                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
+                                        <ogc:Literal>service</ogc:Literal>
+                                    </ogc:PropertyIsEqualTo>
                                     <ogc:PropertyIsEqualTo>
                                         <ogc:PropertyName>dc:type</ogc:PropertyName>
                                         <ogc:Literal>series</ogc:Literal>

udata/harvest/models.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import logging
 from urllib.parse import urlparse
 
+from udata.core.dataservices.models import Dataservice
 from werkzeug.utils import cached_property
 
 from udata.core.dataset.models import HarvestDatasetMetadata
@@ -49,16 +50,21 @@ class HarvestError(db.EmbeddedDocument):
     message = db.StringField()
     details = db.StringField()
 
+class HarvestLog(db.EmbeddedDocument):
+    level = db.StringField()
+    message = db.StringField()
 
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
     dataset = db.ReferenceField(Dataset)
+    dataservice = db.ReferenceField(Dataservice)
     status = db.StringField(choices=list(HARVEST_ITEM_STATUS),
                             default=DEFAULT_HARVEST_ITEM_STATUS, required=True)
     created = db.DateTimeField(default=datetime.utcnow, required=True)
     started = db.DateTimeField()
     ended = db.DateTimeField()
     errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
+    logs = db.ListField(db.EmbeddedDocumentField(HarvestLog), default=[])
     args = db.ListField(db.StringField())
     kwargs = db.DictField()
 

udata/harvest/tests/dcat/bnodes.xml

@@ -14,7 +14,7 @@
 >
   <dcat:Catalog rdf:about="http://data.test.org/">
     <dcat:dataset>
-      <dcat:Dataset>
+      <dcat:Dataset rdf:about="dataset-3">
         <dcterms:title>Dataset 3</dcterms:title>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <owl:versionInfo>1.0</owl:versionInfo>
@@ -73,7 +73,7 @@
       </dcat:Dataset>
     </dcat:dataset>
     <dcat:dataset>
-      <dcat:Dataset>
+      <dcat:Dataset rdf:about="dataset-2">
         <dcat:keyword>Tag 1</dcat:keyword>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/2/resources/1"/>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
@@ -107,6 +107,17 @@
         <dct:conformsTo rdf:nodeID="Ne0189e93917c4f67a412fc44883322e7"/>
       </dcat:Dataset>
     </dcat:dataset>
+    <dcat:service>
+      <dcat:DataService rdf:about="https://data.paris2024.org/api/explore/v2.1/">
+        <dcterms:title xml:lang="en"><![CDATA[Explore API v2]]></dcterms:title>
+        <dcterms:identifier>https://data.paris2024.org/api/explore/v2.1/</dcterms:identifier>
+        <dcat:endpointURL rdf:resource="https://data.paris2024.org/api/explore/v2.1/" />
+        <dcat:endpointDescription rdf:resource="https://data.paris2024.org/api/explore/v2.1/swagger.json" />
+        <dcat:landingPage rdf:resource="https://data.paris2024.org/api/explore/v2.1/console" />
+        <dcat:servesDataset rdf:resource="dataset-2" />
+        <dcat:servesDataset rdf:resource="dataset-3" />
+      </dcat:DataService>
+    </dcat:service>
     <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-15T09:19:51.723691</dcterms:modified>
     <foaf:homepage>http://data.test.org</foaf:homepage>
     <dcterms:language>en</dcterms:language>

udata/harvest/tests/test_dcat_backend.py

@@ -9,6 +9,7 @@ import boto3
 from flask import current_app
 import xml.etree.ElementTree as ET
 
+from udata.core.dataservices.models import Dataservice
 from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
@@ -161,6 +162,26 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
+    def test_harvest_dataservices(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+
+        actions.run(source.slug)
+
+        dataservices = Dataservice.objects
+
+        assert len(dataservices) == 1
+        assert dataservices[0].title == "Explore API v2"
+        assert dataservices[0].base_api_url == "https://data.paris2024.org/api/explore/v2.1/"
+        assert dataservices[0].endpoint_description_url == "https://data.paris2024.org/api/explore/v2.1/swagger.json"
+        assert dataservices[0].harvest.remote_url == "https://data.paris2024.org/api/explore/v2.1/console"
+
     def test_harvest_literal_spatial(self, rmock):
         url = mock_dcat(rmock, 'evian.json')
         org = OrganizationFactory()

udata/migrations/2024-06-11-fix-reuse-datasets-references.py

@@ -0,0 +1,35 @@
+'''
+Add a default topic to all reuses in db
+'''
+import logging
+
+from bson import DBRef
+import mongoengine
+
+from udata.models import Reuse
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info('Processing Reuse.')
+
+    reuses = Reuse.objects().no_cache().timeout(False)
+    count = 0
+    errors = 0
+
+    for reuse in reuses:
+        datasets_ids = []
+        for dataset in reuse.datasets:
+            if not isinstance(dataset, DBRef):
+                datasets_ids.append(dataset.id)
+            else:
+                errors += 1
+
+        if len(datasets_ids) != len(reuse.datasets):
+            reuse.datasets = datasets_ids
+            reuse.save()
+            count += 1
+
+    log.info(f'Modified {count} Reuses objects (removed {errors} datasets)')
+    log.info('Done')

udata/models/__init__.py

@@ -21,6 +21,7 @@ from udata.core.post.models import *  # noqa
 from udata.core.jobs.models import *  # noqa
 from udata.core.tags.models import *  # noqa
 from udata.core.spam.models import *  # noqa
+from udata.core.reports.models import *  # noqa
 
 from udata.features.transfer.models import *  # noqa
 from udata.features.territories.models import *  # noqa

udata/rdf.py

@@ -1,10 +1,11 @@
 '''
 This module centralize udata-wide RDF helpers and configuration
 '''
+from html.parser import HTMLParser
 import logging
 import re
 
-from flask import request, url_for, abort
+from flask import request, url_for, abort, current_app
 
 from rdflib import Graph, Literal, URIRef
 from rdflib.resource import Resource as RdfResource
@@ -13,14 +14,18 @@ from rdflib.namespace import (
 )
 from rdflib.util import SUFFIX_FORMAT_MAP, guess_format as raw_guess_format
 from udata import uris
+from udata.core.contact_point.models import ContactPoint
 from udata.models import Schema
 from udata.mongo.errors import FieldValidationError
+from udata.frontend.markdown import parse_html
+from udata.tags import slug as slugify_tag
 
 log = logging.getLogger(__name__)
 
 # Extra Namespaces
 ADMS = Namespace('http://www.w3.org/ns/adms#')
 DCAT = Namespace('http://www.w3.org/ns/dcat#')
+DCATAP = Namespace('http://data.europa.eu/r5r/')
 HYDRA = Namespace('http://www.w3.org/ns/hydra/core#')
 SCHEMA = Namespace('http://schema.org/')
 SCV = Namespace('http://purl.org/NET/scovo#')
@@ -35,6 +40,7 @@ VCARD = Namespace('http://www.w3.org/2006/vcard/ns#')
 
 namespace_manager = NamespaceManager(Graph())
 namespace_manager.bind('dcat', DCAT)
+namespace_manager.bind('dcatap', DCATAP)
 namespace_manager.bind('dct', DCT)
 namespace_manager.bind('foaf', FOAF)
 namespace_manager.bind('foaf', FOAF)
@@ -98,6 +104,17 @@ RDF_EXTENSIONS = {
 # Includes control characters, unicode surrogate characters and unicode end-of-plane non-characters
 ILLEGAL_XML_CHARS = '[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]'
 
+# Map High Value Datasets URIs to keyword categories
+EU_HVD_CATEGORIES = {
+    "http://data.europa.eu/bna/c_164e0bf5": "Météorologiques",
+    "http://data.europa.eu/bna/c_a9135398": "Entreprises et propriété d'entreprises",
+    "http://data.europa.eu/bna/c_ac64a52d": "Géospatiales",
+    "http://data.europa.eu/bna/c_b79e35eb": "Mobilité",
+    "http://data.europa.eu/bna/c_dd313021": "Observation de la terre et environnement",
+    "http://data.europa.eu/bna/c_e1da4e07": "Statistiques"
+}
+HVD_LEGISLATION = 'http://data.europa.eu/eli/reg_impl/2023/138/oj'
+TAG_TO_EU_HVD_CATEGORIES = {slugify_tag(EU_HVD_CATEGORIES[uri]): uri for uri in EU_HVD_CATEGORIES}
 
 def guess_format(string):
     '''Guess format given an extension or a mime-type'''
@@ -212,6 +229,42 @@ CONTEXT = {
     'totalItems': 'hydra:totalItems',
 }
 
+def serialize_value(value):
+    if isinstance(value, (URIRef, Literal)):
+        return value.toPython()
+    elif isinstance(value, RdfResource):
+        return value.identifier.toPython()
+
+
+def rdf_value(obj, predicate, default=None):
+    value = obj.value(predicate)
+    return serialize_value(value) if value else default
+
+class HTMLDetector(HTMLParser):
+    def __init__(self, *args, **kwargs):
+        HTMLParser.__init__(self, *args, **kwargs)
+        self.elements = set()
+
+    def handle_starttag(self, tag, attrs):
+        self.elements.add(tag)
+
+    def handle_endtag(self, tag):
+        self.elements.add(tag)
+
+
+def is_html(text):
+    parser = HTMLDetector()
+    parser.feed(text)
+    return bool(parser.elements)
+
+
+def sanitize_html(text):
+    text = text.toPython() if isinstance(text, Literal) else ''
+    if is_html(text):
+        return parse_html(text)
+    else:
+        return text.strip()
+
 
 def url_from_rdf(rdf, prop):
     '''
@@ -224,6 +277,65 @@ def url_from_rdf(rdf, prop):
     elif isinstance(value, RdfResource):
         return value.identifier.toPython()
 
+def theme_labels_from_rdf(rdf):
+    '''
+    Get theme labels to use as keywords.
+    Map HVD keywords from known URIs resources if HVD support is activated.
+    '''
+    for theme in rdf.objects(DCAT.theme):
+        if isinstance(theme, RdfResource):
+            uri = theme.identifier.toPython()
+            if current_app.config['HVD_SUPPORT'] and uri in EU_HVD_CATEGORIES:
+                label = EU_HVD_CATEGORIES[uri]
+                # Additionnally yield hvd keyword
+                yield 'hvd'
+            else:
+                label = rdf_value(theme, SKOS.prefLabel)
+        else:
+            label = theme.toPython()
+        if label:
+            yield label
+
+def themes_from_rdf(rdf):
+    tags = [tag.toPython() for tag in rdf.objects(DCAT.keyword)]
+    tags += theme_labels_from_rdf(rdf)
+    return list(set(tags))
+
+def contact_point_from_rdf(rdf, dataset):
+    contact_point = rdf.value(DCAT.contactPoint)
+    if contact_point:
+        name = rdf_value(contact_point, VCARD.fn) or ''
+        email = (rdf_value(contact_point, VCARD.hasEmail)
+                 or rdf_value(contact_point, VCARD.email)
+                 or rdf_value(contact_point, DCAT.email))
+        if not email:
+            return
+        email = email.replace('mailto:', '').strip()
+        if dataset.organization:
+            contact_point = ContactPoint.objects(
+                name=name, email=email, organization=dataset.organization).first()
+            return (contact_point or
+                    ContactPoint(name=name, email=email, organization=dataset.organization).save())
+        elif dataset.owner:
+            contact_point = ContactPoint.objects(
+                name=name, email=email, owner=dataset.owner).first()
+            return (contact_point or
+                    ContactPoint(name=name, email=email, owner=dataset.owner).save())
+
+def remote_url_from_rdf(rdf):
+    '''
+    Return DCAT.landingPage if found and uri validation succeeds.
+    Use RDF identifier as fallback if uri validation succeeds.
+    '''
+    landing_page = url_from_rdf(rdf, DCAT.landingPage)
+    uri = rdf.identifier.toPython()
+    for candidate in [landing_page, uri]:
+        if candidate:
+            try:
+                uris.validate(candidate)
+                return candidate
+            except uris.ValidationError:
+                pass
 
 def schema_from_rdf(rdf):
     '''
@@ -252,7 +364,6 @@ def schema_from_rdf(rdf):
                     url = uris.validate(type.identifier.toPython())
             except uris.ValidationError:
                 pass
-            pass
 
         if url is None:
             return None

udata/routing.py

@@ -217,7 +217,7 @@ def lazy_raise_or_redirect():
             new_args = request.view_args
             new_args[name] = value.arg
             new_url = url_for(request.endpoint, **new_args)
-            return redirect(new_url, code=308)
+            return redirect(new_url, code=204 if request.method == 'OPTIONS' else 308)
 
 
 def init_app(app):