udata 9.0.1.dev29504__py2.py3-none-any.whl → 9.0.1.dev29536__py2.py3-none-any.whl

udata/api_fields.py

@@ -70,10 +70,14 @@ def convert_db_to_field(key, field, info = {}):
         constructor_write = restx_fields.String
     elif isinstance(field, mongo_fields.EmbeddedDocumentField):
         nested_fields = info.get('nested_fields')
-        if nested_fields is None:
-            raise ValueError(f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize.")
+        if nested_fields is not None:
+            constructor = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
+        elif hasattr(field.document_type_obj, '__read_fields__'):
+            constructor_read = lambda **kwargs: restx_fields.Nested(field.document_type_obj.__read_fields__, **kwargs)
+            constructor_write = lambda **kwargs: restx_fields.Nested(field.document_type_obj.__write_fields__, **kwargs)
+        else:
+            raise ValueError(f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize or a `@generate_fields()` definition.")
 
-        constructor = lambda **kwargs: restx_fields.Nested(nested_fields, **kwargs)
     else:
         raise ValueError(f"Unsupported MongoEngine field type {field.__class__.__name__}")
     

udata/core/dataservices/models.py

@@ -31,6 +31,35 @@ class DataserviceQuerySet(OwnedQuerySet):
                     db.Q(deleted_at__ne=None) |
                     db.Q(archived_at__ne=None))
 
+@generate_fields()
+class HarvestMetadata(db.EmbeddedDocument):
+    backend = field(db.StringField())
+    domain = field(db.StringField())
+
+    source_id = field(db.StringField())
+    source_url = field(db.URLField())
+
+    remote_id = field(db.StringField())
+    remote_url = field(db.URLField())
+
+    # If the node ID is a `URIRef` it means it links to something external, if it's not an `URIRef` it's often a
+    # auto-generated ID just to link multiple RDF node togethers. When exporting as RDF to other catalogs, we 
+    # want to re-use this node ID (only if it's not auto-generated) to improve compatibility.
+    uri = field(
+        db.URLField(),
+        description="RDF node ID if it's an `URIRef`. `None` if it's not present or if it's a random auto-generated ID inside the graph.",
+    )
+
+    created_at = field(
+        db.DateTimeField(),
+        description="Date of the creation as provided by the harvested catalog"
+    )
+    last_update = field(
+        db.DateTimeField(),
+        description="Date of the last harvesting"
+    )
+    archived_at = field(db.DateTimeField())
+
 @generate_fields()
 class Dataservice(WithMetrics, Owned, db.Document):
     meta = {
@@ -119,12 +148,18 @@ class Dataservice(WithMetrics, Owned, db.Document):
         },
     )
 
+    harvest = field(
+        db.EmbeddedDocumentField(HarvestMetadata),
+        readonly=True,
+    )
+
     @function_field(description="Link to the API endpoint for this dataservice")
     def self_api_url(self):
         return endpoint_for('api.dataservice', dataservice=self, _external=True)
 
-    def self_web_url():
-        pass
+    @function_field(description="Link to the udata web page for this dataservice")
+    def self_web_url(self):
+        return endpoint_for('dataservices.show', dataservice=self, _external=True)
 
     # TODO
     # frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys()))

udata/core/dataservices/rdf.py

@@ -0,0 +1,58 @@
+
+from datetime import datetime
+from typing import List, Optional
+from rdflib import RDF, Graph, URIRef
+
+from udata.core.dataservices.models import Dataservice, HarvestMetadata as HarvestDataserviceMetadata
+from udata.core.dataset.models import Dataset, License
+from udata.core.dataset.rdf import sanitize_html
+from udata.harvest.models import HarvestSource
+from udata.rdf import DCAT, DCT, contact_point_from_rdf, rdf_value, remote_url_from_rdf, theme_labels_from_rdf, themes_from_rdf, url_from_rdf
+
+def dataservice_from_rdf(graph: Graph, dataservice: Dataservice, node, all_datasets: List[Dataset]) -> Dataservice :
+    '''
+    Create or update a dataset from a RDF/DCAT graph
+    '''
+    if node is None:  # Assume first match is the only match
+        node = graph.value(predicate=RDF.type, object=DCAT.DataService)
+
+    d = graph.resource(node)
+
+    dataservice.title = rdf_value(d, DCT.title)
+    dataservice.description = sanitize_html(d.value(DCT.description) or d.value(DCT.abstract))
+
+    dataservice.base_api_url = url_from_rdf(d, DCAT.endpointURL)
+    dataservice.endpoint_description_url = url_from_rdf(d, DCAT.endpointDescription)
+
+    dataservice.contact_point = contact_point_from_rdf(d, dataservice) or dataservice.contact_point
+
+    datasets = []
+    for dataset_node in d.objects(DCAT.servesDataset):
+        id = dataset_node.value(DCT.identifier)
+        dataset = next((d for d in all_datasets if d is not None and d.harvest.remote_id == id), None)
+
+        if dataset is None:
+            # We try with `endswith` because Europe XSLT have problems with IDs. Sometimes they are prefixed with the domain of the catalog, sometimes not.
+            dataset = next((d for d in all_datasets if d is not None and d.harvest.remote_id.endswith(id)), None)
+
+        if dataset is not None:
+            datasets.append(dataset.id)
+
+    if datasets:
+        dataservice.datasets = datasets
+
+    license = rdf_value(d, DCT.license)
+    if license is not None:
+        dataservice.license = License.guess(license)
+
+    if not dataservice.harvest:
+        dataservice.harvest = HarvestDataserviceMetadata()
+
+    dataservice.harvest.uri = d.identifier.toPython() if isinstance(d.identifier, URIRef) else None
+    dataservice.harvest.remote_url = remote_url_from_rdf(d)
+    dataservice.harvest.created_at = rdf_value(d, DCT.issued)
+    dataservice.metadata_modified_at = rdf_value(d, DCT.modified)
+
+    dataservice.tags = themes_from_rdf(d)
+
+    return dataservice

udata/core/dataset/rdf.py

@@ -6,7 +6,6 @@ import json
 import logging
 
 from datetime import date
-from html.parser import HTMLParser
 from typing import Optional
 from dateutil.parser import parse as parse_dt
 from flask import current_app
@@ -18,14 +17,14 @@ from mongoengine.errors import ValidationError
 
 from udata import i18n, uris
 from udata.core.spatial.models import SpatialCoverage
-from udata.frontend.markdown import parse_html
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
-from udata.models import db, ContactPoint
+from udata.harvest.exceptions import HarvestSkipException
+from udata.models import db
 from udata.rdf import (
-    DCAT, DCATAP, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, VCARD, RDFS,
-    HVD_LEGISLATION, namespace_manager, schema_from_rdf, url_from_rdf
+    DCAT, DCATAP, DCT, FREQ, SCV, SKOS, SPDX, SCHEMA, EUFREQ, EUFORMAT, IANAFORMAT, TAG_TO_EU_HVD_CATEGORIES, RDFS, 
+    namespace_manager, rdf_value, remote_url_from_rdf, sanitize_html, schema_from_rdf, themes_from_rdf, url_from_rdf, HVD_LEGISLATION,
+    contact_point_from_rdf,
 )
-from udata.tags import slug as slugify_tag
 from udata.utils import get_by, safe_unicode
 from udata.uris import endpoint_for
 
@@ -77,44 +76,6 @@ EU_RDF_REQUENCIES = {
     EUFREQ.NEVER: 'punctual',
 }
 
-# Map High Value Datasets URIs to keyword categories
-EU_HVD_CATEGORIES = {
-    "http://data.europa.eu/bna/c_164e0bf5": "Météorologiques",
-    "http://data.europa.eu/bna/c_a9135398": "Entreprises et propriété d'entreprises",
-    "http://data.europa.eu/bna/c_ac64a52d": "Géospatiales",
-    "http://data.europa.eu/bna/c_b79e35eb": "Mobilité",
-    "http://data.europa.eu/bna/c_dd313021": "Observation de la terre et environnement",
-    "http://data.europa.eu/bna/c_e1da4e07": "Statistiques"
-}
-TAG_TO_EU_HVD_CATEGORIES = {slugify_tag(EU_HVD_CATEGORIES[uri]): uri for uri in EU_HVD_CATEGORIES}
-
-
-class HTMLDetector(HTMLParser):
-    def __init__(self, *args, **kwargs):
-        HTMLParser.__init__(self, *args, **kwargs)
-        self.elements = set()
-
-    def handle_starttag(self, tag, attrs):
-        self.elements.add(tag)
-
-    def handle_endtag(self, tag):
-        self.elements.add(tag)
-
-
-def is_html(text):
-    parser = HTMLDetector()
-    parser.feed(text)
-    return bool(parser.elements)
-
-
-def sanitize_html(text):
-    text = text.toPython() if isinstance(text, Literal) else ''
-    if is_html(text):
-        return parse_html(text)
-    else:
-        return text.strip()
-
-
 def temporal_to_rdf(daterange, graph=None):
     if not daterange:
         return
@@ -255,18 +216,6 @@ CHECKSUM_ALGORITHMS = {
 }
 
 
-def serialize_value(value):
-    if isinstance(value, (URIRef, Literal)):
-        return value.toPython()
-    elif isinstance(value, RdfResource):
-        return value.identifier.toPython()
-
-
-def rdf_value(obj, predicate, default=None):
-    value = obj.value(predicate)
-    return serialize_value(value) if value else default
-
-
 def temporal_from_literal(text):
     '''
     Parse a temporal coverage from a literal ie. either:
@@ -341,29 +290,6 @@ def temporal_from_rdf(period_of_time):
         # so we log the error for future investigation and improvement
         log.warning('Unable to parse temporal coverage', exc_info=True)
 
-
-def contact_point_from_rdf(rdf, dataset):
-    contact_point = rdf.value(DCAT.contactPoint)
-    if contact_point:
-        name = rdf_value(contact_point, VCARD.fn) or ''
-        email = (rdf_value(contact_point, VCARD.hasEmail)
-                 or rdf_value(contact_point, VCARD.email)
-                 or rdf_value(contact_point, DCAT.email))
-        if not email:
-            return
-        email = email.replace('mailto:', '').strip()
-        if dataset.organization:
-            contact_point = ContactPoint.objects(
-                name=name, email=email, organization=dataset.organization).first()
-            return (contact_point or
-                    ContactPoint(name=name, email=email, organization=dataset.organization).save())
-        elif dataset.owner:
-            contact_point = ContactPoint.objects(
-                name=name, email=email, owner=dataset.owner).first()
-            return (contact_point or
-                    ContactPoint(name=name, email=email, owner=dataset.owner).save())
-
-
 def spatial_from_rdf(graph):
     geojsons = []
     for term in graph.objects(DCT.spatial):
@@ -503,43 +429,6 @@ def title_from_rdf(rdf, url):
         else:
             return i18n._('Nameless resource')
 
-
-def remote_url_from_rdf(rdf):
-    '''
-    Return DCAT.landingPage if found and uri validation succeeds.
-    Use RDF identifier as fallback if uri validation succeeds.
-    '''
-    landing_page = url_from_rdf(rdf, DCAT.landingPage)
-    uri = rdf.identifier.toPython()
-    for candidate in [landing_page, uri]:
-        if candidate:
-            try:
-                uris.validate(candidate)
-                return candidate
-            except uris.ValidationError:
-                pass
-
-
-def theme_labels_from_rdf(rdf):
-    '''
-    Get theme labels to use as keywords.
-    Map HVD keywords from known URIs resources if HVD support is activated.
-    '''
-    for theme in rdf.objects(DCAT.theme):
-        if isinstance(theme, RdfResource):
-            uri = theme.identifier.toPython()
-            if current_app.config['HVD_SUPPORT'] and uri in EU_HVD_CATEGORIES:
-                label = EU_HVD_CATEGORIES[uri]
-                # Additionnally yield hvd keyword
-                yield 'hvd'
-            else:
-                label = rdf_value(theme, SKOS.prefLabel)
-        else:
-            label = theme.toPython()
-        if label:
-            yield label
-
-
 def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     '''
     Map a Resource domain model to a DCAT/RDF graph
@@ -617,6 +506,9 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     d = graph.resource(node)
 
     dataset.title = rdf_value(d, DCT.title)
+    if not dataset.title:
+        raise HarvestSkipException("missing title on dataset")
+
     # Support dct:abstract if dct:description is missing (sometimes used instead)
     description = d.value(DCT.description) or d.value(DCT.abstract)
     dataset.description = sanitize_html(description)
@@ -634,9 +526,7 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None):
     if acronym:
         dataset.acronym = acronym
 
-    tags = [tag.toPython() for tag in d.objects(DCAT.keyword)]
-    tags += theme_labels_from_rdf(d)
-    dataset.tags = list(set(tags))
+    dataset.tags = themes_from_rdf(d)
 
     temporal_coverage = temporal_from_rdf(d.value(DCT.temporal))
     if temporal_coverage:

udata/harvest/api.py

@@ -5,6 +5,7 @@ from flask import request
 from udata.api import api, API, fields
 from udata.auth import admin_permission
 
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_ref_fields, dataset_fields
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.organization.permissions import EditOrganizationPermission
@@ -45,6 +46,9 @@ item_fields = api.model('HarvestItem', {
     'dataset': fields.Nested(dataset_ref_fields,
                              description='The processed dataset',
                              allow_null=True),
+    'dataservice': fields.Nested(Dataservice.__read_fields__,
+                             description='The processed dataservice',
+                             allow_null=True),
     'status': fields.String(description='The item status',
                             required=True,
                             enum=list(HARVEST_ITEM_STATUS)),

udata/harvest/backends/base.py

@@ -8,9 +8,11 @@ from uuid import UUID
 import requests
 
 from flask import current_app
+from udata.core.dataservices.models import Dataservice
 from voluptuous import MultipleInvalid, RequiredFieldInvalid
 
 from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.models import Dataset
 from udata.utils import safe_unicode
 
@@ -72,8 +74,6 @@ class BaseBackend(object):
     """
     Base class that wrap children methods to add error management and debug logs.
     Also provides a few helpers needed on all or some backends.
-
-
     """
 
     name = None
@@ -139,6 +139,9 @@ class BaseBackend(object):
     def inner_process_dataset(self, item: HarvestItem) -> Dataset:
         raise NotImplementedError
 
+    def inner_process_dataservice(self, item: HarvestItem) -> Dataservice:
+        raise NotImplementedError
+
     def harvest(self):
         log.debug(f'Starting harvesting {self.source.name} ({self.source.url})…')
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
@@ -195,7 +198,7 @@ class BaseBackend(object):
             dataset = self.inner_process_dataset(item, **kwargs)
 
             # Use `item.remote_id` because `inner_process_dataset` could have modified it.
-            dataset.harvest = self.update_harvest_info(dataset.harvest, item.remote_id)
+            dataset.harvest = self.update_dataset_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
             # TODO: Apply editable mappings
@@ -232,17 +235,85 @@ class BaseBackend(object):
         '''Should be called after process_dataset to know if we reach the max items'''
         return self.max_items and len(self.job.items) >= self.max_items
 
-    def update_harvest_info(self, harvest: Optional[HarvestDatasetMetadata], remote_id: int):
+    def process_dataservice(self, remote_id: str, **kwargs) -> bool :
+        '''
+        Return `True` if the parent should stop iterating because we exceed the number
+        of items to process.
+        '''
+        log.debug(f'Processing dataservice {remote_id}…')
+
+        # TODO add `type` to `HarvestItem` to differentiate `Dataset` from `Dataservice`
+        item = HarvestItem(status='started', started=datetime.utcnow(), remote_id=remote_id)
+        self.job.items.append(item)
+        self.save_job()
+
+        try:
+            if not remote_id:
+                raise HarvestSkipException("missing identifier")
+
+            dataservice = self.inner_process_dataservice(item, **kwargs)
+
+            dataservice.harvest = self.update_dataservice_harvest_info(dataservice.harvest, remote_id)
+            dataservice.archived_at = None
+
+            # TODO: Apply editable mappings
+
+            if self.dryrun:
+                dataservice.validate()
+            else:
+                dataservice.save()
+            item.dataservice = dataservice
+            item.status = 'done'
+        except HarvestSkipException as e:
+            item.status = 'skipped'
+
+            log.info(f'Skipped item {item.remote_id} : {safe_unicode(e)}')
+            item.errors.append(HarvestError(message=safe_unicode(e)))
+        except HarvestValidationError as e:
+            item.status = 'failed'
+
+            log.info(f'Error validating item {item.remote_id} : {safe_unicode(e)}')
+            item.errors.append(HarvestError(message=safe_unicode(e)))
+        except Exception as e:
+            item.status = 'failed'
+            log.exception(f'Error while processing {item.remote_id} : {safe_unicode(e)}')
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            item.errors.append(error)
+        finally:
+            item.ended = datetime.utcnow()
+            self.save_job()
+
+    def update_dataset_harvest_info(self, harvest: Optional[HarvestDatasetMetadata], remote_id: int):
         if not harvest:
             harvest = HarvestDatasetMetadata()
-        harvest.domain = self.source.domain
-        harvest.remote_id = remote_id
+
+        harvest.backend = self.display_name
         harvest.source_id = str(self.source.id)
+        harvest.remote_id = remote_id
+        harvest.domain = self.source.domain
         harvest.last_update = datetime.utcnow()
+        harvest.archived_at = None
+        harvest.archived = None
+
+        # created_at, modified_at, remote_url, uri, dct_identifier are set in `dataset_from_rdf`
+
+        return harvest
+
+    def update_dataservice_harvest_info(self, harvest: Optional[HarvestDataserviceMetadata], remote_id: int):
+        if not harvest:
+            harvest = HarvestDataserviceMetadata()
+
         harvest.backend = self.display_name
+        harvest.domain = self.source.domain
+
+        harvest.source_id = str(self.source.id)
+        harvest.source_url = str(self.source.url)
+
+        harvest.remote_id = remote_id
+        harvest.last_update = datetime.utcnow()
 
         harvest.archived_at = None
-        harvest.archived = None
 
         return harvest
 
@@ -307,6 +378,28 @@ class BaseBackend(object):
             return Dataset(owner=self.source.owner)
 
         return Dataset()
+    
+    def get_dataservice(self, remote_id):
+        '''Get or create a dataservice given its remote ID (and its source)
+        We first try to match `source_id` to be source domain independent
+        '''
+        dataservice = Dataservice.objects(__raw__={
+            'harvest.remote_id': remote_id,
+            '$or': [
+                {'harvest.domain': self.source.domain},
+                {'harvest.source_id': str(self.source.id)},
+            ],
+        }).first()
+
+        if dataservice:
+            return dataservice
+
+        if self.source.organization:
+            return Dataservice(organization=self.source.organization)
+        elif self.source.owner:
+            return Dataservice(owner=self.source.owner)
+
+        return Dataservice()
 
     def validate(self, data, schema):
         '''Perform a data validation against a given schema.

udata/harvest/backends/dcat.py

@@ -1,20 +1,19 @@
 import logging
 
-from rdflib import Graph, URIRef
+from rdflib import Graph
 from rdflib.namespace import RDF
 import lxml.etree as ET
-import boto3
 from flask import current_app
 from datetime import date
-import json
-from typing import Generator, List
+from typing import Generator
+from typing import Generator
 
-from udata.core.dataset.models import Dataset
 from udata.rdf import (
     DCAT, DCT, HYDRA, SPDX, namespace_manager, guess_format, url_from_rdf
 )
 from udata.core.dataset.rdf import dataset_from_rdf
-from udata.storage.s3 import store_as_json, get_from_json
+from udata.core.dataservices.rdf import dataservice_from_rdf
+from udata.storage.s3 import store_as_json
 from udata.harvest.models import HarvestItem
 
 from .base import BaseBackend
@@ -71,7 +70,8 @@ class DcatBackend(BaseBackend):
             self.process_one_datasets_page(page_number, page)
             serialized_graphs.append(page.serialize(format=fmt, indent=None))
 
-        # TODO call `walk_graph` with `process_dataservices`
+        for page_number, page in self.walk_graph(self.source.url, fmt):
+            self.process_one_dataservices_page(page_number, page)
 
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
         max_harvest_graph_size_in_mongo = current_app.config.get('HARVEST_MAX_CATALOG_SIZE_IN_MONGO')
@@ -145,6 +145,14 @@ class DcatBackend(BaseBackend):
 
             if self.is_done():
                 return
+
+    def process_one_dataservices_page(self, page_number: int, page: Graph):
+        for node in page.subjects(RDF.type, DCAT.DataService):
+            remote_id = page.value(node, DCT.identifier)
+            self.process_dataservice(remote_id, page_number=page_number, page=page, node=node)
+
+            if self.is_done():
+                return
             
     def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
         item.kwargs['page_number'] = page_number
@@ -152,6 +160,12 @@ class DcatBackend(BaseBackend):
         dataset = self.get_dataset(item.remote_id)
         return dataset_from_rdf(page, dataset, node=node)
 
+    def inner_process_dataservice(self, item: HarvestItem, page_number: int, page: Graph, node):
+        item.kwargs['page_number'] = page_number
+
+        dataservice = self.get_dataservice(item.remote_id)
+        return dataservice_from_rdf(page, dataservice, node, [item.dataset for item in self.job.items])
+
     def get_node_from_item(self, graph, item):
         for node in graph.subjects(RDF.type, DCAT.Dataset):
             if str(graph.value(node, DCT.identifier)) == item.remote_id:
@@ -263,6 +277,10 @@ class CswIso19139DcatBackend(DcatBackend):
     def walk_graph(self, url: str, fmt: str) -> Generator[tuple[int, Graph], None, None]:
         """
         Yield all RDF pages as `Graph` from the source
+
+        Parse CSW graph querying ISO schema.
+        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
+        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
         """
         # Load XSLT
         xsl = ET.fromstring(self.get(self.XSL_URL).content)
@@ -284,6 +302,10 @@ class CswIso19139DcatBackend(DcatBackend):
                                         <ogc:PropertyName>dc:type</ogc:PropertyName>
                                         <ogc:Literal>dataset</ogc:Literal>
                                     </ogc:PropertyIsEqualTo>
+                                    <ogc:PropertyIsEqualTo>
+                                        <ogc:PropertyName>dc:type</ogc:PropertyName>
+                                        <ogc:Literal>service</ogc:Literal>
+                                    </ogc:PropertyIsEqualTo>
                                     <ogc:PropertyIsEqualTo>
                                         <ogc:PropertyName>dc:type</ogc:PropertyName>
                                         <ogc:Literal>series</ogc:Literal>

udata/harvest/models.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import logging
 from urllib.parse import urlparse
 
+from udata.core.dataservices.models import Dataservice
 from werkzeug.utils import cached_property
 
 from udata.core.dataset.models import HarvestDatasetMetadata
@@ -56,6 +57,7 @@ class HarvestLog(db.EmbeddedDocument):
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
     dataset = db.ReferenceField(Dataset)
+    dataservice = db.ReferenceField(Dataservice)
     status = db.StringField(choices=list(HARVEST_ITEM_STATUS),
                             default=DEFAULT_HARVEST_ITEM_STATUS, required=True)
     created = db.DateTimeField(default=datetime.utcnow, required=True)

udata/harvest/tests/dcat/bnodes.xml

@@ -14,7 +14,7 @@
 >
   <dcat:Catalog rdf:about="http://data.test.org/">
     <dcat:dataset>
-      <dcat:Dataset>
+      <dcat:Dataset rdf:about="dataset-3">
         <dcterms:title>Dataset 3</dcterms:title>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <owl:versionInfo>1.0</owl:versionInfo>
@@ -73,7 +73,7 @@
       </dcat:Dataset>
     </dcat:dataset>
     <dcat:dataset>
-      <dcat:Dataset>
+      <dcat:Dataset rdf:about="dataset-2">
         <dcat:keyword>Tag 1</dcat:keyword>
         <dcat:distribution rdf:resource="http://data.test.org/datasets/2/resources/1"/>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
@@ -107,6 +107,17 @@
         <dct:conformsTo rdf:nodeID="Ne0189e93917c4f67a412fc44883322e7"/>
       </dcat:Dataset>
     </dcat:dataset>
+    <dcat:service>
+      <dcat:DataService rdf:about="https://data.paris2024.org/api/explore/v2.1/">
+        <dcterms:title xml:lang="en"><![CDATA[Explore API v2]]></dcterms:title>
+        <dcterms:identifier>https://data.paris2024.org/api/explore/v2.1/</dcterms:identifier>
+        <dcat:endpointURL rdf:resource="https://data.paris2024.org/api/explore/v2.1/" />
+        <dcat:endpointDescription rdf:resource="https://data.paris2024.org/api/explore/v2.1/swagger.json" />
+        <dcat:landingPage rdf:resource="https://data.paris2024.org/api/explore/v2.1/console" />
+        <dcat:servesDataset rdf:resource="dataset-2" />
+        <dcat:servesDataset rdf:resource="dataset-3" />
+      </dcat:DataService>
+    </dcat:service>
     <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-15T09:19:51.723691</dcterms:modified>
     <foaf:homepage>http://data.test.org</foaf:homepage>
     <dcterms:language>en</dcterms:language>

udata/harvest/tests/test_dcat_backend.py

@@ -9,6 +9,7 @@ import boto3
 from flask import current_app
 import xml.etree.ElementTree as ET
 
+from udata.core.dataservices.models import Dataservice
 from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
@@ -161,6 +162,26 @@ class DcatBackendTest:
         assert len(datasets['1'].resources) == 2
         assert len(datasets['2'].resources) == 2
 
+    def test_harvest_dataservices(self, rmock):
+        rmock.get('https://example.com/schemas', json=ResourceSchemaMockData.get_mock_data())
+
+        filename = 'bnodes.xml'
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+
+        actions.run(source.slug)
+
+        dataservices = Dataservice.objects
+
+        assert len(dataservices) == 1
+        assert dataservices[0].title == "Explore API v2"
+        assert dataservices[0].base_api_url == "https://data.paris2024.org/api/explore/v2.1/"
+        assert dataservices[0].endpoint_description_url == "https://data.paris2024.org/api/explore/v2.1/swagger.json"
+        assert dataservices[0].harvest.remote_url == "https://data.paris2024.org/api/explore/v2.1/console"
+
     def test_harvest_literal_spatial(self, rmock):
         url = mock_dcat(rmock, 'evian.json')
         org = OrganizationFactory()