udata 10.2.1.dev34683__py2.py3-none-any.whl → 10.2.1.dev34728__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -722,6 +722,10 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
             resource.checksum.type = algorithm
     if is_additionnal:
         resource.type = "other"
+    elif distrib.value(DCAT.accessService):
+        # The distribution has a DCAT.accessService property, we deduce
+        # that the distribution is of type API
+        resource.type = "api"
 
     identifier = rdf_value(distrib, DCT.identifier)
     uri = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
@@ -751,6 +755,8 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
 
     dataset.title = rdf_value(d, DCT.title)
     if not dataset.title:
+        # If the dataset is externaly defined (so without title and just with a link to the dataset XML)
+        # we should have skipped it way before in :ExcludeExternalyDefinedDataset
         raise HarvestSkipException("missing title on dataset")
 
     # Support dct:abstract if dct:description is missing (sometimes used instead)

udata/harvest/backends/base.py

@@ -256,7 +256,7 @@ class BaseBackend(object):
             ]
             self.save_job()
 
-    def is_done(self) -> bool:
+    def has_reached_max_items(self) -> bool:
         """Should be called after process_dataset to know if we reach the max items"""
         return self.max_items and len(self.job.items) >= self.max_items
 

udata/harvest/backends/dcat.py

@@ -9,7 +9,7 @@ from rdflib.namespace import RDF
 
 from udata.core.dataservices.rdf import dataservice_from_rdf
 from udata.core.dataset.rdf import dataset_from_rdf
-from udata.harvest.models import HarvestItem
+from udata.harvest.models import HarvestError, HarvestItem
 from udata.i18n import gettext as _
 from udata.rdf import (
     DCAT,
@@ -18,6 +18,7 @@ from udata.rdf import (
     SPDX,
     guess_format,
     namespace_manager,
+    rdf_value,
     url_from_rdf,
 )
 from udata.storage.s3 import store_as_json
@@ -77,9 +78,19 @@ class DcatBackend(BaseBackend):
             self.process_one_datasets_page(page_number, page)
             serialized_graphs.append(page.serialize(format=fmt, indent=None))
 
+        # We do a second pass to have all datasets in memory and attach datasets
+        # to dataservices. It could be better to be one pass of graph walking and
+        # then one pass of attaching datasets to dataservices.
         for page_number, page in self.walk_graph(self.source.url, fmt):
             self.process_one_dataservices_page(page_number, page)
 
+        if not self.dryrun and self.has_reached_max_items():
+            # We have reached the max_items limit. Warn the user that all the datasets may not be present.
+            error = HarvestError(
+                message=f"{self.max_items} max items reached, not all datasets/dataservices were retrieved"
+            )
+            self.job.errors.append(error)
+
         # The official MongoDB document size in 16MB. The default value here is 15MB to account for other fields in the document (and for difference between * 1024 vs * 1000).
         max_harvest_graph_size_in_mongo = current_app.config.get(
             "HARVEST_MAX_CATALOG_SIZE_IN_MONGO"
@@ -146,7 +157,7 @@ class DcatBackend(BaseBackend):
                     break
 
             yield page_number, subgraph
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
             page_number += 1
@@ -154,17 +165,48 @@ class DcatBackend(BaseBackend):
     def process_one_datasets_page(self, page_number: int, page: Graph):
         for node in page.subjects(RDF.type, DCAT.Dataset):
             remote_id = page.value(node, DCT.identifier)
+            if self.is_dataset_external_to_this_page(page, node):
+                continue
+
             self.process_dataset(remote_id, page_number=page_number, page=page, node=node)
 
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
+    def is_dataset_external_to_this_page(self, page: Graph, node) -> bool:
+        # In dataservice nodes we have `servesDataset` or `hasPart` that can contains nodes
+        # with type=dataset. We don't want to process them because these nodes are empty (they
+        # only contains a link to the dataset definition).
+        # These datasets are either present in the catalog in previous or next pages or
+        # external from the catalog we are currently harvesting (so we don't want to harvest them).
+        # First we thought of skipping them inside `dataset_from_rdf` (see :ExcludeExternalyDefinedDataset)
+        # but it creates a lot of "fake" items in the job and raising problems (reaching the max harvest item for
+        # example and not getting to the "real" datasets/dataservices in subsequent pages)
+        # So to prevent creating a lot of useless items in the job we first thought about checking to see if there is no title and
+        # if `isPrimaryTopicOf` is present. But it may be better to check if the only link of the node with the current page is a
+        # `servesDataset` or `hasPart`. If it's the case, the node is only present in a dataservice. (maybe we could also check that
+        # the `_other_node` is a dataservice?)
+        # `isPrimaryTopicOf` is the tag present in the first harvester raising the problem, it may exists other
+        # values of the same sort we need to check here.
+
+        # This is not dangerous because we check for missing title in `dataset_from_rdf` later so we would have skipped
+        # this dataset anyway.
+        resource = page.resource(node)
+        title = rdf_value(resource, DCT.title)
+        if title:
+            return False
+
+        predicates = [link_type for (_other_node, link_type) in page.subject_predicates(node)]
+        return len(predicates) == 1 and (
+            predicates[0] == DCAT.servesDataset or predicates[0] == DCT.hasPart
+        )
+
     def process_one_dataservices_page(self, page_number: int, page: Graph):
         for node in page.subjects(RDF.type, DCAT.DataService):
             remote_id = page.value(node, DCT.identifier)
             self.process_dataservice(remote_id, page_number=page_number, page=page, node=node)
 
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
     def inner_process_dataset(self, item: HarvestItem, page_number: int, page: Graph, node):
@@ -266,7 +308,7 @@ class CswDcatBackend(DcatBackend):
                 subgraph.parse(data=ET.tostring(child), format=fmt)
 
                 yield page_number, subgraph
-                if self.is_done():
+                if self.has_reached_max_items():
                     return
 
             next_record = self.next_record_if_should_continue(start, search_results)
@@ -375,7 +417,7 @@ class CswIso19139DcatBackend(DcatBackend):
                 raise ValueError("Failed to fetch CSW content")
 
             yield page_number, subgraph
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
             next_record = self.next_record_if_should_continue(start, search_results)

udata/harvest/tests/dcat/catalog.xml

@@ -61,16 +61,17 @@
         <dcat:theme>Theme 1</dcat:theme>
         <dcterms:publisher rdf:resource="http://data.test.org/organizations/1"/>
         <owl:versionInfo>1.0</owl:versionInfo>
-        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcat:keyword>Tag 4</dcat:keyword>
         <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 1</dcat:keyword>
-        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
         <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:identifier>1</dcterms:identifier>
+        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
+        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcterms:hasPart rdf:resource="http://data.test.org/datasets/1/resources/3"/>
+        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/4"/>
       </dcat:Dataset>
     </dcat:dataset>
     <dcat:dataset>
@@ -152,6 +153,18 @@
     <dcterms:title>Resource 1-3</dcterms:title>
     <dcterms:format>JSON</dcterms:format>
   </foaf:Document>
+  <dcat:Distribution rdf:about="http://data.test.org/datasets/1/resources/4">
+    <dcterms:description>A resource pointing towards a Geo Service</dcterms:description>
+    <dcterms:title>Resource 1-4</dcterms:title>
+    <dcat:accessURL>http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0</dcat:accessURL>
+    <dcat:accessService>
+      <dcat:DataService>
+        <dcterms:title xml:lang="fr">Geo Service</dcterms:title>
+        <dcat:endpointURL rdf:resource="http://data.test.org/datasets/1/resources/4/services"/>
+        <dcat:endpointDescription rdf:resource="http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0"/>
+      </dcat:DataService>
+    </dcat:accessService>
+  </dcat:Distribution>
   <!-- resources for dataset 2 -->
   <dcat:Distribution rdf:about="http://data.test.org/datasets/2/resources/1">
     <dcat:accessURL>http://data.test.org/datasets/2/resources/1/file.json</dcat:accessURL>

udata/harvest/tests/factories.py

@@ -61,7 +61,7 @@ class FactoryBackend(backends.BaseBackend):
         mock_initialize.send(self)
         for i in range(self.config.get("count", DEFAULT_COUNT)):
             self.process_dataset(str(i))
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
     def inner_process_dataset(self, item: HarvestItem):

udata/harvest/tests/test_base_backend.py

@@ -44,12 +44,12 @@ class FakeBackend(BaseBackend):
     def inner_harvest(self):
         for remote_id in self.source.config.get("dataset_remote_ids", []):
             self.process_dataset(remote_id)
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
         for remote_id in self.source.config.get("dataservice_remote_ids", []):
             self.process_dataservice(remote_id)
-            if self.is_done():
+            if self.has_reached_max_items():
                 return
 
     def inner_process_dataset(self, item: HarvestItem):

udata/harvest/tests/test_dcat_backend.py

@@ -137,6 +137,7 @@ class DcatBackendTest:
         assert datasets["1"].resources[0].description == "A JSON resource"
         assert datasets["1"].resources[0].format == "json"
         assert datasets["1"].resources[0].mime == "application/json"
+        assert datasets["1"].resources[0].type == "main"
 
     @pytest.mark.options(
         SCHEMA_CATALOG_URL="https://example.com/schemas",
@@ -403,6 +404,7 @@ class DcatBackendTest:
         assert len(dataset.resources) == 1
 
         resource = dataset.resources[0]
+        assert resource.type == "main"
         assert resource.checksum is not None
         assert resource.checksum.type == "sha1"
         assert resource.checksum.value == "fb4106aa286a53be44ec99515f0f0421d4d7ad7d"
@@ -476,7 +478,12 @@ class DcatBackendTest:
 
         assert job.status == "done"
         assert job.errors == []
-        assert len(job.items) == 4
+        assert len(job.items) == 5
+        # 4 datasets and one Dataservice mentionned but not described
+        # because it appears in a distribution as DCAT.accessService
+        # but is missing a proper DCT.identifier
+        assert len([item for item in job.items if item.status == "done"]) == 4
+        assert len([item for item in job.items if item.status == "skipped"]) == 1
 
     def test_xml_catalog(self, rmock):
         LicenseFactory(id="lov2", title="Licence Ouverte Version 2.0")
@@ -524,7 +531,7 @@ class DcatBackendTest:
         # test dct:license nested in distribution
         assert dataset.license.id == "lov1"
 
-        assert len(dataset.resources) == 3
+        assert len(dataset.resources) == 4
 
         resource_1 = next(res for res in dataset.resources if res.title == "Resource 1-1")
         assert resource_1.filetype == "remote"
@@ -549,6 +556,16 @@ class DcatBackendTest:
         assert resource_3.url == "http://data.test.org/datasets/1/resources/3"
         assert resource_3.type == "other"
 
+        # Make sure a resource with an accessService is of type api
+        resource_4 = next(res for res in dataset.resources if res.title == "Resource 1-4")
+        assert resource_4.format is None
+        assert resource_4.description == "A resource pointing towards a Geo Service"
+        assert (
+            resource_4.url
+            == "http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0"
+        )
+        assert resource_4.type == "api"
+
         # test dct:rights -> license support from dataset
         dataset = Dataset.objects.get(harvest__dct_identifier="2")
         assert dataset.license.id == "lov2"
@@ -838,6 +855,7 @@ class CswDcatBackendTest:
         assert resource.title == "accidento_hdf_L93"
         assert resource.url == "https://www.geo2france.fr/geoserver/cr_hdf/ows"
         assert resource.format == "ogc:wms"
+        assert resource.type == "main"
 
     def test_user_agent_post(self, rmock):
         url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetworkv4-page-{}.xml")
@@ -949,6 +967,7 @@ class CswIso19139DcatBackendTest:
             resource.url
             == "http://atom.geo-ide.developpement-durable.gouv.fr/atomArchive/GetResource?id=fr-120066022-ldd-cab63273-b3ae-4e8a-ae1c-6192e45faa94&datasetAggregate=true"
         )
+        assert resource.type == "main"
 
         # Sadly resource format is parsed as a blank node. Format parsing should be improved.
         assert re.match(r"n[0-9a-f]{32}", resource.format)