udata 6.2.1.dev26916__py2.py3-none-any.whl → 6.2.1.dev26928__py2.py3-none-any.whl

udata/core/dataset/apiv2.py

@@ -232,7 +232,7 @@ class DatasetExtrasAPI(API):
             data.pop(key)
         # then update the extras with the remaining payload
         dataset.extras.update(data)
-        dataset.save()
+        dataset.save(signal_kwargs={'ignores': ['post_save']})
         return dataset.extras
 
     @apiv2.secure
@@ -250,7 +250,7 @@ class DatasetExtrasAPI(API):
                 del dataset.extras[key]
         except KeyError:
             apiv2.abort(404, 'Key not found in existing extras')
-        dataset.save()
+        dataset.save(signal_kwargs={'ignores': ['post_save']})
         return dataset.extras, 204
 
 
@@ -348,7 +348,7 @@ class ResourceExtrasAPI(ResourceMixin, API):
             data.pop(key)
         # then update the extras with the remaining payload
         resource.extras.update(data)
-        resource.save()
+        resource.save(signal_kwargs={'ignores': ['post_save']})
         return resource.extras
 
     @apiv2.secure
@@ -367,5 +367,5 @@ class ResourceExtrasAPI(ResourceMixin, API):
                 del resource.extras[key]
         except KeyError:
             apiv2.abort(404, 'Key not found in existing extras')
-        resource.save()
+        resource.save(signal_kwargs={'ignores': ['post_save']})
         return resource.extras, 204

udata/core/dataset/rdf.py

@@ -389,6 +389,7 @@ def remote_url_from_rdf(rdf):
             except uris.ValidationError:
                 pass
 
+
 def theme_labels_from_rdf(rdf):
     for theme in rdf.objects(DCAT.theme):
         if isinstance(theme, RdfResource):
@@ -399,7 +400,7 @@ def theme_labels_from_rdf(rdf):
             yield label
 
 
-def resource_from_rdf(graph_or_distrib, dataset=None):
+def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     '''
     Map a Resource domain model to a DCAT/RDF graph
     '''
@@ -410,9 +411,12 @@ def resource_from_rdf(graph_or_distrib, dataset=None):
                                       object=DCAT.Distribution)
         distrib = graph_or_distrib.resource(node)
 
-    download_url = url_from_rdf(distrib, DCAT.downloadURL)
-    access_url = url_from_rdf(distrib, DCAT.accessURL)
-    url = safe_unicode(download_url or access_url)
+    if not is_additionnal:
+        download_url = url_from_rdf(distrib, DCAT.downloadURL)
+        access_url = url_from_rdf(distrib, DCAT.accessURL)
+        url = safe_unicode(download_url or access_url)
+    else:
+        url = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
     # we shouldn't create resources without URLs
     if not url:
         log.warning(f'Resource without url: {distrib}')
@@ -439,6 +443,8 @@ def resource_from_rdf(graph_or_distrib, dataset=None):
             resource.checksum = Checksum()
             resource.checksum.value = rdf_value(checksum, SPDX.checksumValue)
             resource.checksum.type = algorithm
+    if is_additionnal:
+        resource.type = 'other'
 
     identifier = rdf_value(distrib, DCT.identifier)
     uri = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
@@ -494,6 +500,9 @@ def dataset_from_rdf(graph, dataset=None, node=None):
             elif isinstance(value, RdfResource):
                 licenses.add(value.identifier.toPython())
 
+    for additionnal in d.objects(DCT.hasPart):
+        resource_from_rdf(additionnal, dataset, is_additionnal=True)
+
     default_license = dataset.license or License.default()
     dataset_license = rdf_value(d, DCT.license)
     dataset.license = License.guess(dataset_license, *licenses, default=default_license)

udata/harvest/tests/dcat/catalog.xml

@@ -61,6 +61,7 @@
         <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
         <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:identifier>1</dcterms:identifier>
+        <dcterms:hasPart rdf:resource="http://data.test.org/datasets/1/resources/3"/>
       </dcat:Dataset>
     </dcat:dataset>
     <dcat:dataset>
@@ -134,6 +135,10 @@
     <dcterms:description>A JSON resource</dcterms:description>
     <dcat:accessURL>http://data.test.org/datasets/3/resources/1/file.json</dcat:accessURL>
   </dcat:Distribution>
+  <foaf:Document rdf:about="http://data.test.org/datasets/1/resources/3">
+    <dcterms:title>Resource 1-3</dcterms:title>
+    <dcterms:format>JSON</dcterms:format>
+  </foaf:Document>
   <dcterms:Location rdf:about="http://wuEurope.com/"/>
   <foaf:Organization rdf:about="http://data.test.org/organizations/1">
     <foaf:name>An Organization</foaf:name>

udata/harvest/tests/test_dcat_backend.py

@@ -297,7 +297,7 @@ class DcatBackendTest:
         assert dataset.temporal_coverage.start == date(2016, 1, 1)
         assert dataset.temporal_coverage.end == date(2016, 12, 5)
 
-        assert len(dataset.resources) == 2
+        assert len(dataset.resources) == 3
 
         resource_1 = next(res for res in dataset.resources if res.title == 'Resource 1-1')
         assert resource_1.filetype == 'remote'
@@ -307,11 +307,20 @@ class DcatBackendTest:
         assert resource_1.filesize == 12323
         assert resource_1.description == 'A JSON resource'
         assert resource_1.url == 'http://data.test.org/datasets/1/resources/1/file.json'
+        assert resource_1.type == 'main'
 
         resource_2 = next(res for res in dataset.resources if res.title == 'Resource 1-2')
         assert resource_2.format == 'json'
         assert resource_2.description == 'A JSON resource'
         assert resource_2.url == 'http://data.test.org/datasets/1/resources/2/file.json'
+        assert resource_2.type == 'main'
+
+        # Make sure additionnal resource is correctly harvested
+        resource_3 = next(res for res in dataset.resources if res.title == 'Resource 1-3')
+        assert resource_3.format == 'json'
+        assert resource_3.description == ''
+        assert resource_3.url == 'http://data.test.org/datasets/1/resources/3'
+        assert resource_3.type == 'other'
 
     def test_geonetwork_xml_catalog(self, rmock):
         url = mock_dcat(rmock, 'geonetwork.xml', path='catalog.xml')
@@ -413,7 +422,7 @@ class DcatBackendTest:
         error = job.errors[0]
         expected = 'Unable to detect format from extension or mime type'
         assert error.message == expected
-        
+
     def test_use_replaced_uris(self, rmock, mocker):
         mocker.patch.dict(
             URIS_TO_REPLACE,
@@ -465,7 +474,7 @@ class DcatBackendTest:
         assert len(job.errors) == 1
         assert "404 Client Error" in job.errors[0].message
 
-        
+
 @pytest.mark.usefixtures('clean_db')
 @pytest.mark.options(PLUGINS=['csw-dcat'])
 class CswDcatBackendTest: