udata 6.2.1.dev26921__py2.py3-none-any.whl → 6.2.1.dev26928__py2.py3-none-any.whl

udata/core/dataset/rdf.py

@@ -389,6 +389,7 @@ def remote_url_from_rdf(rdf):
             except uris.ValidationError:
                 pass
 
+
 def theme_labels_from_rdf(rdf):
     for theme in rdf.objects(DCAT.theme):
         if isinstance(theme, RdfResource):
@@ -399,7 +400,7 @@ def theme_labels_from_rdf(rdf):
             yield label
 
 
-def resource_from_rdf(graph_or_distrib, dataset=None):
+def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     '''
     Map a Resource domain model to a DCAT/RDF graph
     '''
@@ -410,9 +411,12 @@ def resource_from_rdf(graph_or_distrib, dataset=None):
                                       object=DCAT.Distribution)
         distrib = graph_or_distrib.resource(node)
 
-    download_url = url_from_rdf(distrib, DCAT.downloadURL)
-    access_url = url_from_rdf(distrib, DCAT.accessURL)
-    url = safe_unicode(download_url or access_url)
+    if not is_additionnal:
+        download_url = url_from_rdf(distrib, DCAT.downloadURL)
+        access_url = url_from_rdf(distrib, DCAT.accessURL)
+        url = safe_unicode(download_url or access_url)
+    else:
+        url = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
     # we shouldn't create resources without URLs
     if not url:
         log.warning(f'Resource without url: {distrib}')
@@ -439,6 +443,8 @@ def resource_from_rdf(graph_or_distrib, dataset=None):
             resource.checksum = Checksum()
             resource.checksum.value = rdf_value(checksum, SPDX.checksumValue)
             resource.checksum.type = algorithm
+    if is_additionnal:
+        resource.type = 'other'
 
     identifier = rdf_value(distrib, DCT.identifier)
     uri = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None
@@ -494,6 +500,9 @@ def dataset_from_rdf(graph, dataset=None, node=None):
             elif isinstance(value, RdfResource):
                 licenses.add(value.identifier.toPython())
 
+    for additionnal in d.objects(DCT.hasPart):
+        resource_from_rdf(additionnal, dataset, is_additionnal=True)
+
     default_license = dataset.license or License.default()
     dataset_license = rdf_value(d, DCT.license)
     dataset.license = License.guess(dataset_license, *licenses, default=default_license)

udata/harvest/tests/dcat/catalog.xml

@@ -61,6 +61,7 @@
         <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
         <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:identifier>1</dcterms:identifier>
+        <dcterms:hasPart rdf:resource="http://data.test.org/datasets/1/resources/3"/>
       </dcat:Dataset>
     </dcat:dataset>
     <dcat:dataset>
@@ -134,6 +135,10 @@
     <dcterms:description>A JSON resource</dcterms:description>
     <dcat:accessURL>http://data.test.org/datasets/3/resources/1/file.json</dcat:accessURL>
   </dcat:Distribution>
+  <foaf:Document rdf:about="http://data.test.org/datasets/1/resources/3">
+    <dcterms:title>Resource 1-3</dcterms:title>
+    <dcterms:format>JSON</dcterms:format>
+  </foaf:Document>
   <dcterms:Location rdf:about="http://wuEurope.com/"/>
   <foaf:Organization rdf:about="http://data.test.org/organizations/1">
     <foaf:name>An Organization</foaf:name>

udata/harvest/tests/test_dcat_backend.py

@@ -297,7 +297,7 @@ class DcatBackendTest:
         assert dataset.temporal_coverage.start == date(2016, 1, 1)
         assert dataset.temporal_coverage.end == date(2016, 12, 5)
 
-        assert len(dataset.resources) == 2
+        assert len(dataset.resources) == 3
 
         resource_1 = next(res for res in dataset.resources if res.title == 'Resource 1-1')
         assert resource_1.filetype == 'remote'
@@ -307,11 +307,20 @@ class DcatBackendTest:
         assert resource_1.filesize == 12323
         assert resource_1.description == 'A JSON resource'
         assert resource_1.url == 'http://data.test.org/datasets/1/resources/1/file.json'
+        assert resource_1.type == 'main'
 
         resource_2 = next(res for res in dataset.resources if res.title == 'Resource 1-2')
         assert resource_2.format == 'json'
         assert resource_2.description == 'A JSON resource'
         assert resource_2.url == 'http://data.test.org/datasets/1/resources/2/file.json'
+        assert resource_2.type == 'main'
+
+        # Make sure additionnal resource is correctly harvested
+        resource_3 = next(res for res in dataset.resources if res.title == 'Resource 1-3')
+        assert resource_3.format == 'json'
+        assert resource_3.description == ''
+        assert resource_3.url == 'http://data.test.org/datasets/1/resources/3'
+        assert resource_3.type == 'other'
 
     def test_geonetwork_xml_catalog(self, rmock):
         url = mock_dcat(rmock, 'geonetwork.xml', path='catalog.xml')
@@ -413,7 +422,7 @@ class DcatBackendTest:
         error = job.errors[0]
         expected = 'Unable to detect format from extension or mime type'
         assert error.message == expected
-        
+
     def test_use_replaced_uris(self, rmock, mocker):
         mocker.patch.dict(
             URIS_TO_REPLACE,
@@ -465,7 +474,7 @@ class DcatBackendTest:
         assert len(job.errors) == 1
         assert "404 Client Error" in job.errors[0].message
 
-        
+
 @pytest.mark.usefixtures('clean_db')
 @pytest.mark.options(PLUGINS=['csw-dcat'])
 class CswDcatBackendTest:

{udata-6.2.1.dev26921.dist-info → udata-6.2.1.dev26928.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 6.2.1.dev26921
+Version: 6.2.1.dev26928
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -141,6 +141,7 @@ It is collectively taken care of by members of the
 - Topics: API v2 endpoints [#2913](https://github.com/opendatateam/udata/pull/2913)
 - Allow for discussions on Topics [#2922](https://github.com/opendatateam/udata/pull/2922)
 - Raise for status on DCAT harvester calls [#2927](https://github.com/opendatateam/udata/pull/2927)
+- Harvest dcterms:hasPart as 'other' resource in DCAT [#2928](https://github.com/opendatateam/udata/pull/2928)
 - Make sure harvested resources are marked as remote [#2931](https://github.com/opendatateam/udata/pull/2931)
 - Use GET and POST harvest BaseBackend utility to have user-agent [#2930](https://github.com/opendatateam/udata/pull/2930)
 - Use LazyReferenceField on Topic datasets and reuses [#2924](https://github.com/opendatateam/udata/pull/2924)

{udata-6.2.1.dev26921.dist-info → udata-6.2.1.dev26928.dist-info}/RECORD

@@ -84,7 +84,7 @@ udata/core/dataset/forms.py,sha256=HXlH9lKTc2mXQi1L-ulBokJPBCGLKXYxG5iB0-kM9-w,6
 udata/core/dataset/models.py,sha256=GDagMZM2FKJLDxg6bD8zu_Hiz0GfLeBrNB5alScLG5U,33676
 udata/core/dataset/permissions.py,sha256=3F2J7le3_rEYNhh88o3hSRWHAAt01_yHJM6RPmvCrRo,1090
 udata/core/dataset/preview.py,sha256=puPKT3fBD7ezAcT6owh0JK1_rGNDFZOqgT223qGn3LY,2597
-udata/core/dataset/rdf.py,sha256=6SDNXv9I45f60ZmfA0Ko5V1gQgdRzm4l3CjySeyb8HM,17592
+udata/core/dataset/rdf.py,sha256=zvCE9L3d8lgObUPOQQl3wDoK8tM7eovt9gqKOJkb_uY,17932
 udata/core/dataset/search.py,sha256=6Za7DMU18kfTMKELHCTjy13enbVRuFt8zyqyEkBrQg0,5561
 udata/core/dataset/signals.py,sha256=TK6dfrOUitZZkGGOh6XmhYqYvIjzZpI70JTLV4k-JRM,161
 udata/core/dataset/tasks.py,sha256=VB1sQ6Fwbax46IRLGyZUDPGgGOWBYrzAlKzV3npDCyM,8412
@@ -256,7 +256,7 @@ udata/harvest/tests/factories.py,sha256=l1FRbM6-iblzIm4HDwQ23EnmoSDguQFGN4l_ygzS
 udata/harvest/tests/test_actions.py,sha256=7xSpouCAcf5p_bd38zHCyPN7sKWUUZXA7IlpI-yNVrQ,27603
 udata/harvest/tests/test_api.py,sha256=QXhseHfnkBEmMbIJzroMdDYGLDj6Njal1s-2sn0xhEM,14888
 udata/harvest/tests/test_base_backend.py,sha256=JA8Df1Eu-lEPLZfxyK81bsmT6exOjV_3PtKHJekAp5g,12092
-udata/harvest/tests/test_dcat_backend.py,sha256=cOtRJoMgU4skhS5cNfQw6NaTS5gEx-OvudqUKvE_v98,20520
+udata/harvest/tests/test_dcat_backend.py,sha256=-6x5msEX1Z0eeAWzcNA8oE1OQbh3-mC0oF9F2Vnxvdw,20950
 udata/harvest/tests/test_filters.py,sha256=V2HFZlexIJa6r1DX6g2ktvIgjg4gSY11QPfPOd3_Oug,2370
 udata/harvest/tests/test_models.py,sha256=Wh2_Bwvfdrj0Fsh8Q8j4yTfLwmYv5WiGbGhrvf6zbII,905
 udata/harvest/tests/test_notifications.py,sha256=ZwtwioittW3XcZc0x6zbHjs1dVaAxPytlVymnJa5w0E,817
@@ -266,7 +266,7 @@ udata/harvest/tests/csw_dcat/geonetworkv4-page-3.xml,sha256=fsN0E4TVd_ts-sYA612y
 udata/harvest/tests/csw_dcat/geonetworkv4-page-5.xml,sha256=0VmPp1kspik7YAmOFyr-3yJLzWGA6kuQp_x_w-W385o,21213
 udata/harvest/tests/dcat/bnodes.jsonld,sha256=Leqny-ccp30564yojQYYckw_HKbhR0f5qUCaavc2ruE,7964
 udata/harvest/tests/dcat/bnodes.xml,sha256=xTycYrpe8Kq7CAXchV7IsVNQpkNp1xh3300dcEFt9uo,7850
-udata/harvest/tests/dcat/catalog.xml,sha256=DY-lIDWcpZZe55SJqWvfLlsrnFPM6vTA7ArIwaoA7fc,8391
+udata/harvest/tests/dcat/catalog.xml,sha256=InHzKoiz217m98WK5zdNd6KWZT5iHYWGhIx0EU1TOWw,8660
 udata/harvest/tests/dcat/flat.jsonld,sha256=BAw08MDhtW9Px3q6RAoTIqO_OwJmAwBS9EpC8BY_x98,8459
 udata/harvest/tests/dcat/geonetwork.xml,sha256=9_pksE74Zzkbgs9okj6hEbo8CJS0FZjEnIdvopKfm7k,7928
 udata/harvest/tests/dcat/nested.jsonld,sha256=GxGIsMcMv2PfxYj-e_gAnCXodxCgpwJKMA3wIb7xA7g,3911
@@ -630,9 +630,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=N4XoaycpcOJnsi2qJobZa8lEAYVpu7
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=sv-oskJPJ6SStK6TWA-Wa28ssmrJweWLw4M__Sln-_Q,41902
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=TfrjXLjrnuIo5N3Px0mMnrlWa-n4HmJUBP5pbLsGjkA,29086
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=QsVYsk6FRVtKmfxKv427qPmYcw2mx_wuE79aaWNeaYA,48743
-udata-6.2.1.dev26921.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-6.2.1.dev26921.dist-info/METADATA,sha256=Z1xSLblDNH_q0r8hddWXRes3sMGhKBv7sbtx5u8ITJY,113363
-udata-6.2.1.dev26921.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-udata-6.2.1.dev26921.dist-info/entry_points.txt,sha256=ZqIUHhOth0MMQvMIeuhODbUCDwjR-Hvo7PaKrMwTKuQ,384
-udata-6.2.1.dev26921.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-6.2.1.dev26921.dist-info/RECORD,,
+udata-6.2.1.dev26928.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-6.2.1.dev26928.dist-info/METADATA,sha256=NkxZ3ziL1apUekBk-6M8L1RYKJ5RUxDMCSKBzbNlOo8,113474
+udata-6.2.1.dev26928.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+udata-6.2.1.dev26928.dist-info/entry_points.txt,sha256=ZqIUHhOth0MMQvMIeuhODbUCDwjR-Hvo7PaKrMwTKuQ,384
+udata-6.2.1.dev26928.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-6.2.1.dev26928.dist-info/RECORD,,