udata 10.2.0__py2.py3-none-any.whl → 10.2.1.dev34693__py2.py3-none-any.whl

udata/__init__.py

@@ -4,5 +4,5 @@
 udata
 """
 
-__version__ = "10.2.0"
+__version__ = "10.2.1.dev"
 __description__ = "Open data portal"

udata/core/dataset/rdf.py

@@ -722,6 +722,10 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
             resource.checksum.type = algorithm
     if is_additionnal:
         resource.type = "other"
+    elif distrib.value(DCAT.accessService):
+        # The distribution has a DCAT.accessService property, we deduce
+        # that the distribution is of type API
+        resource.type = "api"
 
     identifier = rdf_value(distrib, DCT.identifier)
     uri = distrib.identifier.toPython() if isinstance(distrib.identifier, URIRef) else None

udata/harvest/tests/dcat/catalog.xml

@@ -61,16 +61,17 @@
         <dcat:theme>Theme 1</dcat:theme>
         <dcterms:publisher rdf:resource="http://data.test.org/organizations/1"/>
         <owl:versionInfo>1.0</owl:versionInfo>
-        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcat:keyword>Tag 4</dcat:keyword>
         <dcterms:spatial rdf:resource="http://wuEurope.com/"/>
         <dcterms:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T19:01:24.184120</dcterms:modified>
         <dcat:keyword>Tag 2</dcat:keyword>
         <dcat:keyword>Tag 1</dcat:keyword>
-        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
         <dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2016-12-14T18:59:02.737480</dcterms:issued>
         <dcterms:identifier>1</dcterms:identifier>
+        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/1"/>
+        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/2"/>
         <dcterms:hasPart rdf:resource="http://data.test.org/datasets/1/resources/3"/>
+        <dcat:distribution rdf:resource="http://data.test.org/datasets/1/resources/4"/>
       </dcat:Dataset>
     </dcat:dataset>
     <dcat:dataset>
@@ -152,6 +153,18 @@
     <dcterms:title>Resource 1-3</dcterms:title>
     <dcterms:format>JSON</dcterms:format>
   </foaf:Document>
+  <dcat:Distribution rdf:about="http://data.test.org/datasets/1/resources/4">
+    <dcterms:description>A resource pointing towards a Geo Service</dcterms:description>
+    <dcterms:title>Resource 1-4</dcterms:title>
+    <dcat:accessURL>http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0</dcat:accessURL>
+    <dcat:accessService>
+      <dcat:DataService>
+        <dcterms:title xml:lang="fr">Geo Service</dcterms:title>
+        <dcat:endpointURL rdf:resource="http://data.test.org/datasets/1/resources/4/services"/>
+        <dcat:endpointDescription rdf:resource="http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0"/>
+      </dcat:DataService>
+    </dcat:accessService>
+  </dcat:Distribution>
   <!-- resources for dataset 2 -->
   <dcat:Distribution rdf:about="http://data.test.org/datasets/2/resources/1">
     <dcat:accessURL>http://data.test.org/datasets/2/resources/1/file.json</dcat:accessURL>

udata/harvest/tests/test_dcat_backend.py

@@ -137,6 +137,7 @@ class DcatBackendTest:
         assert datasets["1"].resources[0].description == "A JSON resource"
         assert datasets["1"].resources[0].format == "json"
         assert datasets["1"].resources[0].mime == "application/json"
+        assert datasets["1"].resources[0].type == "main"
 
     @pytest.mark.options(
         SCHEMA_CATALOG_URL="https://example.com/schemas",
@@ -403,6 +404,7 @@ class DcatBackendTest:
         assert len(dataset.resources) == 1
 
         resource = dataset.resources[0]
+        assert resource.type == "main"
         assert resource.checksum is not None
         assert resource.checksum.type == "sha1"
         assert resource.checksum.value == "fb4106aa286a53be44ec99515f0f0421d4d7ad7d"
@@ -476,7 +478,12 @@ class DcatBackendTest:
 
         assert job.status == "done"
         assert job.errors == []
-        assert len(job.items) == 4
+        assert len(job.items) == 5
+        # 4 datasets and one Dataservice mentionned but not described
+        # because it appears in a distribution as DCAT.accessService
+        # but is missing a proper DCT.identifier
+        assert len([item for item in job.items if item.status == "done"]) == 4
+        assert len([item for item in job.items if item.status == "skipped"]) == 1
 
     def test_xml_catalog(self, rmock):
         LicenseFactory(id="lov2", title="Licence Ouverte Version 2.0")
@@ -524,7 +531,7 @@ class DcatBackendTest:
         # test dct:license nested in distribution
         assert dataset.license.id == "lov1"
 
-        assert len(dataset.resources) == 3
+        assert len(dataset.resources) == 4
 
         resource_1 = next(res for res in dataset.resources if res.title == "Resource 1-1")
         assert resource_1.filetype == "remote"
@@ -549,6 +556,16 @@ class DcatBackendTest:
         assert resource_3.url == "http://data.test.org/datasets/1/resources/3"
         assert resource_3.type == "other"
 
+        # Make sure a resource with an accessService is of type api
+        resource_4 = next(res for res in dataset.resources if res.title == "Resource 1-4")
+        assert resource_4.format is None
+        assert resource_4.description == "A resource pointing towards a Geo Service"
+        assert (
+            resource_4.url
+            == "http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0"
+        )
+        assert resource_4.type == "api"
+
         # test dct:rights -> license support from dataset
         dataset = Dataset.objects.get(harvest__dct_identifier="2")
         assert dataset.license.id == "lov2"
@@ -838,6 +855,7 @@ class CswDcatBackendTest:
         assert resource.title == "accidento_hdf_L93"
         assert resource.url == "https://www.geo2france.fr/geoserver/cr_hdf/ows"
         assert resource.format == "ogc:wms"
+        assert resource.type == "main"
 
     def test_user_agent_post(self, rmock):
         url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetworkv4-page-{}.xml")
@@ -949,6 +967,7 @@ class CswIso19139DcatBackendTest:
             resource.url
             == "http://atom.geo-ide.developpement-durable.gouv.fr/atomArchive/GetResource?id=fr-120066022-ldd-cab63273-b3ae-4e8a-ae1c-6192e45faa94&datasetAggregate=true"
         )
+        assert resource.type == "main"
 
         # Sadly resource format is parsed as a blank node. Format parsing should be improved.
         assert re.match(r"n[0-9a-f]{32}", resource.format)