udata 7.0.6.dev28263__py2.py3-none-any.whl → 7.0.6.dev28300__py2.py3-none-any.whl

udata/harvest/tests/test_dcat_backend.py

@@ -1,21 +1,22 @@
+from datetime import date
 import logging
 import os
-from flask import current_app
+import re
 
 import pytest
 
-from datetime import date
 import boto3
+from flask import current_app
 import xml.etree.ElementTree as ET
-from udata.harvest.models import HarvestJob
 
+from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory, ResourceSchemaMockData
 from udata.storage.s3 import get_from_json
 
 from .factories import HarvestSourceFactory
-from ..backends.dcat import URIS_TO_REPLACE
+from ..backends.dcat import URIS_TO_REPLACE, CswIso19139DcatBackend
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -629,7 +630,7 @@ class DcatBackendTest:
 
 
 @pytest.mark.usefixtures('clean_db')
-@pytest.mark.options(PLUGINS=['csw-dcat'])
+@pytest.mark.options(PLUGINS=['csw'])
 class CswDcatBackendTest:
 
     def test_geonetworkv4(self, rmock):
@@ -677,3 +678,60 @@ class CswDcatBackendTest:
 
         assert 'User-Agent' in get_mock.last_request.headers
         assert get_mock.last_request.headers['User-Agent'] == 'uData/0.1 csw-dcat'
+
+
+@pytest.mark.usefixtures('clean_db')
+@pytest.mark.options(PLUGINS=['csw'])
+class CswIso19139DcatBackendTest:
+
+    def test_geo2france(self, rmock):
+
+        with open(os.path.join(CSW_DCAT_FILES_DIR, "XSLT.xml"), "r") as f:
+            xslt = f.read()
+        url = mock_csw_pagination(rmock, 'geonetwork/srv/eng/csw.rdf', 'geonetwork-iso-page-{}.xml')
+        rmock.get(CswIso19139DcatBackend.XSL_URL, text=xslt)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='csw-iso-19139',
+                                      url=url,
+                                      organization=org)
+
+        actions.run(source.slug)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 6
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert len(datasets) == 6
+
+        # First dataset
+        # dataset identifier is gmd:RS_Identifier > gmd:codeSpace + gmd:code
+        dataset = datasets['http://catalogue.geo-ide.developpement-durable.gouv.fr/fr-120066022-orphan-residentifier-140d31c6-643d-42a9-85df-2737a118e144']
+        assert dataset.title == "Plan local d'urbanisme de la commune de Cartigny"
+        assert dataset.description == "Le présent standard de données COVADIS concerne les documents de plans locaux d'urbanisme (PLU) et les plans d'occupation des sols (POS qui valent PLU)."
+        assert set(dataset.tags) == set([
+            'amenagement-urbanisme-zonages-planification', 'cartigny',
+            'document-durbanisme', 'donnees-ouvertes', 'plu', 'usage-des-sols'
+        ])
+        assert dataset.harvest.created_at.date() == date(2017, 10, 7)
+        assert dataset.spatial.geom == {'type': 'MultiPolygon', 'coordinates':
+            [[[[3.28133559, 50.48188019], [1.31279111, 50.48188019], [1.31279111, 49.38547516], [3.28133559, 49.38547516], [3.28133559, 50.48188019]]]]
+        }
+        assert dataset.contact_point.name == 'DDTM 80 (Direction Départementale des Territoires et de la Mer de la Somme)'
+        assert dataset.contact_point.email == 'ddtm-sap-bsig@somme.gouv.fr'
+
+        # License is not properly mapped in XSLT conversion
+        assert dataset.license is None
+
+        # Distributions don't get properly mapped to distribution with this XSLT if missing CI_OnLineFunctionCode.
+        # A CI_OnLineFunctionCode was added explicitely on one of the Online Resources.
+        # (See mapping at: https://semiceu.github.io/GeoDCAT-AP/releases/2.0.0/#resource-locator---on-line-resource)
+        assert len(dataset.resources) == 1
+        resource = dataset.resources[0]
+        assert resource.title == 'Téléchargement direct du lot et des documents associés'
+        assert resource.url == 'http://atom.geo-ide.developpement-durable.gouv.fr/atomArchive/GetResource?id=fr-120066022-ldd-cab63273-b3ae-4e8a-ae1c-6192e45faa94&datasetAggregate=true'
+        
+        # Sadly resource format is parsed as a blank node. Format parsing should be improved.
+        assert re.match(r'n[0-9a-f]{32}', resource.format)

udata/rdf.py

@@ -76,6 +76,7 @@ ACCEPTED_MIME_TYPES = {
     'application/ld+json': 'json-ld',
     'application/json': 'json-ld',
     'application/trig': 'trig',
+    'text/xml': 'xml',
     # Available but not activated
     # 'application/n-quads': 'nquads',
     # 'text/xml': 'trix',

{udata-7.0.6.dev28263.dist-info → udata-7.0.6.dev28300.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 7.0.6.dev28263
+Version: 7.0.6.dev28300
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -138,6 +138,7 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
+- Add new harvester for ISO DCAT with XSLT transform [#2982](https://github.com/opendatateam/udata/pull/2982)
 - Fix, do not fail on spatial coverage harvesting exception and allow literal spatial BBOX from Arcgis [2998](https://github.com/opendatateam/udata/pull/2998)
 - Mock calls to example.com [#3000](https://github.com/opendatateam/udata/pull/3000)
 - Fix duplicate logs in console commands [#2996](https://github.com/opendatateam/udata/pull/2996)

{udata-7.0.6.dev28263.dist-info → udata-7.0.6.dev28300.dist-info}/RECORD

@@ -8,7 +8,7 @@ udata/errors.py,sha256=chc1880TPGteIL-Rfb9hdeAyZI4vdWZQU3JQ_5VF5t0,116
 udata/factories.py,sha256=KKPQeYzMdCxwGVLu9_Y-LxMx14cIgXA0jG2pHZ7uvFY,496
 udata/i18n.py,sha256=E-JoY57Cv4vgPvChkWjpwZAbuYVI8e6BO6raeu_3_Pw,8352
 udata/mail.py,sha256=dAMcbEtk5e54alpQezvF5adDrRPgdaT36QEdHD_5v50,2145
-udata/rdf.py,sha256=ClFUcb_jqQCkTme7LUD_yTACkjz97Q_5Oa5BNOxI7SE,9896
+udata/rdf.py,sha256=eu_tKOGQ9Bs8yyf5QGQNE9yyXGiUxevDYAIIun_KqqI,9919
 udata/routing.py,sha256=ztmFx4YzqlEqa-HG9a43WfGYfULC95ZxTlkr4iHkzVQ,7045
 udata/sentry.py,sha256=KiZz0PpmYpZMvykH9UAbHpF4xBY0Q-8DeiEbXEHDUdw,2683
 udata/settings.py,sha256=nnFcD1DBt3IbO2a10kE5ux9Q2KyeyOvXYLSOLWlF7nw,17142
@@ -39,7 +39,7 @@ udata/auth/views.py,sha256=wxk2GEBHHqgyUk81RdeZMBWPjDn69crmtb_blkhXsNs,6164
 udata/commands/__init__.py,sha256=mTRAsySo7lH4BlLs_IP0obyhU_hUMxLNB7N8Vh1HkMY,7756
 udata/commands/cache.py,sha256=uVpb3WQdqg8NZRSiWEa7cYdcx3QyFF53UAI_f2pXtV8,341
 udata/commands/db.py,sha256=1gQwffdopW2l6BMfj0MiJ7cKLN8zfi3WLFRaIrPxQn8,11131
-udata/commands/dcat.py,sha256=lbID57nogPsQ1EHbZ_sysRNokNE2D8H8B33BhgIGSHw,3136
+udata/commands/dcat.py,sha256=F8GVCT_arz5R_xwIygxbiis6cUSPM89C79Wlu46fR6g,3354
 udata/commands/fixtures.py,sha256=RrzRdUBJ2FbS-iDsEMPu5Nfp0WJXzU2rk3GB9t6JlRc,5852
 udata/commands/images.py,sha256=bzVvLj9LZ_nkyTdPlPe4g5U26k53VTH5ADCWmOfM8e4,2037
 udata/commands/info.py,sha256=Gf0AT8yP1tScbKLz8aXurlro55ot19sefrbOhELi034,1548
@@ -263,17 +263,21 @@ udata/harvest/signals.py,sha256=wlXTi1E7rIVyNvxw0yUqyN5gF3thg276LAOmAF9vDJY,1338
 udata/harvest/tasks.py,sha256=0VhefKCQJSU_puTpdKOpvt3WORXHAFWGEB-R_MhB12M,1981
 udata/harvest/backends/__init__.py,sha256=qcLhHKWO97TeWd93ZwymG_Cc9FO7sMM7h4fs6XYdtS8,447
 udata/harvest/backends/base.py,sha256=oaPQcQ0onIXH5ofUtWH5sM6_5_wSBLawHSOjeeoG6jQ,12258
-udata/harvest/backends/dcat.py,sha256=U7PjYfpBTYEEmk-lQdkXfMcaCa39gSlL5Gy4ywqhAQo,10504
+udata/harvest/backends/dcat.py,sha256=SCbWorkuNzM7OvzFzBh1Is4QTWquOcnCLa_yd3HbgW0,14846
 udata/harvest/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/harvest/tests/factories.py,sha256=CbQORC1OJ1_Agtv_3LjCXysNumjMYlROwZPSEAHo8sM,2005
 udata/harvest/tests/test_actions.py,sha256=7xSpouCAcf5p_bd38zHCyPN7sKWUUZXA7IlpI-yNVrQ,27603
 udata/harvest/tests/test_api.py,sha256=QXhseHfnkBEmMbIJzroMdDYGLDj6Njal1s-2sn0xhEM,14888
 udata/harvest/tests/test_base_backend.py,sha256=JA8Df1Eu-lEPLZfxyK81bsmT6exOjV_3PtKHJekAp5g,12092
-udata/harvest/tests/test_dcat_backend.py,sha256=T0cz5jPHr7h4dxxeq7-aJe53hRwNY_HGMiKVWwZeuMI,29379
+udata/harvest/tests/test_dcat_backend.py,sha256=zZea9e2awtFiLrdREeaHcp9d6tSHhAHEuXr_fc5fKjk,32462
 udata/harvest/tests/test_filters.py,sha256=V2HFZlexIJa6r1DX6g2ktvIgjg4gSY11QPfPOd3_Oug,2370
 udata/harvest/tests/test_models.py,sha256=p2VazyrPXSArBuf8Kf19TGPcQ86SnOGCGmvjcMOw0s0,924
 udata/harvest/tests/test_notifications.py,sha256=ZwtwioittW3XcZc0x6zbHjs1dVaAxPytlVymnJa5w0E,817
 udata/harvest/tests/test_tasks.py,sha256=OfUnsg6x3DtbS2s3ToRDZY46aqqmbowN1sqASmsLaN4,680
+udata/harvest/tests/csw_dcat/XSLT.xml,sha256=vnJF2rom7oPVpzI9jwzwG6ek1TDXTY8GfbuQq_3vFxs,180475
+udata/harvest/tests/csw_dcat/geonetwork-iso-page-1.xml,sha256=cANO-kr4NEDgq7n6pG_9hnOcPrYPinG6GuKoZWuPV3g,82058
+udata/harvest/tests/csw_dcat/geonetwork-iso-page-3.xml,sha256=sLzItw0Pxl68qn-XB64IqZGHJGcdVPJvCNliYK9GFV4,67901
+udata/harvest/tests/csw_dcat/geonetwork-iso-page-5.xml,sha256=uzCyMx0HsoVJhTFvh8hyFxWDdOUuiRzPYiDXv_zVmfY,81713
 udata/harvest/tests/csw_dcat/geonetworkv4-page-1.xml,sha256=k2pKidlQvJpoltGFm9HNh6KHai7nrPcpinDSQS4dGkQ,17040
 udata/harvest/tests/csw_dcat/geonetworkv4-page-3.xml,sha256=fsN0E4TVd_ts-sYA612yBP-gRAwpyQWqJdNm7ohczbs,20945
 udata/harvest/tests/csw_dcat/geonetworkv4-page-5.xml,sha256=0VmPp1kspik7YAmOFyr-3yJLzWGA6kuQp_x_w-W385o,21213
@@ -660,9 +664,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=zCVMB-a4-mLM1jNyYMk58rgVRaVIwQ
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=avfWczvlLBKSohyB55-4TLmUGMU_Rze4XmAo4OTk2v0,43513
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=ScuqdpaV4y1ZIpBAEfxeaKdzkyGZL0mJmKMoG6w0iRQ,28553
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=QpgEXh1eHjztPa7oNLXd_sds1DC95A-STTtZyTE4m-E,50093
-udata-7.0.6.dev28263.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-7.0.6.dev28263.dist-info/METADATA,sha256=2zJTXq_j2szvACqYhZsB6GFJNzIZf1KQXM-EuVgcvvQ,119888
-udata-7.0.6.dev28263.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-udata-7.0.6.dev28263.dist-info/entry_points.txt,sha256=ZqIUHhOth0MMQvMIeuhODbUCDwjR-Hvo7PaKrMwTKuQ,384
-udata-7.0.6.dev28263.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-7.0.6.dev28263.dist-info/RECORD,,
+udata-7.0.6.dev28300.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-7.0.6.dev28300.dist-info/METADATA,sha256=kN_pLEvZ8kkXbSlaOlSjWupU6AMUYiXpVygQD3wVZqw,119999
+udata-7.0.6.dev28300.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+udata-7.0.6.dev28300.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-7.0.6.dev28300.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-7.0.6.dev28300.dist-info/RECORD,,

{udata-7.0.6.dev28263.dist-info → udata-7.0.6.dev28300.dist-info}/entry_points.txt

@@ -11,4 +11,5 @@ robohash = udata.features.identicon.backends:robohash
 
 [udata.harvesters]
 csw-dcat = udata.harvest.backends.dcat:CswDcatBackend
+csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend
 dcat = udata.harvest.backends.dcat:DcatBackend