udata 7.0.6.dev28248__py2.py3-none-any.whl → 7.0.6.dev28300__py2.py3-none-any.whl

udata/harvest/tests/test_dcat_backend.py

@@ -1,21 +1,22 @@
+from datetime import date
 import logging
 import os
-from flask import current_app
+import re
 
 import pytest
 
-from datetime import date
 import boto3
+from flask import current_app
 import xml.etree.ElementTree as ET
-from udata.harvest.models import HarvestJob
 
+from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory, ResourceSchemaMockData
 from udata.storage.s3 import get_from_json
 
 from .factories import HarvestSourceFactory
-from ..backends.dcat import URIS_TO_REPLACE
+from ..backends.dcat import URIS_TO_REPLACE, CswIso19139DcatBackend
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -629,7 +630,7 @@ class DcatBackendTest:
 
 
 @pytest.mark.usefixtures('clean_db')
-@pytest.mark.options(PLUGINS=['csw-dcat'])
+@pytest.mark.options(PLUGINS=['csw'])
 class CswDcatBackendTest:
 
     def test_geonetworkv4(self, rmock):
@@ -677,3 +678,60 @@ class CswDcatBackendTest:
 
         assert 'User-Agent' in get_mock.last_request.headers
         assert get_mock.last_request.headers['User-Agent'] == 'uData/0.1 csw-dcat'
+
+
+@pytest.mark.usefixtures('clean_db')
+@pytest.mark.options(PLUGINS=['csw'])
+class CswIso19139DcatBackendTest:
+
+    def test_geo2france(self, rmock):
+
+        with open(os.path.join(CSW_DCAT_FILES_DIR, "XSLT.xml"), "r") as f:
+            xslt = f.read()
+        url = mock_csw_pagination(rmock, 'geonetwork/srv/eng/csw.rdf', 'geonetwork-iso-page-{}.xml')
+        rmock.get(CswIso19139DcatBackend.XSL_URL, text=xslt)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='csw-iso-19139',
+                                      url=url,
+                                      organization=org)
+
+        actions.run(source.slug)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 6
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert len(datasets) == 6
+
+        # First dataset
+        # dataset identifier is gmd:RS_Identifier > gmd:codeSpace + gmd:code
+        dataset = datasets['http://catalogue.geo-ide.developpement-durable.gouv.fr/fr-120066022-orphan-residentifier-140d31c6-643d-42a9-85df-2737a118e144']
+        assert dataset.title == "Plan local d'urbanisme de la commune de Cartigny"
+        assert dataset.description == "Le présent standard de données COVADIS concerne les documents de plans locaux d'urbanisme (PLU) et les plans d'occupation des sols (POS qui valent PLU)."
+        assert set(dataset.tags) == set([
+            'amenagement-urbanisme-zonages-planification', 'cartigny',
+            'document-durbanisme', 'donnees-ouvertes', 'plu', 'usage-des-sols'
+        ])
+        assert dataset.harvest.created_at.date() == date(2017, 10, 7)
+        assert dataset.spatial.geom == {'type': 'MultiPolygon', 'coordinates':
+            [[[[3.28133559, 50.48188019], [1.31279111, 50.48188019], [1.31279111, 49.38547516], [3.28133559, 49.38547516], [3.28133559, 50.48188019]]]]
+        }
+        assert dataset.contact_point.name == 'DDTM 80 (Direction Départementale des Territoires et de la Mer de la Somme)'
+        assert dataset.contact_point.email == 'ddtm-sap-bsig@somme.gouv.fr'
+
+        # License is not properly mapped in XSLT conversion
+        assert dataset.license is None
+
+        # Distributions don't get properly mapped to distribution with this XSLT if missing CI_OnLineFunctionCode.
+        # A CI_OnLineFunctionCode was added explicitely on one of the Online Resources.
+        # (See mapping at: https://semiceu.github.io/GeoDCAT-AP/releases/2.0.0/#resource-locator---on-line-resource)
+        assert len(dataset.resources) == 1
+        resource = dataset.resources[0]
+        assert resource.title == 'Téléchargement direct du lot et des documents associés'
+        assert resource.url == 'http://atom.geo-ide.developpement-durable.gouv.fr/atomArchive/GetResource?id=fr-120066022-ldd-cab63273-b3ae-4e8a-ae1c-6192e45faa94&datasetAggregate=true'
+        
+        # Sadly resource format is parsed as a blank node. Format parsing should be improved.
+        assert re.match(r'n[0-9a-f]{32}', resource.format)

udata/migrations/2024-03-22-migrate-activity-kwargs-to-extras.py

@@ -0,0 +1,16 @@
+"""
+Migrates `Activity.kwargs` to `Activity.extras`.
+"""
+import logging
+
+from mongoengine.connection import get_db
+
+log = logging.getLogger(__name__)
+
+def migrate(db):
+    log.info('Processing activity objects')
+
+    result = get_db().activity.update_many(filter={}, update={'$rename': {'kwargs': 'extras'}})
+
+    log.info(f'{result.modified_count} activity object(s) migrated')
+    log.info('Done')

udata/rdf.py

@@ -76,6 +76,7 @@ ACCEPTED_MIME_TYPES = {
     'application/ld+json': 'json-ld',
     'application/json': 'json-ld',
     'application/trig': 'trig',
+    'text/xml': 'xml',
     # Available but not activated
     # 'application/n-quads': 'nquads',
     # 'text/xml': 'trix',

udata/tests/test_activity.py

@@ -27,6 +27,18 @@ class ActivityTest(WebTestMixin, DBTestMixin, TestCase):
         self.assertEqual(len(activities), 1)
         self.assertIsInstance(activities[0], FakeActivity)
 
+    def test_create_and_retrieve_with_extras(self):
+        FakeActivity.objects.create(actor=self.user, related_to=self.fake, extras={'some_key': 'some_value',
+                                                                                   'other_key': 'other_value'})
+
+        activities = Activity.objects(actor=self.user)
+
+        self.assertEqual(len(activities), 1)
+        self.assertEqual(activities[0].extras['some_key'], 'some_value')
+        self.assertEqual(activities[0].extras['other_key'], 'other_value')
+        self.assertIsInstance(activities[0], FakeActivity)
+
+
     def test_create_and_retrieve_for_org(self):
         org = OrganizationFactory()
         FakeActivity.objects.create(

{udata-7.0.6.dev28248.dist-info → udata-7.0.6.dev28300.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 7.0.6.dev28248
+Version: 7.0.6.dev28300
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -138,9 +138,11 @@ It is collectively taken care of by members of the
 
 ## Current (in progress)
 
+- Add new harvester for ISO DCAT with XSLT transform [#2982](https://github.com/opendatateam/udata/pull/2982)
 - Fix, do not fail on spatial coverage harvesting exception and allow literal spatial BBOX from Arcgis [2998](https://github.com/opendatateam/udata/pull/2998)
 - Mock calls to example.com [#3000](https://github.com/opendatateam/udata/pull/3000)
 - Fix duplicate logs in console commands [#2996](https://github.com/opendatateam/udata/pull/2996)
+- Refactor `Activity.kwargs` into `Activity.extras` to facilitate its usage [#2999](https://github.com/opendatateam/udata/pull/2999)
 
 ## 7.0.5 (2024-03-20)
 

{udata-7.0.6.dev28248.dist-info → udata-7.0.6.dev28300.dist-info}/RECORD

@@ -8,7 +8,7 @@ udata/errors.py,sha256=chc1880TPGteIL-Rfb9hdeAyZI4vdWZQU3JQ_5VF5t0,116
 udata/factories.py,sha256=KKPQeYzMdCxwGVLu9_Y-LxMx14cIgXA0jG2pHZ7uvFY,496
 udata/i18n.py,sha256=E-JoY57Cv4vgPvChkWjpwZAbuYVI8e6BO6raeu_3_Pw,8352
 udata/mail.py,sha256=dAMcbEtk5e54alpQezvF5adDrRPgdaT36QEdHD_5v50,2145
-udata/rdf.py,sha256=ClFUcb_jqQCkTme7LUD_yTACkjz97Q_5Oa5BNOxI7SE,9896
+udata/rdf.py,sha256=eu_tKOGQ9Bs8yyf5QGQNE9yyXGiUxevDYAIIun_KqqI,9919
 udata/routing.py,sha256=ztmFx4YzqlEqa-HG9a43WfGYfULC95ZxTlkr4iHkzVQ,7045
 udata/sentry.py,sha256=KiZz0PpmYpZMvykH9UAbHpF4xBY0Q-8DeiEbXEHDUdw,2683
 udata/settings.py,sha256=nnFcD1DBt3IbO2a10kE5ux9Q2KyeyOvXYLSOLWlF7nw,17142
@@ -39,7 +39,7 @@ udata/auth/views.py,sha256=wxk2GEBHHqgyUk81RdeZMBWPjDn69crmtb_blkhXsNs,6164
 udata/commands/__init__.py,sha256=mTRAsySo7lH4BlLs_IP0obyhU_hUMxLNB7N8Vh1HkMY,7756
 udata/commands/cache.py,sha256=uVpb3WQdqg8NZRSiWEa7cYdcx3QyFF53UAI_f2pXtV8,341
 udata/commands/db.py,sha256=1gQwffdopW2l6BMfj0MiJ7cKLN8zfi3WLFRaIrPxQn8,11131
-udata/commands/dcat.py,sha256=lbID57nogPsQ1EHbZ_sysRNokNE2D8H8B33BhgIGSHw,3136
+udata/commands/dcat.py,sha256=F8GVCT_arz5R_xwIygxbiis6cUSPM89C79Wlu46fR6g,3354
 udata/commands/fixtures.py,sha256=RrzRdUBJ2FbS-iDsEMPu5Nfp0WJXzU2rk3GB9t6JlRc,5852
 udata/commands/images.py,sha256=bzVvLj9LZ_nkyTdPlPe4g5U26k53VTH5ADCWmOfM8e4,2037
 udata/commands/info.py,sha256=Gf0AT8yP1tScbKLz8aXurlro55ot19sefrbOhELi034,1548
@@ -53,10 +53,10 @@ udata/commands/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 udata/commands/tests/fixtures.py,sha256=h78grSpZDgOwvmezyD3qBpiSnpxaQ9tDeQOr3sB8EiY,1581
 udata/core/__init__.py,sha256=n7TBP0lkYJl-BLoc67V5fyaawrWmsrWG_mLIF49UFOY,385
 udata/core/activity/__init__.py,sha256=4vB92owvzwn2VVxbFWGNcfESb6izDzvbj6lmLH4ssrU,299
-udata/core/activity/api.py,sha256=CeVU0ZtQZwQZMLh8Yt13LRGUohFo7iRf1bogbIoiI_4,3268
-udata/core/activity/models.py,sha256=z5qe7Hu3IoGCGeGyI31WaFPCg4C6q9BpUrTKK_ik4XI,2073
+udata/core/activity/api.py,sha256=ohDEbzhbsmJ6YHQh3Aa8wYctaDQGuwKdg0kjh3lC028,3276
+udata/core/activity/models.py,sha256=oxv8U45WNbzyZy8hgm8OGSn1zcRobOkhvOYTKpBwmrs,2119
 udata/core/activity/signals.py,sha256=nplrZFDDuGVXwLWpI_Lhe64mBRMRtwooJG1KcCH6Ses,191
-udata/core/activity/tasks.py,sha256=vBmxD96-svZpuEwIdm7vwgqMIZq0JgUv8e8dH1YaKLU,1168
+udata/core/activity/tasks.py,sha256=yKkDgBXYmVX-DS4P9hBgZcweEOU8mlihSj7pqQYOmkg,1242
 udata/core/badges/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/core/badges/api.py,sha256=BVHULesVK3godViXVOFSjGPCLYmHqLXXywsf0EaJpr8,993
 udata/core/badges/commands.py,sha256=NMihQH6ABUlkYpR2lNFW4dkeaWOjpijW7voudww8zW4,1129
@@ -263,17 +263,21 @@ udata/harvest/signals.py,sha256=wlXTi1E7rIVyNvxw0yUqyN5gF3thg276LAOmAF9vDJY,1338
 udata/harvest/tasks.py,sha256=0VhefKCQJSU_puTpdKOpvt3WORXHAFWGEB-R_MhB12M,1981
 udata/harvest/backends/__init__.py,sha256=qcLhHKWO97TeWd93ZwymG_Cc9FO7sMM7h4fs6XYdtS8,447
 udata/harvest/backends/base.py,sha256=oaPQcQ0onIXH5ofUtWH5sM6_5_wSBLawHSOjeeoG6jQ,12258
-udata/harvest/backends/dcat.py,sha256=U7PjYfpBTYEEmk-lQdkXfMcaCa39gSlL5Gy4ywqhAQo,10504
+udata/harvest/backends/dcat.py,sha256=SCbWorkuNzM7OvzFzBh1Is4QTWquOcnCLa_yd3HbgW0,14846
 udata/harvest/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/harvest/tests/factories.py,sha256=CbQORC1OJ1_Agtv_3LjCXysNumjMYlROwZPSEAHo8sM,2005
 udata/harvest/tests/test_actions.py,sha256=7xSpouCAcf5p_bd38zHCyPN7sKWUUZXA7IlpI-yNVrQ,27603
 udata/harvest/tests/test_api.py,sha256=QXhseHfnkBEmMbIJzroMdDYGLDj6Njal1s-2sn0xhEM,14888
 udata/harvest/tests/test_base_backend.py,sha256=JA8Df1Eu-lEPLZfxyK81bsmT6exOjV_3PtKHJekAp5g,12092
-udata/harvest/tests/test_dcat_backend.py,sha256=T0cz5jPHr7h4dxxeq7-aJe53hRwNY_HGMiKVWwZeuMI,29379
+udata/harvest/tests/test_dcat_backend.py,sha256=zZea9e2awtFiLrdREeaHcp9d6tSHhAHEuXr_fc5fKjk,32462
 udata/harvest/tests/test_filters.py,sha256=V2HFZlexIJa6r1DX6g2ktvIgjg4gSY11QPfPOd3_Oug,2370
 udata/harvest/tests/test_models.py,sha256=p2VazyrPXSArBuf8Kf19TGPcQ86SnOGCGmvjcMOw0s0,924
 udata/harvest/tests/test_notifications.py,sha256=ZwtwioittW3XcZc0x6zbHjs1dVaAxPytlVymnJa5w0E,817
 udata/harvest/tests/test_tasks.py,sha256=OfUnsg6x3DtbS2s3ToRDZY46aqqmbowN1sqASmsLaN4,680
+udata/harvest/tests/csw_dcat/XSLT.xml,sha256=vnJF2rom7oPVpzI9jwzwG6ek1TDXTY8GfbuQq_3vFxs,180475
+udata/harvest/tests/csw_dcat/geonetwork-iso-page-1.xml,sha256=cANO-kr4NEDgq7n6pG_9hnOcPrYPinG6GuKoZWuPV3g,82058
+udata/harvest/tests/csw_dcat/geonetwork-iso-page-3.xml,sha256=sLzItw0Pxl68qn-XB64IqZGHJGcdVPJvCNliYK9GFV4,67901
+udata/harvest/tests/csw_dcat/geonetwork-iso-page-5.xml,sha256=uzCyMx0HsoVJhTFvh8hyFxWDdOUuiRzPYiDXv_zVmfY,81713
 udata/harvest/tests/csw_dcat/geonetworkv4-page-1.xml,sha256=k2pKidlQvJpoltGFm9HNh6KHai7nrPcpinDSQS4dGkQ,17040
 udata/harvest/tests/csw_dcat/geonetworkv4-page-3.xml,sha256=fsN0E4TVd_ts-sYA612yBP-gRAwpyQWqJdNm7ohczbs,20945
 udata/harvest/tests/csw_dcat/geonetworkv4-page-5.xml,sha256=0VmPp1kspik7YAmOFyr-3yJLzWGA6kuQp_x_w-W385o,21213
@@ -315,6 +319,7 @@ udata/migrations/2022-10-10-add-fs_uniquifier-to-user-model.py,sha256=ZKfrhPIeIr
 udata/migrations/2022-10-10-migrate-harvest-extras.py,sha256=HyBy7f1epYuBry68lUYTe7fMWoiyuIQ6xVwzwljtGqI,2254
 udata/migrations/2023-02-08-rename-internal-dates.py,sha256=AYewRwdVCfHsJ5APbhQWfihH-nerT3jOzQsSrXW9RQ0,1651
 udata/migrations/2024-01-29-fix-reuse-and-dataset-with-private-None.py,sha256=173Zf67G-oJgLwXeQUpzoCzcpAOu1ifcLb4eUMJ9nfU,590
+udata/migrations/2024-03-22-migrate-activity-kwargs-to-extras.py,sha256=n5-udd1IdaD-e3q5cZx9nC4uMUSyJyT1TkI6I4gFgh0,400
 udata/migrations/__init__.py,sha256=3YP8ppVG5Jx_u6fWJ099X6m3JDYsbd_wetpu4arl6_U,10882
 udata/models/__init__.py,sha256=1jVsYyZqtPKOuGgbhHmJd3rWJjAv5t7k51_j5C43m8c,5528
 udata/models/badges_field.py,sha256=NnHA6OT40UysQzO3qnxiDNLR-SCj19KlqD1c-p1DP2Q,984
@@ -545,7 +550,7 @@ udata/tests/helpers.py,sha256=aaifyevJ1Z8CZ8htRrl8OCG5hGcaHfj0lL8iMEKds9w,6022
 udata/tests/models.py,sha256=_V0smMb1Z6p3aZv6PorzNN-HiNt_B46Ox1fqXrTJEqk,238
 udata/tests/plugin.py,sha256=hI9Z5SIxUT8UhqZMLcNPzBqdM8-nUgtCW6Pbw2bnNsQ,11045
 udata/tests/schemas.json,sha256=ZRHijbvYaQBrKuTJzCY5ZeQXdIdaZkv3fo7VMbDjsNg,3483
-udata/tests/test_activity.py,sha256=4ZHRG9GYv5I2gtMYQAnhgONP9YBqwMdKKq9T2vua6-I,2674
+udata/tests/test_activity.py,sha256=spWfhueuLze0kD-pAnegiL3_Kv5io155jQuFI4sjN7I,3258
 udata/tests/test_discussions.py,sha256=zPvKOdcTNGXrvHFp9zqjhKE2fqgUkhb_1F98egXYCL0,31036
 udata/tests/test_i18n.py,sha256=BU9E7OoIkJw5tv5JYGLjDGBDsti2HuQ_3OWDKnBxnaM,3191
 udata/tests/test_linkchecker.py,sha256=KxV1-PuuuqogkHf3jP6JhRsc2QG2dFmFB-vSHOiHkuU,10374
@@ -659,9 +664,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=zCVMB-a4-mLM1jNyYMk58rgVRaVIwQ
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=avfWczvlLBKSohyB55-4TLmUGMU_Rze4XmAo4OTk2v0,43513
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=ScuqdpaV4y1ZIpBAEfxeaKdzkyGZL0mJmKMoG6w0iRQ,28553
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=QpgEXh1eHjztPa7oNLXd_sds1DC95A-STTtZyTE4m-E,50093
-udata-7.0.6.dev28248.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-7.0.6.dev28248.dist-info/METADATA,sha256=cvPvj6V8F4fFMpGW4I6qftu6DenALP-KUuylkJ27cBA,119755
-udata-7.0.6.dev28248.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-udata-7.0.6.dev28248.dist-info/entry_points.txt,sha256=ZqIUHhOth0MMQvMIeuhODbUCDwjR-Hvo7PaKrMwTKuQ,384
-udata-7.0.6.dev28248.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-7.0.6.dev28248.dist-info/RECORD,,
+udata-7.0.6.dev28300.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-7.0.6.dev28300.dist-info/METADATA,sha256=kN_pLEvZ8kkXbSlaOlSjWupU6AMUYiXpVygQD3wVZqw,119999
+udata-7.0.6.dev28300.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+udata-7.0.6.dev28300.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-7.0.6.dev28300.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-7.0.6.dev28300.dist-info/RECORD,,

{udata-7.0.6.dev28248.dist-info → udata-7.0.6.dev28300.dist-info}/entry_points.txt

@@ -11,4 +11,5 @@ robohash = udata.features.identicon.backends:robohash
 
 [udata.harvesters]
 csw-dcat = udata.harvest.backends.dcat:CswDcatBackend
+csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend
 dcat = udata.harvest.backends.dcat:DcatBackend