udata 6.1.7.dev25937__py2.py3-none-any.whl → 6.1.7.dev25970__py2.py3-none-any.whl

udata/harvest/backends/dcat.py

@@ -35,6 +35,12 @@ KNOWN_PAGINATION = (
     (HYDRA.PagedCollection, HYDRA.nextPage)
 )
 
+# Useful to patch essential failing URIs
+URIS_TO_REPLACE = {
+    # See https://github.com/etalab/data.gouv.fr/issues/1151
+    'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld': 'https://gist.githubusercontent.com/maudetes/f019586185d6f59dcfb07f97148a1973/raw/585c3c7bf602b5a4e635b137257d0619792e2c1f/gistfile1.txt'  # noqa
+}
+
 
 def extract_graph(source, target, node, specs):
     for p, o in source.predicate_objects(node):
@@ -80,7 +86,10 @@ class DcatBackend(BaseBackend):
         page = 0
         while url:
             subgraph = Graph(namespace_manager=namespace_manager)
-            subgraph.parse(data=requests.get(url).text, format=fmt)
+            data = requests.get(url).text
+            for old_uri, new_uri in URIS_TO_REPLACE.items():
+                data = data.replace(old_uri, new_uri)
+            subgraph.parse(data=data, format=fmt)
 
             url = None
             for cls, prop in KNOWN_PAGINATION:

udata/harvest/tests/test_dcat_backend.py

@@ -10,6 +10,7 @@ from udata.core.organization.factories import OrganizationFactory
 from udata.core.dataset.factories import LicenseFactory
 
 from .factories import HarvestSourceFactory
+from ..backends.dcat import URIS_TO_REPLACE
 from .. import actions
 
 log = logging.getLogger(__name__)
@@ -333,7 +334,7 @@ class DcatBackendTest:
         assert 'geodesy' in dataset.tags  # support dcat:theme
         assert dataset.license.id == 'fr-lo'
         assert len(dataset.resources) == 1
-        assert dataset.description.startswith('Data from the \'National network')
+        assert dataset.description.startswith("Data from the 'National network")
         assert dataset.harvest is not None
         assert dataset.harvest.dct_identifier == '0437a976-cff1-4fa6-807a-c23006df2f8f'
         assert dataset.harvest.remote_id == '0437a976-cff1-4fa6-807a-c23006df2f8f'
@@ -383,3 +384,27 @@ class DcatBackendTest:
         error = job.errors[0]
         expected = 'Unable to detect format from extension or mime type'
         assert error.message == expected
+
+    def test_use_replaced_uris(self, rmock, mocker):
+        mocker.patch.dict(
+            URIS_TO_REPLACE,
+            {'http://example.org/this-url-does-not-exist': 'https://json-ld.org/contexts/person.jsonld'}
+        )
+        url = DCAT_URL_PATTERN.format(path='', domain=TEST_DOMAIN)
+        rmock.get(url, json={
+            '@context': 'http://example.org/this-url-does-not-exist',
+            '@type': 'dcat:Catalog',
+            'dataset': []
+        })
+        rmock.head(url, headers={'Content-Type': 'application/json'})
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend='dcat',
+                                      url=url,
+                                      organization=org)
+        actions.run(source.slug)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 0
+        assert job.status == 'done'

{udata-6.1.7.dev25937.dist-info → udata-6.1.7.dev25970.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 6.1.7.dev25937
+Version: 6.1.7.dev25970
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -142,6 +142,7 @@ It is collectively taken care of by members of the
 - Add `business_number_id` metadata for organizations [#2871](https://github.com/opendatateam/udata/pull/2871)
 - Return 403 when posting comment on discussion closed [#2881](https://github.com/opendatateam/udata/pull/2881)
 - Ensure rdf parsed frequency is lowercase [#2883](https://github.com/opendatateam/udata/pull/2883)
+- Add a dict of URIs to replace in a RDF graph at harvest time [#2884](https://github.com/opendatateam/udata/pull/2884)
 
 ## 6.1.6 (2023-07-19)
 

{udata-6.1.7.dev25937.dist-info → udata-6.1.7.dev25970.dist-info}/RECORD

@@ -248,13 +248,13 @@ udata/harvest/signals.py,sha256=wlXTi1E7rIVyNvxw0yUqyN5gF3thg276LAOmAF9vDJY,1338
 udata/harvest/tasks.py,sha256=3KkHuyFzGrjoBvLoT1V1VPJooyOIbv2Ze9QSXXWJ0ec,1949
 udata/harvest/backends/__init__.py,sha256=qcLhHKWO97TeWd93ZwymG_Cc9FO7sMM7h4fs6XYdtS8,447
 udata/harvest/backends/base.py,sha256=q0cA38RI_U07JaY69DDEE_31lPGI4sfFPyFupkwy9ks,11888
-udata/harvest/backends/dcat.py,sha256=a0wsLzvcUEnxH13-zTVEbeaEvMUCAJoYnBiFFppCzhI,4031
+udata/harvest/backends/dcat.py,sha256=4V9uBocONeUGfvHqzWtoG2kTkBUYBE4_S-N5NLGEyR8,4509
 udata/harvest/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/harvest/tests/factories.py,sha256=l1FRbM6-iblzIm4HDwQ23EnmoSDguQFGN4l_ygzSdGY,1981
 udata/harvest/tests/test_actions.py,sha256=7xSpouCAcf5p_bd38zHCyPN7sKWUUZXA7IlpI-yNVrQ,27603
 udata/harvest/tests/test_api.py,sha256=QXhseHfnkBEmMbIJzroMdDYGLDj6Njal1s-2sn0xhEM,14888
 udata/harvest/tests/test_base_backend.py,sha256=JA8Df1Eu-lEPLZfxyK81bsmT6exOjV_3PtKHJekAp5g,12092
-udata/harvest/tests/test_dcat_backend.py,sha256=C-0XS-yFL_3OF_1YkLmjjGPfVbc6T1jJ8ydIqKuD06o,15248
+udata/harvest/tests/test_dcat_backend.py,sha256=TzEIbUvbsQ3vJHzDxBunH0-G17xK9sb_p5cuIb9e0VU,16186
 udata/harvest/tests/test_filters.py,sha256=V2HFZlexIJa6r1DX6g2ktvIgjg4gSY11QPfPOd3_Oug,2370
 udata/harvest/tests/test_models.py,sha256=Wh2_Bwvfdrj0Fsh8Q8j4yTfLwmYv5WiGbGhrvf6zbII,905
 udata/harvest/tests/test_notifications.py,sha256=ZwtwioittW3XcZc0x6zbHjs1dVaAxPytlVymnJa5w0E,817
@@ -621,9 +621,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=QA_9yeYr22WR-o8T-Aix_78FJ0TBab
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=VLs_Q8RFHBn6Cw8w9r7BAgop_XBk9QwprZVObuzx1_g,41327
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=k_WnQxuJkopM1eWzKJFOok_OAWOjZhSLlB3O_ggzv3U,29086
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=xJJpHjSGJbpuKhVO8AhdDdbL6F7ILmWqJkz-6-fCgu4,48168
-udata-6.1.7.dev25937.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-6.1.7.dev25937.dist-info/METADATA,sha256=_F1xyYpMQuyMQhyVy5dKSsXzIpp6BcZJ1gnSRNumIyI,109082
-udata-6.1.7.dev25937.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
-udata-6.1.7.dev25937.dist-info/entry_points.txt,sha256=XACUgLu26a0tcYEoS6v8Av8yA-LSsH1XwB5AEM4jOPg,330
-udata-6.1.7.dev25937.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-6.1.7.dev25937.dist-info/RECORD,,
+udata-6.1.7.dev25970.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-6.1.7.dev25970.dist-info/METADATA,sha256=ucpePQHWzY5gt3hR5ySqRdanoFYYCV24pMi5U8XZgWM,109202
+udata-6.1.7.dev25970.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
+udata-6.1.7.dev25970.dist-info/entry_points.txt,sha256=XACUgLu26a0tcYEoS6v8Av8yA-LSsH1XwB5AEM4jOPg,330
+udata-6.1.7.dev25970.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-6.1.7.dev25970.dist-info/RECORD,,