udata 10.8.3.dev37131__py2.py3-none-any.whl → 10.8.3.dev37170__py2.py3-none-any.whl

udata/harvest/backends/dcat.py

@@ -72,16 +72,16 @@ class DcatBackend(BaseBackend):
         fmt = self.get_format()
         self.job.data = {"format": fmt}
 
-        serialized_graphs = []
+        pages = []
 
         for page_number, page in self.walk_graph(self.source.url, fmt):
             self.process_one_datasets_page(page_number, page)
-            serialized_graphs.append(page.serialize(format=fmt, indent=None))
+            pages.append((page_number, page))
 
         # We do a second pass to have all datasets in memory and attach datasets
         # to dataservices. It could be better to be one pass of graph walking and
         # then one pass of attaching datasets to dataservices.
-        for page_number, page in self.walk_graph(self.source.url, fmt):
+        for page_number, page in pages:
             self.process_one_dataservices_page(page_number, page)
 
         if not self.dryrun and self.has_reached_max_items():
@@ -100,6 +100,8 @@ class DcatBackend(BaseBackend):
 
         bucket = current_app.config.get("HARVEST_GRAPHS_S3_BUCKET")
 
+        serialized_graphs = [p.serialize(format=fmt, indent=None) for _, p in pages]
+
         if (
             bucket is not None
             and sum([len(g.encode("utf-8")) for g in serialized_graphs])
@@ -202,7 +204,10 @@ class DcatBackend(BaseBackend):
         )
 
     def process_one_dataservices_page(self, page_number: int, page: Graph):
+        access_services = {o for _, _, o in page.triples((None, DCAT.accessService, None))}
         for node in page.subjects(RDF.type, DCAT.DataService):
+            if node in access_services:
+                continue
             remote_id = page.value(node, DCT.identifier)
             self.process_dataservice(remote_id, page_number=page_number, page=page, node=node)
 

udata/harvest/tests/dcat/catalog.xml

@@ -159,6 +159,7 @@
     <dcat:accessURL>http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0</dcat:accessURL>
     <dcat:accessService>
       <dcat:DataService>
+        <rdf:type rdf:resource="http://www.w3.org/ns/dcat#DataService"/>
         <dcterms:title xml:lang="fr">Geo Service</dcterms:title>
         <dcat:endpointURL rdf:resource="http://data.test.org/datasets/1/resources/4/services"/>
         <dcat:endpointDescription rdf:resource="http://data.test.org/datasets/1/resources/4/services?SERVICE=WMS&amp;REQUEST=GetCapabilities&amp;VERSION=1.3.0"/>

udata/harvest/tests/test_dcat_backend.py

@@ -187,6 +187,23 @@ class DcatBackendTest:
             == "https://data.paris2024.org/api/explore/v2.1/console"
         )
 
+    def test_harvest_dataservices_ignore_accessservices(self, rmock):
+        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
+
+        url = mock_dcat(rmock, "catalog.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+
+        actions.run(source)
+
+        source.reload()
+
+        job = source.get_last_job()
+        assert len(job.items) == 4
+
+        dataservices = Dataservice.objects
+        assert len(dataservices) == 0
+
     def test_harvest_literal_spatial(self, rmock):
         url = mock_dcat(rmock, "evian.json")
         org = OrganizationFactory()
@@ -478,12 +495,8 @@ class DcatBackendTest:
 
         assert job.status == "done"
         assert job.errors == []
-        assert len(job.items) == 5
-        # 4 datasets and one Dataservice mentionned but not described
-        # because it appears in a distribution as DCAT.accessService
-        # but is missing a proper DCT.identifier
+        assert len(job.items) == 4
         assert len([item for item in job.items if item.status == "done"]) == 4
-        assert len([item for item in job.items if item.status == "skipped"]) == 1
 
     def test_xml_catalog(self, rmock):
         LicenseFactory(id="lov2", title="Licence Ouverte Version 2.0")

{udata-10.8.3.dev37131.dist-info → udata-10.8.3.dev37170.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.8.3.dev37131
+Version: 10.8.3.dev37170
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -145,7 +145,9 @@ It is collectively taken care of by members of the
 
 - Count dataservices and reuses for datasets based on signal [#3335](https://github.com/opendatateam/udata/pull/3335)
   :warning: the job `update-datasets-reuses-metrics` disappears, you should unschedule it before installing this version
- 
+- Cache dcat harvest pages to avoid two rounds of requests [#3398](https://github.com/opendatateam/udata/pull/3398)
+- Ignore Dataset.accessService when processing DataService [#3399](https://github.com/opendatateam/udata/pull/3399)
+
 ## 10.8.2 (2025-07-31)
 
 - Add integrity checks on non existent list field instead of empty [#3242](https://github.com/opendatateam/udata/pull/3242)

{udata-10.8.3.dev37131.dist-info → udata-10.8.3.dev37170.dist-info}/RECORD

@@ -300,7 +300,7 @@ udata/harvest/signals.py,sha256=3AhFHMPIFH5vz01NX5ycR_RWH14MXFWnCT6__LSa-QI,1338
 udata/harvest/tasks.py,sha256=id5YmHIhnkgez0LVC1fNg_6Yz1Sp0jrvQ1caslVOWdY,1722
 udata/harvest/backends/__init__.py,sha256=QjoFfBJfpw_xgk5YYWI1SgKJOMEmTMlxSfW79GNkSTI,459
 udata/harvest/backends/base.py,sha256=2wyfw83e3xGQcHnQI-z26g1dg-uVtWcDgzsBk7iGX3Y,17480
-udata/harvest/backends/dcat.py,sha256=IfUmqHy68Tt30E_kn4riExfgb-d7qPa--7KjCLCsia0,18637
+udata/harvest/backends/dcat.py,sha256=aBdCiKcmZoDmjsmZVP_S1DW5MO0h1Um8Zqj8knJwN-k,18804
 udata/harvest/backends/maaf.py,sha256=N7ty8ZWO9pfKPtZRk1wTaJ5pY6qi-0-GtF1p8jiYiY4,8102
 udata/harvest/backends/maaf.xsd,sha256=vEyG8Vqw7Yn_acjRdXjqUJgxOj4pv8bibep-FX-f3BQ,18322
 udata/harvest/backends/ckan/__init__.py,sha256=JE7Qa7kX7Yd8OvmJnAO_NupZe0tqYyhhkgJ-iGNxX64,35
@@ -314,7 +314,7 @@ udata/harvest/tests/person.jsonld,sha256=I7Ynh-PQlNeD51I1LrCgYOEjhL-WBeb65xzIE_s
 udata/harvest/tests/test_actions.py,sha256=oXNTwDuSHtlVANRcsNyq3J33u4XU_jM9o67FbZjLCys,24860
 udata/harvest/tests/test_api.py,sha256=gSuICkPy3KVRUhHAyudXVf_gLwiB7SoriUp3DLXWDdA,21611
 udata/harvest/tests/test_base_backend.py,sha256=ow8ecGtD836mUqyPWYjkS5nx0STyT5RMLgBdDyOhts4,19233
-udata/harvest/tests/test_dcat_backend.py,sha256=6CKQmhlPf5Ceipj6Nj0qKb-myb48f003kwlgIvHQAi8,43568
+udata/harvest/tests/test_dcat_backend.py,sha256=xe8X4Q88-Y50IQYT3Q9lz2rcFeb2ADIg1wLz-I7-ewc,43835
 udata/harvest/tests/test_filters.py,sha256=PT2qopEIoXsqi8MsNDRuhNH7jGXiQo8r0uJrCOUd4aM,2465
 udata/harvest/tests/test_models.py,sha256=f9NRR2_S4oZFgF8qOumg0vv-lpnEBJbI5vNtcwFdSqM,831
 udata/harvest/tests/test_notifications.py,sha256=MMzTzkv-GXMNFeOwAi31rdTsAXyLCLOSna41zOtaJG0,816
@@ -335,7 +335,7 @@ udata/harvest/tests/csw_dcat/geonetworkv4-page-3.xml,sha256=fsN0E4TVd_ts-sYA612y
 udata/harvest/tests/csw_dcat/geonetworkv4-page-5.xml,sha256=0VmPp1kspik7YAmOFyr-3yJLzWGA6kuQp_x_w-W385o,21213
 udata/harvest/tests/dcat/bnodes.jsonld,sha256=Leqny-ccp30564yojQYYckw_HKbhR0f5qUCaavc2ruE,7964
 udata/harvest/tests/dcat/bnodes.xml,sha256=bjG-pE2jDuJ7ZNDzQV4JEiMeAHCeX5eMQyUcDecVQ08,11333
-udata/harvest/tests/dcat/catalog.xml,sha256=ZBBH3JntmnmK9F9tGABvlfoXU4gBPOJTWL4942mhsto,12339
+udata/harvest/tests/dcat/catalog.xml,sha256=1tfktnFDEcJrJWSj_yt0zZVE5j8p-kiUanW9V-oxOrs,12412
 udata/harvest/tests/dcat/evian.json,sha256=R3RxP5azUuf9aZ9fU7n6iJkfbJ6oj-Zej2cjOtkYr8M,16647
 udata/harvest/tests/dcat/flat.jsonld,sha256=BAw08MDhtW9Px3q6RAoTIqO_OwJmAwBS9EpC8BY_x98,8459
 udata/harvest/tests/dcat/geonetwork.xml,sha256=9_pksE74Zzkbgs9okj6hEbo8CJS0FZjEnIdvopKfm7k,7928
@@ -758,9 +758,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=ogtEGsjwIkOEtzrI49e57izi-NUim5
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=1omSlqdMEdwYtzHGalXrtptVk14mlb6JS_8xR8BsznU,48428
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=V7unDjO31PNLiRhfEkYU3pU47wWWct-otdL0GDma0S4,28692
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=wVBipfpPl-DmYeVm5wrlzWWM3oEvuruDVbJHx4I1p8s,55170
-udata-10.8.3.dev37131.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.8.3.dev37131.dist-info/METADATA,sha256=IxyytporWPEDZ9gAw6TzHusA2ienapaEBgen9d7R1m4,153302
-udata-10.8.3.dev37131.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
-udata-10.8.3.dev37131.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
-udata-10.8.3.dev37131.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.8.3.dev37131.dist-info/RECORD,,
+udata-10.8.3.dev37170.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.8.3.dev37170.dist-info/METADATA,sha256=TOsb22KX4silyxYmNmOHWZPoYAIP54nZn2ZKsijar_k,153533
+udata-10.8.3.dev37170.dist-info/WHEEL,sha256=Kh9pAotZVRFj97E15yTA4iADqXdQfIVTHcNaZTjxeGM,110
+udata-10.8.3.dev37170.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
+udata-10.8.3.dev37170.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.8.3.dev37170.dist-info/RECORD,,