udata 10.0.7.dev33415__py2.py3-none-any.whl → 10.0.7.dev33440__py2.py3-none-any.whl

udata/harvest/backends/base.py

@@ -7,6 +7,7 @@ import requests
 from flask import current_app
 from voluptuous import MultipleInvalid, RequiredFieldInvalid
 
+import udata.uris as uris
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.models import HarvestDatasetMetadata
@@ -388,15 +389,19 @@ class BaseBackend(object):
         """Get or create a dataset given its remote ID (and its source)
         We first try to match `source_id` to be source domain independent
         """
-        dataset = Dataset.objects(
-            __raw__={
-                "harvest.remote_id": remote_id,
-                "$or": [
-                    {"harvest.domain": self.source.domain},
-                    {"harvest.source_id": str(self.source.id)},
-                ],
-            }
-        ).first()
+        try:
+            uris.validate(remote_id)
+            dataset = Dataset.objects(harvest__remote_id=remote_id).first()
+        except uris.ValidationError:
+            dataset = Dataset.objects(
+                __raw__={
+                    "harvest.remote_id": remote_id,
+                    "$or": [
+                        {"harvest.domain": self.source.domain},
+                        {"harvest.source_id": str(self.source.id)},
+                    ],
+                }
+            ).first()
 
         if dataset:
             return dataset

udata/harvest/tests/test_base_backend.py

@@ -20,6 +20,11 @@ class Unknown:
     pass
 
 
+def gen_remote_IDs(num: int) -> list[str]:
+    """Generate remote IDs."""
+    return [f"fake-{i}" for i in range(num)]
+
+
 class FakeBackend(BaseBackend):
     filters = (
         HarvestFilter("First filter", "first", str),
@@ -35,8 +40,7 @@ class FakeBackend(BaseBackend):
     )
 
     def inner_harvest(self):
-        for i in range(self.source.config.get("nb_datasets", 3)):
-            remote_id = f"fake-{i}"
+        for remote_id in self.source.config.get("dataset_remote_ids", []):
             self.process_dataset(remote_id)
             if self.is_done():
                 return
@@ -45,7 +49,8 @@ class FakeBackend(BaseBackend):
         dataset = self.get_dataset(item.remote_id)
 
         for key, value in DatasetFactory.as_dict(visible=True).items():
-            setattr(dataset, key, value)
+            if getattr(dataset, key) is None:
+                setattr(dataset, key, value)
         if self.source.config.get("last_modified"):
             dataset.last_modified_internal = self.source.config["last_modified"]
         return dataset
@@ -76,7 +81,7 @@ class BaseBackendTest:
     def test_simple_harvest(self):
         now = datetime.utcnow()
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"nb_datasets": nb_datasets})
+        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
         backend = FakeBackend(source)
 
         job = backend.harvest()
@@ -155,7 +160,7 @@ class BaseBackendTest:
 
     def test_harvest_source_id(self):
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"nb_datasets": nb_datasets})
+        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
         backend = FakeBackend(source)
 
         job = backend.harvest()
@@ -176,7 +181,9 @@ class BaseBackendTest:
 
     def test_dont_overwrite_last_modified(self, mocker):
         last_modified = faker.date_time_between(start_date="-30y", end_date="-1y")
-        source = HarvestSourceFactory(config={"nb_datasets": 1, "last_modified": last_modified})
+        source = HarvestSourceFactory(
+            config={"dataset_remote_ids": gen_remote_IDs(1), "last_modified": last_modified}
+        )
         backend = FakeBackend(source)
 
         backend.harvest()
@@ -188,7 +195,9 @@ class BaseBackendTest:
 
     def test_dont_overwrite_last_modified_even_if_set_to_same(self, mocker):
         last_modified = faker.date_time_between(start_date="-30y", end_date="-1y")
-        source = HarvestSourceFactory(config={"nb_datasets": 1, "last_modified": last_modified})
+        source = HarvestSourceFactory(
+            config={"dataset_remote_ids": gen_remote_IDs(1), "last_modified": last_modified}
+        )
         backend = FakeBackend(source)
 
         backend.harvest()
@@ -201,7 +210,7 @@ class BaseBackendTest:
 
     def test_autoarchive(self, app):
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"nb_datasets": nb_datasets})
+        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
         backend = FakeBackend(source)
 
         # create a dangling dataset to be archived
@@ -263,7 +272,7 @@ class BaseBackendTest:
 
     def test_harvest_datasets_get_deleted(self):
         nb_datasets = 3
-        source = HarvestSourceFactory(config={"nb_datasets": nb_datasets})
+        source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
         backend = FakeBackend(source)
 
         job = backend.harvest()
@@ -281,6 +290,69 @@ class BaseBackendTest:
         for item in job.items:
             assert item.dataset is None
 
+    def test_no_datasets_duplication(self, app):
+        duplicated_remote_id_uri = "http://example.com/duplicated_remote_id_uri"
+        nb_datasets = 3
+        remote_ids = gen_remote_IDs(nb_datasets)
+        remote_ids.append(duplicated_remote_id_uri)
+        source = HarvestSourceFactory(config={"dataset_remote_ids": remote_ids})
+        backend = FakeBackend(source)
+
+        # Create a dataset that should be reused by the harvest, which will update it
+        # instead of creating a new one, as it has the same remote_id, domain and source_id.
+        dataset_reused = DatasetFactory(
+            title="Reused Dataset",
+            harvest={
+                "domain": source.domain,
+                "remote_id": "fake-0",  # the FakeBackend harvest should reuse this dataset
+                "source_id": str(source.id),
+            },
+        )
+
+        # Create a dataset that should be reused even though it's a different `source_id` and `domain,
+        # because the remote_id is the same and an URI.
+        dataset_reused_uri = DatasetFactory(
+            title="Reused Dataset with URI",
+            harvest={
+                "domain": "some-other-domain",
+                # the FakeBackend harvest above should reuse this dataset with the same remote_id URI
+                "remote_id": duplicated_remote_id_uri,
+                "source_id": "some-other-source-id",
+            },
+        )
+
+        # Create a dataset that should not be reused even though it's the same `remote_id`,
+        # as it's not an URI, and has a different domain and source id.
+        dataset_not_reused = DatasetFactory(
+            title="Duplicated Dataset",
+            harvest={
+                "domain": "some-other-domain",
+                "remote_id": "fake-0",  # the "source" harvest above should create another dataset with the same remote_id
+                "source_id": "some-other-source-id",
+            },
+        )
+
+        job = backend.harvest()
+
+        # 3 (nb_datasets) + 1 (dataset_remote_ids) created by the HarvestSourceFactory
+        assert len(job.items) == nb_datasets + 1
+        # all datasets : 4 mocks (3 nb_datasets + 1 dataset_remote_ids) + 3 created with DatasetFactory - 2 reused
+        assert Dataset.objects.count() == nb_datasets + 1 + 3 - 2
+        assert (
+            # and not 3, data_reused was not duplicated
+            Dataset.objects(harvest__remote_id="fake-0").count() == 2
+        )
+        # The dataset not reused wasn't overwritten nor updated by the harvest.
+        dataset_not_reused.reload()
+        assert dataset_not_reused.harvest.domain == "some-other-domain"
+        assert dataset_not_reused.harvest.source_id == "some-other-source-id"
+        # The "reused dataset" was overwritten and updated by the harvest.
+        dataset_reused.reload()
+        # The "reused dataset with uri" was overwritten and updated by the harvest.
+        dataset_reused_uri.reload()
+        assert dataset_reused_uri.harvest.domain == source.domain
+        assert dataset_reused_uri.harvest.source_id == str(source.id)
+
 
 @pytest.mark.usefixtures("clean_db")
 class BaseBackendValidateTest:

{udata-10.0.7.dev33415.dist-info → udata-10.0.7.dev33440.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata
-Version: 10.0.7.dev33415
+Version: 10.0.7.dev33440
 Summary: Open data portal
 Home-page: https://github.com/opendatateam/udata
 Author: Opendata Team
@@ -146,6 +146,7 @@ It is collectively taken care of by members of the
 - fix `function_field` wrong name [#3236](https://github.com/opendatateam/udata/pull/3236)
 - Allow dataservice archive [#3238](https://github.com/opendatateam/udata/pull/3238)
 - Improve OGC service detection to expose in RDF [#3241](https://github.com/opendatateam/udata/pull/3241)
+- Don’t duplicate datasets with a harvest.remote_id URI when harvesting [#3219](https://github.com/opendatateam/udata/pull/3219)
 
 ## 10.0.6 (2024-12-19)
 
@@ -157,6 +158,7 @@ It is collectively taken care of by members of the
 - Fix failing dataset save in update reuses metrics [#3230](https://github.com/opendatateam/udata/pull/3230)
 - Fix catalog RDF by preventing memory increase on getting dataservice hvd tags [#3231](https://github.com/opendatateam/udata/pull/3231)
 - Update purge tasks [#3167](https://github.com/opendatateam/udata/pull/3167)
+- Trigger GitLab infra deployment through simple-scaffolding script [#3232](https://github.com/opendatateam/udata/pull/3232)
 
 ## 10.0.5 (2024-12-09)
 

{udata-10.0.7.dev33415.dist-info → udata-10.0.7.dev33440.dist-info}/RECORD

@@ -291,13 +291,13 @@ udata/harvest/notifications.py,sha256=8WkHtD68v6rfZC6jCmAuuuRp4NN6q71ZkksZU5m9oJ
 udata/harvest/signals.py,sha256=3AhFHMPIFH5vz01NX5ycR_RWH14MXFWnCT6__LSa-QI,1338
 udata/harvest/tasks.py,sha256=ddJtvE0s-kAYt27-rKH6n8U8vKD8qKczlJtdBJzMUns,1718
 udata/harvest/backends/__init__.py,sha256=QjoFfBJfpw_xgk5YYWI1SgKJOMEmTMlxSfW79GNkSTI,459
-udata/harvest/backends/base.py,sha256=gq39-0SN3DSXfXIv4qjhIkEmPkV-2g6z53V0FikooSs,16413
+udata/harvest/backends/base.py,sha256=K-pfjYc7GiAQFprO-2Wknq3cmlEPImCIHmWdFK2yGVM,16638
 udata/harvest/backends/dcat.py,sha256=wbznPWCS9UJzpCWPAnmeUhUCREJg-yoQAHtEfr88RqU,15963
 udata/harvest/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 udata/harvest/tests/factories.py,sha256=Jb_RwsYogWhquG8aj2bzT3JLySq3530OmZV9cofeWlE,2269
 udata/harvest/tests/test_actions.py,sha256=s5yK4YQXVvrbLgKMx8_-ifFBC5QmUYBJWvkfNAZuy_Y,26504
 udata/harvest/tests/test_api.py,sha256=pCIoBCWZYr1IKgGARMe5fZArTsTnFQXUy5ur11vs3ho,20363
-udata/harvest/tests/test_base_backend.py,sha256=Rkr6P-rHJ2OWuyEkFPeEXwnYvI5C0B5M6SkUEUss2N8,13056
+udata/harvest/tests/test_base_backend.py,sha256=DrBwIr6uMweQ6S9oCpBS9tS54VlIcHMdPswGXAIDz9c,16446
 udata/harvest/tests/test_dcat_backend.py,sha256=IauCoeSylEH-Wyk3oXdC7sIk5bEEdnqBBxOiMJyIrSc,39877
 udata/harvest/tests/test_filters.py,sha256=2nXGnLTlM0iSOZBDBxmJp1KO3_59RXLS0vmXvEs2-KE,2468
 udata/harvest/tests/test_models.py,sha256=f9NRR2_S4oZFgF8qOumg0vv-lpnEBJbI5vNtcwFdSqM,831
@@ -710,9 +710,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=18Y5YtzVKInDejw-R-45HNzsB3OVwJ
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=6IQvFk0NTDV5Jq-kLkkzpioWfrMaLDa1oQSevKFbxKQ,44943
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=O4zKHNkiX-2GUfLLa0kwbxIA5M1jxiqkHzaMh1t2wKs,29169
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=W9C447pW0O-Q28ji8wGLgPNrnqlPYXaMD0AOWJPcpZc,51918
-udata-10.0.7.dev33415.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-10.0.7.dev33415.dist-info/METADATA,sha256=Cu_0UIVNWzR3xiGS41CezB2EBc9dKYkbBkd0Ikd6704,138651
-udata-10.0.7.dev33415.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-udata-10.0.7.dev33415.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
-udata-10.0.7.dev33415.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
-udata-10.0.7.dev33415.dist-info/RECORD,,
+udata-10.0.7.dev33440.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-10.0.7.dev33440.dist-info/METADATA,sha256=2fFeFMw--Coc91igNCZY4oslE5pF3ND5JTlLHLIMkHg,138907
+udata-10.0.7.dev33440.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+udata-10.0.7.dev33440.dist-info/entry_points.txt,sha256=3SKiqVy4HUqxf6iWspgMqH8d88Htk6KoLbG1BU-UddQ,451
+udata-10.0.7.dev33440.dist-info/top_level.txt,sha256=39OCg-VWFWOq4gCKnjKNu-s3OwFlZIu_dVH8Gl6ndHw,12
+udata-10.0.7.dev33440.dist-info/RECORD,,