udata 11.0.2.dev20__py3-none-any.whl → 11.0.2.dev22__py3-none-any.whl

udata/core/activity/models.py

@@ -9,7 +9,7 @@ from mongoengine.signals import post_save
 from udata.api_fields import get_fields
 from udata.auth import current_user
 from udata.mongo import db
-from udata.utils import get_field_value_from_path
+from udata.utils import filter_changed_fields, get_field_value_from_path
 
 from .signals import new_activity
 
@@ -130,16 +130,8 @@ class Auditable(object):
             cls.on_create.send(document)
         elif len(changed_fields):
             previous = getattr(document, "_previous_changed_fields", None)
-            # make sure that changed fields have actually changed when comparing the document
-            # once it has been reloaded. It may have been cleaned or normalized when saved to mongo.
-            # We compare them one by one with the previous value stored in _previous_changed_fields.
-            # See https://github.com/opendatateam/udata/pull/3412 for more context.
-            document.reload()
-            changed_fields = [
-                field
-                for field in changed_fields
-                if previous[field] != get_field_value_from_path(document, field)
-            ]
+            # Filter changed_fields since mongoengine raises some false positive occurences
+            changed_fields = filter_changed_fields(document, previous, changed_fields)
             if changed_fields:
                 cls.on_update.send(document, changed_fields=changed_fields, previous=previous)
         if getattr(document, "deleted_at", None) or getattr(document, "deleted", None):

udata/core/dataservices/rdf.py

@@ -32,7 +32,7 @@ def dataservice_from_rdf(
     remote_url_prefix: str | None = None,
 ) -> Dataservice:
     """
-    Create or update a dataset from a RDF/DCAT graph
+    Create or update a dataservice from a RDF/DCAT graph
     """
     if node is None:  # Assume first match is the only match
         node = graph.value(predicate=RDF.type, object=DCAT.DataService)
@@ -57,7 +57,6 @@ def dataservice_from_rdf(
         contact_point for role in roles for contact_point in role
     ] or dataservice.contact_points
 
-    datasets = []
     for dataset_node in d.objects(DCAT.servesDataset):
         id = dataset_node.value(DCT.identifier)
         dataset = next(
@@ -71,11 +70,9 @@ def dataservice_from_rdf(
                 None,
             )
 
-        if dataset is not None:
-            datasets.append(dataset.id)
-
-    if datasets:
-        dataservice.datasets = datasets
+        # We append the dataset to the list of the current attached ones if not already attached
+        if dataset is not None and dataset not in dataservice.datasets:
+            dataservice.datasets.append(dataset)
 
     license = rdf_value(d, DCT.license)
     if license is not None:

udata/harvest/tests/test_dcat_backend.py

@@ -8,8 +8,9 @@ from flask import current_app
 from lxml import etree
 from rdflib import Graph
 
+from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import Dataservice
-from udata.core.dataset.factories import LicenseFactory, ResourceSchemaMockData
+from udata.core.dataset.factories import DatasetFactory, LicenseFactory, ResourceSchemaMockData
 from udata.core.dataset.rdf import dataset_from_rdf
 from udata.core.organization.factories import OrganizationFactory
 from udata.harvest.models import HarvestJob
@@ -187,6 +188,49 @@ class DcatBackendTest:
             == "https://data.paris2024.org/api/explore/v2.1/console"
         )
 
+    def test_harvest_dataservices_keep_attached_associated_datasets(self, rmock):
+        """It should update the existing list of dataservice.datasets and not overwrite existing ones"""
+        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
+
+        filename = "bnodes.xml"
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+
+        previously_attached_dataset = DatasetFactory()
+        previously_harvested_dataset = DatasetFactory(
+            harvest={
+                "remote_id": "2",
+                "domain": source.domain,
+                "source_id": str(source.id),
+            }
+        )
+        existing_dataservice = DataserviceFactory(
+            # Two datasets are already attached, the first one NOT connected via harvesting
+            # when the second one is connected with dcat:servesDataset in harvest graph
+            datasets=[
+                previously_attached_dataset,
+                previously_harvested_dataset,
+            ],
+            harvest={
+                "remote_id": "https://data.paris2024.org/api/explore/v2.1/",
+                "domain": source.domain,
+                "source_id": str(source.id),
+            },
+        )
+
+        actions.run(source)
+
+        existing_dataservice.reload()
+
+        assert len(Dataservice.objects) == 1
+        assert existing_dataservice.title == "Explore API v2"
+        assert (
+            len(existing_dataservice.datasets) == 2 + 1
+        )  # The previsouly harvested dataset, the previously attached one and a new harvested dataset
+        assert previously_attached_dataset in existing_dataservice.datasets
+        assert previously_harvested_dataset in existing_dataservice.datasets
+
     def test_harvest_dataservices_ignore_accessservices(self, rmock):
         rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
 

udata/tests/test_activity.py

@@ -26,6 +26,7 @@ class FakeAuditableSubject(Auditable, db.Document):
     tags = field(db.TagListField())
     some_date = field(db.DateField())
     daterange_embedded = field(db.EmbeddedDocumentField(db.DateRange))
+    some_list = field(db.ListField(db.StringField()))
     embedded_list = field(db.ListField(db.EmbeddedDocumentField("FakeEmbedded")))
     ref_list = field(db.ListField(db.ReferenceField("FakeSubject")))
     not_auditable = field(db.StringField(), auditable=False)
@@ -128,6 +129,7 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
                 tags=["some", "tags"],
                 some_date=date(2020, 1, 1),
                 daterange_embedded={"start": date(2020, 1, 1), "end": date(2020, 12, 31)},
+                some_list=["some", "list"],
                 embedded_list=[FakeEmbedded(name=f"fake_embedded_{i}") for i in range(3)],
                 ref_list=[FakeSubject.objects.create(name=f"fake_ref_{i}") for i in range(3)],
                 not_auditable="original",
@@ -150,6 +152,10 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
         with assert_emit(post_save, FakeAuditableSubject.on_update):
             fake.save()
 
+        fake.some_list = ["other", "list"]
+        with assert_emit(post_save, FakeAuditableSubject.on_update):
+            fake.save()
+
         fake.embedded_list[1].name = "other"
         with assert_emit(post_save, FakeAuditableSubject.on_update):
             fake.save()
@@ -181,6 +187,13 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
             fake.reload()
             self.assertEqual(fake.some_date, date(2027, 7, 7))
 
+        # 3. Reordering of some elements in a list
+        fake.some_list = ["list", "other"]
+        with assert_not_emit(FakeAuditableSubject.on_update):
+            fake.save()
+            fake.reload()
+            self.assertEqual(fake.some_list, ["list", "other"])
+
         # The deletion should trigger a delete signal
         with assert_not_emit(FakeAuditableSubject.on_update):
             fake.delete()
@@ -192,6 +205,7 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
             tags=["some", "tags"],
             some_date=date(2020, 1, 1),
             daterange_embedded={"start": date(2020, 1, 1), "end": date(2020, 12, 31)},
+            some_list=["some", "list"],
             embedded_list=[FakeEmbedded(name=f"fake_embedded_{i}") for i in range(3)],
             ref_list=[FakeSubject.objects.create(name=f"fake_ref_{i}") for i in range(3)],
             not_auditable="original",
@@ -204,6 +218,7 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
                     "name",
                     "tags",
                     "some_date",
+                    "some_list",
                     "daterange_embedded.start",
                     "daterange_embedded.end",
                     "embedded_list.1.name",
@@ -214,6 +229,7 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
             self.assertEqual(args[1]["previous"]["some_date"], date(2020, 1, 1))
             self.assertEqual(args[1]["previous"]["daterange_embedded.start"], date(2020, 1, 1))
             self.assertEqual(args[1]["previous"]["daterange_embedded.end"], date(2020, 12, 31))
+            self.assertEqual(args[1]["previous"]["some_list"], ["some", "list"])
             self.assertEqual(args[1]["previous"]["embedded_list.1.name"], "fake_embedded_1")
 
         with assert_emit(FakeAuditableSubject.on_update, assertions_callback=check_signal_update):
@@ -222,6 +238,7 @@ class AuditableTest(WebTestMixin, DBTestMixin, TestCase):
             fake.some_date = date(2027, 7, 7)
             fake.daterange_embedded.start = date(2017, 7, 7)
             fake.daterange_embedded.end = date(2027, 7, 7)
+            fake.some_list = ["other", "list"]
             fake.embedded_list[1].name = "other"
             # Modification of a reference document should not be taken into account in changed_fields
             fake.ref_list[1].name = "other"

udata/utils.py

@@ -1,9 +1,11 @@
 import hashlib
+import itertools
 import math
 import re
+from collections import Counter
 from datetime import date, datetime
 from math import ceil
-from typing import Any
+from typing import Any, Hashable
 from uuid import UUID, uuid4
 from xml.sax.saxutils import escape
 
@@ -53,6 +55,35 @@ def get_field_value_from_path(document, field_path: str):
     return doc_field
 
 
+def filter_changed_fields(document, previous, changed_fields: list[str]):
+    # Make sure that changed fields have actually changed.
+    # We compare the document values once it has been reloaded.
+    # It may have been cleaned or normalized when saved to mongo.
+    # We compare the field values one by one with the previous value stored in _previous_changed_fields.
+    # We also ignore reordering in the case of list, ex tags or contact points.
+    # See https://github.com/opendatateam/udata/pull/3412 for more context.
+    document.reload()
+    filtered_changed_fields = []
+    for field in changed_fields:
+        previous_value = previous[field]
+        current_value = get_field_value_from_path(document, field)
+        # Filter out special case of list reordering, does not support unhashable types
+        if (
+            isinstance(previous_value, list)
+            and isinstance(current_value, list)
+            and all(
+                isinstance(value, Hashable)
+                for value in itertools.chain(previous_value, current_value)
+            )
+        ):
+            if Counter(previous_value) != Counter(current_value):
+                filtered_changed_fields.append(field)
+        # Direct comparison for the rest of the fields
+        elif previous_value != current_value:
+            filtered_changed_fields.append(field)
+    return filtered_changed_fields
+
+
 FIRST_CAP_RE = re.compile("(.)([A-Z][a-z]+)")
 ALL_CAP_RE = re.compile("([a-z0-9])([A-Z])")
 UUID_LENGTH = 36

{udata-11.0.2.dev20.dist-info → udata-11.0.2.dev22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: udata
-Version: 11.0.2.dev20
+Version: 11.0.2.dev22
 Summary: Open data portal
 Author-email: Opendata Team <opendatateam@data.gouv.fr>
 Maintainer-email: Opendata Team <opendatateam@data.gouv.fr>

{udata-11.0.2.dev20.dist-info → udata-11.0.2.dev22.dist-info}/RECORD

@@ -18,7 +18,7 @@ udata/tasks.py,sha256=Sv01dhvATtq_oHOBp3J1j1VT1HQe0Pab7zxwIeIdKoo,5122
 udata/terms.md,sha256=nFx978tUQ3vTEv6POykXaZvcQ5e_gcvmO4ZgcfbSWXo,187
 udata/tracking.py,sha256=WOcqA1RlHN8EPFuEc2kNau54mec4-pvi-wUFrMXevzg,345
 udata/uris.py,sha256=1wOrsxu6lmZJ1h4634kNHjqOjaOO0D5cIWKF_v_Gtn4,4264
-udata/utils.py,sha256=W2-pZwpEYQULrovX4msGNjBvj6EvP9eJm6ueQ2ujbP0,9681
+udata/utils.py,sha256=mtosjF91SPuSM-63EyxjLLnxs5DT0iSBdz-ECNeTYGU,11128
 udata/worker.py,sha256=K-Wafye5-uXP4kQlffRKws2J9YbJ6m6n2QjcVsY8Nsg,118
 udata/wsgi.py,sha256=MY8en9K9eDluvJYUxTdzqSDoYaDgCVZ69ZcUvxAvgqA,77
 udata/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -57,7 +57,7 @@ udata/core/linkable.py,sha256=8YuTzZo5Y36CVIYxESC-EPJ-zKsQTRbi4OIZTp-wWig,587
 udata/core/owned.py,sha256=OQT7wdk7dAqGvWDiJRVkKJxerDc9_Io910nvLmfBAVI,5561
 udata/core/activity/__init__.py,sha256=dLTseBmVYbQzxB7OR7Dz0LtIwEjNQvgUSR7SwfInb68,399
 udata/core/activity/api.py,sha256=dwdEkJxxwbQ2PsCSnv6fEE8Ps1FiJ4fEHpTog_gk4Rs,4291
-udata/core/activity/models.py,sha256=qZ0BMwrtXu_SZLICaVauDvcWA5CDMgZij2Hxro6CbTQ,5326
+udata/core/activity/models.py,sha256=wJ5SyvTJXsaTUbJRQGK1ZvkJCG8bmlHSDrro1afsrhM,4926
 udata/core/activity/signals.py,sha256=Io2A43as3yR-DZ5R3wzM_bTpn528pxWsZDUFZ9xtj2Y,191
 udata/core/activity/tasks.py,sha256=F3PY12dnpT5Z8VuYfuOyDP6VPKPJmq1Sm4lSiPfmUCA,1498
 udata/core/badges/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -88,7 +88,7 @@ udata/core/dataservices/csv.py,sha256=HWI2JrN_Vuw0te9FHlJ6eyqcRcKHOKXuzg45D4Ti6F
 udata/core/dataservices/factories.py,sha256=pKVoArNSCIbvGA-cWUc7vr8TmjYsUvOXzzcuUB5JyF4,964
 udata/core/dataservices/models.py,sha256=Kkgf_9TfuuijtckoD6c4mM6bN7DZq508HmL_vUkNoqI,12384
 udata/core/dataservices/permissions.py,sha256=98zM_R4v2ZtRubflB7ajaVQz-DVc-pZBMgtKUYy34oI,169
-udata/core/dataservices/rdf.py,sha256=MKzoHDcsEu_Be5gI5l41vPrrlHCql8AnqTbrxhXv2D0,7949
+udata/core/dataservices/rdf.py,sha256=lbOeH-2IG_4zz6WQtD4L_ADq9iZfdSQLAiutanxd8gU,8023
 udata/core/dataservices/search.py,sha256=Tt7CUqb49Rl4hvkfGO3AiNs1Oc7HhTeBp80xQK8wwXc,4911
 udata/core/dataservices/tasks.py,sha256=fHG1r5ymfJRXJ_Lug6je3VKZoK30XKXE2rQ8x0R-jUk,1068
 udata/core/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -318,7 +318,7 @@ udata/harvest/tests/person.jsonld,sha256=I7Ynh-PQlNeD51I1LrCgYOEjhL-WBeb65xzIE_s
 udata/harvest/tests/test_actions.py,sha256=d5TTFTbs4PdBydWICqDtfoeo3zLyzcNDzv4aMH9spxo,25881
 udata/harvest/tests/test_api.py,sha256=gSuICkPy3KVRUhHAyudXVf_gLwiB7SoriUp3DLXWDdA,21611
 udata/harvest/tests/test_base_backend.py,sha256=ow8ecGtD836mUqyPWYjkS5nx0STyT5RMLgBdDyOhts4,19233
-udata/harvest/tests/test_dcat_backend.py,sha256=yREZBXgghpb4SFWdOWKLV_teJPe3A3D4wjHCTSeS6Ns,48645
+udata/harvest/tests/test_dcat_backend.py,sha256=8w_wZepgkb0_j9Ah5f4qMwJIpuH7bTqmc4u5MaaP7G4,50551
 udata/harvest/tests/test_filters.py,sha256=PT2qopEIoXsqi8MsNDRuhNH7jGXiQo8r0uJrCOUd4aM,2465
 udata/harvest/tests/test_models.py,sha256=f9NRR2_S4oZFgF8qOumg0vv-lpnEBJbI5vNtcwFdSqM,831
 udata/harvest/tests/test_notifications.py,sha256=MMzTzkv-GXMNFeOwAi31rdTsAXyLCLOSna41zOtaJG0,816
@@ -631,7 +631,7 @@ udata/tests/helpers.py,sha256=VP8cz13WBxkMoTUbOwnMV2-bzE4nVHI6l_z6m3u1DIE,5975
 udata/tests/models.py,sha256=5oTC-cgKSL0sUdlqjUiJ6U8-YZBQanObb-MhZhQIV3M,238
 udata/tests/plugin.py,sha256=QDY6fqjozck1_KrNGqN4wkIUAdACpAnUDOW6GTjKmqQ,12480
 udata/tests/schemas.json,sha256=szM1jDpkogfOG4xWbjIGjLgG8l9-ZyE3JKQtecJyD1E,4990
-udata/tests/test_activity.py,sha256=u-gao2T7Q2N-isw3_qCh6z5nFiXmMV3N8jEjqJlP2cQ,9164
+udata/tests/test_activity.py,sha256=6VpJbSNPSwBOcPXg1gHZF7kpEMzpvfd9TOvyqePQzRA,9871
 udata/tests/test_api_fields.py,sha256=NCUTtOMEaTM5-tK-YUxhjEud2IPIDOHR3vbZWAQdECg,12786
 udata/tests/test_cors.py,sha256=b_pyxKeIyqhnsXxXryPf4d0V0QxaLQ1P_VjY89Q_j3g,3233
 udata/tests/test_dcat_commands.py,sha256=fDAnAjkja8AXw_qzaAWnVTgglkBAvK2mjPMHUCtqrrU,919
@@ -764,9 +764,9 @@ udata/translations/pt/LC_MESSAGES/udata.mo,sha256=U0abG-nBwCIoYxRZNsc4KOLeIRSqTV
 udata/translations/pt/LC_MESSAGES/udata.po,sha256=eCG35rMzYLHXyLbsnLSexS1g0N_K-WpNHqrt_8y6I4E,48590
 udata/translations/sr/LC_MESSAGES/udata.mo,sha256=IBcCAdmcvkeK7ZeRBNRI-wJ0jzWNM0eXM5VXAc1frWI,28692
 udata/translations/sr/LC_MESSAGES/udata.po,sha256=yFxHEEB4behNwQ7JnyoYheiCKLNnMS_NV4guzgyzWcE,55332
-udata-11.0.2.dev20.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-udata-11.0.2.dev20.dist-info/METADATA,sha256=TCEyw8Z7EE7cNZwnWV3jJoEOJboAJYOWEDXjv3OMel8,6769
-udata-11.0.2.dev20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-udata-11.0.2.dev20.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
-udata-11.0.2.dev20.dist-info/top_level.txt,sha256=EF6CE6YSHd_og-8LCEA4q25ALUpWVe8D0okOLdMAE3A,6
-udata-11.0.2.dev20.dist-info/RECORD,,
+udata-11.0.2.dev22.dist-info/licenses/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+udata-11.0.2.dev22.dist-info/METADATA,sha256=TjMQIGWeKQ9tIiKPyr0UxYK0XFjjBU9hGQiJAxrQj0I,6769
+udata-11.0.2.dev22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+udata-11.0.2.dev22.dist-info/entry_points.txt,sha256=v2u12qO11i2lyLNIp136WmLJ-NHT-Kew3Duu8J-AXPM,614
+udata-11.0.2.dev22.dist-info/top_level.txt,sha256=EF6CE6YSHd_og-8LCEA4q25ALUpWVe8D0okOLdMAE3A,6
+udata-11.0.2.dev22.dist-info/RECORD,,