udata 10.9.1.dev0__py3-none-any.whl → 11.0.2.dev8__py3-none-any.whl

udata/core/activity/models.py

@@ -1,6 +1,8 @@
+import logging
 from datetime import datetime
 
 from blinker import Signal
+from flask import g
 from mongoengine.errors import DoesNotExist
 from mongoengine.signals import post_save
 
@@ -13,6 +15,8 @@ from .signals import new_activity
 
 __all__ = ("Activity",)
 
+log = logging.getLogger(__name__)
+
 
 _registered_activities = {}
 
@@ -70,10 +74,15 @@ class Activity(db.Document, metaclass=EmitNewActivityMetaClass):
 
     @classmethod
     def emit(cls, related_to, organization=None, changed_fields=None, extras=None):
+        if hasattr(g, "harvest_activity_user"):
+            # We're in the context of a harvest action with a harvest activity user to use as actor
+            actor = g.harvest_activity_user
+        else:
+            actor = current_user._get_current_object()
         new_activity.send(
             cls,
             related_to=related_to,
-            actor=current_user._get_current_object(),
+            actor=actor,
             organization=organization,
             changes=changed_fields,
             extras=extras,
@@ -110,7 +119,9 @@ class Auditable(object):
             # for backward compatibility, all fields are treated as auditable for classes not using field() function
             auditable_fields = document._get_changed_fields()
         changed_fields = [
-            field for field in document._get_changed_fields() if field in auditable_fields
+            field
+            for field in document._get_changed_fields()
+            if field.split(".")[0] in auditable_fields
         ]
         if "post_save" in kwargs.get("ignores", []):
             return
@@ -119,6 +130,17 @@ class Auditable(object):
             cls.on_create.send(document)
         elif len(changed_fields):
             previous = getattr(document, "_previous_changed_fields", None)
-            cls.on_update.send(document, changed_fields=changed_fields, previous=previous)
+            # make sure that changed fields have actually changed when comparing the document
+            # once it has been reloaded. It may have been cleaned or normalized when saved to mongo.
+            # We compare them one by one with the previous value stored in _previous_changed_fields.
+            # See https://github.com/opendatateam/udata/pull/3412 for more context.
+            document.reload()
+            changed_fields = [
+                field
+                for field in changed_fields
+                if previous[field] != get_field_value_from_path(document, field)
+            ]
+            if changed_fields:
+                cls.on_update.send(document, changed_fields=changed_fields, previous=previous)
         if getattr(document, "deleted_at", None) or getattr(document, "deleted", None):
             cls.on_delete.send(document)

udata/core/dataservices/activities.py

@@ -1,3 +1,5 @@
+from flask import g
+
 from udata.auth import current_user
 from udata.i18n import lazy_gettext as _
 from udata.models import Activity, Dataservice, db
@@ -36,18 +38,18 @@ class UserDeletedDataservice(DataserviceRelatedActivity, Activity):
 
 @Dataservice.on_create.connect
 def on_user_created_dataservice(dataservice):
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserCreatedDataservice.emit(dataservice, dataservice.organization)
 
 
 @Dataservice.on_update.connect
 def on_user_updated_dataservice(dataservice, **kwargs):
     changed_fields = kwargs.get("changed_fields", [])
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserUpdatedDataservice.emit(dataservice, dataservice.organization, changed_fields)
 
 
 @Dataservice.on_delete.connect
 def on_user_deleted_dataservice(dataservice):
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserDeletedDataservice.emit(dataservice, dataservice.organization)

udata/core/dataset/activities.py

@@ -1,3 +1,5 @@
+from flask import g
+
 from udata.auth import current_user
 from udata.i18n import lazy_gettext as _
 from udata.models import Activity, Dataset, db
@@ -55,7 +57,7 @@ class UserRemovedResourceFromDataset(DatasetRelatedActivity, Activity):
 
 @Dataset.on_resource_added.connect
 def on_user_added_resource_to_dataset(sender, document, **kwargs):
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserAddedResourceToDataset.emit(
             document, document.organization, None, {"resource_id": str(kwargs["resource_id"])}
         )
@@ -64,7 +66,7 @@ def on_user_added_resource_to_dataset(sender, document, **kwargs):
 @Dataset.on_resource_updated.connect
 def on_user_updated_resource(sender, document, **kwargs):
     changed_fields = kwargs.get("changed_fields", [])
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserUpdatedResource.emit(
             document,
             document.organization,
@@ -75,7 +77,7 @@ def on_user_updated_resource(sender, document, **kwargs):
 
 @Dataset.on_resource_removed.connect
 def on_user_removed_resource_from_dataset(sender, document, **kwargs):
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserRemovedResourceFromDataset.emit(
             document, document.organization, None, {"resource_id": str(kwargs["resource_id"])}
         )
@@ -83,18 +85,18 @@ def on_user_removed_resource_from_dataset(sender, document, **kwargs):
 
 @Dataset.on_create.connect
 def on_user_created_dataset(dataset):
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserCreatedDataset.emit(dataset, dataset.organization)
 
 
 @Dataset.on_update.connect
 def on_user_updated_dataset(dataset, **kwargs):
     changed_fields = kwargs.get("changed_fields", [])
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserUpdatedDataset.emit(dataset, dataset.organization, changed_fields)
 
 
 @Dataset.on_delete.connect
 def on_user_deleted_dataset(dataset):
-    if current_user and current_user.is_authenticated:
+    if (current_user and current_user.is_authenticated) or hasattr(g, "harvest_activity_user"):
         UserDeletedDataset.emit(dataset, dataset.organization)

udata/harvest/backends/base.py

@@ -4,14 +4,14 @@ from datetime import date, datetime, timedelta
 from uuid import UUID
 
 import requests
-from flask import current_app
+from flask import current_app, g
 from voluptuous import MultipleInvalid, RequiredFieldInvalid
 
 import udata.uris as uris
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.models import HarvestDatasetMetadata
-from udata.models import Dataset
+from udata.models import Dataset, User
 from udata.utils import safe_unicode
 
 from ..exceptions import HarvestException, HarvestSkipException, HarvestValidationError
@@ -168,6 +168,17 @@ class BaseBackend(object):
         self.job = factory(status="initialized", started=datetime.utcnow(), source=self.source)
 
         before_harvest_job.send(self)
+        # Set harvest_activity_user on global context during the run
+        if current_app.config["HARVEST_ACTIVITY_USER_ID"]:
+            try:
+                # Try to fetch the existing harvest activity user
+                g.harvest_activity_user = User.objects.get(
+                    id=current_app.config["HARVEST_ACTIVITY_USER_ID"]
+                )
+            except User.DoesNotExist:
+                log.exception(
+                    "HARVEST_ACTIVITY_USER_ID does not seem to match an existing user id."
+                )
 
         try:
             self.inner_harvest()
@@ -199,6 +210,9 @@ class BaseBackend(object):
             self.job.errors.append(error)
         finally:
             self.end_job()
+            # Clean harvest_activity_user on global context
+            if hasattr(g, "harvest_activity_user"):
+                delattr(g, "harvest_activity_user")
 
         return self.job
 

udata/harvest/tests/test_actions.py

@@ -4,16 +4,19 @@ from datetime import datetime, timedelta
 from tempfile import NamedTemporaryFile
 
 import pytest
+from flask import current_app
 from mock import patch
 
+from udata.core.activity.models import new_activity
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
+from udata.core.dataset.activities import UserCreatedDataset
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import UserFactory
 from udata.models import Dataset, PeriodicTask
-from udata.tests.helpers import assert_emit, assert_equal_dates
+from udata.tests.helpers import assert_emit, assert_equal_dates, assert_not_emit
 from udata.utils import faker
 
 from .. import actions, signals
@@ -578,6 +581,24 @@ class ExecutionTestMixin(MockBackendsMixin):
         self.action(source)
         assert len(Dataset.objects) == 5
 
+    @pytest.mark.options(HARVEST_ACTIVITY_USER_ID="68b860182728e27218dd7c72")
+    def test_harvest_emit_activity(self):
+        # We need to init dataset activities module
+        import udata.core.dataset.activities  # noqa
+
+        user = UserFactory(id=current_app.config["HARVEST_ACTIVITY_USER_ID"])
+        source = HarvestSourceFactory(backend="factory")
+        with assert_emit(Dataset.on_create, new_activity):
+            self.action(source)
+
+        # We have an activity for each dataset created by the source action
+        activities = UserCreatedDataset.objects(actor=user)
+        assert activities.count() == Dataset.objects().count()
+
+        # On a second run, we don't expect any signal sent (no creation, update or deletion)
+        with assert_not_emit(Dataset.on_create, Dataset.on_update, new_activity):
+            self.action(source)
+
 
 class HarvestLaunchTest(ExecutionTestMixin):
     def action(self, *args, **kwargs):

udata/settings.py

@@ -261,6 +261,8 @@ class Defaults(object):
 
     DELAY_BEFORE_REMINDER_NOTIFICATION = 30  # Days
 
+    # Harvest settings
+    ###########################################################################
     HARVEST_ENABLE_MANUAL_RUN = False
 
     HARVEST_PREVIEW_MAX_ITEMS = 20
@@ -285,7 +287,12 @@ class Defaults(object):
 
     HARVEST_ISO19139_XSLT_URL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/refs/heads/geodcat-ap-2.0.0/iso-19139-to-dcat-ap.xsl"
 
+    # If set, harvest emit activities associated with this user as actor
+    # It should be a dedicated service account
+    HARVEST_ACTIVITY_USER_ID = None
+
     # S3 connection details
+    ###########################################################################
     S3_URL = None
     S3_ACCESS_KEY_ID = None
     S3_SECRET_ACCESS_KEY = None
@@ -626,6 +633,7 @@ class Testing(object):
         "check_deliverability": False
     }  # Disables deliverability for email domain name
     PUBLISH_ON_RESOURCE_EVENTS = False
+    HARVEST_ACTIVITY_USER_ID = None
 
 
 class Debug(Defaults):