udata 10.4.1.dev35211__py2.py3-none-any.whl → 10.4.2__py2.py3-none-any.whl

udata/core/topic/models.py

@@ -1,6 +1,9 @@
+from blinker import Signal
 from flask import url_for
-from mongoengine.signals import pre_save
+from mongoengine.signals import post_save, pre_save
 
+from udata.api_fields import field
+from udata.core.activity.models import Auditable
 from udata.core.owned import Owned, OwnedQuerySet
 from udata.models import SpatialCoverage, db
 from udata.search import reindex
@@ -8,24 +11,24 @@ from udata.search import reindex
 __all__ = ("Topic",)
 
 
-class Topic(db.Document, Owned, db.Datetimed):
-    name = db.StringField(required=True)
-    slug = db.SlugField(
-        max_length=255, required=True, populate_from="name", update=True, follow=True
+class Topic(db.Datetimed, Auditable, db.Document, Owned):
+    name = field(db.StringField(required=True))
+    slug = field(
+        db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
+        auditable=False,
     )
-    description = db.StringField()
-    tags = db.ListField(db.StringField())
-    color = db.IntField()
+    description = field(db.StringField())
+    tags = field(db.ListField(db.StringField()))
+    color = field(db.IntField())
 
-    tags = db.ListField(db.StringField())
-    datasets = db.ListField(db.LazyReferenceField("Dataset", reverse_delete_rule=db.PULL))
-    reuses = db.ListField(db.LazyReferenceField("Reuse", reverse_delete_rule=db.PULL))
+    datasets = field(db.ListField(db.LazyReferenceField("Dataset", reverse_delete_rule=db.PULL)))
+    reuses = field(db.ListField(db.LazyReferenceField("Reuse", reverse_delete_rule=db.PULL)))
 
-    featured = db.BooleanField(default=False)
-    private = db.BooleanField()
-    extras = db.ExtrasField()
+    featured = field(db.BooleanField(default=False), auditable=False)
+    private = field(db.BooleanField())
+    extras = field(db.ExtrasField(), auditable=False)
 
-    spatial = db.EmbeddedDocumentField(SpatialCoverage)
+    spatial = field(db.EmbeddedDocumentField(SpatialCoverage))
 
     meta = {
         "indexes": ["$name", "created_at", "slug"] + Owned.meta["indexes"],
@@ -34,6 +37,10 @@ class Topic(db.Document, Owned, db.Datetimed):
         "queryset_class": OwnedQuerySet,
     }
 
+    after_save = Signal()
+    on_create = Signal()
+    on_update = Signal()
+
     def __str__(self):
         return self.name
 
@@ -60,3 +67,4 @@ class Topic(db.Document, Owned, db.Datetimed):
 
 
 pre_save.connect(Topic.pre_save, sender=Topic)
+post_save.connect(Topic.post_save, sender=Topic)

udata/core/user/activities.py

@@ -1,5 +1,7 @@
 from flask_security import current_user
 
+from udata.core.dataservices.activities import DataserviceRelatedActivity
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.activities import DatasetRelatedActivity
 from udata.core.discussions.signals import on_new_discussion, on_new_discussion_comment
 from udata.core.followers.signals import on_follow
@@ -28,11 +30,6 @@ class DiscussActivity(object):
     badge_type = "warning"
 
 
-class UserStarredOrganization(FollowActivity, OrgRelatedActivity, Activity):
-    key = "organization:followed"
-    label = _("followed an organization")
-
-
 class UserFollowedUser(FollowActivity, Activity):
     key = "user:followed"
     label = _("followed a user")
@@ -40,6 +37,11 @@ class UserFollowedUser(FollowActivity, Activity):
     template = "activity/user.html"
 
 
+class UserDiscussedDataservice(DiscussActivity, DataserviceRelatedActivity, Activity):
+    key = "dataservice:discussed"
+    label = _("discussed a dataservice")
+
+
 class UserDiscussedDataset(DiscussActivity, DatasetRelatedActivity, Activity):
     key = "dataset:discussed"
     label = _("discussed a dataset")
@@ -50,6 +52,11 @@ class UserDiscussedReuse(DiscussActivity, ReuseRelatedActivity, Activity):
     label = _("discussed a reuse")
 
 
+class UserFollowedDataservice(FollowActivity, DataserviceRelatedActivity, Activity):
+    key = "dataservice:followed"
+    label = _("followed a dataservice")
+
+
 class UserFollowedDataset(FollowActivity, DatasetRelatedActivity, Activity):
     key = "dataset:followed"
     label = _("followed a dataset")
@@ -68,7 +75,9 @@ class UserFollowedOrganization(FollowActivity, OrgRelatedActivity, Activity):
 @on_follow.connect
 def write_activity_on_follow(follow, **kwargs):
     if current_user.is_authenticated:
-        if isinstance(follow.following, Dataset):
+        if isinstance(follow.following, Dataservice):
+            UserFollowedDataservice.emit(follow.following)
+        elif isinstance(follow.following, Dataset):
             UserFollowedDataset.emit(follow.following)
         elif isinstance(follow.following, Reuse):
             UserFollowedReuse.emit(follow.following)
@@ -82,6 +91,8 @@ def write_activity_on_follow(follow, **kwargs):
 @on_new_discussion_comment.connect
 def write_activity_on_discuss(discussion, **kwargs):
     if current_user.is_authenticated:
+        if isinstance(discussion.subject, Dataservice):
+            UserDiscussedDataservice.emit(discussion.subject)
         if isinstance(discussion.subject, Dataset):
             UserDiscussedDataset.emit(discussion.subject)
         elif isinstance(discussion.subject, Reuse):

udata/core/user/api.py

@@ -409,6 +409,7 @@ class SuggestUsersAPI(API):
                 "first_name": user.first_name,
                 "last_name": user.last_name,
                 "avatar_url": user.avatar,
+                "email": user.email,
                 "slug": user.slug,
             }
             for user in users.order_by(DEFAULT_SORTING).limit(args["size"])

udata/core/user/api_fields.py

@@ -30,7 +30,7 @@ user_ref_fields = api.inherit(
     },
 )
 
-from udata.core.organization.api_fields import org_ref_fields  # noqa
+from udata.core.organization.api_fields import member_email_with_visibility_check, org_ref_fields  # noqa
 
 user_fields = api.model(
     "User",
@@ -126,6 +126,11 @@ user_suggestion_fields = api.model(
         "avatar_url": fields.ImageField(
             size=BIGGEST_AVATAR_SIZE, description="The user avatar URL", readonly=True
         ),
+        "email": fields.Raw(
+            attribute=lambda o: member_email_with_visibility_check(o["email"]),
+            description="The user email (only the domain for non-admin user)",
+            readonly=True,
+        ),
         "slug": fields.String(description="The user permalink string", readonly=True),
     },
 )

udata/core/user/models.py

@@ -12,6 +12,7 @@ from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
 from udata import mail
+from udata.api_fields import field
 from udata.core import storages
 from udata.core.discussions.models import Discussion
 from udata.core.storages import avatars, default_image_basename
@@ -42,49 +43,53 @@ class UserSettings(db.EmbeddedDocument):
 
 
 class User(WithMetrics, UserMixin, db.Document):
-    slug = db.SlugField(max_length=255, required=True, populate_from="fullname")
-    email = db.StringField(max_length=255, required=True, unique=True)
-    password = db.StringField()
-    active = db.BooleanField()
-    fs_uniquifier = db.StringField(max_length=64, unique=True, sparse=True)
-    roles = db.ListField(db.ReferenceField(Role), default=[])
+    slug = field(
+        db.SlugField(max_length=255, required=True, populate_from="fullname"), auditable=False
+    )
+    email = field(db.StringField(max_length=255, required=True, unique=True))
+    password = field(db.StringField())
+    active = field(db.BooleanField())
+    fs_uniquifier = field(db.StringField(max_length=64, unique=True, sparse=True))
+    roles = field(db.ListField(db.ReferenceField(Role), default=[]))
 
-    first_name = db.StringField(max_length=255, required=True)
-    last_name = db.StringField(max_length=255, required=True)
+    first_name = field(db.StringField(max_length=255, required=True))
+    last_name = field(db.StringField(max_length=255, required=True))
 
-    avatar_url = db.URLField()
-    avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
-    website = db.URLField()
-    about = db.StringField()
+    avatar_url = field(db.URLField())
+    avatar = field(
+        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
+    )
+    website = field(db.URLField())
+    about = field(db.StringField())
 
-    prefered_language = db.StringField()
+    prefered_language = field(db.StringField())
 
-    apikey = db.StringField()
+    apikey = field(db.StringField())
 
-    created_at = db.DateTimeField(default=datetime.utcnow, required=True)
+    created_at = field(db.DateTimeField(default=datetime.utcnow, required=True), auditable=False)
 
     # The field below is required for Flask-security
     # when SECURITY_CONFIRMABLE is True
-    confirmed_at = db.DateTimeField()
+    confirmed_at = field(db.DateTimeField(), auditable=False)
 
-    password_rotation_demanded = db.DateTimeField()
-    password_rotation_performed = db.DateTimeField()
+    password_rotation_demanded = field(db.DateTimeField(), auditable=False)
+    password_rotation_performed = field(db.DateTimeField(), auditable=False)
 
     # The 5 fields below are required for Flask-security
     # when SECURITY_TRACKABLE is True
-    last_login_at = db.DateTimeField()
-    current_login_at = db.DateTimeField()
-    last_login_ip = db.StringField()
-    current_login_ip = db.StringField()
-    login_count = db.IntField()
+    last_login_at = field(db.DateTimeField(), auditable=False)
+    current_login_at = field(db.DateTimeField(), auditable=False)
+    last_login_ip = field(db.StringField(), auditable=False)
+    current_login_ip = field(db.StringField(), auditable=False)
+    login_count = field(db.IntField(), auditable=False)
 
-    deleted = db.DateTimeField()
-    ext = db.MapField(db.GenericEmbeddedDocumentField())
-    extras = db.ExtrasField()
+    deleted = field(db.DateTimeField())
+    ext = field(db.MapField(db.GenericEmbeddedDocumentField()))
+    extras = field(db.ExtrasField(), auditable=False)
 
     # Used to track notification for automatic inactive users deletion
     # when YEARS_OF_INACTIVITY_BEFORE_DELETION is set
-    inactive_deletion_notified_at = db.DateTimeField()
+    inactive_deletion_notified_at = field(db.DateTimeField(), auditable=False)
 
     before_save = Signal()
     after_save = Signal()
@@ -209,6 +214,8 @@ class User(WithMetrics, UserMixin, db.Document):
 
     @classmethod
     def post_save(cls, sender, document, **kwargs):
+        if "post_save" in kwargs.get("ignores", []):
+            return
         cls.after_save.send(document)
         if kwargs.get("created"):
             cls.on_create.send(document)
@@ -294,31 +301,31 @@ class User(WithMetrics, UserMixin, db.Document):
         from udata.models import Dataset
 
         self.metrics["datasets"] = Dataset.objects(owner=self).visible().count()
-        self.save()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
     def count_reuses(self):
         from udata.models import Reuse
 
         self.metrics["reuses"] = Reuse.objects(owner=self).visible().count()
-        self.save()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
     def count_dataservices(self):
         from udata.core.dataservices.models import Dataservice
 
         self.metrics["dataservices"] = Dataservice.objects(owner=self).visible().count()
-        self.save()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
     def count_followers(self):
         from udata.models import Follow
 
         self.metrics["followers"] = Follow.objects(until=None).followers(self).count()
-        self.save()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
     def count_following(self):
         from udata.models import Follow
 
         self.metrics["following"] = Follow.objects.following(self).count()
-        self.save()
+        self.save(signal_kwargs={"ignores": ["post_save"]})
 
 
 datastore = MongoEngineUserDatastore(db, User, Role)

udata/migrations/2025-05-22-purge-duplicate-activities.py

@@ -0,0 +1,101 @@
+"""
+This migration updates Topic.featured to False when it is None.
+"""
+
+import logging
+from datetime import datetime, timedelta
+
+from mongoengine.connection import get_db
+
+from udata.core.dataset.activities import UserCreatedDataset, UserDeletedDataset, UserUpdatedDataset
+from udata.core.organization.activities import UserUpdatedOrganization
+from udata.core.reuse.activities import UserCreatedReuse, UserDeletedReuse, UserUpdatedReuse
+from udata.core.user.activities import (
+    UserDiscussedDataset,
+    UserDiscussedReuse,
+    UserFollowedDataset,
+    UserFollowedOrganization,
+    UserFollowedReuse,
+)
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    # Remove legacy fields (`as_organization`, `kwargs`) from old activities
+    result = get_db().activity.update_many({}, {"$unset": {"as_organization": ""}})
+    log.info(
+        f"Legacy field `as_organization` removed from {result.modified_count} activity objects"
+    )
+
+    result = get_db().activity.update_many({}, {"$unset": {"kwargs": ""}})
+    log.info(f"Legacy field `kwargs` removed from {result.modified_count} activity objects")
+
+    # Clean duplicate activities in case of discussion or following
+    # - remove the "updated" activity on the discussed/followed object
+    # - remove the activity on the organization
+    # The heuristic is to look for specific activities by the same actor on the targeted object
+    # within a -1 +1 second timespan
+    for action_related_activity, object_updated_activity in [
+        (UserDiscussedDataset, UserUpdatedDataset),
+        (UserDiscussedReuse, UserUpdatedReuse),
+        (UserFollowedDataset, UserUpdatedDataset),
+        (UserFollowedReuse, UserUpdatedReuse),
+    ]:
+        org_activity_count = 0
+        object_activity_count = 0
+        activities = (
+            action_related_activity.objects()
+            .no_dereference()  # We use no_dereference in query to prevent DBref DoesNotExist errors
+            .no_cache()
+            .timeout(False)
+        )
+        log.info(
+            f"{datetime.utcnow()}: Processing {activities.count()} {action_related_activity} activities..."
+        )
+        for act in activities:
+            object_activity_count += object_updated_activity.objects(
+                actor=act.actor.id,
+                related_to=act.related_to.id,
+                created_at__gte=act.created_at - timedelta(seconds=1),
+                created_at__lte=act.created_at + timedelta(seconds=1),
+            ).delete()
+            if act.organization:
+                org_activity_count += UserUpdatedOrganization.objects(
+                    actor=act.actor.id,
+                    related_to=act.organization,
+                    created_at__gte=act.created_at - timedelta(seconds=1),
+                    created_at__lte=act.created_at + timedelta(seconds=1),
+                ).delete()
+        log.info(
+            f"{datetime.utcnow()}: Deleted {object_activity_count} {object_updated_activity} and {org_activity_count} UserUpdatedOrganization activities"
+        )
+
+    # Clean duplicated UserUpdatedOrganization activities on organization for any object related activity
+    for object_related_activity in [
+        UserCreatedDataset,
+        UserUpdatedDataset,
+        UserDeletedDataset,
+        UserCreatedReuse,
+        UserUpdatedReuse,
+        UserDeletedReuse,
+        UserFollowedOrganization,
+    ]:
+        count = 0
+        activities = (
+            object_related_activity.objects(organization__exists=True)
+            .no_dereference()  # We use no_dereference in query to prevent DBref DoesNotExist errors
+            .no_cache()
+            .timeout(False)
+        )
+        log.info(
+            f"{datetime.utcnow()}: Processing {activities.count()} {object_related_activity} activities..."
+        )
+        for act in activities:
+            count += UserUpdatedOrganization.objects(
+                actor=act.actor.id,
+                related_to=act.organization,
+                created_at__gte=act.created_at - timedelta(seconds=1),
+                created_at__lte=act.created_at + timedelta(seconds=1),
+            ).delete()
+        log.info(f"{datetime.utcnow()}: Deleted {count} UserUpdatedOrganization activities")

udata/mongo/datetime_fields.py

@@ -67,6 +67,7 @@ class Datetimed(object):
         ),
         sortable=True,
         readonly=True,
+        auditable=False,
     )
 
 

udata/settings.py

@@ -574,6 +574,10 @@ class Defaults(object):
     ###########################################################################
     MAX_RESOURCES_IN_JSON_LD = 20
 
+    # Metrics settings
+    ###########################################################################
+    METRICS_API = None
+
 
 class Testing(object):
     """Sane values for testing. Should be applied as override"""