udata 14.0.3.dev1__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl

udata/core/reuse/apiv2.py

@@ -1,9 +1,6 @@
-from flask import request
-
 from udata import search
 from udata.api import API, apiv2
 from udata.core.reuse.models import Reuse
-from udata.utils import multi_to_dict
 
 from .api_fields import reuse_permissions_fields
 from .search import ReuseSearch
@@ -14,7 +11,7 @@ apiv2.inherit("Reuse (read)", Reuse.__read_fields__)
 
 ns = apiv2.namespace("reuses", "Reuse related operations")
 
-search_parser = ReuseSearch.as_request_parser()
+search_parser = ReuseSearch.as_request_parser(store_missing=False)
 
 DEFAULT_SORTING = "-created_at"
 
@@ -28,5 +25,5 @@ class ReuseSearchAPI(API):
     @apiv2.marshal_with(Reuse.__page_fields__)
     def get(self):
         """Search all reuses"""
-        search_parser.parse_args()
-        return search.query(ReuseSearch, **multi_to_dict(request.args))
+        args = search_parser.parse_args()
+        return search.query(ReuseSearch, **args)

udata/core/reuse/models.py

@@ -199,7 +199,7 @@ class Reuse(db.Datetimed, Auditable, WithMetrics, ReuseBadgeMixin, Linkable, Own
         cls.before_save.send(document)
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/reuses/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/reuses/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(

udata/core/spatial/forms.py

@@ -64,8 +64,8 @@ class GeomField(Field):
                     self.data = geojson.GeoJSON.to_instance(value)
             except Exception:
                 self.data = None
-                log.exception("Unable to parse GeoJSON")
-                raise ValueError(self.gettext("Not a valid GeoJSON"))
+                log.warning(f"Unable to parse GeoJSON: {value}")
+                raise validators.ValidationError(self.gettext("Not a valid GeoJSON"))
 
     def pre_validate(self, form):
         if self.data:

udata/core/topic/models.py

@@ -16,7 +16,10 @@ __all__ = ("Topic", "TopicElement")
 
 class TopicElement(Auditable, db.Document):
     title = field(db.StringField(required=False))
-    description = field(db.StringField(required=False))
+    description = field(
+        db.StringField(required=False),
+        markdown=True,
+    )
     tags = field(db.ListField(db.StringField()))
     extras = field(db.ExtrasField())
     element = field(db.GenericReferenceField(choices=["Dataset", "Reuse", "Dataservice"]))
@@ -63,7 +66,10 @@ class Topic(db.Datetimed, Auditable, Linkable, db.Document, Owned):
         db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
         auditable=False,
     )
-    description = field(db.StringField())
+    description = field(
+        db.StringField(),
+        markdown=True,
+    )
     tags = field(db.ListField(db.StringField()))
     color = field(db.IntField())
 

udata/core/user/api.py

@@ -8,6 +8,7 @@ from udata.core.dataset.api_fields import community_resource_fields, dataset_fie
 from udata.core.discussions.actions import discussions_for
 from udata.core.discussions.api import discussion_fields
 from udata.core.followers.api import FollowAPI
+from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
 from udata.core.storages.api import (
     image_parser,
     parse_uploaded_image,
@@ -265,11 +266,14 @@ class UserAvatarAPI(API):
         return {"image": user.avatar}
 
 
-delete_parser = api.parser()
+delete_parser = add_send_legal_notice_argument(api.parser())
 delete_parser.add_argument(
     "no_mail",
     type=bool,
-    help="Do not send a mail to notify the user of the deletion",
+    help=(
+        "Do not send the simple deletion notification email. "
+        "Note: automatically set to True when send_legal_notice=True to avoid sending duplicate emails."
+    ),
     location="args",
     default=False,
 )
@@ -321,8 +325,11 @@ class UserAPI(API):
             api.abort(
                 403, "You cannot delete yourself with this API. " + 'Use the "me" API instead.'
             )
+        send_legal_notice_on_deletion(user, args)
 
-        user.mark_as_deleted(notify=not args["no_mail"], delete_comments=args["delete_comments"])
+        # Skip simple notification if legal notice is sent (to avoid duplicate emails)
+        skip_notification = args["no_mail"] or args["send_legal_notice"]
+        user.mark_as_deleted(notify=not skip_notification, delete_comments=args["delete_comments"])
         return "", 204
 
 

udata/core/user/api_fields.py

@@ -9,7 +9,7 @@ user_ref_fields = api.inherit(
     {
         "first_name": fields.String(description="The user first name", readonly=True),
         "last_name": fields.String(description="The user larst name", readonly=True),
-        "slug": fields.String(description="The user permalink string", required=True),
+        "slug": fields.String(description="The user permalink string", readonly=True),
         "uri": fields.String(
             attribute=lambda u: u.self_api_url(),
             description="The API URI for this user",
@@ -35,8 +35,8 @@ from udata.core.organization.api_fields import member_email_with_visibility_chec
 user_fields = api.model(
     "User",
     {
-        "id": fields.String(description="The user identifier", required=True),
-        "slug": fields.String(description="The user permalink string", required=True),
+        "id": fields.String(description="The user identifier", readonly=True),
+        "slug": fields.String(description="The user permalink string", readonly=True),
         "first_name": fields.String(description="The user first name", required=True),
         "last_name": fields.String(description="The user last name", required=True),
         "email": fields.Raw(

udata/core/user/models.py

@@ -12,17 +12,18 @@ from flask_security import MongoEngineUserDatastore, RoleMixin, UserMixin
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.core import storages
 from udata.core.discussions.models import Discussion
 from udata.core.linkable import Linkable
 from udata.core.storages import avatars, default_image_basename
 from udata.frontend.markdown import mdstrip
+from udata.i18n import lazy_gettext as _
 from udata.models import Follow, WithMetrics, db
 from udata.uris import cdata_url
 
 from . import mails
-from .constants import AVATAR_SIZES
+from .constants import AVATAR_SIZES, BIGGEST_AVATAR_SIZE
 
 __all__ = ("User", "Role", "datastore")
 
@@ -44,9 +45,12 @@ class UserSettings(db.EmbeddedDocument):
     prefered_language = db.StringField()
 
 
+@generate_fields()
 class User(WithMetrics, UserMixin, Linkable, db.Document):
     slug = field(
-        db.SlugField(max_length=255, required=True, populate_from="fullname"), auditable=False
+        db.SlugField(max_length=255, required=True, populate_from="fullname"),
+        auditable=False,
+        show_as_ref=True,
     )
     email = field(db.StringField(max_length=255, required=True, unique=True))
     password = field(db.StringField())
@@ -54,15 +58,22 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
     fs_uniquifier = field(db.StringField(max_length=64, unique=True, sparse=True))
     roles = field(db.ListField(db.ReferenceField(Role), default=[]))
 
-    first_name = field(db.StringField(max_length=255, required=True))
-    last_name = field(db.StringField(max_length=255, required=True))
+    first_name = field(db.StringField(max_length=255, required=True), show_as_ref=True)
+    last_name = field(db.StringField(max_length=255, required=True), show_as_ref=True)
 
     avatar_url = field(db.URLField())
     avatar = field(
-        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
+        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES),
+        show_as_ref=True,
+        thumbnail_info={
+            "size": BIGGEST_AVATAR_SIZE,
+        },
     )
     website = field(db.URLField())
-    about = field(db.StringField())
+    about = field(
+        db.StringField(),
+        markdown=True,
+    )
 
     prefered_language = field(db.StringField())
 
@@ -116,6 +127,8 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         "auto_create_index_on_save": True,
     }
 
+    verbose_name = _("account")
+
     __metrics_keys__ = [
         "datasets",
         "reuses",
@@ -142,7 +155,7 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         return self.has_role("admin")
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/users/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/users/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(
@@ -193,6 +206,14 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         """Return the number of followers of the user."""
         return self.metrics.get("followers", 0)
 
+    @field(description="Link to the API endpoint for this user", show_as_ref=True)
+    def uri(self, *args, **kwargs):
+        return self.self_api_url(*args, **kwargs)
+
+    @field(description="Link to the udata web page for this user", show_as_ref=True)
+    def page(self, *args, **kwargs):
+        return self.self_web_url(*args, **kwargs)
+
     def generate_api_key(self):
         payload = {
             "user": str(self.id),
@@ -297,6 +318,10 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
                 discussion.save()
         Follow.objects(follower=self).delete()
         Follow.objects(following=self).delete()
+        # Remove related notifications
+        from udata.features.notifications.models import Notification
+
+        Notification.objects.with_user_in_details(self).delete()
 
         from udata.models import ContactPoint
 

udata/features/notifications/api.py

@@ -1,30 +1,19 @@
-from udata.api import API, api, fields
+from udata.api import API, api
 from udata.auth import current_user
 
-from .actions import get_notifications
+from .models import Notification
 
 notifs = api.namespace("notifications", "Notifications API")
 
-notifications_fields = api.model(
-    "Notification",
-    {
-        "type": fields.String(description="The notification type", readonly=True),
-        "created_on": fields.ISODateTime(
-            description="The notification creation datetime", readonly=True
-        ),
-        "details": fields.Raw(
-            description="Key-Value details depending on notification type", readonly=True
-        ),
-    },
-)
-
 
 @notifs.route("/", endpoint="notifications")
 class NotificationsAPI(API):
     @api.secure
-    @api.doc("get_notifications")
-    @api.marshal_list_with(notifications_fields)
+    @api.doc("list_notifications")
+    @api.expect(Notification.__index_parser__)
+    @api.marshal_with(Notification.__page_fields__)
     def get(self):
         """List all current user pending notifications"""
         user = current_user._get_current_object()
-        return get_notifications(user)
+        notifications = Notification.objects(user=user)
+        return Notification.apply_pagination(Notification.apply_sort_filters(notifications))

udata/features/notifications/models.py

@@ -0,0 +1,59 @@
+from flask_restx.inputs import boolean
+from mongoengine import NULLIFY
+
+from udata.api_fields import field, generate_fields
+from udata.core.organization.notifications import MembershipRequestNotificationDetails
+from udata.core.user.api_fields import user_ref_fields
+from udata.core.user.models import User
+from udata.features.transfer.notifications import TransferRequestNotificationDetails
+from udata.models import db
+from udata.mongo.datetime_fields import Datetimed
+from udata.mongo.queryset import UDataQuerySet
+
+
+class NotificationQuerySet(UDataQuerySet):
+    def with_organization_in_details(self, organization):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_organization=organization)
+
+    def with_user_in_details(self, user):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_user=user)
+
+
+def is_handled(base_query, filter_value):
+    if filter_value is None:
+        return base_query
+    if filter_value is True:
+        return base_query.filter(handled_at__ne=None)
+    return base_query.filter(handled_at=None)
+
+
+@generate_fields()
+class Notification(Datetimed, db.Document):
+    meta = {
+        "ordering": ["-created_at"],
+        "queryset_class": NotificationQuerySet,
+    }
+
+    id = field(db.AutoUUIDField(primary_key=True))
+    handled_at = field(
+        db.DateTimeField(),
+        sortable=True,
+        auditable=False,
+        filterable={"key": "handled", "query": is_handled, "type": boolean},
+    )
+    user = field(
+        db.ReferenceField(User, reverse_delete_rule=NULLIFY),
+        nested_fields=user_ref_fields,
+        readonly=True,
+        allow_null=True,
+        auditable=False,
+        filterable={},
+    )
+    details = field(
+        db.GenericEmbeddedDocumentField(
+            choices=(MembershipRequestNotificationDetails, TransferRequestNotificationDetails)
+        ),
+        generic=True,
+    )

udata/features/notifications/tasks.py

@@ -0,0 +1,25 @@
+import logging
+from datetime import datetime, timedelta
+
+from flask import current_app
+
+from udata.features.notifications.models import Notification
+from udata.tasks import job
+
+log = logging.getLogger(__name__)
+
+
+@job("delete-expired-notifications")
+def delete_expired_notifications(self):
+    # Delete expired notifications
+    handled_at = datetime.utcnow() - timedelta(
+        days=current_app.config["DAYS_AFTER_NOTIFICATION_EXPIRED"]
+    )
+    notifications_to_delete = Notification.objects(
+        handled_at__lte=handled_at,
+    )
+    count = notifications_to_delete.count()
+    for notification in notifications_to_delete:
+        notification.delete()
+
+    log.info(f"Deleted {count} expired notifications")

udata/features/transfer/actions.py

@@ -36,6 +36,7 @@ def accept_transfer(transfer, comment=None):
     transfer.status = "accepted"
     transfer.response_comment = comment
     transfer.save()
+    Transfer.after_handle.send(transfer)
 
     subject = transfer.subject
     recipient = transfer.recipient
@@ -59,5 +60,6 @@ def refuse_transfer(transfer, comment=None):
     transfer.status = "refused"
     transfer.response_comment = comment
     transfer.save()
+    Transfer.after_handle.send(transfer)
 
     return transfer

udata/features/transfer/models.py

@@ -1,6 +1,9 @@
 import logging
 from datetime import datetime
 
+from blinker import Signal
+from mongoengine.signals import post_save
+
 from udata.i18n import lazy_gettext as _
 from udata.mongo import db
 
@@ -30,6 +33,9 @@ class Transfer(db.Document):
     responder = db.ReferenceField("User")
     response_comment = db.StringField()
 
+    on_create = Signal()
+    after_handle = Signal()
+
     meta = {
         "indexes": [
             "owner",
@@ -38,3 +44,14 @@ class Transfer(db.Document):
             "status",
         ]
     }
+
+    @classmethod
+    def post_save(cls, sender, document, **kwargs):
+        """Handle post save signal for Transfer documents."""
+        # Only trigger on_create signal on creation, not on every save
+        if kwargs.get("created"):
+            cls.on_create.send(document)
+
+
+# Connect the post_save signal
+post_save.connect(Transfer.post_save, sender=Transfer)

udata/features/transfer/notifications.py

@@ -1,11 +1,107 @@
 import logging
+from datetime import datetime
 
+from udata.api_fields import field, generate_fields
+from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.models import Dataset
+from udata.core.organization.models import Organization
+from udata.core.reuse.models import Reuse
+from udata.core.user.models import User
 from udata.features.notifications.actions import notifier
 from udata.models import Transfer
+from udata.mongo import db
 
 log = logging.getLogger(__name__)
 
 
+@generate_fields()
+class TransferRequestNotificationDetails(db.EmbeddedDocument):
+    transfer_owner = field(
+        db.GenericReferenceField(choices=(User, Organization), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    transfer_recipient = field(
+        db.GenericReferenceField(choices=(User, Organization), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    transfer_subject = field(
+        db.GenericReferenceField(choices=(Dataset, Dataservice, Reuse), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+
+
+@Transfer.on_create.connect
+def on_transfer_created(transfer, **kwargs):
+    """Create notification when a new transfer request is created"""
+
+    from udata.features.notifications.models import Notification
+
+    recipient = transfer.recipient
+    owner = transfer.owner
+    users = []
+
+    if isinstance(recipient, User):
+        users = [recipient]
+    elif isinstance(recipient, Organization):
+        users = [member.user for member in recipient.members if member.role == "admin"]
+
+    for user in users:
+        try:
+            # we don't want notifications for the same transfer, if the previous one is stil no handled
+            existing = Notification.objects(
+                user=user,
+                details__transfer_recipient=recipient,
+                details__transfer_owner=owner,
+                details__transfer_subject=transfer.subject,
+                handled_at=None,
+            ).first()
+
+            if not existing:
+                notification = Notification(
+                    user=user,
+                    details=TransferRequestNotificationDetails(
+                        transfer_owner=owner,
+                        transfer_recipient=recipient,
+                        transfer_subject=transfer.subject,
+                    ),
+                )
+                notification.created_at = transfer.created
+                notification.save()
+        except Exception as e:
+            log.error(
+                f"Error creating notification for admin user {user.id} "
+                f"and recipient {recipient.id}: {e}"
+            )
+
+
+@Transfer.after_handle.connect
+def on_handle_transfer(transfer, **kwargs):
+    """Update handled_at timestamp on related notifications when a transfer is handled"""
+    from udata.features.notifications.models import Notification
+
+    # Find all non handled notifications related to this transfer
+    notifications = Notification.objects(
+        details__transfer_subject=transfer.subject,
+        details__transfer_owner=transfer.owner,
+        details__transfer_recipient=transfer.recipient,
+        handled_at=None,
+    )
+
+    # Update handled_at for all matching notifications
+    for notification in notifications:
+        notification.handled_at = datetime.utcnow()
+        notification.save()
+
+
 @notifier("transfer_request")
 def transfer_request_notifications(user):
     """Notify user about pending transfer requests"""

udata/flask_mongoengine/engine.py

@@ -7,7 +7,6 @@ from mongoengine.errors import DoesNotExist
 from mongoengine.queryset import QuerySet
 
 from .connection import create_connections
-from .json import override_json_encoder
 from .pagination import ListFieldPagination, Pagination
 from .wtf import WtfBaseField
 
@@ -108,9 +107,6 @@ class MongoEngine(object):
 
         app.extensions = getattr(app, "extensions", {})
 
-        # Make documents JSON serializable
-        override_json_encoder(app)
-
         if "mongoengine" not in app.extensions:
             app.extensions["mongoengine"] = {}
 

udata/flask_mongoengine/pagination.py

@@ -6,7 +6,7 @@ from mongoengine.queryset import QuerySet
 
 class Pagination(object):
     def __init__(self, iterable, page, per_page):
-        if page < 1:
+        if page < 1 or per_page < 1:
             abort(404)
 
         self.iterable = iterable

udata/frontend/markdown.py

@@ -7,8 +7,9 @@ import html2text
 import mistune
 from bleach.css_sanitizer import CSSSanitizer
 from bleach.linkifier import LinkifyFilter
-from flask import Markup, current_app, request
+from flask import current_app, request
 from jinja2.filters import do_striptags, do_truncate
+from markupsafe import Markup
 from werkzeug.local import LocalProxy
 
 from udata.i18n import _

udata/harvest/actions.py

@@ -317,3 +317,23 @@ def attach(domain, filename):
             count += 1
 
     return AttachResult(count, errors)
+
+
+def detach(dataset: Dataset):
+    """Detach a dataset from its harvest source
+
+    The dataset will be cleaned from harvested information
+    and will no longer be updated or archived by harvesting.
+    """
+    dataset.harvest = None
+    for resource in dataset.resources:
+        resource.harvest = None
+    dataset.save()
+
+
+def detach_all_from_source(source: HarvestSource):
+    """Detach all datasets linked to a harvest source"""
+    datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
+    for dataset in datasets:
+        detach(dataset)
+    return len(datasets)