udata 14.0.0__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

udata/core/dataset/tasks.py

@@ -1,4 +1,5 @@
 import collections
+import gzip
 import os
 from datetime import date, datetime
 from tempfile import NamedTemporaryFile
@@ -15,6 +16,7 @@ from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.constants import INSPIRE
 from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
 from udata.core.organization.models import Organization
+from udata.core.pages.models import Page
 from udata.harvest.models import HarvestJob
 from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
 from udata.storage.s3 import store_bytes
@@ -54,6 +56,12 @@ def purge_datasets(self):
             dataservice.update(datasets=datasets)
         # Remove HarvestItem references
         HarvestJob.objects(items__dataset=dataset).update(set__items__S__dataset=None)
+        # Remove datasets in pages (mongoengine doesn't support updating a field in a generic embed)
+        Page._get_collection().update_many(
+            {"blocs.datasets": dataset.id},
+            {"$pull": {"blocs.$[b].datasets": dataset.id}},
+            array_filters=[{"b.datasets": dataset.id}],
+        )
         # Remove associated Transfers
         Transfer.objects(subject=dataset).delete()
         # Remove each dataset's resource's file
@@ -87,8 +95,7 @@ def get_queryset(model_cls):
     for attr in attrs:
         if getattr(model_cls, attr, None):
             params[attr] = False
-    # no_cache to avoid eating up too much RAM
-    return model_cls.objects.filter(**params).no_cache()
+    return model_cls.objects.filter(**params)
 
 
 def get_resource_for_csv_export_model(model, dataset):
@@ -166,7 +173,12 @@ def export_csv_for_model(model, dataset, replace: bool = False):
             dataset.save()
         # remove previous catalog if exists and replace is True
         if replace and fs_filename_to_remove:
-            storages.resources.delete(fs_filename_to_remove)
+            try:
+                storages.resources.delete(fs_filename_to_remove)
+            except FileNotFoundError:
+                log.error(
+                    f"File not found while deleting resource #{resource.id} ({fs_filename_to_remove}) in export_csv_for_model cleanup"
+                )
         return resource
     finally:
         csvfile.close()
@@ -210,8 +222,8 @@ def export_csv(self, model=None):
             with storages.resources.open(resource.fs_filename, "rb") as f:
                 store_bytes(
                     bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
-                    filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}",
-                    bytes=f.read(),
+                    filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}.gz",
+                    bytes=gzip.compress(f.read()),
                 )
 
 

udata/core/discussions/models.py

@@ -14,6 +14,7 @@ log = logging.getLogger(__name__)
 
 
 class Message(SpamMixin, db.EmbeddedDocument):
+    id = db.AutoUUIDField()
     content = db.StringField(required=True)
     posted_on = db.DateTimeField(default=datetime.utcnow, required=True)
     posted_by = db.ReferenceField("User")

udata/core/organization/api.py

@@ -383,12 +383,13 @@ class MembershipRequestAPI(API):
 
         form = api.validate(MembershipRequestForm, membership_request)
 
-        if not membership_request:
+        if membership_request:
+            form.populate_obj(membership_request)
+            org.save()
+        else:
             membership_request = MembershipRequest()
-            org.requests.append(membership_request)
-
-        form.populate_obj(membership_request)
-        org.save()
+            form.populate_obj(membership_request)
+            org.add_membership_request(membership_request)
 
         notify_membership_request.delay(str(org.id), str(membership_request.id))
 
@@ -424,6 +425,7 @@ class MembershipAcceptAPI(MembershipAPI):
         org.members.append(member)
         org.count_members()
         org.save()
+        MembershipRequest.after_handle.send(membership_request, org=org)
 
         notify_membership_response.delay(str(org.id), str(membership_request.id))
 
@@ -446,6 +448,7 @@ class MembershipRefuseAPI(MembershipAPI):
         membership_request.refusal_comment = form.comment.data
 
         org.save()
+        MembershipRequest.after_handle.send(membership_request, org=org)
 
         notify_membership_response.delay(str(org.id), str(membership_request.id))
 

udata/core/organization/mails.py

@@ -16,7 +16,7 @@ def new_membership_request(org: Organization, request: MembershipRequest) -> Mai
                 )
             ),
             LabelledContent(_("Reason for the request:"), request.comment),
-            MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members/")),
+            MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members")),
         ],
     )
 

udata/core/organization/models.py

@@ -81,6 +81,9 @@ class MembershipRequest(db.EmbeddedDocument):
     comment = db.StringField()
     refusal_comment = db.StringField()
 
+    after_create = Signal()
+    after_handle = Signal()
+
     @property
     def status_label(self):
         return MEMBERSHIP_STATUS[self.status]
@@ -198,7 +201,7 @@ class Organization(
         cls.before_save.send(document)
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/organizations/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/organizations/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(
@@ -304,6 +307,11 @@ class Organization(
     def views_count(self):
         return self.metrics.get("views", 0)
 
+    def add_membership_request(self, membership_request):
+        self.requests.append(membership_request)
+        self.save()
+        MembershipRequest.after_create.send(membership_request, org=self)
+
     def count_members(self):
         self.metrics["members"] = len(self.members)
         self.save(signal_kwargs={"ignores": ["post_save"]})

udata/core/organization/notifications.py

@@ -1,10 +1,94 @@
 import logging
 
+from udata.api_fields import field, generate_fields
+from udata.core.organization.api_fields import org_ref_fields
+from udata.core.organization.models import MembershipRequest, Organization
+from udata.core.user.api_fields import user_ref_fields
+from udata.core.user.models import User
 from udata.features.notifications.actions import notifier
+from udata.models import db
 
 log = logging.getLogger(__name__)
 
 
+@generate_fields()
+class MembershipRequestNotificationDetails(db.EmbeddedDocument):
+    request_organization = field(
+        db.ReferenceField(Organization),
+        readonly=True,
+        nested_fields=org_ref_fields,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    request_user = field(
+        db.ReferenceField(User),
+        nested_fields=user_ref_fields,
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+
+
+@MembershipRequest.after_create.connect
+def on_new_membership_request(request: MembershipRequest, **kwargs):
+    from udata.features.notifications.models import Notification
+
+    """Create notification when a new membership request is created"""
+    organization = kwargs.get("org")
+
+    if organization is None:
+        return
+
+    # Get all admin users for the organization
+    admin_users = [member.user for member in organization.members if member.role == "admin"]
+
+    # For each pending request, check if a notification already exists
+    for admin_user in admin_users:
+        try:
+            # Check if notification already exists
+            existing = Notification.objects(
+                user=admin_user,
+                details__request_organization=organization,
+                details__request_user=request.user,
+            ).first()
+
+            if not existing:
+                notification = Notification(
+                    user=admin_user,
+                    details=MembershipRequestNotificationDetails(
+                        request_organization=organization, request_user=request.user
+                    ),
+                )
+                notification.created_at = request.created
+                notification.save()
+        except Exception as e:
+            log.error(
+                f"Error creating notification for user {admin_user.id} "
+                f"and organization {organization.id}: {e}"
+            )
+
+
+@MembershipRequest.after_handle.connect
+def on_handle_membership_request(request: MembershipRequest, **kwargs):
+    from udata.features.notifications.models import Notification
+
+    organization = kwargs.get("org")
+
+    if organization is None:
+        return
+
+    notifications = Notification.objects(
+        details__request_organization=organization,
+        details__request_user=request.user,
+    )
+
+    for notification in notifications:
+        notification.handled_at = request.handled_on
+        notification.save()
+
+
 @notifier("membership_request")
 def membership_request_notifications(user):
     """Notify user about pending membership requests"""

udata/core/organization/permissions.py

@@ -2,7 +2,7 @@ from collections import namedtuple
 from functools import partial
 
 from udata.auth import Permission, current_user, identity_loaded
-from udata.models import Organization
+from udata.core.organization.models import Organization
 from udata.utils import get_by
 
 OrganizationNeed = namedtuple("organization", ("role", "value"))

udata/core/organization/tasks.py

@@ -1,5 +1,6 @@
 from udata.core import storages
 from udata.core.badges.tasks import notify_new_badge
+from udata.features.notifications.models import Notification
 from udata.models import Activity, ContactPoint, Dataset, Follow, Transfer
 from udata.search import reindex
 from udata.tasks import get_logger, job, task
@@ -25,6 +26,8 @@ def purge_organizations(self):
         Transfer.objects(owner=organization).delete()
         # Remove related contact points
         ContactPoint.objects(organization=organization).delete()
+        # Remove related notifications
+        Notification.objects.with_organization_in_details(organization).delete()
         # Store datasets for later reindexation
         d_ids = [d.id for d in Dataset.objects(organization=organization)]
         # Remove organization's logo in all sizes

udata/core/pages/tests/test_api.py

@@ -1,7 +1,9 @@
 from flask import url_for
 
+from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
 from udata.core.pages.models import Page
+from udata.core.user.factories import AdminFactory
 from udata.tests.api import APITestCase
 
 
@@ -71,3 +73,33 @@ class PageAPITest(APITestCase):
         self.assertEqual("more information", response.json["blocs"][0]["subtitle"])
         self.assertEqual(len(response.json["blocs"][0]["datasets"]), 1)
         self.assertEqual(str(datasets[2].id), response.json["blocs"][0]["datasets"][0]["id"])
+
+    def test_page_with_deleted_dataset(self):
+        self.login(AdminFactory())
+        datasets = DatasetFactory.create_batch(3)
+
+        response = self.post(
+            url_for("api.pages"),
+            {
+                "blocs": [
+                    {
+                        "class": "DatasetsListBloc",
+                        "title": "My awesome title",
+                        "datasets": [str(d.id) for d in datasets],
+                    }
+                ],
+            },
+        )
+        self.assert201(response)
+        page_id = response.json["id"]
+
+        response = self.delete(url_for("api.dataset", dataset=datasets[0].id))
+        self.assert204(response)
+
+        response = self.get(url_for("api.page", page=page_id))
+        self.assert200(response)
+
+        tasks.purge_datasets()
+
+        response = self.get(url_for("api.page", page=page_id))
+        self.assert200(response)

udata/core/post/api.py

@@ -2,70 +2,27 @@ from datetime import datetime
 
 from feedgenerator.django.utils.feedgenerator import Atom1Feed
 from flask import make_response, request
+from flask_login import current_user
 
-from udata.api import API, api, fields
+from udata.api import API, api
+from udata.api_fields import patch, patch_and_save
 from udata.auth import Permission as AdminPermission
 from udata.auth import admin_permission
-from udata.core.dataset.api_fields import dataset_fields
-from udata.core.reuse.models import Reuse
 from udata.core.storages.api import (
     image_parser,
     parse_uploaded_image,
     uploaded_image_fields,
 )
-from udata.core.user.api_fields import user_ref_fields
 from udata.frontend.markdown import md
 from udata.i18n import gettext as _
 
-from .forms import PostForm
 from .models import Post
 
 DEFAULT_SORTING = "-published"
 
 ns = api.namespace("posts", "Posts related operations")
 
-post_fields = api.model(
-    "Post",
-    {
-        "id": fields.String(description="The post identifier"),
-        "name": fields.String(description="The post name", required=True),
-        "slug": fields.String(description="The post permalink string", readonly=True),
-        "headline": fields.String(description="The post headline", required=True),
-        "content": fields.Markdown(description="The post content in Markdown", required=True),
-        "image": fields.ImageField(description="The post image", readonly=True),
-        "credit_to": fields.String(description="An optional credit line (associated to the image)"),
-        "credit_url": fields.String(description="An optional link associated to the credits"),
-        "tags": fields.List(fields.String, description="Some keywords to help in search"),
-        "datasets": fields.List(fields.Nested(dataset_fields), description="The post datasets"),
-        "reuses": fields.List(fields.Nested(Reuse.__read_fields__), description="The post reuses"),
-        "owner": fields.Nested(
-            user_ref_fields, description="The owner user", readonly=True, allow_null=True
-        ),
-        "created_at": fields.ISODateTime(description="The post creation date", readonly=True),
-        "last_modified": fields.ISODateTime(
-            description="The post last modification date", readonly=True
-        ),
-        "published": fields.ISODateTime(description="The post publication date", readonly=True),
-        "body_type": fields.String(description="HTML or markdown body type", default="markdown"),
-        "uri": fields.String(
-            attribute=lambda p: p.self_api_url(),
-            description="The API URI for this post",
-            readonly=True,
-        ),
-        "page": fields.String(
-            attribute=lambda p: p.self_web_url(),
-            description="The post web page URL",
-            readonly=True,
-        ),
-    },
-    mask="*,datasets{id,title,acronym,uri,page},reuses{id,title,image,image_thumbnail,uri,page}",
-)
-
-post_page_fields = api.model("PostPage", fields.pager(post_fields))
-
-parser = api.page_parser()
-
-parser.add_argument("sort", type=str, location="args", help="The sorting attribute")
+parser = Post.__index_parser__
 parser.add_argument(
     "with_drafts",
     type=bool,
@@ -73,16 +30,13 @@ parser.add_argument(
     location="args",
     help="`True` also returns the unpublished posts (only for super-admins)",
 )
-parser.add_argument(
-    "q", type=str, location="args", help="query string to search through resources titles"
-)
 
 
 @ns.route("/", endpoint="posts")
 class PostsAPI(API):
     @api.doc("list_posts")
     @api.expect(parser)
-    @api.marshal_with(post_page_fields)
+    @api.marshal_with(Post.__page_fields__)
     def get(self):
         """List all posts"""
         args = parser.parse_args()
@@ -92,22 +46,23 @@ class PostsAPI(API):
         if not (AdminPermission().can() and args["with_drafts"]):
             posts = posts.published()
 
-        if args["q"]:
-            phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
-            posts = posts.search_text(phrase_query)
-
-        sort = args["sort"] or ("$text_score" if args["q"] else None) or DEFAULT_SORTING
-        return posts.order_by(sort).paginate(args["page"], args["page_size"])
+        # The search is already handled by apply_sort_filters if searchable=True
+        return Post.apply_pagination(Post.apply_sort_filters(posts))
 
     @api.doc("create_post")
     @api.secure(admin_permission)
-    @api.expect(post_fields)
-    @api.marshal_with(post_fields)
+    @api.expect(Post.__write_fields__)
+    @api.marshal_with(Post.__read_fields__)
     @api.response(400, "Validation error")
     def post(self):
         """Create a post"""
-        form = api.validate(PostForm)
-        return form.save(), 201
+        post = patch(Post(), request)
+
+        if not post.owner:
+            post.owner = current_user._get_current_object()
+
+        post.save()
+        return post, 201
 
 
 @ns.route("/recent.atom", endpoint="recent_posts_atom_feed")
@@ -143,20 +98,20 @@ class PostsAtomFeedAPI(API):
 @api.param("post", "The post ID or slug")
 class PostAPI(API):
     @api.doc("get_post")
-    @api.marshal_with(post_fields)
+    @api.marshal_with(Post.__read_fields__)
     def get(self, post):
         """Get a given post"""
         return post
 
     @api.doc("update_post")
     @api.secure(admin_permission)
-    @api.expect(post_fields)
-    @api.marshal_with(post_fields)
+    @api.expect(Post.__write_fields__)
+    @api.marshal_with(Post.__read_fields__)
     @api.response(400, "Validation error")
     def put(self, post):
         """Update a given post"""
-        form = api.validate(PostForm, post)
-        return form.save()
+        post = patch_and_save(post, request)
+        return post
 
     @api.secure(admin_permission)
     @api.doc("delete_post")
@@ -171,7 +126,7 @@ class PostAPI(API):
 class PublishPostAPI(API):
     @api.secure(admin_permission)
     @api.doc("publish_post")
-    @api.marshal_with(post_fields)
+    @api.marshal_with(Post.__read_fields__)
     def post(self, post):
         """Publish an existing post"""
         post.modify(published=datetime.utcnow())
@@ -179,9 +134,9 @@ class PublishPostAPI(API):
 
     @api.secure(admin_permission)
     @api.doc("unpublish_post")
-    @api.marshal_with(post_fields)
+    @api.marshal_with(Post.__read_fields__)
     def delete(self, post):
-        """Publish an existing post"""
+        """Unpublish an existing post"""
         post.modify(published=None)
         return post
 

udata/core/post/models.py

@@ -1,5 +1,7 @@
 from flask import url_for
 
+from udata.api_fields import field, generate_fields
+from udata.core.dataset.api_fields import dataset_fields
 from udata.core.linkable import Linkable
 from udata.core.storages import default_image_basename, images
 from udata.i18n import lazy_gettext as _
@@ -16,27 +18,85 @@ class PostQuerySet(db.BaseQuerySet):
         return self(published__ne=None).order_by("-published")
 
 
+@generate_fields(
+    searchable=True,
+    additional_sorts=[
+        {"key": "created_at", "value": "created_at"},
+        {"key": "modified", "value": "last_modified"},
+    ],
+    default_sort="-published",
+)
 class Post(db.Datetimed, Linkable, db.Document):
-    name = db.StringField(max_length=255, required=True)
-    slug = db.SlugField(
-        max_length=255, required=True, populate_from="name", update=True, follow=True
+    name = field(
+        db.StringField(max_length=255, required=True),
+        sortable=True,
+        show_as_ref=True,
+    )
+    slug = field(
+        db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
+        readonly=True,
+    )
+    headline = field(
+        db.StringField(),
+        sortable=True,
+    )
+    content = field(
+        db.StringField(required=True),
+        markdown=True,
+    )
+    image_url = field(
+        db.StringField(),
+    )
+    image = field(
+        db.ImageField(fs=images, basename=default_image_basename, thumbnails=IMAGE_SIZES),
+        readonly=True,
+        thumbnail_info={"size": 100},
     )
-    headline = db.StringField()
-    content = db.StringField(required=True)
-    image_url = db.StringField()
-    image = db.ImageField(fs=images, basename=default_image_basename, thumbnails=IMAGE_SIZES)
 
-    credit_to = db.StringField()
-    credit_url = db.URLField()
+    credit_to = field(
+        db.StringField(),
+        description="An optional credit line (associated to the image)",
+    )
+    credit_url = field(
+        db.URLField(),
+        description="An optional link associated to the credits",
+    )
 
-    tags = db.ListField(db.StringField())
-    datasets = db.ListField(db.ReferenceField("Dataset", reverse_delete_rule=db.PULL))
-    reuses = db.ListField(db.ReferenceField("Reuse", reverse_delete_rule=db.PULL))
+    tags = field(
+        db.ListField(db.StringField()),
+        description="Some keywords to help in search",
+    )
+    datasets = field(
+        db.ListField(
+            field(
+                db.ReferenceField("Dataset", reverse_delete_rule=db.PULL),
+                nested_fields=dataset_fields,
+            )
+        ),
+        description="The post datasets",
+    )
+    reuses = field(
+        db.ListField(db.ReferenceField("Reuse", reverse_delete_rule=db.PULL)),
+        description="The post reuses",
+    )
 
-    owner = db.ReferenceField("User")
-    published = db.DateTimeField()
+    owner = field(
+        db.ReferenceField("User"),
+        readonly=True,
+        allow_null=True,
+        description="The owner user",
+    )
+    published = field(
+        db.DateTimeField(),
+        readonly=True,
+        sortable=True,
+        description="The post publication date",
+    )
 
-    body_type = db.StringField(choices=list(BODY_TYPES), default="markdown", required=False)
+    body_type = field(
+        db.StringField(choices=list(BODY_TYPES), default="markdown", required=False),
+        description="HTML or markdown body type",
+    )
 
     meta = {
         "ordering": ["-created_at"],
@@ -58,13 +118,21 @@ class Post(db.Datetimed, Linkable, db.Document):
         return self.name or ""
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/posts/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/posts/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(
             "api.post", post=self._link_id(**kwargs), **self._self_api_url_kwargs(**kwargs)
         )
 
+    @field(description="The API URI for this post")
+    def uri(self):
+        return self.self_api_url()
+
+    @field(description="The post web page URL")
+    def page(self):
+        return self.self_web_url()
+
     def count_discussions(self):
         # There are no metrics on Post to store discussions count
         pass

udata/core/post/tests/test_api.py

@@ -4,7 +4,7 @@ from udata.core.dataset.factories import DatasetFactory
 from udata.core.post.factories import PostFactory
 from udata.core.post.models import Post
 from udata.core.reuse.factories import ReuseFactory
-from udata.core.user.factories import AdminFactory
+from udata.core.user.factories import AdminFactory, UserFactory
 from udata.tests.api import APITestCase
 from udata.tests.helpers import assert200, assert201, assert204
 
@@ -136,3 +136,26 @@ class PostsAPITest(APITestCase):
 
         post.reload()
         assert post.published is None
+
+    def test_post_api_create_with_empty_credit_url(self):
+        """It should create a post with an empty credit_url (converted to None)"""
+        data = PostFactory.as_dict()
+        data["datasets"] = [str(d.id) for d in data["datasets"]]
+        data["reuses"] = [str(r.id) for r in data["reuses"]]
+        data["credit_url"] = ""
+        self.login(AdminFactory())
+        response = self.post(url_for("api.posts"), data)
+        assert201(response)
+        assert Post.objects.count() == 1
+        post = Post.objects.first()
+        assert post.credit_url is None
+
+    def test_post_api_list_with_drafts_non_admin(self):
+        """Non-admin users should not see drafts even with with_drafts=True"""
+        PostFactory.create_batch(3)
+        PostFactory(published=None)
+
+        self.login(UserFactory())
+        response = self.get(url_for("api.posts", with_drafts=True))
+        assert200(response)
+        assert len(response.json["data"]) == 3

udata/core/reports/api.py

@@ -42,6 +42,24 @@ class ReportAPI(API):
     def get(self, report):
         return report
 
+    @api.doc("update_report", responses={400: "Validation error"})
+    @api.secure(admin_permission)
+    @api.expect(Report.__write_fields__)
+    @api.marshal_with(Report.__read_fields__, code=200)
+    def patch(self, report):
+        dismiss_has_changed = (
+            "dismissed_at" in request.json and request.json["dismissed_at"] != report.dismissed_at
+        )
+
+        report = patch(report, request)
+        if dismiss_has_changed:
+            report.dismissed_by = (
+                current_user._get_current_object() if report.dismissed_at else None
+            )
+
+        report.save()
+        return report, 200
+
 
 @ns.route("/reasons/", endpoint="reports_reasons")
 class ReportsReasonsAPI(API):