udata 14.5.1.dev9__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl

udata/core/post/models.py

@@ -3,12 +3,14 @@ from flask import url_for
 from udata.api_fields import field, generate_fields
 from udata.core.dataset.api_fields import dataset_fields
 from udata.core.linkable import Linkable
+from udata.core.pages.models import Page
 from udata.core.storages import default_image_basename, images
+from udata.core.user.api_fields import user_ref_fields
 from udata.i18n import lazy_gettext as _
 from udata.mongo import db
 from udata.uris import cdata_url
 
-from .constants import BODY_TYPES, IMAGE_SIZES
+from .constants import BODY_TYPES, IMAGE_SIZES, POST_KINDS
 
 __all__ = ("Post",)
 
@@ -41,9 +43,15 @@ class Post(db.Datetimed, Linkable, db.Document):
         sortable=True,
     )
     content = field(
-        db.StringField(required=True),
+        db.StringField(),
         markdown=True,
     )
+    content_as_page = field(
+        db.ReferenceField("Page", reverse_delete_rule=db.DENY),
+        nested_fields=Page.__read_fields__,
+        allow_null=True,
+        description="Reference to a Page when body_type is 'blocs'",
+    )
     image_url = field(
         db.StringField(),
     )
@@ -82,6 +90,7 @@ class Post(db.Datetimed, Linkable, db.Document):
 
     owner = field(
         db.ReferenceField("User"),
+        nested_fields=user_ref_fields,
         readonly=True,
         allow_null=True,
         description="The owner user",
@@ -95,7 +104,12 @@ class Post(db.Datetimed, Linkable, db.Document):
 
     body_type = field(
         db.StringField(choices=list(BODY_TYPES), default="markdown", required=False),
-        description="HTML or markdown body type",
+    )
+
+    kind = field(
+        db.StringField(choices=list(POST_KINDS), default="news", required=False),
+        filterable={},
+        description="Post kind (news or page)",
     )
 
     meta = {
@@ -114,6 +128,16 @@ class Post(db.Datetimed, Linkable, db.Document):
 
     verbose_name = _("post")
 
+    def clean(self):
+        if self.body_type == "blocs":
+            if not self.content_as_page:
+                raise db.ValidationError("content_as_page is required when body_type is 'blocs'")
+        else:
+            if not self.content:
+                raise db.ValidationError(
+                    "content is required when body_type is 'markdown' or 'html'"
+                )
+
     def __str__(self):
         return self.name or ""
 

udata/core/post/tests/test_api.py

@@ -1,12 +1,14 @@
 from flask import url_for
 
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.pages.factories import PageFactory
+from udata.core.pages.models import DatasetsListBloc
 from udata.core.post.factories import PostFactory
 from udata.core.post.models import Post
 from udata.core.reuse.factories import ReuseFactory
 from udata.core.user.factories import AdminFactory, UserFactory
 from udata.tests.api import APITestCase
-from udata.tests.helpers import assert200, assert201, assert204
+from udata.tests.helpers import assert200, assert201, assert204, assert400
 
 
 class PostsAPITest(APITestCase):
@@ -56,9 +58,13 @@ class PostsAPITest(APITestCase):
 
     def test_post_api_get(self):
         """It should fetch a post from the API"""
-        post = PostFactory()
+        admin = AdminFactory()
+        post = PostFactory(owner=admin)
         response = self.get(url_for("api.post", post=post))
         assert200(response)
+        owner = response.json["owner"]
+        assert isinstance(owner, dict)
+        assert owner["id"] == str(admin.id)
 
     def test_post_api_create(self):
         """It should create a post from the API"""
@@ -159,3 +165,111 @@ class PostsAPITest(APITestCase):
         response = self.get(url_for("api.posts", with_drafts=True))
         assert200(response)
         assert len(response.json["data"]) == 3
+
+    def test_post_api_create_with_blocs_body_type_and_page(self):
+        """It should create a post with body_type='blocs' when content_as_page is provided"""
+        page = PageFactory()
+        data = PostFactory.as_dict()
+        data["datasets"] = [str(d.id) for d in data["datasets"]]
+        data["reuses"] = [str(r.id) for r in data["reuses"]]
+        data["body_type"] = "blocs"
+        data["content_as_page"] = str(page.id)
+        self.login(AdminFactory())
+        response = self.post(url_for("api.posts"), data)
+        assert201(response)
+        assert Post.objects.count() == 1
+        post = Post.objects.first()
+        assert post.body_type == "blocs"
+        assert post.content_as_page.id == page.id
+
+    def test_post_api_create_with_blocs_body_type_without_page(self):
+        """It should fail to create a post with body_type='blocs' without content_as_page"""
+        data = PostFactory.as_dict()
+        data["datasets"] = [str(d.id) for d in data["datasets"]]
+        data["reuses"] = [str(r.id) for r in data["reuses"]]
+        data["body_type"] = "blocs"
+        self.login(AdminFactory())
+        response = self.post(url_for("api.posts"), data)
+        assert400(response)
+
+    def test_post_api_get_with_blocs_returns_page_blocs(self):
+        """It should return blocs from the associated page when fetching a post"""
+        datasets = DatasetFactory.create_batch(2)
+        bloc = DatasetsListBloc(title="Featured datasets", datasets=datasets)
+        page = PageFactory(blocs=[bloc])
+        post = PostFactory(body_type="blocs", content_as_page=page)
+        response = self.get(url_for("api.post", post=post))
+        assert200(response)
+        assert response.json["body_type"] == "blocs"
+        assert "content_as_page" in response.json
+        page_data = response.json["content_as_page"]
+        assert "blocs" in page_data
+        assert len(page_data["blocs"]) == 1
+        assert page_data["blocs"][0]["class"] == "DatasetsListBloc"
+        assert page_data["blocs"][0]["title"] == "Featured datasets"
+        assert len(page_data["blocs"][0]["datasets"]) == 2
+
+    def test_post_api_update_to_blocs_without_content_as_page(self):
+        """It should fail to update body_type to 'blocs' without providing content_as_page"""
+        post = PostFactory(body_type="markdown")
+        self.login(AdminFactory())
+        response = self.put(url_for("api.post", post=post), {"body_type": "blocs"})
+        assert400(response)
+
+    def test_post_api_update_to_blocs_with_content_as_page(self):
+        """It should update body_type to 'blocs' when content_as_page is provided"""
+        post = PostFactory(body_type="markdown")
+        page = PageFactory()
+        self.login(AdminFactory())
+        response = self.put(
+            url_for("api.post", post=post), {"body_type": "blocs", "content_as_page": str(page.id)}
+        )
+        assert200(response)
+        post.reload()
+        assert post.body_type == "blocs"
+        assert post.content_as_page.id == page.id
+
+    def test_post_api_update_remove_content_as_page_from_blocs_post(self):
+        """It should fail to remove content_as_page from a post with body_type='blocs'"""
+        page = PageFactory()
+        post = PostFactory(body_type="blocs", content_as_page=page)
+        self.login(AdminFactory())
+        response = self.put(url_for("api.post", post=post), {"content_as_page": None})
+        assert400(response)
+
+    def test_post_api_update_body_type_preserves_content_as_page(self):
+        """Switching from 'blocs' to 'markdown' preserves content_as_page so user can switch back"""
+        page = PageFactory()
+        post = PostFactory(body_type="blocs", content_as_page=page)
+        self.login(AdminFactory())
+        response = self.put(url_for("api.post", post=post), {"body_type": "markdown"})
+        assert200(response)
+        post.reload()
+        assert post.body_type == "markdown"
+        assert post.content_as_page.id == page.id
+
+    def test_post_api_filter_by_kind(self):
+        """It should filter posts by kind"""
+        news_post = PostFactory(kind="news")
+        page_post = PostFactory(kind="page")
+
+        response = self.get(url_for("api.posts", kind="news"))
+        assert200(response)
+        assert len(response.json["data"]) == 1
+        assert response.json["data"][0]["id"] == str(news_post.id)
+
+        response = self.get(url_for("api.posts", kind="page"))
+        assert200(response)
+        assert len(response.json["data"]) == 1
+        assert response.json["data"][0]["id"] == str(page_post.id)
+
+    def test_rss_feed_only_returns_news(self):
+        """RSS feed should only return posts with kind=news"""
+        news_post = PostFactory(kind="news")
+        page_post = PostFactory(kind="page")
+
+        response = self.get(url_for("api.recent_posts_atom_feed"))
+        assert200(response)
+        content = response.data.decode("utf-8")
+        assert news_post.name in content
+        assert page_post.name not in content

udata/core/post/tests/test_models.py

@@ -0,0 +1,24 @@
+import mongoengine
+import pytest
+
+from udata.core.pages.factories import PageFactory
+from udata.core.pages.models import Page
+from udata.core.post.factories import PostFactory
+from udata.tests.api import PytestOnlyDBTestCase
+
+
+class PostTest(PytestOnlyDBTestCase):
+    def test_page_deletion_raises_if_reference_still_exists(self):
+        page = PageFactory()
+        post = PostFactory(body_type="blocs", content_as_page=page)
+
+        assert Page.objects().count() == 1
+
+        with pytest.raises(mongoengine.errors.OperationError):
+            page.delete()
+
+        # Delete the post referencing the page before being able to delete the page itself
+        post.delete()
+        page.delete()
+
+        assert Page.objects().count() == 0

udata/core/reuse/apiv2.py

@@ -11,7 +11,7 @@ apiv2.inherit("Reuse (read)", Reuse.__read_fields__)
 
 ns = apiv2.namespace("reuses", "Reuse related operations")
 
-search_parser = ReuseSearch.as_request_parser()
+search_parser = ReuseSearch.as_request_parser(store_missing=False)
 
 DEFAULT_SORTING = "-created_at"
 

udata/core/user/models.py

@@ -12,7 +12,7 @@ from flask_security import MongoEngineUserDatastore, RoleMixin, UserMixin
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.core import storages
 from udata.core.discussions.models import Discussion
 from udata.core.linkable import Linkable
@@ -23,7 +23,7 @@ from udata.models import Follow, WithMetrics, db
 from udata.uris import cdata_url
 
 from . import mails
-from .constants import AVATAR_SIZES
+from .constants import AVATAR_SIZES, BIGGEST_AVATAR_SIZE
 
 __all__ = ("User", "Role", "datastore")
 
@@ -45,9 +45,12 @@ class UserSettings(db.EmbeddedDocument):
     prefered_language = db.StringField()
 
 
+@generate_fields()
 class User(WithMetrics, UserMixin, Linkable, db.Document):
     slug = field(
-        db.SlugField(max_length=255, required=True, populate_from="fullname"), auditable=False
+        db.SlugField(max_length=255, required=True, populate_from="fullname"),
+        auditable=False,
+        show_as_ref=True,
     )
     email = field(db.StringField(max_length=255, required=True, unique=True))
     password = field(db.StringField())
@@ -55,12 +58,16 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
     fs_uniquifier = field(db.StringField(max_length=64, unique=True, sparse=True))
     roles = field(db.ListField(db.ReferenceField(Role), default=[]))
 
-    first_name = field(db.StringField(max_length=255, required=True))
-    last_name = field(db.StringField(max_length=255, required=True))
+    first_name = field(db.StringField(max_length=255, required=True), show_as_ref=True)
+    last_name = field(db.StringField(max_length=255, required=True), show_as_ref=True)
 
     avatar_url = field(db.URLField())
     avatar = field(
-        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
+        db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES),
+        show_as_ref=True,
+        thumbnail_info={
+            "size": BIGGEST_AVATAR_SIZE,
+        },
     )
     website = field(db.URLField())
     about = field(
@@ -199,6 +206,14 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         """Return the number of followers of the user."""
         return self.metrics.get("followers", 0)
 
+    @field(description="Link to the API endpoint for this user", show_as_ref=True)
+    def uri(self, *args, **kwargs):
+        return self.self_api_url(*args, **kwargs)
+
+    @field(description="Link to the udata web page for this user", show_as_ref=True)
+    def page(self, *args, **kwargs):
+        return self.self_web_url(*args, **kwargs)
+
     def generate_api_key(self):
         payload = {
             "user": str(self.id),

udata/features/notifications/models.py

@@ -5,6 +5,7 @@ from udata.api_fields import field, generate_fields
 from udata.core.organization.notifications import MembershipRequestNotificationDetails
 from udata.core.user.api_fields import user_ref_fields
 from udata.core.user.models import User
+from udata.features.transfer.notifications import TransferRequestNotificationDetails
 from udata.models import db
 from udata.mongo.datetime_fields import Datetimed
 from udata.mongo.queryset import UDataQuerySet
@@ -51,6 +52,8 @@ class Notification(Datetimed, db.Document):
         filterable={},
     )
     details = field(
-        db.GenericEmbeddedDocumentField(choices=(MembershipRequestNotificationDetails,)),
+        db.GenericEmbeddedDocumentField(
+            choices=(MembershipRequestNotificationDetails, TransferRequestNotificationDetails)
+        ),
         generic=True,
     )

udata/features/transfer/actions.py

@@ -36,6 +36,7 @@ def accept_transfer(transfer, comment=None):
     transfer.status = "accepted"
     transfer.response_comment = comment
     transfer.save()
+    Transfer.after_handle.send(transfer)
 
     subject = transfer.subject
     recipient = transfer.recipient
@@ -59,5 +60,6 @@ def refuse_transfer(transfer, comment=None):
     transfer.status = "refused"
     transfer.response_comment = comment
     transfer.save()
+    Transfer.after_handle.send(transfer)
 
     return transfer

udata/features/transfer/models.py

@@ -1,6 +1,9 @@
 import logging
 from datetime import datetime
 
+from blinker import Signal
+from mongoengine.signals import post_save
+
 from udata.i18n import lazy_gettext as _
 from udata.mongo import db
 
@@ -30,6 +33,9 @@ class Transfer(db.Document):
     responder = db.ReferenceField("User")
     response_comment = db.StringField()
 
+    on_create = Signal()
+    after_handle = Signal()
+
     meta = {
         "indexes": [
             "owner",
@@ -38,3 +44,14 @@ class Transfer(db.Document):
             "status",
         ]
     }
+
+    @classmethod
+    def post_save(cls, sender, document, **kwargs):
+        """Handle post save signal for Transfer documents."""
+        # Only trigger on_create signal on creation, not on every save
+        if kwargs.get("created"):
+            cls.on_create.send(document)
+
+
+# Connect the post_save signal
+post_save.connect(Transfer.post_save, sender=Transfer)

udata/features/transfer/notifications.py

@@ -1,11 +1,107 @@
 import logging
+from datetime import datetime
 
+from udata.api_fields import field, generate_fields
+from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.models import Dataset
+from udata.core.organization.models import Organization
+from udata.core.reuse.models import Reuse
+from udata.core.user.models import User
 from udata.features.notifications.actions import notifier
 from udata.models import Transfer
+from udata.mongo import db
 
 log = logging.getLogger(__name__)
 
 
+@generate_fields()
+class TransferRequestNotificationDetails(db.EmbeddedDocument):
+    transfer_owner = field(
+        db.GenericReferenceField(choices=(User, Organization), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    transfer_recipient = field(
+        db.GenericReferenceField(choices=(User, Organization), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+    transfer_subject = field(
+        db.GenericReferenceField(choices=(Dataset, Dataservice, Reuse), required=True),
+        readonly=True,
+        auditable=False,
+        allow_null=True,
+        filterable={},
+    )
+
+
+@Transfer.on_create.connect
+def on_transfer_created(transfer, **kwargs):
+    """Create notification when a new transfer request is created"""
+
+    from udata.features.notifications.models import Notification
+
+    recipient = transfer.recipient
+    owner = transfer.owner
+    users = []
+
+    if isinstance(recipient, User):
+        users = [recipient]
+    elif isinstance(recipient, Organization):
+        users = [member.user for member in recipient.members if member.role == "admin"]
+
+    for user in users:
+        try:
+            # we don't want notifications for the same transfer, if the previous one is stil no handled
+            existing = Notification.objects(
+                user=user,
+                details__transfer_recipient=recipient,
+                details__transfer_owner=owner,
+                details__transfer_subject=transfer.subject,
+                handled_at=None,
+            ).first()
+
+            if not existing:
+                notification = Notification(
+                    user=user,
+                    details=TransferRequestNotificationDetails(
+                        transfer_owner=owner,
+                        transfer_recipient=recipient,
+                        transfer_subject=transfer.subject,
+                    ),
+                )
+                notification.created_at = transfer.created
+                notification.save()
+        except Exception as e:
+            log.error(
+                f"Error creating notification for admin user {user.id} "
+                f"and recipient {recipient.id}: {e}"
+            )
+
+
+@Transfer.after_handle.connect
+def on_handle_transfer(transfer, **kwargs):
+    """Update handled_at timestamp on related notifications when a transfer is handled"""
+    from udata.features.notifications.models import Notification
+
+    # Find all non handled notifications related to this transfer
+    notifications = Notification.objects(
+        details__transfer_subject=transfer.subject,
+        details__transfer_owner=transfer.owner,
+        details__transfer_recipient=transfer.recipient,
+        handled_at=None,
+    )
+
+    # Update handled_at for all matching notifications
+    for notification in notifications:
+        notification.handled_at = datetime.utcnow()
+        notification.save()
+
+
 @notifier("transfer_request")
 def transfer_request_notifications(user):
     """Notify user about pending transfer requests"""

udata/harvest/backends/ckan/harvesters.py

@@ -3,6 +3,8 @@ import logging
 from urllib.parse import urljoin
 from uuid import UUID
 
+from dateutil.parser import ParserError
+
 from udata import uris
 from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
@@ -202,10 +204,16 @@ class CkanBackend(BaseBackend):
                         log.debug("frequency value not handled: %s", value)
             # Temporal coverage start
             elif key == "temporal_start":
-                temporal_start = daterange_start(value)
+                try:
+                    temporal_start = daterange_start(value)
+                except ParserError:
+                    log.warning(f"Unparseable temporal_start value: '{value}'")
             # Temporal coverage end
             elif key == "temporal_end":
-                temporal_end = daterange_end(value)
+                try:
+                    temporal_end = daterange_end(value)
+                except ParserError:
+                    log.warning(f"Unparseable temporal_end value: '{value}'")
             else:
                 dataset.extras[extra["key"]] = value
 

udata/migrations/2021-08-17-harvest-integrity.py

@@ -5,6 +5,7 @@ Remove Harvest db integrity problems
 
 import logging
 
+import click
 import mongoengine
 
 from udata.core.jobs.models import PeriodicTask
@@ -16,29 +17,35 @@ log = logging.getLogger(__name__)
 def migrate(db):
     log.info("Processing HarvestJob source references.")
 
-    harvest_jobs = HarvestJob.objects().no_cache().all()
+    harvest_jobs = HarvestJob.objects().no_cache()
+    total = harvest_jobs.count()
     count = 0
-    for harvest_job in harvest_jobs:
-        try:
-            harvest_job.source.id
-        except mongoengine.errors.DoesNotExist:
-            count += 1
-            harvest_job.delete()
+    with click.progressbar(harvest_jobs, length=total, label="Checking sources refs") as jobs:
+        for harvest_job in jobs:
+            try:
+                if harvest_job.source is None:
+                    raise mongoengine.errors.DoesNotExist()
+                harvest_job.source.id
+            except mongoengine.errors.DoesNotExist:
+                count += 1
+                harvest_job.delete()
 
     log.info(f"Completed, removed {count} HarvestJob objects")
 
     log.info("Processing HarvestJob items references.")
 
-    harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache().all()
+    harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache()
+    total = harvest_jobs.count()
     count = 0
-    for harvest_job in harvest_jobs:
-        for item in harvest_job.items:
-            try:
-                item.dataset and item.dataset.id
-            except mongoengine.errors.DoesNotExist:
-                count += 1
-                item.dataset = None
-                harvest_job.save()
+    with click.progressbar(harvest_jobs, length=total, label="Checking items refs") as jobs:
+        for harvest_job in jobs:
+            for item in harvest_job.items:
+                try:
+                    item.dataset and item.dataset.id
+                except mongoengine.errors.DoesNotExist:
+                    count += 1
+                    item.dataset = None
+                    harvest_job.save()
 
     log.info(f"Completed, modified {count} HarvestJob objects")
 

udata/migrations/2025-12-16-create-transfer-request-notifications.py

@@ -0,0 +1,69 @@
+"""
+Create TransferRequestNotification for all pending transfer requests
+"""
+
+import logging
+
+import click
+
+from udata.features.notifications.models import Notification
+from udata.features.transfer.models import Transfer
+from udata.features.transfer.notifications import TransferRequestNotificationDetails
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Processing pending transfer requests...")
+
+    created_count = 0
+
+    # Get all pending transfers
+    transfers = Transfer.objects(status="pending")
+
+    with click.progressbar(transfers, length=transfers.count()) as transfer_list:
+        for transfer in transfer_list:
+            try:
+                # Get the recipient (could be a user or an organization)
+                recipient = transfer.recipient
+
+                # For organizations, we need to find admins who should receive notifications
+                if recipient._cls == "Organization":
+                    # Get all admin users for this organization
+                    recipient_users = [
+                        member.user for member in recipient.members if member.role == "admin"
+                    ]
+                else:
+                    # For users, just use the recipient directly
+                    recipient_users = [recipient]
+
+                # Create a notification for each recipient user
+                for recipient_user in recipient_users:
+                    try:
+                        # Check if notification already exists
+                        existing = Notification.objects(
+                            user=recipient_user,
+                            details__transfer_recipient=recipient,
+                            details__transfer_owner=transfer.owner,
+                            details__transfer_subject=transfer.subject,
+                        ).first()
+                        if not existing:
+                            notification = Notification(user=recipient_user)
+                            notification.details = TransferRequestNotificationDetails(
+                                transfer_owner=transfer.owner,
+                                transfer_recipient=recipient,
+                                transfer_subject=transfer.subject,
+                            )
+                            # Set the created_at to match the transfer creation date
+                            notification.created_at = transfer.created
+                            notification.save()
+                            created_count += 1
+                    except Exception as e:
+                        log.error(
+                            f"Error creating notification for user {recipient_user.id} "
+                            f"and transfer {transfer.id}: {e}"
+                        )
+            except Exception as e:
+                log.error(f"Error creating notification for transfer {transfer.id}: {e}")
+
+    log.info(f"Created {created_count} TransferRequestNotifications")

udata/migrations/2026-01-14-add-default-kind-to-posts.py

@@ -0,0 +1,17 @@
+"""
+This migration sets Post.kind to "news" for posts that don't have a kind.
+This is necessary because the default value is only applied to new documents,
+not existing ones.
+"""
+
+import logging
+
+from udata.models import Post
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Processing posts without kind...")
+    count = Post.objects(kind__exists=False).update(kind="news")
+    log.info(f"\tSet kind='news' for {count} posts")

udata/tasks.py

@@ -161,6 +161,7 @@ def init_app(app):
     import udata.core.metrics.tasks  # noqa
     import udata.core.tags.tasks  # noqa
     import udata.core.activity.tasks  # noqa
+    import udata.core.dataservices.tasks  # noqa
     import udata.core.dataset.tasks  # noqa
     import udata.core.dataset.transport  # noqa
     import udata.core.dataset.recommendations  # noqa