udata 12.0.2.dev15__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

udata/core/dataset/search.py

@@ -54,8 +54,8 @@ class DatasetSearch(ModelSearchAdapter):
     }
 
     @classmethod
-    def is_indexable(cls, dataset):
-        return dataset.deleted is None and dataset.archived is None and not dataset.private
+    def is_indexable(cls, dataset: Dataset):
+        return dataset.is_visible
 
     @classmethod
     def mongo_search(cls, args):

udata/core/dataset/tasks.py

@@ -1,6 +1,6 @@
 import collections
 import os
-from datetime import datetime
+from datetime import date, datetime
 from tempfile import NamedTemporaryFile
 
 from celery.utils.log import get_task_logger
@@ -9,9 +9,15 @@ from mongoengine import ValidationError
 
 from udata import models as udata_models
 from udata.core import csv, storages
+from udata.core.badges import tasks as badge_tasks
+from udata.core.constants import HVD
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.constants import INSPIRE
+from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
+from udata.core.organization.models import Organization
 from udata.harvest.models import HarvestJob
 from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
+from udata.storage.s3 import store_bytes
 from udata.tasks import job
 
 from .models import Checksum, CommunityResource, Dataset, Resource
@@ -85,12 +91,14 @@ def get_queryset(model_cls):
     return model_cls.objects.filter(**params).no_cache()
 
 
+def get_resource_for_csv_export_model(model, dataset):
+    for resource in dataset.resources:
+        if resource.extras.get("csv-export:model", "") == model:
+            return resource
+
+
 def get_or_create_resource(r_info, model, dataset):
-    resource = None
-    for r in dataset.resources:
-        if r.extras.get("csv-export:model", "") == model:
-            resource = r
-            break
+    resource = get_resource_for_csv_export_model(model, dataset)
     if resource:
         for k, v in r_info.items():
             setattr(resource, k, v)
@@ -121,11 +129,16 @@ def store_resource(csvfile, model, dataset):
     return get_or_create_resource(r_info, model, dataset)
 
 
-def export_csv_for_model(model, dataset):
+def export_csv_for_model(model, dataset, replace: bool = False):
     model_cls = getattr(udata_models, model.capitalize(), None)
     if not model_cls:
         log.error("Unknow model %s" % model)
         return
+
+    fs_filename_to_remove = None
+    if existing_resource := get_resource_for_csv_export_model(model, dataset):
+        fs_filename_to_remove = existing_resource.fs_filename
+
     queryset = get_queryset(model_cls)
     adapter = csv.get_adapter(model_cls)
     if not adapter:
@@ -151,6 +164,10 @@ def export_csv_for_model(model, dataset):
         else:
             dataset.last_modified_internal = datetime.utcnow()
             dataset.save()
+        # remove previous catalog if exists and replace is True
+        if replace and fs_filename_to_remove:
+            storages.resources.delete(fs_filename_to_remove)
+        return resource
     finally:
         csvfile.close()
         os.unlink(csvfile.name)
@@ -179,7 +196,23 @@ def export_csv(self, model=None):
 
     models = (model,) if model else ALLOWED_MODELS
     for model in models:
-        export_csv_for_model(model, dataset)
+        resource = export_csv_for_model(model, dataset, replace=True)
+
+        # If we are the first day of the month, archive today catalogs
+        if (
+            current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"]
+            and resource
+            and date.today().day == 1
+        ):
+            log.info(
+                f"Archiving {model} csv catalog on {current_app.config['EXPORT_CSV_ARCHIVE_S3_BUCKET']} bucket"
+            )
+            with storages.resources.open(resource.fs_filename, "rb") as f:
+                store_bytes(
+                    bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
+                    filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}",
+                    bytes=f.read(),
+                )
 
 
 @job("bind-tabular-dataservice")
@@ -213,3 +246,48 @@ def bind_tabular_dataservice(self):
         log.error(exc_info=e)
 
     log.info(f"Bound {datasets.count()} datasets to TabularAPI dataservice")
+
+
+@badge_tasks.register(model=Dataset, badge=HVD)
+def update_dataset_hvd_badge() -> None:
+    """
+    Update HVD badges to candidate datasets, based on the hvd tag.
+    Only datasets owned by certified and public service organizations are candidate to have a HVD badge.
+    """
+    if not current_app.config["HVD_SUPPORT"]:
+        log.error("You need to set HVD_SUPPORT if you want to update dataset hvd badge")
+        return
+    public_certified_orgs = (
+        Organization.objects(badges__kind=PUBLIC_SERVICE).filter(badges__kind=CERTIFIED).only("id")
+    )
+
+    datasets = Dataset.objects(
+        tags="hvd", badges__kind__ne="hvd", organization__in=public_certified_orgs
+    )
+    log.info(f"Adding HVD badge to {datasets.count()} datasets")
+    for dataset in datasets:
+        dataset.add_badge(HVD)
+
+    datasets = Dataset.objects(tags__nin=["hvd"], badges__kind="hvd")
+    log.info(f"Removing HVD badge from {datasets.count()} datasets")
+    for dataset in datasets:
+        dataset.remove_badge(HVD)
+
+
+@badge_tasks.register(model=Dataset, badge=INSPIRE)
+def update_dataset_inspire_badge() -> None:
+    """
+    Update INSPIRE badges to candidate datasets, based on the inspire tag.
+    """
+    if not current_app.config["INSPIRE_SUPPORT"]:
+        log.error("You need to set INSPIRE_SUPPORT if you want to update dataset INSPIRE badge")
+        return
+    datasets = Dataset.objects(tags="inspire", badges__kind__ne="inspire")
+    log.info(f"Adding INSPIRE badge to {datasets.count()} datasets")
+    for dataset in datasets:
+        dataset.add_badge(INSPIRE)
+
+    datasets = Dataset.objects(tags__nin=["inspire"], badges__kind="inspire")
+    log.info(f"Removing INSPIRE badge from {datasets.count()} datasets")
+    for dataset in datasets:
+        dataset.remove_badge(INSPIRE)

udata/core/discussions/mails.py

@@ -0,0 +1,63 @@
+from udata.core.discussions.models import Discussion, Message
+from udata.i18n import lazy_gettext as _
+from udata.mail import LabelledContent, MailCTA, MailMessage, ParagraphWithLinks
+
+
+def new_discussion(discussion: Discussion) -> MailMessage:
+    return MailMessage(
+        subject=_(
+            "A new discussion has been opened on your %(type)s",
+            type=discussion.subject.verbose_name,
+        ),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "You have a new discussion from %(user_or_org)s on your %(type)s %(object)s",
+                    user_or_org=discussion.organization or discussion.user,
+                    type=discussion.subject.verbose_name,
+                    object=discussion.subject,
+                )
+            ),
+            LabelledContent(_("Discussion title:"), discussion.title, inline=True),
+            LabelledContent(_("Comment:"), discussion.discussion[0].content),
+            MailCTA(_("Reply"), discussion.url_for()),
+        ],
+    )
+
+
+def new_discussion_comment(discussion: Discussion, comment: Message) -> MailMessage:
+    return MailMessage(
+        subject=_("A new comment has been added to a discussion"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "You have a new comment from %(user_or_org)s on your %(type)s %(object)s",
+                    user_or_org=comment.posted_by_org_or_user,
+                    type=discussion.subject.verbose_name,
+                    object=discussion.subject,
+                )
+            ),
+            LabelledContent(_("Discussion title:"), discussion.title, inline=True),
+            LabelledContent(_("Comment:"), comment.content),
+            MailCTA(_("Reply"), discussion.url_for()),
+        ],
+    )
+
+
+def discussion_closed(discussion: Discussion, comment: Message | None) -> MailMessage:
+    return MailMessage(
+        subject=_("A discussion has been closed"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "The discussion you participated in on the %(type)s %(object)s has been closed by %(user_or_org)s.",
+                    user_or_org=discussion.closed_by_org_or_user,
+                    type=discussion.subject.verbose_name,
+                    object=discussion.subject,
+                )
+            ),
+            LabelledContent(_("Discussion title:"), discussion.title, inline=True),
+            LabelledContent(_("Comment:"), comment.content) if comment else None,
+            MailCTA(_("View the discussion"), discussion.url_for()),
+        ],
+    )

udata/core/discussions/tasks.py

@@ -1,7 +1,6 @@
-from udata import mail
-from udata.i18n import lazy_gettext as _
 from udata.tasks import connect, get_logger
 
+from . import mails
 from .constants import NOTIFY_DISCUSSION_SUBJECTS
 from .models import Discussion
 from .signals import on_discussion_closed, on_new_discussion, on_new_discussion_comment
@@ -22,15 +21,7 @@ def owner_recipients(discussion):
 def notify_new_discussion(discussion_id):
     discussion = Discussion.objects.get(pk=discussion_id)
     if isinstance(discussion.subject, NOTIFY_DISCUSSION_SUBJECTS):
-        recipients = owner_recipients(discussion)
-        subject = _("Your %(type)s have a new discussion", type=discussion.subject.verbose_name)
-        mail.send(
-            subject,
-            recipients,
-            "new_discussion",
-            discussion=discussion,
-            message=discussion.discussion[0],
-        )
+        mails.new_discussion(discussion).send(owner_recipients(discussion))
     else:
         log.warning("Unrecognized discussion subject type %s", type(discussion.subject))
 
@@ -42,11 +33,7 @@ def notify_new_discussion_comment(discussion_id, message=None):
     if isinstance(discussion.subject, NOTIFY_DISCUSSION_SUBJECTS):
         recipients = owner_recipients(discussion) + [m.posted_by for m in discussion.discussion]
         recipients = list({u.id: u for u in recipients if u != message.posted_by}.values())
-        subject = _("%(user)s commented your discussion", user=message.posted_by_name)
-
-        mail.send(
-            subject, recipients, "new_discussion_comment", discussion=discussion, message=message
-        )
+        mails.new_discussion_comment(discussion, message).send(recipients)
     else:
         log.warning("Unrecognized discussion subject type %s", type(discussion.subject))
 
@@ -58,7 +45,6 @@ def notify_discussion_closed(discussion_id, message=None):
     if isinstance(discussion.subject, NOTIFY_DISCUSSION_SUBJECTS):
         recipients = owner_recipients(discussion) + [m.posted_by for m in discussion.discussion]
         recipients = list({u.id: u for u in recipients if u != discussion.closed_by}.values())
-        subject = _("A discussion has been closed")
-        mail.send(subject, recipients, "discussion_closed", discussion=discussion, message=message)
+        mails.discussion_closed(discussion, message).send(recipients)
     else:
         log.warning("Unrecognized discussion subject type %s", type(discussion.subject))

udata/core/metrics/__init__.py

@@ -1,6 +1,3 @@
-from udata import entrypoints
-
-
 def init_app(app):
     # Load all core metrics
     import udata.core.user.metrics  # noqa
@@ -9,6 +6,3 @@ def init_app(app):
     import udata.core.dataset.metrics  # noqa
     import udata.core.reuse.metrics  # noqa
     import udata.core.followers.metrics  # noqa
-
-    # Load metrics from plugins
-    entrypoints.get_enabled("udata.metrics", app)

udata/core/organization/api.py

@@ -49,7 +49,7 @@ from .forms import (
 from .models import Member, MembershipRequest, Organization
 from .permissions import EditOrganizationPermission, OrganizationPrivatePermission
 from .rdf import build_org_catalog
-from .tasks import notify_membership_request, notify_membership_response
+from .tasks import notify_membership_request, notify_membership_response, notify_new_member
 
 DEFAULT_SORTING = "-created_at"
 SUGGEST_SORTING = "-metrics.followers"
@@ -472,6 +472,8 @@ class MemberAPI(API):
         org.count_members()
         org.save()
 
+        notify_new_member.delay(str(org.id), str(member.user.email))
+
         return member, 201
 
     @api.secure

udata/core/organization/mails.py

@@ -0,0 +1,144 @@
+from udata.core.organization.models import MembershipRequest, Organization
+from udata.i18n import lazy_gettext as _
+from udata.mail import LabelledContent, MailCTA, MailMessage, ParagraphWithLinks
+from udata.uris import cdata_url
+
+
+def new_membership_request(org: Organization, request: MembershipRequest) -> MailMessage:
+    return MailMessage(
+        subject=_("New membership request"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "You received a membership request from %(user)s for your organization %(org)s",
+                    user=request.user,
+                    org=org,
+                )
+            ),
+            LabelledContent(_("Reason for the request:"), request.comment),
+            MailCTA(_("See the request"), cdata_url(f"/admin/organizations/{org.id}/members/")),
+        ],
+    )
+
+
+def membership_refused(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Membership refused"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Your membership for the organization %(org)s has been refused",
+                    org=org,
+                )
+            ),
+        ],
+    )
+
+
+def membership_accepted(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Your invitation to join an organization has been accepted"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Good news! Your request to join the organization %(org)s has been approved.",
+                    org=org,
+                )
+            ),
+            MailCTA(
+                _("View the organization"), cdata_url(f"/admin/organizations/{org.id}/datasets")
+            ),
+        ],
+    )
+
+
+def new_member(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("You have been added as a member of an organization"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Good news! You are now a member of %(org)s.",
+                    org=org,
+                )
+            ),
+            MailCTA(
+                _("View the organization"), cdata_url(f"/admin/organizations/{org.id}/datasets")
+            ),
+        ],
+    )
+
+
+def badge_added_certified(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Your organization has been certified"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Good news! Your organization %(org)s has been certified by our team. A badge is now associated with your organization.",
+                    org=org,
+                )
+            ),
+            MailCTA(_("View the organization"), org.self_web_url()),
+        ],
+    )
+
+
+def badge_added_public_service(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Your organization has been identified as a public service"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Good news! Your organization %(org)s has been identified by our team as a public service. A badge is now associated with your organization.",
+                    org=org,
+                )
+            ),
+            MailCTA(_("View the organization"), org.self_web_url()),
+        ],
+    )
+
+
+def badge_added_local_authority(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Your organization has been identified as a local authority"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Good news! Your organization %(org)s has been identified by our team as a local authority. A badge is now associated with your organization.",
+                    org=org,
+                )
+            ),
+            MailCTA(_("View the organization"), org.self_web_url()),
+        ],
+    )
+
+
+def badge_added_company(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Your organization has been identified as a company"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Your organization %(org)s has been identified by our team as a company. A badge is now associated with your organization.",
+                    org=org,
+                )
+            ),
+            MailCTA(_("View the organization"), org.self_web_url()),
+        ],
+    )
+
+
+def badge_added_association(org: Organization) -> MailMessage:
+    return MailMessage(
+        subject=_("Your organization has been identified as an association"),
+        paragraphs=[
+            ParagraphWithLinks(
+                _(
+                    "Your organization %(org)s has been identified by our team as an association. A badge is now associated with your organization.",
+                    org=org,
+                )
+            ),
+            MailCTA(_("View the organization"), org.self_web_url()),
+        ],
+    )

udata/core/organization/models.py

@@ -3,6 +3,7 @@ from itertools import chain
 
 from blinker import Signal
 from flask import url_for
+from flask_babel import LazyString
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
@@ -34,7 +35,7 @@ from .constants import (
 
 __all__ = ("Organization", "Team", "Member", "MembershipRequest")
 
-BADGES: dict[str, str] = {
+BADGES: dict[str, LazyString] = {
     PUBLIC_SERVICE: _("Public Service"),
     CERTIFIED: _("Certified"),
     ASSOCIATION: _("Association"),

udata/core/organization/search.py

@@ -29,7 +29,7 @@ class OrganizationSearch(search.ModelSearchAdapter):
     }
 
     @classmethod
-    def is_indexable(cls, org):
+    def is_indexable(cls, org: Organization):
         return org.deleted is None
 
     @classmethod

udata/core/organization/tasks.py

@@ -1,11 +1,10 @@
-from udata import mail
 from udata.core import storages
 from udata.core.badges.tasks import notify_new_badge
-from udata.i18n import lazy_gettext as _
 from udata.models import Activity, ContactPoint, Dataset, Follow, Transfer
 from udata.search import reindex
 from udata.tasks import get_logger, job, task
 
+from . import mails
 from .constants import ASSOCIATION, CERTIFIED, COMPANY, LOCAL_AUTHORITY, PUBLIC_SERVICE
 from .models import Organization
 
@@ -47,10 +46,11 @@ def notify_membership_request(org_id, request_id):
     org = Organization.objects.get(pk=org_id)
     request = next((r for r in org.requests if str(r.id) == request_id), None)
 
+    if request is None:
+        return
+
     recipients = [m.user for m in org.by_role("admin")]
-    mail.send(
-        _("New membership request"), recipients, "membership_request", org=org, request=request
-    )
+    mails.new_membership_request(org, request).send(recipients)
 
 
 @task(route="high.mail")
@@ -58,12 +58,13 @@ def notify_membership_response(org_id, request_id):
     org = Organization.objects.get(pk=org_id)
     request = next((r for r in org.requests if str(r.id) == request_id), None)
 
+    if request is None:
+        return
+
     if request.status == "accepted":
-        subject = _('You are now a member of the organization "%(org)s"', org=org)
-        template = "new_member"
+        mails.membership_accepted(org).send(request.user)
     else:
-        subject, template = _("Membership refused"), "membership_refused"
-    mail.send(subject, request.user, template, org=org, request=request)
+        mails.membership_refused(org).send(request.user)
 
 
 @task
@@ -71,8 +72,10 @@ def notify_new_member(org_id, email):
     org = Organization.objects.get(pk=org_id)
     member = next((m for m in org.members if m.user.email == email), None)
 
-    subject = _('You are now a member of the organization "%(org)s"', org=org)
-    mail.send(subject, member.user, "new_member", org=org)
+    if member is None:
+        return
+
+    mails.new_member(org).send(member.user)
 
 
 @notify_new_badge(Organization, CERTIFIED)
@@ -82,14 +85,8 @@ def notify_badge_certified(org_id):
     """
     org = Organization.objects.get(pk=org_id)
     recipients = [member.user for member in org.members]
-    subject = _('Your organization "%(name)s" has been certified', name=org.name)
-    mail.send(
-        subject,
-        recipients,
-        "badge_added_certified",
-        organization=org,
-        badge=org.get_badge(CERTIFIED),
-    )
+
+    mails.badge_added_certified(org).send(recipients)
 
 
 @notify_new_badge(Organization, PUBLIC_SERVICE)
@@ -99,14 +96,8 @@ def notify_badge_public_service(org_id):
     """
     org = Organization.objects.get(pk=org_id)
     recipients = [member.user for member in org.members]
-    subject = _('Your organization "%(name)s" has been identified as public service', name=org.name)
-    mail.send(
-        subject,
-        recipients,
-        "badge_added_public_service",
-        organization=org,
-        badge=org.get_badge(PUBLIC_SERVICE),
-    )
+
+    mails.badge_added_public_service(org).send(recipients)
 
 
 @notify_new_badge(Organization, COMPANY)
@@ -116,10 +107,7 @@ def notify_badge_company(org_id):
     """
     org = Organization.objects.get(pk=org_id)
     recipients = [member.user for member in org.members]
-    subject = _('Your organization "%(name)s" has been identified as a company', name=org.name)
-    mail.send(
-        subject, recipients, "badge_added_company", organization=org, badge=org.get_badge(COMPANY)
-    )
+    mails.badge_added_company(org).send(recipients)
 
 
 @notify_new_badge(Organization, ASSOCIATION)
@@ -129,14 +117,7 @@ def notify_badge_association(org_id):
     """
     org = Organization.objects.get(pk=org_id)
     recipients = [member.user for member in org.members]
-    subject = _('Your organization "%(name)s" has been identified as an association', name=org.name)
-    mail.send(
-        subject,
-        recipients,
-        "badge_added_association",
-        organization=org,
-        badge=org.get_badge(ASSOCIATION),
-    )
+    mails.badge_added_association(org).send(recipients)
 
 
 @notify_new_badge(Organization, LOCAL_AUTHORITY)
@@ -146,13 +127,4 @@ def notify_badge_local_authority(org_id):
     """
     org = Organization.objects.get(pk=org_id)
     recipients = [member.user for member in org.members]
-    subject = _(
-        'Your organization "%(name)s" has been identified as a local authority', name=org.name
-    )
-    mail.send(
-        subject,
-        recipients,
-        "badge_added_local_authority",
-        organization=org,
-        badge=org.get_badge(LOCAL_AUTHORITY),
-    )
+    mails.badge_added_local_authority(org).send(recipients)

udata/core/pages/tests/test_api.py

@@ -6,8 +6,6 @@ from udata.tests.api import APITestCase
 
 
 class PageAPITest(APITestCase):
-    modules = []
-
     def test_create_get_update(self):
         self.login()
         datasets = DatasetFactory.create_batch(3)

udata/core/reuse/api.py

@@ -12,6 +12,7 @@ from udata.api_fields import patch, patch_and_save
 from udata.auth import admin_permission
 from udata.core.badges import api as badges_api
 from udata.core.badges.fields import badge_fields
+from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_ref_fields
 from udata.core.followers.api import FollowAPI
 from udata.core.organization.models import Organization
@@ -227,7 +228,8 @@ class ReuseDatasetsAPI(API):
             dataset = Dataset.objects.get_or_404(id=id_or_404(request.json["id"]))
         except Dataset.DoesNotExist:
             msg = "Dataset {0} does not exists".format(request.json["id"])
-            api.abort(404, msg)
+            return api.abort(404, msg)
+
         if dataset in reuse.datasets:
             return reuse
         reuse.datasets.append(dataset)
@@ -235,6 +237,30 @@ class ReuseDatasetsAPI(API):
         return reuse, 201
 
 
+@ns.route("/<reuse:reuse>/dataservices/", endpoint="reuse_add_dataservice")
+class ReuseDataservicesAPI(API):
+    @api.secure
+    @api.doc("reuse_add_dataservice", **common_doc)
+    @api.expect(Dataservice.__ref_fields__)
+    @api.response(200, "The dataservice is already present", Reuse.__read_fields__)
+    @api.marshal_with(Reuse.__read_fields__, code=201)
+    def post(self, reuse):
+        """Add a dataservice to a given reuse"""
+        if "id" not in request.json:
+            api.abort(400, "Expect a dataservice identifier")
+        try:
+            dataservice = Dataservice.objects.get_or_404(id=id_or_404(request.json["id"]))
+        except Dataservice.DoesNotExist:
+            msg = "Dataservice {0} does not exists".format(request.json["id"])
+            return api.abort(404, msg)
+
+        if dataservice in reuse.dataservices:
+            return reuse
+        reuse.dataservices.append(dataservice)
+        reuse.save()
+        return reuse, 201
+
+
 @ns.route("/badges/", endpoint="available_reuse_badges")
 class AvailableDatasetBadgesAPI(API):
     @api.doc("available_reuse_badges")