udata 12.0.2.dev15__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

udata/api/__init__.py

@@ -333,6 +333,7 @@ def marshal_page_with(func):
 
 def init_app(app):
     # Load all core APIs
+    import udata.core.access_type.api  # noqa
     import udata.core.activity.api  # noqa
     import udata.core.spatial.api  # noqa
     import udata.core.user.api  # noqa

udata/api_fields.py

@@ -93,12 +93,14 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         # TODO: this is currently never used. We may remove it if the auto-conversion
         # is always good enough.
         return info.get("convert_to"), info.get("convert_to")
-    elif isinstance(field, mongo_fields.StringField):
+    elif isinstance(field, mongo_fields.StringField) or isinstance(field, mongo_fields.EnumField):
         constructor = (
             custom_restx_fields.Markdown if info.get("markdown", False) else restx_fields.String
         )
-        params["min_length"] = field.min_length
-        params["max_length"] = field.max_length
+        if isinstance(field, mongo_fields.StringField):
+            params["min_length"] = field.min_length
+            params["max_length"] = field.max_length
+        # EnumField define choices too from the enum values.
         params["enum"] = field.choices
         if field.validation:
             params["validation"] = validation_to_type(field.validation)
@@ -215,6 +217,9 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         # the referenced model, if not we return a String (and RestX will call the `str()` of the model
         # when returning from an endpoint)
         nested_fields: dict | None = info.get("nested_fields")
+        if nested_fields is None and hasattr(field.document_type_obj, "__ref_fields__"):
+            nested_fields = field.document_type_obj.__ref_fields__
+
         if nested_fields is None:
             # If there is no `nested_fields` convert the object to the string representation.
             constructor_read = restx_fields.String
@@ -323,7 +328,8 @@ def generate_fields(**kwargs) -> Callable:
             kwargs.get("nested_filters", {})
         )
 
-        read_fields["id"] = restx_fields.String(required=True, readonly=True)
+        if issubclass(cls, db.Document) or issubclass(cls, db.DynamicDocument):
+            read_fields["id"] = restx_fields.String(required=True, readonly=True)
 
         classes_by_names[cls.__name__] = cls
         save_class_by_parents(cls)

udata/app.py

@@ -20,6 +20,7 @@ from flask import (
 )
 from flask_caching import Cache
 from flask_wtf.csrf import CSRFProtect
+from importlib_metadata import entry_points
 from speaklater import is_lazy_string
 from werkzeug.exceptions import NotFound
 from werkzeug.middleware.proxy_fix import ProxyFix
@@ -200,12 +201,21 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
 def standalone(app):
     """Factory for an all in one application"""
     from udata import api, core, frontend
+    from udata.features import notifications
 
     core.init_app(app)
     frontend.init_app(app)
     api.init_app(app)
+    notifications.init_app(app)
+
+    eps = entry_points(group="udata.plugins")
+    for ep in eps:
+        plugin_module = ep.load()
 
-    register_features(app)
+        if hasattr(plugin_module, "init_app"):
+            plugin_module.init_app(app)
+        else:
+            log.error(f"Plugin {ep.name} ({ep.value}) doesn't expose an `init_app()` function.")
 
     return app
 
@@ -278,12 +288,3 @@ def page_not_found(e: NotFound):
         return render_template("404.html", homepage_url=homepage_url()), 404
 
     return jsonify({"error": e.description, "status": 404}), 404
-
-
-def register_features(app):
-    from udata.features import notifications
-
-    notifications.init_app(app)
-
-    for ep in entrypoints.get_enabled("udata.plugins", app).values():
-        ep.init_app(app)

udata/auth/__init__.py

@@ -1,6 +1,6 @@
 import logging
 
-from flask import current_app, render_template
+from flask import render_template
 from flask_principal import Permission as BasePermission
 from flask_principal import PermissionDenied as PermissionDenied
 from flask_principal import RoleNeed as RoleNeed
@@ -10,17 +10,18 @@ from flask_security import Security as Security
 from flask_security import current_user as current_user
 from flask_security import login_required as login_required
 from flask_security import login_user as login_user
-from werkzeug.utils import import_string
+
+from . import mails
 
 log = logging.getLogger(__name__)
 
 
-def render_security_template(*args, **kwargs):
-    try:
-        render = import_string(current_app.config.get("SECURITY_RENDER"))
-    except Exception:
-        render = render_template
-    return render(*args, **kwargs)
+def render_security_template(template_name_or_list, **kwargs):
+    result = mails.render_mail_template(template_name_or_list, **kwargs)
+    if result is not None:
+        return result
+
+    return render_template(template_name_or_list, **kwargs)
 
 
 security = Security()
@@ -44,7 +45,6 @@ def init_app(app):
         ExtendedRegisterForm,
         ExtendedResetPasswordForm,
     )
-    from .mails import UdataMailUtil
     from .password_validation import UdataPasswordUtil
     from .views import create_security_blueprint
 
@@ -71,7 +71,6 @@ def init_app(app):
         register_form=ExtendedRegisterForm,
         reset_password_form=ExtendedResetPasswordForm,
         forgot_password_form=ExtendedForgotPasswordForm,
-        mail_util_cls=UdataMailUtil,
         password_util_cls=UdataPasswordUtil,
     )
 

udata/auth/mails.py

@@ -1,54 +1,146 @@
+"""
+We have our own system to build mails without Jinja templates with `MailMessage`.
+To connect our system with the system from flask_security we need to override the Jinja
+`render_template` function, create our MailMessage, and generate the HTML or text version.
+`flask_security` then call the standard mail method from flask to send these strings.
+
+In `render_mail_template` we support a few mails but not all. We could fallback to the regular
+Jinja render function, but since we don't have any mails' templates defined in our application
+the render function will crash, so we crash early in the `render_mail_template` function.
+
+Note that `flask_security` have default templates for all mails but we create our own blueprint
+specifying our `template` folder for templates and the system is not intelligent enough to try
+our folder before fallbacking to the templates inside the `flask_security` package.
+"""
+
 import logging
 
-import email_validator
 from flask import current_app
 
-from udata.tasks import task
+from udata.mail import MailCTA, MailMessage
 
 log = logging.getLogger(__name__)
 
 
-@task
-def sendmail(msg):
-    debug = current_app.config.get("DEBUG", False)
-    send_mail = current_app.config.get("SEND_MAIL", not debug)
-    if send_mail:
-        mail = current_app.extensions.get("mail")
-        mail.send(msg)
+def render_mail_template(template_name_or_list: str | list[str], **kwargs):
+    if not isinstance(template_name_or_list, str):
+        return None
+
+    if not template_name_or_list.startswith("security/email/"):
+        return None
+
+    if not template_name_or_list.endswith(".txt") and not template_name_or_list.endswith(".html"):
+        return None
+
+    (name, format) = template_name_or_list.removeprefix("security/email/").split(".")
+
+    mail_message = None
+
+    match name:
+        case "welcome":
+            mail_message = welcome(**kwargs)
+        case "welcome_existing":
+            mail_message = welcome_existing(**kwargs)
+        case "confirmation_instructions":
+            mail_message = confirmation_instructions(**kwargs)
+        case "reset_instructions":
+            mail_message = reset_instructions(**kwargs)
+        case "reset_notice":
+            mail_message = reset_notice(**kwargs)
+        case "change_notice":
+            mail_message = change_notice(**kwargs)
+        case _:
+            raise Exception(f"Unknown mail message template: {name}")
+
+    if format == "txt":
+        return mail_message.text(kwargs.get("user"))
+    elif format == "html":
+        return mail_message.html(kwargs.get("user"))
     else:
-        log.warning(msg)
-
-
-class UdataMailUtil:
-    def __init__(self, app):
-        self.app = app
-
-    def send_mail(self, template, subject, recipient, sender, body, html, user, **kwargs):
-        from flask_mail import Message
-
-        # In Flask-Mail, sender can be a two element tuple -- (name, address)
-        if isinstance(sender, tuple) and len(sender) == 2:
-            sender = (str(sender[0]), str(sender[1]))
-        else:
-            sender = str(sender)
-        msg = Message(str(subject), sender=sender, recipients=[recipient])
-        msg.body = body
-        msg.html = html
-
-        sendmail.delay(msg)
-
-    def normalize(self, email):
-        # Called at registration and login
-        # Calls validate method with deliverability check disabled to prevent
-        # login failure for existing users without a valid email domain name
-        return self.validate(email, validation_args={"check_deliverability": False})
-
-    def validate(self, email, validation_args=None):
-        # Called at registration only
-        # Checks email domain name deliverability
-        # To prevent false email to register
-        validator_args = self.app.config["SECURITY_EMAIL_VALIDATOR_ARGS"] or {}
-        if validation_args:
-            validator_args.update(validation_args)
-        valid = email_validator.validate_email(email, **validator_args)
-        return valid.email
+        raise Exception(f"Mail message with unknown format: {name} (txt or html supported)")
+
+
+def welcome(confirmation_link: str, **kwargs) -> MailMessage:
+    from udata.i18n import lazy_gettext as _
+
+    return MailMessage(
+        subject=_("Confirm your email address"),
+        paragraphs=[
+            _("Welcome to %(site)s!", site=current_app.config["SITE_TITLE"]),
+            _("Please confirm your email address."),
+            MailCTA(_("Confirm your email address"), confirmation_link),
+        ],
+    )
+
+
+def welcome_existing(recovery_link: str, **kwargs) -> MailMessage:
+    from udata.i18n import lazy_gettext as _
+
+    return MailMessage(
+        subject=_("Your email address is already associated with an account"),
+        paragraphs=[
+            _(
+                "Someone (you?) tried to create an account on %(site)s with your email.",
+                site=current_app.config["SITE_TITLE"],
+            ),
+            _("If you forgot your password, you can reset it."),
+            MailCTA(_("Reset your password"), recovery_link),
+        ],
+    )
+
+
+def confirmation_instructions(confirmation_link: str, **kwargs) -> MailMessage:
+    from udata.i18n import lazy_gettext as _
+
+    return MailMessage(
+        subject=_("Confirm your email address"),
+        paragraphs=[
+            _("Please confirm your email address."),
+            MailCTA(_("Confirm your email address"), confirmation_link),
+        ],
+    )
+
+
+def reset_instructions(reset_token: str, **kwargs) -> MailMessage:
+    from udata.i18n import lazy_gettext as _
+    from udata.uris import cdata_url
+
+    return MailMessage(
+        subject=_("Reset your password"),
+        paragraphs=[
+            _(
+                "Someone requested a password reset for your %(site)s account.",
+                site=current_app.config["SITE_TITLE"],
+            ),
+            _("If this wasn't you, please ignore this email."),
+            MailCTA(_("Reset your password"), cdata_url(f"/reset/{reset_token}")),
+        ],
+    )
+
+
+def reset_notice(**kwargs) -> MailMessage:
+    from udata.i18n import lazy_gettext as _
+
+    return MailMessage(
+        subject=_("Your password has been reset"),
+        paragraphs=[
+            _("Your data.gouv.fr password has been reset."),
+        ],
+    )
+
+
+def change_notice(**kwargs) -> MailMessage:
+    from udata.i18n import lazy_gettext as _
+    from udata.uris import cdata_url
+
+    return MailMessage(
+        subject=_("Your password has been changed"),
+        paragraphs=[
+            _(
+                "Your %(site)s account password has been changed.",
+                site=current_app.config["SITE_TITLE"],
+            ),
+            _("If you did not change your password, please reset it."),
+            MailCTA(_("Reset your password"), cdata_url("/reset/")),
+        ],
+    )

udata/auth/views.py

@@ -6,7 +6,6 @@ from flask_security.utils import (
     hash_data,
     login_user,
     logout_user,
-    send_mail,
     verify_hash,
 )
 from flask_security.views import (
@@ -25,10 +24,10 @@ from flask_wtf.csrf import generate_csrf
 from werkzeug.local import LocalProxy
 
 from udata.auth.proconnect import get_logout_url
-from udata.i18n import lazy_gettext as _
 from udata.uris import homepage_url
 from udata.utils import wants_json
 
+from . import mails
 from .forms import ChangeEmailForm
 
 _security = LocalProxy(lambda: current_app.extensions["security"])
@@ -47,16 +46,10 @@ def slash_url_suffix(url, suffix):
 def send_change_email_confirmation_instructions(user, new_email):
     data = [str(current_user.fs_uniquifier), hash_data(current_user.email), new_email]
     token = _security.confirm_serializer.dumps(data)
-    confirmation_link = url_for("security.confirm_change_email", token=token, _external=True)
-
-    subject = _("Confirm change of email instructions")
-    send_mail(
-        subject=subject,
-        recipient=new_email,
-        template="confirmation_instructions",
-        user=current_user,
-        confirmation_link=confirmation_link,
-    )
+
+    mails.confirmation_instructions(
+        confirmation_link=url_for("security.confirm_change_email", token=token, _external=True)
+    ).send(new_email)
 
 
 def confirm_change_email_token_status(token):

udata/commands/__init__.py

@@ -4,12 +4,11 @@ import sys
 from glob import iglob
 
 import click
-import pkg_resources
 from flask.cli import FlaskGroup, ScriptInfo, shell_command
 
 from udata import entrypoints
 from udata.app import VERBOSE_LOGGERS, create_app, standalone
-from udata.utils import safe_unicode
+from udata.utils import get_udata_version, safe_unicode
 
 log = logging.getLogger(__name__)
 
@@ -253,7 +252,7 @@ class UdataGroup(FlaskGroup):
 def print_version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
-    click.echo(pkg_resources.get_distribution("udata").version)
+    click.echo(get_udata_version())
     ctx.exit()
 
 

udata/commands/info.py

@@ -43,9 +43,7 @@ def plugins():
     plugins = current_app.config["PLUGINS"]
     for name, description in entrypoints.ENTRYPOINTS.items():
         echo("{0} ({1})".format(white(description), name))
-        if name == "udata.themes":
-            actives = [current_app.config["THEME"]]
-        elif name == "udata.avatars":
+        if name == "udata.avatars":
             actives = [avatar_config("provider")]
         else:
             actives = plugins

udata/commands/tests/test_fixtures.py

@@ -17,12 +17,12 @@ from udata.core.discussions.factories import DiscussionFactory, MessageDiscussio
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.organization.models import Member
 from udata.core.reuse.factories import ReuseFactory
+from udata.core.spam.models import SpamMixin
 from udata.core.user.factories import UserFactory
+from udata.tests.api import PytestOnlyAPITestCase
 
 
-@pytest.mark.usefixtures("clean_db")
-class FixturesTest:
-    @pytest.mark.frontend
+class FixturesTest(PytestOnlyAPITestCase):
     @pytest.mark.options(FIXTURE_DATASET_SLUGS=["some-test-dataset-slug"])
     def test_generate_fixtures_file_then_import(self, app, cli, api, monkeypatch):
         """Test generating fixtures from the current env, then importing them back."""
@@ -108,7 +108,10 @@ class FixturesTest:
 
     def test_import_fixtures_from_default_file(self, cli):
         """Test importing fixtures from udata.commands.fixture.DEFAULT_FIXTURE_FILE."""
+        # Deactivate spam detection when testing import fixtures
+        SpamMixin.detect_spam_enabled = False
         cli("import-fixtures")
+        SpamMixin.detect_spam_enabled = True
         assert models.Organization.objects.count() > 0
         assert models.Dataset.objects.count() > 0
         assert models.Reuse.objects.count() > 0

udata/core/access_type/api.py

@@ -0,0 +1,18 @@
+from flask import Blueprint
+
+from udata.api import API, api
+from udata.core.access_type.constants import InspireLimitationCategory
+
+blueprint = Blueprint("access_type", __name__, url_prefix="/api/1/access_type")
+ns = api.namespace("access_type", "Access type related operations")
+
+
+@ns.route("/reason_categories/", endpoint="reason_categories")
+class ReasonCategoriesAPI(API):
+    @api.doc("reason_categories")
+    def get(self):
+        """List all limitation reason categories"""
+        return [
+            {"value": category.value, "label": category.label}
+            for category in InspireLimitationCategory
+        ]

udata/core/access_type/constants.py

@@ -0,0 +1,98 @@
+from enum import StrEnum, auto
+from typing import assert_never
+
+from udata.i18n import lazy_gettext as _
+
+
+class AccessType(StrEnum):
+    OPEN = auto()
+    OPEN_WITH_ACCOUNT = auto()
+    RESTRICTED = auto()
+
+
+class AccessAudienceType(StrEnum):
+    ADMINISTRATION = "local_authority_and_administration"
+    COMPANY = "company_and_association"
+    PRIVATE = "private"
+
+
+class AccessAudienceCondition(StrEnum):
+    YES = "yes"
+    NO = "no"
+    UNDER_CONDITIONS = "under_condition"
+
+
+class InspireLimitationCategory(StrEnum):
+    """INSPIRE Directive Article 13(1) limitation reason categories"""
+
+    PUBLIC_AUTHORITIES = "confidentiality_of_proceedings_of_public_authorities"
+    INTERNATIONAL_RELATIONS = "international_relations_public_security_or_national_defence"
+    COURSE_OF_JUSTICE = "course_of_justice_or_fair_trial"
+    COMMERCIAL_CONFIDENTIALITY = "confidentiality_of_commercial_or_industrial_information"
+    INTELLECTUAL_PROPERTY = "intellectual_property_rights"
+    PERSONAL_DATA = "confidentiality_of_personal_data"
+    VOLUNTARY_SUPPLIER = "protection_of_voluntary_information_suppliers"
+    ENVIRONMENTAL_PROTECTION = "protection_of_environment"
+
+    @property
+    def label(self):
+        """Returns the label for this limitation category."""
+        match self:
+            case InspireLimitationCategory.PUBLIC_AUTHORITIES:
+                return _("Confidentiality of public authorities proceedings")
+            case InspireLimitationCategory.INTERNATIONAL_RELATIONS:
+                return _("International relations, public security or national defence")
+            case InspireLimitationCategory.COURSE_OF_JUSTICE:
+                return _("Course of justice")
+            case InspireLimitationCategory.COMMERCIAL_CONFIDENTIALITY:
+                return _("Commercial or industrial confidentiality")
+            case InspireLimitationCategory.INTELLECTUAL_PROPERTY:
+                return _("Intellectual property rights")
+            case InspireLimitationCategory.PERSONAL_DATA:
+                return _("Personal data confidentiality")
+            case InspireLimitationCategory.VOLUNTARY_SUPPLIER:
+                return _("Protection of voluntary information suppliers")
+            case InspireLimitationCategory.ENVIRONMENTAL_PROTECTION:
+                return _("Environmental protection")
+            case _:
+                assert_never(self)
+
+    @property
+    def definition(self):
+        """Returns the definition for this limitation category."""
+        match self:
+            case InspireLimitationCategory.PUBLIC_AUTHORITIES:
+                return _(
+                    "The confidentiality of the proceedings of public authorities, where such confidentiality is provided for by law."
+                )
+            case InspireLimitationCategory.INTERNATIONAL_RELATIONS:
+                return _("International relations, public security or national defence.")
+            case InspireLimitationCategory.COURSE_OF_JUSTICE:
+                return _(
+                    "The course of justice, the ability of any person to receive a fair trial or the ability of a public authority to conduct an enquiry of a criminal or disciplinary nature."
+                )
+            case InspireLimitationCategory.COMMERCIAL_CONFIDENTIALITY:
+                return _(
+                    "The confidentiality of commercial or industrial information, where such confidentiality is provided for by national or Community law to protect a legitimate economic interest."
+                )
+            case InspireLimitationCategory.INTELLECTUAL_PROPERTY:
+                return _("Intellectual property rights.")
+            case InspireLimitationCategory.PERSONAL_DATA:
+                return _(
+                    "The confidentiality of personal data and/or files relating to a natural person where that person has not consented to the disclosure."
+                )
+            case InspireLimitationCategory.VOLUNTARY_SUPPLIER:
+                return _(
+                    "The interests or protection of any person who supplied the information requested on a voluntary basis without being under a legal obligation."
+                )
+            case InspireLimitationCategory.ENVIRONMENTAL_PROTECTION:
+                return _(
+                    "The protection of the environment to which such information relates, such as the location of rare species."
+                )
+            case _:
+                assert_never(self)
+
+    @classmethod
+    def get_labels(cls):
+        """Returns a dictionary of all INSPIRE limitation category labels."""
+        return {member: member.label for member in cls}

udata/core/access_type/models.py

@@ -0,0 +1,44 @@
+from udata.api_fields import field, generate_fields
+from udata.core.access_type.constants import (
+    AccessAudienceCondition,
+    AccessAudienceType,
+    AccessType,
+    InspireLimitationCategory,
+)
+from udata.i18n import lazy_gettext as _
+from udata.models import db
+from udata.mongo.errors import FieldValidationError
+
+
+@generate_fields()
+class AccessAudience(db.EmbeddedDocument):
+    role = field(db.StringField(choices=[e.value for e in AccessAudienceType]), filterable={})
+    condition = field(
+        db.StringField(choices=[e.value for e in AccessAudienceCondition]), filterable={}
+    )
+
+
+def check_only_one_condition_per_role(access_audiences, **_kwargs):
+    roles = set(e["role"] for e in access_audiences)
+    if len(roles) != len(access_audiences):
+        raise FieldValidationError(
+            _("You can only set one condition for a given access audience role"),
+            field="access_audiences",
+        )
+
+
+class WithAccessType:
+    access_type = field(
+        db.EnumField(AccessType, default=AccessType.OPEN),
+        filterable={},
+    )
+    access_audiences = field(
+        db.EmbeddedDocumentListField(AccessAudience),
+        checks=[check_only_one_condition_per_role],
+    )
+
+    authorization_request_url = field(db.URLField())
+    access_type_reason_category = field(
+        db.StringField(choices=[e.value for e in InspireLimitationCategory]), allow_null=True
+    )
+    access_type_reason = field(db.StringField())

udata/core/activity/models.py

@@ -129,7 +129,7 @@ class Auditable(object):
         if kwargs.get("created"):
             cls.on_create.send(document)
         elif len(changed_fields):
-            previous = getattr(document, "_previous_changed_fields", None)
+            previous = getattr(document, "_previous_changed_fields", {})
             # Filter changed_fields since mongoengine raises some false positive occurences
             changed_fields = filter_changed_fields(document, previous, changed_fields)
             if changed_fields:

udata/core/badges/models.py

@@ -57,7 +57,7 @@ class BadgeMixin:
             msg = "Unknown badge type for {model}: {kind}"
             raise db.ValidationError(msg.format(model=self.__class__.__name__, kind=kind))
         badge = self._fields["badges"].field.document_type(kind=kind)
-        if current_user.is_authenticated:
+        if current_user and current_user.is_authenticated:
             badge.created_by = current_user.id
 
         self.update(__raw__={"$push": {"badges": {"$each": [badge.to_mongo()], "$position": 0}}})

udata/core/badges/tasks.py

@@ -1,9 +1,15 @@
-from udata.tasks import get_logger, task
+from typing import Any
+
+from mongoengine.base import TopLevelDocumentMetaclass
+
+from udata.tasks import get_logger, job, task
 
 from .signals import on_badge_added
 
 log = get_logger(__name__)
 
+_badge_jobs: dict[tuple[TopLevelDocumentMetaclass, str], Any] = {}
+
 
 def notify_new_badge(cls, kind):
     def wrapper(func):
@@ -17,3 +23,31 @@ def notify_new_badge(cls, kind):
         return t
 
     return wrapper
+
+
+def register(model: TopLevelDocumentMetaclass, badge: str):
+    """Register a job to update some badge"""
+
+    def inner(func):
+        _badge_jobs[(model, badge)] = func
+        return func
+
+    return inner
+
+
+def get_badge_job(model, badge):
+    return _badge_jobs.get((model, badge))
+
+
+@job(name="update-badges")
+def update_badges(self, badges: list[str] = []) -> None:
+    from udata.core.dataservices.models import Dataservice
+    from udata.models import Dataset, Organization, Reuse
+
+    for model in [Dataset, Reuse, Organization, Dataservice]:
+        for badge in model.__badges__:
+            if badges and badge not in badges:
+                continue
+            if adapter := get_badge_job(model, badge):
+                log.info(f"Running {model.__name__} {badge} job")
+                adapter()