udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

udata/api/__init__.py

@@ -1,8 +1,6 @@
-import inspect
 import logging
 import urllib.parse
 from functools import wraps
-from importlib import import_module
 
 import mongoengine
 from flask import (
@@ -19,7 +17,7 @@ from flask_restx import Api, Resource
 from flask_restx.reqparse import RequestParser
 from flask_storage import UnauthorizedFileType
 
-from udata import entrypoints, tracking
+from udata import tracking
 from udata.app import csrf
 from udata.auth import Permission, PermissionDenied, RoleNeed, current_user, login_user
 from udata.i18n import get_locale
@@ -358,13 +356,9 @@ def init_app(app):
     import udata.core.contact_point.api  # noqa
     import udata.features.transfer.api  # noqa
     import udata.features.notifications.api  # noqa
-    import udata.features.identicon.api  # noqa
-    import udata.features.territories.api  # noqa
+    import udata.core.avatars.api  # noqa
     import udata.harvest.api  # noqa
 
-    for module in entrypoints.get_enabled("udata.apis", app).values():
-        module if inspect.ismodule(module) else import_module(module)
-
     # api.init_app(app)
     app.register_blueprint(apiv1_blueprint)
     app.register_blueprint(apiv2_blueprint)

udata/api_fields.py

@@ -29,7 +29,7 @@ import flask_restx.fields as restx_fields
 import mongoengine
 import mongoengine.fields as mongo_fields
 from bson import DBRef, ObjectId
-from flask import Request
+from flask import Request, request
 from flask_restx import marshal
 from flask_restx.inputs import boolean
 from flask_restx.reqparse import RequestParser
@@ -55,8 +55,8 @@ classes_by_parents = {}
 
 
 class GenericField(restx_fields.Raw):
-    def __init__(self, fields_by_type):
-        super().__init__(self)
+    def __init__(self, fields_by_type, **kwargs):
+        super(GenericField, self).__init__(**kwargs)
         self.default = None
         self.fields_by_type = fields_by_type
 
@@ -230,6 +230,23 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
 
         write_params["description"] = "ID of the reference"
         constructor_write = restx_fields.String
+    elif isinstance(field, mongo_fields.GenericEmbeddedDocumentField):
+        generic_fields = {
+            cls.__name__: convert_db_to_field(
+                f"{key}.{cls.__name__}",
+                # Instead of having GenericEmbeddedDocumentField() we'll create fields for each
+                # of the subclasses with EmbededdDocumentField(MembershipRequestNotificationDetails)…
+                mongoengine.fields.EmbeddedDocumentField(cls),
+                info,
+            )
+            for cls in field.choices
+        }
+
+        def constructor_read(**kwargs):
+            return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
+
+        def constructor_write(**kwargs):
+            return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
     elif isinstance(field, mongo_fields.EmbeddedDocumentField):
         nested_fields = info.get("nested_fields")
         if nested_fields is not None:
@@ -322,12 +339,12 @@ def generate_fields(**kwargs) -> Callable:
         write_fields: dict = {}
         ref_fields: dict = {}
         sortables: list = kwargs.get("additional_sorts", [])
+        default_sort: list = kwargs.get("default_sort", None)
 
         filterables: list[dict] = kwargs.get("standalone_filters", [])
         nested_filters: dict[str, dict] = get_fields_with_nested_filters(
             kwargs.get("nested_filters", {})
         )
-
         if issubclass(cls, db.Document) or issubclass(cls, db.DynamicDocument):
             read_fields["id"] = restx_fields.String(required=True, readonly=True)
 
@@ -472,6 +489,7 @@ def generate_fields(**kwargs) -> Callable:
                 type=str,
                 location="args",
                 choices=choices,
+                default=default_sort,
                 help="The field (and direction) on which sorting apply",
             )
 
@@ -511,6 +529,9 @@ def generate_fields(**kwargs) -> Callable:
                 phrase_query: str = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
                 base_query = base_query.search_text(phrase_query)
 
+                if "sort" not in request.args:
+                    base_query = base_query.order_by("$text_score")
+
             for filterable in filterables:
                 # If it's from an `nested_filter`, use the custom label instead of the key,
                 # eg use `organization_badge` instead of `organization.badges` which is
@@ -652,6 +673,9 @@ def patch(obj, request) -> type:
             model_attribute = getattr(obj.__class__, key)
             info = getattr(model_attribute, "__additional_field_info__", {})
 
+            if value == "" and isinstance(model_attribute, mongo_fields.StringField):
+                value = None
+
             if hasattr(model_attribute, "from_input"):
                 value = model_attribute.from_input(value)
             elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
@@ -677,6 +701,13 @@ def patch(obj, request) -> type:
                     value["id"],
                     document_type=db.resolve_model(value["class"]),
                 )
+            elif value and isinstance(
+                model_attribute,
+                mongoengine.fields.GenericEmbeddedDocumentField,
+            ):
+                generic_key = info.get("generic_key", DEFAULT_GENERIC_KEY)
+                embedded_field = classes_by_names[value[generic_key]]
+                value = patch(embedded_field(), value)
             elif value and isinstance(
                 model_attribute,
                 mongoengine.fields.EmbeddedDocumentField,

udata/app.py

@@ -1,30 +1,32 @@
-import datetime
-import importlib
 import logging
 import os
 import types
+from datetime import datetime
+from importlib.metadata import entry_points
 from os.path import abspath, dirname, exists, isfile, join
 
 import bson
+from bson import json_util
 from flask import Blueprint as BaseBlueprint
 from flask import (
     Flask,
     abort,
     g,
-    json,
     jsonify,
     make_response,
     render_template,
     request,
     send_from_directory,
 )
+from flask.json.provider import DefaultJSONProvider
 from flask_caching import Cache
 from flask_wtf.csrf import CSRFProtect
-from speaklater import is_lazy_string
+from mongoengine import EmbeddedDocument
+from mongoengine.base import BaseDocument
 from werkzeug.exceptions import NotFound
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from udata import cors, entrypoints
+from udata import cors
 
 APP_NAME = __name__.split(".")[0]
 ROOT_DIR = abspath(join(dirname(__file__)))
@@ -108,9 +110,9 @@ class Blueprint(BaseBlueprint):
         return wrapper
 
 
-class UDataJsonEncoder(json.JSONEncoder):
+class UdataJsonProvider(DefaultJSONProvider):
     """
-    A JSONEncoder subclass to encode unsupported types:
+    A JSONProvider subclass to encode unsupported types:
 
         - ObjectId
         - datetime
@@ -120,21 +122,16 @@ class UDataJsonEncoder(json.JSONEncoder):
     Ensure an app context is always present.
     """
 
-    def default(self, obj):
-        if is_lazy_string(obj):
+    @staticmethod
+    def default(obj):
+        if isinstance(obj, BaseDocument) or isinstance(obj, EmbeddedDocument):
+            return json_util._json_convert(obj.to_mongo())
+        elif isinstance(obj, bson.ObjectId):
             return str(obj)
-        elif isinstance(obj, bson.objectid.ObjectId):
-            return str(obj)
-        elif isinstance(obj, datetime.datetime):
+        elif isinstance(obj, datetime):
             return obj.isoformat()
-        elif hasattr(obj, "to_dict"):
-            return obj.to_dict()
-        elif hasattr(obj, "serialize"):
-            return obj.serialize()
-        # Serialize Raw data for Document and EmbeddedDocument.
-        elif hasattr(obj, "_data"):
-            return obj._data
-        return super(UDataJsonEncoder, self).default(obj)
+
+        return super(UdataJsonProvider, UdataJsonProvider).default(obj)
 
 
 # These loggers are very verbose
@@ -148,8 +145,6 @@ def init_logging(app):
     debug = app.debug or app.config.get("TESTING")
     log_level = logging.DEBUG if debug else logging.WARNING
     app.logger.setLevel(log_level)
-    for name in entrypoints.get_roots():  # Entrypoints loggers
-        logging.getLogger(name).setLevel(log_level)
     for logger in VERBOSE_LOGGERS:
         logging.getLogger(logger).setLevel(logging.WARNING)
     return app
@@ -168,24 +163,11 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
     if override:
         app.config.from_object(override)
 
-    # Loads defaults from plugins
-    for pkg in entrypoints.get_roots(app):
-        if pkg == "udata":
-            continue  # Defaults are already loaded
-        module = "{}.settings".format(pkg)
-        try:
-            settings = importlib.import_module(module)
-        except ImportError:
-            continue
-        for key, default in settings.__dict__.items():
-            if key.startswith("__"):
-                continue
-            app.config.setdefault(key, default)
-
-    app.json_encoder = UDataJsonEncoder
+    app.json_provider_class = UdataJsonProvider
+    app.json = app.json_provider_class(app)
 
     # `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
-    app.config["RESTX_JSON"] = {"cls": UDataJsonEncoder}
+    app.config["RESTX_JSON"] = {"default": UdataJsonProvider.default}
 
     app.debug = app.config["DEBUG"] and not app.config["TESTING"]
 
@@ -200,12 +182,21 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
 def standalone(app):
     """Factory for an all in one application"""
     from udata import api, core, frontend
+    from udata.features import notifications
 
     core.init_app(app)
     frontend.init_app(app)
     api.init_app(app)
+    notifications.init_app(app)
+
+    eps = entry_points(group="udata.plugins")
+    for ep in eps:
+        plugin_module = ep.load()
 
-    register_features(app)
+        if hasattr(plugin_module, "init_app"):
+            plugin_module.init_app(app)
+        else:
+            log.error(f"Plugin {ep.name} ({ep.value}) doesn't expose an `init_app()` function.")
 
     return app
 
@@ -215,7 +206,6 @@ def register_extensions(app):
         auth,
         i18n,
         mail,
-        models,
         mongo,
         notifications,  # noqa
         routing,
@@ -229,7 +219,6 @@ def register_extensions(app):
     tasks.init_app(app)
     i18n.init_app(app)
     mongo.init_app(app)
-    models.init_app(app)
     routing.init_app(app)
     auth.init_app(app)
     cache.init_app(app)
@@ -278,12 +267,3 @@ def page_not_found(e: NotFound):
         return render_template("404.html", homepage_url=homepage_url()), 404
 
     return jsonify({"error": e.description, "status": 404}), 404
-
-
-def register_features(app):
-    from udata.features import notifications
-
-    notifications.init_app(app)
-
-    for ep in entrypoints.get_enabled("udata.plugins", app).values():
-        ep.init_app(app)

udata/auth/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import typing as t
 
 from flask import render_template
 from flask_principal import Permission as BasePermission
@@ -10,6 +11,7 @@ from flask_security import Security as Security
 from flask_security import current_user as current_user
 from flask_security import login_required as login_required
 from flask_security import login_user as login_user
+from flask_security import mail_util
 
 from . import mails
 
@@ -24,9 +26,6 @@ def render_security_template(template_name_or_list, **kwargs):
     return render_template(template_name_or_list, **kwargs)
 
 
-security = Security()
-
-
 class Permission(BasePermission):
     def __init__(self, *needs):
         """Let administrator bypass all permissions"""
@@ -36,6 +35,24 @@ class Permission(BasePermission):
 admin_permission = Permission()
 
 
+class NoopMailUtil(mail_util.MailUtil):
+    def send_mail(
+        self,
+        template: str,
+        subject: str,
+        recipient: str,
+        sender: t.Union[str, tuple],
+        body: str,
+        html: t.Optional[str],
+        **kwargs: t.Any,
+    ) -> None:
+        log.debug(f"Sending mail {subject} to {recipient}")
+        log.debug(body)
+        log.debug(html)
+
+        return None
+
+
 def init_app(app):
     from udata.models import datastore
 
@@ -61,19 +78,25 @@ def init_app(app):
             app.config["CDATA_BASE_URL"] + "?flash=confirm_error",
         )
 
-    security.init_app(
-        app,
+    # Same logic as in our own mail system :DisableMail
+    debug = app.config.get("DEBUG", False)
+    send_mail = app.config.get("SEND_MAIL", not debug)
+    mail_util_cls = mail_util.MailUtil if send_mail else NoopMailUtil
+
+    security = Security(
         datastore,
         register_blueprint=False,
         render_template=render_security_template,
         login_form=ExtendedLoginForm,
-        confirm_register_form=ExtendedRegisterForm,
         register_form=ExtendedRegisterForm,
         reset_password_form=ExtendedResetPasswordForm,
         forgot_password_form=ExtendedForgotPasswordForm,
         password_util_cls=UdataPasswordUtil,
+        mail_util_cls=mail_util_cls,
     )
 
+    security.init_app(app, datastore, register_blueprint=False)
+
     security_bp = create_security_blueprint(app, app.extensions["security"], "security_blueprint")
 
     app.register_blueprint(security_bp)

udata/auth/forms.py

@@ -8,7 +8,7 @@ from flask_security.forms import (
     ForgotPasswordForm,
     Form,
     LoginForm,
-    RegisterForm,
+    RegisterFormV2,
     ResetPasswordForm,
 )
 
@@ -31,7 +31,7 @@ class WithCaptcha:
         return False
 
 
-class ExtendedRegisterForm(WithCaptcha, RegisterForm):
+class ExtendedRegisterForm(WithCaptcha, RegisterFormV2):
     first_name = fields.StringField(
         _("First name"),
         [
@@ -54,13 +54,15 @@ class ExtendedRegisterForm(WithCaptcha, RegisterForm):
     )
 
     def validate(self, **kwargs):
-        # no register allowed when read only mode is on
-        if not super().validate(**kwargs) or current_app.config.get("READ_ONLY_MODE"):
+        if current_app.config.get("READ_ONLY_MODE"):
             return False
 
         if not self.validate_captcha():
             return False
 
+        if not super().validate(**kwargs):
+            return False
+
         return True
 
 
@@ -91,10 +93,10 @@ class ExtendedResetPasswordForm(ResetPasswordForm):
 
 class ExtendedForgotPasswordForm(WithCaptcha, ForgotPasswordForm):
     def validate(self, **kwargs):
-        if not super().validate(**kwargs):
+        if not self.validate_captcha():
             return False
 
-        if not self.validate_captcha():
+        if not super().validate(**kwargs):
             return False
 
         return True

udata/auth/views.py

@@ -84,6 +84,11 @@ def confirm_change_email(token):
     if flash:
         return redirect(homepage_url(flash=flash, flash_data=flash_data))
 
+    # Check if the new email is already taken by another user
+    existing_user = _datastore.find_user(email=new_email)
+    if existing_user and existing_user.id != user.id:
+        return redirect(homepage_url(flash="change_email_already_taken"))
+
     if user != current_user:
         logout_user()
         login_user(user)
@@ -140,10 +145,8 @@ def create_security_blueprint(app, state, import_name):
     This creates an I18nBlueprint to use as a base.
     """
     bp = I18nBlueprint(
-        state.blueprint_name,
+        "security",
         import_name,
-        url_prefix=state.url_prefix,
-        subdomain=state.subdomain,
         template_folder="templates",
     )
 

udata/commands/__init__.py

@@ -4,12 +4,10 @@ import sys
 from glob import iglob
 
 import click
-import pkg_resources
 from flask.cli import FlaskGroup, ScriptInfo, shell_command
 
-from udata import entrypoints
 from udata.app import VERBOSE_LOGGERS, create_app, standalone
-from udata.utils import safe_unicode
+from udata.utils import get_udata_version, safe_unicode
 
 log = logging.getLogger(__name__)
 
@@ -149,11 +147,6 @@ def init_logging(app):
     logger.handlers = []
     logger.addHandler(handler)
 
-    for name in entrypoints.get_roots():  # Entrypoints loggers
-        logger = logging.getLogger(name)
-        logger.setLevel(log_level)
-        logger.handlers = []
-
     app.logger.setLevel(log_level)
     app.logger.handlers = []
     app.logger.addHandler(handler)
@@ -208,7 +201,6 @@ class UdataGroup(FlaskGroup):
         Load udata commands from:
         - `udata.commands.*` module
         - known internal modules with commands
-        - plugins exporting a `udata.commands` entrypoint
         """
         if self._udata_commands_loaded:
             return
@@ -229,10 +221,6 @@ class UdataGroup(FlaskGroup):
             except Exception as e:
                 error("Unable to import {0}".format(module), e)
 
-        # Load commands from entry points for enabled plugins
-        app = ctx.ensure_object(ScriptInfo).load_app()
-        entrypoints.get_enabled("udata.commands", app)
-
         # Ensure loading happens once
         self._udata_commands_loaded = False
 
@@ -253,7 +241,7 @@ class UdataGroup(FlaskGroup):
 def print_version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
-    click.echo(pkg_resources.get_distribution("udata").version)
+    click.echo(get_udata_version())
     ctx.exit()
 
 

udata/commands/db.py

@@ -11,9 +11,9 @@ import click
 import mongoengine
 from bson import DBRef
 
-from udata import migrations
 from udata.commands import cli, cyan, echo, green, magenta, red, white, yellow
 from udata.core.dataset.models import Dataset, Resource
+from udata.db import migrations
 from udata.mongo.document import get_all_models
 
 # Date format used to for display
@@ -31,8 +31,7 @@ def grp():
 def log_status(migration, status):
     """Properly display a migration status line"""
     name = os.path.splitext(migration.filename)[0]
-    display = ":".join((migration.plugin, name)) + " "
-    log.info("%s [%s]", "{:.<70}".format(display), status)
+    echo("{:.<70} [{}]".format(name + " ", status))
 
 
 def status_label(record):
@@ -78,11 +77,6 @@ def migrate(record, dry_run=False):
             log_status(migration, status)
             try:
                 output = migration.execute(recordonly=record, dryrun=dry_run)
-            except migrations.RollbackError as re:
-                format_output(re.migrate_exc.output, False)
-                log_status(migration, red("Rollback"))
-                format_output(re.output, not re.exc)
-                success = False
             except migrations.MigrationError as me:
                 format_output(me.output, False, traceback=me.traceback)
                 success = False
@@ -92,35 +86,29 @@ def migrate(record, dry_run=False):
 
 
 @grp.command()
-@click.argument("plugin_or_specs")
-@click.argument("filename", default=None, required=False, metavar="[FILENAME]")
-def unrecord(plugin_or_specs, filename):
+@click.argument("filename")
+def unrecord(filename):
     """
     Remove a database migration record.
 
-    \b
-    A record can be expressed with the following syntaxes:
-     - plugin filename
-     - plugin filename.js
-     - plugin:filename
-     - plugin:fliename.js
+    FILENAME is the migration filename (e.g., 2024-01-01-my-migration.py)
     """
-    migration = migrations.get(plugin_or_specs, filename)
-    removed = migration.unrecord()
+    removed = migrations.unrecord(filename)
     if removed:
-        log.info("Removed migration %s", migration.label)
+        echo("Removed migration {}".format(filename))
     else:
-        log.error("Migration not found %s", migration.label)
+        echo(red("Migration not found {}".format(filename)))
 
 
 @grp.command()
-@click.argument("plugin_or_specs")
-@click.argument("filename", default=None, required=False, metavar="[FILENAME]")
-def info(plugin_or_specs, filename):
+@click.argument("filename")
+def info(filename):
     """
     Display detailed info about a migration
+
+    FILENAME is the migration filename (e.g., 2024-01-01-my-migration.py)
     """
-    migration = migrations.get(plugin_or_specs, filename)
+    migration = migrations.get(filename)
     log_status(migration, status_label(migration.record))
     try:
         echo(migration.module.__doc__)

udata/commands/info.py

@@ -3,9 +3,7 @@ import logging
 from click import echo
 from flask import current_app
 
-from udata import entrypoints
 from udata.commands import KO, OK, cli, green, red, white
-from udata.features.identicon.backends import get_config as avatar_config
 
 log = logging.getLogger(__name__)
 
@@ -35,17 +33,3 @@ def config():
         if key.startswith("__") or not key.isupper():
             continue
         echo("{0}: {1}".format(white(key), current_app.config[key]))
-
-
-@grp.command()
-def plugins():
-    """Display some details about the local plugins"""
-    plugins = current_app.config["PLUGINS"]
-    for name, description in entrypoints.ENTRYPOINTS.items():
-        echo("{0} ({1})".format(white(description), name))
-        if name == "udata.avatars":
-            actives = [avatar_config("provider")]
-        else:
-            actives = plugins
-        for ep in sorted(entrypoints.iter_all(name), key=by_name):
-            echo("> {0}: {1}".format(ep.name, is_active(ep, actives)))

udata/commands/serve.py

@@ -3,7 +3,7 @@ import os
 
 import click
 from flask import current_app
-from flask.cli import DispatchingApp, pass_script_info
+from flask.cli import pass_script_info
 from werkzeug.serving import run_simple
 
 from udata.commands import cli
@@ -26,17 +26,11 @@ log = logging.getLogger(__name__)
     default=None,
     help="Enable or disable the debugger.  By default the debugger is active if debug is enabled.",
 )
-@click.option(
-    "--eager-loading/--lazy-loader",
-    default=None,
-    help="Enable or disable eager loading.  By default eager "
-    "loading is enabled if the reloader is disabled.",
-)
 @click.option(
     "--with-threads/--without-threads", default=True, help="Enable or disable multithreading."
 )
 @pass_script_info
-def serve(info, host, port, reload, debugger, eager_loading, with_threads):
+def serve(info, host, port, reload, debugger, with_threads):
     """
     Runs a local udata development server.
 
@@ -62,10 +56,8 @@ def serve(info, host, port, reload, debugger, eager_loading, with_threads):
         reload = bool(debug)
     if debugger is None:
         debugger = bool(debug)
-    if eager_loading is None:
-        eager_loading = not reload
 
-    app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
+    app = info.load_app()
 
     settings = os.environ.get("UDATA_SETTINGS", os.path.join(os.getcwd(), "udata.cfg"))
     extra_files = [settings]