udata 13.0.1.dev12__py3-none-any.whl → 14.0.0__py3-none-any.whl

udata/api/__init__.py

@@ -1,8 +1,6 @@
-import inspect
 import logging
 import urllib.parse
 from functools import wraps
-from importlib import import_module
 
 import mongoengine
 from flask import (
@@ -19,7 +17,7 @@ from flask_restx import Api, Resource
 from flask_restx.reqparse import RequestParser
 from flask_storage import UnauthorizedFileType
 
-from udata import entrypoints, tracking
+from udata import tracking
 from udata.app import csrf
 from udata.auth import Permission, PermissionDenied, RoleNeed, current_user, login_user
 from udata.i18n import get_locale
@@ -358,13 +356,9 @@ def init_app(app):
     import udata.core.contact_point.api  # noqa
     import udata.features.transfer.api  # noqa
     import udata.features.notifications.api  # noqa
-    import udata.features.identicon.api  # noqa
-    import udata.features.territories.api  # noqa
+    import udata.core.avatars.api  # noqa
     import udata.harvest.api  # noqa
 
-    for module in entrypoints.get_enabled("udata.apis", app).values():
-        module if inspect.ismodule(module) else import_module(module)
-
     # api.init_app(app)
     app.register_blueprint(apiv1_blueprint)
     app.register_blueprint(apiv2_blueprint)

udata/app.py

@@ -1,8 +1,8 @@
 import datetime
-import importlib
 import logging
 import os
 import types
+from importlib.metadata import entry_points
 from os.path import abspath, dirname, exists, isfile, join
 
 import bson
@@ -24,7 +24,7 @@ from speaklater import is_lazy_string
 from werkzeug.exceptions import NotFound
 from werkzeug.middleware.proxy_fix import ProxyFix
 
-from udata import cors, entrypoints
+from udata import cors
 
 APP_NAME = __name__.split(".")[0]
 ROOT_DIR = abspath(join(dirname(__file__)))
@@ -148,8 +148,6 @@ def init_logging(app):
     debug = app.debug or app.config.get("TESTING")
     log_level = logging.DEBUG if debug else logging.WARNING
     app.logger.setLevel(log_level)
-    for name in entrypoints.get_roots():  # Entrypoints loggers
-        logging.getLogger(name).setLevel(log_level)
     for logger in VERBOSE_LOGGERS:
         logging.getLogger(logger).setLevel(logging.WARNING)
     return app
@@ -168,20 +166,6 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
     if override:
         app.config.from_object(override)
 
-    # Loads defaults from plugins
-    for pkg in entrypoints.get_roots(app):
-        if pkg == "udata":
-            continue  # Defaults are already loaded
-        module = "{}.settings".format(pkg)
-        try:
-            settings = importlib.import_module(module)
-        except ImportError:
-            continue
-        for key, default in settings.__dict__.items():
-            if key.startswith("__"):
-                continue
-            app.config.setdefault(key, default)
-
     app.json_encoder = UDataJsonEncoder
 
     # `ujson` doesn't support `cls` parameter https://github.com/ultrajson/ultrajson/issues/124
@@ -200,12 +184,21 @@ def create_app(config="udata.settings.Defaults", override=None, init_logging=ini
 def standalone(app):
     """Factory for an all in one application"""
     from udata import api, core, frontend
+    from udata.features import notifications
 
     core.init_app(app)
     frontend.init_app(app)
     api.init_app(app)
+    notifications.init_app(app)
 
-    register_features(app)
+    eps = entry_points(group="udata.plugins")
+    for ep in eps:
+        plugin_module = ep.load()
+
+        if hasattr(plugin_module, "init_app"):
+            plugin_module.init_app(app)
+        else:
+            log.error(f"Plugin {ep.name} ({ep.value}) doesn't expose an `init_app()` function.")
 
     return app
 
@@ -215,7 +208,6 @@ def register_extensions(app):
         auth,
         i18n,
         mail,
-        models,
         mongo,
         notifications,  # noqa
         routing,
@@ -229,7 +221,6 @@ def register_extensions(app):
     tasks.init_app(app)
     i18n.init_app(app)
     mongo.init_app(app)
-    models.init_app(app)
     routing.init_app(app)
     auth.init_app(app)
     cache.init_app(app)
@@ -278,12 +269,3 @@ def page_not_found(e: NotFound):
         return render_template("404.html", homepage_url=homepage_url()), 404
 
     return jsonify({"error": e.description, "status": 404}), 404
-
-
-def register_features(app):
-    from udata.features import notifications
-
-    notifications.init_app(app)
-
-    for ep in entrypoints.get_enabled("udata.plugins", app).values():
-        ep.init_app(app)

udata/auth/forms.py

@@ -54,13 +54,15 @@ class ExtendedRegisterForm(WithCaptcha, RegisterForm):
     )
 
     def validate(self, **kwargs):
-        # no register allowed when read only mode is on
-        if not super().validate(**kwargs) or current_app.config.get("READ_ONLY_MODE"):
+        if current_app.config.get("READ_ONLY_MODE"):
             return False
 
         if not self.validate_captcha():
             return False
 
+        if not super().validate(**kwargs):
+            return False
+
         return True
 
 
@@ -91,10 +93,10 @@ class ExtendedResetPasswordForm(ResetPasswordForm):
 
 class ExtendedForgotPasswordForm(WithCaptcha, ForgotPasswordForm):
     def validate(self, **kwargs):
-        if not super().validate(**kwargs):
+        if not self.validate_captcha():
             return False
 
-        if not self.validate_captcha():
+        if not super().validate(**kwargs):
             return False
 
         return True

udata/commands/__init__.py

@@ -4,12 +4,10 @@ import sys
 from glob import iglob
 
 import click
-import pkg_resources
 from flask.cli import FlaskGroup, ScriptInfo, shell_command
 
-from udata import entrypoints
 from udata.app import VERBOSE_LOGGERS, create_app, standalone
-from udata.utils import safe_unicode
+from udata.utils import get_udata_version, safe_unicode
 
 log = logging.getLogger(__name__)
 
@@ -149,11 +147,6 @@ def init_logging(app):
     logger.handlers = []
     logger.addHandler(handler)
 
-    for name in entrypoints.get_roots():  # Entrypoints loggers
-        logger = logging.getLogger(name)
-        logger.setLevel(log_level)
-        logger.handlers = []
-
     app.logger.setLevel(log_level)
     app.logger.handlers = []
     app.logger.addHandler(handler)
@@ -208,7 +201,6 @@ class UdataGroup(FlaskGroup):
         Load udata commands from:
         - `udata.commands.*` module
         - known internal modules with commands
-        - plugins exporting a `udata.commands` entrypoint
         """
         if self._udata_commands_loaded:
             return
@@ -229,10 +221,6 @@ class UdataGroup(FlaskGroup):
             except Exception as e:
                 error("Unable to import {0}".format(module), e)
 
-        # Load commands from entry points for enabled plugins
-        app = ctx.ensure_object(ScriptInfo).load_app()
-        entrypoints.get_enabled("udata.commands", app)
-
         # Ensure loading happens once
         self._udata_commands_loaded = False
 
@@ -253,7 +241,7 @@ class UdataGroup(FlaskGroup):
 def print_version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
-    click.echo(pkg_resources.get_distribution("udata").version)
+    click.echo(get_udata_version())
     ctx.exit()
 
 

udata/commands/db.py

@@ -11,9 +11,9 @@ import click
 import mongoengine
 from bson import DBRef
 
-from udata import migrations
 from udata.commands import cli, cyan, echo, green, magenta, red, white, yellow
 from udata.core.dataset.models import Dataset, Resource
+from udata.db import migrations
 from udata.mongo.document import get_all_models
 
 # Date format used to for display
@@ -31,8 +31,7 @@ def grp():
 def log_status(migration, status):
     """Properly display a migration status line"""
     name = os.path.splitext(migration.filename)[0]
-    display = ":".join((migration.plugin, name)) + " "
-    log.info("%s [%s]", "{:.<70}".format(display), status)
+    echo("{:.<70} [{}]".format(name + " ", status))
 
 
 def status_label(record):
@@ -78,11 +77,6 @@ def migrate(record, dry_run=False):
             log_status(migration, status)
             try:
                 output = migration.execute(recordonly=record, dryrun=dry_run)
-            except migrations.RollbackError as re:
-                format_output(re.migrate_exc.output, False)
-                log_status(migration, red("Rollback"))
-                format_output(re.output, not re.exc)
-                success = False
             except migrations.MigrationError as me:
                 format_output(me.output, False, traceback=me.traceback)
                 success = False
@@ -92,35 +86,29 @@ def migrate(record, dry_run=False):
 
 
 @grp.command()
-@click.argument("plugin_or_specs")
-@click.argument("filename", default=None, required=False, metavar="[FILENAME]")
-def unrecord(plugin_or_specs, filename):
+@click.argument("filename")
+def unrecord(filename):
     """
     Remove a database migration record.
 
-    \b
-    A record can be expressed with the following syntaxes:
-     - plugin filename
-     - plugin filename.js
-     - plugin:filename
-     - plugin:fliename.js
+    FILENAME is the migration filename (e.g., 2024-01-01-my-migration.py)
     """
-    migration = migrations.get(plugin_or_specs, filename)
-    removed = migration.unrecord()
+    removed = migrations.unrecord(filename)
     if removed:
-        log.info("Removed migration %s", migration.label)
+        echo("Removed migration {}".format(filename))
     else:
-        log.error("Migration not found %s", migration.label)
+        echo(red("Migration not found {}".format(filename)))
 
 
 @grp.command()
-@click.argument("plugin_or_specs")
-@click.argument("filename", default=None, required=False, metavar="[FILENAME]")
-def info(plugin_or_specs, filename):
+@click.argument("filename")
+def info(filename):
     """
     Display detailed info about a migration
+
+    FILENAME is the migration filename (e.g., 2024-01-01-my-migration.py)
     """
-    migration = migrations.get(plugin_or_specs, filename)
+    migration = migrations.get(filename)
     log_status(migration, status_label(migration.record))
     try:
         echo(migration.module.__doc__)

udata/commands/info.py

@@ -3,9 +3,7 @@ import logging
 from click import echo
 from flask import current_app
 
-from udata import entrypoints
 from udata.commands import KO, OK, cli, green, red, white
-from udata.features.identicon.backends import get_config as avatar_config
 
 log = logging.getLogger(__name__)
 
@@ -35,17 +33,3 @@ def config():
         if key.startswith("__") or not key.isupper():
             continue
         echo("{0}: {1}".format(white(key), current_app.config[key]))
-
-
-@grp.command()
-def plugins():
-    """Display some details about the local plugins"""
-    plugins = current_app.config["PLUGINS"]
-    for name, description in entrypoints.ENTRYPOINTS.items():
-        echo("{0} ({1})".format(white(description), name))
-        if name == "udata.avatars":
-            actives = [avatar_config("provider")]
-        else:
-            actives = plugins
-        for ep in sorted(entrypoints.iter_all(name), key=by_name):
-            echo("> {0}: {1}".format(ep.name, is_active(ep, actives)))

udata/core/avatars/api.py

@@ -0,0 +1,43 @@
+import hashlib
+import io
+
+import pydenticon
+from flask import current_app, send_file
+
+from udata.api import API, api
+from udata.app import cache
+
+ns = api.namespace("avatars", "Avatars")
+
+
+@cache.memoize()
+def generate_pydenticon(identifier, size):
+    """
+    Use pydenticon to generate an identicon image.
+    All parameters are extracted from configuration.
+    """
+    blocks_size = current_app.config["AVATAR_INTERNAL_SIZE"]
+    foreground = current_app.config["AVATAR_INTERNAL_FOREGROUND"]
+    background = current_app.config["AVATAR_INTERNAL_BACKGROUND"]
+    generator = pydenticon.Generator(
+        blocks_size, blocks_size, digest=hashlib.sha1, foreground=foreground, background=background
+    )
+
+    # Pydenticon adds padding to the size and as a consequence
+    # we need to compute the size without the padding
+    padding = int(round(current_app.config["AVATAR_INTERNAL_PADDING"] * size / 100.0))
+    size = size - 2 * padding
+    padding = (padding,) * 4
+    return generator.generate(identifier, size, size, padding=padding, output_format="png")
+
+
+@ns.route("/<identifier>/<int:size>/", endpoint="avatar")
+class IdenticonAPI(API):
+    @api.doc("avatars")
+    def get(self, identifier, size):
+        """Get a deterministic avatar given an identifier at a given size"""
+        identicon = generate_pydenticon(identifier, size)
+        response = send_file(io.BytesIO(identicon), mimetype="image/png")
+        etag = hashlib.sha1(identicon).hexdigest()
+        response.set_etag(etag)
+        return response

udata/core/avatars/test_avatar_api.py

@@ -0,0 +1,30 @@
+from flask import url_for
+
+from udata.tests.api import PytestOnlyAPITestCase
+from udata.tests.helpers import assert200
+from udata.utils import faker
+
+
+def assert_stream_equal(response1, response2):
+    __tracebackhide__ = True
+    stream1 = list(response1.iter_encoded())
+    stream2 = list(response2.iter_encoded())
+    assert stream1 == stream2
+
+
+class InternalBackendTest(PytestOnlyAPITestCase):
+    def test_base_rendering(self):
+        response = self.get(url_for("api.avatar", identifier=faker.word(), size=32))
+
+        assert200(response)
+        assert response.mimetype == "image/png"
+        assert response.is_streamed
+        etag, weak = response.get_etag()
+        assert etag is not None
+
+    def test_render_twice_the_same(self):
+        identifier = faker.word()
+        stream_a = self.get(url_for("api.avatar", identifier=identifier, size=32))
+        stream_b = self.get(url_for("api.avatar", identifier=identifier, size=32))
+
+        assert_stream_equal(stream_a, stream_b)

udata/core/dataservices/models.py

@@ -130,7 +130,7 @@ def filter_by_topic(base_query, filter_value):
     try:
         topic = Topic.objects.get(id=filter_value)
     except Topic.DoesNotExist:
-        pass
+        return base_query
     else:
         return base_query.filter(
             id__in=[
@@ -140,11 +140,23 @@ def filter_by_topic(base_query, filter_value):
         )
 
 
+def filter_by_reuse(base_query, filter_value):
+    from udata.core.reuse.models import Reuse
+
+    try:
+        reuse = Reuse.objects.get(id=filter_value)
+    except Reuse.DoesNotExist:
+        return base_query
+    else:
+        return base_query.filter(id__in=[dataservice.id for dataservice in reuse.dataservices])
+
+
 @generate_fields(
     searchable=True,
     nested_filters={"organization_badge": "organization.badges"},
     standalone_filters=[
-        {"key": "topic", "constraints": "objectid", "query": filter_by_topic, "type": str}
+        {"key": "topic", "constraints": ["objectid"], "query": filter_by_topic, "type": str},
+        {"key": "reuse", "constraints": ["objectid"], "query": filter_by_reuse, "type": str},
     ],
     additional_sorts=[
         {"key": "followers", "value": "metrics.followers"},

udata/core/dataset/tasks.py

@@ -1,6 +1,6 @@
 import collections
 import os
-from datetime import datetime
+from datetime import date, datetime
 from tempfile import NamedTemporaryFile
 
 from celery.utils.log import get_task_logger
@@ -17,6 +17,7 @@ from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
 from udata.core.organization.models import Organization
 from udata.harvest.models import HarvestJob
 from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
+from udata.storage.s3 import store_bytes
 from udata.tasks import job
 
 from .models import Checksum, CommunityResource, Dataset, Resource
@@ -90,12 +91,14 @@ def get_queryset(model_cls):
     return model_cls.objects.filter(**params).no_cache()
 
 
+def get_resource_for_csv_export_model(model, dataset):
+    for resource in dataset.resources:
+        if resource.extras.get("csv-export:model", "") == model:
+            return resource
+
+
 def get_or_create_resource(r_info, model, dataset):
-    resource = None
-    for r in dataset.resources:
-        if r.extras.get("csv-export:model", "") == model:
-            resource = r
-            break
+    resource = get_resource_for_csv_export_model(model, dataset)
     if resource:
         for k, v in r_info.items():
             setattr(resource, k, v)
@@ -126,11 +129,16 @@ def store_resource(csvfile, model, dataset):
     return get_or_create_resource(r_info, model, dataset)
 
 
-def export_csv_for_model(model, dataset):
+def export_csv_for_model(model, dataset, replace: bool = False):
     model_cls = getattr(udata_models, model.capitalize(), None)
     if not model_cls:
         log.error("Unknow model %s" % model)
         return
+
+    fs_filename_to_remove = None
+    if existing_resource := get_resource_for_csv_export_model(model, dataset):
+        fs_filename_to_remove = existing_resource.fs_filename
+
     queryset = get_queryset(model_cls)
     adapter = csv.get_adapter(model_cls)
     if not adapter:
@@ -156,6 +164,10 @@ def export_csv_for_model(model, dataset):
         else:
             dataset.last_modified_internal = datetime.utcnow()
             dataset.save()
+        # remove previous catalog if exists and replace is True
+        if replace and fs_filename_to_remove:
+            storages.resources.delete(fs_filename_to_remove)
+        return resource
     finally:
         csvfile.close()
         os.unlink(csvfile.name)
@@ -184,7 +196,23 @@ def export_csv(self, model=None):
 
     models = (model,) if model else ALLOWED_MODELS
     for model in models:
-        export_csv_for_model(model, dataset)
+        resource = export_csv_for_model(model, dataset, replace=True)
+
+        # If we are the first day of the month, archive today catalogs
+        if (
+            current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"]
+            and resource
+            and date.today().day == 1
+        ):
+            log.info(
+                f"Archiving {model} csv catalog on {current_app.config['EXPORT_CSV_ARCHIVE_S3_BUCKET']} bucket"
+            )
+            with storages.resources.open(resource.fs_filename, "rb") as f:
+                store_bytes(
+                    bucket=current_app.config["EXPORT_CSV_ARCHIVE_S3_BUCKET"],
+                    filename=f"{current_app.config['EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX']}{resource.title}",
+                    bytes=f.read(),
+                )
 
 
 @job("bind-tabular-dataservice")

udata/core/metrics/__init__.py

@@ -1,6 +1,3 @@
-from udata import entrypoints
-
-
 def init_app(app):
     # Load all core metrics
     import udata.core.user.metrics  # noqa
@@ -9,6 +6,3 @@ def init_app(app):
     import udata.core.dataset.metrics  # noqa
     import udata.core.reuse.metrics  # noqa
     import udata.core.followers.metrics  # noqa
-
-    # Load metrics from plugins
-    entrypoints.get_enabled("udata.metrics", app)

udata/core/site/models.py

@@ -8,6 +8,7 @@ from udata.core.metrics.helpers import get_metrics_for_model, get_stock_metrics
 from udata.core.organization.models import Organization
 from udata.core.reuse.models import Reuse
 from udata.models import WithMetrics, db
+from udata.utils import get_udata_version
 
 __all__ = ("Site", "SiteSettings")
 
@@ -66,12 +67,7 @@ class Site(WithMetrics, db.Document):
 
     @field(description="The current version of udata")
     def version(self):
-        try:
-            from importlib.metadata import version
-
-            return version("udata")
-        except Exception:
-            return None
+        return get_udata_version()
 
     def count_users(self):
         from udata.models import User

udata/core/spatial/commands.py

@@ -21,10 +21,8 @@ from udata.core.spatial.models import GeoLevel, GeoZone, SpatialCoverage
 log = logging.getLogger(__name__)
 
 
-DEFAULT_GEOZONES_FILE = (
-    "https://www.data.gouv.fr/fr/datasets/r/a1bb263a-6cc7-4871-ab4f-2470235a67bf"
-)
-DEFAULT_LEVELS_FILE = "https://www.data.gouv.fr/fr/datasets/r/e0206442-78b3-4a00-b71c-c065d20561c8"
+DEFAULT_GEOZONES_FILE = "https://www.data.gouv.fr/datasets/r/a1bb263a-6cc7-4871-ab4f-2470235a67bf"
+DEFAULT_LEVELS_FILE = "https://www.data.gouv.fr/datasets/r/e0206442-78b3-4a00-b71c-c065d20561c8"
 
 
 @cli.group("spatial")

udata/core/spatial/models.py

@@ -1,5 +1,4 @@
 import geojson
-from flask import current_app
 from werkzeug.local import LocalProxy
 from werkzeug.utils import cached_property
 
@@ -85,10 +84,6 @@ class GeoZone(WithMetrics, db.Document):
                 return name
         return self.level_name  # Fallback that should never happen.
 
-    @property
-    def handled_level(self):
-        return self.level in current_app.config.get("HANDLED_LEVELS")
-
     @property
     def url(self):
         return None
@@ -158,11 +153,6 @@ class SpatialCoverage(db.EmbeddedDocument):
                 continue
         return _(top.name)
 
-    @property
-    def handled_zones(self):
-        """Return only zones with a dedicated page."""
-        return [zone for zone in self.zones if zone.handled_level]
-
     def clean(self):
         if self.zones and self.geom:
             raise db.ValidationError(

udata/core/spatial/tests/test_api.py

@@ -1,4 +1,3 @@
-import pytest
 from flask import url_for
 
 from udata.core.dataset.factories import DatasetFactory
@@ -13,9 +12,7 @@ from udata.core.spatial.models import spatial_granularities
 from udata.core.spatial.tasks import compute_geozones_metrics
 from udata.tests.api import APITestCase
 from udata.tests.api.test_datasets_api import SAMPLE_GEOM
-from udata.tests.features.territories import (
-    create_geozones_fixtures,
-)
+from udata.tests.helpers import create_geozones_fixtures
 from udata.utils import faker
 
 
@@ -258,38 +255,6 @@ class SpatialApiTest(APITestCase):
         self.assertEqual(response.json["features"][1]["properties"]["datasets"], 3)
 
 
-@pytest.mark.options(
-    ACTIVATE_TERRITORIES=True,
-    HANDLED_LEVELS=("fr:commune", "fr:departement", "fr:region", "country"),
-)
-class SpatialTerritoriesApiTest(APITestCase):
-    def test_zone_datasets_with_dynamic_and_setting(self):
-        paca, bdr, arles = create_geozones_fixtures()
-        organization = OrganizationFactory()
-        for _ in range(3):
-            DatasetFactory(
-                organization=organization, spatial=SpatialCoverageFactory(zones=[paca.id])
-            )
-
-        response = self.get(url_for("api.zone_datasets", id=paca.id, dynamic=1))
-        self.assert200(response)
-        # No dynamic datasets given that they are added by udata-front extension.
-        self.assertEqual(len(response.json), 3)
-
-    def test_zone_datasets_with_dynamic_and_setting_and_size(self):
-        paca, bdr, arles = create_geozones_fixtures()
-        organization = OrganizationFactory()
-        for _ in range(3):
-            DatasetFactory(
-                organization=organization, spatial=SpatialCoverageFactory(zones=[paca.id])
-            )
-
-        response = self.get(url_for("api.zone_datasets", id=paca.id, dynamic=1, size=2))
-        self.assert200(response)
-        # No dynamic datasets given that they are added by udata-front extension.
-        self.assertEqual(len(response.json), 2)
-
-
 class DatasetsSpatialAPITest(APITestCase):
     def test_create_spatial_zones(self):
         paca, _, _ = create_geozones_fixtures()

udata/core/user/models.py

@@ -102,7 +102,16 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
     on_delete = Signal()
 
     meta = {
-        "indexes": ["$slug", "-created_at", "slug", "apikey"],
+        "indexes": [
+            {
+                "fields": ["$last_name", "$first_name", "$email"],
+                "default_language": "french",
+                "weights": {"last_name": 10, "email": 10, "first_name": 5},
+            },
+            "-created_at",
+            "slug",
+            "apikey",
+        ],
         "ordering": ["-created_at"],
         "auto_create_index_on_save": True,
     }