udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

udata/core/spatial/models.py

@@ -1,5 +1,4 @@
 import geojson
-from flask import current_app
 from werkzeug.local import LocalProxy
 from werkzeug.utils import cached_property
 
@@ -85,10 +84,6 @@ class GeoZone(WithMetrics, db.Document):
                 return name
         return self.level_name  # Fallback that should never happen.
 
-    @property
-    def handled_level(self):
-        return self.level in current_app.config.get("HANDLED_LEVELS")
-
     @property
     def url(self):
         return None
@@ -158,11 +153,6 @@ class SpatialCoverage(db.EmbeddedDocument):
                 continue
         return _(top.name)
 
-    @property
-    def handled_zones(self):
-        """Return only zones with a dedicated page."""
-        return [zone for zone in self.zones if zone.handled_level]
-
     def clean(self):
         if self.zones and self.geom:
             raise db.ValidationError(

udata/core/spatial/tests/test_api.py

@@ -1,4 +1,3 @@
-import pytest
 from flask import url_for
 
 from udata.core.dataset.factories import DatasetFactory
@@ -13,9 +12,7 @@ from udata.core.spatial.models import spatial_granularities
 from udata.core.spatial.tasks import compute_geozones_metrics
 from udata.tests.api import APITestCase
 from udata.tests.api.test_datasets_api import SAMPLE_GEOM
-from udata.tests.features.territories import (
-    create_geozones_fixtures,
-)
+from udata.tests.helpers import create_geozones_fixtures
 from udata.utils import faker
 
 
@@ -258,38 +255,6 @@ class SpatialApiTest(APITestCase):
         self.assertEqual(response.json["features"][1]["properties"]["datasets"], 3)
 
 
-@pytest.mark.options(
-    ACTIVATE_TERRITORIES=True,
-    HANDLED_LEVELS=("fr:commune", "fr:departement", "fr:region", "country"),
-)
-class SpatialTerritoriesApiTest(APITestCase):
-    def test_zone_datasets_with_dynamic_and_setting(self):
-        paca, bdr, arles = create_geozones_fixtures()
-        organization = OrganizationFactory()
-        for _ in range(3):
-            DatasetFactory(
-                organization=organization, spatial=SpatialCoverageFactory(zones=[paca.id])
-            )
-
-        response = self.get(url_for("api.zone_datasets", id=paca.id, dynamic=1))
-        self.assert200(response)
-        # No dynamic datasets given that they are added by udata-front extension.
-        self.assertEqual(len(response.json), 3)
-
-    def test_zone_datasets_with_dynamic_and_setting_and_size(self):
-        paca, bdr, arles = create_geozones_fixtures()
-        organization = OrganizationFactory()
-        for _ in range(3):
-            DatasetFactory(
-                organization=organization, spatial=SpatialCoverageFactory(zones=[paca.id])
-            )
-
-        response = self.get(url_for("api.zone_datasets", id=paca.id, dynamic=1, size=2))
-        self.assert200(response)
-        # No dynamic datasets given that they are added by udata-front extension.
-        self.assertEqual(len(response.json), 2)
-
-
 class DatasetsSpatialAPITest(APITestCase):
     def test_create_spatial_zones(self):
         paca, _, _ = create_geozones_fixtures()

udata/core/user/models.py

@@ -102,7 +102,16 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
     on_delete = Signal()
 
     meta = {
-        "indexes": ["$slug", "-created_at", "slug", "apikey"],
+        "indexes": [
+            {
+                "fields": ["$last_name", "$first_name", "$email"],
+                "default_language": "french",
+                "weights": {"last_name": 10, "email": 10, "first_name": 5},
+            },
+            "-created_at",
+            "slug",
+            "apikey",
+        ],
         "ordering": ["-created_at"],
         "auto_create_index_on_save": True,
     }
@@ -133,7 +142,7 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         return self.has_role("admin")
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/users/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/users/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(
@@ -288,6 +297,10 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
                 discussion.save()
         Follow.objects(follower=self).delete()
         Follow.objects(following=self).delete()
+        # Remove related notifications
+        from udata.features.notifications.models import Notification
+
+        Notification.objects.with_user_in_details(self).delete()
 
         from udata.models import ContactPoint
 

udata/cors.py

@@ -1,6 +1,6 @@
 import logging
 
-from flask import current_app, g, request
+from flask import current_app, request
 from werkzeug.datastructures import Headers
 
 log = logging.getLogger(__name__)
@@ -32,10 +32,7 @@ def is_preflight_request() -> bool:
 
 
 def is_allowed_cors_route():
-    if g and hasattr(g, "lang_code"):
-        path: str = request.path.removeprefix(f"/{g.lang_code}")
-    else:
-        path: str = request.path
+    path: str = request.path
 
     # Allow to keep clean CORS when `udata` and the frontend are on the same domain
     # (as it's the case in data.gouv with cdata/udata).

udata/db/migrations.py

@@ -0,0 +1,279 @@
+"""
+Data migrations logic
+"""
+
+import importlib.util
+import inspect
+import logging
+import os
+import queue
+import traceback
+from datetime import datetime
+from logging.handlers import QueueHandler
+
+from mongoengine.connection import get_db
+from pymongo import ReturnDocument
+
+log = logging.getLogger(__name__)
+
+
+class MigrationError(Exception):
+    """
+    Raised on migration execution error.
+
+    :param msg str: A human readable message (a reason)
+    :param output str: An optionnal array of logging output
+    :param exc Exception: An optionnal underlying exception
+    """
+
+    def __init__(self, msg, output=None, exc=None, traceback=None):
+        super().__init__(msg)
+        self.msg = msg
+        self.output = output
+        self.exc = exc
+        self.traceback = traceback
+
+
+class Record(dict):
+    """
+    A simple wrapper to migrations document
+    """
+
+    __getattr__ = dict.get
+
+    def load(self):
+        specs = {"filename": self["filename"]}
+        self.clear()
+        data = get_db().migrations.find_one(specs)
+        self.update(data or specs)
+
+    def exists(self):
+        return bool(self._id)
+
+    def __bool__(self):
+        return self.exists()
+
+    @property
+    def collection(self):
+        return get_db().migrations
+
+    @property
+    def status(self):
+        """
+        Status is the status of the last operation.
+
+        Will be `None` if the record doesn't exist.
+        Returns "success" or "error".
+        """
+        if not self.exists():
+            return None
+        op = self.ops[-1]
+        return "success" if op["success"] else "error"
+
+    @property
+    def last_date(self):
+        if not self.exists():
+            return
+        op = self.ops[-1]
+        return op["date"]
+
+    @property
+    def ok(self):
+        """
+        Is true if the migration is considered as successfully applied
+        """
+        if not self.exists():
+            return False
+        op = self.ops[-1]
+        return op["success"] and op["type"] in ("migrate", "record")
+
+    def add(self, _type, migration, output, state, success):
+        script = inspect.getsource(migration)
+        return Record(
+            self.collection.find_one_and_update(
+                {"filename": self.filename},
+                {
+                    "$push": {
+                        "ops": {
+                            "date": datetime.utcnow(),
+                            "type": _type,
+                            "script": script,
+                            "output": output,
+                            "state": state,
+                            "success": success,
+                        }
+                    }
+                },
+                upsert=True,
+                return_document=ReturnDocument.AFTER,
+            )
+        )
+
+    def delete(self):
+        return self.collection.delete_one({"_id": self._id})
+
+
+class Migration:
+    def __init__(self, filename):
+        self.filename = filename
+        self._record = None
+        # Load module immediately - migration must exist on disk
+        module = load_migration(self.filename)
+        if module is None:
+            raise FileNotFoundError(f"Migration {self.filename} file not found")
+        # Extract and store the migrate function
+        if not hasattr(module, "migrate"):
+            raise MigrationError(
+                f"Migration {self.filename} is missing required migrate() function"
+            )
+        self.module = module
+        self.migrate = module.migrate
+
+    @property
+    def collection(self):
+        return get_db().migrations
+
+    @property
+    def db_query(self):
+        return {"filename": self.filename}
+
+    @property
+    def label(self):
+        return self.filename
+
+    @property
+    def record(self):
+        if self._record is None:
+            specs = {"filename": self.filename}
+            data = get_db().migrations.find_one(specs)
+            self._record = Record(data or specs)
+        return self._record
+
+    def __eq__(self, value):
+        return isinstance(value, Migration) and getattr(value, "filename") == self.filename
+
+    def execute(self, recordonly=False, dryrun=False):
+        """
+        Execute a migration
+
+        If recordonly is True, the migration is only recorded
+        If dryrun is True, the migration is neither executed nor recorded
+        """
+        q = queue.Queue()
+        logger = getattr(self.module, "log", logging.getLogger(self.module.__name__))
+
+        # Logs only go to the queue handler are not shown.
+        # They will be formatted below to be shown all at once at the end
+        # of the migration.
+        logger.addHandler(QueueHandler(q))
+        logger.propagate = False
+
+        out = [["info", "Recorded only"]] if recordonly else []
+
+        if not recordonly and not dryrun:
+            db = get_db()
+            try:
+                self.migrate(db)
+                out = _extract_output(q)
+            except Exception as e:
+                out = _extract_output(q)
+                tb = traceback.format_exc()
+                self.add_record("migrate", out, False, traceback=tb)
+                raise MigrationError(
+                    "Error while executing migration", output=out, exc=e, traceback=tb
+                )
+
+        if not dryrun:
+            self.add_record("migrate", out, True)
+
+        return out
+
+    def add_record(self, type, output, success, traceback=None):
+        script = inspect.getsource(self.module)
+        return Record(
+            self.collection.find_one_and_update(
+                self.db_query,
+                {
+                    "$push": {
+                        "ops": {
+                            "date": datetime.utcnow(),
+                            "type": type,
+                            "script": script,
+                            "output": output,
+                            "success": success,
+                            "traceback": traceback,
+                        }
+                    }
+                },
+                upsert=True,
+                return_document=ReturnDocument.AFTER,
+            )
+        )
+
+
+def get(filename):
+    """Get a migration"""
+    return Migration(filename)
+
+
+def unrecord(filename):
+    """
+    Delete a migration record from database
+
+    :returns: True if record was deleted, False if it didn't exist
+    """
+    specs = {"filename": filename}
+    db = get_db()
+    record = db.migrations.find_one(specs)
+    if not record:
+        return False
+    return bool(db.migrations.delete_one(specs).deleted_count)
+
+
+def list_available():
+    """
+    List available migrations from udata/migrations
+
+    Returns a list of Migration objects sorted by filename
+    """
+    from importlib.resources import files
+
+    migrations_path = files("udata").joinpath("migrations")
+
+    migrations = [Migration(item.name) for item in migrations_path.iterdir() if item.is_file()]
+
+    return sorted(migrations, key=lambda m: m.filename)
+
+
+def load_migration(filename):
+    """
+    Load a migration from its python file
+
+    :returns: the loaded module or None if file doesn't exist
+    """
+    from importlib.resources import files
+
+    basename = os.path.splitext(os.path.basename(filename))[0]
+    name = f"udata.migrations.{basename}"
+
+    try:
+        script = files("udata").joinpath("migrations", filename).read_bytes()
+    except Exception:
+        # Return None if file doesn't exist instead of raising
+        return None
+
+    spec = importlib.util.spec_from_loader(name, loader=None)
+    module = importlib.util.module_from_spec(spec)
+    exec(script, module.__dict__)
+    module.__file__ = str(files("udata").joinpath("migrations", filename))
+    return module
+
+
+def _extract_output(q):
+    """Extract log output from a QueueHandler queue"""
+    out = []
+    while not q.empty():
+        record = q.get()
+        # Use list instead of tuple to have the same data before and after mongo persist
+        out.append([record.levelname.lower(), record.getMessage()])
+    return out

udata/features/notifications/api.py

@@ -1,30 +1,19 @@
-from udata.api import API, api, fields
+from udata.api import API, api
 from udata.auth import current_user
 
-from .actions import get_notifications
+from .models import Notification
 
 notifs = api.namespace("notifications", "Notifications API")
 
-notifications_fields = api.model(
-    "Notification",
-    {
-        "type": fields.String(description="The notification type", readonly=True),
-        "created_on": fields.ISODateTime(
-            description="The notification creation datetime", readonly=True
-        ),
-        "details": fields.Raw(
-            description="Key-Value details depending on notification type", readonly=True
-        ),
-    },
-)
-
 
 @notifs.route("/", endpoint="notifications")
 class NotificationsAPI(API):
     @api.secure
-    @api.doc("get_notifications")
-    @api.marshal_list_with(notifications_fields)
+    @api.doc("list_notifications")
+    @api.expect(Notification.__index_parser__)
+    @api.marshal_with(Notification.__page_fields__)
     def get(self):
         """List all current user pending notifications"""
         user = current_user._get_current_object()
-        return get_notifications(user)
+        notifications = Notification.objects(user=user)
+        return Notification.apply_pagination(Notification.apply_sort_filters(notifications))

udata/features/notifications/models.py

@@ -0,0 +1,56 @@
+from flask_restx.inputs import boolean
+from mongoengine import NULLIFY
+
+from udata.api_fields import field, generate_fields
+from udata.core.organization.notifications import MembershipRequestNotificationDetails
+from udata.core.user.api_fields import user_ref_fields
+from udata.core.user.models import User
+from udata.models import db
+from udata.mongo.datetime_fields import Datetimed
+from udata.mongo.queryset import UDataQuerySet
+
+
+class NotificationQuerySet(UDataQuerySet):
+    def with_organization_in_details(self, organization):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_organization=organization)
+
+    def with_user_in_details(self, user):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_user=user)
+
+
+def is_handled(base_query, filter_value):
+    if filter_value is None:
+        return base_query
+    if filter_value is True:
+        return base_query.filter(handled_at__ne=None)
+    return base_query.filter(handled_at=None)
+
+
+@generate_fields()
+class Notification(Datetimed, db.Document):
+    meta = {
+        "ordering": ["-created_at"],
+        "queryset_class": NotificationQuerySet,
+    }
+
+    id = field(db.AutoUUIDField(primary_key=True))
+    handled_at = field(
+        db.DateTimeField(),
+        sortable=True,
+        auditable=False,
+        filterable={"key": "handled", "query": is_handled, "type": boolean},
+    )
+    user = field(
+        db.ReferenceField(User, reverse_delete_rule=NULLIFY),
+        nested_fields=user_ref_fields,
+        readonly=True,
+        allow_null=True,
+        auditable=False,
+        filterable={},
+    )
+    details = field(
+        db.GenericEmbeddedDocumentField(choices=(MembershipRequestNotificationDetails,)),
+        generic=True,
+    )

udata/features/notifications/tasks.py

@@ -0,0 +1,25 @@
+import logging
+from datetime import datetime, timedelta
+
+from flask import current_app
+
+from udata.features.notifications.models import Notification
+from udata.tasks import job
+
+log = logging.getLogger(__name__)
+
+
+@job("delete-expired-notifications")
+def delete_expired_notifications(self):
+    # Delete expired notifications
+    handled_at = datetime.utcnow() - timedelta(
+        days=current_app.config["DAYS_AFTER_NOTIFICATION_EXPIRED"]
+    )
+    notifications_to_delete = Notification.objects(
+        handled_at__lte=handled_at,
+    )
+    count = notifications_to_delete.count()
+    for notification in notifications_to_delete:
+        notification.delete()
+
+    log.info(f"Deleted {count} expired notifications")

udata/flask_mongoengine/engine.py

@@ -7,7 +7,6 @@ from mongoengine.errors import DoesNotExist
 from mongoengine.queryset import QuerySet
 
 from .connection import create_connections
-from .json import override_json_encoder
 from .pagination import ListFieldPagination, Pagination
 from .wtf import WtfBaseField
 
@@ -108,9 +107,6 @@ class MongoEngine(object):
 
         app.extensions = getattr(app, "extensions", {})
 
-        # Make documents JSON serializable
-        override_json_encoder(app)
-
         if "mongoengine" not in app.extensions:
             app.extensions["mongoengine"] = {}