udata 11.1.2.dev7__py3-none-any.whl → 11.1.2.dev11__py3-none-any.whl

udata/core/dataset/tasks.py

@@ -1,22 +1,19 @@
 import collections
 import os
-from datetime import datetime, timedelta
+from datetime import datetime
 from tempfile import NamedTemporaryFile
 
 from celery.utils.log import get_task_logger
 from flask import current_app
 from mongoengine import ValidationError
 
-from udata import mail
 from udata import models as udata_models
 from udata.core import csv, storages
 from udata.core.dataservices.models import Dataservice
 from udata.harvest.models import HarvestJob
-from udata.i18n import lazy_gettext as _
-from udata.models import Activity, Discussion, Follow, Organization, TopicElement, Transfer, db
+from udata.models import Activity, Discussion, Follow, TopicElement, Transfer, db
 from udata.tasks import job
 
-from .constants import UPDATE_FREQUENCIES
 from .models import Checksum, CommunityResource, Dataset, Resource
 
 log = get_task_logger(__name__)
@@ -75,51 +72,6 @@ def purge_datasets(self):
         dataset.delete()
 
 
-@job("send-frequency-reminder")
-def send_frequency_reminder(self):
-    # We exclude irrelevant frequencies.
-    frequencies = [
-        f
-        for f in UPDATE_FREQUENCIES.keys()
-        if f not in ("unknown", "realtime", "punctual", "irregular", "continuous")
-    ]
-    now = datetime.utcnow()
-    reminded_orgs = {}
-    reminded_people = []
-    allowed_delay = current_app.config["DELAY_BEFORE_REMINDER_NOTIFICATION"]
-    for org in Organization.objects.visible():
-        outdated_datasets = []
-        for dataset in Dataset.objects.filter(
-            frequency__in=frequencies, organization=org
-        ).visible():
-            if dataset.next_update + timedelta(days=allowed_delay) < now:
-                dataset.outdated = now - dataset.next_update
-                dataset.frequency_str = UPDATE_FREQUENCIES[dataset.frequency]
-                outdated_datasets.append(dataset)
-        if outdated_datasets:
-            reminded_orgs[org] = outdated_datasets
-    for reminded_org, datasets in reminded_orgs.items():
-        print(
-            "{org.name} will be emailed for {datasets_nb} datasets".format(
-                org=reminded_org, datasets_nb=len(datasets)
-            )
-        )
-        recipients = [m.user for m in reminded_org.members]
-        reminded_people.append(recipients)
-        subject = _("You need to update some frequency-based datasets")
-        mail.send(subject, recipients, "frequency_reminder", org=reminded_org, datasets=datasets)
-
-    print("{nb_orgs} orgs concerned".format(nb_orgs=len(reminded_orgs)))
-    reminded_people = list(flatten(reminded_people))
-    print(
-        "{nb_emails} people contacted ({nb_emails_twice} twice)".format(
-            nb_emails=len(reminded_people),
-            nb_emails_twice=len(reminded_people) - len(set(reminded_people)),
-        )
-    )
-    print("Done")
-
-
 def get_queryset(model_cls):
     # special case for resources
     if model_cls.__name__ == "Resource":

udata/core/metrics/helpers.py

@@ -1,7 +1,6 @@
 import logging
 from collections import OrderedDict
 from datetime import datetime, timedelta
-from typing import Dict, List, Union
 from urllib.parse import urlencode
 
 import requests
@@ -14,13 +13,13 @@ from pymongo.command_cursor import CommandCursor
 log = logging.getLogger(__name__)
 
 
-def get_last_13_months() -> List[str]:
+def get_last_13_months() -> list[str]:
     dstart = datetime.today().replace(day=1) - timedelta(days=365)
     months = rrule(freq=MONTHLY, count=13, dtstart=dstart)
     return [month.strftime("%Y-%m") for month in months]
 
 
-def compute_monthly_metrics(metrics_data: List[Dict], metrics_labels: List[str]) -> OrderedDict:
+def compute_monthly_metrics(metrics_data: list[dict], metrics_labels: list[str]) -> OrderedDict:
     # Initialize default monthly_metrics
     monthly_metrics = OrderedDict(
         (month, {label: 0 for label in metrics_labels}) for month in get_last_13_months()
@@ -35,7 +34,7 @@ def compute_monthly_metrics(metrics_data: List[Dict], metrics_labels: List[str])
     return monthly_metrics
 
 
-def metrics_by_label(monthly_metrics: Dict, metrics_labels: List[str]) -> List[OrderedDict]:
+def metrics_by_label(monthly_metrics: dict, metrics_labels: list[str]) -> list[OrderedDict]:
     metrics_by_label = []
     for label in metrics_labels:
         metrics_by_label.append(
@@ -45,8 +44,8 @@ def metrics_by_label(monthly_metrics: Dict, metrics_labels: List[str]) -> List[O
 
 
 def get_metrics_for_model(
-    model: str, id: Union[str, ObjectId, None], metrics_labels: List[str]
-) -> List[OrderedDict]:
+    model: str, id: str | ObjectId | None, metrics_labels: list[str]
+) -> list[OrderedDict]:
     """
     Get distant metrics for a particular model object
     """
@@ -69,7 +68,7 @@ def get_metrics_for_model(
         return [{} for _ in range(len(metrics_labels))]
 
 
-def get_download_url(model: str, id: Union[str, ObjectId, None]) -> str:
+def get_download_url(model: str, id: str | ObjectId | None) -> str:
     api_namespace = model + "s" if model != "site" else model
     base_url = f"{current_app.config['METRICS_API']}/{api_namespace}/data/csv/"
     args = {"metric_month__sort": "asc"}

udata/core/metrics/tasks.py

@@ -1,7 +1,6 @@
 import logging
 import time
 from functools import wraps
-from typing import Dict, List
 
 import requests
 from flask import current_app
@@ -17,9 +16,7 @@ log = logging.getLogger(__name__)
 def log_timing(func):
     @wraps(func)
     def timeit_wrapper(*args, **kwargs):
-        # Better log if we're using Python 3.9
-        name = func.__name__
-        model = name.removeprefix("update_") if hasattr(name, "removeprefix") else name
+        model = func.__name__.removeprefix("update_")
 
         log.info(f"Processing {model}…")
         start_time = time.perf_counter()
@@ -31,7 +28,7 @@ def log_timing(func):
     return timeit_wrapper
 
 
-def save_model(model: db.Document, model_id: str, metrics: Dict[str, int]) -> None:
+def save_model(model: db.Document, model_id: str, metrics: dict[str, int]) -> None:
     try:
         result = model.objects(id=model_id).update(
             **{f"set__metrics__{key}": value for key, value in metrics.items()}
@@ -43,7 +40,7 @@ def save_model(model: db.Document, model_id: str, metrics: Dict[str, int]) -> No
         log.exception(e)
 
 
-def iterate_on_metrics(target: str, value_keys: List[str], page_size: int = 50) -> dict:
+def iterate_on_metrics(target: str, value_keys: list[str], page_size: int = 50) -> dict:
     """
     Yield all elements with not zero values for the keys inside `value_keys`.
     If you pass ['visit', 'download_resource'], it will do a `OR` and get

udata/core/post/api.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import List
 
 from feedgenerator.django.utils.feedgenerator import Atom1Feed
 from flask import make_response, request
@@ -122,7 +121,7 @@ class PostsAtomFeedAPI(API):
             link=request.url_root,
         )
 
-        posts: List[Post] = Post.objects().published().order_by("-published").limit(15)
+        posts: list[Post] = Post.objects().published().order_by("-published").limit(15)
         for post in posts:
             feed.add_item(
                 post.name,

udata/core/reuse/api.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import List
 
 import mongoengine
 from bson.objectid import ObjectId
@@ -144,7 +143,7 @@ class ReusesAtomFeedAPI(API):
             link=request.url_root,
         )
 
-        reuses: List[Reuse] = Reuse.objects.visible().order_by("-created_at").limit(15)
+        reuses: list[Reuse] = Reuse.objects.visible().order_by("-created_at").limit(15)
         for reuse in reuses:
             author_name = None
             author_uri = None

udata/core/user/api.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from flask_security import current_user, logout_user
 from slugify import slugify
 
@@ -370,7 +368,7 @@ suggest_parser.add_argument(
 )
 
 
-def suggest_size(value: str) -> Optional[int]:
+def suggest_size(value: str) -> int | None:
     """Parse an integer that must be between 1 and 20."""
     help_message = "The size must be an integer between 1 and 20."
     try:

udata/cors.py

@@ -1,6 +1,6 @@
 import logging
 
-from flask import g, request
+from flask import current_app, g, request
 from werkzeug.datastructures import Headers
 
 log = logging.getLogger(__name__)
@@ -36,10 +36,27 @@ def is_allowed_cors_route():
         path: str = request.path.removeprefix(f"/{g.lang_code}")
     else:
         path: str = request.path
+
+    # Allow to keep clean CORS when `udata` and the frontend are on the same domain
+    # (as it's the case in data.gouv with cdata/udata).
+    if not current_app.config["SECURITY_SPA_ON_SAME_DOMAIN"] and (
+        path.startswith("/login")
+        or path.startswith("/logout")
+        or path.startswith("/reset")
+        or path.startswith("/register")
+        or path.startswith("/confirm")
+        or path.startswith("/change")
+        or path.startswith("/change-email")
+        or path.startswith("/oauth")
+        or path.startswith("/get-csrf")
+    ):
+        return True
+
     return (
         path.endswith((".js", ".css", ".woff", ".woff2", ".png", ".jpg", ".jpeg", ".svg"))
         or path.startswith("/api")
-        or path.startswith("/oauth")
+        or path.startswith("/oauth/token")
+        or path.startswith("/oauth/revoke")
         or path.startswith("/datasets/r/")
     )
 

udata/harvest/backends/ckan/harvesters.py

@@ -4,19 +4,14 @@ from urllib.parse import urljoin
 from uuid import UUID
 
 from udata import uris
-from udata.harvest.models import HarvestItem
-from udata.i18n import lazy_gettext as _
-
-try:
-    from udata.core.dataset.constants import UPDATE_FREQUENCIES
-except ImportError:
-    # legacy import of constants in udata
-    from udata.models import UPDATE_FREQUENCIES
+from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.core.dataset.rdf import frequency_from_rdf
 from udata.frontend.markdown import parse_html
 from udata.harvest.backends.base import BaseBackend, HarvestFilter
 from udata.harvest.exceptions import HarvestException, HarvestSkipException
+from udata.harvest.models import HarvestItem
+from udata.i18n import lazy_gettext as _
 from udata.models import GeoZone, License, Resource, SpatialCoverage, db
 from udata.utils import daterange_end, daterange_start, get_by
 
@@ -193,14 +188,15 @@ class CkanBackend(BaseBackend):
                 log.debug("spatial-uri value not handled: %s", value)
             elif key == "frequency":
                 # Update frequency
-                freq = frequency_from_rdf(value)
-                if freq:
+                if freq := frequency_from_rdf(value):
                     dataset.frequency = freq
-                elif value in UPDATE_FREQUENCIES:
-                    dataset.frequency = value
                 else:
-                    dataset.extras["ckan:frequency"] = value
-                    log.debug("frequency value not handled: %s", value)
+                    # FIXME(python 3.12+): prefer `if value in UpdateFrequency`
+                    try:
+                        dataset.frequency = UpdateFrequency(value)
+                    except ValueError:
+                        dataset.extras["ckan:frequency"] = value
+                        log.debug("frequency value not handled: %s", value)
             # Temporal coverage start
             elif key == "temporal_start":
                 temporal_start = daterange_start(value)

udata/harvest/backends/maaf.py

@@ -7,6 +7,7 @@ from urllib.parse import urljoin
 from lxml import etree, html
 from voluptuous import All, Any, In, Length, Lower, Optional, Schema
 
+from udata.core.dataset.constants import UpdateFrequency
 from udata.harvest.backends import BaseBackend
 from udata.harvest.filters import (
     boolean,
@@ -38,19 +39,19 @@ ZONES = {
 
 
 FREQUENCIES = {
-    "ponctuelle": "punctual",
-    "temps réel": "continuous",
-    "quotidienne": "daily",
-    "hebdomadaire": "weekly",
-    "bimensuelle": "semimonthly",
-    "mensuelle": "monthly",
-    "bimestrielle": "bimonthly",
-    "trimestrielle": "quarterly",
-    "semestrielle": "semiannual",
-    "annuelle": "annual",
-    "triennale": "triennial",
-    "quinquennale": "quinquennial",
-    "aucune": "unknown",
+    "ponctuelle": UpdateFrequency.PUNCTUAL,
+    "temps réel": UpdateFrequency.CONTINUOUS,
+    "quotidienne": UpdateFrequency.DAILY,
+    "hebdomadaire": UpdateFrequency.WEEKLY,
+    "bimensuelle": UpdateFrequency.SEMIMONTHLY,
+    "mensuelle": UpdateFrequency.MONTHLY,
+    "bimestrielle": UpdateFrequency.BIMONTHLY,
+    "trimestrielle": UpdateFrequency.QUARTERLY,
+    "semestrielle": UpdateFrequency.SEMIANNUAL,
+    "annuelle": UpdateFrequency.ANNUAL,
+    "triennale": UpdateFrequency.TRIENNIAL,
+    "quinquennale": UpdateFrequency.QUINQUENNIAL,
+    "aucune": UpdateFrequency.UNKNOWN,
 }
 
 XSD_PATH = os.path.join(os.path.dirname(__file__), "maaf.xsd")
@@ -161,7 +162,7 @@ class MaafBackend(BaseBackend):
         dataset = self.get_dataset(item.remote_id)
 
         dataset.title = metadata["title"]
-        dataset.frequency = FREQUENCIES.get(metadata["frequency"], "unknown")
+        dataset.frequency = FREQUENCIES.get(metadata["frequency"], UpdateFrequency.UNKNOWN)
         dataset.description = metadata["notes"]
         dataset.private = metadata["private"]
         dataset.tags = sorted(set(metadata["tags"]))

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -5,6 +5,7 @@ from datetime import date
 import pytest
 
 from udata.app import create_app
+from udata.core.dataset.constants import UpdateFrequency
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.spatial.factories import GeoZoneFactory
 from udata.harvest import actions
@@ -336,7 +337,7 @@ def frequency_as_rdf_uri(resource_data):
         "resources": [resource_data],
         "extras": [{"key": "frequency", "value": "http://purl.org/cld/freq/daily"}],
     }
-    return data, {"expected": "daily"}
+    return data, {"expected": UpdateFrequency.DAILY}
 
 
 @pytest.fixture
@@ -348,7 +349,7 @@ def frequency_as_exact_match(resource_data):
         "resources": [resource_data],
         "extras": [{"key": "frequency", "value": "daily"}],
     }
-    return data, {"expected": "daily"}
+    return data, {"expected": UpdateFrequency.DAILY}
 
 
 @pytest.fixture
@@ -486,7 +487,7 @@ def test_can_parse_frequency_as_exact_match(result, kwargs):
 
 
 @pytest.mark.ckan_data("frequency_as_unknown_value")
-def test_can_parse_frequency_as_unkown_value(result, kwargs):
+def test_can_parse_frequency_as_unknown_value(result, kwargs):
     dataset = dataset_for(result)
     assert dataset.extras["ckan:frequency"] == kwargs["expected"]
     assert dataset.frequency is None

udata/harvest/tests/test_dcat_backend.py

@@ -10,6 +10,7 @@ from rdflib import Graph
 
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.factories import DatasetFactory, LicenseFactory, ResourceSchemaMockData
 from udata.core.dataset.rdf import dataset_from_rdf
 from udata.core.organization.factories import OrganizationFactory
@@ -560,7 +561,7 @@ class DcatBackendTest:
         assert dataset.harvest.issued_at.date() == date(2016, 12, 14)
         assert dataset.harvest.created_at.date() == date(2016, 12, 12)
         assert dataset.harvest.modified_at.date() == date(2016, 12, 14)
-        assert dataset.frequency == "daily"
+        assert dataset.frequency == UpdateFrequency.DAILY
         assert dataset.description == "Dataset 3 description"
 
         assert dataset.temporal_coverage is not None
@@ -681,7 +682,7 @@ class DcatBackendTest:
         dataset = Dataset.objects.filter(organization=org).first()
 
         assert dataset is not None
-        assert dataset.frequency == "irregular"
+        assert dataset.frequency == UpdateFrequency.IRREGULAR
         assert "gravi" in dataset.tags  # support dcat:keyword
         assert "geodesy" in dataset.tags  # support dcat:theme
         assert dataset.license.id == "fr-lo"

udata/i18n.py

@@ -206,38 +206,13 @@ class I18nBlueprintSetupState(BlueprintSetupState):
         if "defaults" in options:
             defaults = dict(defaults, **options.pop("defaults"))
 
-        if options.pop("localize", True):
-            self.app.add_url_rule(
-                "/<lang:lang_code>" + rule,
-                "%s.%s" % (self.blueprint.name, endpoint),
-                view_func,
-                defaults=defaults,
-                **options,
-            )
-
-            self.app.add_url_rule(
-                rule,
-                "%s.%s_redirect" % (self.blueprint.name, endpoint),
-                redirect_to_lang,
-                defaults=defaults,
-                **options,
-            )
-        else:
-            self.app.add_url_rule(
-                rule,
-                "%s.%s" % (self.blueprint.name, endpoint),
-                view_func,
-                defaults=defaults,
-                **options,
-            )
-
-            self.app.add_url_rule(
-                "/<lang:lang_code>" + rule,
-                "%s.%s_redirect" % (self.blueprint.name, endpoint),
-                redirect_to_unlocalized,
-                defaults=defaults,
-                **options,
-            )
+        self.app.add_url_rule(
+            rule,
+            "%s.%s" % (self.blueprint.name, endpoint),
+            view_func,
+            defaults=defaults,
+            **options,
+        )
 
 
 class I18nBlueprint(Blueprint):

udata/migrations/2025-01-05-dataservices-fields-changes.py

@@ -3,7 +3,6 @@ This migration keeps only the "Local authority" badge if the organization also h
 """
 
 import logging
-from typing import List
 
 from mongoengine.connection import get_db
 
@@ -84,7 +83,7 @@ def migrate(db):
     )
     log.info(f"\t{count.modified_count} open dataservices to DATASERVICE_ACCESS_TYPE_OPEN")
 
-    dataservices: List[Dataservice] = get_db().dataservice.find()
+    dataservices: list[Dataservice] = get_db().dataservice.find()
     for dataservice in dataservices:
         if (
             "endpoint_description_url" not in dataservice

udata/migrations/2025-09-04-update-legacy-frequencies.py

@@ -0,0 +1,36 @@
+"""
+Convert legacy frequencies to latest vocabulary
+"""
+
+import logging
+
+from udata.core.dataset.constants import UpdateFrequency
+from udata.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Updating datasets legacy frequencies:")
+
+    for legacy_value, frequency in UpdateFrequency._LEGACY_FREQUENCIES.items():
+        count = 0
+        for dataset in Dataset.objects(frequency=legacy_value).no_cache().timeout(False):
+            # Explicitly call update() to force writing the new frequency string to mongo.
+            # We can't rely on save() here because:
+            # 1. save() only writes modified fields to mongo, basically comparing the Dataset's
+            #    object state initially returned by the query with its state when save() is called,
+            #    and only sending the diffset to mongo.
+            # 2. At the ODM layer, the Dataset.frequency field has already been instantiated as an
+            #    UpdateFrequency object, and so legacy frequency strings have already been
+            #    mapped to their new ids (via UpdateFrequency._missing_).
+            # => While the raw frequency string has changed, the save() function sees the same
+            #    UpdateFrequency and therefore ignores the field in is diffset.
+            dataset.update(frequency=frequency.value)
+            # Still call save() afterwards so computed fields like quality_cached are updated if
+            # necessary, e.g. if moving from a predictable timedelta to an unpredictable one.
+            dataset.save()
+            count += 1
+        log.info(f"- {legacy_value} -> {frequency.value}: {count} updated")
+
+    log.info("Completed.")

udata/settings.py

@@ -101,6 +101,24 @@ class Defaults(object):
     SECURITY_RESET_URL = "/reset/"
     SECURITY_CHANGE_EMAIL_URL = "/change-email/"
 
+    # See https://flask-security.readthedocs.io/en/stable/configuration.html#SECURITY_REDIRECT_BEHAVIOR
+    # We do not define all the URLs requested in the documentation because most of the time we do JSON requests in cdata
+    # and catch errors instead of followings the redirects.
+    # The only place where we don't have control over the redirect is when the user is clicking a link directly to udata
+    # (instead of a link to `cdata`) as in /confirm. When the user is clicking on the confirmation link, he's redirected
+    # to `confirm_change_email` endpoint, and then udata redirect him to the homepage of `cdata` with a custom flash message.
+    SECURITY_REDIRECT_BEHAVIOR = "spa"
+    # SECURITY_POST_OAUTH_LOGIN_VIEW = ""    # SECURITY_OAUTH_ENABLE is disabled
+    # SECURITY_LOGIN_ERROR_VIEW = ""         # We don't follow the redirects since we do JSON POST requests during login
+    # SECURITY_CONFIRM_ERROR_VIEW = ""       # Manually changed in `confirm_change_email`
+    # SECURITY_POST_CHANGE_EMAIL_VIEW = ""   # We don't follow the redirects since we do JSON POST requests during change email
+    # SECURITY_CHANGE_EMAIL_ERROR_VIEW = ""  # We don't follow the redirects since we do JSON POST requests during change email
+    # SECURITY_POST_CONFIRM_VIEW = ""        # Set at runtime. See :SecurityPostConfirmViewAtRuntime
+    # SECURITY_RESET_ERROR_VIEW = ""         # We don't follow the redirects since we do JSON POST requests during request reset
+    # SECURITY_RESET_VIEW = ""               # We don't follow the redirects since we do JSON POST requests during request reset
+
+    SECURITY_SPA_ON_SAME_DOMAIN = False
+
     SECURITY_PASSWORD_SALT = "Default uData secret password salt"
     SECURITY_CONFIRM_SALT = "Default uData secret confirm salt"
     SECURITY_RESET_SALT = "Default uData secret reset salt"
@@ -122,6 +140,15 @@ class Defaults(object):
     DAYS_BEFORE_ACCOUNT_INACTIVITY_NOTIFY_DELAY = 30
     MAX_NUMBER_OF_USER_INACTIVITY_NOTIFICATIONS = 200
 
+    # You can activate CaptchEtat, a captcha.com integration by providing
+    # CAPTCHETAT_BASE_URL, CAPTCHETAT_OAUTH_BASE_URL, CAPTCHETAT_CLIENT_ID and CAPTCHETAT_CLIENT_SECRET
+    CAPTCHETAT_BASE_URL = None
+    CAPTCHETAT_OAUTH_BASE_URL = None
+    CAPTCHETAT_CLIENT_ID = None
+    CAPTCHETAT_CLIENT_SECRET = None
+    CAPTCHETAT_TOKEN_CACHE_KEY = "captchetat-bearer-token"
+    CAPTCHETAT_STYLE_NAME = "captchaFR"
+
     # Sentry configuration
     SENTRY_DSN = None
     SENTRY_TAGS = {}

udata/templates/security/email/reset_instructions.html

@@ -16,7 +16,7 @@
         <td align="center">
             {{ mail_button(
                 _('Reset my password'),
-                reset_link
+                cdata_url("/reset/" ~ reset_token) or reset_link
             ) }}
         </td>
     </tr>

udata/templates/security/email/reset_instructions.txt

@@ -4,6 +4,6 @@
 
 {{ _('To reset your password, please confirm your request through the link below:') }}
 
-{{ reset_link }}
+{{ cdata_url("/reset/" ~ reset_token) or reset_link }}
 
 {% endblock %}