udata 12.0.2.dev10__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

udata/rdf.py

@@ -128,17 +128,29 @@ EU_HVD_CATEGORIES = {
 HVD_LEGISLATION = "http://data.europa.eu/eli/reg_impl/2023/138/oj"
 TAG_TO_EU_HVD_CATEGORIES = {slugify_tag(EU_HVD_CATEGORIES[uri]): uri for uri in EU_HVD_CATEGORIES}
 
+INSPIRE_GEMET_THEME_NAMESPACE = "http://inspire.ec.europa.eu/theme"
+INSPIRE_GEMET_SCHEME_URIS = [
+    INSPIRE_GEMET_THEME_NAMESPACE,
+    "http://www.eionet.europa.eu/gemet/inspire_themes",
+]
+
 AGENT_ROLE_TO_RDF_PREDICATE = {
     "contact": DCAT.contactPoint,
-    "publisher": DCT.publisher,
     "creator": DCT.creator,
+    "publisher": DCT.publisher,
+    "rightsHolder": DCT.rightsHolder,
+    "custodian": GEODCAT.custodian,
+    "distributor": GEODCAT.distributor,
+    "originator": GEODCAT.originator,
+    "principalInvestigator": GEODCAT.principalInvestigator,
+    "processor": GEODCAT.processor,
+    "resourceProvider": GEODCAT.resourceProvider,
+    "user": GEODCAT.user,
 }
 
 # Map rdf contact point entity to role
 CONTACT_POINT_ENTITY_TO_ROLE = {
-    DCAT.contactPoint: "contact",
-    DCT.publisher: "publisher",
-    DCT.creator: "creator",
+    predicate: role for role, predicate in AGENT_ROLE_TO_RDF_PREDICATE.items()
 }
 
 
@@ -303,16 +315,37 @@ def theme_labels_from_rdf(rdf):
     """
     Get theme labels to use as keywords.
     Map HVD keywords from known URIs resources if HVD support is activated.
+    Map INSPIRE keyword from known themes if INSPIRE support is activated.
+    - An INSPIRE dataset is a dataset with a theme INSPIRE encoded with gmd:descriptiveKeywords/gmd:MD_Keywords.
+      In DCAT, it is shown as a DCAT.theme with a SKOS.inScheme pointing to to the INSPIRE thesaurus.
+      We filter on this thesaurus based on its name (expecting "GEMET - INSPIRE themes, version 1.0") or its uri.
     """
     for theme in rdf.objects(DCAT.theme):
         if isinstance(theme, RdfResource):
+            label = rdf_value(theme, SKOS.prefLabel)
             uri = theme.identifier.toPython()
             if current_app.config["HVD_SUPPORT"] and uri in EU_HVD_CATEGORIES:
+                # Map label from EU HVD categories
                 label = EU_HVD_CATEGORIES[uri]
                 # Additionnally yield hvd keyword
                 yield "hvd"
-            else:
-                label = rdf_value(theme, SKOS.prefLabel)
+            if current_app.config["INSPIRE_SUPPORT"]:
+                if uri.startswith(INSPIRE_GEMET_THEME_NAMESPACE):
+                    yield "inspire"
+                else:
+                    # Check if the theme belongs to the GEMET INSPIRE scheme
+                    if scheme := theme.value(SKOS.inScheme):
+                        scheme_title = (
+                            rdf_value(scheme, DCT.title)
+                            or rdf_value(scheme, SKOS.prefLabel)
+                            or rdf_value(scheme, RDFS.label)
+                        )
+                        scheme_uri = scheme.identifier.toPython()
+                        if (
+                            scheme_title
+                            and scheme_title.lower() == "gemet - inspire themes, version 1.0"
+                        ) or scheme_uri in INSPIRE_GEMET_SCHEME_URIS:
+                            yield "inspire"
         else:
             label = theme.toPython()
         if label:
@@ -326,6 +359,8 @@ def themes_from_rdf(rdf):
 
 
 def contact_points_from_rdf(rdf, prop, role, dataset):
+    if not dataset.organization and not dataset.owner:
+        return
     for contact_point in rdf.objects(prop):
         # Read contact point information
         if isinstance(contact_point, Literal):
@@ -338,7 +373,7 @@ def contact_points_from_rdf(rdf, prop, role, dataset):
             email = (
                 rdf_value(contact_point, VCARD.hasEmail)
                 or rdf_value(contact_point, VCARD.email)
-                or rdf_value(contact_point, DCAT.email)
+                or None
             )
             email = email.replace("mailto:", "").strip() if email else None
             contact_form = rdf_value(contact_point, VCARD.hasUrl)
@@ -357,8 +392,6 @@ def contact_points_from_rdf(rdf, prop, role, dataset):
         #         continue
 
         # Create of get contact point object
-        if not dataset.organization and not dataset.owner:
-            continue
         org_or_owner = {}
         if dataset.organization:
             org_or_owner = {"organization": dataset.organization}
@@ -393,14 +426,25 @@ def contact_points_to_rdf(contacts, graph=None):
             id = BNode()
 
         node = graph.resource(id)
-        node.set(RDF.type, VCARD.Kind)
-        if contact.name:
-            node.set(VCARD.fn, Literal(contact.name))
-        if contact.email:
-            node.set(VCARD.hasEmail, URIRef(f"mailto:{contact.email}"))
-        if contact.contact_form:
-            node.set(VCARD.hasUrl, URIRef(contact.contact_form))
-        yield node, AGENT_ROLE_TO_RDF_PREDICATE.get(contact.role, DCAT.contactPoint)
+        role = AGENT_ROLE_TO_RDF_PREDICATE.get(contact.role, DCAT.contactPoint)
+        # GeoDCAT-AP spec: Only contactPoint is a VCARD.Kind (like in DCAT). Other roles are FOAF.Agent.
+        if role == DCAT.contactPoint:
+            node.set(RDF.type, VCARD.Kind)
+            if contact.name:
+                node.set(VCARD.fn, Literal(contact.name))
+            if contact.email:
+                node.set(VCARD.hasEmail, URIRef(f"mailto:{contact.email}"))
+            if contact.contact_form:
+                node.set(VCARD.hasUrl, URIRef(contact.contact_form))
+        else:
+            node.set(RDF.type, FOAF.Agent)
+            node.set(FOAF.name, Literal(contact.name))
+            if contact.email:
+                node.set(FOAF.mbox, URIRef(f"mailto:{contact.email}"))
+            if contact.contact_form:
+                node.set(FOAF.page, URIRef(contact.contact_form))
+
+        yield node, role
 
 
 def primary_topic_identifier_from_rdf(graph: Graph, resource: RdfResource):
@@ -490,17 +534,18 @@ def escape_xml_illegal_chars(val, replacement="?"):
     return illegal_xml_chars_RE.sub(replacement, val)
 
 
-def paginate_catalog(catalog, graph, datasets, format, rdf_catalog_endpoint, **values):
+def paginate_catalog(catalog, graph, datasets, _format, rdf_catalog_endpoint, **values):
     if not format:
         raise ValueError("Pagination requires format")
     catalog.add(RDF.type, HYDRA.Collection)
     catalog.set(HYDRA.totalItems, Literal(datasets.total))
     kwargs = {
-        "format": format,
+        "_format": _format,
         "page_size": datasets.page_size,
         "_external": True,
     }
-
+    values.pop("page", None)
+    values.pop("page_size", None)
     kwargs.update(values)
 
     first_url = url_for(rdf_catalog_endpoint, page=1, **kwargs)

udata/sentry.py

@@ -2,15 +2,14 @@ import logging
 import re
 import warnings
 
-import pkg_resources
 from werkzeug.exceptions import HTTPException
 
 from udata import entrypoints
 from udata.core.storages.api import UploadProgress
+from udata.utils import get_udata_version
 
 from .app import UDataApp
 from .auth import PermissionDenied
-from .frontend import package_version
 
 log = logging.getLogger(__name__)
 
@@ -64,7 +63,7 @@ def init_app(app: UDataApp):
             dsn=app.config["SENTRY_PUBLIC_DSN"],
             integrations=[FlaskIntegration(), CeleryIntegration()],
             ignore_errors=list(exceptions),
-            release=f"udata@{package_version('udata')}",
+            release=f"udata@{get_udata_version()}",
             environment=app.config.get("SITE_ID", None),
             # Set traces_sample_rate to 1.0 to capture 100%
             # of transactions for performance monitoring.
@@ -88,4 +87,4 @@ def init_app(app: UDataApp):
             if dist.version:
                 sentry_sdk.set_tag(dist.project_name, dist.version)
         # Do not forget udata itself
-        sentry_sdk.set_tag("udata", pkg_resources.get_distribution("udata").version)
+        sentry_sdk.set_tag("udata", get_udata_version())

udata/settings.py

@@ -1,4 +1,3 @@
-import pkg_resources
 from kombu import Exchange, Queue
 from tlds import tld_set
 
@@ -173,11 +172,11 @@ class Defaults(object):
     SITE_AUTHOR_URL = None
     SITE_AUTHOR = "Udata"
     SITE_GITHUB_URL = "https://github.com/etalab/udata"
-    SITE_TERMS_LOCATION = pkg_resources.resource_filename(__name__, "terms.md")
 
     UDATA_INSTANCE_NAME = "udata"
 
     PLUGINS = []
+    HARVESTER_BACKENDS = []
     THEME = None
 
     STATIC_DIRS = []
@@ -288,6 +287,8 @@ class Defaults(object):
 
     DELAY_BEFORE_REMINDER_NOTIFICATION = 30  # Days
 
+    DELAY_BEFORE_APPEARING_IN_RSS_FEED = 10  # Hours
+
     # Harvest settings
     ###########################################################################
     HARVEST_ENABLE_MANUAL_RUN = False
@@ -324,23 +325,18 @@ class Defaults(object):
     S3_ACCESS_KEY_ID = None
     S3_SECRET_ACCESS_KEY = None
 
-    # Specific support for hvd (map HVD categories URIs to keywords)
+    # Specific support for hvd:
+    # - map HVD categories URIs to keywords
     HVD_SUPPORT = True
 
+    # Specific support for inspire:
+    # - add inspire keyword during harvest if GEMETE INSPIRE thesaurus is used in DCAT.theme
+    INSPIRE_SUPPORT = True
+
     ACTIVATE_TERRITORIES = False
     # The order is important to compute parents/children, smaller first.
     HANDLED_LEVELS = tuple()
 
-    LINKCHECKING_ENABLED = True
-    # Resource types ignored by linkchecker
-    LINKCHECKING_UNCHECKED_TYPES = ("api",)
-    LINKCHECKING_IGNORE_DOMAINS = []
-    LINKCHECKING_IGNORE_PATTERNS = ["format=shp"]
-    LINKCHECKING_MIN_CACHE_DURATION = 60  # in minutes
-    LINKCHECKING_MAX_CACHE_DURATION = 1080  # in minutes (1 week)
-    LINKCHECKING_UNAVAILABLE_THRESHOLD = 100
-    LINKCHECKING_DEFAULT_LINKCHECKER = "no_check"
-
     # Ignore some endpoint from API tracking
     # By default ignore the 3 most called APIs
     TRACKING_BLACKLIST = [
@@ -535,6 +531,8 @@ class Defaults(object):
         "harvest",
     )
     EXPORT_CSV_DATASET_ID = None
+    EXPORT_CSV_ARCHIVE_S3_BUCKET = None  # If this setting is set, an archive is uploaded to the corresponding S3 bucket every first day of the month (if export-csv is scheduled to run daily)
+    EXPORT_CSV_ARCHIVE_S3_FILENAME_PREFIX = ""  # Useful to store the csv archives inside a subfolder of the bucket, ie setting 'csv-catalog-archives/'`
 
     # Autocomplete parameters
     #########################
@@ -642,6 +640,7 @@ class Testing(object):
     CELERY_TASK_EAGER_PROPAGATES = True
     TEST_WITH_PLUGINS = False
     PLUGINS = []
+    HARVESTER_BACKENDS = ["factory"]
     TEST_WITH_THEME = False
     THEME = "testing"
     CACHE_TYPE = "flask_caching.backends.null"
@@ -661,6 +660,9 @@ class Testing(object):
     }  # Disables deliverability for email domain name
     PUBLISH_ON_RESOURCE_EVENTS = False
     HARVEST_ACTIVITY_USER_ID = None
+    SEARCH_SERVICE_API_URL = None
+    CDATA_BASE_URL = None
+    SCHEMA_CATALOG_URL = None
 
 
 class Debug(Defaults):

udata/tags.py

@@ -2,8 +2,8 @@ from flask import current_app
 from slugify import slugify
 from werkzeug.local import LocalProxy
 
-MIN_TAG_LENGTH = LocalProxy(lambda: current_app.config["TAG_MIN_LENGTH"])
-MAX_TAG_LENGTH = LocalProxy(lambda: current_app.config["TAG_MAX_LENGTH"])
+TAG_MIN_LENGTH = LocalProxy(lambda: current_app.config["TAG_MIN_LENGTH"])
+TAG_MAX_LENGTH = LocalProxy(lambda: current_app.config["TAG_MAX_LENGTH"])
 
 
 def slug(value: str) -> str:
@@ -12,10 +12,10 @@ def slug(value: str) -> str:
 
 def normalize(value: str) -> str:
     value = slug(value)
-    if len(value) < MIN_TAG_LENGTH:
+    if len(value) < TAG_MIN_LENGTH:
         value = ""
-    elif len(value) > MAX_TAG_LENGTH:
-        value = value[:MAX_TAG_LENGTH]
+    elif len(value) > TAG_MAX_LENGTH:
+        value = value[:TAG_MAX_LENGTH]
     return value
 
 

udata/tasks.py

@@ -1,12 +1,11 @@
 import logging
+from importlib.metadata import entry_points
 from urllib.parse import urlparse
 
 from celery import Celery, Task
 from celery.utils.log import get_task_logger
 from celerybeatmongo.schedulers import MongoScheduler
 
-from udata import entrypoints
-
 log = logging.getLogger(__name__)
 
 
@@ -176,6 +175,7 @@ def init_app(app):
     import udata.harvest.tasks  # noqa
     import udata.db.tasks  # noqa
 
-    entrypoints.get_enabled("udata.tasks", app)
+    for ep in entry_points(group="udata.tasks"):
+        ep.load()
 
     return celery

udata/templates/mail/message.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{{ message.subject }}</title>
+</head>
+<body style="font-family: Marianne, Helvetica, Arial, sans-serif; line-height: 1.6; color: #333; background-color: #ffffff; max-width: 600px; margin: 0 auto; padding: 20px;">
+    <div style="margin-bottom: 30px;">
+        <svg width="186" height="59" viewBox="0 0 186 59" fill="none" xmlns="http://www.w3.org/2000/svg">
+            <path d="M143.469 29.5086H145.86L152.728 46.4699L159.558 29.5086H161.986L154.208 48.6328H151.248L143.469 29.5086Z" fill="#3558A2"/>
+            <path d="M138.218 40.5126V29.5086H140.419V40.4746C140.419 46.1664 137.155 49.3917 132.26 49.3917C127.328 49.3917 124.026 46.1664 124.026 40.4746V29.5086H126.265V40.5126C126.265 44.8383 128.656 47.2668 132.26 47.2668C135.789 47.2668 138.218 44.8383 138.218 40.5126Z" fill="#3558A2"/>
+            <path d="M109.662 28.7498C115.809 28.7498 119.945 33.6067 119.945 39.0708C119.945 44.5348 115.809 49.3918 109.662 49.3918C103.553 49.3918 99.4167 44.5348 99.4167 39.0708C99.4167 33.6067 103.553 28.7498 109.662 28.7498ZM109.7 47.2669C114.215 47.2669 117.554 43.3965 117.554 39.0708C117.554 34.6692 114.215 30.8747 109.7 30.8747C105.033 30.8747 101.731 34.7071 101.731 39.0708C101.731 43.4344 105.033 47.2669 109.7 47.2669Z" fill="#3558A2"/>
+            <path d="M79.7158 52.1617C79.7158 50.1886 80.7024 48.6328 82.6755 47.1909C81.7269 46.5459 81.2716 45.5214 81.2716 44.4968C81.2716 43.017 82.1822 41.7269 83.6621 40.8541C81.9546 39.602 80.968 37.6288 80.968 35.5039C80.968 31.8991 83.8139 28.7118 88.2155 28.7118C89.5435 28.7118 90.7578 29.0153 91.7823 29.5086H98.9539V31.4817H94.1728C95.0076 32.6201 95.5009 34.024 95.5009 35.5039C95.5009 39.1087 92.655 42.296 88.2155 42.296C87.0771 42.296 86.0526 42.0684 85.1419 41.7269C84.0036 42.3719 83.3965 43.3205 83.3965 44.1553C83.3965 45.0281 83.8139 45.7111 85.2178 45.7111H91.0613C96.2598 45.7111 98.4227 48.1016 98.4227 51.4028C98.4227 55.2732 94.4764 58.1191 88.8605 58.1191C83.4724 58.1191 79.7158 55.8424 79.7158 52.1617ZM88.2534 40.3988C91.4787 40.3988 93.2621 38.0842 93.2621 35.5039C93.2621 32.8478 91.4787 30.609 88.2534 30.609C84.9522 30.609 83.1688 32.8478 83.1688 35.5039C83.1688 38.1221 84.9522 40.3988 88.2534 40.3988ZM81.9546 51.972C81.9546 54.5522 84.7625 56.1839 88.8226 56.1839C93.2242 56.1839 96.1839 54.4005 96.1839 51.5166C96.1839 49.4297 95.0456 47.7601 90.9854 47.7601H84.6487C82.9032 48.9364 81.9546 50.2645 81.9546 51.972Z" fill="#3558A2"/>
+            <path d="M65.9257 49.2021C62.0933 49.2021 59.4751 47.0013 59.4751 43.4724C59.4751 40.5886 61.7138 38.4257 65.8119 37.7427L71.6554 36.7561V36.2628C71.6554 34.2518 70.1376 32.9616 67.9368 32.9616C66.0775 32.9616 64.6356 33.8344 63.6111 35.2383L60.0822 32.5442C61.7897 30.1917 64.5977 28.7498 68.0886 28.7498C73.6285 28.7498 76.4744 32.051 76.4744 36.2628V48.6329H71.6554V46.7736C70.4412 48.2534 68.1645 49.2021 65.9257 49.2021ZM64.2562 43.2447C64.2562 44.5348 65.2807 45.3696 66.9123 45.3696C69.1131 45.3696 70.7068 44.3451 71.6554 42.8273V40.1332L67.102 40.8921C65.0909 41.2336 64.2562 42.0684 64.2562 43.2447Z" fill="#3558A2"/>
+            <path d="M46.6593 41.6509V33.8343H43.0925V29.5086H46.6593V24.7275H51.5163V29.5086H57.3598V33.8343H51.5163V41.6509C51.5163 43.7759 52.6546 44.6106 54.5519 44.6106C55.88 44.6106 56.7527 44.4589 57.3978 44.1932V48.4051C56.4491 48.8225 55.3108 49.0123 53.7171 49.0123C48.936 49.0123 46.6593 46.3182 46.6593 41.6509Z" fill="#3558A2"/>
+            <path d="M30.5161 49.2021C26.6836 49.2021 24.0654 47.0013 24.0654 43.4724C24.0654 40.5886 26.3042 38.4257 30.4022 37.7427L36.2457 36.7561V36.2628C36.2457 34.2518 34.7279 32.9616 32.5271 32.9616C30.6678 32.9616 29.2259 33.8344 28.2014 35.2383L24.6725 32.5442C26.3801 30.1917 29.188 28.7498 32.6789 28.7498C38.2189 28.7498 41.0647 32.051 41.0647 36.2628V48.6329H36.2457V46.7736C35.0315 48.2534 32.7548 49.2021 30.5161 49.2021ZM28.8465 43.2447C28.8465 44.5348 29.871 45.3696 31.5026 45.3696C33.7034 45.3696 35.2971 44.3451 36.2457 42.8273V40.1332L31.6924 40.8921C29.6813 41.2336 28.8465 42.0684 28.8465 43.2447Z" fill="#3558A2"/>
+            <path d="M0 39.0706C0 33.4927 3.68066 28.7496 9.71389 28.7496C12.2941 28.7496 14.1534 29.5465 15.6333 31.0263V20.1741H20.4902V48.6327H15.6333V47.115C14.1534 48.5948 12.2941 49.3916 9.71389 49.3916C3.68066 49.3916 0 44.6485 0 39.0706ZM5.04667 39.0706C5.04667 42.4098 7.13364 44.8383 10.3969 44.8383C12.5598 44.8383 14.3432 43.9276 15.6333 42.1821V35.9591C14.3432 34.2137 12.5598 33.303 10.3969 33.303C7.13364 33.303 5.04667 35.7315 5.04667 39.0706Z" fill="#3558A2"/>
+            <path d="M179.017 11.7431C179.017 11.7431 179.017 11.7297 179.017 11.7207C179.008 10.9944 178.619 10.3218 177.994 9.96316L174.864 8.1518C174.573 7.98142 174.238 7.89175 173.894 7.88278C173.845 7.88278 173.8 7.88278 173.751 7.88727C173.509 7.90072 173.281 7.94555 173.067 8.03074C172.982 8.06661 172.901 8.10696 172.821 8.1518L170.916 9.25476L169.731 9.94074C169.66 9.98109 169.593 10.0259 169.53 10.0752C169.53 10.0752 169.503 10.0932 169.499 10.0932C169.248 10.277 169.052 10.5146 168.913 10.7792C168.904 10.7971 168.891 10.815 168.882 10.833C168.873 10.8509 168.864 10.8688 168.859 10.8868C168.734 11.1603 168.667 11.4696 168.667 11.7969V15.2672V15.321C168.667 15.3838 168.672 15.4511 168.681 15.5318C168.681 15.5721 168.69 15.6169 168.699 15.6663C168.734 15.9084 168.815 16.137 168.927 16.3433C169.11 16.684 169.383 16.9665 169.718 17.1593L172.794 18.9393C172.906 19.002 172.991 19.0424 173.076 19.0783C173.29 19.1679 173.518 19.2173 173.751 19.2262C173.76 19.2262 173.769 19.2262 173.777 19.2262C173.997 19.2307 174.22 19.2083 174.435 19.141C174.582 19.0962 174.73 19.0334 174.864 18.9527L177.971 17.1548C178.289 16.971 178.552 16.7065 178.736 16.3881C178.919 16.0698 179.017 15.7066 179.017 15.3434V11.7431Z" fill="#3558A2"/>
+            <path d="M168.667 18.4043V22.8378V24.1148L173.602 26.9722V21.2617L172.451 20.5965L168.667 18.4043Z" fill="#3558A2"/>
+            <path d="M178.359 9.68272C178.359 9.68272 178.4 9.71397 178.418 9.72736C178.418 9.72736 178.427 9.73183 178.431 9.73629C178.462 9.75862 178.494 9.78541 178.525 9.80773C178.525 9.80773 178.53 9.8122 178.534 9.81666C178.539 9.82112 178.548 9.82559 178.552 9.83005C178.588 9.86131 178.628 9.89256 178.66 9.92382C178.669 9.93275 178.678 9.94167 178.687 9.9506C178.709 9.97293 178.732 9.99525 178.754 10.0176C178.763 10.0265 178.772 10.0354 178.776 10.0399C178.808 10.0756 178.839 10.1113 178.871 10.1471C178.889 10.1649 178.902 10.1872 178.915 10.2051C178.924 10.2185 178.938 10.2319 178.947 10.2453C178.956 10.2542 178.965 10.2676 178.974 10.2765C179.005 10.3167 179.032 10.3614 179.059 10.406C179.086 10.4507 179.113 10.4908 179.14 10.5355C179.167 10.5846 179.189 10.6337 179.216 10.6828C179.238 10.7275 179.256 10.7677 179.274 10.8123C179.292 10.8525 179.306 10.8971 179.319 10.9373C179.319 10.9463 179.324 10.9507 179.328 10.9597C179.328 10.9686 179.333 10.9775 179.337 10.982C179.351 11.0222 179.364 11.0623 179.377 11.1025C179.395 11.1695 179.409 11.2365 179.418 11.3079C179.418 11.3079 179.418 11.3124 179.418 11.3168C179.418 11.3258 179.422 11.3392 179.422 11.3481C179.422 11.3704 179.431 11.3883 179.436 11.4106C179.449 11.5133 179.458 11.6204 179.458 11.7276V15.1968L180.149 15.5941V9.88363L175.179 7.02615V7.84321L178.207 9.58449C178.256 9.61128 178.306 9.64253 178.355 9.67825L178.359 9.68272Z" fill="#3558A2"/>
+            <path d="M161.984 7.02615V12.7595L166.954 15.6284V9.89506L161.984 7.02615Z" fill="#3558A2"/>
+            <path d="M178.969 16.8342C178.929 16.8925 178.884 16.9419 178.84 16.9958C178.809 17.0317 178.777 17.0676 178.746 17.1035C178.697 17.1573 178.639 17.2022 178.586 17.2516C178.559 17.274 178.532 17.3009 178.505 17.3279L179.352 17.8215C179.419 17.8619 179.464 17.9337 179.464 18.0145V22.8032L180.15 23.2026V17.4625L179.005 16.7938C179.005 16.7938 178.982 16.8207 178.973 16.8342H178.969Z" fill="#3558A2"/>
+            <path d="M172.779 6.64593V7.67751C172.779 7.67751 172.811 7.66858 172.828 7.65965C172.917 7.61946 173.015 7.5882 173.109 7.5614C173.185 7.53908 173.26 7.52121 173.341 7.50782C173.416 7.49442 173.488 7.48102 173.563 7.47209C173.59 7.47209 173.612 7.46316 173.639 7.45869V6.11451L168.704 3.25644V4.15405L172.668 6.44944C172.735 6.48963 172.779 6.56108 172.779 6.64146V6.64593Z" fill="#3558A2"/>
+            <path d="M161.984 20.3141L166.954 23.1686V17.4551L161.984 14.6006V20.3141Z" fill="#3558A2"/>
+            <path d="M185.53 20.314V14.6005L180.594 17.455V23.1685L185.53 20.314Z" fill="#3558A2"/>
+            <path d="M174.048 26.9723L179.018 24.1148V22.9138V18.4043L174.048 21.2618V26.9723Z" fill="#3558A2"/>
+            <path d="M167.4 15.6284L168.215 15.1533V11.7554C168.215 10.9754 168.585 10.2313 169.209 9.76061C169.209 9.76061 169.217 9.75613 169.222 9.75165C169.222 9.75165 169.226 9.75165 169.231 9.74716C169.24 9.74268 169.249 9.73372 169.253 9.73372C169.333 9.67544 169.391 9.6351 169.458 9.59475L172.335 7.92272V7.02618L167.4 9.89509V15.6284Z" fill="#3558A2"/>
+            <path d="M185.53 12.7595V7.02618L180.594 9.89509V15.6284L185.53 12.7595Z" fill="#3558A2"/>
+            <path d="M174.169 7.46644C174.218 7.47089 174.262 7.48426 174.311 7.48871C174.369 7.49762 174.431 7.51099 174.489 7.52435C174.543 7.53772 174.592 7.55554 174.641 7.57336C174.681 7.58672 174.721 7.59563 174.761 7.609V6.63336C174.761 6.55317 174.806 6.48189 174.872 6.44179L179.055 4.02718V3.25647L174.12 6.10767V7.45753C174.12 7.45753 174.155 7.46644 174.173 7.46644H174.169Z" fill="#3558A2"/>
+            <path d="M168.26 17.9037C168.273 17.8815 168.296 17.8637 168.318 17.8459C168.323 17.8459 168.327 17.837 168.332 17.8325L169.182 17.343C169.182 17.343 169.146 17.3074 169.124 17.2896C169.052 17.2273 168.981 17.165 168.914 17.0937C168.882 17.0626 168.855 17.027 168.829 16.9958C168.775 16.9335 168.726 16.8712 168.681 16.8045C168.672 16.7911 168.658 16.7778 168.645 16.7599L168.623 16.7733L167.4 17.4765V23.1688L168.22 22.697V18.0239C168.22 17.9883 168.233 17.9527 168.246 17.9216C168.246 17.9127 168.255 17.9082 168.26 17.8993V17.9037Z" fill="#3558A2"/>
+            <path d="M172.132 6.629L167.182 3.76953L162.227 6.629L167.182 9.49294L172.132 6.629Z" fill="#3558A2"/>
+            <path d="M178.847 2.86394L173.878 0L168.908 2.86394L173.878 5.72341L178.847 2.86394Z" fill="#3558A2"/>
+            <path d="M180.045 9.30052L180.371 9.49294L185.326 6.629L180.371 3.76953L179.396 4.33337L175.421 6.629L177.514 7.8417L180.045 9.30052Z" fill="#3558A2"/>
+            <path d="M180.264 16.1917L179.438 15.7158C179.433 15.7469 179.42 15.7825 179.415 15.8136C179.402 15.8892 179.384 15.9648 179.366 16.036C179.353 16.0805 179.335 16.125 179.321 16.165C179.299 16.2317 179.277 16.294 179.245 16.3607C179.236 16.3785 179.232 16.3918 179.228 16.4096L180.376 17.0679L185.328 14.2213L184.617 13.8121L180.492 16.1828C180.457 16.2006 180.421 16.2139 180.381 16.2139C180.34 16.2139 180.305 16.205 180.269 16.1828L180.264 16.1917Z" fill="#3558A2"/>
+            <path d="M175.068 19.3326C174.902 19.426 174.723 19.5016 174.544 19.5594C174.419 19.5994 174.29 19.6217 174.16 19.6395C174.115 19.6439 174.071 19.6439 174.026 19.6484C173.959 19.6528 173.896 19.6617 173.829 19.6617C173.793 19.6617 173.757 19.6617 173.722 19.6617C173.722 19.6617 173.717 19.6617 173.713 19.6617C173.57 19.6573 173.431 19.6395 173.292 19.6083C173.154 19.5772 173.02 19.5372 172.89 19.4838C172.787 19.4393 172.684 19.3904 172.581 19.337L169.585 17.6157L169.349 17.7536L168.875 18.0249L173.829 20.8715L174.366 20.5602L174.871 20.2711L175.881 19.6884L178.779 18.0249L178.305 17.7536L178.068 17.6157L175.072 19.337L175.068 19.3326Z" fill="#3558A2"/>
+            <path d="M168.256 15.7246C168.256 15.6935 168.247 15.6668 168.242 15.6357L167.29 16.1828C167.254 16.2006 167.218 16.2139 167.178 16.2139C167.138 16.2139 167.102 16.205 167.066 16.1828L162.938 13.8121L162.227 14.2213L167.182 17.0679L168.43 16.3518C168.354 16.1739 168.296 15.9693 168.26 15.7424C168.26 15.7424 168.26 15.7335 168.26 15.7291L168.256 15.7246Z" fill="#3558A2"/>
+        </svg>
+    </div>
+
+    <p style="margin-bottom: 20px;">{{ _('Hi %(user)s', user=recipient.first_name) }},</p>
+
+    {% for item in message.paragraphs %}
+        {% if item.__class__.__name__ == 'MailCTA' %}
+            <p style="margin: 25px 0;">
+                <a href="{{ item.link }}" style="color: #3558A2; text-decoration: underline; font-weight: 700;">{{ item.label }}</a>
+            </p>
+        {% elif item.__class__.__name__ == 'LabelledContent' %}
+            {% if item.inline %}
+                <p style="margin-bottom: 15px;"><strong>{{ item.label }}</strong> {{ item.truncated_content }}</p>
+            {% else %}
+                <p style="margin-bottom: 5px; font-weight: bold;">{{ item.label }}</p>
+                <p style="margin-bottom: 15px; margin-top: 0;">{{ item.truncated_content }}</p>
+            {% endif %}
+        {% elif item.__class__.__name__ == 'ParagraphWithLinks' %}
+            <p style="margin-bottom: 15px;">{{ item.html|safe }}</p>
+        {% else %}
+            <p style="margin-bottom: 15px;">{{ item }}</p>
+        {% endif %}
+    {% endfor %}
+
+    <p style="margin-top: 30px; margin-bottom: 5px;">{{ _('Have a great day') }},</p>
+    <p style="margin: 0;">{{ _('The %(site)s team', site=config.SITE_TITLE) }}</p>
+</body>
+</html>

udata/templates/mail/message.txt

@@ -0,0 +1,16 @@
+{{ _('Hi %(user)s', user=recipient.first_name) }},
+
+{% for item in message.paragraphs %}
+{% if item.__class__.__name__ == 'MailCTA' %}
+{{ item.label }}: {{ item.link }}
+{% elif item.__class__.__name__ == 'LabelledContent' %}
+{% if item.inline %}{{ item.label }} {{ item.truncated_content }}{% else %}{{ item.label }}
+{{ item.truncated_content }}{% endif %}
+{% else %}
+{{ item }}
+{% endif %}
+
+{% endfor %}
+
+{{ _('Have a great day') }},
+{{ _('The %(site)s team', site=config.SITE_TITLE) }}

udata/tests/__init__.py

@@ -1,24 +1,48 @@
 import unittest
 
 import pytest
+from werkzeug import Response
 
 from udata import settings
+from udata.app import UDataApp, create_app
+from udata.tests.plugin import TestClient
 
 from . import helpers
 
 
-class TestCase(unittest.TestCase):
-    settings = settings.Testing
+class TestCaseMixin:
+    app: UDataApp
 
-    @pytest.fixture(autouse=True)
-    def inject_app(self, app):
-        self.app = app
-        return self.create_app()
-
-    def create_app(self):
+    def get_settings(self, request):
         """
-        Here for compatibility legacy test classes
+        This seems really complicated for what it's doing. I think we want to create
+        an app with the default setting being `settings.Testing` and some overrides
+        from the "options" markers. But to do that `settings.Testing` should inherit from
+        `settings.Default`?
+
+        We may also want to prevent loading `udata.cfg` for testing to avoid failing tests
+        locally because some config is changed on our computer. Tests should work only with
+        default settings (or overrides for a specific test) but not with a local `udata.cfg`.
+
+        Not sure if the plugin situation is still relevant now that plugins are integrated
+        into udata.
         """
+        _settings = settings.Testing
+        # apply the options(plugins) marker from pytest_flask as soon as app is created
+        # https://github.com/pytest-dev/pytest-flask/blob/a62ea18cb0fe89e3f3911192ab9ea4f9b12f8a16/pytest_flask/plugin.py#L126
+        # this lets us have default settings for plugins applied while testing
+        plugins = getattr(_settings, "PLUGINS", [])
+        for options in request.node.iter_markers("options"):
+            option = options.kwargs.get("plugins", []) or options.kwargs.get("PLUGINS", [])
+            plugins += option
+        setattr(_settings, "PLUGINS", plugins)
+        return _settings
+
+    @pytest.fixture(autouse=True, name="app")
+    def _app(self, request):
+        test_settings = self.get_settings(request)
+        self.app = create_app(settings.Defaults, override=test_settings)
+        self.app.test_client_class = TestClient
         return self.app
 
     def assertEqualDates(self, datetime1, datetime2, limit=1):  # Seconds.
@@ -26,58 +50,16 @@ class TestCase(unittest.TestCase):
         __tracebackhide__ = True
         helpers.assert_equal_dates(datetime1, datetime2, limit=1)
 
-
-class WebTestMixin(object):
-    user = None
-
-    @pytest.fixture(autouse=True)
-    def inject_client(self, client):
-        """
-        Inject test client for compatibility with Flask-Testing.
-        """
-        self.client = client
-
-    def get(self, url, **kwargs):
-        return self.client.get(url, **kwargs)
-
-    def post(self, url, data=None, **kwargs):
-        return self.client.post(url, data=data, **kwargs)
-
-    def put(self, url, data=None, **kwargs):
-        return self.client.put(url, data=data, **kwargs)
-
-    def delete(self, url, data=None, **kwargs):
-        return self.client.delete(url, data=data, **kwargs)
-
-    def assertRedirects(self, response, location, message=None):
-        """
-        Checks if response is an HTTP redirect to the
-        given location.
-        :param response: Flask response
-        :param location: relative URL path to SERVER_NAME or an absolute URL
-        """
-        __tracebackhide__ = True
-        helpers.assert_redirects(response, location, message=message)
-
-    def assertStatus(self, response, status_code, message=None):
-        __tracebackhide__ = True
-        helpers.assert_status(response, status_code, message=message)
-
-    def full_url(self, *args, **kwargs):
+    def assertStreamEqual(self, response1: Response, response2: Response):
         __tracebackhide__ = True
-        return helpers.full_url(*args, **kwargs)
+        stream1 = list(response1.iter_encoded())
+        stream2 = list(response2.iter_encoded())
+        assert stream1 == stream2
 
-    def login(self, user=None):
-        self.user = self.client.login(user)
-        return self.user
 
-
-for code in 200, 201, 204, 400, 401, 403, 404, 410, 500:
-    name = "assert{0}".format(code)
-    helper = getattr(helpers, name)
-    setattr(WebTestMixin, name, lambda s, r, h=helper: h(r))
+class TestCase(TestCaseMixin, unittest.TestCase):
+    pass
 
 
-@pytest.mark.usefixtures("clean_db")
-class DBTestMixin(object):
+class PytestOnlyTestCase(TestCaseMixin):
     pass

udata/tests/api/__init__.py

@@ -1,12 +1,30 @@
 from contextlib import contextmanager
+from urllib.parse import urlparse
 
 import pytest
 
-from ..frontend import FrontTestCase
+from udata.mongo import db
+from udata.mongo.document import get_all_models
+from udata.tests import PytestOnlyTestCase, TestCase, helpers
 
 
 @pytest.mark.usefixtures("instance_path")
-class APITestCase(FrontTestCase):
+class APITestCaseMixin:
+    """
+    See explanation about `get`, `post` overrides in :TestClientOverride
+
+    (switch from `data` in kwargs to `data` in args to avoid doing `data=data` and default to `json=True`)
+    """
+
+    user = None
+
+    @pytest.fixture(autouse=True)
+    def load_api_routes(self, app):
+        from udata import api, frontend
+
+        api.init_app(app)
+        frontend.init_app(app)
+
     @pytest.fixture(autouse=True)
     def inject_api(self, api):
         """
@@ -14,11 +32,22 @@ class APITestCase(FrontTestCase):
         """
         self.api = api
 
+    @pytest.fixture(autouse=True)
+    def inject_client(self, client):
+        """
+        Inject test client for compatibility with Flask-Testing.
+        """
+        self.client = client
+
     @contextmanager
     def api_user(self, user=None):
         with self.api.user(user) as user:
             yield user
 
+    def login(self, user=None):
+        self.user = self.client.login(user)
+        return self.user
+
     def get(self, url, *args, **kwargs):
         return self.api.get(url, *args, **kwargs)
 
@@ -36,3 +65,59 @@ class APITestCase(FrontTestCase):
 
     def options(self, url, data=None, *args, **kwargs):
         return self.api.options(url, data=data, *args, **kwargs)
+
+    def assertStatus(self, response, status_code, message=None):
+        __tracebackhide__ = True
+        helpers.assert_status(response, status_code, message=message)
+
+
+for code in 200, 201, 204, 400, 401, 403, 404, 410, 500:
+    name = "assert{0}".format(code)
+    helper = getattr(helpers, name)
+    setattr(APITestCaseMixin, name, lambda s, r, h=helper: h(r))
+
+
+class _CleanDBMixin:
+    """
+    This is only for internal use. We shouldn't inherit from this mixin but
+    from `DBTestCase` or `PytestOnlyDBTestCase` (or `*APITestCase`)
+    This is temporary while we have two hierarchies.
+    """
+
+    def drop_db(self, app):
+        """Clear the database"""
+        parsed_url = urlparse(app.config["MONGODB_HOST"])
+
+        # drop the leading /
+        db_name = parsed_url.path[1:]
+        db.connection.drop_database(db_name)
+
+    @pytest.fixture(autouse=True)
+    def _clean_db(self, app):
+        self.drop_db(app)
+        for model in get_all_models():
+            # When dropping the database, MongoEngine will keep the collection cached inside
+            # `_collection` (in memory). This cache is used to call `ensure_indexes` only on the
+            # first call to `_get_collection()`, on subsequent calls the value inside `_collection`
+            # is returned without calling `ensure_indexes`.
+            # In tests, the first test will have a clean memory state, so MongoEngine will initialise
+            # the collection and create the indexes, then the following test, with a clean database (no indexes)
+            # will have the collection cached, so MongoEngine will never create the indexes (except if `auto_create_index_on_save`
+            # is set on the model, which may be the reason it is present on most of the big models, we may remove it?)
+            model._collection = None
+
+
+class DBTestCase(_CleanDBMixin, TestCase):
+    pass
+
+
+class PytestOnlyDBTestCase(_CleanDBMixin, PytestOnlyTestCase):
+    pass
+
+
+class APITestCase(APITestCaseMixin, DBTestCase):
+    pass
+
+
+class PytestOnlyAPITestCase(APITestCaseMixin, PytestOnlyDBTestCase):
+    pass