udata 12.0.2.dev10__py3-none-any.whl → 13.0.1.dev21__py3-none-any.whl

udata/harvest/tests/test_api.py

@@ -7,7 +7,9 @@ from pytest_mock import MockerFixture
 
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import AdminFactory, UserFactory
+from udata.harvest.backends import get_enabled_backends
 from udata.models import Member, PeriodicTask
+from udata.tests.api import PytestOnlyAPITestCase
 from udata.tests.helpers import assert200, assert201, assert204, assert400, assert403, assert404
 from udata.tests.plugin import ApiClient
 from udata.utils import faker
@@ -25,15 +27,12 @@ from .factories import HarvestSourceFactory, MockBackendsMixin
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestAPITest(MockBackendsMixin):
-    modules = []
-
+class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
     def test_list_backends(self, api):
         """It should fetch the harvest backends list from the API"""
         response = api.get(url_for("api.harvest_backends"))
         assert200(response)
-        assert len(response.json) == len(actions.list_backends())
+        assert len(response.json) == len(get_enabled_backends())
         for data in response.json:
             assert "id" in data
             assert "label" in data

udata/harvest/tests/test_base_backend.py

@@ -10,6 +10,7 @@ from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
 from udata.harvest.models import HarvestItem
 from udata.models import Dataset
+from udata.tests.api import PytestOnlyDBTestCase
 from udata.tests.helpers import assert_equal_dates
 from udata.utils import faker
 
@@ -28,6 +29,8 @@ def gen_remote_IDs(num: int, prefix: str = "") -> list[str]:
 
 
 class FakeBackend(BaseBackend):
+    name = "fake-backend"
+    display_name = "Fake Backend"
     filters = (
         HarvestFilter("First filter", "first", str),
         HarvestFilter("Second filter", "second", str),
@@ -93,8 +96,7 @@ class HarvestFilterTest:
             HarvestFilter(faker.word(), faker.word(), type, faker.sentence())
 
 
-@pytest.mark.usefixtures("clean_db")
-class BaseBackendTest:
+class BaseBackendTest(PytestOnlyDBTestCase):
     def test_simple_harvest(self):
         now = datetime.utcnow()
         nb_datasets = 3
@@ -420,8 +422,7 @@ class BaseBackendTest:
         assert dataset_reused_uri.harvest.source_id == str(source.id)
 
 
-@pytest.mark.usefixtures("clean_db")
-class BaseBackendValidateTest:
+class BaseBackendValidateTest(PytestOnlyDBTestCase):
     @pytest.fixture
     def validate(self):
         return FakeBackend(HarvestSourceFactory()).validate

udata/harvest/tests/test_dcat_backend.py

@@ -18,6 +18,7 @@ from udata.harvest.models import HarvestJob
 from udata.models import Dataset
 from udata.rdf import DCAT, RDF, namespace_manager
 from udata.storage.s3 import get_from_json
+from udata.tests.api import PytestOnlyDBTestCase
 
 from .. import actions
 from ..backends.dcat import URIS_TO_REPLACE
@@ -67,9 +68,8 @@ def mock_csw_pagination(rmock, path, pattern):
     return url
 
 
-@pytest.mark.usefixtures("clean_db")
-@pytest.mark.options(PLUGINS=["dcat"])
-class DcatBackendTest:
+@pytest.mark.options(HARVESTER_BACKENDS=["dcat"])
+class DcatBackendTest(PytestOnlyDBTestCase):
     def test_simple_flat(self, rmock):
         filename = "flat.jsonld"
         url = mock_dcat(rmock, filename)
@@ -191,7 +191,6 @@ class DcatBackendTest:
 
     def test_harvest_dataservices_keep_attached_associated_datasets(self, rmock):
         """It should update the existing list of dataservice.datasets and not overwrite existing ones"""
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
 
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
@@ -359,10 +358,8 @@ class DcatBackendTest:
             is None
         )
 
-    @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas", HARVEST_MAX_ITEMS=2)
+    @pytest.mark.options(HARVEST_MAX_ITEMS=2)
     def test_harvest_max_items(self, rmock):
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
-
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
         org = OrganizationFactory()
@@ -373,10 +370,7 @@ class DcatBackendTest:
         assert Dataset.objects.count() == 2
         assert HarvestJob.objects.first().status == "done"
 
-    @pytest.mark.options(SCHEMA_CATALOG_URL="https://example.com/schemas")
     def test_harvest_spatial(self, rmock):
-        rmock.get("https://example.com/schemas", json=ResourceSchemaMockData.get_mock_data())
-
         filename = "bnodes.xml"
         url = mock_dcat(rmock, filename)
         org = OrganizationFactory()
@@ -445,6 +439,20 @@ class DcatBackendTest:
         assert resources_by_title["Resource 3-1"].schema.url is None
         assert resources_by_title["Resource 3-1"].schema.version == "2.2.0"
 
+    def test_harvest_inspire_themese(self, rmock):
+        filename = "bnodes.xml"
+        url = mock_dcat(rmock, filename)
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+
+        actions.run(source)
+
+        datasets = {d.harvest.dct_identifier: d for d in Dataset.objects}
+
+        assert set(datasets["1"].tags).issuperset(set(["repartition-des-especes", "inspire"]))
+        assert set(datasets["2"].tags).issuperset(set(["hydrographie", "inspire"]))
+        assert "inspire" not in datasets["3"].tags
+
     def test_simple_nested_attributes(self, rmock):
         filename = "nested.jsonld"
         url = mock_dcat(rmock, filename)
@@ -672,6 +680,9 @@ class DcatBackendTest:
         assert dataset.temporal_coverage is not None
         assert dataset.temporal_coverage.start == date(2004, 11, 3)
         assert dataset.temporal_coverage.end == date(2005, 3, 30)
+        assert set(dataset.tags) == set(
+            ["inspire", "biodiversity-dynamics"]
+        )  # The DCAT.theme with rdf:resource don't have labels properly defined
 
     def test_sigoreme_xml_catalog(self, rmock):
         LicenseFactory(id="fr-lo", title="Licence ouverte / Open Licence")
@@ -703,6 +714,48 @@ class DcatBackendTest:
         )  # noqa
         assert dataset.harvest.last_update.date() == date.today()
 
+    def test_datara_extended_roles_foaf(self, rmock):
+        # Converted manually from ISO-19139 using SEMICeu XSLT (tag geodcat-ap-2.0.0)
+        url = mock_dcat(rmock, "datara--5a26b0f6-0ccf-46ad-ac58-734054b91977.rdf.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+        actions.run(source)
+        dataset = Dataset.objects.filter(organization=org).first()
+
+        assert dataset is not None
+        assert len(dataset.contact_points) == 2
+
+        assert dataset.contact_points[0].name == "IGN"
+        assert dataset.contact_points[0].email == "sav.bd@ign.fr"
+        assert dataset.contact_points[0].role == "rightsHolder"
+
+        assert dataset.contact_points[1].name == "Administrateur de Données"
+        assert dataset.contact_points[1].email == "sig.dreal-ara@developpement-durable.gouv.fr"
+        assert dataset.contact_points[1].role == "user"
+
+    def test_datara_extended_roles_vcard(self, rmock):
+        # Converted manually from ISO-19139 using SEMICeu XSLT (tag geodcat-ap-2.0.0)
+        url = mock_dcat(rmock, "datara--f40c3860-7236-4b30-a141-23b8ae33f7b2.rdf.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+        actions.run(source)
+        dataset = Dataset.objects.filter(organization=org).first()
+
+        assert dataset is not None
+        assert len(dataset.contact_points) == 3
+
+        assert dataset.contact_points[0].name == "Administrateur de Données"
+        assert dataset.contact_points[0].email == "sig.dreal-ara@developpement-durable.gouv.fr"
+        assert dataset.contact_points[0].role == "contact"
+
+        assert dataset.contact_points[1].name == "Jean-Michel GENIS"
+        assert dataset.contact_points[1].email == "jm.genis@cbn-alpin.fr"
+        assert dataset.contact_points[1].role == "rightsHolder"
+
+        assert dataset.contact_points[2].name == "Conservatoire Botanique National Massif Central"
+        assert dataset.contact_points[2].email == "Benoit.Renaux@cbnmc.fr"
+        assert dataset.contact_points[2].role == "rightsHolder"
+
     def test_udata_xml_catalog(self, rmock):
         LicenseFactory(id="fr-lo", title="Licence ouverte / Open Licence")
         url = mock_dcat(rmock, "udata.xml")
@@ -873,9 +926,8 @@ class DcatBackendTest:
         assert "404 Client Error" in job.errors[0].message
 
 
-@pytest.mark.usefixtures("clean_db")
-@pytest.mark.options(PLUGINS=["csw"])
-class CswDcatBackendTest:
+@pytest.mark.options(HARVESTER_BACKENDS=["csw*"])
+class CswDcatBackendTest(PytestOnlyDBTestCase):
     def test_geonetworkv4(self, rmock):
         url = mock_csw_pagination(rmock, "geonetwork/srv/eng/csw.rdf", "geonetworkv4-page-{}.xml")
         org = OrganizationFactory()
@@ -911,6 +963,7 @@ class CswDcatBackendTest:
                 "oise",
                 "somme",
                 "aisne",
+                # "inspire",  TODO: the geonetwork v4 examples use broken URI as theme resources, check if this is still a problem or not
             ]
         )
         assert dataset.harvest.issued_at.date() == date(2017, 1, 1)
@@ -1023,9 +1076,8 @@ class CswDcatBackendTest:
         assert len(job.items) == 1
 
 
-@pytest.mark.usefixtures("clean_db")
-@pytest.mark.options(PLUGINS=["csw"])
-class CswIso19139DcatBackendTest:
+@pytest.mark.options(HARVESTER_BACKENDS=["csw*"])
+class CswIso19139DcatBackendTest(PytestOnlyDBTestCase):
     @pytest.mark.parametrize(
         "remote_url_prefix",
         [
@@ -1085,6 +1137,7 @@ class CswIso19139DcatBackendTest:
                 "donnees-ouvertes",
                 "plu",
                 "usage-des-sols",
+                "inspire",
             ]
         )
         assert dataset.harvest.issued_at.date() == date(2017, 10, 7)
@@ -1195,3 +1248,6 @@ class CswIso19139DcatBackendTest:
         assert dataset.extras["dcat"].get("rights") is None
         for resource in dataset.resources:
             assert resource.extras["dcat"].get("rights") is None
+
+        # Additional INSPIRE tag due to the dataset having a GEMET INSPIRE theme
+        assert "inspire" in dataset.tags

udata/harvest/tests/test_models.py

@@ -1,7 +1,6 @@
 import logging
 
-import pytest
-
+from udata.tests.api import PytestOnlyDBTestCase
 from udata.utils import faker
 
 from ..models import HarvestSource
@@ -9,8 +8,7 @@ from ..models import HarvestSource
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestSourceTest:
+class HarvestSourceTest(PytestOnlyDBTestCase):
     def test_defaults(self):
         source = HarvestSource.objects.create(name="Test", url=faker.url(), backend="factory")
         assert source.name == "Test"

udata/harvest/tests/test_notifications.py

@@ -1,14 +1,12 @@
-import pytest
-
 from udata.core.user.factories import AdminFactory, UserFactory
 from udata.harvest.notifications import validate_harvester_notifications
+from udata.tests.api import PytestOnlyDBTestCase
 from udata.tests.helpers import assert_equal_dates
 
 from .factories import HarvestSourceFactory
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestNotificationsTest:
+class HarvestNotificationsTest(PytestOnlyDBTestCase):
     def test_pending_harvester_validations(self):
         source = HarvestSourceFactory()
         admin = AdminFactory()

udata/harvest/tests/test_tasks.py

@@ -1,14 +1,13 @@
 import logging
 
-import pytest
+from udata.tests.api import PytestOnlyDBTestCase
 
 from ..tasks import purge_harvest_jobs, purge_harvest_sources
 
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.usefixtures("clean_db")
-class HarvestActionsTest:
+class HarvestActionsTest(PytestOnlyDBTestCase):
     def test_purge_sources(self, mocker):
         """It should purge from DB sources flagged as deleted"""
         mock = mocker.patch("udata.harvest.actions.purge_sources")

udata/mail.py

@@ -1,9 +1,11 @@
+import copy
 import logging
-from contextlib import contextmanager
-from smtplib import SMTPException
+from dataclasses import dataclass
+from html import escape
 
 from blinker import signal
 from flask import current_app, render_template
+from flask_babel import LazyString
 from flask_mail import Mail, Message
 
 from udata import i18n
@@ -15,69 +17,104 @@ mail = Mail()
 mail_sent = signal("mail-sent")
 
 
-class FakeMailer(object):
-    """Display sent mail in logging output"""
+@dataclass
+class MailCTA:
+    label: LazyString
+    link: str | None
 
-    def send(self, msg):
-        log.debug(msg.body)
-        log.debug(msg.html)
-        mail_sent.send(msg)
 
+@dataclass
+class LabelledContent:
+    label: LazyString
+    content: str
+    inline: bool = False
+    truncated_at: int = 50
 
-@contextmanager
-def dummyconnection(*args, **kw):
-    """Allow to test email templates rendering without actually send emails."""
-    yield FakeMailer()
+    @property
+    def truncated_content(self) -> str:
+        return (
+            self.content[: self.truncated_at] + "…"
+            if len(self.content) > self.truncated_at
+            else self.content
+        )
 
 
-def init_app(app):
-    mail.init_app(app)
+@dataclass
+class ParagraphWithLinks:
+    paragraph: LazyString
 
+    def __str__(self):
+        return str(self.paragraph)
 
-def send(subject, recipients, template_base, **kwargs):
-    """
-    Send a given email to multiple recipients.
+    @property
+    def html(self):
+        new_paragraph = copy.deepcopy(self.paragraph)
 
-    User prefered language is taken in account.
-    To translate the subject in the right language, you should ugettext_lazy
-    """
-    sender = kwargs.pop("sender", None)
-    if not isinstance(recipients, (list, tuple)):
-        recipients = [recipients]
+        for key, value in new_paragraph._kwargs.items():
+            if hasattr(value, "url_for"):
+                new_paragraph._kwargs[key] = (
+                    f'<a href="{value.url_for(_mailCampaign=True)}" style="color: #000000; text-decoration: underline;">{escape(str(value))}</a>'
+                )
+
+        return str(new_paragraph)
+
+
+@dataclass
+class MailMessage:
+    subject: LazyString
+    paragraphs: list[LazyString | MailCTA | ParagraphWithLinks | LabelledContent | None]
+
+    def __post_init__(self):
+        self.paragraphs = [p for p in self.paragraphs if p is not None]
+
+    def text(self, recipient) -> str:
+        return render_template(
+            "mail/message.txt",
+            message=self,
+            recipient=recipient,
+        )
+
+    def html(self, recipient) -> str:
+        return render_template(
+            "mail/message.html",
+            message=self,
+            recipient=recipient,
+        )
 
-    tpl_path = f"mail/{template_base}"
+    def send(self, recipients):
+        send_mail(recipients, self)
 
+
+def init_app(app):
+    mail.init_app(app)
+
+
+def send_mail(recipients: object | list, message: MailMessage):
     debug = current_app.config.get("DEBUG", False)
     send_mail = current_app.config.get("SEND_MAIL", not debug)
-    connection = mail.connect if send_mail else dummyconnection
-    extras = get_mail_campaign_dict()
-
-    with connection() as conn:
-        for recipient in recipients:
-            lang = i18n._default_lang(recipient)
-            with i18n.language(lang):
-                log.debug('Sending mail "%s" to recipient "%s"', subject, recipient)
-                msg = Message(subject, sender=sender, recipients=[recipient.email])
-                msg.body = render_template(
-                    f"{tpl_path}.txt",
-                    subject=subject,
-                    sender=sender,
-                    recipient=recipient,
-                    extras=extras,
-                    **kwargs,
-                )
-                msg.html = render_template(
-                    f"{tpl_path}.html",
-                    subject=subject,
-                    sender=sender,
-                    recipient=recipient,
-                    extras=extras,
-                    **kwargs,
-                )
-                try:
-                    conn.send(msg)
-                except SMTPException as e:
-                    log.error(f"Error sending mail {e}")
+
+    if not isinstance(recipients, list):
+        recipients = [recipients]
+
+    for recipient in recipients:
+        lang = i18n._default_lang(recipient)
+        to = recipient if isinstance(recipient, str) else recipient.email
+        with i18n.language(lang):
+            msg = Message(
+                subject=str(message.subject),
+                body=message.text(recipient),
+                html=message.html(recipient),
+                recipients=[to],
+            )
+
+        if send_mail:
+            with mail.connect() as conn:
+                conn.send(msg)
+        else:
+            log.debug(f"Sending mail {message.subject} to {to}")
+            log.debug(msg.body)
+            log.debug(msg.html)
+            mail_sent.send(msg)
 
 
 def get_mail_campaign_dict() -> dict:

udata/migrations/2025-01-05-dataservices-fields-changes.py

@@ -6,11 +6,7 @@ import logging
 
 from mongoengine.connection import get_db
 
-from udata.core.dataservices.constants import (
-    DATASERVICE_ACCESS_TYPE_OPEN,
-    DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT,
-    DATASERVICE_ACCESS_TYPE_RESTRICTED,
-)
+from udata.core.access_type.constants import AccessType
 from udata.core.dataservices.models import Dataservice
 
 log = logging.getLogger(__name__)
@@ -47,7 +43,7 @@ def migrate(db):
 
     for dataservice in get_db().dataservice.find({"is_restricted": True, "has_token": False}):
         log.info(
-            f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={DATASERVICE_ACCESS_TYPE_RESTRICTED})"
+            f"\tDataservice #{dataservice['_id']} {dataservice['title']} is restricted but without token. (will be set to access_type={AccessType.RESTRICTED})"
         )
 
     log.info("Processing dataservices…")
@@ -57,21 +53,19 @@ def migrate(db):
             "is_restricted": True,
             # `has_token` could be True or False, we don't care
         },
-        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_RESTRICTED}},
-    )
-    log.info(
-        f"\t{count.modified_count} restricted dataservices to DATASERVICE_ACCESS_TYPE_RESTRICTED"
+        update={"$set": {"access_type": AccessType.RESTRICTED}},
     )
+    log.info(f"\t{count.modified_count} restricted dataservices to AccessType.RESTRICTED")
 
     count = get_db().dataservice.update_many(
         filter={
             "is_restricted": False,
             "has_token": True,
         },
-        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT}},
+        update={"$set": {"access_type": AccessType.OPEN_WITH_ACCOUNT}},
     )
     log.info(
-        f"\t{count.modified_count} dataservices not restricted but with token to DATASERVICE_ACCESS_TYPE_OPEN_WITH_ACCOUNT"
+        f"\t{count.modified_count} dataservices not restricted but with token to AccessType.OPEN_WITH_ACCOUNT"
     )
 
     count = get_db().dataservice.update_many(
@@ -79,9 +73,9 @@ def migrate(db):
             "is_restricted": False,
             "has_token": False,
         },
-        update={"$set": {"access_type": DATASERVICE_ACCESS_TYPE_OPEN}},
+        update={"$set": {"access_type": AccessType.OPEN}},
     )
-    log.info(f"\t{count.modified_count} open dataservices to DATASERVICE_ACCESS_TYPE_OPEN")
+    log.info(f"\t{count.modified_count} open dataservices to AccessType.OPEN")
 
     dataservices: list[Dataservice] = get_db().dataservice.find()
     for dataservice in dataservices:

udata/migrations/2025-10-21-remove-ckan-harvest-modified-at.py

@@ -0,0 +1,28 @@
+"""
+This migration empties harvest.modified_at field in the case of CKAN datasets.
+Indeed, the value that was stored in this field was the *metadata* modification data
+and not the *data* one, contrary to other backends.
+"""
+
+import logging
+
+import click
+
+from udata.core.dataset.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    datasets = Dataset.objects(harvest__backend="CKAN", harvest__modified_at__exists=True)
+    count = datasets.count()
+
+    with click.progressbar(datasets, length=count) as datasets:
+        for dataset in datasets:
+            dataset.harvest.modified_at = None
+            try:
+                dataset.save()
+            except Exception as err:
+                log.error(f"Cannot save dataset {dataset.id} {err}")
+    log.info(f"Updated {count} datasets")
+    log.info("Done")

udata/migrations/2025-10-29-harvesters-sources-integrity.py

@@ -0,0 +1,27 @@
+"""
+Fix HarvestSource with removed `periodic_task`
+"""
+
+import logging
+
+import mongoengine
+
+from udata.harvest.models import HarvestSource
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Processing HarvestSource periodic_task.")
+
+    sources = HarvestSource.objects
+    count = 0
+    for source in sources:
+        try:
+            source.schedule  # query periodic_task
+        except mongoengine.errors.DoesNotExist:
+            count += 1
+            source.periodic_task = None
+            source.save()
+
+    log.info(f"Modified {count} sources")

udata/models/__init__.py

@@ -31,8 +31,6 @@ from udata.features.territories.models import *  # noqa
 # Load HarvestSource model as harvest for catalog
 from udata.harvest.models import HarvestSource as Harvest  # noqa
 
-import udata.linkchecker.models  # noqa
-
 
 def init_app(app):
     entrypoints.get_enabled("udata.models", app)

udata/mongo/extras_fields.py

@@ -11,15 +11,16 @@ ALLOWED_TYPES = (str, int, float, bool, datetime, date, list, dict)
 
 
 class ExtrasField(DictField):
-    def __init__(self, **kwargs):
+    def __init__(self, keys_types={}, **kwargs):
         self.registered = {}
+        for key, dbtype in keys_types.items():
+            self.register(key, dbtype)
         super(ExtrasField, self).__init__()
 
     def register(self, key, dbtype):
         """Register a DB type to add constraint on a given extra key"""
         if not issubclass(dbtype, (BaseField, EmbeddedDocument)):
-            msg = "ExtrasField can only register MongoEngine fields"
-            raise TypeError(msg)
+            raise TypeError("ExtrasField can only register MongoEngine fields")
         self.registered[key] = dbtype
 
     def validate(self, values):

udata/mongo/taglist_field.py

@@ -32,12 +32,12 @@ class TagListField(ListField):
         super(TagListField, self).validate(values)
 
         for tag in values:
-            if not tags.MIN_TAG_LENGTH <= len(tag) <= tags.MAX_TAG_LENGTH:
+            if not tags.TAG_MIN_LENGTH <= len(tag) <= tags.TAG_MAX_LENGTH:
                 self.error(
                     _(
                         'Tag "%(tag)s" must be between %(min)d and %(max)d characters long.',
-                        min=tags.MIN_TAG_LENGTH,
-                        max=tags.MAX_TAG_LENGTH,
+                        min=tags.TAG_MIN_LENGTH,
+                        max=tags.TAG_MAX_LENGTH,
                         tag=tag,
                     )
                 )