udata 14.0.3.dev1__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl

udata/harvest/tests/test_api.py

@@ -5,6 +5,8 @@ import pytest
 from flask import url_for
 from pytest_mock import MockerFixture
 
+from udata.core.dataservices.factories import DataserviceFactory
+from udata.core.dataset.factories import DatasetFactory
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import AdminFactory, UserFactory
 from udata.harvest.backends import get_enabled_backends
@@ -18,10 +20,11 @@ from ..models import (
     VALIDATION_ACCEPTED,
     VALIDATION_PENDING,
     VALIDATION_REFUSED,
+    HarvestItem,
     HarvestSource,
     HarvestSourceValidation,
 )
-from .factories import HarvestSourceFactory, MockBackendsMixin
+from .factories import HarvestJobFactory, HarvestSourceFactory, MockBackendsMixin
 
 log = logging.getLogger(__name__)
 
@@ -383,7 +386,7 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
         assert source["config"] == {"custom": "value"}
 
     def test_update_source(self):
-        """It should update a source if owner or orga member"""
+        """It should update a source if owner or orga admin"""
         user = self.login()
         source = HarvestSourceFactory(owner=user)
         new_url = faker.url()
@@ -398,8 +401,8 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
         assert200(response)
         assert response.json["url"] == new_url
 
-        # Source is now owned by orga, with user as member
-        source.organization = OrganizationFactory(members=[Member(user=user)])
+        # Source is now owned by orga, with user as admin
+        source.organization = OrganizationFactory(members=[Member(user=user, role="admin")])
         source.save()
         api_url = url_for("api.harvest_source", source=source)
         response = self.put(api_url, data)
@@ -473,8 +476,8 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
         assert404(response)
 
     def test_source_preview(self):
-        self.login()
-        source = HarvestSourceFactory(backend="factory")
+        user = self.login()
+        source = HarvestSourceFactory(backend="factory", owner=user)
 
         url = url_for("api.preview_harvest_source", source=source)
         response = self.get(url)
@@ -648,3 +651,155 @@ class HarvestAPITest(MockBackendsMixin, PytestOnlyAPITestCase):
 
         source.reload()
         assert source.periodic_task is not None
+
+    def test_list_items(self):
+        """It should fetch the harvest items list from the API for a specific job"""
+        job = HarvestJobFactory(
+            items=[
+                HarvestItem(dataset=DatasetFactory()),
+                HarvestItem(dataservice=DataserviceFactory()),
+                HarvestItem(dataset=DatasetFactory(), remote_url="https://my.remote.example.com"),
+            ],
+        )
+        response = self.get(url_for("api.harvest_job", ident=str(job.id)))
+        assert200(response)
+        assert len(response.json["items"]) == 3
+        assert set(response.json["items"][0].keys()) == set(
+            [
+                "created",
+                "started",
+                "ended",
+                "dataset",
+                "dataservice",
+                "remote_url",
+                "remote_id",
+                "args",
+                "errors",
+                "kwargs",
+                "logs",
+                "status",
+            ]
+        )
+        # Make sure appropriate dataset or dataservice is set
+        assert response.json["items"][0]["dataset"] is not None
+        assert response.json["items"][0]["dataservice"] is None
+        assert response.json["items"][1]["dataset"] is None
+        assert response.json["items"][1]["dataservice"] is not None
+        # Make sure remote_url is exposed if exists
+        assert response.json["items"][1]["remote_url"] is None
+        assert response.json["items"][2]["remote_url"] == "https://my.remote.example.com"
+
+    def test_get_source_permissions_as_anonymous(self):
+        """It should return all permissions as False for anonymous users"""
+        source = HarvestSourceFactory()
+
+        url = url_for("api.harvest_source", source=source)
+        response = self.get(url)
+        assert200(response)
+
+        assert "permissions" in response.json
+        permissions = response.json["permissions"]
+        assert permissions["edit"] is False
+        assert permissions["delete"] is False
+        assert permissions["run"] is False
+        assert permissions["preview"] is False
+        assert permissions["validate"] is False
+        assert permissions["schedule"] is False
+
+    def test_get_source_permissions_as_owner(self):
+        """It should return owner permissions as True for source owner"""
+        user = self.login()
+        source = HarvestSourceFactory(owner=user)
+
+        url = url_for("api.harvest_source", source=source)
+        response = self.get(url)
+        assert200(response)
+
+        permissions = response.json["permissions"]
+        assert permissions["edit"] is True
+        assert permissions["delete"] is True
+        assert permissions["run"] is True
+        assert permissions["preview"] is True
+        assert permissions["validate"] is False
+        assert permissions["schedule"] is False
+
+    def test_get_source_permissions_as_org_admin(self):
+        """It should return owner permissions as True for org admins"""
+        user = self.login()
+        member = Member(user=user, role="admin")
+        org = OrganizationFactory(members=[member])
+        source = HarvestSourceFactory(organization=org)
+
+        url = url_for("api.harvest_source", source=source)
+        response = self.get(url)
+        assert200(response)
+
+        permissions = response.json["permissions"]
+        assert permissions["edit"] is True
+        assert permissions["delete"] is True
+        assert permissions["run"] is True
+        assert permissions["preview"] is True
+        assert permissions["validate"] is False
+        assert permissions["schedule"] is False
+
+    def test_get_source_permissions_as_org_editor(self):
+        """It should return only preview permission as True for org editors"""
+        user = self.login()
+        member = Member(user=user, role="editor")
+        org = OrganizationFactory(members=[member])
+        source = HarvestSourceFactory(organization=org)
+
+        url = url_for("api.harvest_source", source=source)
+        response = self.get(url)
+        assert200(response)
+
+        permissions = response.json["permissions"]
+        assert permissions["edit"] is False
+        assert permissions["delete"] is False
+        assert permissions["run"] is False
+        assert permissions["preview"] is True
+        assert permissions["validate"] is False
+        assert permissions["schedule"] is False
+
+    def test_get_source_permissions_as_superadmin(self):
+        """It should return all permissions as True for admin users"""
+        self.login(AdminFactory())
+        source = HarvestSourceFactory()
+
+        url = url_for("api.harvest_source", source=source)
+        response = self.get(url)
+        assert200(response)
+
+        permissions = response.json["permissions"]
+        assert permissions["edit"] is True
+        assert permissions["delete"] is True
+        assert permissions["run"] is True
+        assert permissions["preview"] is True
+        assert permissions["validate"] is True
+        assert permissions["schedule"] is True
+
+    def test_get_source_permissions_as_other_user(self):
+        """It should return all permissions as False for non-owner users"""
+        self.login()
+        source = HarvestSourceFactory()  # owned by another user
+
+        url = url_for("api.harvest_source", source=source)
+        response = self.get(url)
+        assert200(response)
+
+        permissions = response.json["permissions"]
+        assert permissions["edit"] is False
+        assert permissions["delete"] is False
+        assert permissions["run"] is False
+        assert permissions["preview"] is False
+        assert permissions["validate"] is False
+        assert permissions["schedule"] is False
+
+    def test_preview_source_require_permission(self):
+        """It should not allow preview if not the owner"""
+        self.login()
+        source = HarvestSourceFactory()  # owned by another user
+
+        url = url_for("api.preview_harvest_source", source=source)
+        response = self.get(url)
+        assert403(response)

udata/harvest/tests/test_base_backend.py

@@ -6,15 +6,23 @@ from voluptuous import Schema
 
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import Dataservice
+from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.models import HarvestDatasetMetadata
 from udata.harvest.models import HarvestItem
 from udata.models import Dataset
 from udata.tests.api import PytestOnlyDBTestCase
 from udata.tests.helpers import assert_equal_dates
 from udata.utils import faker
 
-from ..backends import BaseBackend, HarvestExtraConfig, HarvestFeature, HarvestFilter
+from ..backends import (
+    BaseBackend,
+    HarvestExtraConfig,
+    HarvestFeature,
+    HarvestFilter,
+    get_all_backends,
+)
 from ..exceptions import HarvestException
 from .factories import HarvestSourceFactory
 
@@ -63,6 +71,11 @@ class FakeBackend(BaseBackend):
                 setattr(dataset, key, value)
         if self.source.config.get("last_modified"):
             dataset.last_modified_internal = self.source.config["last_modified"]
+        if not dataset.harvest:
+            dataset.harvest = HarvestDatasetMetadata()
+        dataset.harvest.remote_url = (
+            f"http://www.example.com/records/dataset-url-{len(self.job.items)}"
+        )
         return dataset
 
     def inner_process_dataservice(self, item: HarvestItem):
@@ -73,6 +86,11 @@ class FakeBackend(BaseBackend):
                 setattr(dataservice, key, value)
         if self.source.config.get("last_modified"):
             dataservice.last_modified_internal = self.source.config["last_modified"]
+        if not dataservice.harvest:
+            dataservice.harvest = HarvestDataserviceMetadata()
+        dataservice.harvest.remote_url = (
+            f"http://www.example.com/records/dataservice-url-{len(self.job.items)}"
+        )
         return dataservice
 
 
@@ -177,6 +195,21 @@ class BaseBackendTest(PytestOnlyDBTestCase):
         backend = FakeBackend(source)
         assert backend.get_extra_config_value("test_str") == "test"
 
+    def test_harvest_item_remote_url(self):
+        n = 3
+        source = HarvestSourceFactory(
+            config={
+                "dataset_remote_ids": gen_remote_IDs(n),
+                "dataservice_remote_ids": gen_remote_IDs(n),
+            }
+        )
+        backend = FakeBackend(source)
+
+        job = backend.harvest()
+
+        assert len(job.items) == 2 * n
+        assert all([item.remote_url for item in job.items])
+
     def test_harvest_source_id(self):
         nb_datasets = 3
         source = HarvestSourceFactory(config={"dataset_remote_ids": gen_remote_IDs(nb_datasets)})
@@ -421,6 +454,42 @@ class BaseBackendTest(PytestOnlyDBTestCase):
         assert dataset_reused_uri.harvest.domain == source.domain
         assert dataset_reused_uri.harvest.source_id == str(source.id)
 
+    def test_duplicate_remote_ids(self):
+        dataset_remote_ids = [
+            "dataset-id-1",
+            "dataset-id-2",
+            "dataset-id-3",
+            "dataset-id-3",
+            "dataset-id-1",
+        ]
+        dataservice_remote_ids = [
+            "dataservice-id-1",
+            "dataservice-id-2",
+            "dataservice-id-2",
+        ]
+        source = HarvestSourceFactory(
+            config={
+                "dataset_remote_ids": dataset_remote_ids,
+                "dataservice_remote_ids": dataservice_remote_ids,
+            }
+        )
+        backend = FakeBackend(source)
+
+        job = backend.harvest()
+
+        assert job.status == "done-errors"
+        assert len(job.items) == len(dataset_remote_ids) + len(dataservice_remote_ids)
+        assert Dataset.objects.count() == len(set(dataset_remote_ids))
+        assert Dataservice.objects.count() == len(set(dataservice_remote_ids))
+        seen = set()
+        for job in job.items:
+            if job.remote_id not in seen:
+                assert job.status == "done"
+                seen.add(job.remote_id)
+            else:
+                assert job.status == "failed"
+                assert job.remote_id in job.errors[0].message
+
 
 class BaseBackendValidateTest(PytestOnlyDBTestCase):
     @pytest.fixture
@@ -497,3 +566,19 @@ class BaseBackendValidateTest(PytestOnlyDBTestCase):
         assert "[nested.0.other-bad-value] expected int: wrong" in msg
         assert "[nested.1.bad-value] expected str: 43" in msg
         assert "[nested.1.other-bad-value] expected int: bad" in msg
+
+
+class AllBackendsTest:
+    def test_all_backends_have_unique_display_name(self):
+        """Ensure all harvest backends have unique display_name values."""
+        backends = get_all_backends()
+
+        display_names = {}
+        for name, backend in backends.items():
+            display_name = backend.display_name
+            assert display_name is not None, f"Backend '{name}' has no display_name"
+            assert display_name not in display_names, (
+                f"Duplicate display_name '{display_name}' found in backends "
+                f"'{display_names[display_name]}' and '{name}'"
+            )
+            display_names[display_name] = name

udata/harvest/tests/test_dcat_backend.py

@@ -4,6 +4,7 @@ import xml.etree.ElementTree as ET
 from datetime import date
 
 import pytest
+import requests
 from flask import current_app
 from lxml import etree
 from rdflib import Graph
@@ -729,7 +730,10 @@ class DcatBackendTest(PytestOnlyDBTestCase):
         assert dataset.contact_points[0].email == "sav.bd@ign.fr"
         assert dataset.contact_points[0].role == "rightsHolder"
 
-        assert dataset.contact_points[1].name == "Administrateur de Données"
+        assert (
+            dataset.contact_points[1].name
+            == "Administrateur de Données (Direction Régionale de l’Environnement de l’Aménagement et du Logement d'Auvergne-Rhône-Alpes (DREAL Auvergne-Rhône-Alpes))"
+        )
         assert dataset.contact_points[1].email == "sig.dreal-ara@developpement-durable.gouv.fr"
         assert dataset.contact_points[1].role == "user"
 
@@ -744,11 +748,17 @@ class DcatBackendTest(PytestOnlyDBTestCase):
         assert dataset is not None
         assert len(dataset.contact_points) == 3
 
-        assert dataset.contact_points[0].name == "Administrateur de Données"
+        assert (
+            dataset.contact_points[0].name
+            == "Administrateur de Données (Direction Régionale de l’Environnement de l’Aménagement et du Logement d'Auvergne-Rhône-Alpes (DREAL Auvergne-Rhône-Alpes))"
+        )
         assert dataset.contact_points[0].email == "sig.dreal-ara@developpement-durable.gouv.fr"
         assert dataset.contact_points[0].role == "contact"
 
-        assert dataset.contact_points[1].name == "Jean-Michel GENIS"
+        assert (
+            dataset.contact_points[1].name
+            == "Jean-Michel GENIS (Conservatoire Botanique National Alpin)"
+        )
         assert dataset.contact_points[1].email == "jm.genis@cbn-alpin.fr"
         assert dataset.contact_points[1].role == "rightsHolder"
 
@@ -931,6 +941,61 @@ class DcatBackendTest(PytestOnlyDBTestCase):
         assert len(job.errors) == 1
         assert "404 Client Error" in job.errors[0].message
 
+    @pytest.mark.parametrize(
+        "exception",
+        [
+            requests.exceptions.ConnectTimeout("Connection timed out"),
+            requests.exceptions.ConnectionError(
+                "Failed to resolve 'example.com' (Name resolution failed)"
+            ),
+            requests.exceptions.SSLError("SSL: CERTIFICATE_VERIFY_FAILED"),
+        ],
+    )
+    def test_connection_errors_are_handled_without_sentry(self, rmock, mocker, exception):
+        """Connection exceptions should be logged as warning, not sent to Sentry."""
+        url = DCAT_URL_PATTERN.format(path="test.jsonld", domain=TEST_DOMAIN)
+        rmock.get(url, exc=exception)
+
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=OrganizationFactory())
+
+        mock_warning = mocker.patch("udata.harvest.backends.base.log.warning")
+        mock_exception = mocker.patch("udata.harvest.backends.base.log.exception")
+
+        actions.run(source)
+        source.reload()
+
+        job = source.get_last_job()
+        assert job.status == "failed"
+        assert len(job.errors) == 1
+        assert str(exception) in job.errors[0].message
+        mock_warning.assert_called_once()
+        assert "connection error" in mock_warning.call_args[0][0].lower()
+        mock_exception.assert_not_called()
+
+    def test_preview_does_not_create_contact_points(self, rmock):
+        """Preview should not create ContactPoints in DB."""
+        from udata.core.contact_point.models import ContactPoint
+
+        LicenseFactory(id="lov2", title="Licence Ouverte Version 2.0")
+        LicenseFactory(id="lov1", title="Licence Ouverte Version 1.0")
+
+        url = mock_dcat(rmock, "catalog.xml", path="catalog.xml")
+        org = OrganizationFactory()
+        source = HarvestSourceFactory(backend="dcat", url=url, organization=org)
+
+        assert ContactPoint.objects.count() == 0
+
+        job = actions.preview(source)
+
+        assert job.status == "done"
+        assert len(job.items) == 4
+
+        # No ContactPoints should have been created in the database
+        assert ContactPoint.objects.count() == 0
+
+        # No datasets should have been created either
+        assert Dataset.objects.count() == 0
+
 
 @pytest.mark.options(HARVESTER_BACKENDS=["csw*"])
 class CswDcatBackendTest(PytestOnlyDBTestCase):

udata/harvest/tests/test_filters.py

@@ -40,6 +40,12 @@ class FiltersTest:
         with pytest.raises(Invalid):
             filters.boolean("vrai")
 
+        with pytest.raises(Invalid):
+            filters.boolean("42")
+
+        with pytest.raises(Invalid):
+            filters.boolean({"key": "value"})
+
     def test_empty_none(self):
         empty_values = 0, "", [], {}
         non_empty_values = "hello", "  hello  "

udata/i18n.py

@@ -1,5 +1,4 @@
 from contextlib import contextmanager
-from importlib import resources
 from importlib.metadata import entry_points
 
 import flask_babel
@@ -18,9 +17,7 @@ def get_translation_directories_and_domains():
 
     for pkg in entry_points(group="udata.i18n"):
         module = pkg.load()
-        path = resources.files(module)
-        # `/ ""` is  here to transform MultiplexedPath to a simple str
-        translations_dirs.append(str(path / ""))
+        translations_dirs.append(module.__path__[0])
         domains.append(pkg.name)
 
     return translations_dirs, domains

udata/mail.py

@@ -39,6 +39,20 @@ class LabelledContent:
         )
 
 
+@dataclass
+class Link:
+    """Simple linkable object for use in ParagraphWithLinks"""
+
+    label: str
+    url: str
+
+    def __str__(self):
+        return str(self.label)
+
+    def url_for(self, **kwargs):
+        return self.url
+
+
 @dataclass
 class ParagraphWithLinks:
     paragraph: LazyString

udata/migrations/2021-08-17-harvest-integrity.py

@@ -5,6 +5,7 @@ Remove Harvest db integrity problems
 
 import logging
 
+import click
 import mongoengine
 
 from udata.core.jobs.models import PeriodicTask
@@ -16,29 +17,35 @@ log = logging.getLogger(__name__)
 def migrate(db):
     log.info("Processing HarvestJob source references.")
 
-    harvest_jobs = HarvestJob.objects().no_cache().all()
+    harvest_jobs = HarvestJob.objects().no_cache()
+    total = harvest_jobs.count()
     count = 0
-    for harvest_job in harvest_jobs:
-        try:
-            harvest_job.source.id
-        except mongoengine.errors.DoesNotExist:
-            count += 1
-            harvest_job.delete()
+    with click.progressbar(harvest_jobs, length=total, label="Checking sources refs") as jobs:
+        for harvest_job in jobs:
+            try:
+                if harvest_job.source is None:
+                    raise mongoengine.errors.DoesNotExist()
+                harvest_job.source.id
+            except mongoengine.errors.DoesNotExist:
+                count += 1
+                harvest_job.delete()
 
     log.info(f"Completed, removed {count} HarvestJob objects")
 
     log.info("Processing HarvestJob items references.")
 
-    harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache().all()
+    harvest_jobs = HarvestJob.objects.filter(items__0__exists=True).no_cache()
+    total = harvest_jobs.count()
     count = 0
-    for harvest_job in harvest_jobs:
-        for item in harvest_job.items:
-            try:
-                item.dataset and item.dataset.id
-            except mongoengine.errors.DoesNotExist:
-                count += 1
-                item.dataset = None
-                harvest_job.save()
+    with click.progressbar(harvest_jobs, length=total, label="Checking items refs") as jobs:
+        for harvest_job in jobs:
+            for item in harvest_job.items:
+                try:
+                    item.dataset and item.dataset.id
+                except mongoengine.errors.DoesNotExist:
+                    count += 1
+                    item.dataset = None
+                    harvest_job.save()
 
     log.info(f"Completed, modified {count} HarvestJob objects")
 

udata/migrations/2025-10-31-create-membership-request-notifications.py

@@ -0,0 +1,55 @@
+"""
+Create MembershipRequestNotification for all pending membership requests
+"""
+
+import logging
+
+import click
+
+from udata.core.organization.models import Organization
+from udata.core.organization.notifications import MembershipRequestNotificationDetails
+from udata.features.notifications.models import Notification
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Processing pending membership requests...")
+
+    created_count = 0
+
+    with click.progressbar(
+        Organization.objects, length=Organization.objects().count()
+    ) as organizations:
+        for org in organizations:
+            # Get all admin users for this organization
+            admin_users = [member.user for member in org.members if member.role == "admin"]
+
+            # Process each pending request
+            for request in org.pending_requests:
+                # Create a notification for each admin user
+                for admin_user in admin_users:
+                    try:
+                        # Check if notification already exists
+                        existing = Notification.objects(
+                            user=admin_user,
+                            details__request_organization=org,
+                            details__request_user=request.user,
+                        ).first()
+                        if not existing:
+                            notification = Notification(user=admin_user)
+                            notification.details = MembershipRequestNotificationDetails(
+                                request_organization=org, request_user=request.user
+                            )
+                            # Set the created_at to match the request creation date
+                            notification.created_at = request.created
+                            notification.save()
+                            created_count += 1
+                    except Exception as e:
+                        log.error(
+                            f"Error creating notification for user {admin_user.id} "
+                            f"and organization {org.id}: {e}"
+                        )
+                        continue
+
+    log.info(f"Created {created_count} MembershipRequestNotifications")

udata/migrations/2025-12-04-add-uuid-to-discussion-messages.py

@@ -0,0 +1,28 @@
+"""
+This migration adds UUIDs to existing discussion messages that don't have one.
+"""
+
+import logging
+
+import click
+
+from udata.core.discussions.models import Discussion
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Adding UUIDs to discussion messages...")
+
+    # Find all discussions that have at least one message without an id
+    discussions = Discussion.objects(
+        __raw__={"discussion": {"$elemMatch": {"id": {"$exists": False}}}}
+    )
+    count = discussions.count()
+
+    with click.progressbar(discussions, length=count) as progress:
+        for discussion in progress:
+            discussion._mark_as_changed("discussion")
+            discussion.save()
+
+    log.info(f"Migration complete. {count} discussions updated.")