udata 13.0.1.dev12__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

udata/harvest/filters.py

@@ -3,6 +3,9 @@ from voluptuous import Invalid
 
 from udata import tags, uris
 
+TRUTHY_STRINGS = ("on", "t", "true", "y", "yes", "1")
+FALSY_STRINGS = ("f", "false", "n", "no", "off", "0")
+
 
 def boolean(value):
     """
@@ -15,17 +18,25 @@ def boolean(value):
     if value is None or isinstance(value, bool):
         return value
 
-    try:
-        return bool(int(value))
-    except ValueError:
+    if isinstance(value, int):
+        return bool(value)
+
+    if isinstance(value, str):
         lower_value = value.strip().lower()
+
         if not lower_value:
             return None
-        if lower_value in ("f", "false", "n", "no", "off"):
+        if lower_value in FALSY_STRINGS:
             return False
-        if lower_value in ("on", "t", "true", "y", "yes"):
+        if lower_value in TRUTHY_STRINGS:
             return True
-        raise Invalid("Unable to parse boolean {0}".format(value))
+        raise Invalid(
+            f"Unable to parse string '{value}' as boolean. Supported values are {','.join(TRUTHY_STRINGS)} for `True` and {','.join(FALSY_STRINGS)} for `False`."
+        )
+
+    raise Invalid(
+        f"Cannot convert value {value} of type {type(value)} to boolean. Supported types are `bool`, `int` and `str`"
+    )
 
 
 def to_date(value):

udata/harvest/forms.py

@@ -1,8 +1,8 @@
 from udata.forms import Form, fields, validators
+from udata.harvest.backends import get_backend, get_enabled_backends
 from udata.i18n import lazy_gettext as _
 from udata.utils import safe_unicode
 
-from .actions import list_backends
 from .models import VALIDATION_REFUSED, VALIDATION_STATES
 
 __all__ = "HarvestSourceForm", "HarvestSourceValidationForm"
@@ -13,9 +13,6 @@ class HarvestConfigField(fields.DictField):
     A DictField with extras validations on known configurations
     """
 
-    def get_backend(self, form):
-        return next(b for b in list_backends() if b.name == form.backend.data)
-
     def get_filter_specs(self, backend, key):
         candidates = (f for f in backend.filters if f.key == key)
         return next(candidates, None)
@@ -30,7 +27,10 @@ class HarvestConfigField(fields.DictField):
 
     def pre_validate(self, form):
         if self.data:
-            backend = self.get_backend(form)
+            backend = get_backend(form.backend.data)
+            if backend is None:
+                return  # Should have been catch by the enum check for `form.backend`
+
             # Validate filters
             for f in self.data.get("filters") or []:
                 if not ("key" in f and "value" in f):
@@ -49,6 +49,7 @@ class HarvestConfigField(fields.DictField):
                     msg = '"{0}" filter should of type "{1}"'
                     msg = msg.format(specs.key, specs.type.__name__)
                     raise validators.ValidationError(msg)
+
             # Validate extras configs
             for f in self.data.get("extra_configs") or []:
                 if not ("key" in f and "value" in f):
@@ -63,6 +64,7 @@ class HarvestConfigField(fields.DictField):
                     msg = '"{0}" extra config should be of type "{1}"'
                     msg = msg.format(specs.key, specs.type.__name__)
                     raise validators.ValidationError(msg)
+
             # Validate features
             for key, value in (self.data.get("features") or {}).items():
                 if not isinstance(value, bool):
@@ -81,7 +83,8 @@ class HarvestSourceForm(Form):
     )
     url = fields.URLField(_("URL"), [validators.DataRequired()])
     backend = fields.SelectField(
-        _("Backend"), choices=lambda: [(b.name, b.display_name) for b in list_backends()]
+        _("Backend"),
+        choices=lambda: [(b.name, b.display_name) for b in get_enabled_backends().values()],
     )
     owner = fields.CurrentUserField()
     organization = fields.PublishAsField(_("Publish as"))

udata/harvest/models.py

@@ -66,6 +66,7 @@ class HarvestLog(db.EmbeddedDocument):
 
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
+    remote_url = db.StringField()
     dataset = db.ReferenceField(Dataset)
     dataservice = db.ReferenceField(Dataservice)
     status = db.StringField(
@@ -172,6 +173,21 @@ class HarvestSource(Owned, db.Document):
     def __str__(self):
         return self.name or ""
 
+    @property
+    def permissions(self):
+        from udata.auth import admin_permission
+
+        from .permissions import HarvestSourceAdminPermission, HarvestSourcePermission
+
+        return {
+            "edit": HarvestSourceAdminPermission(self),
+            "delete": HarvestSourceAdminPermission(self),
+            "run": HarvestSourceAdminPermission(self),
+            "preview": HarvestSourcePermission(self),
+            "validate": admin_permission,
+            "schedule": admin_permission,
+        }
+
 
 class HarvestJob(db.Document):
     """Keep track of harvestings"""

udata/harvest/permissions.py

@@ -0,0 +1,27 @@
+from udata.auth import Permission, UserNeed
+from udata.core.dataset.permissions import OwnablePermission
+from udata.core.organization.permissions import OrganizationAdminNeed
+
+
+class HarvestSourcePermission(OwnablePermission):
+    """Permission for basic harvest source operations (preview)
+    Allows organization admins, editors, or owner.
+    """
+
+    pass
+
+
+class HarvestSourceAdminPermission(Permission):
+    """Permission for sensitive harvest source operations (edit, delete, run)
+    Allows only organization admins or owner (not editors).
+    """
+
+    def __init__(self, source) -> None:
+        needs = []
+
+        if source.organization:
+            needs.append(OrganizationAdminNeed(source.organization.id))
+        elif source.owner:
+            needs.append(UserNeed(source.owner.fs_uniquifier))
+
+        super(HarvestSourceAdminPermission, self).__init__(*needs)

udata/harvest/tasks.py

@@ -1,5 +1,3 @@
-from flask import current_app
-
 from udata.tasks import get_logger, job, task
 
 from . import backends
@@ -16,7 +14,7 @@ def harvest(self, ident):
     if source.deleted or not source.active:
         log.info('Ignoring inactive or deleted source "%s"', source.id)
         return  # Ignore deleted and inactive sources
-    Backend = backends.get(current_app, source.backend)
+    Backend = backends.get_backend(source.backend)
     backend = Backend(source)
 
     backend.harvest()
@@ -27,7 +25,7 @@ def harvest_job_item(job_id, item_id):
     log.info('Harvesting item %s for job "%s"', item_id, job_id)
 
     job = HarvestJob.objects.get(pk=job_id)
-    Backend = backends.get(current_app, job.source.backend)
+    Backend = backends.get_backend(job.source.backend)
     backend = Backend(job)
 
     item = next(i for i in job.items if i.remote_id == item_id)
@@ -40,7 +38,7 @@ def harvest_job_item(job_id, item_id):
 def harvest_job_finalize(results, job_id):
     log.info('Finalize harvesting for job "%s"', job_id)
     job = HarvestJob.objects.get(pk=job_id)
-    Backend = backends.get(current_app, job.source.backend)
+    Backend = backends.get_backend(job.source.backend)
     backend = Backend(job)
     backend.finalize()
 

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -200,6 +200,24 @@ def spatial_geom_multipolygon(resource_data):
     return data, {"multipolygon": multipolygon}
 
 
+@pytest.fixture
+def spatial_geom_polygon_as_dict(resource_data):
+    """
+    Test case where extra["value"] is already a dict in CKAN (e.g., datasud.fr).
+    In some CKAN instances, the spatial value is returned as a dict directly
+    instead of a JSON string, so json.loads() would fail.
+    """
+    polygon = faker.polygon()
+    data = {
+        "name": faker.unique_string(),
+        "title": faker.sentence(),
+        "notes": faker.paragraph(),
+        "resources": [resource_data],
+        "extras": [{"key": "spatial", "value": polygon}],
+    }
+    return data, {"polygon": polygon}
+
+
 @pytest.fixture
 def known_spatial_text_name(resource_data):
     zone = GeoZoneFactory()
@@ -364,7 +382,7 @@ def empty_extras(resource_data):
 ##############################################################################
 
 
-@pytest.mark.options(PLUGINS=["ckan"])
+@pytest.mark.options(HARVESTER_BACKENDS=["ckan"])
 class CkanBackendTest(PytestOnlyDBTestCase):
     @pytest.mark.ckan_data("minimal")
     def test_minimal_metadata(self, data, result, kwargs):
@@ -422,6 +440,21 @@ class CkanBackendTest(PytestOnlyDBTestCase):
         dataset = dataset_for(result)
         assert dataset.spatial.geom == multipolygon
 
+    @pytest.mark.ckan_data("spatial_geom_polygon_as_dict")
+    def test_geospatial_geom_polygon_as_dict(self, result, kwargs):
+        """
+        Test that spatial geometry works when the value is already a dict.
+        Some CKAN instances (e.g., datasud.fr) return the spatial value as a dict
+        directly instead of a JSON string.
+        """
+        polygon = kwargs["polygon"]
+        dataset = dataset_for(result)
+
+        assert dataset.spatial.geom == {
+            "type": "MultiPolygon",
+            "coordinates": [polygon["coordinates"]],
+        }
+
     @pytest.mark.ckan_data("skipped_no_resources")
     def test_skip_no_resources(self, source, result):
         job = source.get_last_job()
@@ -505,7 +538,7 @@ class CkanBackendTest(PytestOnlyDBTestCase):
 ##############################################################################
 
 
-@pytest.mark.options(PLUGINS=["ckan"])
+@pytest.mark.options(HARVESTER_BACKENDS=["ckan"])
 class CkanBackendEdgeCasesTest(PytestOnlyDBTestCase):
     def test_minimal_ckan_response(self, rmock):
         """CKAN Harvester should accept the minimum dataset payload"""

udata/harvest/tests/ckan/test_ckan_backend_errors.py

@@ -13,7 +13,7 @@ API_URL = "{}api/3/action/package_list".format(CKAN_URL)
 STATUS_CODE = (400, 500)
 
 
-@pytest.mark.options(PLUGINS=["ckan"])
+@pytest.mark.options(HARVESTER_BACKENDS=["ckan"])
 class CkanBackendErrorsTest(PytestOnlyDBTestCase):
     @pytest.mark.parametrize("code", STATUS_CODE)
     def test_html_error(self, rmock, code):

udata/harvest/tests/ckan/test_ckan_backend_filters.py

@@ -8,7 +8,7 @@ from udata.tests.api import PytestOnlyDBTestCase
 from udata.utils import faker
 
 
-@pytest.mark.options(PLUGINS=["ckan"])
+@pytest.mark.options(HARVESTER_BACKENDS=["ckan"])
 class CkanBackendFilterTest(PytestOnlyDBTestCase):
     def test_include_org_filter(self, ckan, rmock):
         source = HarvestSourceFactory(

udata/harvest/tests/ckan/test_dkan_backend.py

@@ -16,7 +16,7 @@ def data_path(filename):
     return os.path.join(os.path.dirname(__file__), "data", filename)
 
 
-@pytest.mark.options(PLUGINS=["dkan"])
+@pytest.mark.options(HARVESTER_BACKENDS=["dkan"])
 class DkanBackendTest(PytestOnlyDBTestCase):
     def test_dkan_french_w_license(self, rmock):
         """CKAN Harvester should accept the minimum dataset payload"""

udata/harvest/tests/dcat/udata.xml

@@ -37,7 +37,7 @@
     <dct:title>bureau-de-vote-vanves.csv</dct:title>
     <dct:description>Bureaux de vote - Vanves (csv)</dct:description>
     <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/csv?use_labels=true"/>
-    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/5dd4e0b2-4d96-4f36-b73e-b78ec993703c"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/datasets/r/5dd4e0b2-4d96-4f36-b73e-b78ec993703c"/>
     <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
     <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
     <dct:rights>License Not Specified</dct:rights>
@@ -62,7 +62,7 @@
     <dct:title>bureau-de-vote-vanves.geojson</dct:title>
     <dct:description>Bureaux de vote - Vanves (geojson)</dct:description>
     <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/geojson"/>
-    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/d78d1245-6e8e-44d8-88cd-87b1dd66034f"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/datasets/r/d78d1245-6e8e-44d8-88cd-87b1dd66034f"/>
     <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
     <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
     <dct:rights>License Not Specified</dct:rights>
@@ -86,7 +86,7 @@
     <dct:title>vfe_public_219200755_20240506.json</dct:title>
     <dct:description>Ville de Vanves - Part des véhicules à faibles émissions dans le renouvellement du parc (json)</dct:description>
     <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/vfe_public_219200755_20240506/exports/json"/>
-    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/482677b8-379c-45a5-83ef-0361fecb4cc3"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/datasets/r/482677b8-379c-45a5-83ef-0361fecb4cc3"/>
     <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:issued>
     <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:modified>
     <dct:rights>License Not Specified</dct:rights>
@@ -99,7 +99,7 @@
     <dct:title>bureau-de-vote-vanves.zip</dct:title>
     <dct:description>Bureaux de vote - Vanves (shp)</dct:description>
     <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/shp"/>
-    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/b18fb2bd-6c8b-47a8-84fb-cdc8e29f4f1a"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/datasets/r/b18fb2bd-6c8b-47a8-84fb-cdc8e29f4f1a"/>
     <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
     <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
     <dct:rights>License Not Specified</dct:rights>
@@ -112,7 +112,7 @@
     <dct:title>vfe_public_219200755_20240506.csv</dct:title>
     <dct:description>Ville de Vanves - Part des véhicules à faibles émissions dans le renouvellement du parc (csv)</dct:description>
     <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/vfe_public_219200755_20240506/exports/csv?use_labels=true"/>
-    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/aab4d337-a617-4c18-a045-291d68787a7d"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/datasets/r/aab4d337-a617-4c18-a045-291d68787a7d"/>
     <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:issued>
     <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2024-05-06T15:15:18</dct:modified>
     <dct:rights>License Not Specified</dct:rights>
@@ -161,7 +161,7 @@
     <dct:title>bureau-de-vote-vanves.json</dct:title>
     <dct:description>Bureaux de vote - Vanves (json)</dct:description>
     <dcat:downloadURL rdf:resource="https://vanves-seineouest.opendatasoft.com/api/explore/v2.1/catalog/datasets/bureau-de-vote-vanves/exports/json"/>
-    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/fr/datasets/r/0f80d285-72f8-49f8-a691-1dad6bd2f6db"/>
+    <dcat:accessURL rdf:resource="https://www.data.gouv.fr/datasets/r/0f80d285-72f8-49f8-a691-1dad6bd2f6db"/>
     <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:issued>
     <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-04-19T12:21:56</dct:modified>
     <dct:rights>License Not Specified</dct:rights>

udata/harvest/tests/factories.py

@@ -79,4 +79,4 @@ class MockBackendsMixin(object):
     @pytest.fixture(autouse=True)
     def mock_backend(self, mocker):
         return_value = {"factory": FactoryBackend}
-        mocker.patch("udata.harvest.backends.get_all", return_value=return_value)
+        mocker.patch("udata.harvest.backends.get_all_backends", return_value=return_value)

udata/harvest/tests/test_actions.py

@@ -11,17 +11,18 @@ from udata.core.activity.models import new_activity
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.activities import UserCreatedDataset
-from udata.core.dataset.factories import DatasetFactory
-from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.core.dataset.factories import DatasetFactory, ResourceFactory
+from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import UserFactory
+from udata.harvest.backends import get_enabled_backends
+from udata.harvest.backends.base import BaseBackend
 from udata.models import Dataset, PeriodicTask
 from udata.tests.api import PytestOnlyDBTestCase
 from udata.tests.helpers import assert_emit, assert_equal_dates, assert_not_emit
 from udata.utils import faker
 
 from .. import actions, signals
-from ..backends import BaseBackend
 from ..models import (
     VALIDATION_ACCEPTED,
     VALIDATION_PENDING,
@@ -42,9 +43,10 @@ from .factories import (
 log = logging.getLogger(__name__)
 
 
-class HarvestActionsTest(PytestOnlyDBTestCase):
+class HarvestActionsTest(MockBackendsMixin, PytestOnlyDBTestCase):
     def test_list_backends(self):
-        for backend in actions.list_backends():
+        assert len(get_enabled_backends()) > 0
+        for backend in get_enabled_backends().values():
             assert issubclass(backend, BaseBackend)
 
     def test_list_sources(self):
@@ -269,7 +271,16 @@ class HarvestActionsTest(PytestOnlyDBTestCase):
         assert periodic_task.crontab.day_of_month == "*"
         assert periodic_task.crontab.month_of_year == "*"
         assert periodic_task.enabled
-        assert periodic_task.name == "Harvest {0}".format(source.name)
+        assert periodic_task.name == f"Harvest {source.name} ({source.id})"
+
+    def test_double_schedule_with_same_name(self):
+        source_1 = HarvestSourceFactory(name="A")
+        source_2 = HarvestSourceFactory(name="A")
+
+        actions.schedule(source_1, hour=0)
+        actions.schedule(source_2, hour=0)
+
+        assert len(PeriodicTask.objects) == 2
 
     def test_schedule_from_cron(self):
         source = HarvestSourceFactory()
@@ -286,7 +297,7 @@ class HarvestActionsTest(PytestOnlyDBTestCase):
         assert periodic_task.crontab.month_of_year == "3"
         assert periodic_task.crontab.day_of_week == "sunday"
         assert periodic_task.enabled
-        assert periodic_task.name == "Harvest {0}".format(source.name)
+        assert periodic_task.name == f"Harvest {source.name} ({source.id})"
 
     def test_reschedule(self):
         source = HarvestSourceFactory()
@@ -306,7 +317,7 @@ class HarvestActionsTest(PytestOnlyDBTestCase):
         assert periodic_task.crontab.day_of_month == "*"
         assert periodic_task.crontab.month_of_year == "*"
         assert periodic_task.enabled
-        assert periodic_task.name == "Harvest {0}".format(source.name)
+        assert periodic_task.name == f"Harvest {source.name} ({source.id})"
 
     def test_unschedule(self):
         periodic_task = PeriodicTask.objects.create(
@@ -449,6 +460,50 @@ class HarvestActionsTest(PytestOnlyDBTestCase):
         assert result.success == len(datasets)
         assert result.errors == 1
 
+    def test_detach(self):
+        dataset = DatasetFactory(
+            harvest=HarvestDatasetMetadata(
+                source_id="source id", domain="test.org", remote_id="id"
+            ),
+            resources=[
+                ResourceFactory(
+                    harvest=HarvestResourceMetadata(issued_at=datetime.now(), uri="test.org")
+                )
+            ],
+        )
+
+        actions.detach(dataset)
+
+        dataset.reload()
+        assert dataset.harvest is None
+        for resource in dataset.resources:
+            assert resource.harvest is None
+
+    def test_detach_all(self):
+        source = HarvestSourceFactory()
+        datasets = [
+            DatasetFactory(
+                harvest=HarvestDatasetMetadata(
+                    source_id=str(source.id), domain="test.org", remote_id=str(i)
+                ),
+                resources=[
+                    ResourceFactory(
+                        harvest=HarvestResourceMetadata(issued_at=datetime.now(), uri="test.org")
+                    )
+                ],
+            )
+            for i in range(3)
+        ]
+
+        result = actions.detach_all_from_source(source)
+
+        assert result == len(datasets)
+        for dataset in datasets:
+            dataset.reload()
+            assert dataset.harvest is None
+            for resource in dataset.resources:
+                assert resource.harvest is None
+
 
 class ExecutionTestMixin(MockBackendsMixin, PytestOnlyDBTestCase):
     def action(self, *args, **kwargs):