udata 14.0.0__py3-none-any.whl → 14.5.1.dev6__py3-none-any.whl

udata/harvest/models.py

@@ -66,6 +66,7 @@ class HarvestLog(db.EmbeddedDocument):
 
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
+    remote_url = db.StringField()
     dataset = db.ReferenceField(Dataset)
     dataservice = db.ReferenceField(Dataservice)
     status = db.StringField(
@@ -172,6 +173,21 @@ class HarvestSource(Owned, db.Document):
     def __str__(self):
         return self.name or ""
 
+    @property
+    def permissions(self):
+        from udata.auth import admin_permission
+
+        from .permissions import HarvestSourceAdminPermission, HarvestSourcePermission
+
+        return {
+            "edit": HarvestSourceAdminPermission(self),
+            "delete": HarvestSourceAdminPermission(self),
+            "run": HarvestSourceAdminPermission(self),
+            "preview": HarvestSourcePermission(self),
+            "validate": admin_permission,
+            "schedule": admin_permission,
+        }
+
 
 class HarvestJob(db.Document):
     """Keep track of harvestings"""

udata/harvest/permissions.py

@@ -0,0 +1,27 @@
+from udata.auth import Permission, UserNeed
+from udata.core.dataset.permissions import OwnablePermission
+from udata.core.organization.permissions import OrganizationAdminNeed
+
+
+class HarvestSourcePermission(OwnablePermission):
+    """Permission for basic harvest source operations (preview)
+    Allows organization admins, editors, or owner.
+    """
+
+    pass
+
+
+class HarvestSourceAdminPermission(Permission):
+    """Permission for sensitive harvest source operations (edit, delete, run)
+    Allows only organization admins or owner (not editors).
+    """
+
+    def __init__(self, source) -> None:
+        needs = []
+
+        if source.organization:
+            needs.append(OrganizationAdminNeed(source.organization.id))
+        elif source.owner:
+            needs.append(UserNeed(source.owner.fs_uniquifier))
+
+        super(HarvestSourceAdminPermission, self).__init__(*needs)

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -200,6 +200,24 @@ def spatial_geom_multipolygon(resource_data):
     return data, {"multipolygon": multipolygon}
 
 
+@pytest.fixture
+def spatial_geom_polygon_as_dict(resource_data):
+    """
+    Test case where extra["value"] is already a dict in CKAN (e.g., datasud.fr).
+    In some CKAN instances, the spatial value is returned as a dict directly
+    instead of a JSON string, so json.loads() would fail.
+    """
+    polygon = faker.polygon()
+    data = {
+        "name": faker.unique_string(),
+        "title": faker.sentence(),
+        "notes": faker.paragraph(),
+        "resources": [resource_data],
+        "extras": [{"key": "spatial", "value": polygon}],
+    }
+    return data, {"polygon": polygon}
+
+
 @pytest.fixture
 def known_spatial_text_name(resource_data):
     zone = GeoZoneFactory()
@@ -422,6 +440,21 @@ class CkanBackendTest(PytestOnlyDBTestCase):
         dataset = dataset_for(result)
         assert dataset.spatial.geom == multipolygon
 
+    @pytest.mark.ckan_data("spatial_geom_polygon_as_dict")
+    def test_geospatial_geom_polygon_as_dict(self, result, kwargs):
+        """
+        Test that spatial geometry works when the value is already a dict.
+        Some CKAN instances (e.g., datasud.fr) return the spatial value as a dict
+        directly instead of a JSON string.
+        """
+        polygon = kwargs["polygon"]
+        dataset = dataset_for(result)
+
+        assert dataset.spatial.geom == {
+            "type": "MultiPolygon",
+            "coordinates": [polygon["coordinates"]],
+        }
+
     @pytest.mark.ckan_data("skipped_no_resources")
     def test_skip_no_resources(self, source, result):
         job = source.get_last_job()

udata/harvest/tests/test_actions.py

@@ -11,8 +11,8 @@ from udata.core.activity.models import new_activity
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.activities import UserCreatedDataset
-from udata.core.dataset.factories import DatasetFactory
-from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.core.dataset.factories import DatasetFactory, ResourceFactory
+from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import UserFactory
 from udata.harvest.backends import get_enabled_backends
@@ -271,7 +271,16 @@ class HarvestActionsTest(MockBackendsMixin, PytestOnlyDBTestCase):
         assert periodic_task.crontab.day_of_month == "*"
         assert periodic_task.crontab.month_of_year == "*"
         assert periodic_task.enabled
-        assert periodic_task.name == "Harvest {0}".format(source.name)
+        assert periodic_task.name == f"Harvest {source.name} ({source.id})"
+
+    def test_double_schedule_with_same_name(self):
+        source_1 = HarvestSourceFactory(name="A")
+        source_2 = HarvestSourceFactory(name="A")
+
+        actions.schedule(source_1, hour=0)
+        actions.schedule(source_2, hour=0)
+
+        assert len(PeriodicTask.objects) == 2
 
     def test_schedule_from_cron(self):
         source = HarvestSourceFactory()
@@ -288,7 +297,7 @@ class HarvestActionsTest(MockBackendsMixin, PytestOnlyDBTestCase):
         assert periodic_task.crontab.month_of_year == "3"
         assert periodic_task.crontab.day_of_week == "sunday"
         assert periodic_task.enabled
-        assert periodic_task.name == "Harvest {0}".format(source.name)
+        assert periodic_task.name == f"Harvest {source.name} ({source.id})"
 
     def test_reschedule(self):
         source = HarvestSourceFactory()
@@ -308,7 +317,7 @@ class HarvestActionsTest(MockBackendsMixin, PytestOnlyDBTestCase):
         assert periodic_task.crontab.day_of_month == "*"
         assert periodic_task.crontab.month_of_year == "*"
         assert periodic_task.enabled
-        assert periodic_task.name == "Harvest {0}".format(source.name)
+        assert periodic_task.name == f"Harvest {source.name} ({source.id})"
 
     def test_unschedule(self):
         periodic_task = PeriodicTask.objects.create(
@@ -451,6 +460,50 @@ class HarvestActionsTest(MockBackendsMixin, PytestOnlyDBTestCase):
         assert result.success == len(datasets)
         assert result.errors == 1
 
+    def test_detach(self):
+        dataset = DatasetFactory(
+            harvest=HarvestDatasetMetadata(
+                source_id="source id", domain="test.org", remote_id="id"
+            ),
+            resources=[
+                ResourceFactory(
+                    harvest=HarvestResourceMetadata(issued_at=datetime.now(), uri="test.org")
+                )
+            ],
+        )
+
+        actions.detach(dataset)
+
+        dataset.reload()
+        assert dataset.harvest is None
+        for resource in dataset.resources:
+            assert resource.harvest is None
+
+    def test_detach_all(self):
+        source = HarvestSourceFactory()
+        datasets = [
+            DatasetFactory(
+                harvest=HarvestDatasetMetadata(
+                    source_id=str(source.id), domain="test.org", remote_id=str(i)
+                ),
+                resources=[
+                    ResourceFactory(
+                        harvest=HarvestResourceMetadata(issued_at=datetime.now(), uri="test.org")
+                    )
+                ],
+            )
+            for i in range(3)
+        ]
+
+        result = actions.detach_all_from_source(source)
+
+        assert result == len(datasets)
+        for dataset in datasets:
+            dataset.reload()
+            assert dataset.harvest is None
+            for resource in dataset.resources:
+                assert resource.harvest is None
+
 
 class ExecutionTestMixin(MockBackendsMixin, PytestOnlyDBTestCase):
     def action(self, *args, **kwargs):