udata 14.0.3.dev1__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl

udata/harvest/api.py

@@ -6,7 +6,6 @@ from udata.api import API, api, fields
 from udata.auth import admin_permission
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_fields, dataset_ref_fields
-from udata.core.dataset.permissions import OwnablePermission
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.organization.permissions import EditOrganizationPermission
 from udata.core.user.api_fields import user_ref_fields
@@ -55,6 +54,7 @@ item_fields = api.model(
     "HarvestItem",
     {
         "remote_id": fields.String(description="The item remote ID to process", required=True),
+        "remote_url": fields.String(description="The item remote url (if available)"),
         "dataset": fields.Nested(
             dataset_ref_fields, description="The processed dataset", allow_null=True
         ),
@@ -115,6 +115,18 @@ validation_fields = api.model(
     },
 )
 
+source_permissions_fields = api.model(
+    "HarvestSourcePermissions",
+    {
+        "edit": fields.Permission(),
+        "delete": fields.Permission(),
+        "run": fields.Permission(),
+        "preview": fields.Permission(),
+        "validate": fields.Permission(),
+        "schedule": fields.Permission(),
+    },
+)
+
 source_fields = api.model(
     "HarvestSource",
     {
@@ -153,6 +165,7 @@ source_fields = api.model(
         "schedule": fields.String(
             description="The source schedule (interval or cron expression)", readonly=True
         ),
+        "permissions": fields.Nested(source_permissions_fields, readonly=True),
     },
 )
 
@@ -313,7 +326,7 @@ class SourceAPI(API):
     @api.marshal_with(source_fields)
     def put(self, source: HarvestSource):
         """Update a harvest source"""
-        OwnablePermission(source).test()
+        source.permissions["edit"].test()
         form = api.validate(HarvestSourceForm, source)
         source = actions.update_source(source, form.data)
         return source
@@ -322,18 +335,19 @@ class SourceAPI(API):
     @api.doc("delete_harvest_source")
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
-        OwnablePermission(source).test()
+        source.permissions["delete"].test()
         return actions.delete_source(source), 204
 
 
 @ns.route("/source/<harvest_source:source>/validate/", endpoint="validate_harvest_source")
 class ValidateSourceAPI(API):
     @api.doc("validate_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect(validation_fields)
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Validate or reject an harvest source"""
+        source.permissions["validate"].test()
         form = api.validate(HarvestSourceValidationForm)
         if form.state.data == VALIDATION_ACCEPTED:
             return actions.validate_source(source, form.comment.data)
@@ -354,7 +368,7 @@ class RunSourceAPI(API):
                 "Cannot run source manually. Please contact the platform if you need to reschedule the harvester.",
             )
 
-        OwnablePermission(source).test()
+        source.permissions["run"].test()
 
         if source.validation.state != VALIDATION_ACCEPTED:
             api.abort(400, "Source is not validated. Please validate the source before running.")
@@ -367,11 +381,12 @@ class RunSourceAPI(API):
 @ns.route("/source/<harvest_source:source>/schedule/", endpoint="schedule_harvest_source")
 class ScheduleSourceAPI(API):
     @api.doc("schedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect((str, "A cron expression"))
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Schedule an harvest source"""
+        source.permissions["schedule"].test()
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
@@ -380,10 +395,11 @@ class ScheduleSourceAPI(API):
         return actions.schedule(source, data)
 
     @api.doc("unschedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
         """Unschedule an harvest source"""
+        source.permissions["schedule"].test()
         return actions.unschedule(source), 204
 
 
@@ -408,6 +424,7 @@ class PreviewSourceAPI(API):
     @api.marshal_with(preview_job_fields)
     def get(self, source: HarvestSource):
         """Preview a single harvest source given an ID or a slug"""
+        source.permissions["preview"].test()
         return actions.preview(source)
 
 

udata/harvest/backends/base.py

@@ -166,6 +166,7 @@ class BaseBackend(object):
         log.debug(f"Starting harvesting {self.source.name} ({self.source.url})…")
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
         self.job = factory(status="initialized", started=datetime.utcnow(), source=self.source)
+        self.remote_ids = set()
 
         before_harvest_job.send(self)
         # Set harvest_activity_user on global context during the run
@@ -190,6 +191,7 @@ class BaseBackend(object):
 
             if any(i.status == "failed" for i in self.job.items):
                 self.job.status += "-errors"
+
         except HarvestValidationError as e:
             log.exception(
                 f'Harvesting validation failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -199,6 +201,15 @@ class BaseBackend(object):
 
             error = HarvestError(message=safe_unicode(e))
             self.job.errors.append(error)
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+            log.warning(
+                f'Harvesting connection error for "{safe_unicode(self.source.name)}" ({self.source.backend}): {e}'
+            )
+
+            self.job.status = "failed"
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            self.job.errors.append(error)
         except Exception as e:
             log.exception(
                 f'Harvesting failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -232,8 +243,13 @@ class BaseBackend(object):
 
             current_app.logger.addHandler(log_catcher)
             dataset = self.inner_process_dataset(item, **kwargs)
+            if dataset.harvest:
+                item.remote_url = dataset.harvest.remote_url
+
+            # Use `item.remote_id` from this point, because `inner_process_dataset` could have modified it.
+
+            self.ensure_unique_remote_id(item)
 
-            # Use `item.remote_id` because `inner_process_dataset` could have modified it.
             dataset.harvest = self.update_dataset_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
@@ -291,6 +307,10 @@ class BaseBackend(object):
                 raise HarvestSkipException("missing identifier")
 
             dataservice = self.inner_process_dataservice(item, **kwargs)
+            if dataservice.harvest:
+                item.remote_url = dataservice.harvest.remote_url
+
+            self.ensure_unique_remote_id(item)
 
             dataservice.harvest = self.update_dataservice_harvest_info(
                 dataservice.harvest, remote_id
@@ -325,6 +345,12 @@ class BaseBackend(object):
             item.ended = datetime.utcnow()
             self.save_job()
 
+    def ensure_unique_remote_id(self, item):
+        if item.remote_id in self.remote_ids:
+            raise HarvestValidationError(f"Identifier '{item.remote_id}' already exists")
+
+        self.remote_ids.add(item.remote_id)
+
     def update_dataset_harvest_info(self, harvest: HarvestDatasetMetadata | None, remote_id: int):
         if not harvest:
             harvest = HarvestDatasetMetadata()

udata/harvest/backends/ckan/harvesters.py

@@ -3,6 +3,8 @@ import logging
 from urllib.parse import urljoin
 from uuid import UUID
 
+from dateutil.parser import ParserError
+
 from udata import uris
 from udata.core.dataset.constants import UpdateFrequency
 from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
@@ -173,7 +175,10 @@ class CkanBackend(BaseBackend):
                 continue
             elif key == "spatial":
                 # GeoJSON representation (Polygon or Point)
-                spatial_geom = json.loads(value)
+                if isinstance(value, dict):
+                    spatial_geom = value
+                else:
+                    spatial_geom = json.loads(value)
             elif key == "spatial-text":
                 # Textual representation of the extent / location
                 qs = GeoZone.objects(db.Q(name=value) | db.Q(slug=value))
@@ -199,10 +204,16 @@ class CkanBackend(BaseBackend):
                         log.debug("frequency value not handled: %s", value)
             # Temporal coverage start
             elif key == "temporal_start":
-                temporal_start = daterange_start(value)
+                try:
+                    temporal_start = daterange_start(value)
+                except ParserError:
+                    log.warning(f"Unparseable temporal_start value: '{value}'")
             # Temporal coverage end
             elif key == "temporal_end":
-                temporal_end = daterange_end(value)
+                try:
+                    temporal_end = daterange_end(value)
+                except ParserError:
+                    log.warning(f"Unparseable temporal_end value: '{value}'")
             else:
                 dataset.extras[extra["key"]] = value
 
@@ -213,12 +224,17 @@ class CkanBackend(BaseBackend):
             dataset.spatial.zones = [spatial_zone]
 
         if spatial_geom:
+            if "type" not in spatial_geom:
+                raise HarvestException(f"Spatial geometry {spatial_geom} without `type`")
+
             if spatial_geom["type"] == "Polygon":
                 coordinates = [spatial_geom["coordinates"]]
             elif spatial_geom["type"] == "MultiPolygon":
                 coordinates = spatial_geom["coordinates"]
             else:
-                raise HarvestException("Unsupported spatial geometry")
+                raise HarvestException(
+                    f"Unsupported spatial geometry {spatial_geom['type']} in {spatial_geom}. (Supported types are `Polygon` and `MultiPolygon`)"
+                )
             dataset.spatial.geom = {"type": "MultiPolygon", "coordinates": coordinates}
 
         if temporal_start and temporal_end:
@@ -267,5 +283,6 @@ class CkanBackend(BaseBackend):
 
 class DkanBackend(CkanBackend):
     name = "dkan"
+    display_name = "DKAN"
     schema = dkan_schema
     filters = []

udata/harvest/backends/dcat.py

@@ -225,7 +225,9 @@ class DcatBackend(BaseBackend):
 
         dataset = self.get_dataset(item.remote_id)
         remote_url_prefix = self.get_extra_config_value("remote_url_prefix")
-        dataset = dataset_from_rdf(page, dataset, node=node, remote_url_prefix=remote_url_prefix)
+        dataset = dataset_from_rdf(
+            page, dataset, node=node, remote_url_prefix=remote_url_prefix, dryrun=self.dryrun
+        )
         if dataset.organization:
             dataset.organization.compute_aggregate_metrics = False
             self.organizations_to_update.add(dataset.organization)
@@ -242,6 +244,7 @@ class DcatBackend(BaseBackend):
             node,
             [item.dataset for item in self.job.items],
             remote_url_prefix=remote_url_prefix,
+            dryrun=self.dryrun,
         )
 
     def get_node_from_item(self, graph, item):

udata/harvest/commands.py

@@ -4,6 +4,7 @@ import click
 
 from udata.commands import KO, OK, cli, green, red
 from udata.harvest.backends import get_all_backends, is_backend_enabled
+from udata.models import Dataset
 
 from . import actions
 
@@ -156,3 +157,35 @@ def attach(domain, filename):
     log.info("Attaching datasets for domain %s", domain)
     result = actions.attach(domain, filename)
     log.info("Attached %s datasets to %s", result.success, domain)
+
+
+@grp.command()
+@click.argument("dataset_id")
+def detach(dataset_id):
+    """
+    Detach a dataset_id from its harvest source
+
+    The dataset will be cleaned from harvested information
+    """
+    log.info(f"Detaching dataset {dataset_id}")
+    dataset = Dataset.get(dataset_id)
+    actions.detach(dataset)
+    log.info("Done")
+
+
+@grp.command()
+@click.argument("identifier")
+def detach_all_from_source(identifier):
+    """
+    Detach all datasets from a harvest source
+
+    All the datasets will be cleaned from harvested information.
+    Make sure the harvest source won't create new duplicate datasets,
+    either by deactivating it or filtering its scope, etc.
+    """
+    log.info(f"Detaching datasets from harvest source {identifier}")
+    count = actions.detach_all_from_source(actions.get_source(identifier))
+    log.info(f"Detached {count} datasets")
+    log.warning(
+        "Make sure the harvest source won't create new duplicate datasets, either by deactivating it or filtering its scope, etc."
+    )

udata/harvest/filters.py

@@ -3,6 +3,9 @@ from voluptuous import Invalid
 
 from udata import tags, uris
 
+TRUTHY_STRINGS = ("on", "t", "true", "y", "yes", "1")
+FALSY_STRINGS = ("f", "false", "n", "no", "off", "0")
+
 
 def boolean(value):
     """
@@ -15,17 +18,25 @@ def boolean(value):
     if value is None or isinstance(value, bool):
         return value
 
-    try:
-        return bool(int(value))
-    except ValueError:
+    if isinstance(value, int):
+        return bool(value)
+
+    if isinstance(value, str):
         lower_value = value.strip().lower()
+
         if not lower_value:
             return None
-        if lower_value in ("f", "false", "n", "no", "off"):
+        if lower_value in FALSY_STRINGS:
             return False
-        if lower_value in ("on", "t", "true", "y", "yes"):
+        if lower_value in TRUTHY_STRINGS:
             return True
-        raise Invalid("Unable to parse boolean {0}".format(value))
+        raise Invalid(
+            f"Unable to parse string '{value}' as boolean. Supported values are {','.join(TRUTHY_STRINGS)} for `True` and {','.join(FALSY_STRINGS)} for `False`."
+        )
+
+    raise Invalid(
+        f"Cannot convert value {value} of type {type(value)} to boolean. Supported types are `bool`, `int` and `str`"
+    )
 
 
 def to_date(value):

udata/harvest/models.py

@@ -66,6 +66,7 @@ class HarvestLog(db.EmbeddedDocument):
 
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
+    remote_url = db.StringField()
     dataset = db.ReferenceField(Dataset)
     dataservice = db.ReferenceField(Dataservice)
     status = db.StringField(
@@ -172,6 +173,21 @@ class HarvestSource(Owned, db.Document):
     def __str__(self):
         return self.name or ""
 
+    @property
+    def permissions(self):
+        from udata.auth import admin_permission
+
+        from .permissions import HarvestSourceAdminPermission, HarvestSourcePermission
+
+        return {
+            "edit": HarvestSourceAdminPermission(self),
+            "delete": HarvestSourceAdminPermission(self),
+            "run": HarvestSourceAdminPermission(self),
+            "preview": HarvestSourcePermission(self),
+            "validate": admin_permission,
+            "schedule": admin_permission,
+        }
+
 
 class HarvestJob(db.Document):
     """Keep track of harvestings"""

udata/harvest/permissions.py

@@ -0,0 +1,27 @@
+from udata.auth import Permission, UserNeed
+from udata.core.dataset.permissions import OwnablePermission
+from udata.core.organization.permissions import OrganizationAdminNeed
+
+
+class HarvestSourcePermission(OwnablePermission):
+    """Permission for basic harvest source operations (preview)
+    Allows organization admins, editors, or owner.
+    """
+
+    pass
+
+
+class HarvestSourceAdminPermission(Permission):
+    """Permission for sensitive harvest source operations (edit, delete, run)
+    Allows only organization admins or owner (not editors).
+    """
+
+    def __init__(self, source) -> None:
+        needs = []
+
+        if source.organization:
+            needs.append(OrganizationAdminNeed(source.organization.id))
+        elif source.owner:
+            needs.append(UserNeed(source.owner.fs_uniquifier))
+
+        super(HarvestSourceAdminPermission, self).__init__(*needs)

udata/harvest/tests/ckan/test_ckan_backend.py

@@ -200,6 +200,24 @@ def spatial_geom_multipolygon(resource_data):
     return data, {"multipolygon": multipolygon}
 
 
+@pytest.fixture
+def spatial_geom_polygon_as_dict(resource_data):
+    """
+    Test case where extra["value"] is already a dict in CKAN (e.g., datasud.fr).
+    In some CKAN instances, the spatial value is returned as a dict directly
+    instead of a JSON string, so json.loads() would fail.
+    """
+    polygon = faker.polygon()
+    data = {
+        "name": faker.unique_string(),
+        "title": faker.sentence(),
+        "notes": faker.paragraph(),
+        "resources": [resource_data],
+        "extras": [{"key": "spatial", "value": polygon}],
+    }
+    return data, {"polygon": polygon}
+
+
 @pytest.fixture
 def known_spatial_text_name(resource_data):
     zone = GeoZoneFactory()
@@ -422,6 +440,21 @@ class CkanBackendTest(PytestOnlyDBTestCase):
         dataset = dataset_for(result)
         assert dataset.spatial.geom == multipolygon
 
+    @pytest.mark.ckan_data("spatial_geom_polygon_as_dict")
+    def test_geospatial_geom_polygon_as_dict(self, result, kwargs):
+        """
+        Test that spatial geometry works when the value is already a dict.
+        Some CKAN instances (e.g., datasud.fr) return the spatial value as a dict
+        directly instead of a JSON string.
+        """
+        polygon = kwargs["polygon"]
+        dataset = dataset_for(result)
+
+        assert dataset.spatial.geom == {
+            "type": "MultiPolygon",
+            "coordinates": [polygon["coordinates"]],
+        }
+
     @pytest.mark.ckan_data("skipped_no_resources")
     def test_skip_no_resources(self, source, result):
         job = source.get_last_job()

udata/harvest/tests/test_actions.py

@@ -11,8 +11,8 @@ from udata.core.activity.models import new_activity
 from udata.core.dataservices.factories import DataserviceFactory
 from udata.core.dataservices.models import HarvestMetadata as HarvestDataserviceMetadata
 from udata.core.dataset.activities import UserCreatedDataset
-from udata.core.dataset.factories import DatasetFactory
-from udata.core.dataset.models import HarvestDatasetMetadata
+from udata.core.dataset.factories import DatasetFactory, ResourceFactory
+from udata.core.dataset.models import HarvestDatasetMetadata, HarvestResourceMetadata
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import UserFactory
 from udata.harvest.backends import get_enabled_backends
@@ -460,6 +460,50 @@ class HarvestActionsTest(MockBackendsMixin, PytestOnlyDBTestCase):
         assert result.success == len(datasets)
         assert result.errors == 1
 
+    def test_detach(self):
+        dataset = DatasetFactory(
+            harvest=HarvestDatasetMetadata(
+                source_id="source id", domain="test.org", remote_id="id"
+            ),
+            resources=[
+                ResourceFactory(
+                    harvest=HarvestResourceMetadata(issued_at=datetime.now(), uri="test.org")
+                )
+            ],
+        )
+
+        actions.detach(dataset)
+
+        dataset.reload()
+        assert dataset.harvest is None
+        for resource in dataset.resources:
+            assert resource.harvest is None
+
+    def test_detach_all(self):
+        source = HarvestSourceFactory()
+        datasets = [
+            DatasetFactory(
+                harvest=HarvestDatasetMetadata(
+                    source_id=str(source.id), domain="test.org", remote_id=str(i)
+                ),
+                resources=[
+                    ResourceFactory(
+                        harvest=HarvestResourceMetadata(issued_at=datetime.now(), uri="test.org")
+                    )
+                ],
+            )
+            for i in range(3)
+        ]
+
+        result = actions.detach_all_from_source(source)
+
+        assert result == len(datasets)
+        for dataset in datasets:
+            dataset.reload()
+            assert dataset.harvest is None
+            for resource in dataset.resources:
+                assert resource.harvest is None
+
 
 class ExecutionTestMixin(MockBackendsMixin, PytestOnlyDBTestCase):
     def action(self, *args, **kwargs):