udata 14.0.0__py3-none-any.whl → 14.5.1.dev6__py3-none-any.whl

udata/core/spatial/forms.py

@@ -64,8 +64,8 @@ class GeomField(Field):
                     self.data = geojson.GeoJSON.to_instance(value)
             except Exception:
                 self.data = None
-                log.exception("Unable to parse GeoJSON")
-                raise ValueError(self.gettext("Not a valid GeoJSON"))
+                log.warning(f"Unable to parse GeoJSON: {value}")
+                raise validators.ValidationError(self.gettext("Not a valid GeoJSON"))
 
     def pre_validate(self, form):
         if self.data:

udata/core/topic/models.py

@@ -16,7 +16,10 @@ __all__ = ("Topic", "TopicElement")
 
 class TopicElement(Auditable, db.Document):
     title = field(db.StringField(required=False))
-    description = field(db.StringField(required=False))
+    description = field(
+        db.StringField(required=False),
+        markdown=True,
+    )
     tags = field(db.ListField(db.StringField()))
     extras = field(db.ExtrasField())
     element = field(db.GenericReferenceField(choices=["Dataset", "Reuse", "Dataservice"]))
@@ -63,7 +66,10 @@ class Topic(db.Datetimed, Auditable, Linkable, db.Document, Owned):
         db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
         auditable=False,
     )
-    description = field(db.StringField())
+    description = field(
+        db.StringField(),
+        markdown=True,
+    )
     tags = field(db.ListField(db.StringField()))
     color = field(db.IntField())
 

udata/core/user/api.py

@@ -8,6 +8,7 @@ from udata.core.dataset.api_fields import community_resource_fields, dataset_fie
 from udata.core.discussions.actions import discussions_for
 from udata.core.discussions.api import discussion_fields
 from udata.core.followers.api import FollowAPI
+from udata.core.legal.mails import add_send_legal_notice_argument, send_legal_notice_on_deletion
 from udata.core.storages.api import (
     image_parser,
     parse_uploaded_image,
@@ -265,11 +266,14 @@ class UserAvatarAPI(API):
         return {"image": user.avatar}
 
 
-delete_parser = api.parser()
+delete_parser = add_send_legal_notice_argument(api.parser())
 delete_parser.add_argument(
     "no_mail",
     type=bool,
-    help="Do not send a mail to notify the user of the deletion",
+    help=(
+        "Do not send the simple deletion notification email. "
+        "Note: automatically set to True when send_legal_notice=True to avoid sending duplicate emails."
+    ),
     location="args",
     default=False,
 )
@@ -321,8 +325,11 @@ class UserAPI(API):
             api.abort(
                 403, "You cannot delete yourself with this API. " + 'Use the "me" API instead.'
             )
+        send_legal_notice_on_deletion(user, args)
 
-        user.mark_as_deleted(notify=not args["no_mail"], delete_comments=args["delete_comments"])
+        # Skip simple notification if legal notice is sent (to avoid duplicate emails)
+        skip_notification = args["no_mail"] or args["send_legal_notice"]
+        user.mark_as_deleted(notify=not skip_notification, delete_comments=args["delete_comments"])
         return "", 204
 
 

udata/core/user/models.py

@@ -18,6 +18,7 @@ from udata.core.discussions.models import Discussion
 from udata.core.linkable import Linkable
 from udata.core.storages import avatars, default_image_basename
 from udata.frontend.markdown import mdstrip
+from udata.i18n import lazy_gettext as _
 from udata.models import Follow, WithMetrics, db
 from udata.uris import cdata_url
 
@@ -62,7 +63,10 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES)
     )
     website = field(db.URLField())
-    about = field(db.StringField())
+    about = field(
+        db.StringField(),
+        markdown=True,
+    )
 
     prefered_language = field(db.StringField())
 
@@ -116,6 +120,8 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         "auto_create_index_on_save": True,
     }
 
+    verbose_name = _("account")
+
     __metrics_keys__ = [
         "datasets",
         "reuses",
@@ -142,7 +148,7 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         return self.has_role("admin")
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/users/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/users/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(
@@ -297,6 +303,10 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
                 discussion.save()
         Follow.objects(follower=self).delete()
         Follow.objects(following=self).delete()
+        # Remove related notifications
+        from udata.features.notifications.models import Notification
+
+        Notification.objects.with_user_in_details(self).delete()
 
         from udata.models import ContactPoint
 

udata/features/notifications/api.py

@@ -1,30 +1,19 @@
-from udata.api import API, api, fields
+from udata.api import API, api
 from udata.auth import current_user
 
-from .actions import get_notifications
+from .models import Notification
 
 notifs = api.namespace("notifications", "Notifications API")
 
-notifications_fields = api.model(
-    "Notification",
-    {
-        "type": fields.String(description="The notification type", readonly=True),
-        "created_on": fields.ISODateTime(
-            description="The notification creation datetime", readonly=True
-        ),
-        "details": fields.Raw(
-            description="Key-Value details depending on notification type", readonly=True
-        ),
-    },
-)
-
 
 @notifs.route("/", endpoint="notifications")
 class NotificationsAPI(API):
     @api.secure
-    @api.doc("get_notifications")
-    @api.marshal_list_with(notifications_fields)
+    @api.doc("list_notifications")
+    @api.expect(Notification.__index_parser__)
+    @api.marshal_with(Notification.__page_fields__)
     def get(self):
         """List all current user pending notifications"""
         user = current_user._get_current_object()
-        return get_notifications(user)
+        notifications = Notification.objects(user=user)
+        return Notification.apply_pagination(Notification.apply_sort_filters(notifications))

udata/features/notifications/models.py

@@ -0,0 +1,56 @@
+from flask_restx.inputs import boolean
+from mongoengine import NULLIFY
+
+from udata.api_fields import field, generate_fields
+from udata.core.organization.notifications import MembershipRequestNotificationDetails
+from udata.core.user.api_fields import user_ref_fields
+from udata.core.user.models import User
+from udata.models import db
+from udata.mongo.datetime_fields import Datetimed
+from udata.mongo.queryset import UDataQuerySet
+
+
+class NotificationQuerySet(UDataQuerySet):
+    def with_organization_in_details(self, organization):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_organization=organization)
+
+    def with_user_in_details(self, user):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_user=user)
+
+
+def is_handled(base_query, filter_value):
+    if filter_value is None:
+        return base_query
+    if filter_value is True:
+        return base_query.filter(handled_at__ne=None)
+    return base_query.filter(handled_at=None)
+
+
+@generate_fields()
+class Notification(Datetimed, db.Document):
+    meta = {
+        "ordering": ["-created_at"],
+        "queryset_class": NotificationQuerySet,
+    }
+
+    id = field(db.AutoUUIDField(primary_key=True))
+    handled_at = field(
+        db.DateTimeField(),
+        sortable=True,
+        auditable=False,
+        filterable={"key": "handled", "query": is_handled, "type": boolean},
+    )
+    user = field(
+        db.ReferenceField(User, reverse_delete_rule=NULLIFY),
+        nested_fields=user_ref_fields,
+        readonly=True,
+        allow_null=True,
+        auditable=False,
+        filterable={},
+    )
+    details = field(
+        db.GenericEmbeddedDocumentField(choices=(MembershipRequestNotificationDetails,)),
+        generic=True,
+    )

udata/features/notifications/tasks.py

@@ -0,0 +1,25 @@
+import logging
+from datetime import datetime, timedelta
+
+from flask import current_app
+
+from udata.features.notifications.models import Notification
+from udata.tasks import job
+
+log = logging.getLogger(__name__)
+
+
+@job("delete-expired-notifications")
+def delete_expired_notifications(self):
+    # Delete expired notifications
+    handled_at = datetime.utcnow() - timedelta(
+        days=current_app.config["DAYS_AFTER_NOTIFICATION_EXPIRED"]
+    )
+    notifications_to_delete = Notification.objects(
+        handled_at__lte=handled_at,
+    )
+    count = notifications_to_delete.count()
+    for notification in notifications_to_delete:
+        notification.delete()
+
+    log.info(f"Deleted {count} expired notifications")

udata/flask_mongoengine/engine.py

@@ -7,7 +7,6 @@ from mongoengine.errors import DoesNotExist
 from mongoengine.queryset import QuerySet
 
 from .connection import create_connections
-from .json import override_json_encoder
 from .pagination import ListFieldPagination, Pagination
 from .wtf import WtfBaseField
 
@@ -108,9 +107,6 @@ class MongoEngine(object):
 
         app.extensions = getattr(app, "extensions", {})
 
-        # Make documents JSON serializable
-        override_json_encoder(app)
-
         if "mongoengine" not in app.extensions:
             app.extensions["mongoengine"] = {}
 

udata/flask_mongoengine/pagination.py

@@ -6,7 +6,7 @@ from mongoengine.queryset import QuerySet
 
 class Pagination(object):
     def __init__(self, iterable, page, per_page):
-        if page < 1:
+        if page < 1 or per_page < 1:
             abort(404)
 
         self.iterable = iterable

udata/frontend/markdown.py

@@ -7,8 +7,9 @@ import html2text
 import mistune
 from bleach.css_sanitizer import CSSSanitizer
 from bleach.linkifier import LinkifyFilter
-from flask import Markup, current_app, request
+from flask import current_app, request
 from jinja2.filters import do_striptags, do_truncate
+from markupsafe import Markup
 from werkzeug.local import LocalProxy
 
 from udata.i18n import _

udata/harvest/actions.py

@@ -254,7 +254,7 @@ def schedule(
         source.modify(
             periodic_task=PeriodicTask.objects.create(
                 task="harvest",
-                name="Harvest {0}".format(source.name),
+                name=f"Harvest {source.name} ({source.id})",
                 description="Periodic Harvesting",
                 enabled=True,
                 args=[str(source.id)],
@@ -317,3 +317,23 @@ def attach(domain, filename):
             count += 1
 
     return AttachResult(count, errors)
+
+
+def detach(dataset: Dataset):
+    """Detach a dataset from its harvest source
+
+    The dataset will be cleaned from harvested information
+    and will no longer be updated or archived by harvesting.
+    """
+    dataset.harvest = None
+    for resource in dataset.resources:
+        resource.harvest = None
+    dataset.save()
+
+
+def detach_all_from_source(source: HarvestSource):
+    """Detach all datasets linked to a harvest source"""
+    datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
+    for dataset in datasets:
+        detach(dataset)
+    return len(datasets)

udata/harvest/api.py

@@ -6,7 +6,6 @@ from udata.api import API, api, fields
 from udata.auth import admin_permission
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_fields, dataset_ref_fields
-from udata.core.dataset.permissions import OwnablePermission
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.organization.permissions import EditOrganizationPermission
 from udata.core.user.api_fields import user_ref_fields
@@ -55,6 +54,7 @@ item_fields = api.model(
     "HarvestItem",
     {
         "remote_id": fields.String(description="The item remote ID to process", required=True),
+        "remote_url": fields.String(description="The item remote url (if available)"),
         "dataset": fields.Nested(
             dataset_ref_fields, description="The processed dataset", allow_null=True
         ),
@@ -115,6 +115,18 @@ validation_fields = api.model(
     },
 )
 
+source_permissions_fields = api.model(
+    "HarvestSourcePermissions",
+    {
+        "edit": fields.Permission(),
+        "delete": fields.Permission(),
+        "run": fields.Permission(),
+        "preview": fields.Permission(),
+        "validate": fields.Permission(),
+        "schedule": fields.Permission(),
+    },
+)
+
 source_fields = api.model(
     "HarvestSource",
     {
@@ -153,6 +165,7 @@ source_fields = api.model(
         "schedule": fields.String(
             description="The source schedule (interval or cron expression)", readonly=True
         ),
+        "permissions": fields.Nested(source_permissions_fields, readonly=True),
     },
 )
 
@@ -313,7 +326,7 @@ class SourceAPI(API):
     @api.marshal_with(source_fields)
     def put(self, source: HarvestSource):
         """Update a harvest source"""
-        OwnablePermission(source).test()
+        source.permissions["edit"].test()
         form = api.validate(HarvestSourceForm, source)
         source = actions.update_source(source, form.data)
         return source
@@ -322,18 +335,19 @@ class SourceAPI(API):
     @api.doc("delete_harvest_source")
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
-        OwnablePermission(source).test()
+        source.permissions["delete"].test()
         return actions.delete_source(source), 204
 
 
 @ns.route("/source/<harvest_source:source>/validate/", endpoint="validate_harvest_source")
 class ValidateSourceAPI(API):
     @api.doc("validate_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect(validation_fields)
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Validate or reject an harvest source"""
+        source.permissions["validate"].test()
         form = api.validate(HarvestSourceValidationForm)
         if form.state.data == VALIDATION_ACCEPTED:
             return actions.validate_source(source, form.comment.data)
@@ -354,7 +368,7 @@ class RunSourceAPI(API):
                 "Cannot run source manually. Please contact the platform if you need to reschedule the harvester.",
             )
 
-        OwnablePermission(source).test()
+        source.permissions["run"].test()
 
         if source.validation.state != VALIDATION_ACCEPTED:
             api.abort(400, "Source is not validated. Please validate the source before running.")
@@ -367,11 +381,12 @@ class RunSourceAPI(API):
 @ns.route("/source/<harvest_source:source>/schedule/", endpoint="schedule_harvest_source")
 class ScheduleSourceAPI(API):
     @api.doc("schedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect((str, "A cron expression"))
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Schedule an harvest source"""
+        source.permissions["schedule"].test()
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
@@ -380,10 +395,11 @@ class ScheduleSourceAPI(API):
         return actions.schedule(source, data)
 
     @api.doc("unschedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
         """Unschedule an harvest source"""
+        source.permissions["schedule"].test()
         return actions.unschedule(source), 204
 
 
@@ -408,6 +424,7 @@ class PreviewSourceAPI(API):
     @api.marshal_with(preview_job_fields)
     def get(self, source: HarvestSource):
         """Preview a single harvest source given an ID or a slug"""
+        source.permissions["preview"].test()
         return actions.preview(source)
 
 
@@ -437,7 +454,7 @@ class JobAPI(API):
     @api.expect(parser)
     @api.marshal_with(job_fields)
     def get(self, ident):
-        """List all jobs for a given source"""
+        """Get a single job given an ID"""
         return actions.get_job(ident)
 
 

udata/harvest/backends/base.py

@@ -166,6 +166,7 @@ class BaseBackend(object):
         log.debug(f"Starting harvesting {self.source.name} ({self.source.url})…")
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
         self.job = factory(status="initialized", started=datetime.utcnow(), source=self.source)
+        self.remote_ids = set()
 
         before_harvest_job.send(self)
         # Set harvest_activity_user on global context during the run
@@ -190,6 +191,7 @@ class BaseBackend(object):
 
             if any(i.status == "failed" for i in self.job.items):
                 self.job.status += "-errors"
+
         except HarvestValidationError as e:
             log.exception(
                 f'Harvesting validation failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -199,6 +201,15 @@ class BaseBackend(object):
 
             error = HarvestError(message=safe_unicode(e))
             self.job.errors.append(error)
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+            log.warning(
+                f'Harvesting connection error for "{safe_unicode(self.source.name)}" ({self.source.backend}): {e}'
+            )
+
+            self.job.status = "failed"
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            self.job.errors.append(error)
         except Exception as e:
             log.exception(
                 f'Harvesting failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -232,8 +243,13 @@ class BaseBackend(object):
 
             current_app.logger.addHandler(log_catcher)
             dataset = self.inner_process_dataset(item, **kwargs)
+            if dataset.harvest:
+                item.remote_url = dataset.harvest.remote_url
+
+            # Use `item.remote_id` from this point, because `inner_process_dataset` could have modified it.
+
+            self.ensure_unique_remote_id(item)
 
-            # Use `item.remote_id` because `inner_process_dataset` could have modified it.
             dataset.harvest = self.update_dataset_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
@@ -291,6 +307,10 @@ class BaseBackend(object):
                 raise HarvestSkipException("missing identifier")
 
             dataservice = self.inner_process_dataservice(item, **kwargs)
+            if dataservice.harvest:
+                item.remote_url = dataservice.harvest.remote_url
+
+            self.ensure_unique_remote_id(item)
 
             dataservice.harvest = self.update_dataservice_harvest_info(
                 dataservice.harvest, remote_id
@@ -325,6 +345,12 @@ class BaseBackend(object):
             item.ended = datetime.utcnow()
             self.save_job()
 
+    def ensure_unique_remote_id(self, item):
+        if item.remote_id in self.remote_ids:
+            raise HarvestValidationError(f"Identifier '{item.remote_id}' already exists")
+
+        self.remote_ids.add(item.remote_id)
+
     def update_dataset_harvest_info(self, harvest: HarvestDatasetMetadata | None, remote_id: int):
         if not harvest:
             harvest = HarvestDatasetMetadata()

udata/harvest/backends/ckan/harvesters.py

@@ -173,7 +173,10 @@ class CkanBackend(BaseBackend):
                 continue
             elif key == "spatial":
                 # GeoJSON representation (Polygon or Point)
-                spatial_geom = json.loads(value)
+                if isinstance(value, dict):
+                    spatial_geom = value
+                else:
+                    spatial_geom = json.loads(value)
             elif key == "spatial-text":
                 # Textual representation of the extent / location
                 qs = GeoZone.objects(db.Q(name=value) | db.Q(slug=value))
@@ -213,12 +216,17 @@ class CkanBackend(BaseBackend):
             dataset.spatial.zones = [spatial_zone]
 
         if spatial_geom:
+            if "type" not in spatial_geom:
+                raise HarvestException(f"Spatial geometry {spatial_geom} without `type`")
+
             if spatial_geom["type"] == "Polygon":
                 coordinates = [spatial_geom["coordinates"]]
             elif spatial_geom["type"] == "MultiPolygon":
                 coordinates = spatial_geom["coordinates"]
             else:
-                raise HarvestException("Unsupported spatial geometry")
+                raise HarvestException(
+                    f"Unsupported spatial geometry {spatial_geom['type']} in {spatial_geom}. (Supported types are `Polygon` and `MultiPolygon`)"
+                )
             dataset.spatial.geom = {"type": "MultiPolygon", "coordinates": coordinates}
 
         if temporal_start and temporal_end:
@@ -267,5 +275,6 @@ class CkanBackend(BaseBackend):
 
 class DkanBackend(CkanBackend):
     name = "dkan"
+    display_name = "DKAN"
     schema = dkan_schema
     filters = []

udata/harvest/backends/dcat.py

@@ -225,7 +225,9 @@ class DcatBackend(BaseBackend):
 
         dataset = self.get_dataset(item.remote_id)
         remote_url_prefix = self.get_extra_config_value("remote_url_prefix")
-        dataset = dataset_from_rdf(page, dataset, node=node, remote_url_prefix=remote_url_prefix)
+        dataset = dataset_from_rdf(
+            page, dataset, node=node, remote_url_prefix=remote_url_prefix, dryrun=self.dryrun
+        )
         if dataset.organization:
             dataset.organization.compute_aggregate_metrics = False
             self.organizations_to_update.add(dataset.organization)
@@ -242,6 +244,7 @@ class DcatBackend(BaseBackend):
             node,
             [item.dataset for item in self.job.items],
             remote_url_prefix=remote_url_prefix,
+            dryrun=self.dryrun,
         )
 
     def get_node_from_item(self, graph, item):

udata/harvest/commands.py

@@ -4,6 +4,7 @@ import click
 
 from udata.commands import KO, OK, cli, green, red
 from udata.harvest.backends import get_all_backends, is_backend_enabled
+from udata.models import Dataset
 
 from . import actions
 
@@ -156,3 +157,35 @@ def attach(domain, filename):
     log.info("Attaching datasets for domain %s", domain)
     result = actions.attach(domain, filename)
     log.info("Attached %s datasets to %s", result.success, domain)
+
+
+@grp.command()
+@click.argument("dataset_id")
+def detach(dataset_id):
+    """
+    Detach a dataset_id from its harvest source
+
+    The dataset will be cleaned from harvested information
+    """
+    log.info(f"Detaching dataset {dataset_id}")
+    dataset = Dataset.get(dataset_id)
+    actions.detach(dataset)
+    log.info("Done")
+
+
+@grp.command()
+@click.argument("identifier")
+def detach_all_from_source(identifier):
+    """
+    Detach all datasets from a harvest source
+
+    All the datasets will be cleaned from harvested information.
+    Make sure the harvest source won't create new duplicate datasets,
+    either by deactivating it or filtering its scope, etc.
+    """
+    log.info(f"Detaching datasets from harvest source {identifier}")
+    count = actions.detach_all_from_source(actions.get_source(identifier))
+    log.info(f"Detached {count} datasets")
+    log.warning(
+        "Make sure the harvest source won't create new duplicate datasets, either by deactivating it or filtering its scope, etc."
+    )

udata/harvest/filters.py

@@ -3,6 +3,9 @@ from voluptuous import Invalid
 
 from udata import tags, uris
 
+TRUTHY_STRINGS = ("on", "t", "true", "y", "yes", "1")
+FALSY_STRINGS = ("f", "false", "n", "no", "off", "0")
+
 
 def boolean(value):
     """
@@ -15,17 +18,25 @@ def boolean(value):
     if value is None or isinstance(value, bool):
         return value
 
-    try:
-        return bool(int(value))
-    except ValueError:
+    if isinstance(value, int):
+        return bool(value)
+
+    if isinstance(value, str):
         lower_value = value.strip().lower()
+
         if not lower_value:
             return None
-        if lower_value in ("f", "false", "n", "no", "off"):
+        if lower_value in FALSY_STRINGS:
             return False
-        if lower_value in ("on", "t", "true", "y", "yes"):
+        if lower_value in TRUTHY_STRINGS:
             return True
-        raise Invalid("Unable to parse boolean {0}".format(value))
+        raise Invalid(
+            f"Unable to parse string '{value}' as boolean. Supported values are {','.join(TRUTHY_STRINGS)} for `True` and {','.join(FALSY_STRINGS)} for `False`."
+        )
+
+    raise Invalid(
+        f"Cannot convert value {value} of type {type(value)} to boolean. Supported types are `bool`, `int` and `str`"
+    )
 
 
 def to_date(value):