udata 14.0.0__py3-none-any.whl → 14.4.1.dev7__py3-none-any.whl

udata/core/reports/models.py

@@ -2,7 +2,8 @@ from datetime import datetime
 
 from bson import DBRef
 from flask import url_for
-from mongoengine import DO_NOTHING, NULLIFY, signals
+from flask_restx import inputs
+from mongoengine import DO_NOTHING, NULLIFY, Q, signals
 
 from udata.api_fields import field, generate_fields
 from udata.core.user.api_fields import user_ref_fields
@@ -12,7 +13,32 @@ from udata.mongo import db
 from .constants import REPORT_REASONS_CHOICES, REPORTABLE_MODELS
 
 
-@generate_fields()
+class ReportQuerySet(db.BaseQuerySet):
+    def unhandled(self):
+        return self.filter(dismissed_at=None, subject_deleted_at=None)
+
+    def handled(self):
+        return self.filter(Q(dismissed_at__ne=None) | Q(subject_deleted_at__ne=None))
+
+
+def filter_by_handled(base_query, filter_value):
+    if filter_value is True:
+        return base_query.handled()
+    elif filter_value is False:
+        return base_query.unhandled()
+    else:
+        return base_query
+
+
+@generate_fields(
+    standalone_filters=[
+        {
+            "key": "handled",
+            "query": filter_by_handled,
+            "type": inputs.boolean,
+        },
+    ],
+)
 class Report(db.Document):
     by = field(
         db.ReferenceField(User, reverse_delete_rule=NULLIFY),
@@ -44,8 +70,22 @@ class Report(db.Document):
     reported_at = field(
         db.DateTimeField(default=datetime.utcnow, required=True),
         readonly=True,
+        sortable=True,
     )
 
+    dismissed_at = field(
+        db.DateTimeField(),
+    )
+    dismissed_by = field(
+        db.ReferenceField(User, reverse_delete_rule=NULLIFY),
+        nested_fields=user_ref_fields,
+        allow_null=True,
+    )
+
+    meta = {
+        "queryset_class": ReportQuerySet,
+    }
+
     @field(description="Link to the API endpoint for this report")
     def self_api_url(self):
         return url_for("api.report", report=self, _external=True)

udata/core/reuse/models.py

@@ -199,7 +199,7 @@ class Reuse(db.Datetimed, Auditable, WithMetrics, ReuseBadgeMixin, Linkable, Own
         cls.before_save.send(document)
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/reuses/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/reuses/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(

udata/core/reuse/tasks.py

@@ -1,4 +1,5 @@
 from udata.core import storages
+from udata.core.pages.models import Page
 from udata.core.topic.models import TopicElement
 from udata.models import Activity, Discussion, Follow, Transfer
 from udata.tasks import get_logger, job, task
@@ -23,6 +24,12 @@ def purge_reuses(self) -> None:
         Transfer.objects(subject=reuse).delete()
         # Remove reuses references in Topics
         TopicElement.objects(element=reuse).update(element=None)
+        # Remove reuses in pages (mongoengine doesn't support updating a field in a generic embed)
+        Page._get_collection().update_many(
+            {"blocs.reuses": reuse.id},
+            {"$pull": {"blocs.$[b].reuses": reuse.id}},
+            array_filters=[{"b.reuses": reuse.id}],
+        )
         # Remove reuse's logo in all sizes
         if reuse.image.filename is not None:
             storage = storages.images

udata/core/spatial/forms.py

@@ -64,8 +64,8 @@ class GeomField(Field):
                     self.data = geojson.GeoJSON.to_instance(value)
             except Exception:
                 self.data = None
-                log.exception("Unable to parse GeoJSON")
-                raise ValueError(self.gettext("Not a valid GeoJSON"))
+                log.warning(f"Unable to parse GeoJSON: {value}")
+                raise validators.ValidationError(self.gettext("Not a valid GeoJSON"))
 
     def pre_validate(self, form):
         if self.data:

udata/core/user/models.py

@@ -142,7 +142,7 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
         return self.has_role("admin")
 
     def self_web_url(self, **kwargs):
-        return cdata_url(f"/users/{self._link_id(**kwargs)}/", **kwargs)
+        return cdata_url(f"/users/{self._link_id(**kwargs)}", **kwargs)
 
     def self_api_url(self, **kwargs):
         return url_for(
@@ -297,6 +297,10 @@ class User(WithMetrics, UserMixin, Linkable, db.Document):
                 discussion.save()
         Follow.objects(follower=self).delete()
         Follow.objects(following=self).delete()
+        # Remove related notifications
+        from udata.features.notifications.models import Notification
+
+        Notification.objects.with_user_in_details(self).delete()
 
         from udata.models import ContactPoint
 

udata/features/notifications/api.py

@@ -1,30 +1,19 @@
-from udata.api import API, api, fields
+from udata.api import API, api
 from udata.auth import current_user
 
-from .actions import get_notifications
+from .models import Notification
 
 notifs = api.namespace("notifications", "Notifications API")
 
-notifications_fields = api.model(
-    "Notification",
-    {
-        "type": fields.String(description="The notification type", readonly=True),
-        "created_on": fields.ISODateTime(
-            description="The notification creation datetime", readonly=True
-        ),
-        "details": fields.Raw(
-            description="Key-Value details depending on notification type", readonly=True
-        ),
-    },
-)
-
 
 @notifs.route("/", endpoint="notifications")
 class NotificationsAPI(API):
     @api.secure
-    @api.doc("get_notifications")
-    @api.marshal_list_with(notifications_fields)
+    @api.doc("list_notifications")
+    @api.expect(Notification.__index_parser__)
+    @api.marshal_with(Notification.__page_fields__)
     def get(self):
         """List all current user pending notifications"""
         user = current_user._get_current_object()
-        return get_notifications(user)
+        notifications = Notification.objects(user=user)
+        return Notification.apply_pagination(Notification.apply_sort_filters(notifications))

udata/features/notifications/models.py

@@ -0,0 +1,56 @@
+from flask_restx.inputs import boolean
+from mongoengine import NULLIFY
+
+from udata.api_fields import field, generate_fields
+from udata.core.organization.notifications import MembershipRequestNotificationDetails
+from udata.core.user.api_fields import user_ref_fields
+from udata.core.user.models import User
+from udata.models import db
+from udata.mongo.datetime_fields import Datetimed
+from udata.mongo.queryset import UDataQuerySet
+
+
+class NotificationQuerySet(UDataQuerySet):
+    def with_organization_in_details(self, organization):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_organization=organization)
+
+    def with_user_in_details(self, user):
+        """This function must be updated to handle new details cases"""
+        return self(details__request_user=user)
+
+
+def is_handled(base_query, filter_value):
+    if filter_value is None:
+        return base_query
+    if filter_value is True:
+        return base_query.filter(handled_at__ne=None)
+    return base_query.filter(handled_at=None)
+
+
+@generate_fields()
+class Notification(Datetimed, db.Document):
+    meta = {
+        "ordering": ["-created_at"],
+        "queryset_class": NotificationQuerySet,
+    }
+
+    id = field(db.AutoUUIDField(primary_key=True))
+    handled_at = field(
+        db.DateTimeField(),
+        sortable=True,
+        auditable=False,
+        filterable={"key": "handled", "query": is_handled, "type": boolean},
+    )
+    user = field(
+        db.ReferenceField(User, reverse_delete_rule=NULLIFY),
+        nested_fields=user_ref_fields,
+        readonly=True,
+        allow_null=True,
+        auditable=False,
+        filterable={},
+    )
+    details = field(
+        db.GenericEmbeddedDocumentField(choices=(MembershipRequestNotificationDetails,)),
+        generic=True,
+    )

udata/features/notifications/tasks.py

@@ -0,0 +1,25 @@
+import logging
+from datetime import datetime, timedelta
+
+from flask import current_app
+
+from udata.features.notifications.models import Notification
+from udata.tasks import job
+
+log = logging.getLogger(__name__)
+
+
+@job("delete-expired-notifications")
+def delete_expired_notifications(self):
+    # Delete expired notifications
+    handled_at = datetime.utcnow() - timedelta(
+        days=current_app.config["DAYS_AFTER_NOTIFICATION_EXPIRED"]
+    )
+    notifications_to_delete = Notification.objects(
+        handled_at__lte=handled_at,
+    )
+    count = notifications_to_delete.count()
+    for notification in notifications_to_delete:
+        notification.delete()
+
+    log.info(f"Deleted {count} expired notifications")

udata/flask_mongoengine/engine.py

@@ -7,7 +7,6 @@ from mongoengine.errors import DoesNotExist
 from mongoengine.queryset import QuerySet
 
 from .connection import create_connections
-from .json import override_json_encoder
 from .pagination import ListFieldPagination, Pagination
 from .wtf import WtfBaseField
 
@@ -108,9 +107,6 @@ class MongoEngine(object):
 
         app.extensions = getattr(app, "extensions", {})
 
-        # Make documents JSON serializable
-        override_json_encoder(app)
-
         if "mongoengine" not in app.extensions:
             app.extensions["mongoengine"] = {}
 

udata/frontend/markdown.py

@@ -7,8 +7,9 @@ import html2text
 import mistune
 from bleach.css_sanitizer import CSSSanitizer
 from bleach.linkifier import LinkifyFilter
-from flask import Markup, current_app, request
+from flask import current_app, request
 from jinja2.filters import do_striptags, do_truncate
+from markupsafe import Markup
 from werkzeug.local import LocalProxy
 
 from udata.i18n import _

udata/harvest/actions.py

@@ -254,7 +254,7 @@ def schedule(
         source.modify(
             periodic_task=PeriodicTask.objects.create(
                 task="harvest",
-                name="Harvest {0}".format(source.name),
+                name=f"Harvest {source.name} ({source.id})",
                 description="Periodic Harvesting",
                 enabled=True,
                 args=[str(source.id)],
@@ -317,3 +317,23 @@ def attach(domain, filename):
             count += 1
 
     return AttachResult(count, errors)
+
+
+def detach(dataset: Dataset):
+    """Detach a dataset from its harvest source
+
+    The dataset will be cleaned from harvested information
+    and will no longer be updated or archived by harvesting.
+    """
+    dataset.harvest = None
+    for resource in dataset.resources:
+        resource.harvest = None
+    dataset.save()
+
+
+def detach_all_from_source(source: HarvestSource):
+    """Detach all datasets linked to a harvest source"""
+    datasets = Dataset.objects.filter(harvest__source_id=str(source.id))
+    for dataset in datasets:
+        detach(dataset)
+    return len(datasets)

udata/harvest/api.py

@@ -6,7 +6,6 @@ from udata.api import API, api, fields
 from udata.auth import admin_permission
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api_fields import dataset_fields, dataset_ref_fields
-from udata.core.dataset.permissions import OwnablePermission
 from udata.core.organization.api_fields import org_ref_fields
 from udata.core.organization.permissions import EditOrganizationPermission
 from udata.core.user.api_fields import user_ref_fields
@@ -55,6 +54,7 @@ item_fields = api.model(
     "HarvestItem",
     {
         "remote_id": fields.String(description="The item remote ID to process", required=True),
+        "remote_url": fields.String(description="The item remote url (if available)"),
         "dataset": fields.Nested(
             dataset_ref_fields, description="The processed dataset", allow_null=True
         ),
@@ -115,6 +115,18 @@ validation_fields = api.model(
     },
 )
 
+source_permissions_fields = api.model(
+    "HarvestSourcePermissions",
+    {
+        "edit": fields.Permission(),
+        "delete": fields.Permission(),
+        "run": fields.Permission(),
+        "preview": fields.Permission(),
+        "validate": fields.Permission(),
+        "schedule": fields.Permission(),
+    },
+)
+
 source_fields = api.model(
     "HarvestSource",
     {
@@ -153,6 +165,7 @@ source_fields = api.model(
         "schedule": fields.String(
             description="The source schedule (interval or cron expression)", readonly=True
         ),
+        "permissions": fields.Nested(source_permissions_fields, readonly=True),
     },
 )
 
@@ -313,7 +326,7 @@ class SourceAPI(API):
     @api.marshal_with(source_fields)
     def put(self, source: HarvestSource):
         """Update a harvest source"""
-        OwnablePermission(source).test()
+        source.permissions["edit"].test()
         form = api.validate(HarvestSourceForm, source)
         source = actions.update_source(source, form.data)
         return source
@@ -322,18 +335,19 @@ class SourceAPI(API):
     @api.doc("delete_harvest_source")
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
-        OwnablePermission(source).test()
+        source.permissions["delete"].test()
         return actions.delete_source(source), 204
 
 
 @ns.route("/source/<harvest_source:source>/validate/", endpoint="validate_harvest_source")
 class ValidateSourceAPI(API):
     @api.doc("validate_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect(validation_fields)
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Validate or reject an harvest source"""
+        source.permissions["validate"].test()
         form = api.validate(HarvestSourceValidationForm)
         if form.state.data == VALIDATION_ACCEPTED:
             return actions.validate_source(source, form.comment.data)
@@ -354,7 +368,7 @@ class RunSourceAPI(API):
                 "Cannot run source manually. Please contact the platform if you need to reschedule the harvester.",
             )
 
-        OwnablePermission(source).test()
+        source.permissions["run"].test()
 
         if source.validation.state != VALIDATION_ACCEPTED:
             api.abort(400, "Source is not validated. Please validate the source before running.")
@@ -367,11 +381,12 @@ class RunSourceAPI(API):
 @ns.route("/source/<harvest_source:source>/schedule/", endpoint="schedule_harvest_source")
 class ScheduleSourceAPI(API):
     @api.doc("schedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.expect((str, "A cron expression"))
     @api.marshal_with(source_fields)
     def post(self, source: HarvestSource):
         """Schedule an harvest source"""
+        source.permissions["schedule"].test()
         # Handle both syntax: quoted and unquoted
         try:
             data = request.json
@@ -380,10 +395,11 @@ class ScheduleSourceAPI(API):
         return actions.schedule(source, data)
 
     @api.doc("unschedule_harvest_source")
-    @api.secure(admin_permission)
+    @api.secure
     @api.marshal_with(source_fields)
     def delete(self, source: HarvestSource):
         """Unschedule an harvest source"""
+        source.permissions["schedule"].test()
         return actions.unschedule(source), 204
 
 
@@ -408,6 +424,7 @@ class PreviewSourceAPI(API):
     @api.marshal_with(preview_job_fields)
     def get(self, source: HarvestSource):
         """Preview a single harvest source given an ID or a slug"""
+        source.permissions["preview"].test()
         return actions.preview(source)
 
 
@@ -437,7 +454,7 @@ class JobAPI(API):
     @api.expect(parser)
     @api.marshal_with(job_fields)
     def get(self, ident):
-        """List all jobs for a given source"""
+        """Get a single job given an ID"""
         return actions.get_job(ident)
 
 

udata/harvest/backends/base.py

@@ -166,6 +166,7 @@ class BaseBackend(object):
         log.debug(f"Starting harvesting {self.source.name} ({self.source.url})…")
         factory = HarvestJob if self.dryrun else HarvestJob.objects.create
         self.job = factory(status="initialized", started=datetime.utcnow(), source=self.source)
+        self.remote_ids = set()
 
         before_harvest_job.send(self)
         # Set harvest_activity_user on global context during the run
@@ -190,6 +191,7 @@ class BaseBackend(object):
 
             if any(i.status == "failed" for i in self.job.items):
                 self.job.status += "-errors"
+
         except HarvestValidationError as e:
             log.exception(
                 f'Harvesting validation failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -199,6 +201,15 @@ class BaseBackend(object):
 
             error = HarvestError(message=safe_unicode(e))
             self.job.errors.append(error)
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+            log.warning(
+                f'Harvesting connection error for "{safe_unicode(self.source.name)}" ({self.source.backend}): {e}'
+            )
+
+            self.job.status = "failed"
+
+            error = HarvestError(message=safe_unicode(e), details=traceback.format_exc())
+            self.job.errors.append(error)
         except Exception as e:
             log.exception(
                 f'Harvesting failed for "{safe_unicode(self.source.name)}" ({self.source.backend})'
@@ -232,8 +243,13 @@ class BaseBackend(object):
 
             current_app.logger.addHandler(log_catcher)
             dataset = self.inner_process_dataset(item, **kwargs)
+            if dataset.harvest:
+                item.remote_url = dataset.harvest.remote_url
+
+            # Use `item.remote_id` from this point, because `inner_process_dataset` could have modified it.
+
+            self.ensure_unique_remote_id(item)
 
-            # Use `item.remote_id` because `inner_process_dataset` could have modified it.
             dataset.harvest = self.update_dataset_harvest_info(dataset.harvest, item.remote_id)
             dataset.archived = None
 
@@ -291,6 +307,10 @@ class BaseBackend(object):
                 raise HarvestSkipException("missing identifier")
 
             dataservice = self.inner_process_dataservice(item, **kwargs)
+            if dataservice.harvest:
+                item.remote_url = dataservice.harvest.remote_url
+
+            self.ensure_unique_remote_id(item)
 
             dataservice.harvest = self.update_dataservice_harvest_info(
                 dataservice.harvest, remote_id
@@ -325,6 +345,12 @@ class BaseBackend(object):
             item.ended = datetime.utcnow()
             self.save_job()
 
+    def ensure_unique_remote_id(self, item):
+        if item.remote_id in self.remote_ids:
+            raise HarvestValidationError(f"Identifier '{item.remote_id}' already exists")
+
+        self.remote_ids.add(item.remote_id)
+
     def update_dataset_harvest_info(self, harvest: HarvestDatasetMetadata | None, remote_id: int):
         if not harvest:
             harvest = HarvestDatasetMetadata()

udata/harvest/backends/ckan/harvesters.py

@@ -173,7 +173,10 @@ class CkanBackend(BaseBackend):
                 continue
             elif key == "spatial":
                 # GeoJSON representation (Polygon or Point)
-                spatial_geom = json.loads(value)
+                if isinstance(value, dict):
+                    spatial_geom = value
+                else:
+                    spatial_geom = json.loads(value)
             elif key == "spatial-text":
                 # Textual representation of the extent / location
                 qs = GeoZone.objects(db.Q(name=value) | db.Q(slug=value))
@@ -213,12 +216,17 @@ class CkanBackend(BaseBackend):
             dataset.spatial.zones = [spatial_zone]
 
         if spatial_geom:
+            if "type" not in spatial_geom:
+                raise HarvestException(f"Spatial geometry {spatial_geom} without `type`")
+
             if spatial_geom["type"] == "Polygon":
                 coordinates = [spatial_geom["coordinates"]]
             elif spatial_geom["type"] == "MultiPolygon":
                 coordinates = spatial_geom["coordinates"]
             else:
-                raise HarvestException("Unsupported spatial geometry")
+                raise HarvestException(
+                    f"Unsupported spatial geometry {spatial_geom['type']} in {spatial_geom}. (Supported types are `Polygon` and `MultiPolygon`)"
+                )
             dataset.spatial.geom = {"type": "MultiPolygon", "coordinates": coordinates}
 
         if temporal_start and temporal_end:
@@ -267,5 +275,6 @@ class CkanBackend(BaseBackend):
 
 class DkanBackend(CkanBackend):
     name = "dkan"
+    display_name = "DKAN"
     schema = dkan_schema
     filters = []

udata/harvest/commands.py

@@ -4,6 +4,7 @@ import click
 
 from udata.commands import KO, OK, cli, green, red
 from udata.harvest.backends import get_all_backends, is_backend_enabled
+from udata.models import Dataset
 
 from . import actions
 
@@ -156,3 +157,35 @@ def attach(domain, filename):
     log.info("Attaching datasets for domain %s", domain)
     result = actions.attach(domain, filename)
     log.info("Attached %s datasets to %s", result.success, domain)
+
+
+@grp.command()
+@click.argument("dataset_id")
+def detach(dataset_id):
+    """
+    Detach a dataset_id from its harvest source
+
+    The dataset will be cleaned from harvested information
+    """
+    log.info(f"Detaching dataset {dataset_id}")
+    dataset = Dataset.get(dataset_id)
+    actions.detach(dataset)
+    log.info("Done")
+
+
+@grp.command()
+@click.argument("identifier")
+def detach_all_from_source(identifier):
+    """
+    Detach all datasets from a harvest source
+
+    All the datasets will be cleaned from harvested information.
+    Make sure the harvest source won't create new duplicate datasets,
+    either by deactivating it or filtering its scope, etc.
+    """
+    log.info(f"Detaching datasets from harvest source {identifier}")
+    count = actions.detach_all_from_source(actions.get_source(identifier))
+    log.info(f"Detached {count} datasets")
+    log.warning(
+        "Make sure the harvest source won't create new duplicate datasets, either by deactivating it or filtering its scope, etc."
+    )

udata/harvest/filters.py

@@ -3,6 +3,9 @@ from voluptuous import Invalid
 
 from udata import tags, uris
 
+TRUTHY_STRINGS = ("on", "t", "true", "y", "yes", "1")
+FALSY_STRINGS = ("f", "false", "n", "no", "off", "0")
+
 
 def boolean(value):
     """
@@ -15,17 +18,25 @@ def boolean(value):
     if value is None or isinstance(value, bool):
         return value
 
-    try:
-        return bool(int(value))
-    except ValueError:
+    if isinstance(value, int):
+        return bool(value)
+
+    if isinstance(value, str):
         lower_value = value.strip().lower()
+
         if not lower_value:
             return None
-        if lower_value in ("f", "false", "n", "no", "off"):
+        if lower_value in FALSY_STRINGS:
             return False
-        if lower_value in ("on", "t", "true", "y", "yes"):
+        if lower_value in TRUTHY_STRINGS:
             return True
-        raise Invalid("Unable to parse boolean {0}".format(value))
+        raise Invalid(
+            f"Unable to parse string '{value}' as boolean. Supported values are {','.join(TRUTHY_STRINGS)} for `True` and {','.join(FALSY_STRINGS)} for `False`."
+        )
+
+    raise Invalid(
+        f"Cannot convert value {value} of type {type(value)} to boolean. Supported types are `bool`, `int` and `str`"
+    )
 
 
 def to_date(value):

udata/harvest/models.py

@@ -66,6 +66,7 @@ class HarvestLog(db.EmbeddedDocument):
 
 class HarvestItem(db.EmbeddedDocument):
     remote_id = db.StringField()
+    remote_url = db.StringField()
     dataset = db.ReferenceField(Dataset)
     dataservice = db.ReferenceField(Dataservice)
     status = db.StringField(
@@ -172,6 +173,21 @@ class HarvestSource(Owned, db.Document):
     def __str__(self):
         return self.name or ""
 
+    @property
+    def permissions(self):
+        from udata.auth import admin_permission
+
+        from .permissions import HarvestSourceAdminPermission, HarvestSourcePermission
+
+        return {
+            "edit": HarvestSourceAdminPermission(self),
+            "delete": HarvestSourceAdminPermission(self),
+            "run": HarvestSourceAdminPermission(self),
+            "preview": HarvestSourcePermission(self),
+            "validate": admin_permission,
+            "schedule": admin_permission,
+        }
+
 
 class HarvestJob(db.Document):
     """Keep track of harvestings"""