udata 10.2.1.dev34728__py2.py3-none-any.whl → 10.2.1.dev34767__py2.py3-none-any.whl

udata/api/__init__.py

@@ -4,6 +4,7 @@ import urllib.parse
 from functools import wraps
 from importlib import import_module
 
+import mongoengine
 from flask import (
     Blueprint,
     current_app,
@@ -22,7 +23,6 @@ from udata import entrypoints, tracking
 from udata.app import csrf
 from udata.auth import Permission, PermissionDenied, RoleNeed, current_user, login_user
 from udata.i18n import get_locale
-from udata.mongo.errors import FieldValidationError
 from udata.utils import safe_unicode
 
 from . import fields
@@ -258,17 +258,50 @@ def handle_unauthorized_file_type(error):
     return {"message": msg}, 400
 
 
-validation_error_fields = api.model("ValidationError", {"errors": fields.Raw})
+validation_error_fields = api.model(
+    "ValidationError",
+    {"errors": fields.Raw, "message": fields.String},
+)
 
+validation_error_fields_v2 = apiv2.inherit("ValidationError", validation_error_fields)
 
-@api.errorhandler(FieldValidationError)
-@api.marshal_with(validation_error_fields, code=400)
-def handle_validation_error(error: FieldValidationError):
-    """A validation error"""
+
+def convert_object_of_exceptions_to_object_of_strings(exceptions: dict):
     errors = {}
-    errors[error.field] = [error.message]
+    for key, exception in exceptions.items():
+        if isinstance(exception, Exception):
+            errors[key] = str(exception)
+        elif isinstance(exception, dict):
+            errors[key] = convert_object_of_exceptions_to_object_of_strings(exception)
+        elif isinstance(exception, str):
+            errors[key] = exception
+        else:
+            log.warning(
+                f"Unknown type in `convert_object_of_exceptions_to_object_of_strings`: {exception}"
+            )
+            errors[key] = str(exception)
+
+    return errors
+
+
+@api.errorhandler(mongoengine.errors.ValidationError)
+@api.marshal_with(validation_error_fields, code=400)
+def handle_validation_error(error: mongoengine.errors.ValidationError):
+    """Error returned when validation failed."""
+    return (
+        {
+            "errors": convert_object_of_exceptions_to_object_of_strings(error.errors),
+            "message": str(error),
+        },
+        400,
+    )
+
 
-    return {"errors": errors}, 400
+@apiv2.errorhandler(mongoengine.errors.ValidationError)
+@apiv2.marshal_with(validation_error_fields_v2, code=400)
+def handle_validation_error_v2(error: mongoengine.errors.ValidationError):
+    """Error returned when validation failed."""
+    return handle_validation_error(error)
 
 
 class API(Resource):  # Avoid name collision as resource is a core model

udata/api_fields.py

@@ -567,11 +567,7 @@ def is_value_modified(old_value, new_value) -> bool:
 
 def patch_and_save(obj, request) -> type:
     obj = patch(obj, request)
-
-    try:
-        obj.save()
-    except mongoengine.errors.ValidationError as e:
-        api.abort(400, e.message)
+    obj.save()
 
     return obj
 

udata/core/contact_point/api.py

@@ -1,5 +1,3 @@
-import mongoengine
-
 from udata.api import API, api
 from udata.api.parsers import ModelApiParser
 from udata.core.dataset.permissions import OwnablePermission
@@ -33,10 +31,8 @@ class ContactPointsListAPI(API):
     def post(self):
         """Creates a contact point"""
         form = api.validate(ContactPointForm)
-        try:
-            contact_point = form.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+        contact_point = form.save()
+
         return contact_point, 201
 
 

udata/core/dataservices/api.py

@@ -45,11 +45,7 @@ class DataservicesAPI(API):
         if not dataservice.owner and not dataservice.organization:
             dataservice.owner = current_user._get_current_object()
 
-        try:
-            dataservice.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
-
+        dataservice.save()
         return dataservice, 201
 
 
@@ -78,11 +74,8 @@ class DataserviceAPI(API):
         patch(dataservice, request)
         dataservice.metadata_modified_at = datetime.utcnow()
 
-        try:
-            dataservice.save()
-            return dataservice
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+        dataservice.save()
+        return dataservice
 
     @api.secure
     @api.doc("delete_dataservice")

udata/core/dataset/api.py

@@ -264,12 +264,8 @@ class DatasetAPI(API):
         DatasetEditPermission(dataset).test()
         dataset.last_modified_internal = datetime.utcnow()
         form = api.validate(DatasetForm, dataset)
-        # As validation for some fields (ie. extras) is at model
-        # level instead form level, we use mongoengine errors here.
-        try:
-            return form.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+
+        return form.save()
 
     @api.secure
     @api.doc("delete_dataset")

udata/core/dataset/apiv2.py

@@ -340,10 +340,7 @@ class DatasetExtrasAPI(API):
             data.pop(key)
         # then update the extras with the remaining payload
         dataset.extras.update(data)
-        try:
-            dataset.save(signal_kwargs={"ignores": ["post_save"]})
-        except mongoengine.errors.ValidationError as e:
-            apiv2.abort(400, e.message)
+        dataset.save(signal_kwargs={"ignores": ["post_save"]})
         return dataset.extras
 
     @apiv2.secure

udata/core/dataset/forms.py

@@ -54,7 +54,7 @@ class SchemaForm(ModelForm):
             )
         except FieldValidationError as err:
             field = getattr(self, err.field)
-            field.errors.append(err.message)
+            field.errors.append(str(err))
             return False
 
         return validation

udata/core/dataset/models.py

@@ -384,7 +384,7 @@ class ResourceMixin(object):
             return to_naive_datetime(self.harvest.modified_at)
         if self.filetype == "remote" and self.extras.get("analysis:last-modified-at"):
             return to_naive_datetime(self.extras.get("analysis:last-modified-at"))
-        return self.last_modified_internal
+        return to_naive_datetime(self.last_modified_internal)
 
     def clean(self):
         super(ResourceMixin, self).clean()
@@ -565,6 +565,8 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
     extras = db.ExtrasField()
     harvest = db.EmbeddedDocumentField(HarvestDatasetMetadata)
 
+    quality_cached = db.DictField()
+
     featured = db.BooleanField(required=True, default=False)
 
     contact_points = db.ListField(db.ReferenceField("ContactPoint", reverse_delete_rule=db.PULL))
@@ -672,6 +674,8 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         if len(set(res.id for res in self.resources)) != len(self.resources):
             raise MongoEngineValidationError(f"Duplicate resource ID in dataset #{self.id}.")
 
+        self.quality_cached = self.compute_quality()
+
         for key, value in self.extras.items():
             if not key.startswith("custom:"):
                 continue
@@ -763,13 +767,9 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
 
     @property
     def last_modified(self):
-        if (
-            self.harvest
-            and self.harvest.modified_at
-            and to_naive_datetime(self.harvest.modified_at) < datetime.utcnow()
-        ):
+        if self.harvest and self.harvest.modified_at:
             return to_naive_datetime(self.harvest.modified_at)
-        return self.last_modified_internal
+        return to_naive_datetime(self.last_modified_internal)
 
     @property
     def last_update(self):
@@ -824,8 +824,34 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         else:
             return self.last_update + delta
 
-    @cached_property
+    @property
     def quality(self):
+        # `quality_cached` should always be set, except during the migration
+        # creating this property. We could remove `or self.compute_quality()`
+        # after the migration but since we need to keep the computed property for
+        # `update_fulfilled_in_time`, maybe we leave it here? Just in case?
+        quality = self.quality_cached or self.compute_quality()
+
+        # :UpdateFulfilledInTime
+        # `next_update_for_update_fulfilled_in_time` is only useful to compute the
+        # real `update_fulfilled_in_time` check, so we pop it to not polute the `quality`
+        # object for users.
+        next_update = quality.pop("next_update_for_update_fulfilled_in_time", None)
+        if next_update:
+            # Allow for being one day late on update.
+            # We may have up to one day delay due to harvesting for example
+            quality["update_fulfilled_in_time"] = (next_update - datetime.utcnow()).days >= -1
+        elif self.frequency in ["continuous", "irregular", "punctual"]:
+            # For these frequencies, we don't expect regular updates or can't quantify them.
+            # Thus we consider the update_fulfilled_in_time quality criterion to be true.
+            quality["update_fulfilled_in_time"] = True
+
+        # Since `update_fulfilled_in_time` cannot be precomputed, `score` cannot either.
+        quality["score"] = self.compute_quality_score(quality)
+
+        return quality
+
+    def compute_quality(self):
         """Return a dict filled with metrics related to the inner
 
         quality of the dataset:
@@ -835,25 +861,18 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
             * and so on
         """
         result = {}
-        if not self.id:
-            # Quality is only relevant on saved Datasets
-            return result
 
         result["license"] = True if self.license else False
         result["temporal_coverage"] = True if self.temporal_coverage else False
         result["spatial"] = True if self.spatial else False
 
         result["update_frequency"] = self.frequency and self.frequency != "unknown"
-        if self.next_update:
-            # Allow for being one day late on update.
-            # We may have up to one day delay due to harvesting for example
-            result["update_fulfilled_in_time"] = (
-                True if (self.next_update - datetime.utcnow()).days >= -1 else False
-            )
-        elif self.frequency in ["continuous", "irregular", "punctual"]:
-            # For these frequencies, we don't expect regular updates or can't quantify them.
-            # Thus we consider the update_fulfilled_in_time quality criterion to be true.
-            result["update_fulfilled_in_time"] = True
+
+        # We only save the next_update here because it is based on resources
+        # We cannot save the `update_fulfilled_in_time` because it is time
+        # sensitive (so setting it on save is not really useful…)
+        # See :UpdateFulfilledInTime
+        result["next_update_for_update_fulfilled_in_time"] = self.next_update
 
         result["dataset_description_quality"] = (
             True
@@ -876,7 +895,6 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
                     resource_desc = True
             result["resources_documentation"] = resource_doc or resource_desc
 
-        result["score"] = self.compute_quality_score(result)
         return result
 
     @property
@@ -934,8 +952,16 @@ class Dataset(WithMetrics, DatasetBadgeMixin, Owned, db.Document):
         if resource.id in [r.id for r in self.resources]:
             raise MongoEngineValidationError("Cannot add resource with already existing ID")
 
+        self.resources.insert(0, resource)
         self.update(
-            __raw__={"$push": {"resources": {"$each": [resource.to_mongo()], "$position": 0}}}
+            __raw__={
+                "$set": {
+                    "quality_cached": self.compute_quality(),
+                },
+                "$push": {
+                    "resources": {"$each": [resource.to_mongo()], "$position": 0},
+                },
+            }
         )
         self.reload()
         self.on_resource_added.send(self.__class__, document=self, resource_id=resource.id)

udata/core/dataset/rdf.py

@@ -5,7 +5,7 @@ This module centralize dataset helpers for RDF/DCAT serialization and parsing
 import calendar
 import json
 import logging
-from datetime import date
+from datetime import date, datetime
 from typing import Optional
 
 from dateutil.parser import parse as parse_dt
@@ -50,7 +50,7 @@ from udata.rdf import (
     url_from_rdf,
 )
 from udata.uris import endpoint_for
-from udata.utils import get_by, safe_unicode
+from udata.utils import get_by, safe_unicode, to_naive_datetime
 
 from .constants import OGC_SERVICE_FORMATS, UPDATE_FREQUENCIES
 from .models import Checksum, Dataset, License, Resource
@@ -735,7 +735,14 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
     if not resource.harvest:
         resource.harvest = HarvestResourceMetadata()
     resource.harvest.created_at = created_at
-    resource.harvest.modified_at = modified_at
+
+    # In the past, we've encountered future `modified_at` during harvesting
+    # do not save it. :FutureHarvestModifiedAt
+    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
+        log.warning(f"Future `DCT.modified` date '{modified_at}' in resource")
+    else:
+        resource.harvest.modified_at = modified_at
+
     resource.harvest.dct_identifier = identifier
     resource.harvest.uri = uri
 
@@ -836,7 +843,13 @@ def dataset_from_rdf(graph: Graph, dataset=None, node=None, remote_url_prefix: s
     dataset.harvest.uri = uri
     dataset.harvest.remote_url = remote_url
     dataset.harvest.created_at = created_at
-    dataset.harvest.modified_at = modified_at
+
+    # In the past, we've encountered future `modified_at` during harvesting
+    # do not save it. :FutureHarvestModifiedAt
+    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
+        log.warning(f"Future `DCT.modified` date '{modified_at}' in dataset")
+    else:
+        dataset.harvest.modified_at = modified_at
 
     return dataset
 

udata/core/organization/apiv2.py

@@ -1,4 +1,3 @@
-import mongoengine
 from flask import request
 
 from udata import search
@@ -65,10 +64,7 @@ class OrganizationExtrasAPI(API):
 
         # then update the extras with the remaining payload
         org.extras.update(data)
-        try:
-            org.save()
-        except mongoengine.errors.ValidationError as e:
-            apiv2.abort(400, e.message)
+        org.save()
         return org.extras
 
     @apiv2.secure

udata/core/reports/api.py

@@ -1,4 +1,3 @@
-import mongoengine
 from flask import request
 from flask_login import current_user
 
@@ -31,11 +30,7 @@ class ReportsAPI(API):
         if current_user.is_authenticated:
             report.by = current_user._get_current_object()
 
-        try:
-            report.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
-
+        report.save()
         return report, 201
 
 

udata/core/reuse/api.py

@@ -125,10 +125,7 @@ class ReuseListAPI(API):
         if not reuse.owner and not reuse.organization:
             reuse.owner = current_user._get_current_object()
 
-        try:
-            reuse.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+        reuse.save()
 
         return patch_and_save(reuse, request), 201
 

udata/core/spatial/tests/test_api.py

@@ -1,14 +1,17 @@
 from flask import url_for
 
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.spatial.factories import (
     GeoLevelFactory,
     GeoZoneFactory,
     SpatialCoverageFactory,
 )
+from udata.core.spatial.models import spatial_granularities
 from udata.core.spatial.tasks import compute_geozones_metrics
 from udata.tests.api import APITestCase
+from udata.tests.api.test_datasets_api import SAMPLE_GEOM
 from udata.tests.features.territories import (
     TerritoriesSettings,
     create_geozones_fixtures,
@@ -286,3 +289,56 @@ class SpatialTerritoriesApiTest(APITestCase):
         self.assert200(response)
         # No dynamic datasets given that they are added by udata-front extension.
         self.assertEqual(len(response.json), 2)
+
+
+class DatasetsSpatialAPITest(APITestCase):
+    modules = []
+
+    def test_create_spatial_zones(self):
+        paca, _, _ = create_geozones_fixtures()
+        granularity = spatial_granularities[0][0]
+        data = DatasetFactory.as_dict()
+        data["spatial"] = {
+            "zones": [paca.id],
+            "granularity": granularity,
+        }
+        self.login()
+        response = self.post(url_for("api.datasets"), data)
+        self.assert201(response)
+        self.assertEqual(Dataset.objects.count(), 1)
+        dataset = Dataset.objects.first()
+        self.assertEqual([str(z) for z in dataset.spatial.zones], [paca.id])
+        self.assertEqual(dataset.spatial.geom, None)
+        self.assertEqual(dataset.spatial.granularity, granularity)
+
+    def test_create_spatial_geom(self):
+        granularity = spatial_granularities[0][0]
+        data = DatasetFactory.as_dict()
+        data["spatial"] = {
+            "geom": SAMPLE_GEOM,
+            "granularity": granularity,
+        }
+        self.login()
+        response = self.post(url_for("api.datasets"), data)
+        self.assert201(response)
+        self.assertEqual(Dataset.objects.count(), 1)
+        dataset = Dataset.objects.first()
+        self.assertEqual(dataset.spatial.zones, [])
+        self.assertEqual(dataset.spatial.geom, SAMPLE_GEOM)
+        self.assertEqual(dataset.spatial.granularity, granularity)
+
+    def test_cannot_create_both_geom_and_zones(self):
+        paca, _, _ = create_geozones_fixtures()
+
+        granularity = spatial_granularities[0][0]
+        data = DatasetFactory.as_dict()
+        data["spatial"] = {
+            "zones": [paca.id],
+            "geom": SAMPLE_GEOM,
+            "granularity": granularity,
+        }
+        self.login()
+
+        response = self.post(url_for("api.datasets"), data)
+        self.assert400(response)
+        self.assertEqual(Dataset.objects.count(), 0)

udata/migrations/2025-03-20-save-quality-for-datasets.py

@@ -0,0 +1,25 @@
+"""
+This migration keeps only the "Local authority" badge if the organization also has the "Public service" badge.
+"""
+
+import logging
+
+import click
+
+from udata.core.dataset.models import Dataset
+
+log = logging.getLogger(__name__)
+
+
+def migrate(db):
+    log.info("Saving all datasets")
+
+    count = Dataset.objects().count()
+    with click.progressbar(Dataset.objects(), length=count) as datasets:
+        for dataset in datasets:
+            try:
+                dataset.save()
+            except Exception as err:
+                log.error(f"Cannot save dataset {dataset.id} {err}")
+
+    log.info("Done")

udata/mongo/errors.py

@@ -3,7 +3,14 @@ from mongoengine.errors import ValidationError
 
 class FieldValidationError(ValidationError):
     field: str
+    raw_message: str
 
-    def __init__(self, *args, field: str, **kwargs):
-        self.field = field
+    def __init__(self, message: str, *args, field: str, **kwargs):
         super().__init__(*args, **kwargs)
+
+        self.raw_message = message  # It's sad but ValidationError do some stuff with the message…
+        self.field = field
+        self.errors[self.field] = message
+
+    def __str__(self):
+        return str(self.raw_message)