udata 10.2.1.dev34761__py2.py3-none-any.whl → 10.2.1.dev34775__py2.py3-none-any.whl

udata/api/__init__.py

@@ -4,6 +4,7 @@ import urllib.parse
 from functools import wraps
 from importlib import import_module
 
+import mongoengine
 from flask import (
     Blueprint,
     current_app,
@@ -22,7 +23,6 @@ from udata import entrypoints, tracking
 from udata.app import csrf
 from udata.auth import Permission, PermissionDenied, RoleNeed, current_user, login_user
 from udata.i18n import get_locale
-from udata.mongo.errors import FieldValidationError
 from udata.utils import safe_unicode
 
 from . import fields
@@ -258,17 +258,50 @@ def handle_unauthorized_file_type(error):
     return {"message": msg}, 400
 
 
-validation_error_fields = api.model("ValidationError", {"errors": fields.Raw})
+validation_error_fields = api.model(
+    "ValidationError",
+    {"errors": fields.Raw, "message": fields.String},
+)
 
+validation_error_fields_v2 = apiv2.inherit("ValidationError", validation_error_fields)
 
-@api.errorhandler(FieldValidationError)
-@api.marshal_with(validation_error_fields, code=400)
-def handle_validation_error(error: FieldValidationError):
-    """A validation error"""
+
+def convert_object_of_exceptions_to_object_of_strings(exceptions: dict):
     errors = {}
-    errors[error.field] = [error.message]
+    for key, exception in exceptions.items():
+        if isinstance(exception, Exception):
+            errors[key] = str(exception)
+        elif isinstance(exception, dict):
+            errors[key] = convert_object_of_exceptions_to_object_of_strings(exception)
+        elif isinstance(exception, str):
+            errors[key] = exception
+        else:
+            log.warning(
+                f"Unknown type in `convert_object_of_exceptions_to_object_of_strings`: {exception}"
+            )
+            errors[key] = str(exception)
+
+    return errors
+
+
+@api.errorhandler(mongoengine.errors.ValidationError)
+@api.marshal_with(validation_error_fields, code=400)
+def handle_validation_error(error: mongoengine.errors.ValidationError):
+    """Error returned when validation failed."""
+    return (
+        {
+            "errors": convert_object_of_exceptions_to_object_of_strings(error.errors),
+            "message": str(error),
+        },
+        400,
+    )
+
 
-    return {"errors": errors}, 400
+@apiv2.errorhandler(mongoengine.errors.ValidationError)
+@apiv2.marshal_with(validation_error_fields_v2, code=400)
+def handle_validation_error_v2(error: mongoengine.errors.ValidationError):
+    """Error returned when validation failed."""
+    return handle_validation_error(error)
 
 
 class API(Resource):  # Avoid name collision as resource is a core model

udata/api_fields.py

@@ -567,11 +567,7 @@ def is_value_modified(old_value, new_value) -> bool:
 
 def patch_and_save(obj, request) -> type:
     obj = patch(obj, request)
-
-    try:
-        obj.save()
-    except mongoengine.errors.ValidationError as e:
-        api.abort(400, e.message)
+    obj.save()
 
     return obj
 

udata/core/contact_point/api.py

@@ -1,5 +1,3 @@
-import mongoengine
-
 from udata.api import API, api
 from udata.api.parsers import ModelApiParser
 from udata.core.dataset.permissions import OwnablePermission
@@ -33,10 +31,8 @@ class ContactPointsListAPI(API):
     def post(self):
         """Creates a contact point"""
         form = api.validate(ContactPointForm)
-        try:
-            contact_point = form.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+        contact_point = form.save()
+
         return contact_point, 201
 
 

udata/core/dataservices/api.py

@@ -45,11 +45,7 @@ class DataservicesAPI(API):
         if not dataservice.owner and not dataservice.organization:
             dataservice.owner = current_user._get_current_object()
 
-        try:
-            dataservice.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
-
+        dataservice.save()
         return dataservice, 201
 
 
@@ -78,11 +74,8 @@ class DataserviceAPI(API):
         patch(dataservice, request)
         dataservice.metadata_modified_at = datetime.utcnow()
 
-        try:
-            dataservice.save()
-            return dataservice
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+        dataservice.save()
+        return dataservice
 
     @api.secure
     @api.doc("delete_dataservice")

udata/core/dataset/api.py

@@ -264,12 +264,8 @@ class DatasetAPI(API):
         DatasetEditPermission(dataset).test()
         dataset.last_modified_internal = datetime.utcnow()
         form = api.validate(DatasetForm, dataset)
-        # As validation for some fields (ie. extras) is at model
-        # level instead form level, we use mongoengine errors here.
-        try:
-            return form.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+
+        return form.save()
 
     @api.secure
     @api.doc("delete_dataset")

udata/core/dataset/apiv2.py

@@ -340,10 +340,7 @@ class DatasetExtrasAPI(API):
             data.pop(key)
         # then update the extras with the remaining payload
         dataset.extras.update(data)
-        try:
-            dataset.save(signal_kwargs={"ignores": ["post_save"]})
-        except mongoengine.errors.ValidationError as e:
-            apiv2.abort(400, e.message)
+        dataset.save(signal_kwargs={"ignores": ["post_save"]})
         return dataset.extras
 
     @apiv2.secure

udata/core/dataset/forms.py

@@ -54,7 +54,7 @@ class SchemaForm(ModelForm):
             )
         except FieldValidationError as err:
             field = getattr(self, err.field)
-            field.errors.append(err.message)
+            field.errors.append(str(err))
             return False
 
         return validation

udata/core/organization/apiv2.py

@@ -1,4 +1,3 @@
-import mongoengine
 from flask import request
 
 from udata import search
@@ -65,10 +64,7 @@ class OrganizationExtrasAPI(API):
 
         # then update the extras with the remaining payload
         org.extras.update(data)
-        try:
-            org.save()
-        except mongoengine.errors.ValidationError as e:
-            apiv2.abort(400, e.message)
+        org.save()
         return org.extras
 
     @apiv2.secure

udata/core/reports/api.py

@@ -1,4 +1,3 @@
-import mongoengine
 from flask import request
 from flask_login import current_user
 
@@ -31,11 +30,7 @@ class ReportsAPI(API):
         if current_user.is_authenticated:
             report.by = current_user._get_current_object()
 
-        try:
-            report.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
-
+        report.save()
         return report, 201
 
 

udata/core/reuse/api.py

@@ -125,10 +125,7 @@ class ReuseListAPI(API):
         if not reuse.owner and not reuse.organization:
             reuse.owner = current_user._get_current_object()
 
-        try:
-            reuse.save()
-        except mongoengine.errors.ValidationError as e:
-            api.abort(400, e.message)
+        reuse.save()
 
         return patch_and_save(reuse, request), 201
 

udata/core/spatial/tests/test_api.py

@@ -1,14 +1,17 @@
 from flask import url_for
 
 from udata.core.dataset.factories import DatasetFactory
+from udata.core.dataset.models import Dataset
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.spatial.factories import (
     GeoLevelFactory,
     GeoZoneFactory,
     SpatialCoverageFactory,
 )
+from udata.core.spatial.models import spatial_granularities
 from udata.core.spatial.tasks import compute_geozones_metrics
 from udata.tests.api import APITestCase
+from udata.tests.api.test_datasets_api import SAMPLE_GEOM
 from udata.tests.features.territories import (
     TerritoriesSettings,
     create_geozones_fixtures,
@@ -286,3 +289,56 @@ class SpatialTerritoriesApiTest(APITestCase):
         self.assert200(response)
         # No dynamic datasets given that they are added by udata-front extension.
         self.assertEqual(len(response.json), 2)
+
+
+class DatasetsSpatialAPITest(APITestCase):
+    modules = []
+
+    def test_create_spatial_zones(self):
+        paca, _, _ = create_geozones_fixtures()
+        granularity = spatial_granularities[0][0]
+        data = DatasetFactory.as_dict()
+        data["spatial"] = {
+            "zones": [paca.id],
+            "granularity": granularity,
+        }
+        self.login()
+        response = self.post(url_for("api.datasets"), data)
+        self.assert201(response)
+        self.assertEqual(Dataset.objects.count(), 1)
+        dataset = Dataset.objects.first()
+        self.assertEqual([str(z) for z in dataset.spatial.zones], [paca.id])
+        self.assertEqual(dataset.spatial.geom, None)
+        self.assertEqual(dataset.spatial.granularity, granularity)
+
+    def test_create_spatial_geom(self):
+        granularity = spatial_granularities[0][0]
+        data = DatasetFactory.as_dict()
+        data["spatial"] = {
+            "geom": SAMPLE_GEOM,
+            "granularity": granularity,
+        }
+        self.login()
+        response = self.post(url_for("api.datasets"), data)
+        self.assert201(response)
+        self.assertEqual(Dataset.objects.count(), 1)
+        dataset = Dataset.objects.first()
+        self.assertEqual(dataset.spatial.zones, [])
+        self.assertEqual(dataset.spatial.geom, SAMPLE_GEOM)
+        self.assertEqual(dataset.spatial.granularity, granularity)
+
+    def test_cannot_create_both_geom_and_zones(self):
+        paca, _, _ = create_geozones_fixtures()
+
+        granularity = spatial_granularities[0][0]
+        data = DatasetFactory.as_dict()
+        data["spatial"] = {
+            "zones": [paca.id],
+            "geom": SAMPLE_GEOM,
+            "granularity": granularity,
+        }
+        self.login()
+
+        response = self.post(url_for("api.datasets"), data)
+        self.assert400(response)
+        self.assertEqual(Dataset.objects.count(), 0)

udata/harvest/actions.py

@@ -34,25 +34,14 @@ def list_backends():
     return backends.get_all(current_app).values()
 
 
-def _sources_queryset(owner=None, deleted=False):
+def list_sources(owner=None, deleted=False):
+    """List all harvest sources"""
     sources = HarvestSource.objects
     if not deleted:
         sources = sources.visible()
     if owner:
         sources = sources.owned_by(owner)
-    return sources
-
-
-def list_sources(owner=None, deleted=False):
-    """List all harvest sources"""
-    return list(_sources_queryset(owner=owner, deleted=deleted))
-
-
-def paginate_sources(owner=None, page=1, page_size=DEFAULT_PAGE_SIZE, deleted=False):
-    """Paginate harvest sources"""
-    sources = _sources_queryset(owner=owner, deleted=deleted)
-    page = max(page or 1, 1)
-    return sources.paginate(page, page_size)
+    return list(sources)
 
 
 def get_source(ident):

udata/harvest/api.py

@@ -1,4 +1,3 @@
-from bson import ObjectId
 from flask import current_app, request
 from werkzeug.exceptions import BadRequest
 
@@ -253,6 +252,7 @@ source_parser.add_argument(
 source_parser.add_argument(
     "deleted", type=bool, location="args", default=False, help="Include sources flaggued as deleted"
 )
+source_parser.add_argument("q", type=str, location="args", help="The search query")
 
 
 @ns.route("/sources/", endpoint="harvest_sources")
@@ -264,15 +264,19 @@ class SourcesAPI(API):
         """List all harvest sources"""
         args = source_parser.parse_args()
 
-        if args.get("owner") and not ObjectId.is_valid(args.get("owner")):
-            api.abort(400, "`owner` arg must be an identifier")
+        sources = HarvestSource.objects()
 
-        return actions.paginate_sources(
-            args.get("owner"),
-            page=args["page"],
-            page_size=args["page_size"],
-            deleted=args["deleted"],
-        )
+        if not args["deleted"]:
+            sources = sources.visible()
+
+        if args["owner"]:
+            sources = sources.owned_by(args["owner"])
+
+        if args["q"]:
+            phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
+            sources = sources.search_text(phrase_query)
+
+        return sources.paginate(args["page"], args["page_size"])
 
     @api.secure
     @api.doc("create_harvest_source")

udata/harvest/models.py

@@ -155,6 +155,11 @@ class HarvestSource(Owned, db.Document):
 
     meta = {
         "indexes": [
+            {
+                "fields": ["$name", "$url"],
+                "default_language": "french",
+                "weights": {"name": 10, "url": 5},
+            },
             "-created_at",
             "slug",
             ("deleted", "-created_at"),

udata/harvest/tests/test_actions.py

@@ -14,7 +14,7 @@ from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import UserFactory
 from udata.models import Dataset, PeriodicTask
 from udata.tests.helpers import assert_emit, assert_equal_dates
-from udata.utils import Paginable, faker
+from udata.utils import faker
 
 from .. import actions, signals
 from ..backends import BaseBackend
@@ -113,59 +113,6 @@ class HarvestActionsTest:
         for source in sources:
             assert source in result
 
-    def test_paginate_sources(self):
-        result = actions.paginate_sources()
-        assert isinstance(result, Paginable)
-        assert result.page == 1
-        assert result.page_size == actions.DEFAULT_PAGE_SIZE
-        assert result.total == 0
-        assert len(result.objects) == 0
-
-        HarvestSourceFactory.create_batch(3)
-
-        result = actions.paginate_sources(page_size=2)
-        assert isinstance(result, Paginable)
-        assert result.page == 1
-        assert result.page_size == 2
-        assert result.total == 3
-        assert len(result.objects) == 2
-
-        result = actions.paginate_sources(page=2, page_size=2)
-        assert isinstance(result, Paginable)
-        assert result.page == 2
-        assert result.page_size == 2
-        assert result.total == 3
-        assert len(result.objects) == 1
-
-    def test_paginate_sources_exclude_deleted(self):
-        HarvestSourceFactory.create_batch(2)
-        HarvestSourceFactory(deleted=datetime.utcnow())
-
-        result = actions.paginate_sources(page_size=2)
-        assert isinstance(result, Paginable)
-        assert result.page == 1
-        assert result.page_size == 2
-        assert result.total == 2
-        assert len(result.objects) == 2
-
-    def test_paginate_sources_include_deleted(self):
-        HarvestSourceFactory.create_batch(2)
-        HarvestSourceFactory(deleted=datetime.utcnow())
-
-        result = actions.paginate_sources(page_size=2, deleted=True)
-        assert isinstance(result, Paginable)
-        assert result.page == 1
-        assert result.page_size == 2
-        assert result.total == 3
-        assert len(result.objects) == 2
-
-        result = actions.paginate_sources(page=2, page_size=2, deleted=True)
-        assert isinstance(result, Paginable)
-        assert result.page == 2
-        assert result.page_size == 2
-        assert result.total == 3
-        assert len(result.objects) == 1
-
     def test_create_source(self):
         source_url = faker.url()
 

udata/harvest/tests/test_api.py

@@ -8,7 +8,7 @@ from pytest_mock import MockerFixture
 from udata.core.organization.factories import OrganizationFactory
 from udata.core.user.factories import AdminFactory, UserFactory
 from udata.models import Member, PeriodicTask
-from udata.tests.helpers import assert200, assert201, assert204, assert400, assert403
+from udata.tests.helpers import assert200, assert201, assert204, assert400, assert403, assert404
 from udata.tests.plugin import ApiClient
 from udata.utils import faker
 
@@ -85,6 +85,38 @@ class HarvestAPITest(MockBackendsMixin):
 
         assert len(response.json["data"]) == len(sources)
 
+    def test_list_sources_search(self, api):
+        HarvestSourceFactory.create_batch(3)
+        source = HarvestSourceFactory(name="Moissonneur GeoNetwork de la ville de Rennes")
+
+        url = url_for("api.harvest_sources", q="geonetwork rennes")
+        response = api.get(url)
+        assert200(response)
+
+        assert len(response.json["data"]) == 1
+        assert response.json["data"][0]["id"] == str(source.id)
+
+    def test_list_sources_paginate(self, api):
+        total = 25
+        page_size = 20
+        HarvestSourceFactory.create_batch(total)
+
+        url = url_for("api.harvest_sources", page=1, page_size=page_size)
+        response = api.get(url)
+        assert200(response)
+        assert len(response.json["data"]) == page_size
+        assert response.json["total"] == total
+
+        url = url_for("api.harvest_sources", page=2, page_size=page_size)
+        response = api.get(url)
+        assert200(response)
+        assert len(response.json["data"]) == total - page_size
+        assert response.json["total"] == total
+
+        url = url_for("api.harvest_sources", page=3, page_size=page_size)
+        response = api.get(url)
+        assert404(response)
+
     def test_create_source_with_owner(self, api):
         """It should create and attach a new source to an owner"""
         user = api.login()

udata/mongo/errors.py

@@ -3,7 +3,14 @@ from mongoengine.errors import ValidationError
 
 class FieldValidationError(ValidationError):
     field: str
+    raw_message: str
 
-    def __init__(self, *args, field: str, **kwargs):
-        self.field = field
+    def __init__(self, message: str, *args, field: str, **kwargs):
         super().__init__(*args, **kwargs)
+
+        self.raw_message = message  # It's sad but ValidationError do some stuff with the message…
+        self.field = field
+        self.errors[self.field] = message
+
+    def __str__(self):
+        return str(self.raw_message)