udata 10.8.4.dev37423__py2.py3-none-any.whl → 10.8.4.dev37435__py2.py3-none-any.whl

udata/api/__init__.py

@@ -268,6 +268,7 @@ validation_error_fields_v2 = apiv2.inherit("ValidationError", validation_error_f
 
 def convert_object_of_exceptions_to_object_of_strings(exceptions: dict):
     errors = {}
+
     for key, exception in exceptions.items():
         if isinstance(exception, Exception):
             errors[key] = str(exception)
@@ -290,7 +291,9 @@ def handle_validation_error(error: mongoengine.errors.ValidationError):
     """Error returned when validation failed."""
     return (
         {
-            "errors": convert_object_of_exceptions_to_object_of_strings(error.errors),
+            "errors": convert_object_of_exceptions_to_object_of_strings(error.errors)
+            if error.errors
+            else {},
             "message": str(error),
         },
         400,
@@ -343,6 +346,7 @@ def init_app(app):
     import udata.core.reuse.apiv2  # noqa
     import udata.core.organization.api  # noqa
     import udata.core.organization.apiv2  # noqa
+    import udata.core.pages.api  # noqa
     import udata.core.followers.api  # noqa
     import udata.core.jobs.api  # noqa
     import udata.core.reports.api  # noqa

udata/api_fields.py

@@ -43,6 +43,8 @@ import flask_restx.fields as restx_fields
 import mongoengine
 import mongoengine.fields as mongo_fields
 from bson import DBRef, ObjectId
+from flask import Request
+from flask_restx import marshal
 from flask_restx.inputs import boolean
 from flask_restx.reqparse import RequestParser
 from flask_storage.mongo import ImageField as FlaskStorageImageField
@@ -60,6 +62,24 @@ lazy_reference = api.model(
     },
 )
 
+DEFAULT_GENERIC_KEY = "class"
+
+classes_by_names = {}
+classes_by_parents = {}
+
+
+class GenericField(restx_fields.Raw):
+    def __init__(self, fields_by_type):
+        super().__init__(self)
+        self.default = None
+        self.fields_by_type = fields_by_type
+
+    def format(self, value):
+        # Value is one of the generic object
+        data = marshal(value, self.fields_by_type[value.__class__.__name__])
+        data[DEFAULT_GENERIC_KEY] = value.__class__.__name__
+        return data
+
 
 def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | None]:
     """Map a Mongo field to a Flask RestX field.
@@ -96,12 +116,19 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         params["enum"] = field.choices
         if field.validation:
             params["validation"] = validation_to_type(field.validation)
+    elif isinstance(field, mongo_fields.UUIDField):
+        # TODO add validation?
+        constructor = restx_fields.String
     elif isinstance(field, mongo_fields.ObjectIdField):
         constructor = restx_fields.String
     elif isinstance(field, mongo_fields.FloatField):
         constructor = restx_fields.Float
         params["min"] = field.min  # TODO min_value?
         params["max"] = field.max
+    elif isinstance(field, mongo_fields.IntField):
+        constructor = restx_fields.Integer
+        params["min"] = field.min_value
+        params["max"] = field.max_value
     elif isinstance(field, mongo_fields.BooleanField):
         constructor = restx_fields.Boolean
     elif isinstance(field, mongo_fields.DateTimeField):
@@ -125,6 +152,7 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         # For lists, we can expose them only by showing a link to the API
         # with the results of the list to avoid listing a lot of sub-ressources
         # (for example for a dataservices with thousands of datasets).
+
         href = info.get("href", None)
         if href:
 
@@ -147,9 +175,36 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         #     2. `__additional_field_info__` of the inner field
         #     3. `__additional_field_info__` of the parent
         inner_info: dict = getattr(field.field, "__additional_field_info__", {})
-        field_read, field_write = convert_db_to_field(
-            f"{key}.inner", field.field, {**info, **inner_info, **info.get("inner_field_info", {})}
+        nested_info = {**info, **inner_info, **info.get("inner_field_info", {})}
+
+        generic = info.get("generic", False)
+
+        allowed_classes = (
+            classes_by_parents[field.field.document_type_obj]
+            if isinstance(field.field, mongoengine.fields.EmbeddedDocumentField)
+            and field.field.document_type_obj in classes_by_parents
+            else set()
         )
+        if generic and allowed_classes:
+            generic_fields = {
+                cls.__name__: convert_db_to_field(
+                    f"{key}.{cls.__name__}",
+                    # Instead of having EmbeddedDocumentField(Bloc) we'll create fields for each
+                    # of the subclasses with EmbededdDocumentField(DatasetsListBloc), EmbeddedDocumentFied(DataservicesListBloc)…
+                    mongoengine.fields.EmbeddedDocumentField(cls),
+                    nested_info,
+                )
+                for cls in allowed_classes
+            }
+
+            field_read = GenericField({k: v[0].model for k, v in generic_fields.items()})
+            field_write = GenericField({k: v[1].model for k, v in generic_fields.items()})
+        else:
+            field_read, field_write = convert_db_to_field(
+                f"{key}.inner",
+                field.field,
+                nested_info,
+            )
 
         if constructor_read is None:
             # We don't want to set the `constructor_read` if it's already set
@@ -243,6 +298,19 @@ def get_fields(cls) -> Iterable[tuple[str, Callable, dict]]:
             )
 
 
+def save_class_by_parents(cls):
+    from udata.mongo.engine import db
+
+    for parent in cls.__bases__:
+        if parent == db.Document:
+            return
+
+        classes_by_parents[parent] = (
+            classes_by_parents[parent] if parent in classes_by_parents else set()
+        )
+        classes_by_parents[parent].add(cls)
+
+
 def generate_fields(**kwargs) -> Callable:
     """Mongoengine document decorator.
 
@@ -271,6 +339,9 @@ def generate_fields(**kwargs) -> Callable:
 
         read_fields["id"] = restx_fields.String(required=True, readonly=True)
 
+        classes_by_names[cls.__name__] = cls
+        save_class_by_parents(cls)
+
         for key, field, info in get_fields(cls):
             sortable_key: bool = info.get("sortable", False)
             if sortable_key:
@@ -513,10 +584,13 @@ def patch(obj, request) -> type:
     """
     from udata.mongo.engine import db
 
-    for key, value in request.json.items():
+    data = request.json if isinstance(request, Request) else request
+    for key, value in data.items():
         field = obj.__write_fields__.get(key)
         if field is not None and not field.readonly:
             model_attribute = getattr(obj.__class__, key)
+            info = getattr(model_attribute, "__additional_field_info__", {})
+
             if hasattr(model_attribute, "from_input"):
                 value = model_attribute.from_input(value)
             elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
@@ -546,18 +620,27 @@ def patch(obj, request) -> type:
                 model_attribute,
                 mongoengine.fields.EmbeddedDocumentField,
             ):
-                embedded_field = model_attribute.document_type()
-                value = embedded_field._from_son(value)
+                embedded_field = model_attribute.document_type().__class__
+                value = patch(embedded_field(), value)
             elif value and isinstance(
                 model_attribute,
                 mongoengine.fields.EmbeddedDocumentListField,
             ):
-                embedded_field = model_attribute.field.document_type()
-                # MongoEngine BaseDocument has a `from_json` method for string and a private `_from_son`
-                # but there is no public `from_son` to use
-                value = [embedded_field._from_son(embedded_value) for embedded_value in value]
-
-            info = getattr(model_attribute, "__additional_field_info__", {})
+                base_embedded_field = model_attribute.field.document_type().__class__
+                generic = info.get("generic", False)
+                generic_key = info.get("generic_key", DEFAULT_GENERIC_KEY)
+
+                objects = []
+                for embedded_value in value:
+                    # TODO add validation on generic_key presence and value
+                    embedded_field = (
+                        classes_by_names[embedded_value[generic_key]]
+                        if generic
+                        else base_embedded_field
+                    )
+                    objects.append(patch(embedded_field(), embedded_value))
+
+                value = objects
 
             # `checks` field attribute allows to do validation from the request before setting
             # the attribute

udata/core/dataservices/models.py

@@ -6,7 +6,6 @@ from mongoengine import Q
 from mongoengine.signals import post_save
 
 import udata.core.contact_point.api_fields as contact_api_fields
-import udata.core.dataset.api_fields as datasets_api_fields
 from udata.api import api, fields
 from udata.api_fields import field, function_field, generate_fields
 from udata.core.activity.models import Auditable
@@ -16,6 +15,7 @@ from udata.core.dataservices.constants import (
     DATASERVICE_ACCESS_TYPES,
     DATASERVICE_FORMATS,
 )
+from udata.core.dataset.api_fields import dataset_ref_fields
 from udata.core.dataset.models import Dataset
 from udata.core.linkable import Linkable
 from udata.core.metrics.helpers import get_stock_metrics
@@ -264,7 +264,7 @@ class Dataservice(Auditable, WithMetrics, Linkable, Owned, db.Document):
         db.ListField(
             field(
                 db.LazyReferenceField(Dataset, passthrough=True),
-                nested_fields=datasets_api_fields.dataset_ref_fields,
+                nested_fields=dataset_ref_fields,
             )
         ),
         filterable={

udata/core/pages/api.py

@@ -0,0 +1,54 @@
+from flask import request
+from flask_login import current_user
+
+from udata.api import API, api
+from udata.api_fields import patch
+from udata.auth import admin_permission
+
+from .models import Page
+
+ns = api.namespace("pages", "Pages related operations (beta)")
+
+common_doc = {"params": {"page": "The page ID or slug"}}
+
+
+@ns.route("/", endpoint="pages")
+class PagesAPI(API):
+    @api.secure(admin_permission)
+    @api.doc("list_pages")
+    @api.expect(Page.__index_parser__)
+    @api.marshal_with(Page.__page_fields__)
+    def get(self):
+        """List or search all pages"""
+        return Page.apply_pagination(Page.objects)
+
+    @api.secure
+    @api.doc("create_page", responses={400: "Validation error"})
+    @api.expect(Page.__write_fields__)
+    @api.marshal_with(Page.__read_fields__, code=201)
+    def post(self):
+        page = patch(Page(), request)
+        if not page.owner and not page.organization:
+            page.owner = current_user._get_current_object()
+
+        page.save()
+        return page, 201
+
+
+@ns.route("/<page:page>/", endpoint="page")
+class PageAPI(API):
+    @api.doc("get_page")
+    @api.marshal_with(Page.__read_fields__)
+    def get(self, page: Page):
+        return page
+
+    @api.secure
+    @api.doc("update_page")
+    @api.expect(Page.__write_fields__)
+    @api.marshal_with(Page.__read_fields__)
+    def put(self, page: Page):
+        page.permissions["edit"].test()
+        patch(page, request)
+
+        page.save()
+        return page

udata/core/pages/factories.py

@@ -0,0 +1,10 @@
+from udata.factories import ModelFactory
+
+from .models import Page
+
+
+class PageFactory(ModelFactory):
+    class Meta:
+        model = Page
+
+    blocs = []

udata/core/pages/models.py

@@ -0,0 +1,103 @@
+from udata.api import api, fields
+from udata.api_fields import field, function_field, generate_fields
+from udata.core.activity.models import Auditable
+from udata.core.dataservices.models import Dataservice
+from udata.core.dataset.api_fields import dataset_fields
+from udata.core.owned import Owned
+from udata.core.reuse.models import Reuse
+from udata.models import db
+from udata.mongo.datetime_fields import Datetimed
+
+page_permissions_fields = api.model(
+    "PagePermissions",
+    {
+        "delete": fields.Permission(),
+        "edit": fields.Permission(),
+    },
+)
+
+
+@generate_fields()
+class Bloc(db.EmbeddedDocument):
+    meta = {"allow_inheritance": True}
+
+    id = field(db.AutoUUIDField(primary_key=True))
+
+
+class BlocWithTitleMixin:
+    title = field(db.StringField(required=True))
+    subtitle = field(db.StringField())
+
+
+@generate_fields(
+    mask="*,datasets{id,title,uri,page,private,archived,organization,owner,last_update,quality,metrics,description}"
+)
+class DatasetsListBloc(BlocWithTitleMixin, Bloc):
+    datasets = field(
+        db.ListField(
+            field(
+                db.ReferenceField("Dataset"),
+                nested_fields=dataset_fields,
+            )
+        )
+    )
+
+
+@generate_fields()
+class ReusesListBloc(BlocWithTitleMixin, Bloc):
+    reuses = field(
+        db.ListField(
+            field(
+                db.ReferenceField("Reuse"),
+                nested_fields=Reuse.__read_fields__,
+            )
+        )
+    )
+
+
+@generate_fields()
+class DataservicesListBloc(BlocWithTitleMixin, Bloc):
+    dataservices = field(
+        db.ListField(
+            field(
+                db.ReferenceField("Dataservice"),
+                nested_fields=Dataservice.__read_fields__,
+            )
+        )
+    )
+
+
+@generate_fields()
+class LinkInBloc(db.EmbeddedDocument):
+    title = field(db.StringField(required=True))
+    color = field(db.StringField())
+    url = field(db.StringField())
+
+
+@generate_fields()
+class LinksListBloc(BlocWithTitleMixin, Bloc):
+    paragraph = field(db.StringField())
+    main_link_url = field(db.StringField())
+    main_link_title = field(db.StringField())
+
+    links = field(db.EmbeddedDocumentListField(LinkInBloc))
+
+
+@generate_fields()
+class Page(Auditable, Owned, Datetimed, db.Document):
+    blocs = field(
+        db.EmbeddedDocumentListField(Bloc),
+        generic=True,
+    )
+
+    @property
+    @function_field(
+        nested_fields=page_permissions_fields,
+    )
+    def permissions(self):
+        from .permissions import PageEditPermission
+
+        return {
+            "delete": PageEditPermission(self),
+            "edit": PageEditPermission(self),
+        }

udata/core/pages/permissions.py

@@ -0,0 +1,7 @@
+from udata.core.dataset.permissions import OwnablePermission
+
+
+class PageEditPermission(OwnablePermission):
+    """Permissions to edit a Page"""
+
+    pass

udata/core/pages/tests/test_api.py

@@ -0,0 +1,75 @@
+from flask import url_for
+
+from udata.core.dataset.factories import DatasetFactory
+from udata.core.pages.models import Page
+from udata.tests.api import APITestCase
+
+
+class PageAPITest(APITestCase):
+    modules = []
+
+    def test_create_get_update(self):
+        self.login()
+        datasets = DatasetFactory.create_batch(3)
+
+        response = self.post(
+            url_for("api.pages"),
+            {
+                "blocs": [
+                    {
+                        "class": "DatasetsListBloc",
+                        "title": "My awesome title",
+                        "datasets": [str(d.id) for d in datasets],
+                    }
+                ],
+            },
+        )
+        self.assert201(response)
+
+        self.assertEqual(len(response.json["blocs"][0]["datasets"]), 3)
+        self.assertEqual("DatasetsListBloc", response.json["blocs"][0]["class"])
+        self.assertEqual("My awesome title", response.json["blocs"][0]["title"])
+        self.assertIsNone(response.json["blocs"][0]["subtitle"])
+        self.assertEqual(str(datasets[0].id), response.json["blocs"][0]["datasets"][0]["id"])
+        self.assertEqual(str(datasets[1].id), response.json["blocs"][0]["datasets"][1]["id"])
+        self.assertEqual(str(datasets[2].id), response.json["blocs"][0]["datasets"][2]["id"])
+
+        page = Page.objects().first()
+        self.assertEqual(str(page.id), response.json["id"])
+        self.assertEqual("My awesome title", page.blocs[0].title)
+        self.assertIsNone(page.blocs[0].subtitle)
+        self.assertEqual(datasets[0].id, page.blocs[0].datasets[0].id)
+        self.assertEqual(datasets[1].id, page.blocs[0].datasets[1].id)
+        self.assertEqual(datasets[2].id, page.blocs[0].datasets[2].id)
+
+        response = self.get(url_for("api.page", page=page))
+        self.assert200(response)
+
+        self.assertEqual(len(response.json["blocs"][0]["datasets"]), 3)
+        self.assertEqual("DatasetsListBloc", response.json["blocs"][0]["class"])
+        self.assertEqual("My awesome title", response.json["blocs"][0]["title"])
+        self.assertIsNone(response.json["blocs"][0]["subtitle"])
+        self.assertEqual(str(datasets[0].id), response.json["blocs"][0]["datasets"][0]["id"])
+        self.assertEqual(str(datasets[1].id), response.json["blocs"][0]["datasets"][1]["id"])
+        self.assertEqual(str(datasets[2].id), response.json["blocs"][0]["datasets"][2]["id"])
+
+        response = self.put(
+            url_for("api.page", page=page),
+            {
+                "blocs": [
+                    {
+                        "class": "DatasetsListBloc",
+                        "title": "My awesome title",
+                        "subtitle": "more information",
+                        "datasets": [{"id": str(datasets[2].id)}],
+                    }
+                ],
+            },
+        )
+        self.assert200(response)
+
+        self.assertEqual("DatasetsListBloc", response.json["blocs"][0]["class"])
+        self.assertEqual("My awesome title", response.json["blocs"][0]["title"])
+        self.assertEqual("more information", response.json["blocs"][0]["subtitle"])
+        self.assertEqual(len(response.json["blocs"][0]["datasets"]), 1)
+        self.assertEqual(str(datasets[2].id), response.json["blocs"][0]["datasets"][0]["id"])

udata/core/site/api.py

@@ -1,13 +1,12 @@
-from bson import ObjectId
 from flask import current_app, json, make_response, redirect, request, url_for
 
-from udata.api import API, api, fields
+from udata.api import API, api
+from udata.api_fields import patch
 from udata.auth import admin_permission
 from udata.core import csv
 from udata.core.dataservices.csv import DataserviceCsvAdapter
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.api import DatasetApiParser
-from udata.core.dataset.api_fields import dataset_fields
 from udata.core.dataset.csv import ResourcesCsvAdapter
 from udata.core.dataset.search import DatasetSearch
 from udata.core.dataset.tasks import get_queryset as get_csv_queryset
@@ -22,74 +21,28 @@ from udata.models import Dataset, Reuse
 from udata.rdf import CONTEXT, RDF_EXTENSIONS, graph_response, negociate_content
 from udata.utils import multi_to_dict
 
-from .models import current_site
+from .models import Site, current_site
 from .rdf import build_catalog
 
-site_fields = api.model(
-    "Site",
-    {
-        "id": fields.String(description="The Site unique identifier", required=True),
-        "title": fields.String(description="The site display title", required=True),
-        "metrics": fields.Raw(
-            attribute=lambda o: o.get_metrics(), description="The associated metrics", default={}
-        ),
-    },
-)
-
 
 @api.route("/site/", endpoint="site")
 class SiteAPI(API):
     @api.doc(id="get_site")
-    @api.marshal_with(site_fields)
+    @api.marshal_with(Site.__read_fields__)
     def get(self):
         """Site-wide variables"""
         return current_site
 
-
-@api.route("/site/home/datasets/", endpoint="home_datasets")
-class SiteHomeDatasetsAPI(API):
-    @api.doc("get_home_datasets")
-    # @api.secure(admin_permission)
-    @api.marshal_list_with(dataset_fields)
-    def get(self):
-        """List homepage datasets"""
-        return current_site.settings.home_datasets
-
     @api.secure(admin_permission)
-    @api.doc("set_home_datasets")
-    @api.expect(([str], "Dataset IDs to put in homepage"))
-    @api.marshal_list_with(dataset_fields)
-    def put(self):
-        """Set the homepage datasets editorial selection"""
-        if not isinstance(request.json, list):
-            api.abort(400, "Expect a list of dataset IDs")
-        ids = [ObjectId(id) for id in request.json]
-        current_site.settings.home_datasets = Dataset.objects.bulk_list(ids)
-        current_site.save()
-        return current_site.settings.home_datasets
-
+    @api.doc(id="set_site")
+    @api.expect(Site.__write_fields__)
+    @api.marshal_with(Site.__read_fields__)
+    def patch(self):
+        patch(current_site, request)
 
-@api.route("/site/home/reuses/", endpoint="home_reuses")
-class SiteHomeReusesAPI(API):
-    @api.doc("get_home_reuses")
-    @api.secure(admin_permission)
-    @api.marshal_list_with(Reuse.__read_fields__)
-    def get(self):
-        """List homepage featured reuses"""
-        return current_site.settings.home_reuses
-
-    @api.secure(admin_permission)
-    @api.doc("set_home_reuses")
-    @api.expect(([str], "Reuse IDs to put in homepage"))
-    @api.marshal_list_with(Reuse.__read_fields__)
-    def put(self):
-        """Set the homepage reuses editorial selection"""
-        if not isinstance(request.json, list):
-            api.abort(400, "Expect a list of reuse IDs")
-        ids = [ObjectId(id) for id in request.json]
-        current_site.settings.home_reuses = Reuse.objects.bulk_list(ids)
         current_site.save()
-        return current_site.settings.home_reuses
+        current_site.reload()
+        return current_site
 
 
 @api.route("/site/data.<format>", endpoint="site_dataportal")

udata/core/site/models.py

@@ -1,6 +1,7 @@
 from flask import current_app, g
 from werkzeug.local import LocalProxy
 
+from udata.api_fields import field, generate_fields
 from udata.core.dataservices.models import Dataservice
 from udata.core.dataset.models import Dataset
 from udata.core.metrics.helpers import get_metrics_for_model, get_stock_metrics
@@ -19,14 +20,18 @@ class SiteSettings(db.EmbeddedDocument):
     home_reuses = db.ListField(db.ReferenceField(Reuse))
 
 
+@generate_fields()
 class Site(WithMetrics, db.Document):
-    id = db.StringField(primary_key=True)
-    title = db.StringField(required=True)
-    keywords = db.ListField(db.StringField())
-    feed_size = db.IntField(required=True, default=DEFAULT_FEED_SIZE)
+    id = field(db.StringField(primary_key=True), readonly=True)
+    title = field(db.StringField(required=True), description="The site display title")
+    keywords = field(db.ListField(db.StringField()))
+    feed_size = field(db.IntField(required=True, default=DEFAULT_FEED_SIZE))
     configs = db.DictField()
     themes = db.DictField()
     settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings)
+    datasets_page = field(db.ReferenceField("Page"), attribute="datasets_page.id")
+    reuses_page = field(db.ReferenceField("Page"), attribute="reuses_page.id")
+    dataservices_page = field(db.ReferenceField("Page"), attribute="dataservices_page.id")
 
     __metrics_keys__ = [
         "max_dataset_followers",

udata/models/__init__.py

@@ -23,6 +23,7 @@ from udata.core.tags.models import *  # noqa
 from udata.core.spam.models import *  # noqa
 from udata.core.reports.models import *  # noqa
 from udata.core.dataservices.models import *  # noqa
+from udata.core.pages.models import *  # noqa
 
 from udata.features.transfer.models import *  # noqa
 from udata.features.territories.models import *  # noqa

udata/routing.py

@@ -126,6 +126,10 @@ class DatasetConverter(ModelConverter):
     model = models.Dataset
 
 
+class PageConverter(ModelConverter):
+    model = models.Page
+
+
 class DatasetWithoutResourcesConverter(ModelConverter):
     model = models.Dataset
 
@@ -250,6 +254,7 @@ def init_app(app):
     app.url_map.converters["reuse"] = ReuseConverter
     app.url_map.converters["user"] = UserConverter
     app.url_map.converters["topic"] = TopicConverter
+    app.url_map.converters["page"] = PageConverter
     app.url_map.converters["post"] = PostConverter
     app.url_map.converters["territory"] = TerritoryConverter
     app.url_map.converters["contact_point"] = ContactPointConverter