udata 14.5.1.dev9__py3-none-any.whl → 14.7.3.dev4__py3-none-any.whl

udata/api_fields.py

@@ -40,6 +40,32 @@ from udata.api import api, base_reference
 from udata.mongo.errors import FieldValidationError
 from udata.mongo.queryset import DBPaginator, UDataQuerySet
 
+
+def required_if(**conditions):
+    """Check helper that makes a field required when other fields have specific values.
+
+    Usage:
+        page_id = field(
+            db.ReferenceField("Page"),
+            checks=[required_if(body_type="blocs")],
+        )
+    """
+
+    def check(value, data, field, obj, **_kwargs):
+        if value is not None:
+            return
+        for condition_field, condition_value in conditions.items():
+            actual_condition = data.get(condition_field, getattr(obj, condition_field, None))
+            if actual_condition == condition_value:
+                raise FieldValidationError(
+                    f"'{field}' is required when '{condition_field}' is '{condition_value}'",
+                    field=field,
+                )
+
+    check.run_even_if_missing = True
+    return check
+
+
 lazy_reference = api.model(
     "LazyReference",
     {
@@ -79,13 +105,15 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
     user-supplied overrides, setting the readonly flag…), it's easier to have to do this only once at the end of the function.
 
     """
+    from udata.mongo.engine import db
+
     params: dict = {}
     params["required"] = field.required
 
     read_params: dict = {}
     write_params: dict = {}
 
-    constructor: Callable
+    constructor: Callable | None = None
     constructor_read: Callable | None = None
     constructor_write: Callable | None = None
 
@@ -204,13 +232,34 @@ def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | N
         def constructor_write(**kwargs):
             return restx_fields.List(field_write, **kwargs)
 
-    elif isinstance(
-        field, (mongo_fields.GenericReferenceField, mongoengine.fields.GenericLazyReferenceField)
-    ):
+    elif isinstance(field, mongoengine.fields.GenericLazyReferenceField):
 
         def constructor(**kwargs):
             return restx_fields.Nested(lazy_reference, **kwargs)
 
+    elif isinstance(field, mongo_fields.GenericReferenceField):
+        if field.choices:
+            generic_fields = {}
+            for cls in field.choices:
+                cls = db.resolve_model(cls) if isinstance(cls, str) else cls
+                generic_fields[cls.__name__] = convert_db_to_field(
+                    f"{key}.{cls.__name__}",
+                    # Instead of having GenericReferenceField() we'll create fields for each
+                    # of the subclasses with ReferenceField(Organization)…
+                    mongoengine.fields.ReferenceField(cls),
+                    info,
+                )
+
+            def constructor_read(**kwargs):
+                return GenericField({k: v[0].model for k, v in generic_fields.items()}, **kwargs)
+
+            def constructor_write(**kwargs):
+                return GenericField({k: v[1].model for k, v in generic_fields.items()}, **kwargs)
+        else:
+
+            def constructor(**kwargs):
+                return restx_fields.Nested(lazy_reference, **kwargs)
+
     elif isinstance(field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField):
         # For reference we accept while writing a String representing the ID of the referenced model.
         # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
@@ -658,6 +707,19 @@ def field(
         return inner
 
 
+def run_check(check, value, key, obj, data):
+    check(
+        value,
+        **{
+            "is_creation": obj._created,
+            "is_update": not obj._created,
+            "field": key,
+            "obj": obj,
+            "data": data,
+        },
+    )
+
+
 def patch(obj, request) -> type:
     """Patch the object with the data from the request.
 
@@ -667,6 +729,7 @@ def patch(obj, request) -> type:
     from udata.mongo.engine import db
 
     data = request.json if isinstance(request, Request) else request
+
     for key, value in data.items():
         field = obj.__write_fields__.get(key)
         if field is not None and not field.readonly:
@@ -734,23 +797,30 @@ def patch(obj, request) -> type:
 
                 value = objects
 
-            # `checks` field attribute allows to do validation from the request before setting
-            # the attribute
+            # Run checks if value is modified.
+            # We run checks here (before setattr) to compare old vs new value.
             checks = info.get("checks", [])
-
             if is_value_modified(getattr(obj, key), value):
                 for check in checks:
-                    check(
-                        value,
-                        **{
-                            "is_creation": obj._created,
-                            "is_update": not obj._created,
-                            "field": key,
-                        },
-                    )  # TODO add other model attributes in function parameters
+                    run_check(check, value, key, obj, data)
 
             setattr(obj, key, value)
 
+    # Run checks marked with `run_even_if_missing` on fields not in request.
+    # Some checks (like `required_if`) need to run even when their field is absent
+    # from the request, because they validate cross-field constraints based on
+    # other fields in the request (e.g. "page_id is required if body_type is blocs").
+    for key, _, info in get_fields(obj.__class__):
+        if key in data:
+            continue
+        checks = info.get("checks", [])
+        value = getattr(obj, key, None)
+
+        for check in checks:
+            if not getattr(check, "run_even_if_missing", False):
+                continue
+            run_check(check, value, key, obj, data)
+
     return obj
 
 

udata/auth/forms.py

@@ -49,7 +49,7 @@ class ExtendedRegisterForm(WithCaptcha, RegisterFormV2):
     accept_conditions = fields.BooleanField(
         _("J'accepte les conditions générales d'utilisation"),
         validators=[
-            validators.DataRequired(message=_("Vous devez accepter les CGU pour continuer."))
+            validators.DataRequired(message=_("You must accept the terms of use to continue."))
         ],
     )
 

udata/core/badges/tests/test_tasks.py

@@ -1,5 +1,3 @@
-import udata.core.dataservices.tasks  # noqa
-import udata.core.dataset.tasks  # noqa
 from udata.core.badges.tasks import update_badges
 from udata.core.constants import HVD
 from udata.core.dataservices.factories import DataserviceFactory

udata/core/dataservices/apiv2.py

@@ -15,7 +15,7 @@ apiv2.inherit("AccessAudience (read)", AccessAudience.__read_fields__)
 
 ns = apiv2.namespace("dataservices", "Dataservice related operations")
 
-search_parser = DataserviceSearch.as_request_parser()
+search_parser = DataserviceSearch.as_request_parser(store_missing=False)
 
 
 @ns.route("/search/", endpoint="dataservice_search")

udata/core/dataset/models.py

@@ -15,17 +15,19 @@ from mongoengine.fields import DateTimeField
 from mongoengine.signals import post_save, pre_init, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.app import cache
 from udata.core import storages
 from udata.core.access_type.constants import AccessType
 from udata.core.access_type.models import WithAccessType, check_only_one_condition_per_role
 from udata.core.activity.models import Auditable
 from udata.core.constants import HVD
+from udata.core.dataset.api_fields import temporal_coverage_fields
 from udata.core.dataset.preview import TabularAPIPreview
 from udata.core.linkable import Linkable
 from udata.core.metrics.helpers import get_stock_metrics
 from udata.core.owned import Owned, OwnedQuerySet
+from udata.core.spatial.api_fields import spatial_coverage_fields
 from udata.frontend.markdown import mdstrip
 from udata.i18n import lazy_gettext as _
 from udata.models import Badge, BadgeMixin, BadgesList, SpatialCoverage, WithMetrics, db
@@ -89,6 +91,7 @@ def get_json_ld_extra(key, value):
     }
 
 
+@generate_fields()
 class HarvestDatasetMetadata(db.EmbeddedDocument):
     backend = db.StringField()
     created_at = db.DateTimeField()
@@ -114,6 +117,7 @@ class HarvestResourceMetadata(db.EmbeddedDocument):
     dct_identifier = db.StringField()
 
 
+@generate_fields()
 class Schema(db.EmbeddedDocument):
     """
     Schema can only be two things right now:
@@ -482,6 +486,7 @@ class ResourceMixin(object):
         return result
 
 
+@generate_fields()
 class Resource(ResourceMixin, WithMetrics, db.EmbeddedDocument):
     """
     Local file, remote file or API provided by the original provider of the
@@ -533,6 +538,7 @@ class DatasetBadgeMixin(BadgeMixin):
     __badges__ = BADGES
 
 
+@generate_fields()
 class Dataset(
     Auditable, WithMetrics, WithAccessType, DatasetBadgeMixin, Owned, Linkable, db.Document
 ):
@@ -560,8 +566,14 @@ class Dataset(
 
     frequency = field(db.EnumField(UpdateFrequency))
     frequency_date = field(db.DateTimeField(verbose_name=_("Future date of update")))
-    temporal_coverage = field(db.EmbeddedDocumentField(db.DateRange))
-    spatial = field(db.EmbeddedDocumentField(SpatialCoverage))
+    temporal_coverage = field(
+        db.EmbeddedDocumentField(db.DateRange),
+        nested_fields=temporal_coverage_fields,
+    )
+    spatial = field(
+        db.EmbeddedDocumentField(SpatialCoverage),
+        nested_fields=spatial_coverage_fields,
+    )
     schema = field(db.EmbeddedDocumentField(Schema))
 
     ext = field(db.MapField(db.GenericEmbeddedDocumentField()), auditable=False)

udata/core/dataset/rdf.py

@@ -5,7 +5,7 @@ This module centralize dataset helpers for RDF/DCAT serialization and parsing
 import calendar
 import json
 import logging
-from datetime import date, datetime
+from datetime import date
 
 from dateutil.parser import parse as parse_dt
 from flask import current_app
@@ -51,7 +51,7 @@ from udata.rdf import (
     themes_from_rdf,
     url_from_rdf,
 )
-from udata.utils import get_by, safe_unicode, to_naive_datetime
+from udata.utils import get_by, safe_harvest_datetime, safe_unicode
 
 from .constants import OGC_SERVICE_FORMATS, UpdateFrequency
 from .models import Checksum, Dataset, License, Resource
@@ -729,12 +729,10 @@ def resource_from_rdf(graph_or_distrib, dataset=None, is_additionnal=False):
         resource.harvest = HarvestResourceMetadata()
     resource.harvest.issued_at = issued_at
 
-    # In the past, we've encountered future `modified_at` during harvesting
-    # do not save it. :FutureHarvestModifiedAt
-    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
-        log.warning(f"Future `DCT.modified` date '{modified_at}' in resource")
-    else:
-        resource.harvest.modified_at = modified_at
+    # :FutureHarvestModifiedAt
+    resource.harvest.modified_at = safe_harvest_datetime(
+        modified_at, "DCT.modified (resource)", refuse_future=True
+    )
 
     resource.harvest.dct_identifier = identifier
     resource.harvest.uri = uri
@@ -845,12 +843,10 @@ def dataset_from_rdf(
     dataset.harvest.created_at = created_at
     dataset.harvest.issued_at = issued_at
 
-    # In the past, we've encountered future `modified_at` during harvesting
-    # do not save it. :FutureHarvestModifiedAt
-    if modified_at and to_naive_datetime(modified_at) > datetime.utcnow():
-        log.warning(f"Future `DCT.modified` date '{modified_at}' in dataset")
-    else:
-        dataset.harvest.modified_at = modified_at
+    # :FutureHarvestModifiedAt
+    dataset.harvest.modified_at = safe_harvest_datetime(
+        modified_at, "DCT.modified (dataset)", refuse_future=True
+    )
 
     return dataset
 

udata/core/organization/apiv2.py

@@ -15,7 +15,7 @@ apiv2.inherit("ContactPoint", contact_point_fields)
 
 
 ns = apiv2.namespace("organizations", "Organization related operations")
-search_parser = OrganizationSearch.as_request_parser()
+search_parser = OrganizationSearch.as_request_parser(store_missing=False)
 
 DEFAULT_SORTING = "-created_at"
 

udata/core/organization/models.py

@@ -7,7 +7,7 @@ from flask_babel import LazyString
 from mongoengine.signals import post_save, pre_save
 from werkzeug.utils import cached_property
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.core.activity.models import Auditable
 from udata.core.badges.models import Badge, BadgeMixin, BadgesList
 from udata.core.linkable import Linkable
@@ -21,6 +21,7 @@ from udata.uris import cdata_url
 
 from .constants import (
     ASSOCIATION,
+    BIGGEST_LOGO_SIZE,
     CERTIFIED,
     COMPANY,
     DEFAULT_ROLE,
@@ -44,6 +45,7 @@ BADGES: dict[str, LazyString] = {
 }
 
 
+@generate_fields()
 class Team(db.EmbeddedDocument):
     name = db.StringField(required=True)
     slug = db.SlugField(
@@ -54,6 +56,7 @@ class Team(db.EmbeddedDocument):
     members = db.ListField(db.ReferenceField("User"))
 
 
+@generate_fields()
 class Member(db.EmbeddedDocument):
     user = db.ReferenceField("User")
     role = db.StringField(choices=list(ORG_ROLES), default=DEFAULT_ROLE)
@@ -64,6 +67,7 @@ class Member(db.EmbeddedDocument):
         return ORG_ROLES[self.role]
 
 
+@generate_fields()
 class MembershipRequest(db.EmbeddedDocument):
     """
     Pending organization membership requests
@@ -113,18 +117,22 @@ class OrganizationBadge(Badge):
 
 
 class OrganizationBadgeMixin(BadgeMixin):
-    badges = field(BadgesList(OrganizationBadge), **BadgeMixin.default_badges_list_params)
+    badges = field(
+        BadgesList(OrganizationBadge), show_as_ref=True, **BadgeMixin.default_badges_list_params
+    )
     __badges__ = BADGES
 
 
+@generate_fields()
 class Organization(
     Auditable, WithMetrics, OrganizationBadgeMixin, Linkable, db.Datetimed, db.Document
 ):
-    name = field(db.StringField(required=True))
-    acronym = field(db.StringField(max_length=128))
+    name = field(db.StringField(required=True), show_as_ref=True)
+    acronym = field(db.StringField(max_length=128), show_as_ref=True)
     slug = field(
         db.SlugField(max_length=255, required=True, populate_from="name", update=True, follow=True),
         auditable=False,
+        show_as_ref=True,
     )
     description = field(
         db.StringField(required=True),
@@ -138,7 +146,11 @@ class Organization(
             basename=default_image_basename,
             max_size=LOGO_MAX_SIZE,
             thumbnails=LOGO_SIZES,
-        )
+        ),
+        show_as_ref=True,
+        thumbnail_info={
+            "size": BIGGEST_LOGO_SIZE,
+        },
     )
     business_number_id = field(db.StringField(max_length=ORG_BID_SIZE_LIMIT))
 
@@ -265,6 +277,14 @@ class Organization(
                 return request
         return None
 
+    @field(description="Link to the API endpoint for this organization", show_as_ref=True)
+    def uri(self, *args, **kwargs):
+        return self.self_api_url(*args, **kwargs)
+
+    @field(description="Link to the udata web page for this organization", show_as_ref=True)
+    def page(self, *args, **kwargs):
+        return self.self_web_url(*args, **kwargs)
+
     @classmethod
     def get(cls, id_or_slug):
         obj = cls.objects(slug=id_or_slug).first()

udata/core/pages/models.py

@@ -83,6 +83,55 @@ class LinksListBloc(BlocWithTitleMixin, Bloc):
     links = field(db.EmbeddedDocumentListField(LinkInBloc))
 
 
+HERO_COLORS = ("primary", "green", "purple")
+
+
+@generate_fields()
+class HeroBloc(Bloc):
+    title = field(db.StringField(required=True))
+    description = field(db.StringField())
+    color = field(db.StringField(choices=HERO_COLORS))
+
+
+@generate_fields()
+class MarkdownBloc(Bloc):
+    # Not using BlocWithTitleMixin because title should be optional here
+    title = field(db.StringField())
+    subtitle = field(db.StringField())
+    content = field(
+        db.StringField(required=True),
+        markdown=True,
+    )
+
+
+BLOCS_DISALLOWED_IN_ACCORDION = ("AccordionListBloc", "HeroBloc")
+
+
+def check_no_recursive_blocs(blocs, **kwargs):
+    for bloc in blocs:
+        if bloc.__class__.__name__ in BLOCS_DISALLOWED_IN_ACCORDION:
+            raise db.ValidationError(
+                f"{bloc.__class__.__name__} cannot be nested inside an accordion"
+            )
+
+
+@generate_fields()
+class AccordionItemBloc(db.EmbeddedDocument):
+    title = field(db.StringField(required=True))
+    content = field(
+        db.EmbeddedDocumentListField(Bloc),
+        generic=True,
+        checks=[check_no_recursive_blocs],
+    )
+
+
+@generate_fields()
+class AccordionListBloc(Bloc):
+    title = field(db.StringField())
+    description = field(db.StringField())
+    items = field(db.EmbeddedDocumentListField(AccordionItemBloc))
+
+
 @generate_fields()
 class Page(Auditable, Owned, Datetimed, db.Document):
     blocs = field(

udata/core/pages/tests/test_api.py

@@ -2,7 +2,7 @@ from flask import url_for
 
 from udata.core.dataset import tasks
 from udata.core.dataset.factories import DatasetFactory
-from udata.core.pages.models import Page
+from udata.core.pages.models import AccordionItemBloc, AccordionListBloc, Page
 from udata.core.user.factories import AdminFactory
 from udata.tests.api import APITestCase
 
@@ -103,3 +103,167 @@ class PageAPITest(APITestCase):
 
         response = self.get(url_for("api.page", page=page_id))
         self.assert200(response)
+
+    def test_hero_bloc(self):
+        self.login()
+
+        response = self.post(
+            url_for("api.pages"),
+            {
+                "blocs": [
+                    {
+                        "class": "HeroBloc",
+                        "title": "Welcome to our portal",
+                        "description": "Discover our datasets",
+                        "color": "primary",
+                    }
+                ],
+            },
+        )
+        self.assert201(response)
+
+        self.assertEqual("HeroBloc", response.json["blocs"][0]["class"])
+        self.assertEqual("Welcome to our portal", response.json["blocs"][0]["title"])
+        self.assertEqual("Discover our datasets", response.json["blocs"][0]["description"])
+        self.assertEqual("primary", response.json["blocs"][0]["color"])
+
+        page = Page.objects().first()
+        response = self.get(url_for("api.page", page=page))
+        self.assert200(response)
+
+        self.assertEqual("HeroBloc", response.json["blocs"][0]["class"])
+        self.assertEqual("Welcome to our portal", response.json["blocs"][0]["title"])
+        self.assertEqual("Discover our datasets", response.json["blocs"][0]["description"])
+        self.assertEqual("primary", response.json["blocs"][0]["color"])
+
+    def test_accordion_bloc(self):
+        self.login()
+        datasets = DatasetFactory.create_batch(2)
+
+        response = self.post(
+            url_for("api.pages"),
+            {
+                "blocs": [
+                    {
+                        "class": "AccordionListBloc",
+                        "title": "FAQ",
+                        "description": "Frequently asked questions",
+                        "items": [
+                            {
+                                "title": "What is udata?",
+                                "content": [
+                                    {
+                                        "class": "LinksListBloc",
+                                        "title": "Related links",
+                                        "links": [
+                                            {
+                                                "title": "Documentation",
+                                                "url": "https://doc.data.gouv.fr",
+                                            },
+                                        ],
+                                    }
+                                ],
+                            },
+                            {
+                                "title": "How to use datasets?",
+                                "content": [
+                                    {
+                                        "class": "DatasetsListBloc",
+                                        "title": "Example datasets",
+                                        "datasets": [str(d.id) for d in datasets],
+                                    }
+                                ],
+                            },
+                            {
+                                "title": "What is markdown?",
+                                "content": [
+                                    {
+                                        "class": "MarkdownBloc",
+                                        "content": "# Hello\n\nThis is **bold** text.",
+                                    }
+                                ],
+                            },
+                        ],
+                    }
+                ],
+            },
+        )
+        self.assert201(response)
+
+        bloc = response.json["blocs"][0]
+        self.assertEqual("AccordionListBloc", bloc["class"])
+        self.assertEqual("FAQ", bloc["title"])
+        self.assertEqual("Frequently asked questions", bloc["description"])
+        self.assertEqual(3, len(bloc["items"]))
+
+        self.assertEqual("What is udata?", bloc["items"][0]["title"])
+        self.assertEqual("LinksListBloc", bloc["items"][0]["content"][0]["class"])
+        self.assertEqual("Documentation", bloc["items"][0]["content"][0]["links"][0]["title"])
+
+        self.assertEqual("How to use datasets?", bloc["items"][1]["title"])
+        self.assertEqual("DatasetsListBloc", bloc["items"][1]["content"][0]["class"])
+        self.assertEqual(2, len(bloc["items"][1]["content"][0]["datasets"]))
+
+        self.assertEqual("What is markdown?", bloc["items"][2]["title"])
+        self.assertEqual("MarkdownBloc", bloc["items"][2]["content"][0]["class"])
+
+        page = Page.objects().first()
+        self.assertIsInstance(page.blocs[0], AccordionListBloc)
+        self.assertIsInstance(page.blocs[0].items[0], AccordionItemBloc)
+        self.assertEqual("What is udata?", page.blocs[0].items[0].title)
+
+        response = self.get(url_for("api.page", page=page))
+        self.assert200(response)
+        self.assertEqual("AccordionListBloc", response.json["blocs"][0]["class"])
+        self.assertEqual(3, len(response.json["blocs"][0]["items"]))
+
+        response = self.put(
+            url_for("api.page", page=page),
+            {
+                "blocs": [
+                    {
+                        "class": "AccordionListBloc",
+                        "title": "Updated FAQ",
+                        "items": [
+                            {
+                                "title": "Single question",
+                                "content": [],
+                            }
+                        ],
+                    }
+                ],
+            },
+        )
+        self.assert200(response)
+        self.assertEqual("Updated FAQ", response.json["blocs"][0]["title"])
+        self.assertIsNone(response.json["blocs"][0]["description"])
+        self.assertEqual(1, len(response.json["blocs"][0]["items"]))
+        self.assertEqual("Single question", response.json["blocs"][0]["items"][0]["title"])
+
+    def test_accordion_bloc_cannot_be_nested(self):
+        self.login()
+
+        response = self.post(
+            url_for("api.pages"),
+            {
+                "blocs": [
+                    {
+                        "class": "AccordionListBloc",
+                        "title": "FAQ",
+                        "items": [
+                            {
+                                "title": "Question",
+                                "content": [
+                                    {
+                                        "class": "AccordionListBloc",
+                                        "title": "Nested accordion",
+                                        "items": [],
+                                    }
+                                ],
+                            },
+                        ],
+                    }
+                ],
+            },
+        )
+        self.assert400(response)

udata/core/post/api.py

@@ -76,7 +76,7 @@ class PostsAtomFeedAPI(API):
             link=request.url_root,
         )
 
-        posts: list[Post] = Post.objects().published().order_by("-published").limit(15)
+        posts: list[Post] = Post.objects(kind="news").published().order_by("-published").limit(15)
         for post in posts:
             feed.add_item(
                 post.name,

udata/core/post/constants.py

@@ -8,5 +8,13 @@ BODY_TYPES = OrderedDict(
     [
         ("markdown", _("Markdown")),
         ("html", _("HTML")),
+        ("blocs", _("Blocs")),
+    ]
+)
+
+POST_KINDS = OrderedDict(
+    [
+        ("news", _("News")),
+        ("page", _("Page")),
     ]
 )