udata 9.2.5.dev32170__py2.py3-none-any.whl → 9.2.5.dev32222__py2.py3-none-any.whl

udata/api/__init__.py

@@ -303,6 +303,7 @@ def init_app(app):
     import udata.core.dataset.api  # noqa
     import udata.core.dataset.apiv2  # noqa
     import udata.core.dataservices.api  # noqa
+    import udata.core.dataservices.apiv2  # noqa
     import udata.core.discussions.api  # noqa
     import udata.core.reuse.api  # noqa
     import udata.core.reuse.apiv2  # noqa

udata/api_fields.py

@@ -1,3 +1,6 @@
+import functools
+from typing import Any, Dict
+
 import flask_restx.fields as restx_fields
 import mongoengine
 import mongoengine.fields as mongo_fields
@@ -114,13 +117,15 @@ def convert_db_to_field(key, field, info):
         # But we want to keep the `constructor_write` to allow changing the list.
         def constructor_write(**kwargs):
             return restx_fields.List(field_write, **kwargs)
+
     elif isinstance(
         field, (mongo_fields.GenericReferenceField, mongoengine.fields.GenericLazyReferenceField)
     ):
 
         def constructor(**kwargs):
             return restx_fields.Nested(lazy_reference, **kwargs)
-    elif isinstance(field, (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField)):
+
+    elif isinstance(field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField):
         # For reference we accept while writing a String representing the ID of the referenced model.
         # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
         # the referenced model, if not we return a String (and RestX will call the `str()` of the model
@@ -142,6 +147,7 @@ def convert_db_to_field(key, field, info):
 
             def constructor(**kwargs):
                 return restx_fields.Nested(nested_fields, **kwargs)
+
         elif hasattr(field.document_type_obj, "__read_fields__"):
 
             def constructor_read(**kwargs):
@@ -149,6 +155,7 @@ def convert_db_to_field(key, field, info):
 
             def constructor_write(**kwargs):
                 return restx_fields.Nested(field.document_type_obj.__write_fields__, **kwargs)
+
         else:
             raise ValueError(
                 f"EmbeddedDocumentField `{key}` requires a `nested_fields` param to serialize/deserialize or a `@generate_fields()` definition."
@@ -200,8 +207,12 @@ def generate_fields(**kwargs):
         read_fields = {}
         write_fields = {}
         ref_fields = {}
-        sortables = kwargs.get("additionalSorts", [])
+        sortables = kwargs.get("additional_sorts", [])
+
         filterables = []
+        additional_filters = get_fields_with_additional_filters(
+            kwargs.get("additional_filters", {})
+        )
 
         read_fields["id"] = restx_fields.String(required=True, readonly=True)
 
@@ -209,38 +220,48 @@ def generate_fields(**kwargs):
             sortable_key = info.get("sortable", False)
             if sortable_key:
                 sortables.append(
-                    {"key": sortable_key if isinstance(sortable_key, str) else key, "value": key}
+                    {
+                        "key": sortable_key if isinstance(sortable_key, str) else key,
+                        "value": key,
+                    }
                 )
 
             filterable = info.get("filterable", None)
+
             if filterable is not None:
-                if "key" not in filterable:
-                    filterable["key"] = key
-                if "column" not in filterable:
-                    filterable["column"] = key
-
-                if "constraints" not in filterable:
-                    filterable["constraints"] = []
-                    if isinstance(
-                        field, (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField)
-                    ) or (
-                        isinstance(field, mongo_fields.ListField)
-                        and isinstance(
-                            field.field,
-                            (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField),
-                        )
-                    ):
-                        filterable["constraints"].append("objectid")
+                filterables.append(compute_filter(key, field, info, filterable))
+
+            additional_filter = additional_filters.get(key, None)
+            if additional_filter:
+                if not isinstance(
+                    field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField
+                ):
+                    raise Exception("Cannot use additional_filters on not a ref.")
+
+                ref_model = field.document_type
+
+                for child in additional_filter.get("children", []):
+                    inner_field = getattr(ref_model, child["key"])
 
-                if "type" not in filterable:
-                    filterable["type"] = str
-                    if isinstance(field, mongo_fields.BooleanField):
-                        filterable["type"] = boolean
+                    column = f"{key}__{child['key']}"
+                    child["key"] = f"{key}_{child['key']}"
+                    filterable = compute_filter(column, inner_field, info, child)
 
-                # We may add more information later here:
-                # - type of mongo query to execute (right now only simple =)
+                    # Since MongoDB is not capable of doing joins with a column like `organization__slug` we need to
+                    # do a custom filter by splitting the query in two.
 
-                filterables.append(filterable)
+                    def query(filterable, query, value):
+                        # We use the computed `filterable["column"]` here because the `compute_filter` function
+                        # could have added a default filter at the end (for example `organization__badges` converted
+                        # in `organization__badges__kind`)
+                        parts = filterable["column"].split("__", 1)
+                        models = ref_model.objects.filter(**{parts[1]: value}).only("id")
+                        return query.filter(**{f"{parts[0]}__in": models})
+
+                    # do a query-based filter instead of a column based one
+                    filterable["query"] = functools.partial(query, filterable)
+
+                    filterables.append(filterable)
 
             read, write = convert_db_to_field(key, field, info)
 
@@ -330,9 +351,10 @@ def generate_fields(**kwargs):
 
         for filterable in filterables:
             parser.add_argument(
-                filterable["key"],
+                filterable.get("label", filterable["key"]),
                 type=filterable["type"],
                 location="args",
+                choices=filterable.get("choices", None),
             )
 
         cls.__index_parser__ = parser
@@ -360,18 +382,23 @@ def generate_fields(**kwargs):
                 base_query = base_query.search_text(phrase_query)
 
             for filterable in filterables:
-                if args.get(filterable["key"]) is not None:
-                    for constraint in filterable["constraints"]:
+                filter = args.get(filterable.get("label", filterable["key"]))
+                if filter is not None:
+                    for constraint in filterable.get("constraints", []):
                         if constraint == "objectid" and not ObjectId.is_valid(
                             args[filterable["key"]]
                         ):
                             api.abort(400, f'`{filterable["key"]}` must be an identifier')
 
-                    base_query = base_query.filter(
-                        **{
-                            filterable["column"]: args[filterable["key"]],
-                        }
-                    )
+                    query = filterable.get("query", None)
+                    if query:
+                        base_query = filterable["query"](base_query, filter)
+                    else:
+                        base_query = base_query.filter(
+                            **{
+                                filterable["column"]: filter,
+                            }
+                        )
 
             if paginable:
                 base_query = base_query.paginate(args["page"], args["page_size"])
@@ -379,6 +406,7 @@ def generate_fields(**kwargs):
             return base_query
 
         cls.apply_sort_filters_and_pagination = apply_sort_filters_and_pagination
+        cls.__additional_class_info__ = kwargs
         return cls
 
     return wrapper
@@ -417,12 +445,12 @@ def patch(obj, request):
                 value = model_attribute.from_input(value)
             elif isinstance(model_attribute, mongoengine.fields.ListField) and isinstance(
                 model_attribute.field,
-                (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField),
+                mongo_fields.ReferenceField | mongo_fields.LazyReferenceField,
             ):
                 # TODO `wrap_primary_key` do Mongo request, do a first pass to fetch all documents before calling it (to avoid multiple queries).
                 value = [wrap_primary_key(key, model_attribute.field, id) for id in value]
             elif isinstance(
-                model_attribute, (mongo_fields.ReferenceField, mongo_fields.LazyReferenceField)
+                model_attribute, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField
             ):
                 value = wrap_primary_key(key, model_attribute, value)
             elif isinstance(
@@ -517,3 +545,81 @@ def wrap_primary_key(
         raise ValueError(
             f"Unknown ID field type {id_field.__class__} for {document_type} (ID field name is {id_field_name}, value was {value})"
         )
+
+
+def get_fields_with_additional_filters(additional_filters: Dict[str, str]) -> Dict[str, Any]:
+    """
+    Right now we only support additional filters like "organization.badges".
+
+    The goal of this function is to key the additional filters by the first part (`organization`) to
+    be able to compute them when we loop over all the fields (`title`, `organization`…)
+
+    The `additional_filters` property is a dict: {"label": "key"}, for example {"organization_badge": "organization.badges"}.
+    The `label` will be the name of the parser arg, like `?organization_badge=public-service`, which makes more
+    sense than `?organization_badges=public-service`.
+    """
+    results: dict = {}
+    for label, key in additional_filters.items():
+        parts = key.split(".")
+        if len(parts) == 2:
+            parent = parts[0]
+            child = parts[1]
+
+            if parent not in results:
+                results[parent] = {"children": []}
+
+            results[parent]["children"].append(
+                {
+                    "label": label,
+                    "key": child,
+                    "type": str,
+                }
+            )
+        else:
+            raise Exception(f"Do not support `additional_filters` without two parts: {key}.")
+
+    return results
+
+
+def compute_filter(column: str, field, info, filterable):
+    # "key" is the param key in the URL
+    if "key" not in filterable:
+        filterable["key"] = column
+
+    # If we do a filter on a embed document, get the class info
+    # of this document to see if there is a default filter value
+    embed_info = None
+    if isinstance(field, mongo_fields.EmbeddedDocumentField):
+        embed_info = field.get("__additional_class_info__", None)
+    elif isinstance(field, mongo_fields.EmbeddedDocumentListField):
+        embed_info = getattr(field.field.document_type, "__additional_class_info__", None)
+
+    if embed_info and embed_info.get("default_filterable_field", None):
+        # There is a default filterable field so append it to the column and replace the
+        # field to use the inner one (for example using the `kind` `StringField` instead of
+        # the embed `Badge` field.)
+        filterable["column"] = f"{column}__{embed_info['default_filterable_field']}"
+        field = getattr(field.field.document_type, embed_info["default_filterable_field"])
+    else:
+        filterable["column"] = column
+
+    if "constraints" not in filterable:
+        filterable["constraints"] = []
+
+    if isinstance(field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField) or (
+        isinstance(field, mongo_fields.ListField)
+        and isinstance(field.field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField)
+    ):
+        filterable["constraints"].append("objectid")
+
+    if "type" not in filterable:
+        if isinstance(field, mongo_fields.BooleanField):
+            filterable["type"] = boolean
+        else:
+            filterable["type"] = str
+
+    filterable["choices"] = info.get("choices", None)
+    if hasattr(field, "choices") and field.choices:
+        filterable["choices"] = field.choices
+
+    return filterable

udata/core/badges/factories.py

@@ -2,14 +2,12 @@ from factory.fuzzy import FuzzyChoice
 
 from udata.factories import ModelFactory
 
-from .models import Badge
 
-
-def badge_factory(model):
+def badge_factory(model_):
     class BadgeFactory(ModelFactory):
         class Meta:
-            model = Badge
+            model = model_._fields["badges"].field.document_type
 
-        kind = FuzzyChoice(model.__badges__.keys())
+        kind = FuzzyChoice(model_.__badges__)
 
     return BadgeFactory

udata/core/badges/forms.py

@@ -1,6 +1,5 @@
 from udata.forms import ModelForm, fields, validators
 from udata.i18n import lazy_gettext as _
-from udata.models import Badge
 
 __all__ = ("badge_form",)
 
@@ -9,8 +8,6 @@ def badge_form(model):
     """A form factory for a given model badges"""
 
     class BadgeForm(ModelForm):
-        model_class = Badge
-
         kind = fields.RadioField(
             _("Kind"),
             [validators.DataRequired()],

udata/core/badges/models.py

@@ -3,7 +3,7 @@ from datetime import datetime
 
 from mongoengine.signals import post_save
 
-from udata.api_fields import field
+from udata.api_fields import field, generate_fields
 from udata.auth import current_user
 from udata.core.badges.fields import badge_fields
 from udata.mongo import db
@@ -12,10 +12,19 @@ from .signals import on_badge_added, on_badge_removed
 
 log = logging.getLogger(__name__)
 
-__all__ = ("Badge", "BadgeMixin")
 
+__all__ = ["Badge", "BadgeMixin", "BadgesList"]
 
+DEFAULT_BADGES_LIST_PARAMS = {
+    "readonly": True,
+    "inner_field_info": {"nested_fields": badge_fields},
+}
+
+
+@generate_fields(default_filterable_field="kind")
 class Badge(db.EmbeddedDocument):
+    meta = {"allow_inheritance": True}
+    # The following field should be overloaded in descendants.
     kind = db.StringField(required=True)
     created = db.DateTimeField(default=datetime.utcnow, required=True)
     created_by = db.ReferenceField("User")
@@ -23,30 +32,16 @@ class Badge(db.EmbeddedDocument):
     def __str__(self):
         return self.kind
 
-    def validate(self, clean=True):
-        badges = getattr(self._instance, "__badges__", {})
-        if self.kind not in badges.keys():
-            raise db.ValidationError("Unknown badge type %s" % self.kind)
-        return super(Badge, self).validate(clean=clean)
-
 
 class BadgesList(db.EmbeddedDocumentListField):
-    def __init__(self, *args, **kwargs):
-        return super(BadgesList, self).__init__(Badge, *args, **kwargs)
-
-    def validate(self, value):
-        kinds = [b.kind for b in value]
-        if len(kinds) > len(set(kinds)):
-            raise db.ValidationError("Duplicate badges for a given kind is not allowed")
-        return super(BadgesList, self).validate(value)
+    def __init__(self, badge_model, *args, **kwargs):
+        return super(BadgesList, self).__init__(badge_model, *args, **kwargs)
 
 
-class BadgeMixin(object):
-    badges = field(
-        BadgesList(),
-        readonly=True,
-        inner_field_info={"nested_fields": badge_fields},
-    )
+class BadgeMixin:
+    default_badges_list_params = DEFAULT_BADGES_LIST_PARAMS
+    # The following field should be overloaded in descendants.
+    badges = field(BadgesList(Badge), **DEFAULT_BADGES_LIST_PARAMS)
 
     def get_badge(self, kind):
         """Get a badge given its kind if present"""
@@ -61,7 +56,7 @@ class BadgeMixin(object):
         if kind not in getattr(self, "__badges__", {}):
             msg = "Unknown badge type for {model}: {kind}"
             raise db.ValidationError(msg.format(model=self.__class__.__name__, kind=kind))
-        badge = Badge(kind=kind)
+        badge = self._fields["badges"].field.document_type(kind=kind)
         if current_user.is_authenticated:
             badge.created_by = current_user.id
 
@@ -88,5 +83,5 @@ class BadgeMixin(object):
 
     def badge_label(self, badge):
         """Display the badge label for a given kind"""
-        kind = badge.kind if isinstance(badge, Badge) else badge
+        kind = badge.kind if isinstance(badge, self.badge) else badge
         return self.__badges__[kind]

udata/core/badges/tests/test_commands.py

@@ -4,7 +4,6 @@ import pytest
 
 from udata.core.organization.constants import CERTIFIED, PUBLIC_SERVICE
 from udata.core.organization.factories import OrganizationFactory
-from udata.models import Badge
 
 
 @pytest.mark.usefixtures("clean_db")
@@ -21,9 +20,9 @@ class BadgeCommandTest:
         assert org.badges[0].kind == PUBLIC_SERVICE
 
     def test_toggle_badge_off(self, cli):
-        ps_badge = Badge(kind=PUBLIC_SERVICE)
-        certified_badge = Badge(kind=CERTIFIED)
-        org = OrganizationFactory(badges=[ps_badge, certified_badge])
+        org = OrganizationFactory()
+        org.add_badge(PUBLIC_SERVICE)
+        org.add_badge(CERTIFIED)
 
         cli("badges", "toggle", str(org.id), PUBLIC_SERVICE)
 

udata/core/badges/tests/test_model.py

@@ -1,19 +1,31 @@
+from udata.api_fields import field
 from udata.auth import login_user
 from udata.core.user.factories import UserFactory
 from udata.mongo import db
 from udata.tests import DBTestMixin, TestCase
 
-from ..models import Badge, BadgeMixin
+from ..models import Badge, BadgeMixin, BadgesList
 
 TEST = "test"
 OTHER = "other"
 
+BADGES = {
+    TEST: "Test",
+    OTHER: "Other",
+}
 
-class Fake(db.Document, BadgeMixin):
-    __badges__ = {
-        TEST: "Test",
-        OTHER: "Other",
-    }
+
+class FakeBadge(Badge):
+    kind = db.StringField(required=True, choices=list(BADGES.keys()))
+
+
+class FakeBadgeMixin(BadgeMixin):
+    badges = field(BadgesList(FakeBadge), **BadgeMixin.default_badges_list_params)
+    __badges__ = BADGES
+
+
+class Fake(db.Document, FakeBadgeMixin):
+    pass
 
 
 class BadgeMixinTest(DBTestMixin, TestCase):
@@ -22,15 +34,24 @@ class BadgeMixinTest(DBTestMixin, TestCase):
         fake = Fake.objects.create()
         self.assertIsInstance(fake.badges, (list, tuple))
 
+    def test_choices(self):
+        """It should have a choice list on the badge field."""
+        self.assertEqual(
+            Fake._fields["badges"].field.document_type.kind.choices, list(Fake.__badges__.keys())
+        )
+
     def test_get_badge_found(self):
         """It allow to get a badge by kind if present"""
-        fake = Fake.objects.create(badges=[Badge(kind=TEST), Badge(kind=OTHER)])
+        fake = Fake.objects.create()
+        fake.add_badge(TEST)
+        fake.add_badge(OTHER)
         badge = fake.get_badge(TEST)
         self.assertEqual(badge.kind, TEST)
 
     def test_get_badge_not_found(self):
         """It should return None if badge is absent"""
-        fake = Fake.objects.create(badges=[Badge(kind=OTHER)])
+        fake = Fake.objects.create()
+        fake.add_badge(OTHER)
         badge = fake.get_badge(TEST)
         self.assertIsNone(badge)
 
@@ -49,7 +70,8 @@ class BadgeMixinTest(DBTestMixin, TestCase):
 
     def test_add_2nd_badge(self):
         """It should add badges to the top of the list"""
-        fake = Fake.objects.create(badges=[Badge(kind=OTHER)])
+        fake = Fake.objects.create()
+        fake.add_badge(OTHER)
 
         result = fake.add_badge(TEST)
 
@@ -86,8 +108,8 @@ class BadgeMixinTest(DBTestMixin, TestCase):
 
     def test_remove_badge(self):
         """It should remove a badge given its kind"""
-        badge = Badge(kind=TEST)
-        fake = Fake.objects.create(badges=[badge])
+        fake = Fake.objects.create()
+        fake.add_badge(TEST)
 
         fake.remove_badge(TEST)
 
@@ -121,28 +143,15 @@ class BadgeMixinTest(DBTestMixin, TestCase):
 
     def test_toggle_remove_badge(self):
         """Toggle should remove a badge given its kind if present"""
-        badge = Badge(kind=TEST)
-        fake = Fake.objects.create(badges=[badge])
+        fake = Fake.objects.create()
+        fake.add_badge(TEST)
 
         fake.toggle_badge(TEST)
 
         self.assertEqual(len(fake.badges), 0)
 
-    def test_create_with_badges(self):
-        """It should allow object creation with badges"""
-        fake = Fake.objects.create(badges=[Badge(kind=TEST), Badge(kind=OTHER)])
-
-        self.assertEqual(len(fake.badges), 2)
-        for badge, kind in zip(fake.badges, (TEST, OTHER)):
-            self.assertEqual(badge.kind, kind)
-            self.assertIsNotNone(badge.created)
-
-    def test_create_disallow_duplicate_badges(self):
-        """It should not allow object creation with duplicate badges"""
-        with self.assertRaises(db.ValidationError):
-            Fake.objects.create(badges=[Badge(kind=TEST), Badge(kind=TEST)])
-
     def test_create_disallow_unknown_badges(self):
         """It should not allow object creation with unknown badges"""
         with self.assertRaises(db.ValidationError):
-            Fake.objects.create(badges=[Badge(kind="unknown")])
+            fake = Fake.objects.create()
+            fake.add_badge("unknown")

udata/core/dataservices/api.py

@@ -7,8 +7,8 @@ from flask_login import current_user
 
 from udata.api import API, api, fields
 from udata.api_fields import patch
+from udata.core.dataservices.permissions import OwnablePermission
 from udata.core.dataset.models import Dataset
-from udata.core.dataset.permissions import OwnablePermission
 from udata.core.followers.api import FollowAPI
 from udata.rdf import RDF_EXTENSIONS, graph_response, negociate_content
 

udata/core/dataservices/apiv2.py

@@ -0,0 +1,29 @@
+from flask import request
+
+from udata import search
+from udata.api import API, apiv2
+from udata.core.dataservices.models import Dataservice, HarvestMetadata
+from udata.utils import multi_to_dict
+
+from .search import DataserviceSearch
+
+apiv2.inherit("DataservicePage", Dataservice.__page_fields__)
+apiv2.inherit("Dataservice (read)", Dataservice.__read_fields__)
+apiv2.inherit("HarvestMetadata (read)", HarvestMetadata.__read_fields__)
+
+ns = apiv2.namespace("dataservices", "Dataservice related operations")
+
+search_parser = DataserviceSearch.as_request_parser()
+
+
+@ns.route("/search/", endpoint="dataservice_search")
+class DataserviceSearchAPI(API):
+    """Dataservices collection search endpoint"""
+
+    @apiv2.doc("search_dataservices")
+    @apiv2.expect(search_parser)
+    @apiv2.marshal_with(Dataservice.__page_fields__)
+    def get(self):
+        """Search all dataservices"""
+        search_parser.parse_args()
+        return search.query(DataserviceSearch, **multi_to_dict(request.args))

udata/core/dataservices/models.py

@@ -95,7 +95,10 @@ class HarvestMetadata(db.EmbeddedDocument):
     archived_at = field(db.DateTimeField())
 
 
-@generate_fields(searchable=True)
+@generate_fields(
+    searchable=True,
+    additional_filters={"organization_badge": "organization.badges"},
+)
 class Dataservice(WithMetrics, Owned, db.Document):
     meta = {
         "indexes": [