udata 10.0.1.dev32334__py2.py3-none-any.whl → 10.0.1.dev32350__py2.py3-none-any.whl

udata/api_fields.py

@@ -1,16 +1,56 @@
+"""Enhance a MongoEngine document class to give it super powers by decorating it with @generate_fields.
+
+The main goal of `generate_fields` is to remove duplication: we used to have fields declaration in
+- models.py
+- forms.py
+- api_fields.py
+
+Now they're defined in models.py, and adding the `generate_fields` decorator makes them available in the format we need them for the forms or the API.
+
+- default_filterable_field: which field in this document should be the default filter, eg when filtering by Badge, you're actually filtering on `Badge.kind`
+- searchable: boolean, if True, the document can be full-text searched using MongoEngine text search
+- additional_sorts: add more sorts than the already available ones based on fields (see below). Eg, sort by metrics.
+- additional_filters: filter on a field of a field (aka "join"), eg filter on `Reuse__organization__badge=PUBLIC_SERVICE`.
+
+
+On top of those functionalities added to the document by the `@generate_fields` decorator parameters,
+the document fields are parsed and enhanced if they are wrapped in the `field` helper.
+
+- sortable: boolean, if True, it'll be available in the list of sort options
+- show_as_ref: add to the list of `ref_fields` (see below)
+- readonly: don't add this field to the `write_fields`
+- markdown: use Mardown to format this field instead of plain old text
+- filterable: this field can be filtered on. It's either an empty dictionnary, either {`key`: `field_name`} if the `field_name` to use is different from the original field, eg `dataset` instead of `datasets`.
+- description: use as the info on the field in the swagger forms.
+- check: provide a function to validate the content of the field.
+- thumbnail_info: add additional info for a thumbnail, eg `{ "size": BIGGEST_IMAGE_SIZE }`.
+
+You may also use the `@function_field` decorator to treat a document method as a field.
+
+
+The following fields are added on the document class once decorated:
+
+- ref_fields: list of fields to return when embedding/referencing a document, eg when querying Reuse.organization, only return a subset of the org fields
+- read_fields: all of the fields to return when querying a document
+- write_fields: list of fields to provide when creating a document, eg when creating a Reuse, we only provide organization IDs, not all the org fields
+
+"""
+
 import functools
-from typing import Any, Dict
+from typing import Any, Callable, Iterable
 
 import flask_restx.fields as restx_fields
 import mongoengine
 import mongoengine.fields as mongo_fields
 from bson import ObjectId
 from flask_restx.inputs import boolean
+from flask_restx.reqparse import RequestParser
 from flask_storage.mongo import ImageField as FlaskStorageImageField
 
 import udata.api.fields as custom_restx_fields
 from udata.api import api, base_reference
 from udata.mongo.errors import FieldValidationError
+from udata.mongo.queryset import DBPaginator, UDataQuerySet
 
 lazy_reference = api.model(
     "LazyReference",
@@ -21,26 +61,27 @@ lazy_reference = api.model(
 )
 
 
-def convert_db_to_field(key, field, info):
-    """
-    This function maps a Mongo field to a Flask RestX field.
+def convert_db_to_field(key, field, info) -> tuple[Callable | None, Callable | None]:
+    """Map a Mongo field to a Flask RestX field.
+
     Most of the types are a simple 1-to-1 mapping except lists and references that requires
     more work.
     We currently only map the params that we use from Mongo to RestX (for example min_length / max_length…).
 
     In the first part of the function we save the RestX constructor as a lambda because we need to call it with the
     params. Since merging the params involve a litte bit of work (merging default params with read/write params and then with
-    user-supplied overrides, setting the readonly flag…), it's easier to have do this one time at the end of the function.
+    user-supplied overrides, setting the readonly flag…), it's easier to have to do this only once at the end of the function.
+
     """
-    params = {}
+    params: dict = {}
     params["required"] = field.required
 
-    read_params = {}
-    write_params = {}
+    read_params: dict = {}
+    write_params: dict = {}
 
-    constructor = None
-    constructor_read = None
-    constructor_write = None
+    constructor: Callable
+    constructor_read: Callable | None = None
+    constructor_write: Callable | None = None
 
     if info.get("convert_to"):
         # TODO: this is currently never used. We may remove it if the auto-conversion
@@ -66,7 +107,7 @@ def convert_db_to_field(key, field, info):
     elif isinstance(field, mongo_fields.DictField):
         constructor = restx_fields.Raw
     elif isinstance(field, mongo_fields.ImageField) or isinstance(field, FlaskStorageImageField):
-        size = info.get("size", None)
+        size: int | None = info.get("size", None)
         if size:
             params["description"] = f"URL of the cropped and squared image ({size}x{size})"
         else:
@@ -103,7 +144,7 @@ def convert_db_to_field(key, field, info):
         #     1. `inner_field_info` inside `__additional_field_info__` on the parent
         #     2. `__additional_field_info__` of the inner field
         #     3. `__additional_field_info__` of the parent
-        inner_info = getattr(field.field, "__additional_field_info__", {})
+        inner_info: dict = getattr(field.field, "__additional_field_info__", {})
         field_read, field_write = convert_db_to_field(
             f"{key}.inner", field.field, {**info, **inner_info, **info.get("inner_field_info", {})}
         )
@@ -130,7 +171,7 @@ def convert_db_to_field(key, field, info):
         # For reading, if the user supplied a `nested_fields` (RestX model), we use it to convert
         # the referenced model, if not we return a String (and RestX will call the `str()` of the model
         # when returning from an endpoint)
-        nested_fields = info.get("nested_fields")
+        nested_fields: dict | None = info.get("nested_fields")
         if nested_fields is None:
             # If there is no `nested_fields` convert the object to the string representation.
             constructor_read = restx_fields.String
@@ -167,9 +208,11 @@ def convert_db_to_field(key, field, info):
     read_params = {**params, **read_params, **info}
     write_params = {**params, **write_params, **info}
 
-    read = constructor_read(**read_params) if constructor_read else constructor(**read_params)
+    read: Callable = (
+        constructor_read(**read_params) if constructor_read else constructor(**read_params)
+    )
     if write_params.get("readonly", False) or (constructor_write is None and constructor is None):
-        write = None
+        write: Callable | None = None
     else:
         write = (
             constructor_write(**write_params) if constructor_write else constructor(**write_params)
@@ -177,10 +220,11 @@ def convert_db_to_field(key, field, info):
     return read, write
 
 
-def get_fields(cls):
-    """
-    Returns all the exposed fields of the class (fields decorated with `field()`)
-    It also expends image fields to add thumbnail fields.
+def get_fields(cls) -> Iterable[tuple[str, Callable, dict]]:
+    """Return all the document fields that are wrapped with the `field()` helper.
+
+    Also expand image fields to add thumbnail fields.
+
     """
     for key, field in cls._fields.items():
         info: dict | None = getattr(field, "__additional_field_info__", None)
@@ -197,27 +241,35 @@ def get_fields(cls):
             )
 
 
-def generate_fields(**kwargs):
-    """
+def generate_fields(**kwargs) -> Callable:
+    """Mongoengine document decorator.
+
     This decorator will create two auto-generated attributes on the class `__read_fields__` and `__write_fields__`
     that can be used in API endpoint inside `expect()` and `marshall_with()`.
+
+    It will also
+    - generate an API parameter parser
+    - sort and filter a list of documents with the provided params using the `apply_sort_filters_and_pagination` helper
+
     """
 
-    def wrapper(cls):
-        read_fields = {}
-        write_fields = {}
-        ref_fields = {}
-        sortables = kwargs.get("additional_sorts", [])
+    def wrapper(cls) -> Callable:
+        from udata.models import db
 
-        filterables = []
-        additional_filters = get_fields_with_additional_filters(
+        read_fields: dict = {}
+        write_fields: dict = {}
+        ref_fields: dict = {}
+        sortables: list = kwargs.get("additional_sorts", [])
+
+        filterables: list[dict] = []
+        additional_filters: dict[str, dict] = get_fields_with_additional_filters(
             kwargs.get("additional_filters", {})
         )
 
         read_fields["id"] = restx_fields.String(required=True, readonly=True)
 
         for key, field, info in get_fields(cls):
-            sortable_key = info.get("sortable", False)
+            sortable_key: bool = info.get("sortable", False)
             if sortable_key:
                 sortables.append(
                     {
@@ -226,41 +278,41 @@ def generate_fields(**kwargs):
                     }
                 )
 
-            filterable = info.get("filterable", None)
-
+            filterable: dict[str, Any] | None = info.get("filterable", None)
             if filterable is not None:
                 filterables.append(compute_filter(key, field, info, filterable))
 
-            additional_filter = additional_filters.get(key, None)
+            additional_filter: dict | None = additional_filters.get(key, None)
             if additional_filter:
                 if not isinstance(
                     field, mongo_fields.ReferenceField | mongo_fields.LazyReferenceField
                 ):
                     raise Exception("Cannot use additional_filters on not a ref.")
 
-                ref_model = field.document_type
+                ref_model: db.Document = field.document_type
 
                 for child in additional_filter.get("children", []):
-                    inner_field = getattr(ref_model, child["key"])
+                    inner_field: str = getattr(ref_model, child["key"])
 
-                    column = f"{key}__{child['key']}"
+                    column: str = f"{key}__{child['key']}"
                     child["key"] = f"{key}_{child['key']}"
                     filterable = compute_filter(column, inner_field, info, child)
 
                     # Since MongoDB is not capable of doing joins with a column like `organization__slug` we need to
                     # do a custom filter by splitting the query in two.
 
-                    def query(filterable, query, value):
+                    def query(filterable, query, value) -> UDataQuerySet:
                         # We use the computed `filterable["column"]` here because the `compute_filter` function
                         # could have added a default filter at the end (for example `organization__badges` converted
                         # in `organization__badges__kind`)
                         parts = filterable["column"].split("__", 1)
-                        models = ref_model.objects.filter(**{parts[1]: value}).only("id")
+                        models: UDataQuerySet = ref_model.objects.filter(**{parts[1]: value}).only(
+                            "id"
+                        )
                         return query.filter(**{f"{parts[0]}__in": models})
 
                     # do a query-based filter instead of a column based one
                     filterable["query"] = functools.partial(query, filterable)
-
                     filterables.append(filterable)
 
             read, write = convert_db_to_field(key, field, info)
@@ -289,8 +341,8 @@ def generate_fields(**kwargs):
             if not callable(method):
                 continue
 
-            info = getattr(method, "__additional_field_info__", None)
-            if info is None:
+            additional_field_info = getattr(method, "__additional_field_info__", None)
+            if additional_field_info is None:
                 continue
 
             def make_lambda(method):
@@ -302,16 +354,16 @@ def generate_fields(**kwargs):
                 return lambda o: method(o)
 
             read_fields[method_name] = restx_fields.String(
-                attribute=make_lambda(method), **{"readonly": True, **info}
+                attribute=make_lambda(method), **{"readonly": True, **additional_field_info}
             )
-            if info.get("show_as_ref", False):
+            if additional_field_info.get("show_as_ref", False):
                 ref_fields[key] = read_fields[method_name]
 
         cls.__read_fields__ = api.model(f"{cls.__name__} (read)", read_fields, **kwargs)
         cls.__write_fields__ = api.model(f"{cls.__name__} (write)", write_fields, **kwargs)
         cls.__ref_fields__ = api.inherit(f"{cls.__name__}Reference", base_reference, ref_fields)
 
-        mask = kwargs.pop("mask", None)
+        mask: str | None = kwargs.pop("mask", None)
         if mask is not None:
             mask = "data{{{0}}},*".format(mask)
         cls.__page_fields__ = api.model(
@@ -322,8 +374,8 @@ def generate_fields(**kwargs):
         )
 
         # Parser for index sort/filters
-        paginable = kwargs.get("paginable", True)
-        parser = api.parser()
+        paginable: bool = kwargs.get("paginable", True)
+        parser: RequestParser = api.parser()
 
         if paginable:
             parser.add_argument(
@@ -334,7 +386,7 @@ def generate_fields(**kwargs):
             )
 
         if sortables:
-            choices = [sortable["key"] for sortable in sortables] + [
+            choices: list[str] = [sortable["key"] for sortable in sortables] + [
                 "-" + sortable["key"] for sortable in sortables
             ]
             parser.add_argument(
@@ -345,12 +397,13 @@ def generate_fields(**kwargs):
                 help="The field (and direction) on which sorting apply",
             )
 
-        searchable = kwargs.pop("searchable", False)
+        searchable: bool = kwargs.pop("searchable", False)
         if searchable:
             parser.add_argument("q", type=str, location="args")
 
         for filterable in filterables:
             parser.add_argument(
+                # Use the custom label from `additional_filters` if there's one.
                 filterable.get("label", filterable["key"]),
                 type=filterable["type"],
                 location="args",
@@ -359,18 +412,17 @@ def generate_fields(**kwargs):
 
         cls.__index_parser__ = parser
 
-        def apply_sort_filters_and_pagination(base_query):
+        def apply_sort_filters_and_pagination(base_query) -> DBPaginator:
             args = cls.__index_parser__.parse_args()
 
             if sortables and args["sort"]:
-                negate = args["sort"].startswith("-")
-                sort_key = args["sort"][1:] if negate else args["sort"]
+                negate: bool = args["sort"].startswith("-")
+                sort_key: str = args["sort"][1:] if negate else args["sort"]
 
-                sort_by = next(
+                sort_by: str | None = next(
                     (sortable["value"] for sortable in sortables if sortable["key"] == sort_key),
                     None,
                 )
-
                 if sort_by:
                     if negate:
                         sort_by = "-" + sort_by
@@ -378,10 +430,13 @@ def generate_fields(**kwargs):
                     base_query = base_query.order_by(sort_by)
 
             if searchable and args.get("q"):
-                phrase_query = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
+                phrase_query: str = " ".join([f'"{elem}"' for elem in args["q"].split(" ")])
                 base_query = base_query.search_text(phrase_query)
 
             for filterable in filterables:
+                # If it's from an `additional_filter`, use the custom label instead of the key,
+                # eg use `organization_badge` instead of `organization.badges` which is
+                # computed to `organization_badges`.
                 filter = args.get(filterable.get("label", filterable["key"]))
                 if filter is not None:
                     for constraint in filterable.get("constraints", []):
@@ -412,7 +467,7 @@ def generate_fields(**kwargs):
     return wrapper
 
 
-def function_field(**info):
+def function_field(**info) -> Callable:
     def inner(func):
         func.__additional_field_info__ = info
         return func
@@ -421,18 +476,20 @@ def function_field(**info):
 
 
 def field(inner, **kwargs):
-    """
-    Simple decorator to mark a field as visible for the API fields.
-    We can pass additional arguments that will be forward to the RestX field constructor.
+    """Simple wrapper to make a document field visible for the API.
+
+    We can pass additional arguments that will be forwarded to the RestX field constructor.
+
     """
     inner.__additional_field_info__ = kwargs
     return inner
 
 
-def patch(obj, request):
-    """
-    Patch the object with the data from the request.
+def patch(obj, request) -> type:
+    """Patch the object with the data from the request.
+
     Only fields decorated with the `field()` decorator will be read (and not readonly).
+
     """
     from udata.mongo.engine import db
 
@@ -480,7 +537,7 @@ def patch(obj, request):
     return obj
 
 
-def patch_and_save(obj, request):
+def patch_and_save(obj, request) -> type:
     obj = patch(obj, request)
 
     try:
@@ -497,11 +554,12 @@ def wrap_primary_key(
     value: str | None,
     document_type=None,
 ):
-    """
-    We need to wrap the `String` inside an `ObjectId` most of the time. If the foreign ID is a `String` we need to get
-    a `DBRef` from the database.
+    """Wrap the `String` inside an `ObjectId`.
+
+    If the foreign ID is a `String`, get a `DBRef` from the database.
 
     TODO: we only check the document reference if the ID is a `String` field (not in the case of a classic `ObjectId`).
+
     """
     if value is None:
         return value
@@ -547,13 +605,15 @@ def wrap_primary_key(
         )
 
 
-def get_fields_with_additional_filters(additional_filters: Dict[str, str]) -> Dict[str, Any]:
-    """
-    Right now we only support additional filters like "organization.badges".
+def get_fields_with_additional_filters(additional_filters: dict[str, str]) -> dict[str, Any]:
+    """Filter on additional related fields.
 
-    The goal of this function is to key the additional filters by the first part (`organization`) to
+    Right now we only support additional filters with a depth of two, eg "organization.badges".
+
+    The goal of this function is to key by the additional filters by the first part (`organization`) to
     be able to compute them when we loop over all the fields (`title`, `organization`…)
 
+
     The `additional_filters` property is a dict: {"label": "key"}, for example {"organization_badge": "organization.badges"}.
     The `label` will be the name of the parser arg, like `?organization_badge=public-service`, which makes more
     sense than `?organization_badges=public-service`.
@@ -570,6 +630,7 @@ def get_fields_with_additional_filters(additional_filters: Dict[str, str]) -> Di
 
             results[parent]["children"].append(
                 {
+                    # The name for the parser argument, so `organization_badge` instead of `organization_badges`.
                     "label": label,
                     "key": child,
                     "type": str,
@@ -581,7 +642,7 @@ def get_fields_with_additional_filters(additional_filters: Dict[str, str]) -> Di
     return results
 
 
-def compute_filter(column: str, field, info, filterable):
+def compute_filter(column: str, field, info, filterable) -> dict:
     # "key" is the param key in the URL
     if "key" not in filterable:
         filterable["key"] = column