wbcore 2.2.1__py2.py3-none-any.whl

wbcore/models/fields.py

@@ -0,0 +1,29 @@
+from django.core.exceptions import ValidationError
+from django.db.models import DecimalField, Field, FloatField, PositiveIntegerField
+
+
+class AbstractDynamicField(Field):
+    dependencies = []
+
+    def __init__(self, *args, dependencies=list(), **kwargs):
+        blank = kwargs.pop("blank", True)
+        null = kwargs.pop("null", True)
+        self.dependencies = dependencies
+        super().__init__(*args, blank=blank, null=null, **kwargs)
+
+
+class DynamicDecimalField(DecimalField, AbstractDynamicField):
+    description = "Use a custom callback to compute this field if None"
+
+
+class DynamicFloatField(FloatField, AbstractDynamicField):
+    description = "Use a custom callback to compute this field if None"
+
+
+def validate_year(year: int | None):
+    if year and not len(str(year)) == 4:
+        raise ValidationError("Please provide a valid 4-digit year.")
+
+
+class YearField(PositiveIntegerField):
+    default_validators = [validate_year]

wbcore/models/orderable.py

@@ -0,0 +1,6 @@
+from ordered_model.models import OrderedModel as BaseOrderedModel
+
+
+class OrderableModel(BaseOrderedModel):
+    class Meta(BaseOrderedModel.Meta):
+        abstract = True

wbcore/pagination.py

@@ -0,0 +1,65 @@
+from typing import TYPE_CHECKING, Callable
+
+from rest_framework import pagination
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+    from rest_framework.request import Request
+    from rest_framework.views import View
+
+
+class EndlessPaginationMixin(pagination.BasePagination):
+    """Custom pagination mixin to support disabling pagination through a query parameter."""
+
+    def paginate_queryset(self, queryset: "QuerySet", request: "Request", view: "View | None" = None):
+        """
+        Paginate the queryset if pagination is not explicitly disabled.
+
+        Args:
+            queryset (QuerySet): The queryset to paginate.
+            request (Request): The request object.
+            view (View, optional): The view calling this method.
+
+        Returns:
+            QuerySet: The paginated queryset or None if pagination is disabled.
+        """
+        if request.query_params.get("disable_pagination") == "true":
+            return None
+        return super().paginate_queryset(queryset, request, view)
+
+
+class InitialPaginationMixin:
+    """Mixin to determine whether the current page is the initial page."""
+
+    get_previous_link: Callable[[], str | None]
+
+    def is_initial(self) -> bool:
+        """
+        Determine if the current page is the initial page.
+
+        Returns:
+            bool: True if it's the initial page, False otherwise.
+        """
+        return self.get_previous_link() is None
+
+
+class CursorPagination(EndlessPaginationMixin, InitialPaginationMixin, pagination.CursorPagination):
+    def _get_position_from_instance(self, instance, ordering) -> str:
+        """
+        Returns the super _get_position_from_instance but adjusts the first item of ordering
+        to ensure that no foreignkey is included to not break the frontend.
+
+        Args:
+            instance: The instance to get the position from.
+            ordering (list): The ordering criteria.
+
+        Returns:
+            Any: The position from the instance.
+        """
+        new_ordering = [*ordering]
+        new_ordering[0] = new_ordering[0].split("__")[0]
+        return super()._get_position_from_instance(instance, new_ordering)
+
+
+class LimitOffsetPagination(EndlessPaginationMixin, InitialPaginationMixin, pagination.LimitOffsetPagination):
+    ...

wbcore/pandas/fields.py

@@ -0,0 +1,136 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import List
+
+from wbcore.serializers.fields.types import DisplayMode
+
+
+@dataclass
+class _BaseField:
+    key: str
+    label: str
+    type: str = None
+    decorators: List = None
+    help_text: str = None
+
+    def to_dict(self):
+        base = {"key": self.key, "label": self.label, "type": self.type}
+        if self.decorators:
+            base["decorators"] = self.decorators
+
+        for _attr in ["help_text", "extra"]:
+            attr = getattr(self, _attr, None)
+            if attr:
+                base[_attr] = attr
+        return base
+
+
+@dataclass
+class PKField(_BaseField):
+    type: str = "primary_key"
+
+
+@dataclass
+class CharField(_BaseField):
+    type: str = "text"
+
+
+@dataclass
+class DateField(_BaseField):
+    type: str = "date"
+
+
+@dataclass
+class DateRangeField(_BaseField):
+    type: str = "daterange"
+
+
+@dataclass
+class BooleanField(_BaseField):
+    type: str = "boolean"
+
+
+@dataclass
+class TextField(_BaseField):
+    type: str = "texteditor"
+
+
+@dataclass
+class EmojiRatingField(_BaseField):
+    type: str = "emojirating"
+
+
+@dataclass
+class FloatField(_BaseField):
+    type: str = "number"
+    precision: int = 2
+    delimiter: str = ","
+    decimal_mark: str = "."
+    percent: bool = False
+    display_mode: DisplayMode = None
+
+    def to_dict(self):
+        base = super().to_dict()
+        base.update(
+            {
+                "precision": self.precision,
+                "delimiter": self.delimiter,
+                "decimal_mark": self.decimal_mark,
+            }
+        )
+        if self.percent:
+            base["type"] = "percent"
+        if self.display_mode:
+            base["display_mode"] = self.display_mode.value
+        return base
+
+
+@dataclass
+class IntegerField(FloatField):
+    type: str = "number"
+    precision: int = 0
+
+
+@dataclass
+class YearField(IntegerField):
+    precision: int = 0
+    delimiter: str = ""
+    decimal_mark: str = "."
+
+    def __post_init__(self):
+        self.precision = 0
+        self.delimiter = ""
+        self.decimal_mark = "."
+
+
+@dataclass
+class ListField(_BaseField):
+    type: str = "list"
+
+
+@dataclass
+class JsonField(_BaseField):
+    type: str = "json"
+
+
+@dataclass
+class SparklineField(ListField):
+    type: str = "sparkline"
+
+    def to_dict(self):
+        rv = super().to_dict()
+        rv["sparkline_type"] = "column"
+        return rv
+
+
+@dataclass(unsafe_hash=True)
+class PandasFields:
+    fields: List[_BaseField]
+
+    def to_dict(self):
+        fields = defaultdict(dict)
+
+        for field in self.fields:
+            fields[field.key] = field.to_dict()
+
+        return fields

wbcore/pandas/filters.py

@@ -0,0 +1,113 @@
+"""
+Provide Filters for Pandas based views
+"""
+import operator
+from functools import reduce
+
+from rest_framework.filters import OrderingFilter, SearchFilter
+from wbcore import filters as wb_filters
+from wbcore.filters import DjangoFilterBackend
+from wbcore.filters.utils import check_required_filters
+
+
+class PandasDjangoFilterBackend(DjangoFilterBackend):
+    lookups_operator = {
+        "lte": operator.le,
+        "lt": operator.lt,
+        "gte": operator.ge,
+        "gt": operator.gt,
+        "exact": operator.eq,
+    }
+
+    def filter_queryset(self, request, queryset, view):
+        filterset_class = self.get_filterset_class(view, queryset)
+        if filterset_class and callable(filterset_class):
+            kwargs = self.get_filterset_kwargs(request, queryset, view)
+            filterset = filterset_class(**kwargs)
+            if filterset.is_valid():
+                check_required_filters(request, view, filterset)
+        return super().filter_queryset(request, queryset, view)
+
+    def filter_dataframe(self, request, df, view):
+        if not df.empty:
+            filterset_class = self.get_filterset_class(view, view.get_queryset())
+            if filterset_class and callable(filterset_class):
+                kwargs = self.get_filterset_kwargs(request, view.get_queryset(), view)
+                filter_terms = filterset_class(**kwargs).form.data
+                conditions = []
+                for filter_term, value in filter_terms.items():
+                    if _filter := getattr(filterset_class.Meta, "df_fields", {}).get(filter_term, None):
+                        # We support only number for now
+                        lookup_expr = getattr(_filter, "lookup_expr", "exact")
+                        if isinstance(_filter, wb_filters.NumberFilter):
+                            try:
+                                conditions.append(
+                                    self.lookups_operator[lookup_expr](df[_filter.field_name], float(value))
+                                )
+                            except ValueError:
+                                pass
+                if conditions:
+                    df = df[reduce(operator.and_, conditions)]
+        return df
+
+
+class PandasSearchFilter(SearchFilter):
+    def filter_queryset(self, request, queryset, view):
+        return queryset
+
+    def filter_dataframe(self, request, df, view):
+        search_fields = self.get_search_fields(view, request)
+        search_terms = self.get_search_terms(request)
+        search_fields = [field for field in search_fields if field in df.columns]
+
+        if not search_fields or not search_terms or df.empty:
+            return df
+
+        conditions = []
+
+        for search_term in search_terms:
+            queries = [
+                df[field].str.contains(search_term, na=False, case=False, regex=False) for field in search_fields
+            ]
+            conditions.append(reduce(operator.or_, queries))
+
+        df = df[reduce(operator.and_, conditions)]
+
+        return df
+
+
+class PandasOrderingFilter(OrderingFilter):
+    def get_ordering(self, request, df, view) -> tuple[list[str], list[bool]]:
+        """
+        Ordering is set by a comma delimited ?ordering=... query parameter.
+
+        The `ordering` query parameter can be overridden by setting
+        the `ordering_param` value on the OrderingFilter or by
+        specifying an `ORDERING_PARAM` value in the API settings.
+        """
+        params = request.query_params.get(self.ordering_param)
+        base_ordering_fields = view.get_ordering_fields()
+        potential_ordering_fields = params.split(",") if params else self.get_default_ordering(view)
+        ordering_res = []
+        ascending_res = []
+        if potential_ordering_fields:
+            for ordering_field in potential_ordering_fields:
+                ordering_field = ordering_field.strip()
+                ascending = True
+                if ordering_field.startswith("-"):
+                    ordering_field = ordering_field[1:]
+                    ascending = False
+
+                if ordering_field in df.columns and ordering_field in base_ordering_fields:
+                    ordering_res.append(ordering_field)
+                    ascending_res.append(ascending)
+        return ordering_res, ascending_res
+
+    def filter_queryset(self, request, queryset, view):
+        return queryset
+
+    def filter_dataframe(self, request, df, view):
+        ordering_by, ascending_list = self.get_ordering(request, df, view)
+        if ordering_by and ascending_list:
+            df.sort_values(by=list(ordering_by), ascending=list(ascending_list), inplace=True)
+        return df

wbcore/pandas/filterset.py

@@ -0,0 +1,26 @@
+from django.core.exceptions import FieldError
+from django.db import models
+
+
+class PandasFilterSetMixin:
+    def filter_queryset(self, queryset):
+        """
+        Filter the queryset with the underlying form's `cleaned_data`. You must
+        call `is_valid()` or `errors` before calling this method.
+
+        This method should be overridden if additional filtering needs to be
+        applied to the queryset before it is cached.
+        """
+        for name, value in self.form.cleaned_data.items():
+            try:
+                queryset = self.filters[name].filter(queryset, value)
+            except FieldError:
+                pass
+            assert isinstance(
+                queryset, models.QuerySet
+            ), "Expected '%s.%s' to return a QuerySet, but got a %s instead." % (
+                type(self).__name__,
+                name,
+                type(queryset).__name__,
+            )
+        return queryset

wbcore/pandas/metadata.py

@@ -0,0 +1,14 @@
+from slugify import slugify
+from wbcore.metadata.metadata import WBCoreMetadata
+
+
+class PandasMetadata(WBCoreMetadata):
+    def determine_metadata(self, request, view):
+        metadata = super().determine_metadata(request, view)
+        metadata["identifier"] += f"-{slugify(view.__class__.__name__)}"
+        metadata["fields"] = view.get_pandas_fields(request).to_dict()
+        # if ordering fields is not included in the metadata (e.g. dev didn't explicitly specify a list), then we inject the pandas fields by default
+        # it is safe to do that because for pandas table, the ordering happens on AgGrid
+        if not metadata["ordering_fields"]:
+            metadata["ordering_fields"] = {key: key for key in metadata["fields"].keys()}
+        return metadata

wbcore/pandas/utils.py

@@ -0,0 +1,129 @@
+import operator
+from functools import reduce
+from typing import Any, Iterable
+
+from pandas import DataFrame
+
+OverwriteRule = dict[str, dict[str, str] | dict[str, Any]]
+
+
+def rule(key: str, operator: str, value: str) -> dict[str, str]:
+    return {
+        "key": key,
+        "operator": operator,
+        "value": value,
+    }
+
+
+def overwrite(**overwrites) -> dict[str, Any]:
+    return {**overwrites}
+
+
+def overwrite_rule(key: str, operator: str, value: str, **overwrites) -> OverwriteRule:
+    return {
+        "rule": rule(key, operator, value),
+        "overwrite": overwrite(**overwrites),
+    }
+
+
+def overwrite_row(fields: Iterable[OverwriteRule] | None = None) -> dict[str, list[OverwriteRule]]:
+    overwrites_dict = {}
+
+    if fields:
+        overwrites_dict["fields"] = [*fields]
+
+    return overwrites_dict
+
+
+def overwrite_row_df(df: DataFrame, row_cond, fields: Iterable[OverwriteRule]):
+    df.loc[row_cond, "_overwrites"] = [{"fields": fields}]
+
+
+def override_number_to_percent(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    decorators=[{"position": "right", "value": "%"}],
+                    precision=1,
+                    display_mode="decimal",
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_with_currency(df: DataFrame, currency: str, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    decorators=[{"position": "left", "value": currency}],
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_to_x(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    decorators=[{"position": "right", "value": "x"}],
+                    precision=1,
+                    display_mode="decimal",
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_precision(df: DataFrame, precision=2, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    precision=precision,
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_to_decimal(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    precision=0,
+                    displayMode="decimal",
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()

wbcore/pandas/views.py

@@ -0,0 +1,157 @@
+import json
+from functools import cached_property
+
+import numpy as np
+import pandas as pd
+from django.core.cache import cache
+from django.db.models import QuerySet
+from rest_framework.mixins import ListModelMixin
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from rest_framework.viewsets import GenericViewSet
+from wbcore.metadata.mixins import WBCoreMetadataConfigViewMixin
+from wbcore.pandas import fields
+from wbcore.utils.renderers import BrowsableAPIRendererWithoutForms
+from wbcore.viewsets.encoders import PandasDataCustomRenderer
+from wbcore.viewsets.mixins import DocumentationMixin, MessageMixin
+
+from ..cache.mixins import CacheMixin
+from .filters import PandasDjangoFilterBackend, PandasOrderingFilter, PandasSearchFilter
+from .metadata import PandasMetadata
+
+
+class PandasMixin(CacheMixin):
+    filter_backends = [
+        PandasOrderingFilter,
+        PandasSearchFilter,
+        PandasDjangoFilterBackend,
+    ]
+
+    metadata_class = PandasMetadata
+    pagination_class = None
+    filterset_fields = {}
+    search_fields = []
+    ordering_fields = []
+
+    renderer_classes = [PandasDataCustomRenderer, BrowsableAPIRendererWithoutForms]
+
+    # CACHE FRAMEWORK METHODS
+    @cached_property
+    def df(self) -> pd.DataFrame:
+        if not hasattr(self, "_df"):
+            setattr(self, "_df", self._get_dataframe())
+        return getattr(self, "_df")
+
+    # BASIC DATAFRAME GENERATION FRAMEWORK METHODS
+    def filter_queryset(self, queryset: QuerySet) -> QuerySet:
+        for backend in list(self.filter_backends):
+            queryset = backend().filter_queryset(self.request, queryset, self)
+        return queryset
+
+    def filter_dataframe(self, df, **kwargs):
+        for backend in list(self.filter_backends):
+            if hasattr(backend, "filter_dataframe"):
+                df = backend().filter_dataframe(self.request, df, self)
+        return df
+
+    def _sanitize_dataframe(self, df):
+        pd_fields = self.get_pandas_fields(self.request)
+        # Ensure the returned data satisfy the primitive field type
+        for field in pd_fields.fields:
+            if field.key in df.columns:
+                if isinstance(field, fields.FloatField):
+                    df[field.key] = df[field.key].astype("float", errors="ignore")
+                if isinstance(field, fields.CharField) or isinstance(field, fields.TextField):
+                    df[field.key] = df[field.key].astype("string", errors="ignore")
+                if isinstance(field, fields.IntegerField):
+                    df[field.key] = df[field.key].astype("int", errors="ignore")
+                elif isinstance(field, fields.DateField):
+                    df[field.key] = pd.to_datetime(df[field.key])
+        with pd.option_context("future.no_silent_downcasting", True):
+            return (
+                df.drop(columns=df.columns.difference(pd_fields.to_dict().keys()))
+                .infer_objects(copy=False)
+                .replace(
+                    {
+                        np.nan: None,
+                        np.inf: "Infinity",
+                        -np.inf: "-Infinity",
+                    }
+                )
+            )
+
+    def get_queryset(self):
+        assert hasattr(self, "queryset"), "Either specify a queryset or implement the get_queryset method."
+        return self.queryset
+
+    def get_dataframe(self, request, queryset, **kwargs):
+        assert self.get_pandas_fields(request), "No pandas_fields specified"
+        return pd.DataFrame(queryset.values(*self.get_pandas_fields(request).to_dict().keys()))
+
+    def manipulate_dataframe(self, df):
+        return df
+
+    def _get_dataframe(self, **kwargs):
+        df = pd.DataFrame()
+        queryset = self.get_queryset()
+        if queryset.exists():
+            if self.cache_enabled:
+                df = self._get_cached_res()
+            if df.empty:
+                self.extra_cache_kwargs = {}
+                queryset = self.filter_queryset(queryset)
+                df = self.manipulate_dataframe(self.get_dataframe(self.request, queryset, **kwargs))
+                # we loop over the other cached kwargs and set them as class attributes
+                for k, v in self.extra_cache_kwargs.items():
+                    setattr(self, k, v)
+                if self.cache_enabled:
+                    cached_res = self.serialize_cache_results(df)
+                    cache.set(self._get_cache_key(), cached_res, timeout=self._get_cache_timeout())
+        else:
+            df = pd.DataFrame(
+                columns=[field.key for field in self.get_pandas_fields(self.request).fields]
+            )  # if queryset is empty, we make sure the returning df contains all the columns to avoid keyerrors exception
+        setattr(self, "_df", df)
+        df = self.filter_dataframe(df, **kwargs)
+        return df
+
+    def get_aggregates(self, request, df):
+        return {}
+
+    def get_pandas_fields(self, request):
+        return getattr(self, "pandas_fields", None)
+
+
+class PandasAPIView(MessageMixin, PandasMixin, WBCoreMetadataConfigViewMixin, DocumentationMixin, APIView):
+    """
+    IMPORTANT: This view will be deprecated. Please us PandasAPIViewSet and register it though the router
+    """
+
+    def get(self, request, **kwargs):
+        self.request = request
+        df = self._sanitize_dataframe(self._get_dataframe(**kwargs))
+        aggregates = self.get_aggregates(request, df) if not df.empty else {}
+        results = json.loads(
+            df.to_json(orient="records", force_ascii=False)
+        )  # We do that to benefiate from the PandasJSONEncoder
+        data = {"results": results, "aggregates": aggregates}
+        data["messages"] = list(self._get_messages(request))
+        return Response()
+
+
+class PandasAPIViewSet(
+    MessageMixin, PandasMixin, WBCoreMetadataConfigViewMixin, DocumentationMixin, ListModelMixin, GenericViewSet
+):
+    def list(self, request, *args, **kwargs):
+        self.request = request
+        df = self._sanitize_dataframe(self._get_dataframe(**kwargs))
+        aggregates = self.get_aggregates(request, df) if not df.empty else {}
+        results = json.loads(
+            df.to_json(orient="records", force_ascii=False)
+        )  # We do that to benefiate from the PandasJSONEncoder
+        data = {"results": results, "aggregates": aggregates}
+        data["messages"] = list(self._get_messages(request))
+        return Response(data)
+
+    def get_serializer(self, *args, **kwargs):
+        return None