wbcore 1.59.9__py2.py3-none-any.whl

wbcore/contrib/pandas/tests/test_filters/test_pandas.py

@@ -0,0 +1,115 @@
+from collections import namedtuple
+
+import pandas as pd
+import pytest
+from rest_framework.test import APIRequestFactory
+
+from wbcore.contrib.pandas.filters import PandasOrderingFilter
+from wbcore.viewsets import ViewSet
+
+
+class TestPandasOrderingFilter:
+    @pytest.fixture
+    def ordering_filter(self):
+        return PandasOrderingFilter()
+
+    @pytest.fixture
+    def view(self, base_ordering_fields, ordering_fields):
+        class BaseView(ViewSet):
+            def get_ordering_fields(self):
+                return base_ordering_fields
+
+            ordering = ordering_fields
+
+        return BaseView()
+
+    @pytest.fixture
+    def df(self, ordering_fields):
+        return pd.DataFrame(
+            [[1] * len(ordering_fields), [2] * len(ordering_fields)],
+            columns=map(lambda x: x.replace("-", ""), ordering_fields),
+        )
+
+    @pytest.fixture
+    def queryset(self):
+        # dummy iterable to assert that the filter queryset method doesn't touch the queryset
+        return [namedtuple("Point", "x y")]
+
+    @pytest.fixture
+    def filter_request(self, ordering_fields):
+        factory = APIRequestFactory()
+        request = factory.get(f'/?ordering={",".join(ordering_fields)}')
+        request.query_params = request.GET
+        return request
+
+    @pytest.mark.parametrize(
+        "base_ordering_fields, ordering_fields, expected_ordering_fields, expected_ascending",
+        [
+            (["a", "b"], ["a"], ["a"], [True]),
+            (["a", "b"], ["c"], [], []),
+            (["a", "b"], ["-a"], ["a"], [False]),
+            (["a", "b"], ["-a", "b"], ["a", "b"], [False, True]),
+        ],
+    )
+    def test_get_ordering(
+        self,
+        base_ordering_fields,
+        ordering_fields,
+        expected_ordering_fields,
+        expected_ascending,
+        ordering_filter,
+        filter_request,
+        df,
+        view,
+    ):
+        res_ordering_fields, res_ascending = ordering_filter.get_ordering(filter_request, df, view)
+        assert list(res_ordering_fields) == expected_ordering_fields
+        assert list(res_ascending) == expected_ascending
+
+    @pytest.mark.parametrize(
+        "base_ordering_fields, ordering_fields, expected_ordering_fields, expected_ascending",
+        [
+            (["a", "b"], ["a"], ["a"], [True]),
+            (["a", "b"], ["c"], [], []),
+            (["a", "b"], ["-a"], ["a"], [False]),
+            (["a", "b"], ["-a", "b"], ["a", "b"], [False, True]),
+        ],
+    )
+    def test_get_ordering_without_query_params(
+        self,
+        base_ordering_fields,
+        ordering_fields,
+        expected_ordering_fields,
+        ordering_filter,
+        expected_ascending,
+        df,
+        rf,
+        view,
+    ):
+        rf.query_params = {}
+        res_ordering_fields, res_ascending = ordering_filter.get_ordering(rf, df, view)
+        assert list(res_ordering_fields) == expected_ordering_fields
+        assert list(res_ascending) == expected_ascending
+
+    @pytest.mark.parametrize(
+        "base_ordering_fields, ordering_fields",
+        [
+            (["a", "b"], ["a"]),
+        ],
+    )
+    def test_filter_queryset(
+        self, base_ordering_fields, ordering_fields, ordering_filter, filter_request, queryset, view
+    ):
+        assert ordering_filter.filter_queryset(filter_request, queryset, view) == queryset
+
+    @pytest.mark.parametrize(
+        "base_ordering_fields, ordering_fields",
+        [
+            (["a", "b"], ["-a"]),
+        ],
+    )
+    def test_filter_dataframe(self, base_ordering_fields, ordering_fields, ordering_filter, filter_request, df, view):
+        assert list(ordering_filter.filter_dataframe(filter_request, df, view)["a"].values) == [
+            2,
+            1,
+        ]  # we just check that ordering is applied

wbcore/contrib/pandas/tests/test_views.py

@@ -0,0 +1,22 @@
+from wbcore.test.mixins import TestViewSet
+from wbcore.test.utils import get_or_create_superuser
+
+
+class TestPandasView(TestViewSet):
+    def test_get_request(self):
+        if self.factory is None:
+            self._raise_warning_factory("test_get_request")
+        else:
+            obj, request, kwargs, _ = self._get_mixins_data()
+            request.user = get_or_create_superuser()
+            request.query_params = dict()
+            request.parser_context = {}
+            self.mvs.kwargs = kwargs
+            response = self.mvs.as_view({"get": "list"})(request, **kwargs)
+            assert response.status_code == 200
+            assert "results" in response.data
+            assert "aggregates" in response.data
+            assert "messages" in response.data
+
+    def execute_test(self):
+        self.test_get_request()

wbcore/contrib/pandas/utils.py

@@ -0,0 +1,170 @@
+import operator
+from functools import reduce
+from typing import Any, Iterable
+
+from pandas import DataFrame, Series
+
+OverwriteRule = dict[str, dict[str, str] | dict[str, Any]]
+
+
+def rule(key: str, operator: str, value: str) -> dict[str, str]:
+    return {
+        "key": key,
+        "operator": operator,
+        "value": value,
+    }
+
+
+def overwrite(**overwrites) -> dict[str, Any]:
+    return {**overwrites}
+
+
+def overwrite_rule(key: str, operator: str, value: str, **overwrites) -> OverwriteRule:
+    return {
+        "rule": rule(key, operator, value),
+        "overwrite": overwrite(**overwrites),
+    }
+
+
+def overwrite_row(fields: Iterable[OverwriteRule] | None = None) -> dict[str, list[OverwriteRule]]:
+    overwrites_dict = {}
+
+    if fields:
+        overwrites_dict["fields"] = [*fields]
+
+    return overwrites_dict
+
+
+def overwrite_row_df(df: DataFrame, row_cond, fields: Iterable[OverwriteRule]):
+    df.loc[row_cond, "_overwrites"] = [{"fields": fields}]
+
+
+def override_number_to_percent(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    decorators=[{"position": "right", "value": "%"}],
+                    precision=1,
+                    display_mode="decimal",
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_with_currency(df: DataFrame, currency: str, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    decorators=[{"position": "left", "value": currency}],
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_to_x(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    decorators=[{"position": "right", "value": "x"}],
+                    precision=1,
+                    display_mode="decimal",
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_precision(df: DataFrame, precision=2, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    precision=precision,
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_to_decimal(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    precision=0,
+                    displayMode="decimal",
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def override_number_to_integer_without_decorations(df: DataFrame, *conditions, filter_operator=operator.or_):
+    reduced_conditions = reduce(filter_operator, conditions)
+
+    df.loc[reduced_conditions, "_overwrites"] = [
+        {
+            "fields": [
+                overwrite_rule(
+                    key="type",
+                    operator="=",
+                    value="number",
+                    precision=0,
+                    displayMode="decimal",
+                    disable_formatting=True,
+                )
+            ]
+        }
+    ] * reduced_conditions.sum()
+
+
+def pct_change_with_negative_values(df: DataFrame, field: str) -> DataFrame:
+    """Calculate percentage change with negative values"""
+    return df[field].diff() / df[field].abs().shift()
+
+
+def get_empty_series(df: DataFrame) -> Series:
+    """Return empty Series with the same index as the DataFrame"""
+    return Series(dtype="float64", index=df.index)
+
+
+def sanitize_field(df: DataFrame, field: str):
+    """Ensure field exists in DataFrame"""
+    empty_series = get_empty_series(df)
+    df[field] = df.get(field, empty_series)
+
+
+def sanitize_fields(df: DataFrame, fields: Iterable[str]):
+    """Ensure fields exist in DataFrame"""
+    for field in fields:
+        sanitize_field(df, field)

wbcore/contrib/pandas/views.py

@@ -0,0 +1,158 @@
+import json
+from contextlib import suppress
+from functools import cached_property
+
+import numpy as np
+import pandas as pd
+from django.core.cache import cache
+from django.db.models import QuerySet
+from rest_framework.mixins import ListModelMixin
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from rest_framework.viewsets import GenericViewSet
+
+from wbcore.cache.mixins import CacheMixin
+from wbcore.contrib.pandas import fields
+from wbcore.contrib.pandas.filters import PandasDjangoFilterBackend, PandasOrderingFilter, PandasSearchFilter
+from wbcore.contrib.pandas.metadata import PandasMetadata
+from wbcore.metadata.mixins import WBCoreMetadataConfigViewMixin
+from wbcore.utils.renderers import BrowsableAPIRendererWithoutForms
+from wbcore.viewsets.encoders import PandasDataCustomRenderer
+from wbcore.viewsets.mixins import DocumentationMixin, MessageMixin
+
+
+class PandasMixin(CacheMixin):
+    filter_backends = [
+        PandasOrderingFilter,
+        PandasSearchFilter,
+        PandasDjangoFilterBackend,
+    ]
+
+    metadata_class = PandasMetadata
+    pagination_class = None
+    filterset_fields = {}
+    search_fields = []
+    ordering_fields = []
+
+    renderer_classes = [PandasDataCustomRenderer, BrowsableAPIRendererWithoutForms]
+
+    # CACHE FRAMEWORK METHODS
+    @cached_property
+    def df(self) -> pd.DataFrame:
+        if not hasattr(self, "_df"):
+            self._df = self._get_dataframe()
+        return self._df
+
+    # BASIC DATAFRAME GENERATION FRAMEWORK METHODS
+    def filter_queryset(self, queryset: QuerySet) -> QuerySet:
+        for backend in list(self.filter_backends):
+            queryset = backend().filter_queryset(self.request, queryset, self)
+        return queryset
+
+    def filter_dataframe(self, df, **kwargs):
+        for backend in list(self.filter_backends):
+            if hasattr(backend, "filter_dataframe"):
+                df = backend().filter_dataframe(self.request, df, self)
+        return df
+
+    def _sanitize_dataframe(self, df):
+        pd_fields = self.get_pandas_fields(self.request)
+        # Ensure the returned data satisfy the primitive field type
+        for field in pd_fields.fields:
+            if isinstance(field, fields.PKField) and field.key not in df.columns:
+                with suppress(
+                    IndexError
+                ):  # if df is empty with an empty multi-index, the pandas subrountine fails with an IndexError
+                    df = df.reset_index(names=field.key)
+            if field.key in df.columns:
+                df.loc[:, field.key] = field.to_representation(df[field.key])
+        with pd.option_context("future.no_silent_downcasting", True):
+            return (
+                df.drop(columns=df.columns.difference(pd_fields.to_dict().keys()))
+                .infer_objects(copy=False)
+                .replace(
+                    {
+                        np.nan: None,
+                        np.inf: "Infinity",
+                        -np.inf: "-Infinity",
+                    }
+                )
+            )
+
+    def get_queryset(self):
+        if not hasattr(self, "queryset"):
+            raise AssertionError("Either specify a queryset or implement the get_queryset method.")
+        return self.queryset
+
+    def get_dataframe(self, request, queryset, **kwargs):
+        if not self.get_pandas_fields(request):
+            raise AssertionError("No pandas_fields specified")
+        return pd.DataFrame(queryset.values(*self.get_pandas_fields(request).to_dict().keys()))
+
+    def manipulate_dataframe(self, df):
+        return df
+
+    def _get_dataframe(self, **kwargs):
+        df = pd.DataFrame()
+        queryset = self.get_queryset()
+        if queryset.exists():
+            if self.cache_enabled:
+                df = self._get_cached_res()
+            if df.empty:
+                self.extra_cache_kwargs = {}
+                queryset = self.filter_queryset(queryset)
+                df = self.manipulate_dataframe(self.get_dataframe(self.request, queryset, **kwargs))
+                # we loop over the other cached kwargs and set them as class attributes
+                for k, v in self.extra_cache_kwargs.items():
+                    setattr(self, k, v)
+                if self.cache_enabled:
+                    cached_res = self.serialize_cache_results(df)
+                    cache.set(self._get_cache_key(), cached_res, timeout=self._get_cache_timeout())
+        else:
+            df = pd.DataFrame(
+                columns=[field.key for field in self.get_pandas_fields(self.request).fields]
+            )  # if queryset is empty, we make sure the returning df contains all the columns to avoid keyerrors exception
+        self._df = df
+        df = self.filter_dataframe(df, **kwargs)
+        return df
+
+    def get_aggregates(self, request, df):
+        return {}
+
+    def get_pandas_fields(self, request):
+        return getattr(self, "pandas_fields", None)
+
+
+class PandasAPIView(MessageMixin, PandasMixin, WBCoreMetadataConfigViewMixin, DocumentationMixin, APIView):
+    """
+    IMPORTANT: This view will be deprecated. Please us PandasAPIViewSet and register it though the router
+    """
+
+    def get(self, request, **kwargs):
+        self.request = request
+        df = self._sanitize_dataframe(self._get_dataframe(**kwargs))
+        aggregates = self.get_aggregates(request, df) if not df.empty else {}
+        results = json.loads(
+            df.to_json(orient="records", force_ascii=False)
+        )  # We do that to benefiate from the PandasJSONEncoder
+        data = {"results": results, "aggregates": aggregates}
+        data["messages"] = list(self._get_messages(request))
+        return Response()
+
+
+class PandasAPIViewSet(
+    MessageMixin, PandasMixin, WBCoreMetadataConfigViewMixin, DocumentationMixin, ListModelMixin, GenericViewSet
+):
+    def list(self, request, *args, **kwargs):
+        self.request = request
+        df = self._sanitize_dataframe(self._get_dataframe(**kwargs))
+        aggregates = self.get_aggregates(request, df) if not df.empty else {}
+        results = json.loads(
+            df.to_json(orient="records", force_ascii=False)
+        )  # We do that to benefiate from the PandasJSONEncoder
+        data = {"results": results, "aggregates": aggregates}
+        data["messages"] = list(self._get_messages(request))
+        return Response(data)
+
+    def get_serializer(self, *args, **kwargs):
+        return None

wbcore/contrib/tags/admin.py

@@ -0,0 +1,17 @@
+from django.contrib import admin
+
+from .models import Tag, TagGroup
+
+
+@admin.register(Tag)
+class TagModelAdmin(admin.ModelAdmin):
+    list_display = ("title", "slug", "color", "content_type")
+    search_fields = ("title", "slug")
+    readonly_fields = ("managed",)
+
+
+@admin.register(TagGroup)
+class TagGroupModelAdmin(admin.ModelAdmin):
+    list_display = ("title",)
+    search_fields = ("title",)
+    readonly_fields = ("managed",)

wbcore/contrib/tags/apps.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class TagsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "wbcore.contrib.tags"
+
+    def ready(self):
+        import wbcore.contrib.tags.signals  # noqa: F401

wbcore/contrib/tags/factories.py

@@ -0,0 +1,28 @@
+import factory
+
+from .models import Tag, TagGroup
+
+
+class TagFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Tag
+        skip_postgeneration_save = True
+
+    title = factory.Faker("name")
+    description = factory.Faker("paragraph")
+
+    @factory.post_generation
+    def groups(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for group in extracted:
+                self.groups.add(group)
+
+
+class TagGroupFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = TagGroup
+
+    title = factory.Faker("name")

wbcore/contrib/tags/filters.py

@@ -0,0 +1,43 @@
+from itertools import chain
+
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import Q
+
+from wbcore import filters
+from wbcore.content_type.utils import (
+    get_ancestors_content_type,
+    get_view_content_type_id,
+)
+
+from .models import Tag
+
+
+class TagFilterMixin(filters.FilterSet):
+    tags = filters.ModelMultipleChoiceFilter(
+        endpoint=Tag.get_representation_endpoint(),
+        value_key=Tag.get_representation_value_key(),
+        label_key=Tag.get_representation_label_key(),
+        label="Tags",
+        filter_params=lambda request, view: {"content_type": get_view_content_type_id(view)},
+        queryset=Tag.objects.all(),
+    )
+
+
+class TagFilterSet(filters.FilterSet):
+    content_type = filters.ModelMultipleChoiceFilter(
+        endpoint="wbcore:contenttyperepresentation-list",
+        value_key="id",
+        label_key="{{app_label}} | {{model}}",
+        queryset=ContentType.objects.all(),
+        method="filter_content_type",
+    )
+
+    def filter_content_type(self, queryset, label, value):
+        if value:
+            content_types = list(chain(*[get_ancestors_content_type(content_type) for content_type in value]))
+            return queryset.filter(Q(content_type__isnull=True) | Q(content_type__in=content_types))
+        return queryset
+
+    class Meta:
+        model = Tag
+        fields = {"title": ["icontains", "exact"], "description": ["icontains"], "groups": ["exact"]}

wbcore/contrib/tags/migrations/0001_initial.py

@@ -0,0 +1,62 @@
+# Generated by Django 4.1.8 on 2023-04-06 14:14
+
+import django.db.models.deletion
+import wbcore.models.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="TagGroup",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+                ),
+                ("managed", models.BooleanField(default=False)),
+                ("title", models.CharField(max_length=255, unique=True)),
+            ],
+            options={
+                "verbose_name": "Tag Group",
+                "verbose_name_plural": "Tag Groups",
+            },
+        ),
+        migrations.CreateModel(
+            name="Tag",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+                ),
+                ("computed_str", models.CharField(blank=True, max_length=512, null=True, verbose_name="Name")),
+                ("managed", models.BooleanField(default=False)),
+                ("title", models.CharField(max_length=255)),
+                ("slug", models.CharField(blank=True, max_length=255, null=True)),
+                ("color", wbcore.contrib.color.fields.ColorField(default="#D3D3D3")),
+                ("description", models.TextField(blank=True, default="")),
+                (
+                    "content_type",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="associated_tags",
+                        to="contenttypes.contenttype",
+                    ),
+                ),
+                ("groups", models.ManyToManyField(blank=True, to="tags.taggroup")),
+            ],
+            options={
+                "verbose_name": "Tag",
+                "verbose_name_plural": "Tags",
+                "unique_together": {("slug", "content_type")},
+            },
+        ),
+    ]

wbcore/contrib/tags/models/__init__.py

@@ -0,0 +1,2 @@
+from .mixins import TagModelMixin
+from .tags import Tag, TagGroup

wbcore/contrib/tags/models/mixins.py

@@ -0,0 +1,27 @@
+from django.db import models
+
+
+class TagModelMixin(models.Model):
+    tag_detail_endpoint = models.CharField(max_length=255, null=True, blank=True)
+    tag_representation = models.CharField(max_length=512, null=True, blank=True)
+    tags = models.ManyToManyField(
+        to="tags.Tag",
+        related_name="%(app_label)s_%(class)s_items",
+        blank=True,
+    )
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        self.__class__.objects.filter(id=self.id).update(
+            tag_detail_endpoint=self.get_tag_detail_endpoint(),
+            tag_representation=self.get_tag_representation(),
+        )
+
+    def get_tag_detail_endpoint(self):
+        raise NotImplementedError("When using Tags, you must implement get_tag_detail_endpoint")
+
+    def get_tag_representation(self):
+        raise NotImplementedError("When using Tags, you must implement get_tag_representation")
+
+    class Meta:
+        abstract = True