wbcore 1.59.9__py2.py3-none-any.whl

wbcore/contrib/io/release_notes/1_0_0.md

@@ -0,0 +1,13 @@
+Date: 2023-04-26
+Module: I/O
+Summary: Initial Release Note
+Version: 1.0.0
+
+## New Features
+- Bootstrap module versioning
+
+## Enhancements
+- Allow for an automatic view resource created from the view serializer for export
+
+## Bugfixes
+./.

wbcore/contrib/io/resources.py

@@ -0,0 +1,176 @@
+import re
+from contextlib import suppress
+from inspect import isfunction
+
+import tablib
+from django.template import Context, Template
+from import_export import resources
+
+from wbcore.serializers import ListSerializer
+from wbcore.serializers.fields.related import ListSerializer as RelatedListSerializer
+from wbcore.utils.importlib import import_from_dotted_path
+
+
+class ExportResourceMixin:
+    DUMMY_FIELD_MAP = {}
+
+    def get_export_dummy_data(self) -> list[str]:
+        """
+        Returns a list in the same order as the headers with the fixture data that correspond to the header
+        """
+        res = []
+        for header in self.get_export_headers():
+            if value := self.DUMMY_FIELD_MAP.get(header):
+                if callable(value):
+                    value = value()
+                res.append(value)
+            else:
+                res.append("")
+        return res
+
+    def export(self, queryset=None, *args, **kwargs):
+        """
+        override the functionality from import-export in order to allow empty queryset, which correspond to template export
+        """
+        if queryset is None:
+            headers = self.get_export_headers()
+            data = self.get_export_dummy_data()
+            return tablib.Dataset(data, headers=headers)
+        return super().export(*args, queryset=queryset, **kwargs)
+
+
+class FilterModelResource(ExportResourceMixin, resources.ModelResource):
+    """
+    This class inherit from modelResource and provides few functionality out of the box:
+    - Allow to define a template export and a map of field to corresponding dummy data to fill in the template with
+    - Can instantiate the resource with a list of keyword argument that will be injected as field value before instance saving
+    - Filter the queryset with the specified keyword argument
+    """
+
+    def __init__(self, filter_kwargs=None, request=None, **kwargs):
+        super().__init__(**kwargs)
+        if not filter_kwargs:
+            filter_kwargs = dict()
+        self.filter_kwargs = filter_kwargs
+        self.request = request
+
+    def before_save_instance(self, instance, *args):
+        """
+        For all specified `filter_kwargs`, set the attribute as field
+        """
+        for k, v in self.filter_kwargs.items():
+            if not getattr(self, k, None):
+                setattr(instance, k, v)
+
+    def get_queryset(self):
+        """
+        Filter the queryset with the given keyword arguments
+        """
+        return self._meta.model.objects.filter(**self.filter_kwargs)
+
+
+class ViewResource(ExportResourceMixin, resources.Resource):
+    DEFAULT_PAGINATION_LIMIT: int = 200
+
+    def __init__(
+        self,
+        columns_map: dict[str, str] | None = None,
+        serializer_class=None,
+        request=None,
+        serializer_class_path: str | None = None,
+        **kwargs,
+    ):
+        serializer_class_method_args = kwargs.pop("serializer_class_method_args", list())
+        super().__init__(**kwargs)
+        # in that case, serializer_class is lazy loaded,
+        if serializer_class_path:
+            serializer_class = import_from_dotted_path(serializer_class_path)
+            if isfunction(serializer_class):
+                serializer_class = serializer_class(*serializer_class_method_args)
+        if not serializer_class:
+            raise ValueError("ViewResource needs a serializer_class")
+        self.serializer_class = serializer_class
+        if not columns_map:
+            columns_map = dict()
+        self.columns_map = columns_map
+        self.request = request
+
+    def iter_queryset(self, queryset):
+        serializer = self.serializer_class(queryset, many=True)
+        return super().iter_queryset(serializer.data)
+
+    def get_export_fields(self, selected_fields=None):
+        res = []
+        # Get default fields from the list display
+        if "id" not in self.columns_map:
+            res.append(
+                resources.Field(
+                    attribute="id",
+                    column_name="id",
+                    readonly=True,
+                )
+            )
+        if self.columns_map:
+            for key, label in self.columns_map.items():
+                res.append(
+                    resources.Field(
+                        attribute=key,
+                        column_name=label,
+                        readonly=True,
+                    )
+                )
+        else:
+            for field_name, field in self.serializer_class().fields.items():
+                res.append(
+                    resources.Field(
+                        attribute=field_name,
+                        column_name=field.label,
+                        readonly=True,
+                    )
+                )
+        return res
+
+    def export_resource(self, obj, selected_fields=None, **kwargs):
+        if not selected_fields:
+            selected_fields = self.get_export_fields()
+
+        def _parse_representation(representation, template):
+            template = re.sub(r"({{\W*}})", " ", template)
+            t = Template(template).render(Context(representation))
+            return t
+
+        validated_data = []
+        obj = dict(obj)
+        for field in selected_fields:
+            value = None
+            if field.attribute in obj:
+                value = obj[field.attribute]
+                # Try to get the representation of this field derivated from the serializer field label key
+                with suppress(KeyError):
+                    representation_field_name = f"_{field.attribute}"
+                    representation_field = self.serializer_class._declared_fields[representation_field_name]
+                    if isinstance(representation_field, ListSerializer) or isinstance(
+                        representation_field, RelatedListSerializer
+                    ):
+                        label_key = getattr(representation_field.child, "label_key", None)
+                    else:
+                        label_key = getattr(representation_field, "label_key", None)
+                    if representation_field := obj.get(representation_field_name):
+                        if isinstance(representation_field, list):
+                            value = [
+                                _parse_representation(representation, label_key)
+                                for representation in representation_field
+                            ]
+                        else:
+                            value = _parse_representation(representation_field, label_key)
+            validated_data.append(value)
+        return validated_data
+
+    @classmethod
+    def get_columns_map(cls, view):
+        columns_map = {"id": "id"}
+        # Get default fields from the list display
+        if list_display := view.display_config_class(view, view.request, instance=None).get_list_display():
+            for key, label in list_display.flatten_fields:
+                columns_map[key] = label
+        return columns_map

wbcore/contrib/io/serializers.py

@@ -0,0 +1,137 @@
+import re
+
+from django.db import transaction
+from rest_framework.serializers import ValidationError
+
+from wbcore import serializers as wb_serializers
+from wbcore.serializers import ModelSerializer, RepresentationSerializer
+
+from .models import (
+    DataBackend,
+    ImportSource,
+    ParserHandler,
+    Source,
+    import_data_as_task,
+)
+
+
+class ImportSourceRepresentationSerializer(RepresentationSerializer):
+    class Meta:
+        model = ImportSource
+        fields = ("id", "file")
+
+
+class SourceRepresentationSerializer(RepresentationSerializer):
+    def __init__(self, *args, label_key="{{id}} - {{title}}", **kwargs):
+        self.allowed_sources = kwargs.pop("allowed_sources", Source.objects.none())
+        super().__init__(*args, label_key=label_key, **kwargs)
+
+    def get_filter_params(self, request):
+        filter_params = {}
+        if self.allowed_sources.exists():
+            filter_params["id__in"] = ",".join(map(str, self.allowed_sources.values_list("id", flat=True)))
+        return filter_params
+
+    class Meta:
+        model = Source
+        fields = ("id", "title")
+
+
+class ParserHandlerRepresentationSerializer(RepresentationSerializer):
+    def __init__(self, *args, **kwargs):
+        self.allowed_handler = kwargs.pop("allowed_handler", None)
+        super().__init__(*args, **kwargs)
+
+    def get_filter_params(self, request):
+        filter_params = {}
+        if self.allowed_handler:
+            filter_params["handler__icontains"] = self.allowed_handler
+        return filter_params
+
+    class Meta:
+        model = ParserHandler
+        fields = ("id", "parser", "handler")
+
+
+class DataBackendRepresentationSerializer(RepresentationSerializer):
+    def __init__(self, *args, **kwargs):
+        self.allowed_backend_class_path = kwargs.pop("allowed_backend_class_path", ParserHandler.objects.none())
+        super().__init__(*args, **kwargs)
+
+    def get_filter_params(self, request):
+        filter_params = {}
+        if self.allowed_backend_class_path:
+            filter_params["backend_class_path__icontains"] = self.allowed_backend_class_path
+        return filter_params
+
+    class Meta:
+        model = DataBackend
+        fields = ("id", "title", "backend_class_path")
+
+
+class ParserHandlerModelSerializer(ModelSerializer):
+    class Meta:
+        model = ParserHandler
+        fields = ("id", "parser", "handler")
+
+
+class ImportSourceModelSerializer(ModelSerializer):
+    parser_handler = wb_serializers.CharField(required=False)
+
+    def validate(self, data):
+        if "creator" not in data and "request" in self.context:
+            data["creator"] = self.context["request"].user
+        if parser_handler := data.get("parser_handler"):
+            if match := re.search(r"([\w\.]+)::(\w+\.\w+)", parser_handler):
+                parser_lookup = dict(parser=match.group(1), handler=match.group(2))
+            else:
+                parser_lookup = dict(id=parser_handler)
+            try:
+                data["parser_handler"] = ParserHandler.objects.get(**parser_lookup)
+            except (ParserHandler.DoesNotExist, ValueError) as e:
+                raise ValidationError({"parser_handler": "Invalid parser handler"}) from e
+        return data
+
+    def create(self, validated_data):
+        if "parser_handler" not in validated_data and "source" in validated_data:
+            # if parser handler is not provided, we default it to the first source parser handler
+            validated_data["parser_handler"] = validated_data["source"].parser_handler.first()
+        instance = super().create(validated_data)
+        transaction.on_commit(import_data_as_task.s(instance.id).delay)
+        return instance
+
+    class Meta:
+        model = ImportSource
+        read_only_fields = ("id", "status")
+        fields = (
+            "id",
+            "status",
+            "origin",
+            "creator",
+            "file",
+            "save_data",
+            "parser_handler",
+            "source",
+        )
+
+
+class SourceModelSerializer(wb_serializers.ModelSerializer):
+    _parser_handler = ParserHandlerRepresentationSerializer(source="parser_handler", many=True)
+    _data_backend = DataBackendRepresentationSerializer(source="data_backend")
+    crontab = wb_serializers.StringRelatedField()
+
+    class Meta:
+        model = Source
+        fields = (
+            "title",
+            "uuid",
+            "parser_handler",
+            "_parser_handler",
+            "is_active",
+            "connection_parameters",
+            "import_parameters",
+            "data_backend",
+            "_data_backend",
+            "crontab",
+            "import_timedelta_interval",
+        )

wbcore/contrib/io/signals.py

@@ -0,0 +1,4 @@
+from django.db.models.signals import ModelSignal
+
+# Signal fired after a import source was imported
+post_import = ModelSignal(use_caching=False)

wbcore/contrib/io/tasks.py

@@ -0,0 +1,27 @@
+from datetime import date, timedelta
+
+from celery import shared_task
+from django.db import connection
+from django.utils import timezone
+from dynamic_preferences.registries import global_preferences_registry
+
+from wbcore.workers import Queue
+
+
+@shared_task(queue=Queue.BACKGROUND.value)
+def clean_up_import_source(today: date | None = None):
+    if not today:
+        today = timezone.now()
+    global_preferences = global_preferences_registry.manager()
+    retention_period = global_preferences["io__import_source_retention_period"]
+    retention_until = today - timedelta(days=retention_period)
+    with connection.cursor() as cursor:
+        cursor.execute(
+            """
+            UPDATE io_importsource
+            SET data = '{}'::jsonb,
+                log = ''
+            WHERE created < %s;
+        """,
+            (retention_until,),
+        )

wbcore/contrib/io/tests/conftest.py

@@ -0,0 +1,42 @@
+import pytest
+from pytest_factoryboy import register
+from wbcore.contrib.authentication.factories import SuperUserFactory, UserFactory
+from wbcore.contrib.directory.factories import PersonFactory
+from wbcore.tests.conftest import *  # noqa
+
+from ..factories import (
+    CrontabScheduleFactory,
+    DataBackendFactory,
+    ExportSourceFactory,
+    ImportCredentialFactory,
+    ImportModelFactory,
+    ImportSourceFactory,
+    ParserHandlerFactory,
+    ProviderFactory,
+    SourceFactory,
+)
+
+register(CrontabScheduleFactory)
+register(ImportSourceFactory)
+register(ImportCredentialFactory)
+register(SourceFactory)
+register(ParserHandlerFactory)
+register(ExportSourceFactory)
+register(ImportModelFactory)
+register(ProviderFactory)
+register(DataBackendFactory)
+register(UserFactory)
+register(SuperUserFactory, "superuser")
+register(PersonFactory)
+
+
+@pytest.fixture(
+    autouse=True, scope="session"
+)  # Might want to find a way to registered default conftest logic automatically
+def django_test_environment(django_test_environment):
+    from django.apps import apps
+
+    get_models = apps.get_models
+
+    for m in [m for m in get_models() if not m._meta.managed]:
+        m._meta.managed = True

wbcore/contrib/io/tests/test_backends.py

@@ -0,0 +1,134 @@
+from unittest.mock import patch
+
+import pytest
+from anymail.inbound import AnymailInboundMessage
+from anymail.signals import AnymailInboundEvent, EventType, inbound
+from django.utils import timezone
+from faker import Faker
+
+from ..models import Source
+from .test_models import get_byte_stream
+
+fake = Faker()
+
+
+def conn(sftpsrv):
+    """return a dictionary holding argument info for the sftp client"""
+    return {
+        "host": sftpsrv.host,
+        "port": sftpsrv.port,
+        "username": "user",
+        "password": "pw",
+        "default_path": "/home/test",
+    }
+
+
+# filesystem served by pytest-sftpserver plugin
+VFS = {
+    "home": {
+        "test": {
+            "pub": {
+                "make.txt": "content of make.txt",
+                "foo1.csv": "CSV Content",
+                "image01.jpg": "data for image01.jpg",
+            },
+            "read.me": "contents of read.me",
+        }
+    }
+}
+
+
+@pytest.mark.django_db
+class TestBackend:
+    def test_mail_backend(self):
+        from wbcore.contrib.io.import_export.backends.mail import DataBackend as MailDataBackend
+
+        b1 = get_byte_stream()
+        b2 = get_byte_stream()
+        filename1 = "file_valid_12385.json"
+        filename2 = "file.json"
+        att1 = AnymailInboundMessage.construct_attachment(
+            "application/json", b1.getvalue(), charset="utf-8", filename=filename1
+        )
+
+        att2 = AnymailInboundMessage.construct_attachment(
+            "application/json", b2.getvalue(), charset="utf-8", filename=filename2
+        )
+
+        msg = AnymailInboundMessage.construct(
+            from_email="from@example.com",
+            to="to@example.com",
+            cc="cc@example.com",
+            subject="test subject",
+            attachments=[att1, att2],
+        )
+        backend = MailDataBackend()
+        res = list(backend.get_files(timezone.now(), file_name_regex="file_valid_([A-Z0-9]*).json", message=msg))
+        assert len(res) == 1
+        assert res[0][0] == filename1
+        assert (res[0][1]).getvalue() == b1.getvalue()
+
+    @pytest.mark.parametrize(
+        "source__import_parameters,from_email",
+        [
+            ({"whitelisted_emails": ["test@test.ch"]}, "test@test.ch"),
+            ({"whitelisted_emails": ["test@test.ch"]}, "spam@    test.ch"),
+        ],
+    )
+    @patch.object(Source, "trigger_workflow")
+    def test_inbound_mail(self, mock_process_source, source, from_email):
+        from wbcore.contrib.io.models import DataBackend
+
+        data_backend = DataBackend.objects.get(title="Default Mail")
+
+        assert data_backend.backend_class_path == "wbcore.contrib.io.import_export.backends.mail"
+
+        source.data_backend = data_backend
+        source.save()
+        message = AnymailInboundMessage.construct(
+            from_email=from_email,
+            to="to@example.com",
+            subject=fake.paragraph(),
+            text=fake.paragraph(),
+            html=f"<p>{fake.paragraph()}</p>",
+        )
+        event = AnymailInboundEvent(
+            event_type=EventType.INBOUND,
+            timestamp=None,
+            event_id=fake.uuid4(),
+            esp_event={"from_email": from_email},
+            message=message,
+        )
+        inbound.send(sender="TEST", event=event, esp_name="test")
+        if source.import_parameters.get("whitelisted_emails", [None])[0] == from_email:
+            assert mock_process_source.call_count == 1
+        else:
+            assert mock_process_source.call_count == 0
+
+    def test_sftp_backend_without_credential(self):
+        from wbcore.contrib.io.import_export.backends.sftp import DataBackend as SFTPDataBackend
+
+        with pytest.raises(ValueError):
+            SFTPDataBackend()
+
+    def test_sftp_backend(self, sftpserver, import_credential_factory):
+        from wbcore.contrib.io.import_export.backends.sftp import DataBackend as SFTPDataBackend
+
+        with sftpserver.serve_content(VFS):
+            import_credential = import_credential_factory.create(
+                username="user",
+                password="password",  # noqa
+                additional_resources={
+                    "host": sftpserver.host,
+                    "port": sftpserver.port,
+                },
+            )
+            copied_file_content = VFS["home"]["test"]["pub"]["foo1.csv"]
+            backend = SFTPDataBackend(import_credential=import_credential)
+            res = list(
+                backend.get_files(timezone.now(), "/home/test/pub", file_name_regex=r".+\.csv$", cleanup_files=True)
+            )
+
+            assert len(res) == 1
+            assert res[0][0] == "foo1.csv"
+            assert (res[0][1]).getvalue().decode("utf-8") == copied_file_content

wbcore/contrib/io/tests/test_exports.py

@@ -0,0 +1,130 @@
+from unittest.mock import patch
+
+import pandas as pd
+import pytest
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.test.client import RequestFactory
+from faker import Faker
+from rest_framework.reverse import reverse
+from rest_framework.test import force_authenticate
+
+from ..enums import ExportFormat, get_django_import_export_format
+from ..models import ExportSource, ParserHandler
+from ..resources import ViewResource
+from ..serializers import ParserHandlerModelSerializer
+from ..viewsets import ParserHandlerModelViewSet
+from .test_models import get_test_file
+
+fake = Faker()
+
+
+@pytest.mark.django_db
+class TestExportSourceModel:
+    def test_file_format(self, export_source):
+        assert (
+            export_source.file_format.CONTENT_TYPE
+            == get_django_import_export_format(export_source.format).CONTENT_TYPE
+        )
+
+    def test_resource(self, export_source):
+        r = export_source.resource
+        assert isinstance(r, ViewResource)
+        assert r.columns_map == export_source.resource_kwargs["columns_map"]
+        assert r.serializer_class == ParserHandlerModelSerializer
+
+    def test_empty_queryset(self, export_source):
+        assert set(export_source.queryset) == set()
+
+    def test_queryset(self, export_source_factory, parser_handler_factory):
+        ph1 = parser_handler_factory.create()
+        e1 = export_source_factory()
+        assert set(e1.queryset) == {ph1}
+        ph2 = parser_handler_factory.create()
+        e2 = export_source_factory()
+
+        assert set(e2.queryset) == {ph1, ph2}
+
+    def test_queryset_with_filter(self, export_source_factory, parser_handler_factory):
+        ph1 = parser_handler_factory.create()
+        parser_handler_factory.create()  # noise
+
+        # check with customer queryset and filtering
+        query_str, query_params = ParserHandler.objects.filter(parser=ph1.parser).query.sql_with_params()
+        export_source = export_source_factory.create(query_str=query_str, query_params=query_params)
+
+        assert set(export_source.queryset) == {ph1}
+
+    def test_get_export_filename(self, export_source):
+        filename = export_source.get_export_filename()
+        assert isinstance(filename, str)
+        assert (
+            export_source.file_format.get_extension() in filename
+        )  # check that the filename is with the proper extension
+
+    @patch("wbcore.contrib.io.models.send_notification")
+    def test_notify(self, mock_fct, export_source):
+        export_source.notify()
+        assert mock_fct.call_count == 0  # 0 call count because there is neither no file nor status == processed
+
+        export_source.status = ExportSource.Status.PROCESSED.value
+        export_source.save()
+
+        export_source.notify()
+        assert mock_fct.call_count == 0  # Still need a file
+
+        export_source.file = get_test_file()
+        export_source.save()
+
+        export_source.notify()
+        assert mock_fct.call_count == 1  # Still need a file
+
+    @patch.object(ExportSource, "notify")
+    @pytest.mark.parametrize("export_source__format", [ExportFormat.CSV])
+    def test_export_data(self, fct_mock, export_source, parser_handler):
+        export_source.export_data()
+        assert export_source.status == ExportSource.Status.PROCESSED.value
+        with export_source.file.open() as file:
+            df = pd.read_csv(file)
+            assert df.loc[0, "id"] == parser_handler.id
+            assert df.loc[0, "Parser"] == parser_handler.parser
+            assert df.loc[0, "Handler"] == parser_handler.handler
+        assert fct_mock.call_count == 1
+
+    # INFO This test is obsolete
+    # def test_view_export_under_pagination_limit(self, superuser, parser_handler_factory):
+    #     parser_handler_factory.create()
+    #
+    #     url = reverse("wbcore:io:parserhandler-processexport", args=[])
+    #     request = RequestFactory().get(url, {"export_format": 0})
+    #     force_authenticate(request, user=superuser)
+    #     view = ParserHandlerModelViewSet.as_view({"get": "process_export"})
+    #     response = view(request)
+    #     assert response.headers["Content-Type"] == "text/csv"
+
+    def test_view_export_over_pagination_limit(self, superuser, parser_handler_factory):
+        assert ExportSource.objects.count() == 0
+
+        settings.WBIMPORT_EXPORT_DEFAULT_EXPORT_PAGINATION_LIMIT = 1
+
+        parser_handler_factory.create()
+
+        url = reverse("wbcore:io:parserhandler-processexport", args=[])
+        request = RequestFactory().patch(url, {"format": 0}, content_type="application/json")
+        request.query_params = {}
+
+        force_authenticate(request, user=superuser)
+        view = ParserHandlerModelViewSet.as_view({"patch": "process_export"})
+        response = view(request)
+        assert response.status_code == 200
+
+        _view = ParserHandlerModelViewSet()
+        _view.request = request
+        query_str, query_params = _view.filter_queryset(_view.get_queryset()).query.sql_with_params()
+        assert ExportSource.objects.get(
+            creator=superuser,
+            content_type=ContentType.objects.get_for_model(ParserHandler),
+            resource_path="wbcore.contrib.io.resources.ViewResource",
+            query_str=query_str,
+            query_params=query_params,
+        )