wbcore 1.59.9__py2.py3-none-any.whl

wbcore/contrib/icons/models.py

@@ -0,0 +1,11 @@
+from django.db.models import CharField
+
+
+class IconField(CharField):
+    description = "A field storing an icon"
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if "choices" in kwargs:
+            del kwargs["choices"]
+        return name, path, args, kwargs

wbcore/contrib/icons/serializers.py

@@ -0,0 +1,18 @@
+from wbcore.serializers.fields.text import CharField
+from wbcore.serializers.fields.types import WBCoreType
+
+from .icons import WBIcon
+
+
+class IconSelectField(CharField):
+    field_type = WBCoreType.ICON.value
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def to_representation(self, value):
+        # Try to get its "icon" representation. If the field is not supported, we simply returns the value (This allow custom and hardcoded icon)
+        try:
+            return WBIcon[value].icon
+        except KeyError:
+            return value

wbcore/contrib/io/admin.py

@@ -0,0 +1,151 @@
+from django.contrib import admin
+from django.contrib.contenttypes.admin import GenericTabularInline
+from django_admin_inline_paginator.admin import TabularInlinePaginated
+
+from .models import (
+    DataBackend,
+    ExportSource,
+    ImportCredential,
+    ImportedObjectProviderRelationship,
+    ImportSource,
+    ParserHandler,
+    Provider,
+    Source,
+    import_data_as_task,
+)
+
+
+def reprocess_import_source(modeladmin, request, queryset):
+    for import_source in queryset:
+        import_data_as_task.delay(import_source.id, force_reimport=True)
+
+
+def reprocess_source(modeladmin, request, queryset):
+    for source in queryset:
+        source.trigger_workflow()
+
+
+class ImportedObjectProviderRelationshipInline(TabularInlinePaginated):
+    ordering = ["object_id"]
+    model = ImportedObjectProviderRelationship
+    extra = 0
+    readonly_fields = ("content_type", "object_id")
+    raw_id_fields = ["content_type", "provider"]
+    per_page = 25
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("content_type", "provider")
+
+
+class ProviderGenericInline(GenericTabularInline):
+    model = ImportedObjectProviderRelationship
+
+
+@admin.register(ParserHandler)
+class ParserHandlerModelAdmin(admin.ModelAdmin):
+    list_display = ["parser", "handler"]
+
+    search_fields = ["parser", "handler"]
+
+
+@admin.register(Provider)
+class ProviderModelAdmin(admin.ModelAdmin):
+    list_display = ["title"]
+    search_fields = ["title"]
+    inlines = [ImportedObjectProviderRelationshipInline]
+
+
+@admin.register(DataBackend)
+class DataBackendModelAdmin(admin.ModelAdmin):
+    list_display = [
+        "title",
+        "save_data_in_import_source",
+        "backend_class_path",
+        "backend_class_name",
+        "provider",
+    ]
+    search_fields = ["title"]
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("provider")
+
+
+@admin.register(Source)
+class SourceModelAdmin(admin.ModelAdmin):
+    list_display = ["uuid", "title", "is_active", "crontab", "data_backend"]
+    list_filter = ["data_backend", "is_active"]
+    raw_id_fields = ["crontab", "periodic_task"]
+    search_fields = ["uuid", "data_backend__title", "data_backend__provider__title", "parser_handler__parser"]
+    readonly_fields = ["periodic_task"]
+
+    actions = [reprocess_source]
+    autocomplete_fields = ["parser_handler", "data_backend"]
+
+    def get_queryset(self, request):
+        return (
+            super()
+            .get_queryset(request)
+            .select_related("data_backend", "periodic_task", "crontab")
+            .prefetch_related("parser_handler", "credentials")
+        )
+
+
+@admin.register(ExportSource)
+class ExportSourceModelAdmin(admin.ModelAdmin):
+    list_display = ["status", "file", "resource_path", "content_type", "created", "last_updated", "creator"]
+    list_filter = ["status", "resource_path", "content_type"]
+    autocomplete_fields = ["creator", "content_type"]
+    readonly_fields = ("created", "last_updated", "query_str", "query_params")
+    fieldsets = (
+        (
+            "",
+            {
+                "fields": (
+                    ("status", "created", "last_updated"),
+                    ("content_type", "format"),
+                    ("resource_path", "resource_kwargs"),
+                    ("query_str", "query_params"),
+                    ("file", "origin", "creator"),
+                    ("data",),
+                    ("log",),
+                ),
+            },
+        ),
+    )
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("content_type")
+
+
+@admin.register(ImportSource)
+class ImportSourceModelAdmin(admin.ModelAdmin):
+    list_display = ["status", "file", "parser_handler", "source", "created", "last_updated"]
+    list_filter = ["status", "source", "parser_handler__handler", "parser_handler__parser"]
+    autocomplete_fields = ["source", "parser_handler"]
+    actions = [reprocess_import_source]
+    readonly_fields = ("created", "last_updated")
+    fieldsets = (
+        (
+            "",
+            {
+                "fields": (
+                    ("status", "created", "last_updated"),
+                    ("parser_handler", "source", "origin"),
+                    ("file", "save_data", "progress_index"),
+                    ("creator",),
+                    ("resource_kwargs",),
+                    ("data",),
+                    ("log",),
+                    ("errors_log",),
+                ),
+            },
+        ),
+    )
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("source", "parser_handler")
+
+
+@admin.register(ImportCredential)
+class ImportCredentialModelAdmin(admin.ModelAdmin):
+    list_display = ["key", "type", "username", "validity_start", "validity_end"]

wbcore/contrib/io/apps.py

@@ -0,0 +1,37 @@
+from anymail.signals import inbound
+from django.apps import AppConfig
+from django.apps import apps as global_apps
+from django.db import DEFAULT_DB_ALIAS
+from django.db.models.signals import post_migrate
+from django.utils.module_loading import autodiscover_modules
+
+
+class ImportExportConfig(AppConfig):
+    name = "wbcore.contrib.io"
+
+    def ready(self):
+        """
+        registered source from settings
+        """
+        # Implicitly connect a signal handlers decorated with @receiver.
+        from wbcore.contrib.io.management import load_sources_from_settings
+
+        from .backends.mail import handle_inbound
+
+        # Explicitly connect a signal handler.
+        inbound.connect(handle_inbound)
+
+        def autodiscover_backends(
+            app_config, verbosity=2, interactive=True, using=DEFAULT_DB_ALIAS, apps=global_apps, **kwargs
+        ):
+            # we wrap the autodiscover into a post_migrate receiver because we expect db calls
+            autodiscover_modules("import_export.backends")
+
+        post_migrate.connect(
+            autodiscover_backends,
+            dispatch_uid="wbcore.io.autodiscover_backends",
+        )
+        post_migrate.connect(
+            load_sources_from_settings,
+            dispatch_uid="wbcore.contrib.io.load_sources_from_settings",
+        )

wbcore/contrib/io/backends/__init__.py

@@ -0,0 +1,3 @@
+from .abstract import AbstractDataBackend
+from .mail import *
+from .utils import register

wbcore/contrib/io/backends/abstract.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from io import BytesIO
+from typing import Generator, Optional
+
+from django.db import models
+
+
+class AbstractDataBackend:
+    def __init__(self, import_credential: Optional[models.Model] = None, data_backend=None, **kwargs):
+        self.data_backend = data_backend
+        self.import_credential = import_credential
+
+    def is_object_valid(self, obj: models.Model) -> bool:
+        """
+        Can be overriden to define how an object is consired valid by the backend. Default to return True
+
+        Args:
+            obj: The tested object
+
+        Returns:
+            True if the object is considered valid
+        """
+        default_qs = self.get_default_queryset()
+        if default_qs is not None:
+            return isinstance(obj, default_qs.model) and default_qs.filter(pk=obj.pk).exists()
+        return False
+
+    def get_default_queryset(self) -> models.QuerySet | None:
+        """
+        Returns the defaults queryset to loop over by the backend to retreive import files. Used on daily import when no queryset is explicitly provided.
+
+        Returns:
+            The default queryset if it exists, none otherwise (Default)
+        """
+        return None
+
+    def get_provider_id(self, obj: models.Model) -> str | None:
+        """
+        Override if backend support another and different identifier scheme. We expect this function to the external provider identifier given a certain object.
+
+        Args:
+            obj: The object to extract the external identifier
+
+        Returns:
+            The external provider identifier it it exists. None otherwise (Default)
+        """
+        return None
+
+    def get_files(self, execution_time: datetime, **kwargs) -> Generator[tuple[str, BytesIO], None, None] | None:
+        """
+        Override to implement the import file creation logic
+
+        Args:
+            execution_time: The time at which the import takes place
+            **kwargs: Possible keywords arguments to be passed down to the importer.
+
+        Returns:
+            A generator of bitestream object.
+        """
+        pass

wbcore/contrib/io/backends/mail.py

@@ -0,0 +1,39 @@
+import re
+
+from django.conf import settings
+from django.db.models import Q
+from dynamic_preferences.registries import global_preferences_registry
+
+from ..models import Source
+
+
+def is_sender_allowed(from_email: str, whitelisted_emails: list[str], admin_mails: list[str]) -> bool:
+    if from_email in admin_mails:
+        return True
+    for whitelisted_email in whitelisted_emails:
+        if re.search(whitelisted_email, from_email):
+            return True
+    return False
+
+
+def handle_inbound(sender, event, esp_name, **kwargs):
+    spam_detected = False
+    if (message := event.message) and (from_email := message.from_email.addr_spec) and (subject := message.subject):
+        if spam_score := getattr(settings, "WBIMPORT_EXPORT_MAILBACKEND_SPAMSCORE", None):
+            spam_detected = message.spam_detected or message.spam_score >= spam_score
+        if not spam_detected:
+            admin_emails = global_preferences_registry.manager()["io__administrator_mails"].split(";")
+            conditions = Q(import_parameters__inbound_address__isnull=True)
+            for t in message.to:
+                conditions |= Q(import_parameters__inbound_address__contains=t.addr_spec)
+
+            sources = Source.objects.filter(
+                conditions
+                & Q(data_backend__backend_class_path="wbcore.contrib.io.import_export.backends.mail")
+                & Q(is_active=True)
+            )
+            if s := re.search(r"\[([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\]", subject):
+                sources = sources.filter(uuid=s.group(1))
+            for source in sources:
+                if is_sender_allowed(from_email, source.import_parameters.get("whitelisted_emails", []), admin_emails):
+                    source.trigger_workflow(message=message)

wbcore/contrib/io/backends/utils.py

@@ -0,0 +1,39 @@
+from ..models import DataBackend, Provider
+
+
+def register(
+    backend_title: str,
+    save_data_in_import_source: bool | None = False,
+    passive_only: bool | None = False,
+    provider_key: str | None = None,
+):
+    """
+    Decorator to include when a backend need automatic registration
+    Args:
+        backend_name:
+
+    Returns:
+
+    """
+    if not backend_title:
+        raise ValueError("At least one name must be passed to register.")
+
+    def _decorator(backend_class):
+        provider = None
+        if provider_key:
+            provider, created = Provider.objects.get_or_create(
+                key=provider_key, defaults={"title": provider_key.capitalize()}
+            )
+        DataBackend.objects.update_or_create(
+            backend_class_path=backend_class.__module__,
+            backend_class_name=backend_class.__name__,
+            defaults={
+                "title": f"{backend_title} ({provider.title})" if provider else backend_title,
+                "save_data_in_import_source": save_data_in_import_source,
+                "passive_only": passive_only,
+                "provider": provider,
+            },
+        )
+        return backend_class
+
+    return _decorator

wbcore/contrib/io/configs/endpoints.py

@@ -0,0 +1,13 @@
+from rest_framework.reverse import reverse
+
+from wbcore.metadata.configs.endpoints import EndpointViewConfig
+
+
+class ImportSourceModelViewSetEndpointConfig(EndpointViewConfig):
+    def get_endpoint(self, **kwargs):
+        return reverse("wbcore:io:importsource-list", args=[], request=self.request)
+
+
+class SourceModelViewSetEndpointConfig(EndpointViewConfig):
+    def get_endpoint(self, **kwargs):
+        return reverse("wbcore:io:source-list", args=[], request=self.request)

wbcore/contrib/io/configurations/__init__.py

@@ -0,0 +1 @@
+from .base import ImportExportBaseConfiguration

wbcore/contrib/io/configurations/base.py

@@ -0,0 +1,7 @@
+from configurations import values
+
+
+class ImportExportBaseConfiguration:
+    WBIMPORT_EXPORT_SFTPBACKEND_CLEAN_FILES = values.BooleanValue(False, environ_prefix=None)
+    WBIMPORT_EXPORT_MAILBACKEND_SPAMSCORE = values.IntegerValue(None, environ_prefix=None)
+    WBIMPORT_EXPORT_DEFAULT_EXPORT_PAGINATION_LIMIT = values.IntegerValue(200, environ_prefix=None)

wbcore/contrib/io/dynamic_preferences_registry.py

@@ -0,0 +1,25 @@
+from dynamic_preferences.preferences import Section
+from dynamic_preferences.registries import global_preferences_registry
+from dynamic_preferences.types import IntegerPreference, StringPreference
+
+io = Section("io")
+
+
+@global_preferences_registry.register
+class AdministratorMailsPreference(StringPreference):
+    section = io
+    name = "administrator_mails"
+    default = ""
+
+    verbose_name = "Administrator mails"
+    help_text = "The mails (as comma separated string) of the person allow to send directly to the mail backend"
+
+
+@global_preferences_registry.register
+class ImportSourceDataRetentionPeriod(IntegerPreference):
+    section = io
+    name = "import_source_retention_period"
+    default = 365
+
+    verbose_name = "Import Source Retention Period"
+    help_text = "The number of days to keep the data and log info until cleanup"

wbcore/contrib/io/enums.py

@@ -0,0 +1,15 @@
+from django.db.models import IntegerChoices
+from import_export.admin import ImportExportMixin
+
+
+class ExportFormat(IntegerChoices):
+    CSV = 0, "CSV"
+    XLSX = 1, "XLSX"
+    TSV = 2, "TSV"
+    JSON = 3, "JSON"
+    YAML = 4, "YAML"
+
+
+def get_django_import_export_format(format: int):
+    format_map = {f.__name__: f for f in ImportExportMixin().get_export_formats()}
+    return format_map[ExportFormat(format).name]

wbcore/contrib/io/exceptions.py

@@ -0,0 +1,24 @@
+class DeserializationError(Exception):
+    """
+    Exception excepted when a deserialized error happens during the process object phase in the Handler.
+
+    This exception won't stop the importing process and will only be logged
+    """
+
+    pass
+
+
+class SkipImportError(Exception):
+    """
+    Exception excepted when a deserialized skip happens during the process object phase in the Handler.
+
+    This exception won't stop the importing process and won't trigger a warning
+    """
+
+
+class ImportError(Exception):
+    """
+    Exception returns when something wrong happens during the processing of the import source and means that not all data were succesfully imported
+    """
+
+    pass

wbcore/contrib/io/factories.py

@@ -0,0 +1,205 @@
+import factory
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+from django_celery_beat.models import CrontabSchedule
+from faker import Faker
+
+from wbcore.contrib.io.imports import ImportExportHandler
+from wbcore.contrib.io.mixins import ImportMixin
+
+from .enums import ExportFormat
+from .models import (
+    DataBackend,
+    ExportSource,
+    ImportCredential,
+    ImportSource,
+    ParserHandler,
+    Provider,
+    Source,
+)
+
+
+class ImportExportModelHandler(ImportExportHandler):
+    MODEL_APP_LABEL = "io.ImportModel"
+
+    def _deserialize(self, data):
+        data["relationship"] = ParserHandler.objects.get(id=data["relationship"])
+        data["many_relationships"] = [ParserHandler.objects.get(id=d) for d in data["many_relationships"]]
+
+    def _get_instance(self, data, history=None, **kwargs) -> models.Model | None:
+        if id := data.get("id", None):
+            if history:
+                return history.filter(id=id).first()
+            return self.model.objects.filter(id=id).first()
+
+    def _create_instance(self, data, **kwargs) -> models.Model:
+        data.pop("id", None)
+        many_relationships = data.pop("many_relationships", [])
+        obj = self.model.objects.create(import_source=self.import_source, **data)
+        for rel in many_relationships:
+            obj.many_relationships.add(rel)
+        return obj
+
+    def _get_history(self, history):
+        # Return all object with the same relationship
+        return self.model.objects.filter(relationship=history["relationship"])
+
+
+# TODO remove this class
+class ImportModel(ImportMixin):
+    import_export_handler_class = ImportExportModelHandler
+    relationship = models.ForeignKey(
+        "io.ParserHandler", blank=True, null=True, on_delete=models.SET_NULL, related_name="imports"
+    )
+    import_source = models.ForeignKey(
+        "io.ImportSource", blank=True, null=True, on_delete=models.SET_NULL, related_name="imports"
+    )
+    many_relationships = models.ManyToManyField("io.ParserHandler", blank=True, related_name="many_imports")
+    number = models.FloatField(blank=True, null=True)
+    text = models.TextField(blank=True, null=True)
+    name = models.CharField(max_length=256, blank=True, null=True)
+
+
+fake = Faker()
+
+
+class CrontabScheduleFactory(factory.django.DjangoModelFactory):
+    hour = factory.Iterator(range(0, 24))
+    minute = factory.Iterator(range(0, 60))
+
+    class Meta:
+        model = CrontabSchedule
+
+
+class ProviderFactory(factory.django.DjangoModelFactory):
+    title = factory.Faker("company")
+    key = factory.LazyAttribute(lambda x: x.title.lower())
+
+    class Meta:
+        model = Provider
+
+
+class DataBackendFactory(factory.django.DjangoModelFactory):
+    title = factory.Sequence(lambda n: f"Data Backend {n}")
+    save_data_in_import_source = False
+    passive_only = False
+    backend_class_path = "wbcore.contrib.io.backends.abstract"
+    backend_class_name = "AbstractDataBackend"
+    provider = factory.SubFactory("wbcore.contrib.io.factories.ProviderFactory")
+
+    class Meta:
+        model = DataBackend
+
+
+class ExportSourceFactory(factory.django.DjangoModelFactory):
+    """
+    Default exportt source factory that are attached to the parser handler model and serializer
+    """
+
+    created = factory.Faker("date_time")
+    last_updated = factory.Faker("date_time")
+    log = factory.Faker("paragraph")
+    origin = factory.Faker("company")
+    status = ExportSource.Status.PENDING.value
+    creator = factory.SubFactory("wbcore.contrib.authentication.factories.UserFactory")
+    file = None
+    format = factory.Iterator(ExportFormat.values)
+
+    content_type = factory.LazyAttribute(lambda x: ContentType.objects.get_for_model(ParserHandler))
+    resource_path = "wbcore.contrib.io.resources.ViewResource"
+    resource_kwargs = {
+        "columns_map": {"parser": "Parser", "handler": "Handler"},
+        "serializer_class_path": "wbcore.contrib.io.serializers.ParserHandlerModelSerializer",
+    }
+    query_str = factory.LazyAttribute(lambda x: ParserHandler.objects.all().query.sql_with_params()[0])
+    query_params = factory.LazyAttribute(lambda x: ParserHandler.objects.all().query.sql_with_params()[1])
+
+    class Meta:
+        model = ExportSource
+
+
+class ImportSourceFactory(factory.django.DjangoModelFactory):
+    created = factory.Faker("date_time")
+    last_updated = factory.Faker("date_time")
+    parser_handler = factory.SubFactory("wbcore.contrib.io.factories.ParserHandlerFactory")
+    source = factory.SubFactory("wbcore.contrib.io.factories.SourceFactory")
+    save_data = True
+    log = factory.Faker("paragraph")
+    origin = factory.Faker("company")
+    file = None
+
+    class Meta:
+        model = ImportSource
+
+
+class ImportCredentialFactory(factory.django.DjangoModelFactory):
+    type = ImportCredential.Type.CREDENTIAL
+    username = factory.Sequence(lambda n: f"username-{n}")
+    password = factory.Sequence(lambda n: f"password-{n}")
+    key = factory.Sequence(lambda n: f"credential-{n}")
+    validity_start = None
+    validity_end = None
+
+    class Meta:
+        model = ImportCredential
+
+
+class SourceFactory(factory.django.DjangoModelFactory):
+    crontab = factory.SubFactory("wbcore.contrib.io.factories.CrontabScheduleFactory")
+    data_backend = factory.SubFactory("wbcore.contrib.io.factories.DataBackendFactory")
+    import_parameters = factory.LazyAttribute(lambda o: {fake.name(): fake.name(), fake.name(): fake.name()})
+    connection_parameters = factory.LazyAttribute(lambda o: {fake.name(): fake.name(), fake.name(): fake.name()})
+    is_active = True
+    title = factory.Sequence(lambda n: f"title {n}")
+
+    class Meta:
+        model = Source
+        skip_postgeneration_save = True
+
+    @factory.post_generation
+    def parser_handler(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for parser_handler in extracted:
+                self.parser_handler.add(parser_handler)
+
+    @factory.post_generation
+    def credentials(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for credential in extracted:
+                self.credentials.add(credential)
+
+
+class ParserHandlerFactory(factory.django.DjangoModelFactory):
+    parser = factory.Sequence(lambda n: f"parser_{n}")
+    handler = "io.ImportModel"
+    allow_file_type = None
+
+    class Meta:
+        model = ParserHandler
+
+
+class ImportModelFactory(factory.django.DjangoModelFactory):
+    relationship = factory.SubFactory("wbcore.contrib.io.factories.ParserHandlerFactory")
+    import_source = factory.SubFactory("wbcore.contrib.io.factories.ImportSourceFactory")
+    number = factory.Faker("pyfloat")
+    text = factory.Faker("paragraph")
+    name = factory.Faker("name")
+
+    class Meta:
+        model = ImportModel
+        skip_postgeneration_save = True
+
+    @factory.post_generation
+    def many_relationships(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for relationship in extracted:
+                self.many_relationships.add(relationship)