wbcore 2.2.1__py2.py3-none-any.whl

wbcore/contrib/io/backends/sftp.py

@@ -0,0 +1,63 @@
+import io
+import re
+from datetime import datetime
+from io import BytesIO
+from typing import Generator, Optional
+
+import pysftp
+from django.conf import settings
+
+from ..models import ImportCredential
+from .abstract import AbstractDataBackend
+from .utils import register
+
+
+@register("SFTP", save_data_in_import_source=True, passive_only=True)
+class DataBackend(AbstractDataBackend):
+    def __init__(
+        self,
+        import_credential: Optional[ImportCredential] = None,
+        **kwargs,
+    ):
+        if not import_credential:
+            raise ValueError("The sftp backend needs an import credential")
+
+        self.username = import_credential.username
+        self.password = import_credential.password
+        self.host = import_credential.additional_resources.get("host", "")
+        self.port = import_credential.additional_resources.get("port", "")
+
+    def get_files(
+        self,
+        execution_time: datetime,
+        sftp_folder,
+        file_name_regex: Optional[str] = None,
+        cleanup_files: Optional[bool] = getattr(settings, "WBIMPORT_EXPORT_SFTPBACKEND_CLEAN_FILES", False),
+        **kwargs,
+    ) -> Generator[tuple[str, BytesIO], None, None]:
+        # Create CnOpts for Paramiko
+        cnopts = pysftp.CnOpts()
+        setattr(cnopts, "hostkeys", None)
+
+        # Create Connection to the SFTP Server
+        with pysftp.Connection(
+            self.host, port=self.port, username=self.username, password=self.password, cnopts=cnopts
+        ) as sftp:
+            # Change working directory
+            sftp.cwd(sftp_folder)
+
+            # Filter all the files we need
+            file_names = sftp.listdir()
+            if file_name_regex:
+                # Compile the regex for later filtering the files
+                file_names = filter(lambda x: re.match(file_name_regex, x), file_names)
+
+            for file_name in file_names:
+                # Create a Buffer where we write the file to
+                sftp_file = io.BytesIO()
+                sftp.getfo(file_name, sftp_file)
+                yield file_name, sftp_file
+
+                if cleanup_files:
+                    # Delete the file from the server
+                    sftp.remove(file_name)

wbcore/contrib/io/backends/stream.py

@@ -0,0 +1,92 @@
+import mimetypes
+import os
+from contextlib import suppress
+from datetime import datetime
+from io import BytesIO
+from typing import Generator, Optional
+
+import magic
+import requests
+
+from .abstract import AbstractDataBackend
+from .utils import register
+
+
+@register("Default Stream", save_data_in_import_source=True)
+class DataBackend(AbstractDataBackend):
+    """
+    DataBackend for stream based import
+    """
+
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        Args:
+            url: URL to fetch the stream from
+            **kwargs:
+        Raises:
+            ValueError: If url not found in the keywords argument
+        """
+        if not url:
+            raise ValueError("This backend needs a valid url")
+
+        self.url = url
+
+    @classmethod
+    def _check_content_type(self, output: BytesIO, filename: str) -> bool:
+        """
+        Check if given bytes stream matches the corresponding filename extension
+
+        Args:
+            output: The file bytes stream
+            filename: The file name (with extension)
+
+        Returns:
+            True if the contents matches
+
+        Raises:
+            MagicException: If error happens while extracting content type
+        """
+        try:
+            buffer_content_type = magic.from_buffer(output.read(1024), mime=True)
+            guess_extensions = mimetypes.guess_extension(buffer_content_type)
+            return guess_extensions == os.path.splitext(filename)[1]
+        except magic.MagicException:
+            return False
+
+    def get_files(
+        self,
+        execution_time: datetime,
+        base_filename: Optional[str] = "default_stream",
+        extension: Optional[str] = "xml",
+        validate_content_type: Optional[bool] = True,
+        **kwargs,
+    ) -> Generator[tuple[str, BytesIO], None, None]:
+        """
+        Connect to file stream and download its content as bytes stream
+
+        Args:
+            execution_time: When the import is executed.
+            base_filename: The resulting filename. Defaults to "default_stream".
+            extension: The resulting file extension. Defaults to ".xml".
+            validate_content_type: If true, will sanitize file content with its given extension. Defaults to True.
+            **kwargs:
+
+        Returns:
+            A generator of tuple of filename and bytes stream
+
+        """
+        with suppress(requests.ConnectionError):
+            r = requests.get(self.url)
+            if r.ok and (content := r.content):
+                content_file = BytesIO()
+                content_file.write(content)
+                content_file.seek(0)
+                file_name = f"{base_filename}_{datetime.timestamp(execution_time)}.{extension}"
+                if not validate_content_type or (
+                    validate_content_type and self._check_content_type(content_file, file_name)
+                ):
+                    yield file_name, content_file

wbcore/contrib/io/backends/utils.py

@@ -0,0 +1,42 @@
+from contextlib import suppress
+
+from ..models import DataBackend, Provider
+
+
+def register(
+    backend_title: str,
+    save_data_in_import_source: bool | None = False,
+    passive_only: bool | None = False,
+    provider_key: str | None = None,
+):
+    """
+    Decorator to include when a backend need automatic registration
+    Args:
+        backend_name:
+
+    Returns:
+
+    """
+    if not backend_title:
+        raise ValueError("At least one name must be passed to register.")
+
+    def _decorator(backend_class):
+        with suppress(Exception):
+            provider = None
+            if provider_key:
+                provider, created = Provider.objects.get_or_create(
+                    key=provider_key, defaults={"title": provider_key.capitalize()}
+                )
+            DataBackend.objects.update_or_create(
+                backend_class_path=backend_class.__module__,
+                backend_class_name=backend_class.__name__,
+                defaults={
+                    "title": f"{backend_title} ({provider.title})" if provider else backend_title,
+                    "save_data_in_import_source": save_data_in_import_source,
+                    "passive_only": passive_only,
+                    "provider": provider,
+                },
+            )
+        return backend_class
+
+    return _decorator

wbcore/contrib/io/configs/endpoints.py

@@ -0,0 +1,12 @@
+from rest_framework.reverse import reverse
+from wbcore.metadata.configs.endpoints import EndpointViewConfig
+
+
+class ImportSourceModelViewSetEndpointConfig(EndpointViewConfig):
+    def get_endpoint(self, **kwargs):
+        return reverse("wbcore:io:importsource-list", args=[], request=self.request)
+
+
+class SourceModelViewSetEndpointConfig(EndpointViewConfig):
+    def get_endpoint(self, **kwargs):
+        return reverse("wbcore:io:source-list", args=[], request=self.request)

wbcore/contrib/io/configurations/__init__.py

@@ -0,0 +1 @@
+from .base import ImportExportBaseConfiguration

wbcore/contrib/io/configurations/base.py

@@ -0,0 +1,7 @@
+from configurations import values
+
+
+class ImportExportBaseConfiguration:
+    WBIMPORT_EXPORT_SFTPBACKEND_CLEAN_FILES = values.BooleanValue(False, environ_prefix=None)
+    WBIMPORT_EXPORT_MAILBACKEND_SPAMSCORE = values.IntegerValue(None, environ_prefix=None)
+    WBIMPORT_EXPORT_DEFAULT_EXPORT_PAGINATION_LIMIT = values.IntegerValue(200, environ_prefix=None)

wbcore/contrib/io/dynamic_preferences_registry.py

@@ -0,0 +1,15 @@
+from dynamic_preferences.preferences import Section
+from dynamic_preferences.registries import global_preferences_registry
+from dynamic_preferences.types import StringPreference
+
+io = Section("io")
+
+
+@global_preferences_registry.register
+class AdministratorMailsPreference(StringPreference):
+    section = io
+    name = "administrator_mails"
+    default = ""
+
+    verbose_name = "Administrator mails"
+    help_text = "The mails (as comma separated string) of the person allow to send directly to the mail backend"

wbcore/contrib/io/enums.py

@@ -0,0 +1,15 @@
+from django.db.models import IntegerChoices
+from import_export.admin import ImportExportMixin
+
+
+class ExportFormat(IntegerChoices):
+    CSV = 0, "CSV"
+    XLSX = 1, "XLSX"
+    TSV = 2, "TSV"
+    JSON = 3, "JSON"
+    YAML = 4, "YAML"
+
+
+def get_django_import_export_format(format: int):
+    format_map = {f.__name__: f for f in ImportExportMixin().get_export_formats()}
+    return format_map[ExportFormat(format).name]

wbcore/contrib/io/exceptions.py

@@ -0,0 +1,16 @@
+class DeserializationError(Exception):
+    """
+    Exception excepted when a deserialized error happens during the process object phase in the Handler.
+
+    This exception won't stop the importing process and will only be logged
+    """
+
+    pass
+
+
+class ImportError(Exception):
+    """
+    Exception returns when something wrong happens during the processing of the import source and means that not all data were succesfully imported
+    """
+
+    pass

wbcore/contrib/io/factories.py

@@ -0,0 +1,202 @@
+import factory
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+from django_celery_beat.models import CrontabSchedule
+from faker import Faker
+from wbcore.contrib.io.imports import ImportExportHandler
+from wbcore.contrib.io.mixins import ImportMixin
+
+from .enums import ExportFormat
+from .models import (
+    DataBackend,
+    ExportSource,
+    ImportCredential,
+    ImportSource,
+    ParserHandler,
+    Provider,
+    Source,
+)
+
+
+class ImportExportModelHandler(ImportExportHandler):
+    MODEL_APP_LABEL = "io.ImportModel"
+
+    def _deserialize(self, data):
+        data["relationship"] = ParserHandler.objects.get(id=data["relationship"])
+        data["many_relationships"] = [ParserHandler.objects.get(id=d) for d in data["many_relationships"]]
+
+    def _get_instance(self, data, history=None, **kwargs) -> models.Model | None:
+        if id := data.get("id", None):
+            if history:
+                return history.filter(id=id).first()
+            return self.model.objects.filter(id=id).first()
+
+    def _create_instance(self, data, **kwargs) -> models.Model:
+        data.pop("id", None)
+        many_relationships = data.pop("many_relationships", [])
+        obj = self.model.objects.create(import_source=self.import_source, **data)
+        for rel in many_relationships:
+            obj.many_relationships.add(rel)
+        return obj
+
+    def _get_history(self, history):
+        # Return all object with the same relationship
+        return self.model.objects.filter(relationship=history["relationship"])
+
+
+# TODO remove this class
+class ImportModel(ImportMixin):
+    import_export_handler_class = ImportExportModelHandler
+    relationship = models.ForeignKey(
+        "io.ParserHandler", blank=True, null=True, on_delete=models.SET_NULL, related_name="imports"
+    )
+    import_source = models.ForeignKey(
+        "io.ImportSource", blank=True, null=True, on_delete=models.SET_NULL, related_name="imports"
+    )
+    many_relationships = models.ManyToManyField("io.ParserHandler", blank=True, related_name="many_imports")
+    number = models.FloatField(blank=True, null=True)
+    text = models.TextField(blank=True, null=True)
+    name = models.CharField(max_length=256, blank=True, null=True)
+
+
+fake = Faker()
+
+
+class CrontabScheduleFactory(factory.django.DjangoModelFactory):
+    hour = factory.Iterator(range(0, 24))
+    minute = factory.Iterator(range(0, 60))
+
+    class Meta:
+        model = CrontabSchedule
+
+
+class ProviderFactory(factory.django.DjangoModelFactory):
+    title = factory.Faker("company")
+    key = factory.LazyAttribute(lambda x: x.title.lower())
+
+    class Meta:
+        model = Provider
+
+
+class DataBackendFactory(factory.django.DjangoModelFactory):
+    title = factory.Sequence(lambda n: f"Data Backend {n}")
+    save_data_in_import_source = False
+    passive_only = False
+    backend_class_path = "wbcore.contrib.io.backends.abstract"
+    backend_class_name = "AbstractDataBackend"
+    provider = factory.SubFactory("wbcore.contrib.io.factories.ProviderFactory")
+
+    class Meta:
+        model = DataBackend
+
+
+class ExportSourceFactory(factory.django.DjangoModelFactory):
+    """
+    Default exportt source factory that are attached to the parser handler model and serializer
+    """
+
+    created = factory.Faker("date_time")
+    last_updated = factory.Faker("date_time")
+    log = factory.Faker("paragraph")
+    origin = factory.Faker("company")
+    status = ExportSource.Status.PENDING.value
+    creator = factory.SubFactory("wbcore.contrib.authentication.factories.UserFactory")
+    file = None
+    format = factory.Iterator(ExportFormat.values)
+
+    content_type = factory.LazyAttribute(lambda x: ContentType.objects.get_for_model(ParserHandler))
+    resource_path = "wbcore.contrib.io.resources.ViewResource"
+    resource_kwargs = {
+        "columns_map": {"parser": "Parser", "handler": "Handler"},
+        "serializer_class_path": "wbcore.contrib.io.serializers.ParserHandlerModelSerializer",
+    }
+    query_str = factory.LazyAttribute(lambda x: ParserHandler.objects.all().query.sql_with_params()[0])
+    query_params = factory.LazyAttribute(lambda x: ParserHandler.objects.all().query.sql_with_params()[1])
+
+    class Meta:
+        model = ExportSource
+
+
+class ImportSourceFactory(factory.django.DjangoModelFactory):
+    created = factory.Faker("date_time")
+    last_updated = factory.Faker("date_time")
+    parser_handler = factory.SubFactory("wbcore.contrib.io.factories.ParserHandlerFactory")
+    source = factory.SubFactory("wbcore.contrib.io.factories.SourceFactory")
+    save_data = True
+    log = factory.Faker("paragraph")
+    origin = factory.Faker("company")
+    file = None
+
+    class Meta:
+        model = ImportSource
+
+
+class ImportCredentialFactory(factory.django.DjangoModelFactory):
+    type = ImportCredential.Type.CREDENTIAL
+    username = factory.Sequence(lambda n: f"username-{n}")
+    password = factory.Sequence(lambda n: f"password-{n}")
+    key = factory.Sequence(lambda n: f"credential-{n}")
+    validity_start = None
+    validity_end = None
+
+    class Meta:
+        model = ImportCredential
+
+
+class SourceFactory(factory.django.DjangoModelFactory):
+    crontab = factory.SubFactory("wbcore.contrib.io.factories.CrontabScheduleFactory")
+    data_backend = factory.SubFactory("wbcore.contrib.io.factories.DataBackendFactory")
+    import_parameters = factory.LazyAttribute(lambda o: {fake.name(): fake.name(), fake.name(): fake.name()})
+    connection_parameters = factory.LazyAttribute(lambda o: {fake.name(): fake.name(), fake.name(): fake.name()})
+    is_active = True
+    title = factory.Sequence(lambda n: f"title {n}")
+
+    class Meta:
+        model = Source
+
+    @factory.post_generation
+    def parser_handler(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for parser_handler in extracted:
+                self.parser_handler.add(parser_handler)
+
+    @factory.post_generation
+    def credentials(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for credential in extracted:
+                self.credentials.add(credential)
+
+
+class ParserHandlerFactory(factory.django.DjangoModelFactory):
+    parser = factory.Sequence(lambda n: f"parser_{n}")
+    handler = factory.Sequence(lambda n: f"handler.Handler-{n}")
+    allow_file_type = None
+
+    class Meta:
+        model = ParserHandler
+
+
+class ImportModelFactory(factory.django.DjangoModelFactory):
+    relationship = factory.SubFactory("wbcore.contrib.io.factories.ParserHandlerFactory")
+    import_source = factory.SubFactory("wbcore.contrib.io.factories.ImportSourceFactory")
+    number = factory.Faker("pyfloat")
+    text = factory.Faker("paragraph")
+    name = factory.Faker("name")
+
+    class Meta:
+        model = ImportModel
+
+    @factory.post_generation
+    def many_relationships(self, create, extracted, **kwargs):
+        if not create:
+            return
+
+        if extracted:
+            for relationship in extracted:
+                self.many_relationships.add(relationship)