wbcore 1.46.0__py2.py3-none-any.whl → 1.58.2__py2.py3-none-any.whl

wbcore/contrib/io/locale/en/LC_MESSAGES/django.po

@@ -0,0 +1,138 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-30 12:10+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: contrib/io/models.py:95
+msgid "Parser-Handler"
+msgstr ""
+
+#: contrib/io/models.py:96
+msgid "Parsers-Handlers"
+msgstr ""
+
+#: contrib/io/models.py:133
+msgid "Content object Provider Identifier relationship"
+msgstr ""
+
+#: contrib/io/models.py:134
+msgid "Content object Provider Identifier relationships"
+msgstr ""
+
+#: contrib/io/models.py:161 contrib/io/models.py:187
+msgid "Provider"
+msgstr ""
+
+#: contrib/io/models.py:162
+msgid "Providers"
+msgstr ""
+
+#: contrib/io/models.py:206 contrib/io/models.py:250
+msgid "Data Backend"
+msgstr ""
+
+#: contrib/io/models.py:207
+msgid "Data Backends"
+msgstr ""
+
+#: contrib/io/models.py:257
+msgid "Crontab Schedule"
+msgstr ""
+
+#: contrib/io/models.py:258
+msgid ""
+"Crontab Schedule to run the task on.  Set only one schedule type, leave the "
+"others null."
+msgstr ""
+
+#: contrib/io/models.py:299
+msgid "Source"
+msgstr ""
+
+#: contrib/io/models.py:300
+msgid "Sources"
+msgstr ""
+
+#: contrib/io/models.py:688
+msgid "Format of file to be exported"
+msgstr ""
+
+#: contrib/io/models.py:691
+msgid "Export job Content Type"
+msgstr ""
+
+#: contrib/io/models.py:694
+msgid "Resource path to use when exporting"
+msgstr ""
+
+#: contrib/io/models.py:697
+msgid "SQL query to be executed"
+msgstr ""
+
+#: contrib/io/models.py:699
+msgid "SQL query parameters to be used with the sql query"
+msgstr ""
+
+#: contrib/io/models.py:703
+msgid "Export Source"
+msgstr ""
+
+#: contrib/io/models.py:704
+msgid "Export Sources"
+msgstr ""
+
+#: contrib/io/models.py:761
+msgid "Your export file is available"
+msgstr ""
+
+#: contrib/io/models.py:762
+msgid ""
+"<p>The export job you requested is finished and available for one hour.</p>"
+msgstr ""
+
+#: contrib/io/models.py:813
+msgid "Import Source"
+msgstr ""
+
+#: contrib/io/models.py:814
+msgid "Import Sources"
+msgstr ""
+
+#: contrib/io/models.py:1022
+msgid "Import Credential"
+msgstr ""
+
+#: contrib/io/models.py:1023
+msgid "Import Credentials"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:240 contrib/io/viewset_mixins.py:241
+#: contrib/io/viewset_mixins.py:242
+msgid "Import"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:243
+msgid "Please provide a valid CSV file (Coma Separated Value)"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:258 contrib/io/viewset_mixins.py:259
+#: contrib/io/viewset_mixins.py:260
+msgid "Export"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:261
+msgid "Select the export format"
+msgstr ""

wbcore/contrib/io/locale/fr/LC_MESSAGES/django.po

@@ -0,0 +1,138 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2025-05-30 12:10+0200\n"
+"PO-Revision-Date: 2025-05-30 09:40+0000\n"
+"Language-Team: French (https://app.transifex.com/stainly/teams/171242/fr/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: fr\n"
+"Plural-Forms: nplurals=3; plural=(n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2;\n"
+
+#: contrib/io/models.py:95
+msgid "Parser-Handler"
+msgstr ""
+
+#: contrib/io/models.py:96
+msgid "Parsers-Handlers"
+msgstr ""
+
+#: contrib/io/models.py:133
+msgid "Content object Provider Identifier relationship"
+msgstr ""
+
+#: contrib/io/models.py:134
+msgid "Content object Provider Identifier relationships"
+msgstr ""
+
+#: contrib/io/models.py:161 contrib/io/models.py:187
+msgid "Provider"
+msgstr ""
+
+#: contrib/io/models.py:162
+msgid "Providers"
+msgstr ""
+
+#: contrib/io/models.py:206 contrib/io/models.py:250
+msgid "Data Backend"
+msgstr ""
+
+#: contrib/io/models.py:207
+msgid "Data Backends"
+msgstr ""
+
+#: contrib/io/models.py:257
+msgid "Crontab Schedule"
+msgstr ""
+
+#: contrib/io/models.py:258
+msgid ""
+"Crontab Schedule to run the task on.  Set only one schedule type, leave the "
+"others null."
+msgstr ""
+
+#: contrib/io/models.py:299
+msgid "Source"
+msgstr ""
+
+#: contrib/io/models.py:300
+msgid "Sources"
+msgstr ""
+
+#: contrib/io/models.py:688
+msgid "Format of file to be exported"
+msgstr ""
+
+#: contrib/io/models.py:691
+msgid "Export job Content Type"
+msgstr ""
+
+#: contrib/io/models.py:694
+msgid "Resource path to use when exporting"
+msgstr ""
+
+#: contrib/io/models.py:697
+msgid "SQL query to be executed"
+msgstr ""
+
+#: contrib/io/models.py:699
+msgid "SQL query parameters to be used with the sql query"
+msgstr ""
+
+#: contrib/io/models.py:703
+msgid "Export Source"
+msgstr ""
+
+#: contrib/io/models.py:704
+msgid "Export Sources"
+msgstr ""
+
+#: contrib/io/models.py:761
+msgid "Your export file is available"
+msgstr ""
+
+#: contrib/io/models.py:762
+msgid ""
+"<p>The export job you requested is finished and available for one hour.</p>"
+msgstr ""
+
+#: contrib/io/models.py:813
+msgid "Import Source"
+msgstr ""
+
+#: contrib/io/models.py:814
+msgid "Import Sources"
+msgstr ""
+
+#: contrib/io/models.py:1022
+msgid "Import Credential"
+msgstr ""
+
+#: contrib/io/models.py:1023
+msgid "Import Credentials"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:240 contrib/io/viewset_mixins.py:241
+#: contrib/io/viewset_mixins.py:242
+msgid "Import"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:243
+msgid "Please provide a valid CSV file (Coma Separated Value)"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:258 contrib/io/viewset_mixins.py:259
+#: contrib/io/viewset_mixins.py:260
+msgid "Export"
+msgstr ""
+
+#: contrib/io/viewset_mixins.py:261
+msgid "Select the export format"
+msgstr ""

wbcore/contrib/io/migrations/0008_importsource_resource_kwargs.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.12 on 2025-02-18 09:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('io', '0007_alter_exportsource_query_params'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importsource',
+            name='resource_kwargs',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]

wbcore/contrib/io/models.py

@@ -1,4 +1,5 @@
 import importlib
+import io
 import json
 import logging
 import os
@@ -40,7 +41,7 @@ from wbcore.utils.importlib import import_from_dotted_path
 from wbcore.utils.models import ComplexToStringMixin
 
 from .enums import ExportFormat, get_django_import_export_format
-from .exceptions import ImportError
+from .signals import post_import
 
 logger = logging.getLogger("io")
 
@@ -60,6 +61,10 @@ class ParserHandler(models.Model):
     def __str__(self) -> str:
         return f"{self.parser}::{self.handler}"
 
+    @cached_property
+    def model(self):
+        return apps.get_model(self.handler)
+
     def parse(self, import_source: "ImportSource") -> Dict[str, Any]:
         """
         call the parser on the provided data
@@ -80,30 +85,29 @@ class ParserHandler(models.Model):
             parsed_data: The parsed_data
         """
         if parsed_data:
-            model: Any = apps.get_model(self.handler)
-            if handler_class := getattr(model, "import_export_handler_class", None):
+            if handler_class := getattr(self.model, "import_export_handler_class", None):
                 handler = handler_class(import_source)
                 handler.process(parsed_data, **kwargs)
 
     class Meta:
         unique_together = ("parser", "handler")
-        verbose_name = "Parser-Handler"
-        verbose_name_plural = "Parsers-Handlers"
+        verbose_name = _("Parser-Handler")
+        verbose_name_plural = _("Parsers-Handlers")
 
     @classmethod
-    def get_representation_value_key(self) -> str:
+    def get_representation_value_key(cls) -> str:
         return "id"
 
     @classmethod
-    def get_representation_label_key(self) -> str:
+    def get_representation_label_key(cls) -> str:
         return "{{parser}}::{{handler}}"
 
     @classmethod
-    def get_representation_endpoint(self) -> str:
+    def get_representation_endpoint(cls) -> str:
         return "wbcore:io:parserhandlerrepresentation-list"
 
 
-class ImportedObjectProviderRelationship(ComplexToStringMixin, models.Model):
+class ImportedObjectProviderRelationship(ComplexToStringMixin):
     """
     A model that represent the relationship/link between the imported object and a provider.
 
@@ -125,8 +129,8 @@ class ImportedObjectProviderRelationship(ComplexToStringMixin, models.Model):
             return f"{self.provider.title} ({self.provider_identifier})"
 
     class Meta:
-        verbose_name = "Content object Provider Identifier relationship"
-        verbose_name_plural = "Content object Provider Identifier relationships"
+        verbose_name = _("Content object Provider Identifier relationship")
+        verbose_name_plural = _("Content object Provider Identifier relationships")
         unique_together = [
             ("content_type", "object_id", "provider"),
             ("content_type", "provider_identifier", "provider"),
@@ -153,8 +157,8 @@ class Provider(models.Model):
         return f"{self.title} ({self.key})"
 
     class Meta:
-        verbose_name = "Provider"
-        verbose_name_plural = "Providers"
+        verbose_name = _("Provider")
+        verbose_name_plural = _("Providers")
 
 
 class DataBackend(models.Model):
@@ -198,8 +202,8 @@ class DataBackend(models.Model):
             return rel.content_object
 
     class Meta:
-        verbose_name = "Data Backend"
-        verbose_name_plural = "Data Backends"
+        verbose_name = _("Data Backend")
+        verbose_name_plural = _("Data Backends")
 
     @cached_property
     def backend_class(self) -> Any:
@@ -214,15 +218,15 @@ class DataBackend(models.Model):
         return self.title
 
     @classmethod
-    def get_representation_value_key(self) -> str:
+    def get_representation_value_key(cls) -> str:
         return "id"
 
     @classmethod
-    def get_representation_label_key(self) -> str:
+    def get_representation_label_key(cls) -> str:
         return "{{title}}"
 
     @classmethod
-    def get_representation_endpoint(self) -> str:
+    def get_representation_endpoint(cls) -> str:
         return "wbcore:io:databackendrepresentation-list"
 
 
@@ -291,8 +295,8 @@ class Source(models.Model):
         return self.title if self.title else f"{self.uuid}"
 
     class Meta:
-        verbose_name = "Source"
-        verbose_name_plural = "Sources"
+        verbose_name = _("Source")
+        verbose_name_plural = _("Sources")
 
     @property
     def current_valid_credential(self) -> "ImportCredential":
@@ -518,7 +522,7 @@ class Source(models.Model):
         return res
 
     @classmethod
-    def load_sources_from_settings(self, settings: list[tuple[list[tuple[str, str]], str, dict[str, Any]]]):
+    def load_sources_from_settings(cls, settings: list[tuple[list[tuple[str, str]], str, dict[str, Any]]]):
         """
         Utility classmethod to parser sources from the settings.
 
@@ -584,15 +588,15 @@ class Source(models.Model):
                 source.save()
 
     @classmethod
-    def get_representation_value_key(self) -> str:
+    def get_representation_value_key(cls) -> str:
         return "id"
 
     @classmethod
-    def get_representation_label_key(self) -> str:
+    def get_representation_label_key(cls) -> str:
         return "{{title}} ({{id}})"
 
     @classmethod
-    def get_representation_endpoint(self) -> str:
+    def get_representation_endpoint(cls) -> str:
         return "wbcore:io:sourcerepresentation-list"
 
 
@@ -635,15 +639,9 @@ def process_import_sources_as_task(import_source_ids: list[int]):
     """
     Call the `import_source` as a celery task
     """
-    errors = []
     for import_source_id in import_source_ids:
         import_source = ImportSource.objects.get(id=import_source_id)
-        try:
-            import_source.import_data(force_reimport=True)
-        except ImportError:
-            errors.append(import_source_id)
-    if len(errors) > 0:
-        raise ImportError(f"Error while processing import source {errors}")
+        import_source.import_data(force_reimport=True)
 
 
 class ImportExportSource(models.Model):
@@ -669,6 +667,7 @@ class ImportExportSource(models.Model):
         "authentication.User", null=True, blank=True, verbose_name="Creator", on_delete=models.SET_NULL
     )
     data = models.JSONField(default=dict, null=True, blank=True)
+    resource_kwargs = models.JSONField(default=dict, blank=True, null=False)
 
     class Meta:
         abstract = True
@@ -688,8 +687,6 @@ class ExportSource(ImportExportSource):
         verbose_name=_("Resource path to use when exporting"), max_length=255, default="", blank=True, null=True
     )
 
-    resource_kwargs = models.JSONField(default=dict, blank=True, null=False)
-
     query_str = models.TextField(verbose_name=_("SQL query to be executed"), blank=True, null=False)
     query_params = PickledObjectField(  # we have to use picklefield because the sql parameters are given as python object. However, no django model should be stored in this field, which should be the case as these objects are given through their IDs
         blank=True, verbose_name=_("SQL query parameters to be used with the sql query"), default=list
@@ -716,6 +713,9 @@ class ExportSource(ImportExportSource):
             )
         ]
 
+    def __str__(self) -> str:
+        return str(self.id)
+
     @property
     def file_format(self):
         return get_django_import_export_format(self.format)()
@@ -793,6 +793,7 @@ class ExportSource(ImportExportSource):
                 self.log = f"{ex_type}: {ex_value}\n"
                 self.log += traceback.format_exc()
                 self.save()
+                logger.error(f"Export source {self.id} error: {e}")
 
 
 class ImportSource(ImportExportSource):
@@ -806,8 +807,8 @@ class ImportSource(ImportExportSource):
     errors_log = models.TextField(null=True, blank=True)
 
     class Meta:
-        verbose_name = "Import Source"
-        verbose_name_plural = "Import Sources"
+        verbose_name = _("Import Source")
+        verbose_name_plural = _("Import Sources")
         notification_types = [
             create_notification_type(
                 code="io.import_done",
@@ -884,29 +885,48 @@ class ImportSource(ImportExportSource):
         try:
             data = self._parse_data()
             self._process_data(data, debug=debug, **kwargs)
+            post_import.send(sender=self.parser_handler.model, import_source=self)
+
         except Exception as e:
             ex_type, ex_value, _ = sys.exc_info()
             self.status = self.Status.ERROR.name
             self.log = f"{ex_type}: {ex_value}\n"
             self.log += traceback.format_exc()
             self.save()
-            if not debug:
-                raise ImportError(
-                    f"Could not import file {self.file.name} for backend {self.source.data_backend.title} and parser/handler {self.parser_handler}"
-                )
-            else:
+            if debug:
                 raise e
+            elif not self.creator:  # if a creator is set in this import source, they will receive a proper notification with feedback. No need then to pollute the logger
+                logger.error(f"Could not import file {self.file.name}  (parser/handler: {self.parser_handler})")
+        self.notify()
+
+    def notify(self):
+        if self.creator:
+            body = f"""<p><strong>File Name Processed:</strong> {self.file.name}</p>
+        <p><strong>Number of Rows:</strong> {self.progress_index}</p>
+        <p><strong>Status:</strong> {self.Status[self.status].label}</p>"""
+            if self.status == self.Status.ERROR:
+                body += "<p>While processing the import, we encountered an unrecoverable error. Please contact a system administrator.</p>"
+            elif self.status == self.Status.WARNING and self.errors_log:
+                body += f"""<p><strong>Warning:</strong> Some rows were ignored during import.</p>
+        <p><strong>Ignored Rows:</strong></p>
+        <ul>{"".join(['<li>' + line + '</li>' for line in io.StringIO(self.errors_log)])}</ul>"""
+            send_notification(
+                code="io.import_done",
+                title=f"Your import finished with status {self.Status[self.status].label}",
+                body=body,
+                user=self.creator,
+            )
 
     @classmethod
-    def get_representation_value_key(self) -> str:
+    def get_representation_value_key(cls) -> str:
         return "id"
 
     @classmethod
-    def get_representation_label_key(self) -> str:
+    def get_representation_label_key(cls) -> str:
         return "{{file}}"
 
     @classmethod
-    def get_representation_endpoint(self) -> str:
+    def get_representation_endpoint(cls) -> str:
         return "wbcore:io:importsourcerepresentation-list"
 
 
@@ -996,8 +1016,8 @@ class ImportCredential(models.Model):
         super().save(*args, **kwargs)
 
     class Meta:
-        verbose_name = "Import Credential"
-        verbose_name_plural = "Import Credentials"
+        verbose_name = _("Import Credential")
+        verbose_name_plural = _("Import Credentials")
 
     def __str__(self) -> str:
         dates_repr = ""

wbcore/contrib/io/resources.py

@@ -171,12 +171,6 @@ class ViewResource(ExportResourceMixin, resources.Resource):
         columns_map = {"id": "id"}
         # Get default fields from the list display
         if list_display := view.display_config_class(view, view.request, instance=None).get_list_display():
-            if list_display.tree_group_field:
-                columns_map[list_display.tree_group_field] = (
-                    list_display.tree_group_label
-                    if list_display.tree_group_label
-                    else list_display.tree_group_field.title()
-                )
             for key, label in list_display.flatten_fields:
                 columns_map[key] = label
         return columns_map

wbcore/contrib/io/serializers.py

@@ -88,8 +88,8 @@ class ImportSourceModelSerializer(ModelSerializer):
                 parser_lookup = dict(id=parser_handler)
             try:
                 data["parser_handler"] = ParserHandler.objects.get(**parser_lookup)
-            except (ParserHandler.DoesNotExist, ValueError):
-                raise ValidationError({"parser_handler": "Invalid parser handler"})
+            except (ParserHandler.DoesNotExist, ValueError) as e:
+                raise ValidationError({"parser_handler": "Invalid parser handler"}) from e
         return data
 
     def create(self, validated_data):

wbcore/contrib/io/signals.py

@@ -0,0 +1,4 @@
+from django.db.models.signals import ModelSignal
+
+# Signal fired after a import source was imported
+post_import = ModelSignal(use_caching=False)

wbcore/contrib/io/tests/test_backends.py

@@ -6,8 +6,6 @@ from anymail.signals import AnymailInboundEvent, EventType, inbound
 from django.utils import timezone
 from faker import Faker
 
-from ..import_export.backends.mail import DataBackend as MailDataBackend
-from ..import_export.backends.sftp import DataBackend as SFTPDataBackend
 from ..models import Source
 from .test_models import get_byte_stream
 
@@ -43,6 +41,8 @@ VFS = {
 @pytest.mark.django_db
 class TestBackend:
     def test_mail_backend(self):
+        from wbcore.contrib.io.import_export.backends.mail import DataBackend as MailDataBackend
+
         b1 = get_byte_stream()
         b2 = get_byte_stream()
         filename1 = "file_valid_12385.json"
@@ -69,15 +69,20 @@ class TestBackend:
         assert (res[0][1]).getvalue() == b1.getvalue()
 
     @pytest.mark.parametrize(
-        "source__import_parameters,data_backend__backend_class_path,from_email",
+        "source__import_parameters,from_email",
         [
-            ({"whitelisted_emails": ["test@test.ch"]}, "wbcore.contrib.io.backends.mail", "test@test.ch"),
-            ({"whitelisted_emails": ["test@test.ch"]}, "wbcore.contrib.io.backends.mail", "spam@test.ch"),
-            ({"whitelisted_emails": ["test@test.ch"]}, "wbcore.contrib.io.backends.unvalid", "test@test.ch"),
+            ({"whitelisted_emails": ["test@test.ch"]}, "test@test.ch"),
+            ({"whitelisted_emails": ["test@test.ch"]}, "spam@    test.ch"),
         ],
     )
-    @patch.object(Source, "generate_import_sources")
-    def test_inbound_mail(self, mock_process_source, source, data_backend, from_email):
+    @patch.object(Source, "trigger_workflow")
+    def test_inbound_mail(self, mock_process_source, source, from_email):
+        from wbcore.contrib.io.models import DataBackend
+
+        data_backend = DataBackend.objects.get(title="Default Mail")
+
+        assert data_backend.backend_class_path == "wbcore.contrib.io.import_export.backends.mail"
+
         source.data_backend = data_backend
         source.save()
         message = AnymailInboundMessage.construct(
@@ -95,23 +100,24 @@ class TestBackend:
             message=message,
         )
         inbound.send(sender="TEST", event=event, esp_name="test")
-        if (
-            source.data_backend.backend_class_path == "wbcore.contrib.io.backends.mail"
-            and source.import_parameters.get("whitelisted_emails", [None])[0] == from_email
-        ):
+        if source.import_parameters.get("whitelisted_emails", [None])[0] == from_email:
             assert mock_process_source.call_count == 1
         else:
             assert mock_process_source.call_count == 0
 
     def test_sftp_backend_without_credential(self):
+        from wbcore.contrib.io.import_export.backends.sftp import DataBackend as SFTPDataBackend
+
         with pytest.raises(ValueError):
             SFTPDataBackend()
 
     def test_sftp_backend(self, sftpserver, import_credential_factory):
+        from wbcore.contrib.io.import_export.backends.sftp import DataBackend as SFTPDataBackend
+
         with sftpserver.serve_content(VFS):
             import_credential = import_credential_factory.create(
                 username="user",
-                password="password",
+                password="password",  # noqa
                 additional_resources={
                     "host": sftpserver.host,
                     "port": sftpserver.port,