wbcore 1.58.2__py2.py3-none-any.whl → 1.58.3__py2.py3-none-any.whl

wbcore/contrib/io/dynamic_preferences_registry.py

@@ -1,6 +1,6 @@
 from dynamic_preferences.preferences import Section
 from dynamic_preferences.registries import global_preferences_registry
-from dynamic_preferences.types import StringPreference
+from dynamic_preferences.types import IntegerPreference, StringPreference
 
 io = Section("io")
 
@@ -13,3 +13,13 @@ class AdministratorMailsPreference(StringPreference):
 
     verbose_name = "Administrator mails"
     help_text = "The mails (as comma separated string) of the person allow to send directly to the mail backend"
+
+
+@global_preferences_registry.register
+class ImportSourceDataRetentionPeriod(IntegerPreference):
+    section = io
+    name = "import_source_retention_period"
+    default = 365
+
+    verbose_name = "Import Source Retention Period"
+    help_text = "The number of days to keep the data and log info until cleanup"

wbcore/contrib/io/models.py

@@ -515,7 +515,9 @@ class Source(models.Model):
                         import_source = ImportSource.objects.create(
                             file=content_file,
                             parser_handler=parser_handler,
-                            save_data=self.data_backend.save_data_in_import_source,
+                            save_data=self.import_parameters.get(
+                                "save_data_in_import_source", self.data_backend.save_data_in_import_source
+                            ),
                             source=self,
                         )
                         res.append(import_source)
@@ -863,6 +865,9 @@ class ImportSource(ImportExportSource):
                 self.parser_handler.handle(self, {"data": data[self.progress_index :], **parsed_data_copy}, **kwargs)
                 if self.errors_log:
                     self.status = self.Status.WARNING.name
+        # clean log if we don't want to use space to save it
+        if not self.save_data:
+            self.log = ""
         self.save()
 
     def import_data(self, force_reimport: Optional[bool] = False, **kwargs):
@@ -897,6 +902,7 @@ class ImportSource(ImportExportSource):
                 raise e
             elif not self.creator:  # if a creator is set in this import source, they will receive a proper notification with feedback. No need then to pollute the logger
                 logger.error(f"Could not import file {self.file.name}  (parser/handler: {self.parser_handler})")
+
         self.notify()
 
     def notify(self):

wbcore/contrib/io/tasks.py

@@ -0,0 +1,23 @@
+from datetime import date, timedelta
+
+from django.db import connection
+from django.utils import timezone
+from dynamic_preferences.registries import global_preferences_registry
+
+
+def clean_up_import_source(today: date | None = None):
+    if not today:
+        today = timezone.now()
+    global_preferences = global_preferences_registry.manager()
+    retention_period = global_preferences["io__import_source_retention_period"]
+    retention_until = today - timedelta(days=retention_period)
+    with connection.cursor() as cursor:
+        cursor.execute(
+            """
+            UPDATE io_importsource
+            SET data = '{}'::jsonb,
+                log = ''
+            WHERE created < %s;
+        """,
+            (retention_until,),
+        )

{wbcore-1.58.2.dist-info → wbcore-1.58.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wbcore
-Version: 1.58.2
+Version: 1.58.3
 Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
 Requires-Dist: boto3==1.35.*
 Requires-Dist: celery[redis]==5.*

{wbcore-1.58.2.dist-info → wbcore-1.58.3.dist-info}/RECORD

@@ -664,16 +664,17 @@ wbcore/contrib/icons/backends/material.py,sha256=hsj_KRjTgHDIFJh4BDWGkx75SmtxklQ
 wbcore/contrib/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wbcore/contrib/io/admin.py,sha256=Xd6LplibZI4azDt0GvS1-vHaMIGT2TDfBufU0LV2YEw,4813
 wbcore/contrib/io/apps.py,sha256=6k6kj1S5ytYSg8YLWAkuRu39BpZJUuCd4RPH1PmaCng,1304
-wbcore/contrib/io/dynamic_preferences_registry.py,sha256=8dyFqiacvwUZOgQl9gvKfGvsINXfcuyCvO7FBYxMRCA,514
+wbcore/contrib/io/dynamic_preferences_registry.py,sha256=9asttbLsmI0x64QGokmLTrs2oCOdLqSYB55mYHmQBPA,844
 wbcore/contrib/io/enums.py,sha256=9Ph2hUn-cjfvTIhPGxlC07qubPS6R3B9ilvC3tpMm8o,417
 wbcore/contrib/io/exceptions.py,sha256=-9pTtBr4oj7qBpKwnsN7sabu5S6gpDkWTXkA4ZaW9EE,690
 wbcore/contrib/io/factories.py,sha256=U5ppoOMM4yd6pla_bscWo9v7ug5W_bWV29ZL4PKLZsU,7053
 wbcore/contrib/io/imports.py,sha256=Hu8ppai06SQ_CDQ2oUbFcwduAhekCp1l1DB89kTn2nQ,13087
 wbcore/contrib/io/mixins.py,sha256=Sy_1mfdJzrIODCRcbfiA6miU8EqKEaJhL7mEjsRhOvY,1297
-wbcore/contrib/io/models.py,sha256=oYGMz0tbR5Eyul-J1tOSqjbMaNDjaOvB56YrZZhPYd8,40230
+wbcore/contrib/io/models.py,sha256=N5umz_bf5PtmrBYhJF5qTGLK-NPx-WbA-tkSVMF1bi0,40469
 wbcore/contrib/io/resources.py,sha256=eGEpmyrtkB3DgFKV6m57OFzyu6jBZUIXkn5Jopeus9M,6953
 wbcore/contrib/io/serializers.py,sha256=oS5od8ni8wUZml1zM_RAdW9VWrw226Ru4v3RBifOnFY,4639
 wbcore/contrib/io/signals.py,sha256=jCGHjt5Qg2T1aIi4BzWYzWYb2YZT82gUMhG68v2rx58,145
+wbcore/contrib/io/tasks.py,sha256=5qCWhoQkqV4dtjoT-odP9FI_Vd5n0l7d3F_7QTPnazM,751
 wbcore/contrib/io/urls.py,sha256=v91WUMYKwQhI9mDnpScP6hvYdGSdQZg0yQzEljCZ4lk,1014
 wbcore/contrib/io/utils.py,sha256=SMjQeUEg_cuBKQfnKGj-qjJDC8Z_xCOQ-t7VZxwCZt4,1359
 wbcore/contrib/io/viewset_mixins.py,sha256=RhO8TE9RPQsubSd2H2XgIJLD92tHu-7_gO8JwROxtHI,11419
@@ -1230,6 +1231,6 @@ wbcore/viewsets/generics.py,sha256=lKDq9UY_Tyc56u1bqaIEvHGgoaXwXxpZ1c3fLVteptI,1
 wbcore/viewsets/mixins.py,sha256=IdHd_uixOv3ExKoHxTgL5Bt8OELIwfYwhBZm0nsvZfc,12054
 wbcore/viewsets/utils.py,sha256=4520Ij3ASM8lOa8QZkCqbBfOexVRiZu688eW-PGqMOA,882
 wbcore/viewsets/viewsets.py,sha256=FPPESunEjlunDr5VFsjTfsquTS3iDSQkw0H6QjMKPqk,6574
-wbcore-1.58.2.dist-info/METADATA,sha256=55SHGonapoL91aKV0bahOOjadcPCIg4vaL7gtXAzPwI,2332
-wbcore-1.58.2.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
-wbcore-1.58.2.dist-info/RECORD,,
+wbcore-1.58.3.dist-info/METADATA,sha256=Cd0pn3fhP4PMkxnWcJ-4t96sWW1pV0C2fSqMT8eRYPk,2332
+wbcore-1.58.3.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
+wbcore-1.58.3.dist-info/RECORD,,