assemblyline-core 4.7.5.dev15__tar.gz → 4.7.5.dev16__tar.gz

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.5.dev15
+Version: 4.7.5.dev16
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.5.dev16/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.7.5.dev16

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev16}/assemblyline_core/expiry/run_expiry.py

@@ -14,7 +14,7 @@ from datemath import dm
 
 from assemblyline_core.server_base import ServerBase
 from assemblyline_core.dispatching.dispatcher import BAD_SID_HASH
-from assemblyline.common import forge
+from assemblyline.common import forge, chunk
 from assemblyline.common.isotime import epoch_to_iso, now_as_iso
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.filestore import FileStore
@@ -52,15 +52,20 @@ def _confirm_delete(store, from_archive, sha256: str) -> tuple[Optional[str], Op
 def _file_delete_worker(logger, filestore, archivestore, file_batch: list[tuple[str, bool]]) -> list[tuple[str, bool]]:
     finished_files: list[tuple[str, bool]] = []
     try:
-        archive_files = [sha256 for sha256, from_archive in file_batch if from_archive]
-        hot_files = [sha256 for sha256, from_archive in file_batch if not from_archive]
 
         with ThreadPoolExecutor(16) as pool:
             # Delete the two batches in parallel if needed
-            futures = [
-                pool.submit(filestore.delete_batch, hot_files),
-                pool.submit(archivestore.delete_batch, archive_files),
-            ]
+            futures: list[Future] = []
+            archive_files = [sha256 for sha256, from_archive in file_batch if from_archive]
+            hot_files = [sha256 for sha256, from_archive in file_batch if not from_archive]
+            futures.extend(
+                pool.submit(archivestore.delete_batch, batch)
+                for batch in chunk.chunk(archive_files, archivestore.delete_batch_chunk_size())
+            )
+            futures.extend(
+                pool.submit(filestore.delete_batch, batch)
+                for batch in chunk.chunk(hot_files, filestore.delete_batch_chunk_size())
+            )
 
             for future in as_completed(futures):
                 try:

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev16}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.5.dev15
+Version: 4.7.5.dev16
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.5.dev15/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.7.5.dev15