assemblyline-core 4.7.5.dev15__tar.gz → 4.7.5.dev17__tar.gz

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.5.dev15
+Version: 4.7.5.dev17
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.5.dev17/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.7.5.dev17

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev17}/assemblyline_core/expiry/run_expiry.py

@@ -14,7 +14,7 @@ from datemath import dm
 
 from assemblyline_core.server_base import ServerBase
 from assemblyline_core.dispatching.dispatcher import BAD_SID_HASH
-from assemblyline.common import forge
+from assemblyline.common import forge, chunk
 from assemblyline.common.isotime import epoch_to_iso, now_as_iso
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.filestore import FileStore
@@ -52,15 +52,20 @@ def _confirm_delete(store, from_archive, sha256: str) -> tuple[Optional[str], Op
 def _file_delete_worker(logger, filestore, archivestore, file_batch: list[tuple[str, bool]]) -> list[tuple[str, bool]]:
     finished_files: list[tuple[str, bool]] = []
     try:
-        archive_files = [sha256 for sha256, from_archive in file_batch if from_archive]
-        hot_files = [sha256 for sha256, from_archive in file_batch if not from_archive]
 
         with ThreadPoolExecutor(16) as pool:
             # Delete the two batches in parallel if needed
-            futures = [
-                pool.submit(filestore.delete_batch, hot_files),
-                pool.submit(archivestore.delete_batch, archive_files),
-            ]
+            futures: list[Future] = []
+            archive_files = [sha256 for sha256, from_archive in file_batch if from_archive]
+            hot_files = [sha256 for sha256, from_archive in file_batch if not from_archive]
+            futures.extend(
+                pool.submit(archivestore.delete_batch, batch)
+                for batch in chunk.chunk(archive_files, archivestore.delete_batch_chunk_size())
+            )
+            futures.extend(
+                pool.submit(filestore.delete_batch, batch)
+                for batch in chunk.chunk(hot_files, filestore.delete_batch_chunk_size())
+            )
 
             for future in as_completed(futures):
                 try:

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev17}/assemblyline_core/scaler/scaler_server.py

@@ -314,6 +314,7 @@ class ScalerServer(ThreadedCoreBase):
         priv_labels = {}
 
         service_defaults_config = self.config.core.scaler.service_defaults
+        self.max_pending = max(1, self.config.core.scaler.max_pending)
 
         # If Scaler has envs that set service-server env, then that should override configured values
         if SERVICE_API_HOST:
@@ -807,7 +808,10 @@ class ScalerServer(ThreadedCoreBase):
                 # Make adjustments to the targets until everything is satisified
                 # or we don't have the resouces to make more adjustments
                 def trim(prof: list[ServiceProfile]):
+                    # only consider services where more are desired
                     prof = [_p for _p in prof if _p.desired_instances > targets[_p.name]]
+                    # only consider services where there is room for more pending containers
+                    prof = [_p for _p in prof if _p.running_instances + self.max_pending > targets[_p.name]]
                     drop = [_p for _p in prof if _p.cpu > free_cpu or _p.ram > free_memory]
                     if drop:
                         summary = {_p.name: (_p.cpu, _p.ram) for _p in drop}
@@ -817,7 +821,7 @@ class ScalerServer(ThreadedCoreBase):
 
                 remaining_profiles: list[ServiceProfile] = trim(list(all_profiles.values()))
                 while remaining_profiles:
-                    # TODO do we need to add balancing metrics other than 'least running' for this? probably
+                    # Prioritize by adding resources to services with the least instances requested first.
                     remaining_profiles.sort(key=lambda _p: targets[_p.name])
 
                     # Add one for the profile at the bottom

{assemblyline_core-4.7.5.dev15 → assemblyline_core-4.7.5.dev17}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.5.dev15
+Version: 4.7.5.dev17
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.5.dev15/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.7.5.dev15