assemblyline-core 4.3.2.dev72__tar.gz → 4.4.0.0__tar.gz

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.3.2.dev72
+Version: 4.4.0.0
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.4.0.0/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.4.0.0

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/assemblyline_core/dispatching/dispatcher.py

@@ -529,7 +529,8 @@ class Dispatcher(ThreadedCoreBase):
         task.file_names[sha256] = submission.files[0].name or sha256
         # Initialize ancestry chain by identifying the root file
         task.file_temporary_data[sha256]['ancestry'] = [[dict(type=self.datastore.file.get(sha256).type,
-                                                              parent_relation="ROOT")]]
+                                                              parent_relation="ROOT",
+                                                              sha256=sha256)]]
         task.active_files.add(sha256)
         action = DispatchAction(kind=Action.dispatch_file, sid=sid, sha=sha256)
         self.find_process_queue(sid).put(action)
@@ -1249,7 +1250,7 @@ class Dispatcher(ThreadedCoreBase):
                     parent_ancestry = parent_data['ancestry']
                     existing_ancestry = task.file_temporary_data.get(extracted_sha256, {}).get('ancestry', [])
                     current_ancestry_node = dict(type=self.datastore.file.get(extracted_sha256).type,
-                                                 parent_relation=parent_relation)
+                                                 parent_relation=parent_relation, sha256=extracted_sha256)
 
                     task.file_temporary_data[extracted_sha256] = dict(parent_data)
                     task.file_temporary_data[extracted_sha256]['ancestry'] = existing_ancestry

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/assemblyline_core/replay/loader/run.py

@@ -84,8 +84,8 @@ class ReplayLoader(ReplayBase):
         self.maintain_threads(threads)
 
     def stop(self):
-        super().stop()
         self.cache.close()
+        return super().stop()
 
 
 if __name__ == '__main__':

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -507,6 +507,9 @@ class KubernetesController(ControllerInterface):
         for event in watch.stream(func=self.apps_api.list_namespaced_deployment,
                                   namespace=self.namespace, label_selector=label_selector,
                                   timeout_seconds=WATCH_TIMEOUT, _request_timeout=WATCH_API_TIMEOUT):
+            if 'dependency_for' in event['raw_object']['metadata']['labels']:
+                continue
+
             if event['type'] in ['ADDED', 'MODIFIED']:
                 name = event['raw_object']['metadata']['labels'].get('component', None)
                 if name is not None:
@@ -769,6 +772,7 @@ class KubernetesController(ControllerInterface):
                         self.add_profile(profile, scale=target)
                     return
                 raise
+        self.logger.error(f"Repeated conflict scaling {service_name} will not retry.")
 
     def stop_container(self, service_name, container_id):
         try:

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/assemblyline_core/scaler/scaler_server.py

@@ -24,7 +24,7 @@ import yaml
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline.remote.datatypes.queues.priority import PriorityQueue, length as pq_length
 from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
-from assemblyline.remote.datatypes.hash import ExpiringHash
+from assemblyline.remote.datatypes.hash import ExpiringHash, Hash
 from assemblyline.remote.datatypes.events import EventWatcher, EventSender
 from assemblyline.odm.models.service import Service, DockerConfig, EnvironmentVariable
 from assemblyline.odm.messages.scaler_heartbeat import Metrics
@@ -628,6 +628,9 @@ class ScalerServer(ThreadedCoreBase):
                 'dependency_for': name
             })
 
+            # Clear related dependency caching from Redis
+            Hash(f'service-updates-{name}', self.redis_persist).delete()
+
             # Mark this service as not running in the shared record
             self._service_stage_hash.set(name, ServiceStage.Off)
 
@@ -660,7 +663,10 @@ class ScalerServer(ThreadedCoreBase):
                     with self.profiles_lock:
                         all_profiles: dict[str, ServiceProfile] = copy.deepcopy(self.profiles)
                     raw_targets = self.controller.get_targets()
+                    # This is the list of targets we will adjust
                     targets = {_p.name: raw_targets.get(_p.name, 0) for _p in all_profiles.values()}
+                    # This represents what the environment is currently running
+                    old_targets = dict(targets)
 
                 for name, profile in all_profiles.items():
                     self.log.debug(f'{name}')
@@ -707,7 +713,8 @@ class ScalerServer(ThreadedCoreBase):
                 used_memory = total_memory - free_memory
                 free_memory = total_memory * self.get_memory_overallocation() - used_memory
 
-                #
+                # Make adjustments to the targets until everything is satisified
+                # or we don't have the resouces to make more adjustments
                 def trim(prof: list[ServiceProfile]):
                     prof = [_p for _p in prof if _p.desired_instances > targets[_p.name]]
                     drop = [_p for _p in prof if _p.cpu > free_cpu or _p.ram > free_memory]
@@ -718,10 +725,6 @@ class ScalerServer(ThreadedCoreBase):
                     return prof
 
                 remaining_profiles: list[ServiceProfile] = trim(list(all_profiles.values()))
-                # The target values up until now should be in sync with the container orchestrator
-                # create a copy, so we can track which ones change in the following loop
-                old_targets = dict(targets)
-
                 while remaining_profiles:
                     # TODO do we need to add balancing metrics other than 'least running' for this? probably
                     remaining_profiles.sort(key=lambda _p: targets[_p.name])
@@ -744,6 +747,7 @@ class ScalerServer(ThreadedCoreBase):
                                 continue
                             self.profiles[name].target_instances = value
                             old = old_targets[name]
+
                             if value != old:
                                 self.log.info(f"Scaling service {name}: {old} -> {value}")
                                 pool.call(self.controller.set_target, name, value)

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.3.2.dev72
+Version: 4.4.0.0
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.3.2.dev72 → assemblyline-core-4.4.0.0}/test/test_dispatcher.py

@@ -161,7 +161,7 @@ def test_simple(clean_redis, clean_datastore):
     job = client.request_work('0', 'extract', '0')
     assert job.temporary_submission_data == [
         {'name': 'cats', 'value': 'big'},
-        {"name": "ancestry", "value": [[{"type": "unknown", "parent_relation": "ROOT"}]]}
+        {"name": "ancestry", "value": [[{"type": "unknown", "parent_relation": "ROOT", "sha256": file.sha256}]]}
     ]
     client.service_failed(sid, 'abc123', make_error(file_hash, 'extract'))
     # Deliberately do in the wrong order to make sure that works

assemblyline-core-4.3.2.dev72/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.3.2.dev72