assemblyline-core 4.7.5.dev121__tar.gz → 4.7.5.dev123__tar.gz

{assemblyline_core-4.7.5.dev121 → assemblyline_core-4.7.5.dev123}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.5.dev121
+Version: 4.7.5.dev123
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.5.dev123/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.7.5.dev123

{assemblyline_core-4.7.5.dev121 → assemblyline_core-4.7.5.dev123}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -356,10 +356,11 @@ class KubernetesController(ControllerInterface):
         node_background = threading.Thread(target=self._loop_forever(self._monitor_node_pool), daemon=True)
         node_background.start()
 
-        self._pod_used_ram: dict[str, float] = defaultdict(float)
-        self._pod_used_cpu: dict[str, float] = defaultdict(float)
-        self._pod_used_namespace_ram: dict[str, float] = defaultdict(float)
-        self._pod_used_namespace_cpu: dict[str, float] = defaultdict(float)
+        self._pod_used_ram: dict[str | None, float] = defaultdict(float)
+        self._pod_used_cpu: dict[str | None, float] = defaultdict(float)
+        self._pod_used_namespace_ram: dict[str | None, float] = defaultdict(float)
+        self._pod_used_namespace_cpu: dict[str | None, float] = defaultdict(float)
+        self.node_count = 0
         pod_background = threading.Thread(target=self._loop_forever(self._monitor_pods), daemon=True)
         pod_background.start()
 
@@ -440,7 +441,7 @@ class KubernetesController(ControllerInterface):
         self._create_deployment(profile.name, self._deployment_name(profile.name),
                                 profile.container_config, profile.shutdown_seconds, scale,
                                 change_key=profile.config_blob, core_mounts=profile.privileged,
-                                security_context=self.security_policy),
+                                security_context=self.security_policy)
         self._external_profiles[profile.name] = profile
 
     def _loop_forever(self, function):
@@ -459,7 +460,7 @@ class KubernetesController(ControllerInterface):
                     self.logger.exception(f"Error in {function.__name__}")
         return _function
 
-    def _monitor_node_pool(self):
+    def _monitor_node_pool(self) -> None:
         self._node_pool_max_cpu = 0
         self._node_pool_max_ram = 0
         self.node_count = 0
@@ -497,19 +498,19 @@ class KubernetesController(ControllerInterface):
 
             # Update the totals
             self.node_count = len(self.ready_nodes)
-            max_cpu = 0
-            max_ram = 0
+            max_cpu = 0.0
+            max_ram = 0.0
             for cpu, ram in self.ready_nodes.values():
                 max_cpu += cpu
                 max_ram += ram
             self._node_pool_max_cpu = max_cpu
             self._node_pool_max_ram = max_ram
 
-    def _monitor_pods(self):
+    def _monitor_pods(self) -> None:
         watch = Watch()
         log_cache = CacheDict(cache_len=8000)
-        per_node_containers: dict[str, dict[str, tuple[float, float]]] = defaultdict(dict)
-        per_node_namespaced_containers: dict[str, dict[str, tuple[float, float]]] = defaultdict(dict)
+        per_node_containers: dict[str | None, dict[str, tuple[float, float]]] = defaultdict(dict)
+        per_node_namespaced_containers: dict[str | None, dict[str, tuple[float, float]]] = defaultdict(dict)
         self._pod_used_cpu = defaultdict(float)
         self._pod_used_ram = defaultdict(float)
         self._pod_used_namespace_cpu = defaultdict(float)
@@ -529,19 +530,34 @@ class KubernetesController(ControllerInterface):
                 continue
 
             pod_name = "Unknown Pod"
+            changed_nodes: list[str | None] = []
             try:
+                # Read some basic information about the pod we are looking at
                 pod_name = event['raw_object']['metadata']['name']
                 uid = event['raw_object']['metadata']['uid']
                 namespace = event['raw_object']['metadata']['namespace']
-                node = event['raw_object']['spec']['nodeName']
+                node = event['raw_object']['spec'].get('nodeName', None)
+                changed_nodes.append(node)
+
                 containers = per_node_containers[node]
                 namespaced_containers = per_node_namespaced_containers[node]
 
                 if event['type'] in ['ADDED', 'MODIFIED']:
                     for container in event['raw_object']['spec']['containers']:
-                        containers[f"{uid}-{container['name']}"] = get_resources(container)
+                        key = f"{uid}-{container['name']}"
+
+                        # If the pod has already been seen without a node assigned we will have records
+                        # for it under the None node, if we see it again with a non-null node we need to
+                        # clean up those duplicate records.
+                        if node is not None:
+                            per_node_namespaced_containers[None].pop(key, None)
+                            if per_node_containers[None].pop(key, None) is not None:
+                                changed_nodes.append(None)
+
+                        # Write resource records under the current node's container -> resource map
+                        containers[key] = get_resources(container)
                         if namespace == self.namespace:
-                            namespaced_containers[f"{uid}-{container['name']}"] = get_resources(container)
+                            namespaced_containers[key] = get_resources(container)
                     for status in event['raw_object']['status'].get('containerStatuses', []):
                         restarts = status['restartCount']
                         if restarts > CONTAINER_RESTART_THRESHOLD and log_cache.get(pod_name, 0) <= restarts:
@@ -553,8 +569,12 @@ class KubernetesController(ControllerInterface):
                                                 f"with reason {detail.get('reason', 'UNKNOWN')}")
                 elif event['type'] == 'DELETED':
                     for container in event['raw_object']['spec']['containers']:
-                        containers.pop(f"{uid}-{container['name']}", None)
-                        namespaced_containers.pop(f"{uid}-{container['name']}", None)
+                        key = f"{uid}-{container['name']}"
+                        containers.pop(key, None)
+                        namespaced_containers.pop(key, None)
+                        per_node_namespaced_containers[None].pop(key, None)
+                        if per_node_containers[None].pop(key, None) is not None:
+                            changed_nodes.append(None)
                 else:
                     continue
             except KeyError as e:
@@ -566,23 +586,27 @@ class KubernetesController(ControllerInterface):
                 self.logger.exception(f"Couldn't parse container information for {pod_name}: {e}")
                 continue
 
-            memory_unrestricted = sum(1 for _, mem in containers.values() if mem is None)
-            cpu_unrestricted = sum(1 for cpu, _ in containers.values() if cpu is None)
+            for node in changed_nodes:
+                containers = per_node_containers[node]
+                namespaced_containers = per_node_namespaced_containers[node]
+
+                memory_unrestricted = sum(1 for _, mem in containers.values() if mem is None)
+                cpu_unrestricted = sum(1 for cpu, _ in containers.values() if cpu is None)
 
-            memory_used = [mem for _, mem in containers.values() if mem is not None]
-            cpu_used = [cpu for cpu, _ in containers.values() if cpu is not None]
+                memory_used = [mem for _, mem in containers.values() if mem is not None]
+                cpu_used = [cpu for cpu, _ in containers.values() if cpu is not None]
 
-            self._pod_used_cpu[node] = sum(cpu_used) + cpu_unrestricted * mean(cpu_used)
-            self._pod_used_ram[node] = sum(memory_used) + memory_unrestricted * mean(memory_used)
+                self._pod_used_cpu[node] = sum(cpu_used) + cpu_unrestricted * mean(cpu_used)
+                self._pod_used_ram[node] = sum(memory_used) + memory_unrestricted * mean(memory_used)
 
-            memory_unrestricted = sum(1 for _, mem in namespaced_containers.values() if mem is None)
-            cpu_unrestricted = sum(1 for cpu, _ in namespaced_containers.values() if cpu is None)
+                memory_unrestricted = sum(1 for _, mem in namespaced_containers.values() if mem is None)
+                cpu_unrestricted = sum(1 for cpu, _ in namespaced_containers.values() if cpu is None)
 
-            memory_used = [mem for _, mem in namespaced_containers.values() if mem is not None]
-            cpu_used = [cpu for cpu, _ in namespaced_containers.values() if cpu is not None]
+                memory_used = [mem for _, mem in namespaced_containers.values() if mem is not None]
+                cpu_used = [cpu for cpu, _ in namespaced_containers.values() if cpu is not None]
 
-            self._pod_used_namespace_cpu[node] = sum(cpu_used) + cpu_unrestricted * mean(cpu_used)
-            self._pod_used_namespace_ram[node] = sum(memory_used) + memory_unrestricted * mean(memory_used)
+                self._pod_used_namespace_cpu[node] = sum(cpu_used) + cpu_unrestricted * mean(cpu_used)
+                self._pod_used_namespace_ram[node] = sum(memory_used) + memory_unrestricted * mean(memory_used)
 
     def _monitor_quotas(self):
         watch = Watch()

{assemblyline_core-4.7.5.dev121 → assemblyline_core-4.7.5.dev123}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.5.dev121
+Version: 4.7.5.dev123
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.5.dev121/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.7.5.dev121