assemblyline-core 4.6.0.dev7__tar.gz → 4.6.0.dev10__tar.gz

{assemblyline_core-4.6.0.dev7 → assemblyline_core-4.6.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.0.dev7
+Version: 4.6.0.dev10
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.6.0.dev10/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.6.0.dev10

{assemblyline_core-4.6.0.dev7 → assemblyline_core-4.6.0.dev10}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -1,36 +1,73 @@
 from __future__ import annotations
+
 import base64
 import functools
 import json
-import uuid
 import os
 import threading
+import uuid
 import weakref
-import urllib3
-
 from base64 import b64encode
-from cryptography import x509
-from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography.hazmat.primitives import serialization, hashes
 from collections import OrderedDict, defaultdict
 from datetime import datetime, timedelta
-from dateutil.tz import tzlocal
-from typing import List, Optional, Tuple
 from time import sleep
-from assemblyline.odm.models.config import Selector
+from typing import List, Optional, Tuple
 
+import urllib3
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from dateutil.tz import tzlocal
 from kubernetes import client, config, watch
-from kubernetes.client import V1Deployment, V1DeploymentSpec, V1PodTemplateSpec, V1DeploymentStrategy, \
-    V1PodSpec, V1ObjectMeta, V1Volume, V1Container, V1VolumeMount, V1EnvVar, V1ConfigMapVolumeSource, \
-    V1PersistentVolumeClaimVolumeSource, V1LabelSelector, V1ResourceRequirements, V1PersistentVolumeClaim, \
-    V1PersistentVolumeClaimSpec, V1NetworkPolicy, V1NetworkPolicySpec, V1NetworkPolicyEgressRule, V1NetworkPolicyPeer, \
-    V1NetworkPolicyIngressRule, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Service, \
-    V1ServiceSpec, V1ServicePort, V1PodSecurityContext, V1Probe, V1ExecAction, V1SecurityContext, \
-    V1Affinity, V1NodeAffinity, V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, V1Toleration, \
-    V1Capabilities, V1SeccompProfile
+from kubernetes.client import (
+    V1Affinity,
+    V1Capabilities,
+    V1ConfigMapVolumeSource,
+    V1Container,
+    V1Deployment,
+    V1DeploymentSpec,
+    V1DeploymentStrategy,
+    V1EnvVar,
+    V1ExecAction,
+    V1LabelSelector,
+    V1LocalObjectReference,
+    V1NetworkPolicy,
+    V1NetworkPolicyEgressRule,
+    V1NetworkPolicyIngressRule,
+    V1NetworkPolicyPeer,
+    V1NetworkPolicySpec,
+    V1NodeAffinity,
+    V1NodeSelector,
+    V1NodeSelectorRequirement,
+    V1NodeSelectorTerm,
+    V1ObjectMeta,
+    V1PersistentVolumeClaim,
+    V1PersistentVolumeClaimSpec,
+    V1PersistentVolumeClaimVolumeSource,
+    V1PodSecurityContext,
+    V1PodSpec,
+    V1PodTemplateSpec,
+    V1Probe,
+    V1ResourceRequirements,
+    V1SeccompProfile,
+    V1Secret,
+    V1SecretVolumeSource,
+    V1SecurityContext,
+    V1Service,
+    V1ServicePort,
+    V1ServiceSpec,
+    V1Toleration,
+    V1Volume,
+    V1VolumeMount,
+)
 from kubernetes.client.rest import ApiException
-from assemblyline.odm.models.service import DependencyConfig, DockerConfig, PersistentVolume
 
+from assemblyline.odm.models.config import Selector
+from assemblyline.odm.models.service import (
+    DependencyConfig,
+    DockerConfig,
+    PersistentVolume,
+)
 from assemblyline_core.scaler.controllers.interface import ControllerInterface
 
 # RESERVE_MEMORY_PER_NODE = os.environ.get('RESERVE_MEMORY_PER_NODE')
@@ -250,7 +287,7 @@ def parse_cpu(string: str) -> float:
 class KubernetesController(ControllerInterface):
     def __init__(self, logger, namespace: str, prefix: str, priority: str, dependency_priority: str,
                  cpu_reservation: float, linux_node_selector: Selector, labels=None, log_level="INFO", core_env={},
-                 default_service_account=None, cluster_pod_list=True, enable_pod_security=False,
+                 cluster_pod_list=True, enable_pod_security=False,
                  default_service_tolerations=[],
                  priv_labels=None):
         # Try loading a kubernetes connection from either the fact that we are running
@@ -295,7 +332,6 @@ class KubernetesController(ControllerInterface):
         self.core_mounts: dict[str, V1VolumeMount] = {}
         self._external_profiles = weakref.WeakValueDictionary()
         self._service_limited_env: dict[str, dict[str, str]] = defaultdict(dict)
-        self.default_service_account: Optional[str] = default_service_account
         self.cluster_pod_list = cluster_pod_list
         self.security_policy = RESTRICTED_POD_SECURITY_CONTEXT if enable_pod_security else None
         self.default_service_tolerations = [V1Toleration(**toleration.as_primitives()) for toleration in default_service_tolerations]
@@ -836,8 +872,7 @@ class KubernetesController(ControllerInterface):
         metadata = V1ObjectMeta(name=deployment_name, labels=all_labels, annotations={CHANGE_KEY_NAME: change_key})
 
         # Figure out which (if any) service account to use
-        service_account = self.default_service_account or \
-            (PRIVILEGED_SERVICE_ACCOUNT_NAME if core_mounts else UNPRIVILEGED_SERVICE_ACCOUNT_NAME)
+        service_account = PRIVILEGED_SERVICE_ACCOUNT_NAME if core_mounts else UNPRIVILEGED_SERVICE_ACCOUNT_NAME
         if docker_config.service_account:
             service_account = docker_config.service_account
 

{assemblyline_core-4.6.0.dev7 → assemblyline_core-4.6.0.dev10}/assemblyline_core/scaler/scaler_server.py

@@ -2,45 +2,55 @@
 An auto-scaling service specific to Assemblyline services.
 """
 from __future__ import annotations
+
+import concurrent.futures
+import copy
 import functools
-import threading
-from collections import defaultdict
-from string import Template
-from typing import Dict, Optional, Any
-import os
+import json
 import math
-import time
+import os
 import platform
-import concurrent.futures
-import copy
+import threading
+import time
+from collections import defaultdict
 from contextlib import contextmanager
+from string import Template
+from typing import Any, Dict, Optional
 
 import elasticapm
-import json
 import yaml
 
-from assemblyline.remote.datatypes.queues.named import NamedQueue
-from assemblyline.remote.datatypes.queues.priority import PriorityQueue, length as pq_length
-from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
-from assemblyline.remote.datatypes.hash import ExpiringHash, Hash
-from assemblyline.remote.datatypes.events import EventWatcher, EventSender
-from assemblyline.odm.models.service import Service, DockerConfig, EnvironmentVariable
-from assemblyline.odm.models.config import Mount
-from assemblyline.odm.messages.scaler_heartbeat import Metrics
-from assemblyline.odm.messages.scaler_status_heartbeat import Status
-from assemblyline.odm.messages.changes import ServiceChange, Operation
-from assemblyline.common.dict_utils import get_recursive_sorted_tuples, flatten
+from assemblyline.common.constants import (
+    SCALER_TIMEOUT_QUEUE,
+    SERVICE_STATE_HASH,
+    ServiceStatus,
+)
+from assemblyline.common.dict_utils import flatten, get_recursive_sorted_tuples
+from assemblyline.common.forge import (
+    get_apm_client,
+    get_classification,
+    get_service_queue,
+)
 from assemblyline.common.uid import get_id_from_data
-from assemblyline.common.forge import get_classification, get_service_queue, get_apm_client
-from assemblyline.common.constants import SCALER_TIMEOUT_QUEUE, SERVICE_STATE_HASH, ServiceStatus
 from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
-from assemblyline_core.updater.helper import get_registry_config
+from assemblyline.odm.messages.changes import Operation, ServiceChange
+from assemblyline.odm.messages.scaler_heartbeat import Metrics
+from assemblyline.odm.messages.scaler_status_heartbeat import Status
+from assemblyline.odm.models.config import Mount
+from assemblyline.odm.models.service import DockerConfig, EnvironmentVariable, Service
+from assemblyline.remote.datatypes.events import EventSender, EventWatcher
+from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
+from assemblyline.remote.datatypes.hash import ExpiringHash, Hash
+from assemblyline.remote.datatypes.queues.named import NamedQueue
+from assemblyline.remote.datatypes.queues.priority import PriorityQueue
+from assemblyline.remote.datatypes.queues.priority import length as pq_length
 from assemblyline_core.scaler.controllers import KubernetesController
 from assemblyline_core.scaler.controllers.interface import ServiceControlError
 from assemblyline_core.server_base import ServiceStage, ThreadedCoreBase
+from assemblyline_core.updater.helper import get_registry_config
 
-from .controllers import DockerController
 from . import collection
+from .controllers import DockerController
 
 APM_SPAN_TYPE = 'scaler'
 
@@ -325,7 +335,6 @@ class ScalerServer(ThreadedCoreBase):
                                                    core_env=core_env,
                                                    cluster_pod_list=self.config.core.scaler.cluster_pod_list,
                                                    enable_pod_security=self.config.core.scaler.enable_pod_security,
-                                                   default_service_account=self.config.services.service_account,
                                                    default_service_tolerations=service_defaults_config.tolerations,
                                                    priv_labels=priv_labels
                                                    )
@@ -347,14 +356,6 @@ class ScalerServer(ThreadedCoreBase):
 
             # Add default mounts for (non-)privileged services
             for mount in service_defaults_config.mounts:
-                # Deprecated configuration for mounting ConfigMap
-                # TODO: Deprecate code on next major change
-                if mount.config_map:
-                    self.controller.add_config_mount(mount.name, config_map=mount.config_map, key=mount.key,
-                                                     target_path=mount.path, read_only=mount.read_only,
-                                                     core=mount.privileged_only)
-                    continue
-
                 if mount.resource_type == 'configmap':
                     # ConfigMap-based mount
                     self.controller.add_config_mount(mount.name, config_map=mount.resource_name, key=mount.resource_key,

{assemblyline_core-4.6.0.dev7 → assemblyline_core-4.6.0.dev10}/assemblyline_core/updater/run_updater.py

@@ -7,17 +7,31 @@ import os
 import re
 import time
 import uuid
-
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, List, Optional
 
 import docker
-
-from kubernetes.client import V1Job, V1ObjectMeta, V1JobSpec, V1PodTemplateSpec, V1PodSpec, V1Volume, \
-    V1VolumeMount, V1EnvVar, V1Container, V1ResourceRequirements, \
-    V1ConfigMapVolumeSource, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Toleration, V1SecurityContext, \
-    V1Capabilities, V1SeccompProfile
 from kubernetes import client, config
+from kubernetes.client import (
+    V1Capabilities,
+    V1ConfigMapVolumeSource,
+    V1Container,
+    V1EnvVar,
+    V1Job,
+    V1JobSpec,
+    V1LocalObjectReference,
+    V1ObjectMeta,
+    V1PodSpec,
+    V1PodTemplateSpec,
+    V1ResourceRequirements,
+    V1SeccompProfile,
+    V1Secret,
+    V1SecretVolumeSource,
+    V1SecurityContext,
+    V1Toleration,
+    V1Volume,
+    V1VolumeMount,
+)
 from kubernetes.client.rest import ApiException
 
 from assemblyline.common import isotime
@@ -26,7 +40,11 @@ from assemblyline.odm.models.config import Mount, Selector
 from assemblyline.odm.models.service import DockerConfig, Service
 from assemblyline.remote.datatypes.events import EventSender, EventWatcher
 from assemblyline.remote.datatypes.hash import Hash
-from assemblyline_core.scaler.controllers.kubernetes_ctl import create_docker_auth_config, selector_to_node_affinity, PRIVILEGED_SERVICE_ACCOUNT_NAME
+from assemblyline_core.scaler.controllers.kubernetes_ctl import (
+    PRIVILEGED_SERVICE_ACCOUNT_NAME,
+    create_docker_auth_config,
+    selector_to_node_affinity,
+)
 from assemblyline_core.server_base import ThreadedCoreBase
 from assemblyline_core.updater.helper import get_latest_tag_for_service
 
@@ -157,7 +175,7 @@ class DockerUpdateInterface:
 
 class KubernetesUpdateInterface:
     def __init__(self, logger, prefix, namespace, priority_class, extra_labels, linux_node_selector: Selector,
-                 log_level="INFO", default_service_account=None, default_service_tolerations=[], enable_pod_security=False):
+                 log_level="INFO", default_service_tolerations=[], enable_pod_security=False):
         # Try loading a kubernetes connection from either the fact that we are running
         # inside of a cluster, or we have a configuration in the normal location
         try:
@@ -187,7 +205,6 @@ class KubernetesUpdateInterface:
         self.priority_class = priority_class
         self.extra_labels = extra_labels
         self.log_level = log_level
-        self.default_service_account = default_service_account
         self.secret_env = []
         self.linux_node_selector = linux_node_selector
         self.default_service_tolerations = [V1Toleration(**toleration.as_primitives()) for toleration in default_service_tolerations]
@@ -268,13 +285,7 @@ class KubernetesUpdateInterface:
                 read_only=mount.read_only,
             )
 
-            if mount.config_map:
-                # Deprecated configuration for mounting ConfigMap
-                # TODO: Deprecate code on next major change
-                vol_kwargs.update(dict(config_map=V1ConfigMapVolumeSource(name=mount.config_map, optional=False)))
-                vol_mount_kwargs.update(dict(sub_path=mount.key))
-
-            elif mount.resource_type == 'secret':
+            if mount.resource_type == 'secret':
                 # Secret-based source
                 vol_kwargs.update(dict(secret=V1SecretVolumeSource(secret_name=mount.resource_name)))
                 vol_mount_kwargs.update(dict(sub_path=mount.resource_key))
@@ -346,7 +357,7 @@ class KubernetesUpdateInterface:
             restart_policy='Never',
             containers=[container],
             priority_class_name=self.priority_class,
-            service_account_name=docker_config.service_account or self.default_service_account or PRIVILEGED_SERVICE_ACCOUNT_NAME,
+            service_account_name=docker_config.service_account or PRIVILEGED_SERVICE_ACCOUNT_NAME,
             affinity=selector_to_node_affinity(self.linux_node_selector),
             tolerations=self.default_service_tolerations
         )
@@ -487,7 +498,6 @@ class ServiceUpdater(ThreadedCoreBase):
                                                         priority_class='al-core-priority',
                                                         extra_labels=extra_labels,
                                                         log_level=self.config.logging.log_level,
-                                                        default_service_account=self.config.services.service_account,
                                                         linux_node_selector=self.config.core.scaler.linux_node_selector,
                                                         default_service_tolerations=self.config.core.scaler.service_defaults.tolerations,
                                                         enable_pod_security=self.config.core.scaler.enable_pod_security)
@@ -512,7 +522,6 @@ class ServiceUpdater(ThreadedCoreBase):
                     tag = 'stable'
                 else:
                     tag = 'latest'
-                service_key = None
                 try:
                     service = Service(
                         {'name': service_name,

{assemblyline_core-4.6.0.dev7 → assemblyline_core-4.6.0.dev10}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.0.dev7
+Version: 4.6.0.dev10
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.6.0.dev7/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.6.0.dev7