assemblyline-core 4.5.0.75__tar.gz → 4.5.0.77__tar.gz

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: assemblyline-core
-Version: 4.5.0.75
+Version: 4.5.0.77
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.5.0.77/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.77

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/assemblyline_core/scaler/controllers/docker_ctl.py

@@ -189,10 +189,10 @@ class DockerController(ControllerInterface):
         volumes = {row[0]: {'bind': row[1], 'mode': 'ro'} for row in self.global_mounts}
 
         # Define environment variables
-        env = [f'{_e.name}={_e.value}' for _e in cfg.environment]
-        env += [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
+        env = [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
         env += [f'LOG_LEVEL={self.log_level}', f'AL_SERVICE_NAME={service_name}']
         env += [f'{_n}={_v}' for _n, _v in self._service_limited_env[service_name].items()]
+        env += [f'{_e.name}={_e.value}' for _e in cfg.environment]
         if prof.privileged:
             env.append('PRIVILEGED=true')
             volumes.update({row[0]: {'bind': row[1], 'mode': 'ro'} for row in self.core_mounts})

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -26,7 +26,8 @@ from kubernetes.client import V1Deployment, V1DeploymentSpec, V1PodTemplateSpec,
     V1PersistentVolumeClaimSpec, V1NetworkPolicy, V1NetworkPolicySpec, V1NetworkPolicyEgressRule, V1NetworkPolicyPeer, \
     V1NetworkPolicyIngressRule, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Service, \
     V1ServiceSpec, V1ServicePort, V1PodSecurityContext, V1Probe, V1ExecAction, V1SecurityContext, \
-    V1Affinity, V1NodeAffinity, V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, V1Toleration
+    V1Affinity, V1NodeAffinity, V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, V1Toleration, \
+    V1Capabilities, V1SeccompProfile
 from kubernetes.client.rest import ApiException
 from assemblyline.odm.models.service import DependencyConfig, DockerConfig, PersistentVolume
 
@@ -45,6 +46,14 @@ SERVICE_LIVENESS_TIMEOUT = int(os.environ.get('SERVICE_LIVENESS_TIMEOUT', 60))
 UNPRIVILEGED_SERVICE_ACCOUNT_NAME = os.environ.get('UNPRIVILEGED_SERVICE_ACCOUNT_NAME', None)
 PRIVILEGED_SERVICE_ACCOUNT_NAME = os.environ.get('PRIVILEGED_SERVICE_ACCOUNT_NAME', None)
 CERTIFICATE_VALIDITY_PERIOD = int(os.environ.get('CERTIFICATE_VALIDITY_PERIOD', '36500'))
+RESTRICTED_POD_SECURITY_CONTEXT = V1SecurityContext(
+    run_as_user=1000,
+    run_as_group=1000,
+    capabilities=V1Capabilities(drop=["ALL"]),
+    run_as_non_root=True,
+    allow_privilege_escalation=False,
+    seccomp_profile=V1SeccompProfile(type="RuntimeDefault")
+)
 
 AL_ROOT_CA = os.environ.get('AL_ROOT_CA', '/etc/assemblyline/ssl/al_root-ca.crt')
 AL_ROOT_CA_PK = os.environ.get('AL_ROOT_CA_PK', '/etc/assemblyline/ssl/al_root-ca.key')
@@ -241,7 +250,8 @@ def parse_cpu(string: str) -> float:
 class KubernetesController(ControllerInterface):
     def __init__(self, logger, namespace: str, prefix: str, priority: str, dependency_priority: str,
                  cpu_reservation: float, linux_node_selector: Selector, labels=None, log_level="INFO", core_env={},
-                 default_service_account=None, cluster_pod_list=True, default_service_tolerations = [],
+                 default_service_account=None, cluster_pod_list=True, enable_pod_security=False,
+                 default_service_tolerations=[],
                  priv_labels=None):
         # Try loading a kubernetes connection from either the fact that we are running
         # inside of a cluster, or have a config file that tells us how
@@ -287,6 +297,7 @@ class KubernetesController(ControllerInterface):
         self._service_limited_env: dict[str, dict[str, str]] = defaultdict(dict)
         self.default_service_account: Optional[str] = default_service_account
         self.cluster_pod_list = cluster_pod_list
+        self.security_policy = RESTRICTED_POD_SECURITY_CONTEXT if enable_pod_security else None
         self.default_service_tolerations = [V1Toleration(**toleration.as_primitives()) for toleration in default_service_tolerations]
 
         # A record of previously reported events so that we don't report the same message repeatedly, fill it with
@@ -396,7 +407,8 @@ class KubernetesController(ControllerInterface):
         """Tell the controller about a service profile it needs to manage."""
         self._create_deployment(profile.name, self._deployment_name(profile.name),
                                 profile.container_config, profile.shutdown_seconds, scale,
-                                change_key=profile.config_blob, core_mounts=profile.privileged)
+                                change_key=profile.config_blob, core_mounts=profile.privileged,
+                                security_context=self.security_policy),
         self._external_profiles[profile.name] = profile
 
     def _loop_forever(self, function):
@@ -679,7 +691,7 @@ class KubernetesController(ControllerInterface):
             return self._quota_mem_limit - self._get_pod_used_namespace_ram(), self._quota_mem_limit
         return self._node_pool_max_ram - self._get_pod_used_ram(), self._node_pool_max_ram
 
-    def _create_containers(self, service_name: str, deployment_name: str, container_config, mounts,
+    def _create_containers(self, service_name: str, deployment_name: str, container_config, mounts, security_context,
                            core_container=False):
         cores = container_config.cpu_cores
         memory = container_config.ram_mb
@@ -701,8 +713,6 @@ class KubernetesController(ControllerInterface):
             environment_variables += [V1EnvVar(name=_n, value=_v) for _n, _v in self.core_env.items()]
             environment_variables.extend(self.core_secret_env)
             environment_variables.append(V1EnvVar(name='PRIVILEGED', value='true'))
-        # Overwrite them with configured special environment variables
-        environment_variables += [V1EnvVar(name=_e.name, value=_e.value) for _e in container_config.environment]
         # Overwrite those with special hard coded variables
         environment_variables += [
             V1EnvVar(name='AL_SERVICE_NAME', value=service_name),
@@ -711,6 +721,8 @@ class KubernetesController(ControllerInterface):
         # Overwrite ones defined dynamically by dependency container launches
         for name, value in self._service_limited_env[service_name].items():
             environment_variables.append(V1EnvVar(name=name, value=value))
+        # Overwrite them with configured special environment variables
+        environment_variables += [V1EnvVar(name=_e.name, value=_e.value) for _e in container_config.environment]
         image_pull_policy = 'Always' if DEV_MODE else 'IfNotPresent'
         return [V1Container(
             name=deployment_name,
@@ -719,6 +731,7 @@ class KubernetesController(ControllerInterface):
             env=environment_variables,
             image_pull_policy=image_pull_policy,
             volume_mounts=mounts,
+            security_context=security_context,
             resources=V1ResourceRequirements(
                 limits={'cpu': cores, 'memory': f'{memory}Mi'},
                 requests={'cpu': cores*self.cpu_reservation, 'memory': f'{min_memory}Mi'},
@@ -731,7 +744,8 @@ class KubernetesController(ControllerInterface):
                            shutdown_seconds: int, scale: int, labels: dict[str, str] = None,
                            volumes: list[V1Volume] = None, mounts: list[V1VolumeMount] = None,
                            core_mounts: bool = False, change_key: str = '', high_priority: bool = False,
-                           deployment_strategy: V1DeploymentStrategy = V1DeploymentStrategy()):
+                           deployment_strategy: V1DeploymentStrategy = V1DeploymentStrategy(),
+                           security_context: V1SecurityContext = None):
         # Build a cache key to check for changes, just trying to only patch what changed
         # will still potentially result in a lot of restarts due to different kubernetes
         # systems returning differently formatted data
@@ -742,7 +756,8 @@ class KubernetesController(ControllerInterface):
         key_labels += sorted(deployment_labels.items())
         change_key = str(f"n={deployment_name}{change_key}dc={docker_config}ss={shutdown_seconds}"
                          f"l={key_labels}v={volumes}m={mounts}cm={core_mounts}senv={svc_env}"
-                         f"nodes={field_selector or ''}{label_selector or ''}")
+                         f"nodes={field_selector or ''}{label_selector or ''}"
+                         f"security_context={security_context or ''}")
         self.logger.debug(f"{deployment_name} actual change_key: {change_key}")
         change_key = str(hash(change_key))
 
@@ -847,7 +862,7 @@ class KubernetesController(ControllerInterface):
             init_containers=init_containers,
             volumes=all_volumes,
             containers=self._create_containers(service_name, deployment_name, docker_config,
-                                               all_mounts, core_container=core_mounts),
+                                               all_mounts, security_context, core_container=core_mounts),
             priority_class_name=self.dependency_priority if high_priority else self.priority,
             termination_grace_period_seconds=shutdown_seconds,
             security_context=V1PodSecurityContext(fs_group=1000),
@@ -952,7 +967,7 @@ class KubernetesController(ControllerInterface):
     def restart(self, service):
         self._create_deployment(service.name, self._deployment_name(service.name), service.container_config,
                                 service.shutdown_seconds, self.get_target(service.name), core_mounts=service.privileged,
-                                change_key=service.config_blob)
+                                change_key=service.config_blob, security_context=self.security_policy)
 
     def get_running_container_names(self):
         pods = self.api.list_pod_for_all_namespaces(field_selector='status.phase==Running',
@@ -1135,7 +1150,7 @@ class KubernetesController(ControllerInterface):
         self._create_deployment(service_name, deployment_name, spec.container,
                                 30, 1, labels, volumes=volumes, mounts=mounts, high_priority=True,
                                 core_mounts=spec.run_as_core, change_key=change_key,
-                                deployment_strategy=deployment_strategy)
+                                deployment_strategy=deployment_strategy, security_context=self.security_policy)
 
         # Setup a service to direct to the deployment
         try:

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/assemblyline_core/scaler/scaler_server.py

@@ -319,6 +319,7 @@ class ScalerServer(ThreadedCoreBase):
                                                    log_level=self.config.logging.log_level,
                                                    core_env=core_env,
                                                    cluster_pod_list=self.config.core.scaler.cluster_pod_list,
+                                                   enable_pod_security=self.config.core.scaler.enable_pod_security,
                                                    default_service_account=self.config.services.service_account,
                                                    default_service_tolerations=service_defaults_config.tolerations,
                                                    priv_labels=priv_labels

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/assemblyline_core/updater/run_updater.py

@@ -15,7 +15,8 @@ import docker
 
 from kubernetes.client import V1Job, V1ObjectMeta, V1JobSpec, V1PodTemplateSpec, V1PodSpec, V1Volume, \
     V1VolumeMount, V1EnvVar, V1Container, V1ResourceRequirements, \
-    V1ConfigMapVolumeSource, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Toleration
+    V1ConfigMapVolumeSource, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Toleration, V1SecurityContext, \
+    V1Capabilities, V1SeccompProfile
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 
@@ -44,6 +45,14 @@ INHERITED_VARIABLES: list[str] = ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http
 CONFIGURATION_HOST_PATH = os.getenv('CONFIGURATION_HOST_PATH', 'service_config')
 CONFIGURATION_CONFIGMAP = os.getenv('KUBERNETES_AL_CONFIG', None)
 AL_CORE_NETWORK = os.environ.get("AL_CORE_NETWORK", 'core')
+RESTRICTED_POD_SECUTITY_CONTEXT = V1SecurityContext(
+    run_as_user=1000,
+    run_as_group=1000,
+    capabilities=V1Capabilities(drop=["ALL"]),
+    run_as_non_root=True,
+    allow_privilege_escalation=False,
+    seccomp_profile=V1SeccompProfile(type="RuntimeDefault")
+)
 
 SERVICE_API_HOST = os.getenv('SERVICE_API_HOST')
 RELEASE_NAME = os.getenv('RELEASE_NAME')
@@ -148,7 +157,7 @@ class DockerUpdateInterface:
 
 class KubernetesUpdateInterface:
     def __init__(self, logger, prefix, namespace, priority_class, extra_labels, linux_node_selector: Selector,
-                 log_level="INFO", default_service_account=None, default_service_tolerations=[]):
+                 log_level="INFO", default_service_account=None, default_service_tolerations=[], enable_pod_security=False):
         # Try loading a kubernetes connection from either the fact that we are running
         # inside of a cluster, or we have a configuration in the normal location
         try:
@@ -182,6 +191,7 @@ class KubernetesUpdateInterface:
         self.secret_env = []
         self.linux_node_selector = linux_node_selector
         self.default_service_tolerations = [V1Toleration(**toleration.as_primitives()) for toleration in default_service_tolerations]
+        self.security_policy = RESTRICTED_POD_SECUTITY_CONTEXT if enable_pod_security else None
 
 
         # Get the deployment of this process. Use that information to fill out the secret info
@@ -324,6 +334,7 @@ class KubernetesUpdateInterface:
             env=environment_variables,
             image_pull_policy='Always',
             volume_mounts=volume_mounts,
+            security_context=self.security_policy,
             resources=V1ResourceRequirements(
                 limits={'cpu': cores, 'memory': f'{memory}Mi'},
                 requests={'cpu': cores / 4, 'memory': f'{memory_min}Mi'},
@@ -478,7 +489,8 @@ class ServiceUpdater(ThreadedCoreBase):
                                                         log_level=self.config.logging.log_level,
                                                         default_service_account=self.config.services.service_account,
                                                         linux_node_selector=self.config.core.scaler.linux_node_selector,
-                                                        default_service_tolerations=self.config.core.scaler.service_defaults.tolerations)
+                                                        default_service_tolerations=self.config.core.scaler.service_defaults.tolerations,
+                                                        enable_pod_security=self.config.core.scaler.enable_pod_security)
             # Add all additional mounts to privileged services
             self.mounts = self.config.core.scaler.service_defaults.mounts
         else:

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/assemblyline_core/workflow/run_workflow.py

@@ -109,6 +109,9 @@ class WorkflowManager(ServerBase):
                     if not workflow.enabled:
                         continue
 
+                    # Trigger a heartbeat to let the system know the workflow manager is still alive between tasks
+                    self.heartbeat()
+
                     # Start of transaction
                     if self.apm_client:
                         self.apm_client.begin_transaction("Execute workflows")

{assemblyline_core-4.5.0.75 → assemblyline_core-4.5.0.77}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: assemblyline-core
-Version: 4.5.0.75
+Version: 4.5.0.77
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.5.0.75/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.75