assemblyline-core 4.5.1.dev501__tar.gz → 4.6.0.0__tar.gz

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.5.1.dev501
+Version: 4.6.0.0
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.6.0.0/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.6.0.0

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core/dispatching/dispatcher.py

@@ -1,42 +1,63 @@
 from __future__ import annotations
-import uuid
+
+import dataclasses
+import enum
+import json
 import os
 import threading
 import time
+import typing
+import uuid
 from collections import defaultdict
 from contextlib import contextmanager
-import typing
-from typing import Optional, Any, TYPE_CHECKING, Iterable
-import json
-import enum
-from queue import PriorityQueue, Empty, Queue
-import dataclasses
 from copy import deepcopy
+from queue import Empty, PriorityQueue, Queue
+from typing import TYPE_CHECKING, Any, Iterable, Optional
 
 import elasticapm
 
 from assemblyline.common import isotime
-from assemblyline.common.constants import make_watcher_list_name, SUBMISSION_QUEUE, \
-    DISPATCH_RUNNING_TASK_HASH, SCALER_TIMEOUT_QUEUE, DISPATCH_TASK_HASH
-from assemblyline.common.forge import get_service_queue, get_apm_client, get_classification
+from assemblyline.common.constants import (
+    DISPATCH_RUNNING_TASK_HASH,
+    DISPATCH_TASK_HASH,
+    SCALER_TIMEOUT_QUEUE,
+    SUBMISSION_QUEUE,
+    make_watcher_list_name,
+)
+from assemblyline.common.forge import (
+    get_apm_client,
+    get_classification,
+    get_service_queue,
+)
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.common.postprocess import ActionWorker
 from assemblyline.datastore.helper import AssemblylineDatastore
-from assemblyline.odm.messages.changes import ServiceChange, Operation
+from assemblyline.odm.messages.changes import Operation, ServiceChange
 from assemblyline.odm.messages.dispatcher_heartbeat import Metrics
+from assemblyline.odm.messages.dispatching import (
+    CREATE_WATCH,
+    LIST_OUTSTANDING,
+    UPDATE_BAD_SID,
+    CreateWatch,
+    DispatcherCommandMessage,
+    ListOutstanding,
+    WatchQueueMessage,
+)
 from assemblyline.odm.messages.service_heartbeat import Metrics as ServiceMetrics
-from assemblyline.odm.messages.dispatching import WatchQueueMessage, CreateWatch, DispatcherCommandMessage, \
-    CREATE_WATCH, LIST_OUTSTANDING, UPDATE_BAD_SID, ListOutstanding
-from assemblyline.odm.messages.submission import SubmissionMessage, from_datastore_submission
-from assemblyline.odm.messages.task import FileInfo, Task as ServiceTask
+from assemblyline.odm.messages.submission import (
+    SubmissionMessage,
+    from_datastore_submission,
+)
+from assemblyline.odm.messages.task import FileInfo
+from assemblyline.odm.messages.task import Task as ServiceTask
 from assemblyline.odm.models.error import Error
 from assemblyline.odm.models.result import Result
 from assemblyline.odm.models.service import Service
 from assemblyline.odm.models.submission import Submission
 from assemblyline.odm.models.user import User
-from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
 from assemblyline.remote.datatypes.events import EventWatcher
+from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
 from assemblyline.remote.datatypes.hash import Hash
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
 from assemblyline.remote.datatypes.queues.named import NamedQueue
@@ -44,14 +65,15 @@ from assemblyline.remote.datatypes.set import ExpiringSet, Set
 from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
 from assemblyline_core.server_base import ThreadedCoreBase
 
+from ..ingester.constants import COMPLETE_QUEUE_NAME
 from .schedules import Scheduler
 from .timeout import TimeoutTable
-from ..ingester.constants import COMPLETE_QUEUE_NAME
 
 if TYPE_CHECKING:
-    from assemblyline.odm.models.file import File
     from redis import Redis
 
+    from assemblyline.odm.models.file import File
+
 
 APM_SPAN_TYPE = 'handle_message'
 
@@ -62,9 +84,6 @@ RESULT_BATCH_SIZE = int(os.environ.get('DISPATCHER_RESULT_BATCH_SIZE', '50'))
 ERROR_BATCH_SIZE = int(os.environ.get('DISPATCHER_ERROR_BATCH_SIZE', '50'))
 DAY_IN_SECONDS = 24 * 60 * 60
 
-# TODO: DYNAMIC_ANALYSIS_CATEGORY can be removed after assemblyline version 4.6+
-DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
-
 
 class KeyType(enum.Enum):
     OVERWRITE = 'overwrite'
@@ -161,17 +180,17 @@ class TemporaryFileData:
 
     def set_value(self, key: str, value: Any) -> bool:
         """Set the value of a temporary data key using the appropriate method for the key.
-        
+
         Return true if this change could mean partial results should be reevaluated.
         """
         if self.config.get(key) == KeyType.UNION.value:
             return self._union_shared_value(key, value)
-        
+
         if self.config.get(key) == KeyType.OVERWRITE.value:
             change = self.shared_values.get(key) != value
             self.shared_values[key] = value
             return change
-        
+
         self.local_values[key] = value
         return False
 
@@ -257,10 +276,6 @@ class SubmissionTask:
                 if not service:
                     continue
 
-                # TODO: the following 2 lines can be removed when assemblyline changed to version 4.6+
-                if service.category == DYNAMIC_ANALYSIS_CATEGORY:
-                    self.forbid_for_children(sha256, service.name)
-
                 prevented_services = scheduler.expand_categories(service.recursion_prevention)
 
                 for service_name in prevented_services:
@@ -393,7 +408,7 @@ class SubmissionTask:
                     entry.dispatch_needed = True
                 else:
                     # If there are results and there is a monitoring entry, the result was partial
-                    # so redispatch it immediately. If there are not partial results the monitoring 
+                    # so redispatch it immediately. If there are not partial results the monitoring
                     # entry will have been cleared.
                     self.redispatch_service(sha256, service)
                     changed.append(sha256)
@@ -836,11 +851,9 @@ class Dispatcher(ThreadedCoreBase):
 
             forbidden_services = None
 
-            # If Dynamic Recursion Prevention is in effect and the file is not part of the bypass list,
+            # If Recursion Prevention is in effect and the file is not part of the bypass list,
             # Find the list of services this file is forbidden from being sent to.
-            # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
-            ignore_drp = submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention
-            if not ignore_drp and sha256 not in task.dynamic_recursion_bypass:
+            if not submission.params.ignore_recursion_prevention and sha256 not in task.dynamic_recursion_bypass:
                 forbidden_services = task.find_recursion_excluded_services(sha256)
 
             task.file_schedules[sha256] = self.scheduler.build_schedule(submission, file_info.type,
@@ -938,10 +951,6 @@ class Dispatcher(ThreadedCoreBase):
                         tag_fields.append('score')
 
                     # Mark this routing for the purposes of recursion prevention
-                    # TODO: The following 2 lines can be removed after assemblyline upgrade to version 4.6+
-                    if service.category == DYNAMIC_ANALYSIS_CATEGORY:
-                        task.forbid_for_children(sha256, service_name)
-
                     prevented_services = self.scheduler.expand_categories(service.recursion_prevention)
 
                     for service_name in prevented_services:
@@ -961,8 +970,7 @@ class Dispatcher(ThreadedCoreBase):
                         max_files=task.submission.params.max_extracted,
                         ttl=submission.params.ttl,
                         ignore_cache=submission.params.ignore_cache,
-                        # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
-                        ignore_recursion_prevention=submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention ,
+                        ignore_recursion_prevention=submission.params.ignore_recursion_prevention,
                         ignore_filtering=ignore_filtering,
                         tags=[{field: x[field] for field in tag_fields} for x in tags],
                         temporary_submission_data=[
@@ -1574,7 +1582,7 @@ class Dispatcher(ThreadedCoreBase):
         # Not worth running if we know we have services in queue
         if not any(_s == sha256 for _s, _ in task.queue_keys.keys()):
             force_redispatch.add(sha256)
-        
+
         # Try to run the next stage
         for sha256 in force_redispatch:
             self.dispatch_file(task, sha256)

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core/ingester/ingester.py

@@ -408,12 +408,13 @@ class Ingester(ThreadedCoreBase):
                     self.apm_client.end_transaction('ingest_submit', 'exception')
 
     def handle_submission_delete(self, sid: Optional[str]):
-        if not sid:
-            return
-
-        # Upon submission deletion, ensure to cleanup the local cache of filescores relative to the SID
         with self.cache_lock:
-            self.cache = {k: v for k, v in self.cache.items() if v.sid != sid}
+            if not sid:
+                # Clear the entire local cache
+                self.cache = {}
+            else:
+                # Ensure to cleanup the local cache of filescores relative to the SID
+                self.cache = {k: v for k, v in self.cache.items() if v.sid != sid}
 
     def handle_complete(self):
         while self.running:

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -1,36 +1,73 @@
 from __future__ import annotations
+
 import base64
 import functools
 import json
-import uuid
 import os
 import threading
+import uuid
 import weakref
-import urllib3
-
 from base64 import b64encode
-from cryptography import x509
-from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography.hazmat.primitives import serialization, hashes
 from collections import OrderedDict, defaultdict
 from datetime import datetime, timedelta
-from dateutil.tz import tzlocal
-from typing import List, Optional, Tuple
 from time import sleep
-from assemblyline.odm.models.config import Selector
+from typing import List, Optional, Tuple
 
+import urllib3
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from dateutil.tz import tzlocal
 from kubernetes import client, config, watch
-from kubernetes.client import V1Deployment, V1DeploymentSpec, V1PodTemplateSpec, V1DeploymentStrategy, \
-    V1PodSpec, V1ObjectMeta, V1Volume, V1Container, V1VolumeMount, V1EnvVar, V1ConfigMapVolumeSource, \
-    V1PersistentVolumeClaimVolumeSource, V1LabelSelector, V1ResourceRequirements, V1PersistentVolumeClaim, \
-    V1PersistentVolumeClaimSpec, V1NetworkPolicy, V1NetworkPolicySpec, V1NetworkPolicyEgressRule, V1NetworkPolicyPeer, \
-    V1NetworkPolicyIngressRule, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Service, \
-    V1ServiceSpec, V1ServicePort, V1PodSecurityContext, V1Probe, V1ExecAction, V1SecurityContext, \
-    V1Affinity, V1NodeAffinity, V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, V1Toleration, \
-    V1Capabilities, V1SeccompProfile
+from kubernetes.client import (
+    V1Affinity,
+    V1Capabilities,
+    V1ConfigMapVolumeSource,
+    V1Container,
+    V1Deployment,
+    V1DeploymentSpec,
+    V1DeploymentStrategy,
+    V1EnvVar,
+    V1ExecAction,
+    V1LabelSelector,
+    V1LocalObjectReference,
+    V1NetworkPolicy,
+    V1NetworkPolicyEgressRule,
+    V1NetworkPolicyIngressRule,
+    V1NetworkPolicyPeer,
+    V1NetworkPolicySpec,
+    V1NodeAffinity,
+    V1NodeSelector,
+    V1NodeSelectorRequirement,
+    V1NodeSelectorTerm,
+    V1ObjectMeta,
+    V1PersistentVolumeClaim,
+    V1PersistentVolumeClaimSpec,
+    V1PersistentVolumeClaimVolumeSource,
+    V1PodSecurityContext,
+    V1PodSpec,
+    V1PodTemplateSpec,
+    V1Probe,
+    V1ResourceRequirements,
+    V1SeccompProfile,
+    V1Secret,
+    V1SecretVolumeSource,
+    V1SecurityContext,
+    V1Service,
+    V1ServicePort,
+    V1ServiceSpec,
+    V1Toleration,
+    V1Volume,
+    V1VolumeMount,
+)
 from kubernetes.client.rest import ApiException
-from assemblyline.odm.models.service import DependencyConfig, DockerConfig, PersistentVolume
 
+from assemblyline.odm.models.config import Selector
+from assemblyline.odm.models.service import (
+    DependencyConfig,
+    DockerConfig,
+    PersistentVolume,
+)
 from assemblyline_core.scaler.controllers.interface import ControllerInterface
 
 # RESERVE_MEMORY_PER_NODE = os.environ.get('RESERVE_MEMORY_PER_NODE')
@@ -250,7 +287,7 @@ def parse_cpu(string: str) -> float:
 class KubernetesController(ControllerInterface):
     def __init__(self, logger, namespace: str, prefix: str, priority: str, dependency_priority: str,
                  cpu_reservation: float, linux_node_selector: Selector, labels=None, log_level="INFO", core_env={},
-                 default_service_account=None, cluster_pod_list=True, enable_pod_security=False,
+                 cluster_pod_list=True, enable_pod_security=False,
                  default_service_tolerations=[],
                  priv_labels=None):
         # Try loading a kubernetes connection from either the fact that we are running
@@ -295,7 +332,6 @@ class KubernetesController(ControllerInterface):
         self.core_mounts: dict[str, V1VolumeMount] = {}
         self._external_profiles = weakref.WeakValueDictionary()
         self._service_limited_env: dict[str, dict[str, str]] = defaultdict(dict)
-        self.default_service_account: Optional[str] = default_service_account
         self.cluster_pod_list = cluster_pod_list
         self.security_policy = RESTRICTED_POD_SECURITY_CONTEXT if enable_pod_security else None
         self.default_service_tolerations = [V1Toleration(**toleration.as_primitives()) for toleration in default_service_tolerations]
@@ -836,8 +872,7 @@ class KubernetesController(ControllerInterface):
         metadata = V1ObjectMeta(name=deployment_name, labels=all_labels, annotations={CHANGE_KEY_NAME: change_key})
 
         # Figure out which (if any) service account to use
-        service_account = self.default_service_account or \
-            (PRIVILEGED_SERVICE_ACCOUNT_NAME if core_mounts else UNPRIVILEGED_SERVICE_ACCOUNT_NAME)
+        service_account = PRIVILEGED_SERVICE_ACCOUNT_NAME if core_mounts else UNPRIVILEGED_SERVICE_ACCOUNT_NAME
         if docker_config.service_account:
             service_account = docker_config.service_account
 

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core/scaler/scaler_server.py

@@ -2,46 +2,55 @@
 An auto-scaling service specific to Assemblyline services.
 """
 from __future__ import annotations
+
+import concurrent.futures
+import copy
 import functools
-import threading
-from collections import defaultdict
-from string import Template
-from typing import Dict, Optional, Any
-import os
-import re
+import json
 import math
-import time
+import os
 import platform
-import concurrent.futures
-import copy
+import threading
+import time
+from collections import defaultdict
 from contextlib import contextmanager
+from string import Template
+from typing import Any, Dict, Optional
 
 import elasticapm
-import json
 import yaml
 
-from assemblyline.remote.datatypes.queues.named import NamedQueue
-from assemblyline.remote.datatypes.queues.priority import PriorityQueue, length as pq_length
-from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
-from assemblyline.remote.datatypes.hash import ExpiringHash, Hash
-from assemblyline.remote.datatypes.events import EventWatcher, EventSender
-from assemblyline.odm.models.service import Service, DockerConfig, EnvironmentVariable
-from assemblyline.odm.models.config import Mount
-from assemblyline.odm.messages.scaler_heartbeat import Metrics
-from assemblyline.odm.messages.scaler_status_heartbeat import Status
-from assemblyline.odm.messages.changes import ServiceChange, Operation
-from assemblyline.common.dict_utils import get_recursive_sorted_tuples, flatten
+from assemblyline.common.constants import (
+    SCALER_TIMEOUT_QUEUE,
+    SERVICE_STATE_HASH,
+    ServiceStatus,
+)
+from assemblyline.common.dict_utils import flatten, get_recursive_sorted_tuples
+from assemblyline.common.forge import (
+    get_apm_client,
+    get_classification,
+    get_service_queue,
+)
 from assemblyline.common.uid import get_id_from_data
-from assemblyline.common.forge import get_classification, get_service_queue, get_apm_client
-from assemblyline.common.constants import SCALER_TIMEOUT_QUEUE, SERVICE_STATE_HASH, ServiceStatus
 from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
-from assemblyline_core.updater.helper import get_registry_config
+from assemblyline.odm.messages.changes import Operation, ServiceChange
+from assemblyline.odm.messages.scaler_heartbeat import Metrics
+from assemblyline.odm.messages.scaler_status_heartbeat import Status
+from assemblyline.odm.models.config import Mount
+from assemblyline.odm.models.service import DockerConfig, EnvironmentVariable, Service
+from assemblyline.remote.datatypes.events import EventSender, EventWatcher
+from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
+from assemblyline.remote.datatypes.hash import ExpiringHash, Hash
+from assemblyline.remote.datatypes.queues.named import NamedQueue
+from assemblyline.remote.datatypes.queues.priority import PriorityQueue
+from assemblyline.remote.datatypes.queues.priority import length as pq_length
 from assemblyline_core.scaler.controllers import KubernetesController
 from assemblyline_core.scaler.controllers.interface import ServiceControlError
 from assemblyline_core.server_base import ServiceStage, ThreadedCoreBase
+from assemblyline_core.updater.helper import get_registry_config
 
-from .controllers import DockerController
 from . import collection
+from .controllers import DockerController
 
 APM_SPAN_TYPE = 'scaler'
 
@@ -326,7 +335,6 @@ class ScalerServer(ThreadedCoreBase):
                                                    core_env=core_env,
                                                    cluster_pod_list=self.config.core.scaler.cluster_pod_list,
                                                    enable_pod_security=self.config.core.scaler.enable_pod_security,
-                                                   default_service_account=self.config.services.service_account,
                                                    default_service_tolerations=service_defaults_config.tolerations,
                                                    priv_labels=priv_labels
                                                    )
@@ -348,14 +356,6 @@ class ScalerServer(ThreadedCoreBase):
 
             # Add default mounts for (non-)privileged services
             for mount in service_defaults_config.mounts:
-                # Deprecated configuration for mounting ConfigMap
-                # TODO: Deprecate code on next major change
-                if mount.config_map:
-                    self.controller.add_config_mount(mount.name, config_map=mount.config_map, key=mount.key,
-                                                     target_path=mount.path, read_only=mount.read_only,
-                                                     core=mount.privileged_only)
-                    continue
-
                 if mount.resource_type == 'configmap':
                     # ConfigMap-based mount
                     self.controller.add_config_mount(mount.name, config_map=mount.resource_name, key=mount.resource_key,

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core/updater/run_updater.py

@@ -7,17 +7,31 @@ import os
 import re
 import time
 import uuid
-
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, List, Optional
 
 import docker
-
-from kubernetes.client import V1Job, V1ObjectMeta, V1JobSpec, V1PodTemplateSpec, V1PodSpec, V1Volume, \
-    V1VolumeMount, V1EnvVar, V1Container, V1ResourceRequirements, \
-    V1ConfigMapVolumeSource, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Toleration, V1SecurityContext, \
-    V1Capabilities, V1SeccompProfile
 from kubernetes import client, config
+from kubernetes.client import (
+    V1Capabilities,
+    V1ConfigMapVolumeSource,
+    V1Container,
+    V1EnvVar,
+    V1Job,
+    V1JobSpec,
+    V1LocalObjectReference,
+    V1ObjectMeta,
+    V1PodSpec,
+    V1PodTemplateSpec,
+    V1ResourceRequirements,
+    V1SeccompProfile,
+    V1Secret,
+    V1SecretVolumeSource,
+    V1SecurityContext,
+    V1Toleration,
+    V1Volume,
+    V1VolumeMount,
+)
 from kubernetes.client.rest import ApiException
 
 from assemblyline.common import isotime
@@ -26,7 +40,11 @@ from assemblyline.odm.models.config import Mount, Selector
 from assemblyline.odm.models.service import DockerConfig, Service
 from assemblyline.remote.datatypes.events import EventSender, EventWatcher
 from assemblyline.remote.datatypes.hash import Hash
-from assemblyline_core.scaler.controllers.kubernetes_ctl import create_docker_auth_config, selector_to_node_affinity, PRIVILEGED_SERVICE_ACCOUNT_NAME
+from assemblyline_core.scaler.controllers.kubernetes_ctl import (
+    PRIVILEGED_SERVICE_ACCOUNT_NAME,
+    create_docker_auth_config,
+    selector_to_node_affinity,
+)
 from assemblyline_core.server_base import ThreadedCoreBase
 from assemblyline_core.updater.helper import get_latest_tag_for_service
 
@@ -157,7 +175,7 @@ class DockerUpdateInterface:
 
 class KubernetesUpdateInterface:
     def __init__(self, logger, prefix, namespace, priority_class, extra_labels, linux_node_selector: Selector,
-                 log_level="INFO", default_service_account=None, default_service_tolerations=[], enable_pod_security=False):
+                 log_level="INFO", default_service_tolerations=[], enable_pod_security=False):
         # Try loading a kubernetes connection from either the fact that we are running
         # inside of a cluster, or we have a configuration in the normal location
         try:
@@ -187,7 +205,6 @@ class KubernetesUpdateInterface:
         self.priority_class = priority_class
         self.extra_labels = extra_labels
         self.log_level = log_level
-        self.default_service_account = default_service_account
         self.secret_env = []
         self.linux_node_selector = linux_node_selector
         self.default_service_tolerations = [V1Toleration(**toleration.as_primitives()) for toleration in default_service_tolerations]
@@ -268,13 +285,7 @@ class KubernetesUpdateInterface:
                 read_only=mount.read_only,
             )
 
-            if mount.config_map:
-                # Deprecated configuration for mounting ConfigMap
-                # TODO: Deprecate code on next major change
-                vol_kwargs.update(dict(config_map=V1ConfigMapVolumeSource(name=mount.config_map, optional=False)))
-                vol_mount_kwargs.update(dict(sub_path=mount.key))
-
-            elif mount.resource_type == 'secret':
+            if mount.resource_type == 'secret':
                 # Secret-based source
                 vol_kwargs.update(dict(secret=V1SecretVolumeSource(secret_name=mount.resource_name)))
                 vol_mount_kwargs.update(dict(sub_path=mount.resource_key))
@@ -346,7 +357,7 @@ class KubernetesUpdateInterface:
             restart_policy='Never',
             containers=[container],
             priority_class_name=self.priority_class,
-            service_account_name=docker_config.service_account or self.default_service_account or PRIVILEGED_SERVICE_ACCOUNT_NAME,
+            service_account_name=docker_config.service_account or PRIVILEGED_SERVICE_ACCOUNT_NAME,
             affinity=selector_to_node_affinity(self.linux_node_selector),
             tolerations=self.default_service_tolerations
         )
@@ -487,7 +498,6 @@ class ServiceUpdater(ThreadedCoreBase):
                                                         priority_class='al-core-priority',
                                                         extra_labels=extra_labels,
                                                         log_level=self.config.logging.log_level,
-                                                        default_service_account=self.config.services.service_account,
                                                         linux_node_selector=self.config.core.scaler.linux_node_selector,
                                                         default_service_tolerations=self.config.core.scaler.service_defaults.tolerations,
                                                         enable_pod_security=self.config.core.scaler.enable_pod_security)
@@ -512,7 +522,6 @@ class ServiceUpdater(ThreadedCoreBase):
                     tag = 'stable'
                 else:
                     tag = 'latest'
-                service_key = None
                 try:
                     service = Service(
                         {'name': service_name,

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core/vacuum/worker.py

@@ -1,52 +1,55 @@
 import contextlib
 import datetime
 import json
-import tempfile
 import logging
 import os
-import time
-import signal
 import shutil
-from copy import deepcopy
-from typing import Optional, Any
-from multiprocessing import Lock, Event
+import signal
+import tempfile
 import threading
+import time
+from copy import deepcopy
+from multiprocessing import Event, Lock
+from typing import Any, Optional
 
-import elasticapm
 import arrow
+import elasticapm
 
-from assemblyline.common.forge import CachedObject, get_classification, get_config, get_datastore, get_filestore, \
-    get_apm_client
+from assemblyline.common import identify
 from assemblyline.common.codec import decode_file
 from assemblyline.common.dict_utils import flatten
+from assemblyline.common.forge import (
+    CachedObject,
+    get_apm_client,
+    get_classification,
+    get_config,
+    get_datastore,
+    get_filestore,
+)
+from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.log import init_logging
 from assemblyline.common.metrics import MetricsFactory
-from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator
-from assemblyline.common import identify
-from assemblyline.common.isotime import now_as_iso
+from assemblyline.common.str_utils import safe_str
 from assemblyline.common.uid import get_random_id
+from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator
+from assemblyline.filestore import FileStore
+from assemblyline.odm.messages.submission import Submission
+from assemblyline.odm.messages.vacuum_heartbeat import Metrics
 from assemblyline.odm.models import user
 from assemblyline.odm.models.config import Config
 from assemblyline.odm.models.submission import DEFAULT_SRV_SEL
 from assemblyline.odm.models.user_settings import UserSettings
-from assemblyline.remote.datatypes.queues.comms import CommsQueue
-from assemblyline.odm.messages.vacuum_heartbeat import Metrics
-
-from assemblyline.filestore import FileStore
-from assemblyline.common.str_utils import safe_str
 from assemblyline.remote.datatypes import get_client as get_redis_client
-from assemblyline.odm.messages.submission import Submission
-from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline.remote.datatypes.hash import Hash
-
-from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
+from assemblyline.remote.datatypes.queues.comms import CommsQueue
+from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.ingester.constants import INGEST_QUEUE_NAME
+from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
 
-from .safelist import VacuumSafelist
-from .department_map import DepartmentMap
-from .stream_map import StreamMap, Stream
 from .crawler import heartbeat
-
+from .department_map import DepartmentMap
+from .safelist import VacuumSafelist
+from .stream_map import Stream, StreamMap
 
 # init_logging('assemblyline.vacuum.worker')
 logger = logging.getLogger('assemblyline.vacuum.worker')
@@ -410,7 +413,7 @@ class FileProcessor(threading.Thread):
                     'deep_scan': False,
                     "priority": 150,
                     "ignore_cache": False,
-                    "ignore_dynamic_recursion_prevention": False,
+                    "ignore_recursion_prevention": False,
                     "ignore_filtering": False,
                     "type": "INGEST"
                 })

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.5.1.dev501
+Version: 4.6.0.0
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline_core-4.5.1.dev501 → assemblyline_core-4.6.0.0}/test/test_dispatcher.py

@@ -1,20 +1,10 @@
+import json
 import logging
 import time
 from unittest import mock
 
-import json
 import pytest
-
-from assemblyline.common.forge import get_service_queue, get_classification
-from assemblyline.odm.models.error import Error
-from assemblyline.odm.models.file import File
-from assemblyline.odm.models.result import Result
-from assemblyline.odm.models.user import User
-from assemblyline.odm.randomizer import random_model_obj, random_minimal_obj, get_random_hash
-from assemblyline.odm import models
-from assemblyline.common.metrics import MetricsFactory
-
-from assemblyline_core.dispatching.client import DispatchClient, DISPATCH_RESULT_QUEUE
+from assemblyline_core.dispatching.client import DISPATCH_RESULT_QUEUE, DispatchClient
 from assemblyline_core.dispatching.dispatcher import Dispatcher, ServiceTask, Submission
 from assemblyline_core.dispatching.schedules import Scheduler as RealScheduler
 
@@ -23,6 +13,18 @@ from assemblyline_core.dispatching.timeout import TimeoutTable
 from mocking import ToggleTrue
 from test_scheduler import dummy_service
 
+from assemblyline.common.forge import get_classification, get_service_queue
+from assemblyline.common.metrics import MetricsFactory
+from assemblyline.odm import models
+from assemblyline.odm.models.error import Error
+from assemblyline.odm.models.file import File
+from assemblyline.odm.models.result import Result
+from assemblyline.odm.models.user import User
+from assemblyline.odm.randomizer import (
+    get_random_hash,
+    random_minimal_obj,
+    random_model_obj,
+)
 
 logger = logging.getLogger('assemblyline.test')
 
@@ -311,8 +313,6 @@ def test_dispatch_extracted_bypass_drp(clean_redis, clean_datastore):
     submission = random_model_obj(Submission)
     submission.to_be_deleted = False
 
-    # the following 1 line can be removed after assemblyline upgrade to version 4.6+
-    submission.params.ignore_dynamic_recursion_prevention = False
     submission.params.ignore_recursion_prevention = False
     submission.params.services.selected = ['extract', 'sandbox']
     submission.files = [dict(name='./file', sha256=file_hash)]

assemblyline_core-4.5.1.dev501/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev501