assemblyline-core 4.6.0.dev5__tar.gz → 4.6.0.dev7__tar.gz

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.0.dev5
+Version: 4.6.0.dev7
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.6.0.dev7/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.6.0.dev7

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/assemblyline_core/dispatching/dispatcher.py

@@ -1,42 +1,63 @@
 from __future__ import annotations
-import uuid
+
+import dataclasses
+import enum
+import json
 import os
 import threading
 import time
+import typing
+import uuid
 from collections import defaultdict
 from contextlib import contextmanager
-import typing
-from typing import Optional, Any, TYPE_CHECKING, Iterable
-import json
-import enum
-from queue import PriorityQueue, Empty, Queue
-import dataclasses
 from copy import deepcopy
+from queue import Empty, PriorityQueue, Queue
+from typing import TYPE_CHECKING, Any, Iterable, Optional
 
 import elasticapm
 
 from assemblyline.common import isotime
-from assemblyline.common.constants import make_watcher_list_name, SUBMISSION_QUEUE, \
-    DISPATCH_RUNNING_TASK_HASH, SCALER_TIMEOUT_QUEUE, DISPATCH_TASK_HASH
-from assemblyline.common.forge import get_service_queue, get_apm_client, get_classification
+from assemblyline.common.constants import (
+    DISPATCH_RUNNING_TASK_HASH,
+    DISPATCH_TASK_HASH,
+    SCALER_TIMEOUT_QUEUE,
+    SUBMISSION_QUEUE,
+    make_watcher_list_name,
+)
+from assemblyline.common.forge import (
+    get_apm_client,
+    get_classification,
+    get_service_queue,
+)
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.common.postprocess import ActionWorker
 from assemblyline.datastore.helper import AssemblylineDatastore
-from assemblyline.odm.messages.changes import ServiceChange, Operation
+from assemblyline.odm.messages.changes import Operation, ServiceChange
 from assemblyline.odm.messages.dispatcher_heartbeat import Metrics
+from assemblyline.odm.messages.dispatching import (
+    CREATE_WATCH,
+    LIST_OUTSTANDING,
+    UPDATE_BAD_SID,
+    CreateWatch,
+    DispatcherCommandMessage,
+    ListOutstanding,
+    WatchQueueMessage,
+)
 from assemblyline.odm.messages.service_heartbeat import Metrics as ServiceMetrics
-from assemblyline.odm.messages.dispatching import WatchQueueMessage, CreateWatch, DispatcherCommandMessage, \
-    CREATE_WATCH, LIST_OUTSTANDING, UPDATE_BAD_SID, ListOutstanding
-from assemblyline.odm.messages.submission import SubmissionMessage, from_datastore_submission
-from assemblyline.odm.messages.task import FileInfo, Task as ServiceTask
+from assemblyline.odm.messages.submission import (
+    SubmissionMessage,
+    from_datastore_submission,
+)
+from assemblyline.odm.messages.task import FileInfo
+from assemblyline.odm.messages.task import Task as ServiceTask
 from assemblyline.odm.models.error import Error
 from assemblyline.odm.models.result import Result
 from assemblyline.odm.models.service import Service
 from assemblyline.odm.models.submission import Submission
 from assemblyline.odm.models.user import User
-from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
 from assemblyline.remote.datatypes.events import EventWatcher
+from assemblyline.remote.datatypes.exporting_counter import export_metrics_once
 from assemblyline.remote.datatypes.hash import Hash
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
 from assemblyline.remote.datatypes.queues.named import NamedQueue
@@ -44,14 +65,15 @@ from assemblyline.remote.datatypes.set import ExpiringSet, Set
 from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
 from assemblyline_core.server_base import ThreadedCoreBase
 
+from ..ingester.constants import COMPLETE_QUEUE_NAME
 from .schedules import Scheduler
 from .timeout import TimeoutTable
-from ..ingester.constants import COMPLETE_QUEUE_NAME
 
 if TYPE_CHECKING:
-    from assemblyline.odm.models.file import File
     from redis import Redis
 
+    from assemblyline.odm.models.file import File
+
 
 APM_SPAN_TYPE = 'handle_message'
 
@@ -62,9 +84,6 @@ RESULT_BATCH_SIZE = int(os.environ.get('DISPATCHER_RESULT_BATCH_SIZE', '50'))
 ERROR_BATCH_SIZE = int(os.environ.get('DISPATCHER_ERROR_BATCH_SIZE', '50'))
 DAY_IN_SECONDS = 24 * 60 * 60
 
-# TODO: DYNAMIC_ANALYSIS_CATEGORY can be removed after assemblyline version 4.6+
-DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
-
 
 class KeyType(enum.Enum):
     OVERWRITE = 'overwrite'
@@ -161,17 +180,17 @@ class TemporaryFileData:
 
     def set_value(self, key: str, value: Any) -> bool:
         """Set the value of a temporary data key using the appropriate method for the key.
-        
+
         Return true if this change could mean partial results should be reevaluated.
         """
         if self.config.get(key) == KeyType.UNION.value:
             return self._union_shared_value(key, value)
-        
+
         if self.config.get(key) == KeyType.OVERWRITE.value:
             change = self.shared_values.get(key) != value
             self.shared_values[key] = value
             return change
-        
+
         self.local_values[key] = value
         return False
 
@@ -257,10 +276,6 @@ class SubmissionTask:
                 if not service:
                     continue
 
-                # TODO: the following 2 lines can be removed when assemblyline changed to version 4.6+
-                if service.category == DYNAMIC_ANALYSIS_CATEGORY:
-                    self.forbid_for_children(sha256, service.name)
-
                 prevented_services = scheduler.expand_categories(service.recursion_prevention)
 
                 for service_name in prevented_services:
@@ -393,7 +408,7 @@ class SubmissionTask:
                     entry.dispatch_needed = True
                 else:
                     # If there are results and there is a monitoring entry, the result was partial
-                    # so redispatch it immediately. If there are not partial results the monitoring 
+                    # so redispatch it immediately. If there are not partial results the monitoring
                     # entry will have been cleared.
                     self.redispatch_service(sha256, service)
                     changed.append(sha256)
@@ -836,11 +851,9 @@ class Dispatcher(ThreadedCoreBase):
 
             forbidden_services = None
 
-            # If Dynamic Recursion Prevention is in effect and the file is not part of the bypass list,
+            # If Recursion Prevention is in effect and the file is not part of the bypass list,
             # Find the list of services this file is forbidden from being sent to.
-            # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
-            ignore_drp = submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention
-            if not ignore_drp and sha256 not in task.dynamic_recursion_bypass:
+            if not submission.params.ignore_recursion_prevention and sha256 not in task.dynamic_recursion_bypass:
                 forbidden_services = task.find_recursion_excluded_services(sha256)
 
             task.file_schedules[sha256] = self.scheduler.build_schedule(submission, file_info.type,
@@ -938,10 +951,6 @@ class Dispatcher(ThreadedCoreBase):
                         tag_fields.append('score')
 
                     # Mark this routing for the purposes of recursion prevention
-                    # TODO: The following 2 lines can be removed after assemblyline upgrade to version 4.6+
-                    if service.category == DYNAMIC_ANALYSIS_CATEGORY:
-                        task.forbid_for_children(sha256, service_name)
-
                     prevented_services = self.scheduler.expand_categories(service.recursion_prevention)
 
                     for service_name in prevented_services:
@@ -961,8 +970,7 @@ class Dispatcher(ThreadedCoreBase):
                         max_files=task.submission.params.max_extracted,
                         ttl=submission.params.ttl,
                         ignore_cache=submission.params.ignore_cache,
-                        # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
-                        ignore_recursion_prevention=submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention ,
+                        ignore_recursion_prevention=submission.params.ignore_recursion_prevention,
                         ignore_filtering=ignore_filtering,
                         tags=[{field: x[field] for field in tag_fields} for x in tags],
                         temporary_submission_data=[
@@ -1574,7 +1582,7 @@ class Dispatcher(ThreadedCoreBase):
         # Not worth running if we know we have services in queue
         if not any(_s == sha256 for _s, _ in task.queue_keys.keys()):
             force_redispatch.add(sha256)
-        
+
         # Try to run the next stage
         for sha256 in force_redispatch:
             self.dispatch_file(task, sha256)

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/assemblyline_core/ingester/ingester.py

@@ -408,12 +408,13 @@ class Ingester(ThreadedCoreBase):
                     self.apm_client.end_transaction('ingest_submit', 'exception')
 
     def handle_submission_delete(self, sid: Optional[str]):
-        if not sid:
-            return
-
-        # Upon submission deletion, ensure to cleanup the local cache of filescores relative to the SID
         with self.cache_lock:
-            self.cache = {k: v for k, v in self.cache.items() if v.sid != sid}
+            if not sid:
+                # Clear the entire local cache
+                self.cache = {}
+            else:
+                # Ensure to cleanup the local cache of filescores relative to the SID
+                self.cache = {k: v for k, v in self.cache.items() if v.sid != sid}
 
     def handle_complete(self):
         while self.running:

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/assemblyline_core/scaler/scaler_server.py

@@ -8,7 +8,6 @@ from collections import defaultdict
 from string import Template
 from typing import Dict, Optional, Any
 import os
-import re
 import math
 import time
 import platform

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/assemblyline_core/vacuum/worker.py

@@ -1,52 +1,55 @@
 import contextlib
 import datetime
 import json
-import tempfile
 import logging
 import os
-import time
-import signal
 import shutil
-from copy import deepcopy
-from typing import Optional, Any
-from multiprocessing import Lock, Event
+import signal
+import tempfile
 import threading
+import time
+from copy import deepcopy
+from multiprocessing import Event, Lock
+from typing import Any, Optional
 
-import elasticapm
 import arrow
+import elasticapm
 
-from assemblyline.common.forge import CachedObject, get_classification, get_config, get_datastore, get_filestore, \
-    get_apm_client
+from assemblyline.common import identify
 from assemblyline.common.codec import decode_file
 from assemblyline.common.dict_utils import flatten
+from assemblyline.common.forge import (
+    CachedObject,
+    get_apm_client,
+    get_classification,
+    get_config,
+    get_datastore,
+    get_filestore,
+)
+from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.log import init_logging
 from assemblyline.common.metrics import MetricsFactory
-from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator
-from assemblyline.common import identify
-from assemblyline.common.isotime import now_as_iso
+from assemblyline.common.str_utils import safe_str
 from assemblyline.common.uid import get_random_id
+from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator
+from assemblyline.filestore import FileStore
+from assemblyline.odm.messages.submission import Submission
+from assemblyline.odm.messages.vacuum_heartbeat import Metrics
 from assemblyline.odm.models import user
 from assemblyline.odm.models.config import Config
 from assemblyline.odm.models.submission import DEFAULT_SRV_SEL
 from assemblyline.odm.models.user_settings import UserSettings
-from assemblyline.remote.datatypes.queues.comms import CommsQueue
-from assemblyline.odm.messages.vacuum_heartbeat import Metrics
-
-from assemblyline.filestore import FileStore
-from assemblyline.common.str_utils import safe_str
 from assemblyline.remote.datatypes import get_client as get_redis_client
-from assemblyline.odm.messages.submission import Submission
-from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline.remote.datatypes.hash import Hash
-
-from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
+from assemblyline.remote.datatypes.queues.comms import CommsQueue
+from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.ingester.constants import INGEST_QUEUE_NAME
+from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
 
-from .safelist import VacuumSafelist
-from .department_map import DepartmentMap
-from .stream_map import StreamMap, Stream
 from .crawler import heartbeat
-
+from .department_map import DepartmentMap
+from .safelist import VacuumSafelist
+from .stream_map import Stream, StreamMap
 
 # init_logging('assemblyline.vacuum.worker')
 logger = logging.getLogger('assemblyline.vacuum.worker')
@@ -410,7 +413,7 @@ class FileProcessor(threading.Thread):
                     'deep_scan': False,
                     "priority": 150,
                     "ignore_cache": False,
-                    "ignore_dynamic_recursion_prevention": False,
+                    "ignore_recursion_prevention": False,
                     "ignore_filtering": False,
                     "type": "INGEST"
                 })

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.0.dev5
+Version: 4.6.0.dev7
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline_core-4.6.0.dev5 → assemblyline_core-4.6.0.dev7}/test/test_dispatcher.py

@@ -1,20 +1,10 @@
+import json
 import logging
 import time
 from unittest import mock
 
-import json
 import pytest
-
-from assemblyline.common.forge import get_service_queue, get_classification
-from assemblyline.odm.models.error import Error
-from assemblyline.odm.models.file import File
-from assemblyline.odm.models.result import Result
-from assemblyline.odm.models.user import User
-from assemblyline.odm.randomizer import random_model_obj, random_minimal_obj, get_random_hash
-from assemblyline.odm import models
-from assemblyline.common.metrics import MetricsFactory
-
-from assemblyline_core.dispatching.client import DispatchClient, DISPATCH_RESULT_QUEUE
+from assemblyline_core.dispatching.client import DISPATCH_RESULT_QUEUE, DispatchClient
 from assemblyline_core.dispatching.dispatcher import Dispatcher, ServiceTask, Submission
 from assemblyline_core.dispatching.schedules import Scheduler as RealScheduler
 
@@ -23,6 +13,18 @@ from assemblyline_core.dispatching.timeout import TimeoutTable
 from mocking import ToggleTrue
 from test_scheduler import dummy_service
 
+from assemblyline.common.forge import get_classification, get_service_queue
+from assemblyline.common.metrics import MetricsFactory
+from assemblyline.odm import models
+from assemblyline.odm.models.error import Error
+from assemblyline.odm.models.file import File
+from assemblyline.odm.models.result import Result
+from assemblyline.odm.models.user import User
+from assemblyline.odm.randomizer import (
+    get_random_hash,
+    random_minimal_obj,
+    random_model_obj,
+)
 
 logger = logging.getLogger('assemblyline.test')
 
@@ -311,8 +313,6 @@ def test_dispatch_extracted_bypass_drp(clean_redis, clean_datastore):
     submission = random_model_obj(Submission)
     submission.to_be_deleted = False
 
-    # the following 1 line can be removed after assemblyline upgrade to version 4.6+
-    submission.params.ignore_dynamic_recursion_prevention = False
     submission.params.ignore_recursion_prevention = False
     submission.params.services.selected = ['extract', 'sandbox']
     submission.files = [dict(name='./file', sha256=file_hash)]

assemblyline_core-4.6.0.dev5/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.6.0.dev5