assemblyline-core 4.5.0.39__tar.gz → 4.5.0.40__tar.gz

{assemblyline-core-4.5.0.39 → assemblyline-core-4.5.0.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.39
+Version: 4.5.0.40
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.40/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.40

{assemblyline-core-4.5.0.39 → assemblyline-core-4.5.0.40}/assemblyline_core/dispatching/dispatcher.py

@@ -60,9 +60,11 @@ AL_SHUTDOWN_QUIT = 60
 FINALIZING_WINDOW = max(AL_SHUTDOWN_GRACE - AL_SHUTDOWN_QUIT, 0)
 RESULT_BATCH_SIZE = int(os.environ.get('DISPATCHER_RESULT_BATCH_SIZE', '50'))
 ERROR_BATCH_SIZE = int(os.environ.get('DISPATCHER_ERROR_BATCH_SIZE', '50'))
-DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
 DAY_IN_SECONDS = 24 * 60 * 60
 
+# TODO: DYNAMIC_ANALYSIS_CATEGORY can be removed after assemblyline version 4.6+
+DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
+
 
 class Action(enum.IntEnum):
     start = 0
@@ -165,9 +167,16 @@ class SubmissionTask:
                 service = scheduler.services.get(service)
                 if not service:
                     continue
+
+                # TODO: the following 2 lines can be removed when assemblyline changed to version 4.6+
                 if service.category == DYNAMIC_ANALYSIS_CATEGORY:
                     self.forbid_for_children(sha256, service.name)
 
+                prevented_services = scheduler.expand_categories(service.recursion_prevention)
+
+                for service_name in prevented_services:
+                    self.forbid_for_children(sha256, service_name)
+
             # Replay the process of receiving results for dispatcher internal state
             for k, result in results.items():
                 sha256, service, _ = k.split('.', 2)
@@ -204,6 +213,7 @@ class SubmissionTask:
         except KeyError:
             self._forbidden_services[sha256] = {service_name}
 
+
     def register_children(self, parent: str, children: list[str]):
         """
         Note for the purposes of dynamic recursion prevention which
@@ -658,7 +668,8 @@ class Dispatcher(ThreadedCoreBase):
 
             # If Dynamic Recursion Prevention is in effect and the file is not part of the bypass list,
             # Find the list of services this file is forbidden from being sent to.
-            ignore_drp = submission.params.ignore_dynamic_recursion_prevention
+            # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
+            ignore_drp = submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention
             if not ignore_drp and sha256 not in task.dynamic_recursion_bypass:
                 forbidden_services = task.find_recursion_excluded_services(sha256)
 
@@ -753,10 +764,17 @@ class Dispatcher(ThreadedCoreBase):
                     if service.uses_tag_scores:
                         tag_fields.append('score')
 
-                    # Mark this routing for the purposes of dynamic recursion prevention
+                    # Mark this routing for the purposes of recursion prevention
+                    # TODO: The following 2 lines can be removed after assemblyline upgrade to version 4.6+
                     if service.category == DYNAMIC_ANALYSIS_CATEGORY:
                         task.forbid_for_children(sha256, service_name)
 
+                    prevented_services = self.scheduler.expand_categories(service.recursion_prevention)
+
+                    for service_name in prevented_services:
+                        task.forbid_for_children(sha256, service_name)
+
+
                     # Build the actual service dispatch message
                     config = self.build_service_config(service, submission)
                     service_task = ServiceTask(dict(
@@ -771,7 +789,8 @@ class Dispatcher(ThreadedCoreBase):
                         max_files=task.submission.params.max_extracted,
                         ttl=submission.params.ttl,
                         ignore_cache=submission.params.ignore_cache,
-                        ignore_dynamic_recursion_prevention=submission.params.ignore_dynamic_recursion_prevention,
+                        # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
+                        ignore_recursion_prevention=submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention ,
                         ignore_filtering=ignore_filtering,
                         tags=[{field: x[field] for field in tag_fields} for x in tags],
                         temporary_submission_data=[

{assemblyline-core-4.5.0.39 → assemblyline-core-4.5.0.40}/assemblyline_core/dispatching/schedules.py

@@ -112,7 +112,7 @@ class Scheduler:
                     # things that we need to evaluate, and mark this
                     # group as having been seen.
                     services.extend(categories[name])
-                    seen_categories.update(name)
+                    seen_categories.add(name)
                 continue
 
             # If it isn't a category, its a service

{assemblyline-core-4.5.0.39 → assemblyline-core-4.5.0.40}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.39
+Version: 4.5.0.40
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.0.39 → assemblyline-core-4.5.0.40}/test/test_dispatcher.py

@@ -310,7 +310,10 @@ def test_dispatch_extracted_bypass_drp(clean_redis, clean_datastore):
     # Inject the fake submission
     submission = random_model_obj(Submission)
     submission.to_be_deleted = False
+
+    # the following 1 line can be removed after assemblyline upgrade to version 4.6+
     submission.params.ignore_dynamic_recursion_prevention = False
+    submission.params.ignore_recursion_prevention = False
     submission.params.services.selected = ['extract', 'sandbox']
     submission.files = [dict(name='./file', sha256=file_hash)]
     sid = submission.sid = 'first-submission'

assemblyline-core-4.5.0.39/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.39