assemblyline-core 4.5.0.38__tar.gz → 4.5.0.40__tar.gz

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.38
+Version: 4.5.0.40
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.40/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.40

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/assemblyline_core/dispatching/dispatcher.py

@@ -60,9 +60,11 @@ AL_SHUTDOWN_QUIT = 60
 FINALIZING_WINDOW = max(AL_SHUTDOWN_GRACE - AL_SHUTDOWN_QUIT, 0)
 RESULT_BATCH_SIZE = int(os.environ.get('DISPATCHER_RESULT_BATCH_SIZE', '50'))
 ERROR_BATCH_SIZE = int(os.environ.get('DISPATCHER_ERROR_BATCH_SIZE', '50'))
-DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
 DAY_IN_SECONDS = 24 * 60 * 60
 
+# TODO: DYNAMIC_ANALYSIS_CATEGORY can be removed after assemblyline version 4.6+
+DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
+
 
 class Action(enum.IntEnum):
     start = 0
@@ -165,9 +167,16 @@ class SubmissionTask:
                 service = scheduler.services.get(service)
                 if not service:
                     continue
+
+                # TODO: the following 2 lines can be removed when assemblyline changed to version 4.6+
                 if service.category == DYNAMIC_ANALYSIS_CATEGORY:
                     self.forbid_for_children(sha256, service.name)
 
+                prevented_services = scheduler.expand_categories(service.recursion_prevention)
+
+                for service_name in prevented_services:
+                    self.forbid_for_children(sha256, service_name)
+
             # Replay the process of receiving results for dispatcher internal state
             for k, result in results.items():
                 sha256, service, _ = k.split('.', 2)
@@ -204,6 +213,7 @@ class SubmissionTask:
         except KeyError:
             self._forbidden_services[sha256] = {service_name}
 
+
     def register_children(self, parent: str, children: list[str]):
         """
         Note for the purposes of dynamic recursion prevention which
@@ -658,7 +668,8 @@ class Dispatcher(ThreadedCoreBase):
 
             # If Dynamic Recursion Prevention is in effect and the file is not part of the bypass list,
             # Find the list of services this file is forbidden from being sent to.
-            ignore_drp = submission.params.ignore_dynamic_recursion_prevention
+            # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
+            ignore_drp = submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention
             if not ignore_drp and sha256 not in task.dynamic_recursion_bypass:
                 forbidden_services = task.find_recursion_excluded_services(sha256)
 
@@ -753,10 +764,17 @@ class Dispatcher(ThreadedCoreBase):
                     if service.uses_tag_scores:
                         tag_fields.append('score')
 
-                    # Mark this routing for the purposes of dynamic recursion prevention
+                    # Mark this routing for the purposes of recursion prevention
+                    # TODO: The following 2 lines can be removed after assemblyline upgrade to version 4.6+
                     if service.category == DYNAMIC_ANALYSIS_CATEGORY:
                         task.forbid_for_children(sha256, service_name)
 
+                    prevented_services = self.scheduler.expand_categories(service.recursion_prevention)
+
+                    for service_name in prevented_services:
+                        task.forbid_for_children(sha256, service_name)
+
+
                     # Build the actual service dispatch message
                     config = self.build_service_config(service, submission)
                     service_task = ServiceTask(dict(
@@ -771,7 +789,8 @@ class Dispatcher(ThreadedCoreBase):
                         max_files=task.submission.params.max_extracted,
                         ttl=submission.params.ttl,
                         ignore_cache=submission.params.ignore_cache,
-                        ignore_dynamic_recursion_prevention=submission.params.ignore_dynamic_recursion_prevention,
+                        # TODO: remove "or submission.params.ignore_dynamic_recursion_prevention" after assemblyline upgrade to version 4.6+
+                        ignore_recursion_prevention=submission.params.ignore_recursion_prevention or submission.params.ignore_dynamic_recursion_prevention ,
                         ignore_filtering=ignore_filtering,
                         tags=[{field: x[field] for field in tag_fields} for x in tags],
                         temporary_submission_data=[

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/assemblyline_core/dispatching/schedules.py

@@ -112,7 +112,7 @@ class Scheduler:
                     # things that we need to evaluate, and mark this
                     # group as having been seen.
                     services.extend(categories[name])
-                    seen_categories.update(name)
+                    seen_categories.add(name)
                 continue
 
             # If it isn't a category, its a service

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/assemblyline_core/replay/creator/run_worker.py

@@ -1,6 +1,9 @@
 import json
 import os
 
+from cart import pack_stream
+from io import BytesIO
+
 from assemblyline.filestore import FileStore
 from assemblyline.common.isotime import now_as_iso
 from assemblyline_core.replay.client import APIClient, DirectClient
@@ -107,11 +110,10 @@ class ReplayCreatorWorker(ReplayBase):
             os.makedirs(self.replay_config.creator.working_directory, exist_ok=True)
 
             # Create the JSON
-            json_fn = f"{collection}_{now_as_iso()}.al_json"
-            json_path = os.path.join(self.replay_config.creator.working_directory, json_fn)
-            with open(json_path, "w") as fp:
-                json.dump(batch, fp)
+            json_fn = f"{collection}_{now_as_iso()}.al_json.cart"
             json_path = os.path.join(self.replay_config.creator.working_directory, json_fn)
+            with open(json_path, "wb") as fp:
+                pack_stream(BytesIO(json.dumps(batch).encode()), fp)
 
             # Move the JSON
             self.filestore.upload(json_path, json_fn)

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/assemblyline_core/replay/loader/run.py

@@ -50,7 +50,9 @@ class ReplayLoader(ReplayBase):
             for root, _, files in os.walk(self.replay_config.loader.input_directory, topdown=False):
                 for name in files:
                     # Unexpected files that could be the result of external transfer mechanisms
-                    if name.startswith('.') or not (name.endswith('.al_bundle') or name.endswith('.al_json')):
+                    if name.startswith('.') or not (name.endswith('.al_bundle') or \
+                                                    name.endswith('.al_json') or \
+                                                    name.endswith('.al_json.cart')):
                         continue
 
                     file_path = os.path.join(root, name)

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/assemblyline_core/replay/loader/run_worker.py

@@ -1,6 +1,8 @@
 import shutil
 import os
 
+from cart import unpack_file
+
 from assemblyline_core.replay.client import APIClient, DirectClient
 from assemblyline_core.replay.replay import ReplayBase
 
@@ -34,6 +36,13 @@ class ReplayLoaderWorker(ReplayBase):
                     elif file_path.endswith(".al_json"):
                         self.client.load_json(file_path)
 
+                    elif file_path.endswith(".al_json.cart"):
+                        cart_path = file_path
+                        file_path = file_path[:-5]
+                        unpack_file(cart_path, file_path)
+                        self.client.load_json(file_path)
+                        os.unlink(cart_path)
+
                     if os.path.exists(file_path):
                         os.unlink(file_path)
                 except OSError as e:

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.38
+Version: 4.5.0.40
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/test/test_dispatcher.py

@@ -310,7 +310,10 @@ def test_dispatch_extracted_bypass_drp(clean_redis, clean_datastore):
     # Inject the fake submission
     submission = random_model_obj(Submission)
     submission.to_be_deleted = False
+
+    # the following 1 line can be removed after assemblyline upgrade to version 4.6+
     submission.params.ignore_dynamic_recursion_prevention = False
+    submission.params.ignore_recursion_prevention = False
     submission.params.services.selected = ['extract', 'sandbox']
     submission.files = [dict(name='./file', sha256=file_hash)]
     sid = submission.sid = 'first-submission'

{assemblyline-core-4.5.0.38 → assemblyline-core-4.5.0.40}/test/test_replay.py

@@ -251,8 +251,8 @@ def test_replay_single_data_collection(datastore, creator, creator_worker, loade
     getattr(creator_worker, f'process_{collection}')(once=True)
     assert creator_worker.client.queues[collection].length() == 0
     filename = os.path.join(output_dir,
-                            ([f for f in os.listdir(output_dir) if f.startswith(
-                                collection) and f.endswith('.al_json')] + ["not_found"])[0])
+                            ([f for f in os.listdir(output_dir) if f.startswith(collection) \
+                               and f.endswith('.al_json.cart')] + ["not_found"])[0])
     assert os.path.exists(filename)
 
     # Delete the item to test the loading process

assemblyline-core-4.5.0.38/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.38