assemblyline-core 4.6.1.dev68__tar.gz → 4.6.1.dev72__tar.gz

{assemblyline_core-4.6.1.dev68 → assemblyline_core-4.6.1.dev72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.1.dev68
+Version: 4.6.1.dev72
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.6.1.dev72/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.6.1.dev72

{assemblyline_core-4.6.1.dev68 → assemblyline_core-4.6.1.dev72}/assemblyline_core/replay/client.py

@@ -4,13 +4,13 @@ import time
 
 from assemblyline.common import forge
 from assemblyline.common.bundling import create_bundle, import_bundle
+from assemblyline.common.classification import InvalidClassification
 from assemblyline.odm import Model
-from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline.remote.datatypes.hash import Hash
-from assemblyline_core.replay.replay import INPUT_TYPES
+from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.badlist_client import BadlistClient
+from assemblyline_core.replay.replay import INPUT_TYPES
 from assemblyline_core.safelist_client import SafelistClient
-from assemblyline_core.signature_client import SignatureClient
 
 EMPTY_WAIT_TIME = int(os.environ.get('EMPTY_WAIT_TIME', '30'))
 REPLAY_REQUESTED = 'requested'
@@ -296,13 +296,14 @@ class APIClient(ClientBase):
     def create_al_bundle(self, id, bundle_path, use_alert=False):
         self.al_client.bundle.create(id, output=bundle_path, use_alert=use_alert)
 
-    def load_bundle(self, bundle_path, min_classification, rescan_services, exist_ok=True):
+    def load_bundle(self, bundle_path, min_classification, rescan_services, exist_ok=True, reclassification=None):
         self.al_client.bundle.import_bundle(bundle_path,
                                             min_classification=min_classification,
                                             rescan_services=rescan_services,
-                                            exist_ok=exist_ok)
+                                            exist_ok=exist_ok,
+                                            reclassification=reclassification)
 
-    def load_json(self, file_path):
+    def load_json(self, file_path, reclassification=None):
         from assemblyline_client import ClientError
 
         # We're assuming all JSON that loaded has an "enabled" field
@@ -374,6 +375,7 @@ class DirectClient(ClientBase):
         # Initialize connection to redis-persistent for checkpointing
         redis_persist = get_client(config.core.redis.persistent.host,
                                    config.core.redis.persistent.port, False)
+        self.classification = forge.get_classification()
         self.datastore = forge.get_datastore(config=config)
         self.queues = {
             queue_type: NamedQueue(f"replay_{queue_type}", host=redis)
@@ -409,13 +411,14 @@ class DirectClient(ClientBase):
         temp_bundle_file = create_bundle(id, working_dir=os.path.dirname(bundle_path), use_alert=use_alert)
         os.rename(temp_bundle_file, bundle_path)
 
-    def load_bundle(self, bundle_path, min_classification, rescan_services, exist_ok=True):
+    def load_bundle(self, bundle_path, min_classification, rescan_services, exist_ok=True, reclassification=None):
         import_bundle(bundle_path,
                       min_classification=min_classification,
                       rescan_services=rescan_services,
-                      exist_ok=exist_ok)
+                      exist_ok=exist_ok,
+                      reclassification=reclassification)
 
-    def load_json(self, file_path):
+    def load_json(self, file_path, reclassification=None):
         # We're assuming all JSON that loaded has an "enabled" field
         collection = os.path.basename(file_path).split('_', 1)[0]
         with open(file_path) as fp:
@@ -428,6 +431,17 @@ class DirectClient(ClientBase):
 
                 # Let's see if there's an existing document with the same ID in the collection
                 obj = es_collection.get_if_exists(id, as_obj=False)
+                if obj:
+                    # Check if the classification of the object is compatible with the system's classification
+                    try:
+                        self.classification.normalize_classification(obj['classification'])
+                    except InvalidClassification:
+                        if reclassification:
+                            # If reclassification is requested, then we can change the classification
+                            obj['classification'] = reclassification
+                        else:
+                            raise
+
 
                 if collection == "workflow":
                     # If there has been any edits by another user, then preserve the enabled state

{assemblyline_core-4.6.1.dev68 → assemblyline_core-4.6.1.dev72}/assemblyline_core/replay/loader/run_worker.py

@@ -1,5 +1,5 @@
-import shutil
 import os
+import shutil
 
 from cart import unpack_file
 
@@ -32,15 +32,16 @@ class ReplayLoaderWorker(ReplayBase):
                     if file_path.endswith(".al_bundle"):
                         self.client.load_bundle(file_path,
                                                 min_classification=self.replay_config.loader.min_classification,
-                                                rescan_services=self.replay_config.loader.rescan)
+                                                rescan_services=self.replay_config.loader.rescan,
+                                                reclassification=self.replay_config.loader.reclassification)
                     elif file_path.endswith(".al_json"):
-                        self.client.load_json(file_path)
+                        self.client.load_json(file_path, reclassification=self.replay_config.loader.reclassification)
 
                     elif file_path.endswith(".al_json.cart"):
                         cart_path = file_path
                         file_path = file_path[:-5]
                         unpack_file(cart_path, file_path)
-                        self.client.load_json(file_path)
+                        self.client.load_json(file_path, reclassification=self.replay_config.loader.reclassification)
                         os.unlink(cart_path)
 
                     if os.path.exists(file_path):
@@ -55,11 +56,11 @@ class ReplayLoaderWorker(ReplayBase):
                         # Terminate on NFS-related error
                         self.log.warning("'Invalid cross-device link' exception detected. Terminating..")
                         self.stop()
-                except Exception:
+                except Exception as e:
                     # Make sure failed directory exists
                     os.makedirs(self.replay_config.loader.failed_directory, exist_ok=True)
 
-                    self.log.error(f"Failed to load the bundle file {file_path}, moving it to the failed directory.")
+                    self.log.error(f"Failed to load the bundle file {file_path}, moving it to the failed directory. Reason: {e}")
                     failed_path = os.path.join(self.replay_config.loader.failed_directory, os.path.basename(file_path))
                     shutil.move(file_path, failed_path)
 

{assemblyline_core-4.6.1.dev68 → assemblyline_core-4.6.1.dev72}/assemblyline_core/scaler/controllers/docker_ctl.py

@@ -500,6 +500,7 @@ class DockerController(ControllerInterface):
             instance_key = uuid.uuid4().hex
 
         volumes = {_n: {'bind': _v.mount_path, 'mode': 'rw'} for _n, _v in spec.volumes.items()}
+        volumes.update({row[0]: {'bind': row[1], 'mode': 'ro'} for row in self.global_mounts})
         if spec.run_as_core:
             volumes.update({row[0]: {'bind': row[1], 'mode': 'ro'} for row in self.core_mounts})
 

{assemblyline_core-4.6.1.dev68 → assemblyline_core-4.6.1.dev72}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.6.1.dev68
+Version: 4.6.1.dev72
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline_core-4.6.1.dev68 → assemblyline_core-4.6.1.dev72}/test/test_replay.py

@@ -2,17 +2,32 @@ import collections
 import json
 import os
 import random
-import time
+import tarfile
+import tempfile
 
+import cart
 import pytest
-
-from assemblyline.common import forge
-from assemblyline.odm.random_data import create_alerts, wipe_alerts, wipe_submissions, create_submission, create_badlists, create_safelists, create_workflows, wipe_badlist, wipe_safelist, wipe_workflows, create_signatures, wipe_signatures
 from assemblyline_core.replay.creator.run import ReplayCreator
 from assemblyline_core.replay.creator.run_worker import ReplayCreatorWorker
 from assemblyline_core.replay.loader.run import ReplayLoader
 from assemblyline_core.replay.loader.run_worker import ReplayLoaderWorker
 
+from assemblyline.common import forge
+from assemblyline.odm.random_data import (
+    create_alerts,
+    create_badlists,
+    create_safelists,
+    create_signatures,
+    create_submission,
+    create_workflows,
+    wipe_alerts,
+    wipe_badlist,
+    wipe_safelist,
+    wipe_signatures,
+    wipe_submissions,
+    wipe_workflows,
+)
+
 NUM_ALERTS = 1
 NUM_BADLIST_ITEMS = 1
 NUM_SAFELIST_ITEMS = 1
@@ -189,6 +204,7 @@ def test_replay_single_alert(config, datastore, creator, creator_worker, loader,
 def test_replay_single_submission(config, datastore, creator, creator_worker, loader, loader_worker):
     output_dir = creator.replay_config.creator.output_filestore.replace('file://', '')
     input_dir = loader.replay_config.loader.input_directory
+    loader_worker.replay_config.loader.reclassification = "TLP:CLEAR"
 
     # Make sure the submission get picked up by the creator
     sub = random.choice(all_submissions).as_primitives()
@@ -213,6 +229,30 @@ def test_replay_single_submission(config, datastore, creator, creator_worker, lo
     datastore.submission.delete(sub['sid'])
     datastore.submission.commit()
 
+    # Manipulate the classfication of the bundle to something that's invalid to the system importing
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Unpack the bundle to manipulate the results.json file
+        tar_file = os.path.join(temp_dir, "bundle.tar.gz")
+        with open(tar_file, 'wb') as fp:
+            with open(filename, 'rb') as ffp:
+                hdr, _ = cart.unpack_stream(ffp, fp)
+
+
+        with tarfile.open(tar_file, 'r:gz') as tar:
+            results = json.load(tar.extractfile("results.json"))
+
+        # Insert a random classification string that isn't recognized by the Assemblyline system
+        results['submission']['classification'] = "ASSEMBLYLINE"
+
+        with tarfile.open(tar_file, 'w:gz') as tar:
+            with tempfile.NamedTemporaryFile("w") as temp_result:
+                json.dump(results, temp_result)
+                temp_result.seek(0)
+                tar.add(temp_result.name, "results.json")
+
+        # Cart the file and overwrite the original file
+        cart.pack_file(tar_file, filename, optional_header=hdr)
+
     # In case the replay.yaml config creator output is not the same as loader input
     new_filename = filename.replace(output_dir, input_dir)
     if filename != new_filename:
@@ -231,10 +271,12 @@ def test_replay_single_submission(config, datastore, creator, creator_worker, lo
 
     loaded_submission = datastore.submission.get(sub['sid'], as_obj=False)
     assert 'bundle.loaded' in loaded_submission['metadata']
+    # Check to see if the reclassification took place and if we're preserving the original classification in the metadata
+    assert loaded_submission['classification'] in ["TLP:CLEAR", "TLP:C"]
+    assert loaded_submission['metadata']['bundle.classification'] == "ASSEMBLYLINE"
     assert sub['sid'] == loaded_submission['sid']
     assert 'replay' not in loaded_submission['metadata']
 
-
 @pytest.mark.parametrize("collection", ["badlist", "safelist", "signature", "workflow"])
 def test_replay_single_data_collection(datastore, creator, creator_worker, loader, loader_worker, collection):
     output_dir = creator.replay_config.creator.output_filestore.replace('file://', '')

assemblyline_core-4.6.1.dev68/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.6.1.dev68