assemblyline-core 4.5.1.dev223__tar.gz → 4.5.1.dev225__tar.gz

{assemblyline-core-4.5.1.dev223 → assemblyline-core-4.5.1.dev225}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev223
+Version: 4.5.1.dev225
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev225/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev225

{assemblyline-core-4.5.1.dev223 → assemblyline-core-4.5.1.dev225}/assemblyline_core/replay/creator/run_worker.py

@@ -1,6 +1,9 @@
 import json
 import os
 
+from cart import pack_stream
+from io import BytesIO
+
 from assemblyline.filestore import FileStore
 from assemblyline.common.isotime import now_as_iso
 from assemblyline_core.replay.client import APIClient, DirectClient
@@ -107,11 +110,10 @@ class ReplayCreatorWorker(ReplayBase):
             os.makedirs(self.replay_config.creator.working_directory, exist_ok=True)
 
             # Create the JSON
-            json_fn = f"{collection}_{now_as_iso()}.al_json"
-            json_path = os.path.join(self.replay_config.creator.working_directory, json_fn)
-            with open(json_path, "w") as fp:
-                json.dump(batch, fp)
+            json_fn = f"{collection}_{now_as_iso()}.al_json.cart"
             json_path = os.path.join(self.replay_config.creator.working_directory, json_fn)
+            with open(json_path, "wb") as fp:
+                pack_stream(BytesIO(json.dumps(batch).encode()), fp)
 
             # Move the JSON
             self.filestore.upload(json_path, json_fn)

{assemblyline-core-4.5.1.dev223 → assemblyline-core-4.5.1.dev225}/assemblyline_core/replay/loader/run.py

@@ -50,7 +50,9 @@ class ReplayLoader(ReplayBase):
             for root, _, files in os.walk(self.replay_config.loader.input_directory, topdown=False):
                 for name in files:
                     # Unexpected files that could be the result of external transfer mechanisms
-                    if name.startswith('.') or not (name.endswith('.al_bundle') or name.endswith('.al_json')):
+                    if name.startswith('.') or not (name.endswith('.al_bundle') or \
+                                                    name.endswith('.al_json') or \
+                                                    name.endswith('.al_json.cart')):
                         continue
 
                     file_path = os.path.join(root, name)

{assemblyline-core-4.5.1.dev223 → assemblyline-core-4.5.1.dev225}/assemblyline_core/replay/loader/run_worker.py

@@ -1,6 +1,8 @@
 import shutil
 import os
 
+from cart import unpack_file
+
 from assemblyline_core.replay.client import APIClient, DirectClient
 from assemblyline_core.replay.replay import ReplayBase
 
@@ -34,6 +36,13 @@ class ReplayLoaderWorker(ReplayBase):
                     elif file_path.endswith(".al_json"):
                         self.client.load_json(file_path)
 
+                    elif file_path.endswith(".al_json.cart"):
+                        cart_path = file_path
+                        file_path = file_path[:-5]
+                        unpack_file(cart_path, file_path)
+                        self.client.load_json(file_path)
+                        os.unlink(cart_path)
+
                     if os.path.exists(file_path):
                         os.unlink(file_path)
                 except OSError as e:

{assemblyline-core-4.5.1.dev223 → assemblyline-core-4.5.1.dev225}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev223
+Version: 4.5.1.dev225
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.1.dev223 → assemblyline-core-4.5.1.dev225}/test/test_replay.py

@@ -251,8 +251,8 @@ def test_replay_single_data_collection(datastore, creator, creator_worker, loade
     getattr(creator_worker, f'process_{collection}')(once=True)
     assert creator_worker.client.queues[collection].length() == 0
     filename = os.path.join(output_dir,
-                            ([f for f in os.listdir(output_dir) if f.startswith(
-                                collection) and f.endswith('.al_json')] + ["not_found"])[0])
+                            ([f for f in os.listdir(output_dir) if f.startswith(collection) \
+                               and f.endswith('.al_json.cart')] + ["not_found"])[0])
     assert os.path.exists(filename)
 
     # Delete the item to test the loading process

assemblyline-core-4.5.1.dev223/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev223