assemblyline-core 4.5.0.37__tar.gz → 4.5.0.39__tar.gz

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.37
+Version: 4.5.0.39
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.39/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.39

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/assemblyline_core/replay/creator/run_worker.py

@@ -1,6 +1,9 @@
 import json
 import os
 
+from cart import pack_stream
+from io import BytesIO
+
 from assemblyline.filestore import FileStore
 from assemblyline.common.isotime import now_as_iso
 from assemblyline_core.replay.client import APIClient, DirectClient
@@ -107,11 +110,10 @@ class ReplayCreatorWorker(ReplayBase):
             os.makedirs(self.replay_config.creator.working_directory, exist_ok=True)
 
             # Create the JSON
-            json_fn = f"{collection}_{now_as_iso()}.al_json"
-            json_path = os.path.join(self.replay_config.creator.working_directory, json_fn)
-            with open(json_path, "w") as fp:
-                json.dump(batch, fp)
+            json_fn = f"{collection}_{now_as_iso()}.al_json.cart"
             json_path = os.path.join(self.replay_config.creator.working_directory, json_fn)
+            with open(json_path, "wb") as fp:
+                pack_stream(BytesIO(json.dumps(batch).encode()), fp)
 
             # Move the JSON
             self.filestore.upload(json_path, json_fn)

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/assemblyline_core/replay/loader/run.py

@@ -50,7 +50,9 @@ class ReplayLoader(ReplayBase):
             for root, _, files in os.walk(self.replay_config.loader.input_directory, topdown=False):
                 for name in files:
                     # Unexpected files that could be the result of external transfer mechanisms
-                    if name.startswith('.') or not (name.endswith('.al_bundle') or name.endswith('.al_json')):
+                    if name.startswith('.') or not (name.endswith('.al_bundle') or \
+                                                    name.endswith('.al_json') or \
+                                                    name.endswith('.al_json.cart')):
                         continue
 
                     file_path = os.path.join(root, name)

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/assemblyline_core/replay/loader/run_worker.py

@@ -1,6 +1,8 @@
 import shutil
 import os
 
+from cart import unpack_file
+
 from assemblyline_core.replay.client import APIClient, DirectClient
 from assemblyline_core.replay.replay import ReplayBase
 
@@ -34,6 +36,13 @@ class ReplayLoaderWorker(ReplayBase):
                     elif file_path.endswith(".al_json"):
                         self.client.load_json(file_path)
 
+                    elif file_path.endswith(".al_json.cart"):
+                        cart_path = file_path
+                        file_path = file_path[:-5]
+                        unpack_file(cart_path, file_path)
+                        self.client.load_json(file_path)
+                        os.unlink(cart_path)
+
                     if os.path.exists(file_path):
                         os.unlink(file_path)
                 except OSError as e:

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/assemblyline_core/tasking_client.py

@@ -8,7 +8,7 @@ from assemblyline.common import forge
 from assemblyline.common.constants import SERVICE_STATE_HASH, ServiceStatus
 from assemblyline.common.dict_utils import flatten, unflatten
 from assemblyline.common.heuristics import HeuristicHandler, InvalidHeuristicException
-from assemblyline.common.isotime import now_as_iso
+from assemblyline.common.isotime import now_as_iso, now_as_utc_datetime
 from assemblyline.common.threading import APMAwareThreadPoolExecutor
 from assemblyline.datastore.helper import AssemblylineDatastore
 from assemblyline.filestore import FileStore
@@ -264,7 +264,7 @@ class TaskingClient:
                 result.archive_ts = None
 
                 if task.ttl and result.expiry_ts:
-                    result.expiry_ts = max(result.expiry_ts, now_as_iso(task.ttl * 24 * 60 * 60))
+                    result.expiry_ts = max(result.expiry_ts, now_as_utc_datetime(task.ttl * 24 * 60 * 60))
 
                 # Create a list of files to freshen
                 freshen_hashes = [task.fileinfo.sha256]

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.37
+Version: 4.5.0.39
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.0.37 → assemblyline-core-4.5.0.39}/test/test_replay.py

@@ -251,8 +251,8 @@ def test_replay_single_data_collection(datastore, creator, creator_worker, loade
     getattr(creator_worker, f'process_{collection}')(once=True)
     assert creator_worker.client.queues[collection].length() == 0
     filename = os.path.join(output_dir,
-                            ([f for f in os.listdir(output_dir) if f.startswith(
-                                collection) and f.endswith('.al_json')] + ["not_found"])[0])
+                            ([f for f in os.listdir(output_dir) if f.startswith(collection) \
+                               and f.endswith('.al_json.cart')] + ["not_found"])[0])
     assert os.path.exists(filename)
 
     # Delete the item to test the loading process

assemblyline-core-4.5.0.37/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.37