assemblyline-core 4.5.1.dev372__tar.gz → 4.5.1.dev374__tar.gz

{assemblyline-core-4.5.1.dev372 → assemblyline-core-4.5.1.dev374}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev372
+Version: 4.5.1.dev374
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev374/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev374

{assemblyline-core-4.5.1.dev372 → assemblyline-core-4.5.1.dev374}/assemblyline_core/dispatching/dispatcher.py

@@ -732,7 +732,9 @@ class Dispatcher(ThreadedCoreBase):
             self.log.info(f"[{sid}] Submission counts towards {submission.params.submitter.upper()} quota")
 
         # Apply initial data parameter
-        temporary_data = TemporaryFileData(sha256, config=self.config.submission.temporary_keys)
+        temp_key_config = dict(self.config.submission.default_temporary_keys)
+        temp_key_config.update(self.config.submission.temporary_keys)
+        temporary_data = TemporaryFileData(sha256, config=temp_key_config)
         task.temporary_data[sha256] = temporary_data
         if submission.params.initial_data:
             try:

{assemblyline-core-4.5.1.dev372 → assemblyline-core-4.5.1.dev374}/assemblyline_core/tasking_client.py

@@ -159,8 +159,7 @@ class TaskingClient:
             if not self.datastore.service_delta.exists(service.name):
                 self.datastore.service_delta.save(service.name, {'version': service.version})
                 self.datastore.service_delta.commit()
-                self.log.info(f"{log_prefix}{service.name} "
-                              f"version ({service.version}) registered")
+                self.log.info(f"{log_prefix}{service.name} version ({service.version}) registered")
 
             new_heuristics = []
             if heuristics:

{assemblyline-core-4.5.1.dev372 → assemblyline-core-4.5.1.dev374}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev372
+Version: 4.5.1.dev374
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.1.dev372 → assemblyline-core-4.5.1.dev374}/test/test_simulation.py

@@ -250,8 +250,15 @@ def core(request, redis, filestore, config, clean_datastore: AssemblylineDatasto
     # Register services
     stages = get_service_stage_hash(redis)
 
+    service_config: list[tuple[str, int, str, dict]] = [
+        ('pre', 1, 'EXTRACT', {'extra_data': True, 'monitored_keys': ['passwords']}),
+        ('core-a', 2, 'CORE', {}),
+        ('core-b', 1, 'CORE', {}),
+        ('finish', 1, 'POST', {'extra_data': True})
+    ]
+
     services = []
-    for svc, stage, details in [('pre', 'EXTRACT', {'extra_data': True, 'monitored_keys': ['passwords']}), ('core-a', 'CORE', {}), ('core-b', 'CORE', {}), ('finish', 'POST', {'extra_data': True})]:
+    for svc, count, stage, details in service_config:
         ds.service.save(f'{svc}_0', dummy_service(svc, stage, docid=f'{svc}_0', **details))
         ds.service_delta.save(svc, ServiceDelta({
             'name': svc,
@@ -259,7 +266,8 @@ def core(request, redis, filestore, config, clean_datastore: AssemblylineDatasto
             'enabled': True
         }))
         stages.set(svc, ServiceStage.Running)
-        services.append(MockService(svc, ds, redis, filestore))
+        for _ in range(count):
+            services.append(MockService(svc, ds, redis, filestore))
 
     user = random_model_obj(User)
     user.uname = "user"
@@ -1272,7 +1280,7 @@ def test_complex_extracted(core: CoreSession, metrics):
     # 1. extract a file that will process to produce a partial result
     # 2. hold a few seconds on the second stage of the root file to let child start
     # 3. on the last stage of the root file produce the password
-    dispatcher.TIMEOUT_EXTRA_TIME = 10
+    dispatcher.TIMEOUT_EXTRA_TIME = 100
 
     child_sha, _ = ready_body(core, {
         'pre': {'partial': {'passwords': 'test_temp_data_monitoring'}},
@@ -1289,7 +1297,7 @@ def test_complex_extracted(core: CoreSession, metrics):
                 }]
             }
         },
-        'core-a': {'lock': 5},
+        'core-a': {'lock': 60},
         'finish': {'temporary_data': {'passwords': ['test_temp_data_monitoring']}},
     })
 
@@ -1315,6 +1323,9 @@ def test_complex_extracted(core: CoreSession, metrics):
 
     # Wait for the extract file to finish
     metrics.expect('dispatcher', 'files_completed', 1)
+    # check that there is a pending result in the dispatcher
+    task = next(iter(core.dispatcher.tasks.values()))
+    assert 1 == sum(int(summary.partial) for summary in task.service_results.values())
     _global_semaphore.release()
 
     # Wait for the entire submission to finish

assemblyline-core-4.5.1.dev372/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev372