assemblyline-core 4.5.1.dev240__tar.gz → 4.5.1.dev244__tar.gz

{assemblyline-core-4.5.1.dev240 → assemblyline-core-4.5.1.dev244}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev240
+Version: 4.5.1.dev244
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev244/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev244

{assemblyline-core-4.5.1.dev240 → assemblyline-core-4.5.1.dev244}/assemblyline_core/dispatching/dispatcher.py

@@ -61,7 +61,7 @@ RESULT_BATCH_SIZE = int(os.environ.get('DISPATCHER_RESULT_BATCH_SIZE', '50'))
 ERROR_BATCH_SIZE = int(os.environ.get('DISPATCHER_ERROR_BATCH_SIZE', '50'))
 DAY_IN_SECONDS = 24 * 60 * 60
 
-# TODO: DYNAMIC_ANALYSIS_CATEGORY can be removed after assemblyline version
+# TODO: DYNAMIC_ANALYSIS_CATEGORY can be removed after assemblyline version 4.6+
 DYNAMIC_ANALYSIS_CATEGORY = 'Dynamic Analysis'
 
 

{assemblyline-core-4.5.1.dev240 → assemblyline-core-4.5.1.dev244}/assemblyline_core/dispatching/schedules.py

@@ -112,7 +112,7 @@ class Scheduler:
                     # things that we need to evaluate, and mark this
                     # group as having been seen.
                     services.extend(categories[name])
-                    seen_categories.update(name)
+                    seen_categories.add(name)
                 continue
 
             # If it isn't a category, its a service

{assemblyline-core-4.5.1.dev240 → assemblyline-core-4.5.1.dev244}/assemblyline_core/vacuum/worker.py

@@ -165,6 +165,11 @@ class FileProcessor(threading.Thread):
         self.config: Config = config
         self.datastore = datastore
         self.metadata_check = MetadataValidator(datastore)
+        self.metadata_check_kwargs = {
+            'validation_scheme': self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {}),
+            'strict': self.config.core.vacuum.ingest_type in self.config.submission.metadata.strict_schemes
+        }
+        self.validation_scheme = self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {})
         self.counter = counter
         self.minimum_classification = self.config.core.vacuum.minimum_classification
         logger.info("Connect to work queue")
@@ -508,7 +513,7 @@ class FileProcessor(threading.Thread):
 
                 # Validate the metadata
                 while metadata:
-                    metadata_error = self.metadata_check.check_metadata(metadata)
+                    metadata_error = self.metadata_check.check_metadata(metadata, **self.metadata_check_kwargs)
                     if metadata_error:
                         logger.error("Could not accept metadata %s on %s: %s", metadata_error[0],
                                      file_sha256, metadata_error[1])

{assemblyline-core-4.5.1.dev240 → assemblyline-core-4.5.1.dev244}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev240
+Version: 4.5.1.dev244
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.1.dev240 → assemblyline-core-4.5.1.dev244}/test/test_vacuum.py

@@ -9,7 +9,7 @@ import time
 import random
 import threading
 
-from assemblyline.odm.models.config import Config
+from assemblyline.odm.models.config import Config, MetadataConfig
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.vacuum import crawler, worker
 
@@ -67,6 +67,17 @@ def test_worker(config: Config, redis_connection):
             }]
             config.core.vacuum.worker_threads = 1
 
+            # Apply strict metadata validation
+            config.submission.metadata.ingest = {
+                config.core.vacuum.ingest_type: {
+                    'stream': {
+                        'validator_type': 'integer',
+                        'required': True
+                    }
+                }
+            }
+            config.submission.metadata.strict_schemes = [config.core.vacuum.ingest_type]
+
             # Place a file that will be safe
             first_file = os.path.join(data_dir, uuid.uuid4().hex + '.meta')
             with open(first_file, 'w') as temp:

assemblyline-core-4.5.1.dev240/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev240