assemblyline-core 4.5.0.41__tar.gz → 4.5.0.43__tar.gz

{assemblyline-core-4.5.0.41 → assemblyline-core-4.5.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.41
+Version: 4.5.0.43
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.43/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.43

{assemblyline-core-4.5.0.41 → assemblyline-core-4.5.0.43}/assemblyline_core/vacuum/worker.py

@@ -165,6 +165,11 @@ class FileProcessor(threading.Thread):
         self.config: Config = config
         self.datastore = datastore
         self.metadata_check = MetadataValidator(datastore)
+        self.metadata_check_kwargs = {
+            'validation_scheme': self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {}),
+            'strict': self.config.core.vacuum.ingest_type in self.config.submission.metadata.strict_schemes
+        }
+        self.validation_scheme = self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {})
         self.counter = counter
         self.minimum_classification = self.config.core.vacuum.minimum_classification
         logger.info("Connect to work queue")
@@ -508,7 +513,7 @@ class FileProcessor(threading.Thread):
 
                 # Validate the metadata
                 while metadata:
-                    metadata_error = self.metadata_check.check_metadata(metadata)
+                    metadata_error = self.metadata_check.check_metadata(metadata, **self.metadata_check_kwargs)
                     if metadata_error:
                         logger.error("Could not accept metadata %s on %s: %s", metadata_error[0],
                                      file_sha256, metadata_error[1])

{assemblyline-core-4.5.0.41 → assemblyline-core-4.5.0.43}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.41
+Version: 4.5.0.43
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.0.41 → assemblyline-core-4.5.0.43}/test/test_vacuum.py

@@ -9,7 +9,7 @@ import time
 import random
 import threading
 
-from assemblyline.odm.models.config import Config
+from assemblyline.odm.models.config import Config, MetadataConfig
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.vacuum import crawler, worker
 
@@ -67,6 +67,17 @@ def test_worker(config: Config, redis_connection):
             }]
             config.core.vacuum.worker_threads = 1
 
+            # Apply strict metadata validation
+            config.submission.metadata.ingest = {
+                config.core.vacuum.ingest_type: {
+                    'stream': {
+                        'validator_type': 'integer',
+                        'required': True
+                    }
+                }
+            }
+            config.submission.metadata.strict_schemes = [config.core.vacuum.ingest_type]
+
             # Place a file that will be safe
             first_file = os.path.join(data_dir, uuid.uuid4().hex + '.meta')
             with open(first_file, 'w') as temp:

assemblyline-core-4.5.0.41/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.41