assemblyline-core 4.5.0.42__tar.gz → 4.5.0.44__tar.gz

{assemblyline-core-4.5.0.42 → assemblyline-core-4.5.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.42
+Version: 4.5.0.44
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.44/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.44

{assemblyline-core-4.5.0.42 → assemblyline-core-4.5.0.44}/assemblyline_core/updater/helper.py

@@ -283,12 +283,12 @@ def _get_dockerhub_tags(image_name, update_channel, prefix, proxies=None, docker
         'ordering': 'last_updated'
     }
     url = f"{base_url}?{urlencode(query_params)}"
+    tags = []
     while True:
         try:
             response = requests.get(url, proxies=proxies)
             response.raise_for_status()  # Raises an HTTPError for bad responses
             response_data = response.json()
-            tags = []
             if namespace == "cccs":
                 # The tags are coming from a repository managed by the Assemblyline team
                 for tag_info in response_data['results']:

{assemblyline-core-4.5.0.42 → assemblyline-core-4.5.0.44}/assemblyline_core/vacuum/worker.py

@@ -165,6 +165,11 @@ class FileProcessor(threading.Thread):
         self.config: Config = config
         self.datastore = datastore
         self.metadata_check = MetadataValidator(datastore)
+        self.metadata_check_kwargs = {
+            'validation_scheme': self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {}),
+            'strict': self.config.core.vacuum.ingest_type in self.config.submission.metadata.strict_schemes
+        }
+        self.validation_scheme = self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {})
         self.counter = counter
         self.minimum_classification = self.config.core.vacuum.minimum_classification
         logger.info("Connect to work queue")
@@ -508,7 +513,7 @@ class FileProcessor(threading.Thread):
 
                 # Validate the metadata
                 while metadata:
-                    metadata_error = self.metadata_check.check_metadata(metadata)
+                    metadata_error = self.metadata_check.check_metadata(metadata, **self.metadata_check_kwargs)
                     if metadata_error:
                         logger.error("Could not accept metadata %s on %s: %s", metadata_error[0],
                                      file_sha256, metadata_error[1])

{assemblyline-core-4.5.0.42 → assemblyline-core-4.5.0.44}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.42
+Version: 4.5.0.44
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.0.42 → assemblyline-core-4.5.0.44}/test/test_vacuum.py

@@ -9,7 +9,7 @@ import time
 import random
 import threading
 
-from assemblyline.odm.models.config import Config
+from assemblyline.odm.models.config import Config, MetadataConfig
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.vacuum import crawler, worker
 
@@ -67,6 +67,17 @@ def test_worker(config: Config, redis_connection):
             }]
             config.core.vacuum.worker_threads = 1
 
+            # Apply strict metadata validation
+            config.submission.metadata.ingest = {
+                config.core.vacuum.ingest_type: {
+                    'stream': {
+                        'validator_type': 'integer',
+                        'required': True
+                    }
+                }
+            }
+            config.submission.metadata.strict_schemes = [config.core.vacuum.ingest_type]
+
             # Place a file that will be safe
             first_file = os.path.join(data_dir, uuid.uuid4().hex + '.meta')
             with open(first_file, 'w') as temp:

assemblyline-core-4.5.0.42/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.42