assemblyline-core 4.5.1.dev20__tar.gz → 4.5.1.dev21__tar.gz

{assemblyline-core-4.5.1.dev20 → assemblyline-core-4.5.1.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev20
+Version: 4.5.1.dev21
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev21/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev21

{assemblyline-core-4.5.1.dev20 → assemblyline-core-4.5.1.dev21}/assemblyline_core/metrics/metrics_server.py

@@ -4,10 +4,14 @@ import tempfile
 import sys
 import time
 from collections import Counter
-from threading import Lock
+from threading import Lock, Thread
+from os import environ, path
+from urllib.parse import urlparse
 
 import elasticapm
 import elasticsearch
+import requests
+from packaging import version
 
 from apscheduler.schedulers.background import BackgroundScheduler
 from assemblyline_core.metrics.heartbeat_formatter import HeartbeatFormatter
@@ -16,9 +20,6 @@ from assemblyline_core.server_base import ServerBase
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common import forge
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
-from os import environ, path
-from packaging import version
-from urllib.parse import urlparse
 
 METRICS_QUEUE = "assemblyline_metrics"
 NON_AGGREGATED = ['scaler', 'scaler_status']

{assemblyline-core-4.5.1.dev20 → assemblyline-core-4.5.1.dev21}/assemblyline_core/vacuum/worker.py

@@ -21,7 +21,7 @@ from assemblyline.common.codec import decode_file
 from assemblyline.common.dict_utils import flatten
 from assemblyline.common.log import init_logging
 from assemblyline.common.metrics import MetricsFactory
-from assemblyline.datastore.helper import AssemblylineDatastore
+from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator
 from assemblyline.common import identify
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.uid import get_random_id
@@ -164,6 +164,7 @@ class FileProcessor(threading.Thread):
         # Anything that can't be copied easily should be initialized in 'run'.
         self.config: Config = config
         self.datastore = datastore
+        self.metadata_check = MetadataValidator(datastore)
         self.counter = counter
         self.minimum_classification = self.config.core.vacuum.minimum_classification
         logger.info("Connect to work queue")
@@ -494,7 +495,7 @@ class FileProcessor(threading.Thread):
                 # Extract email body strings or similar password settings
                 password_strings = metadata.pop("email_strings", [])
                 if not isinstance(password_strings, list):
-                    logger.warning("Unsupported password list format: " + str(password_strings))
+                    logger.warning("Unsupported password list format: %s", password_strings)
                     password_strings = []
 
                 if password_strings:
@@ -504,6 +505,16 @@ class FileProcessor(threading.Thread):
                 # Set description if it does not exists
                 s_params['description'] = f"[{s_params['type']}] Inspection of file: {file_sha256}"
 
+                # Validate the metadata
+                while metadata:
+                    metadata_error = self.metadata_check.check_metadata(metadata)
+                    if metadata_error:
+                        logger.error("Could not accept metadata %s on %s: %s", metadata_error[0],
+                                     file_sha256, metadata_error[1])
+                        metadata.pop(metadata_error[0], None)
+                    else:
+                        break
+
                 # Create submission object
                 try:
                     submission_obj = Submission({

{assemblyline-core-4.5.1.dev20 → assemblyline-core-4.5.1.dev21}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev20
+Version: 4.5.1.dev21
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev20/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev20

{assemblyline-core-4.5.1.dev20 → assemblyline-core-4.5.1.dev21}/assemblyline_core/metrics/es_metrics.py

@@ -1,25 +1,25 @@
 import json
 import tempfile
-
-import elasticapm
+import time
 import sys
+from os import environ, path
+from urllib.parse import urlparse
 
+import elasticapm
 import elasticsearch
-import time
-
-from assemblyline_core.metrics.helper import with_retries, ensure_indexes
-from assemblyline_core.server_base import ServerBase
+from packaging import version
 
 from assemblyline.common import forge
 from assemblyline.common.isotime import now_as_iso
 
-from os import environ, path
-from packaging import version
-from urllib.parse import urlparse
+from assemblyline_core.metrics.helper import with_retries, ensure_indexes
+from assemblyline_core.server_base import ServerBase
+
 
 METRICSTORE_ROOT_CA_PATH = environ.get('METRICSTORE_ROOT_CA_PATH', '/etc/assemblyline/ssl/al_root-ca.crt')
 METRICSTORE_VERIFY_CERTS = environ.get('METRICSTORE_VERIFY_CERTS', 'true').lower() == "true"
 
+
 class ESMetricsServer(ServerBase):
     """
     There can only be one of these type of metrics server running because it gathers elasticsearch metrics for