PyPI - assemblyline-core - Versions diffs - 4.5.1.dev18__tar.gz → 4.5.1.dev21__tar.gz - Mend

@@ -21,7 +21,7 @@ from assemblyline.common.codec import decode_file
 from assemblyline.common.dict_utils import flatten
 from assemblyline.common.log import init_logging
 from assemblyline.common.metrics import MetricsFactory
-from assemblyline.datastore.helper import AssemblylineDatastore
+from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator
 from assemblyline.common import identify
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.uid import get_random_id
@@ -164,6 +164,7 @@ class FileProcessor(threading.Thread):
         # Anything that can't be copied easily should be initialized in 'run'.
         self.config: Config = config
         self.datastore = datastore
+        self.metadata_check = MetadataValidator(datastore)
         self.counter = counter
         self.minimum_classification = self.config.core.vacuum.minimum_classification
         logger.info("Connect to work queue")
@@ -494,7 +495,7 @@ class FileProcessor(threading.Thread):
                 # Extract email body strings or similar password settings
                 password_strings = metadata.pop("email_strings", [])
                 if not isinstance(password_strings, list):
-                    logger.warning("Unsupported password list format: " + str(password_strings))
+                    logger.warning("Unsupported password list format: %s", password_strings)
                     password_strings = []
                 if password_strings:
@@ -504,6 +505,16 @@ class FileProcessor(threading.Thread):
                 # Set description if it does not exists
                 s_params['description'] = f"[{s_params['type']}] Inspection of file: {file_sha256}"
+                # Validate the metadata
+                while metadata:
+                    metadata_error = self.metadata_check.check_metadata(metadata)
+                    if metadata_error:
+                        logger.error("Could not accept metadata %s on %s: %s", metadata_error[0],
+                                     file_sha256, metadata_error[1])
+                        metadata.pop(metadata_error[0], None)
+                    else:
+                        break
                 # Create submission object
                 try:
                     submission_obj = Submission({

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev18
+Version: 4.5.1.dev21
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

@@ -4,10 +4,14 @@ import tempfile
 import sys
 import time
 from collections import Counter
-from threading import Lock
+from threading import Lock, Thread
+from os import environ, path
+from urllib.parse import urlparse
 import elasticapm
 import elasticsearch
+import requests
+from packaging import version
 from apscheduler.schedulers.background import BackgroundScheduler
 from assemblyline_core.metrics.heartbeat_formatter import HeartbeatFormatter
@@ -16,9 +20,6 @@ from assemblyline_core.server_base import ServerBase
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common import forge
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
-from os import environ, path
-from packaging import version
-from urllib.parse import urlparse
 METRICS_QUEUE = "assemblyline_metrics"
 NON_AGGREGATED = ['scaler', 'scaler_status']

@@ -169,12 +169,16 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
                 break
     if service_config.docker_config.registry_username and service_config.docker_config.registry_password:
+        # We're authenticating using Basic Auth
         auth_config = {
             'username': service_config.docker_config.registry_username,
             'password': service_config.docker_config.registry_password
         }
         upass = f"{service_config.docker_config.registry_username}:{service_config.docker_config.registry_password}"
         auth = f"Basic {b64encode(upass.encode()).decode()}"
+    elif service_config.docker_config.registry_password:
+        # We're assuming that if only a password is given, then this is a token
+        auth = f"Bearer {service_config.docker_config.registry_password}"
     proxies = None
     for reg_conf in system_config.core.updater.registry_configs:

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev18
+Version: 4.5.1.dev21
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

@@ -1,25 +1,25 @@
 import json
 import tempfile
-import elasticapm
+import time
 import sys
+from os import environ, path
+from urllib.parse import urlparse
+import elasticapm
 import elasticsearch
-import time
-from assemblyline_core.metrics.helper import with_retries, ensure_indexes
-from assemblyline_core.server_base import ServerBase
+from packaging import version
 from assemblyline.common import forge
 from assemblyline.common.isotime import now_as_iso
-from os import environ, path
-from packaging import version
-from urllib.parse import urlparse
+from assemblyline_core.metrics.helper import with_retries, ensure_indexes
+from assemblyline_core.server_base import ServerBase
 METRICSTORE_ROOT_CA_PATH = environ.get('METRICSTORE_ROOT_CA_PATH', '/etc/assemblyline/ssl/al_root-ca.crt')
 METRICSTORE_VERIFY_CERTS = environ.get('METRICSTORE_VERIFY_CERTS', 'true').lower() == "true"
 class ESMetricsServer(ServerBase):
     """
     There can only be one of these type of metrics server running because it gathers elasticsearch metrics for

assemblyline-core 4.5.1.dev18tar.gz → 4.5.1.dev21tar.gz

Potentially problematic release.

	@@ -0,0 +1 @@
1	+ 4.5.1.dev21

	@@ -1 +0,0 @@
1	- 4.5.1.dev18

assemblyline-core 4.5.1.dev18__tar.gz → 4.5.1.dev21__tar.gz

Potentially problematic release.

assemblyline-core 4.5.1.dev18tar.gz → 4.5.1.dev21tar.gz