assemblyline-core 4.5.0.61__tar.gz → 4.5.0.63__tar.gz

{assemblyline-core-4.5.0.61 → assemblyline-core-4.5.0.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.61
+Version: 4.5.0.63
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.63/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.63

{assemblyline-core-4.5.0.61 → assemblyline-core-4.5.0.63}/assemblyline_core/ingester/ingester.py

@@ -20,6 +20,7 @@ import elasticapm
 
 from assemblyline.common.postprocess import ActionWorker
 from assemblyline_core.server_base import ThreadedCoreBase
+from assemblyline.common.constants import DROP_PRIORITY
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.common.str_utils import dotdump, safe_str
 from assemblyline.common.exceptions import get_stacktrace_info
@@ -48,7 +49,6 @@ from .constants import INGEST_QUEUE_NAME, drop_chance, COMPLETE_QUEUE_NAME
 
 _dup_prefix = 'w-m-'
 _notification_queue_prefix = 'nq-'
-_min_priority = 1
 _max_retries = 10
 _retry_delay = 60 * 4  # Wait 4 minutes to retry
 _max_time = 2 * 24 * 60 * 60  # Wait 2 days for responses.
@@ -622,7 +622,8 @@ class Ingester(ThreadedCoreBase):
         # Reduce the priority by an order of magnitude for very old files.
         current_time = now()
         if priority and self.expired(current_time - task.submission.time.timestamp(), 0):
-            priority = (priority / 10) or 1
+            # Round priority reduction to the nearest integer
+            priority = round(priority / 10)
 
         param.priority = priority
 
@@ -845,7 +846,7 @@ class Ingester(ThreadedCoreBase):
         sample_threshold = self.config.core.ingester.sampling_at
 
         dropped = False
-        if priority <= _min_priority:
+        if priority == DROP_PRIORITY:
             dropped = True
         else:
             for level, rng in self.priority_range.items():

{assemblyline-core-4.5.0.61 → assemblyline-core-4.5.0.63}/assemblyline_core/vacuum/worker.py

@@ -37,8 +37,10 @@ from assemblyline.common.str_utils import safe_str
 from assemblyline.remote.datatypes import get_client as get_redis_client
 from assemblyline.odm.messages.submission import Submission
 from assemblyline.remote.datatypes.queues.named import NamedQueue
+from assemblyline.remote.datatypes.hash import Hash
 
 from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
+from assemblyline_core.ingester.constants import INGEST_QUEUE_NAME
 
 from .safelist import VacuumSafelist
 from .department_map import DepartmentMap
@@ -164,12 +166,16 @@ class FileProcessor(threading.Thread):
         # Anything that can't be copied easily should be initialized in 'run'.
         self.config: Config = config
         self.datastore = datastore
-        self.metadata_check = MetadataValidator(datastore)
+        self.metadata_check = MetadataValidator(datastore, Hash("metadata_suggestions", persistent_redis))
+
+        # Merge the default metadata required for ingestion with those that are required from vacuum
+        validation_scheme = config.submission.metadata.ingest.get('_default', {})
+        validation_scheme.update(config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {}))
+
         self.metadata_check_kwargs = {
-            'validation_scheme': self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {}),
+            'validation_scheme': validation_scheme,
             'strict': self.config.core.vacuum.ingest_type in self.config.submission.metadata.strict_schemes
         }
-        self.validation_scheme = self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {})
         self.counter = counter
         self.minimum_classification = self.config.core.vacuum.minimum_classification
         logger.info("Connect to work queue")
@@ -184,7 +190,7 @@ class FileProcessor(threading.Thread):
         self.identify = identifier
 
         self.traffic_queue = CommsQueue('submissions', redis)
-        self.ingest_queue = NamedQueue("m-ingest", persistent_redis)
+        self.ingest_queue = NamedQueue(INGEST_QUEUE_NAME, persistent_redis)
         self.service_list = CachedObject(self.datastore.list_all_services, kwargs={'as_obj': False, 'full': True})
 
         with self.profile_lock:

{assemblyline-core-4.5.0.61 → assemblyline-core-4.5.0.63}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.61
+Version: 4.5.0.63
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.0.61 → assemblyline-core-4.5.0.63}/test/test_vacuum.py

@@ -12,6 +12,7 @@ import threading
 from assemblyline.odm.models.config import Config, MetadataConfig
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.vacuum import crawler, worker
+from assemblyline_core.ingester.constants import INGEST_QUEUE_NAME
 
 
 def test_crawler(config: Config, redis_connection):
@@ -110,7 +111,7 @@ def test_worker(config: Config, redis_connection):
             queue.push(second_file)
 
             # Get a message from for the ingested file
-            ingest_queue = NamedQueue("m-ingest", redis_connection)
+            ingest_queue = NamedQueue(INGEST_QUEUE_NAME, redis_connection)
             ingested = ingest_queue.pop(timeout=20)
             assert ingested is not None
             assert ingested['files'][0]['sha256'] == sha256

assemblyline-core-4.5.0.61/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.61