assemblyline-core 4.5.1.dev370__tar.gz → 4.5.1.dev372__tar.gz

{assemblyline-core-4.5.1.dev370 → assemblyline-core-4.5.1.dev372}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev370
+Version: 4.5.1.dev372
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev372/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev372

{assemblyline-core-4.5.1.dev370 → assemblyline-core-4.5.1.dev372}/assemblyline_core/ingester/ingester.py

@@ -20,6 +20,7 @@ import elasticapm
 
 from assemblyline.common.postprocess import ActionWorker
 from assemblyline_core.server_base import ThreadedCoreBase
+from assemblyline.common.constants import DROP_PRIORITY
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.common.str_utils import dotdump, safe_str
 from assemblyline.common.exceptions import get_stacktrace_info
@@ -48,7 +49,6 @@ from .constants import INGEST_QUEUE_NAME, drop_chance, COMPLETE_QUEUE_NAME
 
 _dup_prefix = 'w-m-'
 _notification_queue_prefix = 'nq-'
-_min_priority = 1
 _max_retries = 10
 _retry_delay = 60 * 4  # Wait 4 minutes to retry
 _max_time = 2 * 24 * 60 * 60  # Wait 2 days for responses.
@@ -622,7 +622,8 @@ class Ingester(ThreadedCoreBase):
         # Reduce the priority by an order of magnitude for very old files.
         current_time = now()
         if priority and self.expired(current_time - task.submission.time.timestamp(), 0):
-            priority = (priority / 10) or 1
+            # Round priority reduction to the nearest integer
+            priority = round(priority / 10)
 
         param.priority = priority
 
@@ -845,7 +846,7 @@ class Ingester(ThreadedCoreBase):
         sample_threshold = self.config.core.ingester.sampling_at
 
         dropped = False
-        if priority <= _min_priority:
+        if priority == DROP_PRIORITY:
             dropped = True
         else:
             for level, rng in self.priority_range.items():

{assemblyline-core-4.5.1.dev370 → assemblyline-core-4.5.1.dev372}/assemblyline_core/vacuum/worker.py

@@ -40,6 +40,7 @@ from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline.remote.datatypes.hash import Hash
 
 from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
+from assemblyline_core.ingester.constants import INGEST_QUEUE_NAME
 
 from .safelist import VacuumSafelist
 from .department_map import DepartmentMap
@@ -189,7 +190,7 @@ class FileProcessor(threading.Thread):
         self.identify = identifier
 
         self.traffic_queue = CommsQueue('submissions', redis)
-        self.ingest_queue = NamedQueue("m-ingest", persistent_redis)
+        self.ingest_queue = NamedQueue(INGEST_QUEUE_NAME, persistent_redis)
         self.service_list = CachedObject(self.datastore.list_all_services, kwargs={'as_obj': False, 'full': True})
 
         with self.profile_lock:

{assemblyline-core-4.5.1.dev370 → assemblyline-core-4.5.1.dev372}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev370
+Version: 4.5.1.dev372
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.5.1.dev370 → assemblyline-core-4.5.1.dev372}/test/test_vacuum.py

@@ -12,6 +12,7 @@ import threading
 from assemblyline.odm.models.config import Config, MetadataConfig
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline_core.vacuum import crawler, worker
+from assemblyline_core.ingester.constants import INGEST_QUEUE_NAME
 
 
 def test_crawler(config: Config, redis_connection):
@@ -110,7 +111,7 @@ def test_worker(config: Config, redis_connection):
             queue.push(second_file)
 
             # Get a message from for the ingested file
-            ingest_queue = NamedQueue("m-ingest", redis_connection)
+            ingest_queue = NamedQueue(INGEST_QUEUE_NAME, redis_connection)
             ingested = ingest_queue.pop(timeout=20)
             assert ingested is not None
             assert ingested['files'][0]['sha256'] == sha256

assemblyline-core-4.5.1.dev370/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev370