assemblyline-core 4.4.1.dev296__tar.gz → 4.4.1.dev297__tar.gz

{assemblyline-core-4.4.1.dev296 → assemblyline-core-4.4.1.dev297}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.4.1.dev296
+Version: 4.4.1.dev297
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.4.1.dev297/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.4.1.dev297

{assemblyline-core-4.4.1.dev296 → assemblyline-core-4.4.1.dev297}/assemblyline_core/dispatching/client.py

@@ -15,6 +15,7 @@ from assemblyline.common import forge
 from assemblyline.common.constants import DISPATCH_RUNNING_TASK_HASH, SUBMISSION_QUEUE, \
     make_watcher_list_name, DISPATCH_TASK_HASH
 from assemblyline.common.forge import CachedObject, get_service_queue
+from assemblyline.common.isotime import now_as_iso
 from assemblyline.datastore.exceptions import VersionConflictException
 from assemblyline.odm.base import DATEFORMAT
 from assemblyline.odm.messages.dispatching import DispatcherCommandMessage, CREATE_WATCH, \
@@ -267,6 +268,7 @@ class DispatchClient:
         # Save or freshen the result, the CONTENT of the result shouldn't change, but we need to keep the
         # most distant expiry time to prevent pulling it out from under another submission too early
         if result.is_empty():
+            result.expiry_ts = now_as_iso(self.config.submission.emptyresult_dtl * 24 * 60 * 60)
             self.ds.emptyresult.save(result_key, {"expiry_ts": result.expiry_ts})
         else:
             while True:

{assemblyline-core-4.4.1.dev296 → assemblyline-core-4.4.1.dev297}/assemblyline_core/submission_client.py

@@ -27,7 +27,7 @@ from typing import List, Optional, Tuple, Dict
 from assemblyline.common import forge
 from assemblyline.common.codec import decode_file
 from assemblyline.common.dict_utils import flatten
-from assemblyline.common.isotime import epoch_to_iso, now, now_as_iso
+from assemblyline.common.isotime import epoch_to_iso, now
 from assemblyline.common.str_utils import safe_str
 from assemblyline.datastore.helper import AssemblylineDatastore
 from assemblyline.filestore import FileStore
@@ -135,15 +135,16 @@ class SubmissionClient:
 
         # Figure out the expiry for the submission if none was provided
         if expiry is None:
+            # Enforce maximum DTL
+            if self.config.submission.max_dtl > 0:
+                if int(submission_obj.params.ttl):
+                    submission_obj.params.ttl = min(int(submission_obj.params.ttl), self.config.submission.max_dtl)
+                else:
+                    submission_obj.params.ttl = self.config.submission.max_dtl
+
             if submission_obj.params.ttl:
                 expiry = epoch_to_iso(submission_obj.time.timestamp() + submission_obj.params.ttl * SECONDS_PER_DAY)
 
-        # Enforce the max_dtl
-        if self.config.submission.max_dtl > 0:
-            max_expiry = now_as_iso(self.config.submission.max_dtl * SECONDS_PER_DAY)
-            if not expiry or expiry > max_expiry:
-                expiry = max_expiry
-
         max_size = self.config.submission.max_file_size
 
         for local_file in local_files:

{assemblyline-core-4.4.1.dev296 → assemblyline-core-4.4.1.dev297}/assemblyline_core/tasking_client.py

@@ -263,7 +263,12 @@ class TaskingClient:
 
             if not cache_found:
                 # Checking for previous empty results for this key
-                result = self.datastore.emptyresult.get_if_exists(f"{result_key}.e")
+                try:
+                    result = self.datastore.emptyresult.get_if_exists(f"{result_key}.e")
+                except ValueError:
+                    self.log.warning(f"Got poisoned empty result cache record for key {result_key}.e, cleaning up...")
+                    self.datastore.emptyresult.delete(f"{result_key}.e")
+
                 if result:
                     metric_factory.increment('cache_hit')
                     metric_factory.increment('not_scored')

{assemblyline-core-4.4.1.dev296 → assemblyline-core-4.4.1.dev297}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.4.1.dev296
+Version: 4.4.1.dev297
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.4.1.dev296/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.4.1.dev296