assemblyline-core 4.4.0.41__tar.gz → 4.4.0.43__tar.gz

{assemblyline-core-4.4.0.41 → assemblyline-core-4.4.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.4.0.41
+Version: 4.4.0.43
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.4.0.43/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.4.0.43

{assemblyline-core-4.4.0.41 → assemblyline-core-4.4.0.43}/assemblyline_core/alerter/processing.py

@@ -107,7 +107,7 @@ def get_submission_record(counter, datastore, sid):
 
     if not srecord:
         counter.increment('error')
-        raise SubmissionNotFound("Couldn't find submission: %s" % sid)
+        raise SubmissionNotFound(f"Couldn't find submission: {sid}")
 
     submission_state = srecord.get('state', 'unknown')
     if submission_state != 'completed':
@@ -285,7 +285,8 @@ def perform_alert_update(datastore, logger, alert):
     while True:
         old_alert, version = datastore.alert.get_if_exists(alert_id, as_obj=False, version=True)
         if old_alert is None:
-            raise AlertMissingError(f"{alert_id} is missing from the alert collection.")
+            raise AlertMissingError(
+                f"Alert {alert_id} cannot be updated because it does not exist.")
 
         # Ensure alert keeps original timestamp
         alert['ts'] = old_alert['ts']
@@ -321,14 +322,10 @@ def save_alert(datastore, counter, logger, alert, psid):
         return msg_type, ret_val
 
     if psid:
-        try:
-            msg_type = "AlertUpdated"
-            perform_alert_update(datastore, logger, alert)
-            counter.increment('updated')
-            ret_val = 'update'
-        except AlertMissingError as e:
-            logger.info(f"{str(e)}. Creating a new alert [{alert['alert_id']}]...")
-            msg_type, ret_val = create_alert()
+        msg_type = "AlertUpdated"
+        perform_alert_update(datastore, logger, alert)
+        counter.increment('updated')
+        ret_val = 'update'
     else:
         msg_type, ret_val = create_alert()
 
@@ -381,6 +378,9 @@ def get_alert_update_parts(counter, datastore, alert_data, logger, user_classifi
             }
         }
         cache.add(alert_data['submission']['sid'], (alert_update_p1, alert_update_p2))
+    else:
+        if alert_data['extended_scan'] == 'skipped':
+            alert_update_p1['extended_scan'] = alert_data['extended_scan']
 
     alert_update_p1['reporting_ts'] = now_as_iso()
     alert_update_p1['file'] = {'name': alert_file['name']}

{assemblyline-core-4.4.0.41 → assemblyline-core-4.4.0.43}/assemblyline_core/alerter/run_alerter.py

@@ -9,12 +9,14 @@ from assemblyline.remote.datatypes import get_client
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 from assemblyline.odm.messages.alerter_heartbeat import Metrics
 
-from assemblyline_core.alerter.processing import SubmissionNotFinalized
+from assemblyline_core.alerter.processing import AlertMissingError, SubmissionNotFinalized
 from assemblyline_core.server_base import ServerBase
 
 ALERT_QUEUE_NAME = 'm-alert'
 ALERT_RETRY_QUEUE_NAME = 'm-alert-retry'
 MAX_RETRIES = 10
+UPDATE_RETRY_SEC = 5
+SUBMISSION_RETRY_SEC = 15
 
 
 class Alerter(ServerBase):
@@ -89,17 +91,42 @@ class Alerter(ServerBase):
                 self.apm_client.end_transaction(alert_type, 'success')
 
             return alert_type
+        except AlertMissingError as e:
+            retries = alert['alert_retries'] = alert.get('alert_retries', 0) + 1
+            self.counter.increment('wait')
+            if retries > MAX_RETRIES:
+                self.log.error(f'{str(e)} [Max retries exceeded: {alert}]')
+
+                # End of process alert transaction (wait)
+                if self.apm_client:
+                    self.apm_client.end_transaction('unknown', 'error')
+
+                return 'error'
+            else:
+                self.log.info(f'{str(e)} Waiting {UPDATE_RETRY_SEC}s before retrying...')
+
+                # Wait a bit for the offending alert to be created
+                alert['wait_until'] = now(UPDATE_RETRY_SEC)
+                self.alert_retry_queue.push(alert)
+
+                # End of process alert transaction (wait)
+                if self.apm_client:
+                    self.apm_client.end_transaction('unknown', 'wait')
+
+                return 'wait'
         except SubmissionNotFinalized as error:
             self.counter.increment('wait')
             self.log.error(str(error))
 
-            # Wait another 15 secs for the submission to complete
-            alert['wait_until'] = now(15)
+            # Wait a bit for the submission to complete
+            alert['wait_until'] = now(SUBMISSION_RETRY_SEC)
             self.alert_retry_queue.push(alert)
 
             # End of process alert transaction (wait)
             if self.apm_client:
                 self.apm_client.end_transaction('unknown', 'wait')
+
+            return 'wait'
         except Exception:  # pylint: disable=W0703
             retries = alert['alert_retries'] = alert.get('alert_retries', 0) + 1
             self.counter.increment('error')
@@ -113,6 +140,8 @@ class Alerter(ServerBase):
             if self.apm_client:
                 self.apm_client.end_transaction('unknown', 'exception')
 
+            return 'exception'
+
     def try_run(self):
         while self.running:
             self.heartbeat()

{assemblyline-core-4.4.0.41 → assemblyline-core-4.4.0.43}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.4.0.41
+Version: 4.4.0.43
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

{assemblyline-core-4.4.0.41 → assemblyline-core-4.4.0.43}/test/test_alerter.py

@@ -139,23 +139,27 @@ def test_update_single_alert(config, datastore, delete_original=False):
     child_ingest_msg.submission.time = ingest_msg.submission.time
     child_ingest_msg.ingest_id = ingest_msg.ingest_id
 
-    alert_type_assertion = 'update'
-
     if delete_original:
+        # Delete the original alert
         datastore.alert.delete(original_alert['alert_id'])
-        alert_type_assertion = 'create'
 
-    alert_queue.push(child_ingest_msg.as_primitives())
-    alert_type = alerter.run_once()
-    assert alert_type == alert_type_assertion
+        # If the original alert does not exist, we should wait for a retry
+        alert_queue.push(child_ingest_msg.as_primitives())
+        alert_type = alerter.run_once()
+        assert alert_type == 'wait'
+    else:
+        # If the original alert still exists if should be updated now
+        alert_queue.push(child_ingest_msg.as_primitives())
+        alert_type = alerter.run_once()
+        assert alert_type == 'update'
 
-    datastore.alert.commit()
+        datastore.alert.commit()
 
-    updated_alert = datastore.alert.get(datastore.alert.search(f"sid:{child_submission.sid}",
-                                                               fl="id", as_obj=False)['items'][0]['id'])
-    assert updated_alert is not None
+        updated_alert = datastore.alert.get(datastore.alert.search(f"sid:{child_submission.sid}",
+                                                                   fl="id", as_obj=False)['items'][0]['id'])
+        assert updated_alert is not None
 
-    assert updated_alert != original_alert
+        assert updated_alert != original_alert
 
 
 def test_update_expired_alert(config, datastore):

assemblyline-core-4.4.0.41/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.4.0.41