async-lambda-unstable 0.5.3__tar.gz → 0.5.4__tar.gz

{async-lambda-unstable-0.5.3 → async-lambda-unstable-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: async-lambda-unstable
-Version: 0.5.3
+Version: 0.5.4
 Summary: A framework for creating AWS Lambda Async Workflows. - Unstable Branch
 Author-email: "Nuclei, Inc" <engineering@nuclei.ai>
 Description-Content-Type: text/markdown

{async-lambda-unstable-0.5.3 → async-lambda-unstable-0.5.4}/async_lambda/__init__.py

@@ -2,6 +2,7 @@ from .config import config_set_name as config_set_name
 from .config import config_set_runtime as config_set_runtime
 from .config import config_set_s3_payload_retention as config_set_s3_payload_retention
 from .controller import AsyncLambdaController as AsyncLambdaController
+from .controller import BatchInvokeException as BatchInvokeException
 from .defer import Defer as Defer
 from .env import disable_force_sync_mode as disable_force_sync_mode
 from .env import enable_force_sync_mode as enable_force_sync_mode
@@ -19,4 +20,4 @@ from .models.events.managed_sqs_event import ManagedSQSEvent as ManagedSQSEvent
 from .models.events.scheduled_event import ScheduledEvent as ScheduledEvent
 from .models.events.unmanaged_sqs_event import UnmanagedSQSEvent as UnmanagedSQSEvent
 
-__version__ = "0.5.3"
+__version__ = "0.5.4"

{async-lambda-unstable-0.5.3 → async-lambda-unstable-0.5.4}/async_lambda/controller.py

@@ -1,6 +1,7 @@
 import functools
 import json
 import logging
+import random
 import re
 import time
 from datetime import datetime, timezone
@@ -47,6 +48,14 @@ ScheduledEventT = TypeVar("ScheduledEventT", bound=ScheduledEvent)
 DynamoDBEventT = TypeVar("DynamoDBEventT", bound=DynamoDBEvent)
 
 
+class BatchInvokeException(Exception):
+    failed_payloads: List[int]
+
+    def __init__(self, msg: str, failed_payloads: List[int]):
+        self.failed_payloads = failed_payloads
+        super().__init__(msg)
+
+
 class AsyncLambdaController:
     is_sub: bool
     lane_count: Optional[int] = None
@@ -69,7 +78,7 @@ class AsyncLambdaController:
         lane_count: Optional[int] = None,
         propagate_lane_assignment: Optional[bool] = None,
         middleware: Optional[List[MiddlewareRegistration]] = None,
-        delete_s3_payloads: bool = True,
+        delete_s3_payloads: bool = False,
         controller_name: Optional[str] = None,
     ):
         self.tasks = dict()
@@ -551,7 +560,8 @@ class AsyncLambdaController:
             else:
                 url = destination_task.get_managed_queue_url(lane=lane)
             failed_messages = []
-            for _ in range(5):
+            batch_retry_count = env.get_batch_failure_retry_count() + 1
+            for i in range(batch_retry_count):
                 response = get_sqs_client().send_message_batch(
                     QueueUrl=url,
                     Entries=entries,
@@ -560,13 +570,22 @@ class AsyncLambdaController:
                 if len(failed_messages) == 0:
                     return
                 logger.warning(failed_messages)
-                logger.warning(f"{len(failed_messages)} messages failed to send.")
+                logger.warning(f"{len(failed_messages)} messages failed to send. ")
                 failed_message_ids = {message["Id"] for message in failed_messages}
                 entries = [
                     entry for entry in entries if entry["Id"] in failed_message_ids
                 ]
+                if i < batch_retry_count:
+                    send_delay = 0.5 + random.random()
+                    logger.info(
+                        f"Waiting {send_delay:.3f} before attempting batch failures again."
+                    )
+                    time.sleep(send_delay)
             logger.error(failed_messages)
-            raise Exception(f"Failed to send {len(failed_messages)} messages.")
+            raise BatchInvokeException(
+                f"Failed to send {len(failed_messages)} messages.",
+                failed_payloads=[int(entry["Id"].split("_")[-1]) for entry in entries],
+            )
 
     def new_payload(
         self,

{async-lambda-unstable-0.5.3 → async-lambda-unstable-0.5.4}/async_lambda/env.py

@@ -79,3 +79,7 @@ def disable_force_sync_mode():
 
 def get_force_sync_mode() -> bool:
     return bool(os.environ.get("ASYNC_LAMBDA_FORCE_SYNC", ""))
+
+
+def get_batch_failure_retry_count() -> int:
+    return int(os.environ.get("ASYNC_LAMBDA_BATCH_FAILURE_RETRY_COUNT", 20))

{async-lambda-unstable-0.5.3 → async-lambda-unstable-0.5.4}/async_lambda_unstable.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: async-lambda-unstable
-Version: 0.5.3
+Version: 0.5.4
 Summary: A framework for creating AWS Lambda Async Workflows. - Unstable Branch
 Author-email: "Nuclei, Inc" <engineering@nuclei.ai>
 Description-Content-Type: text/markdown