async-lambda-unstable 0.6.7__tar.gz → 0.6.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/PKG-INFO +1 -1
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/__init__.py +1 -1
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/controller.py +261 -256
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/env.py +16 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/.gitignore +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/README.md +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/build_config.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/cli.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/client.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/config.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/defer.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/middleware.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/__init__.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/api_response.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/case_insensitive_dict.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/__init__.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/api_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/base_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/dynamodb_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/managed_sqs_batch_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/managed_sqs_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/scheduled_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/unmanaged_sqs_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/mock/mock_context.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/mock/mock_event.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/task.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/payload_encoder.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/py.typed +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/util.py +0 -0
- {async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/pyproject.toml +0 -0
|
@@ -21,4 +21,4 @@ from .models.events.managed_sqs_event import ManagedSQSEvent as ManagedSQSEvent
|
|
|
21
21
|
from .models.events.scheduled_event import ScheduledEvent as ScheduledEvent
|
|
22
22
|
from .models.events.unmanaged_sqs_event import UnmanagedSQSEvent as UnmanagedSQSEvent
|
|
23
23
|
|
|
24
|
-
__version__ = "0.6.
|
|
24
|
+
__version__ = "0.6.9"
|
|
@@ -69,30 +69,6 @@ class BatchInvokeException(Exception):
|
|
|
69
69
|
super().__init__(msg)
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
# Exception raised when async_invoke_* is called to a non-internal task with a delay greater than _SQS_MAX_DELAY_SECONDS
|
|
73
|
-
class AsyncInvokeInvalidDelay(Exception):
|
|
74
|
-
pass
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
class _BatchEntry:
|
|
78
|
-
enqueue_id: str
|
|
79
|
-
payload: dict
|
|
80
|
-
delay: int
|
|
81
|
-
message_group_id: Optional[str]
|
|
82
|
-
|
|
83
|
-
def __init__(
|
|
84
|
-
self,
|
|
85
|
-
enqueue_id: str,
|
|
86
|
-
payload: dict,
|
|
87
|
-
delay: int,
|
|
88
|
-
message_group_id: Optional[str],
|
|
89
|
-
):
|
|
90
|
-
self.enqueue_id = enqueue_id
|
|
91
|
-
self.payload = payload
|
|
92
|
-
self.delay = delay
|
|
93
|
-
self.message_group_id = message_group_id
|
|
94
|
-
|
|
95
|
-
|
|
96
72
|
class AsyncLambdaController:
|
|
97
73
|
"""
|
|
98
74
|
AsyncLambdaController manages async tasks, middleware, and invocation logic for the async-lambda framework.
|
|
@@ -449,6 +425,12 @@ class AsyncLambdaController:
|
|
|
449
425
|
"Ref": "AsyncLambdaPayloadBucket"
|
|
450
426
|
},
|
|
451
427
|
"ASYNC_LAMBDA_ACCOUNT_ID": {"Ref": "AWS::AccountId"},
|
|
428
|
+
"ASYNC_LAMBDA_DELAY_SCHEDULER_ROLE_ARN": {
|
|
429
|
+
"Fn::GetAtt": ["AsyncLambdaDelaySchedulerRole", "Arn"]
|
|
430
|
+
},
|
|
431
|
+
"ASYNC_LAMBDA_DELAY_SCHEDULE_GROUP": {
|
|
432
|
+
"Ref": "AsyncLambdaDelayScheduleGroup"
|
|
433
|
+
},
|
|
452
434
|
**build_config.environment_variables,
|
|
453
435
|
},
|
|
454
436
|
},
|
|
@@ -477,102 +459,133 @@ class AsyncLambdaController:
|
|
|
477
459
|
),
|
|
478
460
|
},
|
|
479
461
|
},
|
|
462
|
+
"AsyncLambdaDelayDLQ": {
|
|
463
|
+
"Type": "AWS::SQS::Queue",
|
|
464
|
+
"Properties": {
|
|
465
|
+
"QueueName": f"{config.name}-delay-dlq",
|
|
466
|
+
"MessageRetentionPeriod": 1_209_600, # 14 days
|
|
467
|
+
"Tags": make_cf_tags(
|
|
468
|
+
{
|
|
469
|
+
**build_config.tags,
|
|
470
|
+
"async-lambda-queue-type": "dlq",
|
|
471
|
+
}
|
|
472
|
+
),
|
|
473
|
+
},
|
|
474
|
+
},
|
|
475
|
+
"AsyncLambdaDelayScheduleGroup": {
|
|
476
|
+
"Type": "AWS::Scheduler::ScheduleGroup",
|
|
477
|
+
"Properties": {
|
|
478
|
+
"Name": f"{config.name}-delay",
|
|
479
|
+
"Tags": make_cf_tags(build_config.tags),
|
|
480
|
+
},
|
|
481
|
+
},
|
|
482
|
+
"AsyncLambdaCreateDelaySchedulesPolicy": {
|
|
483
|
+
"Type": "AWS::IAM::ManagedPolicy",
|
|
484
|
+
"Properties": {
|
|
485
|
+
"ManagedPolicyName": {
|
|
486
|
+
"Fn::Sub": "${AWS::StackName}-create-delay-schedules"
|
|
487
|
+
},
|
|
488
|
+
"PolicyDocument": {
|
|
489
|
+
"Version": "2012-10-17",
|
|
490
|
+
"Statement": [
|
|
491
|
+
{
|
|
492
|
+
"Sid": "CreateSchedules",
|
|
493
|
+
"Effect": "Allow",
|
|
494
|
+
"Action": "scheduler:CreateSchedule",
|
|
495
|
+
"Resource": {
|
|
496
|
+
"Fn::Sub": "arn:aws:scheduler:${AWS::Region}:${AWS::AccountId}:schedule/${AsyncLambdaDelayScheduleGroup}/*"
|
|
497
|
+
},
|
|
498
|
+
},
|
|
499
|
+
{
|
|
500
|
+
"Sid": "PassSchedulerExecutionRole",
|
|
501
|
+
"Effect": "Allow",
|
|
502
|
+
"Action": "iam:PassRole",
|
|
503
|
+
"Resource": {
|
|
504
|
+
"Fn::GetAtt": [
|
|
505
|
+
"AsyncLambdaDelaySchedulerRole",
|
|
506
|
+
"Arn",
|
|
507
|
+
]
|
|
508
|
+
},
|
|
509
|
+
"Condition": {
|
|
510
|
+
"StringEquals": {
|
|
511
|
+
"iam:PassedToService": "scheduler.amazonaws.com"
|
|
512
|
+
}
|
|
513
|
+
},
|
|
514
|
+
},
|
|
515
|
+
],
|
|
516
|
+
},
|
|
517
|
+
},
|
|
518
|
+
},
|
|
480
519
|
},
|
|
481
520
|
}
|
|
482
521
|
_task_list = list(self.tasks.values())
|
|
483
|
-
|
|
522
|
+
managed_tasks_resources = [
|
|
484
523
|
resource
|
|
485
524
|
for task in _task_list
|
|
486
525
|
if task.trigger_type in MANAGED_SQS_TASK_TYPES
|
|
487
526
|
for resource in task.get_policy_sqs_resources()
|
|
488
|
-
]
|
|
489
|
-
external_tasks_resources = [
|
|
527
|
+
] + [
|
|
490
528
|
resource
|
|
491
529
|
for external_async_task_id in self.external_async_tasks
|
|
492
530
|
for resource in AsyncLambdaTask.get_policy_external_task_resources(
|
|
493
531
|
external_async_task_id
|
|
494
532
|
)
|
|
495
533
|
]
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
task_ref_policies = self._build_send_to_all_async_lambda_queues_policies(
|
|
534
|
+
|
|
535
|
+
send_all_queues_task_ref_policies = (
|
|
536
|
+
self._build_send_to_all_async_lambda_queues_policies(
|
|
500
537
|
managed_tasks_resources
|
|
501
538
|
)
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
"
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
"Version": "2012-10-17",
|
|
518
|
-
"Statement": [
|
|
519
|
-
{
|
|
520
|
-
"Effect": "Allow",
|
|
521
|
-
"Principal": {"Service": "scheduler.amazonaws.com"},
|
|
522
|
-
"Action": "sts:AssumeRole",
|
|
523
|
-
}
|
|
524
|
-
],
|
|
525
|
-
},
|
|
526
|
-
"ManagedPolicyArns": [
|
|
527
|
-
{"Ref": policy_id} for policy_id in task_ref_policies.keys()
|
|
539
|
+
)
|
|
540
|
+
for key in send_all_queues_task_ref_policies.keys():
|
|
541
|
+
template["Resources"][key] = send_all_queues_task_ref_policies[key]
|
|
542
|
+
|
|
543
|
+
template["Resources"]["AsyncLambdaDelaySchedulerRole"] = {
|
|
544
|
+
"Type": "AWS::IAM::Role",
|
|
545
|
+
"Properties": {
|
|
546
|
+
"AssumeRolePolicyDocument": {
|
|
547
|
+
"Version": "2012-10-17",
|
|
548
|
+
"Statement": [
|
|
549
|
+
{
|
|
550
|
+
"Effect": "Allow",
|
|
551
|
+
"Principal": {"Service": "scheduler.amazonaws.com"},
|
|
552
|
+
"Action": "sts:AssumeRole",
|
|
553
|
+
}
|
|
528
554
|
],
|
|
529
555
|
},
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
"
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
"
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
{
|
|
551
|
-
"Sid": "PassSchedulerExecutionRole",
|
|
552
|
-
"Effect": "Allow",
|
|
553
|
-
"Action": "iam:PassRole",
|
|
554
|
-
"Resource": {
|
|
555
|
-
"Fn::GetAtt": [
|
|
556
|
-
"AsyncLambdaDelaySchedulerRole",
|
|
557
|
-
"Arn",
|
|
558
|
-
]
|
|
559
|
-
},
|
|
560
|
-
"Condition": {
|
|
561
|
-
"StringEquals": {
|
|
562
|
-
"iam:PassedToService": "scheduler.amazonaws.com"
|
|
563
|
-
}
|
|
556
|
+
"ManagedPolicyArns": [
|
|
557
|
+
{"Ref": policy_ref}
|
|
558
|
+
for policy_ref in send_all_queues_task_ref_policies.keys()
|
|
559
|
+
],
|
|
560
|
+
"Policies": [
|
|
561
|
+
{
|
|
562
|
+
"PolicyName": "SendToDelayDLQ",
|
|
563
|
+
"PolicyDocument": {
|
|
564
|
+
"Version": "2012-10-17",
|
|
565
|
+
"Statement": [
|
|
566
|
+
{
|
|
567
|
+
"Sid": "SendToDelayDLQ",
|
|
568
|
+
"Effect": "Allow",
|
|
569
|
+
"Action": "sqs:SendMessage",
|
|
570
|
+
"Resource": {
|
|
571
|
+
"Fn::GetAtt": [
|
|
572
|
+
"AsyncLambdaDelayDLQ",
|
|
573
|
+
"Arn",
|
|
574
|
+
]
|
|
575
|
+
},
|
|
564
576
|
},
|
|
565
|
-
|
|
566
|
-
|
|
577
|
+
],
|
|
578
|
+
},
|
|
567
579
|
},
|
|
568
|
-
|
|
569
|
-
}
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
"
|
|
575
|
-
]
|
|
580
|
+
],
|
|
581
|
+
},
|
|
582
|
+
}
|
|
583
|
+
task_ref_policies = {
|
|
584
|
+
**send_all_queues_task_ref_policies,
|
|
585
|
+
"AsyncLambdaCreateDelaySchedulesPolicy": template["Resources"][
|
|
586
|
+
"AsyncLambdaCreateDelaySchedulesPolicy"
|
|
587
|
+
],
|
|
588
|
+
}
|
|
576
589
|
|
|
577
590
|
has_api_tasks = False
|
|
578
591
|
for task in _task_list:
|
|
@@ -868,6 +881,7 @@ class AsyncLambdaController:
|
|
|
868
881
|
force_sync: bool = False,
|
|
869
882
|
lane: Optional[int] = None,
|
|
870
883
|
message_group_id: Optional[str] = None,
|
|
884
|
+
unique_delay_id: Optional[str] = None,
|
|
871
885
|
):
|
|
872
886
|
"""
|
|
873
887
|
Sends an asynchronous invocation payload to a managed or external task via SQS.
|
|
@@ -883,6 +897,8 @@ class AsyncLambdaController:
|
|
|
883
897
|
delay (int, optional): Delay in seconds before sending the message. Defaults to 0.
|
|
884
898
|
force_sync (bool, optional): If True, invokes the task synchronously. Defaults to False.
|
|
885
899
|
lane (Optional[int], optional): The lane to use for invocation. If None, lane assignment is determined automatically.
|
|
900
|
+
message_group_id (Optional[str], optional): Optional message group ID for enabling SQS Fair Queues. Defaults to None.
|
|
901
|
+
unique_delay_id (Optional[str], optional): Unique name for EventBridge scheduled delays when delay is longer than 900 seconds. Random if not provided.
|
|
886
902
|
|
|
887
903
|
Returns:
|
|
888
904
|
Any: The result of the synchronous invocation if `force_sync` is True; otherwise, None.
|
|
@@ -899,6 +915,7 @@ class AsyncLambdaController:
|
|
|
899
915
|
force_sync=force_sync,
|
|
900
916
|
lane=lane,
|
|
901
917
|
message_group_id=message_group_id,
|
|
918
|
+
unique_delay_id=unique_delay_id,
|
|
902
919
|
)
|
|
903
920
|
is_external_task = False
|
|
904
921
|
if destination_task_id not in self.tasks:
|
|
@@ -909,11 +926,6 @@ class AsyncLambdaController:
|
|
|
909
926
|
f"No such task exists with the task_id {destination_task_id}"
|
|
910
927
|
)
|
|
911
928
|
|
|
912
|
-
if is_external_task and delay > _SQS_MAX_DELAY_SECONDS:
|
|
913
|
-
raise AsyncInvokeInvalidDelay(
|
|
914
|
-
f"Unable to invoke task {destination_task_id} with delay {delay} seconds. External tasks only support delays up to {_SQS_MAX_DELAY_SECONDS} seconds."
|
|
915
|
-
)
|
|
916
|
-
|
|
917
929
|
destination_task = None
|
|
918
930
|
if not is_external_task:
|
|
919
931
|
destination_task = self.tasks[destination_task_id]
|
|
@@ -984,18 +996,7 @@ class AsyncLambdaController:
|
|
|
984
996
|
assert destination_task is not None
|
|
985
997
|
url = destination_task.get_managed_queue_url(lane=lane)
|
|
986
998
|
|
|
987
|
-
if delay
|
|
988
|
-
assert not is_external_task
|
|
989
|
-
|
|
990
|
-
queue_arn = destination_task.get_managed_queue_arn(lane=lane)
|
|
991
|
-
# TODO: Use _message_group_id when AWS tells me how
|
|
992
|
-
self._send_via_scheduler(
|
|
993
|
-
queue_arn=queue_arn,
|
|
994
|
-
message_body=json.dumps(sqs_payload),
|
|
995
|
-
delay=delay,
|
|
996
|
-
message_group_id=_message_group_id,
|
|
997
|
-
)
|
|
998
|
-
else:
|
|
999
|
+
if delay <= _SQS_MAX_DELAY_SECONDS:
|
|
999
1000
|
_kwargs = {}
|
|
1000
1001
|
if _message_group_id:
|
|
1001
1002
|
_kwargs["MessageGroupId"] = _message_group_id
|
|
@@ -1006,8 +1007,62 @@ class AsyncLambdaController:
|
|
|
1006
1007
|
DelaySeconds=delay,
|
|
1007
1008
|
**_kwargs,
|
|
1008
1009
|
)
|
|
1010
|
+
else:
|
|
1011
|
+
self._send_via_scheduler(
|
|
1012
|
+
queue_url=url,
|
|
1013
|
+
message_body=json.dumps(sqs_payload),
|
|
1014
|
+
delay=delay,
|
|
1015
|
+
schedule_name=unique_delay_id,
|
|
1016
|
+
message_group_id=_message_group_id,
|
|
1017
|
+
)
|
|
1018
|
+
|
|
1009
1019
|
return None
|
|
1010
1020
|
|
|
1021
|
+
@staticmethod
|
|
1022
|
+
def _send_via_scheduler(
|
|
1023
|
+
queue_url: str,
|
|
1024
|
+
message_body: str,
|
|
1025
|
+
delay: int,
|
|
1026
|
+
schedule_name: Optional[str] = None,
|
|
1027
|
+
message_group_id: Optional[str] = None,
|
|
1028
|
+
):
|
|
1029
|
+
"""
|
|
1030
|
+
Schedules a one-time EventBridge scheduler SQS sendMessage at some point in the future.
|
|
1031
|
+
|
|
1032
|
+
Args:
|
|
1033
|
+
schedule_name (str): Unique name for the schedule.
|
|
1034
|
+
queue_url (str): URL of the destination SQS queue.
|
|
1035
|
+
message_body (str): JSON-serialized message body to deliver to the queue.
|
|
1036
|
+
delay (int): Delay in seconds before delivering the message.
|
|
1037
|
+
message_group_id (Optional[str]): MessageGroupId for FIFO queues. Defaults to None.
|
|
1038
|
+
"""
|
|
1039
|
+
schedule_time = datetime.now(tz=timezone.utc) + timedelta(seconds=delay)
|
|
1040
|
+
schedule_expression = f"at({schedule_time.strftime('%Y-%m-%dT%H:%M:%S')})"
|
|
1041
|
+
|
|
1042
|
+
sqs_input: dict = {
|
|
1043
|
+
"QueueUrl": queue_url,
|
|
1044
|
+
"MessageBody": message_body,
|
|
1045
|
+
}
|
|
1046
|
+
if message_group_id:
|
|
1047
|
+
sqs_input["MessageGroupId"] = message_group_id
|
|
1048
|
+
|
|
1049
|
+
sqs_universal_target: dict = {
|
|
1050
|
+
"Arn": "arn:aws:scheduler:::aws-sdk:sqs:sendMessage",
|
|
1051
|
+
"RoleArn": env.get_delay_scheduler_role_arn(),
|
|
1052
|
+
"Input": json.dumps(sqs_input),
|
|
1053
|
+
"DeadLetterConfig": {"Arn": env.get_delay_schedule_dlq_arn()},
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
get_scheduler_client().create_schedule(
|
|
1057
|
+
Name=schedule_name or str(uuid4()),
|
|
1058
|
+
GroupName=env.get_delay_schedule_group_name(),
|
|
1059
|
+
ScheduleExpression=schedule_expression,
|
|
1060
|
+
ScheduleExpressionTimezone="UTC",
|
|
1061
|
+
FlexibleTimeWindow={"Mode": "OFF"},
|
|
1062
|
+
Target=sqs_universal_target,
|
|
1063
|
+
ActionAfterCompletion="DELETE",
|
|
1064
|
+
)
|
|
1065
|
+
|
|
1011
1066
|
def send_async_invoke_payload_batch(
|
|
1012
1067
|
self,
|
|
1013
1068
|
destination_task_id: str,
|
|
@@ -1083,35 +1138,6 @@ class AsyncLambdaController:
|
|
|
1083
1138
|
)
|
|
1084
1139
|
lane = 0
|
|
1085
1140
|
|
|
1086
|
-
batch_entries: List[_BatchEntry] = []
|
|
1087
|
-
for i, sqs_payload in enumerate(sqs_payloads):
|
|
1088
|
-
if isinstance(delay, Sequence):
|
|
1089
|
-
_delay = delay[i]
|
|
1090
|
-
else:
|
|
1091
|
-
_delay = delay
|
|
1092
|
-
|
|
1093
|
-
_message_group_id = (
|
|
1094
|
-
message_group_id[i]
|
|
1095
|
-
if isinstance(message_group_id, Sequence)
|
|
1096
|
-
and not isinstance(message_group_id, str)
|
|
1097
|
-
else message_group_id
|
|
1098
|
-
)
|
|
1099
|
-
if (
|
|
1100
|
-
_message_group_id is None
|
|
1101
|
-
and (current_message_group_id := self.get_current_message_group_id())
|
|
1102
|
-
is not None
|
|
1103
|
-
and self.should_propagate_message_group_id()
|
|
1104
|
-
):
|
|
1105
|
-
_message_group_id = current_message_group_id
|
|
1106
|
-
batch_entries.append(
|
|
1107
|
-
_BatchEntry(
|
|
1108
|
-
enqueue_id=f"index_{index + i}",
|
|
1109
|
-
payload=sqs_payload,
|
|
1110
|
-
delay=_delay,
|
|
1111
|
-
message_group_id=_message_group_id,
|
|
1112
|
-
)
|
|
1113
|
-
)
|
|
1114
|
-
|
|
1115
1141
|
if force_sync or env.get_force_sync_mode():
|
|
1116
1142
|
if is_external_task:
|
|
1117
1143
|
raise NotImplementedError(
|
|
@@ -1122,16 +1148,34 @@ class AsyncLambdaController:
|
|
|
1122
1148
|
current_lane = self.get_current_lane()
|
|
1123
1149
|
assert destination_task is not None
|
|
1124
1150
|
queue_arn = destination_task.get_managed_queue_arn(lane=lane)
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1151
|
+
for i, sqs_payload in enumerate(sqs_payloads):
|
|
1152
|
+
if delay:
|
|
1153
|
+
if isinstance(delay, Sequence):
|
|
1154
|
+
time.sleep(delay[i])
|
|
1155
|
+
else:
|
|
1156
|
+
time.sleep(delay)
|
|
1157
|
+
|
|
1158
|
+
_message_group_id = (
|
|
1159
|
+
message_group_id[i]
|
|
1160
|
+
if isinstance(message_group_id, Sequence)
|
|
1161
|
+
and not isinstance(message_group_id, str)
|
|
1162
|
+
else message_group_id
|
|
1163
|
+
)
|
|
1164
|
+
if (
|
|
1165
|
+
_message_group_id is None
|
|
1166
|
+
and (
|
|
1167
|
+
current_message_group_id := self.get_current_message_group_id()
|
|
1168
|
+
)
|
|
1169
|
+
is not None
|
|
1170
|
+
and self.should_propagate_message_group_id()
|
|
1171
|
+
):
|
|
1172
|
+
_message_group_id = current_message_group_id
|
|
1129
1173
|
|
|
1130
1174
|
current_message_group_id = self.get_current_message_group_id()
|
|
1131
1175
|
mock_event = MockSQSLambdaEvent(
|
|
1132
|
-
json.dumps(
|
|
1176
|
+
json.dumps(sqs_payload),
|
|
1133
1177
|
source_queue_arn=queue_arn,
|
|
1134
|
-
message_group_id=
|
|
1178
|
+
message_group_id=_message_group_id,
|
|
1135
1179
|
)
|
|
1136
1180
|
mock_context = MockLambdaContext(destination_task.task_id)
|
|
1137
1181
|
self.handle_invocation(
|
|
@@ -1141,112 +1185,69 @@ class AsyncLambdaController:
|
|
|
1141
1185
|
self.set_current_task_id(current_task_id)
|
|
1142
1186
|
self.set_current_message_group_id(current_message_group_id)
|
|
1143
1187
|
else:
|
|
1144
|
-
|
|
1145
|
-
for
|
|
1146
|
-
if
|
|
1147
|
-
|
|
1148
|
-
raise AsyncInvokeInvalidDelay(
|
|
1149
|
-
f"send_async_invoke_payload_batch does not support a delay longer than {_SQS_MAX_DELAY_SECONDS} seconds for external tasks"
|
|
1150
|
-
)
|
|
1151
|
-
assert destination_task is not None
|
|
1152
|
-
queue_arn = destination_task.get_managed_queue_arn(lane=lane)
|
|
1153
|
-
self._send_via_scheduler(
|
|
1154
|
-
queue_arn=queue_arn,
|
|
1155
|
-
message_body=json.dumps(batch.payload),
|
|
1156
|
-
delay=batch.delay,
|
|
1157
|
-
)
|
|
1188
|
+
entries: List[dict] = []
|
|
1189
|
+
for i, sqs_payload in enumerate(sqs_payloads):
|
|
1190
|
+
if isinstance(delay, Sequence):
|
|
1191
|
+
_delay = delay[i]
|
|
1158
1192
|
else:
|
|
1159
|
-
|
|
1160
|
-
"MessageBody": json.dumps(batch.payload),
|
|
1161
|
-
"DelaySeconds": batch.delay,
|
|
1162
|
-
"Id": batch.enqueue_id,
|
|
1163
|
-
}
|
|
1164
|
-
if batch.message_group_id:
|
|
1165
|
-
entry["MessageGroupId"] = batch.message_group_id
|
|
1166
|
-
sqs_entries.append(entry)
|
|
1167
|
-
if sqs_entries:
|
|
1168
|
-
if is_external_task:
|
|
1169
|
-
url = f"https://sqs.{env.get_aws_region()}.amazonaws.com/{env.get_aws_account_id()}/{destination_task_id}"
|
|
1170
|
-
else:
|
|
1171
|
-
assert destination_task is not None
|
|
1172
|
-
url = destination_task.get_managed_queue_url(lane=lane)
|
|
1173
|
-
failed_messages: List[dict] = []
|
|
1174
|
-
batch_retry_count = env.get_batch_failure_retry_count() + 1
|
|
1175
|
-
entries = sqs_entries
|
|
1176
|
-
for i in range(batch_retry_count):
|
|
1177
|
-
response = get_sqs_client().send_message_batch(
|
|
1178
|
-
QueueUrl=url,
|
|
1179
|
-
Entries=entries,
|
|
1180
|
-
)
|
|
1181
|
-
failed_messages = response.get("Failed", [])
|
|
1182
|
-
if len(failed_messages) == 0:
|
|
1183
|
-
return
|
|
1184
|
-
logger.warning(failed_messages)
|
|
1185
|
-
logger.warning(f"{len(failed_messages)} messages failed to send. ")
|
|
1186
|
-
failed_message_ids = {message["Id"] for message in failed_messages}
|
|
1187
|
-
entries = [
|
|
1188
|
-
entry for entry in entries if entry["Id"] in failed_message_ids
|
|
1189
|
-
]
|
|
1190
|
-
if i < batch_retry_count:
|
|
1191
|
-
send_delay = 0.5 + random.random()
|
|
1192
|
-
logger.info(
|
|
1193
|
-
f"Waiting {send_delay:.3f} before attempting batch failures again."
|
|
1194
|
-
)
|
|
1195
|
-
time.sleep(send_delay)
|
|
1196
|
-
logger.error(failed_messages)
|
|
1197
|
-
raise BatchInvokeException(
|
|
1198
|
-
f"Failed to send {len(failed_messages)} messages.",
|
|
1199
|
-
failed_payloads=[
|
|
1200
|
-
int(entry["Id"].split("_")[-1]) for entry in entries
|
|
1201
|
-
],
|
|
1202
|
-
)
|
|
1203
|
-
|
|
1204
|
-
@staticmethod
|
|
1205
|
-
def _send_via_scheduler(
|
|
1206
|
-
queue_arn: str,
|
|
1207
|
-
message_body: str,
|
|
1208
|
-
delay: int,
|
|
1209
|
-
message_group_id: Optional[str] = None,
|
|
1210
|
-
):
|
|
1211
|
-
"""
|
|
1212
|
-
Schedules a message for future delivery to an SQS queue via EventBridge Scheduler.
|
|
1193
|
+
_delay = delay
|
|
1213
1194
|
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
schedule_time = datetime.now(tz=timezone.utc) + timedelta(seconds=delay)
|
|
1230
|
-
schedule_expression = f"at({schedule_time.strftime('%Y-%m-%dT%H:%M:%S')})"
|
|
1231
|
-
|
|
1232
|
-
target: dict = {
|
|
1233
|
-
"Arn": queue_arn,
|
|
1234
|
-
"RoleArn": env.get_delay_scheduler_role_arn(),
|
|
1235
|
-
"Input": message_body,
|
|
1236
|
-
}
|
|
1237
|
-
# TODO: Set message_group_id based on feedback from AWS
|
|
1238
|
-
# if message_group_id:
|
|
1239
|
-
# target["SqsParameters"] = {"MessageGroupId": message_group_id}
|
|
1195
|
+
_message_group_id = (
|
|
1196
|
+
message_group_id[i]
|
|
1197
|
+
if isinstance(message_group_id, Sequence)
|
|
1198
|
+
and not isinstance(message_group_id, str)
|
|
1199
|
+
else message_group_id
|
|
1200
|
+
)
|
|
1201
|
+
if (
|
|
1202
|
+
_message_group_id is None
|
|
1203
|
+
and (
|
|
1204
|
+
current_message_group_id := self.get_current_message_group_id()
|
|
1205
|
+
)
|
|
1206
|
+
is not None
|
|
1207
|
+
and self.should_propagate_message_group_id()
|
|
1208
|
+
):
|
|
1209
|
+
_message_group_id = current_message_group_id
|
|
1240
1210
|
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1211
|
+
entry = {
|
|
1212
|
+
"MessageBody": json.dumps(sqs_payload),
|
|
1213
|
+
"DelaySeconds": _delay,
|
|
1214
|
+
"Id": f"index_{index + i}",
|
|
1215
|
+
}
|
|
1216
|
+
if _message_group_id:
|
|
1217
|
+
entry["MessageGroupId"] = _message_group_id
|
|
1218
|
+
entries.append(entry)
|
|
1219
|
+
if is_external_task:
|
|
1220
|
+
url = f"https://sqs.{env.get_aws_region()}.amazonaws.com/{env.get_aws_account_id()}/{destination_task_id}"
|
|
1221
|
+
else:
|
|
1222
|
+
assert destination_task is not None
|
|
1223
|
+
url = destination_task.get_managed_queue_url(lane=lane)
|
|
1224
|
+
failed_messages = []
|
|
1225
|
+
batch_retry_count = env.get_batch_failure_retry_count() + 1
|
|
1226
|
+
for i in range(batch_retry_count):
|
|
1227
|
+
response = get_sqs_client().send_message_batch(
|
|
1228
|
+
QueueUrl=url,
|
|
1229
|
+
Entries=entries,
|
|
1230
|
+
)
|
|
1231
|
+
failed_messages: List[dict] = response.get("Failed", [])
|
|
1232
|
+
if len(failed_messages) == 0:
|
|
1233
|
+
return
|
|
1234
|
+
logger.warning(failed_messages)
|
|
1235
|
+
logger.warning(f"{len(failed_messages)} messages failed to send. ")
|
|
1236
|
+
failed_message_ids = {message["Id"] for message in failed_messages}
|
|
1237
|
+
entries = [
|
|
1238
|
+
entry for entry in entries if entry["Id"] in failed_message_ids
|
|
1239
|
+
]
|
|
1240
|
+
if i < batch_retry_count:
|
|
1241
|
+
send_delay = 0.5 + random.random()
|
|
1242
|
+
logger.info(
|
|
1243
|
+
f"Waiting {send_delay:.3f} before attempting batch failures again."
|
|
1244
|
+
)
|
|
1245
|
+
time.sleep(send_delay)
|
|
1246
|
+
logger.error(failed_messages)
|
|
1247
|
+
raise BatchInvokeException(
|
|
1248
|
+
f"Failed to send {len(failed_messages)} messages.",
|
|
1249
|
+
failed_payloads=[int(entry["Id"].split("_")[-1]) for entry in entries],
|
|
1250
|
+
)
|
|
1250
1251
|
|
|
1251
1252
|
def new_payload(
|
|
1252
1253
|
self,
|
|
@@ -1345,6 +1346,7 @@ class AsyncLambdaController:
|
|
|
1345
1346
|
force_sync: bool = False,
|
|
1346
1347
|
lane: Optional[int] = None,
|
|
1347
1348
|
message_group_id: Optional[str] = None,
|
|
1349
|
+
unique_delay_id: Optional[str] = None,
|
|
1348
1350
|
):
|
|
1349
1351
|
"""
|
|
1350
1352
|
Asynchronously invokes a task by sending a payload to the specified destination.
|
|
@@ -1359,6 +1361,8 @@ class AsyncLambdaController:
|
|
|
1359
1361
|
delay (int, optional): Delay in seconds before invoking the task. Defaults to 0.
|
|
1360
1362
|
force_sync (bool, optional): If True, forces synchronous invocation. Defaults to False.
|
|
1361
1363
|
lane (Optional[int], optional): Optional lane identifier for routing. Defaults to None.
|
|
1364
|
+
message_group_id (Optional[str], optional): Optional message group ID. Defaults to None.
|
|
1365
|
+
unique_delay_id (Optional[str], optional): Unique name for EventBridge scheduled delays when delay is longer than 900 seconds. Random if not provided.
|
|
1362
1366
|
|
|
1363
1367
|
Returns:
|
|
1364
1368
|
Any: The result of sending the asynchronous invocation payload.
|
|
@@ -1374,6 +1378,7 @@ class AsyncLambdaController:
|
|
|
1374
1378
|
force_sync=force_sync,
|
|
1375
1379
|
lane=lane,
|
|
1376
1380
|
message_group_id=message_group_id,
|
|
1381
|
+
unique_delay_id=unique_delay_id,
|
|
1377
1382
|
)
|
|
1378
1383
|
|
|
1379
1384
|
def async_invoke_batch(
|
|
@@ -237,3 +237,19 @@ def get_delay_schedule_group_name() -> str:
|
|
|
237
237
|
KeyError: If 'ASYNC_LAMBDA_DELAY_SCHEDULE_GROUP' is not set in the environment.
|
|
238
238
|
"""
|
|
239
239
|
return os.environ["ASYNC_LAMBDA_DELAY_SCHEDULE_GROUP"]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def get_delay_schedule_dlq_arn() -> str:
|
|
243
|
+
"""
|
|
244
|
+
Retrieves the EventBridge Scheduler delay DLQ for this stack.
|
|
245
|
+
|
|
246
|
+
One SQS DLQ is provisioned per CloudFormation stack. EventBridge Scheduler performs a very large number of retries,
|
|
247
|
+
so ending up here would most likely be due to permissions or target deletion.
|
|
248
|
+
|
|
249
|
+
Returns:
|
|
250
|
+
str: The value of the 'ASYNC_LAMBDA_DELAY_SCHEDULE_DLQ_ARN' environment variable.
|
|
251
|
+
|
|
252
|
+
Raises:
|
|
253
|
+
KeyError: If 'ASYNC_LAMBDA_DELAY_SCHEDULE_DLQ_ARN' is not set in the environment.
|
|
254
|
+
"""
|
|
255
|
+
return os.environ["ASYNC_LAMBDA_DELAY_SCHEDULE_DLQ_ARN"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/api_response.py
RENAMED
|
File without changes
|
|
File without changes
|
{async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/__init__.py
RENAMED
|
File without changes
|
{async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/api_event.py
RENAMED
|
File without changes
|
{async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/events/base_event.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/mock/mock_context.py
RENAMED
|
File without changes
|
{async_lambda_unstable-0.6.7 → async_lambda_unstable-0.6.9}/async_lambda/models/mock/mock_event.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|