async-lambda-unstable 0.6.12__tar.gz → 0.6.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/PKG-INFO +42 -2
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/README.md +40 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/__init__.py +2 -1
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/build_config.py +50 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/config.py +18 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/controller.py +267 -31
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/task.py +25 -14
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/.gitignore +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/cli.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/client.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/defer.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/env.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/middleware.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/__init__.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/api_response.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/case_insensitive_dict.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/__init__.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/api_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/base_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/dynamodb_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/managed_sqs_batch_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/managed_sqs_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/scheduled_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/unmanaged_sqs_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/mock/mock_context.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/mock/mock_event.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/payload_encoder.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/py.typed +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/util.py +0 -0
- {async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/pyproject.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: async-lambda-unstable
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.14
|
|
4
4
|
Summary: A framework for creating AWS Lambda Async Workflows. - Unstable Branch
|
|
5
5
|
Author-email: "Nuclei, Inc" <engineering@nuclei.ai>
|
|
6
6
|
Requires-Dist: click>=8.0.0
|
|
@@ -418,6 +418,46 @@ def task(event: ManagedSQSEvent):
|
|
|
418
418
|
...
|
|
419
419
|
```
|
|
420
420
|
|
|
421
|
+
### Shared Function Role
|
|
422
|
+
|
|
423
|
+
By default each task declares `Policies`, so SAM generates one `AWS::IAM::Role` per task.
|
|
424
|
+
|
|
425
|
+
`config_set_shared_function_role()` emits a single `AsyncLambdaSharedFunctionRole` and points every task's `Role` at it instead:
|
|
426
|
+
|
|
427
|
+
```python
|
|
428
|
+
from async_lambda import config_set_shared_function_role
|
|
429
|
+
|
|
430
|
+
config_set_shared_function_role()
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
The shared role carries everything the per-task roles carried, with one difference: SQS consume access is granted on the app's queue-name prefix (`{name}-*`) rather than each task's own queues. Unmanaged SQS queues are still listed individually, since they're named by their producer.
|
|
434
|
+
|
|
435
|
+
Two caveats:
|
|
436
|
+
|
|
437
|
+
- A task with task-specific `policies` keeps its own generated role, since the shared role is built from app-wide and stage-wide config only.
|
|
438
|
+
- Every app-wide policy must be either a managed policy ARN or a `{"Statement": [...]}` document. SAM policy templates can't be expressed in a plain IAM role and will fail the build.
|
|
439
|
+
|
|
440
|
+
### Consolidated Queue-Age Alarms
|
|
441
|
+
|
|
442
|
+
`managed_queue_extras` attaches a copy of each extra resource to *every* managed queue, so using it for a CloudWatch alarm costs one stack resource per queue lane.
|
|
443
|
+
|
|
444
|
+
Setting `queue_age_alarm` in the build config emits a small number of grouped alarms instead, each watching `ApproximateAgeOfOldestMessage` across up to ten queues:
|
|
445
|
+
|
|
446
|
+
```json
|
|
447
|
+
{
|
|
448
|
+
"queue_age_alarm": {
|
|
449
|
+
"threshold": 5000,
|
|
450
|
+
"period": 86400,
|
|
451
|
+
"alarm_actions": ["arn:aws:sns:us-east-1:123456789012:alarms"],
|
|
452
|
+
"ok_actions": ["arn:aws:sns:us-east-1:123456789012:alarms"]
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
Only `threshold` is required and `period` defaults to one day. Coverage is every managed queue lane plus the app DLQ. An app with 104 queues gets 11 alarms rather than 104.
|
|
458
|
+
|
|
459
|
+
CloudWatch rejects an alarm carrying more than 10 metrics.
|
|
460
|
+
|
|
421
461
|
## Known Limitations
|
|
422
462
|
|
|
423
463
|
- Not all Lambda configuration options are supported (see code for extension points)
|
|
@@ -406,6 +406,46 @@ def task(event: ManagedSQSEvent):
|
|
|
406
406
|
...
|
|
407
407
|
```
|
|
408
408
|
|
|
409
|
+
### Shared Function Role
|
|
410
|
+
|
|
411
|
+
By default each task declares `Policies`, so SAM generates one `AWS::IAM::Role` per task.
|
|
412
|
+
|
|
413
|
+
`config_set_shared_function_role()` emits a single `AsyncLambdaSharedFunctionRole` and points every task's `Role` at it instead:
|
|
414
|
+
|
|
415
|
+
```python
|
|
416
|
+
from async_lambda import config_set_shared_function_role
|
|
417
|
+
|
|
418
|
+
config_set_shared_function_role()
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
The shared role carries everything the per-task roles carried, with one difference: SQS consume access is granted on the app's queue-name prefix (`{name}-*`) rather than each task's own queues. Unmanaged SQS queues are still listed individually, since they're named by their producer.
|
|
422
|
+
|
|
423
|
+
Two caveats:
|
|
424
|
+
|
|
425
|
+
- A task with task-specific `policies` keeps its own generated role, since the shared role is built from app-wide and stage-wide config only.
|
|
426
|
+
- Every app-wide policy must be either a managed policy ARN or a `{"Statement": [...]}` document. SAM policy templates can't be expressed in a plain IAM role and will fail the build.
|
|
427
|
+
|
|
428
|
+
### Consolidated Queue-Age Alarms
|
|
429
|
+
|
|
430
|
+
`managed_queue_extras` attaches a copy of each extra resource to *every* managed queue, so using it for a CloudWatch alarm costs one stack resource per queue lane.
|
|
431
|
+
|
|
432
|
+
Setting `queue_age_alarm` in the build config emits a small number of grouped alarms instead, each watching `ApproximateAgeOfOldestMessage` across up to ten queues:
|
|
433
|
+
|
|
434
|
+
```json
|
|
435
|
+
{
|
|
436
|
+
"queue_age_alarm": {
|
|
437
|
+
"threshold": 5000,
|
|
438
|
+
"period": 86400,
|
|
439
|
+
"alarm_actions": ["arn:aws:sns:us-east-1:123456789012:alarms"],
|
|
440
|
+
"ok_actions": ["arn:aws:sns:us-east-1:123456789012:alarms"]
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
Only `threshold` is required and `period` defaults to one day. Coverage is every managed queue lane plus the app DLQ. An app with 104 queues gets 11 alarms rather than 104.
|
|
446
|
+
|
|
447
|
+
CloudWatch rejects an alarm carrying more than 10 metrics.
|
|
448
|
+
|
|
409
449
|
## Known Limitations
|
|
410
450
|
|
|
411
451
|
- Not all Lambda configuration options are supported (see code for extension points)
|
|
@@ -2,6 +2,7 @@ from .config import config_set_default_task_memory as config_set_default_task_me
|
|
|
2
2
|
from .config import config_set_name as config_set_name
|
|
3
3
|
from .config import config_set_runtime as config_set_runtime
|
|
4
4
|
from .config import config_set_s3_payload_retention as config_set_s3_payload_retention
|
|
5
|
+
from .config import config_set_shared_function_role as config_set_shared_function_role
|
|
5
6
|
from .controller import AsyncLambdaController as AsyncLambdaController
|
|
6
7
|
from .controller import BatchInvokeException as BatchInvokeException
|
|
7
8
|
from .defer import Defer as Defer
|
|
@@ -21,4 +22,4 @@ from .models.events.managed_sqs_event import ManagedSQSEvent as ManagedSQSEvent
|
|
|
21
22
|
from .models.events.scheduled_event import ScheduledEvent as ScheduledEvent
|
|
22
23
|
from .models.events.unmanaged_sqs_event import UnmanagedSQSEvent as UnmanagedSQSEvent
|
|
23
24
|
|
|
24
|
-
__version__ = "0.6.
|
|
25
|
+
__version__ = "0.6.14"
|
|
@@ -14,6 +14,46 @@ def make_default_tags() -> Dict[str, str]:
|
|
|
14
14
|
return {"framework": "async-lambda", "framework-version": __version__}
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
@dataclass
|
|
18
|
+
class QueueAgeAlarmConfig:
|
|
19
|
+
"""
|
|
20
|
+
Settings for the consolidated queue-age alarms.
|
|
21
|
+
|
|
22
|
+
One alarm covers a group of queues rather than one alarm per queue.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
threshold (int): Oldest-message age, in seconds, above which the alarm fires.
|
|
26
|
+
period (int): Seconds of data each datapoint covers.
|
|
27
|
+
alarm_actions (List[str]): ARNs notified when an alarm starts firing.
|
|
28
|
+
ok_actions (List[str]): ARNs notified when an alarm recovers.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
threshold: int
|
|
32
|
+
period: int
|
|
33
|
+
alarm_actions: List[str]
|
|
34
|
+
ok_actions: List[str]
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def new(cls, config: dict) -> "QueueAgeAlarmConfig":
|
|
38
|
+
"""
|
|
39
|
+
Creates a new instance from a configuration dictionary.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
config (dict): provided the values of QueueAgeAlarmConfig
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
QueueAgeAlarmConfig: A new instance configured with the provided options.
|
|
46
|
+
"""
|
|
47
|
+
if "threshold" not in config:
|
|
48
|
+
raise ValueError("queue_age_alarm requires a `threshold`.")
|
|
49
|
+
return cls(
|
|
50
|
+
threshold=config["threshold"],
|
|
51
|
+
period=config.get("period", 86400),
|
|
52
|
+
alarm_actions=list(config.get("alarm_actions", list())),
|
|
53
|
+
ok_actions=list(config.get("ok_actions", list())),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
17
57
|
@dataclass
|
|
18
58
|
class AsyncLambdaBuildConfig:
|
|
19
59
|
"""
|
|
@@ -26,6 +66,7 @@ class AsyncLambdaBuildConfig:
|
|
|
26
66
|
subnet_ids (Set[str]): Set of subnet IDs for VPC configuration.
|
|
27
67
|
security_group_ids (Set[str]): Set of security group IDs for VPC configuration.
|
|
28
68
|
managed_queue_extras (List[dict]): Additional configuration for managed queues.
|
|
69
|
+
queue_age_alarm (Optional[QueueAgeAlarmConfig]): Settings for the consolidated queue-age alarms. No alarms are emitted when this is unset.
|
|
29
70
|
method_settings (List[dict]): API Gateway method settings.
|
|
30
71
|
tags (Dict[str, str]): Tags to assign to the Lambda function.
|
|
31
72
|
logging_config (Dict[str, str]): Logging configuration options.
|
|
@@ -57,6 +98,7 @@ class AsyncLambdaBuildConfig:
|
|
|
57
98
|
certificate_arn: Optional[str] = None
|
|
58
99
|
hosted_zone_id: Optional[str] = None
|
|
59
100
|
auto_create_acm_certificate: Optional[bool] = None
|
|
101
|
+
queue_age_alarm: Optional[QueueAgeAlarmConfig] = None
|
|
60
102
|
|
|
61
103
|
@classmethod
|
|
62
104
|
def new(cls, config: dict) -> "AsyncLambdaBuildConfig":
|
|
@@ -71,6 +113,7 @@ class AsyncLambdaBuildConfig:
|
|
|
71
113
|
- subnet_ids (set or list): Set or list of subnet IDs for VPC configuration.
|
|
72
114
|
- security_group_ids (set or list): Set or list of security group IDs for VPC configuration.
|
|
73
115
|
- managed_queue_extras (list): Additional managed queue configuration.
|
|
116
|
+
- queue_age_alarm (dict): Settings for the consolidated queue-age alarms.
|
|
74
117
|
- method_settings (list): List of method settings for API Gateway.
|
|
75
118
|
- tags (dict): Tags to assign to the Lambda function.
|
|
76
119
|
- logging_config (dict): Logging configuration options.
|
|
@@ -89,6 +132,11 @@ class AsyncLambdaBuildConfig:
|
|
|
89
132
|
subnet_ids=set(config.get("subnet_ids", set())),
|
|
90
133
|
security_group_ids=set(config.get("security_group_ids", set())),
|
|
91
134
|
managed_queue_extras=list(config.get("managed_queue_extras", list())),
|
|
135
|
+
queue_age_alarm=(
|
|
136
|
+
QueueAgeAlarmConfig.new(config["queue_age_alarm"])
|
|
137
|
+
if config.get("queue_age_alarm") is not None
|
|
138
|
+
else None
|
|
139
|
+
),
|
|
92
140
|
method_settings=list(config.get("method_settings", list())),
|
|
93
141
|
tags=config.get("tags", dict()),
|
|
94
142
|
logging_config=config.get("logging_config", dict()),
|
|
@@ -118,6 +166,8 @@ class AsyncLambdaBuildConfig:
|
|
|
118
166
|
self.hosted_zone_id = other.hosted_zone_id
|
|
119
167
|
if other.auto_create_acm_certificate is not None:
|
|
120
168
|
self.auto_create_acm_certificate = other.auto_create_acm_certificate
|
|
169
|
+
if other.queue_age_alarm is not None:
|
|
170
|
+
self.queue_age_alarm = other.queue_age_alarm
|
|
121
171
|
|
|
122
172
|
@property
|
|
123
173
|
def function_properties(self):
|
|
@@ -11,6 +11,7 @@ class AsyncLambdaConfig:
|
|
|
11
11
|
s3_payload_retention (int): Days to retain payloads in S3. Defaults to 30.
|
|
12
12
|
default_task_memory (int): Default memory allocation (MB) for tasks. Defaults to 128.
|
|
13
13
|
default_task_ephemeral_storage (int): Default disk allocation (MB) for tasks. Defaults to 512.
|
|
14
|
+
shared_function_role (bool): Whether tasks share a single execution role. Defaults to False.
|
|
14
15
|
"""
|
|
15
16
|
|
|
16
17
|
name: str = "async-lambda"
|
|
@@ -18,6 +19,7 @@ class AsyncLambdaConfig:
|
|
|
18
19
|
s3_payload_retention: Optional[int] = 180
|
|
19
20
|
default_task_memory: int = 128
|
|
20
21
|
default_task_ephemeral_storage: int = 512
|
|
22
|
+
shared_function_role: bool = False
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
config = AsyncLambdaConfig()
|
|
@@ -71,3 +73,19 @@ def config_set_default_task_ephemeral_storage(ephemeral_storage: int = 512):
|
|
|
71
73
|
ephemeral_storage (int, optional): Disk storage in megabytes to set as default for tasks. Defaults to 512.
|
|
72
74
|
"""
|
|
73
75
|
config.default_task_ephemeral_storage = ephemeral_storage
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def config_set_shared_function_role(enabled: bool = True):
|
|
79
|
+
"""
|
|
80
|
+
Use a single execution role for every task instead of one role per task.
|
|
81
|
+
|
|
82
|
+
SAM generates an IAM role per function when a function declares `Policies`. Sharing one
|
|
83
|
+
role trades per-task SQS scoping (the shared role can consume any of the app's own queues)
|
|
84
|
+
for one CloudFormation resource instead of one per task.
|
|
85
|
+
|
|
86
|
+
Tasks with task-specific `policies` keep their own role.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
enabled (bool, optional): Whether to share a single execution role. Defaults to True.
|
|
90
|
+
"""
|
|
91
|
+
config.shared_function_role = enabled
|
|
@@ -20,7 +20,11 @@ from typing import (
|
|
|
20
20
|
from uuid import uuid4
|
|
21
21
|
|
|
22
22
|
from . import env
|
|
23
|
-
from .build_config import
|
|
23
|
+
from .build_config import (
|
|
24
|
+
AsyncLambdaBuildConfig,
|
|
25
|
+
QueueAgeAlarmConfig,
|
|
26
|
+
get_build_config_for_stage,
|
|
27
|
+
)
|
|
24
28
|
from .client import get_s3_client, get_scheduler_client, get_sqs_client
|
|
25
29
|
from .config import config
|
|
26
30
|
from .middleware import MET, RT, MiddlewareFunction, MiddlewareRegistration
|
|
@@ -33,7 +37,12 @@ from .models.events.scheduled_event import ScheduledEvent
|
|
|
33
37
|
from .models.events.unmanaged_sqs_event import UnmanagedSQSEvent
|
|
34
38
|
from .models.mock.mock_context import MockLambdaContext
|
|
35
39
|
from .models.mock.mock_event import MockSQSLambdaEvent
|
|
36
|
-
from .models.task import
|
|
40
|
+
from .models.task import (
|
|
41
|
+
MANAGED_SQS_TASK_TYPES,
|
|
42
|
+
SHARED_FUNCTION_ROLE_LOGICAL_ID,
|
|
43
|
+
AsyncLambdaTask,
|
|
44
|
+
TaskTriggerType,
|
|
45
|
+
)
|
|
37
46
|
from .payload_encoder import PayloadEncoder
|
|
38
47
|
from .util import make_cf_tags
|
|
39
48
|
|
|
@@ -41,6 +50,9 @@ logger = logging.getLogger(__name__)
|
|
|
41
50
|
|
|
42
51
|
_SQS_MAX_DELAY_SECONDS = 900
|
|
43
52
|
|
|
53
|
+
# CloudWatch rejects an alarm carrying more than 10 metrics.
|
|
54
|
+
_MAX_METRICS_PER_ALARM = 10
|
|
55
|
+
|
|
44
56
|
BaseEventT = TypeVar("BaseEventT", bound=BaseEvent)
|
|
45
57
|
APIEventT = TypeVar("APIEventT", bound=APIEvent)
|
|
46
58
|
ManagedSQSEventT = TypeVar("ManagedSQSEventT", bound=ManagedSQSEvent)
|
|
@@ -180,32 +192,26 @@ class AsyncLambdaController:
|
|
|
180
192
|
self.delete_s3_payloads = delete_s3_payloads
|
|
181
193
|
self.controller_name = controller_name
|
|
182
194
|
|
|
183
|
-
# itertools.batched is much cleaner but only available in python 3.12+
|
|
184
|
-
@staticmethod
|
|
185
|
-
def _batched(items: Sequence[dict], batch_size: int) -> List[List[dict]]:
|
|
186
|
-
if batch_size <= 0:
|
|
187
|
-
raise ValueError("batch_size must be greater than 0")
|
|
188
|
-
return [
|
|
189
|
-
list(items[i : i + batch_size]) for i in range(0, len(items), batch_size)
|
|
190
|
-
]
|
|
191
|
-
|
|
192
195
|
@classmethod
|
|
193
196
|
def _build_send_to_all_async_lambda_queues_policies(
|
|
194
197
|
cls,
|
|
195
|
-
|
|
196
|
-
policy_batch_size: int = 30,
|
|
198
|
+
queue_resources: List[dict],
|
|
197
199
|
) -> Dict[str, dict]:
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
200
|
+
"""
|
|
201
|
+
Builds the managed policy granting send access to every managed queue in this app.
|
|
202
|
+
|
|
203
|
+
This app's own queues are covered by a single name-prefix wildcard rather than an
|
|
204
|
+
enumeration, so the policy stays one resource and one document regardless of task
|
|
205
|
+
count. External tasks don't share the prefix, so their ARNs are listed explicitly.
|
|
206
|
+
"""
|
|
207
|
+
if len(queue_resources) == 0:
|
|
208
|
+
return {}
|
|
209
|
+
return {
|
|
210
|
+
"SendToAllAsyncLambdaQueuesPolicy": {
|
|
205
211
|
"Type": "AWS::IAM::ManagedPolicy",
|
|
206
212
|
"Properties": {
|
|
207
213
|
"ManagedPolicyName": {
|
|
208
|
-
"Fn::Sub":
|
|
214
|
+
"Fn::Sub": "${AWS::StackName}-send-to-all-queues"
|
|
209
215
|
},
|
|
210
216
|
"PolicyDocument": {
|
|
211
217
|
"Version": "2012-10-17",
|
|
@@ -214,13 +220,228 @@ class AsyncLambdaController:
|
|
|
214
220
|
"Sid": "SendToAllQueues",
|
|
215
221
|
"Effect": "Allow",
|
|
216
222
|
"Action": ["sqs:SendMessage"],
|
|
217
|
-
"Resource":
|
|
223
|
+
"Resource": queue_resources,
|
|
218
224
|
},
|
|
219
225
|
],
|
|
220
226
|
},
|
|
221
227
|
},
|
|
222
228
|
}
|
|
223
|
-
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
@staticmethod
|
|
232
|
+
def _managed_queue_arn_wildcard() -> dict:
|
|
233
|
+
"""
|
|
234
|
+
An ARN pattern matching every managed queue this app creates.
|
|
235
|
+
"""
|
|
236
|
+
return {
|
|
237
|
+
"Fn::Sub": "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:"
|
|
238
|
+
+ f"{config.name}-*"
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
def _alarmable_queue_logical_ids(self) -> List[str]:
|
|
242
|
+
"""
|
|
243
|
+
Logical IDs of every queue the queue-age alarms should watch.
|
|
244
|
+
|
|
245
|
+
Covers each managed queue lane plus the app DLQ. The delay DLQ is left out
|
|
246
|
+
because it is named deterministically and so can be alarmed on directly.
|
|
247
|
+
"""
|
|
248
|
+
logical_ids = []
|
|
249
|
+
for task in self.tasks.values():
|
|
250
|
+
if task.trigger_type not in MANAGED_SQS_TASK_TYPES:
|
|
251
|
+
continue
|
|
252
|
+
for lane_index in range(task.get_lane_count()):
|
|
253
|
+
logical_ids.append(task.get_managed_queue_logical_id(lane=lane_index))
|
|
254
|
+
logical_ids.append("AsyncLambdaDLQ")
|
|
255
|
+
return logical_ids
|
|
256
|
+
|
|
257
|
+
@staticmethod
|
|
258
|
+
def _build_queue_age_alarms(
|
|
259
|
+
queue_logical_ids: List[str],
|
|
260
|
+
alarm_config: QueueAgeAlarmConfig,
|
|
261
|
+
) -> Dict[str, dict]:
|
|
262
|
+
"""
|
|
263
|
+
Builds alarms watching the oldest message age across every managed queue.
|
|
264
|
+
|
|
265
|
+
Queues are grouped rather than alarmed individually.
|
|
266
|
+
|
|
267
|
+
Each alarm watches `MAX(METRICS())` over its own group. `METRICS()` resolves to just that alarm's
|
|
268
|
+
metrics.
|
|
269
|
+
|
|
270
|
+
Grouping is required because CloudWatch caps an alarm at 10 metrics.
|
|
271
|
+
"""
|
|
272
|
+
alarms = {}
|
|
273
|
+
for alarm_index, group_start in enumerate(
|
|
274
|
+
range(0, len(queue_logical_ids), _MAX_METRICS_PER_ALARM)
|
|
275
|
+
):
|
|
276
|
+
group = queue_logical_ids[
|
|
277
|
+
group_start : group_start + _MAX_METRICS_PER_ALARM
|
|
278
|
+
]
|
|
279
|
+
metrics: List[dict] = [
|
|
280
|
+
{
|
|
281
|
+
"Id": f"q{metric_index}",
|
|
282
|
+
"ReturnData": False,
|
|
283
|
+
"MetricStat": {
|
|
284
|
+
"Metric": {
|
|
285
|
+
"Namespace": "AWS/SQS",
|
|
286
|
+
"MetricName": "ApproximateAgeOfOldestMessage",
|
|
287
|
+
"Dimensions": [
|
|
288
|
+
{
|
|
289
|
+
"Name": "QueueName",
|
|
290
|
+
"Value": {
|
|
291
|
+
"Fn::GetAtt": [queue_logical_id, "QueueName"]
|
|
292
|
+
},
|
|
293
|
+
}
|
|
294
|
+
],
|
|
295
|
+
},
|
|
296
|
+
"Period": alarm_config.period,
|
|
297
|
+
"Stat": "Maximum",
|
|
298
|
+
},
|
|
299
|
+
}
|
|
300
|
+
for metric_index, queue_logical_id in enumerate(group)
|
|
301
|
+
]
|
|
302
|
+
metrics.append(
|
|
303
|
+
{
|
|
304
|
+
"Id": "oldest_message_age",
|
|
305
|
+
"Expression": "MAX(METRICS())",
|
|
306
|
+
"ReturnData": True,
|
|
307
|
+
}
|
|
308
|
+
)
|
|
309
|
+
alarms[f"AsyncLambdaQueueAgeAlarm{alarm_index}"] = {
|
|
310
|
+
"Type": "AWS::CloudWatch::Alarm",
|
|
311
|
+
"Properties": {
|
|
312
|
+
"ActionsEnabled": True,
|
|
313
|
+
"AlarmActions": alarm_config.alarm_actions,
|
|
314
|
+
"OKActions": alarm_config.ok_actions,
|
|
315
|
+
"AlarmDescription": (
|
|
316
|
+
f"Oldest message age across {len(group)} {config.name} queues"
|
|
317
|
+
),
|
|
318
|
+
"ComparisonOperator": "GreaterThanThreshold",
|
|
319
|
+
"EvaluationPeriods": 1,
|
|
320
|
+
"Metrics": metrics,
|
|
321
|
+
"Threshold": alarm_config.threshold,
|
|
322
|
+
"TreatMissingData": "notBreaching",
|
|
323
|
+
},
|
|
324
|
+
}
|
|
325
|
+
return alarms
|
|
326
|
+
|
|
327
|
+
@classmethod
|
|
328
|
+
def _build_shared_function_role(
|
|
329
|
+
cls,
|
|
330
|
+
task_list: List[AsyncLambdaTask],
|
|
331
|
+
build_config: AsyncLambdaBuildConfig,
|
|
332
|
+
task_ref_policy_ids: List[str],
|
|
333
|
+
) -> dict:
|
|
334
|
+
"""
|
|
335
|
+
Builds a single execution role shared by every task.
|
|
336
|
+
"""
|
|
337
|
+
sqs_resources: List[Union[str, dict]] = []
|
|
338
|
+
if any(task.trigger_type in MANAGED_SQS_TASK_TYPES for task in task_list):
|
|
339
|
+
sqs_resources.append(cls._managed_queue_arn_wildcard())
|
|
340
|
+
# Unmanaged queues are named by whoever produces to them, so they fall outside the
|
|
341
|
+
# app-name prefix and have to be listed.
|
|
342
|
+
sqs_resources.extend(
|
|
343
|
+
resource
|
|
344
|
+
for task in task_list
|
|
345
|
+
if task.trigger_type == TaskTriggerType.UNMANAGED_SQS
|
|
346
|
+
for resource in task.get_policy_sqs_resources()
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
statements: List[dict] = [
|
|
350
|
+
{
|
|
351
|
+
"Effect": "Allow",
|
|
352
|
+
"Action": ["s3:DeleteObject", "s3:PutObject", "s3:GetObject"],
|
|
353
|
+
"Resource": {
|
|
354
|
+
"Fn::Join": [
|
|
355
|
+
"",
|
|
356
|
+
[
|
|
357
|
+
"arn:aws:s3:::",
|
|
358
|
+
{"Ref": "AsyncLambdaPayloadBucket"},
|
|
359
|
+
"/*",
|
|
360
|
+
],
|
|
361
|
+
]
|
|
362
|
+
},
|
|
363
|
+
},
|
|
364
|
+
]
|
|
365
|
+
if len(sqs_resources) > 0:
|
|
366
|
+
statements.append(
|
|
367
|
+
{
|
|
368
|
+
"Effect": "Allow",
|
|
369
|
+
"Action": [
|
|
370
|
+
"sqs:ChangeMessageVisibility",
|
|
371
|
+
"sqs:DeleteMessage",
|
|
372
|
+
"sqs:GetQueueAttributes",
|
|
373
|
+
"sqs:GetQueueUrl",
|
|
374
|
+
"sqs:ReceiveMessage",
|
|
375
|
+
],
|
|
376
|
+
"Resource": sqs_resources,
|
|
377
|
+
}
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
inline_policies: List[dict] = [
|
|
381
|
+
{
|
|
382
|
+
"PolicyName": "AsyncLambdaTaskPolicy",
|
|
383
|
+
"PolicyDocument": {
|
|
384
|
+
"Version": "2012-10-17",
|
|
385
|
+
"Statement": statements,
|
|
386
|
+
},
|
|
387
|
+
}
|
|
388
|
+
]
|
|
389
|
+
managed_policy_arns: List[Union[str, dict]] = [
|
|
390
|
+
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
|
|
391
|
+
]
|
|
392
|
+
if "VpcConfig" in build_config.function_properties:
|
|
393
|
+
managed_policy_arns.append(
|
|
394
|
+
"arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
|
|
395
|
+
)
|
|
396
|
+
# SAM attaches this to any lambda function with an SQS event source. Without it, they
|
|
397
|
+
# lose their SQS polling permissions.
|
|
398
|
+
if len(sqs_resources) > 0:
|
|
399
|
+
managed_policy_arns.append(
|
|
400
|
+
"arn:aws:iam::aws:policy/service-role/AWSLambdaSQSQueueExecutionRole"
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
for index, policy in enumerate(build_config.policies):
|
|
404
|
+
if isinstance(policy, str):
|
|
405
|
+
managed_policy_arns.append(policy)
|
|
406
|
+
elif isinstance(policy, dict) and set(policy.keys()) == {"Statement"}:
|
|
407
|
+
inline_policies.append(
|
|
408
|
+
{
|
|
409
|
+
"PolicyName": f"AsyncLambdaBuildConfigPolicy{index}",
|
|
410
|
+
"PolicyDocument": {
|
|
411
|
+
"Version": "2012-10-17",
|
|
412
|
+
"Statement": policy["Statement"],
|
|
413
|
+
},
|
|
414
|
+
}
|
|
415
|
+
)
|
|
416
|
+
else:
|
|
417
|
+
raise Exception(
|
|
418
|
+
f"Unable to build a shared function role. The policy {policy!r} is "
|
|
419
|
+
"neither a managed policy ARN nor a single-key 'Statement' document."
|
|
420
|
+
"SAM policy templates cannot be expressed in a plain IAM role."
|
|
421
|
+
"Rewrite the policy in one of those two forms, or turn off config_set_shared_function_role."
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
managed_policy_arns.extend(
|
|
425
|
+
{"Ref": policy_id} for policy_id in task_ref_policy_ids
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
return {
|
|
429
|
+
"Type": "AWS::IAM::Role",
|
|
430
|
+
"Properties": {
|
|
431
|
+
"AssumeRolePolicyDocument": {
|
|
432
|
+
"Version": "2012-10-17",
|
|
433
|
+
"Statement": [
|
|
434
|
+
{
|
|
435
|
+
"Effect": "Allow",
|
|
436
|
+
"Principal": {"Service": ["lambda.amazonaws.com"]},
|
|
437
|
+
"Action": ["sts:AssumeRole"],
|
|
438
|
+
}
|
|
439
|
+
],
|
|
440
|
+
},
|
|
441
|
+
"ManagedPolicyArns": managed_policy_arns,
|
|
442
|
+
"Policies": inline_policies,
|
|
443
|
+
},
|
|
444
|
+
}
|
|
224
445
|
|
|
225
446
|
def add_middleware(
|
|
226
447
|
self, event_types: List[Type[BaseEvent]], func: MiddlewareFunction[MET, RT]
|
|
@@ -525,22 +746,20 @@ class AsyncLambdaController:
|
|
|
525
746
|
},
|
|
526
747
|
}
|
|
527
748
|
_task_list = list(self.tasks.values())
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
for resource in task.get_policy_sqs_resources()
|
|
533
|
-
] + [
|
|
749
|
+
send_all_queue_resources: List[dict] = []
|
|
750
|
+
if any(task.trigger_type in MANAGED_SQS_TASK_TYPES for task in _task_list):
|
|
751
|
+
send_all_queue_resources.append(self._managed_queue_arn_wildcard())
|
|
752
|
+
send_all_queue_resources.extend(
|
|
534
753
|
resource
|
|
535
754
|
for external_async_task_id in self.external_async_tasks
|
|
536
755
|
for resource in AsyncLambdaTask.get_policy_external_task_resources(
|
|
537
756
|
external_async_task_id
|
|
538
757
|
)
|
|
539
|
-
|
|
758
|
+
)
|
|
540
759
|
|
|
541
760
|
send_all_queues_task_ref_policies = (
|
|
542
761
|
self._build_send_to_all_async_lambda_queues_policies(
|
|
543
|
-
|
|
762
|
+
send_all_queue_resources
|
|
544
763
|
)
|
|
545
764
|
)
|
|
546
765
|
for key in send_all_queues_task_ref_policies.keys():
|
|
@@ -593,6 +812,15 @@ class AsyncLambdaController:
|
|
|
593
812
|
],
|
|
594
813
|
}
|
|
595
814
|
|
|
815
|
+
if config.shared_function_role:
|
|
816
|
+
template["Resources"][SHARED_FUNCTION_ROLE_LOGICAL_ID] = (
|
|
817
|
+
self._build_shared_function_role(
|
|
818
|
+
task_list=_task_list,
|
|
819
|
+
build_config=build_config,
|
|
820
|
+
task_ref_policy_ids=list(task_ref_policies.keys()),
|
|
821
|
+
)
|
|
822
|
+
)
|
|
823
|
+
|
|
596
824
|
has_api_tasks = False
|
|
597
825
|
for task in _task_list:
|
|
598
826
|
if task.trigger_type == TaskTriggerType.API_EVENT:
|
|
@@ -609,6 +837,14 @@ class AsyncLambdaController:
|
|
|
609
837
|
self._dlq_extras_replace_references(extra)
|
|
610
838
|
)
|
|
611
839
|
|
|
840
|
+
if build_config.queue_age_alarm is not None:
|
|
841
|
+
template["Resources"].update(
|
|
842
|
+
self._build_queue_age_alarms(
|
|
843
|
+
queue_logical_ids=self._alarmable_queue_logical_ids(),
|
|
844
|
+
alarm_config=build_config.queue_age_alarm,
|
|
845
|
+
)
|
|
846
|
+
)
|
|
847
|
+
|
|
612
848
|
if has_api_tasks:
|
|
613
849
|
properties: dict = {
|
|
614
850
|
"StageName": "prod",
|
|
@@ -57,6 +57,8 @@ MANAGED_SQS_TASK_TYPES = {
|
|
|
57
57
|
TaskTriggerType.MANAGED_SQS_BATCH,
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
SHARED_FUNCTION_ROLE_LOGICAL_ID = "AsyncLambdaSharedFunctionRole"
|
|
61
|
+
|
|
60
62
|
EventType = TypeVar(
|
|
61
63
|
"EventType",
|
|
62
64
|
bound=Union[
|
|
@@ -610,23 +612,32 @@ class AsyncLambdaTask(Generic[EventType, RT]):
|
|
|
610
612
|
*[{"Ref": policy_id} for policy_id in task_ref_policy_ids],
|
|
611
613
|
]
|
|
612
614
|
|
|
615
|
+
function_properties = {
|
|
616
|
+
"Environment": {
|
|
617
|
+
"Variables": {
|
|
618
|
+
"ASYNC_LAMBDA_TASK_ID": self.task_id,
|
|
619
|
+
**override_config.environment_variables,
|
|
620
|
+
}
|
|
621
|
+
},
|
|
622
|
+
"FunctionName": self.get_function_name(),
|
|
623
|
+
"Timeout": self.timeout,
|
|
624
|
+
"Events": events,
|
|
625
|
+
"CodeUri": ".async_lambda/build/deployment.zip",
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
# cannot use shared role if the task has its own policies
|
|
629
|
+
if config.shared_function_role and len(override_config.policies) == 0:
|
|
630
|
+
function_properties["Role"] = {
|
|
631
|
+
"Fn::GetAtt": [SHARED_FUNCTION_ROLE_LOGICAL_ID, "Arn"]
|
|
632
|
+
}
|
|
633
|
+
else:
|
|
634
|
+
function_properties["Policies"] = policies
|
|
635
|
+
function_properties.update(override_config.function_properties)
|
|
636
|
+
|
|
613
637
|
template = {
|
|
614
638
|
self.get_function_logical_id(): {
|
|
615
639
|
"Type": "AWS::Serverless::Function",
|
|
616
|
-
"Properties":
|
|
617
|
-
"Environment": {
|
|
618
|
-
"Variables": {
|
|
619
|
-
"ASYNC_LAMBDA_TASK_ID": self.task_id,
|
|
620
|
-
**override_config.environment_variables,
|
|
621
|
-
}
|
|
622
|
-
},
|
|
623
|
-
"FunctionName": self.get_function_name(),
|
|
624
|
-
"Timeout": self.timeout,
|
|
625
|
-
"Events": events,
|
|
626
|
-
"Policies": policies,
|
|
627
|
-
"CodeUri": ".async_lambda/build/deployment.zip",
|
|
628
|
-
**override_config.function_properties,
|
|
629
|
-
},
|
|
640
|
+
"Properties": function_properties,
|
|
630
641
|
}
|
|
631
642
|
}
|
|
632
643
|
if self.memory is not None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/__init__.py
RENAMED
|
File without changes
|
{async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/api_response.py
RENAMED
|
File without changes
|
|
File without changes
|
{async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/events/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/models/mock/mock_event.py
RENAMED
|
File without changes
|
{async_lambda_unstable-0.6.12 → async_lambda_unstable-0.6.14}/async_lambda/payload_encoder.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|