async-lambda-unstable 0.6.2__tar.gz → 0.6.3__tar.gz

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: async-lambda-unstable
-Version: 0.6.2
+Version: 0.6.3
 Summary: A framework for creating AWS Lambda Async Workflows. - Unstable Branch
 Author-email: "Nuclei, Inc" <engineering@nuclei.ai>
 Requires-Dist: click>=8.0.0

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/async_lambda/__init__.py

@@ -21,4 +21,4 @@ from .models.events.managed_sqs_event import ManagedSQSEvent as ManagedSQSEvent
 from .models.events.scheduled_event import ScheduledEvent as ScheduledEvent
 from .models.events.unmanaged_sqs_event import UnmanagedSQSEvent as UnmanagedSQSEvent
 
-__version__ = "0.6.2"
+__version__ = "0.6.3"

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/async_lambda/build_config.py

@@ -119,6 +119,23 @@ class AsyncLambdaBuildConfig:
         if other.auto_create_acm_certificate is not None:
             self.auto_create_acm_certificate = other.auto_create_acm_certificate
 
+    @property
+    def function_properties(self):
+        function_properties = {}
+        if len(self.layers) > 0:
+            function_properties["Layers"] = sorted(self.layers)
+        if len(self.security_group_ids) > 0 or len(self.subnet_ids) > 0:
+            function_properties["VpcConfig"] = {}
+            if len(self.security_group_ids) > 0:
+                function_properties["VpcConfig"]["SecurityGroupIds"] = sorted(
+                    self.security_group_ids
+                )
+            if len(self.subnet_ids) > 0:
+                function_properties["VpcConfig"]["SubnetIds"] = sorted(self.subnet_ids)
+        if len(self.logging_config) > 0:
+            function_properties["LoggingConfig"] = self.logging_config
+        return function_properties
+
 
 def get_build_config_for_stage(
     config: dict, stage: Optional[str] = None
@@ -154,8 +171,7 @@ def get_build_config_for_task(
     Retrieves and constructs the build configuration for a specific task and optional stage.
 
     This function applies default build configuration values, then overrides them with
-    task-specific and stage-specific configuration if available. It also updates the build
-    configuration with default tags.
+    task-specific and stage-specific configuration if available.
 
     Args:
         config (dict): The overall configuration dictionary containing tasks and stages.
@@ -166,8 +182,31 @@ def get_build_config_for_task(
         AsyncLambdaBuildConfig: The merged build configuration for the specified task and stage.
     """
     # Apply Defaults
-    build_config = get_build_config_for_stage(config, stage)
+    build_config = get_build_config_for_stage(config=config, stage=stage)
+    # Retrieve Task-Specific Overrides
+    override_config = get_override_build_config_for_task(
+        config=config, task_id=task_id, stage=stage
+    )
+    # Take the super-set of both configurations, prioritizing task-specific values on conflicts
+    build_config.merge(override_config)
+    return build_config
+
+
+def get_override_build_config_for_task(
+    config: dict, task_id: str, stage: Optional[str] = None
+) -> AsyncLambdaBuildConfig:
+    """
+    Retrieves and constructs the build configuration, containing solely task-specific overrides, for a specific task and optional stage.
 
+    Args:
+        config (dict): The overall configuration dictionary containing tasks and stages.
+        task_id (str): The identifier for the task to retrieve configuration for.
+        stage (Optional[str], optional): The stage name to retrieve configuration for. Defaults to None.
+
+    Returns:
+        AsyncLambdaBuildConfig: The build configuration with only override values for the specified task and stage.
+    """
+    build_config = AsyncLambdaBuildConfig.new({})
     if task_id in config.setdefault("tasks", {}):
         # Apply task defaults
         task_config = config["tasks"].setdefault(task_id, {})
@@ -179,5 +218,4 @@ def get_build_config_for_task(
                 stage, {}
             )
             build_config.merge(AsyncLambdaBuildConfig.new(task_stage_config))
-    build_config.tags.update(make_default_tags())
     return build_config

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/async_lambda/cli.py

@@ -140,7 +140,9 @@ def build(
             raise click.ClickException("Extras file contains invalid formatted data.")
 
     click.echo("Generating SAM template...")
-    template = controller.generate_sam_template(module, config, stage=stage)
+    template = controller.generate_sam_template(
+        module=module, config_dict=config, stage=stage
+    )
     if extras_json is not None:
         click.echo("Merging template with extras...")
         nested_update(template, extras_json)

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/async_lambda/config.py

@@ -8,14 +8,16 @@ class AsyncLambdaConfig:
     Attributes:
         name (str): Name of the async lambda application. Defaults to "async-lambda".
         runtime (str): Runtime environment for the lambda function. Defaults to "python3.10".
-        s3_payload_retention (Optional[int]): Days to retain payloads in S3. Defaults to 30.
+        s3_payload_retention (int): Days to retain payloads in S3. Defaults to 30.
         default_task_memory (int): Default memory allocation (MB) for tasks. Defaults to 128.
+        default_task_ephemeral_storage (int): Default disk allocation (MB) for tasks. Defaults to 512.
     """
 
     name: str = "async-lambda"
     runtime: str = "python3.10"
     s3_payload_retention: Optional[int] = 30
     default_task_memory: int = 128
+    default_task_ephemeral_storage: int = 512
 
 
 config = AsyncLambdaConfig()
@@ -41,12 +43,12 @@ def config_set_runtime(runtime: str):
     config.runtime = runtime
 
 
-def config_set_s3_payload_retention(days: Optional[int]):
+def config_set_s3_payload_retention(days: int = 30):
     """
     Set the S3 payload retention policy in days.
 
     Args:
-        days (Optional[int]): Number of days to retain payloads in S3.
+        days (int, optional): Number of days to retain payloads in S3. Defaults to 30.
     """
     config.s3_payload_retention = days
 
@@ -59,3 +61,13 @@ def config_set_default_task_memory(memory: int = 128):
         memory (int, optional): Memory in megabytes to set as default for tasks. Defaults to 128.
     """
     config.default_task_memory = memory
+
+
+def config_set_default_task_ephemeral_storage(ephemeral_storage: int = 512):
+    """
+    Set the default ephemeral_storage (MB) for tasks.
+
+    Args:
+        ephemeral_storage (int, optional): Disk storage in megabytes to set as default for tasks. Defaults to 512.
+    """
+    config.default_task_ephemeral_storage = ephemeral_storage

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/async_lambda/controller.py

@@ -366,9 +366,30 @@ class AsyncLambdaController:
                     }
                 ]
             }
+
         template = {
             "AWSTemplateFormatVersion": "2010-09-09",
             "Transform": "AWS::Serverless-2016-10-31",
+            "Globals": {
+                "Function": {
+                    "Tags": build_config.tags,
+                    "Handler": f"{module}.lambda_handler",
+                    "Runtime": config.runtime,
+                    "Environment": {
+                        "Variables": {
+                            "ASYNC_LAMBDA_PAYLOAD_S3_BUCKET": {
+                                "Ref": "AsyncLambdaPayloadBucket"
+                            },
+                            "ASYNC_LAMBDA_ACCOUNT_ID": {"Ref": "AWS::AccountId"},
+                            **build_config.environment_variables,
+                        },
+                    },
+                    "CodeUri": ".async_lambda/build/deployment.zip",
+                    "MemorySize": config.default_task_memory,
+                    "EphemeralStorage": {"Size": config.default_task_ephemeral_storage},
+                    **build_config.function_properties,
+                }
+            },
             "Resources": {
                 "AsyncLambdaPayloadBucket": {
                     "Type": "AWS::S3::Bucket",
@@ -392,14 +413,53 @@ class AsyncLambdaController:
             },
         }
         _task_list = list(self.tasks.values())
+        managed_tasks_resources = [
+            resource
+            for task in _task_list
+            if task.trigger_type in MANAGED_SQS_TASK_TYPES
+            for resource in task.get_policy_sqs_resources()
+        ] + [
+            resource
+            for external_async_task_id in self.external_async_tasks
+            for resource in AsyncLambdaTask.get_policy_external_task_resources(
+                external_async_task_id
+            )
+        ]
+        task_ref_policies = {}
+        if len(managed_tasks_resources) > 0:
+            task_ref_policies = {
+                "SendToAllAsyncLambdaQueuesPolicy": {
+                    "Type": "AWS::IAM::ManagedPolicy",
+                    "Properties": {
+                        "ManagedPolicyName": {
+                            "Fn::Sub": "${AWS::StackName}-send-to-all-queues"
+                        },
+                        "PolicyDocument": {
+                            "Version": "2012-10-17",
+                            "Statement": [
+                                {
+                                    "Sid": "SendToAllQueues",
+                                    "Effect": "Allow",
+                                    "Action": ["sqs:SendMessage"],
+                                    "Resource": [
+                                        managed_tasks_resource
+                                        for managed_tasks_resource in managed_tasks_resources
+                                    ],
+                                },
+                            ],
+                        },
+                    },
+                }
+            }
+            for key in task_ref_policies.keys():
+                template["Resources"][key] = task_ref_policies[key]
+
         has_api_tasks = False
         for task in _task_list:
             if task.trigger_type == TaskTriggerType.API_EVENT:
                 has_api_tasks = True
             for logical_id, resource in task.get_sam_template(
-                module=module,
-                tasks=_task_list,
-                external_async_tasks=self.external_async_tasks,
+                task_ref_policy_ids=list(task_ref_policies.keys()),
                 config_dict=config_dict,
                 stage=stage,
             ).items():
@@ -1196,7 +1256,7 @@ class AsyncLambdaController:
         propagate_lane_assignment: Optional[bool] = None,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -1273,7 +1333,7 @@ class AsyncLambdaController:
         max_batching_window: Optional[int] = None,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -1345,7 +1405,7 @@ class AsyncLambdaController:
         queue_arn: str,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -1405,7 +1465,7 @@ class AsyncLambdaController:
         schedule_expression: str,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -1466,7 +1526,7 @@ class AsyncLambdaController:
         method: str,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -1531,7 +1591,7 @@ class AsyncLambdaController:
         max_batching_window: int = 0,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -1599,7 +1659,7 @@ class AsyncLambdaController:
         task_id: str,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         **kwargs,
     ):
         """

{async_lambda_unstable-0.6.2 → async_lambda_unstable-0.6.3}/async_lambda/models/task.py

@@ -9,13 +9,12 @@ from typing import (
     Generic,
     List,
     Optional,
-    Set,
     TypeVar,
     Union,
 )
 
 from .. import env
-from ..build_config import get_build_config_for_task
+from ..build_config import get_build_config_for_task, get_override_build_config_for_task
 from ..config import config
 from ..util import make_cf_tags
 
@@ -86,7 +85,7 @@ class AsyncLambdaTask(Generic[EventType, RT]):
         trigger_config (dict): Configuration specific to the trigger type.
         timeout (int): Lambda function timeout (seconds).
         memory (Optional[int]): Memory size for the Lambda function.
-        ephemeral_storage (int): Ephemeral storage size for the Lambda function.
+        ephemeral_storage (Optional[int]): Ephemeral storage size for the Lambda function.
         maximum_concurrency (Optional[Union[int, List[int]]]): Maximum concurrency for the task or per lane.
         init_tasks (List[Union[Callable[[str], Any], Callable[[], Any]]]): Initialization tasks to run before execution.
         _has_run_init_tasks (bool): Flag indicating if initialization tasks have been run.
@@ -100,7 +99,7 @@ class AsyncLambdaTask(Generic[EventType, RT]):
 
     timeout: int
     memory: Optional[int]
-    ephemeral_storage: int
+    ephemeral_storage: Optional[int]
     maximum_concurrency: Optional[Union[int, List[int]]]
     init_tasks: List[Union[Callable[[str], Any], Callable[[], Any]]]
     _has_run_init_tasks: bool
@@ -116,7 +115,7 @@ class AsyncLambdaTask(Generic[EventType, RT]):
         trigger_config: Optional[dict] = None,
         timeout: int = 60,
         memory: Optional[int] = None,
-        ephemeral_storage: int = 512,
+        ephemeral_storage: Optional[int] = None,
         maximum_concurrency: Optional[Union[int, List[int]]] = None,
         init_tasks: Optional[
             List[Union[Callable[[str], Any], Callable[[], Any]]]
@@ -133,7 +132,7 @@ class AsyncLambdaTask(Generic[EventType, RT]):
             trigger_config (Optional[dict], optional): Configuration for the trigger. Defaults to None.
             timeout (int, optional): Maximum execution time (seconds). Defaults to 60.
             memory (Optional[int], optional): Memory allocation (MB). Defaults to None.
-            ephemeral_storage (int, optional): Ephemeral storage (MB). Defaults to 512.
+            ephemeral_storage (Optional[int], optional): Ephemeral storage (MB). Defaults to None.
             maximum_concurrency (Optional[Union[int, List[int]]], optional): Maximum concurrency for the task. Defaults to None.
             init_tasks (Optional[List[Union[Callable[[str], Any], Callable[[], Any]]]], optional): Initialization tasks to run before execution. Defaults to None.
 
@@ -540,26 +539,29 @@ class AsyncLambdaTask(Generic[EventType, RT]):
 
     def get_sam_template(
         self,
-        module: str,
-        tasks: List["AsyncLambdaTask"],
-        external_async_tasks: Set[str],
+        task_ref_policy_ids: List[str],
         config_dict: dict,
         stage: Optional[str] = None,
     ) -> dict:
         """
         Generate an AWS SAM (Serverless Application Model) template for the current async lambda task.
 
+        This template inherits the global SAM values provided via the controller's generate_sam_template(...) method.
+
         Args:
-            module (str): Python module containing the Lambda handler.
-            tasks (List[AsyncLambdaTask]): All async lambda tasks in the deployment.
-            external_async_tasks (Set[str]): External async task IDs referenced by this task.
+            task_ref_policy_ids (List[str]): List of ManagedPolicy logical IDs to attach via CloudFormation Ref.
             config_dict (dict): Configuration dictionary for deployment settings.
             stage (Optional[str], optional): Deployment stage (e.g., 'dev', 'prod'). Defaults to None.
 
         Returns:
             dict: SAM template resources for this task.
         """
-        build_config = get_build_config_for_task(config_dict, self.task_id, stage=stage)
+        build_config = get_build_config_for_task(
+            config=config_dict, task_id=self.task_id, stage=stage
+        )
+        override_config = get_override_build_config_for_task(
+            config=config_dict, task_id=self.task_id, stage=stage
+        )
         events = self.get_template_events()
         policy_sqs_resources = self.get_policy_sqs_resources()
 
@@ -583,29 +585,6 @@ class AsyncLambdaTask(Generic[EventType, RT]):
                 },
             },
         ]
-        managed_tasks_resources = [
-            resource
-            for task in tasks
-            if task.trigger_type in MANAGED_SQS_TASK_TYPES
-            for resource in task.get_policy_sqs_resources()
-        ] + [
-            resource
-            for external_async_task_id in external_async_tasks
-            for resource in self.get_policy_external_task_resources(
-                external_async_task_id
-            )
-        ]
-        if len(managed_tasks_resources) > 0:
-            policy_statements.append(
-                {
-                    "Effect": "Allow",
-                    "Action": ["sqs:SendMessage"],
-                    "Resource": [
-                        managed_tasks_resource
-                        for managed_tasks_resource in managed_tasks_resources
-                    ],
-                },
-            )
         if len(policy_sqs_resources) > 0:
             policy_statements.append(
                 {
@@ -620,55 +599,39 @@ class AsyncLambdaTask(Generic[EventType, RT]):
                     "Resource": policy_sqs_resources,
                 },
             )
-        function_properties = {}
-        if len(build_config.layers) > 0:
-            function_properties["Layers"] = sorted(build_config.layers)
-        if len(build_config.security_group_ids) > 0 or len(build_config.subnet_ids) > 0:
-            function_properties["VpcConfig"] = {}
-            if len(build_config.security_group_ids) > 0:
-                function_properties["VpcConfig"]["SecurityGroupIds"] = sorted(
-                    build_config.security_group_ids
-                )
-            if len(build_config.subnet_ids) > 0:
-                function_properties["VpcConfig"]["SubnetIds"] = sorted(
-                    build_config.subnet_ids
-                )
-        if len(build_config.logging_config) > 0:
-            function_properties["LoggingConfig"] = build_config.logging_config
+
+        policies = [
+            {"Statement": policy_statements},
+            *build_config.policies,
+            *[{"Ref": policy_id} for policy_id in task_ref_policy_ids],
+        ]
 
         template = {
             self.get_function_logical_id(): {
                 "Type": "AWS::Serverless::Function",
                 "Properties": {
-                    "Tags": build_config.tags,
-                    "Handler": f"{module}.lambda_handler",
-                    "Runtime": config.runtime,
                     "Environment": {
                         "Variables": {
-                            "ASYNC_LAMBDA_PAYLOAD_S3_BUCKET": {
-                                "Ref": "AsyncLambdaPayloadBucket"
-                            },
                             "ASYNC_LAMBDA_TASK_ID": self.task_id,
-                            "ASYNC_LAMBDA_ACCOUNT_ID": {"Ref": "AWS::AccountId"},
-                            **build_config.environment_variables,
+                            **override_config.environment_variables,
                         }
                     },
                     "FunctionName": self.get_function_name(),
-                    "CodeUri": ".async_lambda/build/deployment.zip",
-                    "EphemeralStorage": {"Size": self.ephemeral_storage},
-                    "MemorySize": (
-                        self.memory if self.memory else config.default_task_memory
-                    ),
                     "Timeout": self.timeout,
                     "Events": events,
-                    "Policies": [
-                        {"Statement": policy_statements},
-                        *build_config.policies,
-                    ],
-                    **function_properties,
+                    "Policies": policies,
+                    **override_config.function_properties,
                 },
             }
         }
+        if self.memory is not None:
+            template[self.get_function_logical_id()]["Properties"]["MemorySize"] = (
+                self.memory
+            )
+        if self.ephemeral_storage is not None:
+            template[self.get_function_logical_id()]["Properties"][
+                "EphemeralStorage"
+            ] = {"Size": self.ephemeral_storage}
 
         if self.trigger_type in MANAGED_SQS_TASK_TYPES:
             dlq_task = self.get_dlq_task()