localstack-core 4.10.1.dev42__py3-none-any.whl → 4.11.2.dev14__py3-none-any.whl

localstack/testing/pytest/fixtures.py

@@ -1426,6 +1426,34 @@ def create_lambda_function(aws_client, wait_until_lambda_ready, lambda_su_role):
             LOG.debug("Unable to delete log group %s in cleanup", log_group_name)
 
 
+@pytest.fixture
+def lambda_is_function_deleted(aws_client):
+    """Example usage:
+    wait_until(lambda_is_function_deleted(function_name))
+    wait_until(lambda_is_function_deleted(function_name, Qualifier="my-alias"))
+
+    function_name can be a function name, function ARN, or partial function ARN.
+    """
+    return _lambda_is_function_deleted(aws_client.lambda_)
+
+
+def _lambda_is_function_deleted(lambda_client):
+    def _is_function_deleted(
+        function_name: str,
+        **kwargs,
+    ) -> Callable[[], bool]:
+        def _inner() -> bool:
+            try:
+                lambda_client.get_function(FunctionName=function_name, **kwargs)
+                return False
+            except lambda_client.exceptions.ResourceNotFoundException:
+                return True
+
+        return _inner
+
+    return _is_function_deleted
+
+
 @pytest.fixture
 def create_echo_http_server(aws_client, create_lambda_function):
     from localstack.aws.api.lambda_ import Runtime

localstack/testing/snapshots/transformer_utility.py

@@ -791,6 +791,11 @@ class TransformerUtility:
                 "x-amzn-RequestId",
                 replace_reference=False,
             ),
+            JsonpathTransformer(
+                "$..x-amzn-requestid",
+                "x-amzn-requestid",
+                replace_reference=False,
+            ),
             KeyValueBasedTransformer(_transform_stepfunctions_cause_details, "json-input"),
         ]
 

localstack/utils/analytics/publisher.py

@@ -2,11 +2,10 @@ import abc
 import atexit
 import logging
 import threading
-import time
-from queue import Full, Queue
 
 from localstack import config
-from localstack.utils.threads import start_thread, start_worker_thread
+from localstack.utils.batching import AsyncBatcher
+from localstack.utils.threads import FuncThread, start_thread, start_worker_thread
 
 from .client import AnalyticsClient
 from .events import Event, EventHandler
@@ -51,139 +50,36 @@ class Printer(Publisher):
             print(event.asdict())
 
 
-class PublisherBuffer(EventHandler):
-    """
-    A PublisherBuffer is an EventHandler that collects events into a buffer until a flush condition is
-    met, and then flushes the buffer to a Publisher. The condition is either a given buffer size or
-    a time interval, whatever occurs first. The buffer is also flushed when the recorder is stopped
-    via `close()`. Internally it uses a simple event-loop mechanism to multiplex commands on a
-    single thread.
-    """
-
-    flush_size: int
-    flush_interval: float
-
-    _cmd_flush = "__FLUSH__"
-    _cmd_stop = "__STOP__"
-
-    # FIXME: figure out good default values
-    def __init__(
-        self, publisher: Publisher, flush_size: int = 20, flush_interval: float = 10, maxsize=0
-    ):
-        self._publisher = publisher
-        self._queue = Queue(maxsize=maxsize)
-        self._command_queue = Queue()
-
-        self.flush_size = flush_size
-        self.flush_interval = flush_interval
-
-        self._last_flush = time.time()
-        self._stopping = threading.Event()
-        self._stopped = threading.Event()
-
-    def handle(self, event: Event):
-        self._queue.put_nowait(event)
-        self.checked_flush()
-
-    def close(self):
-        if self._stopping.is_set():
-            return
-
-        self._stopping.set()
-        self._command_queue.put(self._cmd_stop)
+class GlobalAnalyticsBus(EventHandler):
+    _batcher: AsyncBatcher[Event]
+    _client: AnalyticsClient
+    _worker_thread: FuncThread | None
 
-    def close_sync(self, timeout: float | None = None):
-        self.close()
-        return self._stopped.wait(timeout)
-
-    def flush(self):
-        self._command_queue.put(self._cmd_flush)
-        self._last_flush = time.time()
-
-    def checked_flush(self):
-        """
-        Runs flush only if a flush condition is met.
-        """
-        if config.DEBUG_ANALYTICS:
-            LOG.debug(
-                "analytics queue size: %d, command queue size: %d, time since last flush: %.1fs",
-                self._queue.qsize(),
-                self._command_queue.qsize(),
-                time.time() - self._last_flush,
-            )
-
-        if self._queue.qsize() >= self.flush_size:
-            self.flush()
-            return
-        if time.time() - self._last_flush >= self.flush_interval:
-            self.flush()
-            return
-
-    def _run_flush_schedule(self, *_):
-        while True:
-            if self._stopping.wait(self.flush_interval):
-                return
-            self.checked_flush()
-
-    def run(self, *_):
-        flush_scheduler = start_thread(self._run_flush_schedule, name="analytics-publishbuffer")
-
-        try:
-            while True:
-                command = self._command_queue.get()
-
-                if command is self._cmd_flush or command is self._cmd_stop:
-                    try:
-                        self._do_flush()
-                    except Exception:
-                        if config.DEBUG_ANALYTICS:
-                            LOG.exception("error while flushing events")
-
-                if command is self._cmd_stop:
-                    return
-        finally:
-            self._stopped.set()
-            flush_scheduler.stop()
-            self._publisher.close()
-            if config.DEBUG_ANALYTICS:
-                LOG.debug("Exit analytics publisher")
-
-    def _do_flush(self):
-        queue = self._queue
-        events = []
-
-        for _ in range(queue.qsize()):
-            event = queue.get_nowait()
-            events.append(event)
-
-        if config.DEBUG_ANALYTICS:
-            LOG.debug("collected %d events to publish", len(events))
-
-        self._publisher.publish(events)
-
-
-class GlobalAnalyticsBus(PublisherBuffer):
-    def __init__(
-        self, client: AnalyticsClient = None, flush_size=20, flush_interval=10, max_buffer_size=1000
-    ) -> None:
+    def __init__(self, client: AnalyticsClient = None, flush_size=20, flush_interval=10) -> None:
         self._client = client or AnalyticsClient()
         self._publisher = AnalyticsClientPublisher(self._client)
-
-        super().__init__(
-            self._publisher,
-            flush_size=flush_size,
-            flush_interval=flush_interval,
-            maxsize=max_buffer_size,
+        self._batcher = AsyncBatcher(
+            self._handle_batch,
+            max_batch_size=flush_size,
+            max_flush_interval=flush_interval,
         )
 
         self._started = False
-        self._startup_complete = False
         self._startup_mutex = threading.Lock()
-        self._buffer_thread = None
+        self._worker_thread = None
 
         self.force_tracking = False  # allow class to ignore all other tracking config
         self.tracking_disabled = False  # disables tracking if global config would otherwise track
 
+    def _handle_batch(self, batch: list[Event]):
+        """Method that satisfies the BatchHandler[Event] protocol and is passed to AsyncBatcher."""
+        try:
+            self._publisher.publish(batch)
+        except Exception:
+            # currently we're just dropping events if something goes wrong during publishing
+            if config.DEBUG_ANALYTICS:
+                LOG.exception("error while publishing analytics events")
+
     @property
     def is_tracking_disabled(self):
         if self.force_tracking:
@@ -200,44 +96,20 @@ class GlobalAnalyticsBus(PublisherBuffer):
 
         return False
 
-    def _do_flush(self):
-        if self.tracking_disabled:
-            # flushing although tracking has been disabled most likely means that _do_start_retry
-            # has failed, tracking is now disabled, and the system tries to flush the queued
-            # events. we use this opportunity to shut down the tracker and clear the queue, since
-            # no tracking should happen from this point on.
-            if config.DEBUG_ANALYTICS:
-                LOG.debug("attempting to flush while tracking is disabled, shutting down tracker")
-            self.close_sync(timeout=10)
-            self._queue.queue.clear()
-            return
-
-        super()._do_flush()
-
-    def flush(self):
-        if not self._startup_complete:
-            # don't flush until _do_start_retry has completed (command queue would fill up)
-            return
-
-        super().flush()
-
     def handle(self, event: Event):
         """
         Publish an event to the global analytics event publisher.
         """
         if self.is_tracking_disabled:
             if config.DEBUG_ANALYTICS:
-                LOG.debug("skipping event %s", event)
+                LOG.debug("tracking disabled, skipping event %s", event)
             return
 
         if not self._started:
+            # we make sure the batching worker is started
             self._start()
 
-        try:
-            super().handle(event)
-        except Full:
-            if config.DEBUG_ANALYTICS:
-                LOG.warning("event queue is full, dropping event %s", event)
+        self._batcher.add(event)
 
     def _start(self):
         with self._startup_mutex:
@@ -267,12 +139,22 @@ class GlobalAnalyticsBus(PublisherBuffer):
             if config.DEBUG_ANALYTICS:
                 LOG.exception("error while registering session. disabling tracking")
             return
-        finally:
-            self._startup_complete = True
 
-        start_thread(self.run, name="global-analytics-bus")
+        self._worker_thread = start_thread(self._run, name="global-analytics-bus")
 
+        # given the "Global" nature of this class, we register a global atexit hook to make sure all events are flushed
+        # when localstack shuts down.
         def _do_close():
             self.close_sync(timeout=2)
 
         atexit.register(_do_close)
+
+    def _run(self, *_):
+        # main control loop, simply runs the batcher
+        self._batcher.run()
+
+    def close_sync(self, timeout=None):
+        self._batcher.close()
+
+        if self._worker_thread:
+            self._worker_thread.join(timeout=timeout)

localstack/utils/analytics/service_request_aggregator.py

@@ -34,7 +34,7 @@ class ServiceRequestAggregator:
         self._flush_interval = flush_interval
         self._flush_scheduler = Scheduler()
         self._mutex = threading.RLock()
-        self._period_start_time = datetime.datetime.utcnow()
+        self._period_start_time = datetime.datetime.now(datetime.UTC)
         self._is_started = False
         self._is_shutdown = False
 
@@ -101,12 +101,14 @@ class ServiceRequestAggregator:
                 self._emit_payload(analytics_payload)
                 self.counter.clear()
             finally:
-                self._period_start_time = datetime.datetime.utcnow()
+                self._period_start_time = datetime.datetime.now(datetime.UTC)
 
     def _create_analytics_payload(self):
         return {
-            "period_start_time": self._period_start_time.isoformat() + "Z",
-            "period_end_time": datetime.datetime.utcnow().isoformat() + "Z",
+            "period_start_time": self._period_start_time.isoformat().replace("+00:00", "Z"),
+            "period_end_time": datetime.datetime.now(datetime.UTC)
+            .isoformat()
+            .replace("+00:00", "Z"),
             "api_calls": self._aggregate_api_calls(self.counter),
         }
 

localstack/utils/aws/arns.py

@@ -284,6 +284,13 @@ def lambda_event_source_mapping_arn(uuid: str, account_id: str, region_name: str
     return _resource_arn(uuid, pattern, account_id=account_id, region_name=region_name)
 
 
+def capacity_provider_arn(capacity_provider_name: str, account_id: str, region_name: str) -> str:
+    pattern = "arn:%s:lambda:%s:%s:capacity-provider:%s"
+    return _resource_arn(
+        capacity_provider_name, pattern, account_id=account_id, region_name=region_name
+    )
+
+
 def lambda_function_or_layer_arn(
     type: str,
     entity_name: str,

localstack/utils/batching.py

@@ -0,0 +1,258 @@
+import copy
+import logging
+import threading
+import time
+from typing import Generic, Protocol, TypeVar, overload
+
+LOG = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# alias to signify whether a batch policy has been triggered
+BatchPolicyTriggered = bool
+
+
+# TODO: Add batching on bytes as well.
+class Batcher(Generic[T]):
+    """
+    A utility for collecting items into batches and flushing them when one or more batch policy conditions are met.
+
+    The batch policy can be created to trigger on:
+    - max_count: Maximum number of items added
+    - max_window: Maximum time window (in seconds)
+
+    If no limits are specified, the batcher is always in triggered state.
+
+    Example usage:
+
+        import time
+
+        # Triggers when 2 (or more) items are added
+        batcher = Batcher(max_count=2)
+        assert batcher.add(["item1", "item2", "item3"])
+        assert batcher.flush() == ["item1", "item2", "item3"]
+
+        # Triggers partially when 2 (or more) items are added
+        batcher = Batcher(max_count=2)
+        assert batcher.add(["item1", "item2", "item3"])
+        assert batcher.flush(partial=True) == ["item1", "item2"]
+        assert batcher.add("item4")
+        assert batcher.flush(partial=True) == ["item3", "item4"]
+
+        # Trigger 2 seconds after the first add
+        batcher = Batcher(max_window=2.0)
+        assert not batcher.add(["item1", "item2", "item3"])
+        time.sleep(2.1)
+        assert not batcher.add(["item4"])
+        assert batcher.flush() == ["item1", "item2", "item3", "item4"]
+    """
+
+    max_count: int | None
+    """
+    Maximum number of items, must be None or positive.
+    """
+
+    max_window: float | None
+    """
+    Maximum time window in seconds, must be None or positive.
+    """
+
+    _triggered: bool
+    _last_batch_time: float
+    _batch: list[T]
+
+    def __init__(self, max_count: int | None = None, max_window: float | None = None):
+        """
+        Initialize a new Batcher instance.
+
+        :param max_count: Maximum number of items that be None or positive.
+        :param max_window: Maximum time window in seconds that must be None or positive.
+        """
+        self.max_count = max_count
+        self.max_window = max_window
+
+        self._triggered = False
+        self._last_batch_time = time.monotonic()
+        self._batch = []
+
+    @property
+    def period(self) -> float:
+        return time.monotonic() - self._last_batch_time
+
+    def _check_batch_policy(self) -> bool:
+        """Check if any batch policy conditions are met"""
+        if self.max_count is not None and len(self._batch) >= self.max_count:
+            self._triggered = True
+        elif self.max_window is not None and self.period >= self.max_window:
+            self._triggered = True
+        elif not self.max_count and not self.max_window:
+            # always return true
+            self._triggered = True
+
+        return self._triggered
+
+    @overload
+    def add(self, item: T, *, deep_copy: bool = False) -> BatchPolicyTriggered: ...
+
+    @overload
+    def add(self, items: list[T], *, deep_copy: bool = False) -> BatchPolicyTriggered: ...
+
+    def add(self, item_or_items: T | list[T], *, deep_copy: bool = False) -> BatchPolicyTriggered:
+        """
+        Add an item or list of items to the collected batch.
+
+        Returns:
+            BatchPolicyTriggered: True if the batch policy was triggered during addition, False otherwise.
+        """
+        if deep_copy:
+            item_or_items = copy.deepcopy(item_or_items)
+
+        if isinstance(item_or_items, list):
+            self._batch.extend(item_or_items)
+        else:
+            self._batch.append(item_or_items)
+
+        # Check if the last addition triggered the batch policy
+        return self.is_triggered()
+
+    def flush(self, *, partial=False) -> list[T]:
+        result = []
+        if not partial or not self.max_count:
+            result = self._batch.copy()
+            self._batch.clear()
+        else:
+            batch_size = min(self.max_count, len(self._batch))
+            result = self._batch[:batch_size].copy()
+            self._batch = self._batch[batch_size:]
+
+        self._last_batch_time = time.monotonic()
+        self._triggered = False
+        self._check_batch_policy()
+
+        return result
+
+    def duration_until_next_batch(self) -> float:
+        if not self.max_window:
+            return -1
+        return max(self.max_window - self.period, -1)
+
+    def get_current_size(self) -> int:
+        return len(self._batch)
+
+    def is_triggered(self):
+        return self._triggered or self._check_batch_policy()
+
+
+class BatchHandler(Protocol[T]):
+    """
+    A BatchHandler is a callable that processes a list of items handed down by the AsyncBatcher.
+    """
+
+    def __call__(self, batch: list[T]) -> None: ...
+
+
+class AsyncBatcher(Generic[T]):
+    """
+    Class for managing asynchronous batching of items.
+
+    This class allows for efficient buffering and processing of items in batches by
+    periodically flushing the buffer to a given handler, or by automatically flushing
+    when the maximum batch size is reached. It is designed to be used in asynchronous
+    scenarios where the caller does not execute the flushing IO call itself, like with ``Batcher``.
+
+    :ivar max_flush_interval: Maximum time interval in seconds between
+        automatic flushes, regardless of the batch size.
+    :ivar max_batch_size: Maximum number of items in a batch. When reached,
+        the batch is flushed automatically.
+    :ivar handler: Callable handler that processes each flushed batch. The handler must
+        be provided during initialization and must accept a list of items as input.
+    """
+
+    max_flush_interval: float
+    max_batch_size: int
+    handler: BatchHandler[T]
+
+    _buffer: list[T]
+    _flush_lock: threading.Condition
+    _closed: bool
+
+    def __init__(
+        self,
+        handler: BatchHandler[T],
+        max_flush_interval: float = 10,
+        max_batch_size: int = 20,
+    ):
+        self.handler = handler
+        self.max_flush_interval = max_flush_interval
+        self.max_batch_size = max_batch_size
+
+        self._buffer = []
+        self._flush_lock = threading.Condition()
+        self._closed = False
+
+    def add(self, item: T):
+        """
+        Adds an item to the buffer.
+
+        :param item: the item to add
+        """
+        with self._flush_lock:
+            if self._closed:
+                raise ValueError("Batcher is stopped, can no longer add items")
+
+            self._buffer.append(item)
+
+            if len(self._buffer) >= self.max_batch_size:
+                self._flush_lock.notify_all()
+
+    @property
+    def current_batch_size(self) -> int:
+        """
+        Returns the current number of items in the buffer waiting to be flushed.
+        """
+        return len(self._buffer)
+
+    def run(self):
+        """
+        Runs the event loop that flushes the buffer to the handler based on the configured rules, and blocks until
+        ``close()`` is called. This method is meant to be run in a separate thread.
+        """
+        while not self._closed:
+            with self._flush_lock:
+                # wait returns once either the condition is notified (in which case wait returns True, indicating that
+                # something has triggered a flush manually), or the timeout expires (in which case wait returns False)
+                self._flush_lock.wait(self.max_flush_interval)
+
+                # if _flush_condition was notified because close() was called, we should still make sure we flush the
+                # last batch
+
+                # perform the flush, if there are any items in the buffer
+                if not self._buffer:
+                    continue
+
+                batch = self._buffer.copy()
+                self._buffer.clear()
+
+            # we can call the processor outside the lock so we can continue adding items into the next batch without
+            # waiting on the processor to return.
+            try:
+                self.handler(batch)
+            except Exception as e:
+                LOG.error(
+                    "Unhandled exception while processing a batch: %s",
+                    e,
+                    exc_info=LOG.isEnabledFor(logging.DEBUG),
+                )
+
+        # this marks that the main control loop is done
+        return
+
+    def close(self):
+        """
+        Triggers a close of the batcher, which will cause one last flush, and then end the main event loop.
+        """
+        with self._flush_lock:
+            if self._closed:
+                return
+            self._closed = True
+            self._flush_lock.notify_all()

localstack/utils/collections.py

@@ -5,7 +5,7 @@ and manipulate python collection (dicts, list, sets).
 
 import logging
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sized
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sized
 from typing import (
     Any,
     Optional,
@@ -24,6 +24,9 @@ LOG = logging.getLogger(__name__)
 # default regex to match an item in a comma-separated list string
 DEFAULT_REGEX_LIST_ITEM = r"[\w-]+"
 
+_E = TypeVar("_E")
+"""TypeVar var used internally for container type parameters."""
+
 
 class AccessTrackingDict(dict):
     """
@@ -101,21 +104,18 @@ class HashableJsonDict(ImmutableDict):
         return hash(canonical_json(self._dict))
 
 
-_ListType = TypeVar("_ListType")
-
-
-class PaginatedList(list[_ListType]):
+class PaginatedList(list[_E]):
     """List which can be paginated and filtered. For usage in AWS APIs with paginated responses"""
 
     DEFAULT_PAGE_SIZE = 50
 
     def get_page(
         self,
-        token_generator: Callable[[_ListType], str],
+        token_generator: Callable[[_E], str],
         next_token: str = None,
         page_size: int = None,
-        filter_function: Callable[[_ListType], bool] = None,
-    ) -> tuple[list[_ListType], str | None]:
+        filter_function: Callable[[_E], bool] = None,
+    ) -> tuple[list[_E], str | None]:
         if filter_function is not None:
             result_list = list(filter(filter_function, self))
         else:
@@ -528,9 +528,6 @@ def is_comma_delimited_list(string: str, item_regex: str | None = None) -> bool:
     return True
 
 
-_E = TypeVar("_E")
-
-
 def optional_list(condition: bool, items: Iterable[_E]) -> list[_E]:
     """
     Given an iterable, either create a list out of the entire iterable (if `condition` is `True`), or return the empty list.
@@ -540,3 +537,18 @@ def optional_list(condition: bool, items: Iterable[_E]) -> list[_E]:
     []
     """
     return list(filter(lambda _: condition, items))
+
+
+def iter_chunks(items: list[_E], chunk_size: int) -> Generator[list[_E], None, None]:
+    """
+    Split a list into smaller chunks of a specified size and iterate over them.
+
+    It is implemented as a generator and yields each chunk as needed, making it memory-efficient for large lists.
+
+    :param items:  A list of elements to be divided into chunks.
+    :param chunk_size: The maximum number of elements that a single chunk can contain.
+    :return: A generator that yields chunks (sublists) of the original list. Each chunk contains up to `chunk_size`
+    elements.
+    """
+    for i in range(0, len(items), chunk_size):
+        yield items[i : i + chunk_size]

localstack/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '4.10.1.dev42'
-__version_tuple__ = version_tuple = (4, 10, 1, 'dev42')
+__version__ = version = '4.11.2.dev14'
+__version_tuple__ = version_tuple = (4, 11, 2, 'dev14')
 
 __commit_id__ = commit_id = None