localstack-core 4.10.1.dev44__tar.gz → 4.10.1.dev47__tar.gz

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localstack-core
-Version: 4.10.1.dev44
+Version: 4.10.1.dev47
 Summary: The core library and runtime of LocalStack
 Author-email: LocalStack Contributors <info@localstack.cloud>
 License-Expression: Apache-2.0

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack/services/cloudformation/engine/template_preparer.py

@@ -6,6 +6,7 @@ from localstack.services.cloudformation.engine.transformers import (
     apply_global_transformations,
     apply_intrinsic_transformations,
 )
+from localstack.services.cloudformation.engine.validations import ValidationError
 from localstack.utils.json import clone_safe
 
 LOG = logging.getLogger(__name__)
@@ -17,9 +18,12 @@ def parse_template(template: str) -> dict:
     except Exception:
         try:
             return clone_safe(yaml_parser.parse_yaml(template))
-        except Exception as e:
-            LOG.debug("Unable to parse CloudFormation template (%s): %s", e, template)
+        except ValidationError:
+            # The error is handled in the yaml parsing helper
             raise
+        except Exception:
+            # TODO: present the user with a better error message including error location
+            raise ValidationError("Template format error: YAML not well-formed.")
 
 
 def template_to_json(template: str) -> str:

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py

@@ -36,7 +36,7 @@ from localstack.services.lambda_.event_source_mapping.senders.sender_utils impor
 )
 from localstack.utils.aws.arns import parse_arn, s3_bucket_name
 from localstack.utils.backoff import ExponentialBackoff
-from localstack.utils.batch_policy import Batcher
+from localstack.utils.batching import Batcher
 from localstack.utils.strings import long_uid
 
 LOG = logging.getLogger(__name__)

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack/services/s3/utils.py

@@ -421,6 +421,7 @@ def get_failed_upload_part_copy_source_preconditions(
     if_none_match = request.get("CopySourceIfNoneMatch")
     if_unmodified_since = request.get("CopySourceIfUnmodifiedSince")
     if_modified_since = request.get("CopySourceIfModifiedSince")
+    last_modified = second_resolution_datetime(last_modified)
 
     if if_match:
         if if_match.strip('"') != etag.strip('"'):
@@ -431,15 +432,15 @@ def get_failed_upload_part_copy_source_preconditions(
         if if_unmodified_since:
             return None
 
-    if if_unmodified_since and if_unmodified_since < last_modified:
+    if if_unmodified_since and second_resolution_datetime(if_unmodified_since) < last_modified:
         return "x-amz-copy-source-If-Unmodified-Since"
 
     if if_none_match and if_none_match.strip('"') == etag.strip('"'):
         return "x-amz-copy-source-If-None-Match"
 
-    if if_modified_since and last_modified < if_modified_since < datetime.datetime.now(
-        tz=_gmt_zone_info
-    ):
+    if if_modified_since and last_modified <= second_resolution_datetime(
+        if_modified_since
+    ) < datetime.datetime.now(tz=_gmt_zone_info):
         return "x-amz-copy-source-If-Modified-Since"
 
 
@@ -701,6 +702,10 @@ def str_to_rfc_1123_datetime(value: str) -> datetime.datetime:
     return datetime.datetime.strptime(value, RFC1123).replace(tzinfo=_gmt_zone_info)
 
 
+def second_resolution_datetime(src: datetime.datetime) -> datetime.datetime:
+    return src.replace(microsecond=0)
+
+
 def add_expiration_days_to_datetime(user_datatime: datetime.datetime, exp_days: int) -> str:
     """
     This adds expiration days to a datetime, rounding to the next day at midnight UTC.
@@ -908,6 +913,7 @@ def get_failed_precondition_copy_source(
     :param etag: source object ETag
     :return str: the failed precondition to raise
     """
+    last_modified = second_resolution_datetime(last_modified)
     if (cs_if_match := request.get("CopySourceIfMatch")) and etag.strip('"') != cs_if_match.strip(
         '"'
     ):
@@ -915,7 +921,7 @@ def get_failed_precondition_copy_source(
 
     elif (
         cs_if_unmodified_since := request.get("CopySourceIfUnmodifiedSince")
-    ) and last_modified > cs_if_unmodified_since:
+    ) and last_modified > second_resolution_datetime(cs_if_unmodified_since):
         return "x-amz-copy-source-If-Unmodified-Since"
 
     elif (cs_if_none_match := request.get("CopySourceIfNoneMatch")) and etag.strip(
@@ -925,7 +931,9 @@ def get_failed_precondition_copy_source(
 
     elif (
         cs_if_modified_since := request.get("CopySourceIfModifiedSince")
-    ) and last_modified < cs_if_modified_since < datetime.datetime.now(tz=_gmt_zone_info):
+    ) and last_modified <= second_resolution_datetime(cs_if_modified_since) < datetime.datetime.now(
+        tz=_gmt_zone_info
+    ):
         return "x-amz-copy-source-If-Modified-Since"
 
 
@@ -943,13 +951,13 @@ def validate_failed_precondition(
     """
     precondition_failed = None
     # last_modified needs to be rounded to a second so that strict equality can be enforced from a RFC1123 header
-    last_modified = last_modified.replace(microsecond=0)
+    last_modified = second_resolution_datetime(last_modified)
     if (if_match := request.get("IfMatch")) and etag != if_match.strip('"'):
         precondition_failed = "If-Match"
 
     elif (
         if_unmodified_since := request.get("IfUnmodifiedSince")
-    ) and last_modified > if_unmodified_since:
+    ) and last_modified > second_resolution_datetime(if_unmodified_since):
         precondition_failed = "If-Unmodified-Since"
 
     if precondition_failed:
@@ -960,7 +968,9 @@ def validate_failed_precondition(
 
     if ((if_none_match := request.get("IfNoneMatch")) and etag == if_none_match.strip('"')) or (
         (if_modified_since := request.get("IfModifiedSince"))
-        and last_modified <= if_modified_since < datetime.datetime.now(tz=_gmt_zone_info)
+        and last_modified
+        <= second_resolution_datetime(if_modified_since)
+        < datetime.datetime.now(tz=_gmt_zone_info)
     ):
         raise CommonServiceException(
             message="Not Modified",

localstack_core-4.10.1.dev47/localstack-core/localstack/utils/analytics/publisher.py

@@ -0,0 +1,160 @@
+import abc
+import atexit
+import logging
+import threading
+
+from localstack import config
+from localstack.utils.batching import AsyncBatcher
+from localstack.utils.threads import FuncThread, start_thread, start_worker_thread
+
+from .client import AnalyticsClient
+from .events import Event, EventHandler
+from .metadata import get_client_metadata
+
+LOG = logging.getLogger(__name__)
+
+
+class Publisher(abc.ABC):
+    """
+    A publisher takes a batch of events and publishes them to a destination.
+    """
+
+    def publish(self, events: list[Event]):
+        raise NotImplementedError
+
+    def close(self):
+        pass
+
+
+class AnalyticsClientPublisher(Publisher):
+    client: AnalyticsClient
+
+    def __init__(self, client: AnalyticsClient = None) -> None:
+        super().__init__()
+        self.client = client or AnalyticsClient()
+
+    def publish(self, events: list[Event]):
+        self.client.append_events(events)
+
+    def close(self):
+        self.client.close()
+
+
+class Printer(Publisher):
+    """
+    Publisher that prints serialized events to stdout.
+    """
+
+    def publish(self, events: list[Event]):
+        for event in events:
+            print(event.asdict())
+
+
+class GlobalAnalyticsBus(EventHandler):
+    _batcher: AsyncBatcher[Event]
+    _client: AnalyticsClient
+    _worker_thread: FuncThread | None
+
+    def __init__(self, client: AnalyticsClient = None, flush_size=20, flush_interval=10) -> None:
+        self._client = client or AnalyticsClient()
+        self._publisher = AnalyticsClientPublisher(self._client)
+        self._batcher = AsyncBatcher(
+            self._handle_batch,
+            max_batch_size=flush_size,
+            max_flush_interval=flush_interval,
+        )
+
+        self._started = False
+        self._startup_mutex = threading.Lock()
+        self._worker_thread = None
+
+        self.force_tracking = False  # allow class to ignore all other tracking config
+        self.tracking_disabled = False  # disables tracking if global config would otherwise track
+
+    def _handle_batch(self, batch: list[Event]):
+        """Method that satisfies the BatchHandler[Event] protocol and is passed to AsyncBatcher."""
+        try:
+            self._publisher.publish(batch)
+        except Exception:
+            # currently we're just dropping events if something goes wrong during publishing
+            if config.DEBUG_ANALYTICS:
+                LOG.exception("error while publishing analytics events")
+
+    @property
+    def is_tracking_disabled(self):
+        if self.force_tracking:
+            return False
+
+        # don't track if event tracking is disabled globally
+        if config.DISABLE_EVENTS:
+            return True
+        # don't track for internal test runs (like integration tests)
+        if config.is_local_test_mode():
+            return True
+        if self.tracking_disabled:
+            return True
+
+        return False
+
+    def handle(self, event: Event):
+        """
+        Publish an event to the global analytics event publisher.
+        """
+        if self.is_tracking_disabled:
+            if config.DEBUG_ANALYTICS:
+                LOG.debug("tracking disabled, skipping event %s", event)
+            return
+
+        if not self._started:
+            # we make sure the batching worker is started
+            self._start()
+
+        self._batcher.add(event)
+
+    def _start(self):
+        with self._startup_mutex:
+            if self._started:
+                return
+            self._started = True
+
+            # startup has to run async, otherwise first call to handle() could block a long time.
+            start_worker_thread(self._do_start_retry)
+
+    def _do_start_retry(self, *_):
+        # TODO: actually retry
+        try:
+            if config.DEBUG_ANALYTICS:
+                LOG.debug("trying to register session with analytics backend")
+            response = self._client.start_session(get_client_metadata())
+            if config.DEBUG_ANALYTICS:
+                LOG.debug("session endpoint returned: %s", response)
+
+            if not response.track_events():
+                if config.DEBUG_ANALYTICS:
+                    LOG.debug("gracefully disabling analytics tracking")
+                self.tracking_disabled = True
+
+        except Exception:
+            self.tracking_disabled = True
+            if config.DEBUG_ANALYTICS:
+                LOG.exception("error while registering session. disabling tracking")
+            return
+
+        self._worker_thread = start_thread(self._run, name="global-analytics-bus")
+
+        # given the "Global" nature of this class, we register a global atexit hook to make sure all events are flushed
+        # when localstack shuts down.
+        def _do_close():
+            self.close_sync(timeout=2)
+
+        atexit.register(_do_close)
+
+    def _run(self, *_):
+        # main control loop, simply runs the batcher
+        self._batcher.run()
+
+    def close_sync(self, timeout=None):
+        self._batcher.close()
+
+        if self._worker_thread:
+            self._worker_thread.join(timeout=timeout)

localstack_core-4.10.1.dev47/localstack-core/localstack/utils/batching.py

@@ -0,0 +1,258 @@
+import copy
+import logging
+import threading
+import time
+from typing import Generic, Protocol, TypeVar, overload
+
+LOG = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# alias to signify whether a batch policy has been triggered
+BatchPolicyTriggered = bool
+
+
+# TODO: Add batching on bytes as well.
+class Batcher(Generic[T]):
+    """
+    A utility for collecting items into batches and flushing them when one or more batch policy conditions are met.
+
+    The batch policy can be created to trigger on:
+    - max_count: Maximum number of items added
+    - max_window: Maximum time window (in seconds)
+
+    If no limits are specified, the batcher is always in triggered state.
+
+    Example usage:
+
+        import time
+
+        # Triggers when 2 (or more) items are added
+        batcher = Batcher(max_count=2)
+        assert batcher.add(["item1", "item2", "item3"])
+        assert batcher.flush() == ["item1", "item2", "item3"]
+
+        # Triggers partially when 2 (or more) items are added
+        batcher = Batcher(max_count=2)
+        assert batcher.add(["item1", "item2", "item3"])
+        assert batcher.flush(partial=True) == ["item1", "item2"]
+        assert batcher.add("item4")
+        assert batcher.flush(partial=True) == ["item3", "item4"]
+
+        # Trigger 2 seconds after the first add
+        batcher = Batcher(max_window=2.0)
+        assert not batcher.add(["item1", "item2", "item3"])
+        time.sleep(2.1)
+        assert not batcher.add(["item4"])
+        assert batcher.flush() == ["item1", "item2", "item3", "item4"]
+    """
+
+    max_count: int | None
+    """
+    Maximum number of items, must be None or positive.
+    """
+
+    max_window: float | None
+    """
+    Maximum time window in seconds, must be None or positive.
+    """
+
+    _triggered: bool
+    _last_batch_time: float
+    _batch: list[T]
+
+    def __init__(self, max_count: int | None = None, max_window: float | None = None):
+        """
+        Initialize a new Batcher instance.
+
+        :param max_count: Maximum number of items that be None or positive.
+        :param max_window: Maximum time window in seconds that must be None or positive.
+        """
+        self.max_count = max_count
+        self.max_window = max_window
+
+        self._triggered = False
+        self._last_batch_time = time.monotonic()
+        self._batch = []
+
+    @property
+    def period(self) -> float:
+        return time.monotonic() - self._last_batch_time
+
+    def _check_batch_policy(self) -> bool:
+        """Check if any batch policy conditions are met"""
+        if self.max_count is not None and len(self._batch) >= self.max_count:
+            self._triggered = True
+        elif self.max_window is not None and self.period >= self.max_window:
+            self._triggered = True
+        elif not self.max_count and not self.max_window:
+            # always return true
+            self._triggered = True
+
+        return self._triggered
+
+    @overload
+    def add(self, item: T, *, deep_copy: bool = False) -> BatchPolicyTriggered: ...
+
+    @overload
+    def add(self, items: list[T], *, deep_copy: bool = False) -> BatchPolicyTriggered: ...
+
+    def add(self, item_or_items: T | list[T], *, deep_copy: bool = False) -> BatchPolicyTriggered:
+        """
+        Add an item or list of items to the collected batch.
+
+        Returns:
+            BatchPolicyTriggered: True if the batch policy was triggered during addition, False otherwise.
+        """
+        if deep_copy:
+            item_or_items = copy.deepcopy(item_or_items)
+
+        if isinstance(item_or_items, list):
+            self._batch.extend(item_or_items)
+        else:
+            self._batch.append(item_or_items)
+
+        # Check if the last addition triggered the batch policy
+        return self.is_triggered()
+
+    def flush(self, *, partial=False) -> list[T]:
+        result = []
+        if not partial or not self.max_count:
+            result = self._batch.copy()
+            self._batch.clear()
+        else:
+            batch_size = min(self.max_count, len(self._batch))
+            result = self._batch[:batch_size].copy()
+            self._batch = self._batch[batch_size:]
+
+        self._last_batch_time = time.monotonic()
+        self._triggered = False
+        self._check_batch_policy()
+
+        return result
+
+    def duration_until_next_batch(self) -> float:
+        if not self.max_window:
+            return -1
+        return max(self.max_window - self.period, -1)
+
+    def get_current_size(self) -> int:
+        return len(self._batch)
+
+    def is_triggered(self):
+        return self._triggered or self._check_batch_policy()
+
+
+class BatchHandler(Protocol[T]):
+    """
+    A BatchHandler is a callable that processes a list of items handed down by the AsyncBatcher.
+    """
+
+    def __call__(self, batch: list[T]) -> None: ...
+
+
+class AsyncBatcher(Generic[T]):
+    """
+    Class for managing asynchronous batching of items.
+
+    This class allows for efficient buffering and processing of items in batches by
+    periodically flushing the buffer to a given handler, or by automatically flushing
+    when the maximum batch size is reached. It is designed to be used in asynchronous
+    scenarios where the caller does not execute the flushing IO call itself, like with ``Batcher``.
+
+    :ivar max_flush_interval: Maximum time interval in seconds between
+        automatic flushes, regardless of the batch size.
+    :ivar max_batch_size: Maximum number of items in a batch. When reached,
+        the batch is flushed automatically.
+    :ivar handler: Callable handler that processes each flushed batch. The handler must
+        be provided during initialization and must accept a list of items as input.
+    """
+
+    max_flush_interval: float
+    max_batch_size: int
+    handler: BatchHandler[T]
+
+    _buffer: list[T]
+    _flush_lock: threading.Condition
+    _closed: bool
+
+    def __init__(
+        self,
+        handler: BatchHandler[T],
+        max_flush_interval: float = 10,
+        max_batch_size: int = 20,
+    ):
+        self.handler = handler
+        self.max_flush_interval = max_flush_interval
+        self.max_batch_size = max_batch_size
+
+        self._buffer = []
+        self._flush_lock = threading.Condition()
+        self._closed = False
+
+    def add(self, item: T):
+        """
+        Adds an item to the buffer.
+
+        :param item: the item to add
+        """
+        with self._flush_lock:
+            if self._closed:
+                raise ValueError("Batcher is stopped, can no longer add items")
+
+            self._buffer.append(item)
+
+            if len(self._buffer) >= self.max_batch_size:
+                self._flush_lock.notify_all()
+
+    @property
+    def current_batch_size(self) -> int:
+        """
+        Returns the current number of items in the buffer waiting to be flushed.
+        """
+        return len(self._buffer)
+
+    def run(self):
+        """
+        Runs the event loop that flushes the buffer to the handler based on the configured rules, and blocks until
+        ``close()`` is called. This method is meant to be run in a separate thread.
+        """
+        while not self._closed:
+            with self._flush_lock:
+                # wait returns once either the condition is notified (in which case wait returns True, indicating that
+                # something has triggered a flush manually), or the timeout expires (in which case wait returns False)
+                self._flush_lock.wait(self.max_flush_interval)
+
+                # if _flush_condition was notified because close() was called, we should still make sure we flush the
+                # last batch
+
+                # perform the flush, if there are any items in the buffer
+                if not self._buffer:
+                    continue
+
+                batch = self._buffer.copy()
+                self._buffer.clear()
+
+            # we can call the processor outside the lock so we can continue adding items into the next batch without
+            # waiting on the processor to return.
+            try:
+                self.handler(batch)
+            except Exception as e:
+                LOG.error(
+                    "Unhandled exception while processing a batch: %s",
+                    e,
+                    exc_info=LOG.isEnabledFor(logging.DEBUG),
+                )
+
+        # this marks that the main control loop is done
+        return
+
+    def close(self):
+        """
+        Triggers a close of the batcher, which will cause one last flush, and then end the main event loop.
+        """
+        with self._flush_lock:
+            if self._closed:
+                return
+            self._closed = True
+            self._flush_lock.notify_all()

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack/utils/collections.py

@@ -5,7 +5,7 @@ and manipulate python collection (dicts, list, sets).
 
 import logging
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sized
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sized
 from typing import (
     Any,
     Optional,
@@ -24,6 +24,9 @@ LOG = logging.getLogger(__name__)
 # default regex to match an item in a comma-separated list string
 DEFAULT_REGEX_LIST_ITEM = r"[\w-]+"
 
+_E = TypeVar("_E")
+"""TypeVar var used internally for container type parameters."""
+
 
 class AccessTrackingDict(dict):
     """
@@ -101,21 +104,18 @@ class HashableJsonDict(ImmutableDict):
         return hash(canonical_json(self._dict))
 
 
-_ListType = TypeVar("_ListType")
-
-
-class PaginatedList(list[_ListType]):
+class PaginatedList(list[_E]):
     """List which can be paginated and filtered. For usage in AWS APIs with paginated responses"""
 
     DEFAULT_PAGE_SIZE = 50
 
     def get_page(
         self,
-        token_generator: Callable[[_ListType], str],
+        token_generator: Callable[[_E], str],
         next_token: str = None,
         page_size: int = None,
-        filter_function: Callable[[_ListType], bool] = None,
-    ) -> tuple[list[_ListType], str | None]:
+        filter_function: Callable[[_E], bool] = None,
+    ) -> tuple[list[_E], str | None]:
         if filter_function is not None:
             result_list = list(filter(filter_function, self))
         else:
@@ -528,9 +528,6 @@ def is_comma_delimited_list(string: str, item_regex: str | None = None) -> bool:
     return True
 
 
-_E = TypeVar("_E")
-
-
 def optional_list(condition: bool, items: Iterable[_E]) -> list[_E]:
     """
     Given an iterable, either create a list out of the entire iterable (if `condition` is `True`), or return the empty list.
@@ -540,3 +537,18 @@ def optional_list(condition: bool, items: Iterable[_E]) -> list[_E]:
     []
     """
     return list(filter(lambda _: condition, items))
+
+
+def iter_chunks(items: list[_E], chunk_size: int) -> Generator[list[_E], None, None]:
+    """
+    Split a list into smaller chunks of a specified size and iterate over them.
+
+    It is implemented as a generator and yields each chunk as needed, making it memory-efficient for large lists.
+
+    :param items:  A list of elements to be divided into chunks.
+    :param chunk_size: The maximum number of elements that a single chunk can contain.
+    :return: A generator that yields chunks (sublists) of the original list. Each chunk contains up to `chunk_size`
+    elements.
+    """
+    for i in range(0, len(items), chunk_size):
+        yield items[i : i + chunk_size]

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '4.10.1.dev44'
-__version_tuple__ = version_tuple = (4, 10, 1, 'dev44')
+__version__ = version = '4.10.1.dev47'
+__version_tuple__ = version_tuple = (4, 10, 1, 'dev47')
 
-__commit_id__ = commit_id = 'gae5c68554'
+__commit_id__ = commit_id = 'g02ce6a22a'

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localstack-core
-Version: 4.10.1.dev44
+Version: 4.10.1.dev47
 Summary: The core library and runtime of LocalStack
 Author-email: LocalStack Contributors <info@localstack.cloud>
 License-Expression: Apache-2.0

{localstack_core-4.10.1.dev44 → localstack_core-4.10.1.dev47}/localstack-core/localstack_core.egg-info/SOURCES.txt

@@ -1252,7 +1252,7 @@ localstack-core/localstack/utils/async_utils.py
 localstack-core/localstack/utils/asyncio.py
 localstack-core/localstack/utils/auth.py
 localstack-core/localstack/utils/backoff.py
-localstack-core/localstack/utils/batch_policy.py
+localstack-core/localstack/utils/batching.py
 localstack-core/localstack/utils/bootstrap.py
 localstack-core/localstack/utils/checksum.py
 localstack-core/localstack/utils/collections.py