localstack-core 4.9.3.dev25__py3-none-any.whl → 4.9.3.dev27__py3-none-any.whl

localstack/aws/handlers/metric_handler.py

@@ -1,9 +1,15 @@
+import csv
 import logging
+import os
+from datetime import datetime
+from pathlib import Path
 
 from localstack import config
 from localstack.aws.api import RequestContext
 from localstack.aws.chain import HandlerChain
+from localstack.constants import ENV_INTERNAL_TEST_STORE_METRICS_PATH
 from localstack.http import Response
+from localstack.utils.strings import short_uid
 
 LOG = logging.getLogger(__name__)
 
@@ -137,6 +143,32 @@ class MetricHandler:
 
     def __init__(self) -> None:
         self.metrics_handler_items = {}
+        self.local_filename = None
+
+        if self.should_store_metric_locally():
+            self.local_filename = self.create_local_file()
+
+    @staticmethod
+    def should_store_metric_locally() -> bool:
+        return config.is_collect_metrics_mode() and config.store_test_metrics_in_local_filesystem()
+
+    @staticmethod
+    def create_local_file():
+        folder = Path(
+            os.environ.get(ENV_INTERNAL_TEST_STORE_METRICS_PATH, "/tmp/localstack-metrics")
+        )
+        if not folder.exists():
+            folder.mkdir(parents=True, exist_ok=True)
+        LOG.debug("Metric reports will be stored in %s", folder)
+        filename = (
+            folder
+            / f"metric-report-raw-data-{datetime.utcnow().strftime('%Y-%m-%d__%H_%M_%S')}-{short_uid()}.csv"
+        )
+        with open(filename, "w") as fd:
+            LOG.debug("Creating new metric data file %s", filename)
+            writer = csv.writer(fd)
+            writer.writerow(Metric.RAW_DATA_HEADER)
+        return filename
 
     def create_metric_handler_item(
         self, chain: HandlerChain, context: RequestContext, response: Response
@@ -194,7 +226,15 @@ class MetricHandler:
         )
         # refrain from adding duplicates
         if metric not in MetricHandler.metric_data:
-            MetricHandler.metric_data.append(metric)
+            self.append_metric(metric)
 
         # cleanup
         del self.metrics_handler_items[context]
+
+    def append_metric(self, metric: Metric):
+        if self.should_store_metric_locally():
+            with open(self.local_filename, "a") as fd:
+                writer = csv.writer(fd)
+                writer.writerow(metric)
+        else:
+            MetricHandler.metric_data.append(metric)

localstack/cli/exceptions.py

@@ -12,7 +12,7 @@ class CLIError(ClickException):
     def format_message(self) -> str:
         return click.style(f"❌ Error: {self.message}", fg="red")
 
-    def show(self, file: t.Optional[t.IO[t.Any]] = None) -> None:
+    def show(self, file: t.IO[t.Any] | None = None) -> None:
         if file is None:
             file = get_text_stderr()
 

localstack/cli/localstack.py

@@ -3,7 +3,7 @@ import logging
 import os
 import sys
 import traceback
-from typing import Optional, TypedDict
+from typing import TypedDict
 
 import click
 import requests
@@ -320,8 +320,8 @@ class DockerStatus(TypedDict, total=False):
     image_tag: str
     image_id: str
     image_created: str
-    container_name: Optional[str]
-    container_ip: Optional[str]
+    container_name: str | None
+    container_ip: str | None
 
 
 def _print_docker_status(format_: str) -> None:
@@ -699,7 +699,7 @@ def cmd_logs(follow: bool, tail: int) -> None:
     metavar="N",
 )
 @publish_invocation
-def cmd_wait(timeout: Optional[float] = None) -> None:
+def cmd_wait(timeout: float | None = None) -> None:
     """
     Wait for the LocalStack runtime to be up and running.
 

localstack/cli/lpm.py

@@ -1,7 +1,6 @@
 import itertools
 import logging
 from multiprocessing.pool import ThreadPool
-from typing import Optional
 
 import click
 from rich.console import Console
@@ -75,9 +74,9 @@ def _do_install_package(package: Package, version: str = None, target: InstallTa
 )
 def install(
     package: list[str],
-    parallel: Optional[int] = 1,
-    version: Optional[str] = None,
-    target: Optional[str] = None,
+    parallel: int | None = 1,
+    version: str | None = None,
+    target: str | None = None,
 ):
     """Install one or more packages."""
     try:

localstack/cli/profiles.py

@@ -1,7 +1,6 @@
 import argparse
 import os
 import sys
-from typing import Optional
 
 # important: this needs to be free of localstack imports
 
@@ -44,7 +43,7 @@ def set_and_remove_profile_from_sys_argv():
         os.environ["CONFIG_PROFILE"] = profile.strip()
 
 
-def parse_p_argument(args) -> Optional[str]:
+def parse_p_argument(args) -> str | None:
     """
     Lightweight arg parsing to find the first occurrence of ``-p <config>``, or ``-p=<config>`` and return the value of
     ``<config>`` from the given arguments.

localstack/config.py

@@ -10,7 +10,7 @@ import time
 import warnings
 from collections import defaultdict
 from collections.abc import Mapping
-from typing import Any, Optional, TypeVar, Union
+from typing import Any, TypeVar
 
 from localstack import constants
 from localstack.constants import (
@@ -19,6 +19,7 @@ from localstack.constants import (
     DEFAULT_VOLUME_DIR,
     ENV_INTERNAL_TEST_COLLECT_METRIC,
     ENV_INTERNAL_TEST_RUN,
+    ENV_INTERNAL_TEST_STORE_METRICS_IN_LOCALSTACK,
     FALSE_STRINGS,
     LOCALHOST,
     LOCALHOST_IP,
@@ -208,13 +209,13 @@ class Directories:
         return str(self.__dict__)
 
 
-def eval_log_type(env_var_name: str) -> Union[str, bool]:
+def eval_log_type(env_var_name: str) -> str | bool:
     """Get the log type from environment variable"""
     ls_log = os.environ.get(env_var_name, "").lower().strip()
     return ls_log if ls_log in LOG_LEVELS else False
 
 
-def parse_boolean_env(env_var_name: str) -> Optional[bool]:
+def parse_boolean_env(env_var_name: str) -> bool | None:
     """Parse the value of the given env variable and return True/False, or None if it is not a boolean value."""
     value = os.environ.get(env_var_name, "").lower().strip()
     if value in TRUE_STRINGS:
@@ -649,7 +650,7 @@ class UniqueHostAndPortList(list[HostAndPort]):
         - Identical identical hosts and ports are de-duped
     """
 
-    def __init__(self, iterable: Union[list[HostAndPort], None] = None):
+    def __init__(self, iterable: list[HostAndPort] | None = None):
         super().__init__(iterable or [])
         self._ensure_unique()
 
@@ -1451,6 +1452,11 @@ def is_collect_metrics_mode() -> bool:
     return is_env_true(ENV_INTERNAL_TEST_COLLECT_METRIC)
 
 
+def store_test_metrics_in_local_filesystem() -> bool:
+    """Returns True if test metrics should be stored in the local filesystem (instead of the system that runs pytest)."""
+    return is_env_true(ENV_INTERNAL_TEST_STORE_METRICS_IN_LOCALSTACK)
+
+
 def collect_config_items() -> list[tuple[str, Any]]:
     """Returns a list of key-value tuples of LocalStack configuration values."""
     none = object()  # sentinel object
@@ -1503,10 +1509,10 @@ def get_protocol() -> str:
 
 
 def external_service_url(
-    host: Optional[str] = None,
-    port: Optional[int] = None,
-    protocol: Optional[str] = None,
-    subdomains: Optional[str] = None,
+    host: str | None = None,
+    port: int | None = None,
+    protocol: str | None = None,
+    subdomains: str | None = None,
 ) -> str:
     """Returns a service URL (e.g., SQS queue URL) to an external client (e.g., boto3) potentially running on another
     machine than LocalStack. The configurations LOCALSTACK_HOST and USE_SSL can customize these returned URLs.
@@ -1523,10 +1529,10 @@ def external_service_url(
 
 
 def internal_service_url(
-    host: Optional[str] = None,
-    port: Optional[int] = None,
-    protocol: Optional[str] = None,
-    subdomains: Optional[str] = None,
+    host: str | None = None,
+    port: int | None = None,
+    protocol: str | None = None,
+    subdomains: str | None = None,
 ) -> str:
     """Returns a service URL for internal use within LocalStack (i.e., same host).
     The configuration USE_SSL can customize these returned URLs but LOCALSTACK_HOST has no effect.

localstack/constants.py

@@ -80,6 +80,10 @@ ENV_INTERNAL_TEST_RUN = "LOCALSTACK_INTERNAL_TEST_RUN"
 # environment variable name to tag collect metrics during a test run
 ENV_INTERNAL_TEST_COLLECT_METRIC = "LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC"
 
+# environment variable name to indicate that metrics should be stored within the container
+ENV_INTERNAL_TEST_STORE_METRICS_IN_LOCALSTACK = "LOCALSTACK_INTERNAL_TEST_METRICS_IN_LOCALSTACK"
+ENV_INTERNAL_TEST_STORE_METRICS_PATH = "LOCALSTACK_INTERNAL_TEST_STORE_METRICS_PATH"
+
 # environment variable that flags whether pro was activated. do not use it for security purposes!
 ENV_PRO_ACTIVATED = "PRO_ACTIVATED"
 

localstack/packages/api.py

@@ -3,10 +3,11 @@ import functools
 import logging
 import os
 from collections import defaultdict
+from collections.abc import Callable
 from enum import Enum
 from inspect import getmodule
-from threading import RLock
-from typing import Any, Callable, Generic, Optional, ParamSpec, TypeVar
+from threading import Lock, RLock
+from typing import Any, Generic, ParamSpec, TypeVar
 
 from plux import Plugin, PluginManager, PluginSpec  # type: ignore
 
@@ -56,7 +57,7 @@ class PackageInstaller(abc.ABC):
     multiple versions).
     """
 
-    def __init__(self, name: str, version: str, install_lock: Optional[RLock] = None):
+    def __init__(self, name: str, version: str, install_lock: Lock | None = None):
         """
         :param name: technical package name, f.e. "opensearch"
         :param version: version of the package to install
@@ -70,7 +71,7 @@ class PackageInstaller(abc.ABC):
         self.install_lock = install_lock or RLock()
         self._setup_for_target: dict[InstallTarget, bool] = defaultdict(lambda: False)
 
-    def install(self, target: Optional[InstallTarget] = None) -> None:
+    def install(self, target: InstallTarget | None = None) -> None:
         """
         Performs the package installation.
 
@@ -210,7 +211,7 @@ class Package(abc.ABC, Generic[T]):
         """
         return self.get_installer(version).get_installed_dir()
 
-    def install(self, version: str | None = None, target: Optional[InstallTarget] = None) -> None:
+    def install(self, version: str | None = None, target: InstallTarget | None = None) -> None:
         """
         Installs the package in the given version in the preferred target location.
         :param version: version of the package to install. If None, the default version of the package will be used.
@@ -274,7 +275,7 @@ class MultiPackageInstaller(PackageInstaller):
         assert len(package_installer) > 0
         self.package_installer = package_installer
 
-    def install(self, target: Optional[InstallTarget] = None) -> None:
+    def install(self, target: InstallTarget | None = None) -> None:
         """
         Installs the different packages this installer is composed of.
 
@@ -356,7 +357,7 @@ class PackagesPluginManager(PluginManager[PackagesPlugin]):  # type: ignore[misc
         )
 
     def get_packages(
-        self, package_names: list[str], version: Optional[str] = None
+        self, package_names: list[str], version: str | None = None
     ) -> list[Package[PackageInstaller]]:
         # Plugin names are unique, but there could be multiple packages with the same name in different scopes
         plugin_specs_per_name = defaultdict(list)
@@ -390,7 +391,7 @@ T2 = TypeVar("T2")
 def package(
     name: str | None = None,
     scope: str = "community",
-    should_load: Optional[Callable[[], bool]] = None,
+    should_load: Callable[[], bool] | None = None,
 ) -> Callable[[Callable[[], Package[Any] | list[Package[Any]]]], PluginSpec]:
     """
     Decorator for marking methods that create Package instances as a PackagePlugin.

localstack/packages/core.py

@@ -4,7 +4,7 @@ import re
 from abc import ABC
 from functools import lru_cache
 from sys import version_info
-from typing import Any, Optional
+from typing import Any
 
 import requests
 
@@ -238,7 +238,7 @@ class NodePackageInstaller(ExecutableInstaller):
         self,
         package_name: str,
         version: str,
-        package_spec: Optional[str] = None,
+        package_spec: str | None = None,
         main_module: str = "main.js",
     ):
         """

localstack/testing/pytest/cloudformation/fixtures.py

@@ -1,7 +1,7 @@
 import json
 from collections import defaultdict
-from collections.abc import Generator
-from typing import Callable, Optional, TypedDict
+from collections.abc import Callable, Generator
+from typing import TypedDict
 
 import pytest
 from botocore.exceptions import WaiterError
@@ -13,7 +13,7 @@ from localstack.utils.strings import short_uid
 
 
 class NormalizedEvent(TypedDict):
-    PhysicalResourceId: Optional[str]
+    PhysicalResourceId: str | None
     LogicalResourceId: str
     ResourceType: str
     ResourceStatus: str

localstack/testing/pytest/container.py

@@ -2,8 +2,7 @@ import logging
 import os
 import shlex
 import threading
-from collections.abc import Generator
-from typing import Callable, Optional
+from collections.abc import Callable, Generator
 
 import pytest
 
@@ -38,8 +37,8 @@ class ContainerFactory:
         self,
         # convenience properties
         pro: bool = False,
-        publish: Optional[list[int]] = None,
-        configurators: Optional[list[ContainerConfigurator]] = None,
+        publish: list[int] | None = None,
+        configurators: list[ContainerConfigurator] | None = None,
         # ContainerConfig properties
         **kwargs,
     ) -> Container:
@@ -172,7 +171,7 @@ def container_factory() -> Generator[ContainerFactory, None, None]:
 
 @pytest.fixture(scope="session")
 def wait_for_localstack_ready():
-    def _wait_for(container: RunningContainer, timeout: Optional[float] = None):
+    def _wait_for(container: RunningContainer, timeout: float | None = None):
         container.wait_until_ready(timeout)
 
         poll_condition(

localstack/testing/pytest/fixtures.py

@@ -6,7 +6,8 @@ import os
 import re
 import textwrap
 import time
-from typing import TYPE_CHECKING, Any, Callable, Optional, Unpack
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, Unpack
 
 import botocore.auth
 import botocore.config
@@ -357,8 +358,8 @@ def sqs_create_queue(aws_client):
 def sqs_receive_messages_delete(aws_client):
     def factory(
         queue_url: str,
-        expected_messages: Optional[int] = None,
-        wait_time: Optional[int] = 5,
+        expected_messages: int | None = None,
+        wait_time: int | None = 5,
     ):
         response = aws_client.sqs.receive_message(
             QueueUrl=queue_url,
@@ -706,7 +707,7 @@ def route53_hosted_zone(aws_client):
 def transcribe_create_job(s3_bucket, aws_client):
     job_names = []
 
-    def _create_job(audio_file: str, params: Optional[dict[str, Any]] = None) -> str:
+    def _create_job(audio_file: str, params: dict[str, Any] | None = None) -> str:
         s3_key = "test-clip.wav"
 
         if not params:
@@ -1085,18 +1086,18 @@ def deploy_cfn_template(
 
     def _deploy(
         *,
-        is_update: Optional[bool] = False,
-        stack_name: Optional[str] = None,
-        change_set_name: Optional[str] = None,
-        template: Optional[str] = None,
-        template_path: Optional[str | os.PathLike] = None,
-        template_mapping: Optional[dict[str, Any]] = None,
-        parameters: Optional[dict[str, str]] = None,
-        role_arn: Optional[str] = None,
-        max_wait: Optional[int] = None,
-        delay_between_polls: Optional[int] = 2,
-        custom_aws_client: Optional[ServiceLevelClientFactory] = None,
-        raw_parameters: Optional[list[Parameter]] = None,
+        is_update: bool | None = False,
+        stack_name: str | None = None,
+        change_set_name: str | None = None,
+        template: str | None = None,
+        template_path: str | os.PathLike | None = None,
+        template_mapping: dict[str, Any] | None = None,
+        parameters: dict[str, str] | None = None,
+        role_arn: str | None = None,
+        max_wait: int | None = None,
+        delay_between_polls: int | None = 2,
+        custom_aws_client: ServiceLevelClientFactory | None = None,
+        raw_parameters: list[Parameter] | None = None,
     ) -> DeployResult:
         if is_update:
             assert stack_name
@@ -1262,7 +1263,7 @@ def _has_stack_status(cfn_client, statuses: list[str]):
 
 @pytest.fixture
 def is_change_set_finished(aws_client):
-    def _is_change_set_finished(change_set_id: str, stack_name: Optional[str] = None):
+    def _is_change_set_finished(change_set_id: str, stack_name: str | None = None):
         def _inner():
             kwargs = {"ChangeSetName": change_set_id}
             if stack_name:
@@ -1993,7 +1994,7 @@ def setup_sender_email_address(ses_verify_identity):
     email address and verify them.
     """
 
-    def inner(sender_email_address: Optional[str] = None) -> str:
+    def inner(sender_email_address: str | None = None) -> str:
         if is_aws_cloud():
             if sender_email_address is None:
                 raise ValueError(
@@ -2252,7 +2253,7 @@ def assert_host_customisation(monkeypatch):
     def asserter(
         url: str,
         *,
-        custom_host: Optional[str] = None,
+        custom_host: str | None = None,
     ):
         if custom_host is not None:
             assert custom_host in url, f"Could not find `{custom_host}` in `{url}`"

localstack/testing/pytest/marking.py

@@ -3,7 +3,8 @@ Custom pytest mark typings
 """
 
 import os
-from typing import TYPE_CHECKING, Callable, Optional
+from collections.abc import Callable
+from typing import TYPE_CHECKING
 
 import pytest
 from _pytest.config import PytestPluginManager
@@ -36,13 +37,13 @@ class SkipSnapshotVerifyMarker:
     def __call__(
         self,
         *,
-        paths: "Optional[list[str]]" = None,
-        condition: "Optional[Callable[[...], bool]]" = None,
+        paths: "list[str] | None" = None,
+        condition: "Callable[[...], bool] | None" = None,
     ): ...
 
 
 class MultiRuntimeMarker:
-    def __call__(self, *, scenario: str, runtimes: Optional[list[str]] = None): ...
+    def __call__(self, *, scenario: str, runtimes: list[str] | None = None): ...
 
 
 class SnapshotMarkers:

localstack/testing/pytest/stepfunctions/utils.py

@@ -1,6 +1,7 @@
 import json
 import logging
-from typing import Callable, Final, Optional
+from collections.abc import Callable
+from typing import Final
 
 from botocore.exceptions import ClientError
 from jsonpath_ng.ext import parse
@@ -402,8 +403,8 @@ def create_state_machine_with_iam_role(
     create_state_machine,
     snapshot,
     definition: Definition,
-    logging_configuration: Optional[LoggingConfiguration] = None,
-    state_machine_name: Optional[str] = None,
+    logging_configuration: LoggingConfiguration | None = None,
+    state_machine_name: str | None = None,
     state_machine_type: StateMachineType = StateMachineType.STANDARD,
 ):
     snf_role_arn = create_state_machine_iam_role(target_aws_client=target_aws_client)

localstack/testing/pytest/util.py

@@ -1,7 +1,7 @@
 import os
 import pwd
+from collections.abc import Callable
 from multiprocessing import Process, ProcessError
-from typing import Callable
 
 
 def run_as_os_user(target: Callable, uid: str | int, gid: str | int = None):

localstack/testing/pytest/validation_tracking.py

@@ -9,7 +9,6 @@ import datetime
 import json
 import os
 from pathlib import Path
-from typing import Optional
 
 import pytest
 from pluggy import Result
@@ -28,7 +27,7 @@ Stores information from call execution phase about whether the test failed.
 """
 
 
-def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]:
+def find_validation_data_for_item(item: pytest.Item) -> dict | None:
     base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
     snapshot_path = f"{base_path}.validation.json"
 

localstack/utils/analytics/events.py

@@ -1,8 +1,8 @@
 import abc
 import dataclasses
-from typing import Any, Union
+from typing import Any
 
-EventPayload = Union[dict[str, Any], Any]  # FIXME: better typing
+EventPayload = dict[str, Any] | Any  # FIXME: better typing
 
 
 @dataclasses.dataclass

localstack/utils/analytics/metadata.py

@@ -2,7 +2,6 @@ import dataclasses
 import logging
 import os
 import platform
-from typing import Optional
 
 from localstack import config
 from localstack.constants import VERSION
@@ -201,7 +200,7 @@ def _generate_machine_id() -> str:
     return f"gen_{long_uid()[:12]}"
 
 
-def get_api_key_or_auth_token() -> Optional[str]:
+def get_api_key_or_auth_token() -> str | None:
     # TODO: this is duplicated code from ext, but should probably migrate that to localstack
     auth_token = os.environ.get("LOCALSTACK_AUTH_TOKEN", "").strip("'\" ")
     if auth_token:

localstack/utils/analytics/metrics/counter.py

@@ -1,7 +1,7 @@
 import threading
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Optional, Union
+from typing import Any
 
 from localstack import config
 
@@ -38,7 +38,7 @@ class LabeledCounterPayload:
     value: int
     type: str
     schema_version: int
-    labels: dict[str, Union[str, float]]
+    labels: dict[str, str | float]
 
     def as_dict(self) -> dict[str, Any]:
         payload_dict = {
@@ -140,10 +140,8 @@ class LabeledCounter(Metric):
 
     _type: str
     _labels: list[str]
-    _label_values: tuple[Optional[Union[str, float]], ...]
-    _counters_by_label_values: defaultdict[
-        tuple[Optional[Union[str, float]], ...], ThreadSafeCounter
-    ]
+    _label_values: tuple[str | float | None, ...]
+    _counters_by_label_values: defaultdict[tuple[str | float | None, ...], ThreadSafeCounter]
 
     def __init__(self, namespace: str, name: str, labels: list[str], schema_version: int = 1):
         super().__init__(namespace=namespace, name=name, schema_version=schema_version)
@@ -162,7 +160,7 @@ class LabeledCounter(Metric):
         self._counters_by_label_values = defaultdict(ThreadSafeCounter)
         MetricRegistry().register(self)
 
-    def labels(self, **kwargs: Union[str, float, None]) -> ThreadSafeCounter:
+    def labels(self, **kwargs: str | float | None) -> ThreadSafeCounter:
         """
         Create a scoped counter instance with specific label values.
 
@@ -198,7 +196,7 @@ class LabeledCounter(Metric):
                 )
 
             # Create labels dictionary
-            labels_dict = dict(zip(self._labels, label_values))
+            labels_dict = dict(zip(self._labels, label_values, strict=False))
 
             payload.append(
                 LabeledCounterPayload(

localstack/utils/analytics/publisher.py

@@ -4,7 +4,6 @@ import logging
 import threading
 import time
 from queue import Full, Queue
-from typing import Optional
 
 from localstack import config
 from localstack.utils.threads import start_thread, start_worker_thread
@@ -93,7 +92,7 @@ class PublisherBuffer(EventHandler):
         self._stopping.set()
         self._command_queue.put(self._cmd_stop)
 
-    def close_sync(self, timeout: Optional[float] = None):
+    def close_sync(self, timeout: float | None = None):
         self.close()
         return self._stopped.wait(timeout)
 

localstack/utils/analytics/service_request_aggregator.py

@@ -2,7 +2,7 @@ import datetime
 import logging
 import threading
 from collections import Counter
-from typing import NamedTuple, Optional
+from typing import NamedTuple
 
 from localstack import config
 from localstack.runtime.shutdown import SHUTDOWN_HANDLERS
@@ -20,7 +20,7 @@ class ServiceRequestInfo(NamedTuple):
     service: str
     operation: str
     status_code: int
-    err_type: Optional[str] = None
+    err_type: str | None = None
 
 
 class ServiceRequestAggregator: