playground-ls-cli 4.14.1.dev8__py3-none-any.whl

localstack_cli/constants.py

@@ -0,0 +1,165 @@
+import os
+
+from localstack_cli.version import __version__
+
+VERSION = __version__
+
+# HTTP headers used to forward proxy request URLs
+HEADER_LOCALSTACK_EDGE_URL = "x-localstack-edge"
+HEADER_LOCALSTACK_REQUEST_URL = "x-localstack-request-url"
+# HTTP header optionally added to LocalStack responses
+HEADER_LOCALSTACK_IDENTIFIER = "x-localstack"
+# xXx custom localstack authorization header only used in ext
+HEADER_LOCALSTACK_AUTHORIZATION = "x-localstack-authorization"
+HEADER_LOCALSTACK_TARGET = "x-localstack-target"
+HEADER_AMZN_ERROR_TYPE = "X-Amzn-Errortype"
+
+# backend service ports, for services that are behind a proxy (counting down from 4566)
+DEFAULT_PORT_EDGE = 4566
+
+# host name for localhost
+LOCALHOST = "localhost"
+LOCALHOST_IP = "127.0.0.1"
+LOCALHOST_HOSTNAME = "localhost.localstack.cloud"
+
+# User-agent string used in outgoing HTTP requests made by LocalStack
+USER_AGENT_STRING = f"localstack/{VERSION}"
+
+# version of the Maven dependency with Java utility code
+LOCALSTACK_MAVEN_VERSION = "0.2.21"
+MAVEN_REPO_URL = "https://repo1.maven.org/maven2"
+
+# URL of localstack's artifacts repository on GitHub
+ARTIFACTS_REPO = "https://github.com/localstack/localstack-artifacts"
+
+# Artifacts endpoint
+ASSETS_ENDPOINT = "https://assets.localstack.cloud"
+
+# Hugging Face endpoint for localstack
+HUGGING_FACE_ENDPOINT = "https://huggingface.co/localstack"
+
+# host to bind to when starting the services
+BIND_HOST = "0.0.0.0"
+
+# root code folder
+MODULE_MAIN_PATH = os.path.dirname(os.path.realpath(__file__))
+# TODO rename to "ROOT_FOLDER"!
+LOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(MODULE_MAIN_PATH, ".."))
+
+# virtualenv folder
+LOCALSTACK_VENV_FOLDER: str = os.environ.get("VIRTUAL_ENV")
+if not LOCALSTACK_VENV_FOLDER:
+    # fallback to the previous logic
+    LOCALSTACK_VENV_FOLDER = os.path.join(LOCALSTACK_ROOT_FOLDER, ".venv")
+    if not os.path.isdir(LOCALSTACK_VENV_FOLDER):
+        # assuming this package lives here: <python>/lib/pythonX.X/site-packages/localstack/
+        LOCALSTACK_VENV_FOLDER = os.path.realpath(
+            os.path.join(LOCALSTACK_ROOT_FOLDER, "..", "..", "..")
+        )
+
+# default volume directory containing shared data
+DEFAULT_VOLUME_DIR = "/var/lib/localstack"
+
+# API Gateway path to indicate a user request sent to the gateway
+PATH_USER_REQUEST = "_user_request_"
+
+# name of LocalStack Docker image
+DOCKER_IMAGE_NAME_PRO = "localstack/localstack-pro"
+
+# backdoor API path used to retrieve or update config variables
+CONFIG_UPDATE_PATH = "/?_config_"
+
+# API path for localstack internal resources
+INTERNAL_RESOURCE_PATH = "/_localstack"
+
+# environment variable name to tag local test runs
+ENV_INTERNAL_TEST_RUN = "LOCALSTACK_INTERNAL_TEST_RUN"
+
+# environment variable name to tag collect metrics during a test run
+ENV_INTERNAL_TEST_COLLECT_METRIC = "LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC"
+
+# environment variable name to indicate that metrics should be stored within the container
+ENV_INTERNAL_TEST_STORE_METRICS_IN_LOCALSTACK = "LOCALSTACK_INTERNAL_TEST_METRICS_IN_LOCALSTACK"
+ENV_INTERNAL_TEST_STORE_METRICS_PATH = "LOCALSTACK_INTERNAL_TEST_STORE_METRICS_PATH"
+
+# environment variable that flags whether pro was activated. do not use it for security purposes!
+ENV_PRO_ACTIVATED = "PRO_ACTIVATED"
+
+# content types / encodings
+HEADER_CONTENT_TYPE = "Content-Type"
+TEXT_XML = "text/xml"
+APPLICATION_AMZ_JSON_1_0 = "application/x-amz-json-1.0"
+APPLICATION_AMZ_JSON_1_1 = "application/x-amz-json-1.1"
+APPLICATION_AMZ_CBOR_1_1 = "application/x-amz-cbor-1.1"
+APPLICATION_CBOR = "application/cbor"
+APPLICATION_JSON = "application/json"
+APPLICATION_XML = "application/xml"
+APPLICATION_OCTET_STREAM = "application/octet-stream"
+APPLICATION_X_WWW_FORM_URLENCODED = "application/x-www-form-urlencoded"
+HEADER_ACCEPT_ENCODING = "Accept-Encoding"
+
+# strings to indicate truthy/falsy values
+TRUE_STRINGS = ("1", "true", "True")
+FALSE_STRINGS = ("0", "false", "False")
+# strings with valid log levels for LS_LOG
+LOG_LEVELS = ("trace-internal", "trace", "debug", "info", "warn", "error", "warning")
+
+# API endpoint for analytics events
+API_ENDPOINT = os.environ.get("API_ENDPOINT") or "https://api.localstack.cloud/v1"
+# new analytics API endpoint
+ANALYTICS_API = os.environ.get("ANALYTICS_API") or "https://analytics.localstack.cloud/v1"
+
+# environment variable to indicate this process should run the localstack infrastructure
+LOCALSTACK_INFRA_PROCESS = "LOCALSTACK_INFRA_PROCESS"
+
+# AWS region us-east-1
+AWS_REGION_US_EAST_1 = "us-east-1"
+
+# AWS region eu-west-1
+AWS_REGION_EU_WEST_1 = "eu-west-1"
+
+# environment variable to override max pool connections
+try:
+    MAX_POOL_CONNECTIONS = int(os.environ["MAX_POOL_CONNECTIONS"])
+except Exception:
+    MAX_POOL_CONNECTIONS = 150
+
+# Fallback Account ID if not available in the client request
+DEFAULT_AWS_ACCOUNT_ID = "000000000000"
+
+# Credentials used for internal calls
+INTERNAL_AWS_ACCESS_KEY_ID = "__internal_call__"
+INTERNAL_AWS_SECRET_ACCESS_KEY = "__internal_call__"
+
+# trace log levels (excluding/including internal API calls), configurable via $LS_LOG
+LS_LOG_TRACE = "trace"
+LS_LOG_TRACE_INTERNAL = "trace-internal"
+TRACE_LOG_LEVELS = [LS_LOG_TRACE, LS_LOG_TRACE_INTERNAL]
+
+# list of official docker images
+OFFICIAL_IMAGES = [
+    "localstack/localstack",
+    "localstack/localstack-pro",
+    "localstack/snowflake"
+]
+
+# port for debug py
+DEFAULT_DEVELOP_PORT = 5678
+
+# Default bucket name of the s3 bucket used for local lambda development
+# This name should be accepted by all IaC tools, so should respect s3 bucket naming conventions
+DEFAULT_BUCKET_MARKER_LOCAL = "hot-reload"
+LEGACY_DEFAULT_BUCKET_MARKER_LOCAL = "__local__"
+
+# output string that indicates that the stack is ready
+READY_MARKER_OUTPUT = "Ready."
+
+# Regex for `Credential` field in the Authorization header in AWS signature version v4
+# The format is as follows:
+#   Credential=<access-key-id>/<date>/<region-name>/<service-name>/aws4_request
+# eg.
+#   Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request
+AUTH_CREDENTIAL_REGEX = r"Credential=(?P<access_key_id>[a-zA-Z0-9-_.]{1,})/(?P<date>\d{8})/(?P<region_name>[a-z0-9-]{1,})/(?P<service_name>[a-z0-9]{1,})/"
+
+# Custom resource tag to override the generated resource ID.
+TAG_KEY_CUSTOM_ID = "_custom_id_"

localstack_cli/logging/format.py

@@ -0,0 +1,194 @@
+"""Tools for formatting localstack logs."""
+
+import logging
+import re
+from functools import lru_cache
+from typing import Any
+
+from localstack_cli.utils.numbers import format_bytes
+from localstack_cli.utils.strings import to_bytes
+
+MAX_THREAD_NAME_LEN = 12
+MAX_NAME_LEN = 26
+
+LOG_FORMAT = f"%(asctime)s.%(msecs)03d %(ls_level)5s --- [%(ls_thread){MAX_THREAD_NAME_LEN}s] %(ls_name)-{MAX_NAME_LEN}s : %(message)s"
+LOG_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
+LOG_INPUT_FORMAT = "%(input_type)s(%(input)s, headers=%(request_headers)s)"
+LOG_OUTPUT_FORMAT = "%(output_type)s(%(output)s, headers=%(response_headers)s)"
+LOG_CONTEXT_FORMAT = "%(account_id)s/%(region)s"
+
+CUSTOM_LEVEL_NAMES = {
+    50: "FATAL",
+    40: "ERROR",
+    30: "WARN",
+    20: "INFO",
+    10: "DEBUG",
+}
+
+
+class DefaultFormatter(logging.Formatter):
+    """
+    A formatter that uses ``LOG_FORMAT`` and ``LOG_DATE_FORMAT``.
+    """
+
+    def __init__(self, fmt=LOG_FORMAT, datefmt=LOG_DATE_FORMAT):
+        super().__init__(fmt=fmt, datefmt=datefmt)
+
+
+class AddFormattedAttributes(logging.Filter):
+    """
+    Filter that adds three attributes to a log record:
+
+    - ls_level: the abbreviated loglevel that's max 5 characters long
+    - ls_name: the abbreviated name of the logger (e.g., `l.bootstrap.install`), trimmed to ``MAX_NAME_LEN``
+    - ls_thread: the abbreviated thread name (prefix trimmed, .e.g, ``omeThread-108``)
+    """
+
+    max_name_len: int
+    max_thread_len: int
+
+    def __init__(self, max_name_len: int = None, max_thread_len: int = None):
+        super().__init__()
+        self.max_name_len = max_name_len if max_name_len else MAX_NAME_LEN
+        self.max_thread_len = max_thread_len if max_thread_len else MAX_THREAD_NAME_LEN
+
+    def filter(self, record):
+        record.ls_level = CUSTOM_LEVEL_NAMES.get(record.levelno, record.levelname)
+        record.ls_name = self._get_compressed_logger_name(record.name)
+        record.ls_thread = record.threadName[-self.max_thread_len :]
+        return True
+
+    @lru_cache(maxsize=256)
+    def _get_compressed_logger_name(self, name):
+        return compress_logger_name(name, self.max_name_len)
+
+
+class MaskSensitiveInputFilter(logging.Filter):
+    """
+    Filter that hides sensitive from a binary json string in a record input.
+    It will find the mathing keys and replace their values with "******"
+
+    For example, if initialized with `sensitive_keys=["my_key"]`, the input
+    b'{"my_key": "sensitive_value"}' would become b'{"my_key": "******"}'.
+    """
+
+    patterns: list[tuple[re.Pattern[bytes], bytes]]
+
+    def __init__(self, sensitive_keys: list[str]):
+        super().__init__()
+
+        self.patterns = [
+            (re.compile(to_bytes(rf'"{key}":\s*"[^"]+"')), to_bytes(f'"{key}": "******"'))
+            for key in sensitive_keys
+        ]
+
+    def filter(self, record):
+        if record.input and isinstance(record.input, bytes):
+            record.input = self.mask_sensitive_msg(record.input)
+        return True
+
+    def mask_sensitive_msg(self, message: bytes) -> bytes:
+        for pattern, replacement in self.patterns:
+            message = re.sub(pattern, replacement, message)
+        return message
+
+
+def compress_logger_name(name: str, length: int) -> str:
+    """
+    Creates a short version of a logger name. For example ``my.very.long.logger.name`` with length=17 turns into
+    ``m.v.l.logger.name``.
+
+    :param name: the logger name
+    :param length: the max length of the logger name
+    :return: the compressed name
+    """
+    if len(name) <= length:
+        return name
+
+    parts = name.split(".")
+    parts.reverse()
+
+    new_parts = []
+
+    # we start by assuming that all parts are collapsed
+    # x.x.x requires 5 = 2n - 1 characters
+    cur_length = (len(parts) * 2) - 1
+
+    for i in range(len(parts)):
+        # try to expand the current part and calculate the resulting length
+        part = parts[i]
+        next_len = cur_length + (len(part) - 1)
+
+        if next_len > length:
+            # if the resulting length would exceed the limit, add only the first letter of the parts of all remaining
+            # parts
+            new_parts += [p[0] for p in parts[i:]]
+
+            # but if this is the first item, that means we would display nothing, so at least display as much of the
+            # max length as possible
+            if i == 0:
+                remaining = length - cur_length
+                if remaining > 0:
+                    new_parts[0] = part[: (remaining + 1)]
+
+            break
+
+        # expanding the current part, i.e., instead of using just the one character, we add the entire part
+        new_parts.append(part)
+        cur_length = next_len
+
+    new_parts.reverse()
+    return ".".join(new_parts)
+
+
+class TraceLoggingFormatter(logging.Formatter):
+    aws_trace_log_format = "; ".join([LOG_FORMAT, LOG_INPUT_FORMAT, LOG_OUTPUT_FORMAT])
+    bytes_length_display_threshold = 512
+
+    def __init__(self):
+        super().__init__(fmt=self.aws_trace_log_format, datefmt=LOG_DATE_FORMAT)
+
+    def _replace_large_payloads(self, input: Any) -> Any:
+        """
+        Replaces large payloads in the logs with placeholders to avoid cluttering the logs with huge bytes payloads.
+        :param input: Input/output extra passed when logging. If it is bytes, it will be replaced if larger than
+            bytes_length_display_threshold
+        :return: Input, unless it is bytes and longer than bytes_length_display_threshold, then `Bytes(length_of_input)`
+        """
+        if isinstance(input, bytes) and len(input) > self.bytes_length_display_threshold:
+            return f"Bytes({format_bytes(len(input))})"
+        return input
+
+    def format(self, record: logging.LogRecord) -> str:
+        record.input = self._replace_large_payloads(record.input)
+        record.output = self._replace_large_payloads(record.output)
+        return super().format(record=record)
+
+
+class AwsTraceLoggingFormatter(TraceLoggingFormatter):
+    aws_trace_log_format = "; ".join(
+        [LOG_FORMAT, LOG_CONTEXT_FORMAT, LOG_INPUT_FORMAT, LOG_OUTPUT_FORMAT]
+    )
+
+    def __init__(self):
+        super().__init__()
+
+    def _copy_service_dict(self, service_dict: dict) -> dict:
+        if not isinstance(service_dict, dict):
+            return service_dict
+        result = {}
+        for key, value in service_dict.items():
+            if isinstance(value, dict):
+                result[key] = self._copy_service_dict(value)
+            elif isinstance(value, bytes) and len(value) > self.bytes_length_display_threshold:
+                result[key] = f"Bytes({format_bytes(len(value))})"
+            elif isinstance(value, list):
+                result[key] = [self._copy_service_dict(item) for item in value]
+            else:
+                result[key] = value
+        return result
+
+    def format(self, record: logging.LogRecord) -> str:
+        record.input = self._copy_service_dict(record.input)
+        record.output = self._copy_service_dict(record.output)
+        return super().format(record=record)

localstack_cli/logging/setup.py

@@ -0,0 +1,142 @@
+import logging
+import sys
+import warnings
+
+from localstack_cli import config, constants
+
+from ..utils.strings import key_value_pairs_to_dict
+from .format import AddFormattedAttributes, DefaultFormatter
+
+# The log levels for modules are evaluated incrementally for logging granularity,
+# from highest (DEBUG) to lowest (TRACE_INTERNAL). Hence, each module below should have
+# higher level which serves as the default.
+
+default_log_levels = {
+    "asyncio": logging.INFO,
+    "boto3": logging.INFO,
+    "botocore": logging.ERROR,
+    "docker": logging.WARNING,
+    "elasticsearch": logging.ERROR,
+    "hpack": logging.ERROR,
+    "moto": logging.WARNING,
+    "requests": logging.WARNING,
+    "s3transfer": logging.INFO,
+    "urllib3": logging.WARNING,
+    "werkzeug": logging.WARNING,
+    "rolo": logging.WARNING,
+    "parse": logging.WARNING,
+    "localstack.aws.accounts": logging.INFO,
+    "localstack.aws.protocol.serializer": logging.INFO,
+    "localstack.aws.serving.wsgi": logging.WARNING,
+    "localstack.request": logging.INFO,
+    "localstack.request.internal": logging.WARNING,
+    "localstack.state.inspect": logging.INFO,
+    "localstack_persistence": logging.INFO,
+}
+
+trace_log_levels = {
+    "rolo": logging.DEBUG,
+    "localstack.aws.protocol.serializer": logging.DEBUG,
+    "localstack.aws.serving.wsgi": logging.DEBUG,
+    "localstack.request": logging.DEBUG,
+    "localstack.request.internal": logging.INFO,
+    "localstack.state.inspect": logging.DEBUG,
+}
+
+trace_internal_log_levels = {
+    "localstack.aws.accounts": logging.DEBUG,
+    "localstack.request.internal": logging.DEBUG,
+}
+
+
+def setup_logging_for_cli(log_level=logging.INFO):
+    logging.basicConfig(level=log_level)
+
+    # set log levels of loggers
+    logging.root.setLevel(log_level)
+    logging.getLogger("localstack").setLevel(log_level)
+    for logger, level in default_log_levels.items():
+        logging.getLogger(logger).setLevel(level)
+
+
+def get_log_level_from_config():
+    # overriding the log level if LS_LOG has been set
+    if config.LS_LOG:
+        log_level = str(config.LS_LOG).upper()
+        if log_level.lower() in constants.TRACE_LOG_LEVELS:
+            log_level = "DEBUG"
+        log_level = logging._nameToLevel[log_level]
+        return log_level
+
+    return logging.DEBUG if config.DEBUG else logging.INFO
+
+
+def setup_logging_from_config():
+    log_level = get_log_level_from_config()
+    setup_logging(log_level)
+
+    if config.is_trace_logging_enabled():
+        for name, level in trace_log_levels.items():
+            logging.getLogger(name).setLevel(level)
+    if config.LS_LOG == constants.LS_LOG_TRACE_INTERNAL:
+        for name, level in trace_internal_log_levels.items():
+            logging.getLogger(name).setLevel(level)
+
+    raw_logging_override = config.LOG_LEVEL_OVERRIDES
+    if raw_logging_override:
+        logging_overrides = key_value_pairs_to_dict(raw_logging_override)
+        for logger, level_name in logging_overrides.items():
+            level = getattr(logging, level_name, None)
+            if not level:
+                raise ValueError(
+                    f"Failed to configure logging overrides ({raw_logging_override}): '{level_name}' is not a valid log level"
+                )
+            logging.getLogger(logger).setLevel(level)
+
+
+def create_default_handler(log_level: int):
+    log_handler = logging.StreamHandler(stream=sys.stderr)
+    log_handler.setLevel(log_level)
+    log_handler.setFormatter(DefaultFormatter())
+    log_handler.addFilter(AddFormattedAttributes())
+    return log_handler
+
+
+def setup_logging(log_level=logging.INFO) -> None:
+    """
+    Configures the python logging environment for LocalStack.
+
+    :param log_level: the optional log level.
+    """
+    # set create a default handler for the root logger (basically logging.basicConfig but explicit)
+    log_handler = create_default_handler(log_level)
+
+    # replace any existing handlers
+    logging.basicConfig(level=log_level, handlers=[log_handler])
+
+    # disable some logs and warnings
+    warnings.filterwarnings("ignore")
+    logging.captureWarnings(True)
+
+    # set log levels of loggers
+    logging.root.setLevel(log_level)
+    logging.getLogger("localstack").setLevel(log_level)
+    for logger, level in default_log_levels.items():
+        logging.getLogger(logger).setLevel(level)
+
+
+def setup_hypercorn_logger(hypercorn_config) -> None:
+    """
+    Sets the hypercorn loggers, which are created in a peculiar way, to the localstack settings.
+
+    :param hypercorn_config: a hypercorn.Config object
+    """
+    logger = hypercorn_config.log.access_logger
+    if logger:
+        logger.handlers[0].addFilter(AddFormattedAttributes())
+        logger.handlers[0].setFormatter(DefaultFormatter())
+
+    logger = hypercorn_config.log.error_logger
+    if logger:
+        logger.handlers[0].addFilter(AddFormattedAttributes())
+        logger.handlers[0].setFormatter(DefaultFormatter())

localstack_cli/packages/__init__.py

@@ -0,0 +1,25 @@
+from .api import (
+    InstallTarget,
+    NoSuchVersionException,
+    Package,
+    PackageException,
+    PackageInstaller,
+    PackagesPlugin,
+    package,
+    packages,
+)
+from .core import DownloadInstaller, GitHubReleaseInstaller, SystemNotSupportedException
+
+__all__ = [
+    "Package",
+    "PackageInstaller",
+    "GitHubReleaseInstaller",
+    "DownloadInstaller",
+    "InstallTarget",
+    "PackageException",
+    "NoSuchVersionException",
+    "SystemNotSupportedException",
+    "PackagesPlugin",
+    "package",
+    "packages",
+]