playground-ls-cli 4.14.1.dev8__py3-none-any.whl

localstack_cli/utils/analytics/metadata.py

@@ -0,0 +1,250 @@
+import dataclasses
+import logging
+import os
+import platform
+
+from localstack_cli import config
+from localstack_cli.constants import VERSION
+from localstack_cli.runtime import get_current_runtime, hooks
+from localstack_cli.utils.bootstrap import Container
+from localstack_cli.utils.files import rm_rf
+from localstack_cli.utils.functions import call_safe
+from localstack_cli.utils.json import FileMappedDocument
+from localstack_cli.utils.objects import singleton_factory
+from localstack_cli.utils.strings import long_uid, md5
+
+LOG = logging.getLogger(__name__)
+
+_PHYSICAL_ID_SALT = "ls"
+
+
+@dataclasses.dataclass
+class ClientMetadata:
+    session_id: str
+    machine_id: str
+    api_key: str
+    system: str
+    version: str
+    is_ci: bool
+    is_docker: bool
+    is_testing: bool
+    product: str
+    edition: str
+
+    def __repr__(self):
+        d = dataclasses.asdict(self)
+
+        # anonymize api_key
+        k = d.get("api_key")
+        if k:
+            k = "*" * len(k)
+        d["api_key"] = k
+
+        return f"ClientMetadata({d})"
+
+
+def get_version_string() -> str:
+    gh = config.LOCALSTACK_BUILD_GIT_HASH
+    if gh:
+        return f"{VERSION}:{gh}"
+    else:
+        return VERSION
+
+
+def read_client_metadata() -> ClientMetadata:
+    return ClientMetadata(
+        session_id=get_session_id(),
+        machine_id=get_machine_id(),
+        api_key=get_api_key_or_auth_token() or "",  # api key should not be None
+        system=get_system(),
+        version=get_version_string(),
+        is_ci=os.getenv("CI") is not None,
+        is_docker=config.is_in_docker,
+        is_testing=config.is_local_test_mode(),
+        product=get_localstack_product(),
+        edition=os.getenv("LOCALSTACK_TELEMETRY_EDITION") or get_localstack_edition(),
+    )
+
+
+@singleton_factory
+def get_session_id() -> str:
+    """
+    Returns the unique ID for this LocalStack session.
+    :return: a UUID
+    """
+    return _generate_session_id()
+
+
+@singleton_factory
+def get_client_metadata() -> ClientMetadata:
+    metadata = read_client_metadata()
+
+    if config.DEBUG_ANALYTICS:
+        LOG.info("resolved client metadata: %s", metadata)
+
+    return metadata
+
+
+@singleton_factory
+def get_machine_id() -> str:
+    cache_path = os.path.join(config.dirs.cache, "machine.json")
+    try:
+        doc = FileMappedDocument(cache_path)
+    except Exception:
+        # it's possible that the file is somehow messed up, so we try to delete the file first and try again.
+        # if that fails, we return a generated ID.
+        call_safe(rm_rf, args=(cache_path,))
+
+        try:
+            doc = FileMappedDocument(cache_path)
+        except Exception:
+            return _generate_machine_id()
+
+    if "machine_id" not in doc:
+        # generate a machine id
+        doc["machine_id"] = _generate_machine_id()
+        # try to cache the machine ID
+        call_safe(doc.save)
+
+    return doc["machine_id"]
+
+
+def get_localstack_edition() -> str:
+    # Generator expression to find the first hidden file ending with '-version'
+    version_file = next(
+        (
+            f
+            for f in os.listdir(config.dirs.static_libs)
+            if f.startswith(".") and f.endswith("-version")
+        ),
+        None,
+    )
+
+    # Return the base name of the version file, or unknown if no file is found
+    return version_file.removesuffix("-version").removeprefix(".") if version_file else "unknown"
+
+
+def get_localstack_product() -> str:
+    """
+    Returns the telemetry product name from the env var, runtime, or "unknown".
+    """
+    runtime_product = None
+    if get_current_runtime is not None:
+        try:
+            runtime_product = get_current_runtime().components.name
+        except (ValueError, AttributeError):
+            pass
+
+    return os.getenv("LOCALSTACK_TELEMETRY_PRODUCT") or runtime_product or "unknown"
+
+
+def is_license_activated() -> bool:
+    try:
+        from localstack_cli.pro.core import config  # noqa
+    except ImportError:
+        return False
+
+    try:
+        from localstack_cli.pro.core.bootstrap import licensingv2
+
+        return licensingv2.get_licensed_environment().activated
+    except Exception:
+        LOG.error(
+            "Could not determine license activation status",
+            exc_info=LOG.isEnabledFor(logging.DEBUG),
+        )
+        return False
+
+
+def _generate_session_id() -> str:
+    return long_uid()
+
+
+def _anonymize_physical_id(physical_id: str) -> str:
+    """
+    Returns 12 digits of the salted hash of the given physical ID.
+
+    :param physical_id: the physical id
+    :return: an anonymized 12 digit value representing the physical ID.
+    """
+    hashed = md5(_PHYSICAL_ID_SALT + physical_id)
+    return hashed[:12]
+
+
+def _generate_machine_id() -> str:
+    try:
+        # try to get a robust ID from the docker socket (which will be the same from the host and the
+        # container)
+        from localstack_cli.utils.docker_utils import DOCKER_CLIENT
+
+        docker_id = DOCKER_CLIENT.get_system_id()
+        # some systems like podman don't return a stable ID, so we double-check that here
+        if docker_id == DOCKER_CLIENT.get_system_id():
+            return f"dkr_{_anonymize_physical_id(docker_id)}"
+    except Exception:
+        pass
+
+    if config.is_in_docker:
+        return f"gen_{long_uid()[:12]}"
+
+    # this can potentially be useful when generated on the host using the CLI and then mounted into the
+    # container via machine.json
+    try:
+        if os.path.exists("/etc/machine-id"):
+            with open("/etc/machine-id") as fd:
+                machine_id = str(fd.read()).strip()
+                if machine_id:
+                    return f"sys_{_anonymize_physical_id(machine_id)}"
+    except Exception:
+        pass
+
+    # always fall back to a generated id
+    return f"gen_{long_uid()[:12]}"
+
+
+def get_api_key_or_auth_token() -> str | None:
+    # TODO: this is duplicated code from ext, but should probably migrate that to localstack
+    auth_token = os.environ.get("LOCALSTACK_AUTH_TOKEN", "").strip("'\" ")
+    if auth_token:
+        return auth_token
+
+    api_key = os.environ.get("LOCALSTACK_API_KEY", "").strip("'\" ")
+    if api_key:
+        return api_key
+
+    return None
+
+
+@singleton_factory
+def get_system() -> str:
+    try:
+        # try to get the system from the docker socket
+        from localstack_cli.utils.docker_utils import DOCKER_CLIENT
+
+        system = DOCKER_CLIENT.get_system_info()
+        if system.get("OSType"):
+            return system.get("OSType").lower()
+    except Exception:
+        pass
+
+    if config.is_in_docker:
+        return "docker"
+
+    return platform.system().lower()
+
+
+@hooks.prepare_host()
+def publish_invocation_metadata():
+    """Lazy-init machine ID into cache on the host, which can then be used in the container."""
+    get_machine_id()
+
+
+@hooks.configure_localstack_container()
+def set_analytics_header(container: Container):
+    """Mount the machine file from the host's CLI cache directory into the container."""
+    from localstack_cli.utils.container_utils.container_client import BindMount
+
+    machine_file = os.path.join(config.dirs.cache, "machine.json")
+    if os.path.isfile(machine_file):
+        target = os.path.join(config.dirs.for_container().cache, "machine.json")
+        container.config.volumes.add(BindMount(machine_file, target, read_only=True))

localstack_cli/utils/analytics/publisher.py

@@ -0,0 +1,160 @@
+import abc
+import atexit
+import logging
+import threading
+
+from localstack_cli import config
+from localstack_cli.utils.batching import AsyncBatcher
+from localstack_cli.utils.threads import FuncThread, start_thread, start_worker_thread
+
+from .client import AnalyticsClient
+from .events import Event, EventHandler
+from .metadata import get_client_metadata
+
+LOG = logging.getLogger(__name__)
+
+
+class Publisher(abc.ABC):
+    """
+    A publisher takes a batch of events and publishes them to a destination.
+    """
+
+    def publish(self, events: list[Event]):
+        raise NotImplementedError
+
+    def close(self):
+        pass
+
+
+class AnalyticsClientPublisher(Publisher):
+    client: AnalyticsClient
+
+    def __init__(self, client: AnalyticsClient = None) -> None:
+        super().__init__()
+        self.client = client or AnalyticsClient()
+
+    def publish(self, events: list[Event]):
+        self.client.append_events(events)
+
+    def close(self):
+        self.client.close()
+
+
+class Printer(Publisher):
+    """
+    Publisher that prints serialized events to stdout.
+    """
+
+    def publish(self, events: list[Event]):
+        for event in events:
+            print(event.asdict())
+
+
+class GlobalAnalyticsBus(EventHandler):
+    _batcher: AsyncBatcher[Event]
+    _client: AnalyticsClient
+    _worker_thread: FuncThread | None
+
+    def __init__(self, client: AnalyticsClient = None, flush_size=20, flush_interval=10) -> None:
+        self._client = client or AnalyticsClient()
+        self._publisher = AnalyticsClientPublisher(self._client)
+        self._batcher = AsyncBatcher(
+            self._handle_batch,
+            max_batch_size=flush_size,
+            max_flush_interval=flush_interval,
+        )
+
+        self._started = False
+        self._startup_mutex = threading.Lock()
+        self._worker_thread = None
+
+        self.force_tracking = False  # allow class to ignore all other tracking config
+        self.tracking_disabled = False  # disables tracking if global config would otherwise track
+
+    def _handle_batch(self, batch: list[Event]):
+        """Method that satisfies the BatchHandler[Event] protocol and is passed to AsyncBatcher."""
+        try:
+            self._publisher.publish(batch)
+        except Exception:
+            # currently we're just dropping events if something goes wrong during publishing
+            if config.DEBUG_ANALYTICS:
+                LOG.exception("error while publishing analytics events")
+
+    @property
+    def is_tracking_disabled(self):
+        if self.force_tracking:
+            return False
+
+        # don't track if event tracking is disabled globally
+        if config.DISABLE_EVENTS:
+            return True
+        # don't track for internal test runs (like integration tests)
+        if config.is_local_test_mode():
+            return True
+        if self.tracking_disabled:
+            return True
+
+        return False
+
+    def handle(self, event: Event):
+        """
+        Publish an event to the global analytics event publisher.
+        """
+        if self.is_tracking_disabled:
+            if config.DEBUG_ANALYTICS:
+                LOG.debug("tracking disabled, skipping event %s", event)
+            return
+
+        if not self._started:
+            # we make sure the batching worker is started
+            self._start()
+
+        self._batcher.add(event)
+
+    def _start(self):
+        with self._startup_mutex:
+            if self._started:
+                return
+            self._started = True
+
+            # startup has to run async, otherwise first call to handle() could block a long time.
+            start_worker_thread(self._do_start_retry)
+
+    def _do_start_retry(self, *_):
+        # TODO: actually retry
+        try:
+            if config.DEBUG_ANALYTICS:
+                LOG.debug("trying to register session with analytics backend")
+            response = self._client.start_session(get_client_metadata())
+            if config.DEBUG_ANALYTICS:
+                LOG.debug("session endpoint returned: %s", response)
+
+            if not response.track_events():
+                if config.DEBUG_ANALYTICS:
+                    LOG.debug("gracefully disabling analytics tracking")
+                self.tracking_disabled = True
+
+        except Exception:
+            self.tracking_disabled = True
+            if config.DEBUG_ANALYTICS:
+                LOG.exception("error while registering session. disabling tracking")
+            return
+
+        self._worker_thread = start_thread(self._run, name="global-analytics-bus")
+
+        # given the "Global" nature of this class, we register a global atexit hook to make sure all events are flushed
+        # when localstack shuts down.
+        def _do_close():
+            self.close_sync(timeout=2)
+
+        atexit.register(_do_close)
+
+    def _run(self, *_):
+        # main control loop, simply runs the batcher
+        self._batcher.run()
+
+    def close_sync(self, timeout=None):
+        self._batcher.close()
+
+        if self._worker_thread:
+            self._worker_thread.join(timeout=timeout)

localstack_cli/utils/analytics/service_request_aggregator.py

@@ -0,0 +1,133 @@
+import datetime
+import logging
+import threading
+from collections import Counter
+from typing import NamedTuple
+
+from localstack_cli import config
+try:
+    from localstack_cli.runtime.shutdown import SHUTDOWN_HANDLERS
+except ImportError:
+    SHUTDOWN_HANDLERS = None
+from localstack_cli.utils import analytics
+try:
+    from localstack_cli.utils.scheduler import Scheduler
+except ImportError:
+    Scheduler = None
+
+LOG = logging.getLogger(__name__)
+
+DEFAULT_FLUSH_INTERVAL_SECS = 15
+EVENT_NAME = "aws_request_agg"
+OPTIONAL_FIELDS = ["err_type"]
+
+
+class ServiceRequestInfo(NamedTuple):
+    service: str
+    operation: str
+    status_code: int
+    err_type: str | None = None
+
+
+class ServiceRequestAggregator:
+    """
+    Collects API call data, aggregates it into small batches, and periodically emits (flushes) it as an
+    analytics event.
+    """
+
+    def __init__(self, flush_interval: float = DEFAULT_FLUSH_INTERVAL_SECS):
+        self.counter = Counter()
+        self._flush_interval = flush_interval
+        self._flush_scheduler = Scheduler()
+        self._mutex = threading.RLock()
+        self._period_start_time = datetime.datetime.now(datetime.UTC)
+        self._is_started = False
+        self._is_shutdown = False
+
+    def start(self):
+        """
+        Start a thread that periodically flushes HTTP response data aggregations as analytics events
+        :returns: the thread containing the running flush scheduler
+        """
+        with self._mutex:
+            if self._is_started:
+                return
+            self._is_started = True
+
+            # schedule flush task
+            self._flush_scheduler.schedule(
+                func=self._flush, period=self._flush_interval, fixed_rate=True
+            )
+
+            # start thread
+            _flush_scheduler_thread = threading.Thread(
+                target=self._flush_scheduler.run, daemon=True
+            )
+            _flush_scheduler_thread.start()
+
+            SHUTDOWN_HANDLERS.register(self.shutdown)
+
+    def shutdown(self):
+        with self._mutex:
+            if not self._is_started:
+                return
+            if self._is_shutdown:
+                return
+            self._is_shutdown = True
+
+            self._flush()
+            self._flush_scheduler.close()
+            SHUTDOWN_HANDLERS.unregister(self.shutdown)
+
+    def add_request(self, request_info: ServiceRequestInfo):
+        """
+        Add an API call for aggregation and collection.
+
+        :param request_info: information about the API call.
+        """
+        if config.DISABLE_EVENTS:
+            return
+
+        if self._is_shutdown:
+            return
+
+        with self._mutex:
+            self.counter[request_info] += 1
+
+    def _flush(self):
+        """
+        Flushes the current batch of HTTP response data as an analytics event.
+        This happens automatically in the background.
+        """
+        with self._mutex:
+            try:
+                if len(self.counter) == 0:
+                    return
+                analytics_payload = self._create_analytics_payload()
+                self._emit_payload(analytics_payload)
+                self.counter.clear()
+            finally:
+                self._period_start_time = datetime.datetime.now(datetime.UTC)
+
+    def _create_analytics_payload(self):
+        return {
+            "period_start_time": self._period_start_time.isoformat().replace("+00:00", "Z"),
+            "period_end_time": datetime.datetime.now(datetime.UTC)
+            .isoformat()
+            .replace("+00:00", "Z"),
+            "api_calls": self._aggregate_api_calls(self.counter),
+        }
+
+    def _aggregate_api_calls(self, counter) -> list:
+        aggregations = []
+        for api_call_info, count in counter.items():
+            doc = api_call_info._asdict()
+            for field in OPTIONAL_FIELDS:
+                if doc.get(field) is None:
+                    del doc[field]
+            doc["count"] = count
+            aggregations.append(doc)
+        return aggregations
+
+    def _emit_payload(self, analytics_payload: dict):
+        analytics.log.event(EVENT_NAME, analytics_payload)