kameleoon-client-python 3.22.0__tar.gz → 3.22.1__tar.gz

{kameleoon_client_python-3.22.0 → kameleoon_client_python-3.22.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kameleoon-client-python
-Version: 3.22.0
+Version: 3.22.1
 Summary: Kameleoon Client Python Software Development Kit.
 Home-page: https://developers.kameleoon.com/python-sdk.html
 Author: Kameleoon
@@ -20,27 +20,27 @@ Requires-Dist: requests<3.0.0,>=2.25
 Requires-Dist: sseclient-py<1.9.0,>=1.7.0
 Requires-Dist: aiohttp<4.0.0,>=3.8.0
 Provides-Extra: test
-Requires-Dist: types-requests==2.25.0; extra == "test"
-Requires-Dist: Django==3.2.4; extra == "test"
+Requires-Dist: types-python-dateutil==2.8.2; extra == "test"
+Requires-Dist: twine==3.4.1; extra == "test"
 Requires-Dist: sseclient-py<1.9.0,>=1.7.0; extra == "test"
-Requires-Dist: pytest-django==4.4.0; extra == "test"
-Requires-Dist: pytest-pythonpath==0.7.3; extra == "test"
-Requires-Dist: pytest-ordering==0.6; extra == "test"
+Requires-Dist: types-PyYAML==5.4.3; extra == "test"
 Requires-Dist: aiohttp<4.0.0,>=3.8.0; extra == "test"
-Requires-Dist: flake8==4.0.1; extra == "test"
-Requires-Dist: pytest-asyncio==0.21.*; extra == "test"
-Requires-Dist: pytest-cov==2.12.1; extra == "test"
-Requires-Dist: mypy==0.991; extra == "test"
-Requires-Dist: pylint==2.12.*; extra == "test"
-Requires-Dist: Werkzeug==2.0.1; extra == "test"
 Requires-Dist: pyyaml<6.1.0,>=5.4.0; extra == "test"
-Requires-Dist: freezegun==1.1.0; extra == "test"
-Requires-Dist: twine==3.4.1; extra == "test"
-Requires-Dist: types-PyYAML==5.4.3; extra == "test"
 Requires-Dist: requests<3.0.0,>=2.25; extra == "test"
+Requires-Dist: pylint==2.12.*; extra == "test"
+Requires-Dist: pytest-pythonpath==0.7.3; extra == "test"
+Requires-Dist: pytest-django==4.5.2; extra == "test"
+Requires-Dist: flake8==4.0.1; extra == "test"
+Requires-Dist: pytest-cov==2.12.1; extra == "test"
 Requires-Dist: wheel==0.36.2; extra == "test"
-Requires-Dist: types-python-dateutil==2.8.2; extra == "test"
+Requires-Dist: freezegun==1.1.0; extra == "test"
+Requires-Dist: types-requests==2.25.0; extra == "test"
+Requires-Dist: pytest-ordering==0.6; extra == "test"
 Requires-Dist: aioresponses==0.7.*; extra == "test"
+Requires-Dist: mypy==0.991; extra == "test"
+Requires-Dist: Werkzeug==2.0.1; extra == "test"
+Requires-Dist: pytest-asyncio==0.21.*; extra == "test"
+Requires-Dist: Django==4.2.*; extra == "test"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{kameleoon_client_python-3.22.0 → kameleoon_client_python-3.22.1}/kameleoon/data/manager/visitor_manager.py

@@ -67,17 +67,15 @@ class VisitorManager:
             KameleoonLogger.debug("RETURN: VisitorManager.get_or_create_visitor(visitor_code: %s)", visitor_code)
 
     def peek_visitor(self, visitor_code: str) -> Optional[Visitor]:
-        """Retrieves a visitor by their code if it exists, otherwise returns `None`."""
-        KameleoonLogger.debug(f"CALL: VisitorManager.peekVisitor(visitor_code: {visitor_code})")
-        if (slot := self.__try_acquire_slot(visitor_code)) is None:
-            visitor = None
-        else:
-            try:
-                visitor = slot.visitor
-            finally:
-                slot.lock.release()
+        """
+        Retrieves a visitor by their code if it exists, otherwise returns `None`. Unlike `get_visitor`, does not
+        count as visitor activity and takes no lock: a plain read needs no protection against the purge.
+        """
+        KameleoonLogger.debug("CALL: VisitorManager.peek_visitor(visitor_code: %s)", visitor_code)
+        slot = self._slots.get(visitor_code)
+        visitor = slot.visitor if slot is not None else None
         KameleoonLogger.debug(
-            f"RETURN: VisitorManager.peekVisitor(visitor_code: {visitor_code}) -> " f"(visitor: {visitor})"
+            "RETURN: VisitorManager.peek_visitor(visitor_code: %s) -> (visitor: %s)", visitor_code, visitor
         )
         return visitor
 
@@ -180,7 +178,7 @@ class VisitorManager:
         while True:
             slot = self._slots[visitor_code]
             slot.lock.acquire()
-            if slot == self._slots.get(visitor_code):
+            if slot is self._slots.get(visitor_code):
                 return slot
             slot.lock.release()
 
@@ -190,7 +188,7 @@ class VisitorManager:
             if slot is None:
                 return None
             slot.lock.acquire()
-            if slot == self._slots.get(visitor_code):
+            if slot is self._slots.get(visitor_code):
                 return slot
             slot.lock.release()
 
@@ -199,7 +197,7 @@ class VisitorManager:
         if slot is None:
             return None
         slot.lock.acquire()
-        if slot == self._slots.get(visitor_code):
+        if slot is self._slots.get(visitor_code):
             return slot
         slot.lock.release()
         return None

{kameleoon_client_python-3.22.0 → kameleoon_client_python-3.22.1}/kameleoon/helpers/multi_threading.py

@@ -12,6 +12,10 @@ class ThreadEventLoop:
     ThreadEventLoop is helper object to handle asynchronous tasks execution in a separate thread.
     """
 
+    # Upper bound for `stop` to wait for the running tasks and for the thread to exit; a task that never
+    # completes (or a loop thread blocked by a synchronous call) must not hang the disposal forever.
+    STOP_TIMEOUT = 10.0
+
     def __init__(self) -> None:
         self.loop: Optional[asyncio.AbstractEventLoop] = None
         self.thread: Optional[threading.Thread] = None
@@ -40,17 +44,38 @@ class ThreadEventLoop:
         return self.loop
 
     def stop(self) -> None:
-        """Stops the thread and the event loop"""
+        """
+        Stops the thread and the event loop.
+
+        Waits (up to `STOP_TIMEOUT`) for the already scheduled tasks to complete before stopping the loop.
+        """
         if self.thread is None:
             return
         self.running_allowed = False
-        time.sleep(0.001)
-        while self.count_tasks() > 0:
-            time.sleep(0.01)
         loop = self.get_loop()
+        if threading.current_thread() is self.thread:
+            # Called from the loop's own thread (e.g. a GC-triggered `__del__`): it cannot wait for or join itself.
+            loop.call_soon(loop.stop)
+            return
+        # The draining runs on the loop itself, after everything already queued (e.g. tasks being created by
+        # `run_coroutine_threadsafe`), so it observes the tasks a cross-thread `all_tasks` poll could miss.
+        drained = asyncio.run_coroutine_threadsafe(self.__wait_for_tasks(), loop)
+        try:
+            drained.result(timeout=self.STOP_TIMEOUT)
+        except Exception:  # pylint: disable=W0703
+            drained.cancel()
         loop.call_soon_threadsafe(loop.stop)
-        self.thread.join()
-        loop.close()
+        self.thread.join(timeout=self.STOP_TIMEOUT)
+        if not self.thread.is_alive():
+            loop.close()
+
+    @staticmethod
+    async def __wait_for_tasks() -> None:
+        while True:
+            tasks = [task for task in asyncio.all_tasks() if task is not asyncio.current_task()]
+            if not tasks:
+                return
+            await asyncio.wait(tasks, timeout=ThreadEventLoop.STOP_TIMEOUT)
 
     def run_coro(self, coro: Coroutine[Any, Any, Any]) -> Optional[concurrent.futures.Future[Any]]:
         """
@@ -66,10 +91,6 @@ class ThreadEventLoop:
             self.start()
         return asyncio.run_coroutine_threadsafe(coro, self.get_loop())
 
-    def count_tasks(self) -> int:
-        """Returns the number of running tasks."""
-        return len(asyncio.all_tasks(self.get_loop()))
-
 
 def run_in_thread(
     func: Callable[[], Any], thread_name: Optional[str] = None, with_event_loop=False
@@ -86,16 +107,15 @@ def run_in_thread(
     :rtype: threading.Thread
     """
     if with_event_loop:
+
         def target():
             loop = asyncio.new_event_loop()
             func()
             loop.close()
+
     else:
         target = func
-    thread = threading.Thread(target=target)
-    thread.daemon = True
-    if thread_name:
-        thread.setName(thread_name)
+    thread = threading.Thread(target=target, name=thread_name or None, daemon=True)
     thread.start()
     return thread
 

{kameleoon_client_python-3.22.0 → kameleoon_client_python-3.22.1}/kameleoon/helpers/scheduler.py

@@ -42,10 +42,14 @@ class Scheduler:
 
     def stop(self) -> None:
         """
-        Starts all scheduled jobs.
+        Stops all scheduled jobs.
 
         Should not be called more than once.
         """
+        if self._loop.is_closed():
+            # The jobs died with the loop; nothing left to stop (e.g. disposal after the host app closed its loop).
+            KameleoonLogger.debug("Scheduler is not stopped: event loop is already closed")
+            return
         try:
             self._loop.call_soon_threadsafe(self._stop)
         except Exception as err:  # pylint: disable=W0703
@@ -53,8 +57,7 @@ class Scheduler:
 
     def _stop(self) -> None:
         for job in self._jobs.values():
-            if job.stop_signal is not None:
-                job.stop_signal.set()
+            job.stop()
 
     class Job:
         """`Job` represents a scheduled job"""
@@ -65,13 +68,24 @@ class Scheduler:
             self.func = func
             self.stop_signal: Optional[asyncio.Event] = None
             self.future: Optional[concurrent.futures.Future[Any]] = None
+            self._stop_requested = False
 
         def run(self, loop: asyncio.AbstractEventLoop) -> None:
             """Starts the job execution in the given event loop."""
             if self.future is None:
                 self.future = asyncio.run_coroutine_threadsafe(self._cycle(), loop)
 
+        def stop(self) -> None:
+            """Requests the job to stop. Must be called on the job's event loop."""
+            # `stop_signal` only exists once `_cycle` has started; a stop posted with `call_soon_threadsafe` right
+            # after `run` is processed before the task's first step, so the request has to be remembered as well.
+            self._stop_requested = True
+            if self.stop_signal is not None:
+                self.stop_signal.set()
+
         async def _cycle(self) -> None:
+            if self._stop_requested:
+                return
             self.stop_signal = asyncio.Event()
             interval = self.interval
             stop_task = asyncio.create_task(self.stop_signal.wait())

{kameleoon_client_python-3.22.0 → kameleoon_client_python-3.22.1}/kameleoon/helpers/string_utils.py

@@ -9,7 +9,7 @@ class StringUtils:
 
     @staticmethod
     def secret(secret: Optional[str]) -> str:
-        """Hide part of the string, keeping only a few initial characters visible."""
+        """Hides the string behind a fixed-width mask, keeping only a few trailing characters visible."""
         hid_ch = '*'
         vis_count = 4
 
@@ -21,8 +21,7 @@ class StringUtils:
         if length <= vis_count:
             return hid_ch * length
 
-        hidden_length = max(length - vis_count, vis_count)
-        return secret[:length - hidden_length] + hid_ch * hidden_length
+        return hid_ch * vis_count + secret[max(length - vis_count, vis_count):]
 
     @staticmethod
     def bool_to_string_lower_case(data: bool) -> str:

{kameleoon_client_python-3.22.0 → kameleoon_client_python-3.22.1}/kameleoon/managers/tracking/tracking_manager.py

@@ -1,5 +1,6 @@
 """Tracking"""
-from typing import Iterable, List
+import asyncio
+from typing import Iterable, List, Optional
 from kameleoon.data.manager.visitor_manager import VisitorManager
 from kameleoon.logging.kameleoon_logger import KameleoonLogger
 from kameleoon.managers.asyncexec.async_executor import AsyncExecutor
@@ -7,7 +8,7 @@ from kameleoon.managers.data.data_manager import DataManager
 from kameleoon.managers.tracking.tracking_builder import TrackingBuilder
 from kameleoon.managers.tracking.visitor_tracking_registry import (
     VisitorTrackingRegistry,
-    LockVisitorTrackingRegistry,
+    ConcurrentVisitorTrackingRegistry,
 )
 from kameleoon.network.net_provider import Response
 from kameleoon.network.network_manager import NetworkManager
@@ -19,6 +20,7 @@ class TrackingManager:
     """Tracking manager"""
     LINES_DELIMITER = "\n"
     REQUEST_SIZE_LIMIT = 2560 * 1024  # 2.5 * 1024^2 characters
+    PROBE_MAX_VISITORS = 1
 
     # pylint: disable=R0913
     def __init__(
@@ -30,7 +32,8 @@ class TrackingManager:
             "CALL: TrackingManager(data_manager, network_manager. visitor_manager, "
             "track_interval_seconds: %s, scheduler)", track_interval_seconds
         )
-        self._tracking_visitors: VisitorTrackingRegistry = LockVisitorTrackingRegistry(visitor_manager)
+        self._tracking_visitors: VisitorTrackingRegistry = ConcurrentVisitorTrackingRegistry(visitor_manager)
+        self._probe_mode = False
         self._data_manager = data_manager
         self._network_manager = network_manager
         self._visitor_manager = visitor_manager
@@ -41,6 +44,11 @@ class TrackingManager:
             "track_interval_seconds: %s, scheduler)", track_interval_seconds
         )
 
+    @property
+    def probe_mode(self) -> bool:
+        """Test purpose only"""
+        return self._probe_mode
+
     def add_visitor_code(self, visitor_code: str) -> None:
         """Adds a visitor code to the tracking registry."""
         KameleoonLogger.debug("CALL: TrackingManager.add_visitor_code(visitor_code: %s)", visitor_code)
@@ -54,7 +62,10 @@ class TrackingManager:
         `track_all` is triggered by the interval tracking timer.
         """
         KameleoonLogger.debug("CALL: TrackingManager.track_all()")
-        self.__track(self._tracking_visitors.extract())
+        if self._probe_mode:
+            self.__track_probe()
+        else:
+            self.__track(self._tracking_visitors.extract())
         KameleoonLogger.debug("RETURN: TrackingManager.track_all()")
 
     def track_visitor(self, visitor_code: str) -> None:
@@ -63,7 +74,14 @@ class TrackingManager:
         self.__track([visitor_code])
         KameleoonLogger.debug("RETURN: TrackingManager.track_visitor(visitor_code: %s)", visitor_code)
 
-    def __track(self, visitor_codes: Iterable[str]) -> None:
+    def __track_probe(self) -> None:
+        # Sends one visitor's data; visitors without data are dropped (as on the normal path) until one is found.
+        while visitor_codes := self._tracking_visitors.extract(self.PROBE_MAX_VISITORS):
+            if self.__track(visitor_codes):
+                return
+
+    def __track(self, visitor_codes: Iterable[str]) -> bool:
+        """Returns whether a request was sent."""
         builder = TrackingBuilder(
             visitor_codes, self._data_manager.data_file, self._visitor_manager, self.REQUEST_SIZE_LIMIT
         )
@@ -75,33 +93,58 @@ class TrackingManager:
                 "If it is not caused by the peak load, decreasing the tracking interval is recommended."
             )
             self._tracking_visitors.add_all(builder.visitor_codes_to_keep)
-        self.__perform_tracking_request(
+        return self.__perform_tracking_request(
             builder.visitor_codes_to_send, builder.unsent_visitor_data, builder.tracking_lines
         )
 
     def __perform_tracking_request(
         self, visitor_codes: List[str], visitors_data: List[Sendable], tracking_lines: List[str]
-    ) -> None:
+    ) -> bool:
+        """Returns whether a request was sent."""
         if not tracking_lines:
-            return
+            return False
         lines = self.LINES_DELIMITER.join(tracking_lines)
         # Mark unsent data as transmitted
         for sendable_data in visitors_data:
             sendable_data.mark_as_transmitting()
 
-        async def call() -> Response:
-            service = self._network_manager.get_service(DataService)
-            response = await service.send_tracking_data(lines)
-            if response.success:
+        async def call() -> Optional[Response]:
+            response: Optional[Response] = None
+            try:
+                service = self._network_manager.get_service(DataService)
+                response = await service.send_tracking_data(lines)
+            except asyncio.CancelledError:
+                # Not a Data API failure: roll the data back for the next request without entering probe mode.
+                self.__handle_failed_request(visitor_codes, visitors_data)
+                raise
+            except Exception as exception:  # pylint: disable=W0703
+                KameleoonLogger.error("Tracking request failed: %s", exception)
+            if (response is not None) and response.success:
                 KameleoonLogger.debug("Successful request for tracking visitors: %s, data: %s",
                                       visitor_codes, visitors_data)
                 for sendable_data in visitors_data:
                     sendable_data.mark_as_sent()
+                self.__leave_probe_mode()
             else:
-                KameleoonLogger.debug("Failed request for tracking visitors: %s, data: %s",
-                                      visitor_codes, visitors_data)
-                for sendable_data in visitors_data:
-                    sendable_data.mark_as_unsent()
-                self._tracking_visitors.add_all(visitor_codes)
+                self.__handle_failed_request(visitor_codes, visitors_data)
+                self.__enter_probe_mode()
             return response
         self._async_executor.run_coro(call(), "send_tracking_data")
+        return True
+
+    def __handle_failed_request(self, visitor_codes: List[str], visitors_data: List[Sendable]) -> None:
+        KameleoonLogger.debug("Failed request for tracking visitors: %s, data: %s", visitor_codes, visitors_data)
+        for sendable_data in visitors_data:
+            sendable_data.mark_as_unsent()
+        self._tracking_visitors.add_all(visitor_codes)
+
+    def __enter_probe_mode(self) -> None:
+        if not self._probe_mode:
+            self._probe_mode = True
+            KameleoonLogger.info("Tracking request failed: only one visitor's data per tracking request"
+                                 " will be sent until a request succeeds")
+
+    def __leave_probe_mode(self) -> None:
+        if self._probe_mode:
+            self._probe_mode = False
+            KameleoonLogger.info("Tracking request succeeded: full-size tracking requests are restored")

kameleoon_client_python-3.22.1/kameleoon/managers/tracking/visitor_tracking_registry.py

@@ -0,0 +1,112 @@
+"""Tracking"""
+
+import sys
+from threading import Lock
+from typing import Dict, Iterable, List, Set
+from kameleoon.data.manager.visitor_manager import VisitorManager
+
+
+class VisitorTrackingRegistry:
+    """
+    Abstract visitor tracking registry.
+
+    Must be thread-safe
+    """
+
+    UNLIMITED = sys.maxsize
+    DEFAULT_STORAGE_LIMIT = 1_000_000
+    DEFAULT_EXTRACTION_LIMIT = 20_000
+    LIMITED_EXTRACTION_THRESHOLD_COEFFICIENT = 2
+    REMOVAL_FACTOR = 0.8
+
+    def __init__(
+        self,
+        visitor_manager: VisitorManager,
+        storage_limit=DEFAULT_STORAGE_LIMIT,
+        extraction_limit=DEFAULT_EXTRACTION_LIMIT,
+    ) -> None:
+        self._visitor_manager = visitor_manager
+        self._storage_limit = storage_limit
+        self._extraction_limit = extraction_limit
+
+    @property
+    def visitors(self) -> Set[str]:
+        """Test purpose only"""
+        raise NotImplementedError()
+
+    def add(self, visitor_code: str) -> None:
+        """Adds a single visitor code to the registry."""
+        raise NotImplementedError()
+
+    def add_all(self, visitor_codes: Iterable[str]) -> None:
+        """Adds multiple visitor codes to the registry."""
+        raise NotImplementedError()
+
+    def extract(self, limit: int = UNLIMITED) -> List[str]:
+        """
+        Removes and returns visitor codes. A backlog under the doubled `extraction_limit` is returned at once
+        (unless `limit` is smaller); otherwise at most `min(limit, extraction_limit)` codes are returned.
+        Codes are extracted in no particular order.
+        """
+        raise NotImplementedError()
+
+    def _extraction_count(self, size: int, limit: int) -> int:
+        if (size <= limit) and (size < self._extraction_limit * self.LIMITED_EXTRACTION_THRESHOLD_COEFFICIENT):
+            return size
+        return min(limit, self._extraction_limit)
+
+    def _storage_limit_after_removal(self) -> int:
+        return int(self._storage_limit * self.REMOVAL_FACTOR)
+
+
+class ConcurrentVisitorTrackingRegistry(VisitorTrackingRegistry):
+    """
+    Lock-free on the hot path: `add` is a single `dict` operation, atomic under the GIL (CPython, PyPy) and under the
+    free-threaded build's per-object locking, so request threads never block on the registry - with the GIL a lock
+    there causes a convoy on every interpreter switch (~5x lower throughput, p99 of hundreds of microseconds at
+    8 threads; see `tests/benchmarks/visitor_tracking_registry_benchmark.py`). The dict is never swapped out;
+    `extract` removes with `popitem()` (O(1)) under a lock that only serializes removers, so a concurrently added
+    code is either returned now or kept for the next extraction, never lost. Nothing here depends on the order in
+    which concurrent operations interleave, only on single operations not corrupting or losing entries.
+    """
+
+    def __init__(
+        self,
+        visitor_manager: VisitorManager,
+        storage_limit=VisitorTrackingRegistry.DEFAULT_STORAGE_LIMIT,
+        extraction_limit=VisitorTrackingRegistry.DEFAULT_EXTRACTION_LIMIT,
+    ) -> None:
+        super().__init__(visitor_manager, storage_limit, extraction_limit)
+        self._visitors: Dict[str, None] = {}
+        self._extract_lock = Lock()
+
+    @property
+    def visitors(self) -> Set[str]:
+        return set(self._visitors)
+
+    def add(self, visitor_code: str) -> None:
+        self._visitors[visitor_code] = None
+
+    def add_all(self, visitor_codes: Iterable[str]) -> None:
+        self._visitors.update(dict.fromkeys(visitor_codes))
+        if len(self._visitors) > self._storage_limit:
+            self.__erase_to_storage_limit()
+
+    def __erase_to_storage_limit(self) -> None:
+        with self._extract_lock:
+            # Drain with `popitem()` instead of a `for` loop over the live dict: request threads keep calling `add`
+            # meanwhile, and a loop over a dict that gets a new key fails with "dictionary changed size during
+            # iteration" - which would abort the eviction. Codes added while draining stay in the live dict.
+            drained = [self._visitors.popitem()[0] for _ in range(len(self._visitors))]
+            # `peek_visitor`: an eviction pass must not count as activity and extend the visitors' sessions.
+            self._visitors.update(
+                dict.fromkeys(vc for vc in drained if self._visitor_manager.peek_visitor(vc) is not None)
+            )
+            visitors_to_remove_count = len(self._visitors) - self._storage_limit_after_removal()
+            for _ in range(visitors_to_remove_count):
+                self._visitors.popitem()
+
+    def extract(self, limit: int = VisitorTrackingRegistry.UNLIMITED) -> List[str]:
+        with self._extract_lock:
+            count = self._extraction_count(len(self._visitors), limit)
+            return [self._visitors.popitem()[0] for _ in range(count)]