leaf-common 1.2.40__tar.gz → 1.2.41__tar.gz

leaf_common-1.2.41/.github/workflows/publish.yml

@@ -0,0 +1,46 @@
+name: Publish to PyPI
+
+permissions:
+  contents: read
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    name: Build and Publish
+    if: github.repository == 'cognizant-ai-lab/leaf-common'
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # Mandatory for PyPI Trusted Publishing
+
+    steps:
+      # v6.0.2
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+
+      - name: Setup Python environment
+        # 1.0.6
+        uses: cognizant-ai-lab/build-common/actions/setup-python-env@e5f749a7ba5de8bb8106cb172e89f21ca581a697
+        with:
+          python-version: '3.10'
+          requirements-file: ''
+          requirements-build-file: ''
+          install-extras: 'build'
+
+      - name: Build wheel and sdist
+        run: python -m build
+
+      # v1.14.0
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b
+
+      - name: Notify Slack
+        if: ${{ !cancelled() }}
+        # 1.0.6
+        uses: cognizant-ai-lab/build-common/actions/slack-notify@e5f749a7ba5de8bb8106cb172e89f21ca581a697
+        with:
+          status: ${{ job.status }}
+          webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}

leaf_common-1.2.41/.github/workflows/tests.yml

@@ -0,0 +1,31 @@
+---
+name: Test
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - "**"
+  pull_request:
+    types: [opened, reopened]
+    branches:
+      - main
+jobs:
+  test:
+    if: >
+      (github.event_name == 'push' && github.ref != 'refs/heads/main') ||
+      (github.event_name == 'pull_request' && github.base_ref == 'main')
+    # 1.0.6
+    uses: cognizant-ai-lab/build-common/.github/workflows/_python-quality-gate.yml@e5f749a7ba5de8bb8106cb172e89f21ca581a697
+    with:
+      python-version: '3.12'
+      lint-toolchain: 'legacy'
+      pylint-command: 'build_scripts/run_pylint.sh'
+      run-shellcheck: true
+      pytest-markers: 'not integration'
+      pytest-extra-args: '--timer-top-n 10 --cov-config=.coveragerc --cov'
+      check-readme-pypi: true
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

{leaf_common-1.2.40 → leaf_common-1.2.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: leaf-common
-Version: 1.2.40
+Version: 1.2.41
 Summary: LEAF team common code library
 Author-email: Dan Fink <Daniel.Fink@cognizant.com>
 License-Expression: Apache-2.0

{leaf_common-1.2.40 → leaf_common-1.2.41}/leaf_common/asyncio/asyncio_executor.py

@@ -22,6 +22,7 @@ from typing import Awaitable
 from typing import Callable
 from typing import Dict
 from typing import List
+from typing import Tuple
 
 import asyncio
 import functools
@@ -36,6 +37,7 @@ from asyncio import Task
 from concurrent import futures
 
 from leaf_common.asyncio.task_executor import TaskExecutor
+from leaf_common.asyncio.asyncio_threadpool_executor import AsyncioThreadPoolExecutor
 
 EXECUTOR_START_TIMEOUT_SECONDS: int = 5
 
@@ -57,8 +59,10 @@ class AsyncioExecutor(TaskExecutor):
         self._thread: threading.Thread = None
         # We are going to start new thread for this Executor,
         # so we need a new event loop bound to this particular thread:
+        self._threadpool_executor: AsyncioThreadPoolExecutor = AsyncioThreadPoolExecutor()
         self._loop: AbstractEventLoop = asyncio.new_event_loop()
         self._loop.set_exception_handler(AsyncioExecutor.loop_exception_handler)
+        self._loop.set_default_executor(self._threadpool_executor)
         self._loop_ready = threading.Event()
         self._init_done = threading.Event()
         self._background_tasks: Dict[int, Dict[str, Any]] = {}
@@ -446,6 +450,16 @@ class AsyncioExecutor(TaskExecutor):
         with self._background_tasks_lock:
             self._background_tasks.pop(task_id, None)
 
+    def get_threads_metrics(self) -> Tuple[int, int]:
+        """
+        For ThreadExecutorPool used by our event loop,
+        get number of created threads and number of currently running threads.
+         :return: Tuple of (number of threads in the pool, number of currently running threads)
+        """
+        if self._threadpool_executor:
+            return self._threadpool_executor.get_threads_metrics()
+        return 0, 0
+
     def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
         """
         Shuts down the event loop.

leaf_common-1.2.41/leaf_common/asyncio/asyncio_executor_pool.py

@@ -0,0 +1,134 @@
+
+# Copyright © 2019-2026 Cognizant Technology Solutions Corp, www.cognizant.com.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# END COPYRIGHT
+"""
+See class comments
+"""
+from typing import Any
+from typing import Dict
+
+import copy
+import logging
+import threading
+from leaf_common.asyncio.asyncio_executor import AsyncioExecutor
+
+
+class AsyncioExecutorPool:
+    """
+    Class maintaining a dynamic set of reusable AsyncioExecutor instances.
+    """
+
+    def __init__(self, reuse_mode: bool = True):
+        """
+        Constructor.
+        :param reuse_mode: True, if requested executor instances
+                                 are taken from pool of available ones (pool mode);
+                           False, if requested executor instances are created new
+                                 and shutdown on return (backward compatible mode)
+        """
+        self.reuse_mode: bool = reuse_mode
+        # List of available (not currently used) AsyncioExecutor instances in the pool.
+        self.pool_available = []
+
+        # List of currently used AsyncioExecutor instances in the pool.
+        self.pool_used = []
+
+        self.lock = threading.Lock()
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger.debug("AsyncioExecutorPool created: %s reuse: %s",
+                          id(self), str(self.reuse_mode))
+
+    def get_executor(self) -> AsyncioExecutor:
+        """
+        Get active (running) executor from the pool
+        :return: AsyncioExecutor instance
+        """
+        if self.reuse_mode:
+            with self.lock:
+                if len(self.pool_available) > 0:
+                    result = self.pool_available.pop(0)
+                    self.logger.debug("Reusing AsyncioExecutor %s", id(result))
+                    self.pool_used.append(result)
+                    return result
+        # Create AsyncioExecutor outside of lock
+        # to avoid potentially longer locked periods
+        result = AsyncioExecutor()
+        result.start()
+        self.logger.debug("Creating AsyncioExecutor %s", id(result))
+        with self.lock:
+            self.pool_used.append(result)
+        return result
+
+    def return_executor(self, executor: AsyncioExecutor):
+        """
+        Return AsyncioExecutor instance back to the pool of available instances.
+        :param executor: AsyncioExecutor to return.
+        """
+        with self.lock:
+            if executor not in self.pool_used:
+                raise ValueError(f"Returned executor {id(executor)} is not in the pool of used executors")
+            self.pool_used.remove(executor)
+
+        # Executor clean up: cancel current tasks and shutdown if not in reuse mode.
+        if self.reuse_mode:
+            executor.cancel_current_tasks()
+        else:
+            self.logger.debug("Shutting down: AsyncioExecutor %s", id(executor))
+            executor.shutdown()
+
+        if self.reuse_mode:
+            with self.lock:
+                self.pool_available.append(executor)
+                self.logger.debug("Returned to pool: AsyncioExecutor %s pool size: %d",
+                                  id(executor), len(self.pool_available))
+
+    def get_threads_metrics(self) -> Dict[str, Any]:
+        """
+        Get metrics related to threads in the pool of executors:
+        for both collections of used and available executors,
+        get the total number of executors in this collection "executors",
+        the total number of work threads across all executors in this collection "work_threads",
+        and the total number of currently running work threads
+        across all executors in this collection "threads_running".
+        """
+        with self.lock:
+            available_copy = copy.copy(self.pool_available)
+            used_copy = copy.copy(self.pool_used)
+        used_threads: int = 0
+        used_running: int = 0
+        available_threads: int = 0
+        available_running: int = 0
+        for executor in used_copy:
+            threads, running = executor.get_threads_metrics()
+            used_threads += threads
+            used_running += running
+        for executor in available_copy:
+            threads, running = executor.get_threads_metrics()
+            available_threads += threads
+            available_running += running
+        result_dict = {
+            "used": {
+                "executors": len(used_copy),
+                "work_threads": used_threads,
+                "threads_running": used_running
+            },
+            "available": {
+                "executors": len(available_copy),
+                "work_threads": available_threads,
+                "threads_running": available_running
+            }
+        }
+        return result_dict

leaf_common-1.2.41/leaf_common/asyncio/asyncio_threadpool_executor.py

@@ -0,0 +1,71 @@
+
+# Copyright © 2019-2026 Cognizant Technology Solutions Corp, www.cognizant.com.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# END COPYRIGHT
+"""
+See class comment for details.
+"""
+from typing import Tuple
+from concurrent.futures import ThreadPoolExecutor
+import logging
+import threading
+
+
+class AsyncioThreadPoolExecutor(ThreadPoolExecutor):
+    """
+    Class instrumenting a ThreadPoolExecutor with some additional run-time metrics
+    such as number of created and active threads.
+    """
+    def __init__(self, *args, **kwargs):
+        """Constructor."""
+        super().__init__(*args, **kwargs)
+        self.running: int = 0
+        self.lock = threading.Lock()
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.no_threads_warning_logged: bool = False
+
+    def submit(self, fn, /, *args, **kwargs):
+        """
+        Override of submit method to wrap the submitted function with additional logic
+        for counting the number of active (running) threads.
+        """
+        def wrapped(*a, **kw):
+            with self.lock:
+                self.running += 1
+            try:
+                return fn(*a, **kw)
+            finally:
+                with self.lock:
+                    self.running -= 1
+
+        return super().submit(wrapped, *args, **kwargs)
+
+    def get_threads_metrics(self) -> Tuple[int, int]:
+        """
+        Get number of threads in the pool and number of currently running threads.
+         :return: Tuple of (number of threads in the pool, number of currently running threads)
+         Note: number of threads in the pool is an approximation, as ThreadPoolExecutor does not provide
+               a direct way to get this information. The method tries to access the protected member _threads,
+               which may not be available in all implementations of ThreadPoolExecutor.
+        """
+        num_threads: int = 0
+        if hasattr(self, "_threads"):
+            num_threads = len(self._threads)
+        elif not self.no_threads_warning_logged:
+            self.logger.warning("ThreadPoolExecutor does not have _threads attribute, "
+                                "number of threads in the pool will be reported as 0")
+            self.no_threads_warning_logged = True
+        with self.lock:
+            return num_threads, self.running

{leaf_common-1.2.40 → leaf_common-1.2.41}/leaf_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: leaf-common
-Version: 1.2.40
+Version: 1.2.41
 Summary: LEAF team common code library
 Author-email: Dan Fink <Daniel.Fink@cognizant.com>
 License-Expression: Apache-2.0

{leaf_common-1.2.40 → leaf_common-1.2.41}/leaf_common.egg-info/SOURCES.txt

@@ -13,7 +13,6 @@ SUPPORT.md
 pyproject.toml
 requirements-build.txt
 requirements.txt
-.github/workflows/codeql.yml
 .github/workflows/publish.yml
 .github/workflows/tests.yml
 build_scripts/copyright_config.json
@@ -30,6 +29,7 @@ leaf_common/asyncio/__init__.py
 leaf_common/asyncio/async_to_sync_generator.py
 leaf_common/asyncio/asyncio_executor.py
 leaf_common/asyncio/asyncio_executor_pool.py
+leaf_common/asyncio/asyncio_threadpool_executor.py
 leaf_common/asyncio/task_executor.py
 leaf_common/candidates/__init__.py
 leaf_common/candidates/constants.py
@@ -220,8 +220,11 @@ leaf_common/utils/atomic_counter.py
 tests/__init__.py
 tests/asyncio/__init__.py
 tests/asyncio/async_test_helpers.py
+tests/asyncio/asyncio_executor_metrics_test.py
 tests/asyncio/asyncio_executor_not_started_test.py
+tests/asyncio/asyncio_executor_pool_metrics_test.py
 tests/asyncio/asyncio_executor_started_test.py
+tests/asyncio/asyncio_threadpool_executor_metrics_test.py
 tests/asyncio/callable_class.py
 tests/asyncio/sync_test_helpers.py
 tests/candidates/__init__.py

leaf_common-1.2.41/tests/asyncio/asyncio_executor_metrics_test.py

@@ -0,0 +1,121 @@
+
+# Copyright © 2019-2026 Cognizant Technology Solutions Corp, www.cognizant.com.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# END COPYRIGHT
+"""
+Unit tests for AsyncioExecutor.get_threads_metrics().
+
+The method delegates to the executor's internal AsyncioThreadPoolExecutor
+when one is present, or returns (0, 0) otherwise. The behavior of the
+threadpool itself is covered in asyncio_threadpool_executor_metrics_test;
+this module isolates AsyncioExecutor's delegation logic.
+"""
+import threading
+
+from unittest import TestCase
+
+from leaf_common.asyncio.asyncio_executor import AsyncioExecutor
+from tests.asyncio.sync_test_helpers import SyncTestHelpers
+
+
+class AsyncioExecutorMetricsTest(TestCase):
+    """
+    Verifies that AsyncioExecutor.get_threads_metrics() returns the same
+    (threads, running) tuple as the underlying AsyncioThreadPoolExecutor,
+    and that the fallback path returns (0, 0) when no threadpool is set.
+    """
+
+    def setUp(self):
+        """Create and start a fresh executor."""
+        self.executor = AsyncioExecutor()
+        self.executor.start()
+
+    def tearDown(self):
+        """Always shutdown so the event-loop thread terminates cleanly."""
+        if self.executor is not None:
+            self.executor.shutdown(wait=True)
+
+    def test_metrics_on_idle_executor_report_zero(self):
+        """
+        Immediately after start(), no work has been dispatched to the
+        threadpool, so both counts are zero.
+        """
+        num_threads, num_running = self.executor.get_threads_metrics()
+        self.assertEqual(
+            0, num_threads,
+            f"Expected 0 worker threads on an idle executor, got {num_threads}.",
+        )
+        self.assertEqual(
+            0, num_running,
+            f"Expected 0 running tasks on an idle executor, got {num_running}.",
+        )
+
+    def test_metrics_reflect_work_dispatched_to_internal_threadpool(self):
+        """
+        AsyncioExecutor.get_threads_metrics() delegates to its internal
+        AsyncioThreadPoolExecutor. Submit work directly to that threadpool
+        (the same one the executor's event loop dispatches sync work to via
+        run_in_executor) and verify the delegated metrics reflect it.
+        """
+        start_event = threading.Event()
+        release_event = threading.Event()
+
+        # pylint: disable=protected-access
+        threadpool = self.executor._threadpool_executor
+        future = threadpool.submit(SyncTestHelpers.block_on_event, start_event, release_event)
+        self.assertTrue(
+            start_event.wait(timeout=5.0),
+            "Helper task never entered its body within 5s.",
+        )
+
+        num_threads, num_running = self.executor.get_threads_metrics()
+
+        # Release and drain before asserting, so failures do not leak
+        # a blocked worker.
+        release_event.set()
+        future.result(timeout=5.0)
+
+        self.assertGreaterEqual(
+            num_threads, 1,
+            f"Expected at least 1 worker thread mid-task, got {num_threads}.",
+        )
+        self.assertEqual(
+            1, num_running,
+            f"Expected exactly 1 running task while the helper was blocked, "
+            f"got {num_running}.",
+        )
+
+    def test_metrics_return_zero_tuple_when_threadpool_is_absent(self):
+        """
+        AsyncioExecutor.get_threads_metrics() short-circuits to (0, 0) if
+        the internal _threadpool_executor reference is falsy. Simulate
+        that branch by clearing the attribute. Avoids needing a real
+        environment where the threadpool failed to construct.
+        """
+        # pylint: disable=protected-access
+        self.executor._threadpool_executor = None
+
+        num_threads, num_running = self.executor.get_threads_metrics()
+
+        self.assertEqual(
+            0, num_threads,
+            f"Expected fallback of 0 worker threads when threadpool is absent, "
+            f"got {num_threads}.",
+        )
+        self.assertEqual(
+            0, num_running,
+            f"Expected 0 running tasks when threadpool is absent, "
+            f"got {num_running}.",
+        )