winipedia-utils 0.1.0__py3-none-any.whl

winipedia_utils/__init__.py

@@ -0,0 +1 @@
+"""__init__ module for winipedia_utils."""

winipedia_utils/concurrent/__init__.py

@@ -0,0 +1 @@
+"""__init__ module for winipedia_utils.concurrent."""

winipedia_utils/concurrent/concurrent.py

@@ -0,0 +1,242 @@
+"""Concurrent processing utilities for parallel execution.
+
+This module provides functions for concurrent processing using both multiprocessing
+and multithreading approaches. It includes utilities for handling timeouts,
+managing process pools, and organizing parallel execution of functions.
+
+Returns:
+    Various utility functions for concurrent processing.
+
+"""
+
+import multiprocessing
+import os
+import threading
+from collections.abc import Callable, Generator, Iterable
+from concurrent.futures import ThreadPoolExecutor
+from copy import deepcopy
+from functools import partial
+from multiprocessing.pool import Pool
+from typing import Any, cast
+
+from tqdm import tqdm
+
+from winipedia_utils.concurrent.multithreading import imap_unordered
+from winipedia_utils.iterating.iterate import get_len_with_default
+from winipedia_utils.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_order_and_func_result(
+    func_order_args: tuple[Any, ...],
+) -> tuple[int, Any]:
+    """Process function for imap with arguments unpacking.
+
+    Helper function that gives back a function that can be used with imap_unordered
+    to execute a function with arguments unpacking.
+
+    Args:
+        func_order_args: Tuple containing the function to be executed,
+            the order index, and the arguments for the function
+
+    Returns:
+        A tuple containing the order index and the result of the function execution
+
+    """
+    function, order, *args = func_order_args
+    return order, function(*args)
+
+
+def generate_process_args(
+    *,
+    process_function: Callable[..., Any],
+    process_args: Iterable[Iterable[Any]],
+    process_args_static: Iterable[Any] | None = None,
+    deepcopy_static_args: Iterable[Any] | None = None,
+) -> Generator[tuple[Any, ...], None, None]:
+    """Prepare arguments for multiprocessing or multithreading execution.
+
+    Converts input arguments into a format suitable for parallel processing,
+    organizing them for efficient unpacking during execution. The function:
+    1. Prepends process func and order indices to arguments
+    2. Handles static arguments (with optional deep copying)
+    3. Restructures arguments into tuples for unpacking
+
+    Args:
+        process_function: Function to be executed
+        process_args: Iterable of argument lists for each parallel call
+        process_args_static: Optional constant arguments to add to each call
+        deepcopy_static_args: Optional constant arguments that should be deep-copied
+
+    Returns:
+        A Genrator that yields one args tuple for each function call
+        First is the process function
+        Second item in the tuple is the order index
+        Second item in the tuple is the function
+        Rest of the items are the arguments for the function
+        The length of the generator
+    """
+    process_args_static = (
+        () if process_args_static is None else tuple(process_args_static)
+    )
+    deepcopy_static_args = (
+        () if deepcopy_static_args is None else tuple(deepcopy_static_args)
+    )
+    for order, process_arg in enumerate(process_args):
+        yield (
+            process_function,
+            order,
+            *process_arg,
+            *process_args_static,
+            *(
+                deepcopy(deepcopy_static_arg)
+                for deepcopy_static_arg in deepcopy_static_args
+            ),
+        )
+
+
+def get_multiprocess_results_with_tqdm(
+    results: Iterable[Any],
+    process_func: Callable[..., Any],
+    process_args_len: int,
+    *,
+    threads: bool,
+) -> list[Any]:
+    """Get multiprocess results with tqdm progress tracking.
+
+    Processes results from parallel execution with a progress bar and ensures
+    they are returned in the original order.
+
+    Args:
+        results: Iterable of results from parallel execution
+        process_func: Function that was executed in parallel
+        process_args_len: Number of items to process in parallel
+        threads: Whether threading (True) or multiprocessing (False) was used
+
+    Returns:
+        list[Any]: Results from parallel execution in original order
+
+    """
+    results = tqdm(
+        results,
+        total=process_args_len,
+        desc=f"Multi{'threading' if threads else 'processing'} {process_func.__name__}",
+        unit=f" {'threads' if threads else 'processes'}",
+    )
+    results_list = list(results)
+    # results list is a tuple of (order, result),
+    # so we need to sort it by order to get the original order
+    results_list = sorted(results_list, key=lambda x: x[0])
+    # now extract the results from the tuple
+    return [result[1] for result in results_list]
+
+
+def find_max_pools(
+    *,
+    threads: bool,
+    process_args_len: int | None = None,
+) -> int:
+    """Find optimal number of worker processes or threads for parallel execution.
+
+    Determines the maximum number of worker processes or threads based on system
+    resources, active tasks, and the number of items to process.
+
+    Args:
+        threads: Whether to use threading (True) or multiprocessing (False)
+        process_args_len: Number of items to process in parallel
+
+    Returns:
+        int: Maximum number of worker processes or threads to use
+
+    """
+    # use tee to find length of process_args
+    cpu_count = os.cpu_count() or 1
+    if threads:
+        active_tasks = threading.active_count()
+        max_tasks = cpu_count * 4
+    else:
+        active_tasks = len(multiprocessing.active_children())
+        max_tasks = cpu_count
+
+    available_tasks = max_tasks - active_tasks
+    max_pools = (
+        min(available_tasks, process_args_len) if process_args_len else available_tasks
+    )
+    max_pools = max(max_pools, 1)
+
+    logger.info(
+        "Multi%s with max_pools: %s",
+        "threading" if threads else "processing",
+        max_pools,
+    )
+
+    return max_pools
+
+
+def concurrent_loop(  # noqa: PLR0913
+    *,
+    threading: bool,
+    process_function: Callable[..., Any],
+    process_args: Iterable[Iterable[Any]],
+    process_args_static: Iterable[Any] | None = None,
+    deepcopy_static_args: Iterable[Any] | None = None,
+    process_args_len: int = 1,
+) -> list[Any]:
+    """Execute a function concurrently with multiple arguments using a pool executor.
+
+    This function is a helper function for multiprocess_loop and multithread_loop.
+    It is not meant to be used directly.
+
+    Args:
+        threading (bool):
+            Whether to use threading (True) or multiprocessing (False)
+        pool_executor (Pool | ThreadPoolExecutor):
+            Pool executor to use for concurrent execution
+        process_function (Callable[..., Any]):
+            Function to be executed concurrently
+        process_args (Iterable[Iterable[Any]]):
+            Arguments for each process
+        process_args_static (Iterable[Any] | None, optional):
+            Static arguments to pass to each process. Defaults to None.
+        deepcopy_static_args (Iterable[Any] | None, optional):
+            Arguments that should be deep-copied for each process. Defaults to None.
+        process_args_len (int | None, optional):
+            Length of process_args. Defaults to None.
+
+    Returns:
+        list[Any]: Results from the process_function executions
+    """
+    process_args_len = get_len_with_default(process_args, process_args_len)
+    process_args = generate_process_args(
+        process_function=process_function,
+        process_args=process_args,
+        process_args_static=process_args_static,
+        deepcopy_static_args=deepcopy_static_args,
+    )
+    max_workers = find_max_pools(threads=threading, process_args_len=process_args_len)
+    pool_executor = (
+        ThreadPoolExecutor(max_workers=max_workers)
+        if threading
+        else Pool(processes=max_workers)
+    )
+    with pool_executor as pool:
+        map_func: Callable[[Callable[..., Any], Iterable[Any]], Any]
+
+        if process_args_len == 1:
+            map_func = map
+        elif threading:
+            pool = cast("ThreadPoolExecutor", pool)
+            map_func = partial(imap_unordered, pool)
+        else:
+            pool = cast("Pool", pool)
+            map_func = pool.imap_unordered
+
+        results = map_func(get_order_and_func_result, process_args)
+
+        return get_multiprocess_results_with_tqdm(
+            results=results,
+            process_func=process_function,
+            process_args_len=process_args_len,
+            threads=threading,
+        )

winipedia_utils/concurrent/multiprocessing.py

@@ -0,0 +1,115 @@
+"""Multiprocessing utilities for concurrent execution.
+
+This module provides functions for parallel processing using both multiprocessing
+and multithreading approaches. It includes utilities for handling timeouts,
+managing process pools, and organizing parallel execution of functions.
+
+Returns:
+    Various utility functions for concurrent processing.
+
+"""
+
+import multiprocessing
+from collections.abc import Callable, Iterable
+from functools import wraps
+from multiprocessing.pool import Pool
+from typing import Any
+
+from winipedia_utils.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def cancel_on_timeout(seconds: float, message: str) -> Callable[..., Any]:
+    """Cancel a function execution if it exceeds a specified timeout.
+
+    Creates a wrapper that executes the decorated function in a separate process
+    and terminates it if execution time exceeds the specified timeout.
+
+    Args:
+        seconds: Maximum execution time in seconds before timeout
+        message: Error message to include in the raised TimeoutError
+
+    Returns:
+        A decorator function that wraps the target function with timeout functionality
+
+    Raises:
+        multiprocessing.TimeoutError: When function execution exceeds the timeout
+
+    Note:
+        Only works with functions that are pickle-able.
+        This means it may not work as a decorator.
+        Instaed you should use it as a wrapper function.
+        Like this:
+        my_func = cancel_on_timeout(seconds=2, message="Test timeout")(my_func)
+
+    """
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args: object, **kwargs: object) -> object:
+            with Pool(processes=1) as pool:
+                async_result = pool.apply_async(func, args, kwargs)
+                try:
+                    return async_result.get(timeout=seconds)
+                except multiprocessing.TimeoutError:
+                    logger.warning(
+                        "%s -> Execution exceeded %s seconds: %s",
+                        func.__name__,
+                        seconds,
+                        message,
+                    )
+                    raise
+                finally:
+                    pool.terminate()  # Ensure the worker process is killed
+                    pool.join()  # Wait for cleanup
+
+        return wrapper
+
+    return decorator
+
+
+def multiprocess_loop(
+    process_function: Callable[..., Any],
+    process_args: Iterable[Iterable[Any]],
+    process_args_static: Iterable[Any] | None = None,
+    deepcopy_static_args: Iterable[Any] | None = None,
+    process_args_len: int = 1,
+) -> list[Any]:
+    """Process a loop using multiprocessing Pool for parallel execution.
+
+    Executes the given process_function with the provided arguments in parallel using
+    multiprocessing Pool, which is suitable for CPU-bound tasks.
+
+    Args:
+        process_function: Function that processes the given process_args
+        process_args: List of args to be processed by the process_function
+                    e.g. [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        process_args_static: Optional constant arguments passed to each function call
+        deepcopy_static_args: Optional arguments that should be
+                              deep-copied for each process
+        process_args_len: Optional length of process_args
+                          If not provided, it will ot be taken into account
+                          when calculating the max number of processes.
+
+    Returns:
+        List of results from the process_function executions
+
+    Note:
+        Pool is used for CPU-bound tasks as it bypasses
+        Python's GIL by creating separate processes.
+        Multiprocessing is not safe for mutable objects unlike ThreadPoolExecutor.
+        When debugging, if ConnectionErrors occur, set max_processes to 1.
+        Also given functions must be pickle-able.
+
+    """
+    from winipedia_utils.concurrent.concurrent import concurrent_loop
+
+    return concurrent_loop(
+        threading=False,
+        process_function=process_function,
+        process_args=process_args,
+        process_args_static=process_args_static,
+        deepcopy_static_args=deepcopy_static_args,
+        process_args_len=process_args_len,
+    )

winipedia_utils/concurrent/multithreading.py

@@ -0,0 +1,93 @@
+"""Multithreading utilities for concurrent execution.
+
+This module provides functions for parallel processing using thread pools.
+It includes utilities for handling thread pools, managing futures, and organizing
+parallel execution of I/O-bound tasks.
+Base helper functions that serve threading and processing are located in the
+multiprocessing module.
+
+Returns:
+    Various utility functions for multithreaded processing.
+
+"""
+
+from collections.abc import Callable, Generator, Iterable
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Any
+
+
+def get_future_results_as_completed(
+    futures: Iterable[Future[Any]],
+) -> Generator[Any, None, None]:
+    """Get future results as they complete.
+
+    Yields results from futures in the order they complete,
+    not in the order they were submitted.
+
+    Args:
+        futures: List of Future objects to get results from
+
+    Yields:
+        The result of each completed future
+
+    """
+    for future in as_completed(futures):
+        yield future.result()
+
+
+def multithread_loop(
+    process_function: Callable[..., Any],
+    process_args: Iterable[Iterable[Any]],
+    process_args_static: Iterable[Any] | None = None,
+    process_args_len: int = 1,
+) -> list[Any]:
+    """Process a loop using ThreadPoolExecutor for parallel execution.
+
+    Executes the given process_function with the provided arguments in parallel using
+    ThreadPoolExecutor, which is suitable for I/O-bound tasks.
+
+    Args:
+        process_function: Function that processes the given process_args
+        process_args: list of args to be processed by the process_function
+                    e.g. [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        process_args_static: Optional constant arguments passed to each function call
+        process_args_len: Optional length of process_args
+                          If not provided, it will ot be taken into account
+                          when calculating the max number of workers.
+
+    Returns:
+        List of results from the process_function executions
+
+    Note:
+        ThreadPoolExecutor is used for I/O-bound tasks, not for CPU-bound tasks.
+
+    """
+    from winipedia_utils.concurrent.concurrent import concurrent_loop
+
+    return concurrent_loop(
+        threading=True,
+        process_function=process_function,
+        process_args=process_args,
+        process_args_static=process_args_static,
+        process_args_len=process_args_len,
+    )
+
+
+def imap_unordered(
+    executor: ThreadPoolExecutor,
+    func: Callable[..., Any],
+    iterable: Iterable[Any],
+) -> Generator[Any, None, None]:
+    """Apply a function to each item in an iterable in parallel.
+
+    Args:
+        executor: ThreadPoolExecutor to use for parallel execution
+        func: Function to apply to each item in the iterable
+        iterable: Iterable of items to apply the function to
+
+    Yields:
+        Results of applying the function to each item in the iterable
+
+    """
+    results = [executor.submit(func, item) for item in iterable]
+    yield from get_future_results_as_completed(results)

winipedia_utils/consts.py

@@ -0,0 +1,22 @@
+"""Constants used throughout the winipedia_utils package.
+
+This module contains package-wide constants that are used by various
+modules within the package. These constants define core configuration
+values and identifiers for the package.
+"""
+
+PACKAGE_NAME = "winipedia_utils"
+
+_DEV_DEPENDENCIES = [
+    "ruff",
+    "pre-commit",
+    "mypy",
+    "pytest",
+    "bandit",
+    "types-setuptools",
+    "types-tqdm",
+    "types-defusedxml",
+    "types-pyyaml",
+    "pytest-mock",
+    "django-stubs",
+]

winipedia_utils/data/__init__.py

@@ -0,0 +1 @@
+"""__init__ module for winipedia_utils.data."""

winipedia_utils/data/dataframe.py

@@ -0,0 +1,7 @@
+"""Dataframe utilities for data manipulation and analysis.
+
+This module provides utility functions for working with pandas DataFrames,
+including data cleaning, transformation, and aggregation operations.
+These utilities help with data preprocessing and analysis tasks.
+
+"""

winipedia_utils/django/__init__.py

@@ -0,0 +1,27 @@
+"""__init__ module for winipedia_utils.django."""
+
+import django
+import django_stubs_ext
+from django.conf import settings
+
+from winipedia_utils.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+django_stubs_ext.monkeypatch()
+logger.info("Monkeypatched django-stubs")
+
+if not settings.configured:
+    logger.info("Configuring minimal django settings")
+    settings.configure(
+        INSTALLED_APPS=[
+            "django.contrib.contenttypes",
+        ],
+        DATABASES={
+            "default": {
+                "ENGINE": "django.db.backends.sqlite3",
+                "NAME": ":memory:",
+            }
+        },
+    )
+    django.setup()