winipedia-utils 0.2.9__py3-none-any.whl → 0.2.17__py3-none-any.whl

winipedia_utils/concurrent/multiprocessing.py

@@ -1,130 +1,130 @@
-"""Multiprocessing utilities for concurrent execution.
-
-This module provides functions for parallel processing using both multiprocessing
-and multithreading approaches. It includes utilities for handling timeouts,
-managing process pools, and organizing parallel execution of functions.
-
-Returns:
-    Various utility functions for concurrent processing.
-
-"""
-
-import multiprocessing
-from collections.abc import Callable, Iterable
-from functools import wraps
-from multiprocessing.pool import Pool
-from typing import Any
-
-from winipedia_utils.logging.logger import get_logger
-
-logger = get_logger(__name__)
-
-
-def get_spwan_pool(*args: Any, **kwargs: Any) -> Pool:
-    """Get a multiprocessing pool with the spawn context.
-
-    Args:
-        *args: Positional arguments to pass to the Pool constructor
-        **kwargs: Keyword arguments to pass to the Pool constructor
-
-    Returns:
-        A multiprocessing pool with the spawn context
-
-    """
-    return multiprocessing.get_context("spawn").Pool(*args, **kwargs)
-
-
-def cancel_on_timeout(seconds: float, message: str) -> Callable[..., Any]:
-    """Cancel a function execution if it exceeds a specified timeout.
-
-    Creates a wrapper that executes the decorated function in a separate process
-    and terminates it if execution time exceeds the specified timeout.
-
-    Args:
-        seconds: Maximum execution time in seconds before timeout
-        message: Error message to include in the raised TimeoutError
-
-    Returns:
-        A decorator function that wraps the target function with timeout functionality
-
-    Raises:
-        multiprocessing.TimeoutError: When function execution exceeds the timeout
-
-    Note:
-        Only works with functions that are pickle-able.
-        This means it may not work as a decorator.
-        Instaed you should use it as a wrapper function.
-        Like this:
-        my_func = cancel_on_timeout(seconds=2, message="Test timeout")(my_func)
-
-    """
-
-    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
-        @wraps(func)
-        def wrapper(*args: object, **kwargs: object) -> object:
-            spawn_pool = get_spwan_pool(processes=1)
-            with spawn_pool as pool:
-                async_result = pool.apply_async(func, args, kwargs)
-                try:
-                    return async_result.get(timeout=seconds)
-                except multiprocessing.TimeoutError:
-                    logger.warning(
-                        "%s -> Execution exceeded %s seconds: %s",
-                        func.__name__,
-                        seconds,
-                        message,
-                    )
-                    raise
-                finally:
-                    pool.terminate()  # Ensure the worker process is killed
-                    pool.join()  # Wait for cleanup
-
-        return wrapper
-
-    return decorator
-
-
-def multiprocess_loop(
-    process_function: Callable[..., Any],
-    process_args: Iterable[Iterable[Any]],
-    process_args_static: Iterable[Any] | None = None,
-    deepcopy_static_args: Iterable[Any] | None = None,
-    process_args_len: int = 1,
-) -> list[Any]:
-    """Process a loop using multiprocessing Pool for parallel execution.
-
-    Executes the given process_function with the provided arguments in parallel using
-    multiprocessing Pool, which is suitable for CPU-bound tasks.
-
-    Args:
-        process_function: Function that processes the given process_args
-        process_args: List of args to be processed by the process_function
-                    e.g. [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
-        process_args_static: Optional constant arguments passed to each function call
-        deepcopy_static_args: Optional arguments that should be
-                              deep-copied for each process
-        process_args_len: Optional length of process_args
-                          If not provided, it will ot be taken into account
-                          when calculating the max number of processes.
-
-    Returns:
-        List of results from the process_function executions
-
-    Note:
-        Pool is used for CPU-bound tasks as it bypasses
-        Python's GIL by creating separate processes.
-        Multiprocessing is not safe for mutable objects unlike ThreadPoolExecutor.
-        When debugging, if ConnectionErrors occur, set max_processes to 1.
-        Also given functions must be pickle-able.
-
-    """
-    from winipedia_utils.concurrent.concurrent import concurrent_loop
-
-    return concurrent_loop(
-        threading=False,
-        process_function=process_function,
-        process_args=process_args,
-        process_args_static=process_args_static,
-        deepcopy_static_args=deepcopy_static_args,
-        process_args_len=process_args_len,
-    )
+"""Multiprocessing utilities for concurrent execution.
+
+This module provides functions for parallel processing using both multiprocessing
+and multithreading approaches. It includes utilities for handling timeouts,
+managing process pools, and organizing parallel execution of functions.
+
+Returns:
+    Various utility functions for concurrent processing.
+
+"""
+
+import multiprocessing
+from collections.abc import Callable, Iterable
+from functools import wraps
+from multiprocessing.pool import Pool
+from typing import Any
+
+from winipedia_utils.logging.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_spwan_pool(*args: Any, **kwargs: Any) -> Pool:
+    """Get a multiprocessing pool with the spawn context.
+
+    Args:
+        *args: Positional arguments to pass to the Pool constructor
+        **kwargs: Keyword arguments to pass to the Pool constructor
+
+    Returns:
+        A multiprocessing pool with the spawn context
+
+    """
+    return multiprocessing.get_context("spawn").Pool(*args, **kwargs)
+
+
+def cancel_on_timeout(seconds: float, message: str) -> Callable[..., Any]:
+    """Cancel a function execution if it exceeds a specified timeout.
+
+    Creates a wrapper that executes the decorated function in a separate process
+    and terminates it if execution time exceeds the specified timeout.
+
+    Args:
+        seconds: Maximum execution time in seconds before timeout
+        message: Error message to include in the raised TimeoutError
+
+    Returns:
+        A decorator function that wraps the target function with timeout functionality
+
+    Raises:
+        multiprocessing.TimeoutError: When function execution exceeds the timeout
+
+    Note:
+        Only works with functions that are pickle-able.
+        This means it may not work as a decorator.
+        Instaed you should use it as a wrapper function.
+        Like this:
+        my_func = cancel_on_timeout(seconds=2, message="Test timeout")(my_func)
+
+    """
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def wrapper(*args: object, **kwargs: object) -> object:
+            spawn_pool = get_spwan_pool(processes=1)
+            with spawn_pool as pool:
+                async_result = pool.apply_async(func, args, kwargs)
+                try:
+                    return async_result.get(timeout=seconds)
+                except multiprocessing.TimeoutError:
+                    logger.warning(
+                        "%s -> Execution exceeded %s seconds: %s",
+                        func.__name__,
+                        seconds,
+                        message,
+                    )
+                    raise
+                finally:
+                    pool.terminate()  # Ensure the worker process is killed
+                    pool.join()  # Wait for cleanup
+
+        return wrapper
+
+    return decorator
+
+
+def multiprocess_loop(
+    process_function: Callable[..., Any],
+    process_args: Iterable[Iterable[Any]],
+    process_args_static: Iterable[Any] | None = None,
+    deepcopy_static_args: Iterable[Any] | None = None,
+    process_args_len: int = 1,
+) -> list[Any]:
+    """Process a loop using multiprocessing Pool for parallel execution.
+
+    Executes the given process_function with the provided arguments in parallel using
+    multiprocessing Pool, which is suitable for CPU-bound tasks.
+
+    Args:
+        process_function: Function that processes the given process_args
+        process_args: List of args to be processed by the process_function
+                    e.g. [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        process_args_static: Optional constant arguments passed to each function call
+        deepcopy_static_args: Optional arguments that should be
+                              deep-copied for each process
+        process_args_len: Optional length of process_args
+                          If not provided, it will ot be taken into account
+                          when calculating the max number of processes.
+
+    Returns:
+        List of results from the process_function executions
+
+    Note:
+        Pool is used for CPU-bound tasks as it bypasses
+        Python's GIL by creating separate processes.
+        Multiprocessing is not safe for mutable objects unlike ThreadPoolExecutor.
+        When debugging, if ConnectionErrors occur, set max_processes to 1.
+        Also given functions must be pickle-able.
+
+    """
+    from winipedia_utils.concurrent.concurrent import concurrent_loop
+
+    return concurrent_loop(
+        threading=False,
+        process_function=process_function,
+        process_args=process_args,
+        process_args_static=process_args_static,
+        deepcopy_static_args=deepcopy_static_args,
+        process_args_len=process_args_len,
+    )

winipedia_utils/concurrent/multithreading.py

@@ -1,93 +1,93 @@
-"""Multithreading utilities for concurrent execution.
-
-This module provides functions for parallel processing using thread pools.
-It includes utilities for handling thread pools, managing futures, and organizing
-parallel execution of I/O-bound tasks.
-Base helper functions that serve threading and processing are located in the
-multiprocessing module.
-
-Returns:
-    Various utility functions for multithreaded processing.
-
-"""
-
-from collections.abc import Callable, Generator, Iterable
-from concurrent.futures import Future, ThreadPoolExecutor, as_completed
-from typing import Any
-
-
-def get_future_results_as_completed(
-    futures: Iterable[Future[Any]],
-) -> Generator[Any, None, None]:
-    """Get future results as they complete.
-
-    Yields results from futures in the order they complete,
-    not in the order they were submitted.
-
-    Args:
-        futures: List of Future objects to get results from
-
-    Yields:
-        The result of each completed future
-
-    """
-    for future in as_completed(futures):
-        yield future.result()
-
-
-def multithread_loop(
-    process_function: Callable[..., Any],
-    process_args: Iterable[Iterable[Any]],
-    process_args_static: Iterable[Any] | None = None,
-    process_args_len: int = 1,
-) -> list[Any]:
-    """Process a loop using ThreadPoolExecutor for parallel execution.
-
-    Executes the given process_function with the provided arguments in parallel using
-    ThreadPoolExecutor, which is suitable for I/O-bound tasks.
-
-    Args:
-        process_function: Function that processes the given process_args
-        process_args: list of args to be processed by the process_function
-                    e.g. [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
-        process_args_static: Optional constant arguments passed to each function call
-        process_args_len: Optional length of process_args
-                          If not provided, it will ot be taken into account
-                          when calculating the max number of workers.
-
-    Returns:
-        List of results from the process_function executions
-
-    Note:
-        ThreadPoolExecutor is used for I/O-bound tasks, not for CPU-bound tasks.
-
-    """
-    from winipedia_utils.concurrent.concurrent import concurrent_loop
-
-    return concurrent_loop(
-        threading=True,
-        process_function=process_function,
-        process_args=process_args,
-        process_args_static=process_args_static,
-        process_args_len=process_args_len,
-    )
-
-
-def imap_unordered(
-    executor: ThreadPoolExecutor,
-    func: Callable[..., Any],
-    iterable: Iterable[Any],
-) -> Generator[Any, None, None]:
-    """Apply a function to each item in an iterable in parallel.
-
-    Args:
-        executor: ThreadPoolExecutor to use for parallel execution
-        func: Function to apply to each item in the iterable
-        iterable: Iterable of items to apply the function to
-
-    Yields:
-        Results of applying the function to each item in the iterable
-
-    """
-    results = [executor.submit(func, item) for item in iterable]
-    yield from get_future_results_as_completed(results)
+"""Multithreading utilities for concurrent execution.
+
+This module provides functions for parallel processing using thread pools.
+It includes utilities for handling thread pools, managing futures, and organizing
+parallel execution of I/O-bound tasks.
+Base helper functions that serve threading and processing are located in the
+multiprocessing module.
+
+Returns:
+    Various utility functions for multithreaded processing.
+
+"""
+
+from collections.abc import Callable, Generator, Iterable
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Any
+
+
+def get_future_results_as_completed(
+    futures: Iterable[Future[Any]],
+) -> Generator[Any, None, None]:
+    """Get future results as they complete.
+
+    Yields results from futures in the order they complete,
+    not in the order they were submitted.
+
+    Args:
+        futures: List of Future objects to get results from
+
+    Yields:
+        The result of each completed future
+
+    """
+    for future in as_completed(futures):
+        yield future.result()
+
+
+def multithread_loop(
+    process_function: Callable[..., Any],
+    process_args: Iterable[Iterable[Any]],
+    process_args_static: Iterable[Any] | None = None,
+    process_args_len: int = 1,
+) -> list[Any]:
+    """Process a loop using ThreadPoolExecutor for parallel execution.
+
+    Executes the given process_function with the provided arguments in parallel using
+    ThreadPoolExecutor, which is suitable for I/O-bound tasks.
+
+    Args:
+        process_function: Function that processes the given process_args
+        process_args: list of args to be processed by the process_function
+                    e.g. [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+        process_args_static: Optional constant arguments passed to each function call
+        process_args_len: Optional length of process_args
+                          If not provided, it will ot be taken into account
+                          when calculating the max number of workers.
+
+    Returns:
+        List of results from the process_function executions
+
+    Note:
+        ThreadPoolExecutor is used for I/O-bound tasks, not for CPU-bound tasks.
+
+    """
+    from winipedia_utils.concurrent.concurrent import concurrent_loop
+
+    return concurrent_loop(
+        threading=True,
+        process_function=process_function,
+        process_args=process_args,
+        process_args_static=process_args_static,
+        process_args_len=process_args_len,
+    )
+
+
+def imap_unordered(
+    executor: ThreadPoolExecutor,
+    func: Callable[..., Any],
+    iterable: Iterable[Any],
+) -> Generator[Any, None, None]:
+    """Apply a function to each item in an iterable in parallel.
+
+    Args:
+        executor: ThreadPoolExecutor to use for parallel execution
+        func: Function to apply to each item in the iterable
+        iterable: Iterable of items to apply the function to
+
+    Yields:
+        Results of applying the function to each item in the iterable
+
+    """
+    results = [executor.submit(func, item) for item in iterable]
+    yield from get_future_results_as_completed(results)

winipedia_utils/consts.py

@@ -1,21 +1,21 @@
-"""Constants used throughout the winipedia_utils package.
-
-This module contains package-wide constants that are used by various
-modules within the package. These constants define core configuration
-values and identifiers for the package.
-"""
-
-PACKAGE_NAME = "winipedia_utils"
-
-_DEV_DEPENDENCIES = [
-    "ruff",
-    "pre-commit",
-    "mypy",
-    "pytest",
-    "bandit",
-    "types-setuptools",
-    "types-tqdm",
-    "types-defusedxml",
-    "types-pyyaml",
-    "pytest-mock",
-]
+"""Constants used throughout the winipedia_utils package.
+
+This module contains package-wide constants that are used by various
+modules within the package. These constants define core configuration
+values and identifiers for the package.
+"""
+
+PACKAGE_NAME = "winipedia_utils"
+
+_DEV_DEPENDENCIES = [
+    "ruff",
+    "pre-commit",
+    "mypy",
+    "pytest",
+    "bandit",
+    "types-setuptools",
+    "types-tqdm",
+    "types-defusedxml",
+    "types-pyyaml",
+    "pytest-mock",
+]

winipedia_utils/data/__init__.py

@@ -1 +1 @@
-"""__init__ module for winipedia_utils.data."""
+"""__init__ module for winipedia_utils.data."""

winipedia_utils/data/dataframe/__init__.py

@@ -1 +1 @@
-"""__init__ module."""
+"""__init__ module."""