tldm 1.0.0__py3-none-any.whl

tldm/__init__.py

@@ -0,0 +1,11 @@
+from importlib.metadata import version
+
+from ._monitor import TMonitor as TMonitor
+from .aliases import tenumerate as tenumerate
+from .aliases import tmap as tmap
+from .aliases import tproduct as tproduct
+from .aliases import trange as trange
+from .aliases import tzip as tzip
+from .std import tldm as tldm
+
+__version__ = version("tldm")

tldm/_monitor.py

@@ -0,0 +1,115 @@
+import atexit
+from collections.abc import Callable
+from threading import Event, Thread, current_thread
+from time import time
+from typing import TYPE_CHECKING
+from warnings import warn
+
+if TYPE_CHECKING:
+    from tldm.std import tldm
+
+
+class TldmSynchronisationWarning(RuntimeWarning):
+    """tldm multi-thread/-process errors which may cause incorrect nesting
+    but otherwise no adverse effects"""
+
+    pass
+
+
+class TMonitor(Thread):
+    """
+    Monitoring thread for tldm bars.
+    Monitors if tldm bars are taking too much time to display
+    and readjusts miniters automatically if necessary.
+
+    Parameters
+    ----------
+    tldm_cls  : class
+        tldm class to use (can be core tldm or a submodule).
+    sleep_interval  : float
+        Time to sleep between monitoring checks.
+    """
+
+    _test: dict[str, Event] = {}  # internal vars for unit testing
+
+    name: str
+    daemon: bool
+    woken: float
+    tldm_cls: type["tldm"]
+    sleep_interval: float
+    _time: Callable[[], float]
+    was_killed: Event
+
+    def __init__(self, tldm_cls: type["tldm"], sleep_interval: float) -> None:
+        Thread.__init__(self)
+        self.name = "tldm_monitor"
+        self.daemon = True  # kill thread when main killed (KeyboardInterrupt)
+        self.woken = 0  # last time woken up, to sync with monitor
+        self.tldm_cls = tldm_cls
+        self.sleep_interval = sleep_interval
+        self._time = self._test.get("time", time)  # type: ignore[assignment]
+        self.was_killed = self._test.get("Event", Event)()  # type: ignore[operator]
+        atexit.register(self.exit)
+        self.start()
+
+    def exit(self) -> bool:
+        self.was_killed.set()
+        if self is not current_thread():
+            self.join()
+        return self.report()
+
+    def get_instances(self) -> list["tldm"]:
+        # returns a copy of started `tldm_cls` instances
+        return [
+            i
+            for i in self.tldm_cls._instances.copy()
+            # Avoid race by checking that the instance started
+            if hasattr(i, "start_t")
+        ]
+
+    def run(self) -> None:
+        cur_t = self._time()
+        while True:
+            # After processing and before sleeping, notify that we woke
+            # Need to be done just before sleeping
+            self.woken = cur_t
+            # Sleep some time...
+            self.was_killed.wait(self.sleep_interval)
+            # Quit if killed
+            if self.was_killed.is_set():
+                return
+            # Then monitor!
+            # Acquire lock (to access _instances)
+            with self.tldm_cls.get_lock():
+                cur_t = self._time()
+                # Check tldm instances are waiting too long to print
+                instances = self.get_instances()
+                for instance in instances:
+                    # Check event in loop to reduce blocking time on exit
+                    if self.was_killed.is_set():
+                        return
+                    # Only if mininterval > 1 (else iterations are just slow)
+                    # and last refresh exceeded maxinterval
+                    if (
+                        instance.miniters > 1
+                        and (cur_t - instance.last_print_t) >= instance.maxinterval
+                    ):
+                        # force bypassing miniters on next iteration
+                        # (dynamic_miniters adjusts mininterval automatically)
+                        instance.miniters = 1
+                        # Refresh now! (works only for manual tldm)
+                        instance.refresh(nolock=True)
+                    # Remove accidental long-lived strong reference
+                    del instance
+                if instances != self.get_instances():  # pragma: nocover
+                    warn(
+                        "Set changed size during iteration"
+                        + " (see https://github.com/tldm/tldm/issues/481)",
+                        TldmSynchronisationWarning,
+                        stacklevel=2,
+                    )
+                # Remove accidental long-lived strong references
+                del instances
+
+    def report(self) -> bool:
+        return not self.was_killed.is_set()

tldm/aliases.py

@@ -0,0 +1,86 @@
+import itertools
+from collections.abc import Callable, Iterable, Iterator
+from operator import length_hint
+from typing import Any, TypeVar
+
+from .std import tldm
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+def tenumerate(
+    iterable: Iterable[T],
+    start: int = 0,
+    total: int | float | None = None,
+    tldm_class: type[tldm] = tldm,
+    **tldm_kwargs: Any,
+) -> Iterator[tuple[int, T]]:
+    """
+    Equivalent of builtin `enumerate`.
+
+    Parameters
+    ----------
+    tldm_class  : [default: tldm.std.tldm].
+    """
+    return enumerate(tldm_class(iterable, total=total, **tldm_kwargs), start)
+
+
+def tzip(
+    iter1: Iterable[T], *iter2plus: Iterable[Any], **tldm_kwargs: Any
+) -> Iterator[tuple[T, ...]]:
+    """
+    Equivalent of builtin `zip`.
+
+    Parameters
+    ----------
+    tldm_class  : [default: tldm.std.tldm].
+    """
+    kwargs = tldm_kwargs.copy()
+    tldm_class = kwargs.pop("tldm_class", tldm)
+    yield from zip(tldm_class(iter1, **kwargs), *iter2plus)
+
+
+def tmap(function: Callable[..., R], *sequences: Iterable[Any], **tldm_kwargs: Any) -> Iterator[R]:
+    """
+    Equivalent of builtin `map`.
+
+    Parameters
+    ----------
+    tldm_class  : [default: tldm.std.tldm].
+    """
+    for i in tzip(*sequences, **tldm_kwargs):
+        yield function(*i)
+
+
+def tproduct(*iterables: Iterable[T], **tldm_kwargs: Any) -> Iterator[tuple[T, ...]]:
+    """
+    Equivalent of `itertools.product`.
+
+    Parameters
+    ----------
+    tldm_class  : [default: tldm.std.tldm].
+    """
+    kwargs = tldm_kwargs.copy()
+    repeat = kwargs.pop("repeat", 1)
+    tldm_class = kwargs.pop("tldm_class", tldm)
+    try:
+        lens = list(map(length_hint, iterables))
+    except TypeError:
+        total = None
+    else:
+        total = 1
+        for i in lens:
+            total *= i
+        total = total**repeat
+        kwargs.setdefault("total", total)
+    with tldm_class(**kwargs) as t:
+        it = itertools.product(*iterables, repeat=repeat)
+        for val in it:
+            yield val
+            t.update()
+
+
+def trange(*args: int, **kwargs: Any) -> tldm:
+    """Shortcut for tldm(range(*args), **kwargs)."""
+    return tldm(range(*args), **kwargs)

tldm/extensions/__init__.py

@@ -0,0 +1 @@
+from .pandas import tldm_pandas as tldm_pandas

tldm/extensions/asyncio.py

@@ -0,0 +1,100 @@
+"""
+Asynchronous progressbar decorator for iterators.
+"""
+
+import asyncio
+
+from ..std import tldm as std_tldm
+
+
+class tldm_asyncio(std_tldm):
+    """
+    Asynchronous-friendly version of tldm.
+    """
+
+    def __init__(self, iterable=None, *args, **kwargs):
+        super().__init__(iterable, *args, **kwargs)
+        self.iterable_awaitable = False
+        if iterable is not None:
+            if hasattr(iterable, "__anext__"):
+                self.iterable_next = iterable.__anext__
+                self.iterable_awaitable = True
+            elif hasattr(iterable, "__next__"):
+                self.iterable_next = iterable.__next__
+
+    def __aiter__(self):
+        return self
+
+    # def __del__(self):
+    #     self.close()
+    #     if len(tldm_asyncio._instances) == 0:
+    #         if hasattr(tldm_asyncio, "_lock"):
+    #             del tldm_asyncio._lock
+    #         if hasattr(tldm_asyncio, "monitor") and tldm_asyncio.monitor is not None:
+    #             tldm_asyncio.monitor.exit()
+
+    async def __anext__(self):
+        try:
+            if self.iterable_awaitable:
+                res = await self.iterable_next()
+            else:
+                if not hasattr(self, "iterable_iterator"):
+                    self.iterable_iterator = iter(self.iterable)
+                    self.iterable_next = self.iterable_iterator.__next__
+                res = self.iterable_next()
+            self.update()
+            return res
+        except StopIteration:
+            self.close()
+            raise StopAsyncIteration from None
+        except BaseException:
+            self.close()
+            raise
+
+    def send(self, *args, **kwargs):
+        return self.iterable.send(*args, **kwargs)
+
+    @classmethod
+    def as_completed(cls, fs, *, loop=None, timeout=None, total=None, **tldm_kwargs):
+        """
+        Wrapper for `asyncio.as_completed`.
+        """
+        if total is None:
+            total = len(fs)
+        yield from cls(
+            asyncio.as_completed(fs, timeout=timeout),
+            total=total,
+            **tldm_kwargs,
+        )
+
+    @classmethod
+    async def gather(
+        cls,
+        *fs,
+        loop=None,
+        timeout=None,
+        total=None,
+        return_exceptions=False,
+        **tldm_kwargs,
+    ):
+        """
+        Wrapper for `asyncio.gather`.
+        """
+        if total is None:
+            total = len(fs)
+
+        async def wrap_awaitable(i, f):
+            try:
+                return i, await f
+            except Exception as e:
+                if return_exceptions:
+                    return i, e
+                raise
+
+        async def aiter_as_completed():
+            ifs = [wrap_awaitable(i, f) for i, f in enumerate(fs)]
+            for r in asyncio.as_completed(ifs, timeout=timeout):
+                yield await r
+
+        res = [f async for f in cls(aiter_as_completed(), total=total, **tldm_kwargs)]
+        return [i for _, i in sorted(res)]

tldm/extensions/concurrent.py

@@ -0,0 +1,125 @@
+"""
+Thin wrappers around `concurrent.futures`.
+"""
+
+from contextlib import contextmanager
+from operator import length_hint
+from os import cpu_count
+
+from ..std import tldm as std_tldm
+from ..utils import TldmWarning
+
+
+@contextmanager
+def ensure_lock(tldm_class, lock_name=""):
+    """get (create if necessary) and then restore `tldm_class`'s lock"""
+    old_lock = getattr(tldm_class, "_lock", None)  # don't create a new lock
+    lock = old_lock or tldm_class.get_lock()  # maybe create a new lock
+    lock = getattr(lock, lock_name, lock)  # maybe subtype
+    tldm_class.set_lock(lock)
+    yield lock
+    if old_lock is None:
+        del tldm_class._lock
+    else:
+        tldm_class.set_lock(old_lock)
+
+
+def _executor_map(PoolExecutor, fn, *iterables, **tldm_kwargs):
+    """
+    Implementation of `thread_map` and `process_map`.
+
+    Parameters
+    ----------
+    tldm_class  : [default: tldm.auto.tldm].
+    max_workers  : [default: min(32, cpu_count() + 4)].
+    timeout  : [default: None].
+    chunksize  : [default: 1].
+    lock_name  : [default: "":str].
+    """
+    kwargs = tldm_kwargs.copy()
+    if "total" not in kwargs:
+        kwargs["total"] = length_hint(iterables[0])
+    tldm_class = kwargs.pop("tldm_class", std_tldm)
+    max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4))
+    timeout = kwargs.pop("timeout", None)
+    chunksize = kwargs.pop("chunksize", 1)
+    lock_name = kwargs.pop("lock_name", "")
+    with ensure_lock(tldm_class, lock_name=lock_name) as lk:
+        # share lock in case workers are already using `tldm`
+        with PoolExecutor(
+            max_workers=max_workers, initializer=tldm_class.set_lock, initargs=(lk,)
+        ) as ex:
+            return list(
+                tldm_class(
+                    ex.map(fn, *iterables, timeout=timeout, chunksize=chunksize),
+                    **kwargs,
+                )
+            )
+
+
+def thread_map(fn, *iterables, **tldm_kwargs):
+    """
+    Equivalent of `list(map(fn, *iterables))`
+    driven by `concurrent.futures.ThreadPoolExecutor`.
+
+    Parameters
+    ----------
+    tldm_class  : optional
+        `tldm` class to use for bars [default: tldm.auto.tldm].
+    max_workers  : int, optional
+        Maximum number of workers to spawn; passed to
+        `concurrent.futures.ThreadPoolExecutor.__init__`.
+        [default: min(32, cpu_count() + 4)].
+    timeout  : int or float, optional
+        The iterator raises a TimeoutError if __next()__ is called and the
+        result isn't available within the timeout specified from the
+        original call to thread_map. [default: None].
+    """
+    from concurrent.futures import ThreadPoolExecutor
+
+    return _executor_map(ThreadPoolExecutor, fn, *iterables, **tldm_kwargs)
+
+
+def process_map(fn, *iterables, **tldm_kwargs):
+    """
+    Equivalent of `list(map(fn, *iterables))`
+    driven by `concurrent.futures.ProcessPoolExecutor`.
+
+    Parameters
+    ----------
+    tldm_class  : optional
+        `tldm` class to use for bars [default: tldm.auto.tldm].
+    max_workers  : int, optional
+        Maximum number of workers to spawn; passed to
+        `concurrent.futures.ProcessPoolExecutor.__init__`.
+        [default: min(32, cpu_count() + 4)].
+    timeout  : int or float, optional
+        The iterator raises a TimeoutError if __next()__ is called and the
+        result isn't available within the timeout specified from the
+        original call to process_map. [default: None].
+    chunksize  : int, optional
+        Size of chunks sent to worker processes; passed to
+        `concurrent.futures.ProcessPoolExecutor.map`. [default: 1].
+    lock_name  : str, optional
+        Member of `tldm_class.get_lock()` to use [default: mp_lock].
+    """
+    from concurrent.futures import ProcessPoolExecutor
+
+    if iterables and "chunksize" not in tldm_kwargs:
+        # default `chunksize=1` has poor performance for large iterables
+        # (most time spent dispatching items to workers).
+        longest_iterable_len = max(map(length_hint, iterables))
+        if longest_iterable_len > 1000:
+            from warnings import warn
+
+            warn(
+                "Iterable length %d > 1000 but `chunksize` is not set."
+                " This may seriously degrade multiprocess performance."
+                " Set `chunksize=1` or more." % longest_iterable_len,
+                TldmWarning,
+                stacklevel=2,
+            )
+    if "lock_name" not in tldm_kwargs:
+        tldm_kwargs = tldm_kwargs.copy()
+        tldm_kwargs["lock_name"] = "mp_lock"
+    return _executor_map(ProcessPoolExecutor, fn, *iterables, **tldm_kwargs)

tldm/extensions/pandas.py

@@ -0,0 +1,179 @@
+"""
+Registration for `tldm` to provide `pandas` progress indicators.
+"""
+
+import contextlib
+from typing import Any
+
+from ..std import tldm as std_tldm
+
+
+def tldm_pandas(**tldm_kwargs: dict[str, Any]) -> None:
+    """
+    Registers the current `tldm` class with
+        pandas.core.
+        ( frame.DataFrame
+        | series.Series
+        | groupby.(generic.)DataFrameGroupBy
+        | groupby.(generic.)SeriesGroupBy
+        ).progress_apply
+
+    A new instance will be created every time `progress_apply` is called,
+    and each instance will automatically `close()` upon completion.
+
+    Parameters
+    ----------
+    tldm_kwargs  : arguments for the tldm instance
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> import numpy as np
+    >>> from tldm import tldm, tldm_pandas
+    >>> from tldm.gui import tldm as tldm_gui
+    >>>
+    >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6)))
+    >>> tldm_pandas(ncols=50)  # can use tldm_gui, optional kwargs, etc
+    >>> # Now you can use `progress_apply` instead of `apply`
+    >>> df.groupby(0).progress_apply(lambda x: x**2)
+
+    References
+    ----------
+    <https://stackoverflow.com/questions/18603270/\
+    progress-indicator-during-pandas-operations-python>
+    """
+    from warnings import catch_warnings, simplefilter
+
+    from pandas.core.frame import DataFrame
+    from pandas.core.series import Series
+
+    try:
+        with catch_warnings():
+            simplefilter("ignore", category=FutureWarning)
+            from pandas import Panel
+    except ImportError:  # pandas>=1.2.0
+        Panel = None
+    Rolling, Expanding = None, None
+    try:  # pandas>=1.0.0
+        from pandas.core.window.rolling import _Rolling_and_Expanding
+    except ImportError:
+        try:  # pandas>=0.18.0
+            from pandas.core.window import _Rolling_and_Expanding
+        except ImportError:  # pandas>=1.2.0
+            try:  # pandas>=1.2.0
+                from pandas.core.window.expanding import Expanding
+                from pandas.core.window.rolling import Rolling
+
+                _Rolling_and_Expanding = Rolling, Expanding
+            except ImportError:  # pragma: no cover
+                _Rolling_and_Expanding = None
+    try:  # pandas>=0.25.0
+        from pandas.core.groupby.generic import (
+            DataFrameGroupBy,
+            SeriesGroupBy,  # , NDFrameGroupBy
+        )
+    except ImportError:  # pragma: no cover
+        try:  # pandas>=0.23.0
+            from pandas.core.groupby.groupby import DataFrameGroupBy, SeriesGroupBy
+        except ImportError:
+            from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy
+    try:  # pandas>=0.23.0
+        from pandas.core.groupby.groupby import GroupBy
+    except ImportError:  # pragma: no cover
+        from pandas.core.groupby import GroupBy
+
+    try:  # pandas>=0.23.0
+        from pandas.core.groupby.groupby import PanelGroupBy
+    except ImportError:
+        try:
+            from pandas.core.groupby import PanelGroupBy
+        except ImportError:  # pandas>=0.25.0
+            PanelGroupBy = None
+
+    tldm_kwargs = tldm_kwargs.copy()
+
+    def inner_generator(df_function="apply"):
+        def inner(df, func, **kwargs):
+            """
+            Parameters
+            ----------
+            df  : (DataFrame|Series)[GroupBy]
+                Data (may be grouped).
+            func  : function
+                To be applied on the (grouped) data.
+            **kwargs  : optional
+                Transmitted to `df.apply()`.
+            """
+
+            # Precompute total iterations
+            total = tldm_kwargs.pop("total", getattr(df, "ngroups", None))
+            if total is None:  # not grouped
+                if df_function == "applymap":
+                    total = df.size
+                elif isinstance(df, Series):
+                    total = len(df)
+                elif _Rolling_and_Expanding is None or not isinstance(df, _Rolling_and_Expanding):
+                    # DataFrame or Panel
+                    axis = kwargs.get("axis", 0)
+                    if axis == "index":
+                        axis = 0
+                    elif axis == "columns":
+                        axis = 1
+                    # when axis=0, total is shape[axis1]
+                    total = df.size // df.shape[axis]
+
+            # Init bar
+            t = std_tldm(total=total, **tldm_kwargs)
+
+            try:  # pandas>=1.3.0
+                from pandas.core.common import is_builtin_func
+            except ImportError:
+                is_builtin_func = df._is_builtin_func
+            with contextlib.suppress(TypeError):
+                func = is_builtin_func(func)
+
+            # Define bar updating wrapper
+            def wrapper(*args, **kwargs):
+                # update tbar correctly
+                # it seems `pandas apply` calls `func` twice
+                # on the first column/row to decide whether it can
+                # take a fast or slow code path; so stop when t.total==t.n
+                t.update(n=1 if not t.total or t.n < t.total else 0)
+                return func(*args, **kwargs)
+
+            # Apply the provided function (in **kwargs)
+            # on the df using our wrapper (which provides bar updating)
+            try:
+                return getattr(df, df_function)(wrapper, **kwargs)
+            finally:
+                t.close()
+
+        return inner
+
+    # Monkeypatch pandas to provide easy methods
+    # Enable custom tldm progress in pandas!
+    Series.progress_apply = inner_generator()
+    SeriesGroupBy.progress_apply = inner_generator()
+    Series.progress_map = inner_generator("map")
+    SeriesGroupBy.progress_map = inner_generator("map")
+
+    DataFrame.progress_apply = inner_generator()
+    DataFrameGroupBy.progress_apply = inner_generator()
+    DataFrame.progress_applymap = inner_generator("applymap")
+    DataFrame.progress_map = inner_generator("map")
+    DataFrameGroupBy.progress_map = inner_generator("map")
+
+    if Panel is not None:
+        Panel.progress_apply = inner_generator()
+    if PanelGroupBy is not None:
+        PanelGroupBy.progress_apply = inner_generator()
+
+    GroupBy.progress_apply = inner_generator()
+    GroupBy.progress_aggregate = inner_generator("aggregate")
+    GroupBy.progress_transform = inner_generator("transform")
+
+    if Rolling is not None and Expanding is not None:
+        Rolling.progress_apply = inner_generator()
+        Expanding.progress_apply = inner_generator()
+    elif _Rolling_and_Expanding is not None:
+        _Rolling_and_Expanding.progress_apply = inner_generator()