dycw-utilities 0.122.1__py3-none-any.whl → 0.124.0__py3-none-any.whl

{dycw_utilities-0.122.1.dist-info → dycw_utilities-0.124.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dycw-utilities
-Version: 0.122.1
+Version: 0.124.0
 Author-email: Derek Wan <d.wan@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.12

{dycw_utilities-0.122.1.dist-info → dycw_utilities-0.124.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-utilities/__init__.py,sha256=HQ1CqsUZ8X8-X0sFT3wvjjj0NMYqAnzvjRBDmvWB2xU,60
+utilities/__init__.py,sha256=HeZfuK-rk7XFCrcSKl_fSwvax77ZgizR9BO2AjEIbLs,60
 utilities/altair.py,sha256=Gpja-flOo-Db0PIPJLJsgzAlXWoKUjPU1qY-DQ829ek,9156
 utilities/astor.py,sha256=xuDUkjq0-b6fhtwjhbnebzbqQZAjMSHR1IIS5uOodVg,777
-utilities/asyncio.py,sha256=cjXpSezv95ZG3t-D9sXLOn6BGBj7w-Wf4udvE5gU4vU,23508
+utilities/asyncio.py,sha256=joGmwv-WiDLYoK4q41TtVgfIT23s8Ok46jT_yEQGjaM,28240
 utilities/atomicwrites.py,sha256=geFjn9Pwn-tTrtoGjDDxWli9NqbYfy3gGL6ZBctiqSo,5393
 utilities/atools.py,sha256=IYMuFSFGSKyuQmqD6v5IUtDlz8PPw0Sr87Cub_gRU3M,1168
 utilities/cachetools.py,sha256=C1zqOg7BYz0IfQFK8e3qaDDgEZxDpo47F15RTfJM37Q,2910
@@ -59,15 +59,15 @@ utilities/pytest_regressions.py,sha256=-SVT9647Dg6-JcdsiaDKXe3NdOmmrvGevLKWwGjxq
 utilities/python_dotenv.py,sha256=iWcnpXbH7S6RoXHiLlGgyuH6udCupAcPd_gQ0eAenQ0,3190
 utilities/random.py,sha256=lYdjgxB7GCfU_fwFVl5U-BIM_HV3q6_urL9byjrwDM8,4157
 utilities/re.py,sha256=5J4d8VwIPFVrX2Eb8zfoxImDv7IwiN_U7mJ07wR2Wvs,3958
-utilities/redis.py,sha256=8ELnXiISVHY1m9elJWhekhLXg6NQSaNIIPvZ_EaGw3s,26624
+utilities/redis.py,sha256=fMVKsVCjCv63m_JgoOKqj3wHVzzHR0s4suvZ-mBy1gk,26615
 utilities/reprlib.py,sha256=Re9bk3n-kC__9DxQmRlevqFA86pE6TtVfWjUgpbVOv0,1849
 utilities/rich.py,sha256=t50MwwVBsoOLxzmeVFSVpjno4OW6Ufum32skXbV8-Bs,1911
 utilities/scipy.py,sha256=X6ROnHwiUhAmPhM0jkfEh0-Fd9iRvwiqtCQMOLmOQF8,945
 utilities/sentinel.py,sha256=3jIwgpMekWgDAxPDA_hXMP2St43cPhciKN3LWiZ7kv0,1248
 utilities/shelve.py,sha256=HZsMwK4tcIfg3sh0gApx4-yjQnrY4o3V3ZRimvRhoW0,738
-utilities/slack_sdk.py,sha256=A2f7-DYOngRoUP6ZdLIaUQ6Lfzgru5Xp3U3k5JfEkQE,3301
+utilities/slack_sdk.py,sha256=Q8UakiB7qo6SUfaBDB0j1N4b8MuFzaD9lG5HGq7rtuw,3200
 utilities/socket.py,sha256=K77vfREvzoVTrpYKo6MZakol0EYu2q1sWJnnZqL0So0,118
-utilities/sqlalchemy.py,sha256=yCUCDhg0wFOCdEh6wwBD7Ma979OksLpz4arck1s653s,35447
+utilities/sqlalchemy.py,sha256=p8vsHaNRoeq5zJouIKyp9piFM26wtm5yR4DkzCMFDSw,35471
 utilities/sqlalchemy_polars.py,sha256=wjJpoUo-yO9E2ujpG_06vV5r2OdvBiQ4yvV6wKCa2Tk,15605
 utilities/statsmodels.py,sha256=koyiBHvpMcSiBfh99wFUfSggLNx7cuAw3rwyfAhoKpQ,3410
 utilities/streamlit.py,sha256=U9PJBaKP1IdSykKhPZhIzSPTZsmLsnwbEPZWzNhJPKk,2955
@@ -88,7 +88,7 @@ utilities/warnings.py,sha256=un1LvHv70PU-LLv8RxPVmugTzDJkkGXRMZTE2-fTQHw,1771
 utilities/whenever.py,sha256=jS31ZAY5OMxFxLja_Yo5Fidi87Pd-GoVZ7Vi_teqVDA,16743
 utilities/zipfile.py,sha256=24lQc9ATcJxHXBPc_tBDiJk48pWyRrlxO2fIsFxU0A8,699
 utilities/zoneinfo.py,sha256=-5j7IQ9nb7gR43rdgA7ms05im-XuqhAk9EJnQBXxCoQ,1874
-dycw_utilities-0.122.1.dist-info/METADATA,sha256=9QN9Ulm7O-TsU1Zg2xrl5ylNadZMkHMp9ktRVFxpFwQ,12943
-dycw_utilities-0.122.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dycw_utilities-0.122.1.dist-info/licenses/LICENSE,sha256=gppZp16M6nSVpBbUBrNL6JuYfvKwZiKgV7XoKKsHzqo,1066
-dycw_utilities-0.122.1.dist-info/RECORD,,
+dycw_utilities-0.124.0.dist-info/METADATA,sha256=-1BZhlwOBb6JkkxdOqznS-XXxhlcvdZfGawiKrjB7Fc,12943
+dycw_utilities-0.124.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dycw_utilities-0.124.0.dist-info/licenses/LICENSE,sha256=gppZp16M6nSVpBbUBrNL6JuYfvKwZiKgV7XoKKsHzqo,1066
+dycw_utilities-0.124.0.dist-info/RECORD,,

utilities/__init__.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
-__version__ = "0.122.1"
+__version__ = "0.124.0"

utilities/asyncio.py

@@ -8,6 +8,7 @@ from asyncio import (
     PriorityQueue,
     Queue,
     QueueEmpty,
+    QueueFull,
     Semaphore,
     StreamReader,
     Task,
@@ -27,6 +28,7 @@ from contextlib import (
 )
 from dataclasses import dataclass, field
 from io import StringIO
+from itertools import chain
 from logging import getLogger
 from subprocess import PIPE
 from sys import stderr, stdout
@@ -43,6 +45,8 @@ from typing import (
     override,
 )
 
+from typing_extensions import deprecated
+
 from utilities.datetime import (
     MINUTE,
     SECOND,
@@ -53,7 +57,6 @@ from utilities.datetime import (
 )
 from utilities.errors import ImpossibleCaseError, repr_error
 from utilities.functions import ensure_int, ensure_not_none, get_class_name
-from utilities.reprlib import get_repr
 from utilities.sentinel import Sentinel, sentinel
 from utilities.types import (
     Coroutine1,
@@ -67,6 +70,7 @@ from utilities.types import (
 if TYPE_CHECKING:
     from asyncio import _CoroutineLike
     from asyncio.subprocess import Process
+    from collections import deque
     from collections.abc import AsyncIterator, Sequence
     from contextvars import Context
     from types import TracebackType
@@ -77,6 +81,164 @@ if TYPE_CHECKING:
 _T = TypeVar("_T")
 
 
+class EnhancedQueue(Queue[_T]):
+    """An asynchronous deque."""
+
+    @override
+    def __init__(self, maxsize: int = 0) -> None:
+        super().__init__(maxsize=maxsize)
+        self._finished: Event
+        self._getters: deque[Any]
+        self._putters: deque[Any]
+        self._queue: deque[_T]
+        self._unfinished_tasks: int
+
+    @override
+    @deprecated("Use `get_left`/`get_right` instead")
+    async def get(self) -> _T:
+        raise RuntimeError  # pragma: no cover
+
+    @override
+    @deprecated("Use `get_left_nowait`/`get_right_nowait` instead")
+    def get_nowait(self) -> _T:
+        raise RuntimeError  # pragma: no cover
+
+    @override
+    @deprecated("Use `put_left`/`put_right` instead")
+    async def put(self, item: _T) -> None:
+        raise RuntimeError(item)  # pragma: no cover
+
+    @override
+    @deprecated("Use `put_left_nowait`/`put_right_nowait` instead")
+    def put_nowait(self, item: _T) -> None:
+        raise RuntimeError(item)  # pragma: no cover
+
+    # get all
+
+    async def get_all(self, *, reverse: bool = False) -> Sequence[_T]:
+        """Remove and return all items from the queue."""
+        first = await (self.get_right() if reverse else self.get_left())
+        return list(chain([first], self.get_all_nowait(reverse=reverse)))
+
+    def get_all_nowait(self, *, reverse: bool = False) -> Sequence[_T]:
+        """Remove and return all items from the queue without blocking."""
+        items: Sequence[_T] = []
+        while True:
+            try:
+                items.append(
+                    self.get_right_nowait() if reverse else self.get_left_nowait()
+                )
+            except QueueEmpty:
+                return items
+
+    # get left/right
+
+    async def get_left(self) -> _T:
+        """Remove and return an item from the start of the queue."""
+        return await self._get_left_or_right(self._get)
+
+    async def get_right(self) -> _T:
+        """Remove and return an item from the end of the queue."""
+        return await self._get_left_or_right(self._get_right)
+
+    def get_left_nowait(self) -> _T:
+        """Remove and return an item from the start of the queue without blocking."""
+        return self._get_left_or_right_nowait(self._get)
+
+    def get_right_nowait(self) -> _T:
+        """Remove and return an item from the end of the queue without blocking."""
+        return self._get_left_or_right_nowait(self._get_right)
+
+    # put left/right
+
+    async def put_left(self, *items: _T) -> None:
+        """Put items into the queue at the start."""
+        return await self._put_left_or_right(self._put_left, *items)
+
+    async def put_right(self, *items: _T) -> None:
+        """Put items into the queue at the end."""
+        return await self._put_left_or_right(self._put, *items)
+
+    def put_left_nowait(self, *items: _T) -> None:
+        """Put items into the queue at the start without blocking."""
+        self._put_left_or_right_nowait(self._put_left, *items)
+
+    def put_right_nowait(self, *items: _T) -> None:
+        """Put items into the queue at the end without blocking."""
+        self._put_left_or_right_nowait(self._put, *items)
+
+    # private
+
+    def _put_left(self, item: _T) -> None:
+        self._queue.appendleft(item)
+
+    def _get_right(self) -> _T:
+        return self._queue.pop()
+
+    async def _get_left_or_right(self, getter_use: Callable[[], _T], /) -> _T:
+        while self.empty():  # pragma: no cover
+            getter = self._get_loop().create_future()  # pyright: ignore[reportAttributeAccessIssue]
+            self._getters.append(getter)
+            try:
+                await getter
+            except:
+                getter.cancel()
+                with suppress(ValueError):
+                    self._getters.remove(getter)
+                if not self.empty() and not getter.cancelled():
+                    self._wakeup_next(self._getters)  # pyright: ignore[reportAttributeAccessIssue]
+                raise
+        return getter_use()
+
+    def _get_left_or_right_nowait(self, getter: Callable[[], _T], /) -> _T:
+        if self.empty():
+            raise QueueEmpty
+        item = getter()
+        self._wakeup_next(self._putters)  # pyright: ignore[reportAttributeAccessIssue]
+        return item
+
+    async def _put_left_or_right(
+        self, putter_use: Callable[[_T], None], /, *items: _T
+    ) -> None:
+        """Put an item into the queue."""
+        for item in items:
+            await self._put_left_or_right_one(putter_use, item)
+
+    async def _put_left_or_right_one(
+        self, putter_use: Callable[[_T], None], item: _T, /
+    ) -> None:
+        """Put an item into the queue."""
+        while self.full():  # pragma: no cover
+            putter = self._get_loop().create_future()  # pyright: ignore[reportAttributeAccessIssue]
+            self._putters.append(putter)
+            try:
+                await putter
+            except:
+                putter.cancel()
+                with suppress(ValueError):
+                    self._putters.remove(putter)
+                if not self.full() and not putter.cancelled():
+                    self._wakeup_next(self._putters)  # pyright: ignore[reportAttributeAccessIssue]
+                raise
+        return putter_use(item)
+
+    def _put_left_or_right_nowait(
+        self, putter: Callable[[_T], None], /, *items: _T
+    ) -> None:
+        for item in items:
+            self._put_left_or_right_nowait_one(putter, item)
+
+    def _put_left_or_right_nowait_one(
+        self, putter: Callable[[_T], None], item: _T, /
+    ) -> None:
+        if self.full():  # pragma: no cover
+            raise QueueFull
+        putter(item)
+        self._unfinished_tasks += 1
+        self._finished.clear()
+        self._wakeup_next(self._getters)  # pyright: ignore[reportAttributeAccessIssue]
+
+
 ##
 
 
@@ -87,7 +249,6 @@ class EnhancedTaskGroup(TaskGroup):
     _timeout: Duration | None
     _error: type[Exception]
     _stack: AsyncExitStack
-    _stack_entered: bool
     _timeout_cm: _AsyncGeneratorContextManager[None] | None
 
     @override
@@ -103,7 +264,6 @@ class EnhancedTaskGroup(TaskGroup):
         self._timeout = timeout
         self._error = error
         self._stack = AsyncExitStack()
-        self._stack_entered = False  # TOOD: no need
         self._timeout_cm = None
 
     @override
@@ -136,8 +296,10 @@ class EnhancedTaskGroup(TaskGroup):
         coroutine = self._wrap_with_timeout(coroutine)
         return super().create_task(coroutine, name=name, context=context)
 
-    async def enter_async_context(self, cm: AbstractAsyncContextManager[_T], /) -> _T:
-        return await self._stack.enter_async_context(cm)
+    def create_task_context(self, cm: AbstractAsyncContextManager[_T], /) -> Task[_T]:
+        """Have the TaskGroup start an asynchronous context manager."""
+        _ = self._stack.push_async_callback(cm.__aexit__, None, None, None)
+        return self.create_task(cm.__aenter__())
 
     async def _wrap_with_semaphore(
         self, semaphore: Semaphore, coroutine: _CoroutineLike[_T], /
@@ -178,18 +340,12 @@ class InfiniteLooper(ABC, Generic[THashable]):
 
     async def __aenter__(self) -> Self:
         """Context manager entry."""
-        if (self._task is None) and (self._depth == 0):
-            _ = await self._stack.__aenter__()
+        if self._depth == 0:
             self._task = create_task(self._run_looper())
             if self._await_upon_aenter:
                 with suppress(CancelledError):
                     await self._task
-        elif (self._task is not None) and (self._depth >= 1):
-            ...
-        else:
-            raise ImpossibleCaseError(  # pragma: no cover
-                case=[f"{self._task=}", f"{self._depth=}"]
-            )
+            _ = await self._stack.__aenter__()
         self._depth += 1
         return self
 
@@ -201,18 +357,16 @@ class InfiniteLooper(ABC, Generic[THashable]):
     ) -> None:
         """Context manager exit."""
         _ = (exc_type, exc_value, traceback)
-        if (self._task is None) or (self._depth == 0):
-            raise ImpossibleCaseError(  # pragma: no cover
-                case=[f"{self._task=}", f"{self._depth=}"]
-            )
-        self._depth -= 1
-        if self._depth == 0:
-            _ = await self._stack.__aexit__(exc_type, exc_value, traceback)
+        self._depth = max(self._depth - 1, 0)
+        if (self._depth == 0) and (self._task is not None):
             with suppress(CancelledError):
                 await self._task
             self._task = None
-            with suppress(Exception):
+            try:
                 await self._teardown()
+            except Exception as error:  # noqa: BLE001
+                self._error_upon_teardown(error)
+            _ = await self._stack.__aexit__(exc_type, exc_value, traceback)
 
     async def stop(self) -> None:
         """Stop the service."""
@@ -436,14 +590,13 @@ class _InfiniteLooperDefaultEventError(InfiniteLooperError):
 class InfiniteQueueLooper(InfiniteLooper[THashable], Generic[THashable, _T]):
     """An infinite loop which processes a queue."""
 
-    queue_type: type[Queue[_T]] = field(default=Queue, repr=False)
     _await_upon_aenter: bool = field(default=False, init=False, repr=False)
-    _queue: Queue[_T] = field(init=False)
+    _queue: EnhancedQueue[_T] = field(init=False, repr=False)
 
     @override
     def __post_init__(self) -> None:
         super().__post_init__()
-        self._queue = self.queue_type()
+        self._queue = EnhancedQueue()
 
     def __len__(self) -> int:
         return self._queue.qsize()
@@ -451,55 +604,32 @@ class InfiniteQueueLooper(InfiniteLooper[THashable], Generic[THashable, _T]):
     @override
     async def _core(self) -> None:
         """Run the core part of the loop."""
-        items = await get_items(self._queue)
-        try:
-            await self._process_items(*items)
-        except Exception as error:  # noqa: BLE001
-            raise InfiniteQueueLooperError(
-                looper=self, items=items, error=error
-            ) from None
+        first = await self._queue.get_left()
+        self._queue.put_left_nowait(first)
+        await self._process_queue()
 
     @abstractmethod
-    async def _process_items(self, *items: _T) -> None:
-        """Process the items."""
+    async def _process_queue(self) -> None:
+        """Process the queue."""
 
     def empty(self) -> bool:
         """Check if the queue is empty."""
         return self._queue.empty()
 
-    def put_items_nowait(self, *items: _T) -> None:
-        """Put items into the queue."""
-        put_items_nowait(items, self._queue)
+    def put_left_nowait(self, *items: _T) -> None:
+        """Put items into the queue at the start without blocking."""
+        self._queue.put_left_nowait(*items)  # pragma: no cover
+
+    def put_right_nowait(self, *items: _T) -> None:
+        """Put items into the queue at the end without blocking."""
+        self._queue.put_right_nowait(*items)  # pragma: no cover
 
     async def run_until_empty(self) -> None:
         """Run until the queue is empty."""
         while not self.empty():
-            await self._process_items(*get_items_nowait(self._queue))
+            await self._process_queue()
         await self.stop()
 
-    @override
-    def _error_upon_core(self, error: Exception, /) -> None:
-        """Handle any errors upon running the core function."""
-        if self.logger is not None:
-            if isinstance(error, InfiniteQueueLooperError):
-                getLogger(name=self.logger).error(
-                    "%r encountered %s whilst processing %d item(s) %s; sleeping %s...",
-                    get_class_name(self),
-                    repr_error(error.error),
-                    len(error.items),
-                    get_repr(error.items),
-                    self._sleep_restart_desc,
-                )
-            else:
-                super()._error_upon_core(error)  # pragma: no cover
-
-
-@dataclass(kw_only=True, slots=True)
-class InfiniteQueueLooperError(Exception, Generic[_T]):
-    looper: InfiniteQueueLooper[Any, Any]
-    items: Sequence[_T]
-    error: Exception
-
 
 ##
 
@@ -723,11 +853,11 @@ async def timeout_dur(
 
 
 __all__ = [
+    "EnhancedQueue",
     "EnhancedTaskGroup",
     "InfiniteLooper",
     "InfiniteLooperError",
     "InfiniteQueueLooper",
-    "InfiniteQueueLooperError",
     "StreamCommandOutput",
     "UniquePriorityQueue",
     "UniqueQueue",

utilities/redis.py

@@ -597,8 +597,8 @@ class Publisher(InfiniteQueueLooper[None, tuple[str, EncodableT]]):
     timeout: Duration = _PUBLISH_TIMEOUT
 
     @override
-    async def _process_items(self, *items: tuple[str, EncodableT]) -> None:
-        for item in items:  # skipif-ci-and-not-linux
+    async def _process_queue(self) -> None:
+        for item in self._queue.get_all_nowait():  # skipif-ci-and-not-linux
             channel, data = item
             _ = await publish(
                 self.redis,

utilities/slack_sdk.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from asyncio import Queue
 from dataclasses import dataclass
 from http import HTTPStatus
 from logging import NOTSET, Handler, LogRecord
@@ -47,15 +46,12 @@ class SlackHandler(Handler, InfiniteQueueLooper[None, str]):
         level: int = NOTSET,
         sleep_core: Duration = _SLEEP,
         sleep_restart: Duration = _SLEEP,
-        queue_type: type[Queue[str]] = Queue,
         sender: Callable[[str, str], Coroutine1[None]] = _send_adapter,
         timeout: Duration = _TIMEOUT,
     ) -> None:
-        InfiniteQueueLooper.__init__(  # InfiniteQueueLooper first
-            self, queue_type=queue_type
-        )
+        InfiniteQueueLooper.__init__(self)  # InfiniteQueueLooper first
         InfiniteQueueLooper.__post_init__(self)
-        Handler.__init__(self, level=level)
+        Handler.__init__(self, level=level)  # Handler next
         self.url = url
         self.sender = sender
         self.timeout = timeout
@@ -65,14 +61,14 @@ class SlackHandler(Handler, InfiniteQueueLooper[None, str]):
     @override
     def emit(self, record: LogRecord) -> None:
         try:
-            self.put_items_nowait(self.format(record))
+            self.put_right_nowait(self.format(record))
         except Exception:  # noqa: BLE001  # pragma: no cover
             self.handleError(record)
 
     @override
-    async def _process_items(self, *items: str) -> None:
-        """Process the first item."""
-        text = "\n".join(items)
+    async def _process_queue(self) -> None:
+        messages = self._queue.get_all_nowait()
+        text = "\n".join(messages)
         async with timeout_dur(duration=self.timeout):
             await self.sender(self.url, text)
 

utilities/sqlalchemy.py

@@ -623,7 +623,8 @@ class Upserter(InfiniteQueueLooper[None, _InsertItem]):
     error_insert: type[Exception] = TimeoutError
 
     @override
-    async def _process_items(self, *items: _InsertItem) -> None:
+    async def _process_queue(self) -> None:
+        items = self._queue.get_all_nowait()
         await upsert_items(
             self.engine,
             *items,