dycw-utilities 0.147.3__tar.gz → 0.148.0__tar.gz

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dycw-utilities
-Version: 0.147.3
+Version: 0.148.0
 Author-email: Derek Wan <d.wan@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.12

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/pyproject.toml

@@ -102,7 +102,7 @@ dependencies = [
 name = "dycw-utilities"
 readme = "README.md"
 requires-python = ">= 3.12"
-version = "0.147.3"
+version = "0.148.0"
 
 [project.entry-points.pytest11]
 pytest-randomly = "utilities.pytest_plugins.pytest_randomly"
@@ -135,7 +135,7 @@ test = [
 # bump-my-version
 [tool.bumpversion]
 allow_dirty = true
-current_version = "0.147.3"
+current_version = "0.148.0"
 
 [[tool.bumpversion.files]]
 filename = "src/utilities/__init__.py"

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/src/tests/test_asyncio.py

@@ -8,11 +8,12 @@ from typing import TYPE_CHECKING, ClassVar
 
 from hypothesis import given
 from hypothesis.strategies import booleans, dictionaries, integers, lists, none
-from pytest import RaisesGroup, raises
+from pytest import LogCaptureFixture, RaisesGroup, mark, param, raises
 
 from utilities.asyncio import (
     AsyncDict,
     EnhancedTaskGroup,
+    get_coroutine_name,
     get_items,
     get_items_nowait,
     loop_until_succeed,
@@ -27,6 +28,7 @@ from utilities.asyncio import (
 )
 from utilities.hypothesis import pairs, text_ascii
 from utilities.pytest import skipif_windows
+from utilities.text import unique_str
 from utilities.timer import Timer
 from utilities.whenever import MILLISECOND, SECOND, get_now
 
@@ -287,6 +289,16 @@ class TestEnhancedTaskGroup:
                 _ = tg.create_task(sleep_td(2 * self.delta))
 
 
+class TestGetCoroutineName:
+    def test_main(self) -> None:
+        async def func() -> None:
+            return None
+
+        result = get_coroutine_name(func)
+        expected = "func"
+        assert result == expected
+
+
 class TestGetItems:
     @given(
         xs=lists(integers(), min_size=1),
@@ -306,7 +318,12 @@ class TestGetItems:
 
 
 class TestLoopUntilSucceed:
-    async def test_main(self) -> None:
+    @mark.parametrize("sleep", [param(MILLISECOND), param(None)])
+    @mark.parametrize("use_logger", [param(True), param(False)])
+    async def test_main(
+        self, *, caplog: LogCaptureFixture, sleep: TimeDelta | None, use_logger: bool
+    ) -> None:
+        caplog.set_level("DEBUG", logger=(name := unique_str()))
         counter = 0
 
         async def func() -> None:
@@ -315,12 +332,22 @@ class TestLoopUntilSucceed:
             if counter <= 3:
                 raise ValueError
 
-        _ = await loop_until_succeed(lambda: func())
+        assert await loop_until_succeed(
+            lambda: func(), logger=name if use_logger else None, sleep=sleep
+        )
         assert counter == 4
 
+        if use_logger:
+            messages = [r.message for r in caplog.records if r.name == name]
+            expected = 3 * (
+                ["Error running 'func'"]
+                + ([] if sleep is None else ["Sleeping for PT0.001S..."])
+                + ["Retrying 'func'..."]
+            )
+            assert messages == expected
+
     async def test_error(self) -> None:
         counter = 0
-        errors: list[Exception] = []
 
         async def func() -> None:
             nonlocal counter
@@ -328,12 +355,8 @@ class TestLoopUntilSucceed:
             if counter <= 3:
                 raise ValueError
 
-        def error(error: Exception, /) -> None:
-            errors.append(error)
-
-        _ = await loop_until_succeed(lambda: func(), error=error)
-        assert counter == 4
-        assert len(errors) == 3
+        assert not await loop_until_succeed(lambda: func(), errors=ValueError)
+        assert counter == 1
 
 
 class TestPutItems:

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/src/tests/test_pottery.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from asyncio import TaskGroup
-from itertools import repeat
 from typing import TYPE_CHECKING, ClassVar
 
 from pottery import AIORedlock
@@ -74,7 +73,7 @@ class TestTryYieldCoroutineLooper:
             redis, key, timeout_acquire=self.delta, logger=logger
         ) as looper:
             if looper is not None:
-                await looper(self.func_main, lst)
+                assert await looper(self.func_main, lst)
 
     async def delayed(
         self,
@@ -102,11 +101,11 @@ class TestTryYieldCoroutineLooper:
             logger=name if use_logger else None,
         ) as looper:
             assert looper is not None
-            await looper(self.func_error, lst)
+            assert await looper(self.func_error, lst)
 
         if use_logger:
             messages = [r.message for r in caplog.records if r.name == name]
-            expected = list(repeat("Error running 'func_error'", times=3))
+            expected = 3 * ["Error running 'func_error'", "Retrying 'func_error'..."]
             assert messages == expected
 
     async def func_error(self, lst: list[None], /) -> None:

dycw_utilities-0.148.0/src/tests/test_yield_access/script.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from asyncio import run
+from logging import getLogger
+from random import randint
+from typing import TYPE_CHECKING
+
+from redis.asyncio import Redis
+
+from utilities.asyncio import sleep_td
+from utilities.logging import setup_logging
+from utilities.pathlib import get_repo_root
+from utilities.pottery import extend_lock, try_yield_coroutine_looper
+from utilities.whenever import SECOND
+
+if TYPE_CHECKING:
+    from pottery import AIORedlock
+
+_LOGGER = getLogger(__name__)
+
+
+async def script(*, lock: AIORedlock | None = None) -> None:
+    total = 1000
+    fail = 30
+    success = 3
+    while True:
+        n = randint(0, total)
+        if n < fail:
+            _LOGGER.info("n = %d; failing...", n)
+            msg = f"n = {n}; failure"
+            raise ValueError(msg)
+        if fail <= n < (fail + success):
+            _LOGGER.info("n = %d; succeeding...", n)
+            return
+        _LOGGER.info("n = %d", n)
+        await extend_lock(lock=lock)
+        await sleep_td(SECOND / 3)
+
+
+async def service() -> None:
+    redis = Redis()
+    async with try_yield_coroutine_looper(
+        redis,
+        "utilities-test",
+        num=1,
+        timeout_release=5 * SECOND,
+        logger=_LOGGER,
+        sleep_error=4 * SECOND,
+    ) as looper:
+        if looper is not None:
+            result = await looper(script, lock=looper.lock)
+            _LOGGER.info("script %s", "succeeded" if result else "failed")
+
+
+def main() -> None:
+    setup_logging(logger=_LOGGER, files_dir=get_repo_root().joinpath(".logs"))
+    run(service())
+
+
+if __name__ == "__main__":
+    main()

dycw_utilities-0.148.0/src/tests/test_yield_access/script.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env sh
+
+# Add to your `crontab` as:
+#     * * * * * /Users/derekwan/work/python-utilities/tests/test_yield_access/script.sh
+
+# helpers
+echo_pid_date() { echo "[$$ | $(date +'%Y-%m-%d %H:%M:%S')] $*"; }
+
+# add to `$PATH`
+for __dir in "${HOME}/.local/bin" '/opt/homebrew/bin' '/opt/homebrew/opt/postgresql@17/bin'; do
+    if [ -d "${__dir}" ]; then
+        case ":${PATH}:" in
+        *:"$__dir":*) ;;
+        *)
+            export PATH="${__dir}:${PATH}"
+            ;;
+        esac
+    fi
+done
+
+# log file
+__package_dir="${HOME}/work/python-utilities"
+__log_file="${__package_dir}/.logs/test-yield-access"
+
+# check if binaries are accessible
+for __app in direnv just uv; do
+    if ! command -v "${__app}" >/dev/null 2>&1; then
+        echo_pid_date "ERROR: Command '${__app}' not found; exiting..." 2>&1 | tee -a "${__log_file}"
+        exit
+    fi
+done
+
+# enter package
+echo_pid_date "Entering package directory '${__package_dir}'..." 2>&1 | tee -a "${__log_file}"
+cd "${__package_dir}" || exit
+
+# trim log
+__threshold=$((10 * 1024 * 1024))
+if [ -f "${__log_file}" ]; then
+    __log_size=$(wc -c <"${__log_file}")
+    if [ "${__log_size}" -gt "${__threshold}" ]; then
+        echo_pid_date "Truncating log file '${__log_file}' (log size = ${__log_size})..." 2>&1 | tee -a "${__log_file}"
+        __total_lines=$(wc -l <"${__log_file}")
+        __keep_lines=$((__total_lines / 2))
+        __tmp_log_file="${__log_file}.tmp.$$"
+        tail -n "${__keep_lines}" "${__log_file}" >"${__tmp_log_file}" && mv "${__tmp_log_file}" "${__log_file}"
+    fi
+fi
+
+# run the script
+echo_pid_date "Running 'PYTHONPATH=src/tests/test_yield_access python -m script'..." 2>&1 | tee -a "${__log_file}"
+__start="$(date +%s)"
+PYTHONPATH=src/tests/test_yield_access direnv exec . python -m script "$*" 2>&1 | tee -a "${__log_file}"
+__exit_code="$?"

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/src/utilities/__init__.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
-__version__ = "0.147.3"
+__version__ = "0.148.0"

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/src/utilities/asyncio.py

@@ -37,8 +37,10 @@ from typing import (
 
 from utilities.errors import ImpossibleCaseError
 from utilities.functions import ensure_int, ensure_not_none, to_bool
+from utilities.logging import get_logger
 from utilities.random import SYSTEM_RANDOM
 from utilities.sentinel import Sentinel, sentinel
+from utilities.warnings import suppress_warnings
 from utilities.whenever import get_now, round_date_or_date_time, to_nanoseconds
 
 if TYPE_CHECKING:
@@ -63,6 +65,7 @@ if TYPE_CHECKING:
     from utilities.types import (
         Coro,
         Delta,
+        LoggerOrName,
         MaybeCallableBool,
         MaybeType,
         SupportsKeysAndGetItem,
@@ -340,6 +343,20 @@ class EnhancedTaskGroup(TaskGroup):
 ##
 
 
+def get_coroutine_name(func: Callable[[], Coro[Any]], /) -> str:
+    """Get the name of a coroutine, and then dispose of it gracefully."""
+    coro = func()
+    name = coro.__name__
+    with suppress_warnings(
+        message="coroutine '.*' was never awaited", category=RuntimeWarning
+    ):
+        del coro
+    return name
+
+
+##
+
+
 async def get_items[T](queue: Queue[T], /, *, max_size: int | None = None) -> list[T]:
     """Get items from a queue; if empty then wait."""
     try:
@@ -380,23 +397,34 @@ async def loop_until_succeed(
     func: Callable[[], Coro[None]],
     /,
     *,
-    error: Callable[[Exception], None] | None = None,
+    logger: LoggerOrName | None = None,
+    errors: type[Exception] | tuple[type[Exception], ...] | None = None,
     sleep: Delta | None = None,
-) -> None:
+) -> bool:
     """Repeatedly call a coroutine until it succeeds."""
+    name = get_coroutine_name(func)
     while True:
         try:
-            return await func()
-        except Exception as err:  # noqa: BLE001
-            if error is not None:
-                error(err)
+            await func()
+        except Exception as error:  # noqa: BLE001
+            if logger is not None:
+                get_logger(logger=logger).error("Error running %r", name, exc_info=True)
             exc_type, exc_value, traceback = sys.exc_info()
             if (exc_type is None) or (exc_value is None):  # pragma: no cover
                 raise ImpossibleCaseError(
                     case=[f"{exc_type=}", f"{exc_value=}"]
                 ) from None
             sys.excepthook(exc_type, exc_value, traceback)
-            await sleep_td(sleep)
+            if (errors is not None) and isinstance(error, errors):
+                return False
+            if sleep is not None:
+                if logger is not None:
+                    get_logger(logger=logger).info("Sleeping for %s...", sleep)
+                await sleep_td(sleep)
+            if logger is not None:
+                get_logger(logger=logger).info("Retrying %r...", name)
+        else:
+            return True
 
 
 ##
@@ -522,6 +550,7 @@ __all__ = [
     "AsyncDict",
     "EnhancedTaskGroup",
     "StreamCommandOutput",
+    "get_coroutine_name",
     "get_items",
     "get_items_nowait",
     "loop_until_succeed",

{dycw_utilities-0.147.3 → dycw_utilities-0.148.0}/src/utilities/pottery.py

@@ -5,16 +5,14 @@ from dataclasses import dataclass
 from sys import maxsize
 from typing import TYPE_CHECKING, override
 
-from pottery import AIORedlock
+from pottery import AIORedlock, ExtendUnlockedLock
 from pottery.exceptions import ReleaseUnlockedLock
 from redis.asyncio import Redis
 
 from utilities.asyncio import loop_until_succeed, sleep_td, timeout_td
 from utilities.contextlib import enhanced_async_context_manager
-from utilities.functools import partial
 from utilities.iterables import always_iterable
 from utilities.logging import get_logger
-from utilities.warnings import suppress_warnings
 from utilities.whenever import MILLISECOND, SECOND, to_seconds
 
 if TYPE_CHECKING:
@@ -72,10 +70,7 @@ async def try_yield_coroutine_looper(
             throttle=throttle,
         ) as lock:
             yield CoroutineLooper(lock=lock, logger=logger, sleep=sleep_error)
-    except (  # skipif-ci-and-not-linux
-        _YieldAccessUnableToAcquireLockError,
-        _YieldAccessAcquiredUnlockedLockError,
-    ) as error:
+    except _YieldAccessUnableToAcquireLockError as error:  # skipif-ci-and-not-linux
         if logger is not None:
             get_logger(logger=logger).info("%s", error)
         async with nullcontext():
@@ -92,27 +87,14 @@ class CoroutineLooper:
 
     async def __call__[**P](
         self, func: Callable[P, Coro[None]], *args: P.args, **kwargs: P.kwargs
-    ) -> None:
+    ) -> bool:
         def make_coro() -> Coro[None]:
             return func(*args, **kwargs)
 
-        await loop_until_succeed(
-            make_coro, error=partial(self._error, func=make_coro), sleep=self.sleep
+        return await loop_until_succeed(
+            make_coro, logger=self.logger, errors=ExtendUnlockedLock, sleep=self.sleep
         )
 
-    def _error(self, error: Exception, /, *, func: Callable[[], Coro[None]]) -> None:
-        _ = error
-        if self.logger is not None:
-            coro = func()
-            name = coro.__name__  # skipif-ci-and-not-linux
-            with suppress_warnings(
-                message="coroutine '.*' was never awaited", category=RuntimeWarning
-            ):
-                del coro
-            get_logger(logger=self.logger).error(
-                "Error running %r", name, exc_info=True
-            )
-
 
 ##
 
@@ -150,8 +132,6 @@ async def yield_access(
         lock = await _get_first_available_lock(
             key, locks, num=num, timeout=timeout_acquire, sleep=sleep
         )
-        if (await lock.locked()) == 0.0:  # pragma: no cover
-            raise _YieldAccessAcquiredUnlockedLockError(key=lock.key)
         yield lock
     finally:  # skipif-ci-and-not-linux
         await sleep_td(throttle)
@@ -175,9 +155,7 @@ async def _get_first_available_lock(
     )
     async with timeout_td(timeout, error=error):  # skipif-ci-and-not-linux
         while True:
-            if (
-                (result := await _get_first_available_lock_if_any(locks)) is not None
-            ) and (await result.locked() > 0.0):
+            if (result := await _get_first_available_lock_if_any(locks)) is not None:
                 return result
             await sleep_td(sleep)
 
@@ -215,13 +193,6 @@ class _YieldAccessUnableToAcquireLockError(YieldAccessError):
         return f"Unable to acquire any 1 of {self.num} locks for {self.key!r} after {self.timeout}"  # skipif-ci-and-not-linux
 
 
-@dataclass(kw_only=True, slots=True)
-class _YieldAccessAcquiredUnlockedLockError(YieldAccessError):
-    @override
-    def __str__(self) -> str:
-        return f"Acquired an unlocked lock {self.key!r}"  # pragma: no cover
-
-
 __all__ = [
     "CoroutineLooper",
     "YieldAccessError",

dycw_utilities-0.148.0/src/utilities/pytest_plugins/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations