dycw-utilities 0.114.2__tar.gz → 0.114.3__tar.gz

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dycw-utilities
-Version: 0.114.2
+Version: 0.114.3
 Author-email: Derek Wan <d.wan@icloud.com>
 License-File: LICENSE
 Requires-Python: >=3.12

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/pyproject.toml

@@ -92,7 +92,7 @@ dependencies = [
 name = "dycw-utilities"
 readme = "README.md"
 requires-python = ">= 3.12"
-version = "0.114.2"
+version = "0.114.3"
 
 [project.optional-dependencies]
 test = [
@@ -334,7 +334,7 @@ zzz-test-zoneinfo = [
 # bump-my-version
 [tool.bumpversion]
 allow_dirty = true
-current_version = "0.114.2"
+current_version = "0.114.3"
 
 [[tool.bumpversion.files]]
 filename = "src/utilities/__init__.py"

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/tests/test_asyncio.py

@@ -21,6 +21,7 @@ from hypothesis.strategies import (
     DataObject,
     data,
     integers,
+    just,
     lists,
     none,
     permutations,
@@ -434,7 +435,8 @@ class TestInfiniteLooper:
         assert 10 <= parent.child.counter <= 15
         assert 3 <= parent.counter <= 7
 
-    async def test_error_upon_initialize(self) -> None:
+    @given(logger=just("logger") | none())
+    async def test_error_upon_initialize(self, *, logger: str | None) -> None:
         class CustomError(Exception): ...
 
         @dataclass(kw_only=True)
@@ -449,9 +451,10 @@ class TestInfiniteLooper:
 
         with raises(TimeoutError):
             async with timeout_dur(duration=0.5):
-                _ = await Example(sleep_core=0.1)()
+                _ = await Example(sleep_core=0.1, logger=logger)()
 
-    async def test_error_upon_core(self) -> None:
+    @given(logger=just("logger") | none())
+    async def test_error_upon_core(self, *, logger: str | None) -> None:
         class CustomError(Exception): ...
 
         @dataclass(kw_only=True)
@@ -468,7 +471,7 @@ class TestInfiniteLooper:
 
         with raises(TimeoutError):
             async with timeout_dur(duration=0.5):
-                _ = await Example(sleep_core=0.1)()
+                _ = await Example(sleep_core=0.1, logger=logger)()
 
     async def test_error_no_event_found(self) -> None:
         @dataclass(kw_only=True)
@@ -536,7 +539,7 @@ class TestInfiniteQueueLooper:
 
             @override
             async def _process_items(self, *items: int) -> None:
-                raise CustomError
+                raise CustomError(*items)
 
         processor = Example(sleep_core=0.1)
         processor.put_items_nowait(1)

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/tests/test_redis.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from asyncio import create_task, get_running_loop, sleep
+from io import BytesIO, StringIO
 from typing import TYPE_CHECKING, Any
 
 from hypothesis import HealthCheck, Phase, given, settings
@@ -17,6 +18,7 @@ from redis.asyncio import Redis
 
 from tests.conftest import SKIPIF_CI_AND_NOT_LINUX
 from tests.test_operator import make_objects
+from utilities.asyncio import EnhancedTaskGroup, sleep_dur
 from utilities.functions import get_class_name
 from utilities.hypothesis import (
     int64s,
@@ -28,6 +30,8 @@ from utilities.hypothesis import (
 from utilities.orjson import deserialize, serialize
 from utilities.redis import (
     Publisher,
+    PublisherIQL,
+    PublisherIQLError,
     publish,
     redis_hash_map_key,
     redis_key,
@@ -118,7 +122,7 @@ class TestPublisher:
     @given(
         data=data(),
         channel=text_ascii(min_size=1).map(
-            lambda c: f"{get_class_name(TestPublishAndSubscribe)}_obj_ser_{c}"
+            lambda c: f"{get_class_name(TestPublisher)}_obj_ser_{c}"
         ),
         obj=make_objects(),
     )
@@ -157,7 +161,7 @@ class TestPublisher:
     @given(
         data=data(),
         channel=text_ascii(min_size=1).map(
-            lambda c: f"{get_class_name(TestPublishAndSubscribe)}_text_no_ser_{c}"
+            lambda c: f"{get_class_name(TestPublisher)}_text_no_ser_{c}"
         ),
         text=text_ascii(min_size=1),
     )
@@ -188,6 +192,93 @@ class TestPublisher:
                 _ = task.cancel()
 
 
+class TestPublisherIQL:
+    @given(
+        data=data(),
+        channel=text_ascii(min_size=1).map(
+            lambda c: f"{get_class_name(TestPublisherIQL)}_main_{c}"
+        ),
+        obj=make_objects(),
+    )
+    @mark.flaky
+    @settings(
+        max_examples=1,
+        phases={Phase.generate},
+        suppress_health_check={HealthCheck.function_scoped_fixture},
+    )
+    @SKIPIF_CI_AND_NOT_LINUX
+    async def test_main(self, *, data: DataObject, channel: str, obj: Any) -> None:
+        buffer = StringIO()
+        async with yield_test_redis(data) as test:
+
+            async def listener() -> None:
+                async for obj_i in subscribe(
+                    test.redis.pubsub(), channel, deserializer=deserialize
+                ):
+                    _ = buffer.write(str(obj_i))
+
+            publisher = PublisherIQL(
+                redis=test.redis, serializer=serialize, sleep_core=0.1
+            )
+
+            async def sleep_then_put() -> None:
+                await sleep_dur(duration=0.1)
+                publisher.put_items_nowait((channel, obj))
+
+            with raises(ExceptionGroup):  # noqa: PT012
+                async with EnhancedTaskGroup(timeout=1.0) as tg:
+                    _ = tg.create_task(publisher())
+                    _ = tg.create_task(listener())
+                    _ = tg.create_task(sleep_then_put())
+
+        assert buffer.getvalue() == str(obj)
+
+    @given(
+        data=data(),
+        channel=text_ascii(min_size=1).map(
+            lambda c: f"{get_class_name(TestPublisherIQL)}_text_without_serialize_{c}"
+        ),
+        text=text_ascii(min_size=1),
+    )
+    @settings(
+        max_examples=1,
+        phases={Phase.generate},
+        suppress_health_check={HealthCheck.function_scoped_fixture},
+    )
+    @SKIPIF_CI_AND_NOT_LINUX
+    async def test_text_without_serialize(
+        self, *, data: DataObject, channel: str, text: str
+    ) -> None:
+        buffer = BytesIO()
+        async with yield_test_redis(data) as test:
+
+            async def listener() -> None:
+                async for bytes_i in subscribe(test.redis.pubsub(), channel):
+                    _ = buffer.write(bytes_i)
+
+            publisher = PublisherIQL(redis=test.redis, sleep_core=0.1)
+
+            async def sleep_then_put() -> None:
+                await sleep_dur(duration=0.1)
+                publisher.put_items_nowait((channel, text))
+
+            with raises(ExceptionGroup):  # noqa: PT012
+                async with EnhancedTaskGroup(timeout=1.0) as tg:
+                    _ = tg.create_task(publisher())
+                    _ = tg.create_task(listener())
+                    _ = tg.create_task(sleep_then_put())
+
+        assert buffer.getvalue() == text.encode()
+
+    @given(data=data())
+    @SKIPIF_CI_AND_NOT_LINUX
+    async def test_error(self, *, data: DataObject) -> None:
+        async with yield_test_redis(data) as test:
+            publisher = PublisherIQL(redis=test.redis)
+            with raises(PublisherIQLError, match="Error running 'PublisherIQL'"):
+                raise PublisherIQLError(publisher=publisher)
+
+
 class TestSubscribeMessages:
     @given(
         channel=text_ascii(min_size=1).map(

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/tests/test_sqlalchemy.py

@@ -27,6 +27,7 @@ from sqlalchemy.exc import DatabaseError, OperationalError, ProgrammingError
 from sqlalchemy.ext.asyncio import AsyncEngine
 from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column
 
+from utilities.asyncio import EnhancedTaskGroup, sleep_dur
 from utilities.hypothesis import (
     int32s,
     pairs,
@@ -44,6 +45,8 @@ from utilities.sqlalchemy import (
     TablenameMixin,
     TableOrORMInstOrClass,
     Upserter,
+    UpserterIQL,
+    UpserterIQLError,
     UpsertItemsError,
     _get_dialect,
     _get_dialect_max_params,
@@ -1177,6 +1180,48 @@ class TestUpserter:
         assert set(res) == set(pairs)
 
 
+class TestUpserterIQL:
+    @given(
+        data=data(),
+        name=_table_names(),
+        triples=_upsert_lists(nullable=True, min_size=1),
+    )
+    @mark.flaky
+    @settings(max_examples=1, phases={Phase.generate})
+    async def test_main(
+        self, *, data: DataObject, name: str, triples: list[tuple[int, bool, bool]]
+    ) -> None:
+        table = Table(
+            name,
+            MetaData(),
+            Column("id_", Integer, primary_key=True),
+            Column("value", Boolean, nullable=True),
+        )
+        engine = await sqlalchemy_engines(data, table)
+        upserter = UpserterIQL(engine=engine, sleep_core=0.1)
+        pairs = [(id_, init) for id_, init, _ in triples]
+
+        async def sleep_then_put() -> None:
+            await sleep_dur(duration=0.1)
+            upserter.put_items_nowait((pairs, table))
+
+        with raises(ExceptionGroup):  # noqa: PT012
+            async with EnhancedTaskGroup(timeout=1.0) as tg:
+                _ = tg.create_task(upserter())
+                _ = tg.create_task(sleep_then_put())
+        sel = select(table)
+        async with engine.begin() as conn:
+            res = (await conn.execute(sel)).all()
+        assert set(res) == set(pairs)
+
+    @given(data=data())
+    async def test_error(self, *, data: DataObject) -> None:
+        engine = await sqlalchemy_engines(data)
+        upserter = UpserterIQL(engine=engine)
+        with raises(UpserterIQLError, match="Error running 'UpserterIQL'"):
+            raise UpserterIQLError(upserter=upserter)
+
+
 class TestUpsertItems:
     @given(data=data(), name=_table_names(), triple=_upsert_triples(nullable=True))
     @settings_with_reduced_examples(phases={Phase.generate})

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/utilities/__init__.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
-__version__ = "0.114.2"
+__version__ = "0.114.3"

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/utilities/asyncio.py

@@ -25,6 +25,7 @@ from contextlib import (
 )
 from dataclasses import dataclass, field
 from io import StringIO
+from logging import getLogger
 from subprocess import PIPE
 from sys import stderr, stdout
 from typing import (
@@ -41,7 +42,7 @@ from typing import (
 
 from utilities.datetime import MILLISECOND, MINUTE, SECOND, datetime_duration_to_float
 from utilities.errors import ImpossibleCaseError
-from utilities.functions import ensure_int, ensure_not_none
+from utilities.functions import ensure_int, ensure_not_none, get_class_name
 from utilities.sentinel import Sentinel, sentinel
 from utilities.types import (
     MaybeCallableEvent,
@@ -59,6 +60,7 @@ if TYPE_CHECKING:
 
     from utilities.types import Duration
 
+
 _T = TypeVar("_T")
 
 
@@ -328,11 +330,12 @@ class ExceptionProcessor(QueueProcessor[Exception | type[Exception]]):
 class InfiniteLooper(ABC, Generic[THashable]):
     """An infinite loop which can throw exceptions by setting events."""
 
+    sleep_core: Duration = SECOND
+    sleep_restart: Duration = MINUTE
+    logger: str | None = None
     _events: Mapping[THashable, Event] = field(
         default_factory=dict, init=False, repr=False
     )
-    sleep_core: Duration = SECOND
-    sleep_restart: Duration = MINUTE
 
     def __post_init__(self) -> None:
         self._events = {
@@ -395,11 +398,23 @@ class InfiniteLooper(ABC, Generic[THashable]):
 
     def _error_upon_initialize(self, error: Exception, /) -> None:
         """Handle any errors upon initializing the looper."""
-        _ = error
+        if self.logger is not None:
+            getLogger(name=self.logger).error(
+                "Error initializing %r due to %s; sleeping for %s...",
+                get_class_name(self),
+                error,
+                self.sleep_restart,
+            )
 
     def _error_upon_core(self, error: Exception, /) -> None:
         """Handle any errors upon running the core function."""
-        _ = error
+        if self.logger is not None:
+            getLogger(name=self.logger).error(
+                "Error running core part of %r due to %s; sleeping for %s...",
+                get_class_name(self),
+                error,
+                self.sleep_restart,
+            )
 
     def _raise_error(self, event: THashable, /) -> NoReturn:
         """Raise the error corresponding to given event."""
@@ -684,7 +699,9 @@ __all__ = [
     "AsyncService",
     "EnhancedTaskGroup",
     "ExceptionProcessor",
+    "InfiniteLooper",
     "InfiniteLooperError",
+    "InfiniteQueueLooper",
     "QueueProcessor",
     "StreamCommandOutput",
     "UniquePriorityQueue",

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/utilities/redis.py

@@ -21,7 +21,7 @@ from uuid import UUID, uuid4
 from redis.asyncio import Redis
 from redis.typing import EncodableT
 
-from utilities.asyncio import QueueProcessor, timeout_dur
+from utilities.asyncio import InfiniteQueueLooper, QueueProcessor, timeout_dur
 from utilities.datetime import (
     MILLISECOND,
     SECOND,
@@ -30,7 +30,7 @@ from utilities.datetime import (
     get_now,
 )
 from utilities.errors import ImpossibleCaseError
-from utilities.functions import ensure_int
+from utilities.functions import ensure_int, get_class_name
 from utilities.iterables import always_iterable, one
 
 if TYPE_CHECKING:
@@ -40,6 +40,7 @@ if TYPE_CHECKING:
         Awaitable,
         Callable,
         Iterable,
+        Iterator,
         Mapping,
         Sequence,
     )
@@ -49,7 +50,7 @@ if TYPE_CHECKING:
     from redis.typing import ResponseT
 
     from utilities.iterables import MaybeIterable
-    from utilities.types import Duration, TypeLike
+    from utilities.types import Duration, MaybeType, TypeLike
 
 
 _K = TypeVar("_K")
@@ -603,6 +604,42 @@ class Publisher(QueueProcessor[tuple[str, EncodableT]]):
         )
 
 
+@dataclass(kw_only=True)
+class PublisherIQL(InfiniteQueueLooper[None, tuple[str, EncodableT]]):
+    """Publish a set of messages to Redis."""
+
+    redis: Redis
+    serializer: Callable[[Any], EncodableT] | None = None
+    timeout: Duration = _PUBLISH_TIMEOUT
+
+    @override
+    async def _process_items(self, *items: tuple[str, EncodableT]) -> None:
+        for item in items:  # skipif-ci-and-not-linux
+            channel, data = item
+            _ = await publish(
+                self.redis,
+                channel,
+                data,
+                serializer=self.serializer,
+                timeout=self.timeout,
+            )
+
+    @override
+    def _yield_events_and_exceptions(
+        self,
+    ) -> Iterator[tuple[None, MaybeType[BaseException]]]:
+        yield (None, PublisherIQLError)  # skipif-ci-and-not-linux
+
+
+@dataclass(kw_only=True)
+class PublisherIQLError(Exception):
+    publisher: PublisherIQL
+
+    @override
+    def __str__(self) -> str:
+        return f"Error running {get_class_name(self.publisher)!r}"  # skipif-ci-and-not-linux
+
+
 ##
 
 
@@ -780,6 +817,8 @@ _ = _TestRedis
 
 __all__ = [
     "Publisher",
+    "PublisherIQL",
+    "PublisherIQLError",
     "RedisHashMapKey",
     "RedisKey",
     "publish",

{dycw_utilities-0.114.2 → dycw_utilities-0.114.3}/src/utilities/sqlalchemy.py

@@ -57,7 +57,7 @@ from sqlalchemy.orm import (
 from sqlalchemy.orm.exc import UnmappedClassError
 from sqlalchemy.pool import NullPool, Pool
 
-from utilities.asyncio import QueueProcessor, timeout_dur
+from utilities.asyncio import InfiniteQueueLooper, QueueProcessor, timeout_dur
 from utilities.functions import (
     ensure_str,
     get_class_name,
@@ -80,7 +80,13 @@ from utilities.iterables import (
 )
 from utilities.reprlib import get_repr
 from utilities.text import snake_case
-from utilities.types import Duration, MaybeIterable, StrMapping, TupleOrStrMapping
+from utilities.types import (
+    Duration,
+    MaybeIterable,
+    MaybeType,
+    StrMapping,
+    TupleOrStrMapping,
+)
 
 _T = TypeVar("_T")
 type _EngineOrConnectionOrAsync = Engine | Connection | AsyncEngine | AsyncConnection
@@ -644,6 +650,51 @@ class Upserter(QueueProcessor[_InsertItem]):
         await self._post_upsert(items)
 
 
+@dataclass(kw_only=True)
+class UpserterIQL(InfiniteQueueLooper[None, _InsertItem]):
+    """Upsert a set of items to a database."""
+
+    engine: AsyncEngine
+    snake: bool = False
+    selected_or_all: _SelectedOrAll = "selected"
+    chunk_size_frac: float = CHUNK_SIZE_FRAC
+    assume_tables_exist: bool = False
+    timeout_create: Duration | None = None
+    error_create: type[Exception] = TimeoutError
+    timeout_insert: Duration | None = None
+    error_insert: type[Exception] = TimeoutError
+
+    @override
+    async def _process_items(self, *items: _InsertItem) -> None:
+        await upsert_items(
+            self.engine,
+            *items,
+            snake=self.snake,
+            selected_or_all=self.selected_or_all,
+            chunk_size_frac=self.chunk_size_frac,
+            assume_tables_exist=self.assume_tables_exist,
+            timeout_create=self.timeout_create,
+            error_create=self.error_create,
+            timeout_insert=self.timeout_insert,
+            error_insert=self.error_insert,
+        )
+
+    @override
+    def _yield_events_and_exceptions(
+        self,
+    ) -> Iterator[tuple[None, MaybeType[BaseException]]]:
+        yield (None, UpserterIQLError)
+
+
+@dataclass(kw_only=True)
+class UpserterIQLError(Exception):
+    upserter: UpserterIQL
+
+    @override
+    def __str__(self) -> str:
+        return f"Error running {get_class_name(self.upserter)!r}"
+
+
 ##
 
 
@@ -1099,6 +1150,9 @@ __all__ = [
     "InsertItemsError",
     "TablenameMixin",
     "UpsertItemsError",
+    "Upserter",
+    "UpserterIQL",
+    "UpserterIQLError",
     "check_engine",
     "columnwise_max",
     "columnwise_min",