haiway 0.19.4__py3-none-any.whl → 0.19.5__py3-none-any.whl

haiway/__init__.py

@@ -22,6 +22,7 @@ from haiway.helpers import (
     LoggerObservability,
     asynchronous,
     cache,
+    process_concurrently,
     retry,
     throttle,
     timeout,
@@ -112,6 +113,7 @@ __all__ = (
     "mimic_function",
     "noop",
     "not_missing",
+    "process_concurrently",
     "retry",
     "setup_logging",
     "throttle",

haiway/helpers/__init__.py

@@ -1,5 +1,6 @@
 from haiway.helpers.asynchrony import asynchronous, wrap_async
 from haiway.helpers.caching import CacheMakeKey, CacheRead, CacheWrite, cache
+from haiway.helpers.concurrent import process_concurrently
 from haiway.helpers.observability import LoggerObservability
 from haiway.helpers.retries import retry
 from haiway.helpers.throttling import throttle
@@ -13,6 +14,7 @@ __all__ = (
     "LoggerObservability",
     "asynchronous",
     "cache",
+    "process_concurrently",
     "retry",
     "throttle",
     "timeout",

haiway/helpers/concurrent.py

@@ -0,0 +1,74 @@
+from asyncio import FIRST_COMPLETED, CancelledError, Task, wait
+from collections.abc import AsyncIterator, Callable, Coroutine
+from concurrent.futures import ALL_COMPLETED
+from typing import Any
+
+from haiway.context import ctx
+
+__all__ = ("process_concurrently",)
+
+
+async def process_concurrently[Element](  # noqa: C901
+    source: AsyncIterator[Element],
+    /,
+    handler: Callable[[Element], Coroutine[Any, Any, None]],
+    *,
+    concurrent_tasks: int = 2,
+    ignore_exceptions: bool = False,
+) -> None:
+    """Process elements from an async iterator concurrently.
+
+    Parameters
+    ----------
+    source: AsyncIterator[Element]
+        An async iterator providing elements to process.
+
+    handler: Callable[[Element], Coroutine[Any, Any, None]]
+        A coroutine function that processes each element.
+
+    concurrent_tasks: int
+        Maximum number of concurrent tasks (must be > 0), default is 2.
+
+    ignore_exceptions: bool
+        If True, exceptions from tasks will be logged but not propagated,
+         default is False.
+
+    """
+    assert concurrent_tasks > 0  # nosec: B101
+    running: set[Task[None]] = set()
+    try:
+        while element := await anext(source, None):
+            if len(running) < concurrent_tasks:
+                running.add(ctx.spawn(handler, element))
+                continue  # keep spawning tasks
+
+            completed, running = await wait(running, return_when=FIRST_COMPLETED)
+
+            for task in completed:
+                if exc := task.exception():
+                    if not ignore_exceptions:
+                        raise exc
+
+                    ctx.log_error(
+                        f"Concurrent processing error - {type(exc)}: {exc}",
+                        exception=exc,
+                    )
+
+    except CancelledError as exc:
+        # Cancel all running tasks
+        for task in running:
+            task.cancel()
+
+        raise exc
+
+    finally:
+        completed, _ = await wait(running, return_when=ALL_COMPLETED)
+        for task in completed:
+            if exc := task.exception():
+                if not ignore_exceptions:
+                    raise exc
+
+                ctx.log_error(
+                    f"Concurrent processing error - {type(exc)}: {exc}",
+                    exception=exc,
+                )

{haiway-0.19.4.dist-info → haiway-0.19.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: haiway
-Version: 0.19.4
+Version: 0.19.5
 Summary: Framework for dependency injection and state management within structured concurrency model.
 Project-URL: Homepage, https://miquido.com
 Project-URL: Repository, https://github.com/miquido/haiway.git

{haiway-0.19.4.dist-info → haiway-0.19.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-haiway/__init__.py,sha256=UjaJeNa5lQGaeOguf2COSakrBDc8sd5zB9ioHiK4uOw,2353
+haiway/__init__.py,sha256=bFRmunbEmBzJy5iH_2WoMzvwhApRk3eo37b23zBe6Bc,2407
 haiway/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 haiway/context/__init__.py,sha256=eoxqxUmFtkWLhc_gH6tqax9m90tSDie-jiP1BHruTdk,1102
 haiway/context/access.py,sha256=vTh5BlduAGBjqnVyTgTUt08DG6SrnAakweAfIDBqx-0,18838
@@ -8,9 +8,10 @@ haiway/context/observability.py,sha256=u9NPejZGyGupqx3eH9IQKbcVcCFpZrg2ZAiov2FQi
 haiway/context/state.py,sha256=0oq7ctNO0urJd7rVzwwNtgpguoXuI6Tp1exfCsxrS2M,5981
 haiway/context/tasks.py,sha256=QOxFdjmMp4IYff0ihHElKLCQrcVksSJmxqTlOKfoH4o,2907
 haiway/context/types.py,sha256=WulPvpqUbI1vYyny-s2NItldDnk3zh1O-n_hGibFZRY,142
-haiway/helpers/__init__.py,sha256=dYqwWSBk8ss9XyXEF6YHZNPrCwU8VDM4nKtcz1tsvZI,583
+haiway/helpers/__init__.py,sha256=_A-JYAPemd-2o4NxxchaZENu8gANL8QDxK5ADzInXUU,670
 haiway/helpers/asynchrony.py,sha256=k_A0yCWUKSFfzYZ8WvqK4wqTMljv6ykMivmERrDLHIU,6266
 haiway/helpers/caching.py,sha256=N60FBFfSoOdSUI6S49l47G6T1j-y9tc3qhU5msSRdPM,13221
+haiway/helpers/concurrent.py,sha256=04pWpS7qOQDD1pa2ifhFwedPcl4sRinhe-KgyDe5GF0,2296
 haiway/helpers/observability.py,sha256=hahA7jgGj5TH6mYBm3bKVwI22Fseuh-IX6X487gueaU,7977
 haiway/helpers/retries.py,sha256=unssUKBDOENvquh6R4Ud65TuSKl4mTHgZ5N_b7mAYa4,7533
 haiway/helpers/throttling.py,sha256=U6HJvSzffw47730VeiXxXSW4VVxpDx48k0oIAOpL-O4,4115
@@ -39,7 +40,7 @@ haiway/utils/mimic.py,sha256=L5AS4WEL2aPMZAQZlvLvRzHl0cipI7ivky60_eL4iwY,1822
 haiway/utils/noop.py,sha256=f54PSLHGEwCQNYXQHkPAW5NDE-tk5yjzkNL1pZj0TJQ,344
 haiway/utils/queue.py,sha256=YTvCn3wgSwLJiLqolMx44sa3304Xkv3tJG77gvfWnZs,4114
 haiway/utils/stream.py,sha256=Mjhy2S-ZDR1g_NsgS_nuBA8AgVbhrGXKvG3wjJ5mCJQ,2826
-haiway-0.19.4.dist-info/METADATA,sha256=viaL8Y3MD5WkZAsNCAUtVEOaH6enHIX-FPXtTstoIhc,4527
-haiway-0.19.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-haiway-0.19.4.dist-info/licenses/LICENSE,sha256=3phcpHVNBP8jsi77gOO0E7rgKeDeu99Pi7DSnK9YHoQ,1069
-haiway-0.19.4.dist-info/RECORD,,
+haiway-0.19.5.dist-info/METADATA,sha256=9zl0LfMAvGxzSzK7owry2SMQfSnitr3LUH6_UeqBqkI,4527
+haiway-0.19.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+haiway-0.19.5.dist-info/licenses/LICENSE,sha256=3phcpHVNBP8jsi77gOO0E7rgKeDeu99Pi7DSnK9YHoQ,1069
+haiway-0.19.5.dist-info/RECORD,,