pyfect-dev 0.1.0__py3-none-any.whl

pyfect/channel/_merge.py

@@ -0,0 +1,246 @@
+"""Channels that run other channels on fibers: ``merge_all`` (behind
+concurrent ``flat_map`` and ``switch_map``), ``merge`` and its
+``merge_effect`` / ``interrupt_when`` forms, ``halt_when``, ``buffer``
+and ``embed_input``.
+
+Every fiber lives in the channel's forked scope, closed as soon as the
+channel fails or ends, so a merged channel never outlives its reader.
+Items reach the reader through a queue that shuts down with that scope.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Literal
+
+from pyfect._concurrency import Concurrency, race_first
+from pyfect._core import Effect, _FailCause, _Succeed, _Suspend
+from pyfect._fiber import Fiber
+from pyfect.cause import Cause
+from pyfect.channel._base import AnyChannel, AnyPull, end_with
+from pyfect.channel._channel import Channel, from_pull, from_transform
+from pyfect.channel._combinators import permits_of
+from pyfect.channel._constructors import bracket, from_effect_drain
+from pyfect.effect import from_exit, never, yield_now
+from pyfect.exit import Exit, Failure, as_void_all
+from pyfect.exit import interrupt as interrupted
+from pyfect.latch import Latch
+from pyfect.pull import done_exit_from_cause, filter_done, is_done_cause
+from pyfect.queue import Queue, Strategy
+from pyfect.result import Ok
+from pyfect.scope import CloseableScope, fork_in
+from pyfect.semaphore import Semaphore
+
+type Channel_ = Channel[Any, Any, Any, Any, Any, Any, Any]
+type HaltStrategy = Literal["left", "right", "both", "either"]
+type Capacity = int | Literal["unbounded"]
+
+_VOID: Effect[None] = _Succeed(None)
+
+
+def merge_all(
+    channels: AnyChannel, concurrency: Concurrency, buffer_size: int, *, switch: bool
+) -> Channel_:
+    n = permits_of(concurrency)
+
+    def build(
+        upstream: AnyPull, scope: CloseableScope, forked: CloseableScope
+    ) -> Effect[AnyPull, Any, Any]:
+        permits = None if switch else Semaphore(n)
+        done_latch = Latch(open=True)
+        fibers: dict[Fiber[Any, Any], CloseableScope] = {}
+        queue: Queue[Any, Any] = Queue(buffer_size)
+
+        def next_channel(pull: AnyPull) -> Effect[Any, Any, Any]:
+            if permits is None:
+                return pull
+            take = permits.take(1)
+            if len(fibers) < n:
+                return take.and_then(pull)
+
+            def raced(fiber: Fiber[Any, Any]) -> Effect[Any, Any, Any]:
+                return race_first(take, fiber.join().and_then(never)).and_then(fiber.join())
+
+            return pull.fork().flat_map(raced)
+
+        def evict() -> Effect[None]:
+            if len(fibers) < n:
+                return _VOID
+            fiber, fiber_scope = next(iter(fibers.items()))
+            del fibers[fiber]
+            if not fibers:
+                done_latch.open_unsafe()
+            # A worker interrupted before it starts never closes its own
+            # scope; failures of replaced workers' finalizers are dropped.
+            closed = fiber_scope.close(interrupted()).ignore_cause()
+            return fiber.interrupt().and_then(closed).and_then(_Suspend(evict))
+
+        def start(channel: AnyChannel) -> Effect[None, Any, Any]:
+            child_scope = forked.fork_unsafe()
+            fiber_box: list[Fiber[Any, Any]] = []
+
+            def finished(cause: Any) -> Effect[None]:
+                ex = done_exit_from_cause(cause)
+
+                def after_close(close_exit: Exit[Any, Any]) -> Effect[None]:
+                    fiber = fiber_box[0]
+                    if fiber not in fibers:
+                        return _VOID
+                    failure = as_void_all([ex, close_exit])
+                    report = (
+                        queue.fail_cause(failure.cause).as_void()
+                        if isinstance(failure, Failure)
+                        else _VOID
+                    )
+
+                    def release() -> Effect[None]:
+                        del fibers[fiber]
+                        freed = _VOID if permits is None else permits.release(1).as_void()
+                        if not fibers:
+                            done_latch.open_unsafe()
+                        return freed
+
+                    return report.and_then(_Suspend(release))
+
+                return child_scope.close(ex).exit().flat_map(after_close)
+
+            def run(child_pull: AnyPull) -> Effect[None, Any, Any]:
+                worker = (
+                    child_pull.tap(lambda _: yield_now())
+                    .flat_map(queue.offer)
+                    .forever(disable_yield=True)
+                    .on_error(finished)
+                )
+
+                def started(fiber: Fiber[Any, Any]) -> None:
+                    fiber_box.append(fiber)
+                    done_latch.close_unsafe()
+                    fibers[fiber] = child_scope
+
+                return evict().and_then(worker.fork()).map(started)
+
+            return channel._transform(upstream, child_scope).flat_map(run)
+
+        def stopped(cause: Any) -> Effect[None]:
+            if isinstance(filter_done(cause), Ok):
+                return done_latch.when_open(queue.fail_cause(cause)).as_void()
+            return queue.fail_cause(cause).as_void()
+
+        def with_pull(pull: AnyPull) -> Effect[AnyPull, Any, Any]:
+            loop = _Suspend(lambda: next_channel(pull)).flat_map(start).forever(disable_yield=True)
+            return fork_in(loop.catch_cause(stopped), forked).as_(queue.take())
+
+        return (
+            forked.add_finalizer(lambda _: queue.shutdown())
+            .and_then(channels._transform(upstream, scope))
+            .flat_map(with_pull)
+        )
+
+    return bracket(build)
+
+
+def merge(left: AnyChannel, right: AnyChannel, halt_strategy: HaltStrategy) -> Channel_:
+    def build(
+        upstream: AnyPull, _scope: CloseableScope, forked: CloseableScope
+    ) -> Effect[AnyPull, Any, Any]:
+        queue: Queue[Any, Any] = Queue(0)
+        done = 0
+
+        def on_exit(side: str, cause: Cause[Any]) -> Effect[None]:
+            nonlocal done
+            done += 1
+            if not is_done_cause(cause):
+                return queue.fail_cause(cause).as_void()
+            halts = done == 2 if halt_strategy == "both" else halt_strategy in ("either", side)
+            return queue.fail_cause(cause).as_void() if halts else _VOID
+
+        def run_side(side: str, channel: AnyChannel) -> Effect[Any]:
+            side_scope = forked.fork_unsafe()
+
+            def stopped(cause: Any) -> Effect[None]:
+                # A failure beside the side's Done (a finalizer's defect)
+                # fails the merge, with any failure closing the scope adds.
+                ex = done_exit_from_cause(cause)
+                closed = from_exit(ex).on_exit(lambda _: side_scope.close(ex))
+                return closed.match_cause_effect(
+                    on_failure=lambda close_cause: on_exit(side, close_cause),
+                    on_success=lambda _: on_exit(side, cause),
+                )
+
+            run = (
+                channel._transform(upstream, side_scope)
+                .flat_map(lambda pull: pull.flat_map(queue.offer).forever())
+                .on_error(stopped)
+            )
+            return fork_in(run, forked)
+
+        return (
+            forked.add_finalizer(lambda _: queue.shutdown())
+            .and_then(run_side("left", left))
+            .and_then(run_side("right", right))
+            .as_(queue.take())
+        )
+
+    return bracket(build)
+
+
+def merge_effect(self: AnyChannel, eff: Effect[Any, Any, Any]) -> Channel_:
+    return merge(self, from_effect_drain(eff), "left")
+
+
+def interrupt_when(self: AnyChannel, eff: Effect[Any, Any, Any]) -> Channel_:
+    return merge(self, from_pull(_Succeed(eff.flat_map(end_with))), "either")
+
+
+def halt_when(self: AnyChannel, eff: Effect[Any, Any, Any]) -> Channel_:
+    def build(
+        upstream: AnyPull, scope: CloseableScope, forked: CloseableScope
+    ) -> Effect[AnyPull, Any, Any]:
+        def with_pull(pull: AnyPull) -> Effect[AnyPull]:
+            def watch(fiber: Fiber[Any, Any]) -> AnyPull:
+                def next_pull() -> AnyPull:
+                    ex = fiber.poll_unsafe()
+                    if ex is None:
+                        return pull
+                    return _FailCause(ex.cause) if isinstance(ex, Failure) else end_with(ex.value)
+
+                return _Suspend(next_pull)
+
+            return fork_in(eff, forked, start_immediately=True).map(watch)
+
+        return self._transform(upstream, scope).flat_map(with_pull)
+
+    return bracket(build)
+
+
+def _make_queue(capacity: Capacity, strategy: Strategy) -> Queue[Any, Any]:
+    return Queue(None) if capacity == "unbounded" else Queue(capacity, strategy)
+
+
+def buffer(self: AnyChannel, capacity: Capacity, strategy: Strategy, *, array: bool) -> Channel_:
+    def transform(upstream: AnyPull, scope: CloseableScope) -> Effect[AnyPull, Any, Any]:
+        queue = _make_queue(capacity, strategy)
+        offer: Callable[[Any], Effect[Any]] = queue.offer_all if array else queue.offer
+
+        def with_pull(pull: AnyPull) -> Effect[AnyPull]:
+            run = pull.flat_map(offer).forever(disable_yield=True).on_error(queue.fail_cause)
+            take = queue.take_all().map(tuple) if array else queue.take()
+            return fork_in(run, scope).as_(take)
+
+        return (
+            scope.add_finalizer(lambda _: queue.shutdown())
+            .and_then(self._transform(upstream, scope))
+            .flat_map(with_pull)
+        )
+
+    return from_transform(transform)
+
+
+def embed_input(self: AnyChannel, input: Callable[[AnyPull], Effect[Any, Any, Any]]) -> Channel_:
+    def build(
+        upstream: AnyPull, scope: CloseableScope, forked: CloseableScope
+    ) -> Effect[AnyPull, Any, Any]:
+        return fork_in(input(upstream), forked).and_then(self._transform(end_with(None), scope))
+
+    return bracket(build)

pyfect/channel/_repeat.py

@@ -0,0 +1,154 @@
+"""Running a channel again: ``repeat``, ``forever``, ``retry``, and pacing
+items with ``schedule``.
+
+Each run of the repeated channel builds its pull in a child of the
+channel's scope, closed with the run's exit before the next run starts,
+so resources do not pile up over repetitions.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from pyfect._core import Effect, _FailCause, _Locally, _Succeed, _Suspend
+from pyfect.cause import Fail
+from pyfect.cause import fail as fail_cause
+from pyfect.channel._base import AnyChannel, AnyPull, end_with
+from pyfect.channel._channel import Channel, from_transform, transform_pull
+from pyfect.channel._constructors import unwrap_effect
+from pyfect.channel._context import provide_service_effect
+from pyfect.channel._lifecycle import on_first
+from pyfect.exit import Exit, Failure
+from pyfect.pull import done_exit_from_cause
+from pyfect.schedule import Schedule
+from pyfect.schedule._drive import resolve_policy
+from pyfect.schedule._model import CurrentMetadata, Metadata, Stop, to_step_with_metadata
+from pyfect.scope import CloseableScope, Scope
+
+type Channel_ = Channel[Any, Any, Any, Any, Any, Any, Any]
+type OnExit = Callable[[Exit[Any, Any], CloseableScope], Effect[Any, Any, Any]]
+type Policy = Schedule[Any, Any, Any, Any] | Callable[[Any], Any]
+
+_VOID: Effect[None] = _Succeed(None)
+
+
+def repeat_loop(self: AnyChannel, on_exit: OnExit) -> Channel_:
+    """Run ``self`` again each time ``on_exit(exit, scope)`` succeeds after
+    a run; its failure (or end) is the channel's."""
+
+    def transform(upstream: AnyPull, scope: CloseableScope) -> Effect[AnyPull]:
+        current: AnyPull | None = None
+
+        def make_pull() -> AnyPull:
+            run_scope = scope.fork_unsafe()
+
+            def recover(cause: Any) -> AnyPull:
+                ex = done_exit_from_cause(cause)
+                return (
+                    run_scope.close(ex).and_then(on_exit(ex, scope)).and_then(_Suspend(make_pull))
+                )
+
+            def started(pull: AnyPull) -> AnyPull:
+                nonlocal current
+                current = pull.catch_cause(recover)
+                return current
+
+            return self._transform(upstream, run_scope).flat_map(started)
+
+        return _Succeed(_Suspend(lambda: make_pull() if current is None else current))
+
+    return from_transform(transform)
+
+
+def _with_metadata(self: AnyChannel, read: Callable[[], Metadata[Any, Any]]) -> AnyChannel:
+    return provide_service_effect(CurrentMetadata, _Suspend(lambda: _Succeed(read())))(self)
+
+
+def repeat(self: AnyChannel, policy: Policy) -> Channel_:
+    schedule = resolve_policy(policy, None, None, None, None)
+
+    def start(step: Callable[[Any], Effect[Any, Any, Any]]) -> AnyChannel:
+        meta: Metadata[Any, Any] = CurrentMetadata.default()
+
+        def stepped(ex: Exit[Any, Any], _: CloseableScope) -> Effect[Any, Any, Any]:
+            if isinstance(ex, Failure):
+                return _FailCause(ex.cause)
+
+            def keep(decision: Metadata[Any, Any] | Stop[Any]) -> Effect[None, Any, Any]:
+                nonlocal meta
+                if isinstance(decision, Stop):
+                    return end_with(ex.value)
+                meta = decision
+                return _VOID
+
+            return step(ex.value).flat_map(keep)
+
+        return repeat_loop(_with_metadata(self, lambda: meta), stepped)
+
+    return unwrap_effect(to_step_with_metadata(schedule).map(start))
+
+
+def forever(self: AnyChannel) -> Channel_:
+    def again(ex: Exit[Any, Any], _: CloseableScope) -> Effect[None, Any, Any]:
+        return _FailCause(ex.cause) if isinstance(ex, Failure) else _VOID
+
+    return repeat_loop(self, again)
+
+
+def retry(self: AnyChannel, policy: Policy) -> Channel_:
+    schedule = resolve_policy(policy, None, None, None, None)
+
+    def build() -> AnyChannel:
+        step: Callable[[Any], Effect[Any, Any, Any]] | None = None
+        meta: Metadata[Any, Any] = CurrentMetadata.default()
+
+        def reset(_: Any) -> Effect[None]:
+            nonlocal step
+            step = None
+            return _VOID
+
+        def stepped(ex: Exit[Any, Any], scope: CloseableScope) -> Effect[Any, Any, Any]:
+            if not isinstance(ex, Failure):
+                return end_with(ex.value)
+            found = ex.cause.find_recoverable_error()
+            if not isinstance(found, Fail):
+                return _FailCause(found)
+            error = found.error
+
+            def keep(decision: Metadata[Any, Any] | Stop[Any]) -> Effect[None, Any, Any]:
+                nonlocal meta
+                if isinstance(decision, Stop):
+                    return _FailCause(fail_cause(error))
+                meta = decision
+                return _VOID
+
+            def with_step(made: Callable[[Any], Effect[Any, Any, Any]]) -> Effect[Any, Any, Any]:
+                nonlocal step
+                step = made
+                return made(error).flat_map(keep)
+
+            make = _Succeed(step) if step is not None else to_step_with_metadata(schedule)
+            return _Locally(make.flat_map(with_step), lambda ctx: ctx.add(Scope, scope))
+
+        with_reset = on_first(_with_metadata(self, lambda: meta), reset)
+        return repeat_loop(with_reset, stepped)
+
+    return from_transform(lambda upstream, scope: build()._transform(upstream, scope))
+
+
+def schedule(self: AnyChannel, policy: Schedule[Any, Any, Any, Any]) -> Channel_:
+    def with_step(pull: AnyPull, step: Callable[[Any], Effect[Any, Any, Any]]) -> AnyPull:
+        def paced(item: Any) -> Effect[Any, Any, Any]:
+            def decided(decision: Metadata[Any, Any] | Stop[Any]) -> Effect[Any, Any, Any]:
+                return end_with(decision.output) if isinstance(decision, Stop) else _Succeed(item)
+
+            return step(item).flat_map(decided)
+
+        return pull.flat_map(paced)
+
+    return transform_pull(
+        self,
+        lambda pull, _: to_step_with_metadata(policy).map(lambda step: with_step(pull, step)),
+    )

pyfect/channel/_run.py

@@ -0,0 +1,149 @@
+"""Running a channel: the ``run_*`` methods, ``to_pull`` and
+``to_pull_scoped``.
+
+A run builds the channel's pull in a fresh scope, with an upstream that
+ends at once, pulls in a ``PullLoop`` until ``Done``, and closes the scope
+with the run's exit.
+"""
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from pyfect._context_effects import provide_context
+from pyfect._core import Effect, _FailCause, _Succeed, _Suspend, _Sync, _WithFiber
+from pyfect._fiber import Fiber
+from pyfect.cause import fail
+from pyfect.channel._base import AnyChannel, AnyPull
+from pyfect.channel._loop import PullLoop
+from pyfect.errors import Done
+from pyfect.option import Option, Some
+from pyfect.pull import catch_done
+from pyfect.scope import CloseableScope, Scope
+from pyfect.semaphore import Semaphore
+
+type AnyEffect = Effect[Any, Any, Any]
+
+ENDED: AnyPull = _FailCause(fail(Done()))
+"""A pull that ends at once with ``Done()``: the upstream of a channel run
+on its own."""
+
+_VOID: Effect[None] = _Succeed(None)
+
+
+def _ignore(_: object) -> None:
+    return None
+
+
+def _run_with(
+    self: AnyChannel,
+    consume: Callable[[AnyPull], AnyEffect],
+    on_done: Callable[[Any], AnyEffect] = _Succeed,
+) -> AnyEffect:
+    def run() -> AnyEffect:
+        scope = CloseableScope()
+        consumed = self._transform(ENDED, scope).flat_map(consume)
+        return catch_done(consumed, on_done).on_exit(scope.close)
+
+    return _Suspend(run)
+
+
+def run_collect(self: AnyChannel) -> Effect[list[Any], Any, Any]:
+    def run() -> AnyEffect:
+        items: list[Any] = []
+        return _run_with(self, lambda pull: PullLoop(pull, items.append), lambda _: _Succeed(items))
+
+    return _Suspend(run)
+
+
+def run_drain(self: AnyChannel) -> AnyEffect:
+    return _run_with(self, lambda pull: PullLoop(pull, _ignore))
+
+
+def run_for_each(self: AnyChannel, f: Callable[[Any], AnyEffect]) -> AnyEffect:
+    return _run_with(self, lambda pull: PullLoop(pull.flat_map(f), _ignore))
+
+
+def run_for_each_while(self: AnyChannel, f: Callable[[Any], Effect[bool, Any, Any]]) -> AnyEffect:
+    def go_on(more: bool) -> AnyEffect:
+        return _VOID if more else ENDED
+
+    return _run_with(self, lambda pull: PullLoop(pull.flat_map(f).flat_map(go_on), _ignore))
+
+
+def run_fold(
+    self: AnyChannel, initial: Callable[[], Any], f: Callable[[Any, Any], Any]
+) -> AnyEffect:
+    def run() -> AnyEffect:
+        state = initial()
+
+        def step(item: Any) -> None:
+            nonlocal state
+            state = f(state, item)
+
+        return _run_with(self, lambda pull: PullLoop(pull, step), lambda _: _Sync(lambda: state))
+
+    return _Suspend(run)
+
+
+def run_fold_effect(
+    self: AnyChannel, initial: Callable[[], Any], f: Callable[[Any, Any], AnyEffect]
+) -> AnyEffect:
+    def run() -> AnyEffect:
+        state = initial()
+
+        def step(item: Any) -> None:
+            nonlocal state
+            state = item
+
+        return _run_with(
+            self,
+            lambda pull: PullLoop(pull.flat_map(lambda item: f(state, item)), step),
+            lambda _: _Sync(lambda: state),
+        )
+
+    return _Suspend(run)
+
+
+def run_head(self: AnyChannel) -> Effect[Option[Any], Any, Any]:
+    def run() -> AnyEffect:
+        head: Option[Any] = None
+
+        def keep(item: Any) -> AnyEffect:
+            nonlocal head
+            head = Some(item)
+            return ENDED
+
+        return _run_with(self, lambda pull: pull.flat_map(keep), lambda _: _Sync(lambda: head))
+
+    return _Suspend(run)
+
+
+def run_last(self: AnyChannel) -> Effect[Option[Any], Any, Any]:
+    def run() -> AnyEffect:
+        last: Option[Any] = None
+
+        def keep(item: Any) -> None:
+            nonlocal last
+            last = Some(item)
+
+        return _run_with(self, lambda pull: PullLoop(pull, keep), lambda _: _Sync(lambda: last))
+
+    return _Suspend(run)
+
+
+def to_pull(self: AnyChannel) -> AnyEffect:
+    def build(fiber: Fiber[Any, Any]) -> AnyEffect:
+        context = fiber._context
+        turns = Semaphore(1)
+        return self._transform(ENDED, context.get_unsafe(Scope)).map(
+            lambda pull: turns.with_permits(pull.pipe(provide_context(context)))
+        )
+
+    return _WithFiber(build)
+
+
+def to_pull_scoped(self: AnyChannel, scope: CloseableScope) -> AnyEffect:
+    return _Suspend(lambda: self._transform(ENDED, scope))