typepipe 0.2.0__tar.gz

typepipe-0.2.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.3
+Name: typepipe
+Version: 0.2.0
+Summary: Create an asynchrounous data processing pipeline that keeps track of the types.
+Author: Daniel Tschertkow
+Author-email: Daniel Tschertkow <daniel.tschertkow@posteo.de>
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+

typepipe-0.2.0/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "typepipe"
+version = "0.2.0"
+description = "Create an asynchrounous data processing pipeline that keeps track of the types."
+readme = "README.md"
+authors = [
+    { name = "Daniel Tschertkow", email = "daniel.tschertkow@posteo.de" }
+]
+requires-python = ">=3.13"
+dependencies = []
+
+[build-system]
+requires = ["uv_build>=0.10.2,<0.11.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.3.0",
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

typepipe-0.2.0/src/typepipe/__init__.py

@@ -0,0 +1,2 @@
+def hello() -> str:
+    return "Hello from pypline!"

typepipe-0.2.0/src/typepipe/links.py

@@ -0,0 +1,250 @@
+import asyncio
+from asyncio import Queue
+from collections.abc import Awaitable, Iterable
+from typing import Callable, Tuple
+
+type Consumer[T] = Callable[[T], None]
+type Function[T, R] = Callable[[T], R]
+type Functor[T, R] = Callable[[T], Iterable[R]]
+type Producer[T] = Callable[[], T]
+type Observer[T] = Callable[[T], None]  # is supposed to not mutate the argument
+type Operator[T] = Callable[[T, T], T]
+type Predicate[T] = Callable[[T], bool]
+type IterableProducer[T] = Callable[[], Iterable[T]]
+
+
+def mapper[T, R](
+    map: Function[T, R], inqueue: Queue[T], /, qsize: int = 1
+) -> Tuple[Queue[R], Awaitable[None]]:
+    """A mapper is a task executed in the background (asynchronously) that
+    receives an input item from the inqueue, applies map to it and puts the
+    result on the outqueue. qsize controls the outquesize. It corresponds to
+    the maxsize parameter of asyncio.Queue, but defaults to 1.
+
+    Returns a tuple of (outqueue, worker). The outqueue contains the results
+    while the worker must be awaited to start processing.
+    """
+
+    outqueue = Queue[R](maxsize=qsize)
+
+    async def worker() -> None:
+        await asyncio.sleep(2)
+        try:
+            while True:
+                item = await inqueue.get()
+                inqueue.task_done()
+                await outqueue.put(map(item))
+        except asyncio.QueueShutDown:
+            outqueue.shutdown()
+    return outqueue, worker()
+
+
+def flatmapper[T, R](
+    map: Functor[T, R], inqueue: Queue[T], /, qsize: int = 1
+) -> Tuple[Queue[R], Awaitable[None]]:
+    """A flatmapper is a task executed in the background (asynchronously) that
+    receives an input item from the inqueue and applies map to it. The result is
+    expected to conform to collections.abc.Sequence where each element of it
+    gets put on the ouqueue individually. qsize controls the outquesize. It
+    corresponds to the maxsize parameter of asyncio.Queue, but defaults to 1.
+
+    Returns a tuple of (outqueue, worker). The outqueue contains the results
+    while the worker must be awaited to start processing.
+    """
+
+    outqueue = Queue[R](maxsize=qsize)
+
+    async def worker() -> None:
+        try:
+            while True:
+                item = await inqueue.get()
+                inqueue.task_done()
+                item_seq: Iterable[R] = map(item)
+                for i in item_seq:
+                    await outqueue.put(i)
+        except asyncio.QueueShutDown:
+            outqueue.shutdown()
+
+    return outqueue, worker()
+
+
+def producer[R](
+    produce: Producer[R], /, qsize: int = 1
+) -> Tuple[Queue[R], Awaitable[None]]:
+    """A producer is a task executed in the background (asynchronously) that
+    generates an input by calling generate() and puts the result on the
+    outqueue. qsize controls the outquesize. It corresponds to the maxsize
+    parameter of asyncio.Queue, but defaults to 1.
+
+    Returns a tuple of (outqueue, worker). The outqueue contains the results
+    while the worker must be awaited to start processing.
+    """
+    outqueue = Queue[R](maxsize=qsize)
+
+    async def worker() -> None:
+        item: R = produce()
+        await outqueue.put(item)
+        outqueue.shutdown()
+
+    return outqueue, worker()
+
+
+def flatproducer[R](
+    produce: IterableProducer[R], /, qsize: int = 1
+) -> Tuple[Queue[R], Awaitable[None]]:
+    """A flatproducer is a task executed in the background (asynchronously) that
+    generates input items by calling generate() and puts the results on the
+    outqueue individually. The result is expected to conform to
+    collections.abc.Sequence. qsize controls the outquesize. It corresponds to
+    the maxsize parameter of asyncio.Queue, but defaults to 1.
+
+    Returns a tuple of (outqueue, worker). The outqueue contains the results
+    while the worker must be awaited to start processing.
+    """
+    outqueue = Queue[R](maxsize=qsize)
+
+    async def worker() -> None:
+        item_seq: Iterable[R] = produce()
+        for item in item_seq:
+            await outqueue.put(item)
+        outqueue.shutdown()
+
+    return outqueue, worker()
+
+
+def iterator[T](
+    iterable: Iterable[T], /, qsize: int = 1
+) -> Tuple[Queue[T], Awaitable[None]]:
+    """A generator is a task executed in the background (asynchronously) that
+    generates an input by iterating through the generate generator and putting
+    the result on the outqueue. qsize controls the outquesize. It corresponds
+    to the maxsize parameter of asyncio.Queue, but defaults to 1.
+
+    Returns a tuple of (outqueue, worker). The outqueue contains the results
+    while the worker must be awaited to start processing.
+    """
+    outqueue = Queue[T](maxsize=qsize)
+
+    async def worker() -> None:
+        item: T
+        for item in iterable:
+            await outqueue.put(item)
+        outqueue.shutdown()
+
+    return outqueue, worker()
+
+
+def reducer[T](
+    reduce: Operator[T], inqueue: Queue[T], /, initial_value: T | None = None
+) -> Awaitable[T | None]:
+    """A reducer is a task executed in the background (asynchronously) that
+    generates an input by calling generate() and puts the result on the
+    outqueue. qsize controls the outquesize. It corresponds to the maxsize
+    parameter of asyncio.Queue, but defaults to 1.
+
+    Returns the result of the reduction as an Awaitable.
+    """
+
+    async def worker() -> T | None:
+        if initial_value is None:
+            try:
+                first: T = await inqueue.get()
+                inqueue.task_done()
+            except asyncio.QueueShutDown:
+                return None
+        else:
+            first = initial_value
+
+        try:
+            second: T = await inqueue.get()
+            inqueue.task_done()
+        except asyncio.QueueShutDown:
+            return first
+
+        result: T = reduce(first, second)
+        try:
+            while True:
+                next = await inqueue.get()
+                inqueue.task_done()
+                result = reduce(result, next)
+        except asyncio.QueueShutDown:
+            return result
+
+    return worker()
+
+
+def filterer[T](
+    predicate: Predicate[T], inqueue: Queue[T], /, qsize: int = 1
+) -> Tuple[Queue[T], Awaitable[None]]:
+    """A filterer is a task executed in the background (asynchronously) that
+    filters items of the inqueue and puts the items for which the predicate is
+    true on the outqueue. qsize controls the outqueue size. It corresponds to
+    the maxsize parameter of asyncio.Queue, but defaults to 1.
+
+    Returns a tuple of (outqueue, worker). The outqueue has the remaining items
+    and the worker is an Awaitable that starts the filtering process.
+    """
+
+    outqueue = Queue[T](maxsize=qsize)
+
+    async def worker() -> None:
+        try:
+            while True:
+                item = await inqueue.get()
+                inqueue.task_done()
+                if predicate(item):
+                    await outqueue.put(item)
+        except asyncio.QueueShutDown:
+            outqueue.shutdown()
+
+    return outqueue, worker()
+
+
+def consumer[T](
+    consume: Consumer[T],
+    inqueue: Queue[T],
+    /,
+) -> Awaitable[None]:
+    """A consumer is a task executed in the background (asynchronously) that
+    consumes items from the inqueue (for side effects).
+
+    Returns a worker. The worker is an Awaitable that starts the filtering
+    process.
+    """
+
+    async def worker() -> None:
+        try:
+            while True:
+                item = await inqueue.get()
+                inqueue.task_done()
+                consume(item)
+        except asyncio.QueueShutDown:
+                return
+
+    return worker()
+
+
+def observer[T](
+    observe: Observer[T], inqueue: Queue[T], /, qsize: int = 1
+) -> Tuple[Queue[T], Awaitable[None]]:
+    """An observer is a task executed in the background (asynchronously) that
+    calls observer on the items from the inqueue (for side effects). The item
+    should not be modified by a call to observe. The item is put back on the
+    outqueue with the assumption that it was not modified.
+
+    Returns a tuple of (outqueue, worker). The outqueue has the remaining items
+    and the worker is an Awaitable that starts the filtering process.
+    """
+    outqueue = Queue[T](maxsize=qsize)
+
+    async def worker() -> None:
+        try:
+            while True:
+                item = await inqueue.get()
+                inqueue.task_done()
+                observe(item)
+                await outqueue.put(item)
+        except asyncio.QueueShutDown:
+            outqueue.shutdown()
+
+    return outqueue, worker()

typepipe-0.2.0/src/typepipe/pipeline.py

@@ -0,0 +1,356 @@
+"""Defines the Pipeline class that builds an asynchronous processing pipeline in
+a functional style.
+"""
+
+import asyncio
+from asyncio import Queue
+from collections.abc import Awaitable, Iterable, MutableSequence
+from typing import List, Self, Tuple
+
+from . import links
+
+
+class PipelineClosedException(Exception):
+    """PipelineClosedException is raised when a pipeline operation is added after
+    the pipeline was closed. A pipeline is closed when a closing operation like
+    reduce or consume has been performed."""
+
+
+class Pipeline[T]:
+    """Asynchronous processing pipeline.
+
+    `Pipeline` orchestrates a series of async stages chained together via queues
+    of type `asyncio.Queue`. Users create a pipeline from an existing queue, a
+    producer function, a sequence, or a generator and then chain transformations
+    via `map`, `flatmap`, `filter`, and finally terminate the pipeline with
+    `reduce`, `consume`, or `collect`.
+
+    The pipeline tracks internal worker coroutines in and ensures they are
+    awaited together. Once a terminating operation is invoked, the pipeline is
+    marked as *closed* and further stage additions raise
+    `PipelineClosedException`.
+
+    Every step that exposes a downstream queue has a `qsize` keyword parameter
+    that sets the size of the queue and defaults to `1` (since most queues are
+    either always empty or always full). Use queues of bigger sizes if your
+    operation is bursty.
+    """
+
+    queue: Queue[T]
+    workers: List[Awaitable[None]]
+    closed: bool
+
+    def __init__(self, queue: Queue[T], workers: List[Awaitable[None]]):
+        """Create a new `Pipeline` from an existing `asyncio.Queue`. Make sure
+        the queue is fed and properly shut down, otherwise the pipeline will not
+        terminate but wait for further input.
+
+        Args:
+            queue: The input queue that provides items to the pipeline.
+
+        The pipeline starts in an *open* state; stages can be added until a
+        terminal operation (reduce/consume/collect) closes it.
+        """
+        self.closed = False
+        self.workers = workers
+        self.queue = queue
+
+    @classmethod
+    def from_queue(cls, inqueue: Queue[T]) -> Self:
+        """Construct a `Pipeline` directly from an existing queue.
+
+        Args:
+            inqueue: The source `asyncio.Queue`.
+
+        Returns:
+            A new `Pipeline` instance.
+        """
+        pipeline = cls(inqueue, list())
+        return pipeline
+
+    @classmethod
+    def from_producer(cls, func: links.Producer[T], /, qsize: int = 1) -> Self:
+        """Create a pipeline from a producer function.
+
+        The producer closure supplies the initial, single item for the pipeline.
+
+        Args:
+            func: A zero-argument callable that returns a single item.
+            qsize: Maximum size of the internal queue (default `1`).
+
+        Returns:
+            A new `Pipeline` instance.
+        """
+        queue, worker = links.producer(func, qsize=qsize)
+        pipeline = cls(queue, list())
+        pipeline.workers.append(worker)
+        return pipeline
+
+    @classmethod
+    def from_producer_flatten(
+        cls, func: links.IterableProducer[T], /, qsize: int = 1
+    ) -> Self:
+        """Create a pipeline from a producer function.
+
+        The producer closure supplies the initial sequence for the pipeline
+        where each item is put into the pipeline individually.
+
+        Args:
+            func: A zero-argument callable that returns a sequence.
+            qsize: Maximum size of the internal queue (default `1`).
+
+        Returns:
+            A new `Pipeline` instance.
+
+        """
+        queue, worker = links.flatproducer(func, qsize=qsize)
+        pipeline = cls(queue, list())
+        pipeline.workers.append(worker)
+        return pipeline
+
+    @classmethod
+    def from_iterable(cls, seq: Iterable[T], /, qsize: int = 1):
+        """Create a pipeline from an existing in-memory iterable.
+
+        Each item from the iterable is put into the pipeline individually.
+
+        Args:
+            seq: Any `collections.abc.Iterable` of items.
+            qsize: Queue size (default `1`).
+
+        Returns:
+            A new `Pipeline` instance.
+        """
+        queue, worker = links.iterator(seq, qsize=qsize)
+        pipeline = cls(queue, list())
+        pipeline.workers.append(worker)
+        return pipeline
+
+    def map[U](self, func: links.Function[T, U], /, qsize: int = 1) -> "Pipeline[U]":
+        """Add a mapping stage to the pipeline.
+
+        Each item from the previous stage is transformed by `func` and the
+        result is the input of the next stage in the pipeline.
+
+        Args:
+            func: Callable that maps an input item to an output item.
+            qsize: Queue size for the downstream queue (default `1`).
+
+        Returns:
+            The pipeline instance to allow method chaining.
+
+        Raises:
+            PipelineClosedException: If `map` is added to a pipeline that already
+            has a closing stage.
+
+        """
+
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        queue, worker = links.mapper(func, self.queue, qsize=qsize)
+
+        self.workers.append(worker)
+        return Pipeline[U](queue, self.workers)
+
+    def flatmap[U](self, func: links.Functor[T, U], /, qsize: int = 1) -> "Pipeline[U]":
+        """Add a mapping stage to the pipeline.
+
+        `func` is expected to return an iterable for each input item. Each
+        element of that iterable is put individually to the downstream
+        queue.
+
+        Args:
+            func: Callable that maps an input item to a sequence of output items.
+            qsize: Queue size for the downstream queue (default `1`).
+
+        Returns:
+            The pipeline instance for method chaining.
+
+        Raises:
+            PipelineClosedException: If `flatmap` is added to a pipeline that
+            already has a closing stage.
+
+        """
+
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        queue, worker = links.flatmapper(func, self.queue, qsize=qsize)
+
+        self.workers.append(worker)
+        return Pipeline[U](queue, self.workers)
+
+    def filter(self, pred: links.Predicate[T], /, qsize: int = 1) -> Self:
+        """Add a filter stage to the pipeline.
+
+        Only items for which `pred` returns `True` are forwarded downstream.
+
+        Args:
+            pred: Predicate callable returning `True` for items to keep.
+            qsize: Queue size for the downstream queue (default `1`).
+
+        Returns:
+            The pipeline instance for chaining.
+
+        Raises:
+            PipelineClosedException: If `filter` is added to a pipeline that
+            already has a closing stage.
+        """
+
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        queue, worker = links.filterer(pred, self.queue, qsize=qsize)
+
+        self.workers.append(worker)
+        self.queue = queue
+        return self
+
+    async def reduce(
+        self, reduce: links.Operator[T], /, initial_value: T | None = None
+    ) -> T | None:
+        """Reduce the pipeline to a single accumulated value.
+
+        Consumes items from the pipeline using `reduce` as a binary operator.
+        After reduction the pipeline is marked as closed.
+
+        `reduce` is a closing operation!
+
+        Args:
+            reduce: Binary function combining two items into one.
+            initial_value: Optional initial accumulator. If omitted the first
+                item from the queue is used as the start value.
+
+        Returns:
+            The final reduced value, or `None` if the queue was empty.
+
+        Raises:
+            PipelineClosedException: If `reduce` is added to a pipeline that
+            already has a closing stage.
+        """
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        self.closed = True
+
+        rworker: Awaitable[T | None] = links.reducer(
+            reduce, self.queue, initial_value=initial_value
+        )
+
+        results: List = await asyncio.gather(*self.workers, rworker)
+        return results[-1]  # rworker holds the awaitable result of the
+                            # reduction and comes last
+
+    async def consume(
+        self,
+        consume: links.Consumer[T],
+        /,
+    ) -> None:
+        """Consume items from the pipeline for side effects.
+
+        The `consume` callable is applied to each item. This is a terminal
+        operation; after calling it the pipeline is closed.
+
+        `consume` is a closing operation!
+
+        Args:
+            consume: Callable that processes each item (e.g., printing or
+                storing).
+
+        Raises:
+            PipelineClosedException: If `consume` is added to a pipeline that
+            already has a closing stage.
+        """
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        self.closed = True
+        cworker: Awaitable[None] = links.consumer(consume, self.queue)
+
+        self.workers.append(cworker)
+        await asyncio.gather(*self.workers)
+
+    def isClosed(self) -> bool:
+        """Return `True` if a closing operation has closed the pipeline."""
+        return self.closed
+
+    def expose(self) -> Tuple[Queue[T], Awaitable[None]]:
+        """Expose the internal queue and a combined worker.
+
+        Allows external code to await the combined workers while receiving the
+        output queue directly.
+
+        `expose` is a closing operation!
+
+        Raises:
+            PipelineClosedException: If `expose` is added to a pipeline that
+            already has a closing stage.
+        """
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        self.closed = True
+
+        # bundle workers into one
+        async def worker() -> None:
+            await asyncio.gather(*self.workers)
+
+        return self.queue, worker()
+
+    async def collect(self, container: MutableSequence[T]) -> MutableSequence[T]:
+        """Collect all items from the pipeline into `container`.
+
+        The pipeline runs to completion, appending each yielded item to the
+        provided mutable sequence. After collection the pipeline is closed.
+
+        `collect` is a closing operation!
+
+        Args:
+            container: A mutable sequence (e.g., list) to which items are added.
+
+        Raises:
+            PipelineClosedException: If `collect` is added to a pipeline that
+            already has a closing stage.
+        """
+
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+        self.closed = True
+
+        async def collector() -> MutableSequence[T]:
+            try:
+                while True:
+                    item: T = await self.queue.get()
+                    self.queue.task_done()
+                    container.append(item)
+            except asyncio.QueueShutDown:
+                return container
+
+        *worker_results, seq = await asyncio.gather(*self.workers, collector())
+        return seq  # seq can't be None
+
+
+    def inspect(self, observe: links.Observer[T], /, qsize: int = 1) -> Self:
+        """Insert an observer stage that runs `observe` on each item.
+
+        Useful for side-effects like logging without modifying the
+        stream. `observe` is expected to not modify its item.
+
+        Args:
+            observe: Callable invoked with each item.
+            qsize: Queue size for the observer's downstream queue.
+
+        Raises:
+            PipelineClosedException: If `inspect` is added to a pipeline that
+            already has a closing stage.
+        """
+        if self.closed:
+            raise PipelineClosedException("The pipeline is already closed")
+
+        outqueue, worker = links.observer(observe, self.queue, qsize=qsize)
+
+        self.workers.append(worker)
+        self.queue = outqueue
+
+        return self