kpipeline 0.1.1__tar.gz

kpipeline-0.1.1/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: kpipeline
+Version: 0.1.1
+Summary: A pipeline library for Python
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# kpipe
+
+`kpipe` is a simple pipeline library that allows you to write your complex application as a pipeline. This helps code organization, debugging, and testing.
+
+## How to use
+
+kpipe is like a DSL language inside Python. It is based on "pipes" which can be combined into pipelines using the pipe primitives provided by the library.
+
+Each pipe represents a function transforming an input into an output. Pipes are stateless and immutable. Below is an example defining two example pipes.
+
+```python
+from kpipeline import Pipe
+
+class AddOnePipe(Pipe[int, int, None]):
+    def apply(self, input: int, metadata: None) -> int:
+        return input + 1
+
+
+class MulByTwoPipe(Pipe[int, int, None]):
+    def apply(self, input: int, metadata: None) -> int:
+        return input * 2
+```
+
+These pipes can be combined using the ChainPipe primitive (aliased into the `|` operator) two form a pipeline that performs these two pipes sequentially:
+
+```python
+pipeline = AddOnePipe() | MulByTwoPipe()
+
+# or
+
+from kpipeline import ChainPipe
+pipeline = ChainPipe(AddOnePipe(), MulByTwoPipe())
+
+print(pipeline.apply(2))  # 6
+```
+
+Currently, the library defines these primitives:
+
+| Primitive | Purpose |
+|---|---|
+|`ChainPipe`|Execute two pipes sequentially|
+|`ConditionalPipe`|Execute a pipe if a given condition is true|
+|`BranchPipe`|Execute one of two pipes depending on whether the condition is true or false|
+|`SelectPipe`|Execute one of multiple pipes based on a selector key|
+|`ParallelPipe`|Execute multiple pipes and combine their results (only async implementation is actually parallel)|
+|`MetadataWrapperPipe`|Transform the metadata given to the pipeline for the subpipe|
+|`MapPipe`|Apply a subpipe into a sequence of inputs|
+|`FilterPipe`|Filter a sequence of inputs using a predicate|
+|`RetryPipe`|Run a pipe multiple times in case it fails|
+
+## Tests
+
+You can run all tests in this repository with
+
+```
+uv run pytest test
+uv run mypy .
+```
+
+## License
+
+TBD
+
+## AI Use Disclosure
+
+I hate writing tests so unit tests have been generated by GPT-OSS 120B for your convenience.
+They are provided in the hope that they are better than nothing.

kpipeline-0.1.1/README.md

@@ -0,0 +1,67 @@
+# kpipe
+
+`kpipe` is a simple pipeline library that allows you to write your complex application as a pipeline. This helps code organization, debugging, and testing.
+
+## How to use
+
+kpipe is like a DSL language inside Python. It is based on "pipes" which can be combined into pipelines using the pipe primitives provided by the library.
+
+Each pipe represents a function transforming an input into an output. Pipes are stateless and immutable. Below is an example defining two example pipes.
+
+```python
+from kpipeline import Pipe
+
+class AddOnePipe(Pipe[int, int, None]):
+    def apply(self, input: int, metadata: None) -> int:
+        return input + 1
+
+
+class MulByTwoPipe(Pipe[int, int, None]):
+    def apply(self, input: int, metadata: None) -> int:
+        return input * 2
+```
+
+These pipes can be combined using the ChainPipe primitive (aliased into the `|` operator) two form a pipeline that performs these two pipes sequentially:
+
+```python
+pipeline = AddOnePipe() | MulByTwoPipe()
+
+# or
+
+from kpipeline import ChainPipe
+pipeline = ChainPipe(AddOnePipe(), MulByTwoPipe())
+
+print(pipeline.apply(2))  # 6
+```
+
+Currently, the library defines these primitives:
+
+| Primitive | Purpose |
+|---|---|
+|`ChainPipe`|Execute two pipes sequentially|
+|`ConditionalPipe`|Execute a pipe if a given condition is true|
+|`BranchPipe`|Execute one of two pipes depending on whether the condition is true or false|
+|`SelectPipe`|Execute one of multiple pipes based on a selector key|
+|`ParallelPipe`|Execute multiple pipes and combine their results (only async implementation is actually parallel)|
+|`MetadataWrapperPipe`|Transform the metadata given to the pipeline for the subpipe|
+|`MapPipe`|Apply a subpipe into a sequence of inputs|
+|`FilterPipe`|Filter a sequence of inputs using a predicate|
+|`RetryPipe`|Run a pipe multiple times in case it fails|
+
+## Tests
+
+You can run all tests in this repository with
+
+```
+uv run pytest test
+uv run mypy .
+```
+
+## License
+
+TBD
+
+## AI Use Disclosure
+
+I hate writing tests so unit tests have been generated by GPT-OSS 120B for your convenience.
+They are provided in the hope that they are better than nothing.

kpipeline-0.1.1/kpipeline/async_pipeline.py

@@ -0,0 +1,442 @@
+import asyncio
+from collections.abc import Awaitable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from .graph import Graph, GraphNode, GraphConnection
+from .pipeline import BasePipe, Pipe
+
+
+class AsyncPipe[Input, Output, Metadata](BasePipe[Input, Awaitable[Output], Metadata]):
+    """
+    Abstract base class of all asynchronous pipes.
+    """
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        """
+        Like batch_apply, but returns Awaitable[Sequence[Output]] instead of Sequence[Awaitable[Output]].
+        The default implementation uses asyncio.gather.
+        """
+        return await asyncio.gather(*self.batch_apply(data, metadata))
+
+    def async_chain[OtherOutput](self, other: "Pipe[Output, OtherOutput, Metadata] | AsyncPipe[Output, OtherOutput, Metadata]") -> "AsyncChainPipe[Input, Output, OtherOutput, Metadata]":
+        """
+        Apply the second pipe to the result of the first pipe. Same as AsyncChainPipe(self, other).
+        """
+        return AsyncChainPipe(self, other)
+
+    def __or__[OtherOutput](self, other: "Pipe[Output, OtherOutput, Metadata] | AsyncPipe[Output, OtherOutput, Metadata]") -> "AsyncChainPipe[Input, Output, OtherOutput, Metadata]":
+        """
+        Alias for AsyncPipe.async_chain.
+        """
+        return self.async_chain(other)
+
+
+type SyncOrAsyncPipe[I, O, M] = Pipe[I, O, M] | AsyncPipe[I, O, M]
+
+
+async def _await_or_return[T](data: T | Awaitable[T]) -> T:
+    if isinstance(data, Awaitable):
+        return await data
+
+    else:
+        return data
+
+
+type SyncOrAsyncPipeOrCallable[I, O, M] = Callable[[I, M], O] | Callable[[I, M], Awaitable[O]] | SyncOrAsyncPipe[I, O, M]
+
+
+async def _call_or_apply[I, O, M](f: SyncOrAsyncPipeOrCallable[I, O, M], d: I, m: M) -> O:
+    if isinstance(f, BasePipe):
+        return await _await_or_return(f.apply(d, m))
+
+    else:
+        return await _await_or_return(f(d, m))
+
+
+async def _batch_apply[I, O, M](p: SyncOrAsyncPipe[I, O, M], d: Sequence[I], m: M) -> Sequence[O]:
+    if isinstance(p, AsyncPipe):
+        return await p.async_batch_apply(d, m)
+    
+    else:
+        return p.batch_apply(d, m)
+
+
+@dataclass(frozen=True)
+class AsyncChainPipe[Input, Middle, Output, Metadata](AsyncPipe[Input, Output, Metadata]):
+    """
+    Connects to pipes so that the second pipe is applied to the result of the first pipe.
+    """
+    pipe1: SyncOrAsyncPipe[Input, Middle, Metadata]
+    pipe2: SyncOrAsyncPipe[Middle, Output, Metadata]
+
+    async def apply(self, data: Input, metadata: Metadata) -> Output:
+        middle = await _await_or_return(self.pipe1.apply(data, metadata))
+        return await _await_or_return(self.pipe2.apply(middle, metadata))
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        middle_batch = await _batch_apply(self.pipe1, data, metadata)
+        return await _batch_apply(self.pipe2, middle_batch, metadata)
+
+    def to_graph(self) -> Graph:
+        graph1 = self.pipe1.to_graph()
+        graph2 = self.pipe2.to_graph()
+        return graph1 >> graph2
+
+
+@dataclass(frozen=True)
+class AsyncIdentityPipe[InputOutput, Metadata](AsyncPipe[InputOutput, InputOutput, Metadata]):
+    """
+    A pipe that returns its input.
+    """
+    async def apply(self, data: InputOutput, metadata: Metadata) -> InputOutput:
+        return data
+
+
+@dataclass(frozen=True)
+class AsyncMerge2Pipe[Input, Output1, Output2, Output, Metadata](AsyncPipe[Input, Output, Metadata]):
+    """
+    Runs two pipes and merges their results.
+    """
+    pipe1: SyncOrAsyncPipe[Input, Output1, Metadata]
+    pipe2: SyncOrAsyncPipe[Input, Output2, Metadata]
+    merge: Callable[[Output1, Output2], Output]
+    description: str = "Combine results"
+
+    async def apply(self, data: Input, metadata: Metadata) -> Output:
+        output1 = await _await_or_return(self.pipe1.apply(data, metadata))
+        output2 = await _await_or_return(self.pipe2.apply(data, metadata))
+        return self.merge(output1, output2)
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        batch1 = await _batch_apply(self.pipe1, data, metadata)
+        batch2 = await _batch_apply(self.pipe2, data, metadata)
+
+        assert len(data) == len(batch1) == len(batch2), "lengths do not match"
+        return await asyncio.gather(*[_await_or_return(self.merge(output1, output2)) for output1, output2 in zip(batch1, batch2)])
+
+    def to_graph(self) -> Graph:
+        graph1 = self.pipe1.to_graph()
+        graph2 = self.pipe2.to_graph()
+        combine_node = self.to_node()._replace(title=self.description, shape="combine")
+        graph = Graph(nodes=(combine_node,), inputs=(combine_node.id,), outputs=(combine_node.id,))
+        return (graph1 | graph2) >> graph
+
+
+@dataclass(frozen=True)
+class AsyncMerge3Pipe[Input, Output1, Output2, Output3, Output, Metadata](AsyncPipe[Input, Output, Metadata]):
+    """
+    Runs three pipes and merges their results.
+    """
+    pipe1: SyncOrAsyncPipe[Input, Output1, Metadata]
+    pipe2: SyncOrAsyncPipe[Input, Output2, Metadata]
+    pipe3: SyncOrAsyncPipe[Input, Output3, Metadata]
+    merge: Callable[[Output1, Output2, Output3], Output] | Callable[[Output1, Output2, Output3], Awaitable[Output]]
+    description: str = "Combine results"
+
+    async def apply(self, data: Input, metadata: Metadata) -> Output:
+        output1 = await _await_or_return(self.pipe1.apply(data, metadata))
+        output2 = await _await_or_return(self.pipe2.apply(data, metadata))
+        output3 = await _await_or_return(self.pipe3.apply(data, metadata))
+        return await _await_or_return(self.merge(output1, output2, output3))
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        batch1 = await _batch_apply(self.pipe1, data, metadata)
+        batch2 = await _batch_apply(self.pipe2, data, metadata)
+        batch3 = await _batch_apply(self.pipe3, data, metadata)
+
+        assert len(data) == len(batch1) == len(batch2) == len(batch3), "lengths do not match"
+        return await asyncio.gather(*[_await_or_return(self.merge(output1, output2, output3)) for output1, output2, output3 in zip(batch1, batch2, batch3)])
+
+    def to_graph(self) -> Graph:
+        graph1 = self.pipe1.to_graph()
+        graph2 = self.pipe2.to_graph()
+        graph3 = self.pipe3.to_graph()
+        combine_node = self.to_node()._replace(title=self.description, shape="combine")
+        graph = Graph(nodes=(combine_node,), inputs=(combine_node.id,), outputs=(combine_node.id,))
+        return (graph1 | graph2 | graph3) >> graph
+
+
+@dataclass(frozen=True)
+class AsyncBranchPipe[Input, Output, Metadata](AsyncPipe[Input, Output, Metadata]):
+    """
+    Selects one of two pipes to apply based on a condition.
+    """
+    condition: SyncOrAsyncPipeOrCallable[Input, bool, Metadata]
+    then_pipe: SyncOrAsyncPipe[Input, Output, Metadata]
+    else_pipe: SyncOrAsyncPipe[Input, Output, Metadata]
+    description: str = ""
+
+    async def apply(self, data: Input, metadata: Metadata) -> Output:
+        if await _call_or_apply(self.condition, data, metadata):
+            return await _await_or_return(self.then_pipe.apply(data, metadata))
+
+        else:
+            return await _await_or_return(self.else_pipe.apply(data, metadata))
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        conds = await asyncio.gather(*[_call_or_apply(self.condition, d, metadata) for d in data])
+        then_batch_future = _batch_apply(self.then_pipe, [d for c, d in zip(conds, data) if c], metadata)
+        else_batch_future = _batch_apply(self.else_pipe, [d for c, d in zip(conds, data) if not c], metadata)
+        then_batch, else_batch = await asyncio.gather(then_batch_future, else_batch_future)
+
+        ans: list[Output] = []
+        t = 0
+        e = 0
+        for cond in conds:
+            if cond:
+                ans.append(then_batch[t])
+                t += 1
+            
+            else:
+                ans.append(else_batch[e])
+                e += 1
+
+        return ans
+
+    def to_graph(self) -> Graph:
+        condition_node = self.to_node()._replace(title=self.description or "Condition", shape="condition", subgraph=self.condition.to_graph() if isinstance(self.condition, Pipe) else None)
+        then_graph = self.then_pipe.to_graph()
+        else_graph = self.else_pipe.to_graph()
+        return (
+            (then_graph | else_graph)
+            .add(nodes=(condition_node,))
+            .add(connections=tuple(GraphConnection(condition_node.id, i, "yes") for i in then_graph.inputs))
+            .add(connections=tuple(GraphConnection(condition_node.id, i, "no") for i in else_graph.inputs))
+            ._replace(
+                inputs=(condition_node.id,),
+                outputs=then_graph.outputs + else_graph.outputs,
+            )
+        )
+
+
+@dataclass(frozen=True)
+class AsyncConditionalPipe[InputOutput, Metadata](AsyncPipe[InputOutput, InputOutput, Metadata]):
+    """
+    Applies the pipe only if the condition is met. Similar to AsyncBranchPipe except that the else branch is an identity function.
+    Can only be used when the subpipe returns the same type that gets in.
+    """
+    condition: Callable[[InputOutput, Metadata], bool | Awaitable[bool]]
+    subpipe: SyncOrAsyncPipe[InputOutput, InputOutput, Metadata]
+    description: str = ""
+
+    async def apply(self, data: InputOutput, metadata: Metadata) -> InputOutput:
+        if await _await_or_return(self.condition(data, metadata)):
+            return await _await_or_return(self.subpipe.apply(data, metadata))
+
+        else:
+            return data
+
+    async def async_batch_apply(self, data: Sequence[InputOutput], metadata: Metadata) -> Sequence[InputOutput]:
+        conds = await asyncio.gather(*[_call_or_apply(self.condition, d, metadata) for d in data])
+        then_batch = await _batch_apply(self.subpipe, [d for c, d in zip(conds, data) if c], metadata)
+
+        ans: list[InputOutput] = []
+        t = 0
+        for i, cond in enumerate(conds):
+            if cond:
+                ans.append(then_batch[t])
+                t += 1
+            
+            else:
+                ans.append(data[i])
+
+        return ans
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+    def is_wrapper(self) -> bool:
+        return True
+
+
+@dataclass(frozen=True)
+class AsyncSelectPipe[Input, Output, Metadata, Key](AsyncPipe[Input, Output, Metadata]):
+    """
+    Selects one of multiple pipes based on a selector function. AsyncBranchPipe generalized to more than two branches.
+    """
+    key: SyncOrAsyncPipeOrCallable[Input, Key, Metadata]
+    subpipes: Mapping[Key, SyncOrAsyncPipe[Input, Output, Metadata]]
+    otherwise_pipe: SyncOrAsyncPipe[Input, Output, Metadata]
+    description: str = ""
+
+    async def apply(self, data: Input, metadata: Metadata) -> Output:
+        key = await _call_or_apply(self.key, data, metadata)
+        if key in self.subpipes:
+            return await _await_or_return(self.subpipes[key].apply(data, metadata))
+
+        else:
+            return await _await_or_return(self.otherwise_pipe.apply(data, metadata))
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        keys = await asyncio.gather(*[_call_or_apply(self.key, d, metadata) for d in data])
+        unique_keys = list(set(keys))
+        batches: dict[Key, list[tuple[int, Input]]] = {key: [] for key in unique_keys}
+        for i, (key, item) in enumerate(zip(keys, data)):
+            batches[key].append((i, item))
+        
+        batch_futures: list[Awaitable[Sequence[Output]]] = []
+        for key in unique_keys:
+            pipe = self.subpipes.get(key, self.otherwise_pipe)
+            batch = [item for _, item in batches[key]]
+            batch_futures.append(_batch_apply(pipe, batch, metadata))
+
+        batch_results = await asyncio.gather(*batch_futures)
+        batch_results_dict = {key: batch_results[i] for i, key in enumerate(unique_keys)}
+        ans_dict = {i: result for key, batch in batches.items() for (i, _), result in zip(batch, batch_results_dict[key])}
+        return [ans_dict[i] for i in range(len(data))]
+
+    def to_graph(self) -> Graph:
+        condition_node = self.to_node()._replace(title=self.description or "Condition", shape="condition", subgraph=self.key.to_graph() if isinstance(self.key, Pipe) else None)
+        otherwise_graph = self.otherwise_pipe.to_graph()
+        graph = otherwise_graph.add(
+            nodes=(condition_node,),
+            connections=tuple(GraphConnection(condition_node.id, i, "otherwise") for i in otherwise_graph.inputs),
+        )
+        for key, subpipe in self.subpipes.items():
+            subpipe_graph = subpipe.to_graph()
+            graph |= subpipe_graph
+            graph = graph.add(connections=tuple(GraphConnection(condition_node.id, i, str(key)) for i in subpipe_graph.inputs))
+
+        return graph._replace(inputs=(condition_node.id,))
+
+
+@dataclass(frozen=True)
+class AsyncParallelPipe[Input, Output, CombinedOutput, Metadata](AsyncPipe[Input, CombinedOutput, Metadata]):
+    """
+    Executes multiple pipes in parallel on the same input and combines the outputs.
+    """
+    subpipes: Sequence[SyncOrAsyncPipe[Input, Output, Metadata]]
+    combine: SyncOrAsyncPipeOrCallable[Sequence[Output], CombinedOutput, Metadata]
+    description: str = ""
+
+    async def apply(self, data: Input, metadata: Metadata) -> CombinedOutput:
+        results: list[Awaitable[Output]] = []
+        for subpipe in self.subpipes:
+            results.append(_await_or_return(subpipe.apply(data, metadata)))
+
+        gathered: Sequence[Output] = await asyncio.gather(*results)
+        return await _call_or_apply(self.combine, gathered, metadata)
+
+    async def async_batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[CombinedOutput]:
+        results: list[Awaitable[Sequence[Output]]] = []
+        for subpipe in self.subpipes:
+            results.append(_batch_apply(subpipe, data, metadata))
+
+        gathered = await asyncio.gather(*results)
+        zipped_results: zip[Sequence[Output]] = zip(*gathered)
+        return await asyncio.gather(*[_call_or_apply(self.combine, seq, metadata) for seq in zipped_results])
+
+    def to_graph(self) -> Graph:
+        combine_node = self.to_node()._replace(title=self.description or "Combine results", shape="combine", subgraph=self.combine.to_graph() if isinstance(self.combine, Pipe) else None)
+        graph = Graph(nodes=(combine_node,))
+        for subpipe in self.subpipes:
+            subpipe_graph = subpipe.to_graph()
+            graph |= subpipe_graph
+            graph = graph.add(connections=tuple(GraphConnection(o, combine_node.id) for o in subpipe_graph.outputs))
+
+        return graph._replace(outputs=(combine_node.id,))
+
+
+@dataclass(frozen=True)
+class AsyncMetadataWrapperPipe[Input, Output, OuterMetadata, InnerMetadata](AsyncPipe[Input, Output, OuterMetadata]):
+    """
+    Executes a pipe with changed metadata.
+
+    Does NOT allow to_inner to be a pipe: this is intentional, metadata is not transformed by pipes.
+    """
+    to_inner: Callable[[OuterMetadata], InnerMetadata | Awaitable[InnerMetadata]]
+    subpipe: SyncOrAsyncPipe[Input, Output, InnerMetadata]
+    description: str = ""
+
+    async def apply(self, data: Input, metadata: OuterMetadata) -> Output:
+        return await _await_or_return(self.subpipe.apply(data, await _await_or_return(self.to_inner(metadata))))
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+    def is_wrapper(self) -> bool:
+        return True
+
+
+@dataclass(frozen=True)
+class AsyncMapPipe[Input, Output, Metadata](AsyncPipe[Sequence[Input], Sequence[Output], Metadata]):
+    """
+    Maps a sequence of input objects into a sequence of output objects with a subpipe.
+
+    Does NOT allow the subpipe to be a callable: this is intentional, functions that transforms Inputs to Outputs should be pipes.
+    """
+    subpipe: SyncOrAsyncPipe[Input, Output, Metadata]
+    description: str = "Map"
+
+    async def apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        if isinstance(self.subpipe, AsyncPipe):
+            return await self.subpipe.async_batch_apply(data, metadata)
+
+        else:
+            return self.subpipe.batch_apply(data, metadata)
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+
+@dataclass(frozen=True)
+class AsyncFilterPipe[Input, Metadata](AsyncPipe[Sequence[Input], Sequence[Input], Metadata]):
+    """
+    Filters a sequence of input object with a predicate.
+    """
+    predicate: SyncOrAsyncPipeOrCallable[Input, bool, Metadata]
+    description: str = "Filter"
+
+    async def apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Input]:
+        return [i for i in data if await _call_or_apply(self.predicate, i, metadata)]
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.predicate.to_graph() if isinstance(self.predicate, BasePipe) else None
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+
+@dataclass(frozen=True)
+class AsyncRetryPipe[Input, Output, Metadata](AsyncPipe[Input, Output, Metadata]):
+    """
+    Tries to execute the subpipe several times in case it fails.
+    """
+    subpipe: SyncOrAsyncPipe[Input, Output, Metadata]
+    retries: int
+    exceptions: type | tuple[type, ...]
+    description: str = "Retry several times"
+
+    async def apply(self, data: Input, metadata: Metadata) -> Output:
+        attempt = 0
+        while True:
+            try:
+                return await _await_or_return(self.subpipe.apply(data, metadata))
+
+            except Exception as e:
+                if not isinstance(e, self.exceptions):
+                    raise e
+
+                attempt += 1
+                if attempt > self.retries:
+                    raise e
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+    def is_wrapper(self) -> bool:
+        return True
+

kpipeline-0.1.1/kpipeline/graph.py

@@ -0,0 +1,109 @@
+from typing import Literal, NamedTuple, Optional, Type
+
+type GraphNodeId = str
+type GraphNodeShape = Literal["default", "combine", "condition"]
+"""
+The shape of the node.
+`combine` is used for nodes that combine multiple inputs into one output.
+`condition` is used for nodes that diverge path based on condition.
+The actual shape depends on the rendering backend.
+"""
+
+
+class GraphNode(NamedTuple):
+    id: GraphNodeId
+    title: str
+    type: Optional[Type] = None
+    shape: GraphNodeShape = "default"
+    subgraph: "Optional[Graph]" = None
+
+    def to_dot(self, indent: int = 0) -> str:
+        shape_map = {
+            "default": "box",
+            "combine": "ellipse",
+            "condition": "diamond",
+        }
+        dot_shape = shape_map.get(self.shape, "box")
+        node_def = f"""{"  "*indent}"{self.id}" [label="{self.title}", shape={dot_shape}];"""
+        if self.subgraph:
+            sub_dot = self.subgraph.to_dot(name=None, indent=indent+1)
+            node_def += f"""{"  "*indent}subgraph "cluster_{self.id}" {{"""
+            node_def += f"""{"  "*indent}  label="{self.title}";"""
+            node_def += sub_dot
+            node_def += "  "*indent + "}"
+        return node_def
+
+
+class GraphConnection(NamedTuple):
+    from_node: GraphNodeId
+    to_node: GraphNodeId
+    label: str = ""
+
+    def to_dot(self, indent: int = 0) -> str:
+        label_part = f' [label="{self.label}"]' if self.label else ""
+        return f"""{"  "*indent}"{self.from_node}" -> "{self.to_node}"{label_part};"""
+
+
+class Graph(NamedTuple):
+    nodes: tuple[GraphNode, ...] = ()
+    connections: tuple[GraphConnection, ...] = ()
+    inputs: tuple[GraphNodeId, ...] = ()
+    outputs: tuple[GraphNodeId, ...] = ()
+
+    def __or__(self, other: "Graph") -> "Graph":
+        return Graph(
+            nodes=self.nodes + other.nodes,
+            connections=self.connections + other.connections,
+            inputs=self.inputs + other.inputs,
+            outputs=self.outputs + other.outputs,
+        )
+
+    def __rshift__(self, other: "Graph") -> "Graph":
+        return Graph(
+            nodes=self.nodes + other.nodes,
+            connections=self.connections + other.connections + tuple(GraphConnection(o, i) for o in self.outputs for i in other.inputs),
+            inputs=self.inputs,
+            outputs=other.outputs,
+        )
+
+    def add(self, nodes: tuple[GraphNode, ...] = (), connections: tuple[GraphConnection, ...] = (), inputs: tuple[GraphNodeId, ...] = (), outputs: tuple[GraphNodeId, ...] = ()) -> "Graph":
+        return (self | Graph(nodes, connections, inputs, outputs))
+
+    def is_valid(self):
+        """
+        Checks that all connections, inputs, and outputs only refer existing nodes.
+        """
+        node_id_set = set()
+        node_stack = list(self.nodes)
+        while len(node_stack):
+            node = node_stack.pop()
+            if node.id in node_id_set:
+                continue
+
+            if node.subgraph is not None:
+                node_stack += node.subgraph.nodes
+
+            node_id_set.add(node.id)
+
+        references = set()
+        references |= {c.from_node for c in self.connections}
+        references |= {c.to_node for c in self.connections}
+        references |= set(self.inputs)
+        references |= set(self.outputs)
+        return references <= node_id_set
+
+    def to_dot(self, name: str | None = "graph", indent: int = 0) -> str:
+        lines: list[str] = []
+        if name is not None:
+            lines.append(f"digraph {name} {{")
+
+        for node in self.nodes:
+            lines.append(node.to_dot(indent=indent+1))
+
+        for conn in self.connections:
+            lines.append(conn.to_dot(indent=indent+1))
+
+        if name is not None:
+            lines.append("}")
+
+        return "\n".join(lines)

kpipeline-0.1.1/kpipeline/pipeline.py

@@ -0,0 +1,451 @@
+from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Callable, Iterable, NamedTuple, Optional
+
+from .graph import Graph, GraphNode, GraphConnection
+
+
+class BasePipe[Input, Output, Metadata](ABC):
+    """
+    Abstract base class of all synchronous and asynchronous pipes.
+    """
+
+    @abstractmethod
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        """
+        Apply the pipe to the given input data and return the result.
+        """
+        ...
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        """
+        Applies the pipe to multiple data points. By default merely calls apply in sequence.
+        """
+        return [self.apply(d, metadata) for d in data]
+
+    def chain[OtherOutput](self, other: "Pipe[Output, OtherOutput, Metadata]") -> "ChainPipe[Input, Output, OtherOutput, Metadata]":
+        """
+        Apply the second pipe to the result of the first pipe. Same as ChainPipe(self, other).
+        """
+        return ChainPipe(self, other)
+
+    def to_node(self) -> GraphNode:
+        return GraphNode(id=str(id(self)), type=type(self), title="Pipe")
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return None
+
+    def is_wrapper(self) -> bool:
+        return False
+
+    def to_graph(self) -> Graph:
+        """
+        Return the pipeline as a graph.
+        """
+        if subgraph := self.get_subgraph():
+            node = self.to_node()._replace(subgraph=subgraph)
+            return Graph(
+                nodes=(node,),
+                connections=(),
+                inputs=subgraph.inputs if self.is_wrapper() else (node.id,),
+                outputs=subgraph.outputs if self.is_wrapper() else (node.id,),
+            )
+
+        else:
+            node = self.to_node()
+            return Graph(nodes=(node,), connections=(), inputs=(node.id,), outputs=(node.id,))
+
+
+class Pipe[Input, Output, Metadata](BasePipe[Input, Output, Metadata]):
+    """
+    Abstract base class of all synchronous pipes.
+    """
+
+    def __or__[OtherOutput](self, other: "Pipe[Output, OtherOutput, Metadata]") -> "ChainPipe[Input, Output, OtherOutput, Metadata]":
+        """
+        Alias for Pipe.chain.
+        """
+        return self.chain(other)
+
+
+type PipeOrCallable[I, O, M] = Callable[[I, M], O] | BasePipe[I, O, M]
+
+
+def _call_or_apply[I, O, M](f: PipeOrCallable[I, O, M], d: I, m: M) -> O:
+    if isinstance(f, BasePipe):
+        return f.apply(d, m)
+
+    else:
+        return f(d, m)
+
+
+@dataclass(frozen=True)
+class ChainPipe[Input, Middle, Output, Metadata](Pipe[Input, Output, Metadata]):
+    """
+    Connects to pipes so that the second pipe is applied to the result of the first pipe.
+    """
+    pipe1: BasePipe[Input, Middle, Metadata]
+    pipe2: BasePipe[Middle, Output, Metadata]
+
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        return self.pipe2.apply(self.pipe1.apply(data, metadata), metadata)
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        middle_batch = self.pipe1.batch_apply(data, metadata)
+        return self.pipe2.batch_apply(middle_batch, metadata)
+
+    def to_graph(self) -> Graph:
+        graph1 = self.pipe1.to_graph()
+        graph2 = self.pipe2.to_graph()
+        return graph1 >> graph2
+
+
+@dataclass(frozen=True)
+class IdentityPipe[InputOutput, Metadata](Pipe[InputOutput, InputOutput, Metadata]):
+    """
+    A pipe that returns its input.
+    """
+    def apply(self, data: InputOutput, metadata: Metadata) -> InputOutput:
+        return data
+
+
+@dataclass(frozen=True)
+class Merge2Pipe[Input, Output1, Output2, Output, Metadata](Pipe[Input, Output, Metadata]):
+    """
+    Runs two pipes and merges their results.
+    """
+    pipe1: BasePipe[Input, Output1, Metadata]
+    pipe2: BasePipe[Input, Output2, Metadata]
+    merge: Callable[[Output1, Output2], Output]
+    description: str = "Combine results"
+
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        output1 = self.pipe1.apply(data, metadata)
+        output2 = self.pipe2.apply(data, metadata)
+        return self.merge(output1, output2)
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        batch1 = self.pipe1.batch_apply(data, metadata)
+        batch2 = self.pipe2.batch_apply(data, metadata)
+        assert len(data) == len(batch1) == len(batch2), "lengths do not match"
+        return [self.merge(output1, output2) for output1, output2 in zip(batch1, batch2)]
+
+    def to_graph(self) -> Graph:
+        graph1 = self.pipe1.to_graph()
+        graph2 = self.pipe2.to_graph()
+        combine_node = self.to_node()._replace(title=self.description, shape="combine")
+        graph = Graph(nodes=(combine_node,), inputs=(combine_node.id,), outputs=(combine_node.id,))
+        return (graph1 | graph2) >> graph
+
+
+@dataclass(frozen=True)
+class Merge3Pipe[Input, Output1, Output2, Output3, Output, Metadata](Pipe[Input, Output, Metadata]):
+    """
+    Runs three pipes and merges their results.
+    """
+    pipe1: BasePipe[Input, Output1, Metadata]
+    pipe2: BasePipe[Input, Output2, Metadata]
+    pipe3: BasePipe[Input, Output3, Metadata]
+    merge: Callable[[Output1, Output2, Output3], Output]
+    description: str = "Combine results"
+
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        output1 = self.pipe1.apply(data, metadata)
+        output2 = self.pipe2.apply(data, metadata)
+        output3 = self.pipe3.apply(data, metadata)
+        return self.merge(output1, output2, output3)
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        batch1 = self.pipe1.batch_apply(data, metadata)
+        batch2 = self.pipe2.batch_apply(data, metadata)
+        batch3 = self.pipe3.batch_apply(data, metadata)
+        assert len(data) == len(batch1) == len(batch2) == len(batch3), "lengths do not match"
+        return [self.merge(output1, output2, output3) for output1, output2, output3 in zip(batch1, batch2, batch3)]
+
+    def to_graph(self) -> Graph:
+        graph1 = self.pipe1.to_graph()
+        graph2 = self.pipe2.to_graph()
+        graph3 = self.pipe3.to_graph()
+        combine_node = self.to_node()._replace(title=self.description, shape="combine")
+        graph = Graph(nodes=(combine_node,), inputs=(combine_node.id,), outputs=(combine_node.id,))
+        return (graph1 | graph2 | graph3) >> graph
+
+
+@dataclass(frozen=True)
+class BranchPipe[Input, Output, Metadata](Pipe[Input, Output, Metadata]):
+    """
+    Selects one of two pipes to apply based on a condition.
+    """
+    condition: PipeOrCallable[Input, bool, Metadata]
+    then_pipe: BasePipe[Input, Output, Metadata]
+    else_pipe: BasePipe[Input, Output, Metadata]
+    description: str = "Condition"
+
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        if _call_or_apply(self.condition, data, metadata):
+            return self.then_pipe.apply(data, metadata)
+
+        else:
+            return self.else_pipe.apply(data, metadata)
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        conds = [_call_or_apply(self.condition, d, metadata) for d in data]
+        then_batch = self.then_pipe.batch_apply([d for c, d in zip(conds, data) if c], metadata)
+        else_batch = self.else_pipe.batch_apply([d for c, d in zip(conds, data) if not c], metadata)
+
+        ans: list[Output] = []
+        t = 0
+        e = 0
+        for cond in conds:
+            if cond:
+                ans.append(then_batch[t])
+                t += 1
+            
+            else:
+                ans.append(else_batch[e])
+                e += 1
+
+        return ans
+
+    def to_graph(self) -> Graph:
+        condition_node = self.to_node()._replace(title=self.description, shape="condition", subgraph=self.condition.to_graph() if isinstance(self.condition, Pipe) else None)
+        then_graph = self.then_pipe.to_graph()
+        else_graph = self.else_pipe.to_graph()
+        return (
+            (then_graph | else_graph)
+            .add(nodes=(condition_node,))
+            .add(connections=tuple(GraphConnection(condition_node.id, i, "yes") for i in then_graph.inputs))
+            .add(connections=tuple(GraphConnection(condition_node.id, i, "no") for i in else_graph.inputs))
+            ._replace(
+                inputs=(condition_node.id,),
+                outputs=then_graph.outputs + else_graph.outputs,
+            )
+        )
+
+
+@dataclass(frozen=True)
+class ConditionalPipe[InputOutput, Metadata](Pipe[InputOutput, InputOutput, Metadata]):
+    """
+    Applies the pipe only if the condition is met. Similar to BranchPipe except that the else branch is an identity function.
+    Can only be used when the subpipe returns the same type that gets in.
+    """
+    condition: Callable[[InputOutput, Metadata], bool]
+    subpipe: BasePipe[InputOutput, InputOutput, Metadata]
+    description: str = "Condition"
+
+    def apply(self, data: InputOutput, metadata: Metadata) -> InputOutput:
+        if self.condition(data, metadata):
+            return self.subpipe.apply(data, metadata)
+
+        else:
+            return data
+
+    def batch_apply(self, data: Sequence[InputOutput], metadata: Metadata) -> Sequence[InputOutput]:
+        conds = [_call_or_apply(self.condition, d, metadata) for d in data]
+        then_batch = self.subpipe.batch_apply([d for c, d in zip(conds, data) if c], metadata)
+
+        ans: list[InputOutput] = []
+        t = 0
+        for i, cond in enumerate(conds):
+            if cond:
+                ans.append(then_batch[t])
+                t += 1
+            
+            else:
+                ans.append(data[i])
+
+        return ans
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+    def is_wrapper(self) -> bool:
+        return True
+
+
+@dataclass(frozen=True)
+class SelectPipe[Input, Output, Metadata, Key](Pipe[Input, Output, Metadata]):
+    """
+    Selects one of multiple pipes based on a selector function. BranchPipe generalized to more than two branches.
+    """
+    key: PipeOrCallable[Input, Key, Metadata]
+    subpipes: Mapping[Key, BasePipe[Input, Output, Metadata]]
+    otherwise_pipe: BasePipe[Input, Output, Metadata]
+    description: str = "Key"
+
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        key = _call_or_apply(self.key, data, metadata)
+        if key in self.subpipes:
+            return self.subpipes[key].apply(data, metadata)
+
+        else:
+            return self.otherwise_pipe.apply(data, metadata)
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        keys = [_call_or_apply(self.key, d, metadata) for d in data]
+        unique_keys = list(set(keys))
+        batches: dict[Key, list[tuple[int, Input]]] = {key: [] for key in unique_keys}
+        for i, (key, item) in enumerate(zip(keys, data)):
+            batches[key].append((i, item))
+        
+        batch_results: list[Sequence[Output]] = []
+        for key in unique_keys:
+            pipe = self.subpipes.get(key, self.otherwise_pipe)
+            batch = [item for _, item in batches[key]]
+            batch_results.append(pipe.batch_apply(batch, metadata))
+
+        batch_results_dict = {key: batch_results[i] for i, key in enumerate(unique_keys)}
+        ans_dict = {i: result for key, batch in batches.items() for (i, _), result in zip(batch, batch_results_dict[key])}
+        return [ans_dict[i] for i in range(len(data))]
+
+    def to_graph(self) -> Graph:
+        condition_node = self.to_node()._replace(title=self.description, shape="condition", subgraph=self.key.to_graph() if isinstance(self.key, Pipe) else None)
+        otherwise_graph = self.otherwise_pipe.to_graph()
+        graph = otherwise_graph.add(
+            nodes=(condition_node,),
+            connections=tuple(GraphConnection(condition_node.id, i, "otherwise") for i in otherwise_graph.inputs),
+        )
+        for key, subpipe in self.subpipes.items():
+            subpipe_graph = subpipe.to_graph()
+            graph |= subpipe_graph
+            graph = graph.add(connections=tuple(GraphConnection(condition_node.id, i, str(key)) for i in subpipe_graph.inputs))
+
+        return graph._replace(inputs=(condition_node.id,))
+
+
+@dataclass(frozen=True)
+class ParallelPipe[Input, Output, CombinedOutput, Metadata](Pipe[Input, CombinedOutput, Metadata]):
+    """
+    Executes multiple pipes on the same input and combines the outputs.
+    The synchronous implementation does not actually execute the pipes in parallel.
+    """
+    subpipes: Sequence[BasePipe[Input, Output, Metadata]]
+    combine: PipeOrCallable[Sequence[Output], CombinedOutput, Metadata]
+    description: str = "Combine results"
+
+    def apply(self, data: Input, metadata: Metadata) -> CombinedOutput:
+        results: list[Output] = []
+        for subpipe in self.subpipes:
+            results.append(subpipe.apply(data, metadata))
+
+        seq: Sequence[Output] = results
+        return _call_or_apply(self.combine, seq, metadata)
+
+    def batch_apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[CombinedOutput]:
+        results: list[Sequence[Output]] = []
+        for subpipe in self.subpipes:
+            results.append(subpipe.batch_apply(data, metadata))
+
+        zipped_results: zip[Sequence[Output]] = zip(*results)
+        return [_call_or_apply(self.combine, seq, metadata) for seq in zipped_results]
+
+    def to_graph(self) -> Graph:
+        combine_node = self.to_node()._replace(title=self.description, shape="combine", subgraph=self.combine.to_graph() if isinstance(self.combine, Pipe) else None)
+        graph = Graph(nodes=(combine_node,))
+        for subpipe in self.subpipes:
+            subpipe_graph = subpipe.to_graph()
+            graph |= subpipe_graph
+            graph = graph.add(connections=tuple(GraphConnection(o, combine_node.id) for o in subpipe_graph.outputs))
+
+        return graph._replace(outputs=(combine_node.id,))
+
+
+@dataclass(frozen=True)
+class MetadataWrapperPipe[Input, Output, OuterMetadata, InnerMetadata](Pipe[Input, Output, OuterMetadata]):
+    """
+    Executes a pipe with changed metadata.
+
+    Does NOT allow to_inner to be a pipe: this is intentional, metadata is not transformed by pipes.
+    """
+    to_inner: Callable[[OuterMetadata], InnerMetadata]
+    subpipe: BasePipe[Input, Output, InnerMetadata]
+    description: str = "Wrap metadata"
+
+    def apply(self, data: Input, metadata: OuterMetadata) -> Output:
+        return self.subpipe.apply(data, self.to_inner(metadata))
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+    def is_wrapper(self) -> bool:
+        return True
+
+
+@dataclass(frozen=True)
+class MapPipe[Input, Output, Metadata](Pipe[Sequence[Input], Sequence[Output], Metadata]):
+    """
+    Maps a sequence of input objects into a sequence of output objects with a subpipe.
+
+    Does NOT allow the subpipe to be a callable: this is intentional, functions that transforms Inputs to Outputs should be pipes.
+    """
+    subpipe: BasePipe[Input, Output, Metadata]
+    description: str = "Map"
+
+    def apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Output]:
+        return self.subpipe.batch_apply(data, metadata)
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+
+@dataclass(frozen=True)
+class FilterPipe[Input, Metadata](Pipe[Sequence[Input], Sequence[Input], Metadata]):
+    """
+    Filters a sequence of input object with a predicate.
+    """
+    predicate: PipeOrCallable[Input, bool, Metadata]
+    description: str = "Filter"
+
+    def apply(self, data: Sequence[Input], metadata: Metadata) -> Sequence[Input]:
+        return [i for i in data if _call_or_apply(self.predicate, i, metadata)]
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.predicate.to_graph() if isinstance(self.predicate, Pipe) else None
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+
+@dataclass(frozen=True)
+class RetryPipe[Input, Output, Metadata](Pipe[Input, Output, Metadata]):
+    """
+    Tries to execute the subpipe several times in case it fails.
+    """
+    subpipe: BasePipe[Input, Output, Metadata]
+    retries: int
+    exceptions: type | tuple[type, ...]
+    description: str = "Retry several times"
+
+    def apply(self, data: Input, metadata: Metadata) -> Output:
+        attempt = 0
+        while True:
+            try:
+                return self.subpipe.apply(data, metadata)
+
+            except Exception as e:
+                if not isinstance(e, self.exceptions):
+                    raise e
+
+                attempt += 1
+                if attempt > self.retries:
+                    raise e
+
+    def get_subgraph(self) -> Optional[Graph]:
+        return self.subpipe.to_graph()
+
+    def to_node(self) -> GraphNode:
+        return super().to_node()._replace(title=self.description)
+
+    def is_wrapper(self) -> bool:
+        return True
+

kpipeline-0.1.1/kpipeline.egg-info/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: kpipeline
+Version: 0.1.1
+Summary: A pipeline library for Python
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# kpipe
+
+`kpipe` is a simple pipeline library that allows you to write your complex application as a pipeline. This helps code organization, debugging, and testing.
+
+## How to use
+
+kpipe is like a DSL language inside Python. It is based on "pipes" which can be combined into pipelines using the pipe primitives provided by the library.
+
+Each pipe represents a function transforming an input into an output. Pipes are stateless and immutable. Below is an example defining two example pipes.
+
+```python
+from kpipeline import Pipe
+
+class AddOnePipe(Pipe[int, int, None]):
+    def apply(self, input: int, metadata: None) -> int:
+        return input + 1
+
+
+class MulByTwoPipe(Pipe[int, int, None]):
+    def apply(self, input: int, metadata: None) -> int:
+        return input * 2
+```
+
+These pipes can be combined using the ChainPipe primitive (aliased into the `|` operator) two form a pipeline that performs these two pipes sequentially:
+
+```python
+pipeline = AddOnePipe() | MulByTwoPipe()
+
+# or
+
+from kpipeline import ChainPipe
+pipeline = ChainPipe(AddOnePipe(), MulByTwoPipe())
+
+print(pipeline.apply(2))  # 6
+```
+
+Currently, the library defines these primitives:
+
+| Primitive | Purpose |
+|---|---|
+|`ChainPipe`|Execute two pipes sequentially|
+|`ConditionalPipe`|Execute a pipe if a given condition is true|
+|`BranchPipe`|Execute one of two pipes depending on whether the condition is true or false|
+|`SelectPipe`|Execute one of multiple pipes based on a selector key|
+|`ParallelPipe`|Execute multiple pipes and combine their results (only async implementation is actually parallel)|
+|`MetadataWrapperPipe`|Transform the metadata given to the pipeline for the subpipe|
+|`MapPipe`|Apply a subpipe into a sequence of inputs|
+|`FilterPipe`|Filter a sequence of inputs using a predicate|
+|`RetryPipe`|Run a pipe multiple times in case it fails|
+
+## Tests
+
+You can run all tests in this repository with
+
+```
+uv run pytest test
+uv run mypy .
+```
+
+## License
+
+TBD
+
+## AI Use Disclosure
+
+I hate writing tests so unit tests have been generated by GPT-OSS 120B for your convenience.
+They are provided in the hope that they are better than nothing.

kpipeline-0.1.1/kpipeline.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+README.md
+pyproject.toml
+kpipeline/__init__.py
+kpipeline/async_pipeline.py
+kpipeline/graph.py
+kpipeline/pipeline.py
+kpipeline/py.typed
+kpipeline.egg-info/PKG-INFO
+kpipeline.egg-info/SOURCES.txt
+kpipeline.egg-info/dependency_links.txt
+kpipeline.egg-info/top_level.txt
+test/test_graph.py
+test/test_graph_dot.py

kpipeline-0.1.1/kpipeline.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

kpipeline-0.1.1/kpipeline.egg-info/top_level.txt

@@ -0,0 +1 @@
+kpipeline

kpipeline-0.1.1/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "kpipeline"
+version = "0.1.1"
+description = "A pipeline library for Python"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = []
+
+[dependency-groups]
+dev = [
+    "mypy>=1.18.2",
+    "pytest>=8.4.2",
+    "pytest-asyncio>=1.2.0",
+]

kpipeline-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

kpipeline-0.1.1/test/test_graph.py

@@ -0,0 +1,67 @@
+import builtins
+import types
+import pytest
+
+from kpipeline.graph import (
+    Graph,
+    GraphNode,
+    GraphConnection,
+)
+
+
+# ----------------------------------------------------------------------
+# Graph unit‑tests
+# ----------------------------------------------------------------------
+
+
+def test_graph_addition_and_validity():
+    """`Graph.__or__` must concatenate the underlying tuples."""
+    n1 = GraphNode(id="a", title="A")
+    n2 = GraphNode(id="b", title="B")
+    c1 = GraphConnection("a", "b", "link")
+
+    g1 = Graph(nodes=(n1,), connections=(c1,), inputs=("a",), outputs=("b",))
+    g2 = Graph(nodes=(n2,), connections=(), inputs=(), outputs=())
+
+    g3 = g1 | g2
+
+    assert isinstance(g3, Graph)
+    assert g3.nodes == (n1, n2)
+    assert g3.connections == (c1,)
+    assert g3.inputs == ("a",)
+    assert g3.outputs == ("b",)
+
+    # The graph is valid because every reference points to an existing node.
+    assert g3.is_valid()
+
+
+def test_graph_rshift_creates_cross_connections():
+    """`Graph.__rshift__` must connect every output of the left graph to every input of the right graph."""
+    n_left = GraphNode(id="l", title="L")
+    n_right = GraphNode(id="r", title="R")
+
+    left = Graph(nodes=(n_left,), connections=(), inputs=("l",), outputs=("l",))
+    right = Graph(nodes=(n_right,), connections=(), inputs=("r",), outputs=("r",))
+
+    composed = left >> right
+
+    # Two original nodes plus the automatically created connection
+    assert composed.nodes == (n_left, n_right)
+    assert len(composed.connections) == 1
+    conn = composed.connections[0]
+    assert conn.from_node == "l"
+    assert conn.to_node == "r"
+    assert conn.label == ""  # default label
+
+    # Inputs come from the left side, outputs from the right side
+    assert composed.inputs == ("l",)
+    assert composed.outputs == ("r",)
+
+    # The composed graph is still valid
+    assert composed.is_valid()
+
+
+def test_graph_is_invalid_when_reference_missing():
+    n = GraphNode(id="x", title="X")
+    g = Graph(nodes=(n,), connections=(GraphConnection("x", "y"),), inputs=("x",), outputs=("x",))
+    assert not g.is_valid()  # "y" does not exist

kpipeline-0.1.1/test/test_graph_dot.py

@@ -0,0 +1,35 @@
+from kpipeline.graph import Graph, GraphNode, GraphConnection
+
+
+def test_simple_graph_dot():
+    node1 = GraphNode(id="n1", title="Node 1")
+    node2 = GraphNode(id="n2", title="Node 2")
+    conn = GraphConnection(from_node="n1", to_node="n2", label="connect")
+    graph = Graph(nodes=(node1, node2), connections=(conn,))
+    
+    dot = graph.to_dot()
+    assert 'digraph graph {' in dot
+    assert '"n1" [label="Node 1", shape=box];' in dot
+    assert '"n2" [label="Node 2", shape=box];' in dot
+    assert '"n1" -> "n2" [label="connect"];' in dot
+
+def test_shapes_dot():
+    node_combine = GraphNode(id="nc", title="Combine", shape="combine")
+    node_cond = GraphNode(id="nd", title="Condition", shape="condition")
+    graph = Graph(nodes=(node_combine, node_cond))
+    
+    dot = graph.to_dot()
+    assert '"nc" [label="Combine", shape=ellipse];' in dot
+    assert '"nd" [label="Condition", shape=diamond];' in dot
+
+def test_subgraph_dot():
+    inner_node = GraphNode(id="in", title="Inner")
+    inner_graph = Graph(nodes=(inner_node,))
+    
+    outer_node = GraphNode(id="out", title="Outer", subgraph=inner_graph)
+    graph = Graph(nodes=(outer_node,))
+    
+    dot = graph.to_dot()
+    assert 'subgraph "cluster_out" {' in dot
+    assert 'label="Outer";' in dot
+    assert '"in" [label="Inner", shape=box];' in dot