khnm 1.0.0__tar.gz

khnm-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Marcin Sawicki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

khnm-1.0.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: khnm
+Version: 1.0.0
+Summary: Micro-framework for implementing data pipelines over RabbitMQ
+Requires-Python: >=3.14
+License-File: LICENSE
+Requires-Dist: aio-pika>=9.5.8
+Requires-Dist: pydantic>=2.12.5
+Dynamic: license-file

khnm-1.0.0/README.md

@@ -0,0 +1,163 @@
+# khnm
+
+A pipeline framework for building message-processing workflows on top of RabbitMQ. Define pipelines as chains of callbacks connected by queues, and run each stage independently with configurable concurrency.
+
+## Requirements
+
+- Python >= 3.14
+- RabbitMQ
+
+## Installation
+
+```bash
+pip install khnm
+```
+
+## Quick start
+
+Define your data models and callbacks, then chain them into a pipeline:
+
+```python
+import khnm
+import pydantic
+from typing import AsyncGenerator, List
+
+
+class Task(pydantic.BaseModel):
+    data: List[int]
+
+
+class Result(pydantic.BaseModel):
+    data: int
+
+
+async def generate() -> AsyncGenerator[Task, None]:
+    for i in range(100):
+        yield Task(data=[i, i + 1, i + 2])
+
+
+def split(task: Task) -> List[Result]:
+    return [Result(data=n) for n in task.data]
+
+
+def print_result(result: Result) -> None:
+    print(result.data)
+
+
+pipeline = (
+    khnm.make_pipeline()
+    .add("generator", generate, pipe_length=128)
+    .add("splitter", split, pipe_length=128, prefetch_count=1)
+    .add("printer", print_result, prefetch_count=1)
+    .build()
+)
+```
+
+Run each stage as a separate process:
+
+```bash
+export RABBITMQ_URL="amqp://guest:guest@localhost/"
+
+khnm my_app:pipeline generator
+khnm my_app:pipeline splitter
+khnm my_app:pipeline printer
+```
+
+## Pipeline structure
+
+A pipeline is an ordered chain of stages:
+
+- **Source** (first stage) -- a sync or async generator that produces messages. Runs until the generator is exhausted, then exits. It does not restart or reconnect; when the source process finishes, no further messages will be produced.
+- **Node** (middle stages) -- a sync or async function that consumes a message and returns one, many, or no output messages
+- **Sink** (last stage) -- a sync or async function that consumes messages without forwarding
+
+All messages are [Pydantic](https://docs.pydantic.dev/) `BaseModel` instances, serialized automatically between stages via RabbitMQ queues.
+
+## Callbacks
+
+Callbacks are automatically detected as sync or async. Both styles work interchangeably:
+
+```python
+# Sync
+def process(task: Task) -> Result:
+    return Result(data=task.data * 2)
+
+# Async
+async def process(task: Task) -> Result:
+    await some_io()
+    return Result(data=task.data * 2)
+```
+
+A callback can return:
+- A single `BaseModel` -- published as one message
+- A list of `BaseModel` -- each item published as a separate message
+- `None` -- nothing is published (filtering)
+
+## Concurrency
+
+### Async tasks
+
+Run multiple concurrent asyncio tasks per process with `--tasks`:
+
+```bash
+khnm my_app:pipeline splitter --tasks 4
+```
+
+Each task gets its own consumer loop. This is the primary concurrency lever for async callbacks.
+
+### Threads
+
+Run sync callbacks on a thread pool with `--threads`:
+
+```bash
+khnm my_app:pipeline splitter --threads 4
+```
+
+This offloads blocking sync callbacks to a `ThreadPoolExecutor`, preventing them from blocking the event loop. Has no effect on async callbacks.
+
+## Per-stage options
+
+Options are passed as keyword arguments to `.add()`:
+
+```python
+khnm.make_pipeline()
+    .add("source", generate, pipe_length=256, durable=True)
+    .add("worker", process,
+         pipe_length=128,
+         prefetch_count=10,
+         backoff_seconds=0.5,
+         max_retries=5,
+         exponential_backoff=True,
+         max_backoff_seconds=30.0,
+         apply_jitter=True,
+         connection_max_retries=10,
+         connection_backoff_seconds=2.0)
+    .add("sink", output, prefetch_count=1)
+    .build()
+```
+
+| Option | Applies to | Description                                                                                                                                |
+|---|---|--------------------------------------------------------------------------------------------------------------------------------------------|
+| `pipe_length` | Source, Node | Max queue length                                                                                                                           |
+| `durable` | Source, Node | Persist messages to disk                                                                                                                   |
+| `prefetch_count` | Node, Sink | RabbitMQ QoS -- max unacknowledged messages per consumer. Important! If you use multiple threads, set this to no lower than threads count! |
+| `backoff_seconds` | Source, Node | Initial backoff for publish retries                                                                                                        |
+| `max_retries` | Source, Node | Max publish retry attempts                                                                                                                 |
+| `exponential_backoff` | Source, Node | Enable exponential backoff                                                                                                                 |
+| `max_backoff_seconds` | Source, Node | Cap on backoff duration                                                                                                                    |
+| `apply_jitter` | Source, Node | Add randomness to backoff                                                                                                                  |
+| `connection_max_retries` | Node, Sink | Max connection retry attempts                                                                                                              |
+| `connection_backoff_seconds` | Node, Sink | Backoff between connection retries                                                                                                         |
+
+## CLI reference:
+
+```
+khnm <module:pipeline> <node> [--tasks N] [--threads N]
+```
+
+- `module:pipeline` -- import path to the built `Pipeline` object (e.g. `my_app:pipeline`)
+- `node` -- name of the stage to run
+- `--tasks N` -- number of concurrent async tasks (default: 1)
+- `--threads N` -- number of threads for sync callbacks (default: 1)
+
+The `RABBITMQ_URL` environment variable must be set.

khnm-1.0.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "khnm"
+version = "1.0.0"
+description = "Micro-framework for implementing data pipelines over RabbitMQ"
+requires-python = ">=3.14"
+dependencies = [
+    "aio-pika>=9.5.8",
+    "pydantic>=2.12.5",
+]
+
+[project.scripts]
+khnm = "khnm.main:main"
+
+[dependency-groups]
+dev = [
+    "httpx>=0.28.1",
+    "pika>=1.3.2",
+    "pre-commit>=4.5.1",
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.3.0",
+    "ruff>=0.14.14",
+    "testcontainers>=4.14.0",
+    "ty>=0.0.13",
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "session"
+asyncio_default_test_loop_scope = "session"
+filterwarnings = [
+    "ignore::DeprecationWarning:testcontainers.*",
+]
+pythonpath = ["src"]
+
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/khnm"]

khnm-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

khnm-1.0.0/src/khnm/__init__.py

@@ -0,0 +1,5 @@
+from khnm.pipelines import make_pipeline
+
+__all__ = [
+    "make_pipeline",
+]

khnm-1.0.0/src/khnm/consumers.py

@@ -0,0 +1,79 @@
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager, AbstractAsyncContextManager
+from typing import Optional, AsyncIterator
+
+from aio_pika.abc import (
+    AbstractRobustConnection,
+    AbstractChannel,
+    AbstractIncomingMessage,
+)
+
+from khnm.exceptions import UpstreamPipeUnavailable
+from khnm.pipes import get_queue, wait_for_pipe, get_queue_name
+from khnm.time import Clock, UtcClock
+from khnm.types import QueueGetterT
+
+
+async def consume(
+    amqp_connection: AbstractRobustConnection,
+    upstream_pipe: str,
+    upstream_connection_max_retries: Optional[int] = None,
+    upstream_connection_backoff_seconds: float = 1.0,
+    upstream_queue_getter: QueueGetterT = get_queue,
+    prefetch_count: Optional[int] = None,
+    clock: Clock = UtcClock(),
+) -> AsyncGenerator[AbstractAsyncContextManager[AbstractIncomingMessage], None]:
+    channel = await _connect(
+        connection=amqp_connection,
+        upstream_pipe=upstream_pipe,
+        upstream_connection_backoff_seconds=upstream_connection_backoff_seconds,
+        upstream_connection_max_retries=upstream_connection_max_retries,
+        prefetch_count=prefetch_count,
+        queue_getter=upstream_queue_getter,
+        clock=clock,
+    )
+    try:
+        queue = await upstream_queue_getter(channel, get_queue_name(upstream_pipe))
+        async with queue.iterator() as queue_iter:
+            async for message in queue_iter:
+                yield _handle_message(message)
+    finally:
+        await channel.close()
+
+
+async def _connect(
+    connection: AbstractRobustConnection,
+    upstream_pipe: str,
+    upstream_connection_backoff_seconds: float = 1.0,
+    upstream_connection_max_retries: Optional[int] = None,
+    prefetch_count: Optional[int] = None,
+    queue_getter: QueueGetterT = get_queue,
+    clock: Clock = UtcClock(),
+) -> AbstractChannel:
+    channel = await connection.channel()
+    if prefetch_count is not None:
+        await channel.set_qos(prefetch_count=prefetch_count)
+    success = await wait_for_pipe(
+        channel,
+        upstream_pipe,
+        upstream_connection_backoff_seconds,
+        upstream_connection_max_retries,
+        clock,
+        queue_getter,
+    )
+    if not success:
+        raise UpstreamPipeUnavailable(f"Upstream unavailable: {upstream_pipe}")
+    return channel
+
+
+@asynccontextmanager
+async def _handle_message(
+    message: AbstractIncomingMessage,
+) -> AsyncIterator[AbstractIncomingMessage]:
+    try:
+        yield message
+    except Exception:
+        await message.nack(requeue=True)
+        raise
+    else:
+        await message.ack()

khnm-1.0.0/src/khnm/exceptions.py

@@ -0,0 +1,10 @@
+class UpstreamPipeUnavailable(Exception):
+    pass
+
+
+class NodeKwargsInvalid(Exception):
+    pass
+
+
+class PipelineDefinitionInvalid(Exception):
+    pass

khnm-1.0.0/src/khnm/main.py

@@ -0,0 +1,61 @@
+import argparse
+import asyncio
+import importlib
+import os
+
+from aio_pika import connect_robust
+
+from khnm.pipelines import Pipeline
+
+
+def load_pipeline_object(import_string: str) -> Pipeline:
+    module_name, attribute_name = import_string.split(":", 1)
+    module = importlib.import_module(module_name)
+    pipeline = getattr(module, attribute_name)
+    return pipeline
+
+
+async def run_node(
+    pipeline: Pipeline,
+    node_name: str,
+    connection_string: str,
+    tasks: int = 1,
+    threads: int = 1,
+) -> None:
+    connection = await connect_robust(connection_string)
+    await asyncio.gather(
+        *[pipeline.run(connection, node_name, threads) for _ in range(tasks)]
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "pipeline",
+        help="Where to look for pipeline object, in format 'module:attribute' (e.g. 'main:pipeline')",
+    )
+    parser.add_argument("node", help="Node name to run")
+    parser.add_argument(
+        "--tasks",
+        type=int,
+        default=1,
+        help="Number of concurrent async tasks to run",
+    )
+    parser.add_argument(
+        "--threads",
+        type=int,
+        default=1,
+        help="Number of concurrent threads to run (sync callbacks only)",
+    )
+    args = parser.parse_args()
+
+    conn_string = os.getenv("RABBITMQ_URL")
+    if not conn_string:
+        raise SystemExit("RABBITMQ_URL not set")
+    if args.tasks < 1:
+        raise SystemExit("Number of tasks must be greater than 0")
+    if args.threads < 1:
+        raise SystemExit("Number of threads must be greater than 0")
+
+    pipeline = load_pipeline_object(args.pipeline)
+    asyncio.run(run_node(pipeline, args.node, conn_string, args.tasks, args.threads))