slipstream-async 0.0.0a0__tar.gz

slipstream_async-0.0.0a0/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.1
+Name: slipstream-async
+Version: 0.0.0a0
+Summary: Streamline your stream processing.
+Author-email: Menziess <stefan_schenk@hotmail.com>
+License: MIT
+Project-URL: repository, https://github.com/Menziess/slipstream
+Project-URL: documentation, https://slipstream.readthedocs.io
+Keywords: kafka,pubsub
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: uvloop
+Provides-Extra: kafka
+Requires-Dist: aiokafka; extra == "kafka"
+Provides-Extra: cache
+Requires-Dist: rocksdict; extra == "cache"
+Provides-Extra: dev
+Requires-Dist: pydocstyle; extra == "dev"
+Requires-Dist: autopep8; extra == "dev"
+Requires-Dist: pyright; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: bandit; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-forked; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
+Requires-Dist: testcontainers[kafka]; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
+Requires-Dist: sphinx-autoapi; extra == "docs"
+Requires-Dist: sphinx-autobuild; extra == "docs"
+
+[![Test Python Package](https://github.com/Menziess/slipstream/actions/workflows/python-test.yml/badge.svg)](https://github.com/Menziess/slipstream/actions/workflows/python-test.yml) [![Documentation Status](https://readthedocs.org/projects/slipstream/badge/?version=latest)](https://slipstream.readthedocs.io/en/latest/?badge=latest) [![Downloads](https://static.pepy.tech/personalized-badge/slipstream?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=downloads/month)](https://pepy.tech/project/slipstream)
+
+# Slipstream
+
+<img src="./res/logo.png" width="25%" height="25%" align="right" />
+
+Slipstream provides a data-flow model to simplify development of stateful streaming applications.
+
+```sh
+pip install slipstream
+```
+
+```py
+from asyncio import run
+
+from slipstream import handle, stream
+
+
+async def messages():
+    for emoji in '🏆📞🐟👌':
+        yield emoji
+
+
+@handle(messages(), sink=[print])
+def handle_message(msg):
+    yield f'Hello {msg}!'
+
+
+if __name__ == '__main__':
+    run(stream())
+```
+
+```sh
+Hello 🏆!
+Hello 📞!
+Hello 🐟!
+Hello 👌!
+```
+
+## Usage
+
+Async `iterables` are sources, (async) `callables` are sinks.
+
+Decorate handler functions using `handle`, then run `stream` to start processing:
+
+![demo](./res/demo.gif)
+
+Multiple sources and sinks can be provided to establish many-to-many relations between them.
+The 4 emoji's were printed using the callable `print`.
+
+## Quickstart
+
+Install `aiokafka` (latest) along with slipstream:
+
+```sh
+pip install slipstream[kafka]
+```
+
+Spin up a local Kafka broker with [docker-compose.yml](docker-compose.yml), using `localhost:29091` to connect:
+
+```sh
+docker compose up broker -d
+```
+
+Follow the docs and set up a Kafka connection: [slipstream.readthedocs.io](https://slipstream.readthedocs.io).
+
+## Features
+
+- [`slipstream.handle`](slipstream/__init__.py): bind streams (iterables) and sinks (callables) to user defined handler functions
+- [`slipstream.stream`](slipstream/__init__.py): start streaming
+- [`slipstream.Topic`](slipstream/core.py): consume from (iterable), and produce to (callable) kafka using [**aiokafka**](https://aiokafka.readthedocs.io/en/stable/index.html)
+- [`slipstream.Cache`](slipstream/caching.py): store data to disk using [**rocksdict**](https://congyuwang.github.io/RocksDict/rocksdict.html)
+- [`slipstream.Conf`](slipstream/core.py): set global kafka configuration (can be overridden per topic)
+- [`slipstream.codecs.JsonCodec`](slipstream/codecs.py): serialize and deserialize json messages

slipstream_async-0.0.0a0/README.md

@@ -0,0 +1,74 @@
+[![Test Python Package](https://github.com/Menziess/slipstream/actions/workflows/python-test.yml/badge.svg)](https://github.com/Menziess/slipstream/actions/workflows/python-test.yml) [![Documentation Status](https://readthedocs.org/projects/slipstream/badge/?version=latest)](https://slipstream.readthedocs.io/en/latest/?badge=latest) [![Downloads](https://static.pepy.tech/personalized-badge/slipstream?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=downloads/month)](https://pepy.tech/project/slipstream)
+
+# Slipstream
+
+<img src="./res/logo.png" width="25%" height="25%" align="right" />
+
+Slipstream provides a data-flow model to simplify development of stateful streaming applications.
+
+```sh
+pip install slipstream
+```
+
+```py
+from asyncio import run
+
+from slipstream import handle, stream
+
+
+async def messages():
+    for emoji in '🏆📞🐟👌':
+        yield emoji
+
+
+@handle(messages(), sink=[print])
+def handle_message(msg):
+    yield f'Hello {msg}!'
+
+
+if __name__ == '__main__':
+    run(stream())
+```
+
+```sh
+Hello 🏆!
+Hello 📞!
+Hello 🐟!
+Hello 👌!
+```
+
+## Usage
+
+Async `iterables` are sources, (async) `callables` are sinks.
+
+Decorate handler functions using `handle`, then run `stream` to start processing:
+
+![demo](./res/demo.gif)
+
+Multiple sources and sinks can be provided to establish many-to-many relations between them.
+The 4 emoji's were printed using the callable `print`.
+
+## Quickstart
+
+Install `aiokafka` (latest) along with slipstream:
+
+```sh
+pip install slipstream[kafka]
+```
+
+Spin up a local Kafka broker with [docker-compose.yml](docker-compose.yml), using `localhost:29091` to connect:
+
+```sh
+docker compose up broker -d
+```
+
+Follow the docs and set up a Kafka connection: [slipstream.readthedocs.io](https://slipstream.readthedocs.io).
+
+## Features
+
+- [`slipstream.handle`](slipstream/__init__.py): bind streams (iterables) and sinks (callables) to user defined handler functions
+- [`slipstream.stream`](slipstream/__init__.py): start streaming
+- [`slipstream.Topic`](slipstream/core.py): consume from (iterable), and produce to (callable) kafka using [**aiokafka**](https://aiokafka.readthedocs.io/en/stable/index.html)
+- [`slipstream.Cache`](slipstream/caching.py): store data to disk using [**rocksdict**](https://congyuwang.github.io/RocksDict/rocksdict.html)
+- [`slipstream.Conf`](slipstream/core.py): set global kafka configuration (can be overridden per topic)
+- [`slipstream.codecs.JsonCodec`](slipstream/codecs.py): serialize and deserialize json messages

slipstream_async-0.0.0a0/pyproject.toml

@@ -0,0 +1,101 @@
+[project]
+name = "slipstream-async"
+dynamic = ["version"]
+description = "Streamline your stream processing."
+authors = [{ name = "Menziess", email = "stefan_schenk@hotmail.com" }]
+readme = "README.md"
+license = { text = "MIT" }
+keywords = ["kafka", "pubsub"]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+requires-python = ">=3.10"
+dependencies = [
+  "uvloop",
+]
+
+[project.optional-dependencies]
+kafka = [
+  "aiokafka",
+]
+cache = [
+  "rocksdict",
+]
+dev = [
+  "pydocstyle",
+  "autopep8",
+  "pyright",
+  "flake8",
+  "bandit",
+  "pre-commit",
+]
+test = [
+  "pytest",
+  "pytest-cov",
+  "pytest-mock",
+  "pytest-forked",
+  "pytest-asyncio",
+  "testcontainers[kafka]",
+]
+docs = [
+  "sphinx",
+  "sphinx-rtd-theme",
+  "sphinx-autoapi",
+  "sphinx-autobuild",
+]
+
+[project.urls]
+repository = "https://github.com/Menziess/slipstream"
+documentation = "https://slipstream.readthedocs.io"
+
+[project.scripts]
+slipstream = "slipstream.__main__:main"
+
+[tool.setuptools.dynamic]
+version = {attr = "slipstream.__version__.VERSION"}
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["slipstream*"]
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.pyright]
+venvPath = "."
+venv = ".venv"
+include = [
+  "slipstream/**/*.py",
+  "tests/**/*.py"
+]
+
+[tool.bandit]
+exclude_dirs = ["tests"]
+
+[tool.ruff]
+line-length = 79
+
+[tool.mypy]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+addopts = "--doctest-modules"
+testpaths = [
+  "slipstream",
+  "tests",
+]
+markers = [
+  "serial"
+]
+asyncio_default_fixture_loop_scope = "function"

slipstream_async-0.0.0a0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

slipstream_async-0.0.0a0/slipstream/__init__.py

@@ -0,0 +1,18 @@
+"""Top level objects."""
+
+import uvloop
+
+from slipstream.__version__ import VERSION
+from slipstream.caching import Cache
+from slipstream.core import Conf, Topic, handle, stream
+
+uvloop.install()
+
+__all__ = (
+    'VERSION',
+    'Conf',
+    'Topic',
+    'Cache',
+    'handle',
+    'stream',
+)

slipstream_async-0.0.0a0/slipstream/__version__.py

@@ -0,0 +1,5 @@
+"""Defines the package version."""
+
+from os import getenv
+
+VERSION = getenv('PACKAGE_VERSION', '0.0.dev0')

slipstream_async-0.0.0a0/slipstream/caching.py

@@ -0,0 +1,308 @@
+"""Slipstream caching."""
+
+import os
+from contextlib import contextmanager
+from threading import RLock
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from rocksdict import (
+    AccessType,
+    ColumnFamily,
+    CompactOptions,
+    DBCompactionStyle,
+    DBCompressionType,
+    FifoCompactOptions,
+    IngestExternalFileOptions,
+    Options,
+    Rdict,
+    RdictIter,
+    ReadOptions,
+    Snapshot,
+    WriteOptions,
+)
+from rocksdict.rocksdict import RdictItems, RdictKeys, RdictValues
+
+MB, MINUTES = 1024 * 1024, 60
+
+
+class Cache:
+    """Create a RocksDB database in the specified folder.
+
+    >>> cache = Cache('db/mycache')            # doctest: +SKIP
+
+    The cache instance acts as a callable to store data:
+
+    >>> cache('key', {'msg': 'Hello World!'})  # doctest: +SKIP
+    >>> cache['key']                           # doctest: +SKIP
+    {'msg': 'Hello World!'}
+    """
+
+    def __init__(
+        self,
+        path: str,
+        options: Union[Options, None] = None,
+        column_families: Union[Dict[str, Options], None] = None,
+        access_type=AccessType.read_write(),
+        target_table_size=25 * MB,
+        number_of_locks=16
+    ) -> None:
+        """Create instance that holds rocksdb reference.
+
+        This configuration setup optimizes for low disk usage (25mb per table).
+        The oldest records may be removed during compaction.
+
+        https://congyuwang.github.io/RocksDict/rocksdict.html
+        """
+        self.name = path
+        self._number_of_locks = number_of_locks
+        self._locks = [RLock() for _ in range(self._number_of_locks)]
+        options = options or self._default_options(target_table_size)
+        column_families = column_families or {
+            key: options
+            for key in Rdict.list_cf(path, options)
+        } if os.path.exists(path + '/CURRENT') else {}
+        self.db = Rdict(path, options, column_families, access_type)
+
+    @staticmethod
+    def _default_options(target_table_size: int):
+        options = Options()
+        compaction_options = FifoCompactOptions()
+        compaction_options.max_table_files_size = target_table_size
+        options.create_if_missing(True)
+        options.set_max_background_jobs(os.cpu_count() or 2)
+        options.increase_parallelism(os.cpu_count() or 2)
+        options.set_log_file_time_to_roll(30 * MINUTES)
+        options.set_keep_log_file_num(1)
+        options.set_max_log_file_size(int(0.1 * MB))
+        options.set_max_manifest_file_size(MB)
+        options.set_fifo_compaction_options(compaction_options)
+        options.set_compaction_style(DBCompactionStyle.fifo())
+        options.set_level_zero_file_num_compaction_trigger(4)
+        options.set_level_zero_slowdown_writes_trigger(6)
+        options.set_level_zero_stop_writes_trigger(8)
+        options.set_max_write_buffer_number(2)
+        options.set_write_buffer_size(1 * MB)
+        options.set_target_file_size_base(256 * MB)
+        options.set_max_bytes_for_level_base(1024 * MB)
+        options.set_max_bytes_for_level_multiplier(4.0)
+        options.set_compression_type(DBCompressionType.lz4())
+        options.set_delete_obsolete_files_period_micros(10 * 1000)
+        return options
+
+    @contextmanager
+    def _get_lock(self, key):
+        """Get lock from a pool of locks based on key."""
+        index = hash(key) % self._number_of_locks
+        with (lock := self._locks[index]):
+            yield lock
+
+    def __call__(self, key, val, *args) -> None:
+        """Call cache to set item."""
+        self.__setitem__(key, val)
+
+    def __contains__(self, key) -> bool:
+        """Key exists in db."""
+        return key in self.db
+
+    def __delitem__(self, key) -> None:
+        """Delete item from db."""
+        del self.db[key]
+
+    def __getitem__(self, key) -> Any:
+        """Get item from db or None."""
+        try:
+            return self.db[key]
+        except KeyError:
+            pass
+
+    def __setitem__(self, key, val) -> None:
+        """Set item in db."""
+        with self._get_lock(key):
+            self.db[key] = val
+
+    def __enter__(self) -> 'Cache':
+        """Contextmanager."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """Exit contextmanager."""
+        self.close()
+
+    def set_dumps(self, dumps: Callable[[Any], bytes]) -> None:
+        """Set custom dumps function."""
+        return self.db.set_dumps(dumps)
+
+    def set_loads(self, dumps: Callable[[bytes], Any]) -> None:
+        """Set custom loads function."""
+        return self.db.set_loads(dumps)
+
+    def set_read_options(self, read_opt: ReadOptions) -> None:
+        """Set custom read options."""
+        return self.db.set_read_options(read_opt)
+
+    def set_write_options(self, write_opt: WriteOptions) -> None:
+        """Set custom write options."""
+        return self.db.set_write_options(write_opt)
+
+    @contextmanager
+    def transaction(self, key) -> Any:
+        """Lock the db entry while using the context manager."""
+        with self._get_lock(key):
+            yield self
+
+    def get(
+        self,
+        key: Union[str, int, float, bytes, bool, List[
+            Union[str, int, float, bytes, bool]]],
+        default: Any = None,
+        read_opt: Union[ReadOptions, None] = None
+    ) -> Optional[Any]:
+        """Get item from database by key."""
+        return self.db.get(key, default, read_opt)
+
+    def put(
+        self,
+        key: Union[str, int, float, bytes, bool],
+        value: Any,
+        write_opt: Union[WriteOptions, None] = None
+    ) -> None:
+        """Put item in database using key."""
+        with self._get_lock(key):
+            return self.db.put(key, value, write_opt)
+
+    def delete(
+        self,
+        key: Union[str, int, float, bytes, bool],
+        write_opt: Union[WriteOptions, None] = None
+    ) -> None:
+        """Delete item from database."""
+        return self.db.delete(key, write_opt)
+
+    def key_may_exist(
+        self,
+        key: Union[str, int, float, bytes, bool],
+        fetch: bool = False,
+        read_opt=None
+    ) -> Union[bool, Tuple[bool, Any]]:
+        """Check if a key exist without performing IO operations."""
+        return self.db.key_may_exist(key, fetch, read_opt)
+
+    def iter(self, read_opt: Union[ReadOptions, None] = None) -> RdictIter:
+        """Get iterable."""
+        return self.db.iter(read_opt)
+
+    def items(
+        self,
+        backwards: bool = False,
+        from_key: Union[str, int, float, bytes, bool, None] = None,
+        read_opt: Union[ReadOptions, None] = None
+    ) -> RdictItems:
+        """Get tuples of key-value pairs."""
+        return self.db.items(backwards, from_key, read_opt)
+
+    def keys(
+        self,
+        backwards: bool = False,
+        from_key: Union[str, int, float, bytes, bool, None] = None,
+        read_opt: Union[ReadOptions, None] = None
+    ) -> RdictKeys:
+        """Get keys."""
+        return self.db.keys(backwards, from_key, read_opt)
+
+    def values(
+        self,
+        backwards: bool = False,
+        from_key: Union[str, int, float, bytes, bool, None] = None,
+        read_opt: Union[ReadOptions, None] = None
+    ) -> RdictValues:
+        """Get values."""
+        return self.db.values(backwards, from_key, read_opt)
+
+    def ingest_external_file(
+        self,
+        paths: List[str],
+        opts: IngestExternalFileOptions = IngestExternalFileOptions()
+    ) -> None:
+        """Load list of SST files into current column family."""
+        return self.db.ingest_external_file(paths, opts)
+
+    def get_column_family(self, name: str) -> Rdict:
+        """Get column family by name."""
+        return self.db.get_column_family(name)
+
+    def get_column_family_handle(self, name: str) -> ColumnFamily:
+        """Get column family handle by name."""
+        return self.db.get_column_family_handle(name)
+
+    def drop_column_family(self, name: str) -> None:
+        """Drop column family by name."""
+        return self.db.drop_column_family(name)
+
+    def create_column_family(
+        self,
+        name: str,
+        options: Options = Options()
+    ) -> Rdict:
+        """Craete column family."""
+        return self.db.create_column_family(name, options)
+
+    def delete_range(
+        self,
+        begin: Union[str, int, float, bytes, bool],
+        end: Union[str, int, float, bytes, bool],
+        write_opt: Union[WriteOptions, None] = None
+    ) -> None:
+        """Delete database items, excluding end."""
+        return self.db.delete_range(begin, end, write_opt)
+
+    def snapshot(self) -> Snapshot:
+        """Create snapshot of current column family."""
+        return self.db.snapshot()
+
+    def path(self) -> str:
+        """Get current database path."""
+        return self.db.path()
+
+    def set_options(self, options: Dict[str, str]) -> None:
+        """Set options for current column family."""
+        return self.db.set_options(options)
+
+    def property_value(self, name: str) -> Union[str, None]:
+        """Get property by name from current column family."""
+        return self.db.property_value(name)
+
+    def property_int_value(self, name: str) -> Union[int, None]:
+        """Get property as int by name from current column family."""
+        return self.db.property_int_value(name)
+
+    def latest_sequence_number(self) -> int:
+        """Get sequence number of the most recent transaction."""
+        return self.db.latest_sequence_number()
+
+    def live_files(self) -> List[Dict[str, Any]]:
+        """Get list of all table files with their level, start- and end key."""
+        return self.db.live_files()
+
+    def compact_range(
+        self, begin: Union[str, int, float, bytes, bool, None],
+        end: Union[str, int, float, bytes, bool, None],
+        compact_opt: CompactOptions = CompactOptions()
+    ) -> None:
+        """Run manual compaction on range for the current column family."""
+        return self.db.compact_range(begin, end, compact_opt)
+
+    def close(self) -> None:
+        """Flush memory to disk, and drop the current column family."""
+        return self.db.close()
+
+    def flush(self, wait: bool = True) -> None:
+        """Manually flush the current column family."""
+        return self.db.flush(wait)
+
+    def flush_wal(self, sync: bool = True) -> None:
+        """Manually flush the WAL buffer."""
+        return self.db.flush_wal(sync)
+
+    def destroy(self, options: Options = Options()) -> None:
+        """Delete the database."""
+        return Rdict.destroy(self.name, options)

slipstream_async-0.0.0a0/slipstream/codecs.py

@@ -0,0 +1,34 @@
+"""Slipstream codecs."""
+
+import logging
+from abc import ABCMeta, abstractmethod
+from json import dumps, loads
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class ICodec(metaclass=ABCMeta):
+    """Base class for codecs."""
+
+    @abstractmethod
+    def encode(self, obj: Any) -> bytes:
+        """Serialize object."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def decode(self, s: bytes) -> object:
+        """Deserialize object."""
+        raise NotImplementedError
+
+
+class JsonCodec(ICodec):
+    """Serialize/deserialize json messages."""
+
+    def encode(self, obj: Any) -> bytes:
+        """Serialize message."""
+        return dumps(obj, default=str).encode()
+
+    def decode(self, s: bytes) -> object:
+        """Deserialize message."""
+        return loads(s.decode())

slipstream_async-0.0.0a0/slipstream/core.py

@@ -0,0 +1,340 @@
+"""Core module."""
+
+import logging
+from asyncio import gather, sleep
+from collections.abc import AsyncIterable
+from inspect import signature
+from re import sub
+from typing import (
+    Any,
+    AsyncIterator,
+    Awaitable,
+    Callable,
+    Generator,
+    Iterable,
+    Optional,
+    Union,
+)
+
+from aiokafka import (
+    AIOKafkaClient,
+    AIOKafkaConsumer,
+    AIOKafkaProducer,
+    ConsumerRecord,
+)
+
+from slipstream.caching import Cache
+from slipstream.codecs import ICodec
+from slipstream.utils import Singleton, get_params_names, iscoroutinecallable
+
+KAFKA_CLASSES_PARAMS = {
+    **get_params_names(AIOKafkaConsumer),
+    **get_params_names(AIOKafkaProducer),
+    **get_params_names(AIOKafkaClient),
+}
+READ_FROM_START = -2
+READ_FROM_END = -1
+
+logger = logging.getLogger(__name__)
+
+
+class Conf(metaclass=Singleton):
+    """Define default kafka configuration, optionally.
+
+    >>> Conf({'bootstrap_servers': 'localhost:29091'})
+    {'bootstrap_servers': 'localhost:29091'}
+    """
+
+    topics: list['Topic'] = []
+    iterables: set[tuple[str, AsyncIterable]] = set()
+    handlers: dict[str, set[Union[
+        Callable[..., Awaitable[None]],
+        Callable[..., None]
+    ]]] = {}
+
+    def register_topic(self, topic: 'Topic'):
+        """Add topic to global conf."""
+        self.topics.append(topic)
+
+    def register_iterable(
+        self,
+        key: str,
+        it: AsyncIterable
+    ):
+        """Add iterable to global Conf."""
+        self.iterables.add((key, it))
+
+    def register_handler(
+        self,
+        key: str,
+        handler: Union[
+            Callable[..., Awaitable[None]],
+            Callable[..., None]
+        ]
+    ):
+        """Add handler to global Conf."""
+        handlers = self.handlers.get(key, set())
+        handlers.add(handler)
+        self.handlers[key] = handlers
+
+    async def _start(self, **kwargs):
+        try:
+            await gather(*[
+                self._distribute_messages(key, it, kwargs)
+                for key, it in self.iterables
+            ])
+        except KeyboardInterrupt:
+            pass
+        finally:
+            await self._shutdown()
+
+    async def _shutdown(self) -> None:
+        # When the program immediately crashes give chance for topic
+        # consumer and producer to be fully initialized before
+        # shutting them down
+        await sleep(0.05)
+        for t in self.topics:
+            await t._shutdown()
+
+    async def _distribute_messages(self, key, it, kwargs):
+        async for msg in it:
+            for h in self.handlers.get(key, []):
+                await h(msg=msg, kwargs=kwargs)  # type: ignore
+
+    def __init__(self, conf: dict = {}) -> None:
+        """Define init behavior."""
+        self.conf: dict[str, Any] = {}
+        self.__update__(conf)
+
+    def __update__(self, conf: dict = {}):
+        """Set default app configuration."""
+        self.conf = {**self.conf, **conf}
+        for key, value in conf.items():
+            key = sub('[^0-9a-zA-Z]+', '_', key)
+            setattr(self, key, value)
+
+    def __repr__(self) -> str:
+        """Represent config."""
+        return str(self.conf)
+
+
+class Topic:
+    """Act as a consumer and producer.
+
+    >>> topic = Topic('emoji', {
+    ...     'bootstrap_servers': 'localhost:29091',
+    ...     'auto_offset_reset': 'earliest',
+    ...     'group_id': 'demo',
+    ... })
+
+    Loop over topic (iterable) to consume from it:
+
+    >>> async for msg in topic:               # doctest: +SKIP
+    ...     print(msg.value)
+
+    Call topic (callable) with data to produce to it:
+
+    >>> await topic({'msg': 'Hello World!'})  # doctest: +SKIP
+    """
+
+    def __init__(
+        self,
+        name: str,
+        conf: dict = {},
+        offset: Optional[int] = None,
+        codec: Optional[ICodec] = None,
+    ):
+        """Create topic instance to produce and consume messages."""
+        c = Conf()
+        c.register_topic(self)
+        self.name = name
+        self.conf = {**c.conf, **conf}
+        self.starting_offset = offset
+        self.codec = codec
+
+        self.consumer: Optional[AIOKafkaConsumer] = None
+        self.producer: Optional[AIOKafkaProducer] = None
+
+        if diff := set(self.conf).difference(KAFKA_CLASSES_PARAMS):
+            logger.warning(
+                f'Unexpected Topic {self.name} conf entries: {",".join(diff)}')
+
+    @property
+    async def admin(self) -> AIOKafkaClient:
+        """Get started instance of Kafka admin client."""
+        params = get_params_names(AIOKafkaClient)
+        return AIOKafkaClient(**{
+            k: v
+            for k, v in self.conf.items()
+            if k in params
+        })
+
+    async def get_consumer(self):
+        """Get started instance of Kafka consumer."""
+        params = get_params_names(AIOKafkaConsumer)
+        if self.codec:
+            self.conf['value_deserializer'] = self.codec.decode
+        consumer = AIOKafkaConsumer(self.name, **{
+            k: v
+            for k, v in self.conf.items()
+            if k in params
+        })
+        await consumer.start()
+        return consumer
+
+    async def get_producer(self):
+        """Get started instance of Kafka producer."""
+        params = get_params_names(AIOKafkaProducer)
+        if self.codec:
+            self.conf['value_serializer'] = self.codec.encode
+        producer = AIOKafkaProducer(**{
+            k: v
+            for k, v in self.conf.items()
+            if k in params
+        })
+        await producer.start()
+        return producer
+
+    async def __call__(self, key, value) -> None:
+        """Produce message to topic."""
+        if not self.producer:
+            self.producer = await self.get_producer()
+        if isinstance(key, str) and not self.conf.get('key_serializer'):
+            key = key.encode()
+        if isinstance(value, str) and not self.conf.get('value_serializer'):
+            value = value.encode()
+        try:
+            await self.producer.send_and_wait(
+                self.name,
+                key=key,
+                value=value,
+            )
+        except Exception as e:
+            logger.error(
+                f'Error raised while producing to Topic {self.name}: '
+                f'{e.args[0]}' if e.args else ''
+            )
+            raise
+
+    async def __aiter__(self) -> AsyncIterator[ConsumerRecord]:
+        """Iterate over messages from topic."""
+        if not self.consumer:
+            self.consumer = await self.get_consumer()
+        try:
+            async for msg in self.consumer:
+                if (
+                    isinstance(msg.key, bytes)
+                    and not self.conf.get('key_deserializer')
+                ):
+                    msg.key = msg.key.decode()
+                if (
+                    isinstance(msg.value, bytes)
+                    and not self.conf.get('value_deserializer')
+                ):
+                    msg.value = msg.value.decode()
+                yield msg
+        except Exception as e:
+            logger.error(
+                f'Error raised while consuming from Topic {self.name}: '
+                f'{e.args[0]}' if e.args else ''
+            )
+            raise
+
+    async def __next__(self):
+        """Get the next message from topic."""
+        iterator = self.__aiter__()
+        return await anext(iterator)
+
+    async def _shutdown(self):
+        """Cleanup and finalization."""
+        if self.consumer:
+            await self.consumer.stop()
+        if self.producer:
+            await self.producer.stop()
+
+
+async def _sink_output(
+    s: Union[
+        Callable[..., Awaitable[None]],
+        Callable[..., None]
+    ],
+    output: Any
+) -> None:
+    is_coroutine = iscoroutinecallable(s)
+    if isinstance(s, Cache):
+        if not isinstance(output, tuple):
+            raise ValueError('Cache sink expects: Tuple[key, val].')
+        else:
+            if isinstance(s, Cache):
+                s(*output)
+    elif isinstance(s, Topic):
+        if not isinstance(output, tuple):
+            await s(b'', output)  # type: ignore
+        else:
+            await s(*output)  # type: ignore
+    else:
+        if is_coroutine:
+            await s(output)  # type: ignore
+        else:
+            s(output)
+
+
+def handle(
+    *iterable: AsyncIterable,
+    sink: Iterable[Union[
+        Callable[..., Awaitable[None]],
+        Callable[..., None]]
+    ] = []
+):
+    """Snaps function to stream.
+
+    Ex:
+        >>> topic = Topic('demo')                 # doctest: +SKIP
+        >>> cache = Cache('state/demo')           # doctest: +SKIP
+
+        >>> @handle(topic, sink=[print, cache])   # doctest: +SKIP
+        ... def handler(msg, **kwargs):
+        ...     return msg.key, msg.value
+    """
+    c = Conf()
+
+    def _deco(f):
+        parameters = signature(f).parameters.values()
+        is_coroutine = iscoroutinecallable(f)
+
+        async def _handler(msg, kwargs={}):
+            if is_coroutine:
+                if any(p.kind == p.VAR_KEYWORD for p in parameters):
+                    output = await f(msg, **kwargs)
+                else:
+                    output = await f(msg) if parameters else await f()
+            else:
+                if any(p.kind == p.VAR_KEYWORD for p in parameters):
+                    output = f(msg, **kwargs)
+                else:
+                    output = f(msg) if parameters else f()
+
+            for val in output if isinstance(output, Generator) else [output]:
+                for s in sink:
+                    await _sink_output(s, val)
+
+        for it in iterable:
+            iterable_key = str(id(it))
+            c.register_iterable(iterable_key, it)
+            c.register_handler(iterable_key, _handler)
+        return _handler
+
+    return _deco
+
+
+def stream(**kwargs):
+    """Start the streams.
+
+    Ex:
+        >>> from asyncio import run
+        >>> args = {
+        ...     'env': 'DEV',
+        ... }
+        >>> run(stream(**args))
+    """
+    return Conf()._start(**kwargs)

slipstream_async-0.0.0a0/slipstream/utils.py

@@ -0,0 +1,42 @@
+"""Slipstream utilities."""
+
+import logging
+from asyncio import iscoroutinefunction
+from inspect import signature
+from typing import Any, Dict
+
+logger = logging.getLogger(__name__)
+
+
+def iscoroutinecallable(o: Any) -> bool:
+    """Check whether function is coroutine."""
+    return iscoroutinefunction(o) or (
+        hasattr(o, '__call__')
+        and iscoroutinefunction(o.__call__)
+    )
+
+
+def get_params_names(o: Any):
+    """Return function parameters."""
+    parameters = signature(o).parameters.values()
+    return getattr(parameters, 'mapping')
+
+
+class Singleton(type):
+    """Maintain a single instance of a class."""
+
+    _instances: Dict['Singleton', Any] = {}
+
+    def __init__(cls, name, bases, dct):
+        """Perform checks before instantiation."""
+        if '__update__' not in dct:
+            raise TypeError('Expected __update__.')
+
+    def __call__(cls, *args, **kwargs):
+        """Apply metaclass singleton action."""
+        if cls not in cls._instances:
+            cls._instances[cls] = super(
+                Singleton, cls).__call__(*args, **kwargs)
+        instance = cls._instances[cls]
+        instance.__update__(*args, **kwargs)
+        return instance

slipstream_async-0.0.0a0/slipstream_async.egg-info/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.1
+Name: slipstream-async
+Version: 0.0.0a0
+Summary: Streamline your stream processing.
+Author-email: Menziess <stefan_schenk@hotmail.com>
+License: MIT
+Project-URL: repository, https://github.com/Menziess/slipstream
+Project-URL: documentation, https://slipstream.readthedocs.io
+Keywords: kafka,pubsub
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: uvloop
+Provides-Extra: kafka
+Requires-Dist: aiokafka; extra == "kafka"
+Provides-Extra: cache
+Requires-Dist: rocksdict; extra == "cache"
+Provides-Extra: dev
+Requires-Dist: pydocstyle; extra == "dev"
+Requires-Dist: autopep8; extra == "dev"
+Requires-Dist: pyright; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: bandit; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-forked; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
+Requires-Dist: testcontainers[kafka]; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
+Requires-Dist: sphinx-autoapi; extra == "docs"
+Requires-Dist: sphinx-autobuild; extra == "docs"
+
+[![Test Python Package](https://github.com/Menziess/slipstream/actions/workflows/python-test.yml/badge.svg)](https://github.com/Menziess/slipstream/actions/workflows/python-test.yml) [![Documentation Status](https://readthedocs.org/projects/slipstream/badge/?version=latest)](https://slipstream.readthedocs.io/en/latest/?badge=latest) [![Downloads](https://static.pepy.tech/personalized-badge/slipstream?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=downloads/month)](https://pepy.tech/project/slipstream)
+
+# Slipstream
+
+<img src="./res/logo.png" width="25%" height="25%" align="right" />
+
+Slipstream provides a data-flow model to simplify development of stateful streaming applications.
+
+```sh
+pip install slipstream
+```
+
+```py
+from asyncio import run
+
+from slipstream import handle, stream
+
+
+async def messages():
+    for emoji in '🏆📞🐟👌':
+        yield emoji
+
+
+@handle(messages(), sink=[print])
+def handle_message(msg):
+    yield f'Hello {msg}!'
+
+
+if __name__ == '__main__':
+    run(stream())
+```
+
+```sh
+Hello 🏆!
+Hello 📞!
+Hello 🐟!
+Hello 👌!
+```
+
+## Usage
+
+Async `iterables` are sources, (async) `callables` are sinks.
+
+Decorate handler functions using `handle`, then run `stream` to start processing:
+
+![demo](./res/demo.gif)
+
+Multiple sources and sinks can be provided to establish many-to-many relations between them.
+The 4 emoji's were printed using the callable `print`.
+
+## Quickstart
+
+Install `aiokafka` (latest) along with slipstream:
+
+```sh
+pip install slipstream[kafka]
+```
+
+Spin up a local Kafka broker with [docker-compose.yml](docker-compose.yml), using `localhost:29091` to connect:
+
+```sh
+docker compose up broker -d
+```
+
+Follow the docs and set up a Kafka connection: [slipstream.readthedocs.io](https://slipstream.readthedocs.io).
+
+## Features
+
+- [`slipstream.handle`](slipstream/__init__.py): bind streams (iterables) and sinks (callables) to user defined handler functions
+- [`slipstream.stream`](slipstream/__init__.py): start streaming
+- [`slipstream.Topic`](slipstream/core.py): consume from (iterable), and produce to (callable) kafka using [**aiokafka**](https://aiokafka.readthedocs.io/en/stable/index.html)
+- [`slipstream.Cache`](slipstream/caching.py): store data to disk using [**rocksdict**](https://congyuwang.github.io/RocksDict/rocksdict.html)
+- [`slipstream.Conf`](slipstream/core.py): set global kafka configuration (can be overridden per topic)
+- [`slipstream.codecs.JsonCodec`](slipstream/codecs.py): serialize and deserialize json messages

slipstream_async-0.0.0a0/slipstream_async.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+pyproject.toml
+slipstream/__init__.py
+slipstream/__version__.py
+slipstream/caching.py
+slipstream/codecs.py
+slipstream/core.py
+slipstream/utils.py
+slipstream_async.egg-info/PKG-INFO
+slipstream_async.egg-info/SOURCES.txt
+slipstream_async.egg-info/dependency_links.txt
+slipstream_async.egg-info/entry_points.txt
+slipstream_async.egg-info/requires.txt
+slipstream_async.egg-info/top_level.txt
+tests/test__init__.py
+tests/test_cache.py

slipstream_async-0.0.0a0/slipstream_async.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

slipstream_async-0.0.0a0/slipstream_async.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+slipstream = slipstream.__main__:main

slipstream_async-0.0.0a0/slipstream_async.egg-info/requires.txt

@@ -0,0 +1,29 @@
+uvloop
+
+[cache]
+rocksdict
+
+[dev]
+pydocstyle
+autopep8
+pyright
+flake8
+bandit
+pre-commit
+
+[docs]
+sphinx
+sphinx-rtd-theme
+sphinx-autoapi
+sphinx-autobuild
+
+[kafka]
+aiokafka
+
+[test]
+pytest
+pytest-cov
+pytest-mock
+pytest-forked
+pytest-asyncio
+testcontainers[kafka]

slipstream_async-0.0.0a0/slipstream_async.egg-info/top_level.txt

@@ -0,0 +1 @@
+slipstream

slipstream_async-0.0.0a0/tests/test__init__.py

@@ -0,0 +1,67 @@
+from asyncio import sleep
+
+import pytest
+
+from slipstream import Conf, handle, stream
+
+
+async def async_iterable(it):
+    """Make synchonous Iterable act like AsyncIterable."""
+    for msg in it:
+        await sleep(0.01)
+        yield msg
+
+
+def test_handle():
+    """Should register iterable."""
+    Conf().iterables = set()
+
+    iterable = async_iterable(range(1))
+    iterable_key = str(id(iterable))
+    iterable_item = (iterable_key, iterable)
+
+    @handle(iterable)
+    def _(msg):
+        return msg
+
+    assert Conf().iterables == set([iterable_item])
+
+
+@pytest.mark.asyncio
+async def test_stream(mocker):
+    """Should start distributing messages for each registered iterable."""
+    Conf().iterables = set()
+    spy = mocker.spy(Conf(), '_distribute_messages')
+
+    it = async_iterable(range(1))
+    iterable_key = str(id(it))
+    Conf().register_iterable(iterable_key, it)
+
+    assert spy.call_count == 0
+    await stream()
+    assert spy.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_kwargable_function():
+    """Should try to pass kwargs to user defined handler function."""
+    my_kwargs = {
+        'my_kwarg': 'kwarg value'
+    }
+    is_kwargable = False
+    is_unkwargable = False
+
+    @handle(async_iterable(range(1)))
+    def kwargable(msg, **kwargs):
+        nonlocal is_kwargable
+        is_kwargable = kwargs == my_kwargs
+
+    @handle(async_iterable(range(1)))
+    def unkwargable(msg):
+        nonlocal is_unkwargable
+        is_unkwargable = msg == 0
+
+    await stream(**my_kwargs)
+
+    assert is_kwargable is True
+    assert is_unkwargable is True

slipstream_async-0.0.0a0/tests/test_cache.py

@@ -0,0 +1,68 @@
+from threading import Thread
+from time import sleep
+
+import pytest
+
+
+@pytest.mark.serial
+@pytest.mark.parametrize('key,val,updated', [
+    (b'123', 'a', 'b'),
+    ('123', 'b', 'c'),
+    (True, 'c', 'd'),
+    (123, 'd', 'e'),
+])
+def test_crud(key, val, updated, cache):
+    """Test create/read/update/delete."""
+    cache[key] = val
+    assert cache[key] == val
+    cache[key] = updated
+    assert cache[key] == updated
+    del cache[key]
+    assert cache[key] is None
+
+
+def test_iterability(cache):
+    """Test iterability."""
+    cache[123] = 123
+    it = cache.iter()
+    it.seek_to_first()
+
+    assert it.valid()
+    while it.valid():
+        assert it.key() == 123
+        assert it.value() == 123
+        it.next()
+
+    assert list(cache.keys()) == [123]
+    assert list(cache.values()) == [123]
+    assert list(cache.items()) == [(123, 123)]
+
+
+def test_transaction(cache):
+    """Test transaction."""
+    key, result = '123', []
+
+    def try_access_locked_cache():
+        result.append(cache[key])
+        cache[key] = 'b'
+        result.append(cache[key])
+
+    t = Thread(target=try_access_locked_cache)
+
+    with cache.transaction(key):
+        cache[key] = 'a'
+
+        # Within the transaction, we read and alter cache[key] and add
+        # its value to the result list, alterations shouldn't work
+        t.start()
+        t.join(timeout=0.01)
+        if t.is_alive():
+            result.append('Timeout')
+
+        assert result == ['a', 'Timeout']
+        assert cache[key] == 'a'
+
+    # The thread is still running here, so outside of the
+    # transaction it will eventually succeed to add 'b'
+    sleep(0.01)
+    assert cache[key] == 'b'