moat-lib-stream 0.1.2__tar.gz

moat_lib_stream-0.1.2/LICENSE.txt

@@ -0,0 +1,14 @@
+The code in this repository, and all MoaT submodules it refers to,
+is part of the MoaT project.
+
+Unless a submodule's LICENSE.txt states otherwise, all included files are
+licensed under the LGPL V3, as published by the FSF at
+https://www.gnu.org/licenses/lgpl-3.0.html .
+
+In addition to the LGPL's terms, the author(s) respectfully ask all users of
+this code to contribute any bug fixes or enhancements. Also, please link back to
+https://M-o-a-T.org.
+
+Thank you.
+
+Copyright © 2021 ff.: the MoaT contributor(s), as per the git changelog(s).

moat_lib_stream-0.1.2/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: moat-lib-stream
+Version: 0.1.2
+Summary: Stream infrastructure for MoaT applications
+Maintainer-email: Matthias Urlichs <matthias@urlichs.de>
+Project-URL: homepage, https://m-o-a-t.org
+Project-URL: repository, https://github.com/M-o-a-T/moat
+Keywords: MoaT
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: AnyIO
+Classifier: Framework :: Trio
+Classifier: Framework :: AsyncIO
+Classifier: Programming Language :: Python :: 3
+Classifier: Intended Audience :: Developers
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: moat-lib-micro~=0.1.0
+Requires-Dist: moat-lib-rpc~=0.1.0
+Dynamic: license-file
+
+# Stream Infrastructure
+
+% start synopsis
+% start main
+
+This module provides base classes for handling data streams in a structured manner.
+
+% end synopsis
+
+## Overview
+
+This package implements infrastructure for building layered communication stacks:
+
+- **BaseMsg** - Message-based communication (Python objects)
+- **BaseBlk** - Block-based communication (delimited bytestrings)
+- **BaseBuf** - Buffer-based communication (undelimited bytestreams)
+
+## Layered Communication Stacks
+
+Build communication stacks by layering protocols:
+
+```python
+from moat.lib.stream import StackedBlk
+
+class Compression(StackedBlk):
+    async def snd(self, blk):
+        compressed = compress(blk)
+        await self.s.send(compressed)
+
+    async def rcv(self):
+        blk = await self.s.recv()
+        return decompress(blk)
+
+# Stack layers
+link = SerialLink(cfg)  # BaseBuf
+link = Packetizer(link, cfg)  # StackedBlk
+link = Compressor(link, cfg)  # StackedBlk
+link = Codec(link, cfg)  # StackedMsg
+async with link:
+    await link.send({"data": "hello"})
+```
+
+% end main
+
+## License
+
+This project is part of the MoaT ecosystem and is licensed under the same terms.

moat_lib_stream-0.1.2/README.md

@@ -0,0 +1,47 @@
+# Stream Infrastructure
+
+% start synopsis
+% start main
+
+This module provides base classes for handling data streams in a structured manner.
+
+% end synopsis
+
+## Overview
+
+This package implements infrastructure for building layered communication stacks:
+
+- **BaseMsg** - Message-based communication (Python objects)
+- **BaseBlk** - Block-based communication (delimited bytestrings)
+- **BaseBuf** - Buffer-based communication (undelimited bytestreams)
+
+## Layered Communication Stacks
+
+Build communication stacks by layering protocols:
+
+```python
+from moat.lib.stream import StackedBlk
+
+class Compression(StackedBlk):
+    async def snd(self, blk):
+        compressed = compress(blk)
+        await self.s.send(compressed)
+
+    async def rcv(self):
+        blk = await self.s.recv()
+        return decompress(blk)
+
+# Stack layers
+link = SerialLink(cfg)  # BaseBuf
+link = Packetizer(link, cfg)  # StackedBlk
+link = Compressor(link, cfg)  # StackedBlk
+link = Codec(link, cfg)  # StackedMsg
+async with link:
+    await link.send({"data": "hello"})
+```
+
+% end main
+
+## License
+
+This project is part of the MoaT ecosystem and is licensed under the same terms.

moat_lib_stream-0.1.2/debian/control

@@ -0,0 +1,21 @@
+Source: moat-lib-stream
+Maintainer: "Matthias Urlichs" <matthias@urlichs.de>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-all, debhelper (>= 13),
+  python3-setuptools,
+  python3-wheel,
+Standards-Version: 3.9.6
+Homepage: https://m-o-a-t.org
+X-DH-Compat: 13
+
+Package: python3-moat-lib-stream
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends},
+  python3-moat-lib-micro (>= 0.1),
+  python3-moat-lib-rpc (>= 0.1),
+Description: Stream infrastructure for MoaT applications
+  This module provides base classes for handling data streams in a
+  structured manner, including message-based (BaseMsg), block-based
+  (BaseBlk), and byte stream-based (BaseBuf) communication with
+  support for stacking and logging.

moat_lib_stream-0.1.2/debian/rules

@@ -0,0 +1,5 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=moat-lib-stream
+%:
+	dh $@ --with python3 --buildsystem=pybuild

moat_lib_stream-0.1.2/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["wheel","setuptools"]
+
+[project]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Framework :: AnyIO",
+    "Framework :: Trio",
+    "Framework :: AsyncIO",
+    "Programming Language :: Python :: 3",
+    "Intended Audience :: Developers",
+]
+dependencies = [
+    "moat-lib-micro ~= 0.1.0",
+    "moat-lib-rpc ~= 0.1.0",
+]
+keywords = ["MoaT"]
+requires-python = ">=3.8"
+name = "moat-lib-stream"
+maintainers = [{email = "matthias@urlichs.de",name = "Matthias Urlichs"}]
+description='Stream infrastructure for MoaT applications'
+readme = "README.md"
+version = "0.1.2"
+
+[project.urls]
+homepage = "https://m-o-a-t.org"
+repository = "https://github.com/M-o-a-T/moat"
+
+[tool.setuptools]
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"*" = ["*.yaml"]

moat_lib_stream-0.1.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

moat_lib_stream-0.1.2/src/moat/lib/stream/__init__.py

@@ -0,0 +1,95 @@
+"""
+Stream infrastructure for handling data streams in a structured manner.
+"""
+
+from __future__ import annotations
+
+# Import base classes directly (not lazy)
+from .base import Base as Base
+from .base import BaseBlk as BaseBlk
+from .base import BaseBuf as BaseBuf
+from .base import BaseConn as BaseConn
+from .base import BaseMsg as BaseMsg
+from .base import StackedBlk as StackedBlk
+from .base import StackedBuf as StackedBuf
+from .base import StackedConn as StackedConn
+from .base import StackedMsg as StackedMsg
+
+from typing import TYPE_CHECKING as _TC
+
+if _TC:
+    from .anyio import AnyioBuf as AnyioBuf
+    from .anyio import BufAnyio as BufAnyio
+    from .anyio import FilenoBuf as FilenoBuf
+    from .anyio import ProcessBuf as ProcessBuf
+    from .anyio import ProcessDeadError as ProcessDeadError
+    from .anyio import RemoteBufAnyio as RemoteBufAnyio
+    from .anyio import SingleAnyioBuf as SingleAnyioBuf
+    from .cbor import CBORMsgBlk as CBORMsgBlk
+    from .cbor import CBORMsgBuf as CBORMsgBuf
+    from .log import LogBlk as LogBlk
+    from .log import LogBuf as LogBuf
+    from .log import LogMsg as LogMsg
+    from .reliable import EphemeralMsg as EphemeralMsg
+    from .reliable import ReliableMsg as ReliableMsg
+    from .tcp import TcpLink as TcpLink
+    from .terminal import FilenoTerm as FilenoTerm
+    from .terminal import TermBuf as TermBuf
+    from .unix import UnixLink as UnixLink
+
+
+# Lazy loading
+_imports = {
+    # Logging
+    "LogMsg": "log",
+    "LogBlk": "log",
+    "LogBuf": "log",
+    # CBOR
+    "CBORMsgBuf": "cbor",
+    "CBORMsgBlk": "cbor",
+    # AnyIO
+    "ProcessDeadError": "anyio",
+    "AnyioBuf": "anyio",
+    "FilenoBuf": "anyio",
+    "RemoteBufAnyio": "anyio",
+    "BufAnyio": "anyio",
+    "SingleAnyioBuf": "anyio",
+    "ProcessBuf": "anyio",
+    # Reliable messaging
+    "ReliableMsg": "reliable",
+    "EphemeralMsg": "reliable",
+    # Network connections
+    "TcpLink": "tcp",
+    "UnixLink": "unix",
+    # Terminal
+    "FilenoTerm": "terminal",
+    "TermBuf": "terminal",
+}
+
+
+def __getattr__(attr: str):
+    try:
+        mod = _imports[attr]
+    except KeyError:
+        raise AttributeError(attr) from None
+    value = getattr(__import__(mod, globals(), None, True, 1), attr)
+    globals()[attr] = value
+    return value
+
+
+__all__ = [
+    # Base classes (not lazy)
+    "Base",
+    "BaseBlk",
+    "BaseBuf",
+    "BaseConn",
+    "BaseMsg",
+    "StackedBlk",
+    "StackedBuf",
+    "StackedConn",
+    "StackedMsg",
+] + list(_imports.keys())
+
+
+def __dir__():
+    return __all__

moat_lib_stream-0.1.2/src/moat/lib/stream/_cbor.py

@@ -0,0 +1,148 @@
+"""
+CBOR message encoding/decoding for stream layers.
+"""
+
+from __future__ import annotations
+
+from moat.lib.micro import AC_use, Lock, log
+from moat.lib.stream import BaseBuf, StackedMsg
+
+from ._console import _CReader
+
+from typing import TYPE_CHECKING  # isort:skip
+
+if TYPE_CHECKING:
+    from moat.lib.codec import Codec
+    from moat.util.liner import Liner
+
+    from collections.abc import Awaitable
+    from typing import Any
+
+
+class _CBORMsgBuf(StackedMsg):
+    """
+    structured messages > CBOR bytestream
+
+    Use this if your stream is reliable (TCP, USB, …) but doesn't support
+    message boundaries.
+
+    If @console is set and a prefix is used, sends data atomically.
+    Otherwise two separate write calls are used to save on message copying.
+
+    Config:
+        console (bool):
+            Flag how to handle non-framed data.
+            True: collect for crd/cwr, False: print incoming, None: ignore.
+        msg_prefix(int):
+            bytecode of prefix for messages (as opposed to console data)
+    """
+
+    cons = False
+    codec: Codec = None
+    liner: Liner = None
+
+    def __init__(self, stream: BaseBuf, cfg: dict):
+        #
+        # console: size of console buffer, 128 if True
+        # msg_prefix: int: code for start-of-packet
+        #
+        super().__init__(stream, cfg)
+        self.w_lock = Lock()
+
+        pref = cfg.get("msg_prefix")
+        if pref is not None:
+            pref = bytes((pref,))
+        self.pref = pref
+
+        cons = cfg.get("console", False)
+        if cons:
+            _CReader.__init__(self, cons)
+
+    async def setup(self):
+        await super().setup()
+        if self.cons is False:
+            from moat.util.liner import Liner  # noqa:PLC0415
+
+            self.liner = await AC_use(self, Liner())
+
+    async def cwr(self, buf):
+        if not self.cons:
+            return
+        return await self.s.wr(buf)
+
+    def crd(self, buf) -> Awaitable:
+        return _CReader.crd(self, buf)
+
+    async def send(self, msg: Any) -> None:
+        try:
+            msg = self.codec.encode(msg)
+        except Exception:
+            log("MSG:\n%r", msg)
+            raise
+        async with self.w_lock:
+            if self.pref is not None:
+                if True:  # self.cons:
+                    msg = self.pref + msg  # must be atomic
+                else:
+                    await self.s.wr(self.pref)
+            await self.s.wr(msg)
+
+    async def recv(self) -> Any:
+        """
+        Receive the next object.
+        """
+        # Pre+postcondition: the codec does not have an object in progress.
+
+        buf = bytearray(64)
+        if self.pref is None:
+            # easy case
+            while True:
+                try:
+                    r = next(self.codec)
+                except StopIteration:
+                    n = await self.s.rd(buf)
+                    self.codec.feed(memoryview(buf)[:n])
+                else:
+                    if self.cons and isinstance(r, int) and r >= 0:
+                        _CReader.cput(self, r)
+                    else:
+                        return r
+
+        while True:
+            b = bytearray(1)
+            # read until we get a prefix byte
+            if self.codec.unfeed(b) == 0:
+                n = await self.s.rd(buf)
+                self.codec.feed(memoryview(buf)[:n])
+            elif b == self.pref:
+                break
+            elif self.cons:
+                _CReader.cput(self, b[0])
+            elif self.liner is not None:
+                self.liner(b)
+
+        while True:
+            # read until we get an object
+            try:
+                return next(self.codec)
+            except StopIteration:
+                pass
+
+            n = await self.s.rd(buf)
+            self.codec.feed(memoryview(buf)[:n])
+
+
+class _CBORMsgBlk(StackedMsg):
+    """
+    structured messages > chunked bytestrings
+
+    Use this if the layer below supports byte boundaries
+    (one bytestring-ized message per call).
+    """
+
+    async def send(self, msg):
+        await self.s.snd(self.codec.encode(msg))
+
+    async def recv(self):
+        m = await self.s.rcv()
+        return self.codec.decode(m)

moat_lib_stream-0.1.2/src/moat/lib/stream/_console.py

@@ -0,0 +1,62 @@
+"""
+Console data handling for stream layers.
+"""
+
+from __future__ import annotations
+
+from moat.lib.micro import Event
+
+
+class _CReader:
+    """
+    A mix-in that processes incoming console data.
+    """
+
+    def __init__(self, cons: bool | int):
+        if cons is True:
+            try:
+                import machine  # noqa: PLC0415,F401
+            except ImportError:
+                cons = 32768
+            else:
+                cons = 240
+        self.cevt = Event()
+        self.cpos = 0
+        self.cbuf = bytearray(cons)
+        self.cons = cons
+        self.intr = 0
+
+    async def crd(self, buf: bytearray):
+        """read waiting console data"""
+        if not self.cons:
+            raise EOFError
+        if not self.cpos:
+            await self.cevt.wait()
+            self.cevt = Event()
+        n = min(len(buf), self.cpos)
+        buf[:n] = self.cbuf[:n]
+        if n < self.cpos:
+            self.cbuf[: self.cpos - n] = self.cbuf[n : self.cpos]
+            self.cpos -= n
+        else:
+            self.cpos = 0
+        return n
+
+    def cput(self, b: int):
+        """store a byte in the console buffer"""
+        if self.cpos == len(self.cbuf):
+            if len(self.cbuf) > 100:
+                bfull = b"\n?BUF\n"
+                self.cbuf[0 : len(bfull)] = bfull
+                self.cpos = len(bfull)
+            else:
+                self.cpos = 0
+        if b != 3:
+            self.intr = 0
+        elif self.intr > 2:
+            raise KeyboardInterrupt
+        else:
+            self.intr += 1
+        self.cbuf[self.cpos] = b
+        self.cpos += 1
+        self.cevt.set()

moat_lib_stream-0.1.2/src/moat/lib/stream/_serial.py

@@ -0,0 +1,72 @@
+"""
+SerialPacker protocol support for stream layers.
+"""
+
+from __future__ import annotations
+
+from moat.lib.micro import Lock
+from moat.lib.stream import BaseBuf, StackedBlk
+
+from ._console import _CReader
+
+
+class SerialPackerBlkBuf(StackedBlk):
+    """
+    chunked bytestrings > SerialPacker-ized stream
+
+    Use this (and a CBORHandler and a Reliable) if your AIO stream
+    is unreliable (TTL serial).
+    """
+
+    cons = False
+
+    def __init__(self, stream: BaseBuf, frame: dict, console: bool | int = False):
+        super().__init__(None)
+
+        from serialpacker import SerialPacker  # noqa: PLC0415
+
+        self.s = stream
+        self.p = SerialPacker(**frame)
+        self.buf = bytearray(16)
+        self.i = 0
+        self.n = 0
+        self.w_lock = Lock()
+        if console:
+            _CReader.__init__(self, console)
+
+    async def crd(self, buf):
+        if not self.cons:
+            raise EOFError
+        return await _CReader.crd(self, buf)
+
+    async def cwr(self, buf):
+        if not self.cons:
+            return
+        return await super().wr(buf)
+
+    async def recv(self):
+        while True:
+            while self.i < self.n:
+                msg = self.p.feed(self.buf[self.i])
+                self.i += 1
+                if isinstance(msg, int):
+                    if self.cons:
+                        _CReader.cput(self, msg)
+                elif msg is not None:
+                    return msg
+
+            n = await self.s.rd(self.buf)
+            if not n:
+                raise EOFError
+            self.i = 0
+            self.n = n
+
+    async def send(self, msg):
+        h, msg, t = self.p.frame(msg)
+        async with self.w_lock:
+            if not self.cons:
+                await self.s.wr(h)
+                await self.s.wr(msg)
+                await self.s.wr(t)
+            else:
+                await self.s.wr(h + msg + t)