decoded-shredstream 0.1.0__py3-none-any.whl

decoded_shredstream/__init__.py

@@ -0,0 +1,85 @@
+"""Client for the Decoded ShredStream of ShredStream.com: pre-execution Solana
+transactions, decoded from shreds.
+"""
+
+from ._client import Client
+from ._codec import (
+    BAD_FRAGMENT,
+    BAD_MAGIC,
+    FRAME_HEADER_LEN,
+    FRAME_MAGIC,
+    FRAME_VERSION,
+    MAX_DATAGRAM,
+    MSG_EVENT,
+    MSG_TRANSACTION,
+    TOO_SHORT,
+    TRUNCATED,
+    UNSUPPORTED_VERSION,
+    FrameError,
+    FrameHeader,
+    Push,
+    StreamDecoder,
+    parse_header,
+)
+from ._config import DEFAULT_GRPC_PORT, GrpcConfig, ReconnectPolicy, UdpConfig
+from ._errors import AuthRefused, ConnectError, InvalidFilter, Kicked, StreamError
+from ._filters import Filter, validate_filters
+from ._grpc import GrpcClient, normalize_endpoint
+from ._notices import (
+    DecodeError,
+    Notice,
+    Reconnected,
+    Reconnecting,
+    RecvBufferClamped,
+)
+from ._shortvec import ShortVecError, decode_len, encode_len
+from ._stats import StatsSnapshot
+from ._transaction import TransactionUpdate
+from ._udp import UdpClient
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Client",
+    "UdpClient",
+    "GrpcClient",
+    "UdpConfig",
+    "GrpcConfig",
+    "ReconnectPolicy",
+    "DEFAULT_GRPC_PORT",
+    "Filter",
+    "validate_filters",
+    "TransactionUpdate",
+    "StatsSnapshot",
+    "Notice",
+    "DecodeError",
+    "RecvBufferClamped",
+    "Reconnecting",
+    "Reconnected",
+    "StreamError",
+    "AuthRefused",
+    "Kicked",
+    "InvalidFilter",
+    "ConnectError",
+    "StreamDecoder",
+    "Push",
+    "FrameHeader",
+    "FrameError",
+    "parse_header",
+    "FRAME_MAGIC",
+    "FRAME_VERSION",
+    "FRAME_HEADER_LEN",
+    "MAX_DATAGRAM",
+    "MSG_EVENT",
+    "MSG_TRANSACTION",
+    "TOO_SHORT",
+    "BAD_MAGIC",
+    "UNSUPPORTED_VERSION",
+    "TRUNCATED",
+    "BAD_FRAGMENT",
+    "decode_len",
+    "encode_len",
+    "ShortVecError",
+    "normalize_endpoint",
+    "__version__",
+]

decoded_shredstream/_b58.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+try:  # pragma: no cover - exercised only when the extra is installed
+    import based58 as _based58
+
+    def b58encode(raw: bytes) -> str:
+        return _based58.b58encode(raw).decode("ascii")
+
+    def b58decode(text: str) -> bytes:
+        return _based58.b58decode(text.encode("ascii"))
+
+except ImportError:
+    import base58 as _base58
+
+    def b58encode(raw: bytes) -> str:
+        return _base58.b58encode(raw).decode("ascii")
+
+    def b58decode(text: str) -> bytes:
+        return _base58.b58decode(text)

decoded_shredstream/_client.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Iterator
+
+from ._config import GrpcConfig, UdpConfig
+from ._notices import NoticeCallback
+from ._stats import StatsSnapshot
+from ._transaction import TransactionUpdate
+
+if TYPE_CHECKING:  # pragma: no cover
+    from ._grpc import GrpcClient
+    from ._udp import UdpClient
+
+class Client:
+
+    __slots__ = ()
+
+    @staticmethod
+    def udp(config: UdpConfig) -> "UdpClient":
+        from ._udp import UdpClient
+
+        return UdpClient(config)
+
+    @staticmethod
+    def grpc(config: GrpcConfig) -> "GrpcClient":
+        from ._grpc import GrpcClient
+
+        return GrpcClient(config)
+
+    def __iter__(self) -> Iterator[TransactionUpdate]:
+        return self
+
+    def __next__(self) -> TransactionUpdate:  # pragma: no cover - abstract
+        raise NotImplementedError
+
+    def on_notice(self, callback: NoticeCallback) -> None:
+        raise NotImplementedError  # pragma: no cover
+
+    def stats(self) -> StatsSnapshot:
+        raise NotImplementedError  # pragma: no cover
+
+    def close(self) -> None:
+        raise NotImplementedError  # pragma: no cover
+
+    def __enter__(self) -> "Client":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()

decoded_shredstream/_codec.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import struct
+import time
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from ._notices import DecodeError, NoticeHook
+from ._stats import Stats
+from ._transaction import TransactionUpdate
+
+FRAME_MAGIC = 0x5AE7
+"""Frame magic (little-endian on the wire)."""
+FRAME_VERSION = 2
+"""Supported frame version."""
+FRAME_HEADER_LEN = 16
+"""Header length, bytes."""
+MAX_DATAGRAM = 1408
+"""Maximum datagram size emitted by the platform."""
+MSG_EVENT = 1
+"""``msg_type`` of event frames (the Shred Event Stream product; skipped
+by this client)."""
+MSG_TRANSACTION = 2
+"""``msg_type`` of decoded-transaction frames (this client's stream)."""
+
+_HEADER = struct.Struct("<HBBBBBBQ")
+_U64 = struct.Struct("<Q")
+
+_Buf = Union[bytes, bytearray, memoryview]
+
+TOO_SHORT = "too_short"
+BAD_MAGIC = "bad_magic"
+UNSUPPORTED_VERSION = "unsupported_version"
+TRUNCATED = "truncated"
+BAD_FRAGMENT = "bad_fragment"
+
+class FrameError(ValueError):
+
+    def __init__(self, code: str) -> None:
+        super().__init__(code)
+        self.code = code
+
+@dataclass(frozen=True, slots=True)
+class FrameHeader:
+    version: int
+    msg_type: int
+    flags: int
+    frag_index: int
+    frag_count: int
+    seq: int
+
+def parse_header(buf: _Buf) -> FrameHeader:
+    if len(buf) < FRAME_HEADER_LEN:
+        raise FrameError(TOO_SHORT)
+    magic, version, msg_type, flags, frag_index, frag_count, _pad, seq = _HEADER.unpack_from(
+        buf, 0
+    )
+    if magic != FRAME_MAGIC:
+        raise FrameError(BAD_MAGIC)
+    if version != FRAME_VERSION:
+        raise FrameError(UNSUPPORTED_VERSION)
+    return FrameHeader(
+        version=version,
+        msg_type=msg_type,
+        flags=flags,
+        frag_index=frag_index,
+        frag_count=frag_count,
+        seq=seq,
+    )
+
+MESSAGE = "message"
+SKIPPED = "skipped"
+ERROR = "error"
+
+class Push:
+
+    __slots__ = ("result", "transaction", "error")
+
+    def __init__(
+        self,
+        result: str,
+        transaction: Optional[TransactionUpdate] = None,
+        error: Optional[str] = None,
+    ):
+        self.result = result
+        self.transaction = transaction
+        self.error = error
+
+    def __repr__(self) -> str:
+        if self.result == MESSAGE:
+            return f"Push(message, {self.transaction!r})"
+        if self.result == ERROR:
+            return f"Push(error, {self.error})"
+        return f"Push({self.result})"
+
+_PUSH_SKIPPED = Push(SKIPPED)
+
+class StreamDecoder:
+
+    __slots__ = ("_stats", "_hook")
+
+    def __init__(self, stats: Optional[Stats] = None, hook: Optional[NoticeHook] = None) -> None:
+        self._stats = stats if stats is not None else Stats()
+        self._hook = hook if hook is not None else NoticeHook()
+
+    def stats(self):
+        return self._stats.snapshot()
+
+    def push(self, datagram: _Buf) -> Push:
+        stats = self._stats
+        stats.datagrams += 1
+        stats.bytes += len(datagram)
+
+        try:
+            header = parse_header(datagram)
+        except FrameError as e:
+            stats.decode_errors += 1
+            self._hook.fire(DecodeError())
+            return Push(ERROR, error=e.code)
+
+        if header.msg_type != MSG_TRANSACTION:
+            stats.skipped_msg_type += 1
+            return _PUSH_SKIPPED
+
+        if header.frag_count != 1:
+            stats.decode_errors += 1
+            self._hook.fire(DecodeError())
+            return Push(ERROR, error=BAD_FRAGMENT)
+
+        payload_len = len(datagram) - FRAME_HEADER_LEN
+        if payload_len < 8:
+            stats.decode_errors += 1
+            self._hook.fire(DecodeError())
+            return Push(ERROR, error=TRUNCATED)
+
+        slot = _U64.unpack_from(datagram, FRAME_HEADER_LEN)[0]
+        data = bytes(memoryview(datagram)[FRAME_HEADER_LEN + 8 :])
+        update = TransactionUpdate(slot=slot, data=data, received_at_s=time.time())
+
+        stats.messages += 1
+        stats.events += 1
+        stats.last_slot = slot
+        return Push(MESSAGE, transaction=update)

decoded_shredstream/_config.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, Optional
+
+from ._filters import Filter
+
+DEFAULT_GRPC_PORT = 9991
+"""Default gRPC port when the endpoint omits one."""
+
+@dataclass(slots=True)
+class UdpConfig:
+
+    port: int = 0
+    """Local UDP port to bind (the port registered in your dashboard)."""
+    host: str = "0.0.0.0"
+    """Local address to bind."""
+    recv_buffer_bytes: int = 64 * 2**20
+    """Requested ``SO_RCVBUF``, in bytes. Default 64 MiB; keep at least
+    8 MiB — slot boundaries produce brutal bursts. If the OS clamps the
+    value the client emits :class:`~decoded_shredstream.RecvBufferClamped`
+    (raise ``net.core.rmem_max`` on Linux, ``kern.ipc.maxsockbuf`` on
+    macOS)."""
+
+@dataclass(slots=True)
+class ReconnectPolicy:
+
+    initial: float = 0.1
+    """First retry delay, seconds."""
+    max: float = 5.0
+    """Delay cap, seconds."""
+    multiplier: float = 2.0
+    """Backoff multiplier."""
+    reset_after: float = 30.0
+    """Stable-connection duration (seconds) after which the backoff
+    resets."""
+
+@dataclass(slots=True)
+class GrpcConfig:
+
+    endpoint: str
+    """``host`` or ``host:port``. Port defaults to **9991** when
+    omitted."""
+    token: str
+    """Access token for this product."""
+    auth_style: str = "bearer"
+    """Token metadata style: ``"bearer"`` (``authorization: Bearer <t>``,
+    default) or ``"x-token"`` (``x-token: <t>``). Both are equivalent
+    server-side."""
+    filters: Dict[str, Filter] = field(default_factory=dict)
+    """Named server-side filters. Must not be empty (an empty map delivers
+    nothing) — use ``{"all": Filter()}`` to receive everything."""
+    reconnect: Optional[ReconnectPolicy] = None
+    """Reconnection policy (defaults to :class:`ReconnectPolicy`)."""
+    connect_timeout: float = 5.0
+    """Connection timeout, seconds."""

decoded_shredstream/_errors.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+class StreamError(Exception):
+    pass
+
+class AuthRefused(StreamError):
+
+    def __init__(self) -> None:
+        super().__init__("authentication refused (check your token and product)")
+
+class Kicked(StreamError):
+
+    def __init__(self) -> None:
+        super().__init__("kicked by operator — do not reconnect in a loop")
+
+class InvalidFilter(StreamError):
+
+    def __init__(self, detail: str) -> None:
+        super().__init__(f"invalid filter: {detail}")
+        self.detail = detail
+
+class ConnectError(Exception):
+    pass

decoded_shredstream/_filters.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, Sequence
+
+from ._b58 import b58decode
+from ._errors import InvalidFilter
+
+MAX_NAMED_FILTERS = 16
+"""Maximum named filters per subscription."""
+MAX_KEYS_PER_FILTER = 1000
+"""Maximum keys per filter (include + exclude + required combined)."""
+
+@dataclass(frozen=True, slots=True)
+class Filter:
+
+    include: Sequence[str] = field(default_factory=tuple)
+    """Match transactions touching AT LEAST ONE of these accounts."""
+    exclude: Sequence[str] = field(default_factory=tuple)
+    """Match transactions touching NONE of these accounts."""
+    required: Sequence[str] = field(default_factory=tuple)
+    """Match transactions touching ALL of these accounts."""
+
+    def key_count(self) -> int:
+        return len(self.include) + len(self.exclude) + len(self.required)
+
+def _validate_keys(name: str, keys: Sequence[str]) -> None:
+    for key in keys:
+        try:
+            raw = b58decode(key)
+        except (ValueError, TypeError):
+            raw = b""
+        if len(raw) != 32:
+            raise InvalidFilter(
+                f"filter {name!r}: key {key!r} is not a base58-encoded 32-byte public key"
+            )
+
+def validate_filters(filters: Dict[str, Filter]) -> None:
+    if not isinstance(filters, dict):
+        raise InvalidFilter("filters must be a dict of name -> Filter")
+    if not filters:
+        raise InvalidFilter(
+            "empty filter map delivers nothing — use Filter() to receive everything"
+        )
+    if len(filters) > MAX_NAMED_FILTERS:
+        raise InvalidFilter(f"{len(filters)} named filters (max {MAX_NAMED_FILTERS})")
+    for name, f in filters.items():
+        if not isinstance(f, Filter):
+            raise InvalidFilter(f"filter {name!r} is not a Filter")
+        count = f.key_count()
+        if count > MAX_KEYS_PER_FILTER:
+            raise InvalidFilter(
+                f"filter {name!r} has {count} keys (max {MAX_KEYS_PER_FILTER} per filter)"
+            )
+        _validate_keys(name, f.include)
+        _validate_keys(name, f.exclude)
+        _validate_keys(name, f.required)