richrelay 1.0.2__py3-none-any.whl

richrelay/__init__.py

@@ -0,0 +1,33 @@
+"""richrelay - ship ``rich`` output, logging and exceptions to syslog and friends."""
+
+from ._version import __version__
+from .buffers import (
+    Buffer,
+    MemoryBuffer,
+    RabbitMQBuffer,
+    RedisBuffer,
+    RedisCache,
+    SpoolError,
+    SQLiteBuffer,
+)
+from .event import Event
+from .formats import FORMATS, Formatter, make_formatter
+from .integrations import (
+    RelayHandler,
+    RichRelayHandler,
+    event_from_record,
+    install_excepthook,
+    patch_rich,
+)
+from .relay import Destination, Relay, get_relay, setup, shutdown, tcp, tls, udp, unix
+from .transports import TCPTransport, Transport, TransportError, UDPTransport, UnixTransport
+from .urls import parse_buffer, parse_destination
+
+__all__ = [
+    "__version__", "Event", "Relay", "Destination", "setup", "shutdown", "get_relay",
+    "udp", "tcp", "tls", "unix", "parse_destination", "parse_buffer",
+    "RelayHandler", "RichRelayHandler", "patch_rich", "install_excepthook", "event_from_record",
+    "Buffer", "MemoryBuffer", "SQLiteBuffer", "RedisBuffer", "RedisCache", "RabbitMQBuffer",
+    "SpoolError", "Formatter", "FORMATS", "make_formatter",
+    "Transport", "TransportError", "UDPTransport", "TCPTransport", "UnixTransport",
+]  # fmt: skip

richrelay/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

richrelay/_version.py

@@ -0,0 +1 @@
+__version__ = "1.0.4"

richrelay/buffers/__init__.py

@@ -0,0 +1,10 @@
+from .base import Buffer, SpoolError
+from .memory import MemoryBuffer
+from .rabbitmq_buffer import RabbitMQBuffer
+from .redis_buffer import RedisBuffer, RedisCache
+from .sqlite import SQLiteBuffer
+
+__all__ = [
+    "Buffer", "SpoolError", "MemoryBuffer", "SQLiteBuffer",
+    "RedisBuffer", "RedisCache", "RabbitMQBuffer",
+]  # fmt: skip

richrelay/buffers/base.py

@@ -0,0 +1,45 @@
+"""Buffer interface used for store-and-forward while a receiver is down."""
+
+from __future__ import annotations
+
+from typing import List
+
+
+class SpoolError(Exception):
+    """The buffer backend itself is unavailable (Redis down, broker down, disk error...)."""
+
+
+class Buffer:
+    """FIFO of already-formatted payloads.
+
+    Contract (single consumer, used from one worker thread):
+    ``peek(n)`` returns up to ``n`` oldest payloads without removing them;
+    ``discard(n)`` removes the first ``n`` of the last peek.
+    """
+
+    overflow = "drop_oldest"
+
+    def __init__(self) -> None:
+        self.dropped = 0
+        self.known_len = 0  # cheap, possibly approximate size for stats
+
+    def push(self, payload: bytes) -> bool:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def peek(self, n: int) -> List[bytes]:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def discard(self, n: int) -> None:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def __len__(self) -> int:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def close(self) -> None:
+        pass
+
+
+def check_overflow(value: str) -> str:
+    if value not in ("drop_oldest", "drop_newest"):
+        raise ValueError("overflow must be 'drop_oldest' or 'drop_newest'")
+    return value

richrelay/buffers/memory.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import threading
+from collections import deque
+from typing import Deque, List, Optional
+
+from .base import Buffer, check_overflow
+
+
+class MemoryBuffer(Buffer):
+    """Bounded in-process buffer. Lost when the process exits."""
+
+    def __init__(self, max_items: int = 10_000, max_bytes: Optional[int] = None,
+                 overflow: str = "drop_oldest") -> None:
+        super().__init__()
+        self.max_items, self.max_bytes = max_items, max_bytes
+        self.overflow = check_overflow(overflow)
+        self._d: Deque[bytes] = deque()
+        self._bytes = 0
+        self._lock = threading.Lock()
+
+    def _over(self) -> bool:
+        return len(self._d) > self.max_items or (
+            self.max_bytes is not None and self._bytes > self.max_bytes
+        )
+
+    def push(self, payload: bytes) -> bool:
+        with self._lock:
+            if self.max_bytes is not None and len(payload) > self.max_bytes:
+                self.dropped += 1
+                return False
+            if self.overflow == "drop_newest":
+                if len(self._d) >= self.max_items or (
+                    self.max_bytes is not None and self._bytes + len(payload) > self.max_bytes
+                ):
+                    self.dropped += 1
+                    return False
+            self._d.append(payload)
+            self._bytes += len(payload)
+            while self._over() and self._d:
+                self._bytes -= len(self._d.popleft())
+                self.dropped += 1
+            self.known_len = len(self._d)
+            return True
+
+    def peek(self, n: int) -> List[bytes]:
+        with self._lock:
+            out = []
+            for i, item in enumerate(self._d):
+                if i >= n:
+                    break
+                out.append(item)
+            return out
+
+    def discard(self, n: int) -> None:
+        with self._lock:
+            for _ in range(min(n, len(self._d))):
+                self._bytes -= len(self._d.popleft())
+            self.known_len = len(self._d)
+
+    def __len__(self) -> int:
+        return len(self._d)

richrelay/buffers/rabbitmq_buffer.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from typing import Any, List, Optional, Tuple
+
+from .base import Buffer, SpoolError, check_overflow
+
+
+class RabbitMQBuffer(Buffer):
+    """Durable store-and-forward queue in RabbitMQ (pika BlockingConnection).
+
+    The queue is declared durable with ``x-max-length``; messages are published persistent
+    and only acknowledged once the real receiver has taken them. If the process dies
+    mid-replay, unacknowledged messages are redelivered (at-least-once).
+
+    Note: with ``drop_oldest`` the *broker* discards the head, so ``dropped`` is not tracked.
+    Changing ``max_items``/``overflow`` for an existing queue requires deleting the queue
+    (RabbitMQ rejects redeclaration with different arguments).
+    """
+
+    def __init__(self, url: str = "amqp://guest:guest@localhost:5672/%2F", *,
+                 queue: str = "richrelay.spool", max_items: int = 100_000,
+                 overflow: str = "drop_oldest", durable: bool = True) -> None:
+        super().__init__()
+        self.url, self.queue, self.max_items, self.durable = url, queue, max_items, durable
+        self.overflow = check_overflow(overflow)
+        self._conn: Any = None
+        self._ch: Any = None
+        self._pending: List[Tuple[int, bytes]] = []  # (delivery_tag, body)
+
+    def _connect(self) -> None:
+        try:
+            import pika
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError("RabbitMQBuffer needs: pip install 'richrelay[rabbitmq]'") from exc
+        params = pika.URLParameters(self.url)
+        params.socket_timeout = 5
+        params.blocked_connection_timeout = 10
+        self._conn = pika.BlockingConnection(params)
+        self._ch = self._conn.channel()
+        self._ch.queue_declare(
+            queue=self.queue, durable=self.durable,
+            arguments={
+                "x-max-length": self.max_items,
+                "x-overflow": "drop-head" if self.overflow == "drop_oldest" else "reject-publish",
+            },
+        )
+        self._ch.confirm_delivery()
+
+    def _reset(self) -> None:
+        self._pending = []  # unacked messages return to the queue when the connection closes
+        try:
+            if self._conn is not None and self._conn.is_open:
+                self._conn.close()
+        except Exception:
+            pass
+        self._conn = self._ch = None
+
+    def _call(self, fn: Any) -> Any:
+        last: Optional[BaseException] = None
+        for _ in range(2):  # one transparent reconnect (idle connections get closed)
+            try:
+                if self._conn is None or self._conn.is_closed:
+                    self._pending = []
+                    self._connect()
+                return fn(self._ch)
+            except SpoolError:
+                raise
+            except Exception as exc:
+                last = exc
+                self._reset()
+        raise SpoolError(f"rabbitmq {self.queue}: {last}") from last
+
+    def push(self, payload: bytes) -> bool:
+        import pika
+
+        def op(ch: Any) -> bool:
+            try:
+                ch.basic_publish(
+                    exchange="", routing_key=self.queue, body=payload,
+                    properties=pika.BasicProperties(delivery_mode=2 if self.durable else 1),
+                    mandatory=False,
+                )
+                return True
+            except (pika.exceptions.NackError, pika.exceptions.UnroutableError):
+                self.dropped += 1  # reject-publish: queue full
+                return False
+
+        ok = self._call(op)
+        if ok:
+            self.known_len += 1
+        return bool(ok)
+
+    def peek(self, n: int) -> List[bytes]:
+        def op(ch: Any) -> List[bytes]:
+            while len(self._pending) < n:
+                method, _props, body = ch.basic_get(queue=self.queue, auto_ack=False)
+                if method is None:
+                    break
+                self._pending.append((method.delivery_tag, bytes(body)))
+            return [b for _t, b in self._pending[:n]]
+
+        return self._call(op)
+
+    def discard(self, n: int) -> None:
+        if n <= 0 or not self._pending:
+            return
+
+        def op(ch: Any) -> None:
+            k = min(n, len(self._pending))
+            ch.basic_ack(delivery_tag=self._pending[k - 1][0], multiple=True)
+            self._pending = self._pending[k:]
+            self.known_len = max(0, self.known_len - k)
+
+        # no retry-by-reconnect here: a reconnect invalidates delivery tags
+        try:
+            if self._ch is None:
+                return
+            op(self._ch)
+        except Exception as exc:
+            self._reset()
+            raise SpoolError(f"rabbitmq {self.queue}: ack failed ({exc}); messages will be redelivered") from exc
+
+    def __len__(self) -> int:
+        def op(ch: Any) -> int:
+            return int(ch.queue_declare(queue=self.queue, passive=True).method.message_count)
+
+        try:
+            ready = self._call(op)
+        except SpoolError:
+            raise
+        self.known_len = ready + len(self._pending)
+        return self.known_len
+
+    def close(self) -> None:
+        self._reset()

richrelay/buffers/redis_buffer.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+from .base import Buffer, SpoolError, check_overflow
+
+
+def _import_redis() -> Any:
+    try:
+        import redis
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError("RedisBuffer needs the extra: pip install 'richrelay[redis]'") from exc
+    return redis
+
+
+class RedisBuffer(Buffer):
+    """Store-and-forward queue backed by a Redis list (RPUSH / LRANGE / LTRIM).
+
+    Bounded by ``max_items`` only (Redis lists have no byte limit).
+    """
+
+    def __init__(self, url: str = "redis://localhost:6379/0", *, key: str = "richrelay:spool",
+                 max_items: int = 100_000, overflow: str = "drop_oldest",
+                 client: Any = None, socket_timeout: float = 3.0) -> None:
+        super().__init__()
+        self.url, self.key, self.max_items = url, key, max_items
+        self.overflow = check_overflow(overflow)
+        self.socket_timeout = socket_timeout
+        self._client = client
+        self._owns_client = client is None  # never close a client the caller injected
+
+    def _c(self) -> Any:
+        if self._client is None:
+            redis = _import_redis()
+            self._client = redis.Redis.from_url(
+                self.url, socket_timeout=self.socket_timeout,
+                socket_connect_timeout=self.socket_timeout,
+            )
+        return self._client
+
+    def _call(self, fn: Any) -> Any:
+        try:
+            return fn(self._c())
+        except SpoolError:
+            raise
+        except Exception as exc:  # redis.RedisError, OSError
+            raise SpoolError(f"redis {self.key}: {exc}") from exc
+
+    def push(self, payload: bytes) -> bool:
+        if self.overflow == "drop_newest":
+            if self._call(lambda c: c.llen(self.key)) >= self.max_items:
+                self.dropped += 1
+                return False
+            self.known_len = self._call(lambda c: c.rpush(self.key, payload))
+            return True
+
+        def op(c: Any) -> int:
+            pipe = c.pipeline(transaction=True)
+            pipe.rpush(self.key, payload)
+            pipe.ltrim(self.key, -self.max_items, -1)
+            return int(pipe.execute()[0])
+
+        new_len = self._call(op)
+        if new_len > self.max_items:
+            self.dropped += new_len - self.max_items
+            new_len = self.max_items
+        self.known_len = new_len
+        return True
+
+    def peek(self, n: int) -> List[bytes]:
+        return [bytes(x) for x in self._call(lambda c: c.lrange(self.key, 0, n - 1))]
+
+    def discard(self, n: int) -> None:
+        if n > 0:
+            self._call(lambda c: c.ltrim(self.key, n, -1))
+            self.known_len = max(0, self.known_len - n)
+
+    def __len__(self) -> int:
+        self.known_len = int(self._call(lambda c: c.llen(self.key)))
+        return self.known_len
+
+    def close(self) -> None:
+        if not self._owns_client:
+            return
+        c, self._client = self._client, None
+        try:
+            if c is not None:
+                c.close()
+        except Exception:
+            pass
+
+
+class RedisCache:
+    """Rolling cache of the most recent events (always written, delivery-independent).
+
+    Keeps the newest ``max_items`` payloads in a list, optionally expiring after ``ttl`` seconds.
+    Read them back with :meth:`recent`.
+    """
+
+    def __init__(self, url: str = "redis://localhost:6379/0", *, key: str = "richrelay:recent",
+                 max_items: int = 1000, ttl: Optional[int] = None, format: str = "json",
+                 client: Any = None, socket_timeout: float = 3.0) -> None:
+        self.url, self.key, self.max_items, self.ttl = url, key, max_items, ttl
+        self.format = format
+        self.socket_timeout = socket_timeout
+        self._client = client
+        self._owns_client = client is None
+
+    def _c(self) -> Any:
+        if self._client is None:
+            redis = _import_redis()
+            self._client = redis.Redis.from_url(
+                self.url, socket_timeout=self.socket_timeout,
+                socket_connect_timeout=self.socket_timeout,
+            )
+        return self._client
+
+    def put(self, payload: bytes) -> None:
+        pipe = self._c().pipeline(transaction=True)
+        pipe.lpush(self.key, payload)
+        pipe.ltrim(self.key, 0, self.max_items - 1)
+        if self.ttl:
+            pipe.expire(self.key, int(self.ttl))
+        pipe.execute()
+
+    def recent(self, n: int = 100) -> List[str]:
+        """Newest first."""
+        return [bytes(x).decode("utf-8", "replace") for x in self._c().lrange(self.key, 0, n - 1)]
+
+    def close(self) -> None:
+        if not self._owns_client:
+            return
+        c, self._client = self._client, None
+        try:
+            if c is not None:
+                c.close()
+        except Exception:
+            pass

richrelay/buffers/sqlite.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import sqlite3
+import threading
+from pathlib import Path
+from typing import List, Optional, Union
+
+from .base import Buffer, SpoolError, check_overflow
+
+
+class SQLiteBuffer(Buffer):
+    """Durable on-disk spool using only the standard library.
+
+    Survives process restarts, bounded by ``max_items`` and ``max_bytes``.
+    Use one process per database file (counters are kept in memory).
+    """
+
+    def __init__(self, path: Optional[Union[str, Path]] = None, *, name: str = "default",
+                 max_items: int = 100_000, max_bytes: Optional[int] = 64 * 1024 * 1024,
+                 overflow: str = "drop_oldest") -> None:
+        super().__init__()
+        self.path = Path(path) if path else Path.home() / ".richrelay" / f"{name}.db"
+        self.max_items, self.max_bytes = max_items, max_bytes
+        self.overflow = check_overflow(overflow)
+        self._lock = threading.RLock()
+        self._peek_ids: List[int] = []
+        try:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            self._db = sqlite3.connect(str(self.path), check_same_thread=False, timeout=10)
+            self._db.execute("PRAGMA journal_mode=WAL")
+            self._db.execute("PRAGMA synchronous=NORMAL")
+            self._db.execute(
+                "CREATE TABLE IF NOT EXISTS spool("
+                "id INTEGER PRIMARY KEY AUTOINCREMENT, payload BLOB NOT NULL)"
+            )
+            self._db.commit()
+            row = self._db.execute(
+                "SELECT COUNT(*), COALESCE(SUM(LENGTH(payload)), 0) FROM spool"
+            ).fetchone()
+        except (sqlite3.Error, OSError) as exc:
+            raise SpoolError(f"sqlite spool {self.path}: {exc}") from exc
+        self._count, self._bytes = int(row[0]), int(row[1])
+        self.known_len = self._count
+
+    def _over(self) -> bool:
+        return self._count > self.max_items or (
+            self.max_bytes is not None and self._bytes > self.max_bytes
+        )
+
+    def push(self, payload: bytes) -> bool:
+        size = len(payload)
+        with self._lock:
+            try:
+                if self.max_bytes is not None and size > self.max_bytes:
+                    self.dropped += 1
+                    return False
+                if self.overflow == "drop_newest" and (
+                    self._count >= self.max_items
+                    or (self.max_bytes is not None and self._bytes + size > self.max_bytes)
+                ):
+                    self.dropped += 1
+                    return False
+                self._db.execute("INSERT INTO spool(payload) VALUES (?)", (payload,))
+                self._count += 1
+                self._bytes += size
+                while self._over():
+                    rows = self._db.execute(
+                        "SELECT id, LENGTH(payload) FROM spool ORDER BY id LIMIT 64"
+                    ).fetchall()
+                    last = None
+                    for rid, ln in rows:
+                        if not self._over():
+                            break
+                        last = rid
+                        self._count -= 1
+                        self._bytes -= ln
+                        self.dropped += 1
+                    if last is None:
+                        break
+                    self._db.execute("DELETE FROM spool WHERE id <= ?", (last,))
+                self._db.commit()
+            except sqlite3.Error as exc:
+                raise SpoolError(f"sqlite spool {self.path}: {exc}") from exc
+            self.known_len = self._count
+            return True
+
+    def peek(self, n: int) -> List[bytes]:
+        with self._lock:
+            try:
+                rows = self._db.execute(
+                    "SELECT id, payload FROM spool ORDER BY id LIMIT ?", (n,)
+                ).fetchall()
+            except sqlite3.Error as exc:
+                raise SpoolError(f"sqlite spool {self.path}: {exc}") from exc
+            self._peek_ids = [r[0] for r in rows]
+            return [bytes(r[1]) for r in rows]
+
+    def discard(self, n: int) -> None:
+        with self._lock:
+            ids = self._peek_ids[:n]
+            if not ids:
+                return
+            last = ids[-1]
+            try:
+                cnt, size = self._db.execute(
+                    "SELECT COUNT(*), COALESCE(SUM(LENGTH(payload)), 0) FROM spool WHERE id <= ?",
+                    (last,),
+                ).fetchone()
+                self._db.execute("DELETE FROM spool WHERE id <= ?", (last,))
+                self._db.commit()
+            except sqlite3.Error as exc:
+                raise SpoolError(f"sqlite spool {self.path}: {exc}") from exc
+            self._count -= int(cnt)
+            self._bytes -= int(size)
+            self._peek_ids = self._peek_ids[n:]
+            self.known_len = self._count
+
+    def __len__(self) -> int:
+        return self._count
+
+    def close(self) -> None:
+        with self._lock:
+            try:
+                self._db.close()
+            except sqlite3.Error:
+                pass