bunqueue-client 0.1.5__py3-none-any.whl

bunqueue/__init__.py

@@ -0,0 +1,69 @@
+"""bunqueue Python SDK — TCP client for the bunqueue job queue server.
+
+Feature parity with the TypeScript client (TCP mode): Queue, Worker,
+FlowProducer, Job, plus the full management/command surface.
+
+Usage::
+
+    from bunqueue import Queue, Worker
+
+    queue = Queue("emails", host="localhost", port=6789)
+    queue.add("send", {"to": "user@example.com"}, priority=5)
+
+    def process(job):
+        job.update_progress(50)
+        return {"sent": True}
+
+    Worker("emails", process, concurrency=10).run()
+"""
+
+import logging as _logging
+
+from .connection import Connection
+from .errors import (
+    AuthError,
+    BunqueueError,
+    CommandError,
+    CommandTimeoutError,
+    ConnectionClosedError,
+    SerializationError,
+    UnrecoverableError,
+)
+from .events import EventEmitter
+from .flow import FlowNode, FlowProducer
+from .job import Job
+from .queue import Queue
+from .simple import Bunqueue, CancellationManager, CancelSignal
+from .telemetry import TelemetryEvent, TelemetryHandler
+from .worker import Worker
+
+# Library logging convention: a NullHandler on the package logger so the SDK
+# stays silent unless the application configures logging. Internal warnings
+# (swallowed command failures, raising listeners, background-task errors) go
+# to logging.getLogger("bunqueue").
+_logging.getLogger("bunqueue").addHandler(_logging.NullHandler())
+
+__version__ = "0.1.4"
+
+__all__ = [
+    "AuthError",
+    "Bunqueue",
+    "BunqueueError",
+    "CancelSignal",
+    "CancellationManager",
+    "CommandError",
+    "CommandTimeoutError",
+    "Connection",
+    "ConnectionClosedError",
+    "EventEmitter",
+    "FlowNode",
+    "FlowProducer",
+    "Job",
+    "Queue",
+    "SerializationError",
+    "TelemetryEvent",
+    "TelemetryHandler",
+    "UnrecoverableError",
+    "Worker",
+    "__version__",
+]

bunqueue/ack_batcher.py

@@ -0,0 +1,93 @@
+"""Batches completed-job ACKs into ACKB round-trips for higher throughput.
+
+Opt-in (see the Worker ``ack_batch`` option; mirrors the TypeScript SDK's
+AckBatcher). Items flush when the buffer reaches ``max_size`` or after
+``max_delay_ms``, whichever comes first; :meth:`flush` drains the rest on
+shutdown. Each item's ``on_settled`` fires after the batch is acked (or
+errored), so the worker keeps a job "active", and its lock renewed, until
+the server has confirmed the ack.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any, Callable, List, Optional
+
+from .wire import _compact
+
+logger = logging.getLogger("bunqueue")
+
+# Called once the batch settles: the error on failure, None on ack.
+SettleCallback = Callable[[Optional[BaseException]], None]
+
+
+class AckItem:
+    """One buffered ACK: job id, lock token, result, settle callback."""
+
+    __slots__ = ("id", "token", "result", "on_settled")
+
+    def __init__(
+        self, id: str, token: str, result: Any, on_settled: SettleCallback
+    ) -> None:
+        self.id = id
+        self.token = token
+        self.result = result
+        self.on_settled = on_settled
+
+
+class AckBatcher:
+    """Thread-safe ACK buffer flushed as a single ACKB command."""
+
+    def __init__(self, connection: Any, max_size: int = 50, max_delay_ms: float = 5) -> None:
+        self._connection = connection
+        self._max_size = max(1, max_size)
+        self._max_delay_s = max_delay_ms / 1000.0
+        self._buffer: List[AckItem] = []
+        self._lock = threading.Lock()
+        self._timer: Optional[threading.Timer] = None
+
+    def add(self, item: AckItem) -> None:
+        with self._lock:
+            self._buffer.append(item)
+            full = len(self._buffer) >= self._max_size
+            if not full and self._timer is None:
+                self._timer = threading.Timer(self._max_delay_s, self.flush)
+                self._timer.daemon = True  # don't keep the process alive
+                self._timer.start()
+        if full:
+            self.flush()
+
+    def flush(self) -> None:
+        """Send the buffered ACKs as one ACKB (no-op when empty)."""
+        with self._lock:
+            if self._timer is not None:
+                self._timer.cancel()
+                self._timer = None
+            batch, self._buffer = self._buffer, []
+        if not batch:
+            return
+        # Capture the wire outcome first; settle callbacks run OUTSIDE the try
+        # so a throwing callback can neither be re-invoked with an error
+        # (double settle) nor starve the remaining items of their callback.
+        error: Optional[BaseException] = None
+        try:
+            self._connection.call(
+                _compact(
+                    {
+                        "cmd": "ACKB",
+                        "ids": [item.id for item in batch],
+                        "tokens": [item.token for item in batch],
+                        "results": [item.result for item in batch],
+                    }
+                )
+            )
+        except BaseException as exc:  # noqa: BLE001 - settle every item, always
+            error = exc
+        for item in batch:
+            try:
+                item.on_settled(error)
+            except Exception:  # noqa: BLE001
+                # A callback error (e.g. a raising listener in the worker) must
+                # never prevent the other items from settling.
+                logger.warning("ACKB settle callback for job %s raised", item.id, exc_info=True)

bunqueue/connection.py

@@ -0,0 +1,240 @@
+"""TCP connection to a bunqueue server.
+
+Wire protocol: each message is a 4-byte big-endian length prefix followed by a
+standard msgpack map. Requests carry a ``reqId`` string; the server echoes it
+back, which allows pipelining (many in-flight commands on one socket).
+
+Thread-safe: any thread may call :meth:`Connection.call`. A single daemon
+reader thread demultiplexes responses to their pending futures.
+"""
+
+from __future__ import annotations
+
+import socket
+import struct
+import threading
+from concurrent.futures import Future
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from typing import Any, Dict, Optional
+
+import msgpack
+
+from .connection_lifecycle import ConnectionLifecycle
+from .errors import (
+    CommandError,
+    CommandTimeoutError,
+    ConnectionClosedError,
+)
+from .telemetry import Telemetry, TelemetryHandler
+from .transport import encode_command
+from .wire import (
+    MAX_FRAME_SIZE,
+    PROTOCOL_VERSION,
+    TlsOption,
+    _compact,
+)
+
+__all__ = ["Connection", "TlsOption", "_compact", "PROTOCOL_VERSION", "MAX_FRAME_SIZE"]
+
+
+class Connection(ConnectionLifecycle):
+    """A single pipelined TCP connection to a bunqueue server."""
+
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 6789,
+        token: Optional[str] = None,
+        tls: TlsOption = None,
+        connect_timeout: float = 5.0,
+        command_timeout: float = 10.0,
+        on_telemetry: Optional[TelemetryHandler] = None,
+    ) -> None:
+        self.host = host
+        self.port = port
+        self.token = token
+        self.tls = tls
+        self.connect_timeout = connect_timeout
+        self.command_timeout = command_timeout
+        self._telemetry = Telemetry(on_telemetry)
+
+        self._sock: Optional[socket.socket] = None
+        self._connected = False
+        self._closed = False
+        # Reentrant: connect() holds this while sending Auth via _send(), which
+        # may itself call _teardown() on a failed send — both need the lock.
+        self._conn_lock = threading.RLock()  # serializes connect/teardown
+        self._write_lock = threading.Lock()
+        self._pending_lock = threading.Lock()
+        self._pending: Dict[str, Future] = {}
+        self._req_counter = 0
+        self._generation = 0
+        self._failed_attempts = 0
+        self._next_attempt_at = 0.0
+        # Half-open recovery (#94): after this many consecutive command timeouts
+        # the socket is presumed dead and torn down so the next call reconnects.
+        self._max_command_timeouts = 3
+        self._consecutive_timeouts = 0
+
+    # ------------------------------------------------------------- commands
+
+    def call(self, command: Dict[str, Any], timeout: Optional[float] = None) -> Dict[str, Any]:
+        """Send a command and wait for its response.
+
+        Raises :class:`CommandError` when the server answers ``ok=false``.
+        Reconnects lazily if the connection was lost.
+        """
+        if not self._connected:
+            self.connect()
+        return self._send(command, timeout)
+
+    def _send(self, command: Dict[str, Any], timeout: Optional[float] = None) -> Dict[str, Any]:
+        """Frame + write a command and await its response on the open socket.
+
+        Assumes the socket is up (used both by :meth:`call` and, during
+        :meth:`connect`, to send the initial Auth before ``_connected`` flips).
+        """
+        gen = self._generation  # snapshot: a timeout must not tear down a newer conn
+        started_at = self._telemetry.now_ms()
+        with self._pending_lock:
+            self._req_counter = (self._req_counter + 1) & 0x7FFFFFFF
+            req_id = str(self._req_counter)
+
+        try:
+            frame = encode_command(command, req_id)
+        except Exception as exc:
+            self._telemetry.emit("error", operation="serialize", error=str(exc))
+            raise
+
+        with self._pending_lock:
+            fut: Future = Future()
+            self._pending[req_id] = fut
+
+        sock = self._sock
+        try:
+            if sock is None:
+                raise OSError("socket gone")
+            with self._write_lock:
+                sock.sendall(frame)
+        except OSError as exc:
+            with self._pending_lock:
+                self._pending.pop(req_id, None)
+            self._teardown()
+            self._telemetry.emit("error", operation="write", error=str(exc))
+            raise ConnectionClosedError(f"send failed: {exc}") from exc
+
+        try:
+            response = fut.result(timeout if timeout is not None else self.command_timeout)
+        except FutureTimeoutError:
+            with self._pending_lock:
+                self._pending.pop(req_id, None)
+            self._note_timeout(gen)
+            self._telemetry.emit(
+                "command_timeout", cmd=command.get("cmd"), req_id=req_id
+            )
+            raise CommandTimeoutError(f"no response for {command.get('cmd')} within timeout") from None
+
+        self._consecutive_timeouts = 0
+        if isinstance(response, BaseException):
+            raise response
+        self._telemetry.emit(
+            "command",
+            cmd=command.get("cmd"),
+            req_id=req_id,
+            duration_ms=self._telemetry.now_ms() - started_at,
+            ok=bool(response.get("ok")),
+        )
+        if not response.get("ok"):
+            raise CommandError(str(response.get("error", "unknown server error")))
+        return response
+
+    def _note_timeout(self, gen: int) -> None:
+        """Tear down after repeated timeouts so a dead/half-open link recovers.
+
+        On a link the peer dropped without FIN/RST, every command times out
+        while the socket still looks connected. Counting consecutive timeouts
+        and forcing a teardown lets the next :meth:`call` reconnect instead of
+        wedging until the OS abandons the writes (~tcp_retries2, minutes).
+
+        ``gen`` is the connection generation at send time: a timeout from an
+        already-replaced connection must not tear down (or miscount against)
+        the current one."""
+        if gen != self._generation:
+            return
+        self._consecutive_timeouts += 1
+        if self._consecutive_timeouts >= self._max_command_timeouts:
+            self._teardown()
+
+    def ping(self) -> bool:
+        try:
+            response = self.call({"cmd": "Ping"})
+        except (ConnectionClosedError, CommandTimeoutError, CommandError):
+            return False
+        data = response.get("data")
+        return bool(isinstance(data, dict) and data.get("pong"))
+
+    def hello(self) -> Dict[str, Any]:
+        """Protocol negotiation; returns server name/version/protocolVersion."""
+        return self.call(
+            {"cmd": "Hello", "protocolVersion": PROTOCOL_VERSION, "capabilities": ["pipelining"]}
+        )
+
+    # ------------------------------------------------------------- internals
+
+    def _read_loop(self, sock: socket.socket) -> None:
+        buffer = bytearray()
+        try:
+            while True:
+                chunk = sock.recv(65536)
+                if not chunk:
+                    break
+                buffer.extend(chunk)
+                while len(buffer) >= 4:
+                    (length,) = struct.unpack_from(">I", buffer)
+                    if length > MAX_FRAME_SIZE:
+                        raise ConnectionClosedError("frame exceeds max size")
+                    if len(buffer) < 4 + length:
+                        break
+                    frame = bytes(buffer[4 : 4 + length])
+                    del buffer[: 4 + length]
+                    self._dispatch(frame)
+        except (OSError, ConnectionClosedError):
+            pass
+        finally:
+            # Only tear down if this reader's socket is still current: a stale
+            # reader from a previous connection must not kill a fresh one.
+            if self._sock is sock:
+                self._teardown()
+
+    def _dispatch(self, frame: bytes) -> None:
+        try:
+            message = msgpack.unpackb(frame, raw=False)
+        except Exception:
+            return  # unparseable frame: skip; stream desync ends via recv error
+        req_id = message.get("reqId") if isinstance(message, dict) else None
+        if req_id is None:
+            return  # server-push messages are not supported yet
+        with self._pending_lock:
+            fut = self._pending.pop(str(req_id), None)
+        if fut is not None and not fut.done():
+            fut.set_result(message)
+
+    def _teardown(self) -> None:
+        with self._conn_lock:
+            was_connected = self._connected
+            self._connected = False
+            sock, self._sock = self._sock, None
+        if sock is not None:
+            try:
+                sock.close()
+            except OSError:
+                pass
+        with self._pending_lock:
+            pending, self._pending = self._pending, {}
+        for fut in pending.values():
+            if not fut.done():
+                fut.set_result(ConnectionClosedError("connection lost"))
+        if was_connected:
+            self._telemetry.emit(
+                "disconnect", host=self.host, port=self.port, generation=self._generation
+            )

bunqueue/connection_lifecycle.py

@@ -0,0 +1,104 @@
+"""Connection establishment, authentication, backoff, and close lifecycle."""
+
+from __future__ import annotations
+
+import socket
+import threading
+import time
+
+from .errors import AuthError, CommandError, CommandTimeoutError, ConnectionClosedError
+from .transport import enable_keepalive
+from .wire import _build_ssl_context
+
+
+class ConnectionLifecycle:
+    @property
+    def generation(self) -> int:
+        """Monotonic counter bumped on every successful reconnect."""
+        return self._generation
+
+    def connect(self) -> None:
+        """Open the socket and authenticate before publishing it as connected."""
+        with self._conn_lock:
+            if self._connected:
+                return
+            if self._closed:
+                raise ConnectionClosedError("connection closed by client")
+            now = time.monotonic()
+            if now < self._next_attempt_at:
+                remaining = int((self._next_attempt_at - now) * 1000)
+                raise ConnectionClosedError(f"server unreachable, retry in {remaining}ms")
+
+            started_at = self._telemetry.now_ms()
+            try:
+                raw = socket.create_connection(
+                    (self.host, self.port), timeout=self.connect_timeout
+                )
+            except OSError as exc:
+                self._note_connect_failure()
+                self._telemetry.emit("error", operation="connect", error=str(exc))
+                raise ConnectionClosedError(f"connect failed: {exc}") from exc
+            raw.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            enable_keepalive(raw)
+            context = _build_ssl_context(self.tls, self.host)
+            if context is not None:
+                try:
+                    raw = context.wrap_socket(raw, server_hostname=self.host)
+                except OSError as exc:
+                    try:
+                        raw.close()
+                    except OSError:
+                        pass
+                    self._note_connect_failure()
+                    self._telemetry.emit("error", operation="tls", error=str(exc))
+                    raise ConnectionClosedError(f"TLS handshake failed: {exc}") from exc
+
+            self._failed_attempts = 0
+            self._next_attempt_at = 0.0
+            raw.settimeout(None)
+            self._sock = raw
+            self._generation += 1
+            self._consecutive_timeouts = 0
+            threading.Thread(target=self._read_loop, args=(raw,), daemon=True).start()
+
+            if self.token:
+                try:
+                    self._send({"cmd": "Auth", "token": self.token})
+                except CommandError as exc:
+                    self._telemetry.emit("auth", ok=False)
+                    self._teardown()
+                    raise AuthError(str(exc)) from exc
+                except (CommandTimeoutError, ConnectionClosedError) as exc:
+                    self._telemetry.emit("auth", ok=False)
+                    self._teardown()
+                    raise ConnectionClosedError(f"auth failed: {exc}") from exc
+                self._telemetry.emit("auth", ok=True)
+
+            self._connected = True
+            self._telemetry.emit(
+                "connect",
+                host=self.host,
+                port=self.port,
+                generation=self._generation,
+                duration_ms=self._telemetry.now_ms() - started_at,
+            )
+
+    def _note_connect_failure(self) -> None:
+        self._failed_attempts += 1
+        backoff = min(0.5 * (2 ** (self._failed_attempts - 1)), 5.0)
+        self._next_attempt_at = time.monotonic() + backoff
+        self._telemetry.emit(
+            "reconnect_scheduled",
+            host=self.host,
+            port=self.port,
+            attempt=self._failed_attempts,
+            delay_ms=backoff * 1000,
+        )
+
+    def close(self) -> None:
+        self._closed = True
+        self._teardown()
+
+    @property
+    def connected(self) -> bool:
+        return self._connected

bunqueue/errors.py

@@ -0,0 +1,38 @@
+"""Exception hierarchy for the bunqueue Python SDK."""
+
+
+class BunqueueError(Exception):
+    """Base class for all bunqueue SDK errors."""
+
+
+class ConnectionClosedError(BunqueueError):
+    """The TCP connection is closed or was lost mid-command."""
+
+
+class CommandTimeoutError(BunqueueError):
+    """No response received for a command within the timeout."""
+
+
+class CommandError(BunqueueError):
+    """The server answered with ok=false; message carries the server error."""
+
+
+class SerializationError(BunqueueError):
+    """A command payload cannot be encoded within the wire constraints.
+
+    Raised before anything is written when msgpack serialization fails or the
+    encoded payload exceeds the protocol frame cap. Serialization failures
+    retain the original error as ``__cause__``.
+    """
+
+
+class AuthError(BunqueueError):
+    """Authentication with the server failed."""
+
+
+class UnrecoverableError(BunqueueError):
+    """Raise inside a Worker processor to fail the job terminally.
+
+    Skips all remaining retry attempts and sends the job straight to the
+    DLQ (mirrors ``UnrecoverableError`` in the TypeScript client).
+    """

bunqueue/events.py

@@ -0,0 +1,56 @@
+"""Minimal thread-safe event emitter (mirrors Node's EventEmitter subset)."""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any, Callable, Dict, List
+
+logger = logging.getLogger("bunqueue")
+
+Listener = Callable[..., None]
+
+
+class EventEmitter:
+    """on/once/off/emit with the semantics the TS client exposes."""
+
+    def __init__(self) -> None:
+        self._listeners: Dict[str, List[Listener]] = {}
+        self._once: Dict[str, List[Listener]] = {}
+        self._lock = threading.Lock()
+
+    def on(self, event: str, listener: Listener) -> "EventEmitter":
+        with self._lock:
+            self._listeners.setdefault(event, []).append(listener)
+        return self
+
+    def once(self, event: str, listener: Listener) -> "EventEmitter":
+        with self._lock:
+            self._once.setdefault(event, []).append(listener)
+        return self
+
+    def off(self, event: str, listener: Listener) -> "EventEmitter":
+        with self._lock:
+            for registry in (self._listeners, self._once):
+                if event in registry and listener in registry[event]:
+                    registry[event].remove(listener)
+        return self
+
+    def remove_all_listeners(self, event: str = None) -> None:
+        with self._lock:
+            if event is None:
+                self._listeners.clear()
+                self._once.clear()
+            else:
+                self._listeners.pop(event, None)
+                self._once.pop(event, None)
+
+    def emit(self, event: str, *args: Any) -> None:
+        with self._lock:
+            listeners = list(self._listeners.get(event, ()))
+            once = self._once.pop(event, [])
+        for listener in listeners + once:
+            try:
+                listener(*args)
+            except Exception:  # noqa: BLE001 - listeners must not kill the emitter
+                logger.warning("listener for %r raised", event, exc_info=True)