bunqueue-client 0.1.5__py3-none-any.whl

bunqueue/simple/rate_gate.py

@@ -0,0 +1,57 @@
+"""Client-side rate limiter for Simple Mode (mirror of rate-gate.ts).
+
+Throttles job starts to ``max`` per ``duration`` ms window, optionally per
+group (``group_key`` names a field of job.data). A job whose window is full
+waits, holding its concurrency slot, until the window frees — the same
+observable semantics as the official client's worker limiter.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Dict, List
+
+
+class RateGate:
+    def __init__(self, options: Dict[str, Any]) -> None:
+        self._max = int(options["max"])
+        self._duration = float(options.get("duration") or 1000)
+        self._group_key = options.get("group_key") or options.get("groupKey")
+        self._windows: Dict[str, List[float]] = {}
+        self._lock = threading.Lock()
+
+    def group_for(self, data: Any) -> str:
+        """Resolve the rate-limit group for a job's data."""
+        if not self._group_key:
+            return ""
+        if isinstance(data, dict):
+            value = data.get(self._group_key)
+            if value is not None:
+                return str(value)
+        return ""
+
+    def acquire(self, group: str) -> None:
+        """Block until the group's window has room, then record the start."""
+        while True:
+            now = time.monotonic() * 1000
+            with self._lock:
+                self._prune_locked(now)
+                window = [t for t in self._windows.get(group, []) if now - t < self._duration]
+                if len(window) < self._max:
+                    window.append(now)
+                    self._windows[group] = window
+                    return
+                self._windows[group] = window
+                oldest = window[0]
+            time.sleep(max((oldest + self._duration - now) / 1000.0, 0.01))
+
+    def _prune_locked(self, now: float) -> None:
+        """Evict fully-expired groups (high-cardinality group_key safety)."""
+        stale = [
+            group
+            for group, window in self._windows.items()
+            if not window or now - window[-1] >= self._duration
+        ]
+        for group in stale:
+            del self._windows[group]

bunqueue/simple/retry.py

@@ -0,0 +1,65 @@
+"""Advanced in-process retry with backoff strategies (mirror of retry.ts).
+
+This retries INSIDE the processor while the job stays active (the worker's
+lock heartbeats keep it owned) — different from the server-side
+attempts/backoff, which re-queues the job.
+"""
+
+from __future__ import annotations
+
+import random
+import time
+from typing import Any, Callable, Dict, TypeVar
+
+R = TypeVar("R")
+
+STRATEGIES = ("fixed", "exponential", "jitter", "fibonacci", "custom")
+
+
+def calculate_backoff(
+    strategy: str,
+    attempt: int,
+    base_delay: float,
+    error: BaseException,
+    config: Dict[str, Any],
+) -> float:
+    """Delay in ms for the given attempt (1-based), same formulas as retry.ts."""
+    if strategy == "fixed":
+        return base_delay
+    if strategy == "exponential":
+        return base_delay * (2 ** (attempt - 1))
+    if strategy == "jitter":
+        exp = base_delay * (2 ** (attempt - 1))
+        return float(int(exp * (0.5 + random.random())))
+    if strategy == "fibonacci":
+        a, b = 1, 1
+        for _ in range(attempt - 1):
+            a, b = b, a + b
+        return base_delay * b
+    if strategy == "custom":
+        custom = config.get("custom_backoff") or config.get("customBackoff")
+        if custom:
+            return float(custom(attempt, error))
+        return base_delay
+    return base_delay
+
+
+def execute_with_retry(fn: Callable[[], R], config: Dict[str, Any]) -> R:
+    """Run ``fn`` retrying on exception, same semantics as executeWithRetry."""
+    max_attempts = int(config.get("max_attempts") or config.get("maxAttempts") or 3)
+    base_delay = float(config.get("delay") or 1000)
+    strategy = str(config.get("strategy") or "exponential")
+    retry_if = config.get("retry_if") or config.get("retryIf")
+
+    attempt = 1
+    while True:
+        try:
+            return fn()
+        except BaseException as exc:  # noqa: BLE001 - retry semantics need broad catch
+            if attempt >= max_attempts:
+                raise
+            if retry_if is not None and not retry_if(exc, attempt):
+                raise
+            delay_ms = calculate_backoff(strategy, attempt, base_delay, exc, config)
+            time.sleep(delay_ms / 1000.0)
+            attempt += 1

bunqueue/simple/triggers.py

@@ -0,0 +1,45 @@
+"""Event triggers: when a job completes/fails, create another job (mirror of
+triggers.ts)."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from ..queue import Queue
+from ..worker import Worker
+
+
+class TriggerManager:
+    def __init__(self, queue: Queue, worker: Worker) -> None:
+        self._queue = queue
+        self._worker = worker
+        self._rules: List[Dict[str, Any]] = []
+        self._active = False
+
+    def add(self, rule: Dict[str, Any]) -> None:
+        """Rule keys: on, create, data(result, job), event ('completed'
+        default | 'failed'), condition(result, job), opts (job options)."""
+        self._rules.append(rule)
+        self._ensure_active()
+
+    def _ensure_active(self) -> None:
+        if self._active:
+            return
+        self._active = True
+        self._worker.on("completed", lambda job, result: self._fire("completed", job, result))
+        self._worker.on("failed", lambda job, error: self._fire("failed", job, error))
+
+    def _fire(self, event: str, job: Any, result_or_error: Any) -> None:
+        for rule in self._rules:
+            if rule["on"] != job.name:
+                continue
+            if rule.get("event", "completed") != event:
+                continue
+            condition = rule.get("condition")
+            if condition and not condition(result_or_error, job):
+                continue
+            data = rule["data"](result_or_error, job)
+            try:
+                self._queue.add(rule["create"], data, **(rule.get("opts") or {}))
+            except Exception:  # noqa: BLE001 - trigger add is fire-and-forget
+                pass

bunqueue/simple/ttl.py

@@ -0,0 +1,34 @@
+"""Job TTL: expire unprocessed jobs at pickup (mirror of ttl.ts).
+
+Resolution order: per_name[job.name] -> default_ttl -> 0 (no TTL).
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict
+
+
+class TtlChecker:
+    def __init__(self, config: Dict[str, Any]) -> None:
+        self._default_ttl = float(
+            config.get("default_ttl") or config.get("defaultTtl") or 0
+        )
+        self._per_name: Dict[str, float] = dict(
+            config.get("per_name") or config.get("perName") or {}
+        )
+
+    def get_ttl(self, job_name: str) -> float:
+        return self._per_name.get(job_name, self._default_ttl)
+
+    def is_expired(self, job_name: str, job_timestamp_ms: float) -> bool:
+        ttl = self.get_ttl(job_name)
+        if ttl <= 0:
+            return False
+        return (time.time() * 1000) - job_timestamp_ms > ttl
+
+    def set_default_ttl(self, ttl_ms: float) -> None:
+        self._default_ttl = ttl_ms
+
+    def set_name_ttl(self, job_name: str, ttl_ms: float) -> None:
+        self._per_name[job_name] = ttl_ms

bunqueue/telemetry.py

@@ -0,0 +1,29 @@
+"""Dependency-free structured telemetry for transport and command lifecycle."""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Callable, Dict, Optional
+
+TelemetryEvent = Dict[str, Any]
+TelemetryHandler = Callable[[TelemetryEvent], None]
+
+
+class Telemetry:
+    """Safely dispatch structured events to an optional application callback."""
+
+    def __init__(self, handler: Optional[TelemetryHandler] = None) -> None:
+        self._handler = handler
+
+    @staticmethod
+    def now_ms() -> float:
+        return time.monotonic() * 1000
+
+    def emit(self, event_type: str, **fields: Any) -> None:
+        if self._handler is None:
+            return
+        try:
+            self._handler({"type": event_type, **fields})
+        except Exception:
+            # Observability must never break connection or worker correctness.
+            pass

bunqueue/transport.py

@@ -0,0 +1,50 @@
+"""Low-level socket and frame helpers for :mod:`bunqueue.connection`."""
+
+from __future__ import annotations
+
+import socket
+import struct
+from typing import Any, Dict
+
+import msgpack
+
+from .errors import SerializationError
+from .wire import MAX_FRAME_SIZE, _compact, _js_safe
+
+
+def encode_command(command: Dict[str, Any], req_id: str) -> bytes:
+    """Serialize and frame a command, enforcing the body cap before framing."""
+    try:
+        normalized = _js_safe({**_compact(command), "reqId": req_id})
+        payload = msgpack.packb(normalized, use_bin_type=True)
+    except SerializationError:
+        raise
+    except Exception as exc:
+        raise SerializationError(
+            f"cannot msgpack-serialize {command.get('cmd')} payload: {exc}"
+        ) from exc
+    if len(payload) > MAX_FRAME_SIZE:
+        raise SerializationError(
+            f"frame size {len(payload)} exceeds maximum {MAX_FRAME_SIZE}"
+        )
+    return struct.pack(">I", len(payload)) + payload
+
+
+def enable_keepalive(raw: socket.socket) -> None:
+    """Enable portable TCP keepalive probes without making support mandatory."""
+    try:
+        raw.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+    except OSError:
+        return
+    for name, value in (
+        ("TCP_KEEPIDLE", 15),
+        ("TCP_KEEPINTVL", 5),
+        ("TCP_KEEPCNT", 3),
+        ("TCP_KEEPALIVE", 15),
+    ):
+        code = getattr(socket, name, None)
+        if code is not None:
+            try:
+                raw.setsockopt(socket.IPPROTO_TCP, code, value)
+            except OSError:
+                pass

bunqueue/wire.py

@@ -0,0 +1,99 @@
+"""Wire-level helpers: payload compaction, JS-safe numbers, TLS contexts."""
+
+from __future__ import annotations
+
+import ssl
+from typing import Any, Dict, Optional, Union
+
+from .errors import SerializationError
+
+PROTOCOL_VERSION = 2
+MAX_FRAME_SIZE = 64 * 1024 * 1024  # mirror server-side limit
+
+TlsOption = Union[bool, ssl.SSLContext, Dict[str, Any], None]
+
+
+def _compact(command: Dict[str, Any]) -> Dict[str, Any]:
+    """Drop None-valued keys so the msgpack frame stays minimal."""
+    return {k: v for k, v in command.items() if v is not None}
+
+
+# msgpackr (server side) decodes int64/uint64 as BigInt, which crashes JS
+# arithmetic (e.g. `now - startedAt`). JS clients never hit this because large
+# Numbers serialize as float64. Mirror that: encode ints outside the int32
+# range as float64 (exact up to 2^53 — fine for ms timestamps).
+_INT32_MIN, _INT32_MAX = -(2**31), 2**31 - 1
+
+
+def _js_safe(
+    value: Any,
+    _active: Optional[set[int]] = None,
+    _path: str = "$",
+) -> Any:
+    """Normalize a command for msgpackr without silently changing map keys.
+
+    Only containers on the active recursion path are tracked. Reusing the same
+    list or map in two independent branches is valid, while an actual cycle is
+    rejected before msgpack serialization or socket writes.
+    """
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, int) and not (_INT32_MIN <= value <= _INT32_MAX):
+        try:
+            return float(value)
+        except OverflowError as exc:
+            raise SerializationError(f"integer at {_path} exceeds float64 range") from exc
+    if _active is None:
+        _active = set()
+    if isinstance(value, dict):
+        identity = id(value)
+        if identity in _active:
+            raise SerializationError(f"cyclic container at {_path}")
+        _active.add(identity)
+        try:
+            normalized = {}
+            for key, item in value.items():
+                if not isinstance(key, str):
+                    raise SerializationError(f"map key at {_path} must be a string")
+                normalized[key] = _js_safe(item, _active, f"{_path}.{key}")
+            return normalized
+        finally:
+            _active.remove(identity)
+    if isinstance(value, list):
+        identity = id(value)
+        if identity in _active:
+            raise SerializationError(f"cyclic container at {_path}")
+        _active.add(identity)
+        try:
+            return [_js_safe(item, _active, f"{_path}[{index}]") for index, item in enumerate(value)]
+        finally:
+            _active.remove(identity)
+    if isinstance(value, tuple):
+        identity = id(value)
+        if identity in _active:
+            raise SerializationError(f"cyclic container at {_path}")
+        _active.add(identity)
+        try:
+            return tuple(
+                _js_safe(item, _active, f"{_path}[{index}]")
+                for index, item in enumerate(value)
+            )
+        finally:
+            _active.remove(identity)
+    return value
+
+
+def _build_ssl_context(tls: TlsOption, host: str) -> Optional[ssl.SSLContext]:
+    if tls is None or tls is False:
+        return None
+    if isinstance(tls, ssl.SSLContext):
+        return tls
+    ctx = ssl.create_default_context()
+    if isinstance(tls, dict):
+        ca_file = tls.get("ca_file")
+        if ca_file:
+            ctx = ssl.create_default_context(cafile=ca_file)
+        if tls.get("verify") is False:
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+    return ctx

bunqueue/worker.py

@@ -0,0 +1,259 @@
+"""Worker: pulls jobs over TCP and runs a processor with bounded concurrency.
+
+Feature parity with the TS client's Worker (TCP mode): events (ready, active,
+completed, failed, progress, drained, error, closed), pause/resume, graceful
+close, lock heartbeats, UnrecoverableError. All queue semantics (retry,
+backoff, DLQ, stall detection, priorities) live in the bunqueue server.
+
+The loop/heartbeat internals live in :mod:`worker_runtime`.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import socket as _socket
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Callable, Dict, Optional
+
+from .ack_batcher import AckBatcher
+from .connection import Connection, TlsOption
+from .errors import CommandTimeoutError, ConnectionClosedError
+from .events import EventEmitter
+from .telemetry import TelemetryHandler
+from .job import Job
+from .worker_runtime import MAX_POLL_TIMEOUT_MS, RECONNECT_BACKOFF_S, WorkerRuntime
+
+logger = logging.getLogger("bunqueue")
+
+Processor = Callable[[Job], Any]
+
+
+class Worker(EventEmitter, WorkerRuntime):
+    """Consumes jobs from one queue and processes them.
+
+    Mirrors the TS Worker: ``autorun=True`` starts a background loop at
+    construction; use ``run()`` for a blocking loop instead.
+    """
+
+    def __init__(
+        self,
+        queue: str,
+        processor: Processor,
+        *,
+        host: str = "localhost",
+        port: int = 6789,
+        token: Optional[str] = None,
+        tls: TlsOption = None,
+        concurrency: int = 4,
+        batch_size: int = 10,
+        poll_timeout_ms: int = 5000,
+        lock_ttl_ms: int = 30000,
+        heartbeat_interval_s: float = 10.0,
+        autorun: bool = True,
+        name: Optional[str] = None,
+        ack_batch: Optional[Dict[str, Any]] = None,
+        on_telemetry: Optional[TelemetryHandler] = None,
+    ) -> None:
+        super().__init__()
+        if concurrency < 1:
+            raise ValueError("concurrency must be >= 1")
+        self.queue = queue
+        self.processor = processor
+        self.concurrency = concurrency
+        # The server rejects PULLB count > 1000 — an unclamped batch_size
+        # would wedge the poll loop in a permanent error cycle.
+        self.batch_size = max(1, min(batch_size, 1000))
+        self.poll_timeout_ms = min(poll_timeout_ms, MAX_POLL_TIMEOUT_MS)
+        self.lock_ttl_ms = lock_ttl_ms
+        self.heartbeat_interval_s = heartbeat_interval_s
+        self.worker_id = f"py-{_socket.gethostname()}-{os.getpid()}-{uuid.uuid4().hex[:8]}"
+        self.name = name or self.worker_id
+
+        self.connection = Connection(
+            host=host, port=port, token=token, tls=tls, on_telemetry=on_telemetry
+        )
+
+        # Opt-in ACKB batching, ack_batch={"max_size": 50, "max_delay_ms": 5}
+        # (TS parity); the dict is the opt-in, {"enabled": False} disables.
+        self._ack_batcher: Optional[AckBatcher] = None
+        if ack_batch and ack_batch.get("enabled", True):
+            self._ack_batcher = AckBatcher(
+                self.connection,
+                max_size=int(ack_batch.get("max_size", 50)),
+                max_delay_ms=float(ack_batch.get("max_delay_ms", 5)),
+            )
+
+        self._active: Dict[str, str] = {}  # job id -> lock token
+        self._cancelled: Dict[str, Optional[str]] = {}  # job id -> reason
+        self._active_lock = threading.Lock()
+        self._stop = threading.Event()
+        self._paused = threading.Event()
+        self._ready = threading.Event()
+        self._closed = False
+        self._loop_started = False  # run() actually entered its loop
+        self._was_busy = False  # for 'drained' edge detection
+        self._executor: Optional[ThreadPoolExecutor] = None
+        self._run_thread: Optional[threading.Thread] = None
+        self._processed = 0
+        self._failed = 0
+        self._registered_generation = -1
+
+        if autorun:
+            self.start()
+
+    def on(self, event: str, listener) -> "Worker":  # type: ignore[override]
+        """Like EventEmitter.on, but replays 'ready' to late listeners.
+
+        With autorun the loop starts inside the constructor, so a listener
+        attached right after construction could otherwise miss the event.
+        """
+        if event == "ready" and self._ready.is_set():
+            try:
+                listener()
+            except Exception:  # noqa: BLE001 - listener errors must not propagate
+                logger.warning("replayed 'ready' listener raised", exc_info=True)
+        super().on(event, listener)
+        return self
+
+    # -------------------------------------------------------------- lifecycle
+
+    def start(self) -> None:
+        """Run the worker in a background thread."""
+        if self._closed or self._run_thread is not None:
+            return
+        self._run_thread = threading.Thread(target=self.run, daemon=True)
+        self._run_thread.start()
+
+    def run(self) -> None:
+        """Blocking main loop; returns after :meth:`close`.
+
+        If the loop is already running (autorun / start()), this joins it
+        instead of starting a second loop, so blocking semantics hold.
+        """
+        if self._run_thread is not None and threading.current_thread() is not self._run_thread:
+            self._run_thread.join()
+            return
+        self._loop_started = True
+        self._executor = ThreadPoolExecutor(
+            max_workers=self.concurrency, thread_name_prefix="bunqueue-job"
+        )
+        self._register()
+        self._ready.set()
+        self.emit("ready")
+        # 0 (or negative) disables heartbeats — Event.wait(0) returns
+        # immediately, so the loop would busy-spin flooding the server.
+        if self.heartbeat_interval_s > 0:
+            heartbeat = threading.Thread(target=self._heartbeat_loop, daemon=True)
+            heartbeat.start()
+        backoff_idx = 0
+        try:
+            while not self._stop.is_set():
+                if self._paused.is_set():
+                    self._stop.wait(0.05)
+                    continue
+                try:
+                    self._poll_once()
+                    backoff_idx = 0
+                except (ConnectionClosedError, CommandTimeoutError) as exc:
+                    self.emit("error", exc)
+                    delay = RECONNECT_BACKOFF_S[min(backoff_idx, len(RECONNECT_BACKOFF_S) - 1)]
+                    backoff_idx += 1
+                    self._stop.wait(delay)
+        finally:
+            self._shutdown()
+
+    def pause(self) -> None:
+        self._paused.set()
+
+    def resume(self) -> None:
+        self._paused.clear()
+
+    def is_running(self) -> bool:
+        return self._run_thread is not None and not self._closed
+
+    def is_paused(self) -> bool:
+        return self._paused.is_set()
+
+    def is_closed(self) -> bool:
+        return self._closed
+
+    def wait_until_ready(self, timeout: float = 10.0) -> None:
+        if not self._ready.wait(timeout):
+            raise TimeoutError("worker did not become ready in time")
+
+    def close(self, timeout: Optional[float] = None) -> bool:
+        """Graceful shutdown: stop pulling, wait for in-flight jobs.
+
+        Returns True once fully shut down; False when ``timeout`` expires with
+        the loop thread still draining (the thread reference is kept alive so
+        state stays honest and a later ``close()`` can join it again).
+        """
+        self._stop.set()
+        thread = self._run_thread
+        if thread is not None:
+            thread.join(timeout)
+            if thread.is_alive():
+                return False  # still draining; do NOT null the live thread
+            self._run_thread = None
+            return True
+        if not self._loop_started and not self._closed:
+            # autorun=False and run() never called: no loop will ever reach
+            # _shutdown, so settle the closed state here.
+            self._closed = True
+            self.connection.close()
+            self.emit("closed")
+        return True
+
+    # `stop` kept as an alias (pre-1.0 SDK name)
+    stop = close
+
+    def __enter__(self) -> "Worker":
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    # ------------------------------------------------------ cooperative cancel
+
+    def cancel_job(self, job_id: str, reason: Optional[str] = None) -> bool:
+        """Mark a locally-active job as cancelled (cooperative, no TCP command).
+
+        The processor must poll :meth:`is_job_cancelled` and abort itself.
+        Returns True if the job is currently active on this worker.
+        """
+        with self._active_lock:
+            if job_id not in self._active:
+                return False
+            self._cancelled[job_id] = reason
+        self.emit("cancelled", {"jobId": job_id, "reason": reason or "cancelled"})
+        return True
+
+    def cancel_all_jobs(self, reason: Optional[str] = None) -> None:
+        with self._active_lock:
+            active = list(self._active.keys())
+            for job_id in active:
+                self._cancelled[job_id] = reason
+        for job_id in active:
+            self.emit("cancelled", {"jobId": job_id, "reason": reason or "cancelled"})
+
+    def is_job_cancelled(self, job_id: str) -> bool:
+        with self._active_lock:
+            return job_id in self._cancelled
+
+    def _shutdown(self) -> None:
+        if self._executor is not None:
+            self._executor.shutdown(wait=True)
+            self._executor = None
+        # Flush AFTER the executor drained: the last in-flight jobs buffer
+        # their ACKs during shutdown(wait=True); send them before closing.
+        if self._ack_batcher is not None:
+            self._ack_batcher.flush()
+        self._safe_call({"cmd": "UnregisterWorker", "workerId": self.worker_id})
+        self.connection.close()
+        already_closed = self._closed
+        self._closed = True
+        if not already_closed:
+            self.emit("closed")