bunqueue-client 0.1.5__py3-none-any.whl

bunqueue/simple/aging.py

@@ -0,0 +1,68 @@
+"""Priority aging: boost old waiting jobs so they never starve (mirror of
+aging.ts, adapted to the TCP query surface)."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from typing import Any, Dict, Optional
+
+from ..queue import Queue
+
+logger = logging.getLogger("bunqueue")
+
+
+class PriorityAger:
+    def __init__(self, config: Dict[str, Any], queue: Queue) -> None:
+        self._config = config
+        self._queue = queue
+        self._timer: Optional[threading.Timer] = None
+        self._stopped = False
+
+    def start(self) -> None:
+        self._schedule()
+
+    def _schedule(self) -> None:
+        if self._stopped:
+            return
+        interval = float(self._config.get("interval") or 60000)
+        self._timer = threading.Timer(interval / 1000.0, self._tick)
+        self._timer.daemon = True
+        self._timer.start()
+
+    def _tick(self) -> None:
+        try:
+            self._boost_old_jobs()
+        except Exception:  # noqa: BLE001 - best-effort background task
+            logger.warning("priority aging tick failed", exc_info=True)
+        finally:
+            self._schedule()
+
+    def _boost_old_jobs(self) -> None:
+        min_age = float(self._config.get("min_age") or self._config.get("minAge") or 60000)
+        boost = int(self._config.get("boost") or 1)
+        max_priority = int(
+            self._config.get("max_priority") or self._config.get("maxPriority") or 100
+        )
+        max_scan = int(self._config.get("max_scan") or self._config.get("maxScan") or 100)
+
+        jobs = self._queue.get_waiting(0, max_scan) + self._queue.get_jobs(
+            "prioritized", 0, max_scan
+        )
+        now = time.time() * 1000
+        for job in jobs:
+            created = job.created_at or now
+            age = now - created
+            if age >= min_age and job.priority < max_priority:
+                new_priority = min(job.priority + boost, max_priority)
+                try:
+                    self._queue.change_job_priority(job.id, new_priority)
+                except Exception:  # noqa: BLE001 - job may have been processed
+                    pass
+
+    def destroy(self) -> None:
+        self._stopped = True
+        if self._timer:
+            self._timer.cancel()
+            self._timer = None

bunqueue/simple/app.py

@@ -0,0 +1,163 @@
+"""Bunqueue Simple Mode: Queue + Worker in one object (mirror of bunqueue.ts).
+
+TCP mode only: the ``embedded`` option of the original requires the in-process
+Bun runtime and raises here with a clear error.
+
+Processing pipeline per job (same order as the original):
+Job -> Circuit Breaker -> TTL check -> CancelSignal -> Retry -> Middleware -> Processor
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional
+
+from ..job import Job
+from ..queue import Queue
+from ..worker import Worker
+from .aging import PriorityAger
+from .batch import BatchAccumulator
+from .cancellation import CancellationManager, CancelSignal
+from .circuit_breaker import WorkerCircuitBreaker
+from .dedup_debounce import DedupDebounceMerger
+from .rate_gate import RateGate
+from .retry import execute_with_retry
+from .triggers import TriggerManager
+from .ttl import TtlChecker
+
+Processor = Callable[[Job], Any]
+
+
+from .app_api import BunqueueApi
+
+
+class Bunqueue(BunqueueApi):
+    """All-in-one Queue + Worker with routes, middleware, cron, batch, retry,
+    circuit breaker, TTL, priority aging, cancellation, dedup/debounce."""
+
+    def __init__(self, name: str, **opts: Any) -> None:
+        if opts.get("embedded"):
+            raise ValueError(
+                "embedded mode requires the Bun runtime and the official bunqueue "
+                "package; this SDK is TCP-only — connect to a bunqueue server instead"
+            )
+        modes = [m for m in (opts.get("processor"), opts.get("routes"), opts.get("batch")) if m]
+        if len(modes) == 0:
+            raise ValueError('Bunqueue requires "processor", "routes", or "batch"')
+        if len(modes) > 1:
+            raise ValueError('Bunqueue: use only one of "processor", "routes", or "batch"')
+
+        self.name = name
+        self._middlewares: List[Callable[[Job, Callable[[], Any]], Any]] = []
+        self._retry_config: Optional[Dict[str, Any]] = opts.get("retry")
+        self._ttl_checker = TtlChecker(opts["ttl"]) if opts.get("ttl") else None
+        self._merger = DedupDebounceMerger(opts.get("deduplication"), opts.get("debounce"))
+        self._cancellation = CancellationManager()
+        self._default_job_options: Dict[str, Any] = dict(opts.get("default_job_options") or {})
+
+        if opts.get("batch"):
+            self._batch_acc: Optional[BatchAccumulator] = BatchAccumulator(opts["batch"])
+            self._base_processor: Processor = self._batch_acc.build_processor()
+        else:
+            self._batch_acc = None
+            routes = opts.get("routes")
+            self._base_processor = (
+                self._build_route_processor(routes) if routes else opts["processor"]
+            )
+
+        connection = {
+            "host": opts.get("host", "localhost"),
+            "port": opts.get("port", 6789),
+            "token": opts.get("token"),
+            "tls": opts.get("tls"),
+        }
+        self.queue = Queue(name, **connection)
+        self.worker = Worker(
+            name,
+            self._process_job,
+            host=connection["host"],
+            port=connection["port"],
+            token=connection["token"],
+            tls=connection["tls"],
+            concurrency=opts.get("concurrency", 4),
+            batch_size=opts.get("batch_size", 10),
+            poll_timeout_ms=opts.get("poll_timeout_ms", 5000),
+            heartbeat_interval_s=opts.get("heartbeat_interval_s", 10.0),
+            autorun=opts.get("autorun", True),
+        )
+
+        if opts.get("dlq"):
+            self.queue.set_dlq_config(opts["dlq"])
+        limiter = opts.get("rate_limit") or opts.get("limiter")
+        self._rate_gate = RateGate(limiter) if limiter else None
+
+        self._cb = (
+            WorkerCircuitBreaker(opts["circuit_breaker"], self.worker)
+            if opts.get("circuit_breaker")
+            else None
+        )
+        self._trigger_mgr = TriggerManager(self.queue, self.worker)
+        self._ager = (
+            PriorityAger(opts["priority_aging"], self.queue)
+            if opts.get("priority_aging")
+            else None
+        )
+        if self._ager:
+            self._ager.start()
+
+    # ---------------------------------------------------------- processing
+
+    def _build_route_processor(self, routes: Dict[str, Processor]) -> Processor:
+        def process(job: Job) -> Any:
+            handler = routes.get(job.name or "")
+            if handler is None:
+                raise ValueError(f'No route for job "{job.name}" in queue "{self.name}"')
+            return handler(job)
+
+        return process
+
+    def _process_job(self, job: Job) -> Any:
+        if self._rate_gate is not None:
+            self._rate_gate.acquire(self._rate_gate.group_for(job.data))
+        if self._cb is not None and self._cb.is_open():
+            raise RuntimeError("Circuit breaker is open")
+        timestamp = job.created_at or 0
+        if self._ttl_checker is not None and self._ttl_checker.is_expired(
+            job.name or "", timestamp
+        ):
+            import time
+
+            raise RuntimeError(f"Job expired (age: {int(time.time() * 1000 - timestamp)}ms)")
+        signal = self._cancellation.register(job.id)
+        try:
+            run = lambda: self._run_middleware_chain(job, signal)  # noqa: E731
+            if self._retry_config:
+                result = execute_with_retry(run, self._retry_config)
+            else:
+                result = run()
+        except BaseException:
+            if self._cb is not None:
+                self._cb.on_failure()
+            raise
+        else:
+            if self._cb is not None:
+                self._cb.on_success()
+            return result
+        finally:
+            self._cancellation.unregister(job.id)
+
+    def _run_middleware_chain(self, job: Job, signal: CancelSignal) -> Any:
+        if not self._middlewares:
+            return self._base_processor(job)
+        index = 0
+
+        def next_fn() -> Any:
+            nonlocal index
+            if signal.aborted:
+                raise RuntimeError("Job cancelled")
+            if index < len(self._middlewares):
+                middleware = self._middlewares[index]
+                index += 1
+                return middleware(job, next_fn)
+            return self._base_processor(job)
+
+        return next_fn()

bunqueue/simple/app_api.py

@@ -0,0 +1,142 @@
+"""Public API surface of Bunqueue Simple Mode (mixin): middleware, queue
+operations, cron shorthands, subsystem accessors, events and control.
+
+Split from :mod:`app` to keep files small; semantics mirror bunqueue.ts.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional
+
+from ..job import Job
+from .cancellation import CancelSignal
+
+
+class BunqueueApi:
+    """Mixin: requires the attributes initialized in ``Bunqueue.__init__``."""
+
+    # ---------------------------------------------------------- middleware
+
+    def use(self, middleware: Callable[[Job, Callable[[], Any]], Any]) -> "Bunqueue":
+        self._middlewares.append(middleware)
+        return self
+
+    # ---------------------------------------------------------- queue ops
+
+    def add(self, name: str, data: Any = None, **opts: Any) -> Job:
+        merged = self._merger.merge(name, {**self._default_job_options, **opts}, data)
+        return self.queue.add(name, data, **merged)
+
+    def add_bulk(self, jobs: List[Dict[str, Any]]) -> List[str]:
+        entries = []
+        for entry in jobs:
+            entry = dict(entry)
+            opts = {k: v for k, v in entry.items() if k not in ("name", "data")}
+            merged = self._merger.merge(
+                entry["name"], {**self._default_job_options, **opts}, entry.get("data")
+            )
+            entries.append({"name": entry["name"], "data": entry.get("data"), **merged})
+        return self.queue.add_bulk(entries)
+
+    def get_job(self, job_id: str) -> Optional[Job]:
+        return self.queue.get_job(job_id)
+
+    def get_job_counts(self) -> Dict[str, int]:
+        return self.queue.get_job_counts()
+
+    def count(self) -> int:
+        return self.queue.count()
+
+    # ---------------------------------------------------------- cron
+
+    def cron(self, scheduler_id: str, pattern: str, data: Any = None, **opts: Any) -> None:
+        self.queue.upsert_job_scheduler(
+            scheduler_id,
+            {"pattern": pattern, "tz": opts.get("timezone"), "limit": opts.get("limit")},
+            {"name": scheduler_id, "data": data, "opts": opts.get("job_opts")},
+        )
+
+    def every(self, scheduler_id: str, interval_ms: int, data: Any = None, **opts: Any) -> None:
+        self.queue.upsert_job_scheduler(
+            scheduler_id,
+            {"every": interval_ms, "limit": opts.get("limit")},
+            {"name": scheduler_id, "data": data, "opts": opts.get("job_opts")},
+        )
+
+    def remove_cron(self, scheduler_id: str) -> None:
+        self.queue.remove_job_scheduler(scheduler_id)
+
+    def list_crons(self) -> List[Dict[str, Any]]:
+        return self.queue.get_job_schedulers()
+
+    # ---------------------------------------------------------- subsystems
+
+    def cancel(self, job_id: str, grace_period_ms: float = 0) -> None:
+        self._cancellation.cancel(job_id, grace_period_ms)
+
+    def is_cancelled(self, job_id: str) -> bool:
+        return self._cancellation.is_cancelled(job_id)
+
+    def get_signal(self, job_id: str) -> Optional[CancelSignal]:
+        return self._cancellation.get_signal(job_id)
+
+    def get_circuit_state(self) -> str:
+        return self._cb.current_state if self._cb else "closed"
+
+    def reset_circuit(self) -> None:
+        if self._cb:
+            self._cb.reset()
+
+    def trigger(self, rule: Dict[str, Any]) -> "Bunqueue":
+        self._trigger_mgr.add(rule)
+        return self
+
+    def set_default_ttl(self, ttl_ms: float) -> None:
+        if self._ttl_checker:
+            self._ttl_checker.set_default_ttl(ttl_ms)
+
+    def set_name_ttl(self, name: str, ttl_ms: float) -> None:
+        if self._ttl_checker:
+            self._ttl_checker.set_name_ttl(name, ttl_ms)
+
+    # ---------------------------------------------------------- events/control
+
+    def on(self, event: str, listener: Callable[..., None]) -> "Bunqueue":
+        self.worker.on(event, listener)
+        return self
+
+    def once(self, event: str, listener: Callable[..., None]) -> "Bunqueue":
+        self.worker.once(event, listener)
+        return self
+
+    def off(self, event: str, listener: Callable[..., None]) -> "Bunqueue":
+        self.worker.off(event, listener)
+        return self
+
+    def pause(self) -> None:
+        self.queue.pause()
+        self.worker.pause()
+
+    def resume(self) -> None:
+        self.queue.resume()
+        self.worker.resume()
+
+    def close(self, force: bool = False) -> None:
+        if self._ager:
+            self._ager.destroy()
+        if self._cb:
+            self._cb.destroy()
+        if self._batch_acc:
+            self._batch_acc.destroy()
+        self._cancellation.destroy_all()
+        self.worker.close(timeout=0 if force else None)
+        self.queue.close()
+
+    def is_running(self) -> bool:
+        return self.worker.is_running()
+
+    def is_paused(self) -> bool:
+        return self.worker.is_paused()
+
+    def is_closed(self) -> bool:
+        return self.worker.is_closed()

bunqueue/simple/batch.py

@@ -0,0 +1,88 @@
+"""Batch processing: accumulate N jobs, process them together (mirror of
+batch.ts). Each job's processor call blocks until its batch flushes, so the
+worker concurrency must be >= batch size for full batches — same as the
+original."""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, Callable, Dict, List, Optional
+
+from ..job import Job
+
+
+class _Entry:
+    __slots__ = ("job", "event", "result", "error")
+
+    def __init__(self, job: Job) -> None:
+        self.job = job
+        self.event = threading.Event()
+        self.result: Any = None
+        self.error: Optional[BaseException] = None
+
+
+class BatchAccumulator:
+    def __init__(self, config: Dict[str, Any]) -> None:
+        self._size = int(config["size"])
+        self._timeout_ms = float(config.get("timeout") or 5000)
+        self._processor: Callable[[List[Job]], List[Any]] = config["processor"]
+        self._buffer: List[_Entry] = []
+        self._timer: Optional[threading.Timer] = None
+        self._lock = threading.Lock()
+
+    def build_processor(self) -> Callable[[Job], Any]:
+        """Processor that buffers jobs; blocks each caller until flush."""
+
+        def process(job: Job) -> Any:
+            entry = _Entry(job)
+            with self._lock:
+                self._buffer.append(entry)
+                if len(self._buffer) >= self._size:
+                    self._flush_locked()
+                elif self._timer is None:
+                    self._timer = threading.Timer(self._timeout_ms / 1000.0, self.flush)
+                    self._timer.daemon = True
+                    self._timer.start()
+            entry.event.wait()
+            if entry.error is not None:
+                raise entry.error
+            return entry.result
+
+        return process
+
+    def flush(self) -> None:
+        with self._lock:
+            self._flush_locked()
+
+    def _flush_locked(self) -> None:
+        if self._timer:
+            self._timer.cancel()
+            self._timer = None
+        batch = self._buffer[:]
+        self._buffer.clear()
+        if not batch:
+            return
+        # Run the batch processor outside the lock via a dedicated thread so
+        # a size-triggered flush doesn't execute inside a caller's lock scope.
+        thread = threading.Thread(target=self._run_batch, args=(batch,), daemon=True)
+        thread.start()
+
+    def _run_batch(self, batch: List[_Entry]) -> None:
+        try:
+            results = self._processor([entry.job for entry in batch])
+        except BaseException as exc:  # noqa: BLE001 - every entry fails together
+            for entry in batch:
+                entry.error = exc
+                entry.event.set()
+            return
+        for i, entry in enumerate(batch):
+            entry.result = results[i] if results and i < len(results) else None
+            entry.event.set()
+
+    def destroy(self) -> None:
+        with self._lock:
+            if self._timer:
+                self._timer.cancel()
+                self._timer = None
+            if self._buffer:
+                self._flush_locked()

bunqueue/simple/cancellation.py

@@ -0,0 +1,79 @@
+"""Graceful job cancellation (mirror of cancellation.ts).
+
+Python has no AbortController; :class:`CancelSignal` exposes the same
+``aborted`` surface, checked cooperatively by processors and the middleware
+chain.
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import Dict, Optional
+
+
+class CancelSignal:
+    """Minimal AbortSignal equivalent: ``signal.aborted`` flips once."""
+
+    def __init__(self) -> None:
+        self._event = threading.Event()
+
+    @property
+    def aborted(self) -> bool:
+        return self._event.is_set()
+
+    def abort(self) -> None:
+        self._event.set()
+
+
+class CancellationManager:
+    def __init__(self) -> None:
+        self._signals: Dict[str, CancelSignal] = {}
+        self._timers: Dict[str, threading.Timer] = {}
+        self._lock = threading.Lock()
+
+    def register(self, job_id: str) -> CancelSignal:
+        signal = CancelSignal()
+        with self._lock:
+            self._signals[job_id] = signal
+        return signal
+
+    def unregister(self, job_id: str) -> None:
+        with self._lock:
+            self._signals.pop(job_id, None)
+            timer = self._timers.pop(job_id, None)
+        if timer:
+            timer.cancel()
+
+    def cancel(self, job_id: str, grace_period_ms: float = 0) -> None:
+        with self._lock:
+            signal = self._signals.get(job_id)
+        if signal is None:
+            return
+        if grace_period_ms > 0:
+            timer = threading.Timer(grace_period_ms / 1000.0, signal.abort)
+            timer.daemon = True
+            with self._lock:
+                self._timers[job_id] = timer
+            timer.start()
+        else:
+            signal.abort()
+
+    def is_cancelled(self, job_id: str) -> bool:
+        with self._lock:
+            signal = self._signals.get(job_id)
+        return signal.aborted if signal else False
+
+    def get_signal(self, job_id: str) -> Optional[CancelSignal]:
+        with self._lock:
+            return self._signals.get(job_id)
+
+    def destroy_all(self) -> None:
+        with self._lock:
+            signals = list(self._signals.values())
+            timers = list(self._timers.values())
+            self._signals.clear()
+            self._timers.clear()
+        for timer in timers:
+            timer.cancel()
+        for signal in signals:
+            signal.abort()

bunqueue/simple/circuit_breaker.py

@@ -0,0 +1,93 @@
+"""Circuit breaker for worker protection (mirror of circuitBreaker.ts).
+
+CLOSED -> failures >= threshold -> OPEN (worker paused)
+                                     |
+        <- success --- HALF-OPEN <- reset timeout expires
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import Any, Dict, Optional
+
+from ..worker import Worker
+
+
+class WorkerCircuitBreaker:
+    def __init__(self, config: Dict[str, Any], worker: Worker) -> None:
+        self._config = config
+        self._worker = worker
+        self._state = "closed"
+        self._failures = 0
+        self._timer: Optional[threading.Timer] = None
+        self._lock = threading.Lock()
+
+    @property
+    def current_state(self) -> str:
+        return self._state
+
+    def is_open(self) -> bool:
+        return self._state == "open"
+
+    def on_success(self) -> None:
+        with self._lock:
+            if self._state == "half-open":
+                self._state = "closed"
+                self._failures = 0
+                callback = self._config.get("on_close") or self._config.get("onClose")
+            elif self._state == "closed":
+                self._failures = 0
+                callback = None
+            else:
+                callback = None
+        if callback:
+            callback()
+
+    def on_failure(self) -> None:
+        with self._lock:
+            self._failures += 1
+            threshold = int(self._config.get("threshold") or 5)
+            should_open = self._state == "half-open" or self._failures >= threshold
+        if should_open:
+            self._open()
+
+    def _open(self) -> None:
+        with self._lock:
+            self._state = "open"
+            failures = self._failures
+            if self._timer:
+                self._timer.cancel()
+            reset_timeout = float(
+                self._config.get("reset_timeout") or self._config.get("resetTimeout") or 30000
+            )
+            self._timer = threading.Timer(reset_timeout / 1000.0, self._half_open)
+            self._timer.daemon = True
+            self._timer.start()
+        on_open = self._config.get("on_open") or self._config.get("onOpen")
+        if on_open:
+            on_open(failures)
+        self._worker.pause()
+
+    def _half_open(self) -> None:
+        with self._lock:
+            self._state = "half-open"
+        on_half_open = self._config.get("on_half_open") or self._config.get("onHalfOpen")
+        if on_half_open:
+            on_half_open()
+        self._worker.resume()
+
+    def reset(self) -> None:
+        with self._lock:
+            self._state = "closed"
+            self._failures = 0
+            if self._timer:
+                self._timer.cancel()
+                self._timer = None
+        if self._worker.is_paused():
+            self._worker.resume()
+
+    def destroy(self) -> None:
+        with self._lock:
+            if self._timer:
+                self._timer.cancel()
+                self._timer = None

bunqueue/simple/dedup_debounce.py

@@ -0,0 +1,42 @@
+"""Deduplication & debounce defaults merger (mirror of dedupDebounce.ts).
+
+Jobs with the same name + data derive the same deduplication id
+(``name:JSON(data)``, compact separators to match JSON.stringify).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, Optional
+
+
+class DedupDebounceMerger:
+    def __init__(
+        self,
+        dedup: Optional[Dict[str, Any]],
+        debounce: Optional[Dict[str, Any]],
+    ) -> None:
+        self._dedup = dedup
+        self._debounce = debounce
+
+    @property
+    def active(self) -> bool:
+        return self._dedup is not None or self._debounce is not None
+
+    def merge(self, name: str, opts: Dict[str, Any], data: Any) -> Dict[str, Any]:
+        if not self.active:
+            return opts
+        merged = dict(opts)
+        if self._dedup is not None and "deduplication" not in merged:
+            data_key = (
+                json.dumps(data, separators=(",", ":")) if data is not None else ""
+            )
+            merged["deduplication"] = {
+                "id": f"{name}:{data_key}",
+                "ttl": self._dedup.get("ttl", 3600000),
+                "extend": self._dedup.get("extend"),
+                "replace": self._dedup.get("replace"),
+            }
+        if self._debounce is not None and "debounce" not in merged:
+            merged["debounce"] = {"id": name, "ttl": self._debounce["ttl"]}
+        return merged