bunqueue-client 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bunqueue/__init__.py +69 -0
- bunqueue/ack_batcher.py +93 -0
- bunqueue/connection.py +240 -0
- bunqueue/connection_lifecycle.py +104 -0
- bunqueue/errors.py +38 -0
- bunqueue/events.py +56 -0
- bunqueue/flow.py +257 -0
- bunqueue/job.py +179 -0
- bunqueue/options.py +126 -0
- bunqueue/py.typed +0 -0
- bunqueue/queue.py +230 -0
- bunqueue/queue_admin.py +201 -0
- bunqueue/queue_query.py +204 -0
- bunqueue/simple/__init__.py +16 -0
- bunqueue/simple/aging.py +68 -0
- bunqueue/simple/app.py +163 -0
- bunqueue/simple/app_api.py +142 -0
- bunqueue/simple/batch.py +88 -0
- bunqueue/simple/cancellation.py +79 -0
- bunqueue/simple/circuit_breaker.py +93 -0
- bunqueue/simple/dedup_debounce.py +42 -0
- bunqueue/simple/rate_gate.py +57 -0
- bunqueue/simple/retry.py +65 -0
- bunqueue/simple/triggers.py +45 -0
- bunqueue/simple/ttl.py +34 -0
- bunqueue/telemetry.py +29 -0
- bunqueue/transport.py +50 -0
- bunqueue/wire.py +99 -0
- bunqueue/worker.py +259 -0
- bunqueue/worker_runtime.py +221 -0
- bunqueue_client-0.1.5.dist-info/METADATA +197 -0
- bunqueue_client-0.1.5.dist-info/RECORD +34 -0
- bunqueue_client-0.1.5.dist-info/WHEEL +4 -0
- bunqueue_client-0.1.5.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Worker runtime mixin: poll loop body, job execution, heartbeats, registry.
|
|
2
|
+
|
|
3
|
+
Split from :mod:`worker` to keep files small; ``Worker`` composes this mixin
|
|
4
|
+
with the lifecycle surface (start/run/pause/close).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import socket as _socket
|
|
12
|
+
import time
|
|
13
|
+
import traceback
|
|
14
|
+
from typing import Any, Dict, Optional
|
|
15
|
+
|
|
16
|
+
from .ack_batcher import AckItem
|
|
17
|
+
from .errors import BunqueueError, UnrecoverableError
|
|
18
|
+
from .job import Job
|
|
19
|
+
from .wire import _compact
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger("bunqueue")
|
|
22
|
+
|
|
23
|
+
MAX_POLL_TIMEOUT_MS = 30000
|
|
24
|
+
# The server persists the FIRST `stackTraceLimit` lines (default 10) of the
|
|
25
|
+
# stack we send. A Python traceback ends with the raise site, so we send the
|
|
26
|
+
# LAST lines — but no more than the server keeps, or the raise site would be
|
|
27
|
+
# truncated away by the server-side first-N cap.
|
|
28
|
+
MAX_STACK_LINES = 10
|
|
29
|
+
RECONNECT_BACKOFF_S = (0.5, 1.0, 2.0, 5.0)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class WorkerRuntime:
|
|
33
|
+
"""Mixin: requires the attributes initialized in ``Worker.__init__``."""
|
|
34
|
+
|
|
35
|
+
def _poll_once(self) -> None:
|
|
36
|
+
with self._active_lock:
|
|
37
|
+
free = self.concurrency - len(self._active)
|
|
38
|
+
if free <= 0:
|
|
39
|
+
self._stop.wait(0.05)
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
# The registration is per-connection server state: after a reconnect
|
|
43
|
+
# the server no longer knows this worker (ListWorkers, skipIfNoWorker
|
|
44
|
+
# crons). Detect the new connection generation and re-register.
|
|
45
|
+
if (
|
|
46
|
+
self.connection.connected
|
|
47
|
+
and self.connection.generation != self._registered_generation
|
|
48
|
+
):
|
|
49
|
+
self._register()
|
|
50
|
+
|
|
51
|
+
response = self.connection.call(
|
|
52
|
+
{
|
|
53
|
+
"cmd": "PULLB",
|
|
54
|
+
"queue": self.queue,
|
|
55
|
+
"count": min(free, self.batch_size),
|
|
56
|
+
"timeout": self.poll_timeout_ms,
|
|
57
|
+
"owner": self.worker_id,
|
|
58
|
+
"lockTtl": self.lock_ttl_ms,
|
|
59
|
+
},
|
|
60
|
+
timeout=self.poll_timeout_ms / 1000 + 10,
|
|
61
|
+
)
|
|
62
|
+
jobs = response.get("jobs") or []
|
|
63
|
+
tokens = response.get("tokens") or []
|
|
64
|
+
if not jobs:
|
|
65
|
+
with self._active_lock:
|
|
66
|
+
idle = not self._active
|
|
67
|
+
if self._was_busy and idle:
|
|
68
|
+
self._was_busy = False
|
|
69
|
+
self.emit("drained")
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
self._was_busy = True
|
|
73
|
+
assert self._executor is not None
|
|
74
|
+
for raw, tok in zip(jobs, tokens):
|
|
75
|
+
job_id = str(raw.get("id"))
|
|
76
|
+
with self._active_lock:
|
|
77
|
+
self._active[job_id] = tok
|
|
78
|
+
self._executor.submit(self._run_job, raw, tok)
|
|
79
|
+
|
|
80
|
+
def _run_job(self, raw: Dict[str, Any], token: str) -> None:
|
|
81
|
+
job = Job(raw, self.connection, token, on_progress=lambda j, p: self.emit("progress", j, p))
|
|
82
|
+
self.emit("active", job)
|
|
83
|
+
try:
|
|
84
|
+
result = self.processor(job)
|
|
85
|
+
except BaseException as exc: # noqa: BLE001 - fail the job, never the worker
|
|
86
|
+
# Send at most as many trailing lines as the server will keep
|
|
87
|
+
# (its cap keeps the FIRST N): honor a per-job stackTraceLimit.
|
|
88
|
+
cap = raw.get("stackTraceLimit")
|
|
89
|
+
cap = cap if isinstance(cap, int) and cap > 0 else MAX_STACK_LINES
|
|
90
|
+
stack = traceback.format_exc().splitlines()[-cap:]
|
|
91
|
+
failed_sent = self._safe_call(
|
|
92
|
+
_compact(
|
|
93
|
+
{
|
|
94
|
+
"cmd": "FAIL",
|
|
95
|
+
"id": job.id,
|
|
96
|
+
"error": str(exc) or exc.__class__.__name__,
|
|
97
|
+
"token": token,
|
|
98
|
+
"stack": stack,
|
|
99
|
+
"unrecoverable": True if isinstance(exc, UnrecoverableError) else None,
|
|
100
|
+
}
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
self._finish_job(job.id)
|
|
104
|
+
# Only count/emit 'failed' when the server recorded the failure;
|
|
105
|
+
# on a wire failure only 'error' fired and the server re-leases
|
|
106
|
+
# the job after lock expiry (same gate as the TS worker).
|
|
107
|
+
if failed_sent:
|
|
108
|
+
self._failed += 1
|
|
109
|
+
self.emit("failed", job, exc)
|
|
110
|
+
return
|
|
111
|
+
if self._ack_batcher is not None:
|
|
112
|
+
# Defer the ACK into a batch; the job stays active (lock renewed by
|
|
113
|
+
# the heartbeat loop) until the ACKB settles. _on_ack_settled frees
|
|
114
|
+
# the slot FIRST so a raising listener can never leak it and
|
|
115
|
+
# permanently shrink the worker's effective concurrency.
|
|
116
|
+
self._ack_batcher.add(
|
|
117
|
+
AckItem(
|
|
118
|
+
id=job.id,
|
|
119
|
+
token=token,
|
|
120
|
+
result=result,
|
|
121
|
+
on_settled=lambda err, job=job, result=result: self._on_ack_settled(
|
|
122
|
+
job, result, err
|
|
123
|
+
),
|
|
124
|
+
)
|
|
125
|
+
)
|
|
126
|
+
return
|
|
127
|
+
ack: Dict[str, Any] = {"cmd": "ACK", "id": job.id, "token": token}
|
|
128
|
+
if result is not None:
|
|
129
|
+
ack["result"] = result
|
|
130
|
+
acked = self._safe_call(ack)
|
|
131
|
+
# Free the slot BEFORE emitting (same rationale as the batched path).
|
|
132
|
+
self._finish_job(job.id)
|
|
133
|
+
# Only count/emit 'completed' when the ACK reached the server: an
|
|
134
|
+
# unconfirmed ack means the job will be redelivered after lock expiry,
|
|
135
|
+
# so reporting success here would double-count (same gate as the
|
|
136
|
+
# batched path and the TS worker).
|
|
137
|
+
if acked:
|
|
138
|
+
self._processed += 1
|
|
139
|
+
self.emit("completed", job, result)
|
|
140
|
+
|
|
141
|
+
def _on_ack_settled(self, job: Job, result: Any, err: Optional[BaseException]) -> None:
|
|
142
|
+
"""Settle callback for a batched ACK: free the slot, then report.
|
|
143
|
+
|
|
144
|
+
On a failed ACKB the job is NOT reported as completed: the server
|
|
145
|
+
never confirmed the ack (the job will be retried after lock expiry),
|
|
146
|
+
so the worker surfaces an 'error' event per job instead."""
|
|
147
|
+
self._finish_job(job.id)
|
|
148
|
+
if err is not None:
|
|
149
|
+
self.emit("error", err)
|
|
150
|
+
else:
|
|
151
|
+
self._processed += 1
|
|
152
|
+
self.emit("completed", job, result)
|
|
153
|
+
|
|
154
|
+
def _finish_job(self, job_id: str) -> None:
|
|
155
|
+
with self._active_lock:
|
|
156
|
+
self._active.pop(job_id, None)
|
|
157
|
+
self._cancelled.pop(job_id, None)
|
|
158
|
+
|
|
159
|
+
def _heartbeat_loop(self) -> None:
|
|
160
|
+
while not self._stop.wait(self.heartbeat_interval_s):
|
|
161
|
+
with self._active_lock:
|
|
162
|
+
ids = list(self._active.keys())
|
|
163
|
+
tokens = [self._active[i] for i in ids]
|
|
164
|
+
active_count = len(ids)
|
|
165
|
+
self._safe_call(
|
|
166
|
+
{
|
|
167
|
+
"cmd": "Heartbeat",
|
|
168
|
+
"id": self.worker_id,
|
|
169
|
+
"activeJobs": active_count,
|
|
170
|
+
"processed": self._processed,
|
|
171
|
+
"failed": self._failed,
|
|
172
|
+
}
|
|
173
|
+
)
|
|
174
|
+
if ids:
|
|
175
|
+
self._safe_call({"cmd": "JobHeartbeatB", "ids": ids, "tokens": tokens})
|
|
176
|
+
|
|
177
|
+
def _register(self) -> None:
|
|
178
|
+
"""Register the worker; mark the generation ONLY on success.
|
|
179
|
+
|
|
180
|
+
A failed RegisterWorker must leave ``_registered_generation`` stale so
|
|
181
|
+
the next poll iteration retries: marking it unconditionally would leave
|
|
182
|
+
the server unaware of this worker (ListWorkers, skipIfNoWorker crons,
|
|
183
|
+
Discussion #103 class) until the next reconnect.
|
|
184
|
+
|
|
185
|
+
The generation is snapshotted BEFORE the call: if the call itself
|
|
186
|
+
triggers a reconnect, the stale snapshot forces one harmless duplicate
|
|
187
|
+
registration (server-side upsert) instead of ever missing one.
|
|
188
|
+
"""
|
|
189
|
+
generation = self.connection.generation
|
|
190
|
+
try:
|
|
191
|
+
self.connection.call(
|
|
192
|
+
{
|
|
193
|
+
"cmd": "RegisterWorker",
|
|
194
|
+
"name": self.name,
|
|
195
|
+
"queues": [self.queue],
|
|
196
|
+
"concurrency": self.concurrency,
|
|
197
|
+
"workerId": self.worker_id,
|
|
198
|
+
"hostname": _socket.gethostname(),
|
|
199
|
+
"pid": os.getpid(),
|
|
200
|
+
"startedAt": int(time.time() * 1000),
|
|
201
|
+
}
|
|
202
|
+
)
|
|
203
|
+
except BunqueueError as exc:
|
|
204
|
+
logger.warning("RegisterWorker failed (will retry next poll): %s", exc)
|
|
205
|
+
self.emit("error", exc)
|
|
206
|
+
return
|
|
207
|
+
self._registered_generation = generation
|
|
208
|
+
|
|
209
|
+
def _safe_call(self, command: Dict[str, Any]) -> bool:
|
|
210
|
+
"""Fire a command, swallowing wire failures into an 'error' event.
|
|
211
|
+
|
|
212
|
+
Returns True only when the command reached the server, so callers can
|
|
213
|
+
gate success-side effects (counters, 'completed'/'failed' emits) on
|
|
214
|
+
the wire outcome, mirroring the TypeScript worker's safeCall."""
|
|
215
|
+
try:
|
|
216
|
+
self.connection.call(command)
|
|
217
|
+
return True
|
|
218
|
+
except BunqueueError as exc:
|
|
219
|
+
logger.warning("swallowed %s failure: %s", command.get("cmd"), exc)
|
|
220
|
+
self.emit("error", exc)
|
|
221
|
+
return False
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bunqueue-client
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: Python client SDK for bunqueue — high-performance job queue server (TCP mode)
|
|
5
|
+
Project-URL: Homepage, https://github.com/egeominotti/bunqueue
|
|
6
|
+
Project-URL: Documentation, https://bunqueue.dev
|
|
7
|
+
Project-URL: Repository, https://github.com/egeominotti/bunqueue
|
|
8
|
+
Project-URL: Changelog, https://github.com/egeominotti/bunqueue/blob/main/sdk/python/CHANGELOG.md
|
|
9
|
+
Author: Egeo Minotti
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: bunqueue,jobs,queue,task-queue,worker
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Requires-Dist: msgpack>=1.0.0
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
<div align="center">
|
|
28
|
+
|
|
29
|
+
<a href="https://bunqueue.dev">
|
|
30
|
+
<img src="https://raw.githubusercontent.com/egeominotti/bunqueue/main/.github/logo.png" alt="bunqueue logo" width="110" />
|
|
31
|
+
</a>
|
|
32
|
+
|
|
33
|
+
# bunqueue-client (Python)
|
|
34
|
+
|
|
35
|
+
**The official Python client for [bunqueue](https://bunqueue.dev), the high performance job queue server.**
|
|
36
|
+
|
|
37
|
+
Native TCP protocol (msgpack, pipelined), one runtime dependency (`msgpack`), sync plus thread based workers.
|
|
38
|
+
|
|
39
|
+
[](https://github.com/egeominotti/bunqueue/blob/main/sdk/python/LICENSE)
|
|
40
|
+
[](https://github.com/egeominotti/bunqueue/tree/main/sdk/python)
|
|
41
|
+
[](https://github.com/egeominotti/bunqueue/tree/main/sdk/conformance)
|
|
42
|
+
|
|
43
|
+
[Documentation](https://bunqueue.dev/guide/sdks/) · [Protocol spec](https://github.com/egeominotti/bunqueue/blob/main/docs/protocol.md) · [Server](https://github.com/egeominotti/bunqueue) · [Changelog](https://github.com/egeominotti/bunqueue/blob/main/sdk/python/CHANGELOG.md)
|
|
44
|
+
|
|
45
|
+
</div>
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
Python client for [bunqueue](https://github.com/egeominotti/bunqueue), the
|
|
50
|
+
high-performance job queue server for Bun. Talks the native TCP protocol
|
|
51
|
+
(msgpack, pipelined) with the same core API as the TypeScript SDK, including
|
|
52
|
+
opt-in ACK batching and dependency-free structured transport telemetry.
|
|
53
|
+
Connection pooling remains TypeScript-only for now.
|
|
54
|
+
|
|
55
|
+
The bunqueue **server** runs on Bun (or as a compiled binary / Docker). This
|
|
56
|
+
SDK lets any Python service produce and consume jobs on it: *one queue, any
|
|
57
|
+
language*.
|
|
58
|
+
|
|
59
|
+
## Install
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
# PyPI release coming soon; install from the repo today:
|
|
63
|
+
pip install "git+https://github.com/egeominotti/bunqueue.git#subdirectory=sdk/python"
|
|
64
|
+
# dependency: msgpack; import name: bunqueue
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Requires Python ≥ 3.9 and a running bunqueue server (`bunx bunqueue start`).
|
|
68
|
+
|
|
69
|
+
## Quick start
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from bunqueue import Queue, Worker
|
|
73
|
+
|
|
74
|
+
# Producer
|
|
75
|
+
queue = Queue("emails", host="localhost", port=6789)
|
|
76
|
+
job = queue.add("send", {"to": "user@example.com"}, priority=5, attempts=3)
|
|
77
|
+
print(job.id)
|
|
78
|
+
|
|
79
|
+
# Consumer
|
|
80
|
+
def process(job):
|
|
81
|
+
job.update_progress(50)
|
|
82
|
+
return {"sent": True}
|
|
83
|
+
|
|
84
|
+
worker = Worker("emails", process, concurrency=10)
|
|
85
|
+
worker.on("completed", lambda job, result: print(job.id, result))
|
|
86
|
+
worker.run() # blocking; or worker.start() for a background thread
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
All queue semantics (retry, backoff, DLQ, priorities, stall detection, cron)
|
|
90
|
+
run **server-side** — the worker only pulls, heartbeats, and acks.
|
|
91
|
+
|
|
92
|
+
## Feature surface
|
|
93
|
+
|
|
94
|
+
- **Queue**: `add`, `add_bulk`, full job options (priority, delay, attempts,
|
|
95
|
+
backoff, ttl, timeout, job_id, deduplication, depends_on, tags, group_id,
|
|
96
|
+
lifo, remove_on_complete/fail, durable, repeat, debounce, …)
|
|
97
|
+
- **Query**: `get_job`, `get_job_by_custom_id`, `get_jobs` (+ per-state
|
|
98
|
+
helpers), `get_state`, `get_result`, `get_progress`, `wait_for_job`
|
|
99
|
+
(raises `CommandTimeoutError` on timeout, BullMQ contract),
|
|
100
|
+
counts (+ per-priority), children values, job logs
|
|
101
|
+
- **Control**: pause/resume/drain/obliterate/clean, remove, discard,
|
|
102
|
+
promote (single/bulk), retry_job / retry_jobs, move to wait/delayed,
|
|
103
|
+
change priority/delay, update data, extend lock
|
|
104
|
+
- **DLQ**: `get_dlq`, `retry_dlq`, `purge_dlq`, DLQ config
|
|
105
|
+
- **Schedulers**: `upsert_job_scheduler` (cron pattern or interval),
|
|
106
|
+
`add_cron` / `every` shorthands, get/list/remove
|
|
107
|
+
- **Admin**: rate limit, global concurrency, stall config, webhooks,
|
|
108
|
+
stats/metrics/list_queues/get_workers
|
|
109
|
+
- **Worker**: events (`ready`, `active`, `completed`, `failed`, `progress`,
|
|
110
|
+
`drained`, `error`, `closed`), pause/resume, graceful `close()` (also a
|
|
111
|
+
context manager), automatic lock heartbeats (jobs longer than the lock TTL
|
|
112
|
+
survive), `UnrecoverableError` to skip retries, opt-in ACK batching
|
|
113
|
+
(`ack_batch={"max_size": 50, "max_delay_ms": 5}`: successful ACKs flush as
|
|
114
|
+
one `ACKB` on size/delay/close; a job stays active until its batch settles)
|
|
115
|
+
- **FlowProducer**: `add` (parent/child trees), `add_bulk`, `add_chain`
|
|
116
|
+
(sequential), `add_bulk_then` (fan-in), `get_flow`, atomic rollback
|
|
117
|
+
- **Simple Mode** (`Bunqueue`): Queue + Worker in one object — routes,
|
|
118
|
+
onion middleware, in-process retry (fixed/exponential/jitter/fibonacci/
|
|
119
|
+
custom + `retry_if`), circuit breaker, batch accumulation, event triggers,
|
|
120
|
+
job TTL, priority aging, cooperative cancellation (`get_signal`),
|
|
121
|
+
dedup/debounce defaults, cron shorthands — 1:1 with the official client
|
|
122
|
+
(TCP mode; `embedded` raises)
|
|
123
|
+
- **Connection**: auth token, TLS (`tls=True`, `{"ca_file": ...}`,
|
|
124
|
+
`{"verify": False}`), pipelining, lazy reconnect, structured telemetry
|
|
125
|
+
|
|
126
|
+
Not applicable outside Bun (by design): embedded mode, sandboxed workers,
|
|
127
|
+
`QueueEvents` (in-process subscription; use webhooks or SSE/WS on the HTTP
|
|
128
|
+
port instead).
|
|
129
|
+
|
|
130
|
+
## Errors
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from bunqueue import (
|
|
134
|
+
BunqueueError, # base
|
|
135
|
+
ConnectionClosedError,
|
|
136
|
+
CommandTimeoutError,
|
|
137
|
+
CommandError, # server answered ok=false
|
|
138
|
+
AuthError,
|
|
139
|
+
SerializationError, # payload not msgpack-serializable (e.g. datetime)
|
|
140
|
+
UnrecoverableError, # raise in a processor: fail terminally, no retries
|
|
141
|
+
)
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
The SDK logs its otherwise-silent failure points (swallowed command errors,
|
|
145
|
+
raising listeners, failed registrations) at warning level on the `bunqueue`
|
|
146
|
+
logger; attach a handler to see them.
|
|
147
|
+
|
|
148
|
+
### Structured telemetry
|
|
149
|
+
|
|
150
|
+
Pass `on_telemetry` to `Connection` or `Queue` to bridge transport metrics to
|
|
151
|
+
OpenTelemetry, Prometheus, or your logger without adding an SDK dependency:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
queue = Queue("emails", on_telemetry=lambda event: metrics.record(event))
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Events cover `connect`, `disconnect`, `reconnect_scheduled`, `auth`, `command`,
|
|
158
|
+
`command_timeout`, and `error`. Command events contain the command name,
|
|
159
|
+
request id, outcome, and monotonic duration. Payloads and authentication tokens
|
|
160
|
+
are never included. A telemetry callback that raises is isolated from the
|
|
161
|
+
transport and cannot fail a queue operation.
|
|
162
|
+
|
|
163
|
+
## Protocol notes
|
|
164
|
+
|
|
165
|
+
- Wire: 4-byte big-endian length prefix + standard msgpack map per message;
|
|
166
|
+
requests carry a `reqId` echoed by the server (pipelining).
|
|
167
|
+
- Integers outside int32 are sent as float64 (exact ≤ 2^53): the server's
|
|
168
|
+
msgpack decoder turns int64 into `BigInt`, which its arithmetic rejects.
|
|
169
|
+
The SDK handles this automatically. Consequence: integers larger than 2^53
|
|
170
|
+
(e.g. 64-bit snowflake IDs) lose precision — pass them as **strings**.
|
|
171
|
+
- Command maps require string keys at every nesting level. Cyclic containers
|
|
172
|
+
and integers outside the float64 range raise `SerializationError` locally,
|
|
173
|
+
before a pending request is registered or any bytes are written. Binary
|
|
174
|
+
values and array-like lists/tuples are preserved by the wire normalizer.
|
|
175
|
+
|
|
176
|
+
## Tests
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
python -m venv .venv && .venv/bin/pip install msgpack
|
|
180
|
+
.venv/bin/python tests/test_integration.py # basic (8)
|
|
181
|
+
.venv/bin/python tests/run_e2e.py # full e2e vs real server (112)
|
|
182
|
+
BUNQUEUE_SDK_SOAK_SECONDS=3600 .venv/bin/python tests/soak.py
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Both spawn a real bunqueue server (`bun src/main.ts` from the repo root). The
|
|
186
|
+
E2E runner emits a monotonic duration for each case so the isolated SDK gate
|
|
187
|
+
can rank slow tests without relying on timing assertions. Hardening covers
|
|
188
|
+
independent-connection idempotency and single-lease contention, fixed-seed
|
|
189
|
+
generated payloads, malformed mutation corpora, 1000-job bursts, and
|
|
190
|
+
crash/restart recovery. The opt-in soak keeps one connection alive; set
|
|
191
|
+
`BUNQUEUE_SDK_SOAK_BATCH` to increase load.
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
MIT. See the [LICENSE](https://github.com/egeominotti/bunqueue/blob/main/sdk/python/LICENSE) file.
|
|
196
|
+
Documentation: [bunqueue.dev/guide/sdks](https://bunqueue.dev/guide/sdks/).
|
|
197
|
+
Issues and feature requests: [GitHub issues](https://github.com/egeominotti/bunqueue/issues).
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
bunqueue/__init__.py,sha256=AVJPSMlGV7TaWIAl_TWyZ_qgqiYurkYW7q9JlkO2PDg,1799
|
|
2
|
+
bunqueue/ack_batcher.py,sha256=VTB42-C4Wwcx4Ja-oI7tZjTXx6jufGtDJArCPdNpZYw,3506
|
|
3
|
+
bunqueue/connection.py,sha256=Kt0-TcdX8lOpimWvIV5W1JvidU7fx8NJ-fny7pH1JJE,9166
|
|
4
|
+
bunqueue/connection_lifecycle.py,sha256=uF8N7T0aMsi2hNnQ4n3FswGOrOdJJs7kwPLuUe0cvyA,3974
|
|
5
|
+
bunqueue/errors.py,sha256=ufaAFPdBlBJR4pyJTEUfdq39lv6oZkuouVAuotfFbzo,1141
|
|
6
|
+
bunqueue/events.py,sha256=R8rKu68nr2tbqpzrji8xKXcLzXNcd6yc8VGa3xmxAeM,1932
|
|
7
|
+
bunqueue/flow.py,sha256=Yh8HCa8hJBc1lYzjsN6dVdmdD_GH2l-CSQuEHjHz37Y,9318
|
|
8
|
+
bunqueue/job.py,sha256=RKWfOiEkvXQ7BEQlFxz8X5T1i1qbtmh71n7jIL6OAtU,6072
|
|
9
|
+
bunqueue/options.py,sha256=CGLMIJheKmQTw2iBnTgKk2Il0sLzvT7j-TfcKZRg3nE,5135
|
|
10
|
+
bunqueue/queue.py,sha256=lBS2_vVxEPUVMaUydBgtTKflEGRg7ae_E2SG6xebpfw,9343
|
|
11
|
+
bunqueue/queue_admin.py,sha256=g9w92oXPgK0O7wH_kJ7bu7HDmtKgCHQ4NGIRGLo7Kxg,9096
|
|
12
|
+
bunqueue/queue_query.py,sha256=bSwtkdC3QaKVGWfUKt-q8c9njZdZli9WwM7_ybfC1lY,8749
|
|
13
|
+
bunqueue/telemetry.py,sha256=Hz-15K_iQJJ7CtNn-MmflE-2RRhABSBx-6aeLHWREjM,864
|
|
14
|
+
bunqueue/transport.py,sha256=0v1_UDbLWCiyga97ZkA1BGZ6DvWRBY0TUW7ZobJk5Oc,1575
|
|
15
|
+
bunqueue/wire.py,sha256=4PeeOeHuYYV_bnUwa3SXrRxJ8fpRy7-3XgKoZdgKYMo,3523
|
|
16
|
+
bunqueue/worker.py,sha256=E3y7ew0Xu328z2RofL0CNEbylj9EJQ1yzcYgKJBv1eQ,9940
|
|
17
|
+
bunqueue/worker_runtime.py,sha256=Axl3ZYbumCyxi0Bx2hPqF0A7tD-q9hIwG9mh-2SCjbY,8958
|
|
18
|
+
bunqueue/simple/__init__.py,sha256=q0NYG9vgGZ6MqyAAAllRTB82RcGZK2ampKLUKKE8emw,487
|
|
19
|
+
bunqueue/simple/aging.py,sha256=pvbRMEQSSGXFt9jqksDCHIs3i5D_aGDc4Pq70kBT_8E,2283
|
|
20
|
+
bunqueue/simple/app.py,sha256=jr1D-zPBcexZjgINaPjy72RP7UeD4CVkCLr2tsCUOEw,6277
|
|
21
|
+
bunqueue/simple/app_api.py,sha256=J57dL34d_-HpeEhnzgMLQ26y5B8Jit64f6HtJVfnluo,4947
|
|
22
|
+
bunqueue/simple/batch.py,sha256=lZBeBhrsGKmpDceZXprjyaxSZd_6rspcKF9V-zPQk2E,3064
|
|
23
|
+
bunqueue/simple/cancellation.py,sha256=u_QdYTv_i5fTG2bkCyC3u9JRrOm6to6wLoLKkNmrCBw,2326
|
|
24
|
+
bunqueue/simple/circuit_breaker.py,sha256=kUSR0CmqYpXoWVsd0G-6M3QedsmbfonDlQAPLGBdPy8,2929
|
|
25
|
+
bunqueue/simple/dedup_debounce.py,sha256=lTea4mCBl3_Vcbf5awqIZu88V3yFlz7XR9X2j7S_3JY,1420
|
|
26
|
+
bunqueue/simple/rate_gate.py,sha256=sVGTJ5A5P4uP_5O8vZyhS71YtySaj8DskI_5uY3fPUY,2157
|
|
27
|
+
bunqueue/simple/retry.py,sha256=BKz1dPMqQTlcKnBSEnA4F23GEb2bDIzZJMV9A6U2uv8,2216
|
|
28
|
+
bunqueue/simple/triggers.py,sha256=SqLdgQpCBASK4cXtnTDYHEAC6-C-80uAn-pU4HpcuKM,1637
|
|
29
|
+
bunqueue/simple/ttl.py,sha256=wX6Sa5XhxPUlIFQ3zqO0tHYnCUkUk5kO9dcCZqEk8Kc,1062
|
|
30
|
+
bunqueue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
|
+
bunqueue_client-0.1.5.dist-info/METADATA,sha256=5vvem67o-0hjERJ-Ror7s1es7sgvi_bCFmArNLyNDxA,8940
|
|
32
|
+
bunqueue_client-0.1.5.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
33
|
+
bunqueue_client-0.1.5.dist-info/licenses/LICENSE,sha256=y5XqDMIC5-PicSAv0CsVSQ_wbUwRiPun9Qzp-gAHte0,1069
|
|
34
|
+
bunqueue_client-0.1.5.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Egeo Minotti
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|