bunqueue-client 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bunqueue/__init__.py +69 -0
- bunqueue/ack_batcher.py +93 -0
- bunqueue/connection.py +240 -0
- bunqueue/connection_lifecycle.py +104 -0
- bunqueue/errors.py +38 -0
- bunqueue/events.py +56 -0
- bunqueue/flow.py +257 -0
- bunqueue/job.py +179 -0
- bunqueue/options.py +126 -0
- bunqueue/py.typed +0 -0
- bunqueue/queue.py +230 -0
- bunqueue/queue_admin.py +201 -0
- bunqueue/queue_query.py +204 -0
- bunqueue/simple/__init__.py +16 -0
- bunqueue/simple/aging.py +68 -0
- bunqueue/simple/app.py +163 -0
- bunqueue/simple/app_api.py +142 -0
- bunqueue/simple/batch.py +88 -0
- bunqueue/simple/cancellation.py +79 -0
- bunqueue/simple/circuit_breaker.py +93 -0
- bunqueue/simple/dedup_debounce.py +42 -0
- bunqueue/simple/rate_gate.py +57 -0
- bunqueue/simple/retry.py +65 -0
- bunqueue/simple/triggers.py +45 -0
- bunqueue/simple/ttl.py +34 -0
- bunqueue/telemetry.py +29 -0
- bunqueue/transport.py +50 -0
- bunqueue/wire.py +99 -0
- bunqueue/worker.py +259 -0
- bunqueue/worker_runtime.py +221 -0
- bunqueue_client-0.1.5.dist-info/METADATA +197 -0
- bunqueue_client-0.1.5.dist-info/RECORD +34 -0
- bunqueue_client-0.1.5.dist-info/WHEEL +4 -0
- bunqueue_client-0.1.5.dist-info/licenses/LICENSE +21 -0
bunqueue/queue.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Queue: producer + control API over the bunqueue TCP protocol.
|
|
2
|
+
|
|
3
|
+
Feature parity with the TypeScript client's Queue (TCP mode). Query and
|
|
4
|
+
admin surfaces live in :mod:`queue_query` / :mod:`queue_admin` mixins.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import traceback
|
|
10
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
|
11
|
+
|
|
12
|
+
from .connection import Connection, TlsOption, _compact
|
|
13
|
+
from .errors import UnrecoverableError
|
|
14
|
+
from .job import Job
|
|
15
|
+
from .options import job_options, job_payload
|
|
16
|
+
from .queue_admin import QueueAdminOps
|
|
17
|
+
from .queue_query import QueueQueryOps
|
|
18
|
+
from .worker_runtime import MAX_STACK_LINES
|
|
19
|
+
from .telemetry import TelemetryHandler
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Queue(QueueQueryOps, QueueAdminOps):
|
|
23
|
+
"""Producer and management client for one named queue."""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
name: str,
|
|
28
|
+
host: str = "localhost",
|
|
29
|
+
port: int = 6789,
|
|
30
|
+
token: Optional[str] = None,
|
|
31
|
+
tls: TlsOption = None,
|
|
32
|
+
connection: Optional[Connection] = None,
|
|
33
|
+
command_timeout: float = 10.0,
|
|
34
|
+
prefix_key: str = "",
|
|
35
|
+
on_telemetry: Optional[TelemetryHandler] = None,
|
|
36
|
+
) -> None:
|
|
37
|
+
self.name = prefix_key + name
|
|
38
|
+
self.connection = connection or Connection(
|
|
39
|
+
host=host,
|
|
40
|
+
port=port,
|
|
41
|
+
token=token,
|
|
42
|
+
tls=tls,
|
|
43
|
+
command_timeout=command_timeout,
|
|
44
|
+
on_telemetry=on_telemetry,
|
|
45
|
+
)
|
|
46
|
+
self._owns_connection = connection is None
|
|
47
|
+
|
|
48
|
+
# --------------------------------------------------------------- produce
|
|
49
|
+
|
|
50
|
+
def add(self, name: str, data: Any = None, **options: Any) -> Job:
|
|
51
|
+
"""Add a job; returns a Job stub carrying the assigned id."""
|
|
52
|
+
payload = job_payload(name, data)
|
|
53
|
+
opts = job_options(**options)
|
|
54
|
+
response = self.connection.call({"cmd": "PUSH", "queue": self.name, "data": payload, **opts})
|
|
55
|
+
return Job({"id": response.get("id"), "queue": self.name, "data": payload, **opts}, self.connection)
|
|
56
|
+
|
|
57
|
+
def add_bulk(self, jobs: Sequence[Dict[str, Any]]) -> List[str]:
|
|
58
|
+
"""Add many jobs in one round-trip.
|
|
59
|
+
|
|
60
|
+
Each entry: ``{"name": str, "data": Any, ...options}`` with the same
|
|
61
|
+
option names accepted by :meth:`add`.
|
|
62
|
+
"""
|
|
63
|
+
inputs = []
|
|
64
|
+
for entry in jobs:
|
|
65
|
+
entry = dict(entry)
|
|
66
|
+
name = entry.pop("name")
|
|
67
|
+
data = entry.pop("data", None)
|
|
68
|
+
opts = job_options(**entry)
|
|
69
|
+
# PUSHB entries are typed JobInput, whose custom-id field is
|
|
70
|
+
# `customId` — unlike single PUSH which renames `jobId`->`customId`
|
|
71
|
+
# server-side. Without this rename the batch custom id is dropped
|
|
72
|
+
# (idempotency/getJobByCustomId broken).
|
|
73
|
+
if "jobId" in opts:
|
|
74
|
+
opts["customId"] = opts.pop("jobId")
|
|
75
|
+
inputs.append({"data": job_payload(name, data), **opts})
|
|
76
|
+
response = self.connection.call({"cmd": "PUSHB", "queue": self.name, "jobs": inputs})
|
|
77
|
+
return [str(job_id) for job_id in response.get("ids", [])]
|
|
78
|
+
|
|
79
|
+
# --------------------------------------------------------------- control
|
|
80
|
+
|
|
81
|
+
def pause(self) -> None:
|
|
82
|
+
self.connection.call({"cmd": "Pause", "queue": self.name})
|
|
83
|
+
|
|
84
|
+
def resume(self) -> None:
|
|
85
|
+
self.connection.call({"cmd": "Resume", "queue": self.name})
|
|
86
|
+
|
|
87
|
+
def is_paused(self) -> bool:
|
|
88
|
+
response = self.connection.call({"cmd": "IsPaused", "queue": self.name})
|
|
89
|
+
return bool(response.get("paused"))
|
|
90
|
+
|
|
91
|
+
def drain(self) -> int:
|
|
92
|
+
"""Remove all waiting/delayed jobs; returns how many were dropped."""
|
|
93
|
+
response = self.connection.call({"cmd": "Drain", "queue": self.name})
|
|
94
|
+
return int(response.get("count", 0))
|
|
95
|
+
|
|
96
|
+
def obliterate(self) -> None:
|
|
97
|
+
self.connection.call({"cmd": "Obliterate", "queue": self.name})
|
|
98
|
+
|
|
99
|
+
def clean(self, grace_ms: int, limit: Optional[int] = None, state: Optional[str] = None) -> int:
|
|
100
|
+
response = self.connection.call(
|
|
101
|
+
_compact({"cmd": "Clean", "queue": self.name, "grace": grace_ms, "state": state, "limit": limit})
|
|
102
|
+
)
|
|
103
|
+
return int(response.get("count", 0))
|
|
104
|
+
|
|
105
|
+
def remove(self, job_id: str) -> None:
|
|
106
|
+
self.connection.call({"cmd": "Cancel", "id": job_id})
|
|
107
|
+
|
|
108
|
+
# `cancel` kept as an alias of remove (bunqueue-native name)
|
|
109
|
+
cancel = remove
|
|
110
|
+
|
|
111
|
+
def discard(self, job_id: str) -> None:
|
|
112
|
+
self.connection.call({"cmd": "Discard", "id": job_id})
|
|
113
|
+
|
|
114
|
+
def promote_job(self, job_id: str) -> None:
|
|
115
|
+
self.connection.call({"cmd": "Promote", "id": job_id})
|
|
116
|
+
|
|
117
|
+
def promote_jobs(self, count: Optional[int] = None) -> int:
|
|
118
|
+
response = self.connection.call(
|
|
119
|
+
_compact({"cmd": "PromoteJobs", "queue": self.name, "count": count})
|
|
120
|
+
)
|
|
121
|
+
return int(response.get("count", 0))
|
|
122
|
+
|
|
123
|
+
def retry_job(self, job_id: str) -> None:
|
|
124
|
+
"""BullMQ contract: failed -> waiting (MoveToWait over TCP)."""
|
|
125
|
+
self.connection.call({"cmd": "MoveToWait", "id": job_id})
|
|
126
|
+
|
|
127
|
+
def retry_jobs(self, state: str = "failed", count: Optional[int] = None) -> int:
|
|
128
|
+
# `count` caps how many DLQ entries are retried (server >= 2.8.29);
|
|
129
|
+
# older servers ignore it and retry the whole DLQ (forward-compatible).
|
|
130
|
+
if state == "failed":
|
|
131
|
+
response = self.connection.call(
|
|
132
|
+
_compact({"cmd": "RetryDlq", "queue": self.name, "count": count})
|
|
133
|
+
)
|
|
134
|
+
elif state == "completed":
|
|
135
|
+
response = self.connection.call({"cmd": "RetryCompleted", "queue": self.name})
|
|
136
|
+
else:
|
|
137
|
+
raise ValueError("state must be 'failed' or 'completed'")
|
|
138
|
+
return int(response.get("count", 0))
|
|
139
|
+
|
|
140
|
+
def retry_completed(self, job_id: Optional[str] = None) -> int:
|
|
141
|
+
response = self.connection.call(
|
|
142
|
+
_compact({"cmd": "RetryCompleted", "queue": self.name, "id": job_id})
|
|
143
|
+
)
|
|
144
|
+
return int(response.get("count", 0))
|
|
145
|
+
|
|
146
|
+
# ----------------------------------------------------- job state changes
|
|
147
|
+
|
|
148
|
+
def update_job_progress(self, job_id: str, progress: int, message: Optional[str] = None) -> None:
|
|
149
|
+
self.connection.call(
|
|
150
|
+
_compact({"cmd": "Progress", "id": job_id, "progress": progress, "message": message})
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
def update_job_data(self, job_id: str, data: Any) -> None:
|
|
154
|
+
self.connection.call({"cmd": "Update", "id": job_id, "data": data})
|
|
155
|
+
|
|
156
|
+
def change_job_priority(self, job_id: str, priority: int, lifo: Optional[bool] = None) -> None:
|
|
157
|
+
self.connection.call(
|
|
158
|
+
_compact({"cmd": "ChangePriority", "id": job_id, "priority": priority, "lifo": lifo})
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
def change_job_delay(self, job_id: str, delay_ms: int) -> None:
|
|
162
|
+
self.connection.call({"cmd": "ChangeDelay", "id": job_id, "delay": delay_ms})
|
|
163
|
+
|
|
164
|
+
def move_job_to_wait(self, job_id: str) -> None:
|
|
165
|
+
self.connection.call({"cmd": "MoveToWait", "id": job_id})
|
|
166
|
+
|
|
167
|
+
def move_job_to_delayed(self, job_id: str, delay_ms: int) -> None:
|
|
168
|
+
self.connection.call({"cmd": "MoveToDelayed", "id": job_id, "delay": delay_ms})
|
|
169
|
+
|
|
170
|
+
def extend_job_lock(self, job_id: str, token: str, duration_ms: int) -> None:
|
|
171
|
+
self.connection.call(
|
|
172
|
+
{"cmd": "ExtendLock", "id": job_id, "token": token, "duration": duration_ms}
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def move_job_to_completed(self, job_id: str, result: Any = None, token: Optional[str] = None) -> None:
|
|
176
|
+
self.connection.call(_compact({"cmd": "ACK", "id": job_id, "result": result, "token": token}))
|
|
177
|
+
|
|
178
|
+
def move_job_to_failed(
|
|
179
|
+
self, job_id: str, error: Union[BaseException, str], token: Optional[str] = None
|
|
180
|
+
) -> None:
|
|
181
|
+
"""Explicit failure path: mirrors the worker's FAIL wire.
|
|
182
|
+
|
|
183
|
+
Passing an exception persists its ``stack`` (traceback lines, bounded
|
|
184
|
+
like the worker path) and the UnrecoverableError "do not retry"
|
|
185
|
+
intent (#111 silent-loss class), not just the message. String errors
|
|
186
|
+
travel as before."""
|
|
187
|
+
if isinstance(error, BaseException):
|
|
188
|
+
message = str(error) or error.__class__.__name__
|
|
189
|
+
# Keep the LAST lines: a Python traceback ends with the raise site
|
|
190
|
+
# and the message (opposite of a JS stack, which leads with them).
|
|
191
|
+
stack = "".join(
|
|
192
|
+
traceback.format_exception(type(error), error, error.__traceback__)
|
|
193
|
+
).splitlines()[-MAX_STACK_LINES:]
|
|
194
|
+
unrecoverable = True if isinstance(error, UnrecoverableError) else None
|
|
195
|
+
else:
|
|
196
|
+
message, stack, unrecoverable = error, None, None
|
|
197
|
+
self.connection.call(
|
|
198
|
+
_compact(
|
|
199
|
+
{
|
|
200
|
+
"cmd": "FAIL",
|
|
201
|
+
"id": job_id,
|
|
202
|
+
"error": message,
|
|
203
|
+
"stack": stack,
|
|
204
|
+
"unrecoverable": unrecoverable,
|
|
205
|
+
"token": token,
|
|
206
|
+
}
|
|
207
|
+
)
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# ------------------------------------------------------------- lifecycle
|
|
211
|
+
|
|
212
|
+
def ping(self) -> bool:
|
|
213
|
+
return self.connection.ping()
|
|
214
|
+
|
|
215
|
+
def wait_until_ready(self) -> None:
|
|
216
|
+
self.connection.connect()
|
|
217
|
+
|
|
218
|
+
def close(self) -> None:
|
|
219
|
+
if self._owns_connection:
|
|
220
|
+
self.connection.close()
|
|
221
|
+
|
|
222
|
+
def disconnect(self) -> None:
|
|
223
|
+
"""BullMQ-parity alias for :meth:`close`."""
|
|
224
|
+
self.close()
|
|
225
|
+
|
|
226
|
+
def __enter__(self) -> "Queue":
|
|
227
|
+
return self
|
|
228
|
+
|
|
229
|
+
def __exit__(self, *exc: Any) -> None:
|
|
230
|
+
self.close()
|
bunqueue/queue_admin.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Admin operations mixin for Queue: DLQ, configs, rate limits, schedulers,
|
|
2
|
+
webhooks, monitoring. Mirrors the TS client's management surface (TCP mode)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from .connection import _compact
|
|
9
|
+
from .errors import CommandError
|
|
10
|
+
from .options import build_cron_job_options
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class QueueAdminOps:
|
|
14
|
+
"""Mixin: requires ``self.name`` and ``self.connection``."""
|
|
15
|
+
|
|
16
|
+
# ------------------------------------------------------------------- dlq
|
|
17
|
+
|
|
18
|
+
def get_dlq(self, count: Optional[int] = None) -> List[Dict[str, Any]]:
|
|
19
|
+
response = self.connection.call(_compact({"cmd": "Dlq", "queue": self.name, "count": count}))
|
|
20
|
+
return list(response.get("jobs") or response.get("data") or [])
|
|
21
|
+
|
|
22
|
+
def retry_dlq(self, job_id: Optional[str] = None, count: Optional[int] = None) -> int:
|
|
23
|
+
# `count` caps how many DLQ entries are retried when no `job_id` is
|
|
24
|
+
# given (server >= 2.8.29); older servers ignore it (forward-compatible).
|
|
25
|
+
response = self.connection.call(
|
|
26
|
+
_compact({"cmd": "RetryDlq", "queue": self.name, "jobId": job_id, "count": count})
|
|
27
|
+
)
|
|
28
|
+
return int(response.get("count", 0))
|
|
29
|
+
|
|
30
|
+
def purge_dlq(self) -> int:
|
|
31
|
+
response = self.connection.call({"cmd": "PurgeDlq", "queue": self.name})
|
|
32
|
+
return int(response.get("count", 0))
|
|
33
|
+
|
|
34
|
+
def set_dlq_config(self, config: Dict[str, Any]) -> None:
|
|
35
|
+
self.connection.call({"cmd": "SetDlqConfig", "queue": self.name, "config": config})
|
|
36
|
+
|
|
37
|
+
def get_dlq_config(self) -> Dict[str, Any]:
|
|
38
|
+
response = self.connection.call({"cmd": "GetDlqConfig", "queue": self.name})
|
|
39
|
+
return dict(response.get("config") or response.get("data") or {})
|
|
40
|
+
|
|
41
|
+
# ----------------------------------------------------------------- stall
|
|
42
|
+
|
|
43
|
+
def set_stall_config(self, config: Dict[str, Any]) -> None:
|
|
44
|
+
self.connection.call({"cmd": "SetStallConfig", "queue": self.name, "config": config})
|
|
45
|
+
|
|
46
|
+
def get_stall_config(self) -> Dict[str, Any]:
|
|
47
|
+
response = self.connection.call({"cmd": "GetStallConfig", "queue": self.name})
|
|
48
|
+
return dict(response.get("config") or response.get("data") or {})
|
|
49
|
+
|
|
50
|
+
# ------------------------------------------------- rate limit/concurrency
|
|
51
|
+
|
|
52
|
+
def set_global_rate_limit(self, max_jobs: int, duration_ms: Optional[int] = None) -> None:
|
|
53
|
+
"""`duration_ms` accepted for TS-signature parity; the wire carries only `limit`."""
|
|
54
|
+
del duration_ms
|
|
55
|
+
self.connection.call({"cmd": "RateLimit", "queue": self.name, "limit": max_jobs})
|
|
56
|
+
|
|
57
|
+
def remove_global_rate_limit(self) -> None:
|
|
58
|
+
self.connection.call({"cmd": "RateLimitClear", "queue": self.name})
|
|
59
|
+
|
|
60
|
+
def set_global_concurrency(self, concurrency: int) -> None:
|
|
61
|
+
self.connection.call({"cmd": "SetConcurrency", "queue": self.name, "limit": concurrency})
|
|
62
|
+
|
|
63
|
+
def remove_global_concurrency(self) -> None:
|
|
64
|
+
self.connection.call({"cmd": "ClearConcurrency", "queue": self.name})
|
|
65
|
+
|
|
66
|
+
# -------------------------------------------------------------- scheduler
|
|
67
|
+
|
|
68
|
+
def upsert_job_scheduler(
|
|
69
|
+
self,
|
|
70
|
+
scheduler_id: str,
|
|
71
|
+
repeat: Dict[str, Any],
|
|
72
|
+
template: Optional[Dict[str, Any]] = None,
|
|
73
|
+
) -> None:
|
|
74
|
+
"""BullMQ-style scheduler.
|
|
75
|
+
|
|
76
|
+
``repeat``: {"pattern": cron, "every": ms, "tz": timezone,
|
|
77
|
+
"immediately": bool, "limit": n}. ``template``: {"name", "data", "opts"}.
|
|
78
|
+
"""
|
|
79
|
+
template = template or {}
|
|
80
|
+
opts = template.get("opts") or {}
|
|
81
|
+
data: Dict[str, Any] = {"name": template.get("name", scheduler_id)}
|
|
82
|
+
tdata = template.get("data")
|
|
83
|
+
if isinstance(tdata, dict):
|
|
84
|
+
data.update(tdata)
|
|
85
|
+
elif tdata is not None:
|
|
86
|
+
data["payload"] = tdata
|
|
87
|
+
# Priority and deduplication of spawned jobs travel as TOP-LEVEL Cron
|
|
88
|
+
# fields (the handler reads cmd.priority/uniqueKey/dedup); inside
|
|
89
|
+
# jobOptions the server's CronJobOptions silently ignores them (#111).
|
|
90
|
+
dedup = opts.get("deduplication") or {}
|
|
91
|
+
dedup_fields = _compact(
|
|
92
|
+
{"ttl": dedup.get("ttl"), "extend": dedup.get("extend"), "replace": dedup.get("replace")}
|
|
93
|
+
)
|
|
94
|
+
command = _compact(
|
|
95
|
+
{
|
|
96
|
+
"cmd": "Cron",
|
|
97
|
+
"name": scheduler_id,
|
|
98
|
+
"queue": self.name,
|
|
99
|
+
"data": data,
|
|
100
|
+
"schedule": repeat.get("pattern"),
|
|
101
|
+
"repeatEvery": repeat.get("every"),
|
|
102
|
+
"priority": opts.get("priority"),
|
|
103
|
+
"timezone": repeat.get("tz"),
|
|
104
|
+
"immediately": repeat.get("immediately"),
|
|
105
|
+
"maxLimit": repeat.get("limit"),
|
|
106
|
+
"uniqueKey": opts.get("unique_key") or dedup.get("id"),
|
|
107
|
+
"dedup": dedup_fields if dedup_fields else None,
|
|
108
|
+
"skipIfNoWorker": repeat.get("skip_if_no_worker"),
|
|
109
|
+
"skipMissedOnRestart": repeat.get("skip_missed_on_restart"),
|
|
110
|
+
"preventOverlap": repeat.get("prevent_overlap"),
|
|
111
|
+
# Map Pythonic template opts (attempts, remove_on_complete, ...)
|
|
112
|
+
# to the wire CronJobOptions, else the server silently drops
|
|
113
|
+
# them and cron-spawned jobs fall back to JOB_DEFAULTS.
|
|
114
|
+
"jobOptions": build_cron_job_options(opts),
|
|
115
|
+
}
|
|
116
|
+
)
|
|
117
|
+
self.connection.call(command)
|
|
118
|
+
|
|
119
|
+
def remove_job_scheduler(self, scheduler_id: str) -> None:
|
|
120
|
+
self.connection.call({"cmd": "CronDelete", "name": scheduler_id})
|
|
121
|
+
|
|
122
|
+
def get_job_scheduler(self, scheduler_id: str) -> Optional[Dict[str, Any]]:
|
|
123
|
+
try:
|
|
124
|
+
response = self.connection.call({"cmd": "CronGet", "name": scheduler_id})
|
|
125
|
+
except CommandError as exc:
|
|
126
|
+
if "not found" in str(exc).lower():
|
|
127
|
+
return None
|
|
128
|
+
raise
|
|
129
|
+
cron = response.get("cron") or response.get("data")
|
|
130
|
+
return dict(cron) if cron else None
|
|
131
|
+
|
|
132
|
+
def get_job_schedulers(self) -> List[Dict[str, Any]]:
|
|
133
|
+
response = self.connection.call({"cmd": "CronList"})
|
|
134
|
+
crons = response.get("crons", [])
|
|
135
|
+
return [dict(c) for c in crons if c.get("queue") == self.name]
|
|
136
|
+
|
|
137
|
+
def get_job_schedulers_count(self) -> int:
|
|
138
|
+
return len(self.get_job_schedulers())
|
|
139
|
+
|
|
140
|
+
def add_cron(self, name: str, schedule: str, data: Any = None, **repeat: Any) -> None:
|
|
141
|
+
"""Shorthand: cron-pattern scheduler (mirrors Bunqueue.cron)."""
|
|
142
|
+
self.upsert_job_scheduler(name, {"pattern": schedule, **repeat}, {"name": name, "data": data})
|
|
143
|
+
|
|
144
|
+
def every(self, name: str, interval_ms: int, data: Any = None, **repeat: Any) -> None:
|
|
145
|
+
"""Shorthand: fixed-interval scheduler (mirrors Bunqueue.every)."""
|
|
146
|
+
self.upsert_job_scheduler(name, {"every": interval_ms, **repeat}, {"name": name, "data": data})
|
|
147
|
+
|
|
148
|
+
# --------------------------------------------------------------- webhooks
|
|
149
|
+
|
|
150
|
+
def add_webhook(
|
|
151
|
+
self,
|
|
152
|
+
url: str,
|
|
153
|
+
events: List[str],
|
|
154
|
+
queue: Optional[str] = None,
|
|
155
|
+
secret: Optional[str] = None,
|
|
156
|
+
) -> Optional[str]:
|
|
157
|
+
response = self.connection.call(
|
|
158
|
+
_compact({"cmd": "AddWebhook", "url": url, "events": events, "queue": queue, "secret": secret})
|
|
159
|
+
)
|
|
160
|
+
data = response.get("data") if isinstance(response.get("data"), dict) else response
|
|
161
|
+
webhook_id = data.get("webhookId") or data.get("id")
|
|
162
|
+
return str(webhook_id) if webhook_id else None
|
|
163
|
+
|
|
164
|
+
def remove_webhook(self, webhook_id: str) -> None:
|
|
165
|
+
self.connection.call({"cmd": "RemoveWebhook", "webhookId": webhook_id})
|
|
166
|
+
|
|
167
|
+
def list_webhooks(self) -> List[Dict[str, Any]]:
|
|
168
|
+
response = self.connection.call({"cmd": "ListWebhooks"})
|
|
169
|
+
data = response.get("data")
|
|
170
|
+
if isinstance(data, dict):
|
|
171
|
+
return list(data.get("webhooks") or [])
|
|
172
|
+
return list(response.get("webhooks") or [])
|
|
173
|
+
|
|
174
|
+
def set_webhook_enabled(self, webhook_id: str, enabled: bool) -> None:
|
|
175
|
+
# wire field is `id` (SetWebhookEnabledCommand), unlike RemoveWebhook's `webhookId`
|
|
176
|
+
self.connection.call({"cmd": "SetWebhookEnabled", "id": webhook_id, "enabled": enabled})
|
|
177
|
+
|
|
178
|
+
# ------------------------------------------------------------- monitoring
|
|
179
|
+
|
|
180
|
+
def get_workers(self) -> List[Dict[str, Any]]:
|
|
181
|
+
response = self.connection.call({"cmd": "ListWorkers"})
|
|
182
|
+
data = response.get("data")
|
|
183
|
+
if isinstance(data, dict):
|
|
184
|
+
return list(data.get("workers") or [])
|
|
185
|
+
return list(response.get("workers") or [])
|
|
186
|
+
|
|
187
|
+
def get_workers_count(self) -> int:
|
|
188
|
+
return len(self.get_workers())
|
|
189
|
+
|
|
190
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
191
|
+
response = self.connection.call({"cmd": "Stats"})
|
|
192
|
+
return dict(response.get("stats", {}))
|
|
193
|
+
|
|
194
|
+
def get_metrics(self) -> Dict[str, Any]:
|
|
195
|
+
response = self.connection.call({"cmd": "Metrics"})
|
|
196
|
+
return dict(response.get("metrics", {}))
|
|
197
|
+
|
|
198
|
+
def list_queues(self) -> List[Any]:
|
|
199
|
+
"""Returns the server's queue list (names or info dicts)."""
|
|
200
|
+
response = self.connection.call({"cmd": "ListQueues"})
|
|
201
|
+
return list(response.get("queues") or response.get("data") or [])
|
bunqueue/queue_query.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Query operations mixin for Queue (jobs, states, counts, logs, children)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
|
6
|
+
|
|
7
|
+
from .connection import _compact
|
|
8
|
+
from .errors import CommandError, CommandTimeoutError
|
|
9
|
+
from .job import Job
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _values_payload(response: Dict[str, Any]) -> Dict[str, Any]:
|
|
13
|
+
"""Unwrap {data: {values: {...}}} (server) or top-level values."""
|
|
14
|
+
data = response.get("data")
|
|
15
|
+
if isinstance(data, dict) and "values" in data:
|
|
16
|
+
return dict(data.get("values") or {})
|
|
17
|
+
return dict(response.get("values") or {})
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class QueueQueryOps:
|
|
21
|
+
"""Mixin: requires ``self.name`` and ``self.connection``."""
|
|
22
|
+
|
|
23
|
+
# ------------------------------------------------------------------ jobs
|
|
24
|
+
|
|
25
|
+
def get_job(self, job_id: str) -> Optional[Job]:
|
|
26
|
+
try:
|
|
27
|
+
response = self.connection.call({"cmd": "GetJob", "id": job_id})
|
|
28
|
+
except CommandError as exc:
|
|
29
|
+
if "not found" in str(exc).lower():
|
|
30
|
+
return None
|
|
31
|
+
raise
|
|
32
|
+
raw = response.get("job")
|
|
33
|
+
return Job(raw, self.connection) if raw else None
|
|
34
|
+
|
|
35
|
+
def get_job_by_custom_id(self, custom_id: str) -> Optional[Job]:
|
|
36
|
+
try:
|
|
37
|
+
response = self.connection.call({"cmd": "GetJobByCustomId", "customId": custom_id})
|
|
38
|
+
except CommandError as exc:
|
|
39
|
+
if "not found" in str(exc).lower():
|
|
40
|
+
return None
|
|
41
|
+
raise
|
|
42
|
+
raw = response.get("job")
|
|
43
|
+
return Job(raw, self.connection) if raw else None
|
|
44
|
+
|
|
45
|
+
def get_jobs(
|
|
46
|
+
self,
|
|
47
|
+
state: Optional[Union[str, Sequence[str]]] = None,
|
|
48
|
+
start: int = 0,
|
|
49
|
+
end: int = 1000,
|
|
50
|
+
) -> List[Job]:
|
|
51
|
+
"""Mirror TS getJobsAsync: offset=start, limit=end-start."""
|
|
52
|
+
command = _compact(
|
|
53
|
+
{
|
|
54
|
+
"cmd": "GetJobs",
|
|
55
|
+
"queue": self.name,
|
|
56
|
+
"state": list(state) if isinstance(state, (list, tuple)) else state,
|
|
57
|
+
"offset": start,
|
|
58
|
+
"limit": max(end - start, 0),
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
response = self.connection.call(command)
|
|
62
|
+
return [Job(raw, self.connection) for raw in response.get("jobs", [])]
|
|
63
|
+
|
|
64
|
+
def get_waiting(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
65
|
+
return self.get_jobs("waiting", start, end)
|
|
66
|
+
|
|
67
|
+
def get_delayed(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
68
|
+
return self.get_jobs("delayed", start, end)
|
|
69
|
+
|
|
70
|
+
def get_active(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
71
|
+
return self.get_jobs("active", start, end)
|
|
72
|
+
|
|
73
|
+
def get_completed(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
74
|
+
return self.get_jobs("completed", start, end)
|
|
75
|
+
|
|
76
|
+
def get_failed(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
77
|
+
return self.get_jobs("failed", start, end)
|
|
78
|
+
|
|
79
|
+
def get_prioritized(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
80
|
+
return self.get_jobs("prioritized", start, end)
|
|
81
|
+
|
|
82
|
+
def get_waiting_children(self, start: int = 0, end: int = 1000) -> List[Job]:
|
|
83
|
+
return self.get_jobs("waiting-children", start, end)
|
|
84
|
+
|
|
85
|
+
# ----------------------------------------------------------------- state
|
|
86
|
+
|
|
87
|
+
def get_state(self, job_id: str) -> str:
|
|
88
|
+
response = self.connection.call({"cmd": "GetState", "id": job_id})
|
|
89
|
+
return str(response.get("state"))
|
|
90
|
+
|
|
91
|
+
def get_result(self, job_id: str) -> Any:
|
|
92
|
+
response = self.connection.call({"cmd": "GetResult", "id": job_id})
|
|
93
|
+
return response.get("result")
|
|
94
|
+
|
|
95
|
+
def get_progress(self, job_id: str) -> Dict[str, Any]:
|
|
96
|
+
response = self.connection.call({"cmd": "GetProgress", "id": job_id})
|
|
97
|
+
return {"progress": response.get("progress"), "message": response.get("message")}
|
|
98
|
+
|
|
99
|
+
def wait_for_job(self, job_id: str, timeout_ms: int = 30000) -> Any:
|
|
100
|
+
"""Block until the job completes; returns its result.
|
|
101
|
+
|
|
102
|
+
The server's ``WaitJob`` waiter resolves only on completion, answering
|
|
103
|
+
``{ok:true, completed:false}`` (no result) otherwise — so returning
|
|
104
|
+
``None`` would be indistinguishable from a genuine ``None`` result. On
|
|
105
|
+
non-completion we probe the job state: a ``failed`` job raises
|
|
106
|
+
:class:`CommandError` (it will not complete), everything else raises
|
|
107
|
+
:class:`CommandTimeoutError`."""
|
|
108
|
+
# The server validates 0 <= timeout <= 600000: clamp instead of erroring.
|
|
109
|
+
timeout_ms = max(0, min(timeout_ms, 600_000))
|
|
110
|
+
response = self.connection.call(
|
|
111
|
+
{"cmd": "WaitJob", "id": job_id, "timeout": timeout_ms},
|
|
112
|
+
timeout=timeout_ms / 1000 + 5,
|
|
113
|
+
)
|
|
114
|
+
if not response.get("completed"):
|
|
115
|
+
try:
|
|
116
|
+
state = self.get_state(job_id)
|
|
117
|
+
except CommandError:
|
|
118
|
+
state = None
|
|
119
|
+
if state == "failed":
|
|
120
|
+
raise CommandError(f"job {job_id} failed before completion")
|
|
121
|
+
raise CommandTimeoutError(f"waitUntilFinished timed out after {timeout_ms}ms")
|
|
122
|
+
return response.get("result")
|
|
123
|
+
|
|
124
|
+
def wait_job_until_finished(
|
|
125
|
+
self, job_id: str, queue_events: Any = None, ttl_ms: Optional[int] = None
|
|
126
|
+
) -> Any:
|
|
127
|
+
"""BullMQ-parity alias for :meth:`wait_for_job` (`queue_events` unused over TCP)."""
|
|
128
|
+
del queue_events
|
|
129
|
+
return self.wait_for_job(job_id, ttl_ms if ttl_ms is not None else 30000)
|
|
130
|
+
|
|
131
|
+
# -------------------------------------------------------------- children
|
|
132
|
+
|
|
133
|
+
def get_children_values(self, job_id: str) -> Dict[str, Any]:
|
|
134
|
+
response = self.connection.call({"cmd": "GetChildrenValues", "id": job_id})
|
|
135
|
+
return _values_payload(response)
|
|
136
|
+
|
|
137
|
+
def get_failed_children_values(self, job_id: str) -> Dict[str, Any]:
|
|
138
|
+
response = self.connection.call({"cmd": "GetFailedChildrenValues", "id": job_id})
|
|
139
|
+
return _values_payload(response)
|
|
140
|
+
|
|
141
|
+
def get_ignored_children_failures(self, job_id: str) -> Dict[str, Any]:
|
|
142
|
+
response = self.connection.call({"cmd": "GetIgnoredChildrenFailures", "id": job_id})
|
|
143
|
+
return _values_payload(response)
|
|
144
|
+
|
|
145
|
+
def remove_child_dependency(self, job_id: str) -> None:
|
|
146
|
+
self.connection.call({"cmd": "RemoveChildDependency", "id": job_id})
|
|
147
|
+
|
|
148
|
+
def remove_unprocessed_children(self, job_id: str) -> None:
|
|
149
|
+
self.connection.call({"cmd": "RemoveUnprocessedChildren", "id": job_id})
|
|
150
|
+
|
|
151
|
+
def get_deduplication_job_id(self, deduplication_id: str) -> Optional[str]:
|
|
152
|
+
"""Id of the job currently holding this deduplication key (customId)."""
|
|
153
|
+
job = self.get_job_by_custom_id(deduplication_id)
|
|
154
|
+
return job.id if job else None
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------- counts
|
|
157
|
+
|
|
158
|
+
def get_job_counts(self) -> Dict[str, int]:
|
|
159
|
+
response = self.connection.call({"cmd": "GetJobCounts", "queue": self.name})
|
|
160
|
+
return dict(response.get("counts", {}))
|
|
161
|
+
|
|
162
|
+
def get_waiting_count(self) -> int:
|
|
163
|
+
return self.get_job_counts().get("waiting", 0)
|
|
164
|
+
|
|
165
|
+
def get_active_count(self) -> int:
|
|
166
|
+
return self.get_job_counts().get("active", 0)
|
|
167
|
+
|
|
168
|
+
def get_completed_count(self) -> int:
|
|
169
|
+
return self.get_job_counts().get("completed", 0)
|
|
170
|
+
|
|
171
|
+
def get_failed_count(self) -> int:
|
|
172
|
+
return self.get_job_counts().get("failed", 0)
|
|
173
|
+
|
|
174
|
+
def get_delayed_count(self) -> int:
|
|
175
|
+
return self.get_job_counts().get("delayed", 0)
|
|
176
|
+
|
|
177
|
+
def get_prioritized_count(self) -> int:
|
|
178
|
+
return self.get_job_counts().get("prioritized", 0)
|
|
179
|
+
|
|
180
|
+
def get_waiting_children_count(self) -> int:
|
|
181
|
+
return self.get_job_counts().get("waiting-children", 0)
|
|
182
|
+
|
|
183
|
+
def count(self) -> int:
|
|
184
|
+
response = self.connection.call({"cmd": "Count", "queue": self.name})
|
|
185
|
+
return int(response.get("count", 0))
|
|
186
|
+
|
|
187
|
+
def get_counts_per_priority(self) -> Dict[str, int]:
|
|
188
|
+
response = self.connection.call({"cmd": "GetCountsPerPriority", "queue": self.name})
|
|
189
|
+
return dict(response.get("counts") or response.get("data") or {})
|
|
190
|
+
|
|
191
|
+
# ------------------------------------------------------------------ logs
|
|
192
|
+
|
|
193
|
+
def add_job_log(self, job_id: str, message: str, level: Optional[str] = None) -> None:
|
|
194
|
+
self.connection.call(_compact({"cmd": "AddLog", "id": job_id, "message": message, "level": level}))
|
|
195
|
+
|
|
196
|
+
def get_job_logs(self, job_id: str, start: Optional[int] = None, end: Optional[int] = None) -> List[Any]:
|
|
197
|
+
response = self.connection.call(_compact({"cmd": "GetLogs", "id": job_id, "start": start, "end": end}))
|
|
198
|
+
data = response.get("data")
|
|
199
|
+
if isinstance(data, dict):
|
|
200
|
+
return list(data.get("logs") or [])
|
|
201
|
+
return list(response.get("logs") or [])
|
|
202
|
+
|
|
203
|
+
def clear_job_logs(self, job_id: str, keep_logs: Optional[int] = None) -> None:
|
|
204
|
+
self.connection.call(_compact({"cmd": "ClearLogs", "id": job_id, "keepLogs": keep_logs}))
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Simple Mode: Queue + Worker in a single object (1:1 with the official
|
|
2
|
+
`Bunqueue` class in src/client/bunqueue.ts, TCP mode only)."""
|
|
3
|
+
|
|
4
|
+
from .app import Bunqueue
|
|
5
|
+
from .cancellation import CancelSignal, CancellationManager
|
|
6
|
+
from .circuit_breaker import WorkerCircuitBreaker
|
|
7
|
+
from .retry import calculate_backoff, execute_with_retry
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"Bunqueue",
|
|
11
|
+
"CancelSignal",
|
|
12
|
+
"CancellationManager",
|
|
13
|
+
"WorkerCircuitBreaker",
|
|
14
|
+
"calculate_backoff",
|
|
15
|
+
"execute_with_retry",
|
|
16
|
+
]
|