queue-jobs-worker 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/LICENSE +21 -0
- package/README.md +821 -0
- package/dist/core/backoff.d.ts +24 -0
- package/dist/core/backoff.d.ts.map +1 -0
- package/dist/core/client.d.ts +93 -0
- package/dist/core/client.d.ts.map +1 -0
- package/dist/core/id.d.ts +9 -0
- package/dist/core/id.d.ts.map +1 -0
- package/dist/core/index.d.ts +7 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/job.d.ts +75 -0
- package/dist/core/job.d.ts.map +1 -0
- package/dist/core/queue.d.ts +70 -0
- package/dist/core/queue.d.ts.map +1 -0
- package/dist/core/worker.d.ts +65 -0
- package/dist/core/worker.d.ts.map +1 -0
- package/dist/events/emitter.d.ts +24 -0
- package/dist/events/emitter.d.ts.map +1 -0
- package/dist/index.cjs +2229 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +39 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2219 -0
- package/dist/index.js.map +1 -0
- package/dist/storage/in-memory.adapter.d.ts +32 -0
- package/dist/storage/in-memory.adapter.d.ts.map +1 -0
- package/dist/storage/index.d.ts +5 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/mysql.adapter.d.ts +37 -0
- package/dist/storage/mysql.adapter.d.ts.map +1 -0
- package/dist/storage/postgres.adapter.d.ts +37 -0
- package/dist/storage/postgres.adapter.d.ts.map +1 -0
- package/dist/storage/redis.adapter.d.ts +44 -0
- package/dist/storage/redis.adapter.d.ts.map +1 -0
- package/dist/types/client.types.d.ts +41 -0
- package/dist/types/client.types.d.ts.map +1 -0
- package/dist/types/events.types.d.ts +22 -0
- package/dist/types/events.types.d.ts.map +1 -0
- package/dist/types/index.d.ts +10 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/job.types.d.ts +97 -0
- package/dist/types/job.types.d.ts.map +1 -0
- package/dist/types/queue.types.d.ts +43 -0
- package/dist/types/queue.types.d.ts.map +1 -0
- package/dist/types/storage.types.d.ts +120 -0
- package/dist/types/storage.types.d.ts.map +1 -0
- package/dist/types/worker.types.d.ts +25 -0
- package/dist/types/worker.types.d.ts.map +1 -0
- package/package.json +97 -0
package/README.md
ADDED
|
@@ -0,0 +1,821 @@
|
|
|
1
|
+
# queue-jobs-worker
|
|
2
|
+
|
|
3
|
+
A production-ready background job queue for Node.js — persistent, reliable, and TypeScript-first.
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/queue-jobs-worker)
|
|
6
|
+
[](./LICENSE)
|
|
7
|
+
[](https://nodejs.org)
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## Overview
|
|
12
|
+
|
|
13
|
+
`queue-jobs-worker` lets you push work into a persistent queue and process it in the background — outside the main request lifecycle. You define the processor; the library handles everything else: queueing, persistence, retries, scheduling, concurrency, and failure recovery.
|
|
14
|
+
|
|
15
|
+
**Supports:**
|
|
16
|
+
- In-memory (dev / testing)
|
|
17
|
+
- Redis (node-redis v4+)
|
|
18
|
+
- PostgreSQL (node-postgres / pg)
|
|
19
|
+
- MySQL (mysql2)
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## Table of Contents
|
|
24
|
+
|
|
25
|
+
- [Installation](#installation)
|
|
26
|
+
- [Quick Start](#quick-start)
|
|
27
|
+
- [Dialects](#dialects)
|
|
28
|
+
- [Memory](#memory-no-setup-required)
|
|
29
|
+
- [Redis](#redis)
|
|
30
|
+
- [PostgreSQL](#postgresql)
|
|
31
|
+
- [MySQL](#mysql)
|
|
32
|
+
- [Core Concepts](#core-concepts)
|
|
33
|
+
- [Configuration](#configuration)
|
|
34
|
+
- [Enqueueing Jobs](#enqueueing-jobs)
|
|
35
|
+
- [Processing Jobs](#processing-jobs)
|
|
36
|
+
- [Workers](#workers)
|
|
37
|
+
- [Events](#events)
|
|
38
|
+
- [Querying Jobs](#querying-jobs)
|
|
39
|
+
- [Retry & Backoff](#retry--backoff)
|
|
40
|
+
- [Scheduling](#scheduling)
|
|
41
|
+
- [Priority](#priority)
|
|
42
|
+
- [Rate Limiting](#rate-limiting)
|
|
43
|
+
- [Dead Letter Queue](#dead-letter-queue)
|
|
44
|
+
- [Graceful Shutdown](#graceful-shutdown)
|
|
45
|
+
- [Custom Storage Adapter](#custom-storage-adapter)
|
|
46
|
+
- [API Reference](#api-reference)
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Installation
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
npm install queue-jobs-worker
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Install only the driver(s) you actually use:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# Redis
|
|
60
|
+
npm install redis
|
|
61
|
+
|
|
62
|
+
# PostgreSQL
|
|
63
|
+
npm install pg
|
|
64
|
+
|
|
65
|
+
# MySQL
|
|
66
|
+
npm install mysql2
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Quick Start
|
|
72
|
+
|
|
73
|
+
**TypeScript**
|
|
74
|
+
```ts
|
|
75
|
+
import { QueueClient } from "queue-jobs-worker";
|
|
76
|
+
|
|
77
|
+
const client = new QueueClient();
|
|
78
|
+
|
|
79
|
+
// Generic type parameter gives you typed job.data
|
|
80
|
+
const emails = client.createQueue<{ to: string; subject: string }>("emails");
|
|
81
|
+
|
|
82
|
+
emails.process("send-email", async (job) => {
|
|
83
|
+
await sendEmail(job.data.to, job.data.subject);
|
|
84
|
+
// Throw to trigger retry; return to mark as completed
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
emails.createWorker({ concurrency: 5 });
|
|
88
|
+
|
|
89
|
+
await emails.enqueue("send-email", {
|
|
90
|
+
to: "user@example.com",
|
|
91
|
+
subject: "Welcome!",
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
process.on("SIGTERM", async () => {
|
|
95
|
+
await client.close();
|
|
96
|
+
process.exit(0);
|
|
97
|
+
});
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
**JavaScript**
|
|
101
|
+
```js
|
|
102
|
+
const { QueueClient } = require("queue-jobs-worker");
|
|
103
|
+
|
|
104
|
+
const client = new QueueClient();
|
|
105
|
+
|
|
106
|
+
// No generic — job.data is untyped
|
|
107
|
+
const emails = client.createQueue("emails");
|
|
108
|
+
|
|
109
|
+
emails.process("send-email", async (job) => {
|
|
110
|
+
await sendEmail(job.data.to, job.data.subject);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
emails.createWorker({ concurrency: 5 });
|
|
114
|
+
|
|
115
|
+
await emails.enqueue("send-email", {
|
|
116
|
+
to: "user@example.com",
|
|
117
|
+
subject: "Welcome!",
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
process.on("SIGTERM", async () => {
|
|
121
|
+
await client.close();
|
|
122
|
+
process.exit(0);
|
|
123
|
+
});
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
## Dialects
|
|
129
|
+
|
|
130
|
+
### Memory (no setup required)
|
|
131
|
+
|
|
132
|
+
Uses an in-process Map. Data is lost on restart. Perfect for development and tests.
|
|
133
|
+
|
|
134
|
+
```js
|
|
135
|
+
const client = new QueueClient();
|
|
136
|
+
// or explicitly:
|
|
137
|
+
const client = new QueueClient({ dialect: "memory" });
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
`init()` is optional for memory — it's a no-op. All other dialects require it.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
### Redis
|
|
145
|
+
|
|
146
|
+
Requires `redis` (node-redis v4+): `npm install redis`
|
|
147
|
+
|
|
148
|
+
```js
|
|
149
|
+
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
150
|
+
const { QueueClient } = require("queue-jobs-worker");
|
|
151
|
+
|
|
152
|
+
const client = new QueueClient({
|
|
153
|
+
dialect: "redis",
|
|
154
|
+
connectionString: "redis://localhost:6379",
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
await client.init(); // connects + PING — throws if unreachable
|
|
158
|
+
|
|
159
|
+
const jobs = client.createQueue("jobs");
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
**With authentication:**
|
|
163
|
+
```js
|
|
164
|
+
const client = new QueueClient({
|
|
165
|
+
dialect: "redis",
|
|
166
|
+
connectionString: "redis://:yourpassword@redis-host:6379/0",
|
|
167
|
+
});
|
|
168
|
+
await client.init();
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
**With TLS (Redis Cloud, Upstash, etc.):**
|
|
172
|
+
```js
|
|
173
|
+
const client = new QueueClient({
|
|
174
|
+
dialect: "redis",
|
|
175
|
+
connectionString: "rediss://user:password@host:6380",
|
|
176
|
+
});
|
|
177
|
+
await client.init();
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
**What `init()` does for Redis:**
|
|
181
|
+
- Creates the node-redis client
|
|
182
|
+
- Calls `client.connect()`
|
|
183
|
+
- Sends `PING` and asserts the response is `PONG`
|
|
184
|
+
- Throws a descriptive error if the connection fails
|
|
185
|
+
|
|
186
|
+
**Key structure in Redis** (prefix: `qjw:`):
|
|
187
|
+
```
|
|
188
|
+
qjw:job:{id} → Hash (all job fields)
|
|
189
|
+
qjw:queue:{name}:waiting → Sorted Set (score = -priority)
|
|
190
|
+
qjw:queue:{name}:delayed → Sorted Set (score = runAt ms)
|
|
191
|
+
qjw:queue:{name}:active → Set
|
|
192
|
+
qjw:queue:{name}:completed → Set
|
|
193
|
+
qjw:queue:{name}:dead → Set
|
|
194
|
+
qjw:rate:{name} → String (rate-limit counter)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Job claiming uses a **Lua script** so it is atomic — two concurrent workers can never claim the same job.
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
### PostgreSQL
|
|
202
|
+
|
|
203
|
+
Requires `pg` (node-postgres): `npm install pg`
|
|
204
|
+
|
|
205
|
+
```js
|
|
206
|
+
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
207
|
+
const { QueueClient } = require("queue-jobs-worker");
|
|
208
|
+
|
|
209
|
+
const client = new QueueClient({
|
|
210
|
+
dialect: "postgres",
|
|
211
|
+
connectionString: "postgresql://user:password@localhost:5432/mydb",
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
await client.init(); // connects + SELECT 1 + creates tables
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
**What `init()` does for PostgreSQL:**
|
|
218
|
+
- Creates a connection pool (`pg.Pool`)
|
|
219
|
+
- Runs `SELECT 1` to verify connectivity
|
|
220
|
+
- Executes `CREATE TABLE IF NOT EXISTS` for `qjw_jobs` and `qjw_rate_limits` — **idempotent, safe to run on every startup**
|
|
221
|
+
- Throws a descriptive error if the connection fails
|
|
222
|
+
|
|
223
|
+
**Tables created automatically** (prefix: `qjw_`):
|
|
224
|
+
```sql
|
|
225
|
+
qjw_jobs -- stores every job and its full lifecycle state
|
|
226
|
+
qjw_rate_limits -- sliding-window rate-limit counters
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Claiming uses `SELECT ... FOR UPDATE SKIP LOCKED` inside a transaction — safe for any number of concurrent workers.
|
|
230
|
+
|
|
231
|
+
**With SSL (Heroku, Supabase, Neon, etc.):**
|
|
232
|
+
```js
|
|
233
|
+
const client = new QueueClient({
|
|
234
|
+
dialect: "postgres",
|
|
235
|
+
connectionString: process.env.DATABASE_URL,
|
|
236
|
+
// pg respects ?sslmode=require in the connection string
|
|
237
|
+
});
|
|
238
|
+
await client.init();
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
---
|
|
242
|
+
|
|
243
|
+
### MySQL
|
|
244
|
+
|
|
245
|
+
Requires `mysql2`: `npm install mysql2`
|
|
246
|
+
|
|
247
|
+
```js
|
|
248
|
+
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
249
|
+
const { QueueClient } = require("queue-jobs-worker");
|
|
250
|
+
|
|
251
|
+
const client = new QueueClient({
|
|
252
|
+
dialect: "mysql",
|
|
253
|
+
connectionString: "mysql://user:password@localhost:3306/mydb",
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
await client.init(); // connects + SELECT 1 + creates tables
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
**What `init()` does for MySQL:**
|
|
260
|
+
- Creates a connection pool (`mysql2.createPool`)
|
|
261
|
+
- Runs `SELECT 1` to verify connectivity
|
|
262
|
+
- Executes `CREATE TABLE IF NOT EXISTS` for `qjw_jobs` and `qjw_rate_limits` — **idempotent**
|
|
263
|
+
- Throws a descriptive error if the connection fails
|
|
264
|
+
|
|
265
|
+
**Tables created automatically** (prefix: `qjw_`):
|
|
266
|
+
```sql
|
|
267
|
+
qjw_jobs -- full job state (InnoDB, utf8mb4)
|
|
268
|
+
qjw_rate_limits -- sliding-window rate-limit counters
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Claiming uses `SELECT ... FOR UPDATE SKIP LOCKED` inside a transaction.
|
|
272
|
+
|
|
273
|
+
---
|
|
274
|
+
|
|
275
|
+
## Core Concepts
|
|
276
|
+
|
|
277
|
+
| Concept | Description |
|
|
278
|
+
|---|---|
|
|
279
|
+
| `QueueClient` | Entry point — holds config, storage, and all queues |
|
|
280
|
+
| `Queue` | An independent stream of jobs with its own config |
|
|
281
|
+
| `Job` | A unit of work — passed to your processor |
|
|
282
|
+
| `Worker` | Claims and executes jobs from a queue |
|
|
283
|
+
| `Processor` | Your function — `async (job) => { ... }` |
|
|
284
|
+
| `StorageAdapter` | Interface between the core and the database |
|
|
285
|
+
| DLQ | Dead Letter Queue — permanently failed jobs land here |
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## Configuration
|
|
290
|
+
|
|
291
|
+
Config is layered — more specific settings override broader ones:
|
|
292
|
+
|
|
293
|
+
```
|
|
294
|
+
Client defaults → Queue options → Worker options → Job options
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
```js
|
|
298
|
+
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
299
|
+
const { QueueClient } = require("queue-jobs-worker");
|
|
300
|
+
|
|
301
|
+
const client = new QueueClient({
|
|
302
|
+
dialect: "redis",
|
|
303
|
+
connectionString: process.env.REDIS_URL,
|
|
304
|
+
|
|
305
|
+
defaults: {
|
|
306
|
+
attempts: 3, // max retry attempts per job
|
|
307
|
+
retryDelay: 1000, // base retry delay in ms
|
|
308
|
+
backoff: "exponential", // "fixed" | "linear" | "exponential"
|
|
309
|
+
timeout: 30_000, // per-attempt timeout in ms
|
|
310
|
+
concurrency: 10, // worker concurrency
|
|
311
|
+
pollInterval: 1_000, // how often workers poll for new jobs (ms)
|
|
312
|
+
stalledInterval: 30_000,// how often to check for stalled jobs (ms)
|
|
313
|
+
lockDuration: 60_000, // how long a job lock is valid (ms)
|
|
314
|
+
rateLimit: {
|
|
315
|
+
max: 100,
|
|
316
|
+
duration: 60_000, // 100 jobs per minute
|
|
317
|
+
},
|
|
318
|
+
},
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
await client.init();
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
---
|
|
325
|
+
|
|
326
|
+
## Enqueueing Jobs
|
|
327
|
+
|
|
328
|
+
**TypeScript**
|
|
329
|
+
```ts
|
|
330
|
+
// Generic type gives you autocomplete and type-safety on job.data
|
|
331
|
+
const queue = client.createQueue<{ userId: string }>("notifications");
|
|
332
|
+
|
|
333
|
+
await queue.enqueue("send-push", { userId: "u_123" });
|
|
334
|
+
|
|
335
|
+
// With options
|
|
336
|
+
await queue.enqueue("send-push", { userId: "u_123" }, {
|
|
337
|
+
attempts: 5,
|
|
338
|
+
retryDelay: 2000,
|
|
339
|
+
backoff: "linear",
|
|
340
|
+
timeout: 10_000,
|
|
341
|
+
priority: 10, // higher = processed first (default: 0)
|
|
342
|
+
});
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
**JavaScript**
|
|
346
|
+
```js
|
|
347
|
+
const queue = client.createQueue("notifications");
|
|
348
|
+
|
|
349
|
+
await queue.enqueue("send-push", { userId: "u_123" });
|
|
350
|
+
|
|
351
|
+
await queue.enqueue("send-push", { userId: "u_123" }, {
|
|
352
|
+
attempts: 5,
|
|
353
|
+
retryDelay: 2000,
|
|
354
|
+
backoff: "linear",
|
|
355
|
+
timeout: 10_000,
|
|
356
|
+
priority: 10,
|
|
357
|
+
});
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
---
|
|
361
|
+
|
|
362
|
+
## Processing Jobs
|
|
363
|
+
|
|
364
|
+
Register a processor before starting the worker:
|
|
365
|
+
|
|
366
|
+
```js
|
|
367
|
+
queue.process("send-push", async (job) => {
|
|
368
|
+
const { userId } = job.data;
|
|
369
|
+
|
|
370
|
+
await pushService.send(userId, "You have a new message");
|
|
371
|
+
|
|
372
|
+
// Return to mark as completed.
|
|
373
|
+
// Throw any error to mark as failed (triggers retry or DLQ).
|
|
374
|
+
});
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
The `job` object exposes:
|
|
378
|
+
|
|
379
|
+
```js
|
|
380
|
+
job.id // unique stable ID
|
|
381
|
+
job.type // job type string
|
|
382
|
+
job.data // your payload (never logged)
|
|
383
|
+
job.attemptsMade // attempts already executed
|
|
384
|
+
job.maxAttempts // maximum allowed
|
|
385
|
+
job.attemptsRemaining
|
|
386
|
+
job.attemptHistory // array of past attempt records
|
|
387
|
+
job.status // current status
|
|
388
|
+
job.priority
|
|
389
|
+
job.runAt
|
|
390
|
+
job.createdAt
|
|
391
|
+
job.updatedAt
|
|
392
|
+
|
|
393
|
+
// helpers
|
|
394
|
+
job.isActive()
|
|
395
|
+
job.isCompleted()
|
|
396
|
+
job.isWaiting()
|
|
397
|
+
job.isDelayed()
|
|
398
|
+
job.isDead()
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
---
|
|
402
|
+
|
|
403
|
+
## Workers
|
|
404
|
+
|
|
405
|
+
```js
|
|
406
|
+
const worker = queue.createWorker({
|
|
407
|
+
concurrency: 10, // max simultaneous jobs
|
|
408
|
+
shutdownTimeout: 30_000, // ms to wait for active jobs during shutdown
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
console.log(worker.status); // "idle" | "running" | "stopping" | "stopped"
|
|
412
|
+
console.log(worker.id); // unique worker ID
|
|
413
|
+
|
|
414
|
+
await worker.stop();
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
You can create multiple workers on the same queue — they coordinate through the storage layer:
|
|
418
|
+
|
|
419
|
+
```js
|
|
420
|
+
const w1 = queue.createWorker({ concurrency: 5 });
|
|
421
|
+
const w2 = queue.createWorker({ concurrency: 5 });
|
|
422
|
+
// Total capacity: 10 concurrent jobs
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
---
|
|
426
|
+
|
|
427
|
+
## Events
|
|
428
|
+
|
|
429
|
+
All lifecycle events are emitted on the client. Subscribe before creating queues/workers:
|
|
430
|
+
|
|
431
|
+
```js
|
|
432
|
+
// Job events
|
|
433
|
+
client.on("job:enqueued", (job) => console.log("Enqueued:", job.id));
|
|
434
|
+
client.on("job:started", (job) => console.log("Started:", job.id));
|
|
435
|
+
client.on("job:completed", (job) => console.log("Done:", job.id));
|
|
436
|
+
client.on("job:failed", (job, err) => console.error("Failed:", job.id, err.message));
|
|
437
|
+
client.on("job:retrying", (job, err, nextRunAt) => {
|
|
438
|
+
console.log(`Retrying ${job.id} at ${nextRunAt}`);
|
|
439
|
+
});
|
|
440
|
+
client.on("job:dead", (job, err) => console.error("DLQ:", job.id, err.message));
|
|
441
|
+
client.on("job:stalled", (jobId) => console.warn("Stalled:", jobId));
|
|
442
|
+
client.on("job:recovered", (jobId) => console.log("Recovered:", jobId));
|
|
443
|
+
|
|
444
|
+
// Worker events
|
|
445
|
+
client.on("worker:started", (workerId) => console.log("Worker started:", workerId));
|
|
446
|
+
client.on("worker:stopped", (workerId) => console.log("Worker stopped:", workerId));
|
|
447
|
+
client.on("worker:status", (workerId, status) => console.log(workerId, "→", status));
|
|
448
|
+
client.on("worker:error", (workerId, err) => console.error("Worker error:", err));
|
|
449
|
+
|
|
450
|
+
// Queue / system errors
|
|
451
|
+
client.on("queue:error", (queueName, err) => console.error("Queue error:", queueName, err));
|
|
452
|
+
client.on("error", (err) => console.error("Error:", err));
|
|
453
|
+
|
|
454
|
+
// Remove a listener
|
|
455
|
+
client.off("job:completed", myListener);
|
|
456
|
+
|
|
457
|
+
// One-time listener
|
|
458
|
+
client.once("job:dead", (job, err) => alertTeam(job, err));
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
---
|
|
462
|
+
|
|
463
|
+
## Querying Jobs
|
|
464
|
+
|
|
465
|
+
```js
|
|
466
|
+
const job = await queue.getJob("job-id-here");
|
|
467
|
+
if (job) {
|
|
468
|
+
console.log(job.status, job.attemptsMade);
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Jobs by status (paginated)
|
|
472
|
+
const waiting = await queue.getJobs("waiting", 50, 0); // limit, offset
|
|
473
|
+
const active = await queue.getJobs("active");
|
|
474
|
+
const completed = await queue.getJobs("completed", 100, 0);
|
|
475
|
+
const dead = await queue.getJobs("dead");
|
|
476
|
+
|
|
477
|
+
// Counts per status
|
|
478
|
+
const counts = await queue.getJobCounts();
|
|
479
|
+
// {
|
|
480
|
+
// waiting: 12,
|
|
481
|
+
// active: 3,
|
|
482
|
+
// completed: 204,
|
|
483
|
+
// delayed: 5,
|
|
484
|
+
// dead: 1
|
|
485
|
+
// }
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
---
|
|
489
|
+
|
|
490
|
+
## Retry & Backoff
|
|
491
|
+
|
|
492
|
+
Control retry behaviour at the client, queue, or job level:
|
|
493
|
+
|
|
494
|
+
```js
|
|
495
|
+
// Queue-level
|
|
496
|
+
const queue = client.createQueue("tasks", {
|
|
497
|
+
attempts: 5,
|
|
498
|
+
retryDelay: 2000,
|
|
499
|
+
backoff: "exponential",
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
// Job-level override
|
|
503
|
+
await queue.enqueue("task", payload, {
|
|
504
|
+
attempts: 3,
|
|
505
|
+
retryDelay: 500,
|
|
506
|
+
backoff: "fixed",
|
|
507
|
+
});
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
**Backoff strategies:**
|
|
511
|
+
|
|
512
|
+
| Strategy | Formula | Example (base = 1 s) |
|
|
513
|
+
|---|---|---|
|
|
514
|
+
| `fixed` | `baseDelay` | 1 s, 1 s, 1 s |
|
|
515
|
+
| `linear` | `baseDelay × attempt` | 1 s, 2 s, 3 s |
|
|
516
|
+
| `exponential` | `baseDelay × 2^(attempt-1)` (max 10 min) | 1 s, 2 s, 4 s, 8 s |
|
|
517
|
+
|
|
518
|
+
Each failed attempt is recorded in `job.attemptHistory`:
|
|
519
|
+
|
|
520
|
+
```js
|
|
521
|
+
job.attemptHistory.forEach((attempt) => {
|
|
522
|
+
console.log(`Attempt ${attempt.attempt}: ${attempt.error}`);
|
|
523
|
+
});
|
|
524
|
+
```
|
|
525
|
+
|
|
526
|
+
---
|
|
527
|
+
|
|
528
|
+
## Scheduling
|
|
529
|
+
|
|
530
|
+
```js
|
|
531
|
+
// Run 30 seconds from now
|
|
532
|
+
await queue.enqueue("reminder", payload, {
|
|
533
|
+
schedule: { delay: 30_000 },
|
|
534
|
+
});
|
|
535
|
+
|
|
536
|
+
// Run at a specific time
|
|
537
|
+
await queue.enqueue("report", payload, {
|
|
538
|
+
schedule: { runAt: "2026-09-01T09:00:00Z" },
|
|
539
|
+
});
|
|
540
|
+
|
|
541
|
+
// Store a cron expression (integration point for recurring jobs)
|
|
542
|
+
await queue.enqueue("cleanup", payload, {
|
|
543
|
+
schedule: { cron: "0 3 * * *" },
|
|
544
|
+
});
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
Delayed jobs are not eligible until their `runAt` time. The Redis adapter promotes them automatically inside the Lua claim script; SQL adapters check `run_at` in the `WHERE` clause.
|
|
548
|
+
|
|
549
|
+
---
|
|
550
|
+
|
|
551
|
+
## Priority
|
|
552
|
+
|
|
553
|
+
Higher values are processed first. Default is `0`.
|
|
554
|
+
|
|
555
|
+
```js
|
|
556
|
+
await queue.enqueue("urgent-task", payload, { priority: 100 });
|
|
557
|
+
await queue.enqueue("normal-task", payload, { priority: 0 });
|
|
558
|
+
await queue.enqueue("low-task", payload, { priority: -10 });
|
|
559
|
+
|
|
560
|
+
// Processing order: urgent → normal → low
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
---
|
|
564
|
+
|
|
565
|
+
## Rate Limiting
|
|
566
|
+
|
|
567
|
+
Limit how many jobs are processed per time window:
|
|
568
|
+
|
|
569
|
+
```js
|
|
570
|
+
// Queue-level rate limit
|
|
571
|
+
const queue = client.createQueue("webhooks", {
|
|
572
|
+
rateLimit: {
|
|
573
|
+
max: 50, // max 50 jobs
|
|
574
|
+
duration: 60_000, // per 60 seconds
|
|
575
|
+
},
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
// Or as a client default
|
|
579
|
+
const client = new QueueClient({
|
|
580
|
+
defaults: {
|
|
581
|
+
rateLimit: { max: 100, duration: 60_000 },
|
|
582
|
+
},
|
|
583
|
+
});
|
|
584
|
+
```
|
|
585
|
+
|
|
586
|
+
When the limit is reached, workers skip claiming until the window resets. Jobs stay in the queue and are not lost.
|
|
587
|
+
|
|
588
|
+
---
|
|
589
|
+
|
|
590
|
+
## Dead Letter Queue
|
|
591
|
+
|
|
592
|
+
When a job exhausts all retry attempts it is moved to the DLQ (status: `"dead"`).
|
|
593
|
+
|
|
594
|
+
```js
|
|
595
|
+
client.on("job:dead", async (job, error) => {
|
|
596
|
+
await alertOncall({
|
|
597
|
+
jobId: job.id,
|
|
598
|
+
type: job.type,
|
|
599
|
+
error: error.message,
|
|
600
|
+
attempts: job.attemptsMade,
|
|
601
|
+
});
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
// Query dead jobs
|
|
605
|
+
const deadJobs = await queue.getJobs("dead");
|
|
606
|
+
|
|
607
|
+
// Inspect a dead job's full attempt history
|
|
608
|
+
const job = await queue.getJob(jobId);
|
|
609
|
+
if (job) {
|
|
610
|
+
job.attemptHistory.forEach((a) => {
|
|
611
|
+
console.log(`Attempt ${a.attempt} failed: ${a.error}`);
|
|
612
|
+
});
|
|
613
|
+
}
|
|
614
|
+
```
|
|
615
|
+
|
|
616
|
+
---
|
|
617
|
+
|
|
618
|
+
## Graceful Shutdown
|
|
619
|
+
|
|
620
|
+
Always call `client.close()` before your process exits:
|
|
621
|
+
|
|
622
|
+
```js
|
|
623
|
+
async function shutdown() {
|
|
624
|
+
await client.close(); // stops workers, releases locks, closes DB connections
|
|
625
|
+
process.exit(0);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
process.on("SIGTERM", shutdown);
|
|
629
|
+
process.on("SIGINT", shutdown);
|
|
630
|
+
```
|
|
631
|
+
|
|
632
|
+
During shutdown each worker:
|
|
633
|
+
1. Stops polling for new jobs
|
|
634
|
+
2. Waits up to `shutdownTimeout` (default 30 s) for active jobs to finish
|
|
635
|
+
3. Releases DB/Redis connections
|
|
636
|
+
|
|
637
|
+
Interrupted jobs remain recoverable — the stalled-job recovery mechanism will reclaim them on the next startup.
|
|
638
|
+
|
|
639
|
+
---
|
|
640
|
+
|
|
641
|
+
## Custom Storage Adapter
|
|
642
|
+
|
|
643
|
+
Implement the `StorageAdapter` interface to add your own backend:
|
|
644
|
+
|
|
645
|
+
**TypeScript**
|
|
646
|
+
```ts
|
|
647
|
+
import type { StorageAdapter } from "queue-jobs-worker";
|
|
648
|
+
|
|
649
|
+
class MongoStorageAdapter implements StorageAdapter {
|
|
650
|
+
async initialize() { /* connect, create indexes */ }
|
|
651
|
+
async close() { /* disconnect */ }
|
|
652
|
+
async enqueue(input) { /* ... */ }
|
|
653
|
+
async claim(input) { /* atomic claim */ }
|
|
654
|
+
async complete(jobId) { /* ... */ }
|
|
655
|
+
async requeue(input) { /* ... */ }
|
|
656
|
+
async moveToDlq(input) { /* ... */ }
|
|
657
|
+
async releaseLock(jobId) { /* ... */ }
|
|
658
|
+
async recoverStalledJobs(queue, now) { /* ... */ }
|
|
659
|
+
async getJob(jobId) { /* ... */ }
|
|
660
|
+
async getJobs(filter) { /* ... */ }
|
|
661
|
+
async getJobCounts(queue) { /* ... */ }
|
|
662
|
+
async checkAndIncrementRateLimit(queue, max, windowMs, now) { /* ... */ }
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const client = QueueClient.withAdapter(new MongoStorageAdapter(), {
|
|
666
|
+
defaults: { attempts: 5 },
|
|
667
|
+
});
|
|
668
|
+
await client.init();
|
|
669
|
+
```
|
|
670
|
+
|
|
671
|
+
**JavaScript**
|
|
672
|
+
```js
|
|
673
|
+
const { QueueClient } = require("queue-jobs-worker");
|
|
674
|
+
|
|
675
|
+
// In JS there's no interface to implement — just match the method signatures
|
|
676
|
+
class MongoStorageAdapter {
|
|
677
|
+
async initialize() { /* connect, create indexes */ }
|
|
678
|
+
async close() { /* disconnect */ }
|
|
679
|
+
async enqueue(input) { /* ... */ }
|
|
680
|
+
async claim(input) { /* atomic claim */ }
|
|
681
|
+
async complete(jobId) { /* ... */ }
|
|
682
|
+
async requeue(input) { /* ... */ }
|
|
683
|
+
async moveToDlq(input) { /* ... */ }
|
|
684
|
+
async releaseLock(jobId) { /* ... */ }
|
|
685
|
+
async recoverStalledJobs(queue, now) { /* ... */ }
|
|
686
|
+
async getJob(jobId) { /* ... */ }
|
|
687
|
+
async getJobs(filter) { /* ... */ }
|
|
688
|
+
async getJobCounts(queue) { /* ... */ }
|
|
689
|
+
async checkAndIncrementRateLimit(queue, max, windowMs, now) { /* ... */ }
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
const client = QueueClient.withAdapter(new MongoStorageAdapter(), {
|
|
693
|
+
defaults: { attempts: 5 },
|
|
694
|
+
});
|
|
695
|
+
await client.init();
|
|
696
|
+
```
|
|
697
|
+
|
|
698
|
+
---
|
|
699
|
+
|
|
700
|
+
## API Reference
|
|
701
|
+
|
|
702
|
+
### `new QueueClient(options?)`
|
|
703
|
+
|
|
704
|
+
| Option | Type | Default | Description |
|
|
705
|
+
|---|---|---|---|
|
|
706
|
+
| `dialect` | `"memory" \| "redis" \| "postgres" \| "mysql"` | `"memory"` | Storage backend |
|
|
707
|
+
| `connectionString` | `string` | — | Required for redis/postgres/mysql |
|
|
708
|
+
| `defaults.attempts` | `number` | `3` | Default max attempts |
|
|
709
|
+
| `defaults.retryDelay` | `number` | `1000` | Default base retry delay (ms) |
|
|
710
|
+
| `defaults.backoff` | `"fixed" \| "linear" \| "exponential"` | `"exponential"` | Default backoff strategy |
|
|
711
|
+
| `defaults.timeout` | `number` | `30000` | Default per-attempt timeout (ms) |
|
|
712
|
+
| `defaults.concurrency` | `number` | `10` | Default worker concurrency |
|
|
713
|
+
| `defaults.pollInterval` | `number` | `1000` | Worker poll interval (ms) |
|
|
714
|
+
| `defaults.stalledInterval` | `number` | `30000` | Stalled-job check interval (ms) |
|
|
715
|
+
| `defaults.lockDuration` | `number` | `60000` | Lock TTL (ms) |
|
|
716
|
+
| `defaults.rateLimit` | `{ max, duration }` | — | Optional rate limit |
|
|
717
|
+
|
|
718
|
+
### `client.init()`
|
|
719
|
+
|
|
720
|
+
Initialises the storage backend. Required for redis/postgres/mysql before any queue operations. Idempotent.
|
|
721
|
+
|
|
722
|
+
### `client.createQueue<TPayload>(name, options?)`
|
|
723
|
+
|
|
724
|
+
Creates and returns a `Queue`. `options` override `defaults` for this queue. The generic `<TPayload>` is TypeScript-only — omit it in JavaScript.
|
|
725
|
+
|
|
726
|
+
### `client.getQueue<TPayload>(name)` / `client.requireQueue<TPayload>(name)`
|
|
727
|
+
|
|
728
|
+
Returns an existing queue by name (`requireQueue` throws if not found).
|
|
729
|
+
|
|
730
|
+
### `client.on(event, listener)` / `client.once(...)` / `client.off(...)`
|
|
731
|
+
|
|
732
|
+
Subscribe/unsubscribe from lifecycle events.
|
|
733
|
+
|
|
734
|
+
### `client.close()`
|
|
735
|
+
|
|
736
|
+
Gracefully shut down. Safe to call multiple times.
|
|
737
|
+
|
|
738
|
+
### `QueueClient.withAdapter(adapter, options?)`
|
|
739
|
+
|
|
740
|
+
Static factory for custom storage adapters.
|
|
741
|
+
|
|
742
|
+
---
|
|
743
|
+
|
|
744
|
+
### `queue.enqueue(type, payload, options?)`
|
|
745
|
+
|
|
746
|
+
| Option | Type | Description |
|
|
747
|
+
|---|---|---|
|
|
748
|
+
| `attempts` | `number` | Max attempts for this job |
|
|
749
|
+
| `retryDelay` | `number` | Base retry delay (ms) |
|
|
750
|
+
| `backoff` | `string` | Backoff strategy |
|
|
751
|
+
| `timeout` | `number` | Per-attempt timeout (ms) |
|
|
752
|
+
| `priority` | `number` | Higher = sooner (default: 0) |
|
|
753
|
+
| `schedule.delay` | `number` | Delay before eligible (ms) |
|
|
754
|
+
| `schedule.runAt` | `string \| number` | Absolute run time |
|
|
755
|
+
| `schedule.cron` | `string` | Cron expression (stored) |
|
|
756
|
+
|
|
757
|
+
### `queue.process(type, processor)`
|
|
758
|
+
|
|
759
|
+
Register an async processor function for a job type.
|
|
760
|
+
|
|
761
|
+
### `queue.createWorker(options?)`
|
|
762
|
+
|
|
763
|
+
| Option | Type | Default | Description |
|
|
764
|
+
|---|---|---|---|
|
|
765
|
+
| `concurrency` | `number` | queue config | Max concurrent jobs |
|
|
766
|
+
| `shutdownTimeout` | `number` | `30000` | Drain timeout on stop (ms) |
|
|
767
|
+
|
|
768
|
+
### `queue.getJob(id)` / `queue.getJobs(status?, limit?, offset?)`
|
|
769
|
+
### `queue.getJobCounts()`
|
|
770
|
+
|
|
771
|
+
---
|
|
772
|
+
|
|
773
|
+
## Storage Support Matrix
|
|
774
|
+
|
|
775
|
+
| Feature | Memory | Redis | PostgreSQL | MySQL |
|
|
776
|
+
|---|:---:|:---:|:---:|:---:|
|
|
777
|
+
| Persistence | | ✓ | ✓ | ✓ |
|
|
778
|
+
| Atomic claim | ✓ | ✓ (Lua) | ✓ (SKIP LOCKED) | ✓ (SKIP LOCKED) |
|
|
779
|
+
| Priority ordering | ✓ | ✓ | ✓ | ✓ |
|
|
780
|
+
| Delayed jobs | ✓ | ✓ | ✓ | ✓ |
|
|
781
|
+
| Retry + backoff | ✓ | ✓ | ✓ | ✓ |
|
|
782
|
+
| DLQ | ✓ | ✓ | ✓ | ✓ |
|
|
783
|
+
| Stalled recovery | ✓ | ✓ | ✓ | ✓ |
|
|
784
|
+
| Rate limiting | ✓ | ✓ | ✓ | ✓ |
|
|
785
|
+
| Connection check on init | — | ✓ PING | ✓ SELECT 1 | ✓ SELECT 1 |
|
|
786
|
+
| Auto-create schema | — | — | ✓ | ✓ |
|
|
787
|
+
|
|
788
|
+
---
|
|
789
|
+
|
|
790
|
+
## Security
|
|
791
|
+
|
|
792
|
+
- Job payloads are never logged by default.
|
|
793
|
+
- Error messages do not include payload data.
|
|
794
|
+
- Connection strings should always come from environment variables, not source code.
|
|
795
|
+
- See [SECURITY.md](./SECURITY.md) for the full policy.
|
|
796
|
+
|
|
797
|
+
```js
|
|
798
|
+
// Good
|
|
799
|
+
const client = new QueueClient({
|
|
800
|
+
dialect: "postgres",
|
|
801
|
+
connectionString: process.env.DATABASE_URL,
|
|
802
|
+
});
|
|
803
|
+
|
|
804
|
+
// Bad — never hard-code credentials
|
|
805
|
+
const client = new QueueClient({
|
|
806
|
+
dialect: "postgres",
|
|
807
|
+
connectionString: "postgresql://admin:secret@prod-db:5432/app",
|
|
808
|
+
});
|
|
809
|
+
```
|
|
810
|
+
|
|
811
|
+
---
|
|
812
|
+
|
|
813
|
+
## Contributing
|
|
814
|
+
|
|
815
|
+
See [CONTRIBUTING.md](./CONTRIBUTING.md).
|
|
816
|
+
|
|
817
|
+
---
|
|
818
|
+
|
|
819
|
+
## License
|
|
820
|
+
|
|
821
|
+
MIT — [LICENSE](./LICENSE)
|