queue-jobs-worker 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -122
- package/README.md +142 -386
- package/assets/queue-jobs-worker-github.png +0 -0
- package/dist/core/worker.d.ts.map +1 -1
- package/dist/index.cjs +430 -137
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +430 -137
- package/dist/index.js.map +1 -1
- package/dist/storage/in-memory.adapter.d.ts +3 -2
- package/dist/storage/in-memory.adapter.d.ts.map +1 -1
- package/dist/storage/mysql.adapter.d.ts +3 -2
- package/dist/storage/mysql.adapter.d.ts.map +1 -1
- package/dist/storage/postgres.adapter.d.ts +3 -2
- package/dist/storage/postgres.adapter.d.ts.map +1 -1
- package/dist/storage/redis.adapter.d.ts +3 -2
- package/dist/storage/redis.adapter.d.ts.map +1 -1
- package/dist/types/storage.types.d.ts +20 -2
- package/dist/types/storage.types.d.ts.map +1 -1
- package/dist/types/worker.types.d.ts +1 -1
- package/dist/types/worker.types.d.ts.map +1 -1
- package/package.json +110 -97
package/README.md
CHANGED
|
@@ -1,49 +1,35 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
1
3
|
# queue-jobs-worker
|
|
2
4
|
|
|
3
|
-
A
|
|
5
|
+
A durable, TypeScript-first job queue for Node.js built for asynchronous work, retries, scheduling, and recovery.
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
<p align="center">
|
|
8
|
+
<a href="https://www.npmjs.com/package/queue-jobs-worker">
|
|
9
|
+
<img src="https://img.shields.io/npm/v/queue-jobs-worker.svg" alt="npm version">
|
|
10
|
+
</a>
|
|
11
|
+
<a href="./LICENSE">
|
|
12
|
+
<img src="https://img.shields.io/npm/l/queue-jobs-worker.svg" alt="license">
|
|
13
|
+
</a>
|
|
14
|
+
<a href="https://nodejs.org">
|
|
15
|
+
<img src="https://img.shields.io/node/v/queue-jobs-worker.svg" alt="node">
|
|
16
|
+
</a>
|
|
17
|
+
</p>
|
|
8
18
|
|
|
9
19
|
---
|
|
10
20
|
|
|
11
21
|
## Overview
|
|
12
22
|
|
|
13
|
-
`queue-jobs-worker`
|
|
23
|
+
`queue-jobs-worker` helps you move background work out of the request lifecycle and into a reliable, persistent queue. Define the processor logic once and let the library handle enqueueing, persistence, retries, schedules, concurrency, rate limiting, and recovery.
|
|
14
24
|
|
|
15
|
-
|
|
16
|
-
- In-memory (dev / testing)
|
|
17
|
-
- Redis (node-redis v4+)
|
|
18
|
-
- PostgreSQL (node-postgres / pg)
|
|
19
|
-
- MySQL (mysql2)
|
|
25
|
+
It supports all major local and production-friendly backends:
|
|
20
26
|
|
|
21
|
-
|
|
27
|
+
- In-memory queue for development and tests
|
|
28
|
+
- Redis via `node-redis` v4+
|
|
29
|
+
- PostgreSQL via `pg`
|
|
30
|
+
- MySQL via `mysql2`
|
|
22
31
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
- [Installation](#installation)
|
|
26
|
-
- [Quick Start](#quick-start)
|
|
27
|
-
- [Dialects](#dialects)
|
|
28
|
-
- [Memory](#memory-no-setup-required)
|
|
29
|
-
- [Redis](#redis)
|
|
30
|
-
- [PostgreSQL](#postgresql)
|
|
31
|
-
- [MySQL](#mysql)
|
|
32
|
-
- [Core Concepts](#core-concepts)
|
|
33
|
-
- [Configuration](#configuration)
|
|
34
|
-
- [Enqueueing Jobs](#enqueueing-jobs)
|
|
35
|
-
- [Processing Jobs](#processing-jobs)
|
|
36
|
-
- [Workers](#workers)
|
|
37
|
-
- [Events](#events)
|
|
38
|
-
- [Querying Jobs](#querying-jobs)
|
|
39
|
-
- [Retry & Backoff](#retry--backoff)
|
|
40
|
-
- [Scheduling](#scheduling)
|
|
41
|
-
- [Priority](#priority)
|
|
42
|
-
- [Rate Limiting](#rate-limiting)
|
|
43
|
-
- [Dead Letter Queue](#dead-letter-queue)
|
|
44
|
-
- [Graceful Shutdown](#graceful-shutdown)
|
|
45
|
-
- [Custom Storage Adapter](#custom-storage-adapter)
|
|
46
|
-
- [API Reference](#api-reference)
|
|
32
|
+
For a detailed feature breakdown, see [FEATURES.md](./FEATURES.md).
|
|
47
33
|
|
|
48
34
|
---
|
|
49
35
|
|
|
@@ -53,7 +39,7 @@ A production-ready background job queue for Node.js — persistent, reliable, an
|
|
|
53
39
|
npm install queue-jobs-worker
|
|
54
40
|
```
|
|
55
41
|
|
|
56
|
-
Install
|
|
42
|
+
Install the driver you plan to use:
|
|
57
43
|
|
|
58
44
|
```bash
|
|
59
45
|
# Redis
|
|
@@ -70,44 +56,15 @@ npm install mysql2
|
|
|
70
56
|
|
|
71
57
|
## Quick Start
|
|
72
58
|
|
|
73
|
-
**TypeScript**
|
|
74
|
-
```ts
|
|
75
|
-
import { QueueClient } from "queue-jobs-worker";
|
|
76
|
-
|
|
77
|
-
const client = new QueueClient();
|
|
78
|
-
|
|
79
|
-
// Generic type parameter gives you typed job.data
|
|
80
|
-
const emails = client.createQueue<{ to: string; subject: string }>("emails");
|
|
81
|
-
|
|
82
|
-
emails.process("send-email", async (job) => {
|
|
83
|
-
await sendEmail(job.data.to, job.data.subject);
|
|
84
|
-
// Throw to trigger retry; return to mark as completed
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
emails.createWorker({ concurrency: 5 });
|
|
88
|
-
|
|
89
|
-
await emails.enqueue("send-email", {
|
|
90
|
-
to: "user@example.com",
|
|
91
|
-
subject: "Welcome!",
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
process.on("SIGTERM", async () => {
|
|
95
|
-
await client.close();
|
|
96
|
-
process.exit(0);
|
|
97
|
-
});
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
**JavaScript**
|
|
101
59
|
```js
|
|
102
60
|
const { QueueClient } = require("queue-jobs-worker");
|
|
103
61
|
|
|
104
62
|
const client = new QueueClient();
|
|
105
|
-
|
|
106
|
-
// No generic — job.data is untyped
|
|
107
63
|
const emails = client.createQueue("emails");
|
|
108
64
|
|
|
109
65
|
emails.process("send-email", async (job) => {
|
|
110
66
|
await sendEmail(job.data.to, job.data.subject);
|
|
67
|
+
// Return to mark the job complete; throw to trigger retry or DLQ handling
|
|
111
68
|
});
|
|
112
69
|
|
|
113
70
|
emails.createWorker({ concurrency: 5 });
|
|
@@ -123,13 +80,15 @@ process.on("SIGTERM", async () => {
|
|
|
123
80
|
});
|
|
124
81
|
```
|
|
125
82
|
|
|
83
|
+
If you are using TypeScript, you can optionally make the queue payload type-safe with a generic like `client.createQueue<{ to: string; subject: string }>("emails")`.
|
|
84
|
+
|
|
126
85
|
---
|
|
127
86
|
|
|
128
|
-
##
|
|
87
|
+
## Supported Backends
|
|
129
88
|
|
|
130
|
-
### Memory
|
|
89
|
+
### Memory
|
|
131
90
|
|
|
132
|
-
|
|
91
|
+
Use the in-memory backend for local development and tests. Data is not persisted across restarts.
|
|
133
92
|
|
|
134
93
|
```js
|
|
135
94
|
const client = new QueueClient();
|
|
@@ -137,16 +96,11 @@ const client = new QueueClient();
|
|
|
137
96
|
const client = new QueueClient({ dialect: "memory" });
|
|
138
97
|
```
|
|
139
98
|
|
|
140
|
-
`init()` is
|
|
141
|
-
|
|
142
|
-
---
|
|
99
|
+
`init()` is effectively a no-op for this backend.
|
|
143
100
|
|
|
144
101
|
### Redis
|
|
145
102
|
|
|
146
|
-
Requires `redis` (node-redis v4+): `npm install redis`
|
|
147
|
-
|
|
148
103
|
```js
|
|
149
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
150
104
|
const { QueueClient } = require("queue-jobs-worker");
|
|
151
105
|
|
|
152
106
|
const client = new QueueClient({
|
|
@@ -154,56 +108,37 @@ const client = new QueueClient({
|
|
|
154
108
|
connectionString: "redis://localhost:6379",
|
|
155
109
|
});
|
|
156
110
|
|
|
157
|
-
await client.init();
|
|
158
|
-
|
|
111
|
+
await client.init();
|
|
159
112
|
const jobs = client.createQueue("jobs");
|
|
160
113
|
```
|
|
161
114
|
|
|
162
|
-
|
|
115
|
+
With authentication:
|
|
116
|
+
|
|
163
117
|
```js
|
|
164
118
|
const client = new QueueClient({
|
|
165
119
|
dialect: "redis",
|
|
166
120
|
connectionString: "redis://:yourpassword@redis-host:6379/0",
|
|
167
121
|
});
|
|
122
|
+
|
|
168
123
|
await client.init();
|
|
169
124
|
```
|
|
170
125
|
|
|
171
|
-
|
|
126
|
+
With TLS:
|
|
127
|
+
|
|
172
128
|
```js
|
|
173
129
|
const client = new QueueClient({
|
|
174
130
|
dialect: "redis",
|
|
175
131
|
connectionString: "rediss://user:password@host:6380",
|
|
176
132
|
});
|
|
177
|
-
await client.init();
|
|
178
|
-
```
|
|
179
|
-
|
|
180
|
-
**What `init()` does for Redis:**
|
|
181
|
-
- Creates the node-redis client
|
|
182
|
-
- Calls `client.connect()`
|
|
183
|
-
- Sends `PING` and asserts the response is `PONG`
|
|
184
|
-
- Throws a descriptive error if the connection fails
|
|
185
133
|
|
|
186
|
-
|
|
187
|
-
```
|
|
188
|
-
qjw:job:{id} → Hash (all job fields)
|
|
189
|
-
qjw:queue:{name}:waiting → Sorted Set (score = -priority)
|
|
190
|
-
qjw:queue:{name}:delayed → Sorted Set (score = runAt ms)
|
|
191
|
-
qjw:queue:{name}:active → Set
|
|
192
|
-
qjw:queue:{name}:completed → Set
|
|
193
|
-
qjw:queue:{name}:dead → Set
|
|
194
|
-
qjw:rate:{name} → String (rate-limit counter)
|
|
134
|
+
await client.init();
|
|
195
135
|
```
|
|
196
136
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
---
|
|
137
|
+
`init()` creates the Redis client, connects to the server, sends `PING`, and verifies the response is `PONG`.
|
|
200
138
|
|
|
201
139
|
### PostgreSQL
|
|
202
140
|
|
|
203
|
-
Requires `pg` (node-postgres): `npm install pg`
|
|
204
|
-
|
|
205
141
|
```js
|
|
206
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
207
142
|
const { QueueClient } = require("queue-jobs-worker");
|
|
208
143
|
|
|
209
144
|
const client = new QueueClient({
|
|
@@ -211,41 +146,14 @@ const client = new QueueClient({
|
|
|
211
146
|
connectionString: "postgresql://user:password@localhost:5432/mydb",
|
|
212
147
|
});
|
|
213
148
|
|
|
214
|
-
await client.init(); // connects + SELECT 1 + creates tables
|
|
215
|
-
```
|
|
216
|
-
|
|
217
|
-
**What `init()` does for PostgreSQL:**
|
|
218
|
-
- Creates a connection pool (`pg.Pool`)
|
|
219
|
-
- Runs `SELECT 1` to verify connectivity
|
|
220
|
-
- Executes `CREATE TABLE IF NOT EXISTS` for `qjw_jobs` and `qjw_rate_limits` — **idempotent, safe to run on every startup**
|
|
221
|
-
- Throws a descriptive error if the connection fails
|
|
222
|
-
|
|
223
|
-
**Tables created automatically** (prefix: `qjw_`):
|
|
224
|
-
```sql
|
|
225
|
-
qjw_jobs -- stores every job and its full lifecycle state
|
|
226
|
-
qjw_rate_limits -- sliding-window rate-limit counters
|
|
227
|
-
```
|
|
228
|
-
|
|
229
|
-
Claiming uses `SELECT ... FOR UPDATE SKIP LOCKED` inside a transaction — safe for any number of concurrent workers.
|
|
230
|
-
|
|
231
|
-
**With SSL (Heroku, Supabase, Neon, etc.):**
|
|
232
|
-
```js
|
|
233
|
-
const client = new QueueClient({
|
|
234
|
-
dialect: "postgres",
|
|
235
|
-
connectionString: process.env.DATABASE_URL,
|
|
236
|
-
// pg respects ?sslmode=require in the connection string
|
|
237
|
-
});
|
|
238
149
|
await client.init();
|
|
239
150
|
```
|
|
240
151
|
|
|
241
|
-
|
|
152
|
+
`init()` verifies connectivity with `SELECT 1` and creates the queue tables if they do not already exist.
|
|
242
153
|
|
|
243
154
|
### MySQL
|
|
244
155
|
|
|
245
|
-
Requires `mysql2`: `npm install mysql2`
|
|
246
|
-
|
|
247
156
|
```js
|
|
248
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
249
157
|
const { QueueClient } = require("queue-jobs-worker");
|
|
250
158
|
|
|
251
159
|
const client = new QueueClient({
|
|
@@ -253,22 +161,10 @@ const client = new QueueClient({
|
|
|
253
161
|
connectionString: "mysql://user:password@localhost:3306/mydb",
|
|
254
162
|
});
|
|
255
163
|
|
|
256
|
-
await client.init();
|
|
257
|
-
```
|
|
258
|
-
|
|
259
|
-
**What `init()` does for MySQL:**
|
|
260
|
-
- Creates a connection pool (`mysql2.createPool`)
|
|
261
|
-
- Runs `SELECT 1` to verify connectivity
|
|
262
|
-
- Executes `CREATE TABLE IF NOT EXISTS` for `qjw_jobs` and `qjw_rate_limits` — **idempotent**
|
|
263
|
-
- Throws a descriptive error if the connection fails
|
|
264
|
-
|
|
265
|
-
**Tables created automatically** (prefix: `qjw_`):
|
|
266
|
-
```sql
|
|
267
|
-
qjw_jobs -- full job state (InnoDB, utf8mb4)
|
|
268
|
-
qjw_rate_limits -- sliding-window rate-limit counters
|
|
164
|
+
await client.init();
|
|
269
165
|
```
|
|
270
166
|
|
|
271
|
-
|
|
167
|
+
`init()` validates the connection and creates the required tables in the database.
|
|
272
168
|
|
|
273
169
|
---
|
|
274
170
|
|
|
@@ -276,26 +172,25 @@ Claiming uses `SELECT ... FOR UPDATE SKIP LOCKED` inside a transaction.
|
|
|
276
172
|
|
|
277
173
|
| Concept | Description |
|
|
278
174
|
|---|---|
|
|
279
|
-
| `QueueClient` | Entry point
|
|
280
|
-
| `Queue` |
|
|
281
|
-
| `Job` | A unit of work
|
|
282
|
-
| `Worker` | Claims and executes jobs
|
|
283
|
-
| `Processor` | Your function
|
|
284
|
-
| `StorageAdapter` |
|
|
285
|
-
| DLQ | Dead Letter Queue
|
|
175
|
+
| `QueueClient` | Entry point that owns configuration, storage, and queues |
|
|
176
|
+
| `Queue` | A separate job stream with its own settings |
|
|
177
|
+
| `Job` | A unit of work passed to your processor |
|
|
178
|
+
| `Worker` | Claims and executes jobs |
|
|
179
|
+
| `Processor` | Your async function, e.g. `async (job) => { ... }` |
|
|
180
|
+
| `StorageAdapter` | A backend abstraction for durable storage |
|
|
181
|
+
| `DLQ` | Dead Letter Queue for permanently failed jobs |
|
|
286
182
|
|
|
287
183
|
---
|
|
288
184
|
|
|
289
185
|
## Configuration
|
|
290
186
|
|
|
291
|
-
|
|
187
|
+
Settings are layered so more specific config overrides broader defaults:
|
|
292
188
|
|
|
293
189
|
```
|
|
294
190
|
Client defaults → Queue options → Worker options → Job options
|
|
295
191
|
```
|
|
296
192
|
|
|
297
193
|
```js
|
|
298
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
299
194
|
const { QueueClient } = require("queue-jobs-worker");
|
|
300
195
|
|
|
301
196
|
const client = new QueueClient({
|
|
@@ -303,17 +198,17 @@ const client = new QueueClient({
|
|
|
303
198
|
connectionString: process.env.REDIS_URL,
|
|
304
199
|
|
|
305
200
|
defaults: {
|
|
306
|
-
attempts: 3,
|
|
307
|
-
retryDelay: 1000,
|
|
308
|
-
backoff: "exponential",
|
|
309
|
-
timeout: 30_000,
|
|
310
|
-
concurrency: 10,
|
|
311
|
-
pollInterval: 1_000,
|
|
312
|
-
stalledInterval: 30_000
|
|
313
|
-
lockDuration: 60_000,
|
|
201
|
+
attempts: 3,
|
|
202
|
+
retryDelay: 1000,
|
|
203
|
+
backoff: "exponential",
|
|
204
|
+
timeout: 30_000,
|
|
205
|
+
concurrency: 10,
|
|
206
|
+
pollInterval: 1_000,
|
|
207
|
+
stalledInterval: 30_000,
|
|
208
|
+
lockDuration: 60_000,
|
|
314
209
|
rateLimit: {
|
|
315
210
|
max: 100,
|
|
316
|
-
duration: 60_000,
|
|
211
|
+
duration: 60_000,
|
|
317
212
|
},
|
|
318
213
|
},
|
|
319
214
|
});
|
|
@@ -325,24 +220,6 @@ await client.init();
|
|
|
325
220
|
|
|
326
221
|
## Enqueueing Jobs
|
|
327
222
|
|
|
328
|
-
**TypeScript**
|
|
329
|
-
```ts
|
|
330
|
-
// Generic type gives you autocomplete and type-safety on job.data
|
|
331
|
-
const queue = client.createQueue<{ userId: string }>("notifications");
|
|
332
|
-
|
|
333
|
-
await queue.enqueue("send-push", { userId: "u_123" });
|
|
334
|
-
|
|
335
|
-
// With options
|
|
336
|
-
await queue.enqueue("send-push", { userId: "u_123" }, {
|
|
337
|
-
attempts: 5,
|
|
338
|
-
retryDelay: 2000,
|
|
339
|
-
backoff: "linear",
|
|
340
|
-
timeout: 10_000,
|
|
341
|
-
priority: 10, // higher = processed first (default: 0)
|
|
342
|
-
});
|
|
343
|
-
```
|
|
344
|
-
|
|
345
|
-
**JavaScript**
|
|
346
223
|
```js
|
|
347
224
|
const queue = client.createQueue("notifications");
|
|
348
225
|
|
|
@@ -357,46 +234,32 @@ await queue.enqueue("send-push", { userId: "u_123" }, {
|
|
|
357
234
|
});
|
|
358
235
|
```
|
|
359
236
|
|
|
237
|
+
If you want TypeScript type safety for `job.data`, pass a generic when creating the queue, such as `client.createQueue<{ userId: string }>("notifications")`.
|
|
238
|
+
|
|
360
239
|
---
|
|
361
240
|
|
|
362
241
|
## Processing Jobs
|
|
363
242
|
|
|
364
|
-
Register a processor before starting the
|
|
243
|
+
Register a processor before creating or starting a worker. Processors receive the `job` instance as well as an `AbortSignal` for cooperative cancellation when a job attempt times out:
|
|
365
244
|
|
|
366
245
|
```js
|
|
367
|
-
queue.process("send-push", async (job) => {
|
|
246
|
+
queue.process("send-push", async (job, signal) => {
|
|
368
247
|
const { userId } = job.data;
|
|
369
248
|
|
|
370
|
-
|
|
249
|
+
// Pass signal to APIs that support cancellation (e.g. fetch, DB queries):
|
|
250
|
+
await pushService.send(userId, "You have a new message", { signal });
|
|
371
251
|
|
|
372
|
-
//
|
|
373
|
-
|
|
252
|
+
// Or check signal.aborted before performing expensive steps:
|
|
253
|
+
if (signal.aborted) return;
|
|
254
|
+
|
|
255
|
+
// Return to mark the job complete.
|
|
256
|
+
// Throw any error to trigger retry logic or DLQ handling.
|
|
374
257
|
});
|
|
375
258
|
```
|
|
376
259
|
|
|
377
|
-
|
|
260
|
+
> **Note on Timeout Cancellation**: In Node.js, asynchronous operations cannot be forcibly terminated from the outside. Processors should cooperate with cancellation by checking `signal.aborted` or forwarding `signal` to abortable APIs to ensure timed-out executions do not continue running in the background.
|
|
378
261
|
|
|
379
|
-
|
|
380
|
-
job.id // unique stable ID
|
|
381
|
-
job.type // job type string
|
|
382
|
-
job.data // your payload (never logged)
|
|
383
|
-
job.attemptsMade // attempts already executed
|
|
384
|
-
job.maxAttempts // maximum allowed
|
|
385
|
-
job.attemptsRemaining
|
|
386
|
-
job.attemptHistory // array of past attempt records
|
|
387
|
-
job.status // current status
|
|
388
|
-
job.priority
|
|
389
|
-
job.runAt
|
|
390
|
-
job.createdAt
|
|
391
|
-
job.updatedAt
|
|
392
|
-
|
|
393
|
-
// helpers
|
|
394
|
-
job.isActive()
|
|
395
|
-
job.isCompleted()
|
|
396
|
-
job.isWaiting()
|
|
397
|
-
job.isDelayed()
|
|
398
|
-
job.isDead()
|
|
399
|
-
```
|
|
262
|
+
See [FEATURES.md](./FEATURES.md) for the full `job` model and helper methods.
|
|
400
263
|
|
|
401
264
|
---
|
|
402
265
|
|
|
@@ -404,57 +267,37 @@ job.isDead()
|
|
|
404
267
|
|
|
405
268
|
```js
|
|
406
269
|
const worker = queue.createWorker({
|
|
407
|
-
concurrency: 10,
|
|
408
|
-
shutdownTimeout: 30_000,
|
|
270
|
+
concurrency: 10,
|
|
271
|
+
shutdownTimeout: 30_000,
|
|
409
272
|
});
|
|
410
273
|
|
|
411
|
-
console.log(worker.status);
|
|
412
|
-
console.log(worker.id);
|
|
274
|
+
console.log(worker.status);
|
|
275
|
+
console.log(worker.id);
|
|
413
276
|
|
|
414
277
|
await worker.stop();
|
|
415
278
|
```
|
|
416
279
|
|
|
417
|
-
|
|
280
|
+
Multiple workers can share the same queue and coordinate through the storage layer:
|
|
418
281
|
|
|
419
282
|
```js
|
|
420
283
|
const w1 = queue.createWorker({ concurrency: 5 });
|
|
421
284
|
const w2 = queue.createWorker({ concurrency: 5 });
|
|
422
|
-
//
|
|
285
|
+
// total capacity: 10 concurrent jobs
|
|
423
286
|
```
|
|
424
287
|
|
|
425
288
|
---
|
|
426
289
|
|
|
427
290
|
## Events
|
|
428
291
|
|
|
429
|
-
|
|
292
|
+
The client emits lifecycle events that are useful for monitoring and alerting:
|
|
430
293
|
|
|
431
294
|
```js
|
|
432
|
-
// Job events
|
|
433
|
-
client.on("job:enqueued", (job) => console.log("Enqueued:", job.id));
|
|
434
|
-
client.on("job:started", (job) => console.log("Started:", job.id));
|
|
435
295
|
client.on("job:completed", (job) => console.log("Done:", job.id));
|
|
436
|
-
client.on("job:failed",
|
|
437
|
-
client.on("job:
|
|
438
|
-
|
|
439
|
-
});
|
|
440
|
-
client.on("job:dead", (job, err) => console.error("DLQ:", job.id, err.message));
|
|
441
|
-
client.on("job:stalled", (jobId) => console.warn("Stalled:", jobId));
|
|
442
|
-
client.on("job:recovered", (jobId) => console.log("Recovered:", jobId));
|
|
443
|
-
|
|
444
|
-
// Worker events
|
|
445
|
-
client.on("worker:started", (workerId) => console.log("Worker started:", workerId));
|
|
446
|
-
client.on("worker:stopped", (workerId) => console.log("Worker stopped:", workerId));
|
|
447
|
-
client.on("worker:status", (workerId, status) => console.log(workerId, "→", status));
|
|
448
|
-
client.on("worker:error", (workerId, err) => console.error("Worker error:", err));
|
|
449
|
-
|
|
450
|
-
// Queue / system errors
|
|
451
|
-
client.on("queue:error", (queueName, err) => console.error("Queue error:", queueName, err));
|
|
452
|
-
client.on("error", (err) => console.error("Error:", err));
|
|
296
|
+
client.on("job:failed", (job, err) => console.error("Failed:", job.id, err.message));
|
|
297
|
+
client.on("job:dead", (job, err) => console.error("DLQ:", job.id, err.message));
|
|
298
|
+
client.on("worker:error", (workerId, err) => console.error("Worker error:", err));
|
|
453
299
|
|
|
454
|
-
// Remove a listener
|
|
455
300
|
client.off("job:completed", myListener);
|
|
456
|
-
|
|
457
|
-
// One-time listener
|
|
458
301
|
client.once("job:dead", (job, err) => alertTeam(job, err));
|
|
459
302
|
```
|
|
460
303
|
|
|
@@ -468,13 +311,11 @@ if (job) {
|
|
|
468
311
|
console.log(job.status, job.attemptsMade);
|
|
469
312
|
}
|
|
470
313
|
|
|
471
|
-
|
|
472
|
-
const
|
|
473
|
-
const active = await queue.getJobs("active");
|
|
314
|
+
const waiting = await queue.getJobs("waiting", 50, 0);
|
|
315
|
+
const active = await queue.getJobs("active");
|
|
474
316
|
const completed = await queue.getJobs("completed", 100, 0);
|
|
475
|
-
const dead
|
|
317
|
+
const dead = await queue.getJobs("dead");
|
|
476
318
|
|
|
477
|
-
// Counts per status
|
|
478
319
|
const counts = await queue.getJobCounts();
|
|
479
320
|
// {
|
|
480
321
|
// waiting: 12,
|
|
@@ -489,17 +330,15 @@ const counts = await queue.getJobCounts();
|
|
|
489
330
|
|
|
490
331
|
## Retry & Backoff
|
|
491
332
|
|
|
492
|
-
|
|
333
|
+
Retries can be configured at the client, queue, or job level:
|
|
493
334
|
|
|
494
335
|
```js
|
|
495
|
-
// Queue-level
|
|
496
336
|
const queue = client.createQueue("tasks", {
|
|
497
337
|
attempts: 5,
|
|
498
338
|
retryDelay: 2000,
|
|
499
339
|
backoff: "exponential",
|
|
500
340
|
});
|
|
501
341
|
|
|
502
|
-
// Job-level override
|
|
503
342
|
await queue.enqueue("task", payload, {
|
|
504
343
|
attempts: 3,
|
|
505
344
|
retryDelay: 500,
|
|
@@ -507,172 +346,90 @@ await queue.enqueue("task", payload, {
|
|
|
507
346
|
});
|
|
508
347
|
```
|
|
509
348
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
| Strategy | Formula | Example (base = 1 s) |
|
|
513
|
-
|---|---|---|
|
|
514
|
-
| `fixed` | `baseDelay` | 1 s, 1 s, 1 s |
|
|
515
|
-
| `linear` | `baseDelay × attempt` | 1 s, 2 s, 3 s |
|
|
516
|
-
| `exponential` | `baseDelay × 2^(attempt-1)` (max 10 min) | 1 s, 2 s, 4 s, 8 s |
|
|
517
|
-
|
|
518
|
-
Each failed attempt is recorded in `job.attemptHistory`:
|
|
519
|
-
|
|
520
|
-
```js
|
|
521
|
-
job.attemptHistory.forEach((attempt) => {
|
|
522
|
-
console.log(`Attempt ${attempt.attempt}: ${attempt.error}`);
|
|
523
|
-
});
|
|
524
|
-
```
|
|
349
|
+
Available strategies are `fixed`, `linear`, and `exponential`. Each failure is tracked in `job.attemptHistory` so you can inspect what happened without losing context.
|
|
525
350
|
|
|
526
351
|
---
|
|
527
352
|
|
|
528
353
|
## Scheduling
|
|
529
354
|
|
|
530
355
|
```js
|
|
531
|
-
|
|
532
|
-
await queue.enqueue("
|
|
533
|
-
|
|
534
|
-
});
|
|
535
|
-
|
|
536
|
-
// Run at a specific time
|
|
537
|
-
await queue.enqueue("report", payload, {
|
|
538
|
-
schedule: { runAt: "2026-09-01T09:00:00Z" },
|
|
539
|
-
});
|
|
540
|
-
|
|
541
|
-
// Store a cron expression (integration point for recurring jobs)
|
|
542
|
-
await queue.enqueue("cleanup", payload, {
|
|
543
|
-
schedule: { cron: "0 3 * * *" },
|
|
544
|
-
});
|
|
356
|
+
await queue.enqueue("reminder", payload, { schedule: { delay: 30_000 } });
|
|
357
|
+
await queue.enqueue("report", payload, { schedule: { runAt: "2026-09-01T09:00:00Z" } });
|
|
358
|
+
await queue.enqueue("cleanup", payload, { schedule: { cron: "0 3 * * *" } });
|
|
545
359
|
```
|
|
546
360
|
|
|
547
|
-
Delayed jobs
|
|
361
|
+
Delayed jobs stay dormant until their scheduled time is reached.
|
|
548
362
|
|
|
549
363
|
---
|
|
550
364
|
|
|
551
365
|
## Priority
|
|
552
366
|
|
|
553
|
-
|
|
367
|
+
Jobs with a higher priority value are processed sooner. The default is `0`.
|
|
554
368
|
|
|
555
369
|
```js
|
|
556
370
|
await queue.enqueue("urgent-task", payload, { priority: 100 });
|
|
557
371
|
await queue.enqueue("normal-task", payload, { priority: 0 });
|
|
558
|
-
await queue.enqueue("low-task",
|
|
559
|
-
|
|
560
|
-
// Processing order: urgent → normal → low
|
|
372
|
+
await queue.enqueue("low-task", payload, { priority: -10 });
|
|
373
|
+
// order: urgent → normal → low
|
|
561
374
|
```
|
|
562
375
|
|
|
563
376
|
---
|
|
564
377
|
|
|
565
378
|
## Rate Limiting
|
|
566
379
|
|
|
567
|
-
Limit how many jobs are processed per time window:
|
|
568
|
-
|
|
569
380
|
```js
|
|
570
|
-
// Queue-level rate limit
|
|
571
381
|
const queue = client.createQueue("webhooks", {
|
|
572
|
-
rateLimit: {
|
|
573
|
-
max: 50, // max 50 jobs
|
|
574
|
-
duration: 60_000, // per 60 seconds
|
|
575
|
-
},
|
|
576
|
-
});
|
|
577
|
-
|
|
578
|
-
// Or as a client default
|
|
579
|
-
const client = new QueueClient({
|
|
580
|
-
defaults: {
|
|
581
|
-
rateLimit: { max: 100, duration: 60_000 },
|
|
582
|
-
},
|
|
382
|
+
rateLimit: { max: 50, duration: 60_000 },
|
|
583
383
|
});
|
|
584
384
|
```
|
|
585
385
|
|
|
586
|
-
When
|
|
386
|
+
When a queue reaches its limit, workers pause claiming new jobs until the time window resets. Jobs are not discarded.
|
|
587
387
|
|
|
588
388
|
---
|
|
589
389
|
|
|
590
390
|
## Dead Letter Queue
|
|
591
391
|
|
|
592
|
-
When a job
|
|
392
|
+
When a job reaches the end of its retry budget, it is moved to the dead-letter queue with status `"dead"`.
|
|
593
393
|
|
|
594
394
|
```js
|
|
595
395
|
client.on("job:dead", async (job, error) => {
|
|
596
|
-
await alertOncall({
|
|
597
|
-
jobId: job.id,
|
|
598
|
-
type: job.type,
|
|
599
|
-
error: error.message,
|
|
600
|
-
attempts: job.attemptsMade,
|
|
601
|
-
});
|
|
396
|
+
await alertOncall({ jobId: job.id, type: job.type, error: error.message });
|
|
602
397
|
});
|
|
603
398
|
|
|
604
|
-
// Query dead jobs
|
|
605
399
|
const deadJobs = await queue.getJobs("dead");
|
|
606
|
-
|
|
607
|
-
// Inspect a dead job's full attempt history
|
|
608
|
-
const job = await queue.getJob(jobId);
|
|
609
|
-
if (job) {
|
|
610
|
-
job.attemptHistory.forEach((a) => {
|
|
611
|
-
console.log(`Attempt ${a.attempt} failed: ${a.error}`);
|
|
612
|
-
});
|
|
613
|
-
}
|
|
614
400
|
```
|
|
615
401
|
|
|
402
|
+
All failure history remains attached to the job record.
|
|
403
|
+
|
|
616
404
|
---
|
|
617
405
|
|
|
618
406
|
## Graceful Shutdown
|
|
619
407
|
|
|
620
|
-
|
|
408
|
+
Call `client.close()` before your process exits:
|
|
621
409
|
|
|
622
410
|
```js
|
|
623
|
-
async
|
|
624
|
-
await client.close();
|
|
411
|
+
process.on("SIGTERM", async () => {
|
|
412
|
+
await client.close();
|
|
625
413
|
process.exit(0);
|
|
626
|
-
}
|
|
414
|
+
});
|
|
627
415
|
|
|
628
|
-
process.on("
|
|
629
|
-
|
|
416
|
+
process.on("SIGINT", async () => {
|
|
417
|
+
await client.close();
|
|
418
|
+
process.exit(0);
|
|
419
|
+
});
|
|
630
420
|
```
|
|
631
421
|
|
|
632
|
-
|
|
633
|
-
1. Stops polling for new jobs
|
|
634
|
-
2. Waits up to `shutdownTimeout` (default 30 s) for active jobs to finish
|
|
635
|
-
3. Releases DB/Redis connections
|
|
636
|
-
|
|
637
|
-
Interrupted jobs remain recoverable — the stalled-job recovery mechanism will reclaim them on the next startup.
|
|
422
|
+
This stops workers cleanly, releases locks, and allows stalled-job recovery to continue safely after restarts.
|
|
638
423
|
|
|
639
424
|
---
|
|
640
425
|
|
|
641
426
|
## Custom Storage Adapter
|
|
642
427
|
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
**TypeScript**
|
|
646
|
-
```ts
|
|
647
|
-
import type { StorageAdapter } from "queue-jobs-worker";
|
|
428
|
+
You can provide a custom backend by implementing the `StorageAdapter` interface. The same idea applies in JavaScript or TypeScript; the main difference is whether you add explicit interface typing in TypeScript.
|
|
648
429
|
|
|
649
|
-
class MongoStorageAdapter implements StorageAdapter {
|
|
650
|
-
async initialize() { /* connect, create indexes */ }
|
|
651
|
-
async close() { /* disconnect */ }
|
|
652
|
-
async enqueue(input) { /* ... */ }
|
|
653
|
-
async claim(input) { /* atomic claim */ }
|
|
654
|
-
async complete(jobId) { /* ... */ }
|
|
655
|
-
async requeue(input) { /* ... */ }
|
|
656
|
-
async moveToDlq(input) { /* ... */ }
|
|
657
|
-
async releaseLock(jobId) { /* ... */ }
|
|
658
|
-
async recoverStalledJobs(queue, now) { /* ... */ }
|
|
659
|
-
async getJob(jobId) { /* ... */ }
|
|
660
|
-
async getJobs(filter) { /* ... */ }
|
|
661
|
-
async getJobCounts(queue) { /* ... */ }
|
|
662
|
-
async checkAndIncrementRateLimit(queue, max, windowMs, now) { /* ... */ }
|
|
663
|
-
}
|
|
664
|
-
|
|
665
|
-
const client = QueueClient.withAdapter(new MongoStorageAdapter(), {
|
|
666
|
-
defaults: { attempts: 5 },
|
|
667
|
-
});
|
|
668
|
-
await client.init();
|
|
669
|
-
```
|
|
670
|
-
|
|
671
|
-
**JavaScript**
|
|
672
430
|
```js
|
|
673
431
|
const { QueueClient } = require("queue-jobs-worker");
|
|
674
432
|
|
|
675
|
-
// In JS there's no interface to implement — just match the method signatures
|
|
676
433
|
class MongoStorageAdapter {
|
|
677
434
|
async initialize() { /* connect, create indexes */ }
|
|
678
435
|
async close() { /* disconnect */ }
|
|
@@ -692,6 +449,7 @@ class MongoStorageAdapter {
|
|
|
692
449
|
const client = QueueClient.withAdapter(new MongoStorageAdapter(), {
|
|
693
450
|
defaults: { attempts: 5 },
|
|
694
451
|
});
|
|
452
|
+
|
|
695
453
|
await client.init();
|
|
696
454
|
```
|
|
697
455
|
|
|
@@ -704,66 +462,64 @@ await client.init();
|
|
|
704
462
|
| Option | Type | Default | Description |
|
|
705
463
|
|---|---|---|---|
|
|
706
464
|
| `dialect` | `"memory" \| "redis" \| "postgres" \| "mysql"` | `"memory"` | Storage backend |
|
|
707
|
-
| `connectionString` | `string` | — | Required for
|
|
708
|
-
| `defaults.attempts` | `number` | `3` |
|
|
709
|
-
| `defaults.retryDelay` | `number` | `1000` |
|
|
710
|
-
| `defaults.backoff` | `"fixed" \| "linear" \| "exponential"` | `"exponential"` |
|
|
711
|
-
| `defaults.timeout` | `number` | `30000` |
|
|
465
|
+
| `connectionString` | `string` | — | Required for Redis/PostgreSQL/MySQL |
|
|
466
|
+
| `defaults.attempts` | `number` | `3` | Max retries per job |
|
|
467
|
+
| `defaults.retryDelay` | `number` | `1000` | Base retry delay in ms |
|
|
468
|
+
| `defaults.backoff` | `"fixed" \| "linear" \| "exponential"` | `"exponential"` | Retry strategy |
|
|
469
|
+
| `defaults.timeout` | `number` | `30000` | Per-attempt timeout in ms |
|
|
712
470
|
| `defaults.concurrency` | `number` | `10` | Default worker concurrency |
|
|
713
|
-
| `defaults.pollInterval` | `number` | `1000` |
|
|
714
|
-
| `defaults.stalledInterval` | `number` | `30000` | Stalled-job check interval
|
|
715
|
-
| `defaults.lockDuration` | `number` | `60000` | Lock TTL
|
|
716
|
-
| `defaults.rateLimit` | `{ max, duration }` | — | Optional rate
|
|
471
|
+
| `defaults.pollInterval` | `number` | `1000` | Poll interval in ms |
|
|
472
|
+
| `defaults.stalledInterval` | `number` | `30000` | Stalled-job check interval in ms |
|
|
473
|
+
| `defaults.lockDuration` | `number` | `60000` | Lock TTL in ms |
|
|
474
|
+
| `defaults.rateLimit` | `{ max, duration }` | — | Optional rate limiting |
|
|
717
475
|
|
|
718
476
|
### `client.init()`
|
|
719
477
|
|
|
720
|
-
|
|
478
|
+
Initializes the configured backend. This is required for Redis, PostgreSQL, and MySQL before queue operations. It is safe to call more than once.
|
|
721
479
|
|
|
722
480
|
### `client.createQueue<TPayload>(name, options?)`
|
|
723
481
|
|
|
724
|
-
Creates and returns a
|
|
482
|
+
Creates and returns a queue. Queue-specific options override client defaults.
|
|
725
483
|
|
|
726
484
|
### `client.getQueue<TPayload>(name)` / `client.requireQueue<TPayload>(name)`
|
|
727
485
|
|
|
728
|
-
|
|
486
|
+
Fetches an existing queue by name. `requireQueue()` throws if none exists.
|
|
729
487
|
|
|
730
488
|
### `client.on(event, listener)` / `client.once(...)` / `client.off(...)`
|
|
731
489
|
|
|
732
|
-
|
|
490
|
+
Registers and removes event listeners for queue and worker lifecycle events.
|
|
733
491
|
|
|
734
492
|
### `client.close()`
|
|
735
493
|
|
|
736
|
-
|
|
494
|
+
Stops workers and closes storage connections gracefully.
|
|
737
495
|
|
|
738
496
|
### `QueueClient.withAdapter(adapter, options?)`
|
|
739
497
|
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
---
|
|
498
|
+
Creates a client using a custom storage backend.
|
|
743
499
|
|
|
744
500
|
### `queue.enqueue(type, payload, options?)`
|
|
745
501
|
|
|
746
502
|
| Option | Type | Description |
|
|
747
503
|
|---|---|---|
|
|
748
|
-
| `attempts` | `number` |
|
|
749
|
-
| `retryDelay` | `number` | Base retry delay
|
|
750
|
-
| `backoff` | `string` |
|
|
751
|
-
| `timeout` | `number` | Per-attempt timeout
|
|
752
|
-
| `priority` | `number` | Higher
|
|
753
|
-
| `schedule.delay` | `number` | Delay before eligible
|
|
504
|
+
| `attempts` | `number` | Maximum attempts for this job |
|
|
505
|
+
| `retryDelay` | `number` | Base retry delay in ms |
|
|
506
|
+
| `backoff` | `string` | Retry backoff strategy |
|
|
507
|
+
| `timeout` | `number` | Per-attempt timeout in ms |
|
|
508
|
+
| `priority` | `number` | Higher values are processed first |
|
|
509
|
+
| `schedule.delay` | `number` | Delay before the job becomes eligible |
|
|
754
510
|
| `schedule.runAt` | `string \| number` | Absolute run time |
|
|
755
|
-
| `schedule.cron` | `string` | Cron expression
|
|
511
|
+
| `schedule.cron` | `string` | Cron expression for recurring jobs |
|
|
756
512
|
|
|
757
513
|
### `queue.process(type, processor)`
|
|
758
514
|
|
|
759
|
-
|
|
515
|
+
Registers an async processor for a job type. The processor signature is `async (job, signal) => ...`, where `signal` is an `AbortSignal` aborted when the per-attempt timeout is reached.
|
|
760
516
|
|
|
761
517
|
### `queue.createWorker(options?)`
|
|
762
518
|
|
|
763
519
|
| Option | Type | Default | Description |
|
|
764
520
|
|---|---|---|---|
|
|
765
|
-
| `concurrency` | `number` | queue config |
|
|
766
|
-
| `shutdownTimeout` | `number` | `30000` |
|
|
521
|
+
| `concurrency` | `number` | queue config | Maximum simultaneous job executions |
|
|
522
|
+
| `shutdownTimeout` | `number` | `30000` | Graceful shutdown wait time in ms |
|
|
767
523
|
|
|
768
524
|
### `queue.getJob(id)` / `queue.getJobs(status?, limit?, offset?)`
|
|
769
525
|
### `queue.getJobCounts()`
|
|
@@ -774,7 +530,7 @@ Register an async processor function for a job type.
|
|
|
774
530
|
|
|
775
531
|
| Feature | Memory | Redis | PostgreSQL | MySQL |
|
|
776
532
|
|---|:---:|:---:|:---:|:---:|
|
|
777
|
-
| Persistence | | ✓ | ✓ | ✓ |
|
|
533
|
+
| Persistence | — | ✓ | ✓ | ✓ |
|
|
778
534
|
| Atomic claim | ✓ | ✓ (Lua) | ✓ (SKIP LOCKED) | ✓ (SKIP LOCKED) |
|
|
779
535
|
| Priority ordering | ✓ | ✓ | ✓ | ✓ |
|
|
780
536
|
| Delayed jobs | ✓ | ✓ | ✓ | ✓ |
|
|
@@ -789,9 +545,9 @@ Register an async processor function for a job type.
|
|
|
789
545
|
|
|
790
546
|
## Security
|
|
791
547
|
|
|
792
|
-
- Job payloads are
|
|
793
|
-
- Error messages do not
|
|
794
|
-
- Connection strings should
|
|
548
|
+
- Job payloads are not logged by default.
|
|
549
|
+
- Error messages do not expose payload data.
|
|
550
|
+
- Connection strings should come from environment variables rather than source code.
|
|
795
551
|
- See [SECURITY.md](./SECURITY.md) for the full policy.
|
|
796
552
|
|
|
797
553
|
```js
|
|
@@ -812,7 +568,7 @@ const client = new QueueClient({
|
|
|
812
568
|
|
|
813
569
|
## Contributing
|
|
814
570
|
|
|
815
|
-
|
|
571
|
+
Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for contribution guidelines.
|
|
816
572
|
|
|
817
573
|
---
|
|
818
574
|
|
|
@@ -820,8 +576,8 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md).
|
|
|
820
576
|
|
|
821
577
|
MIT — [LICENSE](./LICENSE)
|
|
822
578
|
|
|
823
|
-
|
|
579
|
+
## Donation
|
|
824
580
|
|
|
825
|
-
|
|
581
|
+
If this project has been useful to you, consider supporting it with a coffee.
|
|
826
582
|
|
|
827
|
-
BTC
|
|
583
|
+
**BTC:** `12dxgVQ3sRFhc4g7M6oydsN2tTMMthJJqS`
|