queue-jobs-worker 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -128
- package/README.md +133 -291
- package/assets/queue-jobs-worker-github.png +0 -0
- package/dist/core/worker.d.ts.map +1 -1
- package/dist/index.cjs +321 -92
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +321 -92
- package/dist/index.js.map +1 -1
- package/dist/storage/in-memory.adapter.d.ts +3 -2
- package/dist/storage/in-memory.adapter.d.ts.map +1 -1
- package/dist/storage/mysql.adapter.d.ts +3 -2
- package/dist/storage/mysql.adapter.d.ts.map +1 -1
- package/dist/storage/postgres.adapter.d.ts +3 -2
- package/dist/storage/postgres.adapter.d.ts.map +1 -1
- package/dist/storage/redis.adapter.d.ts +3 -2
- package/dist/storage/redis.adapter.d.ts.map +1 -1
- package/dist/types/storage.types.d.ts +20 -2
- package/dist/types/storage.types.d.ts.map +1 -1
- package/dist/types/worker.types.d.ts +1 -1
- package/dist/types/worker.types.d.ts.map +1 -1
- package/package.json +110 -109
package/README.md
CHANGED
|
@@ -1,51 +1,35 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
1
3
|
# queue-jobs-worker
|
|
2
4
|
|
|
3
|
-
A
|
|
5
|
+
A durable, TypeScript-first job queue for Node.js built for asynchronous work, retries, scheduling, and recovery.
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
<p align="center">
|
|
8
|
+
<a href="https://www.npmjs.com/package/queue-jobs-worker">
|
|
9
|
+
<img src="https://img.shields.io/npm/v/queue-jobs-worker.svg" alt="npm version">
|
|
10
|
+
</a>
|
|
11
|
+
<a href="./LICENSE">
|
|
12
|
+
<img src="https://img.shields.io/npm/l/queue-jobs-worker.svg" alt="license">
|
|
13
|
+
</a>
|
|
14
|
+
<a href="https://nodejs.org">
|
|
15
|
+
<img src="https://img.shields.io/node/v/queue-jobs-worker.svg" alt="node">
|
|
16
|
+
</a>
|
|
17
|
+
</p>
|
|
8
18
|
|
|
9
19
|
---
|
|
10
20
|
|
|
11
21
|
## Overview
|
|
12
22
|
|
|
13
|
-
`queue-jobs-worker`
|
|
14
|
-
|
|
15
|
-
For a full breakdown of every feature, see [FEATURES.md](./FEATURES.md).
|
|
23
|
+
`queue-jobs-worker` helps you move background work out of the request lifecycle and into a reliable, persistent queue. Define the processor logic once and let the library handle enqueueing, persistence, retries, schedules, concurrency, rate limiting, and recovery.
|
|
16
24
|
|
|
17
|
-
|
|
18
|
-
- In-memory (dev / testing)
|
|
19
|
-
- Redis (node-redis v4+)
|
|
20
|
-
- PostgreSQL (node-postgres / pg)
|
|
21
|
-
- MySQL (mysql2)
|
|
25
|
+
It supports all major local and production-friendly backends:
|
|
22
26
|
|
|
23
|
-
|
|
27
|
+
- In-memory queue for development and tests
|
|
28
|
+
- Redis via `node-redis` v4+
|
|
29
|
+
- PostgreSQL via `pg`
|
|
30
|
+
- MySQL via `mysql2`
|
|
24
31
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
- [Installation](#installation)
|
|
28
|
-
- [Quick Start](#quick-start)
|
|
29
|
-
- [Dialects](#dialects)
|
|
30
|
-
- [Memory](#memory-no-setup-required)
|
|
31
|
-
- [Redis](#redis)
|
|
32
|
-
- [PostgreSQL](#postgresql)
|
|
33
|
-
- [MySQL](#mysql)
|
|
34
|
-
- [Core Concepts](#core-concepts)
|
|
35
|
-
- [Configuration](#configuration)
|
|
36
|
-
- [Enqueueing Jobs](#enqueueing-jobs)
|
|
37
|
-
- [Processing Jobs](#processing-jobs)
|
|
38
|
-
- [Workers](#workers)
|
|
39
|
-
- [Events](#events)
|
|
40
|
-
- [Querying Jobs](#querying-jobs)
|
|
41
|
-
- [Retry & Backoff](#retry--backoff)
|
|
42
|
-
- [Scheduling](#scheduling)
|
|
43
|
-
- [Priority](#priority)
|
|
44
|
-
- [Rate Limiting](#rate-limiting)
|
|
45
|
-
- [Dead Letter Queue](#dead-letter-queue)
|
|
46
|
-
- [Graceful Shutdown](#graceful-shutdown)
|
|
47
|
-
- [Custom Storage Adapter](#custom-storage-adapter)
|
|
48
|
-
- [API Reference](#api-reference)
|
|
32
|
+
For a detailed feature breakdown, see [FEATURES.md](./FEATURES.md).
|
|
49
33
|
|
|
50
34
|
---
|
|
51
35
|
|
|
@@ -55,7 +39,7 @@ For a full breakdown of every feature, see [FEATURES.md](./FEATURES.md).
|
|
|
55
39
|
npm install queue-jobs-worker
|
|
56
40
|
```
|
|
57
41
|
|
|
58
|
-
Install
|
|
42
|
+
Install the driver you plan to use:
|
|
59
43
|
|
|
60
44
|
```bash
|
|
61
45
|
# Redis
|
|
@@ -72,44 +56,15 @@ npm install mysql2
|
|
|
72
56
|
|
|
73
57
|
## Quick Start
|
|
74
58
|
|
|
75
|
-
**TypeScript**
|
|
76
|
-
```ts
|
|
77
|
-
import { QueueClient } from "queue-jobs-worker";
|
|
78
|
-
|
|
79
|
-
const client = new QueueClient();
|
|
80
|
-
|
|
81
|
-
// Generic type parameter gives you typed job.data
|
|
82
|
-
const emails = client.createQueue<{ to: string; subject: string }>("emails");
|
|
83
|
-
|
|
84
|
-
emails.process("send-email", async (job) => {
|
|
85
|
-
await sendEmail(job.data.to, job.data.subject);
|
|
86
|
-
// Throw to trigger retry; return to mark as completed
|
|
87
|
-
});
|
|
88
|
-
|
|
89
|
-
emails.createWorker({ concurrency: 5 });
|
|
90
|
-
|
|
91
|
-
await emails.enqueue("send-email", {
|
|
92
|
-
to: "user@example.com",
|
|
93
|
-
subject: "Welcome!",
|
|
94
|
-
});
|
|
95
|
-
|
|
96
|
-
process.on("SIGTERM", async () => {
|
|
97
|
-
await client.close();
|
|
98
|
-
process.exit(0);
|
|
99
|
-
});
|
|
100
|
-
```
|
|
101
|
-
|
|
102
|
-
**JavaScript**
|
|
103
59
|
```js
|
|
104
60
|
const { QueueClient } = require("queue-jobs-worker");
|
|
105
61
|
|
|
106
62
|
const client = new QueueClient();
|
|
107
|
-
|
|
108
|
-
// No generic — job.data is untyped
|
|
109
63
|
const emails = client.createQueue("emails");
|
|
110
64
|
|
|
111
65
|
emails.process("send-email", async (job) => {
|
|
112
66
|
await sendEmail(job.data.to, job.data.subject);
|
|
67
|
+
// Return to mark the job complete; throw to trigger retry or DLQ handling
|
|
113
68
|
});
|
|
114
69
|
|
|
115
70
|
emails.createWorker({ concurrency: 5 });
|
|
@@ -125,13 +80,15 @@ process.on("SIGTERM", async () => {
|
|
|
125
80
|
});
|
|
126
81
|
```
|
|
127
82
|
|
|
83
|
+
If you are using TypeScript, you can optionally make the queue payload type-safe with a generic like `client.createQueue<{ to: string; subject: string }>("emails")`.
|
|
84
|
+
|
|
128
85
|
---
|
|
129
86
|
|
|
130
|
-
##
|
|
87
|
+
## Supported Backends
|
|
131
88
|
|
|
132
|
-
### Memory
|
|
89
|
+
### Memory
|
|
133
90
|
|
|
134
|
-
|
|
91
|
+
Use the in-memory backend for local development and tests. Data is not persisted across restarts.
|
|
135
92
|
|
|
136
93
|
```js
|
|
137
94
|
const client = new QueueClient();
|
|
@@ -139,16 +96,11 @@ const client = new QueueClient();
|
|
|
139
96
|
const client = new QueueClient({ dialect: "memory" });
|
|
140
97
|
```
|
|
141
98
|
|
|
142
|
-
`init()` is
|
|
143
|
-
|
|
144
|
-
---
|
|
99
|
+
`init()` is effectively a no-op for this backend.
|
|
145
100
|
|
|
146
101
|
### Redis
|
|
147
102
|
|
|
148
|
-
Requires `redis` (node-redis v4+): `npm install redis`
|
|
149
|
-
|
|
150
103
|
```js
|
|
151
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
152
104
|
const { QueueClient } = require("queue-jobs-worker");
|
|
153
105
|
|
|
154
106
|
const client = new QueueClient({
|
|
@@ -156,56 +108,37 @@ const client = new QueueClient({
|
|
|
156
108
|
connectionString: "redis://localhost:6379",
|
|
157
109
|
});
|
|
158
110
|
|
|
159
|
-
await client.init();
|
|
160
|
-
|
|
111
|
+
await client.init();
|
|
161
112
|
const jobs = client.createQueue("jobs");
|
|
162
113
|
```
|
|
163
114
|
|
|
164
|
-
|
|
115
|
+
With authentication:
|
|
116
|
+
|
|
165
117
|
```js
|
|
166
118
|
const client = new QueueClient({
|
|
167
119
|
dialect: "redis",
|
|
168
120
|
connectionString: "redis://:yourpassword@redis-host:6379/0",
|
|
169
121
|
});
|
|
122
|
+
|
|
170
123
|
await client.init();
|
|
171
124
|
```
|
|
172
125
|
|
|
173
|
-
|
|
126
|
+
With TLS:
|
|
127
|
+
|
|
174
128
|
```js
|
|
175
129
|
const client = new QueueClient({
|
|
176
130
|
dialect: "redis",
|
|
177
131
|
connectionString: "rediss://user:password@host:6380",
|
|
178
132
|
});
|
|
179
|
-
await client.init();
|
|
180
|
-
```
|
|
181
133
|
|
|
182
|
-
|
|
183
|
-
- Creates the node-redis client
|
|
184
|
-
- Calls `client.connect()`
|
|
185
|
-
- Sends `PING` and asserts the response is `PONG`
|
|
186
|
-
- Throws a descriptive error if the connection fails
|
|
187
|
-
|
|
188
|
-
**Key structure in Redis** (prefix: `qjw:`):
|
|
189
|
-
```
|
|
190
|
-
qjw:job:{id} → Hash (all job fields)
|
|
191
|
-
qjw:queue:{name}:waiting → Sorted Set (score = -priority)
|
|
192
|
-
qjw:queue:{name}:delayed → Sorted Set (score = runAt ms)
|
|
193
|
-
qjw:queue:{name}:active → Set
|
|
194
|
-
qjw:queue:{name}:completed → Set
|
|
195
|
-
qjw:queue:{name}:dead → Set
|
|
196
|
-
qjw:rate:{name} → String (rate-limit counter)
|
|
134
|
+
await client.init();
|
|
197
135
|
```
|
|
198
136
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
---
|
|
137
|
+
`init()` creates the Redis client, connects to the server, sends `PING`, and verifies the response is `PONG`.
|
|
202
138
|
|
|
203
139
|
### PostgreSQL
|
|
204
140
|
|
|
205
|
-
Requires `pg` (node-postgres): `npm install pg`
|
|
206
|
-
|
|
207
141
|
```js
|
|
208
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
209
142
|
const { QueueClient } = require("queue-jobs-worker");
|
|
210
143
|
|
|
211
144
|
const client = new QueueClient({
|
|
@@ -213,41 +146,14 @@ const client = new QueueClient({
|
|
|
213
146
|
connectionString: "postgresql://user:password@localhost:5432/mydb",
|
|
214
147
|
});
|
|
215
148
|
|
|
216
|
-
await client.init(); // connects + SELECT 1 + creates tables
|
|
217
|
-
```
|
|
218
|
-
|
|
219
|
-
**What `init()` does for PostgreSQL:**
|
|
220
|
-
- Creates a connection pool (`pg.Pool`)
|
|
221
|
-
- Runs `SELECT 1` to verify connectivity
|
|
222
|
-
- Executes `CREATE TABLE IF NOT EXISTS` for `qjw_jobs` and `qjw_rate_limits` — **idempotent, safe to run on every startup**
|
|
223
|
-
- Throws a descriptive error if the connection fails
|
|
224
|
-
|
|
225
|
-
**Tables created automatically** (prefix: `qjw_`):
|
|
226
|
-
```sql
|
|
227
|
-
qjw_jobs -- stores every job and its full lifecycle state
|
|
228
|
-
qjw_rate_limits -- sliding-window rate-limit counters
|
|
229
|
-
```
|
|
230
|
-
|
|
231
|
-
Claiming uses `SELECT ... FOR UPDATE SKIP LOCKED` inside a transaction — safe for any number of concurrent workers.
|
|
232
|
-
|
|
233
|
-
**With SSL (Heroku, Supabase, Neon, etc.):**
|
|
234
|
-
```js
|
|
235
|
-
const client = new QueueClient({
|
|
236
|
-
dialect: "postgres",
|
|
237
|
-
connectionString: process.env.DATABASE_URL,
|
|
238
|
-
// pg respects ?sslmode=require in the connection string
|
|
239
|
-
});
|
|
240
149
|
await client.init();
|
|
241
150
|
```
|
|
242
151
|
|
|
243
|
-
|
|
152
|
+
`init()` verifies connectivity with `SELECT 1` and creates the queue tables if they do not already exist.
|
|
244
153
|
|
|
245
154
|
### MySQL
|
|
246
155
|
|
|
247
|
-
Requires `mysql2`: `npm install mysql2`
|
|
248
|
-
|
|
249
156
|
```js
|
|
250
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
251
157
|
const { QueueClient } = require("queue-jobs-worker");
|
|
252
158
|
|
|
253
159
|
const client = new QueueClient({
|
|
@@ -255,22 +161,10 @@ const client = new QueueClient({
|
|
|
255
161
|
connectionString: "mysql://user:password@localhost:3306/mydb",
|
|
256
162
|
});
|
|
257
163
|
|
|
258
|
-
await client.init();
|
|
259
|
-
```
|
|
260
|
-
|
|
261
|
-
**What `init()` does for MySQL:**
|
|
262
|
-
- Creates a connection pool (`mysql2.createPool`)
|
|
263
|
-
- Runs `SELECT 1` to verify connectivity
|
|
264
|
-
- Executes `CREATE TABLE IF NOT EXISTS` for `qjw_jobs` and `qjw_rate_limits` — **idempotent**
|
|
265
|
-
- Throws a descriptive error if the connection fails
|
|
266
|
-
|
|
267
|
-
**Tables created automatically** (prefix: `qjw_`):
|
|
268
|
-
```sql
|
|
269
|
-
qjw_jobs -- full job state (InnoDB, utf8mb4)
|
|
270
|
-
qjw_rate_limits -- sliding-window rate-limit counters
|
|
164
|
+
await client.init();
|
|
271
165
|
```
|
|
272
166
|
|
|
273
|
-
|
|
167
|
+
`init()` validates the connection and creates the required tables in the database.
|
|
274
168
|
|
|
275
169
|
---
|
|
276
170
|
|
|
@@ -278,26 +172,25 @@ Claiming uses `SELECT ... FOR UPDATE SKIP LOCKED` inside a transaction.
|
|
|
278
172
|
|
|
279
173
|
| Concept | Description |
|
|
280
174
|
|---|---|
|
|
281
|
-
| `QueueClient` | Entry point
|
|
282
|
-
| `Queue` |
|
|
283
|
-
| `Job` | A unit of work
|
|
284
|
-
| `Worker` | Claims and executes jobs
|
|
285
|
-
| `Processor` | Your function
|
|
286
|
-
| `StorageAdapter` |
|
|
287
|
-
| DLQ | Dead Letter Queue
|
|
175
|
+
| `QueueClient` | Entry point that owns configuration, storage, and queues |
|
|
176
|
+
| `Queue` | A separate job stream with its own settings |
|
|
177
|
+
| `Job` | A unit of work passed to your processor |
|
|
178
|
+
| `Worker` | Claims and executes jobs |
|
|
179
|
+
| `Processor` | Your async function, e.g. `async (job) => { ... }` |
|
|
180
|
+
| `StorageAdapter` | A backend abstraction for durable storage |
|
|
181
|
+
| `DLQ` | Dead Letter Queue for permanently failed jobs |
|
|
288
182
|
|
|
289
183
|
---
|
|
290
184
|
|
|
291
185
|
## Configuration
|
|
292
186
|
|
|
293
|
-
|
|
187
|
+
Settings are layered so more specific config overrides broader defaults:
|
|
294
188
|
|
|
295
189
|
```
|
|
296
190
|
Client defaults → Queue options → Worker options → Job options
|
|
297
191
|
```
|
|
298
192
|
|
|
299
193
|
```js
|
|
300
|
-
// TypeScript: import { QueueClient } from "queue-jobs-worker";
|
|
301
194
|
const { QueueClient } = require("queue-jobs-worker");
|
|
302
195
|
|
|
303
196
|
const client = new QueueClient({
|
|
@@ -305,17 +198,17 @@ const client = new QueueClient({
|
|
|
305
198
|
connectionString: process.env.REDIS_URL,
|
|
306
199
|
|
|
307
200
|
defaults: {
|
|
308
|
-
attempts: 3,
|
|
309
|
-
retryDelay: 1000,
|
|
310
|
-
backoff: "exponential",
|
|
311
|
-
timeout: 30_000,
|
|
312
|
-
concurrency: 10,
|
|
313
|
-
pollInterval: 1_000,
|
|
314
|
-
stalledInterval: 30_000
|
|
315
|
-
lockDuration: 60_000,
|
|
201
|
+
attempts: 3,
|
|
202
|
+
retryDelay: 1000,
|
|
203
|
+
backoff: "exponential",
|
|
204
|
+
timeout: 30_000,
|
|
205
|
+
concurrency: 10,
|
|
206
|
+
pollInterval: 1_000,
|
|
207
|
+
stalledInterval: 30_000,
|
|
208
|
+
lockDuration: 60_000,
|
|
316
209
|
rateLimit: {
|
|
317
210
|
max: 100,
|
|
318
|
-
duration: 60_000,
|
|
211
|
+
duration: 60_000,
|
|
319
212
|
},
|
|
320
213
|
},
|
|
321
214
|
});
|
|
@@ -327,24 +220,6 @@ await client.init();
|
|
|
327
220
|
|
|
328
221
|
## Enqueueing Jobs
|
|
329
222
|
|
|
330
|
-
**TypeScript**
|
|
331
|
-
```ts
|
|
332
|
-
// Generic type gives you autocomplete and type-safety on job.data
|
|
333
|
-
const queue = client.createQueue<{ userId: string }>("notifications");
|
|
334
|
-
|
|
335
|
-
await queue.enqueue("send-push", { userId: "u_123" });
|
|
336
|
-
|
|
337
|
-
// With options
|
|
338
|
-
await queue.enqueue("send-push", { userId: "u_123" }, {
|
|
339
|
-
attempts: 5,
|
|
340
|
-
retryDelay: 2000,
|
|
341
|
-
backoff: "linear",
|
|
342
|
-
timeout: 10_000,
|
|
343
|
-
priority: 10, // higher = processed first (default: 0)
|
|
344
|
-
});
|
|
345
|
-
```
|
|
346
|
-
|
|
347
|
-
**JavaScript**
|
|
348
223
|
```js
|
|
349
224
|
const queue = client.createQueue("notifications");
|
|
350
225
|
|
|
@@ -359,24 +234,32 @@ await queue.enqueue("send-push", { userId: "u_123" }, {
|
|
|
359
234
|
});
|
|
360
235
|
```
|
|
361
236
|
|
|
237
|
+
If you want TypeScript type safety for `job.data`, pass a generic when creating the queue, such as `client.createQueue<{ userId: string }>("notifications")`.
|
|
238
|
+
|
|
362
239
|
---
|
|
363
240
|
|
|
364
241
|
## Processing Jobs
|
|
365
242
|
|
|
366
|
-
Register a processor before starting the
|
|
243
|
+
Register a processor before creating or starting a worker. Processors receive the `job` instance as well as an `AbortSignal` for cooperative cancellation when a job attempt times out:
|
|
367
244
|
|
|
368
245
|
```js
|
|
369
|
-
queue.process("send-push", async (job) => {
|
|
246
|
+
queue.process("send-push", async (job, signal) => {
|
|
370
247
|
const { userId } = job.data;
|
|
371
248
|
|
|
372
|
-
|
|
249
|
+
// Pass signal to APIs that support cancellation (e.g. fetch, DB queries):
|
|
250
|
+
await pushService.send(userId, "You have a new message", { signal });
|
|
251
|
+
|
|
252
|
+
// Or check signal.aborted before performing expensive steps:
|
|
253
|
+
if (signal.aborted) return;
|
|
373
254
|
|
|
374
|
-
// Return to mark
|
|
375
|
-
// Throw any error to
|
|
255
|
+
// Return to mark the job complete.
|
|
256
|
+
// Throw any error to trigger retry logic or DLQ handling.
|
|
376
257
|
});
|
|
377
258
|
```
|
|
378
259
|
|
|
379
|
-
|
|
260
|
+
> **Note on Timeout Cancellation**: In Node.js, asynchronous operations cannot be forcibly terminated from the outside. Processors should cooperate with cancellation by checking `signal.aborted` or forwarding `signal` to abortable APIs to ensure timed-out executions do not continue running in the background.
|
|
261
|
+
|
|
262
|
+
See [FEATURES.md](./FEATURES.md) for the full `job` model and helper methods.
|
|
380
263
|
|
|
381
264
|
---
|
|
382
265
|
|
|
@@ -384,42 +267,40 @@ For the full list of `job` fields and helper methods, see [FEATURES.md → Job I
|
|
|
384
267
|
|
|
385
268
|
```js
|
|
386
269
|
const worker = queue.createWorker({
|
|
387
|
-
concurrency: 10,
|
|
388
|
-
shutdownTimeout: 30_000,
|
|
270
|
+
concurrency: 10,
|
|
271
|
+
shutdownTimeout: 30_000,
|
|
389
272
|
});
|
|
390
273
|
|
|
391
|
-
console.log(worker.status);
|
|
392
|
-
console.log(worker.id);
|
|
274
|
+
console.log(worker.status);
|
|
275
|
+
console.log(worker.id);
|
|
393
276
|
|
|
394
277
|
await worker.stop();
|
|
395
278
|
```
|
|
396
279
|
|
|
397
|
-
|
|
280
|
+
Multiple workers can share the same queue and coordinate through the storage layer:
|
|
398
281
|
|
|
399
282
|
```js
|
|
400
283
|
const w1 = queue.createWorker({ concurrency: 5 });
|
|
401
284
|
const w2 = queue.createWorker({ concurrency: 5 });
|
|
402
|
-
//
|
|
285
|
+
// total capacity: 10 concurrent jobs
|
|
403
286
|
```
|
|
404
287
|
|
|
405
288
|
---
|
|
406
289
|
|
|
407
290
|
## Events
|
|
408
291
|
|
|
409
|
-
|
|
292
|
+
The client emits lifecycle events that are useful for monitoring and alerting:
|
|
410
293
|
|
|
411
294
|
```js
|
|
412
295
|
client.on("job:completed", (job) => console.log("Done:", job.id));
|
|
413
|
-
client.on("job:failed",
|
|
414
|
-
client.on("job:dead",
|
|
415
|
-
client.on("worker:error",
|
|
296
|
+
client.on("job:failed", (job, err) => console.error("Failed:", job.id, err.message));
|
|
297
|
+
client.on("job:dead", (job, err) => console.error("DLQ:", job.id, err.message));
|
|
298
|
+
client.on("worker:error", (workerId, err) => console.error("Worker error:", err));
|
|
416
299
|
|
|
417
|
-
client.off("job:completed", myListener);
|
|
418
|
-
client.once("job:dead", (job, err) => alertTeam(job, err));
|
|
300
|
+
client.off("job:completed", myListener);
|
|
301
|
+
client.once("job:dead", (job, err) => alertTeam(job, err));
|
|
419
302
|
```
|
|
420
303
|
|
|
421
|
-
For the full event reference (all job, worker, and system events), see [FEATURES.md → Event System](./FEATURES.md#event-system).
|
|
422
|
-
|
|
423
304
|
---
|
|
424
305
|
|
|
425
306
|
## Querying Jobs
|
|
@@ -430,13 +311,11 @@ if (job) {
|
|
|
430
311
|
console.log(job.status, job.attemptsMade);
|
|
431
312
|
}
|
|
432
313
|
|
|
433
|
-
|
|
434
|
-
const
|
|
435
|
-
const active = await queue.getJobs("active");
|
|
314
|
+
const waiting = await queue.getJobs("waiting", 50, 0);
|
|
315
|
+
const active = await queue.getJobs("active");
|
|
436
316
|
const completed = await queue.getJobs("completed", 100, 0);
|
|
437
|
-
const dead
|
|
317
|
+
const dead = await queue.getJobs("dead");
|
|
438
318
|
|
|
439
|
-
// Counts per status
|
|
440
319
|
const counts = await queue.getJobCounts();
|
|
441
320
|
// {
|
|
442
321
|
// waiting: 12,
|
|
@@ -451,17 +330,15 @@ const counts = await queue.getJobCounts();
|
|
|
451
330
|
|
|
452
331
|
## Retry & Backoff
|
|
453
332
|
|
|
454
|
-
|
|
333
|
+
Retries can be configured at the client, queue, or job level:
|
|
455
334
|
|
|
456
335
|
```js
|
|
457
|
-
// Queue-level
|
|
458
336
|
const queue = client.createQueue("tasks", {
|
|
459
337
|
attempts: 5,
|
|
460
338
|
retryDelay: 2000,
|
|
461
339
|
backoff: "exponential",
|
|
462
340
|
});
|
|
463
341
|
|
|
464
|
-
// Job-level override
|
|
465
342
|
await queue.enqueue("task", payload, {
|
|
466
343
|
attempts: 3,
|
|
467
344
|
retryDelay: 500,
|
|
@@ -469,57 +346,50 @@ await queue.enqueue("task", payload, {
|
|
|
469
346
|
});
|
|
470
347
|
```
|
|
471
348
|
|
|
472
|
-
|
|
349
|
+
Available strategies are `fixed`, `linear`, and `exponential`. Each failure is tracked in `job.attemptHistory` so you can inspect what happened without losing context.
|
|
473
350
|
|
|
474
351
|
---
|
|
475
352
|
|
|
476
353
|
## Scheduling
|
|
477
354
|
|
|
478
355
|
```js
|
|
479
|
-
// Relative delay
|
|
480
356
|
await queue.enqueue("reminder", payload, { schedule: { delay: 30_000 } });
|
|
481
|
-
|
|
482
|
-
// Absolute timestamp
|
|
483
357
|
await queue.enqueue("report", payload, { schedule: { runAt: "2026-09-01T09:00:00Z" } });
|
|
484
|
-
|
|
485
|
-
// Cron expression (stored for recurring jobs)
|
|
486
358
|
await queue.enqueue("cleanup", payload, { schedule: { cron: "0 3 * * *" } });
|
|
487
359
|
```
|
|
488
360
|
|
|
489
|
-
Delayed jobs
|
|
361
|
+
Delayed jobs stay dormant until their scheduled time is reached.
|
|
490
362
|
|
|
491
363
|
---
|
|
492
364
|
|
|
493
365
|
## Priority
|
|
494
366
|
|
|
495
|
-
|
|
367
|
+
Jobs with a higher priority value are processed sooner. The default is `0`.
|
|
496
368
|
|
|
497
369
|
```js
|
|
498
370
|
await queue.enqueue("urgent-task", payload, { priority: 100 });
|
|
499
371
|
await queue.enqueue("normal-task", payload, { priority: 0 });
|
|
500
|
-
await queue.enqueue("low-task",
|
|
501
|
-
//
|
|
372
|
+
await queue.enqueue("low-task", payload, { priority: -10 });
|
|
373
|
+
// order: urgent → normal → low
|
|
502
374
|
```
|
|
503
375
|
|
|
504
|
-
See [FEATURES.md → Priority](./FEATURES.md#priority).
|
|
505
|
-
|
|
506
376
|
---
|
|
507
377
|
|
|
508
378
|
## Rate Limiting
|
|
509
379
|
|
|
510
380
|
```js
|
|
511
381
|
const queue = client.createQueue("webhooks", {
|
|
512
|
-
rateLimit: { max: 50, duration: 60_000 },
|
|
382
|
+
rateLimit: { max: 50, duration: 60_000 },
|
|
513
383
|
});
|
|
514
384
|
```
|
|
515
385
|
|
|
516
|
-
When
|
|
386
|
+
When a queue reaches its limit, workers pause claiming new jobs until the time window resets. Jobs are not discarded.
|
|
517
387
|
|
|
518
388
|
---
|
|
519
389
|
|
|
520
390
|
## Dead Letter Queue
|
|
521
391
|
|
|
522
|
-
When a job
|
|
392
|
+
When a job reaches the end of its retry budget, it is moved to the dead-letter queue with status `"dead"`.
|
|
523
393
|
|
|
524
394
|
```js
|
|
525
395
|
client.on("job:dead", async (job, error) => {
|
|
@@ -529,64 +399,37 @@ client.on("job:dead", async (job, error) => {
|
|
|
529
399
|
const deadJobs = await queue.getJobs("dead");
|
|
530
400
|
```
|
|
531
401
|
|
|
532
|
-
|
|
402
|
+
All failure history remains attached to the job record.
|
|
533
403
|
|
|
534
404
|
---
|
|
535
405
|
|
|
536
406
|
## Graceful Shutdown
|
|
537
407
|
|
|
538
|
-
|
|
408
|
+
Call `client.close()` before your process exits:
|
|
539
409
|
|
|
540
410
|
```js
|
|
541
411
|
process.on("SIGTERM", async () => {
|
|
542
|
-
await client.close();
|
|
412
|
+
await client.close();
|
|
543
413
|
process.exit(0);
|
|
544
414
|
});
|
|
415
|
+
|
|
545
416
|
process.on("SIGINT", async () => {
|
|
546
417
|
await client.close();
|
|
547
418
|
process.exit(0);
|
|
548
419
|
});
|
|
549
420
|
```
|
|
550
421
|
|
|
551
|
-
|
|
422
|
+
This stops workers cleanly, releases locks, and allows stalled-job recovery to continue safely after restarts.
|
|
552
423
|
|
|
553
424
|
---
|
|
554
425
|
|
|
555
426
|
## Custom Storage Adapter
|
|
556
427
|
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
**TypeScript**
|
|
560
|
-
```ts
|
|
561
|
-
import type { StorageAdapter } from "queue-jobs-worker";
|
|
562
|
-
|
|
563
|
-
class MongoStorageAdapter implements StorageAdapter {
|
|
564
|
-
async initialize() { /* connect, create indexes */ }
|
|
565
|
-
async close() { /* disconnect */ }
|
|
566
|
-
async enqueue(input) { /* ... */ }
|
|
567
|
-
async claim(input) { /* atomic claim */ }
|
|
568
|
-
async complete(jobId) { /* ... */ }
|
|
569
|
-
async requeue(input) { /* ... */ }
|
|
570
|
-
async moveToDlq(input) { /* ... */ }
|
|
571
|
-
async releaseLock(jobId) { /* ... */ }
|
|
572
|
-
async recoverStalledJobs(queue, now) { /* ... */ }
|
|
573
|
-
async getJob(jobId) { /* ... */ }
|
|
574
|
-
async getJobs(filter) { /* ... */ }
|
|
575
|
-
async getJobCounts(queue) { /* ... */ }
|
|
576
|
-
async checkAndIncrementRateLimit(queue, max, windowMs, now) { /* ... */ }
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
const client = QueueClient.withAdapter(new MongoStorageAdapter(), {
|
|
580
|
-
defaults: { attempts: 5 },
|
|
581
|
-
});
|
|
582
|
-
await client.init();
|
|
583
|
-
```
|
|
428
|
+
You can provide a custom backend by implementing the `StorageAdapter` interface. The same idea applies in JavaScript or TypeScript; the main difference is whether you add explicit interface typing in TypeScript.
|
|
584
429
|
|
|
585
|
-
**JavaScript**
|
|
586
430
|
```js
|
|
587
431
|
const { QueueClient } = require("queue-jobs-worker");
|
|
588
432
|
|
|
589
|
-
// In JS there's no interface to implement — just match the method signatures
|
|
590
433
|
class MongoStorageAdapter {
|
|
591
434
|
async initialize() { /* connect, create indexes */ }
|
|
592
435
|
async close() { /* disconnect */ }
|
|
@@ -606,6 +449,7 @@ class MongoStorageAdapter {
|
|
|
606
449
|
const client = QueueClient.withAdapter(new MongoStorageAdapter(), {
|
|
607
450
|
defaults: { attempts: 5 },
|
|
608
451
|
});
|
|
452
|
+
|
|
609
453
|
await client.init();
|
|
610
454
|
```
|
|
611
455
|
|
|
@@ -618,66 +462,64 @@ await client.init();
|
|
|
618
462
|
| Option | Type | Default | Description |
|
|
619
463
|
|---|---|---|---|
|
|
620
464
|
| `dialect` | `"memory" \| "redis" \| "postgres" \| "mysql"` | `"memory"` | Storage backend |
|
|
621
|
-
| `connectionString` | `string` | — | Required for
|
|
622
|
-
| `defaults.attempts` | `number` | `3` |
|
|
623
|
-
| `defaults.retryDelay` | `number` | `1000` |
|
|
624
|
-
| `defaults.backoff` | `"fixed" \| "linear" \| "exponential"` | `"exponential"` |
|
|
625
|
-
| `defaults.timeout` | `number` | `30000` |
|
|
465
|
+
| `connectionString` | `string` | — | Required for Redis/PostgreSQL/MySQL |
|
|
466
|
+
| `defaults.attempts` | `number` | `3` | Max retries per job |
|
|
467
|
+
| `defaults.retryDelay` | `number` | `1000` | Base retry delay in ms |
|
|
468
|
+
| `defaults.backoff` | `"fixed" \| "linear" \| "exponential"` | `"exponential"` | Retry strategy |
|
|
469
|
+
| `defaults.timeout` | `number` | `30000` | Per-attempt timeout in ms |
|
|
626
470
|
| `defaults.concurrency` | `number` | `10` | Default worker concurrency |
|
|
627
|
-
| `defaults.pollInterval` | `number` | `1000` |
|
|
628
|
-
| `defaults.stalledInterval` | `number` | `30000` | Stalled-job check interval
|
|
629
|
-
| `defaults.lockDuration` | `number` | `60000` | Lock TTL
|
|
630
|
-
| `defaults.rateLimit` | `{ max, duration }` | — | Optional rate
|
|
471
|
+
| `defaults.pollInterval` | `number` | `1000` | Poll interval in ms |
|
|
472
|
+
| `defaults.stalledInterval` | `number` | `30000` | Stalled-job check interval in ms |
|
|
473
|
+
| `defaults.lockDuration` | `number` | `60000` | Lock TTL in ms |
|
|
474
|
+
| `defaults.rateLimit` | `{ max, duration }` | — | Optional rate limiting |
|
|
631
475
|
|
|
632
476
|
### `client.init()`
|
|
633
477
|
|
|
634
|
-
|
|
478
|
+
Initializes the configured backend. This is required for Redis, PostgreSQL, and MySQL before queue operations. It is safe to call more than once.
|
|
635
479
|
|
|
636
480
|
### `client.createQueue<TPayload>(name, options?)`
|
|
637
481
|
|
|
638
|
-
Creates and returns a
|
|
482
|
+
Creates and returns a queue. Queue-specific options override client defaults.
|
|
639
483
|
|
|
640
484
|
### `client.getQueue<TPayload>(name)` / `client.requireQueue<TPayload>(name)`
|
|
641
485
|
|
|
642
|
-
|
|
486
|
+
Fetches an existing queue by name. `requireQueue()` throws if none exists.
|
|
643
487
|
|
|
644
488
|
### `client.on(event, listener)` / `client.once(...)` / `client.off(...)`
|
|
645
489
|
|
|
646
|
-
|
|
490
|
+
Registers and removes event listeners for queue and worker lifecycle events.
|
|
647
491
|
|
|
648
492
|
### `client.close()`
|
|
649
493
|
|
|
650
|
-
|
|
494
|
+
Stops workers and closes storage connections gracefully.
|
|
651
495
|
|
|
652
496
|
### `QueueClient.withAdapter(adapter, options?)`
|
|
653
497
|
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
---
|
|
498
|
+
Creates a client using a custom storage backend.
|
|
657
499
|
|
|
658
500
|
### `queue.enqueue(type, payload, options?)`
|
|
659
501
|
|
|
660
502
|
| Option | Type | Description |
|
|
661
503
|
|---|---|---|
|
|
662
|
-
| `attempts` | `number` |
|
|
663
|
-
| `retryDelay` | `number` | Base retry delay
|
|
664
|
-
| `backoff` | `string` |
|
|
665
|
-
| `timeout` | `number` | Per-attempt timeout
|
|
666
|
-
| `priority` | `number` | Higher
|
|
667
|
-
| `schedule.delay` | `number` | Delay before eligible
|
|
504
|
+
| `attempts` | `number` | Maximum attempts for this job |
|
|
505
|
+
| `retryDelay` | `number` | Base retry delay in ms |
|
|
506
|
+
| `backoff` | `string` | Retry backoff strategy |
|
|
507
|
+
| `timeout` | `number` | Per-attempt timeout in ms |
|
|
508
|
+
| `priority` | `number` | Higher values are processed first |
|
|
509
|
+
| `schedule.delay` | `number` | Delay before the job becomes eligible |
|
|
668
510
|
| `schedule.runAt` | `string \| number` | Absolute run time |
|
|
669
|
-
| `schedule.cron` | `string` | Cron expression
|
|
511
|
+
| `schedule.cron` | `string` | Cron expression for recurring jobs |
|
|
670
512
|
|
|
671
513
|
### `queue.process(type, processor)`
|
|
672
514
|
|
|
673
|
-
|
|
515
|
+
Registers an async processor for a job type. The processor signature is `async (job, signal) => ...`, where `signal` is an `AbortSignal` aborted when the per-attempt timeout is reached.
|
|
674
516
|
|
|
675
517
|
### `queue.createWorker(options?)`
|
|
676
518
|
|
|
677
519
|
| Option | Type | Default | Description |
|
|
678
520
|
|---|---|---|---|
|
|
679
|
-
| `concurrency` | `number` | queue config |
|
|
680
|
-
| `shutdownTimeout` | `number` | `30000` |
|
|
521
|
+
| `concurrency` | `number` | queue config | Maximum simultaneous job executions |
|
|
522
|
+
| `shutdownTimeout` | `number` | `30000` | Graceful shutdown wait time in ms |
|
|
681
523
|
|
|
682
524
|
### `queue.getJob(id)` / `queue.getJobs(status?, limit?, offset?)`
|
|
683
525
|
### `queue.getJobCounts()`
|
|
@@ -688,7 +530,7 @@ Register an async processor function for a job type.
|
|
|
688
530
|
|
|
689
531
|
| Feature | Memory | Redis | PostgreSQL | MySQL |
|
|
690
532
|
|---|:---:|:---:|:---:|:---:|
|
|
691
|
-
| Persistence | | ✓ | ✓ | ✓ |
|
|
533
|
+
| Persistence | — | ✓ | ✓ | ✓ |
|
|
692
534
|
| Atomic claim | ✓ | ✓ (Lua) | ✓ (SKIP LOCKED) | ✓ (SKIP LOCKED) |
|
|
693
535
|
| Priority ordering | ✓ | ✓ | ✓ | ✓ |
|
|
694
536
|
| Delayed jobs | ✓ | ✓ | ✓ | ✓ |
|
|
@@ -703,9 +545,9 @@ Register an async processor function for a job type.
|
|
|
703
545
|
|
|
704
546
|
## Security
|
|
705
547
|
|
|
706
|
-
- Job payloads are
|
|
707
|
-
- Error messages do not
|
|
708
|
-
- Connection strings should
|
|
548
|
+
- Job payloads are not logged by default.
|
|
549
|
+
- Error messages do not expose payload data.
|
|
550
|
+
- Connection strings should come from environment variables rather than source code.
|
|
709
551
|
- See [SECURITY.md](./SECURITY.md) for the full policy.
|
|
710
552
|
|
|
711
553
|
```js
|
|
@@ -726,7 +568,7 @@ const client = new QueueClient({
|
|
|
726
568
|
|
|
727
569
|
## Contributing
|
|
728
570
|
|
|
729
|
-
|
|
571
|
+
Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for contribution guidelines.
|
|
730
572
|
|
|
731
573
|
---
|
|
732
574
|
|
|
@@ -736,6 +578,6 @@ MIT — [LICENSE](./LICENSE)
|
|
|
736
578
|
|
|
737
579
|
## Donation
|
|
738
580
|
|
|
739
|
-
If
|
|
581
|
+
If this project has been useful to you, consider supporting it with a coffee.
|
|
740
582
|
|
|
741
583
|
**BTC:** `12dxgVQ3sRFhc4g7M6oydsN2tTMMthJJqS`
|