power-queues 2.0.6 → 2.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/index.cjs +31 -27
  2. package/dist/index.d.cts +4 -4
  3. package/dist/index.d.ts +4 -4
  4. package/dist/index.js +31 -27
  5. package/package.json +1 -1
package/dist/index.cjs CHANGED
@@ -280,7 +280,6 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
280
280
  this.workerSelectionTimeoutMs = 80;
281
281
  this.workerMaxRetries = 5;
282
282
  this.workerClearAttemptsTimeoutMs = 864e5;
283
- this.workerStatusTimeoutMs = 864e5;
284
283
  }
285
284
  async onSelected(data) {
286
285
  return data;
@@ -291,6 +290,12 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
291
290
  }
292
291
  async onSuccess(id, payload, createdAt, job, key) {
293
292
  }
293
+ async onBatchError(err, tasks) {
294
+ }
295
+ async onError(err, id, payload, createdAt, job, key) {
296
+ }
297
+ async onRetry(err, id, payload, createdAt, job, key, attempts) {
298
+ }
294
299
  async runQueue() {
295
300
  await this.createGroup("0-0");
296
301
  await this.consumerLoop();
@@ -351,10 +356,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
351
356
  await this.redis.set(`${queueName}:${job}:ready`, 0);
352
357
  await this.redis.set(`${queueName}:${job}:err`, 0);
353
358
  await this.redis.set(`${queueName}:${job}:ok`, 0);
354
- await this.redis.pexpire(`${queueName}:${job}:total`, this.workerStatusTimeoutMs);
355
- await this.redis.pexpire(`${queueName}:${job}:ready`, this.workerStatusTimeoutMs);
356
- await this.redis.pexpire(`${queueName}:${job}:err`, this.workerStatusTimeoutMs);
357
- await this.redis.pexpire(`${queueName}:${job}:ok`, this.workerStatusTimeoutMs);
359
+ await this.redis.pexpire(`${queueName}:${job}:total`, opts.statusTimeoutMs || 864e5);
360
+ await this.redis.pexpire(`${queueName}:${job}:ready`, opts.statusTimeoutMs || 864e5);
361
+ await this.redis.pexpire(`${queueName}:${job}:err`, opts.statusTimeoutMs || 864e5);
362
+ await this.redis.pexpire(`${queueName}:${job}:ok`, opts.statusTimeoutMs || 864e5);
358
363
  }
359
364
  await Promise.all(runners);
360
365
  return result;
@@ -539,26 +544,25 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
539
544
  }
540
545
  async success(id, payload, createdAt, job, key) {
541
546
  if (this.executeJobStatus) {
542
- await this.status(id, payload, createdAt, job, key);
547
+ const prefix = `${this.stream}:${job}:`;
548
+ const { ready = 0, ok = 0 } = await this.getMany(prefix);
549
+ await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
543
550
  }
544
551
  await this.onSuccess(id, payload, createdAt, job, key);
545
552
  }
546
- async status(id, payload, createdAt, job, key) {
547
- const prefix = `s:${this.stream}:`;
548
- const { ready = 0, ok = 0 } = await this.getMany(prefix);
549
- await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
550
- }
551
553
  async batchError(err, tasks) {
554
+ await this.onBatchError(err, tasks);
552
555
  }
553
- async error(err, id, payload, createdAt, job, key) {
556
+ async error(err, id, payload, createdAt, job, key, attempt) {
557
+ if (this.executeJobStatus && attempt >= this.workerMaxRetries) {
558
+ const prefix = `${this.stream}:${job}:`;
559
+ const { ready = 0, err: err2 = 0 } = await this.getMany(prefix);
560
+ await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}err`, value: err2 + 1 }], this.executeJobStatusTtlSec);
561
+ }
554
562
  await this.onError(err, id, payload, createdAt, job, key);
555
563
  }
556
- async onError(err, id, payload, createdAt, job, key) {
557
- }
558
- async attempt(err, id, payload, createdAt, job, key, attempts) {
559
- await this.onRetry(err, id, payload, createdAt, job, key, attempts);
560
- }
561
- async onRetry(err, id, payload, createdAt, job, key, attempts) {
564
+ async attempt(err, id, payload, createdAt, job, key, attempt) {
565
+ await this.onRetry(err, id, payload, createdAt, job, key, attempt);
562
566
  }
563
567
  async execute(tasks) {
564
568
  const result = [];
@@ -604,10 +608,10 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
604
608
  await this.success(id, payload, createdAt, job, key);
605
609
  return { id };
606
610
  } catch (err) {
607
- const attempts = await this.incrAttempts(id);
608
- await this.attempt(err, id, payload, createdAt, job, key, attempts);
609
- await this.error(err, id, payload, createdAt, job, key);
610
- if (attempts >= this.workerMaxRetries) {
611
+ const attempt = await this.incrAttempts(id);
612
+ await this.attempt(err, id, payload, createdAt, job, key, attempt);
613
+ await this.error(err, id, payload, createdAt, job, key, attempt);
614
+ if (attempt >= this.workerMaxRetries) {
611
615
  await this.addTasks(`${this.stream}:dlq`, [{
612
616
  payload: {
613
617
  ...payload,
@@ -615,7 +619,7 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
615
619
  createdAt,
616
620
  job,
617
621
  id,
618
- attempts
622
+ attempt
619
623
  }
620
624
  }]);
621
625
  await this.clearAttempts(id);
@@ -665,11 +669,11 @@ var PowerQueues = class extends import_power_redis.PowerRedis {
665
669
  await this.success(id, payload, createdAt, job, key);
666
670
  return { id };
667
671
  } catch (err) {
668
- const attempts = await this.incrAttempts(id);
672
+ const attempt = await this.incrAttempts(id);
669
673
  try {
670
- await this.attempt(err, id, payload, createdAt, job, key, attempts);
671
- await this.error(err, id, payload, createdAt, job, key);
672
- if (attempts >= this.workerMaxRetries) {
674
+ await this.attempt(err, id, payload, createdAt, job, key, attempt);
675
+ await this.error(err, id, payload, createdAt, job, key, attempt);
676
+ if (attempt >= this.workerMaxRetries) {
673
677
  await this.addTasks(`${this.stream}:dlq`, [{
674
678
  payload: {
675
679
  ...payload,
package/dist/index.d.cts CHANGED
@@ -14,6 +14,7 @@ type AddTasksOptions = {
14
14
  trimLimit?: number;
15
15
  id?: string;
16
16
  status?: boolean;
17
+ statusTimeoutMs?: number;
17
18
  idem?: boolean;
18
19
  };
19
20
  type Task = {
@@ -53,11 +54,13 @@ declare class PowerQueues extends PowerRedis {
53
54
  readonly workerSelectionTimeoutMs: number;
54
55
  readonly workerMaxRetries: number;
55
56
  readonly workerClearAttemptsTimeoutMs: number;
56
- readonly workerStatusTimeoutMs: number;
57
57
  onSelected(data: Array<[string, any[], number, string, string]>): Promise<[string, any[], number, string, string][]>;
58
58
  onExecute(id: string, payload: any, createdAt: number, job: string, key: string, attempt: number): Promise<void>;
59
59
  onExecuted(data: Array<[string, any[], number, string, string]>): Promise<void>;
60
60
  onSuccess(id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
61
+ onBatchError(err: any, tasks?: Array<[string, any[], number, string, string]>): Promise<void>;
62
+ onError(err: any, id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
63
+ onRetry(err: any, id: string, payload: any, createdAt: number, job: string, key: string, attempts: number): Promise<void>;
61
64
  runQueue(): Promise<void>;
62
65
  consumerLoop(): Promise<void>;
63
66
  addTasks(queueName: string, data: any[], opts?: AddTasksOptions): Promise<string[]>;
@@ -74,12 +77,9 @@ declare class PowerQueues extends PowerRedis {
74
77
  private getAttempts;
75
78
  private clearAttempts;
76
79
  private success;
77
- private status;
78
80
  private batchError;
79
81
  private error;
80
- onError(err: any, id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
81
82
  private attempt;
82
- onRetry(err: any, id: string, payload: any, createdAt: number, job: string, key: string, attempts: number): Promise<void>;
83
83
  private execute;
84
84
  private executeProcess;
85
85
  private approve;
package/dist/index.d.ts CHANGED
@@ -14,6 +14,7 @@ type AddTasksOptions = {
14
14
  trimLimit?: number;
15
15
  id?: string;
16
16
  status?: boolean;
17
+ statusTimeoutMs?: number;
17
18
  idem?: boolean;
18
19
  };
19
20
  type Task = {
@@ -53,11 +54,13 @@ declare class PowerQueues extends PowerRedis {
53
54
  readonly workerSelectionTimeoutMs: number;
54
55
  readonly workerMaxRetries: number;
55
56
  readonly workerClearAttemptsTimeoutMs: number;
56
- readonly workerStatusTimeoutMs: number;
57
57
  onSelected(data: Array<[string, any[], number, string, string]>): Promise<[string, any[], number, string, string][]>;
58
58
  onExecute(id: string, payload: any, createdAt: number, job: string, key: string, attempt: number): Promise<void>;
59
59
  onExecuted(data: Array<[string, any[], number, string, string]>): Promise<void>;
60
60
  onSuccess(id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
61
+ onBatchError(err: any, tasks?: Array<[string, any[], number, string, string]>): Promise<void>;
62
+ onError(err: any, id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
63
+ onRetry(err: any, id: string, payload: any, createdAt: number, job: string, key: string, attempts: number): Promise<void>;
61
64
  runQueue(): Promise<void>;
62
65
  consumerLoop(): Promise<void>;
63
66
  addTasks(queueName: string, data: any[], opts?: AddTasksOptions): Promise<string[]>;
@@ -74,12 +77,9 @@ declare class PowerQueues extends PowerRedis {
74
77
  private getAttempts;
75
78
  private clearAttempts;
76
79
  private success;
77
- private status;
78
80
  private batchError;
79
81
  private error;
80
- onError(err: any, id: string, payload: any, createdAt: number, job: string, key: string): Promise<void>;
81
82
  private attempt;
82
- onRetry(err: any, id: string, payload: any, createdAt: number, job: string, key: string, attempts: number): Promise<void>;
83
83
  private execute;
84
84
  private executeProcess;
85
85
  private approve;
package/dist/index.js CHANGED
@@ -254,7 +254,6 @@ var PowerQueues = class extends PowerRedis {
254
254
  this.workerSelectionTimeoutMs = 80;
255
255
  this.workerMaxRetries = 5;
256
256
  this.workerClearAttemptsTimeoutMs = 864e5;
257
- this.workerStatusTimeoutMs = 864e5;
258
257
  }
259
258
  async onSelected(data) {
260
259
  return data;
@@ -265,6 +264,12 @@ var PowerQueues = class extends PowerRedis {
265
264
  }
266
265
  async onSuccess(id, payload, createdAt, job, key) {
267
266
  }
267
+ async onBatchError(err, tasks) {
268
+ }
269
+ async onError(err, id, payload, createdAt, job, key) {
270
+ }
271
+ async onRetry(err, id, payload, createdAt, job, key, attempts) {
272
+ }
268
273
  async runQueue() {
269
274
  await this.createGroup("0-0");
270
275
  await this.consumerLoop();
@@ -325,10 +330,10 @@ var PowerQueues = class extends PowerRedis {
325
330
  await this.redis.set(`${queueName}:${job}:ready`, 0);
326
331
  await this.redis.set(`${queueName}:${job}:err`, 0);
327
332
  await this.redis.set(`${queueName}:${job}:ok`, 0);
328
- await this.redis.pexpire(`${queueName}:${job}:total`, this.workerStatusTimeoutMs);
329
- await this.redis.pexpire(`${queueName}:${job}:ready`, this.workerStatusTimeoutMs);
330
- await this.redis.pexpire(`${queueName}:${job}:err`, this.workerStatusTimeoutMs);
331
- await this.redis.pexpire(`${queueName}:${job}:ok`, this.workerStatusTimeoutMs);
333
+ await this.redis.pexpire(`${queueName}:${job}:total`, opts.statusTimeoutMs || 864e5);
334
+ await this.redis.pexpire(`${queueName}:${job}:ready`, opts.statusTimeoutMs || 864e5);
335
+ await this.redis.pexpire(`${queueName}:${job}:err`, opts.statusTimeoutMs || 864e5);
336
+ await this.redis.pexpire(`${queueName}:${job}:ok`, opts.statusTimeoutMs || 864e5);
332
337
  }
333
338
  await Promise.all(runners);
334
339
  return result;
@@ -513,26 +518,25 @@ var PowerQueues = class extends PowerRedis {
513
518
  }
514
519
  async success(id, payload, createdAt, job, key) {
515
520
  if (this.executeJobStatus) {
516
- await this.status(id, payload, createdAt, job, key);
521
+ const prefix = `${this.stream}:${job}:`;
522
+ const { ready = 0, ok = 0 } = await this.getMany(prefix);
523
+ await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
517
524
  }
518
525
  await this.onSuccess(id, payload, createdAt, job, key);
519
526
  }
520
- async status(id, payload, createdAt, job, key) {
521
- const prefix = `s:${this.stream}:`;
522
- const { ready = 0, ok = 0 } = await this.getMany(prefix);
523
- await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}ok`, value: ok + 1 }], this.executeJobStatusTtlSec);
524
- }
525
527
  async batchError(err, tasks) {
528
+ await this.onBatchError(err, tasks);
526
529
  }
527
- async error(err, id, payload, createdAt, job, key) {
530
+ async error(err, id, payload, createdAt, job, key, attempt) {
531
+ if (this.executeJobStatus && attempt >= this.workerMaxRetries) {
532
+ const prefix = `${this.stream}:${job}:`;
533
+ const { ready = 0, err: err2 = 0 } = await this.getMany(prefix);
534
+ await this.setMany([{ key: `${prefix}ready`, value: ready + 1 }, { key: `${prefix}err`, value: err2 + 1 }], this.executeJobStatusTtlSec);
535
+ }
528
536
  await this.onError(err, id, payload, createdAt, job, key);
529
537
  }
530
- async onError(err, id, payload, createdAt, job, key) {
531
- }
532
- async attempt(err, id, payload, createdAt, job, key, attempts) {
533
- await this.onRetry(err, id, payload, createdAt, job, key, attempts);
534
- }
535
- async onRetry(err, id, payload, createdAt, job, key, attempts) {
538
+ async attempt(err, id, payload, createdAt, job, key, attempt) {
539
+ await this.onRetry(err, id, payload, createdAt, job, key, attempt);
536
540
  }
537
541
  async execute(tasks) {
538
542
  const result = [];
@@ -578,10 +582,10 @@ var PowerQueues = class extends PowerRedis {
578
582
  await this.success(id, payload, createdAt, job, key);
579
583
  return { id };
580
584
  } catch (err) {
581
- const attempts = await this.incrAttempts(id);
582
- await this.attempt(err, id, payload, createdAt, job, key, attempts);
583
- await this.error(err, id, payload, createdAt, job, key);
584
- if (attempts >= this.workerMaxRetries) {
585
+ const attempt = await this.incrAttempts(id);
586
+ await this.attempt(err, id, payload, createdAt, job, key, attempt);
587
+ await this.error(err, id, payload, createdAt, job, key, attempt);
588
+ if (attempt >= this.workerMaxRetries) {
585
589
  await this.addTasks(`${this.stream}:dlq`, [{
586
590
  payload: {
587
591
  ...payload,
@@ -589,7 +593,7 @@ var PowerQueues = class extends PowerRedis {
589
593
  createdAt,
590
594
  job,
591
595
  id,
592
- attempts
596
+ attempt
593
597
  }
594
598
  }]);
595
599
  await this.clearAttempts(id);
@@ -639,11 +643,11 @@ var PowerQueues = class extends PowerRedis {
639
643
  await this.success(id, payload, createdAt, job, key);
640
644
  return { id };
641
645
  } catch (err) {
642
- const attempts = await this.incrAttempts(id);
646
+ const attempt = await this.incrAttempts(id);
643
647
  try {
644
- await this.attempt(err, id, payload, createdAt, job, key, attempts);
645
- await this.error(err, id, payload, createdAt, job, key);
646
- if (attempts >= this.workerMaxRetries) {
648
+ await this.attempt(err, id, payload, createdAt, job, key, attempt);
649
+ await this.error(err, id, payload, createdAt, job, key, attempt);
650
+ if (attempt >= this.workerMaxRetries) {
647
651
  await this.addTasks(`${this.stream}:dlq`, [{
648
652
  payload: {
649
653
  ...payload,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "power-queues",
3
- "version": "2.0.6",
3
+ "version": "2.0.8",
4
4
  "description": "Base classes for implementing custom queues in redis under high load conditions based on nestjs.",
5
5
  "author": "ihor-bielchenko",
6
6
  "license": "MIT",