power-queues 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Processor.d.ts +4 -15
- package/dist/Processor.js +4 -51
- package/dist/Processor.js.map +1 -1
- package/dist/Queue.d.ts +28 -35
- package/dist/Queue.js +91 -140
- package/dist/Queue.js.map +1 -1
- package/dist/QueueMethod.d.ts +9 -0
- package/dist/QueueMethod.js +29 -0
- package/dist/QueueMethod.js.map +1 -0
- package/dist/RedisManager.d.ts +22 -0
- package/dist/RedisManager.js +120 -0
- package/dist/RedisManager.js.map +1 -0
- package/dist/index.d.ts +5 -6
- package/dist/index.js +7 -11
- package/dist/index.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/package.json +11 -54
- package/dist/QueuePortion.d.ts +0 -7
- package/dist/QueuePortion.js +0 -31
- package/dist/QueuePortion.js.map +0 -1
- package/dist/QueuePortionProcessor.d.ts +0 -8
- package/dist/QueuePortionProcessor.js +0 -75
- package/dist/QueuePortionProcessor.js.map +0 -1
- package/dist/QueuePortionProcessorMethod.d.ts +0 -12
- package/dist/QueuePortionProcessorMethod.js +0 -116
- package/dist/QueuePortionProcessorMethod.js.map +0 -1
- package/dist/QueueProcessor.d.ts +0 -14
- package/dist/QueueProcessor.js +0 -61
- package/dist/QueueProcessor.js.map +0 -1
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface DistLockInterface {
|
|
2
|
+
key: string;
|
|
3
|
+
token: string;
|
|
4
|
+
ttlMs: number;
|
|
5
|
+
extend: (newTtlMs?: number) => Promise<boolean>;
|
|
6
|
+
unlock: () => Promise<boolean>;
|
|
7
|
+
}
|
|
8
|
+
export interface TaskInterface {
|
|
9
|
+
payload: object;
|
|
10
|
+
attempt?: number;
|
|
11
|
+
processor?: string;
|
|
12
|
+
method?: number;
|
|
13
|
+
enqueuedAt?: number;
|
|
14
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "power-queues",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "Base classes for implementing custom queues in redis under high load conditions.",
|
|
3
|
+
"version": "1.0.5",
|
|
4
|
+
"description": "Base classes for implementing custom queues in redis under high load conditions based on nestjs.",
|
|
5
5
|
"author": "ihor-bielchenko",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"repository": {
|
|
@@ -18,63 +18,20 @@
|
|
|
18
18
|
"dist"
|
|
19
19
|
],
|
|
20
20
|
"keywords": [
|
|
21
|
+
"nestjs",
|
|
22
|
+
"ioredis",
|
|
21
23
|
"queue",
|
|
22
24
|
"redis",
|
|
23
25
|
"custom",
|
|
24
26
|
"portions",
|
|
25
27
|
"order"
|
|
26
28
|
],
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
|
|
35
|
-
"test:e2e": "jest --config ./test/jest-e2e.json"
|
|
36
|
-
},
|
|
37
|
-
"peerDependencies": {
|
|
38
|
-
"ioredis": "^5.6.1",
|
|
39
|
-
"uuid": "^11.1.0"
|
|
40
|
-
},
|
|
41
|
-
"devDependencies": {
|
|
42
|
-
"@nestjs/cli": "^10.0.0",
|
|
43
|
-
"@nestjs/schematics": "^10.0.0",
|
|
44
|
-
"@nestjs/testing": "^10.0.0",
|
|
45
|
-
"@types/jest": "^29.5.2",
|
|
46
|
-
"@types/node": "^20.17.31",
|
|
47
|
-
"@types/supertest": "^6.0.0",
|
|
48
|
-
"@typescript-eslint/eslint-plugin": "^7.0.0",
|
|
49
|
-
"@typescript-eslint/parser": "^7.0.0",
|
|
50
|
-
"eslint": "^8.42.0",
|
|
51
|
-
"eslint-config-prettier": "^9.0.0",
|
|
52
|
-
"eslint-plugin-prettier": "^5.0.0",
|
|
53
|
-
"jest": "^29.5.0",
|
|
54
|
-
"prettier": "^3.0.0",
|
|
55
|
-
"source-map-support": "^0.5.21",
|
|
56
|
-
"supertest": "^7.0.0",
|
|
57
|
-
"ts-jest": "^29.1.0",
|
|
58
|
-
"ts-loader": "^9.4.3",
|
|
59
|
-
"ts-node": "^10.9.1",
|
|
60
|
-
"tsconfig-paths": "^4.2.0",
|
|
61
|
-
"typescript": "^5.1.3"
|
|
62
|
-
},
|
|
63
|
-
"jest": {
|
|
64
|
-
"moduleFileExtensions": [
|
|
65
|
-
"js",
|
|
66
|
-
"json",
|
|
67
|
-
"ts"
|
|
68
|
-
],
|
|
69
|
-
"rootDir": "src",
|
|
70
|
-
"testRegex": ".*\\.spec\\.ts$",
|
|
71
|
-
"transform": {
|
|
72
|
-
"^.+\\.(t|j)s$": "ts-jest"
|
|
73
|
-
},
|
|
74
|
-
"collectCoverageFrom": [
|
|
75
|
-
"**/*.(t|j)s"
|
|
76
|
-
],
|
|
77
|
-
"coverageDirectory": "../coverage",
|
|
78
|
-
"testEnvironment": "node"
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@nestjs-labs/nestjs-ioredis": "^11.0.4",
|
|
31
|
+
"@nestjs/common": "^11.1.6",
|
|
32
|
+
"crypto": "^1.0.1",
|
|
33
|
+
"date-fns": "^4.1.0",
|
|
34
|
+
"full-utils": "^1.0.1",
|
|
35
|
+
"ioredis": "^5.7.0"
|
|
79
36
|
}
|
|
80
37
|
}
|
package/dist/QueuePortion.d.ts
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { Queue } from './Queue';
|
|
2
|
-
export declare class QueuePortion extends Queue {
|
|
3
|
-
protected readonly portionSize: number;
|
|
4
|
-
allow(queueName: string, attemptIndex: number, data: Array<any>): Promise<boolean>;
|
|
5
|
-
select(queueName: string, attemptIndex: number): Promise<Array<any>>;
|
|
6
|
-
selectAfter(queueName: string, data?: Array<any>): Promise<Array<any>>;
|
|
7
|
-
}
|
package/dist/QueuePortion.js
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.QueuePortion = void 0;
|
|
4
|
-
const Queue_1 = require("./Queue");
|
|
5
|
-
class QueuePortion extends Queue_1.Queue {
|
|
6
|
-
constructor() {
|
|
7
|
-
super(...arguments);
|
|
8
|
-
this.portionSize = 1;
|
|
9
|
-
}
|
|
10
|
-
async allow(queueName, attemptIndex, data) {
|
|
11
|
-
return data.length > 0 && data.filter((item) => !!item).length === data.length;
|
|
12
|
-
}
|
|
13
|
-
async select(queueName, attemptIndex) {
|
|
14
|
-
const queueKey = this.queueKey(queueName, attemptIndex);
|
|
15
|
-
const data = await this.redis.lpop(queueKey, this.portionSize);
|
|
16
|
-
if (Array.isArray(data) && data.length > 0) {
|
|
17
|
-
return await this.selectAfter(queueName, data);
|
|
18
|
-
}
|
|
19
|
-
return [];
|
|
20
|
-
}
|
|
21
|
-
async selectAfter(queueName, data = []) {
|
|
22
|
-
let i = 0, output = [];
|
|
23
|
-
while (i < data.length) {
|
|
24
|
-
output.push(await super.selectAfter(queueName, data[i]));
|
|
25
|
-
i++;
|
|
26
|
-
}
|
|
27
|
-
return output;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
exports.QueuePortion = QueuePortion;
|
|
31
|
-
//# sourceMappingURL=QueuePortion.js.map
|
package/dist/QueuePortion.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"QueuePortion.js","sourceRoot":"","sources":["../src/QueuePortion.ts"],"names":[],"mappings":";;;AAAA,mCAAiC;AAEjC,MAAa,YAAa,SAAQ,aAAK;IAAvC;;QACoB,gBAAW,GAAW,CAAC,CAAC;IA0B5C,CAAC;IAxBA,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QACpE,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,YAAoB;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE/D,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,EAAE,CAAC;IACX,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,OAAmB,EAAE;QACzD,IAAI,CAAC,GAAG,CAAC,EACR,MAAM,GAAG,EAAE,CAAC;QAEb,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC,EAAE,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;CACD;AA3BD,oCA2BC"}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { QueueProcessor } from './QueueProcessor';
|
|
2
|
-
export declare class QueuePortionProcessor extends QueueProcessor {
|
|
3
|
-
protected readonly portionSize: number;
|
|
4
|
-
allow(queueName: string, attemptIndex: number, data: Array<any>): Promise<boolean>;
|
|
5
|
-
select(queueName: string, attemptIndex: number): Promise<Array<any>>;
|
|
6
|
-
selectAfter(queueName: string, data: Array<any>): Promise<Array<any>>;
|
|
7
|
-
excecuteWrapper(queueName: string, attemptIndex: number, data: Array<any>): Promise<void>;
|
|
8
|
-
}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.QueuePortionProcessor = void 0;
|
|
4
|
-
const QueueProcessor_1 = require("./QueueProcessor");
|
|
5
|
-
class QueuePortionProcessor extends QueueProcessor_1.QueueProcessor {
|
|
6
|
-
constructor() {
|
|
7
|
-
super(...arguments);
|
|
8
|
-
this.portionSize = 1;
|
|
9
|
-
}
|
|
10
|
-
async allow(queueName, attemptIndex, data) {
|
|
11
|
-
return data.length > 0 && data.filter((item) => !!item).length === data.length;
|
|
12
|
-
}
|
|
13
|
-
async select(queueName, attemptIndex) {
|
|
14
|
-
const queueKey = this.queueKey(queueName, attemptIndex);
|
|
15
|
-
const data = await this.redis.lpop(queueKey, this.portionSize);
|
|
16
|
-
const output = await this.selectAfter(queueName, data);
|
|
17
|
-
return output;
|
|
18
|
-
}
|
|
19
|
-
async selectAfter(queueName, data) {
|
|
20
|
-
let i = 0, output = [];
|
|
21
|
-
if (Array.isArray(data) && data.length > 0) {
|
|
22
|
-
while (i < data.length) {
|
|
23
|
-
output.push(await super.selectAfter(queueName, data[i]));
|
|
24
|
-
i++;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
return output;
|
|
28
|
-
}
|
|
29
|
-
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
30
|
-
const keyData = this.keyData(queueName);
|
|
31
|
-
const processorName = keyData.processorName;
|
|
32
|
-
const processor = this.getProcessorByName(processorName);
|
|
33
|
-
if (!processor
|
|
34
|
-
|| !processor.excecute
|
|
35
|
-
|| !Array.isArray(data)
|
|
36
|
-
|| !(data.length > 0)) {
|
|
37
|
-
return;
|
|
38
|
-
}
|
|
39
|
-
await (new Promise((resolve, reject) => {
|
|
40
|
-
let length = data.length, ready = 0, i = 0;
|
|
41
|
-
while (i < length) {
|
|
42
|
-
const item = data[i];
|
|
43
|
-
processor
|
|
44
|
-
.excecute
|
|
45
|
-
.call(processor, queueName, attemptIndex, item)
|
|
46
|
-
.then((result) => {
|
|
47
|
-
this.excecute(queueName, attemptIndex, result)
|
|
48
|
-
.then(() => {
|
|
49
|
-
if (ready >= (length - 1)) {
|
|
50
|
-
resolve(ready);
|
|
51
|
-
}
|
|
52
|
-
ready += 1;
|
|
53
|
-
})
|
|
54
|
-
.catch((err) => {
|
|
55
|
-
this.retry(queueName, attemptIndex, item, err);
|
|
56
|
-
if (ready >= (length - 1)) {
|
|
57
|
-
resolve(ready);
|
|
58
|
-
}
|
|
59
|
-
ready += 1;
|
|
60
|
-
});
|
|
61
|
-
})
|
|
62
|
-
.catch((err) => {
|
|
63
|
-
this.retry(queueName, attemptIndex, item, err);
|
|
64
|
-
if (ready >= (length - 1)) {
|
|
65
|
-
resolve(ready);
|
|
66
|
-
}
|
|
67
|
-
ready += 1;
|
|
68
|
-
});
|
|
69
|
-
i++;
|
|
70
|
-
}
|
|
71
|
-
}));
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
exports.QueuePortionProcessor = QueuePortionProcessor;
|
|
75
|
-
//# sourceMappingURL=QueuePortionProcessor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"QueuePortionProcessor.js","sourceRoot":"","sources":["../src/QueuePortionProcessor.ts"],"names":[],"mappings":";;;AAAA,qDAAmD;AAGnD,MAAa,qBAAsB,SAAQ,+BAAc;IAAzD;;QACoB,gBAAW,GAAW,CAAC,CAAC;IA8E5C,CAAC;IA5EA,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QACpE,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,YAAoB;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAEvD,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,IAAgB;QACpD,IAAI,CAAC,GAAG,CAAC,EACR,MAAM,GAAG,EAAE,CAAC;QAEb,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzD,CAAC,EAAE,CAAC;YACL,CAAC;QACF,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS;eACV,CAAC,SAAS,CAAC,QAAQ;eACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;eACpB,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACxB,OAAO;QACR,CAAC;QACD,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,EACvB,KAAK,GAAG,CAAC,EACT,CAAC,GAAG,CAAC,CAAC;YAEP,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAErB,SAAS;qBACP,QAAQ;qBACR,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC;qBAC9C,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBAChB,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC;yBAC5C,IAAI,CAAC,GAAG,EAAE;wBACV,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;wBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;oBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;wBAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;oBAChB,CAAC;oBACD,KAAK,IAAI,CAAC,CAAC;gBACZ,CAAC,CAAC,CAAC;gBACJ,CAAC,EAAE,CAAC;YACL,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;CACD;AA/ED,sDA+EC"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { QueuePortionProcessor } from './QueuePortionProcessor';
|
|
2
|
-
export declare class QueuePortionProcessorMethod extends QueuePortionProcessor {
|
|
3
|
-
keyData(queueName: string): {
|
|
4
|
-
processorName: string;
|
|
5
|
-
methodIndex: number;
|
|
6
|
-
};
|
|
7
|
-
start(queueName: string): void;
|
|
8
|
-
excecuteWrapper(queueName: string, attemptIndex: number, data: Array<any>): Promise<void>;
|
|
9
|
-
successWrapper(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
10
|
-
successMethod(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
11
|
-
errorWrapper(queueName: string, attemptIndex: number, data: any, err: any): Promise<void>;
|
|
12
|
-
}
|
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.QueuePortionProcessorMethod = void 0;
|
|
4
|
-
const QueuePortionProcessor_1 = require("./QueuePortionProcessor");
|
|
5
|
-
class QueuePortionProcessorMethod extends QueuePortionProcessor_1.QueuePortionProcessor {
|
|
6
|
-
keyData(queueName) {
|
|
7
|
-
const queueNameSplit = queueName.split(`.`);
|
|
8
|
-
const processorName = queueNameSplit[queueNameSplit.length - 2];
|
|
9
|
-
const methodIndex = Number(queueNameSplit[queueNameSplit.length - 1]);
|
|
10
|
-
return {
|
|
11
|
-
processorName,
|
|
12
|
-
methodIndex,
|
|
13
|
-
};
|
|
14
|
-
}
|
|
15
|
-
start(queueName) {
|
|
16
|
-
const processors = this.getProcessors();
|
|
17
|
-
let i = 0;
|
|
18
|
-
while (i < processors.length) {
|
|
19
|
-
const processor = processors[i];
|
|
20
|
-
const methods = processor.getMethods();
|
|
21
|
-
let ii = 0;
|
|
22
|
-
while (ii < methods.length) {
|
|
23
|
-
this.listen(`${queueName}.${processor.name}.${ii}`);
|
|
24
|
-
ii++;
|
|
25
|
-
}
|
|
26
|
-
i++;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
30
|
-
const keyData = this.keyData(queueName);
|
|
31
|
-
const processorName = keyData.processorName;
|
|
32
|
-
const methodIndex = keyData.methodIndex;
|
|
33
|
-
const processor = this.getProcessorByName(processorName);
|
|
34
|
-
if (!processor
|
|
35
|
-
|| !processor.excecute
|
|
36
|
-
|| !Array.isArray(data)
|
|
37
|
-
|| !(methodIndex >= 0)
|
|
38
|
-
|| !(data.length > 0)) {
|
|
39
|
-
return;
|
|
40
|
-
}
|
|
41
|
-
const method = processor.getMethod(methodIndex);
|
|
42
|
-
let i = 0;
|
|
43
|
-
if (!method) {
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
await (new Promise((resolve, reject) => {
|
|
47
|
-
let length = data.length, ready = 0, i = 0;
|
|
48
|
-
while (i < length) {
|
|
49
|
-
const item = data[i];
|
|
50
|
-
method
|
|
51
|
-
.call(processor, queueName, attemptIndex, item)
|
|
52
|
-
.then((result) => {
|
|
53
|
-
this.excecute(queueName, attemptIndex, result)
|
|
54
|
-
.then(() => {
|
|
55
|
-
if (ready >= (length - 1)) {
|
|
56
|
-
resolve(ready);
|
|
57
|
-
}
|
|
58
|
-
ready += 1;
|
|
59
|
-
})
|
|
60
|
-
.catch((err) => {
|
|
61
|
-
this.retry(queueName, attemptIndex, item, err);
|
|
62
|
-
if (ready >= (length - 1)) {
|
|
63
|
-
resolve(ready);
|
|
64
|
-
}
|
|
65
|
-
ready += 1;
|
|
66
|
-
});
|
|
67
|
-
})
|
|
68
|
-
.catch((err) => {
|
|
69
|
-
this.retry(queueName, attemptIndex, item, err);
|
|
70
|
-
if (ready >= (length - 1)) {
|
|
71
|
-
resolve(ready);
|
|
72
|
-
}
|
|
73
|
-
ready += 1;
|
|
74
|
-
});
|
|
75
|
-
i++;
|
|
76
|
-
}
|
|
77
|
-
}));
|
|
78
|
-
}
|
|
79
|
-
async successWrapper(queueName, attemptIndex, data) {
|
|
80
|
-
const keyData = this.keyData(queueName);
|
|
81
|
-
const processorName = keyData.processorName;
|
|
82
|
-
const methodIndex = keyData.methodIndex;
|
|
83
|
-
const processor = this.getProcessorByName(processorName);
|
|
84
|
-
if (!processor) {
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
const methodsLength = processor.getMethodsLength();
|
|
88
|
-
if (methodIndex === (methodsLength - 1)) {
|
|
89
|
-
await super.successWrapper(queueName, attemptIndex, data);
|
|
90
|
-
}
|
|
91
|
-
else {
|
|
92
|
-
const nextMethodIndex = methodIndex + 1;
|
|
93
|
-
const nextMethod = processor.getMethod(nextMethodIndex);
|
|
94
|
-
if (nextMethod) {
|
|
95
|
-
const queueKey = await this.queueKey(`poll.${processorName}.${nextMethodIndex}`, 0);
|
|
96
|
-
const dataProcessed = JSON.stringify(data);
|
|
97
|
-
await this.redis.rpush(queueKey, dataProcessed);
|
|
98
|
-
await this.successMethod(queueName, attemptIndex, data);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
return data;
|
|
102
|
-
}
|
|
103
|
-
async successMethod(queueName, attemptIndex, data) {
|
|
104
|
-
}
|
|
105
|
-
async errorWrapper(queueName, attemptIndex, data, err) {
|
|
106
|
-
const keyData = this.keyData(queueName);
|
|
107
|
-
const processorName = keyData.processorName;
|
|
108
|
-
const processor = this.getProcessorByName(processorName);
|
|
109
|
-
if (!processor) {
|
|
110
|
-
return;
|
|
111
|
-
}
|
|
112
|
-
await super.errorWrapper(queueName, attemptIndex, data, err);
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
exports.QueuePortionProcessorMethod = QueuePortionProcessorMethod;
|
|
116
|
-
//# sourceMappingURL=QueuePortionProcessorMethod.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"QueuePortionProcessorMethod.js","sourceRoot":"","sources":["../src/QueuePortionProcessorMethod.ts"],"names":[],"mappings":";;;AAAA,mEAAgE;AAEhE,MAAa,2BAA4B,SAAQ,6CAAqB;IACrE,OAAO,CAAC,SAAiB;QACxB,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChE,MAAM,WAAW,GAAG,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAEtE,OAAO;YACN,aAAa;YACb,WAAW;SACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;YACvC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEX,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,SAAS,IAAI,SAAS,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;gBACpD,EAAE,EAAE,CAAC;YACN,CAAC;YACD,CAAC,EAAE,CAAC;QACL,CAAC;IACF,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS;eACV,CAAC,SAAS,CAAC,QAAQ;eACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;eACpB,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;eACnB,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACxB,OAAO;QACR,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO;QACR,CAAC;QACD,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,EACvB,KAAK,GAAG,CAAC,EACT,CAAC,GAAG,CAAC,CAAC;YAEP,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAErB,MAAM;qBACJ,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC;qBAC9C,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBAChB,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC;yBAC5C,IAAI,CAAC,GAAG,EAAE;wBACV,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;wBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;oBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;wBAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;oBAChB,CAAC;oBACD,KAAK,IAAI,CAAC,CAAC;gBACZ,CAAC,CAAC,CAAC;gBACJ,CAAC,EAAE,CAAC;YACL,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,OAAO;QACR,CAAC;QACD,MAAM,aAAa,GAAG,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAEnD,IAAI,WAAW,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACL,MAAM,eAAe,GAAG,WAAW,GAAG,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YAExD,IAAI,UAAU,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,aAAa,IAAI,eAAe,EAAE,EAAE,CAAC,CAAC,CAAC;gBACpF,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;YACzD,CAAC;QACF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;IACtE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS,EAAE,GAAG;QACzE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,OAAO;QACR,CAAC;QACD,MAAM,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC9D,CAAC;CACD;AAlID,kEAkIC"}
|
package/dist/QueueProcessor.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { Queue } from './Queue';
|
|
2
|
-
import { Processor } from './Processor';
|
|
3
|
-
export declare class QueueProcessor extends Queue {
|
|
4
|
-
private processors;
|
|
5
|
-
keyData(queueName: string): {
|
|
6
|
-
processorName: string;
|
|
7
|
-
};
|
|
8
|
-
setProcessors(processors: Array<Processor>): QueueProcessor;
|
|
9
|
-
getProcessors(): Array<Processor>;
|
|
10
|
-
getProcessorByName(name: string): Processor | null;
|
|
11
|
-
start(queueName: string): void;
|
|
12
|
-
excecuteWrapper(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
13
|
-
listen(queueName: string): void;
|
|
14
|
-
}
|
package/dist/QueueProcessor.js
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.QueueProcessor = void 0;
|
|
4
|
-
const Queue_1 = require("./Queue");
|
|
5
|
-
class QueueProcessor extends Queue_1.Queue {
|
|
6
|
-
constructor() {
|
|
7
|
-
super(...arguments);
|
|
8
|
-
this.processors = [];
|
|
9
|
-
}
|
|
10
|
-
keyData(queueName) {
|
|
11
|
-
const queueNameSplit = queueName.split(`.`);
|
|
12
|
-
const processorName = queueNameSplit[queueNameSplit.length - 1];
|
|
13
|
-
return {
|
|
14
|
-
processorName,
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
setProcessors(processors) {
|
|
18
|
-
this.processors = [...processors];
|
|
19
|
-
return this;
|
|
20
|
-
}
|
|
21
|
-
getProcessors() {
|
|
22
|
-
return [...this.processors];
|
|
23
|
-
}
|
|
24
|
-
getProcessorByName(name) {
|
|
25
|
-
return this.getProcessors().find((processor) => processor.name === name) ?? null;
|
|
26
|
-
}
|
|
27
|
-
start(queueName) {
|
|
28
|
-
const processors = this.getProcessors();
|
|
29
|
-
let i = 0;
|
|
30
|
-
while (i < processors.length) {
|
|
31
|
-
const processor = processors[i];
|
|
32
|
-
const queueKey = `${queueName}.${processor.name}`;
|
|
33
|
-
this.listen(queueKey);
|
|
34
|
-
i++;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
38
|
-
const keyData = this.keyData(queueName);
|
|
39
|
-
const processorName = keyData.processorName;
|
|
40
|
-
const processor = this.getProcessorByName(processorName);
|
|
41
|
-
if (!processor || !processor.excecute) {
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
await this.excecute(queueName, attemptIndex, await processor.excecute.call(processor, attemptIndex, data));
|
|
45
|
-
}
|
|
46
|
-
listen(queueName) {
|
|
47
|
-
const processors = this.getProcessors();
|
|
48
|
-
let i = 0;
|
|
49
|
-
while (i < processors.length) {
|
|
50
|
-
const processor = processors[i];
|
|
51
|
-
let ii = 0;
|
|
52
|
-
while (ii < this.attempts) {
|
|
53
|
-
this.attempt(queueName, ii);
|
|
54
|
-
ii++;
|
|
55
|
-
}
|
|
56
|
-
i++;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
exports.QueueProcessor = QueueProcessor;
|
|
61
|
-
//# sourceMappingURL=QueueProcessor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"QueueProcessor.js","sourceRoot":"","sources":["../src/QueueProcessor.ts"],"names":[],"mappings":";;;AAAA,mCAAiC;AAGjC,MAAa,cAAe,SAAQ,aAAK;IAAzC;;QACS,eAAU,GAAqB,EAAE,CAAC;IA+D3C,CAAC;IA7DA,OAAO,CAAC,SAAiB;QACxB,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEhE,OAAO;YACN,aAAa;SACb,CAAC;IACH,CAAC;IAED,aAAa,CAAC,UAA4B;QACzC,IAAI,CAAC,UAAU,GAAG,CAAE,GAAG,UAAU,CAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,aAAa;QACZ,OAAO,CAAE,GAAG,IAAI,CAAC,UAAU,CAAE,CAAC;IAC/B,CAAC;IAED,kBAAkB,CAAC,IAAY;QAC9B,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,CAAC,SAAoB,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;IAC7F,CAAC;IAED,KAAK,CAAC,SAAiB;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YAElD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC,EAAE,CAAC;QACL,CAAC;IACF,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YACvC,OAAO;QACR,CAAC;QACD,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5G,CAAC;IAED,MAAM,CAAC,SAAiB;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEX,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gBAC5B,EAAE,EAAE,CAAC;YACN,CAAC;YACD,CAAC,EAAE,CAAC;QACL,CAAC;IACF,CAAC;CACD;AAhED,wCAgEC"}
|