power-queues 0.0.14 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Processor.d.ts +17 -0
- package/dist/Processor.js +63 -0
- package/dist/Processor.js.map +1 -0
- package/dist/Queue.d.ts +37 -0
- package/dist/Queue.js +181 -0
- package/dist/Queue.js.map +1 -0
- package/dist/QueuePortion.d.ts +8 -0
- package/dist/QueuePortion.js +34 -0
- package/dist/QueuePortion.js.map +1 -0
- package/dist/QueuePortionProcessor.d.ts +8 -0
- package/dist/QueuePortionProcessor.js +75 -0
- package/dist/QueuePortionProcessor.js.map +1 -0
- package/dist/QueuePortionProcessorMethod.d.ts +12 -0
- package/dist/QueuePortionProcessorMethod.js +116 -0
- package/dist/QueuePortionProcessorMethod.js.map +1 -0
- package/dist/QueueProcessor.d.ts +14 -0
- package/dist/QueueProcessor.js +61 -0
- package/dist/QueueProcessor.js.map +1 -0
- package/dist/index.d.ts +7 -7
- package/dist/index.js +13 -13
- package/dist/index.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/ProcessorService.d.ts +0 -7
- package/dist/ProcessorService.js +0 -19
- package/dist/ProcessorService.js.map +0 -1
- package/dist/QueuePortionProcessorOrderService.d.ts +0 -5
- package/dist/QueuePortionProcessorOrderService.js +0 -44
- package/dist/QueuePortionProcessorOrderService.js.map +0 -1
- package/dist/QueuePortionProcessorService.d.ts +0 -8
- package/dist/QueuePortionProcessorService.js +0 -26
- package/dist/QueuePortionProcessorService.js.map +0 -1
- package/dist/QueuePortionService.d.ts +0 -7
- package/dist/QueuePortionService.js +0 -38
- package/dist/QueuePortionService.js.map +0 -1
- package/dist/QueueProcessorService.d.ts +0 -15
- package/dist/QueueProcessorService.js +0 -66
- package/dist/QueueProcessorService.js.map +0 -1
- package/dist/QueueService.d.ts +0 -18
- package/dist/QueueService.js +0 -86
- package/dist/QueueService.js.map +0 -1
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export declare class Processor {
|
|
2
|
+
readonly defaultName: string;
|
|
3
|
+
readonly timeoutCoefficient: number;
|
|
4
|
+
readonly timeoutInitial: number;
|
|
5
|
+
get name(): string;
|
|
6
|
+
excecute(queueName: string, attemptIndex: number, data: any): Promise<any>;
|
|
7
|
+
success(data: any): Promise<any>;
|
|
8
|
+
successMethod(data: any): Promise<any>;
|
|
9
|
+
timestamp(date?: Date): string;
|
|
10
|
+
orderMethods(): Array<Function>;
|
|
11
|
+
errorMethods(): Array<Function>;
|
|
12
|
+
getMethods(): Array<Function>;
|
|
13
|
+
getMethodsLength(): number;
|
|
14
|
+
getMethod(index: number): Function;
|
|
15
|
+
isErrorMethod(method: Function): boolean;
|
|
16
|
+
attemptsTimeout(attemptIndex: number): number;
|
|
17
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Processor = void 0;
|
|
4
|
+
class Processor {
|
|
5
|
+
constructor() {
|
|
6
|
+
this.timeoutCoefficient = 3;
|
|
7
|
+
this.timeoutInitial = 3000;
|
|
8
|
+
}
|
|
9
|
+
get name() {
|
|
10
|
+
return this.defaultName ?? this.constructor.name;
|
|
11
|
+
}
|
|
12
|
+
async excecute(queueName, attemptIndex, data) {
|
|
13
|
+
return data;
|
|
14
|
+
}
|
|
15
|
+
async success(data) {
|
|
16
|
+
return data;
|
|
17
|
+
}
|
|
18
|
+
async successMethod(data) {
|
|
19
|
+
return data;
|
|
20
|
+
}
|
|
21
|
+
timestamp(date = new Date()) {
|
|
22
|
+
const pad = (n) => String(n).padStart(2, '0');
|
|
23
|
+
const year = date.getFullYear();
|
|
24
|
+
const month = pad(date.getMonth() + 1);
|
|
25
|
+
const day = pad(date.getDate());
|
|
26
|
+
const hours = pad(date.getHours());
|
|
27
|
+
const minutes = pad(date.getMinutes());
|
|
28
|
+
const seconds = pad(date.getSeconds());
|
|
29
|
+
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
|
|
30
|
+
}
|
|
31
|
+
orderMethods() {
|
|
32
|
+
return [];
|
|
33
|
+
}
|
|
34
|
+
errorMethods() {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
getMethods() {
|
|
38
|
+
return [...this.orderMethods()];
|
|
39
|
+
}
|
|
40
|
+
getMethodsLength() {
|
|
41
|
+
return this.orderMethods().length;
|
|
42
|
+
}
|
|
43
|
+
getMethod(index) {
|
|
44
|
+
return this.orderMethods()[index];
|
|
45
|
+
}
|
|
46
|
+
isErrorMethod(method) {
|
|
47
|
+
return !!this.errorMethods().find((item) => item.name === method.name);
|
|
48
|
+
}
|
|
49
|
+
attemptsTimeout(attemptIndex) {
|
|
50
|
+
const coefficient = this.timeoutCoefficient;
|
|
51
|
+
let i = 0, timeout = this.timeoutInitial;
|
|
52
|
+
while (i < attemptIndex) {
|
|
53
|
+
timeout += (timeout * coefficient);
|
|
54
|
+
i++;
|
|
55
|
+
}
|
|
56
|
+
if (timeout === 0) {
|
|
57
|
+
timeout = this.timeoutInitial;
|
|
58
|
+
}
|
|
59
|
+
return timeout;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
exports.Processor = Processor;
|
|
63
|
+
//# sourceMappingURL=Processor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Processor.js","sourceRoot":"","sources":["../src/Processor.ts"],"names":[],"mappings":";;;AACA,MAAa,SAAS;IAAtB;QAEiB,uBAAkB,GAAW,CAAC,CAAC;QAC/B,mBAAc,GAAW,IAAI,CAAC;IAsE/C,CAAC;IApEA,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QAChE,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAS;QACtB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,IAAS;QAC5B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,SAAS,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE;QAC1B,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAEtD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAEvC,OAAO,GAAG,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;IACjE,CAAC;IAED,YAAY;QACX,OAAO,EAAE,CAAC;IACX,CAAC;IAED,YAAY;QACX,OAAO,EAAE,CAAC;IACX,CAAC;IAED,UAAU;QACT,OAAO,CAAE,GAAG,IAAI,CAAC,YAAY,EAAE,CAAE,CAAC;IACnC,CAAC;IAED,gBAAgB;QACf,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC;IACnC,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,aAAa,CAAC,MAAgB;QAC7B,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC;IAED,eAAe,CAAC,YAAoB;QACnC,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,EACR,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;QAE/B,OAAO,CAAC,GAAG,YAAY,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,OAAO,GAAG,WAAW,CAAC,CAAC;YACnC,CAAC,EAAE,CAAC;QACL,CAAC;QACD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;QAC/B,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;CACD;AAzED,8BAyEC"}
|
package/dist/Queue.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import Redis from 'ioredis';
|
|
2
|
+
export declare class Queue {
|
|
3
|
+
protected readonly redis: Redis | null;
|
|
4
|
+
protected readonly threadId: string;
|
|
5
|
+
protected readonly timeout: number;
|
|
6
|
+
protected readonly attempts: number;
|
|
7
|
+
protected readonly displayLog: boolean;
|
|
8
|
+
protected readonly displayError: boolean;
|
|
9
|
+
protected readonly displayErrorObj: boolean;
|
|
10
|
+
protected readonly displayErrorData: boolean;
|
|
11
|
+
timestamp(date?: Date): string;
|
|
12
|
+
queueKey(queueName: string, attemptIndex: number): string;
|
|
13
|
+
readyKey(queueName: string, attemptIndex: number): string;
|
|
14
|
+
start(queueName: string): void;
|
|
15
|
+
listen(queueName: string): void;
|
|
16
|
+
wait(timeout?: number): Promise<void>;
|
|
17
|
+
attempt(queueName: string, attemptIndex: number): Promise<void>;
|
|
18
|
+
process(queueName: string, attemptIndex: number): Promise<void>;
|
|
19
|
+
processOne(queueName: string, attemptIndex: number): Promise<void>;
|
|
20
|
+
retry(queueName: string, attemptIndex: number, data: any, err: any): Promise<number>;
|
|
21
|
+
select(queueName: string, attemptIndex: number): Promise<any>;
|
|
22
|
+
selectAfter(queueName: string, data: any): Promise<any>;
|
|
23
|
+
allow(queueName: string, attemptIndex: number, data: any): Promise<boolean>;
|
|
24
|
+
excecuteWrapper(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
25
|
+
excecute(queueName: string, attemptIndex: number, data: any): Promise<any>;
|
|
26
|
+
successWrapper(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
27
|
+
success(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
28
|
+
errorWrapper(queueName: string, attemptIndex: number, data: any, err: any): Promise<void>;
|
|
29
|
+
errorMessage(queueName: string, attemptIndex: number, data: any, err: any): Promise<void>;
|
|
30
|
+
error(queueName: string, attemptIndex: number, data: any, err: any): Promise<void>;
|
|
31
|
+
dropKeys(pattern: string, opts?: {
|
|
32
|
+
count?: number;
|
|
33
|
+
batch?: number;
|
|
34
|
+
pauseMs?: number;
|
|
35
|
+
useUnlink?: boolean;
|
|
36
|
+
}): Promise<number>;
|
|
37
|
+
}
|
package/dist/Queue.js
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Queue = void 0;
|
|
4
|
+
const uuid_1 = require("uuid");
|
|
5
|
+
class Queue {
|
|
6
|
+
constructor() {
|
|
7
|
+
this.threadId = (0, uuid_1.v4)();
|
|
8
|
+
this.timeout = 10;
|
|
9
|
+
this.attempts = 1;
|
|
10
|
+
this.displayLog = true;
|
|
11
|
+
this.displayError = true;
|
|
12
|
+
this.displayErrorObj = false;
|
|
13
|
+
this.displayErrorData = false;
|
|
14
|
+
}
|
|
15
|
+
timestamp(date = new Date()) {
|
|
16
|
+
const pad = (n) => String(n).padStart(2, '0');
|
|
17
|
+
const year = date.getFullYear();
|
|
18
|
+
const month = pad(date.getMonth() + 1);
|
|
19
|
+
const day = pad(date.getDate());
|
|
20
|
+
const hours = pad(date.getHours());
|
|
21
|
+
const minutes = pad(date.getMinutes());
|
|
22
|
+
const seconds = pad(date.getSeconds());
|
|
23
|
+
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
|
|
24
|
+
}
|
|
25
|
+
queueKey(queueName, attemptIndex) {
|
|
26
|
+
return `queue.${queueName}.${attemptIndex}`;
|
|
27
|
+
}
|
|
28
|
+
readyKey(queueName, attemptIndex) {
|
|
29
|
+
return `ready.${queueName}.${attemptIndex}`;
|
|
30
|
+
}
|
|
31
|
+
start(queueName) {
|
|
32
|
+
this.listen(queueName);
|
|
33
|
+
}
|
|
34
|
+
listen(queueName) {
|
|
35
|
+
let i = 0;
|
|
36
|
+
while (i < this.attempts) {
|
|
37
|
+
this.attempt(queueName, i);
|
|
38
|
+
i++;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
async wait(timeout) {
|
|
42
|
+
await new Promise((resolve) => setTimeout(resolve, timeout ?? this.timeout));
|
|
43
|
+
}
|
|
44
|
+
async attempt(queueName, attemptIndex) {
|
|
45
|
+
const readyKey = this.readyKey(queueName, attemptIndex);
|
|
46
|
+
const threadId = this.threadId;
|
|
47
|
+
await this.redis.rpush(readyKey, this.threadId);
|
|
48
|
+
await this.wait();
|
|
49
|
+
setImmediate(() => this.process(queueName, attemptIndex));
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
async process(queueName, attemptIndex) {
|
|
53
|
+
try {
|
|
54
|
+
await this.processOne(queueName, attemptIndex);
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
await this.errorWrapper(queueName, attemptIndex, null, err);
|
|
58
|
+
}
|
|
59
|
+
await this.wait();
|
|
60
|
+
setImmediate(() => this.process(queueName, attemptIndex));
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
async processOne(queueName, attemptIndex) {
|
|
64
|
+
const readyKey = this.readyKey(queueName, attemptIndex);
|
|
65
|
+
const readyThreadId = await this.redis.lpop(readyKey);
|
|
66
|
+
if (readyThreadId) {
|
|
67
|
+
if (readyThreadId === this.threadId) {
|
|
68
|
+
const data = await this.select(queueName, attemptIndex);
|
|
69
|
+
const allow = await this.allow(queueName, attemptIndex, data);
|
|
70
|
+
if (allow) {
|
|
71
|
+
try {
|
|
72
|
+
await this.excecuteWrapper(queueName, attemptIndex, data);
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
this.retry(queueName, attemptIndex, data, err);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
await this.redis.rpush(readyKey, this.threadId);
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
console.log('eeeeeeeeeeee', err);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
console.log('????????????????????/', readyKey, readyThreadId, Date.now());
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async retry(queueName, attemptIndex, data, err) {
|
|
91
|
+
try {
|
|
92
|
+
if (attemptIndex <= (this.attempts - 1)) {
|
|
93
|
+
const queueKey = this.queueKey(queueName, attemptIndex + 1);
|
|
94
|
+
const dataProcessed = JSON.stringify(data);
|
|
95
|
+
if (attemptIndex < (this.attempts - 1)) {
|
|
96
|
+
return await this.redis.rpush(queueKey, dataProcessed);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
await this.errorWrapper(queueName, attemptIndex, data, err);
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
}
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
async select(queueName, attemptIndex) {
|
|
106
|
+
const queueKey = this.queueKey(queueName, attemptIndex);
|
|
107
|
+
const data = await this.redis.lpop(queueKey);
|
|
108
|
+
const output = await this.selectAfter(queueName, data);
|
|
109
|
+
return output;
|
|
110
|
+
}
|
|
111
|
+
async selectAfter(queueName, data) {
|
|
112
|
+
try {
|
|
113
|
+
const parsed = JSON.parse(data);
|
|
114
|
+
return parsed;
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
}
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
async allow(queueName, attemptIndex, data) {
|
|
121
|
+
return !!data;
|
|
122
|
+
}
|
|
123
|
+
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
124
|
+
await this.excecute(queueName, attemptIndex, data);
|
|
125
|
+
}
|
|
126
|
+
async excecute(queueName, attemptIndex, data) {
|
|
127
|
+
return await this.successWrapper(queueName, attemptIndex, data);
|
|
128
|
+
}
|
|
129
|
+
async successWrapper(queueName, attemptIndex, data) {
|
|
130
|
+
return await this.success(queueName, attemptIndex, data);
|
|
131
|
+
}
|
|
132
|
+
async success(queueName, attemptIndex, data) {
|
|
133
|
+
return data;
|
|
134
|
+
}
|
|
135
|
+
async errorWrapper(queueName, attemptIndex, data, err) {
|
|
136
|
+
try {
|
|
137
|
+
await this.error(queueName, attemptIndex, data, err);
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
await this.errorMessage(queueName, attemptIndex, data, err);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
async errorMessage(queueName, attemptIndex, data, err) {
|
|
144
|
+
if (this.displayError) {
|
|
145
|
+
console.log(`\n-------------------------------------`);
|
|
146
|
+
console.error(`[ERR]`, this.timestamp());
|
|
147
|
+
console.error(` `, `Очередь:`, queueName);
|
|
148
|
+
console.error(` `, `Попытка:`, attemptIndex);
|
|
149
|
+
console.error(` `, `Результат:`, typeof data);
|
|
150
|
+
console.error(` `, `Сообщение:`, err.message);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async error(queueName, attemptIndex, data, err) {
|
|
154
|
+
}
|
|
155
|
+
async dropKeys(pattern, opts) {
|
|
156
|
+
const count = opts?.count ?? 1000;
|
|
157
|
+
const batch = opts?.batch ?? 5000;
|
|
158
|
+
const pauseMs = opts?.pauseMs ?? 0;
|
|
159
|
+
const cmd = opts?.useUnlink ?? true ? `unlink` : `del`;
|
|
160
|
+
const stream = this.redis.scanStream({ match: pattern, count });
|
|
161
|
+
let buffer = [], deleted = 0;
|
|
162
|
+
for await (const keys of stream) {
|
|
163
|
+
buffer.push(...keys);
|
|
164
|
+
while (buffer.length >= batch) {
|
|
165
|
+
const chunk = buffer.splice(0, batch);
|
|
166
|
+
const n = await this.redis[cmd](...chunk);
|
|
167
|
+
deleted += Number(n) || 0;
|
|
168
|
+
if (pauseMs) {
|
|
169
|
+
await new Promise((r) => setTimeout(r, pauseMs));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (buffer.length) {
|
|
174
|
+
const n = await this.redis[cmd](...buffer);
|
|
175
|
+
deleted += Number(n) || 0;
|
|
176
|
+
}
|
|
177
|
+
return deleted;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
exports.Queue = Queue;
|
|
181
|
+
//# sourceMappingURL=Queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Queue.js","sourceRoot":"","sources":["../src/Queue.ts"],"names":[],"mappings":";;;AACA,+BAAoC;AAEpC,MAAa,KAAK;IAAlB;QAEoB,aAAQ,GAAW,IAAA,SAAM,GAAE,CAAC;QAC5B,YAAO,GAAW,EAAE,CAAC;QACrB,aAAQ,GAAW,CAAC,CAAC;QACrB,eAAU,GAAY,IAAI,CAAC;QAC3B,iBAAY,GAAY,IAAI,CAAC;QAC7B,oBAAe,GAAY,KAAK,CAAC;QACjC,qBAAgB,GAAY,KAAK,CAAC;IAmNtD,CAAC;IAjNA,SAAS,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE;QAC1B,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAEtD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAEvC,OAAO,GAAG,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;IACjE,CAAC;IAED,QAAQ,CAAC,SAAiB,EAAE,YAAoB;QAC/C,OAAO,SAAS,SAAS,IAAI,YAAY,EAAE,CAAC;IAC7C,CAAC;IAED,QAAQ,CAAC,SAAiB,EAAE,YAAoB;QAC/C,OAAO,SAAS,SAAS,IAAI,YAAY,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,SAAiB;QACtB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,SAAiB;QACvB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC,EAAE,CAAC;QACL,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,OAAgB;QAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,SAAiB,EAAE,YAAoB;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE/B,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;QAC1D,OAAO;IACR,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,SAAiB,EAAE,YAAoB;QACpD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;QAC1D,OAAO;IACR,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB,EAAE,YAAoB;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEtD,IAAI,aAAa,EAAE,CAAC;YACnB,IAAI,aAAa,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACrC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;gBACxD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;gBAE9D,IAAI,KAAK,EAAE,CAAC;oBACX,IAAI,CAAC;wBACJ,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;oBAC3D,CAAC;oBACD,OAAO,GAAG,EAAE,CAAC;wBACZ,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;oBAChD,CAAC;gBACF,CAAC;YACF,CAAC;YACD,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC;QACF,CAAC;aACI,CAAC;YACL,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS,EAAE,GAAG;QAClE,IAAI,CAAC;YACJ,IAAI,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;gBAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAE3C,IAAI,YAAY,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;oBACxC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBACxD,CAAC;YACF,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;QACb,CAAC;QACD,OAAO,CAAC,CAAC;IACV,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,YAAoB;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAEvD,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,IAAS;QAC7C,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,OAAO,MAAM,CAAC;QACf,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QAC7D,OAAO,CAAC,CAAC,IAAI,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QACvE,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QAChE,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QACtE,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QAC/D,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS,EAAE,GAAG;QACzE,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7D,CAAC;IACF,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS,EAAE,GAAG;QACzE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YACzC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAC9C,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;YACjD,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,IAAI,CAAC,CAAC;YAClD,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAQnD,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS,EAAE,GAAG;IACnE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,IAAiF;QAChH,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC;QAClC,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,IAAI,MAAM,GAAa,EAAE,EACxB,OAAO,GAAG,CAAC,CAAC;QAEb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAiC,EAAE,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YAErB,OAAO,MAAM,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACtC,MAAM,CAAC,GAAG,MAAO,IAAI,CAAC,KAAa,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;gBAEnD,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAE1B,IAAI,OAAO,EAAE,CAAC;oBACb,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBAClD,CAAC;YACF,CAAC;QACF,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,MAAO,IAAI,CAAC,KAAa,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;YAEpD,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;CACD;AA3ND,sBA2NC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Queue } from './Queue';
|
|
2
|
+
export declare class QueuePortion extends Queue {
|
|
3
|
+
protected readonly portionSize: number;
|
|
4
|
+
allow(queueName: string, attemptIndex: number, data: Array<any>): Promise<boolean>;
|
|
5
|
+
select(queueName: string, attemptIndex: number): Promise<Array<any>>;
|
|
6
|
+
selectAfter(queueName: string, data?: Array<any>): Promise<Array<any>>;
|
|
7
|
+
excecuteWrapper(queueName: string, attemptIndex: number, data: Array<any>): Promise<void>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QueuePortion = void 0;
|
|
4
|
+
const Queue_1 = require("./Queue");
|
|
5
|
+
class QueuePortion extends Queue_1.Queue {
|
|
6
|
+
constructor() {
|
|
7
|
+
super(...arguments);
|
|
8
|
+
this.portionSize = 1;
|
|
9
|
+
}
|
|
10
|
+
async allow(queueName, attemptIndex, data) {
|
|
11
|
+
return data.length > 0 && data.filter((item) => !!item).length === data.length;
|
|
12
|
+
}
|
|
13
|
+
async select(queueName, attemptIndex) {
|
|
14
|
+
const queueKey = this.queueKey(queueName, attemptIndex);
|
|
15
|
+
const data = await this.redis.lpop(queueKey, this.portionSize);
|
|
16
|
+
return await this.selectAfter(queueName, data);
|
|
17
|
+
}
|
|
18
|
+
async selectAfter(queueName, data = []) {
|
|
19
|
+
let i = 0, output = [];
|
|
20
|
+
while (i < data.length) {
|
|
21
|
+
output.push(await super.selectAfter(queueName, data));
|
|
22
|
+
}
|
|
23
|
+
return output;
|
|
24
|
+
}
|
|
25
|
+
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
26
|
+
let i = 0;
|
|
27
|
+
while (i < data.length) {
|
|
28
|
+
await this.excecute(queueName, attemptIndex, data[i]);
|
|
29
|
+
i++;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
exports.QueuePortion = QueuePortion;
|
|
34
|
+
//# sourceMappingURL=QueuePortion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"QueuePortion.js","sourceRoot":"","sources":["../src/QueuePortion.ts"],"names":[],"mappings":";;;AAAA,mCAAiC;AAEjC,MAAa,YAAa,SAAQ,aAAK;IAAvC;;QACoB,gBAAW,GAAW,CAAC,CAAC;IA+B5C,CAAC;IA7BA,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QACpE,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,YAAoB;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE/D,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,OAAmB,EAAE;QACzD,IAAI,CAAC,GAAG,CAAC,EACR,MAAM,GAAG,EAAE,CAAC;QAEb,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QAC9E,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC,EAAE,CAAC;QACL,CAAC;IACF,CAAC;CACD;AAhCD,oCAgCC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { QueueProcessor } from './QueueProcessor';
|
|
2
|
+
export declare class QueuePortionProcessor extends QueueProcessor {
|
|
3
|
+
protected readonly portionSize: number;
|
|
4
|
+
allow(queueName: string, attemptIndex: number, data: Array<any>): Promise<boolean>;
|
|
5
|
+
select(queueName: string, attemptIndex: number): Promise<Array<any>>;
|
|
6
|
+
selectAfter(queueName: string, data: Array<any>): Promise<Array<any>>;
|
|
7
|
+
excecuteWrapper(queueName: string, attemptIndex: number, data: Array<any>): Promise<void>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QueuePortionProcessor = void 0;
|
|
4
|
+
const QueueProcessor_1 = require("./QueueProcessor");
|
|
5
|
+
class QueuePortionProcessor extends QueueProcessor_1.QueueProcessor {
|
|
6
|
+
constructor() {
|
|
7
|
+
super(...arguments);
|
|
8
|
+
this.portionSize = 1;
|
|
9
|
+
}
|
|
10
|
+
async allow(queueName, attemptIndex, data) {
|
|
11
|
+
return data.length > 0 && data.filter((item) => !!item).length === data.length;
|
|
12
|
+
}
|
|
13
|
+
async select(queueName, attemptIndex) {
|
|
14
|
+
const queueKey = this.queueKey(queueName, attemptIndex);
|
|
15
|
+
const data = await this.redis.lpop(queueKey, this.portionSize);
|
|
16
|
+
const output = await this.selectAfter(queueName, data);
|
|
17
|
+
return output;
|
|
18
|
+
}
|
|
19
|
+
async selectAfter(queueName, data) {
|
|
20
|
+
let i = 0, output = [];
|
|
21
|
+
if (Array.isArray(data) && data.length > 0) {
|
|
22
|
+
while (i < data.length) {
|
|
23
|
+
output.push(await super.selectAfter(queueName, data[i]));
|
|
24
|
+
i++;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return output;
|
|
28
|
+
}
|
|
29
|
+
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
30
|
+
const keyData = this.keyData(queueName);
|
|
31
|
+
const processorName = keyData.processorName;
|
|
32
|
+
const processor = this.getProcessorByName(processorName);
|
|
33
|
+
if (!processor
|
|
34
|
+
|| !processor.excecute
|
|
35
|
+
|| !Array.isArray(data)
|
|
36
|
+
|| !(data.length > 0)) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
await (new Promise((resolve, reject) => {
|
|
40
|
+
let length = data.length, ready = 0, i = 0;
|
|
41
|
+
while (i < length) {
|
|
42
|
+
const item = data[i];
|
|
43
|
+
processor
|
|
44
|
+
.excecute
|
|
45
|
+
.call(processor, queueName, attemptIndex, item)
|
|
46
|
+
.then((result) => {
|
|
47
|
+
this.excecute(queueName, attemptIndex, result)
|
|
48
|
+
.then(() => {
|
|
49
|
+
if (ready >= (length - 1)) {
|
|
50
|
+
resolve(ready);
|
|
51
|
+
}
|
|
52
|
+
ready += 1;
|
|
53
|
+
})
|
|
54
|
+
.catch((err) => {
|
|
55
|
+
this.retry(queueName, attemptIndex, item, err);
|
|
56
|
+
if (ready >= (length - 1)) {
|
|
57
|
+
resolve(ready);
|
|
58
|
+
}
|
|
59
|
+
ready += 1;
|
|
60
|
+
});
|
|
61
|
+
})
|
|
62
|
+
.catch((err) => {
|
|
63
|
+
this.retry(queueName, attemptIndex, item, err);
|
|
64
|
+
if (ready >= (length - 1)) {
|
|
65
|
+
resolve(ready);
|
|
66
|
+
}
|
|
67
|
+
ready += 1;
|
|
68
|
+
});
|
|
69
|
+
i++;
|
|
70
|
+
}
|
|
71
|
+
}));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.QueuePortionProcessor = QueuePortionProcessor;
|
|
75
|
+
//# sourceMappingURL=QueuePortionProcessor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"QueuePortionProcessor.js","sourceRoot":"","sources":["../src/QueuePortionProcessor.ts"],"names":[],"mappings":";;;AAAA,qDAAmD;AAGnD,MAAa,qBAAsB,SAAQ,+BAAc;IAAzD;;QACoB,gBAAW,GAAW,CAAC,CAAC;IA8E5C,CAAC;IA5EA,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QACpE,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,YAAoB;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAEvD,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,IAAgB;QACpD,IAAI,CAAC,GAAG,CAAC,EACR,MAAM,GAAG,EAAE,CAAC;QAEb,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzD,CAAC,EAAE,CAAC;YACL,CAAC;QACF,CAAC;QACD,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS;eACV,CAAC,SAAS,CAAC,QAAQ;eACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;eACpB,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACxB,OAAO;QACR,CAAC;QACD,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,EACvB,KAAK,GAAG,CAAC,EACT,CAAC,GAAG,CAAC,CAAC;YAEP,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAErB,SAAS;qBACP,QAAQ;qBACR,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC;qBAC9C,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBAChB,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC;yBAC5C,IAAI,CAAC,GAAG,EAAE;wBACV,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;wBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;oBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;wBAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;oBAChB,CAAC;oBACD,KAAK,IAAI,CAAC,CAAC;gBACZ,CAAC,CAAC,CAAC;gBACJ,CAAC,EAAE,CAAC;YACL,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;CACD;AA/ED,sDA+EC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { QueuePortionProcessor } from './QueuePortionProcessor';
|
|
2
|
+
export declare class QueuePortionProcessorMethod extends QueuePortionProcessor {
|
|
3
|
+
keyData(queueName: string): {
|
|
4
|
+
processorName: string;
|
|
5
|
+
methodIndex: number;
|
|
6
|
+
};
|
|
7
|
+
start(queueName: string): void;
|
|
8
|
+
excecuteWrapper(queueName: string, attemptIndex: number, data: Array<any>): Promise<void>;
|
|
9
|
+
successWrapper(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
10
|
+
successMethod(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
11
|
+
errorWrapper(queueName: string, attemptIndex: number, data: any, err: any): Promise<void>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QueuePortionProcessorMethod = void 0;
|
|
4
|
+
const QueuePortionProcessor_1 = require("./QueuePortionProcessor");
|
|
5
|
+
class QueuePortionProcessorMethod extends QueuePortionProcessor_1.QueuePortionProcessor {
|
|
6
|
+
keyData(queueName) {
|
|
7
|
+
const queueNameSplit = queueName.split(`.`);
|
|
8
|
+
const processorName = queueNameSplit[queueNameSplit.length - 2];
|
|
9
|
+
const methodIndex = Number(queueNameSplit[queueNameSplit.length - 1]);
|
|
10
|
+
return {
|
|
11
|
+
processorName,
|
|
12
|
+
methodIndex,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
start(queueName) {
|
|
16
|
+
const processors = this.getProcessors();
|
|
17
|
+
let i = 0;
|
|
18
|
+
while (i < processors.length) {
|
|
19
|
+
const processor = processors[i];
|
|
20
|
+
const methods = processor.getMethods();
|
|
21
|
+
let ii = 0;
|
|
22
|
+
while (ii < methods.length) {
|
|
23
|
+
this.listen(`${queueName}.${processor.name}.${ii}`);
|
|
24
|
+
ii++;
|
|
25
|
+
}
|
|
26
|
+
i++;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
async excecuteWrapper(queueName, attemptIndex, data) {
|
|
30
|
+
const keyData = this.keyData(queueName);
|
|
31
|
+
const processorName = keyData.processorName;
|
|
32
|
+
const methodIndex = keyData.methodIndex;
|
|
33
|
+
const processor = this.getProcessorByName(processorName);
|
|
34
|
+
if (!processor
|
|
35
|
+
|| !processor.excecute
|
|
36
|
+
|| !Array.isArray(data)
|
|
37
|
+
|| !(methodIndex >= 0)
|
|
38
|
+
|| !(data.length > 0)) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const method = processor.getMethod(methodIndex);
|
|
42
|
+
let i = 0;
|
|
43
|
+
if (!method) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
await (new Promise((resolve, reject) => {
|
|
47
|
+
let length = data.length, ready = 0, i = 0;
|
|
48
|
+
while (i < length) {
|
|
49
|
+
const item = data[i];
|
|
50
|
+
method
|
|
51
|
+
.call(processor, queueName, attemptIndex, item)
|
|
52
|
+
.then((result) => {
|
|
53
|
+
this.excecute(queueName, attemptIndex, result)
|
|
54
|
+
.then(() => {
|
|
55
|
+
if (ready >= (length - 1)) {
|
|
56
|
+
resolve(ready);
|
|
57
|
+
}
|
|
58
|
+
ready += 1;
|
|
59
|
+
})
|
|
60
|
+
.catch((err) => {
|
|
61
|
+
this.retry(queueName, attemptIndex, item, err);
|
|
62
|
+
if (ready >= (length - 1)) {
|
|
63
|
+
resolve(ready);
|
|
64
|
+
}
|
|
65
|
+
ready += 1;
|
|
66
|
+
});
|
|
67
|
+
})
|
|
68
|
+
.catch((err) => {
|
|
69
|
+
this.retry(queueName, attemptIndex, item, err);
|
|
70
|
+
if (ready >= (length - 1)) {
|
|
71
|
+
resolve(ready);
|
|
72
|
+
}
|
|
73
|
+
ready += 1;
|
|
74
|
+
});
|
|
75
|
+
i++;
|
|
76
|
+
}
|
|
77
|
+
}));
|
|
78
|
+
}
|
|
79
|
+
async successWrapper(queueName, attemptIndex, data) {
|
|
80
|
+
const keyData = this.keyData(queueName);
|
|
81
|
+
const processorName = keyData.processorName;
|
|
82
|
+
const methodIndex = keyData.methodIndex;
|
|
83
|
+
const processor = this.getProcessorByName(processorName);
|
|
84
|
+
if (!processor) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
const methodsLength = processor.getMethodsLength();
|
|
88
|
+
if (methodIndex === (methodsLength - 1)) {
|
|
89
|
+
await super.successWrapper(queueName, attemptIndex, data);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
const nextMethodIndex = methodIndex + 1;
|
|
93
|
+
const nextMethod = processor.getMethod(nextMethodIndex);
|
|
94
|
+
if (nextMethod) {
|
|
95
|
+
const queueKey = await this.queueKey(`poll.${processorName}.${nextMethodIndex}`, 0);
|
|
96
|
+
const dataProcessed = JSON.stringify(data);
|
|
97
|
+
await this.redis.rpush(queueKey, dataProcessed);
|
|
98
|
+
await this.successMethod(queueName, attemptIndex, data);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return data;
|
|
102
|
+
}
|
|
103
|
+
async successMethod(queueName, attemptIndex, data) {
|
|
104
|
+
}
|
|
105
|
+
async errorWrapper(queueName, attemptIndex, data, err) {
|
|
106
|
+
const keyData = this.keyData(queueName);
|
|
107
|
+
const processorName = keyData.processorName;
|
|
108
|
+
const processor = this.getProcessorByName(processorName);
|
|
109
|
+
if (!processor) {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
await super.errorWrapper(queueName, attemptIndex, data, err);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
exports.QueuePortionProcessorMethod = QueuePortionProcessorMethod;
|
|
116
|
+
//# sourceMappingURL=QueuePortionProcessorMethod.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"QueuePortionProcessorMethod.js","sourceRoot":"","sources":["../src/QueuePortionProcessorMethod.ts"],"names":[],"mappings":";;;AAAA,mEAAgE;AAEhE,MAAa,2BAA4B,SAAQ,6CAAqB;IACrE,OAAO,CAAC,SAAiB;QACxB,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChE,MAAM,WAAW,GAAG,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAEtE,OAAO;YACN,aAAa;YACb,WAAW;SACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;YACvC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEX,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,SAAS,IAAI,SAAS,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;gBACpD,EAAE,EAAE,CAAC;YACN,CAAC;YACD,CAAC,EAAE,CAAC;QACL,CAAC;IACF,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAgB;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS;eACV,CAAC,SAAS,CAAC,QAAQ;eACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;eACpB,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;eACnB,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACxB,OAAO;QACR,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO;QACR,CAAC;QACD,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtC,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,EACvB,KAAK,GAAG,CAAC,EACT,CAAC,GAAG,CAAC,CAAC;YAEP,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAErB,MAAM;qBACJ,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC;qBAC9C,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBAChB,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC;yBAC5C,IAAI,CAAC,GAAG,EAAE;wBACV,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;wBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChB,CAAC;wBACD,KAAK,IAAI,CAAC,CAAC;oBACZ,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACd,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;oBAE/C,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;wBAC3B,OAAO,CAAC,KAAK,CAAC,CAAC;oBAChB,CAAC;oBACD,KAAK,IAAI,CAAC,CAAC;gBACZ,CAAC,CAAC,CAAC;gBACJ,CAAC,EAAE,CAAC;YACL,CAAC;QACF,CAAC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;QACtE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,OAAO;QACR,CAAC;QACD,MAAM,aAAa,GAAG,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAEnD,IAAI,WAAW,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC3D,CAAC;aACI,CAAC;YACL,MAAM,eAAe,GAAG,WAAW,GAAG,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YAExD,IAAI,UAAU,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,aAAa,IAAI,eAAe,EAAE,EAAE,CAAC,CAAC,CAAC;gBACpF,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAChD,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;YACzD,CAAC;QACF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS;IACtE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,YAAoB,EAAE,IAAS,EAAE,GAAG;QACzE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAEzD,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,OAAO;QACR,CAAC;QACD,MAAM,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC9D,CAAC;CACD;AAlID,kEAkIC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Queue } from './Queue';
|
|
2
|
+
import { Processor } from './Processor';
|
|
3
|
+
export declare class QueueProcessor extends Queue {
|
|
4
|
+
private processors;
|
|
5
|
+
keyData(queueName: string): {
|
|
6
|
+
processorName: string;
|
|
7
|
+
};
|
|
8
|
+
setProcessors(processors: Array<Processor>): QueueProcessor;
|
|
9
|
+
getProcessors(): Array<Processor>;
|
|
10
|
+
getProcessorByName(name: string): Processor | null;
|
|
11
|
+
start(queueName: string): void;
|
|
12
|
+
excecuteWrapper(queueName: string, attemptIndex: number, data: any): Promise<void>;
|
|
13
|
+
listen(queueName: string): void;
|
|
14
|
+
}
|