timequeue 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -12
- package/dist/index.d.ts +82 -0
- package/dist/index.js +190 -0
- package/dist/index.js.map +1 -0
- package/dist/mem-store.d.ts +13 -0
- package/dist/mem-store.js +49 -0
- package/dist/mem-store.js.map +1 -0
- package/package.json +17 -12
- package/lib/index.js +0 -207
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# timequeue.js
|
|
2
2
|
|
|
3
|
-
A queue with custom concurrency and time limits. Inspired by [
|
|
3
|
+
A queue with custom concurrency and time limits. Inspired by [async/queue](https://caolan.github.io/async/v3/docs.html#queue), but also with variable number of arguments in the worker, events, and with optional time limits.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
|
|
6
6
|
[](https://codecov.io/gh/fent/timequeue.js)
|
|
7
7
|
|
|
8
8
|
# Usage
|
|
@@ -16,8 +16,8 @@ const worker = (arg1, arg2, callback) => {
|
|
|
16
16
|
|
|
17
17
|
// Worker can be an async function.
|
|
18
18
|
const worker = async (arg1) => {
|
|
19
|
-
await anotherSyncFunction(arg1);
|
|
20
|
-
return
|
|
19
|
+
let result1 = await anotherSyncFunction(arg1);
|
|
20
|
+
return andAnotherOne(result1);
|
|
21
21
|
};
|
|
22
22
|
|
|
23
23
|
// create a queue with max 5 concurrency every second
|
|
@@ -60,32 +60,33 @@ Creates a new instance of a queue. Worker must be a function with a callback for
|
|
|
60
60
|
// if callback was given to that task, it will be called with the error,
|
|
61
61
|
// otherwise, the returned promise should be `caught`.
|
|
62
62
|
, timeout: 0
|
|
63
|
+
|
|
64
|
+
// You can pass a custom store to share tasks between several queues.
|
|
65
|
+
// Default is MemoryStore from `src/mem-store.ts`.
|
|
66
|
+
// Look at `example/redis-store.js` for an example that saves tasks onto redis.
|
|
67
|
+
, store: MemoryStore
|
|
63
68
|
}
|
|
64
69
|
```
|
|
65
70
|
|
|
66
71
|
All of these options can later be edited on the queue instance.
|
|
67
72
|
|
|
68
73
|
### TimeQueue#active
|
|
69
|
-
|
|
70
74
|
How many tasks are currently active.
|
|
71
75
|
|
|
72
76
|
### TimeQueue#intransit
|
|
73
|
-
|
|
74
77
|
If you use the `every` option to queue up tasks, this property will be delayed from being updated until there are free spots open for new tasks to begin. `active` will be updated as soon as a task finishes, even if the next one is just a timeout around the corner.
|
|
75
78
|
|
|
76
|
-
### TimeQueue#queued
|
|
77
|
-
|
|
78
|
-
How many tasks are currently in the queue.
|
|
79
|
-
|
|
80
79
|
### TimeQueue#finished
|
|
81
|
-
|
|
82
80
|
How many tasks have finished in total.
|
|
83
81
|
|
|
84
|
-
### TimeQueue#push(data..., [callback])
|
|
82
|
+
### async TimeQueue#push(data..., [callback])
|
|
85
83
|
Pushes a new task to the queue. Any number of arguments can be given. An optional callback can also be given as the last parameter. The callback will be called when the task is finished or if there was any error running the worker.
|
|
86
84
|
|
|
87
85
|
If the queue is full, pushed tasks will be ignored.
|
|
88
86
|
|
|
87
|
+
### async TimeQueue#store.getQueued()
|
|
88
|
+
How many tasks are currently in the queue.
|
|
89
|
+
|
|
89
90
|
### TimeQueue#isFull()
|
|
90
91
|
Returns true if queue is full.
|
|
91
92
|
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
3
|
+
declare namespace TimeQueue {
|
|
4
|
+
type Worker = (...args: any[]) => void | Promise<any>;
|
|
5
|
+
interface Options {
|
|
6
|
+
concurrency?: number;
|
|
7
|
+
every?: number;
|
|
8
|
+
maxQueued?: number;
|
|
9
|
+
timeout?: number;
|
|
10
|
+
store?: Store;
|
|
11
|
+
}
|
|
12
|
+
interface TaskError extends Error {
|
|
13
|
+
args: any[];
|
|
14
|
+
}
|
|
15
|
+
interface Store {
|
|
16
|
+
isEmpty: () => Promise<boolean>;
|
|
17
|
+
getQueued: () => Promise<number>;
|
|
18
|
+
getNextTask: () => Promise<any[]>;
|
|
19
|
+
pushTask: (...args: any[]) => void;
|
|
20
|
+
clear: () => void;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
declare class TimeQueue extends EventEmitter {
|
|
24
|
+
worker: TimeQueue.Worker;
|
|
25
|
+
concurrency: number;
|
|
26
|
+
every: number;
|
|
27
|
+
maxQueued: number;
|
|
28
|
+
timeout: number;
|
|
29
|
+
private _isWorkerAsync;
|
|
30
|
+
private _timers;
|
|
31
|
+
store: TimeQueue.Store;
|
|
32
|
+
active: number;
|
|
33
|
+
intransit: number;
|
|
34
|
+
finished: number;
|
|
35
|
+
static TaskError: {
|
|
36
|
+
new (message?: string): {
|
|
37
|
+
args: any[];
|
|
38
|
+
name: string;
|
|
39
|
+
message: string;
|
|
40
|
+
stack?: string;
|
|
41
|
+
};
|
|
42
|
+
captureStackTrace(targetObject: object, constructorOpt?: Function): void;
|
|
43
|
+
prepareStackTrace?: (err: Error, stackTraces: NodeJS.CallSite[]) => any;
|
|
44
|
+
stackTraceLimit: number;
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* @constructor
|
|
48
|
+
* @extends {EventEmitter}
|
|
49
|
+
* @param {Function(..., Function(!Error, ...)} worker
|
|
50
|
+
* @param {Object?} options
|
|
51
|
+
* @param {number?} options.concurrency
|
|
52
|
+
* @param {number?} options.every
|
|
53
|
+
* @param {number?} options.maxQueued
|
|
54
|
+
* @param {number?} options.timeout
|
|
55
|
+
*/
|
|
56
|
+
constructor(worker: TimeQueue.Worker, options?: TimeQueue.Options);
|
|
57
|
+
/**
|
|
58
|
+
* Pushes a task onto the queue.
|
|
59
|
+
*
|
|
60
|
+
* @param {Object} ...args
|
|
61
|
+
* @param {Function(!Error, ...)} callback
|
|
62
|
+
* @return {Promise?}
|
|
63
|
+
*/
|
|
64
|
+
push(...args: any[]): Promise<unknown>;
|
|
65
|
+
/**
|
|
66
|
+
* Starts a task
|
|
67
|
+
*
|
|
68
|
+
* @param {Array.<Object>} args
|
|
69
|
+
*/
|
|
70
|
+
_process(args: any[]): Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* Called when a task finishes. Looks at the queue and processes the next
|
|
73
|
+
* waiting task.
|
|
74
|
+
*/
|
|
75
|
+
_next(): Promise<void>;
|
|
76
|
+
/**
|
|
77
|
+
* Empties the queue and kills the timers.
|
|
78
|
+
* Active tasks will still be completed.
|
|
79
|
+
*/
|
|
80
|
+
die(): void;
|
|
81
|
+
}
|
|
82
|
+
export = TimeQueue;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
const events_1 = require("events");
|
|
15
|
+
const mem_store_1 = __importDefault(require("./mem-store"));
|
|
16
|
+
class TimeQueue extends events_1.EventEmitter {
|
|
17
|
+
/**
|
|
18
|
+
* @constructor
|
|
19
|
+
* @extends {EventEmitter}
|
|
20
|
+
* @param {Function(..., Function(!Error, ...)} worker
|
|
21
|
+
* @param {Object?} options
|
|
22
|
+
* @param {number?} options.concurrency
|
|
23
|
+
* @param {number?} options.every
|
|
24
|
+
* @param {number?} options.maxQueued
|
|
25
|
+
* @param {number?} options.timeout
|
|
26
|
+
*/
|
|
27
|
+
constructor(worker, options = {}) {
|
|
28
|
+
super();
|
|
29
|
+
this.worker = worker;
|
|
30
|
+
this._isWorkerAsync = worker.constructor.name == 'AsyncFunction';
|
|
31
|
+
this.concurrency = options.concurrency || 1;
|
|
32
|
+
this.every = options.every || 0;
|
|
33
|
+
this.maxQueued = options.maxQueued || Infinity;
|
|
34
|
+
this.timeout = options.timeout || 0;
|
|
35
|
+
this.store = options.store || new mem_store_1.default({ maxQueued: this.maxQueued });
|
|
36
|
+
this._timers = [];
|
|
37
|
+
this.active = 0;
|
|
38
|
+
this.intransit = 0;
|
|
39
|
+
this.finished = 0;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Pushes a task onto the queue.
|
|
43
|
+
*
|
|
44
|
+
* @param {Object} ...args
|
|
45
|
+
* @param {Function(!Error, ...)} callback
|
|
46
|
+
* @return {Promise?}
|
|
47
|
+
*/
|
|
48
|
+
push(...args) {
|
|
49
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
50
|
+
// Returns a promise when no `callback` is given.
|
|
51
|
+
if (this._isWorkerAsync && args.length === this.worker.length ||
|
|
52
|
+
!this._isWorkerAsync && args.length < this.worker.length) {
|
|
53
|
+
return new Promise((resolve, reject) => {
|
|
54
|
+
// Add any missing arguments.
|
|
55
|
+
if (!this._isWorkerAsync) {
|
|
56
|
+
while (args.length < this.worker.length - 1) {
|
|
57
|
+
args.push(undefined);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
this.push(...args, (err, results) => {
|
|
61
|
+
if (err)
|
|
62
|
+
return reject(err);
|
|
63
|
+
resolve(results);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
if (this.intransit < this.concurrency) {
|
|
68
|
+
this.intransit++;
|
|
69
|
+
this.active++;
|
|
70
|
+
if (this.intransit === this.concurrency) {
|
|
71
|
+
this.emit('full');
|
|
72
|
+
}
|
|
73
|
+
yield this._process(args);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
yield this.store.pushTask(args);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Starts a task
|
|
82
|
+
*
|
|
83
|
+
* @param {Array.<Object>} args
|
|
84
|
+
*/
|
|
85
|
+
_process(args) {
|
|
86
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
87
|
+
const callback = args.pop();
|
|
88
|
+
let finished = false;
|
|
89
|
+
let every = ~~this.every;
|
|
90
|
+
let tookLongerThanEvery;
|
|
91
|
+
if (every) {
|
|
92
|
+
tookLongerThanEvery = false;
|
|
93
|
+
this._timers.push(setTimeout(() => {
|
|
94
|
+
tookLongerThanEvery = true;
|
|
95
|
+
this._timers.shift();
|
|
96
|
+
if (finished) {
|
|
97
|
+
this._next();
|
|
98
|
+
}
|
|
99
|
+
}, every));
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
tookLongerThanEvery = true;
|
|
103
|
+
}
|
|
104
|
+
// If `timeout` option is set, set a timeout to check the task doesn't lag.
|
|
105
|
+
let taskTimedOut = false;
|
|
106
|
+
let callbackCalled = false;
|
|
107
|
+
let timeout = ~~this.timeout;
|
|
108
|
+
let tid;
|
|
109
|
+
const taskCallback = (err, result) => {
|
|
110
|
+
// If this task has timed out, and the callback is called again
|
|
111
|
+
// from the worker, ignore it.
|
|
112
|
+
if (!taskTimedOut) {
|
|
113
|
+
clearTimeout(tid);
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
// Check that this callback is only called once.
|
|
119
|
+
if (callbackCalled && !taskTimedOut) {
|
|
120
|
+
throw Error('Callback from worker should only be called once');
|
|
121
|
+
}
|
|
122
|
+
callbackCalled = true;
|
|
123
|
+
this.finished++;
|
|
124
|
+
this.active--;
|
|
125
|
+
callback(err, result);
|
|
126
|
+
finished = true;
|
|
127
|
+
if (tookLongerThanEvery) {
|
|
128
|
+
this._next();
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
if (timeout) {
|
|
132
|
+
tid = setTimeout(() => {
|
|
133
|
+
const err = new TimeQueue.TaskError('Task timed out');
|
|
134
|
+
err.args = args;
|
|
135
|
+
taskCallback(err);
|
|
136
|
+
taskTimedOut = true;
|
|
137
|
+
}, timeout);
|
|
138
|
+
}
|
|
139
|
+
// Call the worker.
|
|
140
|
+
if (this._isWorkerAsync) {
|
|
141
|
+
try {
|
|
142
|
+
taskCallback(null, yield this.worker(...args));
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
taskCallback(err);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
// Add custom callback to args.
|
|
150
|
+
const args2 = args.slice();
|
|
151
|
+
args2.push(taskCallback);
|
|
152
|
+
this.worker(...args2);
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Called when a task finishes. Looks at the queue and processes the next
|
|
158
|
+
* waiting task.
|
|
159
|
+
*/
|
|
160
|
+
_next() {
|
|
161
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
162
|
+
let task;
|
|
163
|
+
if (this.intransit <= this.concurrency && (task = yield this.store.getNextTask())) {
|
|
164
|
+
this.active++;
|
|
165
|
+
yield this._process(task);
|
|
166
|
+
if (yield this.store.isEmpty()) {
|
|
167
|
+
this.emit('empty');
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
else if (--this.intransit === 0) {
|
|
171
|
+
this.emit('drain');
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Empties the queue and kills the timers.
|
|
177
|
+
* Active tasks will still be completed.
|
|
178
|
+
*/
|
|
179
|
+
die() {
|
|
180
|
+
this.store.clear();
|
|
181
|
+
this._timers.forEach(clearTimeout);
|
|
182
|
+
this._timers = [];
|
|
183
|
+
this.intransit = 0;
|
|
184
|
+
this.active = 0;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
TimeQueue.TaskError = class TaskError extends Error {
|
|
188
|
+
};
|
|
189
|
+
module.exports = TimeQueue;
|
|
190
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAAsC;AACtC,4DAAsC;AAuBtC,MAAM,SAAU,SAAQ,qBAAY;IA0BlC;;;;;;;;;OASG;IACH,YAAY,MAAwB,EAAE,UAA6B,EAAE;QACnE,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,IAAI,eAAe,CAAC;QACjE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,mBAAW,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;IACpB,CAAC;IAGD;;;;;;OAMG;IACG,IAAI,CAAC,GAAG,IAAW;;YACvB,iDAAiD;YACjD,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC3D,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;gBAC1D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACrC,6BAA6B;oBAC7B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;wBACxB,OAAO,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;4BAC3C,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;yBACtB;qBACF;oBACD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAiB,EAAE,OAAY,EAAE,EAAE;wBACrD,IAAI,GAAG;4BAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC5B,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE;gBACrC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,WAAW,EAAE;oBACvC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACnB;gBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;aAC3B;iBAAM;gBACL,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;aACjC;QACH,CAAC;KAAA;IAGD;;;;OAIG;IACG,QAAQ,CAAC,IAAW;;YACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,mBAA4B,CAAC;YAEjC,IAAI,KAAK,EAAE;gBACT,mBAAmB,GAAG,KAAK,CAAC;gBAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE;oBAChC,mBAAmB,GAAG,IAAI,CAAC;oBAC3B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;oBACrB,IAAI,QAAQ,EAAE;wBACZ,IAAI,CAAC,KAAK,EAAE,CAAC;qBACd;gBACH,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;aAEZ;iBAAM;gBACL,mBAAmB,GAAG,IAAI,CAAC;aAC5B;YAED,2EAA2E;YAC3E,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAC7B,IAAI,GAAiB,CAAC;YAEtB,MAAM,YAAY,GAAG,CAAC,GAAiB,EAAE,MAAY,EAAE,EAAE;gBACvD,+DAA+D;gBAC/D,8BAA8B;gBAC9B,IAAI,CAAC,YAAY,EAAE;oBACjB,YAAY,CAAC,GAAG,CAAC,CAAC;iBACnB;qBAAM;oBACL,OAAO;iBACR;gBAED,gDAAgD;gBAChD,IAAI,cAAc,IAAI,CAAC,YAAY,EAAE;oBACnC,MAAM,KAAK,CAAC,iDAAiD,CAAC,CAAC;iBAChE;gBACD,cAAc,GAAG,IAAI,CAAC;gBACtB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAEtB,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,mBAAmB,EAAE;oBACvB,IAAI,CAAC,KAAK,EAAE,CAAC;iBACd;YACH,CAAC,CAAC;YAEF,IAAI,OAAO,EAAE;gBACX,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE;oBACpB,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;oBACtD,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;oBAChB,YAAY,CAAC,GAAG,CAAC,CAAC;oBAClB,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC,EAAE,OAAO,CAAC,CAAC;aACb;YAED,mBAAmB;YACnB,IAAI,IAAI,CAAC,cAAc,EAAE;gBACvB,IAAI;oBACF,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;iBAChD;gBAAC,OAAO,GAAG,EAAE;oBACZ,YAAY,CAAC,GAAY,CAAC,CAAC;iBAC5B;aACF;iBAAM;gBACL,+BAA+B;gBAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;aACvB;QACH,CAAC;KAAA;IAGD;;;OAGG;IACG,KAAK;;YACT,IAAI,IAAI,CAAC;YACT,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE;gBACjF,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAE1B,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACpB;aAEF;iBAAM,IAAI,EAAE,IAAI,CAAC,SAAS,KAAK,CAAC,EAAE;gBACjC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aACpB;QACH,CAAC;KAAA;IAGD;;;OAGG;IACH,GAAG;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,CAAC;;AApLa,mBAAS,GAAG,MAAM,SAAU,SAAQ,KAAK;CAEtD,CAAA;AAqLH,iBAAS,SAAS,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare class Options {
|
|
2
|
+
maxQueued?: number;
|
|
3
|
+
}
|
|
4
|
+
export default class MemoryStore {
|
|
5
|
+
maxQueued: number;
|
|
6
|
+
private _queue;
|
|
7
|
+
constructor(options: Options);
|
|
8
|
+
isEmpty(): Promise<boolean>;
|
|
9
|
+
getQueued(): Promise<number>;
|
|
10
|
+
getNextTask(): Promise<any[]>;
|
|
11
|
+
pushTask(args: any[]): Promise<void>;
|
|
12
|
+
clear(): Promise<void>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
class Options {
|
|
13
|
+
}
|
|
14
|
+
exports.Options = Options;
|
|
15
|
+
class MemoryStore {
|
|
16
|
+
constructor(options) {
|
|
17
|
+
this.maxQueued = options.maxQueued;
|
|
18
|
+
this._queue = [];
|
|
19
|
+
}
|
|
20
|
+
isEmpty() {
|
|
21
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
22
|
+
return this._queue.length === 0;
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
getQueued() {
|
|
26
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
27
|
+
return this._queue.length;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
getNextTask() {
|
|
31
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
32
|
+
return this._queue.shift();
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
pushTask(args) {
|
|
36
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
37
|
+
if (this._queue.length < this.maxQueued) {
|
|
38
|
+
this._queue.push(args);
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
clear() {
|
|
43
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
44
|
+
this._queue = [];
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
exports.default = MemoryStore;
|
|
49
|
+
//# sourceMappingURL=mem-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mem-store.js","sourceRoot":"","sources":["../src/mem-store.ts"],"names":[],"mappings":";;;;;;;;;;;AAAA,MAAa,OAAO;CAEnB;AAFD,0BAEC;AACD,MAAqB,WAAW;IAG9B,YAAY,OAAgB;QAC1B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,CAAC;IACK,OAAO;;YACX,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;QAClC,CAAC;KAAA;IACK,SAAS;;YACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5B,CAAC;KAAA;IACK,WAAW;;YACf,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;KAAA;IACK,QAAQ,CAAC,IAAW;;YACxB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;gBACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACxB;QACH,CAAC;KAAA;IACK,KAAK;;YACT,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACnB,CAAC;KAAA;CACF;AAxBD,8BAwBC"}
|
package/package.json
CHANGED
|
@@ -6,29 +6,34 @@
|
|
|
6
6
|
"flow",
|
|
7
7
|
"time"
|
|
8
8
|
],
|
|
9
|
-
"version": "
|
|
9
|
+
"version": "3.0.0",
|
|
10
10
|
"repository": {
|
|
11
11
|
"type": "git",
|
|
12
12
|
"url": "git://github.com/fent/timequeue.js.git"
|
|
13
13
|
},
|
|
14
|
-
"author": "fent (https://github.com/fent)",
|
|
15
|
-
"main": "./
|
|
14
|
+
"author": "fent <fentbox@gmail.com> (https://github.com/fent)",
|
|
15
|
+
"main": "./dist/index.js",
|
|
16
16
|
"files": [
|
|
17
|
-
"
|
|
17
|
+
"dist"
|
|
18
18
|
],
|
|
19
19
|
"scripts": {
|
|
20
|
-
"
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
"
|
|
20
|
+
"prepare": "tsc -p tsconfig.build.json",
|
|
21
|
+
"build": "tsc -p tsconfig.build.json",
|
|
22
|
+
"test": "nyc --extension .ts --reporter=lcov --reporter=text-summary npm run test:unit",
|
|
23
|
+
"test:unit": "mocha -- --require ts-node/register test/*-test.ts"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"mocha": "^
|
|
27
|
-
"
|
|
28
|
-
"sinon": "^
|
|
26
|
+
"@types/mocha": "^7.0.0",
|
|
27
|
+
"@types/node": "^13.1.6",
|
|
28
|
+
"@types/sinon": "^9.0.8",
|
|
29
|
+
"mocha": "^7.0.1",
|
|
30
|
+
"nyc": "^15.0.0",
|
|
31
|
+
"sinon": "^9.0.0",
|
|
32
|
+
"ts-node": "^10.4.0",
|
|
33
|
+
"typescript": "^4.5.4"
|
|
29
34
|
},
|
|
30
35
|
"engines": {
|
|
31
|
-
"node": ">=
|
|
36
|
+
"node": ">=10"
|
|
32
37
|
},
|
|
33
38
|
"license": "MIT"
|
|
34
39
|
}
|
package/lib/index.js
DELETED
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
const EventEmitter = require('events').EventEmitter;
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
module.exports = class TimeQueue extends EventEmitter {
|
|
5
|
-
/**
|
|
6
|
-
* @constructor
|
|
7
|
-
* @extends {EventEmitter}
|
|
8
|
-
* @param {Function(..., Function(!Error, ...)} worker
|
|
9
|
-
* @param {Object} options
|
|
10
|
-
* @param {number} options.concurrency
|
|
11
|
-
* @param {number} options.time
|
|
12
|
-
*/
|
|
13
|
-
constructor(worker, options) {
|
|
14
|
-
super();
|
|
15
|
-
|
|
16
|
-
this.worker = worker;
|
|
17
|
-
this._workerAsync = worker.constructor.name == 'AsyncFunction';
|
|
18
|
-
options = options || {};
|
|
19
|
-
this.concurrency = options.concurrency || 1;
|
|
20
|
-
this.every = options.every || 0;
|
|
21
|
-
this.maxQueued = options.maxQueued || Infinity;
|
|
22
|
-
this.timeout = options.timeout || 0;
|
|
23
|
-
this._queue = [];
|
|
24
|
-
this._timers = [];
|
|
25
|
-
|
|
26
|
-
// How many tasks are currently active.
|
|
27
|
-
TimeQueue.prototype.active = 0;
|
|
28
|
-
|
|
29
|
-
// How many tasks are still being waited on,
|
|
30
|
-
// in case the `every` option was used.
|
|
31
|
-
TimeQueue.prototype.intransit = 0;
|
|
32
|
-
|
|
33
|
-
// How many tasks are in the queue.
|
|
34
|
-
TimeQueue.prototype.queued = 0;
|
|
35
|
-
|
|
36
|
-
// How many tasks have finished.
|
|
37
|
-
TimeQueue.prototype.finished = 0;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Pushes a task onto the queue.
|
|
43
|
-
*
|
|
44
|
-
* @param {Object} ...args
|
|
45
|
-
* @param {Function(!Error, ...)} callback
|
|
46
|
-
* @return {Promise?}
|
|
47
|
-
*/
|
|
48
|
-
push(...args) {
|
|
49
|
-
// Returns a promise no `callback` is given.
|
|
50
|
-
if (this._workerAsync && args.length === this.worker.length ||
|
|
51
|
-
!this._workerAsync && args.length < this.worker.length) {
|
|
52
|
-
return new Promise((resolve, reject) => {
|
|
53
|
-
// Add any missing arguments.
|
|
54
|
-
if (!this._workerAsync) {
|
|
55
|
-
while (args.length < this.worker.length - 1) {
|
|
56
|
-
args.push(undefined);
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
this.push(...args, (err, results) => {
|
|
60
|
-
if (err) return reject(err);
|
|
61
|
-
resolve(results);
|
|
62
|
-
});
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
if (this.isFull()) {
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
if (this.intransit < this.concurrency) {
|
|
71
|
-
this.intransit++;
|
|
72
|
-
this.active++;
|
|
73
|
-
if (this.intransit === this.concurrency) {
|
|
74
|
-
this.emit('full');
|
|
75
|
-
}
|
|
76
|
-
this._process(args);
|
|
77
|
-
} else {
|
|
78
|
-
this._queue.push(args);
|
|
79
|
-
this.queued++;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* Returns true if queue is full.
|
|
86
|
-
*
|
|
87
|
-
* @return {boolean}
|
|
88
|
-
*/
|
|
89
|
-
isFull() {
|
|
90
|
-
return this.maxQueued === this.queued;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
/**
|
|
95
|
-
* Starts a task
|
|
96
|
-
*
|
|
97
|
-
* @param {Array.<Object>} args
|
|
98
|
-
*/
|
|
99
|
-
async _process(args) {
|
|
100
|
-
const callback = args.pop();
|
|
101
|
-
let finished = false;
|
|
102
|
-
let every = ~~this.every;
|
|
103
|
-
let timedOut;
|
|
104
|
-
|
|
105
|
-
if (every) {
|
|
106
|
-
timedOut = false;
|
|
107
|
-
|
|
108
|
-
this._timers.push(setTimeout(() => {
|
|
109
|
-
timedOut = true;
|
|
110
|
-
this._timers.shift();
|
|
111
|
-
if (finished) {
|
|
112
|
-
this._next();
|
|
113
|
-
}
|
|
114
|
-
}, every));
|
|
115
|
-
|
|
116
|
-
} else {
|
|
117
|
-
timedOut = true;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
// If `timeout` option is set, set a timeout to check the task doesn't lag.
|
|
121
|
-
let taskTimedOut = false;
|
|
122
|
-
let callbackCalled = false;
|
|
123
|
-
let timeout = ~~this.timeout;
|
|
124
|
-
let tid;
|
|
125
|
-
|
|
126
|
-
const taskCallback = (err, result) => {
|
|
127
|
-
// If this task has timed out, and the callback is called again
|
|
128
|
-
// from the worker, ignore it.
|
|
129
|
-
if (!taskTimedOut) {
|
|
130
|
-
clearTimeout(tid);
|
|
131
|
-
} else {
|
|
132
|
-
return;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// Check that this callback is only called once.
|
|
136
|
-
if (callbackCalled && !taskTimedOut) {
|
|
137
|
-
throw Error('Callback from worker should only be called once');
|
|
138
|
-
}
|
|
139
|
-
callbackCalled = true;
|
|
140
|
-
this.finished++;
|
|
141
|
-
this.active--;
|
|
142
|
-
callback(err, result);
|
|
143
|
-
|
|
144
|
-
finished = true;
|
|
145
|
-
if (timedOut) {
|
|
146
|
-
this._next();
|
|
147
|
-
}
|
|
148
|
-
};
|
|
149
|
-
|
|
150
|
-
if (timeout) {
|
|
151
|
-
tid = setTimeout(() => {
|
|
152
|
-
const err = Error('Task timed out');
|
|
153
|
-
err.args = args;
|
|
154
|
-
taskCallback(err);
|
|
155
|
-
taskTimedOut = true;
|
|
156
|
-
}, timeout);
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// Call the worker.
|
|
160
|
-
if (this._workerAsync) {
|
|
161
|
-
try {
|
|
162
|
-
taskCallback(null, await this.worker(...args));
|
|
163
|
-
} catch (err) {
|
|
164
|
-
taskCallback(err);
|
|
165
|
-
}
|
|
166
|
-
} else {
|
|
167
|
-
// Add custom callback to args.
|
|
168
|
-
const args2 = args.slice();
|
|
169
|
-
args2.push(taskCallback);
|
|
170
|
-
this.worker(...args2);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
/**
|
|
176
|
-
* Called when a task finishes. Looks at the queue and processes the next
|
|
177
|
-
* waiting task.
|
|
178
|
-
*/
|
|
179
|
-
_next() {
|
|
180
|
-
if (this.intransit <= this.concurrency && this._queue.length) {
|
|
181
|
-
this.queued--;
|
|
182
|
-
this.active++;
|
|
183
|
-
this._process(this._queue.shift());
|
|
184
|
-
|
|
185
|
-
if (this._queue.length === 0) {
|
|
186
|
-
this.emit('empty');
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
} else if (--this.intransit === 0) {
|
|
190
|
-
this.emit('drain');
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Empties the queue and kills the timers.
|
|
197
|
-
* Active tasks will still be completed.
|
|
198
|
-
*/
|
|
199
|
-
die() {
|
|
200
|
-
this._queue = [];
|
|
201
|
-
this._timers.forEach(clearTimeout);
|
|
202
|
-
this._timers = [];
|
|
203
|
-
this.intransit = 0;
|
|
204
|
-
this.active = 0;
|
|
205
|
-
this.queued = 0;
|
|
206
|
-
}
|
|
207
|
-
};
|