timequeue 1.2.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -24
- package/dist/index.d.ts +82 -0
- package/dist/index.js +190 -0
- package/dist/index.js.map +1 -0
- package/dist/mem-store.d.ts +13 -0
- package/dist/mem-store.js +50 -0
- package/dist/mem-store.js.map +1 -0
- package/package.json +17 -12
- package/lib/index.js +0 -196
package/README.md
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# timequeue.js
|
|
2
2
|
|
|
3
|
-
A queue with custom concurrency and time limits. Inspired by [
|
|
3
|
+
A queue with custom concurrency and time limits. Inspired by [async/queue](https://caolan.github.io/async/v3/docs.html#queue), but also with variable number of arguments in the worker, events, and with optional time limits.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
[](https://david-dm.org/fent/timequeue.js)
|
|
5
|
+

|
|
7
6
|
[](https://codecov.io/gh/fent/timequeue.js)
|
|
8
7
|
|
|
9
8
|
# Usage
|
|
@@ -15,17 +14,26 @@ const worker = (arg1, arg2, callback) => {
|
|
|
15
14
|
someAsyncFunction(calculation(arg1, arg2), callback);
|
|
16
15
|
};
|
|
17
16
|
|
|
17
|
+
// Worker can be an async function.
|
|
18
|
+
const worker = async (arg1) => {
|
|
19
|
+
let result1 = await anotherSyncFunction(arg1);
|
|
20
|
+
return andAnotherOne(result1);
|
|
21
|
+
};
|
|
22
|
+
|
|
18
23
|
// create a queue with max 5 concurrency every second
|
|
19
24
|
let q = new TimeQueue(worker, { concurrency: 5, every: 1000 });
|
|
20
25
|
|
|
21
|
-
//
|
|
26
|
+
// Push tasks onto the queue.
|
|
22
27
|
q.push(42, 24);
|
|
23
28
|
q.push(2, 74);
|
|
24
29
|
|
|
25
|
-
//
|
|
26
|
-
q.push(2, 2, (err) => {
|
|
30
|
+
// Optional callback when pushing tasks.
|
|
31
|
+
q.push(2, 2, (err, result) => {
|
|
27
32
|
// task finished
|
|
28
33
|
});
|
|
34
|
+
|
|
35
|
+
// Can use promise/await syntax instead.
|
|
36
|
+
let result = await q.push(3, 5);
|
|
29
37
|
```
|
|
30
38
|
|
|
31
39
|
|
|
@@ -40,44 +48,45 @@ Creates a new instance of a queue. Worker must be a function with a callback for
|
|
|
40
48
|
|
|
41
49
|
// How much time in milliseconds to allow no more than
|
|
42
50
|
// the max number of concurrent tasks to run.
|
|
43
|
-
// If the max amount of concurrent tasks
|
|
44
|
-
//
|
|
51
|
+
// If the max amount of concurrent tasks finish faster than this time limit,
|
|
52
|
+
// additional tasks will wait until enough time has passed before starting.
|
|
45
53
|
, every: 0
|
|
46
54
|
|
|
47
55
|
// Maximum number of tasks to keep in the queue.
|
|
48
56
|
// While full, pushed tasks will be ignored.
|
|
49
57
|
, maxQueued: Infinity
|
|
50
58
|
|
|
51
|
-
// If set, will
|
|
52
|
-
// if callback was given to that task,
|
|
53
|
-
//
|
|
59
|
+
// If set, tasks will error if they take too much time.
|
|
60
|
+
// if callback was given to that task, it will be called with the error,
|
|
61
|
+
// otherwise, the returned promise should be `caught`.
|
|
54
62
|
, timeout: 0
|
|
63
|
+
|
|
64
|
+
// You can pass a custom store to share tasks between several queues.
|
|
65
|
+
// Default is MemoryStore from `src/mem-store.ts`.
|
|
66
|
+
// Look at `example/redis-store.js` for an example that saves tasks onto redis.
|
|
67
|
+
, store: MemoryStore
|
|
55
68
|
}
|
|
56
69
|
```
|
|
57
70
|
|
|
58
71
|
All of these options can later be edited on the queue instance.
|
|
59
72
|
|
|
60
73
|
### TimeQueue#active
|
|
61
|
-
|
|
62
74
|
How many tasks are currently active.
|
|
63
75
|
|
|
64
76
|
### TimeQueue#intransit
|
|
65
|
-
|
|
66
77
|
If you use the `every` option to queue up tasks, this property will be delayed from being updated until there are free spots open for new tasks to begin. `active` will be updated as soon as a task finishes, even if the next one is just a timeout around the corner.
|
|
67
78
|
|
|
68
|
-
### TimeQueue#queued
|
|
69
|
-
|
|
70
|
-
How many tasks are currently in the queue.
|
|
71
|
-
|
|
72
79
|
### TimeQueue#finished
|
|
73
|
-
|
|
74
80
|
How many tasks have finished in total.
|
|
75
81
|
|
|
76
|
-
### TimeQueue#push(data..., [callback])
|
|
82
|
+
### async TimeQueue#push(data..., [callback])
|
|
77
83
|
Pushes a new task to the queue. Any number of arguments can be given. An optional callback can also be given as the last parameter. The callback will be called when the task is finished or if there was any error running the worker.
|
|
78
84
|
|
|
79
85
|
If the queue is full, pushed tasks will be ignored.
|
|
80
86
|
|
|
87
|
+
### async TimeQueue#store.getQueued()
|
|
88
|
+
How many tasks are currently in the queue.
|
|
89
|
+
|
|
81
90
|
### TimeQueue#isFull()
|
|
82
91
|
Returns true if queue is full.
|
|
83
92
|
|
|
@@ -86,11 +95,6 @@ Empties queue and clears the timeouts TimeQueue sets to keep track of running ta
|
|
|
86
95
|
|
|
87
96
|
# Events
|
|
88
97
|
|
|
89
|
-
### Event: 'error'
|
|
90
|
-
* `Error`
|
|
91
|
-
|
|
92
|
-
Emitted when there is an error processing a task and a callback isn't given to the `push` method.
|
|
93
|
-
|
|
94
98
|
### Event: 'full'
|
|
95
99
|
|
|
96
100
|
Queue is full.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
3
|
+
declare namespace TimeQueue {
|
|
4
|
+
type Worker = (...args: any[]) => void | Promise<any>;
|
|
5
|
+
interface Options {
|
|
6
|
+
concurrency?: number;
|
|
7
|
+
every?: number;
|
|
8
|
+
maxQueued?: number;
|
|
9
|
+
timeout?: number;
|
|
10
|
+
store?: Store;
|
|
11
|
+
}
|
|
12
|
+
interface TaskError extends Error {
|
|
13
|
+
args: any[];
|
|
14
|
+
}
|
|
15
|
+
interface Store {
|
|
16
|
+
isEmpty: () => Promise<boolean>;
|
|
17
|
+
getQueued: () => Promise<number>;
|
|
18
|
+
getNextTask: () => Promise<any[]>;
|
|
19
|
+
pushTask: (...args: any[]) => void;
|
|
20
|
+
clear: () => void;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
declare class TimeQueue extends EventEmitter {
|
|
24
|
+
worker: TimeQueue.Worker;
|
|
25
|
+
concurrency: number;
|
|
26
|
+
every: number;
|
|
27
|
+
maxQueued: number;
|
|
28
|
+
timeout: number;
|
|
29
|
+
private _isWorkerAsync;
|
|
30
|
+
private _timers;
|
|
31
|
+
store: TimeQueue.Store;
|
|
32
|
+
active: number;
|
|
33
|
+
intransit: number;
|
|
34
|
+
finished: number;
|
|
35
|
+
static TaskError: {
|
|
36
|
+
new (message?: string): {
|
|
37
|
+
args: any[];
|
|
38
|
+
name: string;
|
|
39
|
+
message: string;
|
|
40
|
+
stack?: string;
|
|
41
|
+
};
|
|
42
|
+
captureStackTrace(targetObject: object, constructorOpt?: Function): void;
|
|
43
|
+
prepareStackTrace?: (err: Error, stackTraces: NodeJS.CallSite[]) => any;
|
|
44
|
+
stackTraceLimit: number;
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* @constructor
|
|
48
|
+
* @extends {EventEmitter}
|
|
49
|
+
* @param {Function(..., Function(!Error, ...)} worker
|
|
50
|
+
* @param {Object?} options
|
|
51
|
+
* @param {number?} options.concurrency
|
|
52
|
+
* @param {number?} options.every
|
|
53
|
+
* @param {number?} options.maxQueued
|
|
54
|
+
* @param {number?} options.timeout
|
|
55
|
+
*/
|
|
56
|
+
constructor(worker: TimeQueue.Worker, options?: TimeQueue.Options);
|
|
57
|
+
/**
|
|
58
|
+
* Pushes a task onto the queue.
|
|
59
|
+
*
|
|
60
|
+
* @param {Object} ...args
|
|
61
|
+
* @param {Function(!Error, ...)} callback
|
|
62
|
+
* @return {Promise?}
|
|
63
|
+
*/
|
|
64
|
+
push(...args: any[]): Promise<unknown>;
|
|
65
|
+
/**
|
|
66
|
+
* Starts a task
|
|
67
|
+
*
|
|
68
|
+
* @param {Array.<Object>} args
|
|
69
|
+
*/
|
|
70
|
+
_process(args: any[]): Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* Called when a task finishes. Looks at the queue and processes the next
|
|
73
|
+
* waiting task.
|
|
74
|
+
*/
|
|
75
|
+
_next(): Promise<void>;
|
|
76
|
+
/**
|
|
77
|
+
* Empties the queue and kills the timers.
|
|
78
|
+
* Active tasks will still be completed.
|
|
79
|
+
*/
|
|
80
|
+
die(): void;
|
|
81
|
+
}
|
|
82
|
+
export = TimeQueue;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
const events_1 = require("events");
|
|
15
|
+
const mem_store_1 = __importDefault(require("./mem-store"));
|
|
16
|
+
class TimeQueue extends events_1.EventEmitter {
|
|
17
|
+
/**
|
|
18
|
+
* @constructor
|
|
19
|
+
* @extends {EventEmitter}
|
|
20
|
+
* @param {Function(..., Function(!Error, ...)} worker
|
|
21
|
+
* @param {Object?} options
|
|
22
|
+
* @param {number?} options.concurrency
|
|
23
|
+
* @param {number?} options.every
|
|
24
|
+
* @param {number?} options.maxQueued
|
|
25
|
+
* @param {number?} options.timeout
|
|
26
|
+
*/
|
|
27
|
+
constructor(worker, options = {}) {
|
|
28
|
+
super();
|
|
29
|
+
this.worker = worker;
|
|
30
|
+
this._isWorkerAsync = worker.constructor.name == 'AsyncFunction';
|
|
31
|
+
this.concurrency = options.concurrency || 1;
|
|
32
|
+
this.every = options.every || 0;
|
|
33
|
+
this.maxQueued = options.maxQueued || Infinity;
|
|
34
|
+
this.timeout = options.timeout || 0;
|
|
35
|
+
this.store = options.store || new mem_store_1.default({ maxQueued: this.maxQueued });
|
|
36
|
+
this._timers = [];
|
|
37
|
+
this.active = 0;
|
|
38
|
+
this.intransit = 0;
|
|
39
|
+
this.finished = 0;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Pushes a task onto the queue.
|
|
43
|
+
*
|
|
44
|
+
* @param {Object} ...args
|
|
45
|
+
* @param {Function(!Error, ...)} callback
|
|
46
|
+
* @return {Promise?}
|
|
47
|
+
*/
|
|
48
|
+
push(...args) {
|
|
49
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
50
|
+
// Returns a promise when no `callback` is given.
|
|
51
|
+
if (this._isWorkerAsync && args.length === this.worker.length ||
|
|
52
|
+
!this._isWorkerAsync && args.length < this.worker.length) {
|
|
53
|
+
return new Promise((resolve, reject) => {
|
|
54
|
+
// Add any missing arguments.
|
|
55
|
+
if (!this._isWorkerAsync) {
|
|
56
|
+
while (args.length < this.worker.length - 1) {
|
|
57
|
+
args.push(undefined);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
this.push(...args, (err, results) => {
|
|
61
|
+
if (err)
|
|
62
|
+
return reject(err);
|
|
63
|
+
resolve(results);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
if (this.intransit < this.concurrency) {
|
|
68
|
+
this.intransit++;
|
|
69
|
+
this.active++;
|
|
70
|
+
if (this.intransit === this.concurrency) {
|
|
71
|
+
this.emit('full');
|
|
72
|
+
}
|
|
73
|
+
yield this._process(args);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
yield this.store.pushTask(args);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Starts a task
|
|
82
|
+
*
|
|
83
|
+
* @param {Array.<Object>} args
|
|
84
|
+
*/
|
|
85
|
+
_process(args) {
|
|
86
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
87
|
+
const callback = args.pop();
|
|
88
|
+
let finished = false;
|
|
89
|
+
let every = ~~this.every;
|
|
90
|
+
let tookLongerThanEvery;
|
|
91
|
+
if (every) {
|
|
92
|
+
tookLongerThanEvery = false;
|
|
93
|
+
this._timers.push(setTimeout(() => {
|
|
94
|
+
tookLongerThanEvery = true;
|
|
95
|
+
this._timers.shift();
|
|
96
|
+
if (finished) {
|
|
97
|
+
this._next();
|
|
98
|
+
}
|
|
99
|
+
}, every));
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
tookLongerThanEvery = true;
|
|
103
|
+
}
|
|
104
|
+
// If `timeout` option is set, set a timeout to check the task doesn't lag.
|
|
105
|
+
let taskTimedOut = false;
|
|
106
|
+
let callbackCalled = false;
|
|
107
|
+
let timeout = ~~this.timeout;
|
|
108
|
+
let tid;
|
|
109
|
+
const taskCallback = (err, result) => {
|
|
110
|
+
// If this task has timed out, and the callback is called again
|
|
111
|
+
// from the worker, ignore it.
|
|
112
|
+
if (!taskTimedOut) {
|
|
113
|
+
clearTimeout(tid);
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
// Check that this callback is only called once.
|
|
119
|
+
if (callbackCalled && !taskTimedOut) {
|
|
120
|
+
throw Error('Callback from worker should only be called once');
|
|
121
|
+
}
|
|
122
|
+
callbackCalled = true;
|
|
123
|
+
this.finished++;
|
|
124
|
+
this.active--;
|
|
125
|
+
callback(err, result);
|
|
126
|
+
finished = true;
|
|
127
|
+
if (tookLongerThanEvery) {
|
|
128
|
+
this._next();
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
if (timeout) {
|
|
132
|
+
tid = setTimeout(() => {
|
|
133
|
+
const err = new TimeQueue.TaskError('Task timed out');
|
|
134
|
+
err.args = args;
|
|
135
|
+
taskCallback(err);
|
|
136
|
+
taskTimedOut = true;
|
|
137
|
+
}, timeout);
|
|
138
|
+
}
|
|
139
|
+
// Call the worker.
|
|
140
|
+
if (this._isWorkerAsync) {
|
|
141
|
+
try {
|
|
142
|
+
taskCallback(null, yield this.worker(...args));
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
taskCallback(err);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
// Add custom callback to args.
|
|
150
|
+
const args2 = args.slice();
|
|
151
|
+
args2.push(taskCallback);
|
|
152
|
+
this.worker(...args2);
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Called when a task finishes. Looks at the queue and processes the next
|
|
158
|
+
* waiting task.
|
|
159
|
+
*/
|
|
160
|
+
_next() {
|
|
161
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
162
|
+
let task;
|
|
163
|
+
if (this.intransit <= this.concurrency && (task = yield this.store.getNextTask())) {
|
|
164
|
+
this.active++;
|
|
165
|
+
yield this._process(task);
|
|
166
|
+
if (yield this.store.isEmpty()) {
|
|
167
|
+
this.emit('empty');
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
else if (--this.intransit === 0) {
|
|
171
|
+
this.emit('drain');
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Empties the queue and kills the timers.
|
|
177
|
+
* Active tasks will still be completed.
|
|
178
|
+
*/
|
|
179
|
+
die() {
|
|
180
|
+
this.store.clear();
|
|
181
|
+
this._timers.forEach(clearTimeout);
|
|
182
|
+
this._timers = [];
|
|
183
|
+
this.intransit = 0;
|
|
184
|
+
this.active = 0;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
TimeQueue.TaskError = class TaskError extends Error {
|
|
188
|
+
};
|
|
189
|
+
module.exports = TimeQueue;
|
|
190
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAAsC;AACtC,4DAAsC;AAuBtC,MAAM,SAAU,SAAQ,qBAAY;IA0BlC;;;;;;;;;OASG;IACH,YAAY,MAAwB,EAAE,UAA6B,EAAE;QACnE,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,IAAI,eAAe,CAAC;QACjE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,mBAAW,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;IACpB,CAAC;IAGD;;;;;;OAMG;IACG,IAAI,CAAC,GAAG,IAAW;;YACvB,iDAAiD;YACjD,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC3D,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;gBAC1D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACrC,6BAA6B;oBAC7B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;wBACxB,OAAO,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;4BAC3C,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;yBACtB;qBACF;oBACD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAiB,EAAE,OAAY,EAAE,EAAE;wBACrD,IAAI,GAAG;4BAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC5B,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE;gBACrC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,WAAW,EAAE;oBACvC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACnB;gBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;aAC3B;iBAAM;gBACL,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;aACjC;QACH,CAAC;KAAA;IAGD;;;;OAIG;IACG,QAAQ,CAAC,IAAW;;YACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,mBAA4B,CAAC;YAEjC,IAAI,KAAK,EAAE;gBACT,mBAAmB,GAAG,KAAK,CAAC;gBAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE;oBAChC,mBAAmB,GAAG,IAAI,CAAC;oBAC3B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;oBACrB,IAAI,QAAQ,EAAE;wBACZ,IAAI,CAAC,KAAK,EAAE,CAAC;qBACd;gBACH,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;aAEZ;iBAAM;gBACL,mBAAmB,GAAG,IAAI,CAAC;aAC5B;YAED,2EAA2E;YAC3E,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAC7B,IAAI,GAAiB,CAAC;YAEtB,MAAM,YAAY,GAAG,CAAC,GAAiB,EAAE,MAAY,EAAE,EAAE;gBACvD,+DAA+D;gBAC/D,8BAA8B;gBAC9B,IAAI,CAAC,YAAY,EAAE;oBACjB,YAAY,CAAC,GAAG,CAAC,CAAC;iBACnB;qBAAM;oBACL,OAAO;iBACR;gBAED,gDAAgD;gBAChD,IAAI,cAAc,IAAI,CAAC,YAAY,EAAE;oBACnC,MAAM,KAAK,CAAC,iDAAiD,CAAC,CAAC;iBAChE;gBACD,cAAc,GAAG,IAAI,CAAC;gBACtB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAEtB,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,mBAAmB,EAAE;oBACvB,IAAI,CAAC,KAAK,EAAE,CAAC;iBACd;YACH,CAAC,CAAC;YAEF,IAAI,OAAO,EAAE;gBACX,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE;oBACpB,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;oBACtD,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;oBAChB,YAAY,CAAC,GAAG,CAAC,CAAC;oBAClB,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC,EAAE,OAAO,CAAC,CAAC;aACb;YAED,mBAAmB;YACnB,IAAI,IAAI,CAAC,cAAc,EAAE;gBACvB,IAAI;oBACF,YAAY,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;iBAChD;gBAAC,OAAO,GAAG,EAAE;oBACZ,YAAY,CAAC,GAAY,CAAC,CAAC;iBAC5B;aACF;iBAAM;gBACL,+BAA+B;gBAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;aACvB;QACH,CAAC;KAAA;IAGD;;;OAGG;IACG,KAAK;;YACT,IAAI,IAAI,CAAC;YACT,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE;gBACjF,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAE1B,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACpB;aAEF;iBAAM,IAAI,EAAE,IAAI,CAAC,SAAS,KAAK,CAAC,EAAE;gBACjC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aACpB;QACH,CAAC;KAAA;IAGD;;;OAGG;IACH,GAAG;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,CAAC;;AApLa,mBAAS,GAAG,MAAM,SAAU,SAAQ,KAAK;CAEtD,CAAA;AAqLH,iBAAS,SAAS,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare class Options {
|
|
2
|
+
maxQueued?: number;
|
|
3
|
+
}
|
|
4
|
+
export default class MemoryStore {
|
|
5
|
+
maxQueued: number;
|
|
6
|
+
private _queue;
|
|
7
|
+
constructor(options: Options);
|
|
8
|
+
isEmpty(): Promise<boolean>;
|
|
9
|
+
getQueued(): Promise<number>;
|
|
10
|
+
getNextTask(): Promise<any[]>;
|
|
11
|
+
pushTask(args: any[]): Promise<void>;
|
|
12
|
+
clear(): Promise<void>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.Options = void 0;
|
|
13
|
+
class Options {
|
|
14
|
+
}
|
|
15
|
+
exports.Options = Options;
|
|
16
|
+
class MemoryStore {
|
|
17
|
+
constructor(options) {
|
|
18
|
+
this.maxQueued = options.maxQueued;
|
|
19
|
+
this._queue = [];
|
|
20
|
+
}
|
|
21
|
+
isEmpty() {
|
|
22
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
23
|
+
return this._queue.length === 0;
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
getQueued() {
|
|
27
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
28
|
+
return this._queue.length;
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
getNextTask() {
|
|
32
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
33
|
+
return this._queue.shift();
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
pushTask(args) {
|
|
37
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
38
|
+
if (this._queue.length < this.maxQueued) {
|
|
39
|
+
this._queue.push(args);
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
clear() {
|
|
44
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
45
|
+
this._queue = [];
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
exports.default = MemoryStore;
|
|
50
|
+
//# sourceMappingURL=mem-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mem-store.js","sourceRoot":"","sources":["../src/mem-store.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,MAAa,OAAO;CAEnB;AAFD,0BAEC;AACD,MAAqB,WAAW;IAG9B,YAAY,OAAgB;QAC1B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACnB,CAAC;IACK,OAAO;;YACX,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;QAClC,CAAC;KAAA;IACK,SAAS;;YACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5B,CAAC;KAAA;IACK,WAAW;;YACf,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,CAAC;KAAA;IACK,QAAQ,CAAC,IAAW;;YACxB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;gBACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aACxB;QACH,CAAC;KAAA;IACK,KAAK;;YACT,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACnB,CAAC;KAAA;CACF;AAxBD,8BAwBC"}
|
package/package.json
CHANGED
|
@@ -6,29 +6,34 @@
|
|
|
6
6
|
"flow",
|
|
7
7
|
"time"
|
|
8
8
|
],
|
|
9
|
-
"version": "1.
|
|
9
|
+
"version": "2.1.0",
|
|
10
10
|
"repository": {
|
|
11
11
|
"type": "git",
|
|
12
12
|
"url": "git://github.com/fent/timequeue.js.git"
|
|
13
13
|
},
|
|
14
|
-
"author": "fent (https://github.com/fent)",
|
|
15
|
-
"main": "./
|
|
14
|
+
"author": "fent <fentbox@gmail.com> (https://github.com/fent)",
|
|
15
|
+
"main": "./dist/index.js",
|
|
16
16
|
"files": [
|
|
17
|
-
"
|
|
17
|
+
"dist"
|
|
18
18
|
],
|
|
19
19
|
"scripts": {
|
|
20
|
-
"
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
"
|
|
20
|
+
"prepare": "tsc -p tsconfig.build.json",
|
|
21
|
+
"build": "tsc -p tsconfig.build.json",
|
|
22
|
+
"test": "nyc --extension .ts --reporter=lcov --reporter=text-summary npm run test:unit",
|
|
23
|
+
"test:unit": "mocha -- --require ts-node/register test/*-test.ts"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"sinon": "^
|
|
26
|
+
"@types/mocha": "^7.0.0",
|
|
27
|
+
"@types/node": "^13.1.6",
|
|
28
|
+
"@types/sinon": "^9.0.8",
|
|
29
|
+
"mocha": "^7.0.1",
|
|
30
|
+
"nyc": "^15.0.0",
|
|
31
|
+
"sinon": "^9.0.0",
|
|
32
|
+
"ts-node": "^10.4.0",
|
|
33
|
+
"typescript": "^4.5.4"
|
|
29
34
|
},
|
|
30
35
|
"engines": {
|
|
31
|
-
"node": ">=
|
|
36
|
+
"node": ">=10"
|
|
32
37
|
},
|
|
33
38
|
"license": "MIT"
|
|
34
39
|
}
|
package/lib/index.js
DELETED
|
@@ -1,196 +0,0 @@
|
|
|
1
|
-
const EventEmitter = require('events').EventEmitter;
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
module.exports = class TimeQueue extends EventEmitter {
|
|
5
|
-
/**
|
|
6
|
-
* @constructor
|
|
7
|
-
* @extends {EventEmitter}
|
|
8
|
-
* @param {Function(..., Function(!Error, ...)} worker
|
|
9
|
-
* @param {Object} options
|
|
10
|
-
* @param {number} concurrency
|
|
11
|
-
* @param {number} time
|
|
12
|
-
*/
|
|
13
|
-
constructor(worker, options) {
|
|
14
|
-
super();
|
|
15
|
-
|
|
16
|
-
this.worker = worker;
|
|
17
|
-
options = options || {};
|
|
18
|
-
this.concurrency = options.concurrency || 1;
|
|
19
|
-
this.every = options.every || 0;
|
|
20
|
-
this.maxQueued = options.maxQueued || Infinity;
|
|
21
|
-
this.timeout = options.timeout || 0;
|
|
22
|
-
this._queue = [];
|
|
23
|
-
this._timers = [];
|
|
24
|
-
|
|
25
|
-
// How many tasks are currently active.
|
|
26
|
-
TimeQueue.prototype.active = 0;
|
|
27
|
-
|
|
28
|
-
// How many tasks are still being waited on,
|
|
29
|
-
// in case the `every` option was used.
|
|
30
|
-
TimeQueue.prototype.intransit = 0;
|
|
31
|
-
|
|
32
|
-
// How many tasks are in the queue.
|
|
33
|
-
TimeQueue.prototype.queued = 0;
|
|
34
|
-
|
|
35
|
-
// How many tasks have finished.
|
|
36
|
-
TimeQueue.prototype.finished = 0;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Pushes a task onto the queue.
|
|
42
|
-
*
|
|
43
|
-
* @param {Object} ...args
|
|
44
|
-
* @param {Function(!Error, ...)} callback
|
|
45
|
-
*/
|
|
46
|
-
push(...args) {
|
|
47
|
-
if (this.isFull()) {
|
|
48
|
-
return;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
if (this.intransit < this.concurrency) {
|
|
52
|
-
this.intransit++;
|
|
53
|
-
this.active++;
|
|
54
|
-
if (this.intransit === this.concurrency) {
|
|
55
|
-
this.emit('full');
|
|
56
|
-
}
|
|
57
|
-
this._process(args);
|
|
58
|
-
} else {
|
|
59
|
-
this._queue.push(args);
|
|
60
|
-
this.queued++;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Returns true if queue is full.
|
|
67
|
-
*
|
|
68
|
-
* @return {boolean}
|
|
69
|
-
*/
|
|
70
|
-
isFull() {
|
|
71
|
-
return this.maxQueued === this.queued;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Starts a task
|
|
77
|
-
*
|
|
78
|
-
* @param {Array.<Object>} args
|
|
79
|
-
*/
|
|
80
|
-
_process(args) {
|
|
81
|
-
const callback = args.splice(this.worker.length - 1, 1)[0];
|
|
82
|
-
let finished = false;
|
|
83
|
-
let every = ~~this.every;
|
|
84
|
-
let timedOut;
|
|
85
|
-
|
|
86
|
-
if (every) {
|
|
87
|
-
timedOut = false;
|
|
88
|
-
|
|
89
|
-
this._timers.push(setTimeout(() => {
|
|
90
|
-
timedOut = true;
|
|
91
|
-
this._timers.shift();
|
|
92
|
-
if (finished) {
|
|
93
|
-
this._next();
|
|
94
|
-
}
|
|
95
|
-
}, every));
|
|
96
|
-
|
|
97
|
-
} else {
|
|
98
|
-
timedOut = true;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
// If `timeout` option is set, set a timeout to check the task doesn't lag.
|
|
102
|
-
let taskTimedOut = false;
|
|
103
|
-
let callbackCalled = false;
|
|
104
|
-
let timeout = ~~this.timeout;
|
|
105
|
-
let tid;
|
|
106
|
-
|
|
107
|
-
const taskCallback = (err, ...args) => {
|
|
108
|
-
// If this task has timed out, and the callback is called again
|
|
109
|
-
// from the worker, ignore it.
|
|
110
|
-
if (!taskTimedOut) {
|
|
111
|
-
clearTimeout(tid);
|
|
112
|
-
} else {
|
|
113
|
-
return;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// Check that this callback is only called once.
|
|
117
|
-
if (callbackCalled && !taskTimedOut) {
|
|
118
|
-
throw Error('Callback from worker should only be called once');
|
|
119
|
-
}
|
|
120
|
-
callbackCalled = true;
|
|
121
|
-
|
|
122
|
-
this.finished++;
|
|
123
|
-
this.active--;
|
|
124
|
-
|
|
125
|
-
if (typeof callback === 'function') {
|
|
126
|
-
// If a callback was given with the task,
|
|
127
|
-
// call it when the task is finished.
|
|
128
|
-
callback(err, ...args);
|
|
129
|
-
|
|
130
|
-
} else if (err) {
|
|
131
|
-
// Otherwise emit an `error` event if there was an error with the task.
|
|
132
|
-
this.emit('error', err);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
finished = true;
|
|
136
|
-
if (timedOut) {
|
|
137
|
-
this._next();
|
|
138
|
-
}
|
|
139
|
-
};
|
|
140
|
-
|
|
141
|
-
if (timeout) {
|
|
142
|
-
tid = setTimeout(() => {
|
|
143
|
-
const err = new Error('Task timed out');
|
|
144
|
-
err.args = args;
|
|
145
|
-
taskCallback(err);
|
|
146
|
-
taskTimedOut = true;
|
|
147
|
-
}, timeout);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// Add missing arguments.
|
|
151
|
-
while (args.length < this.worker.length - 1) {
|
|
152
|
-
args.push(undefined);
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// Add custom callback to args.
|
|
156
|
-
const args2 = args.slice();
|
|
157
|
-
args2.push(taskCallback);
|
|
158
|
-
|
|
159
|
-
// Call the worker.
|
|
160
|
-
this.worker(...args2);
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
/**
|
|
165
|
-
* Called when a task finishes. Looks at the queue and processes the next
|
|
166
|
-
* waiting task.
|
|
167
|
-
*/
|
|
168
|
-
_next() {
|
|
169
|
-
if (this.intransit <= this.concurrency && this._queue.length) {
|
|
170
|
-
this.queued--;
|
|
171
|
-
this.active++;
|
|
172
|
-
this._process(this._queue.shift());
|
|
173
|
-
|
|
174
|
-
if (this._queue.length === 0) {
|
|
175
|
-
this.emit('empty');
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
} else if (--this.intransit === 0) {
|
|
179
|
-
this.emit('drain');
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
/**
|
|
185
|
-
* Empties the queue and kills the timers.
|
|
186
|
-
* Active tasks will still be completed.
|
|
187
|
-
*/
|
|
188
|
-
die() {
|
|
189
|
-
this._queue = [];
|
|
190
|
-
this._timers.forEach(clearTimeout);
|
|
191
|
-
this._timers = [];
|
|
192
|
-
this.intransit = 0;
|
|
193
|
-
this.active = 0;
|
|
194
|
-
this.queued = 0;
|
|
195
|
-
}
|
|
196
|
-
};
|