squiss-ts 4.4.1 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/EventEmitterTypesHelper.d.ts +5 -5
- package/dist/EventEmitterTypesHelper.js +2 -1
- package/dist/Message.d.ts +7 -9
- package/dist/Message.js +10 -10
- package/dist/Squiss.d.ts +4 -4
- package/dist/Squiss.js +86 -47
- package/dist/TimeoutExtender.js +3 -2
- package/dist/Types.d.ts +10 -11
- package/dist/Types.js +1 -1
- package/dist/Utils.js +1 -1
- package/dist/attributeUtils.d.ts +4 -4
- package/dist/attributeUtils.js +1 -1
- package/dist/gzipUtils.js +5 -17
- package/dist/index.d.ts +2 -2
- package/dist/index.js +5 -5
- package/dist/messageSizeUtils.d.ts +6 -1
- package/dist/messageSizeUtils.js +1 -1
- package/dist/s3Utils.d.ts +1 -1
- package/dist/s3Utils.js +11 -8
- package/package.json +44 -27
package/README.md
CHANGED
|
@@ -4,15 +4,13 @@
|
|
|
4
4
|
[](https://codeclimate.com/github/PruvoNet/squiss-ts/test_coverage)
|
|
5
5
|
[](https://codeclimate.com/github/PruvoNet/squiss-ts/maintainability)
|
|
6
6
|
[](https://snyk.io/test/github/PruvoNet/squiss-ts?targetFile=package.json)
|
|
7
|
-
[](https://david-dm.org/PruvoNet/squiss-ts)
|
|
8
|
-
[](https://david-dm.org/PruvoNet/squiss-ts?type=dev)
|
|
9
7
|
|
|
10
8
|
<p align="center">
|
|
11
9
|
<a href="https://squiss-ts.pruvo.com"><img src="https://github.com/PruvoNet/squiss-ts/blob/docs/source/images/logo.png?raw=true" /></a>
|
|
12
10
|
</p>
|
|
13
11
|
|
|
14
12
|
# Squiss-TS
|
|
15
|
-
High-volume Amazon SQS Poller and single-queue client for Node.js
|
|
13
|
+
High-volume Amazon SQS Poller and single-queue client for Node.js 16 and up with full typescript support
|
|
16
14
|
The library is production ready and is being stress used in a full blown production environment
|
|
17
15
|
|
|
18
16
|
## Main features
|
|
@@ -33,8 +31,10 @@ Please see full documentation <a href="https://squiss-ts.pruvo.com">here</a>
|
|
|
33
31
|
import {Squiss, Message} from 'squiss-ts';
|
|
34
32
|
|
|
35
33
|
const awsConfig = {
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
credentials: {
|
|
35
|
+
accessKeyId: 'accessKeyId',
|
|
36
|
+
secretAccessKey: 'secretAccessKey',
|
|
37
|
+
},
|
|
38
38
|
region: '<region>',
|
|
39
39
|
};
|
|
40
40
|
|
|
@@ -58,7 +58,7 @@ const messageToSend = {
|
|
|
58
58
|
a: 1,
|
|
59
59
|
b: 2,
|
|
60
60
|
},
|
|
61
|
-
}
|
|
61
|
+
};
|
|
62
62
|
|
|
63
63
|
const propsToSend = {
|
|
64
64
|
p1: 1,
|
|
@@ -4,9 +4,9 @@ interface TypeRecord<T, U, V> {
|
|
|
4
4
|
' _eventsType'?: U;
|
|
5
5
|
' _emitType'?: V;
|
|
6
6
|
}
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
type InnerEEMethodReturnType<T, TValue, FValue> = T extends (...args: any[]) => any ? ReturnType<T> extends void | undefined ? FValue : TValue : FValue;
|
|
8
|
+
type EEMethodReturnType<T, S extends string, TValue, FValue = void> = S extends keyof T ? InnerEEMethodReturnType<T[S], TValue, FValue> : FValue;
|
|
9
|
+
type ListenerType<T> = [T] extends [(...args: infer U) => any] ? U : [T] extends [void] ? [] : [T];
|
|
10
10
|
interface OverriddenMethods<TEmitter, TEventRecord, TEmitRecord = TEventRecord> {
|
|
11
11
|
on<P extends keyof TEventRecord, T>(this: T, event: P, listener: (...args: ListenerType<TEventRecord[P]>) => void): EEMethodReturnType<TEmitter, 'on', T>;
|
|
12
12
|
on(event: typeof assignmentCompatibilityHack, listener: (...args: any[]) => any): void;
|
|
@@ -23,6 +23,6 @@ interface OverriddenMethods<TEmitter, TEventRecord, TEmitRecord = TEventRecord>
|
|
|
23
23
|
emit<P extends keyof TEmitRecord, T>(this: T, event: P, ...args: ListenerType<TEmitRecord[P]>): EEMethodReturnType<TEmitter, 'emit', T>;
|
|
24
24
|
emit(event: typeof assignmentCompatibilityHack, ...args: any[]): void;
|
|
25
25
|
}
|
|
26
|
-
|
|
27
|
-
export
|
|
26
|
+
type OverriddenKeys = keyof OverriddenMethods<any, any, any>;
|
|
27
|
+
export type StrictEventEmitter<TEmitterType, TEventRecord, TEmitRecord = TEventRecord, UnneededMethods extends Exclude<OverriddenKeys, keyof TEmitterType> = Exclude<OverriddenKeys, keyof TEmitterType>, NeededMethods extends Exclude<OverriddenKeys, UnneededMethods> = Exclude<OverriddenKeys, UnneededMethods>> = TypeRecord<TEmitterType, TEventRecord, TEmitRecord> & Pick<TEmitterType, Exclude<keyof TEmitterType, OverriddenKeys>> & Pick<OverriddenMethods<TEmitterType, TEventRecord, TEmitRecord>, NeededMethods>;
|
|
28
28
|
export {};
|
package/dist/Message.d.ts
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import
|
|
3
|
-
import
|
|
4
|
-
import { AWSError } from 'aws-sdk/lib/error';
|
|
2
|
+
import { BatchResultErrorEntry, Message as SQSMessage, SQSServiceException } from '@aws-sdk/client-sqs';
|
|
3
|
+
import { S3 } from '@aws-sdk/client-s3';
|
|
5
4
|
import { BodyFormat, Squiss } from '.';
|
|
6
5
|
import { IMessageAttributes } from './attributeUtils';
|
|
7
6
|
import { EventEmitter } from 'events';
|
|
8
7
|
import { IS3Upload } from './s3Utils';
|
|
9
|
-
import { BatchResultErrorEntry } from 'aws-sdk/clients/sqs';
|
|
10
8
|
import { StrictEventEmitter } from './EventEmitterTypesHelper';
|
|
11
9
|
export interface IMessageOpts {
|
|
12
|
-
msg:
|
|
10
|
+
msg: SQSMessage;
|
|
13
11
|
unwrapSns?: boolean;
|
|
14
12
|
bodyFormat?: BodyFormat;
|
|
15
13
|
squiss: Squiss;
|
|
@@ -26,16 +24,16 @@ interface IMessageEvents {
|
|
|
26
24
|
keep: void;
|
|
27
25
|
delError: BatchResultErrorEntry;
|
|
28
26
|
deleted: string;
|
|
29
|
-
autoExtendFail:
|
|
30
|
-
autoExtendError:
|
|
27
|
+
autoExtendFail: SQSServiceException;
|
|
28
|
+
autoExtendError: SQSServiceException;
|
|
31
29
|
s3Download: IS3Upload;
|
|
32
30
|
s3Delete: IS3Upload;
|
|
33
31
|
}
|
|
34
|
-
|
|
32
|
+
type MessageEmitter = StrictEventEmitter<EventEmitter, IMessageEvents>;
|
|
35
33
|
declare const Message_base: new () => MessageEmitter;
|
|
36
34
|
export declare class Message extends Message_base {
|
|
37
35
|
private static formatMessage;
|
|
38
|
-
raw:
|
|
36
|
+
raw: SQSMessage;
|
|
39
37
|
body?: string | any;
|
|
40
38
|
subject?: string;
|
|
41
39
|
topicArn?: string;
|
package/dist/Message.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Message = void 0;
|
|
4
4
|
const attributeUtils_1 = require("./attributeUtils");
|
|
@@ -7,6 +7,14 @@ const gzipUtils_1 = require("./gzipUtils");
|
|
|
7
7
|
const s3Utils_1 = require("./s3Utils");
|
|
8
8
|
const EMPTY_BODY = '{}';
|
|
9
9
|
class Message extends events_1.EventEmitter {
|
|
10
|
+
static formatMessage(msg, format) {
|
|
11
|
+
if (format === 'json') {
|
|
12
|
+
return JSON.parse(msg || EMPTY_BODY);
|
|
13
|
+
}
|
|
14
|
+
else {
|
|
15
|
+
return msg;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
10
18
|
constructor(opts) {
|
|
11
19
|
super();
|
|
12
20
|
this._opts = opts;
|
|
@@ -28,14 +36,6 @@ class Message extends events_1.EventEmitter {
|
|
|
28
36
|
this._s3Retriever = opts.s3Retriever;
|
|
29
37
|
this._s3Retain = opts.s3Retain;
|
|
30
38
|
}
|
|
31
|
-
static formatMessage(msg, format) {
|
|
32
|
-
if (format === 'json') {
|
|
33
|
-
return JSON.parse(msg || EMPTY_BODY);
|
|
34
|
-
}
|
|
35
|
-
else {
|
|
36
|
-
return msg;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
39
|
parse() {
|
|
40
40
|
if (this.body === undefined || this.body === null) {
|
|
41
41
|
return Promise.resolve();
|
|
@@ -95,7 +95,7 @@ class Message extends events_1.EventEmitter {
|
|
|
95
95
|
if (!this._handled) {
|
|
96
96
|
this._handled = true;
|
|
97
97
|
return this._squiss.releaseMessage(this)
|
|
98
|
-
.catch((
|
|
98
|
+
.catch(() => {
|
|
99
99
|
this._handled = false;
|
|
100
100
|
});
|
|
101
101
|
}
|
package/dist/Squiss.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Message } from './Message';
|
|
2
2
|
import { TimeoutExtender } from './TimeoutExtender';
|
|
3
3
|
import { IMessageAttributes } from './attributeUtils';
|
|
4
|
-
import
|
|
5
|
-
import
|
|
4
|
+
import { SendMessageBatchResult, SQS, SendMessageCommandOutput } from '@aws-sdk/client-sqs';
|
|
5
|
+
import { S3 } from '@aws-sdk/client-s3';
|
|
6
6
|
import { IMessageToSend, SquissEmitter, ISquissOptions } from './Types';
|
|
7
7
|
declare const Squiss_base: new () => SquissEmitter;
|
|
8
8
|
export declare class Squiss extends Squiss_base {
|
|
@@ -32,8 +32,8 @@ export declare class Squiss extends Squiss_base {
|
|
|
32
32
|
handledMessage(msg: Message): void;
|
|
33
33
|
releaseMessage(msg: Message): Promise<void>;
|
|
34
34
|
purgeQueue(): Promise<void>;
|
|
35
|
-
sendMessage(message: IMessageToSend, delay?: number, attributes?: IMessageAttributes): Promise<
|
|
36
|
-
sendMessages(messages: IMessageToSend[] | IMessageToSend, delay?: number, attributes?: IMessageAttributes | IMessageAttributes[]): Promise<
|
|
35
|
+
sendMessage(message: IMessageToSend, delay?: number, attributes?: IMessageAttributes): Promise<SendMessageCommandOutput>;
|
|
36
|
+
sendMessages(messages: IMessageToSend[] | IMessageToSend, delay?: number, attributes?: IMessageAttributes | IMessageAttributes[]): Promise<SendMessageBatchResult>;
|
|
37
37
|
start(): Promise<void>;
|
|
38
38
|
stop(soft?: boolean, timeout?: number): Promise<boolean>;
|
|
39
39
|
getS3(): S3;
|
package/dist/Squiss.js
CHANGED
|
@@ -1,21 +1,27 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Squiss = void 0;
|
|
4
|
-
const url = require("url");
|
|
5
4
|
const events_1 = require("events");
|
|
6
5
|
const Message_1 = require("./Message");
|
|
7
6
|
const TimeoutExtender_1 = require("./TimeoutExtender");
|
|
8
7
|
const attributeUtils_1 = require("./attributeUtils");
|
|
9
8
|
const ts_type_guards_1 = require("ts-type-guards");
|
|
10
|
-
const
|
|
11
|
-
const
|
|
9
|
+
const client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
10
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
12
11
|
const gzipUtils_1 = require("./gzipUtils");
|
|
13
12
|
const s3Utils_1 = require("./s3Utils");
|
|
14
13
|
const messageSizeUtils_1 = require("./messageSizeUtils");
|
|
15
14
|
const Types_1 = require("./Types");
|
|
16
15
|
const Utils_1 = require("./Utils");
|
|
16
|
+
const url_1 = require("url");
|
|
17
17
|
const AWS_MAX_SEND_BATCH = 10;
|
|
18
18
|
class Squiss extends events_1.EventEmitter {
|
|
19
|
+
get inFlight() {
|
|
20
|
+
return this._inFlight;
|
|
21
|
+
}
|
|
22
|
+
get running() {
|
|
23
|
+
return this._running;
|
|
24
|
+
}
|
|
19
25
|
constructor(opts) {
|
|
20
26
|
super();
|
|
21
27
|
this._running = false;
|
|
@@ -29,12 +35,6 @@ class Squiss extends events_1.EventEmitter {
|
|
|
29
35
|
this._queueUrl = this._opts.queueUrl || '';
|
|
30
36
|
this.sqs = this._initSqs();
|
|
31
37
|
}
|
|
32
|
-
get inFlight() {
|
|
33
|
-
return this._inFlight;
|
|
34
|
-
}
|
|
35
|
-
get running() {
|
|
36
|
-
return this._running;
|
|
37
|
-
}
|
|
38
38
|
changeMessageVisibility(msg, timeoutInSeconds) {
|
|
39
39
|
let receiptHandle;
|
|
40
40
|
if (msg instanceof Message_1.Message) {
|
|
@@ -49,7 +49,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
49
49
|
QueueUrl: queueUrl,
|
|
50
50
|
ReceiptHandle: receiptHandle,
|
|
51
51
|
VisibilityTimeout: timeoutInSeconds,
|
|
52
|
-
})
|
|
52
|
+
});
|
|
53
53
|
})
|
|
54
54
|
.then(() => {
|
|
55
55
|
return Promise.resolve();
|
|
@@ -74,7 +74,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
74
74
|
if (this._opts.queuePolicy) {
|
|
75
75
|
params.Attributes.Policy = this._opts.queuePolicy;
|
|
76
76
|
}
|
|
77
|
-
return this.sqs.createQueue(params).
|
|
77
|
+
return this.sqs.createQueue(params).then((res) => {
|
|
78
78
|
this._queueUrl = res.QueueUrl;
|
|
79
79
|
return res.QueueUrl;
|
|
80
80
|
});
|
|
@@ -107,7 +107,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
107
107
|
deleteQueue() {
|
|
108
108
|
return this.getQueueUrl()
|
|
109
109
|
.then((queueUrl) => {
|
|
110
|
-
return this.sqs.deleteQueue({ QueueUrl: queueUrl })
|
|
110
|
+
return this.sqs.deleteQueue({ QueueUrl: queueUrl });
|
|
111
111
|
})
|
|
112
112
|
.then(() => {
|
|
113
113
|
return Promise.resolve();
|
|
@@ -121,13 +121,35 @@ class Squiss extends events_1.EventEmitter {
|
|
|
121
121
|
if (this._opts.accountNumber) {
|
|
122
122
|
params.QueueOwnerAWSAccountId = this._opts.accountNumber.toString();
|
|
123
123
|
}
|
|
124
|
-
return this.sqs.getQueueUrl(params).
|
|
124
|
+
return this.sqs.getQueueUrl(params).then(async (data) => {
|
|
125
125
|
this._queueUrl = data.QueueUrl;
|
|
126
126
|
if (this._opts.correctQueueUrl) {
|
|
127
|
-
|
|
128
|
-
const
|
|
127
|
+
let newUrl;
|
|
128
|
+
const endpoint = this.sqs.config.endpoint;
|
|
129
|
+
/* istanbul ignore else */
|
|
130
|
+
if (typeof endpoint === 'string') {
|
|
131
|
+
newUrl = new url_1.URL(endpoint);
|
|
132
|
+
}
|
|
133
|
+
else if (typeof endpoint === 'function') {
|
|
134
|
+
const retrievedEndpoint = await endpoint();
|
|
135
|
+
if (retrievedEndpoint && typeof retrievedEndpoint === 'object') {
|
|
136
|
+
if ('url' in retrievedEndpoint) {
|
|
137
|
+
newUrl = new url_1.URL(retrievedEndpoint.url.toString());
|
|
138
|
+
}
|
|
139
|
+
if ('hostname' in retrievedEndpoint) {
|
|
140
|
+
const { protocol, hostname, port, path } = retrievedEndpoint;
|
|
141
|
+
// query params are ignored in setting endpoint.
|
|
142
|
+
newUrl = new url_1.URL(`${protocol}//${hostname}${port ? ':' + port : ''}${path}`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/* istanbul ignore if */
|
|
147
|
+
if (!newUrl) {
|
|
148
|
+
throw new Error(`Failed to get configured SQQ endpoint`);
|
|
149
|
+
}
|
|
150
|
+
const parsedQueueUrl = new url_1.URL(this._queueUrl);
|
|
129
151
|
newUrl.pathname = parsedQueueUrl.pathname;
|
|
130
|
-
this._queueUrl =
|
|
152
|
+
this._queueUrl = newUrl.toString();
|
|
131
153
|
}
|
|
132
154
|
return this._queueUrl;
|
|
133
155
|
});
|
|
@@ -140,7 +162,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
140
162
|
return this.sqs.getQueueAttributes({
|
|
141
163
|
AttributeNames: ['VisibilityTimeout'],
|
|
142
164
|
QueueUrl: queueUrl,
|
|
143
|
-
})
|
|
165
|
+
});
|
|
144
166
|
}).then((res) => {
|
|
145
167
|
if (!res.Attributes || !res.Attributes.VisibilityTimeout) {
|
|
146
168
|
throw new Error('SQS.GetQueueAttributes call did not return expected shape. Response: ' +
|
|
@@ -158,7 +180,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
158
180
|
return this.sqs.getQueueAttributes({
|
|
159
181
|
AttributeNames: ['MaximumMessageSize'],
|
|
160
182
|
QueueUrl: queueUrl,
|
|
161
|
-
})
|
|
183
|
+
});
|
|
162
184
|
}).then((res) => {
|
|
163
185
|
if (!res.Attributes || !res.Attributes.MaximumMessageSize) {
|
|
164
186
|
throw new Error('SQS.GetQueueAttributes call did not return expected shape. Response: ' +
|
|
@@ -192,7 +214,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
192
214
|
purgeQueue() {
|
|
193
215
|
return this.getQueueUrl()
|
|
194
216
|
.then((queueUrl) => {
|
|
195
|
-
return this.sqs.purgeQueue({ QueueUrl: queueUrl })
|
|
217
|
+
return this.sqs.purgeQueue({ QueueUrl: queueUrl });
|
|
196
218
|
})
|
|
197
219
|
.then(() => {
|
|
198
220
|
this._inFlight = 0;
|
|
@@ -209,8 +231,11 @@ class Squiss extends events_1.EventEmitter {
|
|
|
209
231
|
.then((data) => {
|
|
210
232
|
const rawParams = data[0];
|
|
211
233
|
const queueUrl = data[1];
|
|
212
|
-
const params =
|
|
213
|
-
|
|
234
|
+
const params = {
|
|
235
|
+
QueueUrl: queueUrl,
|
|
236
|
+
...rawParams,
|
|
237
|
+
};
|
|
238
|
+
return this.sqs.sendMessage(params);
|
|
214
239
|
});
|
|
215
240
|
}
|
|
216
241
|
sendMessages(messages, delay, attributes) {
|
|
@@ -224,12 +249,13 @@ class Squiss extends events_1.EventEmitter {
|
|
|
224
249
|
}));
|
|
225
250
|
})
|
|
226
251
|
.then((results) => {
|
|
227
|
-
const
|
|
252
|
+
const successful = [];
|
|
253
|
+
const failed = [];
|
|
228
254
|
results.forEach((res) => {
|
|
229
|
-
res.Successful
|
|
230
|
-
res.Failed
|
|
255
|
+
res.Successful?.forEach((elem) => successful.push(elem));
|
|
256
|
+
res.Failed?.forEach((elem) => failed.push(elem));
|
|
231
257
|
});
|
|
232
|
-
return
|
|
258
|
+
return { Successful: successful, Failed: failed };
|
|
233
259
|
});
|
|
234
260
|
}
|
|
235
261
|
start() {
|
|
@@ -275,14 +301,15 @@ class Squiss extends events_1.EventEmitter {
|
|
|
275
301
|
_initS3() {
|
|
276
302
|
if (this._opts.S3) {
|
|
277
303
|
if (typeof this._opts.S3 === 'function') {
|
|
278
|
-
return new this._opts.S3(this._opts.awsConfig);
|
|
304
|
+
return new this._opts.S3(this._opts.awsConfig || {});
|
|
279
305
|
}
|
|
280
306
|
else {
|
|
281
307
|
return this._opts.S3;
|
|
282
308
|
}
|
|
283
309
|
}
|
|
284
310
|
else {
|
|
285
|
-
|
|
311
|
+
/* istanbul ignore next */
|
|
312
|
+
return new client_s3_1.S3(this._opts.awsConfig || {});
|
|
286
313
|
}
|
|
287
314
|
}
|
|
288
315
|
_initOpts() {
|
|
@@ -306,7 +333,7 @@ class Squiss extends events_1.EventEmitter {
|
|
|
306
333
|
ReceiptHandle: item.ReceiptHandle,
|
|
307
334
|
};
|
|
308
335
|
}),
|
|
309
|
-
})
|
|
336
|
+
});
|
|
310
337
|
}).then(this._handleBatchDeleteResults(batch))
|
|
311
338
|
.catch((err) => {
|
|
312
339
|
this.emit('error', err);
|
|
@@ -349,10 +376,13 @@ class Squiss extends events_1.EventEmitter {
|
|
|
349
376
|
AttributeNames: this._opts.receiveSqsAttributes,
|
|
350
377
|
VisibilityTimeout: this._opts.visibilityTimeoutSecs,
|
|
351
378
|
});
|
|
352
|
-
|
|
353
|
-
this._activeReq
|
|
379
|
+
const controller = new AbortController();
|
|
380
|
+
this._activeReq = controller;
|
|
381
|
+
this.sqs.receiveMessage(params, {
|
|
382
|
+
abortSignal: controller.signal,
|
|
383
|
+
}).then(this._handleGetBatchResult(queueUrl)).catch((err) => {
|
|
354
384
|
this._activeReq = undefined;
|
|
355
|
-
if (err.
|
|
385
|
+
if (err.name === 'AbortError') {
|
|
356
386
|
this.emit('aborted', err);
|
|
357
387
|
}
|
|
358
388
|
else {
|
|
@@ -384,18 +414,22 @@ class Squiss extends events_1.EventEmitter {
|
|
|
384
414
|
_sendMessageBatch(messages, delay, startIndex) {
|
|
385
415
|
const start = startIndex || 0;
|
|
386
416
|
return this.getQueueUrl().then((queueUrl) => {
|
|
417
|
+
const entries = [];
|
|
387
418
|
const params = {
|
|
388
419
|
QueueUrl: queueUrl,
|
|
389
|
-
Entries:
|
|
420
|
+
Entries: entries,
|
|
390
421
|
};
|
|
391
422
|
const promises = [];
|
|
392
423
|
messages.forEach((msg, idx) => {
|
|
393
|
-
const entry =
|
|
394
|
-
|
|
424
|
+
const entry = {
|
|
425
|
+
Id: (start + idx).toString(),
|
|
426
|
+
...msg,
|
|
427
|
+
};
|
|
428
|
+
entries.push(entry);
|
|
395
429
|
});
|
|
396
430
|
return Promise.all(promises)
|
|
397
431
|
.then(() => {
|
|
398
|
-
return this.sqs.sendMessageBatch(params)
|
|
432
|
+
return this.sqs.sendMessageBatch(params);
|
|
399
433
|
});
|
|
400
434
|
});
|
|
401
435
|
}
|
|
@@ -491,14 +525,14 @@ class Squiss extends events_1.EventEmitter {
|
|
|
491
525
|
_initSqs() {
|
|
492
526
|
if (this._opts.SQS) {
|
|
493
527
|
if (typeof this._opts.SQS === 'function') {
|
|
494
|
-
return new this._opts.SQS(this._opts.awsConfig);
|
|
528
|
+
return new this._opts.SQS(this._opts.awsConfig || {});
|
|
495
529
|
}
|
|
496
530
|
else {
|
|
497
531
|
return this._opts.SQS;
|
|
498
532
|
}
|
|
499
533
|
}
|
|
500
534
|
else {
|
|
501
|
-
return new SQS(this._opts.awsConfig);
|
|
535
|
+
return new client_sqs_1.SQS(this._opts.awsConfig || {});
|
|
502
536
|
}
|
|
503
537
|
}
|
|
504
538
|
_handleBatchDeleteResults(batch) {
|
|
@@ -507,19 +541,24 @@ class Squiss extends events_1.EventEmitter {
|
|
|
507
541
|
return prevByValue;
|
|
508
542
|
}, {});
|
|
509
543
|
return (data) => {
|
|
510
|
-
if (data.Failed
|
|
544
|
+
if (data.Failed?.length) {
|
|
511
545
|
data.Failed.forEach((fail) => {
|
|
512
|
-
|
|
513
|
-
itemById[fail.Id
|
|
514
|
-
|
|
546
|
+
/* istanbul ignore next */
|
|
547
|
+
const item = itemById[fail.Id ?? ''];
|
|
548
|
+
this.emit('delError', { error: fail, message: item.msg });
|
|
549
|
+
item.msg.emit('delError', fail);
|
|
550
|
+
item.reject(fail);
|
|
515
551
|
});
|
|
516
552
|
}
|
|
517
|
-
if (data.Successful
|
|
553
|
+
if (data.Successful?.length) {
|
|
518
554
|
data.Successful.forEach((success) => {
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
555
|
+
/* istanbul ignore next */
|
|
556
|
+
const id = success.Id ?? '';
|
|
557
|
+
const item = itemById[id];
|
|
558
|
+
const msg = item.msg;
|
|
559
|
+
this.emit('deleted', { msg, successId: id });
|
|
560
|
+
msg.emit('deleted', id);
|
|
561
|
+
item.resolve();
|
|
523
562
|
});
|
|
524
563
|
}
|
|
525
564
|
};
|
package/dist/TimeoutExtender.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TimeoutExtender = void 0;
|
|
4
4
|
const LinkedList = require("linked-list");
|
|
@@ -96,7 +96,8 @@ class TimeoutExtender {
|
|
|
96
96
|
this._squiss.emit('timeoutExtended', node.message);
|
|
97
97
|
})
|
|
98
98
|
.catch((err) => {
|
|
99
|
-
if (err.
|
|
99
|
+
if (err.name === 'ReceiptHandleIsInvalid' || err.name === 'MessageNotInflight' ||
|
|
100
|
+
err.message.match(/Message does not exist or is not available/)) {
|
|
100
101
|
this._deleteNode(node);
|
|
101
102
|
node.message.emit('autoExtendFail', err);
|
|
102
103
|
this._squiss.emit('autoExtendFail', { message: node.message, error: err });
|
package/dist/Types.d.ts
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { Message } from './Message';
|
|
3
|
-
import
|
|
4
|
-
import
|
|
5
|
-
import { AWSError } from 'aws-sdk/lib/error';
|
|
6
|
-
import { BatchResultErrorEntry } from 'aws-sdk/clients/sqs';
|
|
3
|
+
import { SQSClientConfig, BatchResultErrorEntry, SQS, SQSServiceException, MessageAttributeValue } from '@aws-sdk/client-sqs';
|
|
4
|
+
import { S3, S3ClientConfig } from '@aws-sdk/client-s3';
|
|
7
5
|
import { IS3Upload } from './s3Utils';
|
|
8
6
|
import { StrictEventEmitter } from './EventEmitterTypesHelper';
|
|
9
7
|
import { EventEmitter } from 'events';
|
|
@@ -13,7 +11,7 @@ export interface IMessageDeletedEventPayload {
|
|
|
13
11
|
}
|
|
14
12
|
export interface IMessageErrorEventPayload {
|
|
15
13
|
message: Message;
|
|
16
|
-
error:
|
|
14
|
+
error: SQSServiceException;
|
|
17
15
|
}
|
|
18
16
|
export interface IMessageDeleteErrorEventPayload {
|
|
19
17
|
message: Message;
|
|
@@ -23,18 +21,19 @@ export interface IMessageS3EventPayload {
|
|
|
23
21
|
message: Message;
|
|
24
22
|
data: IS3Upload;
|
|
25
23
|
}
|
|
24
|
+
export type MessageBodyAttributeMap = Record<string, MessageAttributeValue>;
|
|
26
25
|
export interface ISendMessageRequest {
|
|
27
26
|
MessageBody: string;
|
|
28
27
|
DelaySeconds?: number;
|
|
29
|
-
MessageAttributes?:
|
|
28
|
+
MessageAttributes?: MessageBodyAttributeMap;
|
|
30
29
|
MessageDeduplicationId?: string;
|
|
31
30
|
MessageGroupId?: string;
|
|
32
31
|
}
|
|
33
32
|
export interface IObject {
|
|
34
33
|
[k: string]: any;
|
|
35
34
|
}
|
|
36
|
-
export
|
|
37
|
-
export
|
|
35
|
+
export type IMessageToSend = IObject | string;
|
|
36
|
+
export type BodyFormat = 'json' | 'plain' | undefined;
|
|
38
37
|
export interface ISquissOptions {
|
|
39
38
|
receiveBatchSize?: number;
|
|
40
39
|
receiveAttributes?: string[];
|
|
@@ -58,7 +57,7 @@ export interface ISquissOptions {
|
|
|
58
57
|
autoExtendTimeout?: boolean;
|
|
59
58
|
SQS?: SQS | typeof SQS;
|
|
60
59
|
S3?: S3 | typeof S3;
|
|
61
|
-
awsConfig?:
|
|
60
|
+
awsConfig?: SQSClientConfig & S3ClientConfig;
|
|
62
61
|
queueUrl?: string;
|
|
63
62
|
queueName?: string;
|
|
64
63
|
visibilityTimeoutSecs?: number;
|
|
@@ -98,7 +97,7 @@ export interface ISquissEvents {
|
|
|
98
97
|
deleted: IMessageDeletedEventPayload;
|
|
99
98
|
gotMessages: number;
|
|
100
99
|
error: Error;
|
|
101
|
-
aborted:
|
|
100
|
+
aborted: SQSServiceException;
|
|
102
101
|
delError: IMessageDeleteErrorEventPayload;
|
|
103
102
|
autoExtendFail: IMessageErrorEventPayload;
|
|
104
103
|
autoExtendError: IMessageErrorEventPayload;
|
|
@@ -106,4 +105,4 @@ export interface ISquissEvents {
|
|
|
106
105
|
s3Delete: IMessageS3EventPayload;
|
|
107
106
|
s3Upload: IS3Upload;
|
|
108
107
|
}
|
|
109
|
-
export
|
|
108
|
+
export type SquissEmitter = StrictEventEmitter<EventEmitter, ISquissEvents>;
|
package/dist/Types.js
CHANGED
package/dist/Utils.js
CHANGED
package/dist/attributeUtils.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
export
|
|
1
|
+
import { MessageBodyAttributeMap } from './Types';
|
|
2
|
+
export type IMessageAttribute = number | string | Uint8Array | undefined | boolean;
|
|
3
3
|
export interface IMessageAttributes {
|
|
4
4
|
FIFO_MessageDeduplicationId?: string;
|
|
5
5
|
FIFO_MessageGroupId?: string;
|
|
6
6
|
[k: string]: IMessageAttribute;
|
|
7
7
|
}
|
|
8
|
-
export declare const parseMessageAttributes: (messageAttributes:
|
|
9
|
-
export declare const createMessageAttributes: (messageAttributes: IMessageAttributes) =>
|
|
8
|
+
export declare const parseMessageAttributes: (messageAttributes: MessageBodyAttributeMap | undefined) => IMessageAttributes;
|
|
9
|
+
export declare const createMessageAttributes: (messageAttributes: IMessageAttributes) => MessageBodyAttributeMap | undefined;
|
package/dist/attributeUtils.js
CHANGED
package/dist/gzipUtils.js
CHANGED
|
@@ -1,22 +1,10 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.decompressMessage = exports.compressMessage = exports.GZIP_MARKER = void 0;
|
|
4
|
-
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
/* istanbul ignore next */
|
|
9
|
-
if ((0, semver_1.gte)(process.version, '10.16.0')) {
|
|
10
|
-
const { brotliCompress, brotliDecompress } = require('zlib');
|
|
11
|
-
const { promisify } = require('util');
|
|
12
|
-
compress = promisify(brotliCompress);
|
|
13
|
-
decompress = promisify(brotliDecompress);
|
|
14
|
-
}
|
|
15
|
-
else {
|
|
16
|
-
const iltorb = require('iltorb');
|
|
17
|
-
compress = iltorb.compress;
|
|
18
|
-
decompress = iltorb.decompress;
|
|
19
|
-
}
|
|
4
|
+
const zlib_1 = require("zlib");
|
|
5
|
+
const util_1 = require("util");
|
|
6
|
+
const compress = (0, util_1.promisify)(zlib_1.brotliCompress);
|
|
7
|
+
const decompress = (0, util_1.promisify)(zlib_1.brotliDecompress);
|
|
20
8
|
exports.GZIP_MARKER = '__SQS_GZIP__';
|
|
21
9
|
const compressMessage = (message) => {
|
|
22
10
|
return compress(Buffer.from(message))
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { SQS } from '@aws-sdk/client-sqs';
|
|
2
2
|
export { SQS };
|
|
3
|
-
import
|
|
3
|
+
import { S3 } from '@aws-sdk/client-s3';
|
|
4
4
|
export { S3 };
|
|
5
5
|
export { Squiss } from './Squiss';
|
|
6
6
|
export { BodyFormat, IMessageDeletedEventPayload, IMessageDeleteErrorEventPayload, IMessageErrorEventPayload, IMessageS3EventPayload, IMessageToSend, IObject, ISendMessageRequest, ISquissOptions, } from './Types';
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Message = exports.Squiss = exports.S3 = exports.SQS = void 0;
|
|
4
4
|
/* istanbul ignore file */
|
|
5
|
-
const
|
|
6
|
-
exports
|
|
7
|
-
const
|
|
8
|
-
exports
|
|
5
|
+
const client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
6
|
+
Object.defineProperty(exports, "SQS", { enumerable: true, get: function () { return client_sqs_1.SQS; } });
|
|
7
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
8
|
+
Object.defineProperty(exports, "S3", { enumerable: true, get: function () { return client_s3_1.S3; } });
|
|
9
9
|
var Squiss_1 = require("./Squiss");
|
|
10
10
|
Object.defineProperty(exports, "Squiss", { enumerable: true, get: function () { return Squiss_1.Squiss; } });
|
|
11
11
|
var Message_1 = require("./Message");
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
+
/// <reference types="mocha" />
|
|
3
|
+
/// <reference types="node" />
|
|
4
|
+
/// <reference types="node" />
|
|
5
|
+
/// <reference types="node" />
|
|
6
|
+
/// <reference types="node" />
|
|
2
7
|
import { ISendMessageRequest } from './index';
|
|
3
8
|
export declare const getMessageSize: (message: ISendMessageRequest) => number;
|
|
4
|
-
export declare const getSizeInBytes: (obj?: string |
|
|
9
|
+
export declare const getSizeInBytes: (obj?: string | NodeJS.TypedArray | DataView | ArrayBuffer | SharedArrayBuffer) => number;
|
package/dist/messageSizeUtils.js
CHANGED
package/dist/s3Utils.d.ts
CHANGED
package/dist/s3Utils.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.deleteBlob = exports.getBlob = exports.uploadBlob = exports.S3_MARKER = void 0;
|
|
4
4
|
const uuid_1 = require("uuid");
|
|
@@ -12,7 +12,7 @@ const uploadBlob = (s3, bucket, blob, prefix) => {
|
|
|
12
12
|
Bucket: bucket,
|
|
13
13
|
Key: key,
|
|
14
14
|
ContentLength: size,
|
|
15
|
-
})
|
|
15
|
+
})
|
|
16
16
|
.then(() => {
|
|
17
17
|
return Promise.resolve({
|
|
18
18
|
uploadSize: size,
|
|
@@ -22,16 +22,19 @@ const uploadBlob = (s3, bucket, blob, prefix) => {
|
|
|
22
22
|
});
|
|
23
23
|
};
|
|
24
24
|
exports.uploadBlob = uploadBlob;
|
|
25
|
-
const getBlob = (s3, uploadData) => {
|
|
25
|
+
const getBlob = async (s3, uploadData) => {
|
|
26
26
|
const key = uploadData.key;
|
|
27
27
|
const bucket = uploadData.bucket;
|
|
28
|
-
|
|
28
|
+
const s3Result = await s3.getObject({
|
|
29
29
|
Key: key,
|
|
30
30
|
Bucket: bucket,
|
|
31
|
-
}).promise()
|
|
32
|
-
.then((s3Result) => {
|
|
33
|
-
return s3Result.Body.toString('utf-8');
|
|
34
31
|
});
|
|
32
|
+
const s3ResponseStream = s3Result.Body;
|
|
33
|
+
const chunks = [];
|
|
34
|
+
for await (const chunk of s3ResponseStream) {
|
|
35
|
+
chunks.push(chunk);
|
|
36
|
+
}
|
|
37
|
+
return Buffer.concat(chunks).toString('utf-8');
|
|
35
38
|
};
|
|
36
39
|
exports.getBlob = getBlob;
|
|
37
40
|
const deleteBlob = (s3, uploadData) => {
|
|
@@ -40,7 +43,7 @@ const deleteBlob = (s3, uploadData) => {
|
|
|
40
43
|
return s3.deleteObject({
|
|
41
44
|
Key: key,
|
|
42
45
|
Bucket: bucket,
|
|
43
|
-
})
|
|
46
|
+
})
|
|
44
47
|
.then(() => {
|
|
45
48
|
return Promise.resolve();
|
|
46
49
|
});
|
package/package.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "squiss-ts",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.1.0",
|
|
4
4
|
"description": "High-volume SQS poller",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"engines": {
|
|
8
|
-
"node": "^
|
|
8
|
+
"node": "^16 || ^18 || ^20"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
|
-
"
|
|
12
|
-
"clean": "rm -rf node_modules build coverage dist",
|
|
11
|
+
"clean": "rm -rf .nyc_output coverage dist",
|
|
13
12
|
"build": "yarn lint && yarn compile",
|
|
14
13
|
"compile": "tsc",
|
|
15
14
|
"test": "yarn lint && yarn mocha",
|
|
16
|
-
"
|
|
17
|
-
"cover": "nyc --reporter=lcov --reporter=text-summary npm run mocha",
|
|
15
|
+
"cover": "yarn clean && yarn nyc yarn mocha",
|
|
18
16
|
"lint": "tslint -c tslint.json \"src/**/*.ts\" \"test/**/*.ts\"",
|
|
19
|
-
"benchmark": "tsc --extendedDiagnostics --incremental false"
|
|
17
|
+
"benchmark": "tsc --extendedDiagnostics --incremental false",
|
|
18
|
+
"e2e:start": "docker compose -f e2e/docker-compose.yml up &",
|
|
19
|
+
"e2e:stop": "docker compose -f e2e/docker-compose.yml down &"
|
|
20
20
|
},
|
|
21
21
|
"repository": {
|
|
22
22
|
"type": "git",
|
|
@@ -48,34 +48,51 @@
|
|
|
48
48
|
},
|
|
49
49
|
"homepage": "https://github.com/PruvoNet/squiss-ts#readme",
|
|
50
50
|
"dependencies": {
|
|
51
|
-
"aws-sdk": "
|
|
51
|
+
"@aws-sdk/client-s3": "^3.312.0",
|
|
52
|
+
"@aws-sdk/client-sqs": "^3.312.0",
|
|
53
|
+
"@aws-sdk/types": "^3.310.0",
|
|
52
54
|
"linked-list": "^2.1.0",
|
|
53
|
-
"semver": "
|
|
55
|
+
"semver": "7.4.0",
|
|
54
56
|
"ts-type-guards": "^0.7.0",
|
|
55
|
-
"uuid": "^
|
|
57
|
+
"uuid": "^9.0.0"
|
|
56
58
|
},
|
|
57
59
|
"devDependencies": {
|
|
58
|
-
"@istanbuljs/nyc-config-typescript": "0.
|
|
59
|
-
"@types/chai": "^4.
|
|
60
|
-
"@types/chai-as-promised": "^7.1.
|
|
61
|
-
"@types/
|
|
62
|
-
"@types/
|
|
63
|
-
"@types/node": "^14.0.26",
|
|
60
|
+
"@istanbuljs/nyc-config-typescript": "1.0.2",
|
|
61
|
+
"@types/chai": "^4.3.4",
|
|
62
|
+
"@types/chai-as-promised": "^7.1.5",
|
|
63
|
+
"@types/mocha": "^10.0.1",
|
|
64
|
+
"@types/node": "^14.18.42",
|
|
64
65
|
"@types/proxyquire": "^1.3.28",
|
|
65
|
-
"@types/semver": "7.3.
|
|
66
|
-
"@types/uuid": "^
|
|
67
|
-
"chai": "^4.
|
|
66
|
+
"@types/semver": "^7.3.13",
|
|
67
|
+
"@types/uuid": "^9.0.1",
|
|
68
|
+
"chai": "^4.3.7",
|
|
68
69
|
"chai-as-promised": "^7.1.1",
|
|
69
|
-
"delay": "^
|
|
70
|
+
"delay": "^5.0.0",
|
|
70
71
|
"dirty-chai": "^2.0.1",
|
|
71
|
-
"mocha": "
|
|
72
|
-
"nyc": "
|
|
72
|
+
"mocha": "^10.2.0",
|
|
73
|
+
"nyc": "^15.1.0",
|
|
73
74
|
"proxyquire": "^2.1.3",
|
|
74
|
-
"sinon": "^
|
|
75
|
-
"sinon-chai": "^3.
|
|
76
|
-
"source-map-support": "^0.5.
|
|
77
|
-
"ts-node": "^
|
|
75
|
+
"sinon": "^15.0.3",
|
|
76
|
+
"sinon-chai": "^3.7.0",
|
|
77
|
+
"source-map-support": "^0.5.21",
|
|
78
|
+
"ts-node": "^10.9.1",
|
|
78
79
|
"tslint": "^6.1.3",
|
|
79
|
-
"typescript": "
|
|
80
|
+
"typescript": "^5.0.4"
|
|
81
|
+
},
|
|
82
|
+
"nyc": {
|
|
83
|
+
"extends": "@istanbuljs/nyc-config-typescript",
|
|
84
|
+
"check-coverage": true,
|
|
85
|
+
"all": true,
|
|
86
|
+
"exclude": [
|
|
87
|
+
"src/test/**",
|
|
88
|
+
"e2e**"
|
|
89
|
+
],
|
|
90
|
+
"reporter": [
|
|
91
|
+
"html",
|
|
92
|
+
"lcov",
|
|
93
|
+
"text",
|
|
94
|
+
"text-summary"
|
|
95
|
+
],
|
|
96
|
+
"report-dir": "coverage"
|
|
80
97
|
}
|
|
81
98
|
}
|