serverless-simple-middleware 0.0.52 → 0.0.53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.prettierignore +2 -2
  2. package/README.md +4 -4
  3. package/dist/aws/config.d.ts +15 -15
  4. package/dist/aws/config.js +29 -29
  5. package/dist/aws/define.d.ts +21 -21
  6. package/dist/aws/define.js +8 -8
  7. package/dist/aws/index.d.ts +3 -3
  8. package/dist/aws/index.js +8 -8
  9. package/dist/aws/simple.d.ts +44 -42
  10. package/dist/aws/simple.js +656 -590
  11. package/dist/index.d.ts +3 -3
  12. package/dist/index.js +8 -8
  13. package/dist/middleware/aws.d.ts +25 -25
  14. package/dist/middleware/aws.js +128 -128
  15. package/dist/middleware/base.d.ts +54 -54
  16. package/dist/middleware/base.js +140 -140
  17. package/dist/middleware/build.d.ts +3 -3
  18. package/dist/middleware/build.js +234 -234
  19. package/dist/middleware/index.d.ts +12 -12
  20. package/dist/middleware/index.js +22 -22
  21. package/dist/middleware/logger.d.ts +18 -18
  22. package/dist/middleware/logger.js +71 -71
  23. package/dist/middleware/mysql.d.ts +44 -44
  24. package/dist/middleware/mysql.js +289 -289
  25. package/dist/middleware/trace.d.ts +86 -86
  26. package/dist/middleware/trace.js +255 -255
  27. package/dist/utils/index.d.ts +2 -2
  28. package/dist/utils/index.js +7 -7
  29. package/dist/utils/logger.d.ts +26 -26
  30. package/dist/utils/logger.js +73 -73
  31. package/dist/utils/misc.d.ts +1 -1
  32. package/dist/utils/misc.js +9 -9
  33. package/jest.config.js +7 -7
  34. package/package.json +61 -60
  35. package/src/aws/config.ts +46 -46
  36. package/src/aws/define.ts +29 -29
  37. package/src/aws/index.ts +3 -3
  38. package/src/aws/simple.ts +531 -482
  39. package/src/index.ts +3 -3
  40. package/src/middleware/aws.ts +78 -78
  41. package/src/middleware/base.ts +164 -164
  42. package/src/middleware/build.ts +173 -173
  43. package/src/middleware/index.ts +20 -20
  44. package/src/middleware/logger.ts +28 -28
  45. package/src/middleware/mysql.ts +210 -210
  46. package/src/middleware/trace.ts +269 -269
  47. package/src/utils/index.ts +2 -2
  48. package/src/utils/logger.ts +94 -94
  49. package/src/utils/misc.ts +11 -11
  50. package/tsconfig.json +15 -15
  51. package/tslint.json +12 -12
package/src/aws/simple.ts CHANGED
@@ -1,482 +1,531 @@
1
- import * as AWS from 'aws-sdk'; // tslint:disable-line
2
- import * as fs from 'fs';
3
-
4
- import { getLogger } from '../utils';
5
- import { SimpleAWSConfig } from './config';
6
-
7
- import {
8
- AWSComponent,
9
- S3SignedUrlParams,
10
- S3SignedUrlResult,
11
- SQSMessageBody,
12
- } from './define';
13
-
14
- const logger = getLogger(__filename);
15
-
16
- export class SimpleAWS {
17
- private queueUrls: { [queueName: string]: string };
18
- private config: SimpleAWSConfig;
19
- private lazyS3: AWS.S3 | undefined;
20
- private lazySqs: AWS.SQS | undefined;
21
- private lazyDynamodb: AWS.DynamoDB.DocumentClient | undefined;
22
- private lazyDynamodbAdmin: AWS.DynamoDB | undefined;
23
-
24
- constructor(config?: SimpleAWSConfig) {
25
- this.config = config || new SimpleAWSConfig();
26
- /**
27
- * The simple cache for { queueName: queueUrl }.
28
- * It can help in the only case of launching this project as offline.
29
- * @type { { [queueName: string]: string } }
30
- */
31
- this.queueUrls = {};
32
- }
33
-
34
- get s3() {
35
- if (this.lazyS3 === undefined) {
36
- this.lazyS3 = new AWS.S3(this.config.get(AWSComponent.s3));
37
- }
38
- return this.lazyS3;
39
- }
40
- get sqs() {
41
- if (this.lazySqs === undefined) {
42
- this.lazySqs = new AWS.SQS(this.config.get(AWSComponent.sqs));
43
- }
44
- return this.lazySqs;
45
- }
46
- get dynamodb() {
47
- if (this.lazyDynamodb === undefined) {
48
- this.lazyDynamodb = new AWS.DynamoDB.DocumentClient(
49
- this.config.get(AWSComponent.dynamodb),
50
- );
51
- }
52
- return this.lazyDynamodb;
53
- }
54
- get dynamodbAdmin() {
55
- if (this.lazyDynamodbAdmin === undefined) {
56
- this.lazyDynamodbAdmin = new AWS.DynamoDB(
57
- this.config.get(AWSComponent.dynamodb),
58
- );
59
- }
60
- return this.lazyDynamodbAdmin;
61
- }
62
-
63
- public getQueueUrl = async (queueName: string): Promise<string> => {
64
- if (this.queueUrls[queueName] !== undefined) {
65
- return this.queueUrls[queueName];
66
- }
67
- const urlResult = await this.sqs
68
- .getQueueUrl({
69
- QueueName: queueName,
70
- })
71
- .promise();
72
- logger.stupid(`urlResult`, urlResult);
73
- if (!urlResult.QueueUrl) {
74
- throw new Error(`No queue url with name[${queueName}]`);
75
- }
76
- return (this.queueUrls[queueName] = urlResult.QueueUrl);
77
- };
78
-
79
- public enqueue = async (queueName: string, data: any): Promise<number> => {
80
- logger.debug(`Send message[${data.key}] to queue.`);
81
- logger.stupid(`data`, data);
82
- const queueUrl = await this.getQueueUrl(queueName);
83
- const sendResult = await this.sqs
84
- .sendMessage({
85
- QueueUrl: queueUrl,
86
- MessageBody: JSON.stringify(data),
87
- DelaySeconds: 0,
88
- })
89
- .promise();
90
- logger.stupid(`sendResult`, sendResult);
91
-
92
- const attrResult = await this.sqs
93
- .getQueueAttributes({
94
- QueueUrl: queueUrl,
95
- AttributeNames: ['ApproximateNumberOfMessages'],
96
- })
97
- .promise();
98
- logger.stupid(`attrResult`, attrResult);
99
- if (!attrResult.Attributes) {
100
- return 0;
101
- }
102
- return +attrResult.Attributes.ApproximateNumberOfMessages;
103
- };
104
-
105
- public dequeue = async <T>(
106
- queueName: string,
107
- fetchSize: number = 1,
108
- waitSeconds: number = 1,
109
- visibilityTimeout: number = 15,
110
- ): Promise<Array<SQSMessageBody<T>>> => {
111
- logger.debug(`Receive message from queue[${queueName}].`);
112
- const queueUrl = await this.getQueueUrl(queueName);
113
- const receiveResult = await this.sqs
114
- .receiveMessage({
115
- QueueUrl: queueUrl,
116
- MaxNumberOfMessages: fetchSize,
117
- WaitTimeSeconds: waitSeconds,
118
- VisibilityTimeout: visibilityTimeout,
119
- })
120
- .promise();
121
- logger.stupid(`receiveResult`, receiveResult);
122
- if (
123
- receiveResult.Messages === undefined ||
124
- receiveResult.Messages.length === 0
125
- ) {
126
- return [];
127
- }
128
- const data = [];
129
- for (const each of receiveResult.Messages) {
130
- if (!each.ReceiptHandle) {
131
- logger.warn(`No receipt handler: ${JSON.stringify(each)}`);
132
- continue;
133
- }
134
- const message: SQSMessageBody<T> = {
135
- handle: each.ReceiptHandle,
136
- body: each.Body ? (JSON.parse(each.Body) as T) : undefined,
137
- };
138
- data.push(message);
139
- }
140
- logger.verbose(`Receive a message[${JSON.stringify(data)}] from queue`);
141
- return data;
142
- };
143
-
144
- public dequeueAll = async <T>(
145
- queueName: string,
146
- limitSize: number = Number.MAX_VALUE,
147
- visibilityTimeout: number = 15,
148
- ): Promise<Array<SQSMessageBody<T>>> => {
149
- const messages = [];
150
- const maxFetchSize = 10; // This is max-value for fetching in each time.
151
- while (messages.length < limitSize) {
152
- const eachOfMessages: Array<SQSMessageBody<T>> = await this.dequeue<T>(
153
- queueName,
154
- Math.min(limitSize - messages.length, maxFetchSize),
155
- 0,
156
- visibilityTimeout,
157
- );
158
- if (!eachOfMessages || eachOfMessages.length === 0) {
159
- break;
160
- }
161
- for (const each of eachOfMessages) {
162
- messages.push(each);
163
- }
164
- }
165
- logger.stupid(`messages`, messages);
166
- return messages;
167
- };
168
-
169
- public retainMessage = async (
170
- queueName: string,
171
- handle: string,
172
- seconds: number,
173
- ): Promise<string> =>
174
- new Promise<string>(async (resolve, reject) => {
175
- logger.debug(`Change visibilityTimeout of ${handle} to ${seconds}secs.`);
176
- this.getQueueUrl(queueName)
177
- .then(queueUrl => {
178
- this.sqs.changeMessageVisibility(
179
- {
180
- QueueUrl: queueUrl,
181
- ReceiptHandle: handle,
182
- VisibilityTimeout: seconds,
183
- },
184
- (err, changeResult) => {
185
- if (err) {
186
- reject(err);
187
- } else {
188
- logger.stupid(`changeResult`, changeResult);
189
- resolve(handle);
190
- }
191
- },
192
- );
193
- })
194
- .catch(reject);
195
- });
196
-
197
- public completeMessage = async (
198
- queueName: string,
199
- handle: string,
200
- ): Promise<string> => {
201
- logger.debug(`Complete a message with handle[${handle}]`);
202
- const queueUrl = await this.getQueueUrl(queueName);
203
- const deleteResult = await this.sqs
204
- .deleteMessage({
205
- QueueUrl: queueUrl,
206
- ReceiptHandle: handle,
207
- })
208
- .promise();
209
- logger.stupid(`deleteResult`, deleteResult);
210
- return handle;
211
- };
212
-
213
- public completeMessages = async (queueName: string, handles: string[]) => {
214
- logger.debug(`Complete a message with handle[${handles}]`);
215
- if (!handles) {
216
- return handles;
217
- }
218
-
219
- const chunkSize = 10;
220
- let index = 0;
221
- for (let start = 0; start < handles.length; start += chunkSize) {
222
- const end = Math.min(start + chunkSize, handles.length);
223
- const sublist = handles.slice(start, end);
224
- const queueUrl = await this.getQueueUrl(queueName);
225
- const deletesResult = await this.sqs
226
- .deleteMessageBatch({
227
- QueueUrl: queueUrl,
228
- Entries: sublist.map(handle => ({
229
- Id: (++index).toString(),
230
- ReceiptHandle: handle,
231
- })),
232
- })
233
- .promise();
234
- logger.stupid(`deleteResult`, deletesResult);
235
- }
236
- return handles;
237
- };
238
-
239
- public download = async (
240
- bucketName: string,
241
- key: string,
242
- localPath: string,
243
- ): Promise<string> => {
244
- logger.debug(`Get a stream of item[${key}] from bucket[${bucketName}]`);
245
- const stream = this.s3
246
- .getObject({
247
- Bucket: bucketName,
248
- Key: key,
249
- })
250
- .createReadStream();
251
- return new Promise<string>((resolve, reject) =>
252
- stream
253
- .on('error', error => reject(error))
254
- .pipe(fs.createWriteStream(localPath))
255
- .on('finish', () => resolve(localPath))
256
- .on('error', error => reject(error)),
257
- );
258
- };
259
-
260
- public upload = async (
261
- bucketName: string,
262
- localPath: string,
263
- key: string,
264
- ): Promise<string> => {
265
- logger.debug(`Upload item[${key}] into bucket[${bucketName}]`);
266
- const putResult = await this.s3
267
- .upload({
268
- Bucket: bucketName,
269
- Key: key,
270
- Body: fs.createReadStream(localPath),
271
- })
272
- .promise();
273
- logger.stupid(`putResult`, putResult);
274
- return key;
275
- };
276
-
277
- public getSignedUrl = (
278
- bucketName: string,
279
- key: string,
280
- operation: 'getObject' | 'putObject' = 'getObject',
281
- params?: S3SignedUrlParams,
282
- ): S3SignedUrlResult => {
283
- return {
284
- key,
285
- url: this.s3.getSignedUrl(operation, {
286
- Bucket: bucketName,
287
- Key: key,
288
- Expires: 60 * 10,
289
- ...(params || {}),
290
- }),
291
- };
292
- };
293
-
294
- public getSignedCookie = (
295
- keyPairId: string,
296
- privateKey: string,
297
- url: string,
298
- expires: number,
299
- ): AWS.CloudFront.Signer.CustomPolicy => {
300
- const signer = new AWS.CloudFront.Signer(keyPairId, privateKey);
301
- const policy = {
302
- Statement: [
303
- {
304
- Resource: url,
305
- Condition: {
306
- DateLessThan: { 'AWS:EpochTime': expires },
307
- },
308
- },
309
- ],
310
- };
311
- const ret = signer.getSignedCookie({ policy: JSON.stringify(policy) });
312
- return ret;
313
- };
314
-
315
- public getAttachmentUrl = (
316
- bucketName: string,
317
- key: string,
318
- fileName: string,
319
- params?: S3SignedUrlParams,
320
- ): S3SignedUrlResult => {
321
- return this.getSignedUrl(bucketName, key, 'getObject', {
322
- ...params,
323
- ResponseContentDisposition: `attachment; filename="${fileName}"`,
324
- });
325
- };
326
-
327
- public getDynamoDbItem = async <T>(
328
- tableName: string,
329
- key: { [keyColumn: string]: string },
330
- defaultValue?: T,
331
- ): Promise<T | undefined> => {
332
- logger.debug(
333
- `Read an item with key[${JSON.stringify(key)}] from ${tableName}.`,
334
- );
335
- const getResult = await this.dynamodb
336
- .get({
337
- TableName: tableName,
338
- Key: key,
339
- })
340
- .promise();
341
- logger.stupid(`getResult`, getResult);
342
- const item: T | undefined =
343
- getResult !== undefined && getResult.Item !== undefined
344
- ? ((getResult.Item as any) as T) // Casts forcefully.
345
- : defaultValue;
346
- logger.stupid(`item`, item);
347
- return item;
348
- };
349
-
350
- public updateDynamoDbItem = async (
351
- tableName: string,
352
- key: { [keyColumn: string]: string },
353
- columnValues: { [column: string]: any },
354
- ) => {
355
- logger.debug(
356
- `Update an item with key[${JSON.stringify(key)}] to ${tableName}`,
357
- );
358
- logger.stupid(`keyValues`, columnValues);
359
- const expressions = Object.keys(columnValues)
360
- .map(column => `${column} = :${column}`)
361
- .join(', ');
362
- const attributeValues = Object.keys(columnValues)
363
- .map(column => [`:${column}`, columnValues[column]])
364
- .reduce((obj, pair) => ({ ...obj, [pair[0]]: pair[1] }), {});
365
- logger.stupid(`expressions`, expressions);
366
- logger.stupid(`attributeValues`, attributeValues);
367
- const updateResult = await this.dynamodb
368
- .update({
369
- TableName: tableName,
370
- Key: key,
371
- UpdateExpression: `set ${expressions}`,
372
- ExpressionAttributeValues: attributeValues,
373
- })
374
- .promise();
375
- logger.stupid(`updateResult`, updateResult);
376
- return updateResult;
377
- };
378
-
379
- // Setup
380
-
381
- public setupQueue = async (queueName: string) => {
382
- try {
383
- const listResult = await this.sqs
384
- .listQueues({
385
- QueueNamePrefix: queueName,
386
- })
387
- .promise();
388
- if (listResult.QueueUrls) {
389
- for (const queueUrl of listResult.QueueUrls) {
390
- if (queueUrl.endsWith(queueName)) {
391
- logger.debug(`Queue[${queueName} => ${queueUrl}] already exists.`);
392
- return true;
393
- }
394
- }
395
- }
396
- } catch (error) {
397
- logger.debug(`No Queue[${queueName}] exists due to ${error}`);
398
- }
399
- logger.debug(`Create a queue[${queueName}] newly.`);
400
- const createResult = await this.sqs
401
- .createQueue({
402
- QueueName: queueName,
403
- })
404
- .promise();
405
- logger.stupid(`createResult`, createResult);
406
- return true;
407
- };
408
-
409
- public setupStorage = async (
410
- bucketName: string,
411
- cors: {
412
- methods: Array<'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD'>;
413
- origins: string[];
414
- },
415
- ) => {
416
- try {
417
- const listResult = await this.s3.listBuckets().promise();
418
- if (
419
- listResult.Buckets &&
420
- listResult.Buckets.map(each => each.Name).includes(bucketName)
421
- ) {
422
- logger.debug(`Bucket[${bucketName}] already exists.`);
423
- return true;
424
- }
425
- } catch (error) {
426
- logger.debug(`No bucket[${bucketName}] exists due to ${error}`);
427
- }
428
- logger.debug(`Create a bucket[${bucketName}] newly.`);
429
- const createResult = await this.s3
430
- .createBucket({
431
- Bucket: bucketName,
432
- })
433
- .promise();
434
- logger.stupid(`createResult`, createResult);
435
- if (cors) {
436
- const corsResult = await this.s3
437
- .putBucketCors({
438
- Bucket: bucketName,
439
- CORSConfiguration: {
440
- CORSRules: [
441
- {
442
- AllowedHeaders: ['*'],
443
- AllowedMethods: cors.methods,
444
- AllowedOrigins: cors.origins,
445
- },
446
- ],
447
- },
448
- })
449
- .promise();
450
- logger.stupid(`corsResult`, corsResult);
451
- }
452
- return true;
453
- };
454
-
455
- public setupDynamoDb = async (tableName: string, keyColumn: string) => {
456
- try {
457
- const listResult = await this.dynamodbAdmin.listTables().promise();
458
- if (listResult.TableNames && listResult.TableNames.includes(tableName)) {
459
- logger.debug(`Table[${tableName}] already exists.`);
460
- return true;
461
- }
462
- } catch (error) {
463
- logger.debug(`No table[${tableName}] exists due to ${error}`);
464
- }
465
- logger.debug(`Create a table[${tableName}] newly.`);
466
- const createResult = await this.dynamodbAdmin
467
- .createTable({
468
- TableName: tableName,
469
- KeySchema: [{ AttributeName: keyColumn, KeyType: 'HASH' }],
470
- AttributeDefinitions: [
471
- { AttributeName: keyColumn, AttributeType: 'S' },
472
- ],
473
- ProvisionedThroughput: {
474
- ReadCapacityUnits: 30,
475
- WriteCapacityUnits: 10,
476
- },
477
- })
478
- .promise();
479
- logger.stupid(`createResult`, createResult);
480
- return true;
481
- };
482
- }
1
+ import * as AWS from 'aws-sdk'; // tslint:disable-line
2
+ import * as fs from 'fs';
3
+ import * as os from 'os';
4
+ import { nanoid } from 'nanoid/non-secure';
5
+
6
+ import { getLogger, stringifyError } from '../utils';
7
+ import { SimpleAWSConfig } from './config';
8
+
9
+ import {
10
+ AWSComponent,
11
+ S3SignedUrlParams,
12
+ S3SignedUrlResult,
13
+ SQSMessageBody,
14
+ } from './define';
15
+
16
+ const logger = getLogger(__filename);
17
+
18
+ export class SimpleAWS {
19
+ private queueUrls: { [queueName: string]: string };
20
+ private config: SimpleAWSConfig;
21
+ private lazyS3: AWS.S3 | undefined;
22
+ private lazySqs: AWS.SQS | undefined;
23
+ private lazyDynamodb: AWS.DynamoDB.DocumentClient | undefined;
24
+ private lazyDynamodbAdmin: AWS.DynamoDB | undefined;
25
+
26
+ constructor(config?: SimpleAWSConfig) {
27
+ this.config = config || new SimpleAWSConfig();
28
+ /**
29
+ * The simple cache for { queueName: queueUrl }.
30
+ * It can help in the only case of launching this project as offline.
31
+ * @type { { [queueName: string]: string } }
32
+ */
33
+ this.queueUrls = {};
34
+ }
35
+
36
+ get s3() {
37
+ if (this.lazyS3 === undefined) {
38
+ this.lazyS3 = new AWS.S3(this.config.get(AWSComponent.s3));
39
+ }
40
+ return this.lazyS3;
41
+ }
42
+ get sqs() {
43
+ if (this.lazySqs === undefined) {
44
+ this.lazySqs = new AWS.SQS(this.config.get(AWSComponent.sqs));
45
+ }
46
+ return this.lazySqs;
47
+ }
48
+ get dynamodb() {
49
+ if (this.lazyDynamodb === undefined) {
50
+ this.lazyDynamodb = new AWS.DynamoDB.DocumentClient(
51
+ this.config.get(AWSComponent.dynamodb),
52
+ );
53
+ }
54
+ return this.lazyDynamodb;
55
+ }
56
+ get dynamodbAdmin() {
57
+ if (this.lazyDynamodbAdmin === undefined) {
58
+ this.lazyDynamodbAdmin = new AWS.DynamoDB(
59
+ this.config.get(AWSComponent.dynamodb),
60
+ );
61
+ }
62
+ return this.lazyDynamodbAdmin;
63
+ }
64
+
65
+ public getQueueUrl = async (queueName: string): Promise<string> => {
66
+ if (this.queueUrls[queueName] !== undefined) {
67
+ return this.queueUrls[queueName];
68
+ }
69
+ const urlResult = await this.sqs
70
+ .getQueueUrl({
71
+ QueueName: queueName,
72
+ })
73
+ .promise();
74
+ logger.stupid(`urlResult`, urlResult);
75
+ if (!urlResult.QueueUrl) {
76
+ throw new Error(`No queue url with name[${queueName}]`);
77
+ }
78
+ return (this.queueUrls[queueName] = urlResult.QueueUrl);
79
+ };
80
+
81
+ public enqueue = async (queueName: string, data: any): Promise<number> => {
82
+ logger.debug(`Send message[${data.key}] to queue.`);
83
+ logger.stupid(`data`, data);
84
+ const queueUrl = await this.getQueueUrl(queueName);
85
+ const sendResult = await this.sqs
86
+ .sendMessage({
87
+ QueueUrl: queueUrl,
88
+ MessageBody: JSON.stringify(data),
89
+ DelaySeconds: 0,
90
+ })
91
+ .promise();
92
+ logger.stupid(`sendResult`, sendResult);
93
+
94
+ const attrResult = await this.sqs
95
+ .getQueueAttributes({
96
+ QueueUrl: queueUrl,
97
+ AttributeNames: ['ApproximateNumberOfMessages'],
98
+ })
99
+ .promise();
100
+ logger.stupid(`attrResult`, attrResult);
101
+ if (!attrResult.Attributes) {
102
+ return 0;
103
+ }
104
+ return +attrResult.Attributes.ApproximateNumberOfMessages;
105
+ };
106
+
107
+ public dequeue = async <T>(
108
+ queueName: string,
109
+ fetchSize: number = 1,
110
+ waitSeconds: number = 1,
111
+ visibilityTimeout: number = 15,
112
+ ): Promise<Array<SQSMessageBody<T>>> => {
113
+ logger.debug(`Receive message from queue[${queueName}].`);
114
+ const queueUrl = await this.getQueueUrl(queueName);
115
+ const receiveResult = await this.sqs
116
+ .receiveMessage({
117
+ QueueUrl: queueUrl,
118
+ MaxNumberOfMessages: fetchSize,
119
+ WaitTimeSeconds: waitSeconds,
120
+ VisibilityTimeout: visibilityTimeout,
121
+ })
122
+ .promise();
123
+ logger.stupid(`receiveResult`, receiveResult);
124
+ if (
125
+ receiveResult.Messages === undefined ||
126
+ receiveResult.Messages.length === 0
127
+ ) {
128
+ return [];
129
+ }
130
+ const data = [];
131
+ for (const each of receiveResult.Messages) {
132
+ if (!each.ReceiptHandle) {
133
+ logger.warn(`No receipt handler: ${JSON.stringify(each)}`);
134
+ continue;
135
+ }
136
+ const message: SQSMessageBody<T> = {
137
+ handle: each.ReceiptHandle,
138
+ body: each.Body ? (JSON.parse(each.Body) as T) : undefined,
139
+ };
140
+ data.push(message);
141
+ }
142
+ logger.verbose(`Receive a message[${JSON.stringify(data)}] from queue`);
143
+ return data;
144
+ };
145
+
146
+ public dequeueAll = async <T>(
147
+ queueName: string,
148
+ limitSize: number = Number.MAX_VALUE,
149
+ visibilityTimeout: number = 15,
150
+ ): Promise<Array<SQSMessageBody<T>>> => {
151
+ const messages = [];
152
+ const maxFetchSize = 10; // This is max-value for fetching in each time.
153
+ while (messages.length < limitSize) {
154
+ const eachOfMessages: Array<SQSMessageBody<T>> = await this.dequeue<T>(
155
+ queueName,
156
+ Math.min(limitSize - messages.length, maxFetchSize),
157
+ 0,
158
+ visibilityTimeout,
159
+ );
160
+ if (!eachOfMessages || eachOfMessages.length === 0) {
161
+ break;
162
+ }
163
+ for (const each of eachOfMessages) {
164
+ messages.push(each);
165
+ }
166
+ }
167
+ logger.stupid(`messages`, messages);
168
+ return messages;
169
+ };
170
+
171
+ public retainMessage = async (
172
+ queueName: string,
173
+ handle: string,
174
+ seconds: number,
175
+ ): Promise<string> =>
176
+ new Promise<string>(async (resolve, reject) => {
177
+ logger.debug(`Change visibilityTimeout of ${handle} to ${seconds}secs.`);
178
+ this.getQueueUrl(queueName)
179
+ .then(queueUrl => {
180
+ this.sqs.changeMessageVisibility(
181
+ {
182
+ QueueUrl: queueUrl,
183
+ ReceiptHandle: handle,
184
+ VisibilityTimeout: seconds,
185
+ },
186
+ (err, changeResult) => {
187
+ if (err) {
188
+ reject(err);
189
+ } else {
190
+ logger.stupid(`changeResult`, changeResult);
191
+ resolve(handle);
192
+ }
193
+ },
194
+ );
195
+ })
196
+ .catch(reject);
197
+ });
198
+
199
+ public completeMessage = async (
200
+ queueName: string,
201
+ handle: string,
202
+ ): Promise<string> => {
203
+ logger.debug(`Complete a message with handle[${handle}]`);
204
+ const queueUrl = await this.getQueueUrl(queueName);
205
+ const deleteResult = await this.sqs
206
+ .deleteMessage({
207
+ QueueUrl: queueUrl,
208
+ ReceiptHandle: handle,
209
+ })
210
+ .promise();
211
+ logger.stupid(`deleteResult`, deleteResult);
212
+ return handle;
213
+ };
214
+
215
+ public completeMessages = async (queueName: string, handles: string[]) => {
216
+ logger.debug(`Complete a message with handle[${handles}]`);
217
+ if (!handles) {
218
+ return handles;
219
+ }
220
+
221
+ const chunkSize = 10;
222
+ let index = 0;
223
+ for (let start = 0; start < handles.length; start += chunkSize) {
224
+ const end = Math.min(start + chunkSize, handles.length);
225
+ const sublist = handles.slice(start, end);
226
+ const queueUrl = await this.getQueueUrl(queueName);
227
+ const deletesResult = await this.sqs
228
+ .deleteMessageBatch({
229
+ QueueUrl: queueUrl,
230
+ Entries: sublist.map(handle => ({
231
+ Id: (++index).toString(),
232
+ ReceiptHandle: handle,
233
+ })),
234
+ })
235
+ .promise();
236
+ logger.stupid(`deleteResult`, deletesResult);
237
+ }
238
+ return handles;
239
+ };
240
+
241
+ public download = async (
242
+ bucket: string,
243
+ key: string,
244
+ localPath: string,
245
+ ): Promise<string> => {
246
+ logger.debug(`Get a stream of item[${key}] from bucket[${bucket}]`);
247
+ const stream = this.s3
248
+ .getObject({ Bucket: bucket, Key: key })
249
+ .createReadStream();
250
+ return new Promise<string>((resolve, reject) =>
251
+ stream
252
+ .on('error', error => reject(error))
253
+ .pipe(fs.createWriteStream(localPath))
254
+ .on('finish', () => resolve(localPath))
255
+ .on('error', error => reject(error)),
256
+ );
257
+ };
258
+
259
+ public readFile = async (bucket: string, key: string): Promise<string> => {
260
+ logger.debug(`Read item[${key}] from bucket[${bucket}]`);
261
+ const tempFile = `${os.tmpdir()}/${nanoid()}`;
262
+ try {
263
+ await this.download(bucket, key, tempFile);
264
+ const content = await fs.promises.readFile(tempFile, {
265
+ encoding: 'utf-8',
266
+ });
267
+ return content;
268
+ } finally {
269
+ if (!fs.existsSync(tempFile)) {
270
+ fs.unlink(tempFile, error => {
271
+ if (!error) {
272
+ return;
273
+ }
274
+ const msg = `Error during readFile: unlink ${tempFile}: ${stringifyError(
275
+ error,
276
+ )}`;
277
+ logger.error(msg);
278
+ });
279
+ }
280
+ }
281
+ };
282
+
283
+ public upload = async (
284
+ bucket: string,
285
+ localPath: string,
286
+ key: string,
287
+ ): Promise<string> => {
288
+ logger.debug(`Upload item[${key}] into bucket[${bucket}]`);
289
+ const putResult = await this.s3
290
+ .upload({
291
+ Bucket: bucket,
292
+ Key: key,
293
+ Body: fs.createReadStream(localPath),
294
+ })
295
+ .promise();
296
+ logger.stupid(`putResult`, putResult);
297
+ return key;
298
+ };
299
+
300
+ public writeFile = async (
301
+ bucket: string,
302
+ key: string,
303
+ content: string,
304
+ ): Promise<void> => {
305
+ logger.debug(`Write item[${key}] into bucket[${bucket}]`);
306
+ const tempFile = `${os.tmpdir()}/${nanoid()}`;
307
+ try {
308
+ await fs.promises.writeFile(tempFile, content, 'utf-8');
309
+ await this.upload(bucket, tempFile, key);
310
+ } finally {
311
+ if (!fs.existsSync(tempFile)) {
312
+ return;
313
+ }
314
+ fs.unlink(tempFile, error => {
315
+ if (!error) {
316
+ return;
317
+ }
318
+ const msg = `Error during writeFile: unlink file ${tempFile}: ${stringifyError(
319
+ error,
320
+ )}`;
321
+ logger.error(msg);
322
+ });
323
+ }
324
+ };
325
+
326
+ public getSignedUrl = (
327
+ bucketName: string,
328
+ key: string,
329
+ operation: 'getObject' | 'putObject' = 'getObject',
330
+ params?: S3SignedUrlParams,
331
+ ): S3SignedUrlResult => {
332
+ return {
333
+ key,
334
+ url: this.s3.getSignedUrl(operation, {
335
+ Bucket: bucketName,
336
+ Key: key,
337
+ Expires: 60 * 10,
338
+ ...(params || {}),
339
+ }),
340
+ };
341
+ };
342
+
343
+ public getSignedCookie = (
344
+ keyPairId: string,
345
+ privateKey: string,
346
+ url: string,
347
+ expires: number,
348
+ ): AWS.CloudFront.Signer.CustomPolicy => {
349
+ const signer = new AWS.CloudFront.Signer(keyPairId, privateKey);
350
+ const policy = {
351
+ Statement: [
352
+ {
353
+ Resource: url,
354
+ Condition: {
355
+ DateLessThan: { 'AWS:EpochTime': expires },
356
+ },
357
+ },
358
+ ],
359
+ };
360
+ const ret = signer.getSignedCookie({ policy: JSON.stringify(policy) });
361
+ return ret;
362
+ };
363
+
364
+ public getAttachmentUrl = (
365
+ bucketName: string,
366
+ key: string,
367
+ fileName: string,
368
+ params?: S3SignedUrlParams,
369
+ ): S3SignedUrlResult => {
370
+ return this.getSignedUrl(bucketName, key, 'getObject', {
371
+ ...params,
372
+ ResponseContentDisposition: `attachment; filename="${fileName}"`,
373
+ });
374
+ };
375
+
376
+ public getDynamoDbItem = async <T>(
377
+ tableName: string,
378
+ key: { [keyColumn: string]: string },
379
+ defaultValue?: T,
380
+ ): Promise<T | undefined> => {
381
+ logger.debug(
382
+ `Read an item with key[${JSON.stringify(key)}] from ${tableName}.`,
383
+ );
384
+ const getResult = await this.dynamodb
385
+ .get({
386
+ TableName: tableName,
387
+ Key: key,
388
+ })
389
+ .promise();
390
+ logger.stupid(`getResult`, getResult);
391
+ const item: T | undefined =
392
+ getResult !== undefined && getResult.Item !== undefined
393
+ ? ((getResult.Item as any) as T) // Casts forcefully.
394
+ : defaultValue;
395
+ logger.stupid(`item`, item);
396
+ return item;
397
+ };
398
+
399
+ public updateDynamoDbItem = async (
400
+ tableName: string,
401
+ key: { [keyColumn: string]: string },
402
+ columnValues: { [column: string]: any },
403
+ ) => {
404
+ logger.debug(
405
+ `Update an item with key[${JSON.stringify(key)}] to ${tableName}`,
406
+ );
407
+ logger.stupid(`keyValues`, columnValues);
408
+ const expressions = Object.keys(columnValues)
409
+ .map(column => `${column} = :${column}`)
410
+ .join(', ');
411
+ const attributeValues = Object.keys(columnValues)
412
+ .map(column => [`:${column}`, columnValues[column]])
413
+ .reduce((obj, pair) => ({ ...obj, [pair[0]]: pair[1] }), {});
414
+ logger.stupid(`expressions`, expressions);
415
+ logger.stupid(`attributeValues`, attributeValues);
416
+ const updateResult = await this.dynamodb
417
+ .update({
418
+ TableName: tableName,
419
+ Key: key,
420
+ UpdateExpression: `set ${expressions}`,
421
+ ExpressionAttributeValues: attributeValues,
422
+ })
423
+ .promise();
424
+ logger.stupid(`updateResult`, updateResult);
425
+ return updateResult;
426
+ };
427
+
428
+ // Setup
429
+
430
+ public setupQueue = async (queueName: string) => {
431
+ try {
432
+ const listResult = await this.sqs
433
+ .listQueues({
434
+ QueueNamePrefix: queueName,
435
+ })
436
+ .promise();
437
+ if (listResult.QueueUrls) {
438
+ for (const queueUrl of listResult.QueueUrls) {
439
+ if (queueUrl.endsWith(queueName)) {
440
+ logger.debug(`Queue[${queueName} => ${queueUrl}] already exists.`);
441
+ return true;
442
+ }
443
+ }
444
+ }
445
+ } catch (error) {
446
+ logger.debug(`No Queue[${queueName}] exists due to ${error}`);
447
+ }
448
+ logger.debug(`Create a queue[${queueName}] newly.`);
449
+ const createResult = await this.sqs
450
+ .createQueue({
451
+ QueueName: queueName,
452
+ })
453
+ .promise();
454
+ logger.stupid(`createResult`, createResult);
455
+ return true;
456
+ };
457
+
458
+ public setupStorage = async (
459
+ bucketName: string,
460
+ cors: {
461
+ methods: Array<'GET' | 'POST' | 'PUT' | 'DELETE' | 'HEAD'>;
462
+ origins: string[];
463
+ },
464
+ ) => {
465
+ try {
466
+ const listResult = await this.s3.listBuckets().promise();
467
+ if (
468
+ listResult.Buckets &&
469
+ listResult.Buckets.map(each => each.Name).includes(bucketName)
470
+ ) {
471
+ logger.debug(`Bucket[${bucketName}] already exists.`);
472
+ return true;
473
+ }
474
+ } catch (error) {
475
+ logger.debug(`No bucket[${bucketName}] exists due to ${error}`);
476
+ }
477
+ logger.debug(`Create a bucket[${bucketName}] newly.`);
478
+ const createResult = await this.s3
479
+ .createBucket({
480
+ Bucket: bucketName,
481
+ })
482
+ .promise();
483
+ logger.stupid(`createResult`, createResult);
484
+ if (cors) {
485
+ const corsResult = await this.s3
486
+ .putBucketCors({
487
+ Bucket: bucketName,
488
+ CORSConfiguration: {
489
+ CORSRules: [
490
+ {
491
+ AllowedHeaders: ['*'],
492
+ AllowedMethods: cors.methods,
493
+ AllowedOrigins: cors.origins,
494
+ },
495
+ ],
496
+ },
497
+ })
498
+ .promise();
499
+ logger.stupid(`corsResult`, corsResult);
500
+ }
501
+ return true;
502
+ };
503
+
504
+ public setupDynamoDb = async (tableName: string, keyColumn: string) => {
505
+ try {
506
+ const listResult = await this.dynamodbAdmin.listTables().promise();
507
+ if (listResult.TableNames && listResult.TableNames.includes(tableName)) {
508
+ logger.debug(`Table[${tableName}] already exists.`);
509
+ return true;
510
+ }
511
+ } catch (error) {
512
+ logger.debug(`No table[${tableName}] exists due to ${error}`);
513
+ }
514
+ logger.debug(`Create a table[${tableName}] newly.`);
515
+ const createResult = await this.dynamodbAdmin
516
+ .createTable({
517
+ TableName: tableName,
518
+ KeySchema: [{ AttributeName: keyColumn, KeyType: 'HASH' }],
519
+ AttributeDefinitions: [
520
+ { AttributeName: keyColumn, AttributeType: 'S' },
521
+ ],
522
+ ProvisionedThroughput: {
523
+ ReadCapacityUnits: 30,
524
+ WriteCapacityUnits: 10,
525
+ },
526
+ })
527
+ .promise();
528
+ logger.stupid(`createResult`, createResult);
529
+ return true;
530
+ };
531
+ }