serverless-simple-middleware 0.0.62 → 0.0.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aws/config.js +11 -12
- package/dist/aws/index.d.ts +1 -1
- package/dist/aws/index.js +1 -1
- package/dist/aws/simple.d.ts +1 -1
- package/dist/aws/simple.js +534 -786
- package/dist/internal/AwsError.js +10 -28
- package/dist/middleware/aws.js +42 -125
- package/dist/middleware/base.d.ts +4 -4
- package/dist/middleware/base.js +92 -102
- package/dist/middleware/build.js +101 -244
- package/dist/middleware/database/connectionProxy.d.ts +19 -0
- package/dist/middleware/database/connectionProxy.js +136 -0
- package/dist/middleware/database/index.d.ts +2 -0
- package/dist/middleware/database/index.js +7 -0
- package/dist/middleware/database/sqlClient.d.ts +8 -0
- package/dist/middleware/database/sqlClient.js +65 -0
- package/dist/middleware/index.d.ts +4 -3
- package/dist/middleware/index.js +8 -7
- package/dist/middleware/logger.js +13 -69
- package/dist/middleware/mysql.d.ts +11 -23
- package/dist/middleware/mysql.js +25 -288
- package/dist/middleware/trace.d.ts +1 -1
- package/dist/middleware/trace.js +144 -216
- package/dist/utils/logger.js +29 -30
- package/dist/utils/misc.js +6 -6
- package/package.json +5 -3
- package/src/aws/index.ts +1 -1
- package/src/aws/simple.ts +3 -3
- package/src/middleware/base.ts +24 -7
- package/src/middleware/database/connectionProxy.ts +174 -0
- package/src/middleware/database/index.ts +14 -0
- package/src/middleware/database/sqlClient.ts +95 -0
- package/src/middleware/index.ts +3 -2
- package/src/middleware/mysql.ts +16 -178
- package/src/middleware/trace.ts +1 -1
- package/tsconfig.json +2 -2
package/dist/aws/simple.js
CHANGED
|
@@ -1,809 +1,557 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SimpleAWS = void 0;
|
|
4
|
+
const cloudfront_signer_1 = require("@aws-sdk/cloudfront-signer");
|
|
5
|
+
const simple_staging_1 = require("simple-staging");
|
|
6
|
+
const fs = require("fs");
|
|
7
|
+
const non_secure_1 = require("nanoid/non-secure");
|
|
8
|
+
const os = require("os");
|
|
9
|
+
const utils_1 = require("../utils");
|
|
10
|
+
const config_1 = require("./config");
|
|
11
|
+
const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
|
|
12
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
13
|
+
const client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
14
|
+
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
|
|
15
|
+
const lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
16
|
+
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
|
17
|
+
const define_1 = require("./define");
|
|
18
|
+
const logger = (0, utils_1.getLogger)(__filename);
|
|
19
|
+
class SimpleAWS {
|
|
20
|
+
queueUrls = {};
|
|
21
|
+
config;
|
|
22
|
+
lazyS3;
|
|
23
|
+
lazySqs;
|
|
24
|
+
lazyDynamodb;
|
|
25
|
+
lazyDynamodbAdmin;
|
|
26
|
+
static stageTag = {
|
|
27
|
+
Key: 'STAGE',
|
|
28
|
+
Value: simple_staging_1.envDefault.level,
|
|
10
29
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
});
|
|
21
|
-
};
|
|
22
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
23
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
24
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
25
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
26
|
-
function step(op) {
|
|
27
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
28
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
29
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
30
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
31
|
-
switch (op[0]) {
|
|
32
|
-
case 0: case 1: t = op; break;
|
|
33
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
34
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
35
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
36
|
-
default:
|
|
37
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
38
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
39
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
40
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
41
|
-
if (t[2]) _.ops.pop();
|
|
42
|
-
_.trys.pop(); continue;
|
|
43
|
-
}
|
|
44
|
-
op = body.call(thisArg, _);
|
|
45
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
46
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
30
|
+
static stringifiedStageTag = `STAGE=${simple_staging_1.envDefault.level}`;
|
|
31
|
+
constructor(config) {
|
|
32
|
+
this.config = config || new config_1.SimpleAWSConfig();
|
|
33
|
+
/**
|
|
34
|
+
* The simple cache for { queueName: queueUrl }.
|
|
35
|
+
* It can help in the only case of launching this project as offline.
|
|
36
|
+
* @type { { [queueName: string]: string } }
|
|
37
|
+
*/
|
|
38
|
+
this.queueUrls = {};
|
|
47
39
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (ar || !(i in from)) {
|
|
52
|
-
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
|
|
53
|
-
ar[i] = from[i];
|
|
40
|
+
get s3() {
|
|
41
|
+
if (this.lazyS3 === undefined) {
|
|
42
|
+
this.lazyS3 = new client_s3_1.S3(this.config.get(define_1.AWSComponent.s3) || {});
|
|
54
43
|
}
|
|
44
|
+
return this.lazyS3;
|
|
55
45
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
var client_s3_1 = require("@aws-sdk/client-s3");
|
|
70
|
-
var client_sqs_1 = require("@aws-sdk/client-sqs");
|
|
71
|
-
var lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
|
|
72
|
-
var s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
|
|
73
|
-
var lib_storage_1 = require("@aws-sdk/lib-storage");
|
|
74
|
-
var logger = (0, utils_1.getLogger)(__filename);
|
|
75
|
-
var SimpleAWS = /** @class */ (function () {
|
|
76
|
-
function SimpleAWS(config) {
|
|
77
|
-
var _this = this;
|
|
78
|
-
this.queueUrls = {};
|
|
79
|
-
this.getQueueUrl = function (queueName) { return __awaiter(_this, void 0, void 0, function () {
|
|
80
|
-
var urlResult;
|
|
81
|
-
return __generator(this, function (_a) {
|
|
82
|
-
switch (_a.label) {
|
|
83
|
-
case 0:
|
|
84
|
-
if (this.queueUrls[queueName] !== undefined) {
|
|
85
|
-
return [2 /*return*/, this.queueUrls[queueName]];
|
|
86
|
-
}
|
|
87
|
-
return [4 /*yield*/, this.sqs.getQueueUrl({
|
|
88
|
-
QueueName: queueName,
|
|
89
|
-
})];
|
|
90
|
-
case 1:
|
|
91
|
-
urlResult = _a.sent();
|
|
92
|
-
logger.stupid("urlResult", urlResult);
|
|
93
|
-
if (!urlResult.QueueUrl) {
|
|
94
|
-
throw new Error("No queue url with name[".concat(queueName, "]"));
|
|
95
|
-
}
|
|
96
|
-
return [2 /*return*/, (this.queueUrls[queueName] = urlResult.QueueUrl)];
|
|
97
|
-
}
|
|
98
|
-
});
|
|
99
|
-
}); };
|
|
100
|
-
this.enqueue = function (queueName, data) { return __awaiter(_this, void 0, void 0, function () {
|
|
101
|
-
var queueUrl, sendResult, attrResult;
|
|
102
|
-
var _a;
|
|
103
|
-
return __generator(this, function (_b) {
|
|
104
|
-
switch (_b.label) {
|
|
105
|
-
case 0:
|
|
106
|
-
logger.debug("Send message[".concat(data.key, "] to queue."));
|
|
107
|
-
logger.stupid("data", data);
|
|
108
|
-
return [4 /*yield*/, this.getQueueUrl(queueName)];
|
|
109
|
-
case 1:
|
|
110
|
-
queueUrl = _b.sent();
|
|
111
|
-
return [4 /*yield*/, this.sqs.sendMessage({
|
|
112
|
-
QueueUrl: queueUrl,
|
|
113
|
-
MessageBody: JSON.stringify(data),
|
|
114
|
-
DelaySeconds: 0,
|
|
115
|
-
})];
|
|
116
|
-
case 2:
|
|
117
|
-
sendResult = _b.sent();
|
|
118
|
-
logger.stupid("sendResult", sendResult);
|
|
119
|
-
return [4 /*yield*/, this.sqs.getQueueAttributes({
|
|
120
|
-
QueueUrl: queueUrl,
|
|
121
|
-
AttributeNames: ['ApproximateNumberOfMessages'],
|
|
122
|
-
})];
|
|
123
|
-
case 3:
|
|
124
|
-
attrResult = _b.sent();
|
|
125
|
-
logger.stupid("attrResult", attrResult);
|
|
126
|
-
if (!attrResult.Attributes) {
|
|
127
|
-
return [2 /*return*/, 0];
|
|
128
|
-
}
|
|
129
|
-
return [2 /*return*/, +(((_a = attrResult.Attributes) === null || _a === void 0 ? void 0 : _a.ApproximateNumberOfMessages) || 0)];
|
|
130
|
-
}
|
|
46
|
+
get sqs() {
|
|
47
|
+
if (this.lazySqs === undefined) {
|
|
48
|
+
this.lazySqs = new client_sqs_1.SQS(this.config.get(define_1.AWSComponent.sqs) || {});
|
|
49
|
+
}
|
|
50
|
+
return this.lazySqs;
|
|
51
|
+
}
|
|
52
|
+
get dynamodb() {
|
|
53
|
+
if (this.lazyDynamodb === undefined) {
|
|
54
|
+
this.lazyDynamodb = lib_dynamodb_1.DynamoDBDocument.from(new client_dynamodb_1.DynamoDBClient(this.config.get(define_1.AWSComponent.dynamodb) || {}), {
|
|
55
|
+
marshallOptions: {
|
|
56
|
+
convertEmptyValues: true,
|
|
57
|
+
removeUndefinedValues: true,
|
|
58
|
+
},
|
|
131
59
|
});
|
|
132
|
-
}
|
|
133
|
-
this.
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
60
|
+
}
|
|
61
|
+
return this.lazyDynamodb;
|
|
62
|
+
}
|
|
63
|
+
get dynamodbAdmin() {
|
|
64
|
+
if (this.lazyDynamodbAdmin === undefined) {
|
|
65
|
+
this.lazyDynamodbAdmin = new client_dynamodb_1.DynamoDB(this.config.get(define_1.AWSComponent.dynamodb) || {});
|
|
66
|
+
}
|
|
67
|
+
return this.lazyDynamodbAdmin;
|
|
68
|
+
}
|
|
69
|
+
getQueueUrl = async (queueName) => {
|
|
70
|
+
if (this.queueUrls[queueName] !== undefined) {
|
|
71
|
+
return this.queueUrls[queueName];
|
|
72
|
+
}
|
|
73
|
+
const urlResult = await this.sqs.getQueueUrl({
|
|
74
|
+
QueueName: queueName,
|
|
75
|
+
});
|
|
76
|
+
logger.stupid(`urlResult`, urlResult);
|
|
77
|
+
if (!urlResult.QueueUrl) {
|
|
78
|
+
throw new Error(`No queue url with name[${queueName}]`);
|
|
79
|
+
}
|
|
80
|
+
return (this.queueUrls[queueName] = urlResult.QueueUrl);
|
|
81
|
+
};
|
|
82
|
+
enqueue = async (queueName, data) => {
|
|
83
|
+
logger.debug(`Send message[${data.key}] to queue.`);
|
|
84
|
+
logger.stupid(`data`, data);
|
|
85
|
+
const queueUrl = await this.getQueueUrl(queueName);
|
|
86
|
+
const sendResult = await this.sqs.sendMessage({
|
|
87
|
+
QueueUrl: queueUrl,
|
|
88
|
+
MessageBody: JSON.stringify(data),
|
|
89
|
+
DelaySeconds: 0,
|
|
90
|
+
});
|
|
91
|
+
logger.stupid(`sendResult`, sendResult);
|
|
92
|
+
const attrResult = await this.sqs.getQueueAttributes({
|
|
93
|
+
QueueUrl: queueUrl,
|
|
94
|
+
AttributeNames: ['ApproximateNumberOfMessages'],
|
|
95
|
+
});
|
|
96
|
+
logger.stupid(`attrResult`, attrResult);
|
|
97
|
+
if (!attrResult.Attributes) {
|
|
98
|
+
return 0;
|
|
99
|
+
}
|
|
100
|
+
return +(attrResult.Attributes?.ApproximateNumberOfMessages || 0);
|
|
101
|
+
};
|
|
102
|
+
dequeue = async (queueName, fetchSize = 1, waitSeconds = 1, visibilityTimeout = 15) => {
|
|
103
|
+
logger.debug(`Receive message from queue[${queueName}].`);
|
|
104
|
+
const queueUrl = await this.getQueueUrl(queueName);
|
|
105
|
+
const receiveResult = await this.sqs.receiveMessage({
|
|
106
|
+
QueueUrl: queueUrl,
|
|
107
|
+
MaxNumberOfMessages: fetchSize,
|
|
108
|
+
WaitTimeSeconds: waitSeconds,
|
|
109
|
+
VisibilityTimeout: visibilityTimeout,
|
|
110
|
+
});
|
|
111
|
+
logger.stupid(`receiveResult`, receiveResult);
|
|
112
|
+
if (receiveResult.Messages === undefined ||
|
|
113
|
+
receiveResult.Messages.length === 0) {
|
|
114
|
+
return [];
|
|
115
|
+
}
|
|
116
|
+
const data = [];
|
|
117
|
+
for (const each of receiveResult.Messages) {
|
|
118
|
+
if (!each.ReceiptHandle) {
|
|
119
|
+
logger.warn(`No receipt handler: ${JSON.stringify(each)}`);
|
|
120
|
+
continue;
|
|
137
121
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
VisibilityTimeout: visibilityTimeout,
|
|
155
|
-
})];
|
|
156
|
-
case 2:
|
|
157
|
-
receiveResult = _c.sent();
|
|
158
|
-
logger.stupid("receiveResult", receiveResult);
|
|
159
|
-
if (receiveResult.Messages === undefined ||
|
|
160
|
-
receiveResult.Messages.length === 0) {
|
|
161
|
-
return [2 /*return*/, []];
|
|
162
|
-
}
|
|
163
|
-
data = [];
|
|
164
|
-
for (_a = 0, _b = receiveResult.Messages; _a < _b.length; _a++) {
|
|
165
|
-
each = _b[_a];
|
|
166
|
-
if (!each.ReceiptHandle) {
|
|
167
|
-
logger.warn("No receipt handler: ".concat(JSON.stringify(each)));
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
message = {
|
|
171
|
-
handle: each.ReceiptHandle,
|
|
172
|
-
body: each.Body ? JSON.parse(each.Body) : undefined,
|
|
173
|
-
};
|
|
174
|
-
data.push(message);
|
|
175
|
-
}
|
|
176
|
-
logger.verbose("Receive a message[".concat(JSON.stringify(data), "] from queue"));
|
|
177
|
-
return [2 /*return*/, data];
|
|
178
|
-
}
|
|
179
|
-
});
|
|
180
|
-
});
|
|
181
|
-
};
|
|
182
|
-
this.dequeueAll = function (queueName_1) {
|
|
183
|
-
var args_1 = [];
|
|
184
|
-
for (var _i = 1; _i < arguments.length; _i++) {
|
|
185
|
-
args_1[_i - 1] = arguments[_i];
|
|
122
|
+
const message = {
|
|
123
|
+
handle: each.ReceiptHandle,
|
|
124
|
+
body: each.Body ? JSON.parse(each.Body) : undefined,
|
|
125
|
+
};
|
|
126
|
+
data.push(message);
|
|
127
|
+
}
|
|
128
|
+
logger.verbose(`Receive a message[${JSON.stringify(data)}] from queue`);
|
|
129
|
+
return data;
|
|
130
|
+
};
|
|
131
|
+
dequeueAll = async (queueName, limitSize = Number.MAX_VALUE, visibilityTimeout = 15) => {
|
|
132
|
+
const messages = [];
|
|
133
|
+
const maxFetchSize = 10; // This is max-value for fetching in each time.
|
|
134
|
+
while (messages.length < limitSize) {
|
|
135
|
+
const eachOfMessages = await this.dequeue(queueName, Math.min(limitSize - messages.length, maxFetchSize), 0, visibilityTimeout);
|
|
136
|
+
if (!eachOfMessages || eachOfMessages.length === 0) {
|
|
137
|
+
break;
|
|
186
138
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
case 2:
|
|
232
|
-
_a.sent();
|
|
233
|
-
return [2 /*return*/, handle];
|
|
234
|
-
}
|
|
235
|
-
});
|
|
236
|
-
}); };
|
|
237
|
-
this.completeMessage = function (queueName, handle) { return __awaiter(_this, void 0, void 0, function () {
|
|
238
|
-
var queueUrl, deleteResult;
|
|
239
|
-
return __generator(this, function (_a) {
|
|
240
|
-
switch (_a.label) {
|
|
241
|
-
case 0:
|
|
242
|
-
logger.debug("Complete a message with handle[".concat(handle, "]"));
|
|
243
|
-
return [4 /*yield*/, this.getQueueUrl(queueName)];
|
|
244
|
-
case 1:
|
|
245
|
-
queueUrl = _a.sent();
|
|
246
|
-
return [4 /*yield*/, this.sqs.deleteMessage({
|
|
247
|
-
QueueUrl: queueUrl,
|
|
248
|
-
ReceiptHandle: handle,
|
|
249
|
-
})];
|
|
250
|
-
case 2:
|
|
251
|
-
deleteResult = _a.sent();
|
|
252
|
-
logger.stupid("deleteResult", deleteResult);
|
|
253
|
-
return [2 /*return*/, handle];
|
|
254
|
-
}
|
|
255
|
-
});
|
|
256
|
-
}); };
|
|
257
|
-
this.completeMessages = function (queueName, handles) { return __awaiter(_this, void 0, void 0, function () {
|
|
258
|
-
var chunkSize, index, start, end, sublist, queueUrl, deletesResult;
|
|
259
|
-
return __generator(this, function (_a) {
|
|
260
|
-
switch (_a.label) {
|
|
261
|
-
case 0:
|
|
262
|
-
logger.debug("Complete a message with handle[".concat(handles, "]"));
|
|
263
|
-
if (!handles) {
|
|
264
|
-
return [2 /*return*/, handles];
|
|
265
|
-
}
|
|
266
|
-
chunkSize = 10;
|
|
267
|
-
index = 0;
|
|
268
|
-
start = 0;
|
|
269
|
-
_a.label = 1;
|
|
270
|
-
case 1:
|
|
271
|
-
if (!(start < handles.length)) return [3 /*break*/, 5];
|
|
272
|
-
end = Math.min(start + chunkSize, handles.length);
|
|
273
|
-
sublist = handles.slice(start, end);
|
|
274
|
-
return [4 /*yield*/, this.getQueueUrl(queueName)];
|
|
275
|
-
case 2:
|
|
276
|
-
queueUrl = _a.sent();
|
|
277
|
-
return [4 /*yield*/, this.sqs.deleteMessageBatch({
|
|
278
|
-
QueueUrl: queueUrl,
|
|
279
|
-
Entries: sublist.map(function (handle) { return ({
|
|
280
|
-
Id: (++index).toString(),
|
|
281
|
-
ReceiptHandle: handle,
|
|
282
|
-
}); }),
|
|
283
|
-
})];
|
|
284
|
-
case 3:
|
|
285
|
-
deletesResult = _a.sent();
|
|
286
|
-
logger.stupid("deleteResult", deletesResult);
|
|
287
|
-
_a.label = 4;
|
|
288
|
-
case 4:
|
|
289
|
-
start += chunkSize;
|
|
290
|
-
return [3 /*break*/, 1];
|
|
291
|
-
case 5: return [2 /*return*/, handles];
|
|
292
|
-
}
|
|
293
|
-
});
|
|
294
|
-
}); };
|
|
295
|
-
this.download = function (bucket, key, localPath) { return __awaiter(_this, void 0, void 0, function () {
|
|
296
|
-
var Body;
|
|
297
|
-
return __generator(this, function (_a) {
|
|
298
|
-
switch (_a.label) {
|
|
299
|
-
case 0:
|
|
300
|
-
logger.debug("Get a stream of item[".concat(key, "] from bucket[").concat(bucket, "]"));
|
|
301
|
-
return [4 /*yield*/, this.s3.getObject({ Bucket: bucket, Key: key })];
|
|
302
|
-
case 1:
|
|
303
|
-
Body = (_a.sent()).Body;
|
|
304
|
-
return [2 /*return*/, new Promise(function (resolve, reject) {
|
|
305
|
-
return Body
|
|
306
|
-
.on('error', function (error) { return reject(error); })
|
|
307
|
-
.pipe(fs.createWriteStream(localPath))
|
|
308
|
-
.on('finish', function () { return resolve(localPath); })
|
|
309
|
-
.on('error', function (error) { return reject(error); });
|
|
310
|
-
})];
|
|
311
|
-
}
|
|
312
|
-
});
|
|
313
|
-
}); };
|
|
314
|
-
this.readFile = function (bucket, key) { return __awaiter(_this, void 0, void 0, function () {
|
|
315
|
-
var tempFile, content, error_1;
|
|
316
|
-
return __generator(this, function (_a) {
|
|
317
|
-
switch (_a.label) {
|
|
318
|
-
case 0:
|
|
319
|
-
logger.debug("Read item[".concat(key, "] from bucket[").concat(bucket, "]"));
|
|
320
|
-
tempFile = "".concat(os.tmpdir(), "/").concat((0, non_secure_1.nanoid)());
|
|
321
|
-
_a.label = 1;
|
|
322
|
-
case 1:
|
|
323
|
-
_a.trys.push([1, , 4, 9]);
|
|
324
|
-
return [4 /*yield*/, this.download(bucket, key, tempFile)];
|
|
325
|
-
case 2:
|
|
326
|
-
_a.sent();
|
|
327
|
-
return [4 /*yield*/, fs.promises.readFile(tempFile, {
|
|
328
|
-
encoding: 'utf-8',
|
|
329
|
-
})];
|
|
330
|
-
case 3:
|
|
331
|
-
content = _a.sent();
|
|
332
|
-
return [2 /*return*/, content];
|
|
333
|
-
case 4:
|
|
334
|
-
if (!fs.existsSync(tempFile)) return [3 /*break*/, 8];
|
|
335
|
-
_a.label = 5;
|
|
336
|
-
case 5:
|
|
337
|
-
_a.trys.push([5, 7, , 8]);
|
|
338
|
-
return [4 /*yield*/, fs.promises.unlink(tempFile)];
|
|
339
|
-
case 6:
|
|
340
|
-
_a.sent();
|
|
341
|
-
return [3 /*break*/, 8];
|
|
342
|
-
case 7:
|
|
343
|
-
error_1 = _a.sent();
|
|
344
|
-
logger.error("Failed to delete temp file ".concat(tempFile, ": ").concat((0, utils_1.stringifyError)(error_1)));
|
|
345
|
-
return [3 /*break*/, 8];
|
|
346
|
-
case 8: return [7 /*endfinally*/];
|
|
347
|
-
case 9: return [2 /*return*/];
|
|
348
|
-
}
|
|
349
|
-
});
|
|
350
|
-
}); };
|
|
351
|
-
this.readFileBuffer = function (bucket, key) { return __awaiter(_this, void 0, void 0, function () {
|
|
352
|
-
var Body, buffer;
|
|
353
|
-
return __generator(this, function (_a) {
|
|
354
|
-
switch (_a.label) {
|
|
355
|
-
case 0:
|
|
356
|
-
logger.debug("Read item[".concat(key, "] from bucket[").concat(bucket, "]"));
|
|
357
|
-
return [4 /*yield*/, this.s3.getObject({ Bucket: bucket, Key: key })];
|
|
358
|
-
case 1:
|
|
359
|
-
Body = (_a.sent()).Body;
|
|
360
|
-
return [4 /*yield*/, (Body === null || Body === void 0 ? void 0 : Body.transformToByteArray())];
|
|
361
|
-
case 2:
|
|
362
|
-
buffer = _a.sent();
|
|
363
|
-
if (!buffer) {
|
|
364
|
-
throw new Error("Failed to read file ".concat(key, " from bucket ").concat(bucket));
|
|
365
|
-
}
|
|
366
|
-
return [2 /*return*/, Buffer.from(buffer)];
|
|
367
|
-
}
|
|
368
|
-
});
|
|
369
|
-
}); };
|
|
370
|
-
this.upload = function (bucket, localPath, key, tags) { return __awaiter(_this, void 0, void 0, function () {
|
|
371
|
-
var upload;
|
|
372
|
-
return __generator(this, function (_a) {
|
|
373
|
-
switch (_a.label) {
|
|
374
|
-
case 0:
|
|
375
|
-
logger.debug("Upload item[".concat(key, "] into bucket[").concat(bucket, "]"));
|
|
376
|
-
upload = new lib_storage_1.Upload({
|
|
377
|
-
client: this.s3,
|
|
378
|
-
params: {
|
|
379
|
-
Bucket: bucket,
|
|
380
|
-
Key: key,
|
|
381
|
-
Body: fs.createReadStream(localPath),
|
|
382
|
-
},
|
|
383
|
-
partSize: 5 * 1024 * 1024, // 5MB
|
|
384
|
-
queueSize: 4,
|
|
385
|
-
tags: __spreadArray([SimpleAWS.stageTag], (tags || []), true),
|
|
386
|
-
});
|
|
387
|
-
return [4 /*yield*/, upload.done()];
|
|
388
|
-
case 1:
|
|
389
|
-
_a.sent();
|
|
390
|
-
return [2 /*return*/, key];
|
|
391
|
-
}
|
|
392
|
-
});
|
|
393
|
-
}); };
|
|
394
|
-
this.uploadFromBuffer = function (bucket, key, buffer, tags) { return __awaiter(_this, void 0, void 0, function () {
|
|
395
|
-
var upload;
|
|
396
|
-
return __generator(this, function (_a) {
|
|
397
|
-
switch (_a.label) {
|
|
398
|
-
case 0:
|
|
399
|
-
logger.debug("Upload item[".concat(key, "] into bucket[").concat(bucket, "]"));
|
|
400
|
-
upload = new lib_storage_1.Upload({
|
|
401
|
-
client: this.s3,
|
|
402
|
-
params: {
|
|
403
|
-
Bucket: bucket,
|
|
404
|
-
Key: key,
|
|
405
|
-
Body: buffer,
|
|
406
|
-
},
|
|
407
|
-
partSize: 5 * 1024 * 1024, // 5MB
|
|
408
|
-
queueSize: 4,
|
|
409
|
-
tags: __spreadArray([SimpleAWS.stageTag], (tags || []), true),
|
|
410
|
-
});
|
|
411
|
-
return [4 /*yield*/, upload.done()];
|
|
412
|
-
case 1:
|
|
413
|
-
_a.sent();
|
|
414
|
-
return [2 /*return*/, key];
|
|
415
|
-
}
|
|
416
|
-
});
|
|
417
|
-
}); };
|
|
418
|
-
this.writeFile = function (bucket, key, content) { return __awaiter(_this, void 0, void 0, function () {
|
|
419
|
-
var tempFile, error_2, msg;
|
|
420
|
-
return __generator(this, function (_a) {
|
|
421
|
-
switch (_a.label) {
|
|
422
|
-
case 0:
|
|
423
|
-
logger.debug("Write item[".concat(key, "] into bucket[").concat(bucket, "]"));
|
|
424
|
-
tempFile = "".concat(os.tmpdir(), "/").concat((0, non_secure_1.nanoid)());
|
|
425
|
-
_a.label = 1;
|
|
426
|
-
case 1:
|
|
427
|
-
_a.trys.push([1, , 4, 9]);
|
|
428
|
-
return [4 /*yield*/, fs.promises.writeFile(tempFile, content, 'utf-8')];
|
|
429
|
-
case 2:
|
|
430
|
-
_a.sent();
|
|
431
|
-
return [4 /*yield*/, this.upload(bucket, tempFile, key)];
|
|
432
|
-
case 3:
|
|
433
|
-
_a.sent();
|
|
434
|
-
return [3 /*break*/, 9];
|
|
435
|
-
case 4:
|
|
436
|
-
if (!fs.existsSync(tempFile)) {
|
|
437
|
-
return [2 /*return*/];
|
|
438
|
-
}
|
|
439
|
-
_a.label = 5;
|
|
440
|
-
case 5:
|
|
441
|
-
_a.trys.push([5, 7, , 8]);
|
|
442
|
-
return [4 /*yield*/, fs.promises.unlink(tempFile)];
|
|
443
|
-
case 6:
|
|
444
|
-
_a.sent();
|
|
445
|
-
return [3 /*break*/, 8];
|
|
446
|
-
case 7:
|
|
447
|
-
error_2 = _a.sent();
|
|
448
|
-
msg = "Error during writeFile: unlink file ".concat(tempFile, ": ").concat((0, utils_1.stringifyError)(error_2));
|
|
449
|
-
logger.error(msg);
|
|
450
|
-
return [3 /*break*/, 8];
|
|
451
|
-
case 8: return [7 /*endfinally*/];
|
|
452
|
-
case 9: return [2 /*return*/];
|
|
453
|
-
}
|
|
454
|
-
});
|
|
455
|
-
}); };
|
|
456
|
-
this.getSignedCookie = function (keyPairId, privateKey, url, expires) {
|
|
457
|
-
var policy = JSON.stringify({
|
|
458
|
-
Statement: [
|
|
459
|
-
{
|
|
460
|
-
Resource: url,
|
|
461
|
-
Condition: {
|
|
462
|
-
DateLessThan: { 'AWS:EpochTime': expires },
|
|
463
|
-
},
|
|
464
|
-
},
|
|
465
|
-
],
|
|
466
|
-
});
|
|
467
|
-
return (0, cloudfront_signer_1.getSignedCookies)({
|
|
468
|
-
keyPairId: keyPairId,
|
|
469
|
-
privateKey: privateKey,
|
|
470
|
-
policy: policy,
|
|
471
|
-
});
|
|
472
|
-
};
|
|
473
|
-
this.getDynamoDbItem = function (tableName, key, defaultValue) { return __awaiter(_this, void 0, void 0, function () {
|
|
474
|
-
var getResult, item;
|
|
475
|
-
return __generator(this, function (_a) {
|
|
476
|
-
switch (_a.label) {
|
|
477
|
-
case 0:
|
|
478
|
-
logger.debug("Read an item with key[".concat(JSON.stringify(key), "] from ").concat(tableName, "."));
|
|
479
|
-
return [4 /*yield*/, this.dynamodb.get({
|
|
480
|
-
TableName: tableName,
|
|
481
|
-
Key: key,
|
|
482
|
-
})];
|
|
483
|
-
case 1:
|
|
484
|
-
getResult = _a.sent();
|
|
485
|
-
logger.stupid("getResult", getResult);
|
|
486
|
-
item = getResult !== undefined && getResult.Item !== undefined
|
|
487
|
-
? getResult.Item // Casts forcefully.
|
|
488
|
-
: defaultValue;
|
|
489
|
-
logger.stupid("item", item);
|
|
490
|
-
return [2 /*return*/, item];
|
|
491
|
-
}
|
|
492
|
-
});
|
|
493
|
-
}); };
|
|
494
|
-
this.updateDynamoDbItem = function (tableName, key, columnValues) { return __awaiter(_this, void 0, void 0, function () {
|
|
495
|
-
var expressions, attributeValues, updateResult;
|
|
496
|
-
return __generator(this, function (_a) {
|
|
497
|
-
switch (_a.label) {
|
|
498
|
-
case 0:
|
|
499
|
-
logger.debug("Update an item with key[".concat(JSON.stringify(key), "] to ").concat(tableName));
|
|
500
|
-
logger.stupid("keyValues", columnValues);
|
|
501
|
-
expressions = Object.keys(columnValues)
|
|
502
|
-
.map(function (column) { return "".concat(column, " = :").concat(column); })
|
|
503
|
-
.join(', ');
|
|
504
|
-
attributeValues = Object.keys(columnValues)
|
|
505
|
-
.map(function (column) { return [":".concat(column), columnValues[column]]; })
|
|
506
|
-
.reduce(function (obj, pair) {
|
|
507
|
-
var _a;
|
|
508
|
-
return (__assign(__assign({}, obj), (_a = {}, _a[pair[0]] = pair[1], _a)));
|
|
509
|
-
}, {});
|
|
510
|
-
logger.stupid("expressions", expressions);
|
|
511
|
-
logger.stupid("attributeValues", attributeValues);
|
|
512
|
-
return [4 /*yield*/, this.dynamodb.update({
|
|
513
|
-
TableName: tableName,
|
|
514
|
-
Key: key,
|
|
515
|
-
UpdateExpression: "set ".concat(expressions),
|
|
516
|
-
ExpressionAttributeValues: attributeValues,
|
|
517
|
-
})];
|
|
518
|
-
case 1:
|
|
519
|
-
updateResult = _a.sent();
|
|
520
|
-
logger.stupid("updateResult", updateResult);
|
|
521
|
-
return [2 /*return*/, updateResult];
|
|
522
|
-
}
|
|
139
|
+
for (const each of eachOfMessages) {
|
|
140
|
+
messages.push(each);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
logger.stupid(`messages`, messages);
|
|
144
|
+
return messages;
|
|
145
|
+
};
|
|
146
|
+
retainMessage = async (queueName, handle, seconds) => {
|
|
147
|
+
logger.debug(`Change visibilityTimeout of ${handle} to ${seconds}secs.`);
|
|
148
|
+
const queueUrl = await this.getQueueUrl(queueName);
|
|
149
|
+
await this.sqs.changeMessageVisibility({
|
|
150
|
+
QueueUrl: queueUrl,
|
|
151
|
+
ReceiptHandle: handle,
|
|
152
|
+
VisibilityTimeout: seconds,
|
|
153
|
+
});
|
|
154
|
+
return handle;
|
|
155
|
+
};
|
|
156
|
+
completeMessage = async (queueName, handle) => {
|
|
157
|
+
logger.debug(`Complete a message with handle[${handle}]`);
|
|
158
|
+
const queueUrl = await this.getQueueUrl(queueName);
|
|
159
|
+
const deleteResult = await this.sqs.deleteMessage({
|
|
160
|
+
QueueUrl: queueUrl,
|
|
161
|
+
ReceiptHandle: handle,
|
|
162
|
+
});
|
|
163
|
+
logger.stupid(`deleteResult`, deleteResult);
|
|
164
|
+
return handle;
|
|
165
|
+
};
|
|
166
|
+
completeMessages = async (queueName, handles) => {
|
|
167
|
+
logger.debug(`Complete a message with handle[${handles}]`);
|
|
168
|
+
if (!handles) {
|
|
169
|
+
return handles;
|
|
170
|
+
}
|
|
171
|
+
const chunkSize = 10;
|
|
172
|
+
let index = 0;
|
|
173
|
+
for (let start = 0; start < handles.length; start += chunkSize) {
|
|
174
|
+
const end = Math.min(start + chunkSize, handles.length);
|
|
175
|
+
const sublist = handles.slice(start, end);
|
|
176
|
+
const queueUrl = await this.getQueueUrl(queueName);
|
|
177
|
+
const deletesResult = await this.sqs.deleteMessageBatch({
|
|
178
|
+
QueueUrl: queueUrl,
|
|
179
|
+
Entries: sublist.map((handle) => ({
|
|
180
|
+
Id: (++index).toString(),
|
|
181
|
+
ReceiptHandle: handle,
|
|
182
|
+
})),
|
|
523
183
|
});
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
return [3 /*break*/, 3];
|
|
547
|
-
case 2:
|
|
548
|
-
error_3 = _b.sent();
|
|
549
|
-
logger.debug("No Queue[".concat(queueName, "] exists due to ").concat(error_3));
|
|
550
|
-
return [3 /*break*/, 3];
|
|
551
|
-
case 3:
|
|
552
|
-
logger.debug("Create a queue[".concat(queueName, "] newly."));
|
|
553
|
-
return [4 /*yield*/, this.sqs.createQueue({
|
|
554
|
-
QueueName: queueName,
|
|
555
|
-
})];
|
|
556
|
-
case 4:
|
|
557
|
-
createResult = _b.sent();
|
|
558
|
-
logger.stupid("createResult", createResult);
|
|
559
|
-
return [2 /*return*/, true];
|
|
560
|
-
}
|
|
184
|
+
logger.stupid(`deleteResult`, deletesResult);
|
|
185
|
+
}
|
|
186
|
+
return handles;
|
|
187
|
+
};
|
|
188
|
+
download = async (bucket, key, localPath) => {
|
|
189
|
+
logger.debug(`Get a stream of item[${key}] from bucket[${bucket}]`);
|
|
190
|
+
const { Body } = await this.s3.getObject({ Bucket: bucket, Key: key });
|
|
191
|
+
return new Promise((resolve, reject) => Body
|
|
192
|
+
.on('error', (error) => reject(error))
|
|
193
|
+
.pipe(fs.createWriteStream(localPath))
|
|
194
|
+
.on('finish', () => resolve(localPath))
|
|
195
|
+
.on('error', (error) => reject(error)));
|
|
196
|
+
};
|
|
197
|
+
readFile = async (bucket, key) => {
|
|
198
|
+
logger.debug(`Read item[${key}] from bucket[${bucket}]`);
|
|
199
|
+
const tempFile = `${os.tmpdir()}/${(0, non_secure_1.nanoid)()}`;
|
|
200
|
+
try {
|
|
201
|
+
await this.download(bucket, key, tempFile);
|
|
202
|
+
const content = await fs.promises.readFile(tempFile, {
|
|
203
|
+
encoding: 'utf-8',
|
|
561
204
|
});
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
_a.trys.push([0, 2, , 3]);
|
|
569
|
-
return [4 /*yield*/, this.s3.listBuckets()];
|
|
570
|
-
case 1:
|
|
571
|
-
listResult = _a.sent();
|
|
572
|
-
if (listResult.Buckets &&
|
|
573
|
-
listResult.Buckets.map(function (each) { return each.Name; }).includes(bucketName)) {
|
|
574
|
-
logger.debug("Bucket[".concat(bucketName, "] already exists."));
|
|
575
|
-
return [2 /*return*/, true];
|
|
576
|
-
}
|
|
577
|
-
return [3 /*break*/, 3];
|
|
578
|
-
case 2:
|
|
579
|
-
error_4 = _a.sent();
|
|
580
|
-
logger.debug("No bucket[".concat(bucketName, "] exists due to ").concat(error_4));
|
|
581
|
-
return [3 /*break*/, 3];
|
|
582
|
-
case 3:
|
|
583
|
-
logger.debug("Create a bucket[".concat(bucketName, "] newly."));
|
|
584
|
-
return [4 /*yield*/, this.s3.createBucket({
|
|
585
|
-
Bucket: bucketName,
|
|
586
|
-
})];
|
|
587
|
-
case 4:
|
|
588
|
-
createResult = _a.sent();
|
|
589
|
-
logger.stupid("createResult", createResult);
|
|
590
|
-
if (!cors) return [3 /*break*/, 6];
|
|
591
|
-
return [4 /*yield*/, this.s3.putBucketCors({
|
|
592
|
-
Bucket: bucketName,
|
|
593
|
-
CORSConfiguration: {
|
|
594
|
-
CORSRules: [
|
|
595
|
-
{
|
|
596
|
-
AllowedHeaders: ['*'],
|
|
597
|
-
AllowedMethods: cors.methods,
|
|
598
|
-
AllowedOrigins: cors.origins,
|
|
599
|
-
},
|
|
600
|
-
],
|
|
601
|
-
},
|
|
602
|
-
})];
|
|
603
|
-
case 5:
|
|
604
|
-
corsResult = _a.sent();
|
|
605
|
-
logger.stupid("corsResult", corsResult);
|
|
606
|
-
_a.label = 6;
|
|
607
|
-
case 6: return [2 /*return*/, true];
|
|
205
|
+
return content;
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
if (fs.existsSync(tempFile)) {
|
|
209
|
+
try {
|
|
210
|
+
await fs.promises.unlink(tempFile);
|
|
608
211
|
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
this.setupDynamoDb = function (tableName, keyColumn) { return __awaiter(_this, void 0, void 0, function () {
|
|
612
|
-
var listResult, error_5, createResult;
|
|
613
|
-
return __generator(this, function (_a) {
|
|
614
|
-
switch (_a.label) {
|
|
615
|
-
case 0:
|
|
616
|
-
_a.trys.push([0, 2, , 3]);
|
|
617
|
-
return [4 /*yield*/, this.dynamodbAdmin.listTables()];
|
|
618
|
-
case 1:
|
|
619
|
-
listResult = _a.sent();
|
|
620
|
-
if (listResult.TableNames && listResult.TableNames.includes(tableName)) {
|
|
621
|
-
logger.debug("Table[".concat(tableName, "] already exists."));
|
|
622
|
-
return [2 /*return*/, true];
|
|
623
|
-
}
|
|
624
|
-
return [3 /*break*/, 3];
|
|
625
|
-
case 2:
|
|
626
|
-
error_5 = _a.sent();
|
|
627
|
-
logger.debug("No table[".concat(tableName, "] exists due to ").concat(error_5));
|
|
628
|
-
return [3 /*break*/, 3];
|
|
629
|
-
case 3:
|
|
630
|
-
logger.debug("Create a table[".concat(tableName, "] newly."));
|
|
631
|
-
return [4 /*yield*/, this.dynamodbAdmin.createTable({
|
|
632
|
-
TableName: tableName,
|
|
633
|
-
KeySchema: [{ AttributeName: keyColumn, KeyType: 'HASH' }],
|
|
634
|
-
AttributeDefinitions: [{ AttributeName: keyColumn, AttributeType: 'S' }],
|
|
635
|
-
ProvisionedThroughput: {
|
|
636
|
-
ReadCapacityUnits: 30,
|
|
637
|
-
WriteCapacityUnits: 10,
|
|
638
|
-
},
|
|
639
|
-
})];
|
|
640
|
-
case 4:
|
|
641
|
-
createResult = _a.sent();
|
|
642
|
-
logger.stupid("createResult", createResult);
|
|
643
|
-
return [2 /*return*/, true];
|
|
212
|
+
catch (error) {
|
|
213
|
+
logger.error(`Failed to delete temp file ${tempFile}: ${(0, utils_1.stringifyError)(error)}`);
|
|
644
214
|
}
|
|
645
|
-
});
|
|
646
|
-
}); };
|
|
647
|
-
this.config = config || new config_1.SimpleAWSConfig();
|
|
648
|
-
/**
|
|
649
|
-
* The simple cache for { queueName: queueUrl }.
|
|
650
|
-
* It can help in the only case of launching this project as offline.
|
|
651
|
-
* @type { { [queueName: string]: string } }
|
|
652
|
-
*/
|
|
653
|
-
this.queueUrls = {};
|
|
654
|
-
}
|
|
655
|
-
Object.defineProperty(SimpleAWS.prototype, "s3", {
|
|
656
|
-
get: function () {
|
|
657
|
-
if (this.lazyS3 === undefined) {
|
|
658
|
-
this.lazyS3 = new client_s3_1.S3(this.config.get(define_1.AWSComponent.s3) || {});
|
|
659
215
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
216
|
+
}
|
|
217
|
+
};
|
|
218
|
+
readFileBuffer = async (bucket, key) => {
|
|
219
|
+
logger.debug(`Read item[${key}] from bucket[${bucket}]`);
|
|
220
|
+
const { Body } = await this.s3.getObject({ Bucket: bucket, Key: key });
|
|
221
|
+
const buffer = await Body?.transformToByteArray();
|
|
222
|
+
if (!buffer) {
|
|
223
|
+
throw new Error(`Failed to read file ${key} from bucket ${bucket}`);
|
|
224
|
+
}
|
|
225
|
+
return Buffer.from(buffer);
|
|
226
|
+
};
|
|
227
|
+
upload = async (bucket, localPath, key, tags) => {
|
|
228
|
+
logger.debug(`Upload item[${key}] into bucket[${bucket}]`);
|
|
229
|
+
const upload = new lib_storage_1.Upload({
|
|
230
|
+
client: this.s3,
|
|
231
|
+
params: {
|
|
232
|
+
Bucket: bucket,
|
|
233
|
+
Key: key,
|
|
234
|
+
Body: fs.createReadStream(localPath),
|
|
235
|
+
},
|
|
236
|
+
partSize: 5 * 1024 * 1024, // 5MB
|
|
237
|
+
queueSize: 4,
|
|
238
|
+
tags: [SimpleAWS.stageTag, ...(tags || [])],
|
|
239
|
+
});
|
|
240
|
+
await upload.done();
|
|
241
|
+
return key;
|
|
242
|
+
};
|
|
243
|
+
uploadFromBuffer = async (bucket, key, buffer, tags) => {
|
|
244
|
+
logger.debug(`Upload item[${key}] into bucket[${bucket}]`);
|
|
245
|
+
const upload = new lib_storage_1.Upload({
|
|
246
|
+
client: this.s3,
|
|
247
|
+
params: {
|
|
248
|
+
Bucket: bucket,
|
|
249
|
+
Key: key,
|
|
250
|
+
Body: buffer,
|
|
251
|
+
},
|
|
252
|
+
partSize: 5 * 1024 * 1024, // 5MB
|
|
253
|
+
queueSize: 4,
|
|
254
|
+
tags: [SimpleAWS.stageTag, ...(tags || [])],
|
|
255
|
+
});
|
|
256
|
+
await upload.done();
|
|
257
|
+
return key;
|
|
258
|
+
};
|
|
259
|
+
writeFile = async (bucket, key, content) => {
|
|
260
|
+
logger.debug(`Write item[${key}] into bucket[${bucket}]`);
|
|
261
|
+
const tempFile = `${os.tmpdir()}/${(0, non_secure_1.nanoid)()}`;
|
|
262
|
+
try {
|
|
263
|
+
await fs.promises.writeFile(tempFile, content, 'utf-8');
|
|
264
|
+
await this.upload(bucket, tempFile, key);
|
|
265
|
+
}
|
|
266
|
+
finally {
|
|
267
|
+
if (!fs.existsSync(tempFile)) {
|
|
268
|
+
return;
|
|
669
269
|
}
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
270
|
+
try {
|
|
271
|
+
await fs.promises.unlink(tempFile);
|
|
272
|
+
}
|
|
273
|
+
catch (error) {
|
|
274
|
+
const msg = `Error during writeFile: unlink file ${tempFile}: ${(0, utils_1.stringifyError)(error)}`;
|
|
275
|
+
logger.error(msg);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
async getSignedUrl(options) {
|
|
280
|
+
const { expiresIn = 600, unhoistableHeaders } = options;
|
|
281
|
+
switch (options.operation) {
|
|
282
|
+
case 'putObject': {
|
|
283
|
+
const tagging = options.params?.Tagging
|
|
284
|
+
? SimpleAWS.stringifiedStageTag + '&' + options.params.Tagging
|
|
285
|
+
: SimpleAWS.stringifiedStageTag;
|
|
286
|
+
const cmd = new client_s3_1.PutObjectCommand({
|
|
287
|
+
Bucket: options.bucket,
|
|
288
|
+
Key: options.key,
|
|
289
|
+
...options.params,
|
|
290
|
+
Tagging: tagging,
|
|
291
|
+
});
|
|
292
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
293
|
+
expiresIn: expiresIn,
|
|
294
|
+
unhoistableHeaders,
|
|
683
295
|
});
|
|
684
296
|
}
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
297
|
+
case 'getObject': {
|
|
298
|
+
const cmd = new client_s3_1.GetObjectCommand({
|
|
299
|
+
Bucket: options.bucket,
|
|
300
|
+
Key: options.key,
|
|
301
|
+
...options.params,
|
|
302
|
+
});
|
|
303
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
304
|
+
expiresIn: expiresIn,
|
|
305
|
+
unhoistableHeaders,
|
|
306
|
+
});
|
|
694
307
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
308
|
+
case 'deleteObject': {
|
|
309
|
+
const cmd = new client_s3_1.DeleteObjectCommand({
|
|
310
|
+
Bucket: options.bucket,
|
|
311
|
+
Key: options.key,
|
|
312
|
+
...options.params,
|
|
313
|
+
});
|
|
314
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
315
|
+
expiresIn: expiresIn,
|
|
316
|
+
unhoistableHeaders,
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
case 'headObject': {
|
|
320
|
+
const cmd = new client_s3_1.HeadObjectCommand({
|
|
321
|
+
Bucket: options.bucket,
|
|
322
|
+
Key: options.key,
|
|
323
|
+
...options.params,
|
|
324
|
+
});
|
|
325
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
326
|
+
expiresIn: expiresIn,
|
|
327
|
+
unhoistableHeaders,
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
case 'copyObject': {
|
|
331
|
+
const cmd = new client_s3_1.CopyObjectCommand({
|
|
332
|
+
Bucket: options.bucket,
|
|
333
|
+
Key: options.key,
|
|
334
|
+
...options.params,
|
|
335
|
+
});
|
|
336
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
337
|
+
expiresIn: expiresIn,
|
|
338
|
+
unhoistableHeaders,
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
case 'uploadPart': {
|
|
342
|
+
const cmd = new client_s3_1.UploadPartCommand({
|
|
343
|
+
Bucket: options.bucket,
|
|
344
|
+
Key: options.key,
|
|
345
|
+
...options.params,
|
|
346
|
+
});
|
|
347
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
348
|
+
expiresIn: expiresIn,
|
|
349
|
+
unhoistableHeaders,
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
case 'uploadPartCopy': {
|
|
353
|
+
const cmd = new client_s3_1.UploadPartCopyCommand({
|
|
354
|
+
Bucket: options.bucket,
|
|
355
|
+
Key: options.key,
|
|
356
|
+
...options.params,
|
|
357
|
+
});
|
|
358
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
359
|
+
expiresIn: expiresIn,
|
|
360
|
+
unhoistableHeaders,
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
case 'listObjectsV2': {
|
|
364
|
+
const cmd = new client_s3_1.ListObjectsV2Command({
|
|
365
|
+
Bucket: options.bucket,
|
|
366
|
+
...options.params,
|
|
367
|
+
});
|
|
368
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
369
|
+
expiresIn: expiresIn,
|
|
370
|
+
unhoistableHeaders,
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
case 'createMultipartUpload': {
|
|
374
|
+
const tagging = options.params?.Tagging
|
|
375
|
+
? SimpleAWS.stringifiedStageTag + '&' + options.params.Tagging
|
|
376
|
+
: SimpleAWS.stringifiedStageTag;
|
|
377
|
+
const cmd = new client_s3_1.CreateMultipartUploadCommand({
|
|
378
|
+
Bucket: options.bucket,
|
|
379
|
+
Key: options.key,
|
|
380
|
+
...options.params,
|
|
381
|
+
Tagging: tagging,
|
|
382
|
+
});
|
|
383
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
384
|
+
expiresIn: expiresIn,
|
|
385
|
+
unhoistableHeaders,
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
case 'completeMultipartUpload': {
|
|
389
|
+
const cmd = new client_s3_1.CompleteMultipartUploadCommand({
|
|
390
|
+
Bucket: options.bucket,
|
|
391
|
+
Key: options.key,
|
|
392
|
+
...options.params,
|
|
393
|
+
});
|
|
394
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
395
|
+
expiresIn: expiresIn,
|
|
396
|
+
unhoistableHeaders,
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
case 'abortMultipartUpload': {
|
|
400
|
+
const cmd = new client_s3_1.AbortMultipartUploadCommand({
|
|
401
|
+
Bucket: options.bucket,
|
|
402
|
+
Key: options.key,
|
|
403
|
+
...options.params,
|
|
404
|
+
});
|
|
405
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
406
|
+
expiresIn: expiresIn,
|
|
407
|
+
unhoistableHeaders,
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
case 'listParts': {
|
|
411
|
+
const cmd = new client_s3_1.ListPartsCommand({
|
|
412
|
+
Bucket: options.bucket,
|
|
413
|
+
Key: options.key,
|
|
414
|
+
...options.params,
|
|
415
|
+
});
|
|
416
|
+
return (0, s3_request_presigner_1.getSignedUrl)(this.s3, cmd, {
|
|
417
|
+
expiresIn: expiresIn,
|
|
418
|
+
unhoistableHeaders,
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
getSignedCookie = (keyPairId, privateKey, url, expires) => {
|
|
424
|
+
const policy = JSON.stringify({
|
|
425
|
+
Statement: [
|
|
426
|
+
{
|
|
427
|
+
Resource: url,
|
|
428
|
+
Condition: {
|
|
429
|
+
DateLessThan: { 'AWS:EpochTime': expires },
|
|
430
|
+
},
|
|
431
|
+
},
|
|
432
|
+
],
|
|
433
|
+
});
|
|
434
|
+
return (0, cloudfront_signer_1.getSignedCookies)({
|
|
435
|
+
keyPairId,
|
|
436
|
+
privateKey,
|
|
437
|
+
policy,
|
|
438
|
+
});
|
|
439
|
+
};
|
|
440
|
+
getDynamoDbItem = async (tableName, key, defaultValue) => {
|
|
441
|
+
logger.debug(`Read an item with key[${JSON.stringify(key)}] from ${tableName}.`);
|
|
442
|
+
const getResult = await this.dynamodb.get({
|
|
443
|
+
TableName: tableName,
|
|
444
|
+
Key: key,
|
|
445
|
+
});
|
|
446
|
+
logger.stupid(`getResult`, getResult);
|
|
447
|
+
const item = getResult !== undefined && getResult.Item !== undefined
|
|
448
|
+
? getResult.Item // Casts forcefully.
|
|
449
|
+
: defaultValue;
|
|
450
|
+
logger.stupid(`item`, item);
|
|
451
|
+
return item;
|
|
452
|
+
};
|
|
453
|
+
updateDynamoDbItem = async (tableName, key, columnValues) => {
|
|
454
|
+
logger.debug(`Update an item with key[${JSON.stringify(key)}] to ${tableName}`);
|
|
455
|
+
logger.stupid(`keyValues`, columnValues);
|
|
456
|
+
const expressions = Object.keys(columnValues)
|
|
457
|
+
.map((column) => `${column} = :${column}`)
|
|
458
|
+
.join(', ');
|
|
459
|
+
const attributeValues = Object.keys(columnValues)
|
|
460
|
+
.map((column) => [`:${column}`, columnValues[column]])
|
|
461
|
+
.reduce((obj, pair) => ({ ...obj, [pair[0]]: pair[1] }), {});
|
|
462
|
+
logger.stupid(`expressions`, expressions);
|
|
463
|
+
logger.stupid(`attributeValues`, attributeValues);
|
|
464
|
+
const updateResult = await this.dynamodb.update({
|
|
465
|
+
TableName: tableName,
|
|
466
|
+
Key: key,
|
|
467
|
+
UpdateExpression: `set ${expressions}`,
|
|
468
|
+
ExpressionAttributeValues: attributeValues,
|
|
469
|
+
});
|
|
470
|
+
logger.stupid(`updateResult`, updateResult);
|
|
471
|
+
return updateResult;
|
|
472
|
+
};
|
|
473
|
+
// Setup
|
|
474
|
+
setupQueue = async (queueName) => {
|
|
475
|
+
try {
|
|
476
|
+
const listResult = await this.sqs.listQueues({
|
|
477
|
+
QueueNamePrefix: queueName,
|
|
478
|
+
});
|
|
479
|
+
if (listResult.QueueUrls) {
|
|
480
|
+
for (const queueUrl of listResult.QueueUrls) {
|
|
481
|
+
if (queueUrl.endsWith(queueName)) {
|
|
482
|
+
logger.debug(`Queue[${queueName} => ${queueUrl}] already exists.`);
|
|
483
|
+
return true;
|
|
796
484
|
}
|
|
797
485
|
}
|
|
798
|
-
|
|
799
|
-
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
catch (error) {
|
|
489
|
+
logger.debug(`No Queue[${queueName}] exists due to ${error}`);
|
|
490
|
+
}
|
|
491
|
+
logger.debug(`Create a queue[${queueName}] newly.`);
|
|
492
|
+
const createResult = await this.sqs.createQueue({
|
|
493
|
+
QueueName: queueName,
|
|
800
494
|
});
|
|
495
|
+
logger.stupid(`createResult`, createResult);
|
|
496
|
+
return true;
|
|
801
497
|
};
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
498
|
+
setupStorage = async (bucketName, cors) => {
|
|
499
|
+
try {
|
|
500
|
+
const listResult = await this.s3.listBuckets();
|
|
501
|
+
if (listResult.Buckets &&
|
|
502
|
+
listResult.Buckets.map((each) => each.Name).includes(bucketName)) {
|
|
503
|
+
logger.debug(`Bucket[${bucketName}] already exists.`);
|
|
504
|
+
return true;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
catch (error) {
|
|
508
|
+
logger.debug(`No bucket[${bucketName}] exists due to ${error}`);
|
|
509
|
+
}
|
|
510
|
+
logger.debug(`Create a bucket[${bucketName}] newly.`);
|
|
511
|
+
const createResult = await this.s3.createBucket({
|
|
512
|
+
Bucket: bucketName,
|
|
513
|
+
});
|
|
514
|
+
logger.stupid(`createResult`, createResult);
|
|
515
|
+
if (cors) {
|
|
516
|
+
const corsResult = await this.s3.putBucketCors({
|
|
517
|
+
Bucket: bucketName,
|
|
518
|
+
CORSConfiguration: {
|
|
519
|
+
CORSRules: [
|
|
520
|
+
{
|
|
521
|
+
AllowedHeaders: ['*'],
|
|
522
|
+
AllowedMethods: cors.methods,
|
|
523
|
+
AllowedOrigins: cors.origins,
|
|
524
|
+
},
|
|
525
|
+
],
|
|
526
|
+
},
|
|
527
|
+
});
|
|
528
|
+
logger.stupid(`corsResult`, corsResult);
|
|
529
|
+
}
|
|
530
|
+
return true;
|
|
531
|
+
};
|
|
532
|
+
setupDynamoDb = async (tableName, keyColumn) => {
|
|
533
|
+
try {
|
|
534
|
+
const listResult = await this.dynamodbAdmin.listTables();
|
|
535
|
+
if (listResult.TableNames && listResult.TableNames.includes(tableName)) {
|
|
536
|
+
logger.debug(`Table[${tableName}] already exists.`);
|
|
537
|
+
return true;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
catch (error) {
|
|
541
|
+
logger.debug(`No table[${tableName}] exists due to ${error}`);
|
|
542
|
+
}
|
|
543
|
+
logger.debug(`Create a table[${tableName}] newly.`);
|
|
544
|
+
const createResult = await this.dynamodbAdmin.createTable({
|
|
545
|
+
TableName: tableName,
|
|
546
|
+
KeySchema: [{ AttributeName: keyColumn, KeyType: 'HASH' }],
|
|
547
|
+
AttributeDefinitions: [{ AttributeName: keyColumn, AttributeType: 'S' }],
|
|
548
|
+
ProvisionedThroughput: {
|
|
549
|
+
ReadCapacityUnits: 30,
|
|
550
|
+
WriteCapacityUnits: 10,
|
|
551
|
+
},
|
|
552
|
+
});
|
|
553
|
+
logger.stupid(`createResult`, createResult);
|
|
554
|
+
return true;
|
|
805
555
|
};
|
|
806
|
-
|
|
807
|
-
return SimpleAWS;
|
|
808
|
-
}());
|
|
556
|
+
}
|
|
809
557
|
exports.SimpleAWS = SimpleAWS;
|