swift-multi 0.0.10 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm-shrinkwrap.json +2 -2
- package/package.json +2 -2
- package/src/chunk_uploader.js +18 -2
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "swift-multi",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.11",
|
|
4
4
|
"lockfileVersion": 2,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "swift-multi",
|
|
9
|
-
"version": "0.0.
|
|
9
|
+
"version": "0.0.11",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"async": "3.2.0",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "swift-multi",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.11",
|
|
4
4
|
"description": "Multi stream upload to openstack swift store",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Jim Lake"
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
],
|
|
20
20
|
"scripts": {
|
|
21
21
|
"lint": "eslint src/**.js",
|
|
22
|
-
"pretty": "prettier --write src
|
|
22
|
+
"pretty": "prettier --write src/**/*.js test/**/*.js"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
25
|
"async": "3.2.0",
|
package/src/chunk_uploader.js
CHANGED
|
@@ -9,6 +9,7 @@ exports.sendChunks = sendChunks;
|
|
|
9
9
|
|
|
10
10
|
const REQ_TIMEOUT = 2 * 60 * 1000;
|
|
11
11
|
const BACKOFF_RETRY_MS = 60 * 1000;
|
|
12
|
+
|
|
12
13
|
const g_ipErrorMap = {};
|
|
13
14
|
|
|
14
15
|
function sendChunks(params, done) {
|
|
@@ -25,6 +26,10 @@ function sendChunks(params, done) {
|
|
|
25
26
|
progress,
|
|
26
27
|
} = params;
|
|
27
28
|
|
|
29
|
+
chunk_list.forEach((chunk) => {
|
|
30
|
+
chunk.fatal_error = false;
|
|
31
|
+
});
|
|
32
|
+
|
|
28
33
|
const max_inflight_count = ip_list.length
|
|
29
34
|
? ip_list.length * send_per_ip
|
|
30
35
|
: send_per_ip;
|
|
@@ -36,7 +41,12 @@ function sendChunks(params, done) {
|
|
|
36
41
|
let byte_count = 0;
|
|
37
42
|
|
|
38
43
|
function _startSends() {
|
|
39
|
-
if (
|
|
44
|
+
if (_isFatal(chunk_list)) {
|
|
45
|
+
if (!is_complete) {
|
|
46
|
+
is_complete = true;
|
|
47
|
+
done('send_failed');
|
|
48
|
+
}
|
|
49
|
+
} else if (_isSendDone(chunk_list)) {
|
|
40
50
|
if (!is_complete) {
|
|
41
51
|
is_complete = true;
|
|
42
52
|
done(null, byte_count);
|
|
@@ -81,6 +91,9 @@ function sendChunks(params, done) {
|
|
|
81
91
|
_startSends();
|
|
82
92
|
}
|
|
83
93
|
|
|
94
|
+
function _isFatal(chunk_list) {
|
|
95
|
+
return chunk_list.some((chunk) => chunk.fatal_error);
|
|
96
|
+
}
|
|
84
97
|
function _isSendDone(chunk_list) {
|
|
85
98
|
return chunk_list.every((chunk) => chunk.is_done);
|
|
86
99
|
}
|
|
@@ -235,6 +248,9 @@ function _sendChunk(params, done) {
|
|
|
235
248
|
_send(opts, (err, body) => {
|
|
236
249
|
if (err) {
|
|
237
250
|
errorLog('send chunk err:', err.code ? err.code : err, body);
|
|
251
|
+
if (err === 422 || err === 404) {
|
|
252
|
+
chunk.fatal_error = true;
|
|
253
|
+
}
|
|
238
254
|
} else {
|
|
239
255
|
chunk.is_done = true;
|
|
240
256
|
}
|
|
@@ -244,7 +260,7 @@ function _sendChunk(params, done) {
|
|
|
244
260
|
},
|
|
245
261
|
],
|
|
246
262
|
(err) => {
|
|
247
|
-
if (err) {
|
|
263
|
+
if (err && !chunk.fatal_error) {
|
|
248
264
|
// backoff on retry
|
|
249
265
|
setTimeout(() => {
|
|
250
266
|
g_ipErrorMap[ip] = (g_ipErrorMap[ip] || 0) + 1;
|