teraslice 0.80.0 → 0.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/lib/storage/backends/elasticsearch_store.js +35 -88
  2. package/lib/storage/state.js +2 -0
  3. package/package.json +17 -17
package/lib/storage/backends/elasticsearch_store.js CHANGED
@@ -4,34 +4,20 @@ const ms = require('ms');
4
4
  const fs = require('fs');
5
5
  const path = require('path');
6
6
  const {
7
- TSError,
8
- parseError,
9
- isTest,
10
- pDelay,
11
- pRetry,
12
- logError,
13
- pWhile,
14
- isString,
15
- getTypeOf,
16
- get,
17
- random,
18
- isInteger
7
+ TSError, parseError, isTest, pDelay,
8
+ pRetry, logError, pWhile, isString, getTypeOf,
9
+ get, random, isInteger
19
10
  } = require('@terascope/utils');
20
11
  const elasticsearchApi = require('@terascope/elasticsearch-api');
21
12
  const { getClientAsync } = require('@terascope/job-components');
22
13
  const { makeLogger } = require('../../workers/helpers/terafoundation');
23
14
  const { timeseriesIndex } = require('../../utils/date_utils');
24
15
 
25
- module.exports = function elasticsearchStorage(backendConfig) {
16
+ module.exports = async function elasticsearchStorage(backendConfig) {
26
17
  const {
27
- context,
28
- indexName,
29
- recordType,
30
- idField,
31
- storageName,
32
- bulkSize = 1000,
33
- fullResponse = false,
34
- logRecord = true,
18
+ context, indexName, recordType,
19
+ idField, storageName, bulkSize = 1000,
20
+ fullResponse = false, logRecord = true,
35
21
  forceRefresh = true,
36
22
  } = backendConfig;
37
23
 
@@ -107,6 +93,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
107
93
  index: indexArg,
108
94
  from: from != null ? from : 0,
109
95
  size: size != null ? size : 10000,
96
+ type: recordType,
110
97
  sort,
111
98
  };
112
99
 
@@ -204,6 +191,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
204
191
 
205
192
  const esQuery = {
206
193
  index: indexArg,
194
+ type: recordType,
207
195
  from,
208
196
  sort,
209
197
  };
@@ -214,7 +202,9 @@ module.exports = function elasticsearchStorage(backendConfig) {
214
202
  esQuery.body = query;
215
203
  }
216
204
 
217
- return elasticsearch.count(esQuery);
205
+ const response = await elasticsearch.count(esQuery);
206
+
207
+ return response;
218
208
  }
219
209
 
220
210
  async function update(recordId, updateSpec, indexArg = indexName) {
@@ -523,7 +513,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
523
513
  newIndex = timeseriesIndex(timeseriesFormat, indexName.slice(0, nameSize)).index;
524
514
  }
525
515
 
526
- return new Promise((resolve, reject) => {
516
+ async function setup() {
527
517
  const clientName = JSON.stringify({
528
518
  connection: config.state.connection,
529
519
  index: indexName,
@@ -544,77 +534,34 @@ module.exports = function elasticsearchStorage(backendConfig) {
544
534
  connection,
545
535
  };
546
536
 
547
- Promise.resolve()
548
- .then(() => getClientAsync(context, connectionConfig, 'elasticsearch-next'))
549
- .then((esClient) => {
550
- client = esClient;
551
- if (!client) {
552
- reject(new Error(`Unable to get client for connection: ${config.state.connection}`));
553
- return;
554
- }
537
+ await pWhile(async () => {
538
+ try {
539
+ client = await getClientAsync(context, connectionConfig, 'elasticsearch-next');
555
540
  elasticsearch = elasticsearchApi(client, logger, options);
556
- // eslint-disable-next-line consistent-return
557
- return _createIndex(newIndex);
558
- })
559
- .then(() => elasticsearch.isAvailable(newIndex, recordType))
560
- .then(() => resolve(api))
561
- .catch((err) => {
541
+
542
+ await _createIndex(newIndex);
543
+ await elasticsearch.isAvailable(newIndex, recordType);
544
+
545
+ return true;
546
+ } catch (err) {
562
547
  const error = new TSError(err, {
563
548
  reason: `Failure initializing ${recordType} index: ${indexName}`,
564
549
  });
550
+
565
551
  if (error.statusCode >= 400 && error.statusCode < 500) {
566
- reject(err);
567
- return;
552
+ throw error;
568
553
  }
569
554
 
570
555
  logError(logger, error, `Failed attempt connecting to elasticsearch: ${clientName} (will retry)`);
571
- let running = false;
572
-
573
- const checking = setInterval(() => {
574
- if (isShutdown) {
575
- clearInterval(checking);
576
- return;
577
- }
578
- if (running) return;
579
- running = true;
580
-
581
- _createIndex(newIndex)
582
- .then(() => {
583
- const query = { index: newIndex };
584
- return elasticsearch.index_recovery(query);
585
- })
586
- .then((results) => {
587
- let bool = false;
588
- const shards = get(results, [newIndex, 'shards'], []);
589
- if (shards.length) {
590
- bool = shards
591
- .filter((shard) => shard.primary)
592
- .every((shard) => shard.stage === 'DONE');
593
- }
594
-
595
- if (bool) {
596
- clearInterval(checking);
597
- logger.info('connection to elasticsearch has been established');
598
- return elasticsearch.isAvailable(newIndex, recordType).then(() => {
599
- resolve(api);
600
- });
601
- }
602
- return true;
603
- })
604
- .then(() => {
605
- running = false;
606
- })
607
- .catch((checkingErr) => {
608
- // add a random delay to stagger requests
609
- pDelay(isTest ? 0 : random(0, 1000)).then(() => {
610
- running = false;
611
- const checkingError = new TSError(checkingErr, {
612
- reason: `Attempting to connect to elasticsearch: ${clientName}`,
613
- });
614
- logger.info(checkingError.message);
615
- });
616
- });
617
- }, 3000);
618
- });
619
- });
556
+
557
+ await pDelay(isTest ? 0 : random(2000, 4000));
558
+
559
+ return false;
560
+ }
561
+ });
562
+
563
+ return api;
564
+ }
565
+
566
+ return setup();
620
567
  };
package/lib/storage/state.js CHANGED
@@ -54,6 +54,7 @@ async function stateStorage(context) {
54
54
  await waitForClient();
55
55
 
56
56
  const { record, index } = _createSliceRecord(exId, slice, state, error);
57
+
57
58
  return backend.indexWithId(slice.slice_id, record, index);
58
59
  }
59
60
 
@@ -73,6 +74,7 @@ async function stateStorage(context) {
73
74
  data: record
74
75
  };
75
76
  });
77
+
76
78
  return backend.bulkSend(bulkRequest);
77
79
  }
78
80
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "teraslice",
3
3
  "displayName": "Teraslice",
4
- "version": "0.80.0",
4
+ "version": "0.81.0",
5
5
  "description": "Distributed computing platform for processing JSON data",
6
6
  "homepage": "https://github.com/terascope/teraslice#readme",
7
7
  "bugs": {
@@ -37,46 +37,46 @@
37
37
  "ms": "^2.1.3"
38
38
  },
39
39
  "dependencies": {
40
- "@terascope/elasticsearch-api": "^3.2.0",
41
- "@terascope/job-components": "^0.57.0",
42
- "@terascope/teraslice-messaging": "^0.27.2",
43
- "@terascope/utils": "^0.44.2",
44
- "async-mutex": "^0.3.2",
40
+ "@terascope/elasticsearch-api": "^3.3.1",
41
+ "@terascope/job-components": "^0.58.1",
42
+ "@terascope/teraslice-messaging": "^0.28.1",
43
+ "@terascope/utils": "^0.45.1",
44
+ "async-mutex": "^0.4.0",
45
45
  "barbe": "^3.0.16",
46
46
  "body-parser": "^1.20.0",
47
- "convict": "^4.4.1",
47
+ "convict": "^6.2.3",
48
48
  "decompress": "^4.2.1",
49
49
  "easy-table": "^1.2.0",
50
50
  "event-loop-stats": "^1.4.1",
51
- "express": "^4.17.3",
52
- "fs-extra": "^10.0.1",
51
+ "express": "^4.18.1",
52
+ "fs-extra": "^10.1.0",
53
53
  "gc-stats": "^1.4.0",
54
54
  "got": "^11.8.3",
55
- "ip": "^1.1.5",
55
+ "ip": "^1.1.8",
56
56
  "kubernetes-client": "^9.0.0",
57
57
  "lodash": "^4.17.21",
58
58
  "ms": "^2.1.3",
59
- "nanoid": "^3.3.2",
59
+ "nanoid": "^3.3.4",
60
60
  "porty": "^3.1.1",
61
- "semver": "^7.3.6",
61
+ "semver": "^7.3.8",
62
62
  "socket.io": "^1.7.4",
63
63
  "socket.io-client": "^1.7.4",
64
- "terafoundation": "^0.39.0",
65
- "uuid": "^8.3.2"
64
+ "terafoundation": "^0.41.0",
65
+ "uuid": "^9.0.0"
66
66
  },
67
67
  "devDependencies": {
68
68
  "@terascope/teraslice-op-test-harness": "^1.24.1",
69
- "archiver": "^5.3.0",
69
+ "archiver": "^5.3.1",
70
70
  "bufferstreams": "^3.0.0",
71
71
  "chance": "^1.1.8",
72
72
  "elasticsearch": "^15.4.1",
73
73
  "got": "^11.8.3",
74
74
  "jest-fixtures": "^0.6.0",
75
75
  "js-yaml": "^4.1.0",
76
- "nock": "^13.2.4"
76
+ "nock": "^13.2.9"
77
77
  },
78
78
  "engines": {
79
- "node": "^12.22.0 || >=14.17.0",
79
+ "node": ">=14.17.0",
80
80
  "yarn": ">=1.16.0"
81
81
  },
82
82
  "publishConfig": {