teraslice 0.79.0 → 0.80.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/lib/config/default-sysconfig.js +5 -0
  2. package/lib/storage/backends/elasticsearch_store.js +35 -91
  3. package/package.json +20 -20
package/lib/config/default-sysconfig.js CHANGED
@@ -16,6 +16,11 @@ function getConnectors() {
16
16
  default: {
17
17
  host: ['localhost:9200']
18
18
  }
19
+ },
20
+ 'elasticsearch-next': {
21
+ default: {
22
+ node: ['localhost:9200']
23
+ }
19
24
  }
20
25
  };
21
26
 
package/lib/storage/backends/elasticsearch_store.js CHANGED
@@ -4,34 +4,20 @@ const ms = require('ms');
4
4
  const fs = require('fs');
5
5
  const path = require('path');
6
6
  const {
7
- TSError,
8
- parseError,
9
- isTest,
10
- pDelay,
11
- pRetry,
12
- logError,
13
- pWhile,
14
- isString,
15
- getTypeOf,
16
- get,
17
- random,
18
- isInteger
7
+ TSError, parseError, isTest, pDelay,
8
+ pRetry, logError, pWhile, isString, getTypeOf,
9
+ get, random, isInteger
19
10
  } = require('@terascope/utils');
20
11
  const elasticsearchApi = require('@terascope/elasticsearch-api');
21
- const { getClient } = require('@terascope/job-components');
12
+ const { getClientAsync } = require('@terascope/job-components');
22
13
  const { makeLogger } = require('../../workers/helpers/terafoundation');
23
14
  const { timeseriesIndex } = require('../../utils/date_utils');
24
15
 
25
- module.exports = function elasticsearchStorage(backendConfig) {
16
+ module.exports = async function elasticsearchStorage(backendConfig) {
26
17
  const {
27
- context,
28
- indexName,
29
- recordType,
30
- idField,
31
- storageName,
32
- bulkSize = 1000,
33
- fullResponse = false,
34
- logRecord = true,
18
+ context, indexName, recordType,
19
+ idField, storageName, bulkSize = 1000,
20
+ fullResponse = false, logRecord = true,
35
21
  forceRefresh = true,
36
22
  } = backendConfig;
37
23
 
@@ -83,8 +69,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
83
69
  };
84
70
 
85
71
  if (fields) {
86
- const esVersion = elasticsearch.getESVersion();
87
- if (esVersion > 6) {
72
+ if (!elasticsearch.isElasticsearch6()) {
88
73
  query._sourceIncludes = fields;
89
74
  } else {
90
75
  query._sourceInclude = fields;
@@ -118,8 +103,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
118
103
  }
119
104
 
120
105
  if (fields) {
121
- const esVersion = elasticsearch.getESVersion();
122
- if (esVersion > 6) {
106
+ if (!elasticsearch.isElasticsearch6()) {
123
107
  esQuery._sourceIncludes = fields;
124
108
  } else {
125
109
  esQuery._sourceInclude = fields;
@@ -273,9 +257,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
273
257
  refresh: forceRefresh,
274
258
  };
275
259
 
276
- const esVersion = elasticsearch.getESVersion();
277
-
278
- if (esVersion >= 7) {
260
+ if (!elasticsearch.isElasticsearch6()) {
279
261
  query.if_seq_no = existing._seq_no;
280
262
  query.if_primary_term = existing._primary_term;
281
263
  } else {
@@ -527,7 +509,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
527
509
  newIndex = timeseriesIndex(timeseriesFormat, indexName.slice(0, nameSize)).index;
528
510
  }
529
511
 
530
- return new Promise((resolve, reject) => {
512
+ async function setup() {
531
513
  const clientName = JSON.stringify({
532
514
  connection: config.state.connection,
533
515
  index: indexName,
@@ -537,11 +519,6 @@ module.exports = function elasticsearchStorage(backendConfig) {
537
519
  if (connectionConfig.connection_cache == null) {
538
520
  connectionConfig.connection_cache = true;
539
521
  }
540
- client = getClient(context, connectionConfig, 'elasticsearch');
541
- if (!client) {
542
- reject(new Error(`Unable to get client for connection: ${config.state.connection}`));
543
- return;
544
- }
545
522
 
546
523
  let { connection } = config.state;
547
524
  if (config.state.endpoint) {
@@ -553,67 +530,34 @@ module.exports = function elasticsearchStorage(backendConfig) {
553
530
  connection,
554
531
  };
555
532
 
556
- elasticsearch = elasticsearchApi(client, logger, options);
557
- _createIndex(newIndex)
558
- .then(() => elasticsearch.isAvailable(newIndex, recordType))
559
- .then(() => resolve(api))
560
- .catch((err) => {
533
+ await pWhile(async () => {
534
+ try {
535
+ client = await getClientAsync(context, connectionConfig, 'elasticsearch-next');
536
+ elasticsearch = elasticsearchApi(client, logger, options);
537
+
538
+ await _createIndex(newIndex);
539
+ await elasticsearch.isAvailable(newIndex, recordType);
540
+
541
+ return true;
542
+ } catch (err) {
561
543
  const error = new TSError(err, {
562
544
  reason: `Failure initializing ${recordType} index: ${indexName}`,
563
545
  });
546
+
564
547
  if (error.statusCode >= 400 && error.statusCode < 500) {
565
- reject(err);
566
- return;
548
+ throw error;
567
549
  }
568
550
 
569
551
  logError(logger, error, `Failed attempt connecting to elasticsearch: ${clientName} (will retry)`);
570
- let running = false;
571
-
572
- const checking = setInterval(() => {
573
- if (isShutdown) {
574
- clearInterval(checking);
575
- return;
576
- }
577
- if (running) return;
578
- running = true;
579
-
580
- _createIndex(newIndex)
581
- .then(() => {
582
- const query = { index: newIndex };
583
- return elasticsearch.index_recovery(query);
584
- })
585
- .then((results) => {
586
- let bool = false;
587
- const shards = get(results, [newIndex, 'shards'], []);
588
- if (shards.length) {
589
- bool = shards
590
- .filter((shard) => shard.primary)
591
- .every((shard) => shard.stage === 'DONE');
592
- }
593
-
594
- if (bool) {
595
- clearInterval(checking);
596
- logger.info('connection to elasticsearch has been established');
597
- return elasticsearch.isAvailable(newIndex, recordType).then(() => {
598
- resolve(api);
599
- });
600
- }
601
- return true;
602
- })
603
- .then(() => {
604
- running = false;
605
- })
606
- .catch((checkingErr) => {
607
- // add a random delay to stagger requests
608
- pDelay(isTest ? 0 : random(0, 1000)).then(() => {
609
- running = false;
610
- const checkingError = new TSError(checkingErr, {
611
- reason: `Attempting to connect to elasticsearch: ${clientName}`,
612
- });
613
- logger.info(checkingError.message);
614
- });
615
- });
616
- }, 3000);
617
- });
618
- });
552
+
553
+ await pDelay(isTest ? 0 : random(2000, 4000));
554
+
555
+ return false;
556
+ }
557
+ });
558
+
559
+ return api;
560
+ }
561
+
562
+ return setup();
619
563
  };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "teraslice",
3
3
  "displayName": "Teraslice",
4
- "version": "0.79.0",
4
+ "version": "0.80.1",
5
5
  "description": "Distributed computing platform for processing JSON data",
6
6
  "homepage": "https://github.com/terascope/teraslice#readme",
7
7
  "bugs": {
@@ -33,50 +33,50 @@
33
33
  "test:watch": "ts-scripts test --watch . --"
34
34
  },
35
35
  "resolutions": {
36
- "debug": "^4.3.3",
36
+ "debug": "^4.3.4",
37
37
  "ms": "^2.1.3"
38
38
  },
39
39
  "dependencies": {
40
- "@terascope/elasticsearch-api": "^3.0.2",
41
- "@terascope/job-components": "^0.56.3",
42
- "@terascope/teraslice-messaging": "^0.27.1",
43
- "@terascope/utils": "^0.44.1",
44
- "async-mutex": "^0.3.2",
40
+ "@terascope/elasticsearch-api": "^3.3.0",
41
+ "@terascope/job-components": "^0.58.0",
42
+ "@terascope/teraslice-messaging": "^0.28.0",
43
+ "@terascope/utils": "^0.45.0",
44
+ "async-mutex": "^0.4.0",
45
45
  "barbe": "^3.0.16",
46
- "body-parser": "^1.19.1",
47
- "convict": "^4.4.1",
46
+ "body-parser": "^1.20.0",
47
+ "convict": "^6.2.3",
48
48
  "decompress": "^4.2.1",
49
49
  "easy-table": "^1.2.0",
50
- "event-loop-stats": "^1.2.0",
51
- "express": "^4.17.2",
52
- "fs-extra": "^10.0.0",
50
+ "event-loop-stats": "^1.4.1",
51
+ "express": "^4.18.1",
52
+ "fs-extra": "^10.1.0",
53
53
  "gc-stats": "^1.4.0",
54
54
  "got": "^11.8.3",
55
- "ip": "^1.1.5",
55
+ "ip": "^1.1.8",
56
56
  "kubernetes-client": "^9.0.0",
57
57
  "lodash": "^4.17.21",
58
58
  "ms": "^2.1.3",
59
- "nanoid": "^3.2.0",
59
+ "nanoid": "^3.3.4",
60
60
  "porty": "^3.1.1",
61
- "semver": "^7.3.5",
61
+ "semver": "^7.3.8",
62
62
  "socket.io": "^1.7.4",
63
63
  "socket.io-client": "^1.7.4",
64
- "terafoundation": "^0.37.1",
65
- "uuid": "^8.3.2"
64
+ "terafoundation": "^0.40.0",
65
+ "uuid": "^9.0.0"
66
66
  },
67
67
  "devDependencies": {
68
68
  "@terascope/teraslice-op-test-harness": "^1.24.1",
69
- "archiver": "^5.3.0",
69
+ "archiver": "^5.3.1",
70
70
  "bufferstreams": "^3.0.0",
71
71
  "chance": "^1.1.8",
72
72
  "elasticsearch": "^15.4.1",
73
73
  "got": "^11.8.3",
74
74
  "jest-fixtures": "^0.6.0",
75
75
  "js-yaml": "^4.1.0",
76
- "nock": "^13.2.1"
76
+ "nock": "^13.2.9"
77
77
  },
78
78
  "engines": {
79
- "node": "^12.22.0 || >=14.17.0",
79
+ "node": ">=14.17.0",
80
80
  "yarn": ">=1.16.0"
81
81
  },
82
82
  "publishConfig": {