teraslice 3.0.0-dev.4 → 3.0.0-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/dist/src/lib/storage/backends/elasticsearch_store.js +6 -36
  2. package/dist/src/lib/storage/backends/mappings/analytics.js +37 -42
  3. package/dist/src/lib/storage/backends/mappings/asset.js +29 -34
  4. package/dist/src/lib/storage/backends/mappings/ex.js +48 -53
  5. package/dist/src/lib/storage/backends/mappings/job.js +21 -26
  6. package/dist/src/lib/storage/backends/mappings/state.js +24 -29
  7. package/dist/src/lib/storage/state.js +0 -1
  8. package/package.json +9 -10
package/dist/src/lib/storage/backends/elasticsearch_store.js CHANGED
@@ -127,7 +127,7 @@ export class TerasliceElasticsearchStorage {
127
127
  const client = await getClient(this.context, connectionConfig, 'elasticsearch-next');
128
128
  this.api = elasticsearchApi(client, this.logger, options);
129
129
  await this._createIndex(newIndex);
130
- await this.api.isAvailable(newIndex, this.recordType);
130
+ await this.api.isAvailable(newIndex);
131
131
  return true;
132
132
  }
133
133
  catch (err) {
@@ -157,17 +157,10 @@ export class TerasliceElasticsearchStorage {
157
157
  this.logger.trace(`getting record id: ${recordId}`);
158
158
  const query = {
159
159
  index,
160
- type: this.recordType,
161
160
  id: recordId,
162
161
  };
163
162
  if (fields) {
164
- if (!this.api.isElasticsearch6()) {
165
- query._source_includes = fields;
166
- }
167
- else {
168
- // @ts-expect-error backwards compatible
169
- query._sourceInclude = fields;
170
- }
163
+ query._source_includes = fields;
171
164
  }
172
165
  return this.api.get(query);
173
166
  }
@@ -185,7 +178,6 @@ export class TerasliceElasticsearchStorage {
185
178
  index,
186
179
  from,
187
180
  size,
188
- type: this.recordType,
189
181
  sort,
190
182
  };
191
183
  if (typeof query === 'string') {
@@ -195,13 +187,7 @@ export class TerasliceElasticsearchStorage {
195
187
  esQuery.body = query;
196
188
  }
197
189
  if (fields) {
198
- if (!this.api.isElasticsearch6()) {
199
- esQuery._source_includes = fields;
200
- }
201
- else {
202
- // @ts-expect-error backwards compatible
203
- esQuery._sourceInclude = fields;
204
- }
190
+ esQuery._source_includes = fields;
205
191
  }
206
192
  return this.api.search(esQuery);
207
193
  }
@@ -213,7 +199,6 @@ export class TerasliceElasticsearchStorage {
213
199
  this.logger.trace('indexing record', this.options.logRecord ? record : undefined);
214
200
  const query = {
215
201
  index: indexArg,
216
- type: this.recordType,
217
202
  body: record,
218
203
  refresh: this.options.forceRefresh,
219
204
  };
@@ -228,7 +213,6 @@ export class TerasliceElasticsearchStorage {
228
213
  this.logger.trace(`indexWithId call with id: ${recordId}, record`, this.options.logRecord ? record : null);
229
214
  const query = {
230
215
  index,
231
- type: this.recordType,
232
216
  id: recordId,
233
217
  body: record,
234
218
  refresh: this.options.forceRefresh,
@@ -244,7 +228,6 @@ export class TerasliceElasticsearchStorage {
244
228
  this.logger.trace('creating record', this.options.logRecord ? record : null);
245
229
  const query = {
246
230
  index,
247
- type: this.recordType,
248
231
  id: record[this.idField],
249
232
  body: record,
250
233
  refresh: this.options.forceRefresh,
@@ -261,7 +244,6 @@ export class TerasliceElasticsearchStorage {
261
244
  // TODO: check from
262
245
  const esQuery = {
263
246
  index,
264
- type: this.recordType,
265
247
  // @ts-expect-error
266
248
  from,
267
249
  sort,
@@ -279,7 +261,6 @@ export class TerasliceElasticsearchStorage {
279
261
  this.logger.trace(`updating record ${recordId}, `, this.options.logRecord ? updateSpec : null);
280
262
  const query = {
281
263
  index,
282
- type: this.recordType,
283
264
  id: recordId,
284
265
  body: {
285
266
  doc: updateSpec,
@@ -297,7 +278,6 @@ export class TerasliceElasticsearchStorage {
297
278
  validateId(recordId, this.recordType);
298
279
  const getParams = {
299
280
  index,
300
- type: this.recordType,
301
281
  id: recordId,
302
282
  };
303
283
  const existing = await pRetry(() => this.api.get(getParams, true), {
@@ -311,18 +291,12 @@ export class TerasliceElasticsearchStorage {
311
291
  validateIdAndRecord(recordId, doc, this.recordType, this.idField);
312
292
  const query = {
313
293
  index,
314
- type: this.recordType,
315
294
  id: recordId,
316
295
  body: doc,
317
296
  refresh: this.options.forceRefresh,
318
297
  };
319
- if (!this.api.isElasticsearch6()) {
320
- query.if_seq_no = existing._seq_no;
321
- query.if_primary_term = existing._primary_term;
322
- }
323
- else {
324
- query.version = existing._version;
325
- }
298
+ query.if_seq_no = existing._seq_no;
299
+ query.if_primary_term = existing._primary_term;
326
300
  try {
327
301
  await this.api.indexWithId(query);
328
302
  return doc;
@@ -341,7 +315,6 @@ export class TerasliceElasticsearchStorage {
341
315
  this.logger.trace(`removing record ${recordId}`);
342
316
  const query = {
343
317
  index,
344
- type: this.recordType,
345
318
  id: recordId,
346
319
  refresh: this.options.forceRefresh,
347
320
  };
@@ -357,10 +330,7 @@ export class TerasliceElasticsearchStorage {
357
330
  });
358
331
  }
359
332
  const action = {
360
- [type]: {
361
- _index: index,
362
- _type: this.recordType,
363
- }
333
+ [type]: { _index: index }
364
334
  };
365
335
  this.bulkQueue.push({
366
336
  action,
package/dist/src/lib/storage/backends/mappings/analytics.js CHANGED
@@ -1,48 +1,43 @@
1
1
  export default {
2
2
  template: '__analytics*',
3
3
  mappings: {
4
- analytics: {
5
- _all: {
6
- enabled: false
7
- },
8
- dynamic: false,
9
- properties: {
10
- ex_id: {
11
- type: 'keyword'
12
- },
13
- job_id: {
14
- type: 'keyword'
15
- },
16
- worker_id: {
17
- type: 'keyword'
18
- },
19
- slice_id: {
20
- type: 'keyword'
21
- },
22
- slicer_id: {
23
- type: 'keyword'
24
- },
25
- op: {
26
- type: 'keyword'
27
- },
28
- order: {
29
- type: 'integer'
30
- },
31
- count: {
32
- type: 'integer'
33
- },
34
- state: {
35
- type: 'keyword'
36
- },
37
- time: {
38
- type: 'integer'
39
- },
40
- memory: {
41
- type: 'long'
42
- },
43
- '@timestamp': {
44
- type: 'date'
45
- }
4
+ dynamic: false,
5
+ properties: {
6
+ ex_id: {
7
+ type: 'keyword'
8
+ },
9
+ job_id: {
10
+ type: 'keyword'
11
+ },
12
+ worker_id: {
13
+ type: 'keyword'
14
+ },
15
+ slice_id: {
16
+ type: 'keyword'
17
+ },
18
+ slicer_id: {
19
+ type: 'keyword'
20
+ },
21
+ op: {
22
+ type: 'keyword'
23
+ },
24
+ order: {
25
+ type: 'integer'
26
+ },
27
+ count: {
28
+ type: 'integer'
29
+ },
30
+ state: {
31
+ type: 'keyword'
32
+ },
33
+ time: {
34
+ type: 'integer'
35
+ },
36
+ memory: {
37
+ type: 'long'
38
+ },
39
+ '@timestamp': {
40
+ type: 'date'
46
41
  }
47
42
  }
48
43
  }
package/dist/src/lib/storage/backends/mappings/asset.js CHANGED
@@ -1,39 +1,34 @@
1
1
  export default {
2
2
  mappings: {
3
- asset: {
4
- _all: {
5
- enabled: false
6
- },
7
- dynamic: false,
8
- properties: {
9
- blob: {
10
- type: 'binary',
11
- doc_values: false
12
- },
13
- name: {
14
- type: 'keyword'
15
- },
16
- version: {
17
- type: 'keyword'
18
- },
19
- id: {
20
- type: 'keyword'
21
- },
22
- description: {
23
- type: 'keyword'
24
- },
25
- arch: {
26
- type: 'keyword'
27
- },
28
- platform: {
29
- type: 'keyword'
30
- },
31
- node_version: {
32
- type: 'integer'
33
- },
34
- _created: {
35
- type: 'date'
36
- }
3
+ dynamic: false,
4
+ properties: {
5
+ blob: {
6
+ type: 'binary',
7
+ doc_values: false
8
+ },
9
+ name: {
10
+ type: 'keyword'
11
+ },
12
+ version: {
13
+ type: 'keyword'
14
+ },
15
+ id: {
16
+ type: 'keyword'
17
+ },
18
+ description: {
19
+ type: 'keyword'
20
+ },
21
+ arch: {
22
+ type: 'keyword'
23
+ },
24
+ platform: {
25
+ type: 'keyword'
26
+ },
27
+ node_version: {
28
+ type: 'integer'
29
+ },
30
+ _created: {
31
+ type: 'date'
37
32
  }
38
33
  }
39
34
  }
package/dist/src/lib/storage/backends/mappings/ex.js CHANGED
@@ -1,60 +1,55 @@
1
1
  export default {
2
2
  mappings: {
3
- ex: {
4
- _all: {
3
+ dynamic: false,
4
+ properties: {
5
+ active: {
6
+ type: 'boolean'
7
+ },
8
+ job_id: {
9
+ type: 'keyword'
10
+ },
11
+ ex_id: {
12
+ type: 'keyword'
13
+ },
14
+ _context: {
15
+ type: 'keyword'
16
+ },
17
+ _status: {
18
+ type: 'keyword'
19
+ },
20
+ _has_errors: {
21
+ type: 'keyword'
22
+ },
23
+ slicer_hostname: {
24
+ type: 'keyword'
25
+ },
26
+ slicer_port: {
27
+ type: 'keyword'
28
+ },
29
+ recovered_execution: {
30
+ type: 'keyword'
31
+ },
32
+ recovered_slice_type: {
33
+ type: 'keyword'
34
+ },
35
+ metadata: {
36
+ type: 'object',
5
37
  enabled: false
6
38
  },
7
- dynamic: false,
8
- properties: {
9
- active: {
10
- type: 'boolean'
11
- },
12
- job_id: {
13
- type: 'keyword'
14
- },
15
- ex_id: {
16
- type: 'keyword'
17
- },
18
- _context: {
19
- type: 'keyword'
20
- },
21
- _status: {
22
- type: 'keyword'
23
- },
24
- _has_errors: {
25
- type: 'keyword'
26
- },
27
- slicer_hostname: {
28
- type: 'keyword'
29
- },
30
- slicer_port: {
31
- type: 'keyword'
32
- },
33
- recovered_execution: {
34
- type: 'keyword'
35
- },
36
- recovered_slice_type: {
37
- type: 'keyword'
38
- },
39
- metadata: {
40
- type: 'object',
41
- enabled: false
42
- },
43
- _slicer_stats: {
44
- type: 'object'
45
- },
46
- _created: {
47
- type: 'date'
48
- },
49
- _updated: {
50
- type: 'date'
51
- },
52
- _deleted: {
53
- type: 'boolean'
54
- },
55
- _deleted_on: {
56
- type: 'date'
57
- }
39
+ _slicer_stats: {
40
+ type: 'object'
41
+ },
42
+ _created: {
43
+ type: 'date'
44
+ },
45
+ _updated: {
46
+ type: 'date'
47
+ },
48
+ _deleted: {
49
+ type: 'boolean'
50
+ },
51
+ _deleted_on: {
52
+ type: 'date'
58
53
  }
59
54
  }
60
55
  }
package/dist/src/lib/storage/backends/mappings/job.js CHANGED
@@ -4,33 +4,28 @@ export default {
4
4
  'index.number_of_replicas': 1
5
5
  },
6
6
  mappings: {
7
- job: {
8
- _all: {
9
- enabled: false
7
+ dynamic: false,
8
+ properties: {
9
+ active: {
10
+ type: 'boolean'
10
11
  },
11
- dynamic: false,
12
- properties: {
13
- active: {
14
- type: 'boolean'
15
- },
16
- job_id: {
17
- type: 'keyword'
18
- },
19
- _context: {
20
- type: 'keyword'
21
- },
22
- _created: {
23
- type: 'date'
24
- },
25
- _updated: {
26
- type: 'date'
27
- },
28
- _deleted: {
29
- type: 'boolean'
30
- },
31
- _deleted_on: {
32
- type: 'date'
33
- }
12
+ job_id: {
13
+ type: 'keyword'
14
+ },
15
+ _context: {
16
+ type: 'keyword'
17
+ },
18
+ _created: {
19
+ type: 'date'
20
+ },
21
+ _updated: {
22
+ type: 'date'
23
+ },
24
+ _deleted: {
25
+ type: 'boolean'
26
+ },
27
+ _deleted_on: {
28
+ type: 'date'
34
29
  }
35
30
  }
36
31
  }
package/dist/src/lib/storage/backends/mappings/state.js CHANGED
@@ -1,36 +1,31 @@
1
1
  export default {
2
2
  template: '__state*',
3
3
  mappings: {
4
- state: {
5
- _all: {
6
- enabled: false
4
+ dynamic: false,
5
+ properties: {
6
+ ex_id: {
7
+ type: 'keyword'
7
8
  },
8
- dynamic: false,
9
- properties: {
10
- ex_id: {
11
- type: 'keyword'
12
- },
13
- slice_id: {
14
- type: 'keyword'
15
- },
16
- slicer_id: {
17
- type: 'keyword'
18
- },
19
- slicer_order: {
20
- type: 'integer'
21
- },
22
- state: {
23
- type: 'keyword'
24
- },
25
- _created: {
26
- type: 'date'
27
- },
28
- _updated: {
29
- type: 'date'
30
- },
31
- error: {
32
- type: 'keyword'
33
- }
9
+ slice_id: {
10
+ type: 'keyword'
11
+ },
12
+ slicer_id: {
13
+ type: 'keyword'
14
+ },
15
+ slicer_order: {
16
+ type: 'integer'
17
+ },
18
+ state: {
19
+ type: 'keyword'
20
+ },
21
+ _created: {
22
+ type: 'date'
23
+ },
24
+ _updated: {
25
+ type: 'date'
26
+ },
27
+ error: {
28
+ type: 'keyword'
34
29
  }
35
30
  }
36
31
  }
package/dist/src/lib/storage/state.js CHANGED
@@ -57,7 +57,6 @@ export class StateStorage {
57
57
  action: {
58
58
  index: {
59
59
  _index: index,
60
- _type: this.recordType,
61
60
  _id: record.slice_id,
62
61
  },
63
62
  },
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "teraslice",
3
3
  "displayName": "Teraslice",
4
- "version": "3.0.0-dev.4",
4
+ "version": "3.0.0-dev.5",
5
5
  "description": "Distributed computing platform for processing JSON data",
6
6
  "homepage": "https://github.com/terascope/teraslice#readme",
7
7
  "bugs": {
@@ -27,9 +27,8 @@
27
27
  "start": "node service.js",
28
28
  "test": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
29
29
  "test:debug": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --debug ../teraslice --",
30
- "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='6.8.6' node ../scripts/bin/ts-scripts test ../teraslice --",
31
30
  "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' node ../scripts/bin/ts-scripts test ../teraslice --",
32
- "test:opensearch1": "TEST_OPENSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
31
+ "test:opensearch1": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='1.3.11' node ../scripts/bin/ts-scripts test ../teraslice --",
33
32
  "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' node ../scripts/bin/ts-scripts test ../teraslice --",
34
33
  "test:opensearch3": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='3.1.0' node ../scripts/bin/ts-scripts test ../teraslice --",
35
34
  "test:watch": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --watch ../teraslice --"
@@ -40,11 +39,11 @@
40
39
  },
41
40
  "dependencies": {
42
41
  "@kubernetes/client-node": "~1.3.0",
43
- "@terascope/elasticsearch-api": "~5.0.0-dev.4",
44
- "@terascope/job-components": "~2.0.0-dev.4",
45
- "@terascope/teraslice-messaging": "~2.0.0-dev.4",
46
- "@terascope/types": "~2.0.0-dev.4",
47
- "@terascope/utils": "~2.0.0-dev.4",
42
+ "@terascope/elasticsearch-api": "~5.0.0-dev.5",
43
+ "@terascope/job-components": "~2.0.0-dev.5",
44
+ "@terascope/teraslice-messaging": "~2.0.0-dev.5",
45
+ "@terascope/types": "~2.0.0-dev.5",
46
+ "@terascope/utils": "~2.0.0-dev.5",
48
47
  "async-mutex": "~0.5.0",
49
48
  "barbe": "~3.0.17",
50
49
  "body-parser": "~2.2.0",
@@ -62,11 +61,11 @@
62
61
  "semver": "~7.7.2",
63
62
  "socket.io": "~1.7.4",
64
63
  "socket.io-client": "~1.7.4",
65
- "terafoundation": "~2.0.0-dev.4",
64
+ "terafoundation": "~2.0.0-dev.5",
66
65
  "uuid": "~13.0.0"
67
66
  },
68
67
  "devDependencies": {
69
- "@terascope/opensearch-client": "~2.0.0-dev.4",
68
+ "@terascope/opensearch-client": "~2.0.0-dev.5",
70
69
  "@types/archiver": "~6.0.3",
71
70
  "@types/express": "~5.0.3",
72
71
  "@types/gc-stats": "~1.4.3",