teraslice 3.0.0-dev.4 → 3.0.0-dev.6

package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js

@@ -1,7 +1,7 @@
 import { nanoid } from 'nanoid';
 import { pDelay, Queue, isFunction, isEmpty, get, toNumber, isKey } from '@terascope/utils';
-import socketIOClient from 'socket.io-client';
-import socketIOServer from 'socket.io';
+import { io as socketIOClient } from 'socket.io-client';
+import { Server as socketIOServer } from 'socket.io';
 import { isProcessAssignment, ProcessAssignment } from '../../../../../../interfaces.js';
 // messages send to cluster_master
 const clusterMasterMessages = {
@@ -288,10 +288,11 @@ export class Messaging {
             serveClient: false,
         };
         if (server) {
-            this.io = socketIOServer(server, opts);
+            this.io = new socketIOServer(server, opts);
         }
         else if (port) {
-            this.io = socketIOServer(port, opts);
+            this.io = new socketIOServer(opts);
+            this.io.listen(port);
         }
         this._attachRoomsSocketIO();
         this.io.on('connection', (socket) => {

package/dist/src/lib/config/schemas/system.js

@@ -140,7 +140,7 @@ export const schema = {
         format: 'duration'
     },
     node_disconnect_timeout: {
-        doc: 'time in milliseconds that the cluster  will wait untill it drops that node from state and attempts to provision the lost workers',
+        doc: 'time in milliseconds that the cluster will wait until it drops that node from state and attempts to provision the lost workers',
         default: '5 minutes',
         format: 'duration'
     },

package/dist/src/lib/storage/backends/elasticsearch_store.js

@@ -127,7 +127,7 @@ export class TerasliceElasticsearchStorage {
                 const client = await getClient(this.context, connectionConfig, 'elasticsearch-next');
                 this.api = elasticsearchApi(client, this.logger, options);
                 await this._createIndex(newIndex);
-                await this.api.isAvailable(newIndex, this.recordType);
+                await this.api.isAvailable(newIndex);
                 return true;
             }
             catch (err) {
@@ -157,17 +157,10 @@ export class TerasliceElasticsearchStorage {
         this.logger.trace(`getting record id: ${recordId}`);
         const query = {
             index,
-            type: this.recordType,
             id: recordId,
         };
         if (fields) {
-            if (!this.api.isElasticsearch6()) {
-                query._source_includes = fields;
-            }
-            else {
-                // @ts-expect-error backwards compatible
-                query._sourceInclude = fields;
-            }
+            query._source_includes = fields;
         }
         return this.api.get(query);
     }
@@ -185,7 +178,6 @@ export class TerasliceElasticsearchStorage {
             index,
             from,
             size,
-            type: this.recordType,
             sort,
         };
         if (typeof query === 'string') {
@@ -195,13 +187,7 @@ export class TerasliceElasticsearchStorage {
             esQuery.body = query;
         }
         if (fields) {
-            if (!this.api.isElasticsearch6()) {
-                esQuery._source_includes = fields;
-            }
-            else {
-                // @ts-expect-error backwards compatible
-                esQuery._sourceInclude = fields;
-            }
+            esQuery._source_includes = fields;
         }
         return this.api.search(esQuery);
     }
@@ -213,7 +199,6 @@ export class TerasliceElasticsearchStorage {
         this.logger.trace('indexing record', this.options.logRecord ? record : undefined);
         const query = {
             index: indexArg,
-            type: this.recordType,
             body: record,
             refresh: this.options.forceRefresh,
         };
@@ -228,7 +213,6 @@ export class TerasliceElasticsearchStorage {
         this.logger.trace(`indexWithId call with id: ${recordId}, record`, this.options.logRecord ? record : null);
         const query = {
             index,
-            type: this.recordType,
             id: recordId,
             body: record,
             refresh: this.options.forceRefresh,
@@ -244,7 +228,6 @@ export class TerasliceElasticsearchStorage {
         this.logger.trace('creating record', this.options.logRecord ? record : null);
         const query = {
             index,
-            type: this.recordType,
             id: record[this.idField],
             body: record,
             refresh: this.options.forceRefresh,
@@ -261,7 +244,6 @@ export class TerasliceElasticsearchStorage {
         // TODO: check from
         const esQuery = {
             index,
-            type: this.recordType,
             // @ts-expect-error
             from,
             sort,
@@ -279,7 +261,6 @@ export class TerasliceElasticsearchStorage {
         this.logger.trace(`updating record ${recordId}, `, this.options.logRecord ? updateSpec : null);
         const query = {
             index,
-            type: this.recordType,
             id: recordId,
             body: {
                 doc: updateSpec,
@@ -297,7 +278,6 @@ export class TerasliceElasticsearchStorage {
         validateId(recordId, this.recordType);
         const getParams = {
             index,
-            type: this.recordType,
             id: recordId,
         };
         const existing = await pRetry(() => this.api.get(getParams, true), {
@@ -311,18 +291,12 @@ export class TerasliceElasticsearchStorage {
         validateIdAndRecord(recordId, doc, this.recordType, this.idField);
         const query = {
             index,
-            type: this.recordType,
             id: recordId,
             body: doc,
             refresh: this.options.forceRefresh,
         };
-        if (!this.api.isElasticsearch6()) {
-            query.if_seq_no = existing._seq_no;
-            query.if_primary_term = existing._primary_term;
-        }
-        else {
-            query.version = existing._version;
-        }
+        query.if_seq_no = existing._seq_no;
+        query.if_primary_term = existing._primary_term;
         try {
             await this.api.indexWithId(query);
             return doc;
@@ -341,7 +315,6 @@ export class TerasliceElasticsearchStorage {
         this.logger.trace(`removing record ${recordId}`);
         const query = {
             index,
-            type: this.recordType,
             id: recordId,
             refresh: this.options.forceRefresh,
         };
@@ -357,10 +330,7 @@ export class TerasliceElasticsearchStorage {
             });
         }
         const action = {
-            [type]: {
-                _index: index,
-                _type: this.recordType,
-            }
+            [type]: { _index: index }
         };
         this.bulkQueue.push({
             action,

package/dist/src/lib/storage/backends/mappings/analytics.js

@@ -1,48 +1,43 @@
 export default {
     template: '__analytics*',
     mappings: {
-        analytics: {
-            _all: {
-                enabled: false
-            },
-            dynamic: false,
-            properties: {
-                ex_id: {
-                    type: 'keyword'
-                },
-                job_id: {
-                    type: 'keyword'
-                },
-                worker_id: {
-                    type: 'keyword'
-                },
-                slice_id: {
-                    type: 'keyword'
-                },
-                slicer_id: {
-                    type: 'keyword'
-                },
-                op: {
-                    type: 'keyword'
-                },
-                order: {
-                    type: 'integer'
-                },
-                count: {
-                    type: 'integer'
-                },
-                state: {
-                    type: 'keyword'
-                },
-                time: {
-                    type: 'integer'
-                },
-                memory: {
-                    type: 'long'
-                },
-                '@timestamp': {
-                    type: 'date'
-                }
+        dynamic: false,
+        properties: {
+            ex_id: {
+                type: 'keyword'
+            },
+            job_id: {
+                type: 'keyword'
+            },
+            worker_id: {
+                type: 'keyword'
+            },
+            slice_id: {
+                type: 'keyword'
+            },
+            slicer_id: {
+                type: 'keyword'
+            },
+            op: {
+                type: 'keyword'
+            },
+            order: {
+                type: 'integer'
+            },
+            count: {
+                type: 'integer'
+            },
+            state: {
+                type: 'keyword'
+            },
+            time: {
+                type: 'integer'
+            },
+            memory: {
+                type: 'long'
+            },
+            '@timestamp': {
+                type: 'date'
             }
         }
     }

package/dist/src/lib/storage/backends/mappings/asset.js

@@ -1,39 +1,34 @@
 export default {
     mappings: {
-        asset: {
-            _all: {
-                enabled: false
-            },
-            dynamic: false,
-            properties: {
-                blob: {
-                    type: 'binary',
-                    doc_values: false
-                },
-                name: {
-                    type: 'keyword'
-                },
-                version: {
-                    type: 'keyword'
-                },
-                id: {
-                    type: 'keyword'
-                },
-                description: {
-                    type: 'keyword'
-                },
-                arch: {
-                    type: 'keyword'
-                },
-                platform: {
-                    type: 'keyword'
-                },
-                node_version: {
-                    type: 'integer'
-                },
-                _created: {
-                    type: 'date'
-                }
+        dynamic: false,
+        properties: {
+            blob: {
+                type: 'binary',
+                doc_values: false
+            },
+            name: {
+                type: 'keyword'
+            },
+            version: {
+                type: 'keyword'
+            },
+            id: {
+                type: 'keyword'
+            },
+            description: {
+                type: 'keyword'
+            },
+            arch: {
+                type: 'keyword'
+            },
+            platform: {
+                type: 'keyword'
+            },
+            node_version: {
+                type: 'integer'
+            },
+            _created: {
+                type: 'date'
             }
         }
     }

package/dist/src/lib/storage/backends/mappings/ex.js

@@ -1,60 +1,55 @@
 export default {
     mappings: {
-        ex: {
-            _all: {
+        dynamic: false,
+        properties: {
+            active: {
+                type: 'boolean'
+            },
+            job_id: {
+                type: 'keyword'
+            },
+            ex_id: {
+                type: 'keyword'
+            },
+            _context: {
+                type: 'keyword'
+            },
+            _status: {
+                type: 'keyword'
+            },
+            _has_errors: {
+                type: 'keyword'
+            },
+            slicer_hostname: {
+                type: 'keyword'
+            },
+            slicer_port: {
+                type: 'keyword'
+            },
+            recovered_execution: {
+                type: 'keyword'
+            },
+            recovered_slice_type: {
+                type: 'keyword'
+            },
+            metadata: {
+                type: 'object',
                 enabled: false
             },
-            dynamic: false,
-            properties: {
-                active: {
-                    type: 'boolean'
-                },
-                job_id: {
-                    type: 'keyword'
-                },
-                ex_id: {
-                    type: 'keyword'
-                },
-                _context: {
-                    type: 'keyword'
-                },
-                _status: {
-                    type: 'keyword'
-                },
-                _has_errors: {
-                    type: 'keyword'
-                },
-                slicer_hostname: {
-                    type: 'keyword'
-                },
-                slicer_port: {
-                    type: 'keyword'
-                },
-                recovered_execution: {
-                    type: 'keyword'
-                },
-                recovered_slice_type: {
-                    type: 'keyword'
-                },
-                metadata: {
-                    type: 'object',
-                    enabled: false
-                },
-                _slicer_stats: {
-                    type: 'object'
-                },
-                _created: {
-                    type: 'date'
-                },
-                _updated: {
-                    type: 'date'
-                },
-                _deleted: {
-                    type: 'boolean'
-                },
-                _deleted_on: {
-                    type: 'date'
-                }
+            _slicer_stats: {
+                type: 'object'
+            },
+            _created: {
+                type: 'date'
+            },
+            _updated: {
+                type: 'date'
+            },
+            _deleted: {
+                type: 'boolean'
+            },
+            _deleted_on: {
+                type: 'date'
             }
         }
     }

package/dist/src/lib/storage/backends/mappings/job.js

@@ -4,33 +4,28 @@ export default {
         'index.number_of_replicas': 1
     },
     mappings: {
-        job: {
-            _all: {
-                enabled: false
+        dynamic: false,
+        properties: {
+            active: {
+                type: 'boolean'
             },
-            dynamic: false,
-            properties: {
-                active: {
-                    type: 'boolean'
-                },
-                job_id: {
-                    type: 'keyword'
-                },
-                _context: {
-                    type: 'keyword'
-                },
-                _created: {
-                    type: 'date'
-                },
-                _updated: {
-                    type: 'date'
-                },
-                _deleted: {
-                    type: 'boolean'
-                },
-                _deleted_on: {
-                    type: 'date'
-                }
+            job_id: {
+                type: 'keyword'
+            },
+            _context: {
+                type: 'keyword'
+            },
+            _created: {
+                type: 'date'
+            },
+            _updated: {
+                type: 'date'
+            },
+            _deleted: {
+                type: 'boolean'
+            },
+            _deleted_on: {
+                type: 'date'
             }
         }
     }

package/dist/src/lib/storage/backends/mappings/state.js

@@ -1,36 +1,31 @@
 export default {
     template: '__state*',
     mappings: {
-        state: {
-            _all: {
-                enabled: false
+        dynamic: false,
+        properties: {
+            ex_id: {
+                type: 'keyword'
             },
-            dynamic: false,
-            properties: {
-                ex_id: {
-                    type: 'keyword'
-                },
-                slice_id: {
-                    type: 'keyword'
-                },
-                slicer_id: {
-                    type: 'keyword'
-                },
-                slicer_order: {
-                    type: 'integer'
-                },
-                state: {
-                    type: 'keyword'
-                },
-                _created: {
-                    type: 'date'
-                },
-                _updated: {
-                    type: 'date'
-                },
-                error: {
-                    type: 'keyword'
-                }
+            slice_id: {
+                type: 'keyword'
+            },
+            slicer_id: {
+                type: 'keyword'
+            },
+            slicer_order: {
+                type: 'integer'
+            },
+            state: {
+                type: 'keyword'
+            },
+            _created: {
+                type: 'date'
+            },
+            _updated: {
+                type: 'date'
+            },
+            error: {
+                type: 'keyword'
             }
         }
     }

package/dist/src/lib/storage/state.js

@@ -57,7 +57,6 @@ export class StateStorage {
                 action: {
                     index: {
                         _index: index,
-                        _type: this.recordType,
                         _id: record.slice_id,
                     },
                 },

package/dist/test/node_master-spec.js

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'node:events';
 import { debugLogger } from '@terascope/utils';
-import Socket from 'socket.io';
+import { Server as SocketServer } from 'socket.io';
 import { nodeMaster } from '../src/lib/cluster/node_master.js';
 process.env.assignment = 'node_master';
 describe('Node master', () => {
@@ -49,7 +49,7 @@ describe('Node master', () => {
         __test_job: JSON.stringify(jobJson),
         __test_assignment: 'worker'
     };
-    const fakeClusterMaster = Socket({
+    const fakeClusterMaster = new SocketServer({
         path: '/native-clustering'
     });
     fakeClusterMaster.on('connection', (socket) => {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "teraslice",
   "displayName": "Teraslice",
-  "version": "3.0.0-dev.4",
+  "version": "3.0.0-dev.6",
   "description": "Distributed computing platform for processing JSON data",
   "homepage": "https://github.com/terascope/teraslice#readme",
   "bugs": {
@@ -27,9 +27,8 @@
     "start": "node service.js",
     "test": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
     "test:debug": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --debug ../teraslice --",
-    "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='6.8.6' node ../scripts/bin/ts-scripts test ../teraslice --",
     "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' node ../scripts/bin/ts-scripts test ../teraslice --",
-    "test:opensearch1": "TEST_OPENSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
+    "test:opensearch1": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='1.3.11' node ../scripts/bin/ts-scripts test ../teraslice --",
     "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' node ../scripts/bin/ts-scripts test ../teraslice --",
     "test:opensearch3": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='3.1.0' node ../scripts/bin/ts-scripts test ../teraslice --",
     "test:watch": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --watch ../teraslice --"
@@ -40,11 +39,11 @@
   },
   "dependencies": {
     "@kubernetes/client-node": "~1.3.0",
-    "@terascope/elasticsearch-api": "~5.0.0-dev.4",
-    "@terascope/job-components": "~2.0.0-dev.4",
-    "@terascope/teraslice-messaging": "~2.0.0-dev.4",
-    "@terascope/types": "~2.0.0-dev.4",
-    "@terascope/utils": "~2.0.0-dev.4",
+    "@terascope/elasticsearch-api": "~5.0.0-dev.5",
+    "@terascope/job-components": "~2.0.0-dev.5",
+    "@terascope/teraslice-messaging": "~2.0.0-dev.6",
+    "@terascope/types": "~2.0.0-dev.5",
+    "@terascope/utils": "~2.0.0-dev.5",
     "async-mutex": "~0.5.0",
     "barbe": "~3.0.17",
     "body-parser": "~2.2.0",
@@ -60,13 +59,13 @@
     "ms": "~2.1.3",
     "nanoid": "~5.1.5",
     "semver": "~7.7.2",
-    "socket.io": "~1.7.4",
-    "socket.io-client": "~1.7.4",
-    "terafoundation": "~2.0.0-dev.4",
+    "socket.io": "~4.8.1",
+    "socket.io-client": "~4.8.1",
+    "terafoundation": "~2.0.0-dev.5",
     "uuid": "~13.0.0"
   },
   "devDependencies": {
-    "@terascope/opensearch-client": "~2.0.0-dev.4",
+    "@terascope/opensearch-client": "~2.0.0-dev.5",
     "@types/archiver": "~6.0.3",
     "@types/express": "~5.0.3",
     "@types/gc-stats": "~1.4.3",