teraslice 0.77.1 → 0.80.0

package/lib/cluster/services/cluster/backends/kubernetes/k8s.js

@@ -8,10 +8,10 @@ const Request = require('kubernetes-client/backends/request');
 const { getRetryConfig } = require('./utils');
 
 class K8s {
-    constructor(logger, clientConfig, defaultNamespace = 'default',
+    constructor(logger, clientConfig, defaultNamespace,
         apiPollDelay, shutdownTimeout) {
         this.apiPollDelay = apiPollDelay;
-        this.defaultNamespace = defaultNamespace;
+        this.defaultNamespace = defaultNamespace || 'default';
         this.logger = logger;
         this.shutdownTimeout = shutdownTimeout; // this is in milliseconds
 

package/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -25,6 +25,7 @@ class K8sResource {
     constructor(resourceType, resourceName, terasliceConfig, execution) {
         this.execution = execution;
         this.jobLabelPrefix = 'job.teraslice.terascope.io';
+        this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
         this.nodeType = resourceName;
         this.terasliceConfig = terasliceConfig;
 
@@ -54,9 +55,10 @@ class K8sResource {
         this._setImagePullSecret();
         this._setEphemeralStorage();
         this._setExternalPorts();
+        this._setPriorityClassName();
 
         if (resourceName === 'worker') {
-            this._setAntiAffinity();
+            this._setWorkerAntiAffinity();
         }
 
         // Execution controller targets are required nodeAffinities, if
@@ -124,31 +126,40 @@ class K8sResource {
         };
     }
 
-    _setAntiAffinity() {
+    _setWorkerAntiAffinity() {
         if (this.terasliceConfig.kubernetes_worker_antiaffinity) {
-            this.resource.spec.template.spec.affinity = {
-                podAntiAffinity: {
-                    preferredDuringSchedulingIgnoredDuringExecution: [
-                        {
-                            weight: 1,
-                            podAffinityTerm: {
-                                labelSelector: {
-                                    matchExpressions: [
-                                        {
-                                            key: 'app.kubernetes.io/name',
-                                            operator: 'In',
-                                            values: [
-                                                'teraslice'
-                                            ]
-                                        }
+            const targetKey = 'spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution';
+            if (!_.has(this.resource, targetKey)) {
+                _.set(this.resource, targetKey, []);
+            }
+
+            // eslint-disable-next-line max-len
+            this.resource.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.push(
+                {
+                    weight: 1,
+                    podAffinityTerm: {
+                        labelSelector: {
+                            matchExpressions: [
+                                {
+                                    key: 'app.kubernetes.io/name',
+                                    operator: 'In',
+                                    values: [
+                                        'teraslice'
                                     ]
                                 },
-                                topologyKey: 'kubernetes.io/hostname'
-                            }
-                        }
-                    ]
+                                {
+                                    key: 'app.kubernetes.io/instance',
+                                    operator: 'In',
+                                    values: [
+                                        this.templateConfig.clusterNameLabel
+                                    ]
+                                }
+                            ]
+                        },
+                        topologyKey: 'kubernetes.io/hostname'
+                    }
                 }
-            };
+            );
         }
     }
 
@@ -210,6 +221,25 @@ class K8sResource {
         }
     }
 
+    _setPriorityClassName() {
+        if (this.terasliceConfig.kubernetes_priority_class_name) {
+            if (this.nodeType === 'execution_controller') {
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
+                if (this.execution.stateful) {
+                    // eslint-disable-next-line max-len
+                    this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
+                }
+            }
+            if (this.nodeType === 'worker' && this.execution.stateful) {
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
+            }
+        }
+    }
+
     _setAssetsVolume() {
         if (this.terasliceConfig.assets_directory && this.terasliceConfig.assets_volume) {
             this.resource.spec.template.spec.volumes.push({
@@ -257,14 +287,40 @@ class K8sResource {
     _setResources() {
         let cpu;
         let memory;
+        let maxMemory;
+
+        const container = this.resource.spec.template.spec.containers[0];
 
         // use teraslice config as defaults and execution config will override it
         const envVars = Object.assign({}, this.terasliceConfig.env_vars, this.execution.env_vars);
 
         if (this.nodeType === 'worker') {
-            // The settings on the executions override the cluster configs
-            cpu = this.execution.cpu || this.terasliceConfig.cpu || -1;
-            memory = this.execution.memory || this.terasliceConfig.memory || -1;
+            if (this.execution.resources_requests_cpu
+                || this.execution.resources_limits_cpu) {
+                if (this.execution.resources_requests_cpu) {
+                    _.set(container, 'resources.requests.cpu', this.execution.resources_requests_cpu);
+                }
+                if (this.execution.resources_limits_cpu) {
+                    _.set(container, 'resources.limits.cpu', this.execution.resources_limits_cpu);
+                }
+            } else if (this.execution.cpu || this.terasliceConfig.cpu) {
+                // The settings on the executions override the cluster configs
+                cpu = this.execution.cpu || this.terasliceConfig.cpu || -1;
+                _.set(container, 'resources.requests.cpu', cpu);
+                _.set(container, 'resources.limits.cpu', cpu);
+            }
+            if (this.execution.resources_requests_memory
+                || this.execution.resources_limits_memory) {
+                _.set(container, 'resources.requests.memory', this.execution.resources_requests_memory);
+                _.set(container, 'resources.limits.memory', this.execution.resources_limits_memory);
+                maxMemory = this.execution.resources_limits_memory;
+            } else if (this.execution.memory || this.terasliceConfig.memory) {
+                // The settings on the executions override the cluster configs
+                memory = this.execution.memory || this.terasliceConfig.memory || -1;
+                _.set(container, 'resources.requests.memory', memory);
+                _.set(container, 'resources.limits.memory', memory);
+                maxMemory = memory;
+            }
         }
 
         if (this.nodeType === 'execution_controller') {
@@ -273,21 +329,17 @@ class K8sResource {
                 || this.terasliceConfig.cpu_execution_controller || -1;
             memory = this.execution.memory_execution_controller
                 || this.terasliceConfig.memory_execution_controller || -1;
-        }
-
-        const container = this.resource.spec.template.spec.containers[0];
-
-        if (cpu !== -1) {
             _.set(container, 'resources.requests.cpu', cpu);
             _.set(container, 'resources.limits.cpu', cpu);
-        }
-
-        if (memory !== -1) {
             _.set(container, 'resources.requests.memory', memory);
             _.set(container, 'resources.limits.memory', memory);
+            maxMemory = memory;
         }
 
-        setMaxOldSpaceViaEnv(container.env, envVars, memory);
+        // NOTE: This sucks, this manages the memory env var but it ALSO is
+        // responsible for doing the config and execution env var merge, which
+        // should NOT be in this function
+        setMaxOldSpaceViaEnv(container.env, envVars, maxMemory);
     }
 
     _setTargets() {

package/lib/config/default-sysconfig.js

@@ -16,6 +16,11 @@ function getConnectors() {
             default: {
                 host: ['localhost:9200']
             }
+        },
+        'elasticsearch-next': {
+            default: {
+                node: ['localhost:9200']
+            }
         }
     };
 

package/lib/config/schemas/system.js

@@ -295,6 +295,11 @@ const schema = {
         default: 'default',
         format: 'optional_String'
     },
+    kubernetes_priority_class_name: {
+        doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
+        default: undefined,
+        format: 'optional_String'
+    },
     kubernetes_config_map_name: {
         doc: 'Specify the name of the Kubernetes ConfigMap used to configure worker pods',
         default: 'teraslice-worker',

package/lib/storage/analytics.js

@@ -50,7 +50,7 @@ module.exports = async function analyticsService(context) {
                 time: stats.time[index],
                 memory: stats.memory[index],
             },
-            null,
+            'index',
             esIndex
         ));
 

package/lib/storage/backends/elasticsearch_store.js

@@ -18,7 +18,7 @@ const {
     isInteger
 } = require('@terascope/utils');
 const elasticsearchApi = require('@terascope/elasticsearch-api');
-const { getClient } = require('@terascope/job-components');
+const { getClientAsync } = require('@terascope/job-components');
 const { makeLogger } = require('../../workers/helpers/terafoundation');
 const { timeseriesIndex } = require('../../utils/date_utils');
 
@@ -29,7 +29,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
         recordType,
         idField,
         storageName,
-        bulkSize = 500,
+        bulkSize = 1000,
         fullResponse = false,
         logRecord = true,
         forceRefresh = true,
@@ -83,8 +83,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
         };
 
         if (fields) {
-            const esVersion = elasticsearch.getESVersion();
-            if (esVersion > 6) {
+            if (!elasticsearch.isElasticsearch6()) {
                 query._sourceIncludes = fields;
             } else {
                 query._sourceInclude = fields;
@@ -118,8 +117,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
         }
 
         if (fields) {
-            const esVersion = elasticsearch.getESVersion();
-            if (esVersion > 6) {
+            if (!elasticsearch.isElasticsearch6()) {
                 esQuery._sourceIncludes = fields;
             } else {
                 esQuery._sourceInclude = fields;
@@ -161,13 +159,13 @@ module.exports = function elasticsearchStorage(backendConfig) {
      * index saves a record to elasticsearch with a specified ID.
      * If the document is already there it will be replaced.
      */
-    async function indexWithId(recordId, record, indexArg = indexName, timeout) {
+    async function indexWithId(recordId, record, indexArg, timeout) {
         validateIdAndRecord(recordId, record);
 
         logger.trace(`indexWithId call with id: ${recordId}, record`, logRecord ? record : null);
 
         const query = {
-            index: indexArg,
+            index: indexArg || indexName,
             type: recordType,
             id: recordId,
             body: record,
@@ -273,9 +271,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
             refresh: forceRefresh,
         };
 
-        const esVersion = elasticsearch.getESVersion();
-
-        if (esVersion >= 7) {
+        if (!elasticsearch.isElasticsearch6()) {
             query.if_seq_no = existing._seq_no;
             query.if_primary_term = existing._primary_term;
         } else {
@@ -322,14 +318,17 @@ module.exports = function elasticsearchStorage(backendConfig) {
 
         const type = _type || 'index';
 
-        const indexRequest = {
+        const action = {
             [type]: {
                 _index: indexArg,
                 _type: recordType,
             }
         };
 
-        bulkQueue.push(indexRequest, record);
+        bulkQueue.push({
+            action,
+            data: type === 'delete' ? undefined : record
+        });
 
         // We only flush once enough records have accumulated for it to make sense.
         if (bulkQueue.length >= bulkSize) {
@@ -375,17 +374,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
     }
 
     async function bulkSend(bulkRequest) {
-        const recordCount = (bulkRequest.length / 2);
-
-        await pRetry(async () => elasticsearch.bulkSend(bulkRequest), {
-            reason: `Failure to bulk create "${recordType}"`,
-            logError: logger.warn,
-            delay: isTest ? 100 : 1000,
-            backoff: 5,
-            retries: 100,
-        });
-
-        return recordCount;
+        return elasticsearch.bulkSend(bulkRequest);
     }
 
     async function _flush(shuttingDown = false) {
@@ -544,11 +533,6 @@ module.exports = function elasticsearchStorage(backendConfig) {
         if (connectionConfig.connection_cache == null) {
             connectionConfig.connection_cache = true;
         }
-        client = getClient(context, connectionConfig, 'elasticsearch');
-        if (!client) {
-            reject(new Error(`Unable to get client for connection: ${config.state.connection}`));
-            return;
-        }
 
         let { connection } = config.state;
         if (config.state.endpoint) {
@@ -560,8 +544,18 @@ module.exports = function elasticsearchStorage(backendConfig) {
             connection,
         };
 
-        elasticsearch = elasticsearchApi(client, logger, options);
-        _createIndex(newIndex)
+        Promise.resolve()
+            .then(() => getClientAsync(context, connectionConfig, 'elasticsearch-next'))
+            .then((esClient) => {
+                client = esClient;
+                if (!client) {
+                    reject(new Error(`Unable to get client for connection: ${config.state.connection}`));
+                    return;
+                }
+                elasticsearch = elasticsearchApi(client, logger, options);
+                // eslint-disable-next-line consistent-return
+                return _createIndex(newIndex);
+            })
             .then(() => elasticsearch.isAvailable(newIndex, recordType))
             .then(() => resolve(api))
             .catch((err) => {

package/lib/storage/state.js

@@ -60,17 +60,19 @@ async function stateStorage(context) {
     async function createSlices(exId, slices) {
         await waitForClient();
 
-        const bulkRequest = [];
-        for (const slice of slices) {
+        const bulkRequest = slices.map((slice) => {
             const { record, index } = _createSliceRecord(exId, slice, SliceState.pending);
-            bulkRequest.push({
-                index: {
-                    _index: index,
-                    _type: recordType,
-                    _id: record.slice_id,
+            return {
+                action: {
+                    index: {
+                        _index: index,
+                        _type: recordType,
+                        _id: record.slice_id,
+                    },
                 },
-            }, record);
-        }
+                data: record
+            };
+        });
         return backend.bulkSend(bulkRequest);
     }
 
@@ -242,8 +244,8 @@ async function stateStorage(context) {
         }
     }
 
-    async function search(query, from, size, sort = '_updated:desc', fields) {
-        return backend.search(query, from, size, sort, fields);
+    async function search(query, from, size, sort, fields) {
+        return backend.search(query, from, size, sort || '_updated:desc', fields);
     }
 
     async function count(query, from = 0, sort = '_updated:desc') {

package/lib/utils/port_utils.js

@@ -4,8 +4,8 @@ const shuffle = require('lodash/shuffle');
 const {
     get,
     times,
-    toInteger,
     pDelay,
+    toIntegerOrThrow,
 } = require('@terascope/utils');
 const porty = require('porty');
 
@@ -57,11 +57,11 @@ async function findPort(options = {}) {
 
 function getPorts(context) {
     const portConfig = get(context, 'sysconfig.teraslice.slicer_port_range');
-    const dataArray = portConfig.split(':');
-    const assetsPort = toInteger(dataArray[0]);
+    const dataArray = portConfig.split(':', 2);
+    const assetsPort = toIntegerOrThrow(dataArray[0]);
     const start = assetsPort + 1;
     // range end is exclusive, so we need to add one
-    const end = toInteger(dataArray[1]) + 1;
+    const end = toIntegerOrThrow(dataArray[1]) + 1;
     return { assetsPort, start, end };
 }
 

package/lib/workers/worker/index.js

@@ -192,7 +192,7 @@ class Worker {
         this.slicesProcessed += 1;
     }
 
-    async shutdown(block = true, event, shutdownError) {
+    async shutdown(block, event, shutdownError) {
         if (this.isShutdown) return;
         if (!this.isInitialized) return;
         const { exId } = this.executionContext;
@@ -202,11 +202,11 @@ class Worker {
                 'worker',
                 `shutdown was called for ${exId}`,
                 'but it was already shutting down',
-                block ? ', will block until done' : ''
+                block !== false ? ', will block until done' : ''
             ];
             this.logger.debug(msgs.join(' '));
 
-            if (block) {
+            if (block !== false) {
                 await waitForWorkerShutdown(this.context, 'worker:shutdown:complete');
             }
             return;

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.77.1",
+    "version": "0.80.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -33,50 +33,50 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "resolutions": {
-        "debug": "^4.3.1",
+        "debug": "^4.3.4",
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^2.20.0",
-        "@terascope/job-components": "^0.51.0",
-        "@terascope/teraslice-messaging": "^0.22.0",
-        "@terascope/utils": "^0.39.0",
-        "async-mutex": "^0.3.1",
+        "@terascope/elasticsearch-api": "^3.2.0",
+        "@terascope/job-components": "^0.57.0",
+        "@terascope/teraslice-messaging": "^0.27.2",
+        "@terascope/utils": "^0.44.2",
+        "async-mutex": "^0.3.2",
         "barbe": "^3.0.16",
-        "body-parser": "^1.19.0",
+        "body-parser": "^1.20.0",
         "convict": "^4.4.1",
         "decompress": "^4.2.1",
-        "easy-table": "^1.1.1",
-        "event-loop-stats": "^1.2.0",
-        "express": "^4.17.1",
-        "fs-extra": "^9.1.0",
+        "easy-table": "^1.2.0",
+        "event-loop-stats": "^1.4.1",
+        "express": "^4.17.3",
+        "fs-extra": "^10.0.1",
         "gc-stats": "^1.4.0",
-        "got": "^11.8.2",
+        "got": "^11.8.3",
         "ip": "^1.1.5",
         "kubernetes-client": "^9.0.0",
         "lodash": "^4.17.21",
         "ms": "^2.1.3",
-        "nanoid": "^3.1.21",
+        "nanoid": "^3.3.2",
         "porty": "^3.1.1",
-        "semver": "^7.3.5",
+        "semver": "^7.3.6",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.32.0",
+        "terafoundation": "^0.39.0",
         "uuid": "^8.3.2"
     },
     "devDependencies": {
-        "@terascope/teraslice-op-test-harness": "^1.24.0",
+        "@terascope/teraslice-op-test-harness": "^1.24.1",
         "archiver": "^5.3.0",
         "bufferstreams": "^3.0.0",
-        "chance": "^1.1.7",
+        "chance": "^1.1.8",
         "elasticsearch": "^15.4.1",
-        "got": "^11.8.2",
+        "got": "^11.8.3",
         "jest-fixtures": "^0.6.0",
         "js-yaml": "^4.1.0",
-        "nock": "^13.0.11"
+        "nock": "^13.2.4"
     },
     "engines": {
-        "node": "^12.20.0 || >=14.17.0",
+        "node": "^12.22.0 || >=14.17.0",
         "yarn": ">=1.16.0"
     },
     "publishConfig": {