teraslice 0.78.0 → 0.79.0

package/lib/cluster/services/cluster/backends/kubernetes/k8s.js

@@ -8,10 +8,10 @@ const Request = require('kubernetes-client/backends/request');
 const { getRetryConfig } = require('./utils');
 
 class K8s {
-    constructor(logger, clientConfig, defaultNamespace = 'default',
+    constructor(logger, clientConfig, defaultNamespace,
         apiPollDelay, shutdownTimeout) {
         this.apiPollDelay = apiPollDelay;
-        this.defaultNamespace = defaultNamespace;
+        this.defaultNamespace = defaultNamespace || 'default';
         this.logger = logger;
         this.shutdownTimeout = shutdownTimeout; // this is in milliseconds
 

package/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -25,6 +25,7 @@ class K8sResource {
     constructor(resourceType, resourceName, terasliceConfig, execution) {
         this.execution = execution;
         this.jobLabelPrefix = 'job.teraslice.terascope.io';
+        this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
         this.nodeType = resourceName;
         this.terasliceConfig = terasliceConfig;
 
@@ -54,6 +55,7 @@ class K8sResource {
         this._setImagePullSecret();
         this._setEphemeralStorage();
         this._setExternalPorts();
+        this._setPriorityClassName();
 
         if (resourceName === 'worker') {
             this._setWorkerAntiAffinity();
@@ -219,6 +221,25 @@ class K8sResource {
         }
     }
 
+    _setPriorityClassName() {
+        if (this.terasliceConfig.kubernetes_priority_class_name) {
+            if (this.nodeType === 'execution_controller') {
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
+                if (this.execution.stateful) {
+                    // eslint-disable-next-line max-len
+                    this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
+                }
+            }
+            if (this.nodeType === 'worker' && this.execution.stateful) {
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
+            }
+        }
+    }
+
     _setAssetsVolume() {
         if (this.terasliceConfig.assets_directory && this.terasliceConfig.assets_volume) {
             this.resource.spec.template.spec.volumes.push({
@@ -266,14 +287,40 @@ class K8sResource {
     _setResources() {
         let cpu;
         let memory;
+        let maxMemory;
+
+        const container = this.resource.spec.template.spec.containers[0];
 
         // use teraslice config as defaults and execution config will override it
         const envVars = Object.assign({}, this.terasliceConfig.env_vars, this.execution.env_vars);
 
         if (this.nodeType === 'worker') {
-            // The settings on the executions override the cluster configs
-            cpu = this.execution.cpu || this.terasliceConfig.cpu || -1;
-            memory = this.execution.memory || this.terasliceConfig.memory || -1;
+            if (this.execution.resources_requests_cpu
+                || this.execution.resources_limits_cpu) {
+                if (this.execution.resources_requests_cpu) {
+                    _.set(container, 'resources.requests.cpu', this.execution.resources_requests_cpu);
+                }
+                if (this.execution.resources_limits_cpu) {
+                    _.set(container, 'resources.limits.cpu', this.execution.resources_limits_cpu);
+                }
+            } else if (this.execution.cpu || this.terasliceConfig.cpu) {
+                // The settings on the executions override the cluster configs
+                cpu = this.execution.cpu || this.terasliceConfig.cpu || -1;
+                _.set(container, 'resources.requests.cpu', cpu);
+                _.set(container, 'resources.limits.cpu', cpu);
+            }
+            if (this.execution.resources_requests_memory
+                || this.execution.resources_limits_memory) {
+                _.set(container, 'resources.requests.memory', this.execution.resources_requests_memory);
+                _.set(container, 'resources.limits.memory', this.execution.resources_limits_memory);
+                maxMemory = this.execution.resources_limits_memory;
+            } else if (this.execution.memory || this.terasliceConfig.memory) {
+                // The settings on the executions override the cluster configs
+                memory = this.execution.memory || this.terasliceConfig.memory || -1;
+                _.set(container, 'resources.requests.memory', memory);
+                _.set(container, 'resources.limits.memory', memory);
+                maxMemory = memory;
+            }
         }
 
         if (this.nodeType === 'execution_controller') {
@@ -282,21 +329,17 @@ class K8sResource {
                 || this.terasliceConfig.cpu_execution_controller || -1;
             memory = this.execution.memory_execution_controller
                 || this.terasliceConfig.memory_execution_controller || -1;
-        }
-
-        const container = this.resource.spec.template.spec.containers[0];
-
-        if (cpu !== -1) {
             _.set(container, 'resources.requests.cpu', cpu);
             _.set(container, 'resources.limits.cpu', cpu);
-        }
-
-        if (memory !== -1) {
             _.set(container, 'resources.requests.memory', memory);
             _.set(container, 'resources.limits.memory', memory);
+            maxMemory = memory;
         }
 
-        setMaxOldSpaceViaEnv(container.env, envVars, memory);
+        // NOTE: This sucks, this manages the memory env var but it ALSO is
+        // responsible for doing the config and execution env var merge, which
+        // should NOT be in this function
+        setMaxOldSpaceViaEnv(container.env, envVars, maxMemory);
     }
 
     _setTargets() {

package/lib/config/schemas/system.js

@@ -295,6 +295,11 @@ const schema = {
         default: 'default',
         format: 'optional_String'
     },
+    kubernetes_priority_class_name: {
+        doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
+        default: undefined,
+        format: 'optional_String'
+    },
     kubernetes_config_map_name: {
         doc: 'Specify the name of the Kubernetes ConfigMap used to configure worker pods',
         default: 'teraslice-worker',

package/lib/storage/analytics.js

@@ -50,7 +50,7 @@ module.exports = async function analyticsService(context) {
                 time: stats.time[index],
                 memory: stats.memory[index],
             },
-            null,
+            'index',
             esIndex
         ));
 

package/lib/storage/backends/elasticsearch_store.js

@@ -29,7 +29,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
         recordType,
         idField,
         storageName,
-        bulkSize = 500,
+        bulkSize = 1000,
         fullResponse = false,
         logRecord = true,
         forceRefresh = true,
@@ -161,13 +161,13 @@ module.exports = function elasticsearchStorage(backendConfig) {
      * index saves a record to elasticsearch with a specified ID.
      * If the document is already there it will be replaced.
      */
-    async function indexWithId(recordId, record, indexArg = indexName, timeout) {
+    async function indexWithId(recordId, record, indexArg, timeout) {
         validateIdAndRecord(recordId, record);
 
         logger.trace(`indexWithId call with id: ${recordId}, record`, logRecord ? record : null);
 
         const query = {
-            index: indexArg,
+            index: indexArg || indexName,
             type: recordType,
             id: recordId,
             body: record,
@@ -322,14 +322,17 @@ module.exports = function elasticsearchStorage(backendConfig) {
 
         const type = _type || 'index';
 
-        const indexRequest = {
+        const action = {
             [type]: {
                 _index: indexArg,
                 _type: recordType,
             }
         };
 
-        bulkQueue.push(indexRequest, record);
+        bulkQueue.push({
+            action,
+            data: type === 'delete' ? undefined : record
+        });
 
         // We only flush once enough records have accumulated for it to make sense.
         if (bulkQueue.length >= bulkSize) {
@@ -375,17 +378,7 @@ module.exports = function elasticsearchStorage(backendConfig) {
     }
 
     async function bulkSend(bulkRequest) {
-        const recordCount = (bulkRequest.length / 2);
-
-        await pRetry(async () => elasticsearch.bulkSend(bulkRequest), {
-            reason: `Failure to bulk create "${recordType}"`,
-            logError: logger.warn,
-            delay: isTest ? 100 : 1000,
-            backoff: 5,
-            retries: 100,
-        });
-
-        return recordCount;
+        return elasticsearch.bulkSend(bulkRequest);
     }
 
     async function _flush(shuttingDown = false) {

package/lib/storage/state.js

@@ -60,17 +60,19 @@ async function stateStorage(context) {
     async function createSlices(exId, slices) {
         await waitForClient();
 
-        const bulkRequest = [];
-        for (const slice of slices) {
+        const bulkRequest = slices.map((slice) => {
             const { record, index } = _createSliceRecord(exId, slice, SliceState.pending);
-            bulkRequest.push({
-                index: {
-                    _index: index,
-                    _type: recordType,
-                    _id: record.slice_id,
+            return {
+                action: {
+                    index: {
+                        _index: index,
+                        _type: recordType,
+                        _id: record.slice_id,
+                    },
                 },
-            }, record);
-        }
+                data: record
+            };
+        });
         return backend.bulkSend(bulkRequest);
     }
 
@@ -242,8 +244,8 @@ async function stateStorage(context) {
         }
     }
 
-    async function search(query, from, size, sort = '_updated:desc', fields) {
-        return backend.search(query, from, size, sort, fields);
+    async function search(query, from, size, sort, fields) {
+        return backend.search(query, from, size, sort || '_updated:desc', fields);
     }
 
     async function count(query, from = 0, sort = '_updated:desc') {

package/lib/workers/worker/index.js

@@ -192,7 +192,7 @@ class Worker {
         this.slicesProcessed += 1;
     }
 
-    async shutdown(block = true, event, shutdownError) {
+    async shutdown(block, event, shutdownError) {
         if (this.isShutdown) return;
         if (!this.isInitialized) return;
         const { exId } = this.executionContext;
@@ -202,11 +202,11 @@ class Worker {
                 'worker',
                 `shutdown was called for ${exId}`,
                 'but it was already shutting down',
-                block ? ', will block until done' : ''
+                block !== false ? ', will block until done' : ''
             ];
             this.logger.debug(msgs.join(' '));
 
-            if (block) {
+            if (block !== false) {
                 await waitForWorkerShutdown(this.context, 'worker:shutdown:complete');
             }
             return;

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.78.0",
+    "version": "0.79.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -33,50 +33,50 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "resolutions": {
-        "debug": "^4.3.2",
+        "debug": "^4.3.3",
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^2.23.5",
-        "@terascope/job-components": "^0.54.5",
-        "@terascope/teraslice-messaging": "^0.25.5",
-        "@terascope/utils": "^0.42.5",
+        "@terascope/elasticsearch-api": "^3.0.2",
+        "@terascope/job-components": "^0.56.3",
+        "@terascope/teraslice-messaging": "^0.27.1",
+        "@terascope/utils": "^0.44.1",
         "async-mutex": "^0.3.2",
         "barbe": "^3.0.16",
-        "body-parser": "^1.19.0",
+        "body-parser": "^1.19.1",
         "convict": "^4.4.1",
         "decompress": "^4.2.1",
-        "easy-table": "^1.1.1",
+        "easy-table": "^1.2.0",
         "event-loop-stats": "^1.2.0",
-        "express": "^4.17.1",
-        "fs-extra": "^9.1.0",
+        "express": "^4.17.2",
+        "fs-extra": "^10.0.0",
         "gc-stats": "^1.4.0",
-        "got": "^11.8.2",
+        "got": "^11.8.3",
         "ip": "^1.1.5",
         "kubernetes-client": "^9.0.0",
         "lodash": "^4.17.21",
         "ms": "^2.1.3",
-        "nanoid": "^3.1.25",
+        "nanoid": "^3.2.0",
         "porty": "^3.1.1",
         "semver": "^7.3.5",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.35.5",
+        "terafoundation": "^0.37.1",
         "uuid": "^8.3.2"
     },
     "devDependencies": {
-        "@terascope/teraslice-op-test-harness": "^1.24.0",
+        "@terascope/teraslice-op-test-harness": "^1.24.1",
         "archiver": "^5.3.0",
         "bufferstreams": "^3.0.0",
         "chance": "^1.1.8",
         "elasticsearch": "^15.4.1",
-        "got": "^11.8.2",
+        "got": "^11.8.3",
         "jest-fixtures": "^0.6.0",
         "js-yaml": "^4.1.0",
-        "nock": "^13.1.3"
+        "nock": "^13.2.1"
     },
     "engines": {
-        "node": "^12.20.0 || >=14.17.0",
+        "node": "^12.22.0 || >=14.17.0",
         "yarn": ">=1.16.0"
     },
     "publishConfig": {