teraslice 0.81.0 → 0.83.0

package/lib/cluster/services/cluster/backends/kubernetes/index.js

@@ -94,7 +94,7 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
 
         execution.slicer_port = 45680;
         const exJobResource = new K8sResource(
-            'jobs', 'execution_controller', context.sysconfig.teraslice, execution
+            'jobs', 'execution_controller', context.sysconfig.teraslice, execution, logger
         );
         const exJob = exJobResource.resource;
 
@@ -135,7 +135,7 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
         execution.k8sUid = jobs.items[0].metadata.uid;
 
         const kr = new K8sResource(
-            'deployments', 'worker', context.sysconfig.teraslice, execution
+            'deployments', 'worker', context.sysconfig.teraslice, execution, logger
         );
 
         const workerDeployment = kr.resource;

package/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -22,10 +22,11 @@ class K8sResource {
      * @param {Object} terasliceConfig - teraslice cluster config from context
      * @param {Object} execution - teraslice execution
      */
-    constructor(resourceType, resourceName, terasliceConfig, execution) {
+    constructor(resourceType, resourceName, terasliceConfig, execution, logger) {
         this.execution = execution;
         this.jobLabelPrefix = 'job.teraslice.terascope.io';
         this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
+        this.logger = logger;
         this.nodeType = resourceName;
         this.terasliceConfig = terasliceConfig;
 
@@ -68,6 +69,10 @@ class K8sResource {
         if (resourceName === 'execution_controller') {
             this._setExecutionControllerTargets();
         }
+
+        if (this.terasliceConfig.kubernetes_overrides_enabled) {
+            this._mergePodSpecOverlay();
+        }
     }
 
     _makeConfig() {
@@ -412,6 +417,27 @@ class K8sResource {
             effect: 'NoSchedule'
         });
     }
+
+    /**
+     * _mergePodSpecOverlay - allows the author of the job to override anything
+     * in the pod .spec for both the execution controller and the worker pods
+     * created in Kubernetes.  This can be useful in many ways including these:
+     *
+     *   * add `initContainers` to the pods
+     *   * add `hostAliases` to the pods
+     *
+     * Note that this happens at the end of the process, so anything added by
+     * this overlay will overwrite any other setting set on the job or by the
+     * config.
+     *
+     * Job setting: `pod_spec_override`
+     */
+    _mergePodSpecOverlay() {
+        this.resource.spec.template.spec = _.merge(
+            this.resource.spec.template.spec,
+            this.execution.pod_spec_override
+        );
+    }
 }
 
 module.exports = K8sResource;

package/lib/config/schemas/system.js

@@ -295,6 +295,11 @@ const schema = {
         default: 'default',
         format: 'optional_String'
     },
+    kubernetes_overrides_enabled: {
+        doc: '',
+        default: false,
+        format: Boolean
+    },
     kubernetes_priority_class_name: {
         doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
         default: undefined,

package/lib/storage/backends/elasticsearch_store.js

@@ -377,7 +377,7 @@ module.exports = async function elasticsearchStorage(backendConfig) {
         bulkQueue = [];
 
         try {
-            const recordCount = await bulkSend(bulkRequest);
+            const { recordCount } = await bulkSend(bulkRequest);
             const extraMsg = shuttingDown ? ', on shutdown' : '';
             logger.debug(`flushed ${recordCount}${extraMsg} records to index ${indexName}`);
         } finally {

package/lib/workers/execution-controller/scheduler.js

@@ -393,9 +393,9 @@ class Scheduler {
         this._creating += slices.length;
 
         try {
-            const count = await this.stateStore.createSlices(this.exId, slices);
+            const { recordCount } = await this.stateStore.createSlices(this.exId, slices);
             this.enqueueSlices(slices);
-            this._creating -= count;
+            this._creating -= recordCount;
         } catch (err) {
             const { lifecycle } = this.executionContext.config;
             if (lifecycle === 'once') {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.81.0",
+    "version": "0.83.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -37,18 +37,18 @@
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^3.3.1",
-        "@terascope/job-components": "^0.58.1",
-        "@terascope/teraslice-messaging": "^0.28.1",
-        "@terascope/utils": "^0.45.1",
+        "@terascope/elasticsearch-api": "^3.4.0",
+        "@terascope/job-components": "^0.58.5",
+        "@terascope/teraslice-messaging": "^0.28.5",
+        "@terascope/utils": "^0.45.5",
         "async-mutex": "^0.4.0",
         "barbe": "^3.0.16",
-        "body-parser": "^1.20.0",
-        "convict": "^6.2.3",
+        "body-parser": "^1.20.2",
+        "convict": "^6.2.4",
         "decompress": "^4.2.1",
         "easy-table": "^1.2.0",
         "event-loop-stats": "^1.4.1",
-        "express": "^4.18.1",
+        "express": "^4.18.2",
         "fs-extra": "^10.1.0",
         "gc-stats": "^1.4.0",
         "got": "^11.8.3",
@@ -61,19 +61,19 @@
         "semver": "^7.3.8",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.41.0",
+        "terafoundation": "^0.42.5",
         "uuid": "^9.0.0"
     },
     "devDependencies": {
         "@terascope/teraslice-op-test-harness": "^1.24.1",
         "archiver": "^5.3.1",
         "bufferstreams": "^3.0.0",
-        "chance": "^1.1.8",
+        "chance": "^1.1.10",
         "elasticsearch": "^15.4.1",
         "got": "^11.8.3",
         "jest-fixtures": "^0.6.0",
         "js-yaml": "^4.1.0",
-        "nock": "^13.2.9"
+        "nock": "^13.3.0"
     },
     "engines": {
         "node": ">=14.17.0",