teraslice 0.82.0 → 0.84.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -94,7 +94,7 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
|
|
|
94
94
|
|
|
95
95
|
execution.slicer_port = 45680;
|
|
96
96
|
const exJobResource = new K8sResource(
|
|
97
|
-
'jobs', 'execution_controller', context.sysconfig.teraslice, execution
|
|
97
|
+
'jobs', 'execution_controller', context.sysconfig.teraslice, execution, logger
|
|
98
98
|
);
|
|
99
99
|
const exJob = exJobResource.resource;
|
|
100
100
|
|
|
@@ -135,7 +135,7 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
|
|
|
135
135
|
execution.k8sUid = jobs.items[0].metadata.uid;
|
|
136
136
|
|
|
137
137
|
const kr = new K8sResource(
|
|
138
|
-
'deployments', 'worker', context.sysconfig.teraslice, execution
|
|
138
|
+
'deployments', 'worker', context.sysconfig.teraslice, execution, logger
|
|
139
139
|
);
|
|
140
140
|
|
|
141
141
|
const workerDeployment = kr.resource;
|
|
@@ -22,10 +22,11 @@ class K8sResource {
|
|
|
22
22
|
* @param {Object} terasliceConfig - teraslice cluster config from context
|
|
23
23
|
* @param {Object} execution - teraslice execution
|
|
24
24
|
*/
|
|
25
|
-
constructor(resourceType, resourceName, terasliceConfig, execution) {
|
|
25
|
+
constructor(resourceType, resourceName, terasliceConfig, execution, logger) {
|
|
26
26
|
this.execution = execution;
|
|
27
27
|
this.jobLabelPrefix = 'job.teraslice.terascope.io';
|
|
28
28
|
this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
|
|
29
|
+
this.logger = logger;
|
|
29
30
|
this.nodeType = resourceName;
|
|
30
31
|
this.terasliceConfig = terasliceConfig;
|
|
31
32
|
|
|
@@ -68,6 +69,10 @@ class K8sResource {
|
|
|
68
69
|
if (resourceName === 'execution_controller') {
|
|
69
70
|
this._setExecutionControllerTargets();
|
|
70
71
|
}
|
|
72
|
+
|
|
73
|
+
if (this.terasliceConfig.kubernetes_overrides_enabled) {
|
|
74
|
+
this._mergePodSpecOverlay();
|
|
75
|
+
}
|
|
71
76
|
}
|
|
72
77
|
|
|
73
78
|
_makeConfig() {
|
|
@@ -412,6 +417,27 @@ class K8sResource {
|
|
|
412
417
|
effect: 'NoSchedule'
|
|
413
418
|
});
|
|
414
419
|
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* _mergePodSpecOverlay - allows the author of the job to override anything
|
|
423
|
+
* in the pod .spec for both the execution controller and the worker pods
|
|
424
|
+
* created in Kubernetes. This can be useful in many ways including these:
|
|
425
|
+
*
|
|
426
|
+
* * add `initContainers` to the pods
|
|
427
|
+
* * add `hostAliases` to the pods
|
|
428
|
+
*
|
|
429
|
+
* Note that this happens at the end of the process, so anything added by
|
|
430
|
+
* this overlay will overwrite any other setting set on the job or by the
|
|
431
|
+
* config.
|
|
432
|
+
*
|
|
433
|
+
* Job setting: `pod_spec_override`
|
|
434
|
+
*/
|
|
435
|
+
_mergePodSpecOverlay() {
|
|
436
|
+
this.resource.spec.template.spec = _.merge(
|
|
437
|
+
this.resource.spec.template.spec,
|
|
438
|
+
this.execution.pod_spec_override
|
|
439
|
+
);
|
|
440
|
+
}
|
|
415
441
|
}
|
|
416
442
|
|
|
417
443
|
module.exports = K8sResource;
|
|
@@ -295,6 +295,11 @@ const schema = {
|
|
|
295
295
|
default: 'default',
|
|
296
296
|
format: 'optional_String'
|
|
297
297
|
},
|
|
298
|
+
kubernetes_overrides_enabled: {
|
|
299
|
+
doc: '',
|
|
300
|
+
default: false,
|
|
301
|
+
format: Boolean
|
|
302
|
+
},
|
|
298
303
|
kubernetes_priority_class_name: {
|
|
299
304
|
doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
|
|
300
305
|
default: undefined,
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "teraslice",
|
|
3
3
|
"displayName": "Teraslice",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.84.0",
|
|
5
5
|
"description": "Distributed computing platform for processing JSON data",
|
|
6
6
|
"homepage": "https://github.com/terascope/teraslice#readme",
|
|
7
7
|
"bugs": {
|
|
@@ -37,18 +37,18 @@
|
|
|
37
37
|
"ms": "^2.1.3"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@terascope/elasticsearch-api": "^3.
|
|
41
|
-
"@terascope/job-components": "^0.58.
|
|
42
|
-
"@terascope/teraslice-messaging": "^0.28.
|
|
43
|
-
"@terascope/utils": "^0.45.
|
|
40
|
+
"@terascope/elasticsearch-api": "^3.5.0",
|
|
41
|
+
"@terascope/job-components": "^0.58.5",
|
|
42
|
+
"@terascope/teraslice-messaging": "^0.28.5",
|
|
43
|
+
"@terascope/utils": "^0.45.5",
|
|
44
44
|
"async-mutex": "^0.4.0",
|
|
45
45
|
"barbe": "^3.0.16",
|
|
46
|
-
"body-parser": "^1.20.
|
|
47
|
-
"convict": "^6.2.
|
|
46
|
+
"body-parser": "^1.20.2",
|
|
47
|
+
"convict": "^6.2.4",
|
|
48
48
|
"decompress": "^4.2.1",
|
|
49
49
|
"easy-table": "^1.2.0",
|
|
50
50
|
"event-loop-stats": "^1.4.1",
|
|
51
|
-
"express": "^4.18.
|
|
51
|
+
"express": "^4.18.2",
|
|
52
52
|
"fs-extra": "^10.1.0",
|
|
53
53
|
"gc-stats": "^1.4.0",
|
|
54
54
|
"got": "^11.8.3",
|
|
@@ -61,19 +61,19 @@
|
|
|
61
61
|
"semver": "^7.3.8",
|
|
62
62
|
"socket.io": "^1.7.4",
|
|
63
63
|
"socket.io-client": "^1.7.4",
|
|
64
|
-
"terafoundation": "^0.
|
|
64
|
+
"terafoundation": "^0.43.0",
|
|
65
65
|
"uuid": "^9.0.0"
|
|
66
66
|
},
|
|
67
67
|
"devDependencies": {
|
|
68
68
|
"@terascope/teraslice-op-test-harness": "^1.24.1",
|
|
69
69
|
"archiver": "^5.3.1",
|
|
70
70
|
"bufferstreams": "^3.0.0",
|
|
71
|
-
"chance": "^1.1.
|
|
71
|
+
"chance": "^1.1.10",
|
|
72
72
|
"elasticsearch": "^15.4.1",
|
|
73
73
|
"got": "^11.8.3",
|
|
74
74
|
"jest-fixtures": "^0.6.0",
|
|
75
75
|
"js-yaml": "^4.1.0",
|
|
76
|
-
"nock": "^13.
|
|
76
|
+
"nock": "^13.3.0"
|
|
77
77
|
},
|
|
78
78
|
"engines": {
|
|
79
79
|
"node": ">=14.17.0",
|