teraslice 3.3.0 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/LICENSE +202 -0
  2. package/package.json +25 -28
  3. package/dist/src/interfaces.js +0 -12
  4. package/dist/src/lib/cluster/cluster_master.js +0 -246
  5. package/dist/src/lib/cluster/node_master.js +0 -355
  6. package/dist/src/lib/cluster/services/api.js +0 -663
  7. package/dist/src/lib/cluster/services/assets.js +0 -224
  8. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js +0 -192
  9. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js +0 -2
  10. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js +0 -419
  11. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js +0 -60
  12. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js +0 -55
  13. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js +0 -357
  14. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js +0 -37
  15. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js +0 -60
  16. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js +0 -170
  17. package/dist/src/lib/cluster/services/cluster/backends/native/dispatch.js +0 -13
  18. package/dist/src/lib/cluster/services/cluster/backends/native/index.js +0 -526
  19. package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js +0 -548
  20. package/dist/src/lib/cluster/services/cluster/backends/state-utils.js +0 -26
  21. package/dist/src/lib/cluster/services/cluster/index.js +0 -13
  22. package/dist/src/lib/cluster/services/execution.js +0 -435
  23. package/dist/src/lib/cluster/services/index.js +0 -6
  24. package/dist/src/lib/cluster/services/interfaces.js +0 -2
  25. package/dist/src/lib/cluster/services/jobs.js +0 -458
  26. package/dist/src/lib/config/default-sysconfig.js +0 -25
  27. package/dist/src/lib/config/index.js +0 -20
  28. package/dist/src/lib/config/schemas/system.js +0 -360
  29. package/dist/src/lib/storage/analytics.js +0 -86
  30. package/dist/src/lib/storage/assets.js +0 -401
  31. package/dist/src/lib/storage/backends/elasticsearch_store.js +0 -496
  32. package/dist/src/lib/storage/backends/mappings/analytics.js +0 -20
  33. package/dist/src/lib/storage/backends/mappings/asset.js +0 -32
  34. package/dist/src/lib/storage/backends/mappings/ex.js +0 -53
  35. package/dist/src/lib/storage/backends/mappings/job.js +0 -42
  36. package/dist/src/lib/storage/backends/mappings/state.js +0 -16
  37. package/dist/src/lib/storage/backends/s3_store.js +0 -237
  38. package/dist/src/lib/storage/execution.js +0 -302
  39. package/dist/src/lib/storage/index.js +0 -7
  40. package/dist/src/lib/storage/jobs.js +0 -81
  41. package/dist/src/lib/storage/state.js +0 -254
  42. package/dist/src/lib/utils/api_utils.js +0 -128
  43. package/dist/src/lib/utils/asset_utils.js +0 -94
  44. package/dist/src/lib/utils/date_utils.js +0 -52
  45. package/dist/src/lib/utils/encoding_utils.js +0 -27
  46. package/dist/src/lib/utils/events.js +0 -4
  47. package/dist/src/lib/utils/file_utils.js +0 -124
  48. package/dist/src/lib/utils/id_utils.js +0 -15
  49. package/dist/src/lib/utils/port_utils.js +0 -32
  50. package/dist/src/lib/workers/assets/index.js +0 -3
  51. package/dist/src/lib/workers/assets/loader-executable.js +0 -40
  52. package/dist/src/lib/workers/assets/loader.js +0 -73
  53. package/dist/src/lib/workers/assets/spawn.js +0 -55
  54. package/dist/src/lib/workers/context/execution-context.js +0 -12
  55. package/dist/src/lib/workers/context/terafoundation-context.js +0 -8
  56. package/dist/src/lib/workers/execution-controller/execution-analytics.js +0 -188
  57. package/dist/src/lib/workers/execution-controller/index.js +0 -1024
  58. package/dist/src/lib/workers/execution-controller/recovery.js +0 -151
  59. package/dist/src/lib/workers/execution-controller/scheduler.js +0 -390
  60. package/dist/src/lib/workers/execution-controller/slice-analytics.js +0 -96
  61. package/dist/src/lib/workers/helpers/job.js +0 -80
  62. package/dist/src/lib/workers/helpers/op-analytics.js +0 -22
  63. package/dist/src/lib/workers/helpers/terafoundation.js +0 -34
  64. package/dist/src/lib/workers/helpers/worker-shutdown.js +0 -147
  65. package/dist/src/lib/workers/metrics/index.js +0 -108
  66. package/dist/src/lib/workers/worker/index.js +0 -378
  67. package/dist/src/lib/workers/worker/slice.js +0 -122
  68. package/dist/test/config/schemas/system_schema-spec.js +0 -26
  69. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js +0 -458
  70. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js +0 -818
  71. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js +0 -67
  72. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js +0 -84
  73. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.js +0 -320
  74. package/dist/test/lib/cluster/services/cluster/backends/state-utils-spec.js +0 -37
  75. package/dist/test/node_master-spec.js +0 -194
  76. package/dist/test/services/api-spec.js +0 -79
  77. package/dist/test/services/assets-spec.js +0 -158
  78. package/dist/test/services/messaging-spec.js +0 -440
  79. package/dist/test/storage/assets_storage-spec.js +0 -95
  80. package/dist/test/storage/s3_store-spec.js +0 -149
  81. package/dist/test/test.config.js +0 -23
  82. package/dist/test/test.setup.js +0 -6
  83. package/dist/test/utils/api_utils-spec.js +0 -25
  84. package/dist/test/utils/asset_utils-spec.js +0 -141
  85. package/dist/test/utils/elastic_utils-spec.js +0 -25
  86. package/dist/test/workers/execution-controller/execution-controller-spec.js +0 -371
  87. package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js +0 -519
  88. package/dist/test/workers/execution-controller/execution-test-cases-spec.js +0 -343
  89. package/dist/test/workers/execution-controller/recovery-spec.js +0 -160
  90. package/dist/test/workers/execution-controller/scheduler-spec.js +0 -249
  91. package/dist/test/workers/execution-controller/slice-analytics-spec.js +0 -121
  92. package/dist/test/workers/fixtures/ops/example-op/processor.js +0 -20
  93. package/dist/test/workers/fixtures/ops/example-op/schema.js +0 -19
  94. package/dist/test/workers/fixtures/ops/example-reader/fetcher.js +0 -20
  95. package/dist/test/workers/fixtures/ops/example-reader/schema.js +0 -41
  96. package/dist/test/workers/fixtures/ops/example-reader/slicer.js +0 -37
  97. package/dist/test/workers/fixtures/ops/new-op/processor.js +0 -29
  98. package/dist/test/workers/fixtures/ops/new-op/schema.js +0 -18
  99. package/dist/test/workers/fixtures/ops/new-reader/fetcher.js +0 -19
  100. package/dist/test/workers/fixtures/ops/new-reader/schema.js +0 -23
  101. package/dist/test/workers/fixtures/ops/new-reader/slicer.js +0 -13
  102. package/dist/test/workers/helpers/configs.js +0 -128
  103. package/dist/test/workers/helpers/execution-controller-helper.js +0 -49
  104. package/dist/test/workers/helpers/index.js +0 -5
  105. package/dist/test/workers/helpers/test-context.js +0 -210
  106. package/dist/test/workers/helpers/zip-directory.js +0 -25
  107. package/dist/test/workers/worker/slice-spec.js +0 -333
  108. package/dist/test/workers/worker/worker-spec.js +0 -356
@@ -1,357 +0,0 @@
1
- import { isNumber, get, set, has, isEmpty, merge } from '@terascope/core-utils';
2
- import { safeEncode } from '../../../../../utils/encoding_utils.js';
3
- import { setMaxOldSpaceViaEnv } from './utils.js';
4
- export class K8sResource {
5
- execution;
6
- jobLabelPrefix;
7
- jobPropertyLabelPrefix;
8
- logger;
9
- terasliceConfig;
10
- /**
11
- * K8sResource allows the generation of k8s resources based on templates.
12
- * After creating the object, the k8s resource is accessible on the objects
13
- * .resource property.
14
- *
15
- * @param {Object} terasliceConfig - teraslice cluster config from context
16
- * @param {Object} execution - teraslice execution
17
- * @param {Logger} logger - teraslice logger
18
- */
19
- constructor(terasliceConfig, execution, logger) {
20
- this.execution = execution;
21
- this.jobLabelPrefix = 'job.teraslice.terascope.io';
22
- this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
23
- this.logger = logger;
24
- this.terasliceConfig = terasliceConfig;
25
- }
26
- _setEnvVariables() {
27
- }
28
- _mountLocalTeraslice(resource) {
29
- const devMounts = JSON.parse(Buffer.from(process.env.MOUNT_LOCAL_TERASLICE, 'base64').toString('utf-8'));
30
- resource.spec.template.spec.containers[0].volumeMounts.push(...devMounts.volumeMounts);
31
- resource.spec.template.spec.volumes.push(...devMounts.volumes);
32
- if (resource.spec.template.spec.containers[0]) {
33
- resource.spec.template.spec.containers[0].args = [
34
- 'node',
35
- 'service.js'
36
- ];
37
- }
38
- }
39
- _makeConfig(nameInfix, exName, exUid) {
40
- const clusterName = get(this.terasliceConfig, 'name');
41
- const clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
42
- const configMapName = get(this.terasliceConfig, 'kubernetes_config_map_name', `${this.terasliceConfig.name}-worker`);
43
- const dockerImage = this.execution.kubernetes_image
44
- || this.terasliceConfig.kubernetes_image;
45
- // name needs to be a valid DNS name since it is used in the svc name,
46
- // so we can only permit alphanumeric and - characters. _ is forbidden.
47
- // -> regex used for validation is '[a-z]([-a-z0-9]*[a-z0-9])?'
48
- const jobNameLabel = this.execution.name
49
- .toLowerCase()
50
- .replace(/[^a-zA-Z0-9\-.]/g, '-')
51
- .replace(/^[^a-z]/, 'a')
52
- .replace(/[^a-z0-9]$/, '0')
53
- .substring(0, 63);
54
- const name = `ts-${nameInfix}-${jobNameLabel.substring(0, 35)}-${this.execution.job_id.substring(0, 13)}`;
55
- const shutdownTimeoutMs = get(this.terasliceConfig, 'shutdown_timeout', 60000);
56
- const shutdownTimeoutSeconds = Math.round(shutdownTimeoutMs / 1000);
57
- const config = {
58
- clusterName,
59
- clusterNameLabel,
60
- configMapName,
61
- dockerImage,
62
- execution: safeEncode(this.execution),
63
- exId: this.execution.ex_id,
64
- exName: exName,
65
- exUid: exUid,
66
- jobId: this.execution.job_id,
67
- jobNameLabel,
68
- name,
69
- namespace: get(this.terasliceConfig, 'kubernetes_namespace', 'default'),
70
- nodeType: this.nodeType,
71
- replicas: this.execution.workers,
72
- shutdownTimeout: shutdownTimeoutSeconds
73
- };
74
- return config;
75
- }
76
- _setWorkerAntiAffinity(resource) {
77
- if (this.terasliceConfig.kubernetes_worker_antiaffinity) {
78
- const targetKey = 'spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution';
79
- if (!has(this.resource, targetKey)) {
80
- set(this.resource, targetKey, []);
81
- }
82
- resource.spec.template.spec.affinity?.podAntiAffinity
83
- ?.preferredDuringSchedulingIgnoredDuringExecution?.push({
84
- weight: 1,
85
- podAffinityTerm: {
86
- labelSelector: {
87
- matchExpressions: [
88
- {
89
- key: 'app.kubernetes.io/name',
90
- operator: 'In',
91
- values: [
92
- 'teraslice'
93
- ]
94
- },
95
- {
96
- key: 'app.kubernetes.io/instance',
97
- operator: 'In',
98
- values: [
99
- this.templateConfig.clusterNameLabel
100
- ]
101
- }
102
- ]
103
- },
104
- topologyKey: 'kubernetes.io/hostname'
105
- }
106
- });
107
- }
108
- }
109
- /**
110
- * Execution Controllers get tolerations and required affinities
111
- *
112
- * NOTE: We considered changing `execution_controller_targets` to be an
113
- * object but the inconsistency with `targets` made this awkward. See the
114
- * `teraslice config with execution_controller_targets and job targets set`
115
- * test for an example. If the syntax for this were to change, we should
116
- * also consider changing `execution.targets`, which is a change on the job.
117
- */
118
- _setExecutionControllerTargets(resource) {
119
- if (this.terasliceConfig.execution_controller_targets) {
120
- this.terasliceConfig.execution_controller_targets.forEach((target) => {
121
- this._setTargetRequired(target, resource);
122
- this._setTargetAccepted(target, resource);
123
- });
124
- }
125
- }
126
- _setEphemeralStorage(resource) {
127
- if (this.execution.ephemeral_storage) {
128
- resource.spec.template.spec.containers[0].volumeMounts.push({
129
- name: 'ephemeral-volume',
130
- mountPath: '/ephemeral0'
131
- });
132
- resource.spec.template.spec.volumes.push({
133
- name: 'ephemeral-volume',
134
- emptyDir: {}
135
- });
136
- }
137
- }
138
- _setExternalPorts(resource) {
139
- if (this.execution.external_ports) {
140
- this.execution.external_ports.forEach((portValue) => {
141
- if (isNumber(portValue)) {
142
- resource.spec.template.spec.containers[0].ports
143
- .push({ containerPort: portValue });
144
- }
145
- else {
146
- resource.spec.template.spec.containers[0].ports
147
- .push({
148
- name: portValue.name,
149
- containerPort: portValue.port
150
- });
151
- }
152
- });
153
- }
154
- }
155
- _setImagePullSecret(resource) {
156
- if (this.terasliceConfig.kubernetes_image_pull_secret) {
157
- if (resource.spec.template.spec.imagePullSecrets) {
158
- resource.spec.template.spec.imagePullSecrets.push({ name: this.terasliceConfig.kubernetes_image_pull_secret });
159
- }
160
- else {
161
- resource.spec.template.spec.imagePullSecrets = [
162
- { name: this.terasliceConfig.kubernetes_image_pull_secret }
163
- ];
164
- }
165
- }
166
- }
167
- _setPriorityClassName(resource) {
168
- if (this.terasliceConfig.kubernetes_priority_class_name) {
169
- const className = this.terasliceConfig.kubernetes_priority_class_name;
170
- if (this.nodeType === 'execution_controller') {
171
- resource.spec.template.spec.priorityClassName = className;
172
- if (this.execution.stateful) {
173
- resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
174
- }
175
- }
176
- if (this.nodeType === 'worker' && this.execution.stateful) {
177
- resource.spec.template.spec.priorityClassName = className;
178
- resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
179
- }
180
- }
181
- }
182
- _setAssetsVolume(resource) {
183
- if (this.terasliceConfig.assets_volume
184
- && this.terasliceConfig.assets_directory
185
- && typeof this.terasliceConfig.assets_directory === 'string') {
186
- resource.spec.template.spec.volumes.push({
187
- name: this.terasliceConfig.assets_volume,
188
- persistentVolumeClaim: { claimName: this.terasliceConfig.assets_volume }
189
- });
190
- resource.spec.template.spec.containers[0].volumeMounts.push({
191
- name: this.terasliceConfig.assets_volume,
192
- mountPath: this.terasliceConfig.assets_directory
193
- });
194
- }
195
- }
196
- _setJobLabels(resource) {
197
- if (this.execution.labels != null) {
198
- Object.entries(this.execution.labels).forEach(([k, v]) => {
199
- const key = `${this.jobLabelPrefix}/${k.replace(/[^a-zA-Z0-9\-._]/g, '-').substring(0, 63)}`;
200
- const value = v.replace(/[^a-zA-Z0-9\-._]/g, '-').substring(0, 63);
201
- resource.metadata.labels[key] = value;
202
- resource.spec.template.metadata.labels[key] = value;
203
- });
204
- }
205
- }
206
- _setVolumes(resource) {
207
- if (this.execution.volumes != null) {
208
- this.execution.volumes.forEach((volume) => {
209
- resource.spec.template.spec.volumes.push({
210
- name: volume.name,
211
- persistentVolumeClaim: { claimName: volume.name }
212
- });
213
- resource.spec.template.spec.containers[0].volumeMounts.push({
214
- name: volume.name,
215
- mountPath: volume.path
216
- });
217
- });
218
- }
219
- }
220
- _setResources(resource) {
221
- let cpu;
222
- let memory;
223
- let maxMemory;
224
- const container = resource.spec.template.spec.containers[0];
225
- // use teraslice config as defaults and execution config will override it
226
- const envVars = Object.assign({}, this.terasliceConfig.env_vars, this.execution.env_vars);
227
- if (this.nodeType === 'worker') {
228
- if (this.execution.resources_requests_cpu
229
- || this.execution.resources_limits_cpu) {
230
- if (this.execution.resources_requests_cpu) {
231
- set(container, 'resources.requests.cpu', this.execution.resources_requests_cpu);
232
- }
233
- if (this.execution.resources_limits_cpu) {
234
- set(container, 'resources.limits.cpu', this.execution.resources_limits_cpu);
235
- }
236
- }
237
- else if (this.execution.cpu || this.terasliceConfig.cpu) {
238
- // The settings on the executions override the cluster configs
239
- cpu = this.execution.cpu || this.terasliceConfig.cpu || -1;
240
- set(container, 'resources.requests.cpu', cpu);
241
- set(container, 'resources.limits.cpu', cpu);
242
- }
243
- if (this.execution.resources_requests_memory
244
- || this.execution.resources_limits_memory) {
245
- set(container, 'resources.requests.memory', this.execution.resources_requests_memory);
246
- set(container, 'resources.limits.memory', this.execution.resources_limits_memory);
247
- maxMemory = this.execution.resources_limits_memory;
248
- }
249
- else if (this.execution.memory || this.terasliceConfig.memory) {
250
- // The settings on the executions override the cluster configs
251
- memory = this.execution.memory || this.terasliceConfig.memory || -1;
252
- set(container, 'resources.requests.memory', memory);
253
- set(container, 'resources.limits.memory', memory);
254
- maxMemory = memory;
255
- }
256
- }
257
- if (this.nodeType === 'execution_controller') {
258
- // The settings on the executions override the cluster configs
259
- cpu = this.execution.cpu_execution_controller
260
- || this.terasliceConfig.cpu_execution_controller || -1;
261
- memory = this.execution.memory_execution_controller
262
- || this.terasliceConfig.memory_execution_controller || -1;
263
- set(container, 'resources.requests.cpu', cpu);
264
- set(container, 'resources.limits.cpu', cpu);
265
- set(container, 'resources.requests.memory', memory);
266
- set(container, 'resources.limits.memory', memory);
267
- maxMemory = memory;
268
- }
269
- // NOTE: This sucks, this manages the memory env var but it ALSO is
270
- // responsible for doing the config and execution env var merge, which
271
- // should NOT be in this function
272
- if (container.env === undefined) {
273
- throw new Error('Resource container V1EnvVar[] undefined while setting resources.');
274
- }
275
- setMaxOldSpaceViaEnv(container.env, envVars, maxMemory);
276
- }
277
- _setTargets(resource) {
278
- if (this.execution.targets && !isEmpty(this.execution.targets)) {
279
- this.execution.targets?.forEach((target) => {
280
- // `required` is the default if no `constraint` is provided for
281
- // backwards compatibility and as the most likely case
282
- if (target.constraint === 'required' || !has(target, 'constraint')) {
283
- this._setTargetRequired(target, resource);
284
- }
285
- if (target.constraint === 'preferred') {
286
- this._setTargetPreferred(target, resource);
287
- }
288
- if (target.constraint === 'accepted') {
289
- this._setTargetAccepted(target, resource);
290
- }
291
- });
292
- }
293
- }
294
- _setTargetRequired(target, resource) {
295
- const targetKey = 'spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution';
296
- if (!has(this.resource, targetKey)) {
297
- const nodeSelectorObj = {
298
- nodeSelectorTerms: [{ matchExpressions: [] }]
299
- };
300
- set(this.resource, targetKey, nodeSelectorObj);
301
- }
302
- resource.spec.template.spec.affinity?.nodeAffinity
303
- ?.requiredDuringSchedulingIgnoredDuringExecution
304
- ?.nodeSelectorTerms[0].matchExpressions?.push({
305
- key: target.key,
306
- operator: 'In',
307
- values: [target.value]
308
- });
309
- }
310
- _setTargetPreferred(target, resource) {
311
- const targetKey = 'spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution';
312
- if (!has(this.resource, targetKey)) {
313
- set(this.resource, targetKey, []);
314
- }
315
- resource.spec.template.spec.affinity?.nodeAffinity
316
- ?.preferredDuringSchedulingIgnoredDuringExecution?.push({
317
- weight: 1,
318
- preference: {
319
- matchExpressions: [{
320
- key: target.key,
321
- operator: 'In',
322
- values: [target.value]
323
- }]
324
- }
325
- });
326
- }
327
- _setTargetAccepted(target, resource) {
328
- const targetKey = 'spec.template.spec.tolerations';
329
- if (!has(this.resource, targetKey)) {
330
- set(this.resource, targetKey, []);
331
- }
332
- resource.spec.template.spec.tolerations?.push({
333
- key: target.key,
334
- operator: 'Equal',
335
- value: target.value,
336
- effect: 'NoSchedule'
337
- });
338
- }
339
- /**
340
- * _mergePodSpecOverlay - allows the author of the job to override anything
341
- * in the pod .spec for both the execution controller and the worker pods
342
- * created in Kubernetes. This can be useful in many ways including these:
343
- *
344
- * * add `initContainers` to the pods
345
- * * add `hostAliases` to the pods
346
- *
347
- * Note that this happens at the end of the process, so anything added by
348
- * this overlay will overwrite any other setting set on the job or by the
349
- * config.
350
- *
351
- * Job setting: `pod_spec_override`
352
- */
353
- _mergePodSpecOverlay(resource) {
354
- resource.spec.template.spec = merge(resource.spec.template.spec, this.execution.pod_spec_override);
355
- }
356
- }
357
- //# sourceMappingURL=k8sResource.js.map
@@ -1,37 +0,0 @@
1
- import { V1Service } from '@kubernetes/client-node';
2
- import { convertToTSResource, makeTemplate } from './utils.js';
3
- import { K8sResource } from './k8sResource.js';
4
- export class K8sServiceResource extends K8sResource {
5
- nodeType = 'execution_controller';
6
- nameInfix = 'exc';
7
- templateGenerator;
8
- templateConfig;
9
- resource;
10
- exName;
11
- exUid;
12
- /**
13
- * K8sServiceResource allows the generation of a k8s service based on a template.
14
- * After creating the object, the k8s service is accessible on the objects
15
- * .resource property.
16
- *
17
- * @param {Object} terasliceConfig - teraslice cluster config from context
18
- * @param {Object} execution - teraslice execution
19
- * @param {Logger} logger - teraslice logger
20
- * @param {String} exName - name from execution resource
21
- * @param {String} exUid - uid from execution resource
22
- */
23
- constructor(terasliceConfig, execution, logger, exName, exUid) {
24
- super(terasliceConfig, execution, logger);
25
- this.execution = execution;
26
- this.logger = logger;
27
- this.terasliceConfig = terasliceConfig;
28
- this.exName = exName;
29
- this.exUid = exUid;
30
- this.templateGenerator = makeTemplate('services', this.nodeType);
31
- this.templateConfig = this._makeConfig(this.nameInfix, exName, exUid);
32
- const k8sService = new V1Service();
33
- Object.assign(k8sService, this.templateGenerator(this.templateConfig));
34
- this.resource = convertToTSResource(k8sService);
35
- }
36
- }
37
- //# sourceMappingURL=k8sServiceResource.js.map
@@ -1,60 +0,0 @@
1
- import { get, has, uniq, difference } from '@terascope/core-utils';
2
- /**
3
- * Given the k8s Pods API output generates the appropriate Teraslice cluster
4
- * state. NOTE: This assumes the pods have already been filtered to ensure they
5
- * are teraslice pods and match the cluster in question.
6
- * @param {TSPodList} k8sPods k8s pods API object (k8s v1.10+)
7
- * @param {Object} clusterState Teraslice Cluster State
8
- * @param {String} clusterNameLabel k8s label containing clusterName
9
- * @param {Logger} logger Teraslice logger
10
- */
11
- export function gen(k8sPods, clusterState) {
12
- // Make sure we clean up the old
13
- const hostIPs = uniq(k8sPods.items.map((item) => get(item, 'status.hostIP')));
14
- const oldHostIps = difference(Object.keys(clusterState), hostIPs);
15
- oldHostIps.forEach((ip) => {
16
- delete clusterState[ip];
17
- });
18
- // Loop over the nodes in clusterState and set active = [] so we can append
19
- // later
20
- Object.keys(clusterState).forEach((nodeId) => {
21
- clusterState[nodeId].active = [];
22
- });
23
- // add a worker for each pod
24
- k8sPods.items.forEach((pod) => {
25
- if (!has(clusterState, pod.status.hostIP)) {
26
- // If the node isn't in clusterState, add it
27
- clusterState[pod.status.hostIP] = {
28
- node_id: pod.status.hostIP,
29
- hostname: pod.status.hostIP,
30
- pid: 'N/A',
31
- node_version: 'N/A',
32
- teraslice_version: 'N/A',
33
- total: 'N/A',
34
- state: 'connected',
35
- available: 'N/A',
36
- active: []
37
- };
38
- }
39
- const worker = {
40
- assets: [],
41
- assignment: pod.metadata.labels['app.kubernetes.io/component'],
42
- ex_id: pod.metadata.labels['teraslice.terascope.io/exId'],
43
- // WARNING: This makes the assumption that the first container
44
- // in the pod is the teraslice container. Currently it is the
45
- // only container, so this assumption is safe for now.
46
- image: pod.spec.containers[0].image,
47
- job_id: pod.metadata.labels['teraslice.terascope.io/jobId'],
48
- pod_name: pod.metadata.name,
49
- pod_ip: pod.status.podIP,
50
- worker_id: pod.metadata.name,
51
- };
52
- // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`,
53
- // `Failed`, `Unknown`. We will only add `Running` pods to the
54
- // Teraslice cluster state.
55
- if (pod.status.phase === 'Running') {
56
- clusterState[pod.status.hostIP].active.push(worker);
57
- }
58
- });
59
- }
60
- //# sourceMappingURL=k8sState.js.map
@@ -1,170 +0,0 @@
1
- import fs from 'node:fs';
2
- import path from 'node:path';
3
- // @ts-expect-error
4
- import barbe from 'barbe';
5
- import * as k8s from '@kubernetes/client-node';
6
- import { isTest } from '@terascope/core-utils';
7
- const MAX_RETRIES = isTest ? 2 : 3;
8
- const RETRY_DELAY = isTest ? 50 : 1000; // time in ms
9
- const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/');
10
- export function makeTemplate(folder, fileName) {
11
- const filePath = path.join(resourcePath, folder, `${fileName}.hbs`);
12
- const templateData = fs.readFileSync(filePath, 'utf-8');
13
- const templateKeys = ['{{', '}}'];
14
- return (config) => {
15
- if (folder !== 'jobs' && (config.exName === undefined || config.exUid === undefined)) {
16
- throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`);
17
- }
18
- if (folder !== 'services' && config.dockerImage === undefined) {
19
- throw new Error(`K8s config requires a dockerImage to create a ${folder} template`);
20
- }
21
- const templated = barbe(templateData, templateKeys, config);
22
- return JSON.parse(templated);
23
- };
24
- }
25
- // Convert bytes to MB and reduce by 10%
26
- export function getMaxOldSpace(memory) {
27
- return Math.round(0.9 * (memory / 1024 / 1024));
28
- }
29
- export function setMaxOldSpaceViaEnv(envArr, jobEnv, memory) {
30
- const envObj = {};
31
- if (memory && memory > -1) {
32
- // Set NODE_OPTIONS to override max-old-space-size
33
- const maxOldSpace = getMaxOldSpace(memory);
34
- envObj.NODE_OPTIONS = `--max-old-space-size=${maxOldSpace}`;
35
- }
36
- Object.assign(envObj, jobEnv);
37
- Object.entries(envObj).forEach(([name, value]) => {
38
- envArr.push({
39
- name,
40
- value
41
- });
42
- });
43
- }
44
- export function getRetryConfig() {
45
- return {
46
- retries: MAX_RETRIES,
47
- delay: RETRY_DELAY
48
- };
49
- }
50
- export function isDeployment(resource) {
51
- return resource instanceof k8s.V1Deployment;
52
- }
53
- export function isJob(resource) {
54
- return resource instanceof k8s.V1Job;
55
- }
56
- export function isPod(resource) {
57
- return resource instanceof k8s.V1Pod;
58
- }
59
- export function isReplicaSet(resource) {
60
- return resource instanceof k8s.V1ReplicaSet;
61
- }
62
- export function isService(resource) {
63
- return resource instanceof k8s.V1Service;
64
- }
65
- export function isTSDeployment(manifest) {
66
- return manifest instanceof k8s.V1Deployment
67
- && manifest.metadata?.labels !== undefined
68
- && manifest.metadata.name !== undefined
69
- && manifest.spec?.replicas !== undefined
70
- && manifest.spec.template.metadata?.labels !== undefined
71
- && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined
72
- && manifest.spec.template.spec.volumes !== undefined;
73
- }
74
- export function isTSJob(manifest) {
75
- return manifest instanceof k8s.V1Job
76
- && manifest.metadata?.labels !== undefined
77
- && manifest.metadata.name !== undefined
78
- && manifest.spec?.template.metadata?.labels !== undefined
79
- && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined
80
- && manifest.spec.template.spec.volumes !== undefined;
81
- }
82
- export function isTSPod(manifest) {
83
- return manifest instanceof k8s.V1Pod
84
- && manifest.metadata?.name !== undefined
85
- && manifest.status !== undefined;
86
- }
87
- export function isTSReplicaSet(manifest) {
88
- return manifest instanceof k8s.V1ReplicaSet
89
- && manifest.metadata?.name !== undefined
90
- && manifest.status !== undefined;
91
- }
92
- export function isTSService(manifest) {
93
- return manifest instanceof k8s.V1Service
94
- && manifest.metadata?.name !== undefined
95
- && manifest.spec?.selector !== undefined
96
- && manifest.spec.ports !== undefined;
97
- }
98
- export function convertToTSResource(resource) {
99
- if (isDeployment(resource) && isTSDeployment(resource)) {
100
- return resource;
101
- }
102
- if (isJob(resource) && isTSJob(resource)) {
103
- return resource;
104
- }
105
- if (isPod(resource) && isTSPod(resource)) {
106
- return resource;
107
- }
108
- if (isReplicaSet(resource) && isTSReplicaSet(resource)) {
109
- return resource;
110
- }
111
- if (isService(resource) && isTSService(resource)) {
112
- return resource;
113
- }
114
- throw new Error('K8sResource missing required field(s) to be converted to TSResource.');
115
- }
116
- export function isDeploymentList(manifest) {
117
- return manifest.kind === 'DeploymentList';
118
- }
119
- export function isJobList(manifest) {
120
- return manifest.kind === 'JobList';
121
- }
122
- export function isPodList(manifest) {
123
- return manifest.kind === 'PodList';
124
- }
125
- export function isReplicaSetList(manifest) {
126
- return manifest.kind === 'ReplicaSetList';
127
- }
128
- export function isServiceList(manifest) {
129
- return manifest.kind === 'ServiceList';
130
- }
131
- export function isTSDeploymentList(manifest) {
132
- return manifest.kind === 'DeploymentList'
133
- && (manifest.items[0] ? isTSDeployment(manifest.items[0]) : true);
134
- }
135
- export function isTSJobList(manifest) {
136
- return manifest.kind === 'JobList'
137
- && (manifest.items[0] ? isTSJob(manifest.items[0]) : true);
138
- }
139
- export function isTSPodList(manifest) {
140
- return manifest.kind === 'PodList'
141
- && (manifest.items[0] ? isTSPod(manifest.items[0]) : true);
142
- }
143
- export function isTSReplicaSetList(manifest) {
144
- return manifest.kind === 'ReplicaSetList'
145
- && (manifest.items[0] ? isTSReplicaSet(manifest.items[0]) : true);
146
- }
147
- export function isTSServiceList(manifest) {
148
- return manifest.kind === 'ServiceList'
149
- && (manifest.items[0] ? isTSService(manifest.items[0]) : true);
150
- }
151
- export function convertToTSResourceList(resourceList) {
152
- resourceList.items.map((resource) => convertToTSResource(resource));
153
- if (isDeploymentList(resourceList) && isTSDeploymentList(resourceList)) {
154
- return resourceList;
155
- }
156
- if (isJobList(resourceList) && isTSJobList(resourceList)) {
157
- return resourceList;
158
- }
159
- if (isPodList(resourceList) && isTSPodList(resourceList)) {
160
- return resourceList;
161
- }
162
- if (isReplicaSetList(resourceList) && isTSReplicaSetList(resourceList)) {
163
- return resourceList;
164
- }
165
- if (isServiceList(resourceList) && isTSServiceList(resourceList)) {
166
- return resourceList;
167
- }
168
- throw new Error('K8sResource missing required field(s) to be converted to TSResourceList.');
169
- }
170
- //# sourceMappingURL=utils.js.map
@@ -1,13 +0,0 @@
1
- export class Dispatch {
2
- dispatch = {};
3
- set(nodeId, numOfWorkers) {
4
- if (this.dispatch[nodeId]) {
5
- this.dispatch[nodeId] += numOfWorkers;
6
- }
7
- else {
8
- this.dispatch[nodeId] = numOfWorkers;
9
- }
10
- }
11
- getDispatch = () => this.dispatch;
12
- }
13
- //# sourceMappingURL=dispatch.js.map