teraslice 2.10.0 → 2.12.0

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js

@@ -0,0 +1,67 @@
+import { cloneDeep } from '@terascope/utils';
+import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json';
+import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
+describe('k8sState with pods from multiple clusters', () => {
+    it('should generate cluster state correctly on first call', () => {
+        const podsJobRunning = cloneDeep(_podsJobRunning);
+        const clusterState = {};
+        gen(podsJobRunning, clusterState);
+        // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`);
+        // console.log(JSON.stringify(podsJobRunning, null, 2));
+        expect(clusterState['192.168.99.100'].state).toEqual('connected');
+        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
+        expect(clusterState['192.168.99.100'].active[1])
+            .toEqual({
+            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
+            assignment: 'execution_controller',
+            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.5',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+        expect(clusterState['192.168.99.100'].active[2])
+            .toEqual({
+            worker_id: 'teraslice-worker-123456-8b68v7p8t',
+            assignment: 'worker',
+            pod_name: 'teraslice-worker-123456-8b68v7p8t',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.6',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+    });
+    it('should generate cluster state correctly on second call', () => {
+        const podsJobRunning = cloneDeep(_podsJobRunning);
+        const clusterState = {};
+        gen(podsJobRunning, clusterState);
+        gen(podsJobRunning, clusterState);
+        expect(clusterState['192.168.99.100'].state).toEqual('connected');
+        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
+        expect(clusterState['192.168.99.100'].active[1])
+            .toEqual({
+            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
+            assignment: 'execution_controller',
+            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.5',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+        expect(clusterState['192.168.99.100'].active[2])
+            .toEqual({
+            worker_id: 'teraslice-worker-123456-8b68v7p8t',
+            assignment: 'worker',
+            pod_name: 'teraslice-worker-123456-8b68v7p8t',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.6',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+    });
+});
+//# sourceMappingURL=k8sState-multicluster-v2-spec.js.map

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js

@@ -0,0 +1,84 @@
+import { cloneDeep } from '@terascope/utils';
+import _podsJobRunning from '../files/job-running-v1-k8s-pods.json';
+import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
+describe('k8sState', () => {
+    it('should generate cluster state correctly on first call', () => {
+        const podsJobRunning = cloneDeep(_podsJobRunning);
+        const clusterState = {};
+        gen(podsJobRunning, clusterState);
+        // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`);
+        expect(clusterState['192.168.99.100'].state).toEqual('connected');
+        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
+        expect(clusterState['192.168.99.100'].active[1])
+            .toEqual({
+            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
+            assignment: 'execution_controller',
+            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.5',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+        expect(clusterState['192.168.99.100'].active[2])
+            .toEqual({
+            worker_id: 'teraslice-worker-123456-8b68v7p8t',
+            assignment: 'worker',
+            pod_name: 'teraslice-worker-123456-8b68v7p8t',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.6',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+    });
+    it('should generate cluster state correctly on second call', () => {
+        const podsJobRunning = cloneDeep(_podsJobRunning);
+        const clusterState = {};
+        gen(podsJobRunning, clusterState);
+        gen(podsJobRunning, clusterState);
+        expect(clusterState['192.168.99.100'].state).toEqual('connected');
+        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
+        expect(clusterState['192.168.99.100'].active[1])
+            .toEqual({
+            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
+            assignment: 'execution_controller',
+            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.5',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+        expect(clusterState['192.168.99.100'].active[2])
+            .toEqual({
+            worker_id: 'teraslice-worker-123456-8b68v7p8t',
+            assignment: 'worker',
+            pod_name: 'teraslice-worker-123456-8b68v7p8t',
+            ex_id: '123456',
+            job_id: '654321',
+            pod_ip: '172.17.0.6',
+            assets: [],
+            image: 'docker.registry.example/teraslice:0.0.0'
+        });
+    });
+    it('should remove old host ips', () => {
+        const podsJobRunning = cloneDeep(_podsJobRunning);
+        const clusterState = {};
+        clusterState['2.2.2.2'] = {
+            node_id: '2.2.2.2',
+            hostname: '2.2.2.2',
+            pid: 'N/A',
+            node_version: 'N/A',
+            teraslice_version: 'N/A',
+            total: 'N/A',
+            state: 'idk',
+            available: 'N/A',
+            active: []
+        };
+        gen(podsJobRunning, clusterState);
+        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
+        expect(clusterState['2.2.2.2']).toBeUndefined();
+    });
+});
+//# sourceMappingURL=k8sState-v2-spec.js.map

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.js

@@ -0,0 +1,320 @@
+import { makeTemplate, getMaxOldSpace } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js';
+import { safeEncode } from '../../../../../../../../src/lib/utils/encoding_utils.js';
+describe('K8s Utils', () => {
+    describe('->makeTemplate', () => {
+        describe('execution_controller job', () => {
+            it('should be able to support the execution_controller job', () => {
+                const exJobTemplate = makeTemplate('jobs', 'execution_controller');
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-worker',
+                    exName: 'ts-exc-example-job-74ab9324-9f67',
+                    exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91',
+                    replicas: 1,
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    jobId: 'some-job-id',
+                    nodeType: 'execution_controller',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    shutdownTimeout: 12345
+                };
+                const exJob = exJobTemplate(config);
+                expect(exJob.kind).toEqual('Job');
+                expect(exJob.metadata).toEqual({
+                    labels: {
+                        'app.kubernetes.io/name': 'teraslice',
+                        'app.kubernetes.io/component': config.nodeType,
+                        'teraslice.terascope.io/exId': config.exId,
+                        'teraslice.terascope.io/jobId': config.jobId,
+                        'teraslice.terascope.io/jobName': config.jobNameLabel,
+                        'app.kubernetes.io/instance': config.clusterNameLabel
+                    },
+                    name: config.name,
+                    namespace: config.namespace
+                });
+                expect(exJob.spec.template.metadata.labels).toEqual(exJob.metadata.labels);
+                const templateSpec = exJob.spec.template.spec;
+                expect(templateSpec.containers[0].image).toEqual(config.dockerImage);
+                expect(templateSpec.containers[0].name).toEqual(config.name);
+                expect(templateSpec.containers[0].env).toEqual([
+                    {
+                        name: 'NODE_TYPE',
+                        value: config.nodeType
+                    },
+                    {
+                        name: 'EX',
+                        value: config.execution
+                    },
+                    {
+                        name: 'POD_IP',
+                        valueFrom: {
+                            fieldRef: {
+                                fieldPath: 'status.podIP'
+                            }
+                        }
+                    }
+                ]);
+                expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout);
+            });
+            it('should throw error if docker image undefined on config for job', () => {
+                const exJobTemplate = makeTemplate('jobs', 'execution_controller');
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-worker',
+                    exName: 'ts-exc-example-job-74ab9324-9f67',
+                    exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91',
+                    replicas: 1,
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    jobId: 'some-job-id',
+                    nodeType: 'execution_controller',
+                    namespace: 'some-namespace',
+                    dockerImage: undefined,
+                    execution: safeEncode({ example: 'hello' }),
+                    shutdownTimeout: 12345
+                };
+                expect(() => exJobTemplate(config))
+                    .toThrow('K8s config requires a dockerImage to create a jobs template');
+            });
+        });
+        describe('worker deployment', () => {
+            const workerDeploymentTemplate = makeTemplate('deployments', 'worker');
+            it('should be able to support the worker deployment', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-worker',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: 'example-job-abcd',
+                    exUid: 'UID1',
+                    jobId: 'some-job-id',
+                    nodeType: 'worker',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                const workerDeployment = workerDeploymentTemplate(config);
+                expect(workerDeployment.kind).toEqual('Deployment');
+                expect(workerDeployment.metadata).toEqual({
+                    labels: {
+                        'app.kubernetes.io/name': 'teraslice',
+                        'app.kubernetes.io/component': config.nodeType,
+                        'teraslice.terascope.io/exId': config.exId,
+                        'teraslice.terascope.io/jobId': config.jobId,
+                        'teraslice.terascope.io/jobName': config.jobNameLabel,
+                        'app.kubernetes.io/instance': config.clusterNameLabel
+                    },
+                    name: config.name,
+                    namespace: config.namespace,
+                    ownerReferences: [
+                        {
+                            apiVersion: 'batch/v1',
+                            blockOwnerDeletion: false,
+                            controller: false,
+                            kind: 'Job',
+                            name: 'example-job-abcd',
+                            uid: 'UID1',
+                        },
+                    ],
+                });
+                expect(workerDeployment.spec.replicas).toEqual(config.replicas);
+                const labels = workerDeployment.spec.template.metadata.labels;
+                expect(labels).toEqual(workerDeployment.metadata.labels);
+                const templateSpec = workerDeployment.spec.template.spec;
+                expect(templateSpec.containers[0].image).toEqual(config.dockerImage);
+                expect(templateSpec.containers[0].name).toEqual(config.name);
+                expect(templateSpec.containers[0].env).toEqual([
+                    {
+                        name: 'NODE_TYPE',
+                        value: config.nodeType
+                    },
+                    {
+                        name: 'EX',
+                        value: config.execution
+                    },
+                    {
+                        name: 'POD_IP',
+                        valueFrom: {
+                            fieldRef: {
+                                fieldPath: 'status.podIP'
+                            }
+                        }
+                    }
+                ]);
+                expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout);
+            });
+            it('should throw error if docker image undefined on config for deployment', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-worker',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: 'example-job-abcd',
+                    exUid: 'UID1',
+                    jobId: 'some-job-id',
+                    nodeType: 'worker',
+                    namespace: 'some-namespace',
+                    dockerImage: undefined,
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                expect(() => workerDeploymentTemplate(config))
+                    .toThrow('K8s config requires a dockerImage to create a deployments template');
+            });
+            it('should throw error if exName undefined on config for deployment', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-worker',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: undefined,
+                    exUid: 'UID1',
+                    jobId: 'some-job-id',
+                    nodeType: 'worker',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exName to create a deployments template');
+            });
+            it('should throw error if exUid undefined on config for deployment', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-worker',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: 'example-job-abcd',
+                    exUid: undefined,
+                    jobId: 'some-job-id',
+                    nodeType: 'worker',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exUid to create a deployments template');
+            });
+        });
+        describe('execution_controller service', () => {
+            const exServiceTemplate = makeTemplate('services', 'execution_controller');
+            it('should be able to support the execution_controller service', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-exService',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: 'example-job-abcd',
+                    exUid: 'UID1',
+                    jobId: 'some-job-id',
+                    nodeType: 'execution_controller',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                const exService = exServiceTemplate(config);
+                expect(exService.kind).toEqual('Service');
+                expect(exService.metadata).toEqual({
+                    labels: {
+                        'app.kubernetes.io/name': 'teraslice',
+                        'app.kubernetes.io/component': config.nodeType,
+                        'teraslice.terascope.io/exId': config.exId,
+                        'teraslice.terascope.io/jobId': config.jobId
+                    },
+                    name: `svc-${config.name}`,
+                    namespace: config.namespace,
+                    ownerReferences: [
+                        {
+                            apiVersion: 'batch/v1',
+                            blockOwnerDeletion: false,
+                            controller: false,
+                            kind: 'Job',
+                            name: 'example-job-abcd',
+                            uid: 'UID1',
+                        },
+                    ],
+                });
+                expect(exService.spec.selector).toEqual({
+                    'app.kubernetes.io/component': 'execution_controller',
+                    'teraslice.terascope.io/exId': config.exId
+                });
+                expect(exService.spec.ports).toEqual([
+                    {
+                        port: 45680,
+                        targetPort: 45680
+                    }
+                ]);
+            });
+            it('should throw error if exName undefined on config for service', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-exService',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: undefined,
+                    exUid: 'UID1',
+                    jobId: 'some-job-id',
+                    nodeType: 'execution_controller',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                expect(() => exServiceTemplate(config)).toThrow('K8s config requires exName to create a services template');
+            });
+            it('should throw error if exUid undefined on config for service', () => {
+                const config = {
+                    clusterName: 'teracluster',
+                    configMapName: 'teracluster-exService',
+                    name: 'example',
+                    jobNameLabel: 'example-job',
+                    clusterNameLabel: 'example-cluster',
+                    exId: 'some-ex-id',
+                    exName: 'example-job-abcd',
+                    exUid: undefined,
+                    jobId: 'some-job-id',
+                    nodeType: 'execution_controller',
+                    namespace: 'some-namespace',
+                    dockerImage: 'some/docker-image',
+                    execution: safeEncode({ example: 'hello' }),
+                    replicas: 1,
+                    shutdownTimeout: 12345
+                };
+                expect(() => exServiceTemplate(config)).toThrow('K8s config requires exUid to create a services template');
+            });
+        });
+    });
+    describe('->getMaxOldSpace', () => {
+        it('should convert bytes to mb', () => {
+            const oneGBInBytes = 1024 * 1024 * 1024;
+            expect(getMaxOldSpace(oneGBInBytes)).toEqual(Math.round(0.9 * 1024));
+        });
+    });
+});
+//# sourceMappingURL=utils-v2-spec.js.map

package/dist/test/lib/cluster/services/cluster/backends/state-utils-spec.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as stateUtils from '../../../../../../src/lib/cluster/services/cluster/backends/state-utils.js';
+const dirname = path.dirname(fileURLToPath(import.meta.url));
+const clusterStateFile = path.join(dirname, 'files', 'sample-state.json');
+describe('stateUtils', () => {
+    let clusterState;
+    beforeEach(() => {
+        clusterState = JSON.parse(fs.readFileSync(clusterStateFile, 'utf-8'));
+    });
+    describe('->iterateState', () => {
+        it('can filter worker list and return cluster master node', () => {
+            const workerList = stateUtils.iterateState(clusterState, (worker) => worker.assignment === 'cluster_master');
+            expect(workerList[0].node_id).toEqual('node1.lan');
+        });
+    });
+    describe('->findAllSlicers', () => {
+        it('returns the right number of slicers', () => {
+            const workerList = stateUtils.findAllSlicers(clusterState);
+            expect(workerList[0].assignment).toEqual('execution_controller');
+        });
+    });
+    describe('->findWorkersByExecutionID', () => {
+        it('returns the right number of workers', () => {
+            const workerList = stateUtils.findWorkersByExecutionID(clusterState, '8d7669e1-d351-4a0d-b36e-b11da5ade26a');
+            expect(workerList.length).toEqual(8);
+        });
+    });
+    describe('->findSlicersByExecutionID', () => {
+        it('returns execution_controller worker', () => {
+            const workerList = stateUtils.findSlicersByExecutionID(clusterState, { '8d7669e1-d351-4a0d-b36e-b11da5ade26a': '8d7669e1-d351-4a0d-b36e-b11da5ade26a' });
+            expect(workerList[0].assignment).toEqual('execution_controller');
+        });
+    });
+});
+//# sourceMappingURL=state-utils-spec.js.map