teraslice 3.3.0 → 3.3.2

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js

@@ -1,67 +0,0 @@
-import { cloneDeep } from '@terascope/core-utils';
-import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json';
-import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
-describe('k8sState with pods from multiple clusters', () => {
-    it('should generate cluster state correctly on first call', () => {
-        const podsJobRunning = cloneDeep(_podsJobRunning);
-        const clusterState = {};
-        gen(podsJobRunning, clusterState);
-        // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`);
-        // console.log(JSON.stringify(podsJobRunning, null, 2));
-        expect(clusterState['192.168.99.100'].state).toEqual('connected');
-        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
-        expect(clusterState['192.168.99.100'].active[1])
-            .toEqual({
-            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
-            assignment: 'execution_controller',
-            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.5',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-        expect(clusterState['192.168.99.100'].active[2])
-            .toEqual({
-            worker_id: 'teraslice-worker-123456-8b68v7p8t',
-            assignment: 'worker',
-            pod_name: 'teraslice-worker-123456-8b68v7p8t',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.6',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-    });
-    it('should generate cluster state correctly on second call', () => {
-        const podsJobRunning = cloneDeep(_podsJobRunning);
-        const clusterState = {};
-        gen(podsJobRunning, clusterState);
-        gen(podsJobRunning, clusterState);
-        expect(clusterState['192.168.99.100'].state).toEqual('connected');
-        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
-        expect(clusterState['192.168.99.100'].active[1])
-            .toEqual({
-            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
-            assignment: 'execution_controller',
-            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.5',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-        expect(clusterState['192.168.99.100'].active[2])
-            .toEqual({
-            worker_id: 'teraslice-worker-123456-8b68v7p8t',
-            assignment: 'worker',
-            pod_name: 'teraslice-worker-123456-8b68v7p8t',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.6',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-    });
-});
-//# sourceMappingURL=k8sState-multicluster-v2-spec.js.map

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js

@@ -1,84 +0,0 @@
-import { cloneDeep } from '@terascope/core-utils';
-import _podsJobRunning from '../files/job-running-v1-k8s-pods.json';
-import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
-describe('k8sState', () => {
-    it('should generate cluster state correctly on first call', () => {
-        const podsJobRunning = cloneDeep(_podsJobRunning);
-        const clusterState = {};
-        gen(podsJobRunning, clusterState);
-        // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`);
-        expect(clusterState['192.168.99.100'].state).toEqual('connected');
-        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
-        expect(clusterState['192.168.99.100'].active[1])
-            .toEqual({
-            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
-            assignment: 'execution_controller',
-            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.5',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-        expect(clusterState['192.168.99.100'].active[2])
-            .toEqual({
-            worker_id: 'teraslice-worker-123456-8b68v7p8t',
-            assignment: 'worker',
-            pod_name: 'teraslice-worker-123456-8b68v7p8t',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.6',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-    });
-    it('should generate cluster state correctly on second call', () => {
-        const podsJobRunning = cloneDeep(_podsJobRunning);
-        const clusterState = {};
-        gen(podsJobRunning, clusterState);
-        gen(podsJobRunning, clusterState);
-        expect(clusterState['192.168.99.100'].state).toEqual('connected');
-        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
-        expect(clusterState['192.168.99.100'].active[1])
-            .toEqual({
-            worker_id: 'teraslice-execution_controller-123456-784cbt5mz',
-            assignment: 'execution_controller',
-            pod_name: 'teraslice-execution_controller-123456-784cbt5mz',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.5',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-        expect(clusterState['192.168.99.100'].active[2])
-            .toEqual({
-            worker_id: 'teraslice-worker-123456-8b68v7p8t',
-            assignment: 'worker',
-            pod_name: 'teraslice-worker-123456-8b68v7p8t',
-            ex_id: '123456',
-            job_id: '654321',
-            pod_ip: '172.17.0.6',
-            assets: [],
-            image: 'docker.registry.example/teraslice:0.0.0'
-        });
-    });
-    it('should remove old host ips', () => {
-        const podsJobRunning = cloneDeep(_podsJobRunning);
-        const clusterState = {};
-        clusterState['2.2.2.2'] = {
-            node_id: '2.2.2.2',
-            hostname: '2.2.2.2',
-            pid: 'N/A',
-            node_version: 'N/A',
-            teraslice_version: 'N/A',
-            total: 'N/A',
-            state: 'idk',
-            available: 'N/A',
-            active: []
-        };
-        gen(podsJobRunning, clusterState);
-        expect(clusterState['192.168.99.100'].active.length).toEqual(3);
-        expect(clusterState['2.2.2.2']).toBeUndefined();
-    });
-});
-//# sourceMappingURL=k8sState-v2-spec.js.map

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.js

@@ -1,320 +0,0 @@
-import { makeTemplate, getMaxOldSpace } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js';
-import { safeEncode } from '../../../../../../../../src/lib/utils/encoding_utils.js';
-describe('K8s Utils', () => {
-    describe('->makeTemplate', () => {
-        describe('execution_controller job', () => {
-            it('should be able to support the execution_controller job', () => {
-                const exJobTemplate = makeTemplate('jobs', 'execution_controller');
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-worker',
-                    exName: 'ts-exc-example-job-74ab9324-9f67',
-                    exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91',
-                    replicas: 1,
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    jobId: 'some-job-id',
-                    nodeType: 'execution_controller',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    shutdownTimeout: 12345
-                };
-                const exJob = exJobTemplate(config);
-                expect(exJob.kind).toEqual('Job');
-                expect(exJob.metadata).toEqual({
-                    labels: {
-                        'app.kubernetes.io/name': 'teraslice',
-                        'app.kubernetes.io/component': config.nodeType,
-                        'teraslice.terascope.io/exId': config.exId,
-                        'teraslice.terascope.io/jobId': config.jobId,
-                        'teraslice.terascope.io/jobName': config.jobNameLabel,
-                        'app.kubernetes.io/instance': config.clusterNameLabel
-                    },
-                    name: config.name,
-                    namespace: config.namespace
-                });
-                expect(exJob.spec.template.metadata.labels).toEqual(exJob.metadata.labels);
-                const templateSpec = exJob.spec.template.spec;
-                expect(templateSpec.containers[0].image).toEqual(config.dockerImage);
-                expect(templateSpec.containers[0].name).toEqual(config.name);
-                expect(templateSpec.containers[0].env).toEqual([
-                    {
-                        name: 'NODE_TYPE',
-                        value: config.nodeType
-                    },
-                    {
-                        name: 'EX',
-                        value: config.execution
-                    },
-                    {
-                        name: 'POD_IP',
-                        valueFrom: {
-                            fieldRef: {
-                                fieldPath: 'status.podIP'
-                            }
-                        }
-                    }
-                ]);
-                expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout);
-            });
-            it('should throw error if docker image undefined on config for job', () => {
-                const exJobTemplate = makeTemplate('jobs', 'execution_controller');
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-worker',
-                    exName: 'ts-exc-example-job-74ab9324-9f67',
-                    exUid: '0512dae5-8ca2-437d-b744-fdc50695fd91',
-                    replicas: 1,
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    jobId: 'some-job-id',
-                    nodeType: 'execution_controller',
-                    namespace: 'some-namespace',
-                    dockerImage: undefined,
-                    execution: safeEncode({ example: 'hello' }),
-                    shutdownTimeout: 12345
-                };
-                expect(() => exJobTemplate(config))
-                    .toThrow('K8s config requires a dockerImage to create a jobs template');
-            });
-        });
-        describe('worker deployment', () => {
-            const workerDeploymentTemplate = makeTemplate('deployments', 'worker');
-            it('should be able to support the worker deployment', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-worker',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: 'example-job-abcd',
-                    exUid: 'UID1',
-                    jobId: 'some-job-id',
-                    nodeType: 'worker',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                const workerDeployment = workerDeploymentTemplate(config);
-                expect(workerDeployment.kind).toEqual('Deployment');
-                expect(workerDeployment.metadata).toEqual({
-                    labels: {
-                        'app.kubernetes.io/name': 'teraslice',
-                        'app.kubernetes.io/component': config.nodeType,
-                        'teraslice.terascope.io/exId': config.exId,
-                        'teraslice.terascope.io/jobId': config.jobId,
-                        'teraslice.terascope.io/jobName': config.jobNameLabel,
-                        'app.kubernetes.io/instance': config.clusterNameLabel
-                    },
-                    name: config.name,
-                    namespace: config.namespace,
-                    ownerReferences: [
-                        {
-                            apiVersion: 'batch/v1',
-                            blockOwnerDeletion: false,
-                            controller: false,
-                            kind: 'Job',
-                            name: 'example-job-abcd',
-                            uid: 'UID1',
-                        },
-                    ],
-                });
-                expect(workerDeployment.spec.replicas).toEqual(config.replicas);
-                const labels = workerDeployment.spec.template.metadata.labels;
-                expect(labels).toEqual(workerDeployment.metadata.labels);
-                const templateSpec = workerDeployment.spec.template.spec;
-                expect(templateSpec.containers[0].image).toEqual(config.dockerImage);
-                expect(templateSpec.containers[0].name).toEqual(config.name);
-                expect(templateSpec.containers[0].env).toEqual([
-                    {
-                        name: 'NODE_TYPE',
-                        value: config.nodeType
-                    },
-                    {
-                        name: 'EX',
-                        value: config.execution
-                    },
-                    {
-                        name: 'POD_IP',
-                        valueFrom: {
-                            fieldRef: {
-                                fieldPath: 'status.podIP'
-                            }
-                        }
-                    }
-                ]);
-                expect(templateSpec.terminationGracePeriodSeconds).toEqual(config.shutdownTimeout);
-            });
-            it('should throw error if docker image undefined on config for deployment', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-worker',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: 'example-job-abcd',
-                    exUid: 'UID1',
-                    jobId: 'some-job-id',
-                    nodeType: 'worker',
-                    namespace: 'some-namespace',
-                    dockerImage: undefined,
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                expect(() => workerDeploymentTemplate(config))
-                    .toThrow('K8s config requires a dockerImage to create a deployments template');
-            });
-            it('should throw error if exName undefined on config for deployment', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-worker',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: undefined,
-                    exUid: 'UID1',
-                    jobId: 'some-job-id',
-                    nodeType: 'worker',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exName to create a deployments template');
-            });
-            it('should throw error if exUid undefined on config for deployment', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-worker',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: 'example-job-abcd',
-                    exUid: undefined,
-                    jobId: 'some-job-id',
-                    nodeType: 'worker',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                expect(() => workerDeploymentTemplate(config)).toThrow('K8s config requires exUid to create a deployments template');
-            });
-        });
-        describe('execution_controller service', () => {
-            const exServiceTemplate = makeTemplate('services', 'execution_controller');
-            it('should be able to support the execution_controller service', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-exService',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: 'example-job-abcd',
-                    exUid: 'UID1',
-                    jobId: 'some-job-id',
-                    nodeType: 'execution_controller',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                const exService = exServiceTemplate(config);
-                expect(exService.kind).toEqual('Service');
-                expect(exService.metadata).toEqual({
-                    labels: {
-                        'app.kubernetes.io/name': 'teraslice',
-                        'app.kubernetes.io/component': config.nodeType,
-                        'teraslice.terascope.io/exId': config.exId,
-                        'teraslice.terascope.io/jobId': config.jobId
-                    },
-                    name: `svc-${config.name}`,
-                    namespace: config.namespace,
-                    ownerReferences: [
-                        {
-                            apiVersion: 'batch/v1',
-                            blockOwnerDeletion: false,
-                            controller: false,
-                            kind: 'Job',
-                            name: 'example-job-abcd',
-                            uid: 'UID1',
-                        },
-                    ],
-                });
-                expect(exService.spec.selector).toEqual({
-                    'app.kubernetes.io/component': 'execution_controller',
-                    'teraslice.terascope.io/exId': config.exId
-                });
-                expect(exService.spec.ports).toEqual([
-                    {
-                        port: 45680,
-                        targetPort: 45680
-                    }
-                ]);
-            });
-            it('should throw error if exName undefined on config for service', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-exService',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: undefined,
-                    exUid: 'UID1',
-                    jobId: 'some-job-id',
-                    nodeType: 'execution_controller',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                expect(() => exServiceTemplate(config)).toThrow('K8s config requires exName to create a services template');
-            });
-            it('should throw error if exUid undefined on config for service', () => {
-                const config = {
-                    clusterName: 'teracluster',
-                    configMapName: 'teracluster-exService',
-                    name: 'example',
-                    jobNameLabel: 'example-job',
-                    clusterNameLabel: 'example-cluster',
-                    exId: 'some-ex-id',
-                    exName: 'example-job-abcd',
-                    exUid: undefined,
-                    jobId: 'some-job-id',
-                    nodeType: 'execution_controller',
-                    namespace: 'some-namespace',
-                    dockerImage: 'some/docker-image',
-                    execution: safeEncode({ example: 'hello' }),
-                    replicas: 1,
-                    shutdownTimeout: 12345
-                };
-                expect(() => exServiceTemplate(config)).toThrow('K8s config requires exUid to create a services template');
-            });
-        });
-    });
-    describe('->getMaxOldSpace', () => {
-        it('should convert bytes to mb', () => {
-            const oneGBInBytes = 1024 * 1024 * 1024;
-            expect(getMaxOldSpace(oneGBInBytes)).toEqual(Math.round(0.9 * 1024));
-        });
-    });
-});
-//# sourceMappingURL=utils-v2-spec.js.map

package/dist/test/lib/cluster/services/cluster/backends/state-utils-spec.js

@@ -1,37 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import * as stateUtils from '../../../../../../src/lib/cluster/services/cluster/backends/state-utils.js';
-const dirname = path.dirname(fileURLToPath(import.meta.url));
-const clusterStateFile = path.join(dirname, 'files', 'sample-state.json');
-describe('stateUtils', () => {
-    let clusterState;
-    beforeEach(() => {
-        clusterState = JSON.parse(fs.readFileSync(clusterStateFile, 'utf-8'));
-    });
-    describe('->iterateState', () => {
-        it('can filter worker list and return cluster master node', () => {
-            const workerList = stateUtils.iterateState(clusterState, (worker) => worker.assignment === 'cluster_master');
-            expect(workerList[0].node_id).toEqual('node1.lan');
-        });
-    });
-    describe('->findAllSlicers', () => {
-        it('returns the right number of slicers', () => {
-            const workerList = stateUtils.findAllSlicers(clusterState);
-            expect(workerList[0].assignment).toEqual('execution_controller');
-        });
-    });
-    describe('->findWorkersByExecutionID', () => {
-        it('returns the right number of workers', () => {
-            const workerList = stateUtils.findWorkersByExecutionID(clusterState, '8d7669e1-d351-4a0d-b36e-b11da5ade26a');
-            expect(workerList.length).toEqual(8);
-        });
-    });
-    describe('->findSlicersByExecutionID', () => {
-        it('returns execution_controller worker', () => {
-            const workerList = stateUtils.findSlicersByExecutionID(clusterState, { '8d7669e1-d351-4a0d-b36e-b11da5ade26a': '8d7669e1-d351-4a0d-b36e-b11da5ade26a' });
-            expect(workerList[0].assignment).toEqual('execution_controller');
-        });
-    });
-});
-//# sourceMappingURL=state-utils-spec.js.map