teraslice 2.10.0 → 2.12.0

package/dist/test/utils/api_utils-spec.js

@@ -0,0 +1,86 @@
+import { makePrometheus, isPrometheusTerasliceRequest, createJobActiveQuery, addDeletedToQuery } from '../../src/lib/utils/api_utils.js';
+describe('apiUtils', () => {
+    it('should be able make a prometheus text format', () => {
+        const stats = {
+            controllers: {
+                processed: 1000,
+                failed: 10,
+                queued: 5,
+                job_duration: 10,
+                workers_joined: 20,
+                workers_disconnected: 30,
+                workers_reconnected: 40
+            }
+        };
+        const r = `# TYPE teraslice_slices_processed counter
+teraslice_slices_processed ${stats.controllers.processed}
+# TYPE teraslice_slices_failed counter
+teraslice_slices_failed ${stats.controllers.failed}
+# TYPE teraslice_slices_queued counter
+teraslice_slices_queued ${stats.controllers.queued}
+# TYPE teraslice_workers_joined counter
+teraslice_workers_joined ${stats.controllers.workers_joined}
+# TYPE teraslice_workers_disconnected counter
+teraslice_workers_disconnected ${stats.controllers.workers_disconnected}
+# TYPE teraslice_workers_reconnected counter
+teraslice_workers_reconnected ${stats.controllers.workers_reconnected}
+`;
+        expect(makePrometheus(stats)).toEqual(r);
+    });
+    it('should be able make a prometheus text format with labels', () => {
+        const stats = {
+            controllers: {
+                processed: 1000,
+                failed: 10,
+                queued: 5,
+                job_duration: 10,
+                workers_joined: 20,
+                workers_disconnected: 30,
+                workers_reconnected: 40
+            }
+        };
+        const r = `# TYPE teraslice_slices_processed counter
+teraslice_slices_processed{foo="bar"} ${stats.controllers.processed}
+# TYPE teraslice_slices_failed counter
+teraslice_slices_failed{foo="bar"} ${stats.controllers.failed}
+# TYPE teraslice_slices_queued counter
+teraslice_slices_queued{foo="bar"} ${stats.controllers.queued}
+# TYPE teraslice_workers_joined counter
+teraslice_workers_joined{foo="bar"} ${stats.controllers.workers_joined}
+# TYPE teraslice_workers_disconnected counter
+teraslice_workers_disconnected{foo="bar"} ${stats.controllers.workers_disconnected}
+# TYPE teraslice_workers_reconnected counter
+teraslice_workers_reconnected{foo="bar"} ${stats.controllers.workers_reconnected}
+`;
+        expect(makePrometheus(stats, { foo: 'bar' })).toEqual(r);
+    });
+    it('should be able to detect if a request is prometheus', () => {
+        expect(isPrometheusTerasliceRequest({
+            headers: {
+                accept: 'blah application/openmetrics-text; blah blah'
+            }
+        })).toBeTruthy();
+    });
+    it('should be able to create the proper job queries', () => {
+        let query;
+        query = createJobActiveQuery('true');
+        query = addDeletedToQuery('true', query);
+        expect(query).toBe('job_id:* AND !active:false AND _deleted:true');
+        query = createJobActiveQuery('true');
+        query = addDeletedToQuery('', query);
+        expect(query).toBe('job_id:* AND !active:false AND _deleted:true');
+        query = createJobActiveQuery('true');
+        query = addDeletedToQuery('false', query);
+        expect(query).toBe('job_id:* AND !active:false AND (_deleted:false OR (* AND -_deleted:*))');
+        query = createJobActiveQuery('false');
+        query = addDeletedToQuery('true', query);
+        expect(query).toBe('job_id:* AND active:false AND _deleted:true');
+        query = createJobActiveQuery('false');
+        query = addDeletedToQuery('', query);
+        expect(query).toBe('job_id:* AND active:false AND _deleted:true');
+        query = createJobActiveQuery('false');
+        query = addDeletedToQuery('false', query);
+        expect(query).toBe('job_id:* AND active:false AND (_deleted:false OR (* AND -_deleted:*))');
+    });
+});
+//# sourceMappingURL=api_utils-spec.js.map

package/dist/test/utils/asset_utils-spec.js

@@ -0,0 +1,141 @@
+import 'jest-extended';
+import { toSemverRange, findMatchingAsset, getMajorVersion, toVersionQuery, findSimilarAssets, getInCompatibilityReason, } from '../../src/lib/utils/asset_utils.js';
+describe('Asset Utils', () => {
+    describe('->toSemverRange', () => {
+        test.each([
+            ['latest', '*'],
+            [null, '*'],
+            ['^22.0.0', '^22.0.0'],
+            ['~22.0.0', '~22.0.0'],
+            ['2.0.0', '2.0.0'],
+            ['v2.0.0', '2.0.0'],
+            ['*', '*'],
+            ['1.*', '1.*'],
+            ['2.*', '2.*'],
+            ['2222.*', '2222.*'],
+            ['12.*.1', '12.*.1'],
+            ['12.34*.55', '12.34*.55'],
+            ['12.34.*', '12.34.*'],
+            ['12.34.5*', '12.34.5*'],
+        ])('should convert %p to %p', (input, output) => {
+            expect(toSemverRange(input)).toEqual(output);
+        });
+    });
+    describe('->toVersionQuery', () => {
+        test.each([
+            ['latest', 'version:*'],
+            [null, 'version:*'],
+            ['^22.0.0', 'version:>=22.0.0 AND version:<23.0.0-0'],
+            ['~22.0.0', 'version:>=22.0.0 AND version:<22.1.0-0'],
+            ['2.0.0', 'version:2.0.0'],
+            ['v1.1.1', 'version:1.1.1'],
+            ['*', 'version:*'],
+            ['1.*', 'version:>=1.0.0 AND version:<2.0.0-0'],
+            ['2.*', 'version:>=2.0.0 AND version:<3.0.0-0'],
+            ['2222.*', 'version:>=2222.0.0 AND version:<2223.0.0-0'],
+            ['12.*.1', 'version:>=12.0.0 AND version:<13.0.0-0'],
+            ['12.34*.55', 'version:12.34*.55'],
+            ['12.34.*', 'version:>=12.34.0 AND version:<12.35.0-0'],
+            ['12.34.5*', 'version:12.34.5*'],
+        ])('should convert %p to %p', (input, output) => {
+            expect(toVersionQuery(input)).toEqual(output);
+        });
+    });
+    const wrongPlatform = {
+        darwin: 'linux',
+        linux: 'darwin',
+    };
+    const currentNodeVersion = getMajorVersion(process.version);
+    const assets = [{
+            id: 'foo-1',
+            name: 'foo',
+            version: '2.0.1',
+            platform: process.platform,
+            arch: process.arch,
+            node_version: currentNodeVersion,
+        },
+        {
+            id: 'foo-2',
+            name: 'foo',
+            version: '2.0.0',
+            platform: process.platform,
+            arch: process.arch,
+            node_version: currentNodeVersion,
+        },
+        {
+            id: 'foo-3',
+            name: 'foo',
+            version: '2.0.1',
+            platform: wrongPlatform[process.platform],
+            arch: process.arch,
+            node_version: currentNodeVersion,
+        },
+        {
+            id: 'foo-4',
+            name: 'foo',
+            version: '2.0.2',
+            platform: process.platform,
+            arch: process.arch,
+            node_version: currentNodeVersion + 1,
+        },
+        {
+            id: 'foo-5',
+            name: 'foo',
+            version: '1.0.0',
+            platform: process.platform,
+            arch: process.arch,
+            node_version: currentNodeVersion,
+        },
+        {
+            id: 'foo-6',
+            name: 'foo',
+            version: '1.0.0',
+            node_version: currentNodeVersion,
+        }];
+    describe('->findMatchingAsset', () => {
+        test.each([
+            ['foo',
+                'latest',
+                {
+                    version: '2.0.1'
+                }],
+            ['foo',
+                '2.0.0',
+                {
+                    version: '2.0.0'
+                }],
+            ['foo',
+                '~2.0.0',
+                {
+                    version: '2.0.1'
+                }],
+            ['foo',
+                '1.*',
+                {
+                    version: '1.0.0'
+                }],
+            ['foo', '0.1.0', null],
+            ['foo', '0.1.*', null],
+        ])('should return the correct result for %s:%s', (name, version, result) => {
+            if (result == null) {
+                expect(findMatchingAsset(assets, name, version)).toBeNil();
+            }
+            else {
+                expect(findMatchingAsset(assets, name, version)).toMatchObject(result);
+            }
+        });
+    });
+    describe('->getInCompatibilityReason/->findSimilarAssets', () => {
+        test.each([
+            ['foo', 'latest', 'node_version or platform mismatch'],
+            ['foo', '2.0.1', 'platform mismatch'],
+            ['foo', '~2.0.0', 'node_version or platform mismatch'],
+            ['foo', '3.*', ''],
+            ['foo', '0.1.*', ''],
+        ])('should return the correct result for %s:%s', (name, version, result) => {
+            const reason = getInCompatibilityReason(findSimilarAssets(assets, name, version));
+            expect(reason).toEqual(result);
+        });
+    });
+});
+//# sourceMappingURL=asset_utils-spec.js.map

package/dist/test/utils/elastic_utils-spec.js

@@ -0,0 +1,25 @@
+import { dateOptions } from '../../src/lib/utils/date_utils.js';
+describe('elastic_utils', () => {
+    it('has methods dateOptions and processInterval', () => {
+        expect(dateOptions).toBeDefined();
+        expect(typeof dateOptions).toEqual('function');
+    });
+    it('dateOptions returns a string used for the moment library', () => {
+        expect(() => {
+            dateOptions('Day');
+        }).toThrowError();
+        expect(dateOptions('day')).toEqual('d');
+    });
+    it('dateOptions will throw a new error if not given correct values', () => {
+        expect(() => {
+            dateOptions('hourz');
+        }).toThrowError();
+        expect(() => {
+            dateOptions(3);
+        }).toThrowError();
+        expect(() => {
+            dateOptions({ some: 'obj' });
+        }).toThrowError();
+    });
+});
+//# sourceMappingURL=elastic_utils-spec.js.map

package/dist/test/workers/execution-controller/execution-controller-spec.js

@@ -0,0 +1,371 @@
+import { jest } from '@jest/globals';
+import { pDelay, get, set } from '@terascope/utils';
+import { TestContext } from '../helpers/index.js';
+import { findPort } from '../../../src/lib/utils/port_utils.js';
+import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
+describe('ExecutionController', () => {
+    describe('when the execution context is invalid', () => {
+        let testContext;
+        let exController;
+        let executionStorage;
+        beforeEach(async () => {
+            const port = await findPort();
+            testContext = new TestContext({
+                assignment: 'execution_controller',
+                slicerPort: port,
+            });
+            // needs to be in this order
+            await testContext.initialize(true);
+            await testContext.addClusterMaster();
+            exController = new ExecutionController(testContext.context, testContext.executionContext);
+            set(exController, 'isExecutionFinished', true);
+            await testContext.addExStore();
+            executionStorage = testContext.stores.executionStorage;
+            testContext.attachCleanup(() => exController.shutdown().catch(() => {
+                /* ignore-error */
+            }));
+        });
+        afterEach(() => testContext.cleanup());
+        describe('when the execution does not exist', () => {
+            beforeEach(async () => {
+                await executionStorage.remove(testContext.exId);
+            });
+            it('should throw an error on initialize', async () => {
+                expect.hasAssertions();
+                try {
+                    await exController.initialize();
+                }
+                catch (err) {
+                    expect(err.message).toStartWith(`Cannot get execution status ${testContext.exId}`);
+                }
+            });
+        });
+        describe('when the execution is set to running', () => {
+            beforeEach(async () => {
+                await executionStorage.setStatus(testContext.exId, 'running');
+            });
+            it('should throw an error on initialize', async () => {
+                expect.hasAssertions();
+                await exController.initialize();
+                expect(get(exController, 'isInitialized')).toBeFalse();
+                expect(get(exController, 'isShutdown')).toBeTrue();
+            });
+        });
+        describe('when the execution is in a terminal state', () => {
+            beforeEach(async () => {
+                await executionStorage.setStatus(testContext.exId, 'completed');
+            });
+            it('should throw an error on initialize', async () => {
+                expect.hasAssertions();
+                await exController.initialize();
+                expect(get(exController, 'isInitialized')).toBeFalse();
+                expect(get(exController, 'isShutdown')).toBeTrue();
+            });
+        });
+    });
+    describe('when the slice failure watch dog is started', () => {
+        let testContext;
+        let executionStorage;
+        let exController;
+        const probationWindow = 500;
+        beforeEach(async () => {
+            const port = await findPort();
+            testContext = new TestContext({
+                assignment: 'execution_controller',
+                slicerPort: port,
+                probationWindow
+            });
+            // needs to be in this order
+            await testContext.initialize(true);
+            await testContext.addClusterMaster();
+            exController = new ExecutionController(testContext.context, testContext.executionContext);
+            await exController.initialize();
+            await testContext.addExStore();
+            executionStorage = testContext.stores.executionStorage;
+            testContext.attachCleanup(() => exController.shutdown().catch(() => {
+                /* ignore-error */
+            }));
+        });
+        afterEach(() => testContext.cleanup());
+        it('should handle the probation period correctly', async () => {
+            exController.executionAnalytics.increment('processed');
+            exController.executionAnalytics.increment('failed');
+            // @ts-expect-error
+            expect(exController.sliceFailureInterval).toBeNil();
+            // @ts-expect-error
+            await exController._startSliceFailureWatchDog();
+            // @ts-expect-error
+            expect(exController.sliceFailureInterval).not.toBeNil();
+            // @ts-expect-error
+            // should be able to call slice watch again without resetting the interval
+            const { sliceFailureInterval } = exController;
+            // @ts-expect-error
+            await exController._startSliceFailureWatchDog();
+            // @ts-expect-error
+            expect(exController.sliceFailureInterval).toBe(sliceFailureInterval);
+            await expect(executionStorage.getStatus(testContext.exId)).resolves.toEqual('failing');
+            await pDelay(probationWindow + 100);
+            // should be able to setr the status back to running if more slices are processed
+            exController.executionAnalytics.increment('processed');
+            await pDelay(probationWindow + 100);
+            await expect(executionStorage.getStatus(testContext.exId)).resolves.toEqual('running');
+            // @ts-expect-error
+            expect(exController.sliceFailureInterval).toBeNil();
+            // @ts-expect-error
+            await exController._startSliceFailureWatchDog();
+            // @ts-expect-error
+            expect(exController.sliceFailureInterval).not.toBeNil();
+        });
+    });
+    describe('when testing setFailingStatus', () => {
+        let testContext;
+        let exController;
+        beforeEach(async () => {
+            testContext = new TestContext({
+                assignment: 'execution_controller'
+            });
+            await testContext.initialize();
+            exController = new ExecutionController(testContext.context, testContext.executionContext);
+        });
+        afterEach(() => testContext.cleanup());
+        describe('when it fails to set the status', () => {
+            it('should log the error twice', async () => {
+                const logErr = jest.fn();
+                // @ts-expect-error
+                const setStatus = jest.fn().mockRejectedValue(new Error('Uh oh'));
+                const errMeta = { error: 'metadata' };
+                const executionMetaData = jest.fn().mockReturnValue(errMeta);
+                // @ts-expect-error
+                exController.executionStorage = {
+                    setStatus,
+                    executionMetaData
+                };
+                exController.logger.error = logErr;
+                const stats = exController.executionAnalytics.getAnalytics();
+                await exController.setFailingStatus('some reason');
+                expect(setStatus).toHaveBeenCalledWith(testContext.exId, 'failing', errMeta);
+                expect(executionMetaData).toHaveBeenCalledWith(stats, `execution ${testContext.exId} has encountered a processing error, reason: some reason`);
+                expect(logErr).toHaveBeenCalledTimes(2);
+            });
+        });
+    });
+    describe('when testing _terminalError', () => {
+        let testContext;
+        let exController;
+        beforeEach(async () => {
+            testContext = new TestContext({
+                assignment: 'execution_controller'
+            });
+            await testContext.initialize();
+            exController = new ExecutionController(testContext.context, testContext.executionContext);
+        });
+        afterEach(() => testContext.cleanup());
+        describe('when it fails to set the status', () => {
+            it('should log the error twice', async () => {
+                const logErr = jest.fn();
+                const logFatal = jest.fn();
+                // @ts-expect-error
+                const setStatus = jest.fn().mockRejectedValue(new Error('Uh oh'));
+                const errMeta = { error: 'metadata' };
+                const executionMetaData = jest.fn().mockReturnValue(errMeta);
+                // @ts-expect-error
+                exController.executionStorage = {
+                    setStatus,
+                    executionMetaData
+                };
+                exController.logger.error = logErr;
+                exController.logger.fatal = logFatal;
+                const stats = exController.executionAnalytics.getAnalytics();
+                // @ts-expect-error
+                await exController._terminalError('Uh oh');
+                // @ts-expect-error
+                expect(exController.slicerFailed).toBeTrue();
+                expect(setStatus).toHaveBeenCalledWith(testContext.exId, 'failed', errMeta);
+                const errMsg = `TSError: slicer for ex ${testContext.exId} had an error, shutting down execution, caused by Uh oh`;
+                expect(executionMetaData.mock.calls[0][0]).toEqual(stats);
+                expect(executionMetaData.mock.calls[0][1]).toStartWith(errMsg);
+                expect(logErr).toHaveBeenCalledTimes(2);
+                expect(logFatal).toHaveBeenCalledWith(`execution ${testContext.exId} is ended because of slice failure`);
+            });
+        });
+        describe('when the execution is already done', () => {
+            it('should not do anything', async () => {
+                exController.isExecutionDone = true;
+                const logErr = jest.fn();
+                exController.logger.error = logErr;
+                // @ts-expect-error
+                await exController._terminalError();
+                // @ts-expect-error
+                expect(exController.slicerFailed).toBeFalsy();
+                expect(logErr).not.toHaveBeenCalled();
+            });
+        });
+    });
+    describe('when testing shutdown', () => {
+        let testContext;
+        let exController;
+        beforeEach(async () => {
+            testContext = new TestContext({
+                assignment: 'execution_controller',
+                shutdownTimeout: 100
+            });
+            await testContext.initialize();
+            exController = new ExecutionController(testContext.context, testContext.executionContext);
+            testContext.attachCleanup(() => exController.shutdown().catch(() => {
+                /* ignore-error */
+            }));
+        });
+        afterEach(() => testContext.cleanup());
+        describe('when not initialized', () => {
+            it('should resolve', () => expect(exController.shutdown()).resolves.toBeNil());
+        });
+        describe('when initialized', () => {
+            beforeEach(() => {
+                // @ts-expect-error
+                exController.isInitialized = true;
+            });
+            describe('when controller is already being shutdown', () => {
+                beforeEach(() => {
+                    // @ts-expect-error
+                    exController.isShuttingDown = true;
+                });
+                it('should resolve when shutdown passes', async () => {
+                    setTimeout(() => {
+                        exController.events.emit('worker:shutdown:complete');
+                    });
+                    await expect(exController.shutdown()).resolves.toBeNil();
+                });
+                it('should reject when shutdown fails', async () => {
+                    setTimeout(() => {
+                        exController.events.emit('worker:shutdown:complete', new Error('Uh oh'));
+                    });
+                    await expect(exController.shutdown()).rejects.toThrowError('Uh oh');
+                });
+            });
+            describe('when everything errors', () => {
+                beforeEach(() => {
+                    // @ts-expect-error
+                    exController.isExecutionFinished = true;
+                    exController.stateStorage = {
+                        // @ts-expect-error
+                        shutdown: () => Promise.resolve(true)
+                    };
+                    // @ts-expect-error
+                    exController.executionStorage = {
+                        shutdown: () => Promise.reject(new Error('Store Error'))
+                    };
+                    // @ts-expect-error
+                    exController.executionAnalytics = {
+                        shutdown: () => Promise.reject(new Error('Execution Analytics Error'))
+                    };
+                    // @ts-expect-error
+                    exController.collectAnalytics = true;
+                    // @ts-expect-error
+                    exController.slicerAnalytics = {
+                        shutdown: () => Promise.reject(new Error('Slicer Analytics Error'))
+                    };
+                    // @ts-expect-error
+                    exController.scheduler = {
+                        stop: () => { },
+                        shutdown: () => Promise.reject(new Error('Scheduler Error'))
+                    };
+                    // @ts-expect-error
+                    exController.server = {
+                        shutdown: () => Promise.reject(new Error('Execution Controller Server Error'))
+                    };
+                    // @ts-expect-error
+                    exController.client = {
+                        shutdown: () => Promise.reject(new Error('Cluster Master Client Error'))
+                    };
+                });
+                it('should reject with all of the errors', async () => {
+                    expect.hasAssertions();
+                    try {
+                        await exController.shutdown();
+                    }
+                    catch (err) {
+                        const errMsg = err.toString();
+                        expect(errMsg).toStartWith('Error: Failed to shutdown correctly');
+                        expect(errMsg).toInclude('Store Error');
+                        expect(errMsg).toInclude('Execution Analytics Error');
+                        expect(errMsg).toInclude('Slicer Analytics Error');
+                        expect(errMsg).toInclude('Scheduler Error');
+                        expect(errMsg).toInclude('Execution Controller Server Error');
+                        expect(errMsg).toInclude('Cluster Master Client Error');
+                    }
+                });
+            });
+        });
+    });
+    describe('when testing log_level', () => {
+        beforeAll(() => {
+            process.env.TESTING_LOG_LEVEL = 'true';
+            return;
+        });
+        describe('when there is no log_level set in either job configuration or terafoundation', () => {
+            let testContext;
+            let exController;
+            beforeEach(async () => {
+                testContext = new TestContext({
+                    assignment: 'execution_controller',
+                    shutdownTimeout: 100
+                });
+                await testContext.initialize();
+                exController = new ExecutionController(testContext.context, testContext.executionContext);
+                testContext.attachCleanup(() => exController.shutdown().catch(() => {
+                    /* ignore-error */
+                }));
+            });
+            it('should have a logger with log_level info', async () => {
+                expect(exController.executionContext.context.logger.level()).toBe(30);
+            });
+            afterEach(() => testContext.cleanup());
+        });
+        describe('when no log_level is set in job configuration and terafoundation log level is error', () => {
+            let testContext;
+            let exController;
+            beforeEach(async () => {
+                testContext = new TestContext({
+                    assignment: 'execution_controller',
+                    shutdownTimeout: 100,
+                    log_level_terafoundation: 'error'
+                });
+                await testContext.initialize();
+                exController = new ExecutionController(testContext.context, testContext.executionContext);
+                testContext.attachCleanup(() => exController.shutdown().catch(() => {
+                    /* ignore-error */
+                }));
+            });
+            it('should have a logger with log_level error', async () => {
+                expect(exController.executionContext.context.logger.level()).toBe(50);
+            });
+            afterEach(() => testContext.cleanup());
+        });
+        describe('when log_level set to trace in job configuration and terafoundation is set to error', () => {
+            let testContext;
+            let exController;
+            beforeEach(async () => {
+                testContext = new TestContext({
+                    assignment: 'execution_controller',
+                    shutdownTimeout: 100,
+                    log_level: 'trace',
+                    log_level_terafoundation: 'error'
+                });
+                await testContext.initialize();
+                exController = new ExecutionController(testContext.context, testContext.executionContext);
+                testContext.attachCleanup(() => exController.shutdown().catch(() => {
+                    /* ignore-error */
+                }));
+            });
+            it('should have a logger with log_level trace', async () => {
+                expect(exController.executionContext.context.logger.level()).toBe(10);
+            });
+            afterEach(() => testContext.cleanup());
+        });
+        afterAll(() => {
+            process.env.TESTING_LOG_LEVEL = 'false';
+            return;
+        });
+    });
+});
+//# sourceMappingURL=execution-controller-spec.js.map