teraslice 3.3.0 → 3.3.2

package/dist/test/test.config.js

@@ -1,23 +0,0 @@
-import { SchemaValidator } from '@terascope/core-utils';
-import { newId } from '../src/lib/utils/id_utils.js';
-const terasliceTestEnvSchema = {
-    SEARCH_TEST_HOST: {
-        default: undefined,
-        format: String,
-    },
-    TERASLICE_CLUSTER_NAME: {
-        default: null,
-        format: 'optional_string',
-    },
-    TEST_INDEX_PREFIX: {
-        default: undefined,
-        format: String,
-    },
-};
-const validator = new SchemaValidator(terasliceTestEnvSchema, 'terasliceTestEnvSchema', undefined, 'allow');
-const envConfig = validator.validate(process.env);
-const { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } = envConfig;
-const TERASLICE_CLUSTER_NAME = envConfig.TERASLICE_CLUSTER_NAME
-    || newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2);
-export { TEST_INDEX_PREFIX, SEARCH_TEST_HOST, TERASLICE_CLUSTER_NAME, };
-//# sourceMappingURL=test.config.js.map

package/dist/test/test.setup.js

@@ -1,6 +0,0 @@
-import { jest } from '@jest/globals';
-process.env.USE_DEBUG_LOGGER = 'true';
-process.env.NODE_ENV = 'test';
-// use a larger timeout
-jest.setTimeout(60 * 1000);
-//# sourceMappingURL=test.setup.js.map

package/dist/test/utils/api_utils-spec.js

@@ -1,25 +0,0 @@
-import { createJobActiveQuery, addDeletedToQuery } from '../../src/lib/utils/api_utils.js';
-describe('apiUtils', () => {
-    it('should be able to create the proper job queries', () => {
-        let query;
-        query = createJobActiveQuery('true');
-        query = addDeletedToQuery('true', query);
-        expect(query).toBe('job_id:* AND !active:false AND _deleted:true');
-        query = createJobActiveQuery('true');
-        query = addDeletedToQuery('', query);
-        expect(query).toBe('job_id:* AND !active:false AND _deleted:true');
-        query = createJobActiveQuery('true');
-        query = addDeletedToQuery('false', query);
-        expect(query).toBe('job_id:* AND !active:false AND (_deleted:false OR (* AND -_deleted:*))');
-        query = createJobActiveQuery('false');
-        query = addDeletedToQuery('true', query);
-        expect(query).toBe('job_id:* AND active:false AND _deleted:true');
-        query = createJobActiveQuery('false');
-        query = addDeletedToQuery('', query);
-        expect(query).toBe('job_id:* AND active:false AND _deleted:true');
-        query = createJobActiveQuery('false');
-        query = addDeletedToQuery('false', query);
-        expect(query).toBe('job_id:* AND active:false AND (_deleted:false OR (* AND -_deleted:*))');
-    });
-});
-//# sourceMappingURL=api_utils-spec.js.map

package/dist/test/utils/asset_utils-spec.js

@@ -1,141 +0,0 @@
-import 'jest-extended';
-import { toSemverRange, findMatchingAsset, getMajorVersion, toVersionQuery, findSimilarAssets, getInCompatibilityReason, } from '../../src/lib/utils/asset_utils.js';
-describe('Asset Utils', () => {
-    describe('->toSemverRange', () => {
-        test.each([
-            ['latest', '*'],
-            [null, '*'],
-            ['^22.0.0', '^22.0.0'],
-            ['~22.0.0', '~22.0.0'],
-            ['2.0.0', '2.0.0'],
-            ['v2.0.0', '2.0.0'],
-            ['*', '*'],
-            ['1.*', '1.*'],
-            ['2.*', '2.*'],
-            ['2222.*', '2222.*'],
-            ['12.*.1', '12.*.1'],
-            ['12.34*.55', '12.34*.55'],
-            ['12.34.*', '12.34.*'],
-            ['12.34.5*', '12.34.5*'],
-        ])('should convert %p to %p', (input, output) => {
-            expect(toSemverRange(input)).toEqual(output);
-        });
-    });
-    describe('->toVersionQuery', () => {
-        test.each([
-            ['latest', 'version:*'],
-            [null, 'version:*'],
-            ['^22.0.0', 'version:>=22.0.0 AND version:<23.0.0-0'],
-            ['~22.0.0', 'version:>=22.0.0 AND version:<22.1.0-0'],
-            ['2.0.0', 'version:2.0.0'],
-            ['v1.1.1', 'version:1.1.1'],
-            ['*', 'version:*'],
-            ['1.*', 'version:>=1.0.0 AND version:<2.0.0-0'],
-            ['2.*', 'version:>=2.0.0 AND version:<3.0.0-0'],
-            ['2222.*', 'version:>=2222.0.0 AND version:<2223.0.0-0'],
-            ['12.*.1', 'version:>=12.0.0 AND version:<13.0.0-0'],
-            ['12.34*.55', 'version:12.34*.55'],
-            ['12.34.*', 'version:>=12.34.0 AND version:<12.35.0-0'],
-            ['12.34.5*', 'version:12.34.5*'],
-        ])('should convert %p to %p', (input, output) => {
-            expect(toVersionQuery(input)).toEqual(output);
-        });
-    });
-    const wrongPlatform = {
-        darwin: 'linux',
-        linux: 'darwin',
-    };
-    const currentNodeVersion = getMajorVersion(process.version);
-    const assets = [{
-            id: 'foo-1',
-            name: 'foo',
-            version: '2.0.1',
-            platform: process.platform,
-            arch: process.arch,
-            node_version: currentNodeVersion,
-        },
-        {
-            id: 'foo-2',
-            name: 'foo',
-            version: '2.0.0',
-            platform: process.platform,
-            arch: process.arch,
-            node_version: currentNodeVersion,
-        },
-        {
-            id: 'foo-3',
-            name: 'foo',
-            version: '2.0.1',
-            platform: wrongPlatform[process.platform],
-            arch: process.arch,
-            node_version: currentNodeVersion,
-        },
-        {
-            id: 'foo-4',
-            name: 'foo',
-            version: '2.0.2',
-            platform: process.platform,
-            arch: process.arch,
-            node_version: currentNodeVersion + 1,
-        },
-        {
-            id: 'foo-5',
-            name: 'foo',
-            version: '1.0.0',
-            platform: process.platform,
-            arch: process.arch,
-            node_version: currentNodeVersion,
-        },
-        {
-            id: 'foo-6',
-            name: 'foo',
-            version: '1.0.0',
-            node_version: currentNodeVersion,
-        }];
-    describe('->findMatchingAsset', () => {
-        test.each([
-            ['foo',
-                'latest',
-                {
-                    version: '2.0.1'
-                }],
-            ['foo',
-                '2.0.0',
-                {
-                    version: '2.0.0'
-                }],
-            ['foo',
-                '~2.0.0',
-                {
-                    version: '2.0.1'
-                }],
-            ['foo',
-                '1.*',
-                {
-                    version: '1.0.0'
-                }],
-            ['foo', '0.1.0', null],
-            ['foo', '0.1.*', null],
-        ])('should return the correct result for %s:%s', (name, version, result) => {
-            if (result == null) {
-                expect(findMatchingAsset(assets, name, version)).toBeNil();
-            }
-            else {
-                expect(findMatchingAsset(assets, name, version)).toMatchObject(result);
-            }
-        });
-    });
-    describe('->getInCompatibilityReason/->findSimilarAssets', () => {
-        test.each([
-            ['foo', 'latest', 'node_version or platform mismatch'],
-            ['foo', '2.0.1', 'platform mismatch'],
-            ['foo', '~2.0.0', 'node_version or platform mismatch'],
-            ['foo', '3.*', ''],
-            ['foo', '0.1.*', ''],
-        ])('should return the correct result for %s:%s', (name, version, result) => {
-            const reason = getInCompatibilityReason(findSimilarAssets(assets, name, version));
-            expect(reason).toEqual(result);
-        });
-    });
-});
-//# sourceMappingURL=asset_utils-spec.js.map

package/dist/test/utils/elastic_utils-spec.js

@@ -1,25 +0,0 @@
-import { dateOptions } from '../../src/lib/utils/date_utils.js';
-describe('elastic_utils', () => {
-    it('has methods dateOptions and processInterval', () => {
-        expect(dateOptions).toBeDefined();
-        expect(typeof dateOptions).toEqual('function');
-    });
-    it('dateOptions returns a string used for the moment library', () => {
-        expect(() => {
-            dateOptions('Day');
-        }).toThrow();
-        expect(dateOptions('day')).toEqual('d');
-    });
-    it('dateOptions will throw a new error if not given correct values', () => {
-        expect(() => {
-            dateOptions('hourz');
-        }).toThrow();
-        expect(() => {
-            dateOptions(3);
-        }).toThrow();
-        expect(() => {
-            dateOptions({ some: 'obj' });
-        }).toThrow();
-    });
-});
-//# sourceMappingURL=elastic_utils-spec.js.map

package/dist/test/workers/execution-controller/execution-controller-spec.js

@@ -1,371 +0,0 @@
-import { jest } from '@jest/globals';
-import { pDelay, get, set } from '@terascope/core-utils';
-import { TestContext } from '../helpers/index.js';
-import { findPort } from '../../../src/lib/utils/port_utils.js';
-import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
-describe('ExecutionController', () => {
-    describe('when the execution context is invalid', () => {
-        let testContext;
-        let exController;
-        let executionStorage;
-        beforeEach(async () => {
-            const port = await findPort();
-            testContext = new TestContext({
-                assignment: 'execution_controller',
-                slicerPort: port,
-            });
-            // needs to be in this order
-            await testContext.initialize(true);
-            await testContext.addClusterMaster();
-            exController = new ExecutionController(testContext.context, testContext.executionContext);
-            set(exController, 'isExecutionFinished', true);
-            await testContext.addExStore();
-            executionStorage = testContext.stores.executionStorage;
-            testContext.attachCleanup(() => exController.shutdown().catch(() => {
-                /* ignore-error */
-            }));
-        });
-        afterEach(() => testContext.cleanup());
-        describe('when the execution does not exist', () => {
-            beforeEach(async () => {
-                await executionStorage.remove(testContext.exId);
-            });
-            it('should throw an error on initialize', async () => {
-                expect.hasAssertions();
-                try {
-                    await exController.initialize();
-                }
-                catch (err) {
-                    expect(err.message).toStartWith(`Cannot get execution status ${testContext.exId}`);
-                }
-            });
-        });
-        describe('when the execution is set to running', () => {
-            beforeEach(async () => {
-                await executionStorage.setStatus(testContext.exId, 'running');
-            });
-            it('should throw an error on initialize', async () => {
-                expect.hasAssertions();
-                await exController.initialize();
-                expect(get(exController, 'isInitialized')).toBeFalse();
-                expect(get(exController, 'isShutdown')).toBeTrue();
-            });
-        });
-        describe('when the execution is in a terminal state', () => {
-            beforeEach(async () => {
-                await executionStorage.setStatus(testContext.exId, 'completed');
-            });
-            it('should throw an error on initialize', async () => {
-                expect.hasAssertions();
-                await exController.initialize();
-                expect(get(exController, 'isInitialized')).toBeFalse();
-                expect(get(exController, 'isShutdown')).toBeTrue();
-            });
-        });
-    });
-    describe('when the slice failure watch dog is started', () => {
-        let testContext;
-        let executionStorage;
-        let exController;
-        const probationWindow = 500;
-        beforeEach(async () => {
-            const port = await findPort();
-            testContext = new TestContext({
-                assignment: 'execution_controller',
-                slicerPort: port,
-                probationWindow
-            });
-            // needs to be in this order
-            await testContext.initialize(true);
-            await testContext.addClusterMaster();
-            exController = new ExecutionController(testContext.context, testContext.executionContext);
-            await exController.initialize();
-            await testContext.addExStore();
-            executionStorage = testContext.stores.executionStorage;
-            testContext.attachCleanup(() => exController.shutdown().catch(() => {
-                /* ignore-error */
-            }));
-        });
-        afterEach(() => testContext.cleanup());
-        it('should handle the probation period correctly', async () => {
-            exController.executionAnalytics.increment('processed');
-            exController.executionAnalytics.increment('failed');
-            // @ts-expect-error
-            expect(exController.sliceFailureInterval).toBeNil();
-            // @ts-expect-error
-            await exController._startSliceFailureWatchDog();
-            // @ts-expect-error
-            expect(exController.sliceFailureInterval).not.toBeNil();
-            // @ts-expect-error
-            // should be able to call slice watch again without resetting the interval
-            const { sliceFailureInterval } = exController;
-            // @ts-expect-error
-            await exController._startSliceFailureWatchDog();
-            // @ts-expect-error
-            expect(exController.sliceFailureInterval).toBe(sliceFailureInterval);
-            await expect(executionStorage.getStatus(testContext.exId)).resolves.toEqual('failing');
-            await pDelay(probationWindow + 100);
-            // should be able to setr the status back to running if more slices are processed
-            exController.executionAnalytics.increment('processed');
-            await pDelay(probationWindow + 100);
-            await expect(executionStorage.getStatus(testContext.exId)).resolves.toEqual('running');
-            // @ts-expect-error
-            expect(exController.sliceFailureInterval).toBeNil();
-            // @ts-expect-error
-            await exController._startSliceFailureWatchDog();
-            // @ts-expect-error
-            expect(exController.sliceFailureInterval).not.toBeNil();
-        });
-    });
-    describe('when testing setFailingStatus', () => {
-        let testContext;
-        let exController;
-        beforeEach(async () => {
-            testContext = new TestContext({
-                assignment: 'execution_controller'
-            });
-            await testContext.initialize();
-            exController = new ExecutionController(testContext.context, testContext.executionContext);
-        });
-        afterEach(() => testContext.cleanup());
-        describe('when it fails to set the status', () => {
-            it('should log the error twice', async () => {
-                const logErr = jest.fn();
-                // @ts-expect-error
-                const setStatus = jest.fn().mockRejectedValue(new Error('Uh oh'));
-                const errMeta = { error: 'metadata' };
-                const executionMetaData = jest.fn().mockReturnValue(errMeta);
-                // @ts-expect-error
-                exController.executionStorage = {
-                    setStatus,
-                    executionMetaData
-                };
-                exController.logger.error = logErr;
-                const stats = exController.executionAnalytics.getAnalytics();
-                await exController.setFailingStatus('some reason');
-                expect(setStatus).toHaveBeenCalledWith(testContext.exId, 'failing', errMeta);
-                expect(executionMetaData).toHaveBeenCalledWith(stats, `execution ${testContext.exId} has encountered a processing error, reason: some reason`);
-                expect(logErr).toHaveBeenCalledTimes(2);
-            });
-        });
-    });
-    describe('when testing _terminalError', () => {
-        let testContext;
-        let exController;
-        beforeEach(async () => {
-            testContext = new TestContext({
-                assignment: 'execution_controller'
-            });
-            await testContext.initialize();
-            exController = new ExecutionController(testContext.context, testContext.executionContext);
-        });
-        afterEach(() => testContext.cleanup());
-        describe('when it fails to set the status', () => {
-            it('should log the error twice', async () => {
-                const logErr = jest.fn();
-                const logFatal = jest.fn();
-                // @ts-expect-error
-                const setStatus = jest.fn().mockRejectedValue(new Error('Uh oh'));
-                const errMeta = { error: 'metadata' };
-                const executionMetaData = jest.fn().mockReturnValue(errMeta);
-                // @ts-expect-error
-                exController.executionStorage = {
-                    setStatus,
-                    executionMetaData
-                };
-                exController.logger.error = logErr;
-                exController.logger.fatal = logFatal;
-                const stats = exController.executionAnalytics.getAnalytics();
-                // @ts-expect-error
-                await exController._terminalError('Uh oh');
-                // @ts-expect-error
-                expect(exController.slicerFailed).toBeTrue();
-                expect(setStatus).toHaveBeenCalledWith(testContext.exId, 'failed', errMeta);
-                const errMsg = `TSError: slicer for ex ${testContext.exId} had an error, shutting down execution, caused by Uh oh`;
-                expect(executionMetaData.mock.calls[0][0]).toEqual(stats);
-                expect(executionMetaData.mock.calls[0][1]).toStartWith(errMsg);
-                expect(logErr).toHaveBeenCalledTimes(2);
-                expect(logFatal).toHaveBeenCalledWith(`execution ${testContext.exId} is ended because of slice failure`);
-            });
-        });
-        describe('when the execution is already done', () => {
-            it('should not do anything', async () => {
-                exController.isExecutionDone = true;
-                const logErr = jest.fn();
-                exController.logger.error = logErr;
-                // @ts-expect-error
-                await exController._terminalError();
-                // @ts-expect-error
-                expect(exController.slicerFailed).toBeFalsy();
-                expect(logErr).not.toHaveBeenCalled();
-            });
-        });
-    });
-    describe('when testing shutdown', () => {
-        let testContext;
-        let exController;
-        beforeEach(async () => {
-            testContext = new TestContext({
-                assignment: 'execution_controller',
-                shutdownTimeout: 100
-            });
-            await testContext.initialize();
-            exController = new ExecutionController(testContext.context, testContext.executionContext);
-            testContext.attachCleanup(() => exController.shutdown().catch(() => {
-                /* ignore-error */
-            }));
-        });
-        afterEach(() => testContext.cleanup());
-        describe('when not initialized', () => {
-            it('should resolve', () => expect(exController.shutdown()).resolves.toBeNil());
-        });
-        describe('when initialized', () => {
-            beforeEach(() => {
-                // @ts-expect-error
-                exController.isInitialized = true;
-            });
-            describe('when controller is already being shutdown', () => {
-                beforeEach(() => {
-                    // @ts-expect-error
-                    exController.isShuttingDown = true;
-                });
-                it('should resolve when shutdown passes', async () => {
-                    setTimeout(() => {
-                        exController.events.emit('worker:shutdown:complete');
-                    });
-                    await expect(exController.shutdown()).resolves.toBeNil();
-                });
-                it('should reject when shutdown fails', async () => {
-                    setTimeout(() => {
-                        exController.events.emit('worker:shutdown:complete', new Error('Uh oh'));
-                    });
-                    await expect(exController.shutdown()).rejects.toThrow('Uh oh');
-                });
-            });
-            describe('when everything errors', () => {
-                beforeEach(() => {
-                    // @ts-expect-error
-                    exController.isExecutionFinished = true;
-                    exController.stateStorage = {
-                        // @ts-expect-error
-                        shutdown: () => Promise.resolve(true)
-                    };
-                    // @ts-expect-error
-                    exController.executionStorage = {
-                        shutdown: () => Promise.reject(new Error('Store Error'))
-                    };
-                    // @ts-expect-error
-                    exController.executionAnalytics = {
-                        shutdown: () => Promise.reject(new Error('Execution Analytics Error'))
-                    };
-                    // @ts-expect-error
-                    exController.collectAnalytics = true;
-                    // @ts-expect-error
-                    exController.slicerAnalytics = {
-                        shutdown: () => Promise.reject(new Error('Slicer Analytics Error'))
-                    };
-                    // @ts-expect-error
-                    exController.scheduler = {
-                        stop: () => { },
-                        shutdown: () => Promise.reject(new Error('Scheduler Error'))
-                    };
-                    // @ts-expect-error
-                    exController.server = {
-                        shutdown: () => Promise.reject(new Error('Execution Controller Server Error'))
-                    };
-                    // @ts-expect-error
-                    exController.client = {
-                        shutdown: () => Promise.reject(new Error('Cluster Master Client Error'))
-                    };
-                });
-                it('should reject with all of the errors', async () => {
-                    expect.hasAssertions();
-                    try {
-                        await exController.shutdown();
-                    }
-                    catch (err) {
-                        const errMsg = err.toString();
-                        expect(errMsg).toStartWith('Error: Failed to shutdown correctly');
-                        expect(errMsg).toInclude('Store Error');
-                        expect(errMsg).toInclude('Execution Analytics Error');
-                        expect(errMsg).toInclude('Slicer Analytics Error');
-                        expect(errMsg).toInclude('Scheduler Error');
-                        expect(errMsg).toInclude('Execution Controller Server Error');
-                        expect(errMsg).toInclude('Cluster Master Client Error');
-                    }
-                });
-            });
-        });
-    });
-    describe('when testing log_level', () => {
-        beforeAll(() => {
-            process.env.TESTING_LOG_LEVEL = 'true';
-            return;
-        });
-        describe('when there is no log_level set in either job configuration or terafoundation', () => {
-            let testContext;
-            let exController;
-            beforeEach(async () => {
-                testContext = new TestContext({
-                    assignment: 'execution_controller',
-                    shutdownTimeout: 100
-                });
-                await testContext.initialize();
-                exController = new ExecutionController(testContext.context, testContext.executionContext);
-                testContext.attachCleanup(() => exController.shutdown().catch(() => {
-                    /* ignore-error */
-                }));
-            });
-            it('should have a logger with log_level info', async () => {
-                expect(exController.executionContext.context.logger.level()).toBe(30);
-            });
-            afterEach(() => testContext.cleanup());
-        });
-        describe('when no log_level is set in job configuration and terafoundation log level is error', () => {
-            let testContext;
-            let exController;
-            beforeEach(async () => {
-                testContext = new TestContext({
-                    assignment: 'execution_controller',
-                    shutdownTimeout: 100,
-                    log_level_terafoundation: 'error'
-                });
-                await testContext.initialize();
-                exController = new ExecutionController(testContext.context, testContext.executionContext);
-                testContext.attachCleanup(() => exController.shutdown().catch(() => {
-                    /* ignore-error */
-                }));
-            });
-            it('should have a logger with log_level error', async () => {
-                expect(exController.executionContext.context.logger.level()).toBe(50);
-            });
-            afterEach(() => testContext.cleanup());
-        });
-        describe('when log_level set to trace in job configuration and terafoundation is set to error', () => {
-            let testContext;
-            let exController;
-            beforeEach(async () => {
-                testContext = new TestContext({
-                    assignment: 'execution_controller',
-                    shutdownTimeout: 100,
-                    log_level: 'trace',
-                    log_level_terafoundation: 'error'
-                });
-                await testContext.initialize();
-                exController = new ExecutionController(testContext.context, testContext.executionContext);
-                testContext.attachCleanup(() => exController.shutdown().catch(() => {
-                    /* ignore-error */
-                }));
-            });
-            it('should have a logger with log_level trace', async () => {
-                expect(exController.executionContext.context.logger.level()).toBe(10);
-            });
-            afterEach(() => testContext.cleanup());
-        });
-        afterAll(() => {
-            process.env.TESTING_LOG_LEVEL = 'false';
-            return;
-        });
-    });
-});
-//# sourceMappingURL=execution-controller-spec.js.map