teraslice 3.3.0 → 3.3.2

package/dist/test/workers/execution-controller/scheduler-spec.js

@@ -1,249 +0,0 @@
-import { jest } from '@jest/globals';
-import { v4 as uuidv4 } from 'uuid';
-import { pDelay, once, times } from '@terascope/core-utils';
-import { TestContext } from '../helpers/test-context.js';
-import { Scheduler } from '../../../src/lib/workers/execution-controller/scheduler.js';
-describe('Scheduler', () => {
-    const slicers = 3;
-    const countPerSlicer = 200;
-    let expectedCount;
-    let testContext;
-    let scheduler;
-    let stateStorage;
-    let executionStorage;
-    function getSlices() {
-        const slices = [];
-        return new Promise((resolve) => {
-            const intervalId = setInterval(() => {
-                if (scheduler.isFinished) {
-                    clearInterval(intervalId);
-                    resolve(slices);
-                }
-                else {
-                    const result = scheduler.getSlices(100);
-                    if (result.length > 0) {
-                        slices.push(...result);
-                    }
-                }
-            }, 1);
-        });
-    }
-    beforeEach(async () => {
-        expectedCount = slicers * countPerSlicer;
-        testContext = new TestContext({
-            assignment: 'execution_controller',
-            slicers,
-            newOps: true,
-            countPerSlicer
-        });
-        await testContext.initialize();
-        scheduler = new Scheduler(testContext.context, testContext.executionContext);
-        stateStorage = {
-            async getStartingPoints(exId, numSlicers) {
-                if (numSlicers !== slicers) {
-                    throw new Error(`Got invalid slicer ids, ${numSlicers.join(' ')}`);
-                }
-            },
-            createState: () => pDelay(0),
-            createSlices: async (exId, slices) => {
-                if (!exId || typeof exId !== 'string' || exId !== testContext.exId) {
-                    throw new Error(`Got invalid ex_id ${exId}`);
-                }
-                await pDelay(0);
-                return slices.length;
-            }
-        };
-        executionStorage = {
-            async get() {
-                return { slicers };
-            },
-            setStatus: jest.fn(() => pDelay(0)),
-        };
-        testContext.attachCleanup(() => scheduler.shutdown());
-    });
-    afterEach(() => testContext.cleanup());
-    describe('when testing a normal execution', () => {
-        beforeEach(() => scheduler.initialize(stateStorage, executionStorage));
-        it('should be constructed wih the correct values', async () => {
-            expect(scheduler.slicersDone).toBeFalse();
-            expect(scheduler.ready).toBeFalse();
-            expect(scheduler.paused).toBeTrue();
-            expect(scheduler.stopped).toBeFalse();
-            expect(scheduler.queueLength).toEqual(0);
-            expect(scheduler.isFinished).toBeFalse();
-        });
-        it('should be able to reenqueue a slice', () => {
-            scheduler.enqueueSlices([
-                {
-                    slice_id: 1
-                },
-                {
-                    slice_id: 2
-                }
-            ]);
-            scheduler.enqueueSlice({ slice_id: 1 });
-            scheduler.enqueueSlice({
-                slice_id: 3
-            }, true);
-            const slices = scheduler.getSlices(100);
-            expect(slices).toEqual([
-                {
-                    slice_id: 3
-                },
-                {
-                    slice_id: 1
-                },
-                {
-                    slice_id: 2
-                }
-            ]);
-        });
-        it(`should be able to schedule ${expectedCount} slices`, async () => {
-            let slices = [];
-            await Promise.all([
-                scheduler.run(),
-                getSlices().then((_slices) => {
-                    slices = _slices;
-                })
-            ]);
-            expect(scheduler.paused).toBeFalse();
-            expect(scheduler.slicersDone).toBeTrue();
-            expect(scheduler.queueLength).toEqual(0);
-            expect(slices).toHaveLength(expectedCount);
-            expect(scheduler.isFinished).toBeTrue();
-        });
-        it('should handle pause and resume correctly', async () => {
-            let slices = [];
-            const pause = once(() => {
-                scheduler.pause();
-                setTimeout(() => scheduler.start(), 10);
-            });
-            scheduler.events.on('slicer:done', pause);
-            await Promise.all([
-                scheduler.run(),
-                getSlices().then((_slices) => {
-                    slices = _slices;
-                })
-            ]);
-            expect(slices).toHaveLength(expectedCount);
-            expect(scheduler.isFinished).toBeTrue();
-            expect(scheduler.slicersDone).toBeTrue();
-        });
-        it('should handle stop correctly', async () => {
-            let slices = [];
-            scheduler.events.once('slicer:done', () => scheduler.stop());
-            await Promise.all([
-                scheduler.run(),
-                getSlices().then((_slices) => {
-                    slices = _slices;
-                })
-            ]);
-            // be more flexible
-            const min = expectedCount - slicers * 8;
-            const max = expectedCount + slicers * 8;
-            expect(slices.length).toBeWithin(min, max);
-            expect(scheduler.isFinished).toBeTrue();
-            expect(scheduler.stopped).toBeTrue();
-            expect(scheduler.slicersDone).toBeFalse();
-        });
-    });
-    describe('when testing recovery', () => {
-        let recoveryRecords;
-        let emitDone;
-        let exitAfterComplete = false;
-        beforeEach(() => {
-            emitDone = once(() => {
-                scheduler.events.emit('execution:recovery:complete', []);
-            });
-            // @ts-expect-error
-            scheduler.recoverExecution = true;
-            scheduler.recovering = true;
-            // @ts-expect-error
-            scheduler.recover = {
-                initialize() {
-                    return Promise.resolve();
-                },
-                shutdown() {
-                    return Promise.resolve();
-                },
-                // @ts-expect-error
-                handle() {
-                    return recoveryRecords.length === 0;
-                },
-                getSlices(max = 1) {
-                    const result = recoveryRecords.splice(0, max);
-                    if (!recoveryRecords.length) {
-                        emitDone();
-                    }
-                    return result;
-                },
-                recoveryComplete() {
-                    if (!recoveryRecords.length) {
-                        emitDone();
-                        return true;
-                    }
-                    return false;
-                },
-                sliceCount() {
-                    return 10;
-                },
-                exitAfterComplete() {
-                    return exitAfterComplete;
-                }
-            };
-            return scheduler.initialize(stateStorage, executionStorage);
-        });
-        it('should handle recovery correctly and exit', async () => {
-            recoveryRecords = times(countPerSlicer * slicers, () => ({
-                slice_id: uuidv4(),
-                slicer_id: 1,
-                slicer_order: 0,
-                request: {
-                    id: `recover-${Math.random()}`
-                },
-                _created: new Date().toISOString()
-            }));
-            let slices = [];
-            expectedCount += recoveryRecords.length;
-            await Promise.all([
-                scheduler.run(),
-                getSlices().then((_slices) => {
-                    slices = _slices;
-                })
-            ]);
-            expect(slices).toHaveLength(expectedCount);
-            expect(scheduler.ready).toBeTrue();
-            expect(scheduler.isFinished).toBeTrue();
-            expect(scheduler.stopped).toBeFalse();
-            expect(scheduler.recovering).toBeFalse();
-            expect(scheduler.slicersDone).toBeTrue();
-        });
-        it('should handle recovery with cleanup type correctly and not exit', async () => {
-            exitAfterComplete = true;
-            recoveryRecords = times(countPerSlicer, () => ({
-                slice_id: uuidv4(),
-                slicer_id: 1,
-                slicer_order: 0,
-                request: {
-                    id: `recover-${Math.random()}`
-                },
-                _created: new Date().toISOString()
-            }));
-            let slices = [];
-            expectedCount = recoveryRecords.length;
-            await Promise.all([
-                scheduler.run(),
-                getSlices().then((_slices) => {
-                    slices = _slices;
-                })
-            ]);
-            expect(slices).toHaveLength(expectedCount);
-            expect(scheduler.ready).toBeFalse();
-            expect(scheduler.isFinished).toBeTrue();
-            expect(scheduler.stopped).toBeFalse();
-            expect(scheduler.recovering).toBeFalse();
-            expect(scheduler.slicersDone).toBeTrue();
-        });
-    });
-});
-//# sourceMappingURL=scheduler-spec.js.map

package/dist/test/workers/execution-controller/slice-analytics-spec.js

@@ -1,121 +0,0 @@
-import events from 'node:events';
-import { SliceAnalytics } from '../../../src/lib/workers/execution-controller/slice-analytics.js';
-const eventEmitter = new events.EventEmitter();
-describe('slice_analytics', () => {
-    const logMessages = [];
-    const logger = {
-        error() { },
-        info(msg) {
-            logMessages.push(msg);
-        },
-        warn() { },
-        trace() { },
-        debug() { }
-    };
-    const context = {
-        apis: {
-            foundation: {
-                makeLogger: () => logger,
-                getSystemEvents: () => eventEmitter
-            }
-        }
-    };
-    const executionContext = {
-        config: {
-            slicers: 2,
-            operations: [{ _op: 'config1' }, { _op: 'config2' }, { _op: 'config3' }]
-        },
-        ex_id: '1234',
-        job_id: '5678'
-    };
-    const analytics = new SliceAnalytics(context, executionContext);
-    it('addStats transfers message stats to the statsContainer', () => {
-        const statsObj = analytics.getStats();
-        const data = { time: [234, 125, 1300], size: [2300, 4600], memory: [1234, 4567] };
-        const data2 = { time: [346, 325, 1102], size: [1120, 2240], memory: [12345, 56789] };
-        expect(statsObj.size[0]).toMatchObject({
-            min: 0,
-            max: 0,
-            sum: 0,
-            total: 0,
-            average: 0,
-        });
-        expect(statsObj.time[0]).toMatchObject({
-            min: 0,
-            max: 0,
-            sum: 0,
-            total: 0,
-            average: 0,
-        });
-        expect(statsObj.memory[0]).toMatchObject({
-            min: 0,
-            max: 0,
-            sum: 0,
-            total: 0,
-            average: 0,
-        });
-        analytics.addStats(data);
-        expect(statsObj.size[0]).toMatchObject({
-            min: 2300,
-            max: 2300,
-            sum: 2300,
-            total: 1,
-            average: 2300,
-        });
-        expect(statsObj.time[0]).toMatchObject({
-            min: 234,
-            max: 234,
-            sum: 234,
-            total: 1,
-            average: 234,
-        });
-        expect(statsObj.memory[0]).toMatchObject({
-            min: 1234,
-            max: 1234,
-            sum: 1234,
-            total: 1,
-            average: 1234,
-        });
-        analytics.addStats(data2);
-        expect(statsObj.size[0]).toMatchObject({
-            min: 1120,
-            max: 2300,
-            sum: 3420,
-            total: 2,
-            average: 1710,
-        });
-        expect(statsObj.time[0]).toMatchObject({
-            min: 234,
-            max: 346,
-            sum: 580,
-            total: 2,
-            average: 290,
-        });
-        expect(statsObj.memory[0]).toMatchObject({
-            min: 1234,
-            max: 12345,
-            sum: 13579,
-            total: 2,
-            average: 6789.5,
-        });
-    });
-    it('statsContainer takes in job.operations and returns an object for the number of ops', () => {
-        const results = analytics.getStats();
-        expect(results).toBeDefined();
-        expect(results.time).toBeDefined();
-        expect(results.size).toBeDefined();
-        expect(Array.isArray(results.time)).toBe(true);
-        expect(Array.isArray(results.size)).toBe(true);
-        expect(results.time.length).toEqual(3);
-        expect(results.size.length).toEqual(3);
-        expect(results.memory.length).toEqual(3);
-    });
-    it('analyzeStats will log results', () => {
-        analytics.analyzeStats();
-        expect(logMessages.shift()).toEqual('calculating statistics');
-        expect(logMessages.shift()).toContain('operation config1');
-        expect(logMessages.shift()).toContain('operation config2');
-        expect(logMessages.shift()).toContain('operation config3');
-    });
-});
-//# sourceMappingURL=slice-analytics-spec.js.map

package/dist/test/workers/fixtures/ops/example-op/processor.js

@@ -1,20 +0,0 @@
-import { BatchProcessor } from '@terascope/job-components';
-const defaultResults = Array.from(Array(10)).map(() => ({ hi: true }));
-export default class ExampleBatch extends BatchProcessor {
-    async onBatch(_batch) {
-        const errorAt = this.opConfig.errorAt ?? [];
-        // @ts-expect-error
-        if (this.context._opCalls == null) {
-            // @ts-expect-error
-            this.context._opCalls = -1;
-        }
-        // @ts-expect-error
-        this.context._opCalls += 1;
-        // @ts-expect-error
-        if (errorAt.includes(this.context._opCalls)) {
-            return Promise.reject(new Error('Bad news bears'));
-        }
-        return this.opConfig.results ?? defaultResults;
-    }
-}
-//# sourceMappingURL=processor.js.map

package/dist/test/workers/fixtures/ops/example-op/schema.js

@@ -1,19 +0,0 @@
-import { BaseSchema } from '@terascope/job-components';
-const defaultResults = Array.from(Array(10)).map(() => ({ hi: true }));
-export default class Schema extends BaseSchema {
-    build() {
-        return {
-            errorAt: {
-                doc: 'An array of indexes to error at',
-                default: [],
-                format: 'Array'
-            },
-            results: {
-                doc: 'Op results to return',
-                default: defaultResults,
-                format: 'Array'
-            },
-        };
-    }
-}
-//# sourceMappingURL=schema.js.map

package/dist/test/workers/fixtures/ops/example-reader/fetcher.js

@@ -1,20 +0,0 @@
-import { Fetcher } from '@terascope/job-components';
-const defaultResults = Array.from(Array(10)).map(() => ({ hi: true }));
-export default class ExampleBatch extends Fetcher {
-    async fetch() {
-        const errorAt = this.opConfig.errorAt ?? [];
-        // @ts-expect-error
-        if (this.context._readerCalls == null) {
-            // @ts-expect-error
-            this.context._readerCalls = -1;
-        }
-        // @ts-expect-error
-        this.context._readerCalls += 1;
-        // @ts-expect-error
-        if (errorAt.includes(this.context._readerCalls)) {
-            return Promise.reject(new Error('Bad news bears'));
-        }
-        return this.opConfig.results ?? defaultResults;
-    }
-}
-//# sourceMappingURL=fetcher.js.map

package/dist/test/workers/fixtures/ops/example-reader/schema.js

@@ -1,41 +0,0 @@
-import { BaseSchema } from '@terascope/job-components';
-const defaultResults = Array.from(Array(10)).map(() => ({ hi: true }));
-const defaultSlicerResults = [{ howdy: true }, null];
-const defaultSlicerQueueLength = '10';
-export default class Schema extends BaseSchema {
-    build() {
-        return {
-            errorAt: {
-                doc: 'An array of indexes to error at',
-                default: [],
-                format: 'Array'
-            },
-            results: {
-                doc: 'Reader results to return',
-                default: defaultResults,
-                format: 'Array'
-            },
-            slicerQueueLength: {
-                doc: 'A string for the slicer queue length, anything but QUEUE_MINIMUM_SIZE will be converted to a number',
-                default: defaultSlicerQueueLength,
-                format: 'String'
-            },
-            slicerErrorAt: {
-                doc: 'An array of indexes to error at when newSlicer is invoked',
-                default: [],
-                format: 'Array'
-            },
-            slicerResults: {
-                doc: 'Slicer results to return',
-                default: defaultSlicerResults,
-                format: 'Array'
-            },
-            updateMetadata: {
-                doc: 'Update the metadata on the slicer execution',
-                default: false,
-                format: Boolean
-            }
-        };
-    }
-}
-//# sourceMappingURL=schema.js.map

package/dist/test/workers/fixtures/ops/example-reader/slicer.js

@@ -1,37 +0,0 @@
-import { ParallelSlicer } from '@terascope/job-components';
-import { get } from '@terascope/core-utils';
-const defaultSlicerResults = [{ howdy: true }, null];
-export default class ExampleSlicer extends ParallelSlicer {
-    async newSlicer(_id) {
-        const slicerResults = get(this.opConfig, 'slicerResults', defaultSlicerResults);
-        const errorAt = get(this.opConfig, 'slicerErrorAt', []);
-        const updateMetadata = get(this.opConfig, 'updateMetadata', false);
-        // @ts-expect-error
-        if (!this.context._slicerCalls) {
-            // @ts-expect-error
-            this.context._slicerCalls = -1;
-        }
-        let sliceCalls = 0;
-        return async () => {
-            // @ts-expect-error
-            this.context._slicerCalls += 1;
-            // @ts-expect-error
-            if (errorAt.includes(this.context._slicerCalls)) {
-                return Promise.reject(new Error('Bad news bears'));
-            }
-            const result = slicerResults.shift();
-            sliceCalls++;
-            if (updateMetadata) {
-                await this.context.apis.executionContext.setMetadata('slice_calls', sliceCalls);
-            }
-            if (result && result.error) {
-                throw new Error(result.error);
-            }
-            return result;
-        };
-    }
-    isRecoverable() {
-        return true;
-    }
-}
-//# sourceMappingURL=slicer.js.map

package/dist/test/workers/fixtures/ops/new-op/processor.js

@@ -1,29 +0,0 @@
-import { BatchProcessor } from '@terascope/job-components';
-export default class ExampleBatch extends BatchProcessor {
-    initialized;
-    shutdownCalled;
-    async initialize() {
-        this.initialized = true;
-        this.logger.debug('example map initalized', this.opConfig);
-        return super.initialize();
-    }
-    async shutdown() {
-        this.shutdownCalled = true;
-        return super.shutdown();
-    }
-    async onBatch(batch) {
-        if (this.opConfig.failOnSliceRetry) {
-            throw new Error('Fail slices');
-        }
-        return batch.map((data) => {
-            data.touchedAt = new Date().toISOString();
-            return data;
-        });
-    }
-    onSliceRetry() {
-        if (this.opConfig.failOnSliceRetry) {
-            throw new Error('I will not allow it');
-        }
-    }
-}
-//# sourceMappingURL=processor.js.map

package/dist/test/workers/fixtures/ops/new-op/schema.js

@@ -1,18 +0,0 @@
-import { BaseSchema } from '@terascope/job-components';
-export default class Schema extends BaseSchema {
-    build() {
-        return {
-            example: {
-                default: 'examples are quick and easy',
-                doc: 'A random example schema property',
-                format: 'String',
-            },
-            failOnSliceRetry: {
-                default: false,
-                doc: 'fail on slice retry',
-                format: Boolean,
-            }
-        };
-    }
-}
-//# sourceMappingURL=schema.js.map

package/dist/test/workers/fixtures/ops/new-reader/fetcher.js

@@ -1,19 +0,0 @@
-import { Fetcher } from '@terascope/job-components';
-export default class ExampleFetcher extends Fetcher {
-    async fetch() {
-        const { countPerFetch } = this.opConfig;
-        const result = [];
-        for (let i = 0; i < countPerFetch; i++) {
-            result.push({
-                id: i,
-                data: [
-                    Math.random(),
-                    Math.random(),
-                    Math.random(),
-                ]
-            });
-        }
-        return result;
-    }
-}
-//# sourceMappingURL=fetcher.js.map

package/dist/test/workers/fixtures/ops/new-reader/schema.js

@@ -1,23 +0,0 @@
-import { BaseSchema } from '@terascope/job-components';
-export default class Schema extends BaseSchema {
-    build() {
-        return {
-            example: {
-                default: 'examples are quick and easy',
-                doc: 'A random example schema property',
-                format: 'String',
-            },
-            countPerSlicer: {
-                default: 10,
-                doc: 'The number of slices for the slicer to create',
-                format: 'Number',
-            },
-            countPerFetch: {
-                default: 10,
-                doc: 'The number of records to return from the fetcher',
-                format: 'Number',
-            }
-        };
-    }
-}
-//# sourceMappingURL=schema.js.map

package/dist/test/workers/fixtures/ops/new-reader/slicer.js

@@ -1,13 +0,0 @@
-import { ParallelSlicer } from '@terascope/job-components';
-import { pDelay, times } from '@terascope/core-utils';
-export default class ExampleSlicer extends ParallelSlicer {
-    async newSlicer(id) {
-        const { countPerSlicer } = this.opConfig;
-        const records = times(countPerSlicer, (i) => ({ id: `slicer-${id}-${i}` }));
-        return async () => {
-            await pDelay(0);
-            return records.shift();
-        };
-    }
-}
-//# sourceMappingURL=slicer.js.map