teraslice 3.3.0 → 3.3.2

package/dist/src/lib/workers/execution-controller/recovery.js

@@ -1,151 +0,0 @@
-import { pRaceWithTimeout, logError, Queue } from '@terascope/core-utils';
-import { makeLogger } from '../helpers/terafoundation.js';
-export class RecoveryModule {
-    logger;
-    events;
-    slicersToRecover;
-    recoveryQueue;
-    recoverComplete = true;
-    isShutdown = false;
-    autorecover;
-    cleanupType;
-    slicerID = 0;
-    retryState = new Map();
-    recoverExecution;
-    exId;
-    stateStore;
-    timeout;
-    constructor(context, executionContext) {
-        const { exId } = executionContext;
-        this.events = context.apis.foundation.getSystemEvents();
-        this.slicersToRecover = executionContext.config.slicers;
-        this.recoveryQueue = new Queue();
-        this.cleanupType = executionContext.config.recovered_slice_type;
-        this.recoverExecution = executionContext.config.recovered_execution;
-        this.autorecover = Boolean(executionContext.config.autorecover);
-        this.exId = exId;
-        this.logger = makeLogger(context, 'execution_recovery');
-        this.timeout = context.sysconfig.teraslice.shutdown_timeout;
-    }
-    initialize(stateStore) {
-        this.stateStore = stateStore;
-        this.events.on('slice:success', this._sliceComplete.bind(this));
-        this.recoverComplete = false;
-        // once we have fully recovered, clean up event listeners
-        this.events.once('execution:recovery:complete', () => {
-            this.events.removeListener('slice:success', this._sliceComplete.bind(this));
-        });
-    }
-    // TODO: this is wrong
-    _sliceComplete(sliceData) {
-        this.retryState.set(sliceData.slice.slice_id, false);
-    }
-    _setId(slice) {
-        this.retryState.set(slice.slice_id, true);
-    }
-    async _processIncompleteSlices(slicerID) {
-        const slices = await this.stateStore.recoverSlices(this.recoverExecution, slicerID, this.cleanupType);
-        for (const slice of slices) {
-            this._setId(slice);
-            this.recoveryQueue.enqueue(slice);
-        }
-        return slices.length;
-    }
-    _recoveryBatchCompleted() {
-        return [...this.retryState.values()].every((v) => v === false);
-    }
-    _retryState() {
-        return Object.fromEntries(this.retryState);
-    }
-    // TODO: refactor to use pwhile
-    async _waitForRecoveryBatchCompletion() {
-        return new Promise((resolve) => {
-            const checkingBatch = setInterval(() => {
-                if (this._recoveryBatchCompleted()) {
-                    clearInterval(checkingBatch);
-                    this.events.emit('execution:recovery:complete');
-                    this.recoverComplete = true;
-                    resolve(true);
-                    return;
-                }
-                if (this.isShutdown) {
-                    clearInterval(checkingBatch);
-                    this.recoverComplete = false;
-                    resolve(true);
-                }
-            }, 100);
-        });
-    }
-    async handle() {
-        if (this.recoverComplete) {
-            return true;
-        }
-        const recoveredSlicesCount = await this._processIncompleteSlices(this.slicerID);
-        if (recoveredSlicesCount === 0) {
-            this.slicerID++;
-            // all slicers have been recovered
-            if (this.slicerID > this.slicersToRecover) {
-                this.logger.warn(`recovered data for execution: ${this.exId} has successfully been enqueued`);
-                await this._waitForRecoveryBatchCompletion();
-                return true;
-            }
-        }
-        await this._waitForRecoveryBatchCompletion();
-        return false;
-    }
-    getSlice() {
-        if (this.recoveryQueue.size() > 0) {
-            return this.recoveryQueue.dequeue();
-        }
-        return;
-    }
-    getSlices(max = 1) {
-        const count = max > this.sliceCount() ? this.sliceCount() : max;
-        const slices = [];
-        for (let i = 0; i < count; i++) {
-            const slice = this.recoveryQueue.dequeue();
-            if (!slice)
-                return slices;
-            slices.push(slice);
-        }
-        return slices;
-    }
-    sliceCount() {
-        return this.recoveryQueue.size();
-    }
-    async shutdown() {
-        let checkInterval;
-        try {
-            await pRaceWithTimeout(new Promise((resolve) => {
-                checkInterval = setInterval(() => {
-                    if (this.recoverComplete) {
-                        resolve(true);
-                    }
-                }, 100);
-            }), this.timeout, (err) => {
-                logError(this.logger, err);
-            });
-        }
-        finally {
-            this.isShutdown = true;
-            clearInterval(checkInterval);
-        }
-    }
-    recoveryComplete() {
-        return this.recoverComplete;
-    }
-    /**
-     * Whether or not the execution will continue to process
-     * slices after recovering.
-     *
-     * @returns {boolean}
-    */
-    exitAfterComplete() {
-        if (this.autorecover)
-            return false;
-        if (!this.cleanupType)
-            return false;
-        return true;
-    }
-}
-//# sourceMappingURL=recovery.js.map

package/dist/src/lib/workers/execution-controller/scheduler.js

@@ -1,390 +0,0 @@
-import { noop, pDelay, get, toString, logError, pWhile, Queue, makeISODate } from '@terascope/core-utils';
-import { RecoveryModule } from './recovery.js';
-import { makeLogger } from '../helpers/terafoundation.js';
-export class Scheduler {
-    context;
-    logger;
-    events;
-    executionContext;
-    exId;
-    recoverFromExId;
-    recoverExecution;
-    recovering;
-    autorecover;
-    _creating = 0;
-    ready = false;
-    paused = true;
-    stopped = false;
-    slicersDone = false;
-    slicersFailed = false;
-    queue = new Queue();
-    recover;
-    stateStorage;
-    executionStorage;
-    _resolveRun = noop;
-    _processCleanup = noop;
-    startingPoints = [];
-    constructor(context, executionContext) {
-        this.context = context;
-        this.logger = makeLogger(context, 'execution_scheduler');
-        this.events = context.apis.foundation.getSystemEvents();
-        this.executionContext = executionContext;
-        this.exId = executionContext.exId;
-        this.recover = new RecoveryModule(context, executionContext);
-        const recoverFromExId = get(executionContext.config, 'recovered_execution');
-        const slicerCanRecover = executionContext.slicer().isRecoverable();
-        if (recoverFromExId && !slicerCanRecover) {
-            throw new Error('Slicer is not recoverable');
-        }
-        this.recoverFromExId = recoverFromExId;
-        this.recoverExecution = Boolean(recoverFromExId && slicerCanRecover);
-        this.recovering = Boolean(this.recoverExecution);
-        this.autorecover = Boolean(executionContext.config.autorecover);
-        this._processSlicers();
-    }
-    /**
-     * Initialize the recovery instance or the execution context,
-     * if recovery is initialized, the execution context will not be
-     * initialized until the execution if finished and the cleanup
-     * type is set.
-    */
-    async initialize(stateStorage, executionStorage) {
-        this.stateStorage = stateStorage;
-        this.executionStorage = executionStorage;
-        if (this.recoverExecution) {
-            await this._initializeRecovery();
-            return;
-        }
-        await this._initializeExecution();
-    }
-    /**
-     * Run the execution, this will block until complete (or failed)
-    */
-    async run() {
-        if (this.recoverExecution) {
-            await this.executionStorage.setStatus(this.exId, 'recovering');
-            this.logger.info(`execution: ${this.exId} is starting in recovery mode`);
-            this.ready = true;
-            this.start();
-            await this._waitForRecovery();
-            await this._recoveryComplete();
-            if (this.recover.exitAfterComplete()) {
-                return;
-            }
-            await this._initializeExecution();
-        }
-        await this.executionStorage.setStatus(this.exId, 'running');
-        this.ready = true;
-        const promise = new Promise((resolve) => {
-            const handler = (err) => {
-                if (err) {
-                    this.slicersFailed = true;
-                }
-                else {
-                    this.slicersDone = true;
-                }
-                this._resolveRun();
-                this._resolveRun = noop;
-            };
-            this._resolveRun = () => {
-                this.events.removeListener('slicers:finished', handler);
-                resolve(true);
-            };
-            this.events.once('slicers:finished', handler);
-        });
-        this.start();
-        this.logger.trace('running the scheduler');
-        await promise;
-        const n = this.pendingSlices + this.pendingSlicerCount;
-        this.logger.debug(`execution ${this.exId} is finished scheduling, ${n} remaining slices in the queue`);
-        await pWhile(async () => {
-            if (!this._creating) {
-                this.logger.debug('done creating remaining slices');
-                return true;
-            }
-            this.logger.debug(`waiting for ${this._creating} remaining slices to be created...`);
-            await pDelay(100);
-            return false;
-        });
-    }
-    start() {
-        this.paused = false;
-    }
-    async stop() {
-        if (this.stopped)
-            return;
-        this.logger.debug('stopping scheduler...');
-        this.stopped = true;
-        const drain = this._drainPendingSlices(false);
-        this._processCleanup();
-        this._processCleanup = noop;
-        this._resolveRun();
-        this._resolveRun = noop;
-        await drain;
-    }
-    pause() {
-        this.paused = true;
-    }
-    get maxQueueLength() {
-        return this.executionContext.slicer().maxQueueLength();
-    }
-    get queueLength() {
-        return this.queue.size();
-    }
-    get isQueueFull() {
-        const maxLength = this.maxQueueLength;
-        return this.pendingSlices + this.pendingSlicerCount > maxLength;
-    }
-    get pendingSlicerCount() {
-        if (!this.ready)
-            return 0;
-        if (this.recovering && this.recover) {
-            return this.recover.sliceCount();
-        }
-        return this.executionContext.slicer().sliceCount();
-    }
-    get pendingSlices() {
-        return this.queueLength + this._creating;
-    }
-    get queueRemainder() {
-        const remainder = this.maxQueueLength - this.pendingSlices;
-        return remainder > 0 ? remainder : 0;
-    }
-    get isFinished() {
-        const isDone = this.slicersDone || this.slicersFailed || this.stopped;
-        return isDone && !this.pendingSlices;
-    }
-    canAllocateSlice() {
-        return this.ready && !this.paused && !this.isQueueFull;
-    }
-    canComplete() {
-        const { lifecycle } = this.executionContext.config;
-        return this.ready && lifecycle === 'once' && !this.recovering;
-    }
-    isRecovering() {
-        return this.ready && this.recovering;
-    }
-    async shutdown() {
-        await this.stop();
-        if (this.recover) {
-            try {
-                await this.recover.shutdown();
-            }
-            catch (err) {
-                logError(this.logger, err, 'failed to shutdown recovery');
-            }
-        }
-        this.queue.each((slice) => {
-            this.queue.remove(slice.slice_id, 'slice_id');
-        });
-    }
-    getSlices(limit = 1) {
-        if (this.queue.size() === 0)
-            return [];
-        if (limit < 1)
-            return [];
-        const slices = [];
-        for (let i = 0; i < limit; i += 1) {
-            const slice = this.queue.dequeue();
-            if (slice == null)
-                break;
-            slices.push(slice);
-        }
-        if (slices.length > 0) {
-            this.events.emit('slicers:queued', this.queueLength);
-        }
-        return slices;
-    }
-    enqueueSlice(slice, addToStart) {
-        return this.enqueueSlices([slice], addToStart);
-    }
-    enqueueSlices(slices, addToStart = false) {
-        if (this.stopped)
-            return;
-        slices.forEach((slice) => {
-            if (this.queue.exists('slice_id', slice.slice_id)) {
-                this.logger.warn(`slice ${slice.slice_id} has already been enqueued`);
-                return;
-            }
-            this.logger.trace('enqueuing slice', slice);
-            if (addToStart) {
-                this.queue.unshift(slice);
-            }
-            else {
-                this.queue.enqueue(slice);
-                this.executionContext.onSliceEnqueued(slice);
-            }
-        });
-        this.events.emit('slicers:queued', this.queueLength);
-    }
-    _processSlicers() {
-        const { events, logger, exId } = this;
-        let _handling = false;
-        let _finished = false;
-        let createInterval;
-        let handleInterval;
-        const resetCleanup = () => {
-            this._processCleanup = noop;
-        };
-        const cleanup = () => {
-            clearInterval(createInterval);
-            createInterval = undefined;
-            clearInterval(handleInterval);
-            handleInterval = undefined;
-            resetCleanup();
-        };
-        const drain = () => {
-            const n = this.pendingSlicerCount;
-            if (n) {
-                logger.debug(`draining the remaining ${n} pending slices from the slicer`);
-            }
-            return this._drainPendingSlices(false);
-        };
-        const onSlicerFinished = async () => {
-            cleanup();
-            logger.info(`all slicers for execution: ${exId} have been completed`);
-            await drain();
-            events.emit('slicers:finished');
-        };
-        const onSlicerFailure = async (err) => {
-            cleanup();
-            logger.warn('slicer failed', toString(err));
-            await drain();
-            // before removing listeners make sure we've received all of the events
-            await pDelay(100);
-            events.emit('slicers:finished', err);
-        };
-        const makeSlices = async () => {
-            try {
-                if (this.recovering && this.recover) {
-                    _finished = await this.recover.handle();
-                }
-                else {
-                    _finished = await this.executionContext.slicer().handle();
-                }
-            }
-            catch (err) {
-                await onSlicerFailure(err);
-                return;
-            }
-            if (!_finished) {
-                return;
-            }
-            if (!this.recovering) {
-                clearInterval(handleInterval);
-                handleInterval = undefined;
-            }
-            if (this.canComplete()) {
-                await onSlicerFinished();
-            }
-        };
-        handleInterval = setInterval(() => {
-            if (!this.canAllocateSlice())
-                return;
-            if (_handling)
-                return;
-            _handling = true;
-            makeSlices()
-                .then(() => {
-                _handling = false;
-            })
-                .catch((err) => {
-                _handling = false;
-                logError(this.logger, err, 'failure to run slicers');
-            });
-        }, 3);
-        createInterval = setInterval(() => {
-            if (!this.pendingSlicerCount)
-                return;
-            this._drainPendingSlices().catch(onSlicerFailure);
-        }, 5);
-        this._processCleanup = cleanup;
-    }
-    async _drainPendingSlices(once = true) {
-        const slices = this._getPendingSlices();
-        if (!slices.length)
-            return;
-        await this._createSlices(slices);
-        if (once)
-            return;
-        await this._drainPendingSlices();
-    }
-    _getPendingSlices() {
-        if (!this.ready)
-            return [];
-        if (this.recovering && this.recover) {
-            return this.recover.getSlices(this.queueRemainder);
-        }
-        return this.executionContext.slicer().getSlices(this.queueRemainder);
-    }
-    async _createSlices(allSlices) {
-        // filter out anything that doesn't need to be created
-        const slices = [];
-        for (const slice of allSlices) {
-            // In the case of recovery slices have already been
-            // created, so its important to skip this step
-            if (slice._created) {
-                this.enqueueSlice(slice);
-            }
-            else {
-                slice._created = makeISODate();
-                slices.push(slice);
-            }
-        }
-        if (!slices.length)
-            return;
-        this._creating += slices.length;
-        try {
-            const count = await this.stateStorage.createSlices(this.exId, slices);
-            this.enqueueSlices(slices);
-            this._creating -= count;
-        }
-        catch (err) {
-            const { lifecycle } = this.executionContext.config;
-            if (lifecycle === 'once') {
-                throw err;
-            }
-            else {
-                logError(this.logger, err, 'failure creating slices');
-            }
-        }
-    }
-    async _initializeRecovery() {
-        await this.recover.initialize(this.stateStorage);
-        const { slicers: prevSlicers } = await this.executionStorage.get(this.recoverFromExId);
-        this.events.emit('slicers:registered', prevSlicers);
-    }
-    async _initializeExecution() {
-        await this.executionContext.initialize(this.startingPoints);
-        const slicers = this.executionContext.slicer().slicers();
-        this.events.emit('slicers:registered', slicers);
-    }
-    async _waitForRecovery() {
-        if (!this.recoverExecution)
-            return;
-        if (!this.recover.recoveryComplete()) {
-            await new Promise((resolve) => {
-                this.events.once('execution:recovery:complete', () => {
-                    resolve(true);
-                });
-            });
-        }
-    }
-    async _recoveryComplete() {
-        this.recovering = false;
-        this.ready = false;
-        if (this.recover.exitAfterComplete()) {
-            this.logger.warn('execution recovery has been marked as completed');
-            this.slicersDone = true;
-            this._processCleanup();
-            this._processCleanup = noop;
-            this._resolveRun();
-            this._resolveRun = noop;
-            return;
-        }
-        const { slicers: prevSlicers } = await this.executionStorage.get(this.recoverFromExId);
-        this.startingPoints = await this.stateStorage.getStartingPoints(this.recoverFromExId, prevSlicers);
-        this.logger.info(`execution: ${this.exId} finished its recovery`);
-    }
-}
-//# sourceMappingURL=scheduler.js.map

package/dist/src/lib/workers/execution-controller/slice-analytics.js

@@ -1,96 +0,0 @@
-import { has, isInteger, toNumber } from '@terascope/core-utils';
-import { makeLogger } from '../helpers/terafoundation.js';
-// TODO: more types
-export class SliceAnalytics {
-    logger;
-    events;
-    sliceAnalytics;
-    operations;
-    constructor(context, executionContext) {
-        this.logger = makeLogger(context, 'slice_analytics');
-        this.events = context.apis.foundation.getSystemEvents();
-        const { operations } = executionContext.config;
-        this.operations = operations;
-        // create a container to hold all the slice analytics for this execution
-        this.sliceAnalytics = {
-            time: [],
-            size: [],
-            memory: []
-        };
-        const opsLength = this.operations.length;
-        for (let i = 0; i < opsLength; i += 1) {
-            this.sliceAnalytics.time.push({
-                min: 0,
-                max: 0,
-                sum: 0,
-                total: 0,
-                average: 0
-            });
-            this.sliceAnalytics.size.push({
-                min: 0,
-                max: 0,
-                sum: 0,
-                total: 0,
-                average: 0
-            });
-            this.sliceAnalytics.memory.push({
-                min: 0,
-                max: 0,
-                sum: 0,
-                total: 0,
-                average: 0
-            });
-        }
-        this.events.on('slice:success', this.onSliceSuccess.bind(this));
-    }
-    addStat(input, stat) {
-        if (!has(input, stat) || !has(this.sliceAnalytics, stat)) {
-            this.logger.warn(`unsupported stat "${stat}"`);
-            return;
-        }
-        for (let i = 0; i < this.operations.length; i += 1) {
-            const val = input[stat][i];
-            if (!isInteger(val)) {
-                return;
-            }
-            this.sliceAnalytics[stat][i].sum += val;
-            this.sliceAnalytics[stat][i].total += 1;
-            const { min, max, total, sum } = this.sliceAnalytics[stat][i];
-            this.sliceAnalytics[stat][i].min = min !== 0 ? Math.min(val, min) : val;
-            this.sliceAnalytics[stat][i].max = max !== 0 ? Math.max(val, max) : val;
-            this.sliceAnalytics[stat][i].average = toNumber((sum / total).toFixed(2));
-        }
-    }
-    addStats(data) {
-        this.addStat(data, 'time');
-        this.addStat(data, 'memory');
-        this.addStat(data, 'size');
-    }
-    analyzeStats() {
-        this.logger.info('calculating statistics');
-        for (let i = 0; i < this.operations.length; i += 1) {
-            const name = this.operations[i]._op;
-            const time = this.sliceAnalytics.time[i];
-            const size = this.sliceAnalytics.size[i];
-            const memory = this.sliceAnalytics.memory[i];
-            this.logger.info(`
-operation ${name}
-average completion time of: ${time.average} ms, min: ${time.min} ms, and max: ${time.max} ms
-average size: ${size.average}, min: ${size.min}, and max: ${size.max}
-average memory: ${memory.average}, min: ${memory.min}, and max: ${memory.max}
-            `);
-        }
-    }
-    getStats() {
-        return this.sliceAnalytics;
-    }
-    onSliceSuccess({ analytics }) {
-        if (analytics) {
-            this.addStats(analytics);
-        }
-    }
-    async shutdown() {
-        this.events.removeListener('slice:success', this.onSliceSuccess.bind(this));
-    }
-}
-//# sourceMappingURL=slice-analytics.js.map