teraslice 2.11.0 → 2.12.1

package/dist/src/lib/workers/execution-controller/recovery.js

@@ -0,0 +1,151 @@
+import { pRaceWithTimeout, logError, Queue } from '@terascope/utils';
+import { makeLogger } from '../helpers/terafoundation.js';
+export class RecoveryModule {
+    logger;
+    events;
+    slicersToRecover;
+    recoveryQueue;
+    recoverComplete = true;
+    isShutdown = false;
+    autorecover;
+    cleanupType;
+    slicerID = 0;
+    retryState = new Map();
+    recoverExecution;
+    exId;
+    stateStore;
+    timeout;
+    constructor(context, executionContext) {
+        const { exId } = executionContext;
+        this.events = context.apis.foundation.getSystemEvents();
+        this.slicersToRecover = executionContext.config.slicers;
+        this.recoveryQueue = new Queue();
+        this.cleanupType = executionContext.config.recovered_slice_type;
+        this.recoverExecution = executionContext.config.recovered_execution;
+        this.autorecover = Boolean(executionContext.config.autorecover);
+        this.exId = exId;
+        this.logger = makeLogger(context, 'execution_recovery');
+        this.timeout = context.sysconfig.teraslice.shutdown_timeout;
+    }
+    initialize(stateStore) {
+        this.stateStore = stateStore;
+        this.events.on('slice:success', this._sliceComplete.bind(this));
+        this.recoverComplete = false;
+        // once we have fully recovered, clean up event listeners
+        this.events.once('execution:recovery:complete', () => {
+            this.events.removeListener('slice:success', this._sliceComplete.bind(this));
+        });
+    }
+    // TODO: this is wrong
+    _sliceComplete(sliceData) {
+        this.retryState.set(sliceData.slice.slice_id, false);
+    }
+    _setId(slice) {
+        this.retryState.set(slice.slice_id, true);
+    }
+    async _processIncompleteSlices(slicerID) {
+        const slices = await this.stateStore.recoverSlices(this.recoverExecution, slicerID, this.cleanupType);
+        for (const slice of slices) {
+            this._setId(slice);
+            this.recoveryQueue.enqueue(slice);
+        }
+        return slices.length;
+    }
+    _recoveryBatchCompleted() {
+        return [...this.retryState.values()].every((v) => v === false);
+    }
+    _retryState() {
+        return Object.fromEntries(this.retryState);
+    }
+    // TODO: refactor to use pwhile
+    async _waitForRecoveryBatchCompletion() {
+        return new Promise((resolve) => {
+            const checkingBatch = setInterval(() => {
+                if (this._recoveryBatchCompleted()) {
+                    clearInterval(checkingBatch);
+                    this.events.emit('execution:recovery:complete');
+                    this.recoverComplete = true;
+                    resolve(true);
+                    return;
+                }
+                if (this.isShutdown) {
+                    clearInterval(checkingBatch);
+                    this.recoverComplete = false;
+                    resolve(true);
+                }
+            }, 100);
+        });
+    }
+    async handle() {
+        if (this.recoverComplete) {
+            return true;
+        }
+        const recoveredSlicesCount = await this._processIncompleteSlices(this.slicerID);
+        if (recoveredSlicesCount === 0) {
+            this.slicerID++;
+            // all slicers have been recovered
+            if (this.slicerID > this.slicersToRecover) {
+                this.logger.warn(`recovered data for execution: ${this.exId} has successfully been enqueued`);
+                await this._waitForRecoveryBatchCompletion();
+                return true;
+            }
+        }
+        await this._waitForRecoveryBatchCompletion();
+        return false;
+    }
+    getSlice() {
+        if (this.recoveryQueue.size() > 0) {
+            return this.recoveryQueue.dequeue();
+        }
+        return;
+    }
+    getSlices(max = 1) {
+        const count = max > this.sliceCount() ? this.sliceCount() : max;
+        const slices = [];
+        for (let i = 0; i < count; i++) {
+            const slice = this.recoveryQueue.dequeue();
+            if (!slice)
+                return slices;
+            slices.push(slice);
+        }
+        return slices;
+    }
+    sliceCount() {
+        return this.recoveryQueue.size();
+    }
+    async shutdown() {
+        let checkInterval;
+        try {
+            await pRaceWithTimeout(new Promise((resolve) => {
+                checkInterval = setInterval(() => {
+                    if (this.recoverComplete) {
+                        resolve(true);
+                    }
+                }, 100);
+            }), this.timeout, (err) => {
+                logError(this.logger, err);
+            });
+        }
+        finally {
+            this.isShutdown = true;
+            clearInterval(checkInterval);
+        }
+    }
+    recoveryComplete() {
+        return this.recoverComplete;
+    }
+    /**
+     * Whether or not the execution will continue to process
+     * slices after recovering.
+     *
+     * @returns {boolean}
+    */
+    exitAfterComplete() {
+        if (this.autorecover)
+            return false;
+        if (!this.cleanupType)
+            return false;
+        return true;
+    }
+}
+//# sourceMappingURL=recovery.js.map

package/dist/src/lib/workers/execution-controller/scheduler.js

@@ -0,0 +1,390 @@
+import { Queue, noop, pDelay, get, toString, makeISODate, logError, pWhile } from '@terascope/utils';
+import { RecoveryModule } from './recovery.js';
+import { makeLogger } from '../helpers/terafoundation.js';
+export class Scheduler {
+    context;
+    logger;
+    events;
+    executionContext;
+    exId;
+    recoverFromExId;
+    recoverExecution;
+    recovering;
+    autorecover;
+    _creating = 0;
+    ready = false;
+    paused = true;
+    stopped = false;
+    slicersDone = false;
+    slicersFailed = false;
+    queue = new Queue();
+    recover;
+    stateStorage;
+    executionStorage;
+    _resolveRun = noop;
+    _processCleanup = noop;
+    startingPoints = [];
+    constructor(context, executionContext) {
+        this.context = context;
+        this.logger = makeLogger(context, 'execution_scheduler');
+        this.events = context.apis.foundation.getSystemEvents();
+        this.executionContext = executionContext;
+        this.exId = executionContext.exId;
+        this.recover = new RecoveryModule(context, executionContext);
+        const recoverFromExId = get(executionContext.config, 'recovered_execution');
+        const slicerCanRecover = executionContext.slicer().isRecoverable();
+        if (recoverFromExId && !slicerCanRecover) {
+            throw new Error('Slicer is not recoverable');
+        }
+        this.recoverFromExId = recoverFromExId;
+        this.recoverExecution = Boolean(recoverFromExId && slicerCanRecover);
+        this.recovering = Boolean(this.recoverExecution);
+        this.autorecover = Boolean(executionContext.config.autorecover);
+        this._processSlicers();
+    }
+    /**
+     * Initialize the recovery instance or the execution context,
+     * if recovery is initialized, the execution context will not be
+     * initialized until the execution if finished and the cleanup
+     * type is set.
+    */
+    async initialize(stateStorage, executionStorage) {
+        this.stateStorage = stateStorage;
+        this.executionStorage = executionStorage;
+        if (this.recoverExecution) {
+            await this._initializeRecovery();
+            return;
+        }
+        await this._initializeExecution();
+    }
+    /**
+     * Run the execution, this will block until complete (or failed)
+    */
+    async run() {
+        if (this.recoverExecution) {
+            await this.executionStorage.setStatus(this.exId, 'recovering');
+            this.logger.info(`execution: ${this.exId} is starting in recovery mode`);
+            this.ready = true;
+            this.start();
+            await this._waitForRecovery();
+            await this._recoveryComplete();
+            if (this.recover.exitAfterComplete()) {
+                return;
+            }
+            await this._initializeExecution();
+        }
+        await this.executionStorage.setStatus(this.exId, 'running');
+        this.ready = true;
+        const promise = new Promise((resolve) => {
+            const handler = (err) => {
+                if (err) {
+                    this.slicersFailed = true;
+                }
+                else {
+                    this.slicersDone = true;
+                }
+                this._resolveRun();
+                this._resolveRun = noop;
+            };
+            this._resolveRun = () => {
+                this.events.removeListener('slicers:finished', handler);
+                resolve(true);
+            };
+            this.events.once('slicers:finished', handler);
+        });
+        this.start();
+        this.logger.trace('running the scheduler');
+        await promise;
+        const n = this.pendingSlices + this.pendingSlicerCount;
+        this.logger.debug(`execution ${this.exId} is finished scheduling, ${n} remaining slices in the queue`);
+        await pWhile(async () => {
+            if (!this._creating) {
+                this.logger.debug('done creating remaining slices');
+                return true;
+            }
+            this.logger.debug(`waiting for ${this._creating} remaining slices to be created...`);
+            await pDelay(100);
+            return false;
+        });
+    }
+    start() {
+        this.paused = false;
+    }
+    async stop() {
+        if (this.stopped)
+            return;
+        this.logger.debug('stopping scheduler...');
+        this.stopped = true;
+        const drain = this._drainPendingSlices(false);
+        this._processCleanup();
+        this._processCleanup = noop;
+        this._resolveRun();
+        this._resolveRun = noop;
+        await drain;
+    }
+    pause() {
+        this.paused = true;
+    }
+    get maxQueueLength() {
+        return this.executionContext.slicer().maxQueueLength();
+    }
+    get queueLength() {
+        return this.queue.size();
+    }
+    get isQueueFull() {
+        const maxLength = this.maxQueueLength;
+        return this.pendingSlices + this.pendingSlicerCount > maxLength;
+    }
+    get pendingSlicerCount() {
+        if (!this.ready)
+            return 0;
+        if (this.recovering && this.recover) {
+            return this.recover.sliceCount();
+        }
+        return this.executionContext.slicer().sliceCount();
+    }
+    get pendingSlices() {
+        return this.queueLength + this._creating;
+    }
+    get queueRemainder() {
+        const remainder = this.maxQueueLength - this.pendingSlices;
+        return remainder > 0 ? remainder : 0;
+    }
+    get isFinished() {
+        const isDone = this.slicersDone || this.slicersFailed || this.stopped;
+        return isDone && !this.pendingSlices;
+    }
+    canAllocateSlice() {
+        return this.ready && !this.paused && !this.isQueueFull;
+    }
+    canComplete() {
+        const { lifecycle } = this.executionContext.config;
+        return this.ready && lifecycle === 'once' && !this.recovering;
+    }
+    isRecovering() {
+        return this.ready && this.recovering;
+    }
+    async shutdown() {
+        await this.stop();
+        if (this.recover) {
+            try {
+                await this.recover.shutdown();
+            }
+            catch (err) {
+                logError(this.logger, err, 'failed to shutdown recovery');
+            }
+        }
+        this.queue.each((slice) => {
+            this.queue.remove(slice.slice_id, 'slice_id');
+        });
+    }
+    getSlices(limit = 1) {
+        if (this.queue.size() === 0)
+            return [];
+        if (limit < 1)
+            return [];
+        const slices = [];
+        for (let i = 0; i < limit; i += 1) {
+            const slice = this.queue.dequeue();
+            if (slice == null)
+                break;
+            slices.push(slice);
+        }
+        if (slices.length > 0) {
+            this.events.emit('slicers:queued', this.queueLength);
+        }
+        return slices;
+    }
+    enqueueSlice(slice, addToStart) {
+        return this.enqueueSlices([slice], addToStart);
+    }
+    enqueueSlices(slices, addToStart = false) {
+        if (this.stopped)
+            return;
+        slices.forEach((slice) => {
+            if (this.queue.exists('slice_id', slice.slice_id)) {
+                this.logger.warn(`slice ${slice.slice_id} has already been enqueued`);
+                return;
+            }
+            this.logger.trace('enqueuing slice', slice);
+            if (addToStart) {
+                this.queue.unshift(slice);
+            }
+            else {
+                this.queue.enqueue(slice);
+                this.executionContext.onSliceEnqueued(slice);
+            }
+        });
+        this.events.emit('slicers:queued', this.queueLength);
+    }
+    _processSlicers() {
+        const { events, logger, exId } = this;
+        let _handling = false;
+        let _finished = false;
+        let createInterval;
+        let handleInterval;
+        const resetCleanup = () => {
+            this._processCleanup = noop;
+        };
+        const cleanup = () => {
+            clearInterval(createInterval);
+            createInterval = undefined;
+            clearInterval(handleInterval);
+            handleInterval = undefined;
+            resetCleanup();
+        };
+        const drain = () => {
+            const n = this.pendingSlicerCount;
+            if (n) {
+                logger.debug(`draining the remaining ${n} pending slices from the slicer`);
+            }
+            return this._drainPendingSlices(false);
+        };
+        const onSlicerFinished = async () => {
+            cleanup();
+            logger.info(`all slicers for execution: ${exId} have been completed`);
+            await drain();
+            events.emit('slicers:finished');
+        };
+        const onSlicerFailure = async (err) => {
+            cleanup();
+            logger.warn('slicer failed', toString(err));
+            await drain();
+            // before removing listeners make sure we've received all of the events
+            await pDelay(100);
+            events.emit('slicers:finished', err);
+        };
+        const makeSlices = async () => {
+            try {
+                if (this.recovering && this.recover) {
+                    _finished = await this.recover.handle();
+                }
+                else {
+                    _finished = await this.executionContext.slicer().handle();
+                }
+            }
+            catch (err) {
+                await onSlicerFailure(err);
+                return;
+            }
+            if (!_finished) {
+                return;
+            }
+            if (!this.recovering) {
+                clearInterval(handleInterval);
+                handleInterval = undefined;
+            }
+            if (this.canComplete()) {
+                await onSlicerFinished();
+            }
+        };
+        handleInterval = setInterval(() => {
+            if (!this.canAllocateSlice())
+                return;
+            if (_handling)
+                return;
+            _handling = true;
+            makeSlices()
+                .then(() => {
+                _handling = false;
+            })
+                .catch((err) => {
+                _handling = false;
+                logError(this.logger, err, 'failure to run slicers');
+            });
+        }, 3);
+        createInterval = setInterval(() => {
+            if (!this.pendingSlicerCount)
+                return;
+            this._drainPendingSlices().catch(onSlicerFailure);
+        }, 5);
+        this._processCleanup = cleanup;
+    }
+    async _drainPendingSlices(once = true) {
+        const slices = this._getPendingSlices();
+        if (!slices.length)
+            return;
+        await this._createSlices(slices);
+        if (once)
+            return;
+        await this._drainPendingSlices();
+    }
+    _getPendingSlices() {
+        if (!this.ready)
+            return [];
+        if (this.recovering && this.recover) {
+            return this.recover.getSlices(this.queueRemainder);
+        }
+        return this.executionContext.slicer().getSlices(this.queueRemainder);
+    }
+    async _createSlices(allSlices) {
+        // filter out anything that doesn't need to be created
+        const slices = [];
+        for (const slice of allSlices) {
+            // In the case of recovery slices have already been
+            // created, so its important to skip this step
+            if (slice._created) {
+                this.enqueueSlice(slice);
+            }
+            else {
+                slice._created = makeISODate();
+                slices.push(slice);
+            }
+        }
+        if (!slices.length)
+            return;
+        this._creating += slices.length;
+        try {
+            const count = await this.stateStorage.createSlices(this.exId, slices);
+            this.enqueueSlices(slices);
+            this._creating -= count;
+        }
+        catch (err) {
+            const { lifecycle } = this.executionContext.config;
+            if (lifecycle === 'once') {
+                throw err;
+            }
+            else {
+                logError(this.logger, err, 'failure creating slices');
+            }
+        }
+    }
+    async _initializeRecovery() {
+        await this.recover.initialize(this.stateStorage);
+        const { slicers: prevSlicers } = await this.executionStorage.get(this.recoverFromExId);
+        this.events.emit('slicers:registered', prevSlicers);
+    }
+    async _initializeExecution() {
+        await this.executionContext.initialize(this.startingPoints);
+        const slicers = this.executionContext.slicer().slicers();
+        this.events.emit('slicers:registered', slicers);
+    }
+    async _waitForRecovery() {
+        if (!this.recoverExecution)
+            return;
+        if (!this.recover.recoveryComplete()) {
+            await new Promise((resolve) => {
+                this.events.once('execution:recovery:complete', () => {
+                    resolve(true);
+                });
+            });
+        }
+    }
+    async _recoveryComplete() {
+        this.recovering = false;
+        this.ready = false;
+        if (this.recover.exitAfterComplete()) {
+            this.logger.warn('execution recovery has been marked as completed');
+            this.slicersDone = true;
+            this._processCleanup();
+            this._processCleanup = noop;
+            this._resolveRun();
+            this._resolveRun = noop;
+            return;
+        }
+        const { slicers: prevSlicers } = await this.executionStorage.get(this.recoverFromExId);
+        this.startingPoints = await this.stateStorage.getStartingPoints(this.recoverFromExId, prevSlicers);
+        this.logger.info(`execution: ${this.exId} finished its recovery`);
+    }
+}
+//# sourceMappingURL=scheduler.js.map

package/dist/src/lib/workers/execution-controller/slice-analytics.js

@@ -0,0 +1,96 @@
+import { has, isInteger, toNumber } from '@terascope/utils';
+import { makeLogger } from '../helpers/terafoundation.js';
+// TODO: more types
+export class SliceAnalytics {
+    logger;
+    events;
+    sliceAnalytics;
+    operations;
+    constructor(context, executionContext) {
+        this.logger = makeLogger(context, 'slice_analytics');
+        this.events = context.apis.foundation.getSystemEvents();
+        const { operations } = executionContext.config;
+        this.operations = operations;
+        // create a container to hold all the slice analytics for this execution
+        this.sliceAnalytics = {
+            time: [],
+            size: [],
+            memory: []
+        };
+        const opsLength = this.operations.length;
+        for (let i = 0; i < opsLength; i += 1) {
+            this.sliceAnalytics.time.push({
+                min: 0,
+                max: 0,
+                sum: 0,
+                total: 0,
+                average: 0
+            });
+            this.sliceAnalytics.size.push({
+                min: 0,
+                max: 0,
+                sum: 0,
+                total: 0,
+                average: 0
+            });
+            this.sliceAnalytics.memory.push({
+                min: 0,
+                max: 0,
+                sum: 0,
+                total: 0,
+                average: 0
+            });
+        }
+        this.events.on('slice:success', this.onSliceSuccess.bind(this));
+    }
+    addStat(input, stat) {
+        if (!has(input, stat) || !has(this.sliceAnalytics, stat)) {
+            this.logger.warn(`unsupported stat "${stat}"`);
+            return;
+        }
+        for (let i = 0; i < this.operations.length; i += 1) {
+            const val = input[stat][i];
+            if (!isInteger(val)) {
+                return;
+            }
+            this.sliceAnalytics[stat][i].sum += val;
+            this.sliceAnalytics[stat][i].total += 1;
+            const { min, max, total, sum } = this.sliceAnalytics[stat][i];
+            this.sliceAnalytics[stat][i].min = min !== 0 ? Math.min(val, min) : val;
+            this.sliceAnalytics[stat][i].max = max !== 0 ? Math.max(val, max) : val;
+            this.sliceAnalytics[stat][i].average = toNumber((sum / total).toFixed(2));
+        }
+    }
+    addStats(data) {
+        this.addStat(data, 'time');
+        this.addStat(data, 'memory');
+        this.addStat(data, 'size');
+    }
+    analyzeStats() {
+        this.logger.info('calculating statistics');
+        for (let i = 0; i < this.operations.length; i += 1) {
+            const name = this.operations[i]._op;
+            const time = this.sliceAnalytics.time[i];
+            const size = this.sliceAnalytics.size[i];
+            const memory = this.sliceAnalytics.memory[i];
+            this.logger.info(`
+operation ${name}
+average completion time of: ${time.average} ms, min: ${time.min} ms, and max: ${time.max} ms
+average size: ${size.average}, min: ${size.min}, and max: ${size.max}
+average memory: ${memory.average}, min: ${memory.min}, and max: ${memory.max}
+            `);
+        }
+    }
+    getStats() {
+        return this.sliceAnalytics;
+    }
+    onSliceSuccess({ analytics }) {
+        if (analytics) {
+            this.addStats(analytics);
+        }
+    }
+    async shutdown() {
+        this.events.removeListener('slice:success', this.onSliceSuccess.bind(this));
+    }
+}
+//# sourceMappingURL=slice-analytics.js.map