teraslice 0.87.1 → 0.89.0

package/lib/workers/execution-controller/slice-analytics.js

@@ -1,115 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-const { makeLogger } = require('../helpers/terafoundation');
-
-module.exports = function _sliceAnalytics(context, executionContext) {
-    const logger = makeLogger(context, 'slice_analytics');
-
-    const events = context.apis.foundation.getSystemEvents();
-
-    const { operations } = executionContext.config;
-
-    // create a container to hold all the slice analytics for this execution
-    const sliceAnalytics = {
-        time: [],
-        size: [],
-        memory: []
-    };
-
-    for (let i = 0; i < operations.length; i += 1) {
-        sliceAnalytics.time.push({
-            min: 0,
-            max: 0,
-            sum: 0,
-            total: 0,
-            average: 0
-        });
-        sliceAnalytics.size.push({
-            min: 0,
-            max: 0,
-            sum: 0,
-            total: 0,
-            average: 0
-        });
-        sliceAnalytics.memory.push({
-            min: 0,
-            max: 0,
-            sum: 0,
-            total: 0,
-            average: 0
-        });
-    }
-
-    function addStat(input, stat) {
-        if (!_.has(input, stat) || !_.has(sliceAnalytics, stat)) {
-            logger.warn(`unsupported stat "${stat}"`);
-            return;
-        }
-
-        for (let i = 0; i < operations.length; i += 1) {
-            const val = input[stat][i];
-            if (!_.isSafeInteger(val)) {
-                return;
-            }
-
-            sliceAnalytics[stat][i].sum += val;
-            sliceAnalytics[stat][i].total += 1;
-
-            const {
-                min, max, total, sum
-            } = sliceAnalytics[stat][i];
-
-            sliceAnalytics[stat][i].min = min !== 0 ? _.min([val, min]) : val;
-            sliceAnalytics[stat][i].max = max !== 0 ? _.max([val, max]) : val;
-            sliceAnalytics[stat][i].average = _.round(sum / total, 2);
-        }
-    }
-
-    function addStats(data) {
-        addStat(data, 'time');
-        addStat(data, 'memory');
-        addStat(data, 'size');
-    }
-
-    function analyzeStats() {
-        logger.info('calculating statistics');
-
-        for (let i = 0; i < operations.length; i += 1) {
-            const name = operations[i]._op;
-            const time = sliceAnalytics.time[i];
-            const size = sliceAnalytics.size[i];
-            const memory = sliceAnalytics.memory[i];
-
-            logger.info(`
-operation ${name}
-average completion time of: ${time.average} ms, min: ${time.min} ms, and max: ${time.max} ms
-average size: ${size.average}, min: ${size.min}, and max: ${size.max}
-average memory: ${memory.average}, min: ${memory.min}, and max: ${memory.max}
-            `);
-        }
-    }
-
-    function getStats() {
-        return sliceAnalytics;
-    }
-
-    function onSliceSuccess({ analytics }) {
-        if (analytics) {
-            addStats(analytics);
-        }
-    }
-
-    events.on('slice:success', onSliceSuccess);
-
-    async function shutdown() {
-        events.removeListener('slice:success', onSliceSuccess);
-    }
-
-    return {
-        addStats,
-        analyzeStats,
-        getStats,
-        shutdown
-    };
-};

package/lib/workers/helpers/job.js

@@ -1,93 +0,0 @@
-'use strict';
-
-const { get, makeISODate } = require('@terascope/utils');
-const { JobValidator } = require('@terascope/job-components');
-const { terasliceOpPath } = require('../../config');
-const { makeJobStore, makeExStore, makeStateStore } = require('../../storage');
-
-async function validateJob(context, jobSpec) {
-    const jobValidator = new JobValidator(context, {
-        terasliceOpPath,
-        assetPath: get(context, 'sysconfig.teraslice.assets_directory'),
-    });
-
-    try {
-        return await jobValidator.validateConfig(jobSpec);
-    } catch (error) {
-        throw new Error(`validating job: ${error}`);
-    }
-}
-
-async function initializeTestExecution({
-    context,
-    config,
-    stores = {},
-    isRecovery,
-    cleanupType,
-    createRecovery = true,
-    shutdownStores = false,
-    recoverySlices = [],
-    lastStatus = 'failed'
-}) {
-    stores.jobStore = stores.jobStore || (await makeJobStore(context));
-    stores.exStore = stores.exStore || (await makeExStore(context));
-    stores.stateStore = stores.stateStore || (await makeStateStore(context));
-
-    const validJob = await validateJob(context, config, { skipRegister: true });
-    const jobSpec = await stores.jobStore.create(config);
-
-    const job = Object.assign({}, jobSpec, validJob, {
-        job_id: jobSpec.job_id
-    });
-
-    const slicerHostname = job.slicer_hostname;
-    const slicerPort = job.slicer_port;
-
-    let ex;
-    if (isRecovery) {
-        ex = await stores.exStore.create(job, lastStatus);
-
-        if (recoverySlices.length) {
-            await Promise.all(recoverySlices
-                .map(({ slice, state }) => stores.stateStore.createState(
-                    ex.ex_id,
-                    slice,
-                    state,
-                    slice.error
-                )));
-            await stores.stateStore.refresh();
-        }
-
-        if (createRecovery) {
-            ex = await stores.exStore.createRecoveredExecution(ex, cleanupType);
-        }
-    } else {
-        ex = await stores.exStore.create(job);
-    }
-
-    if (slicerHostname && slicerPort) {
-        ex = await stores.exStore.updatePartial(ex.ex_id, (existing) => Object.assign(existing, {
-            slicer_hostname: slicerHostname,
-            slicer_port: slicerPort,
-            _updated: makeISODate()
-        }));
-    }
-
-    if (shutdownStores) {
-        await Promise.all([
-            stores.exStore.shutdown(true),
-            stores.jobStore.shutdown(true),
-            stores.stateStore.shutdown(true)
-        ]);
-    }
-
-    return {
-        job,
-        ex,
-    };
-}
-
-module.exports = {
-    initializeTestExecution,
-    validateJob,
-};

package/lib/workers/helpers/op-analytics.js

@@ -1,22 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-
-function formatVal(value) {
-    if (_.isString(value)) return `"${value}"`;
-    if (_.isArray(value)) return `[${value.join(', ')}]`;
-
-    return _.truncate(JSON.stringify(value));
-}
-
-function format(input) {
-    return _.map(input, (value, key) => `${key}: ${formatVal(value)}`).join(', ');
-}
-
-function logOpStats(logger, slice, analyticsData) {
-    const obj = Object.assign({}, _.omit(slice, 'request'), analyticsData);
-
-    logger.info(`analytics for slice: ${format(obj)}`);
-}
-
-module.exports = { logOpStats };

package/lib/workers/helpers/terafoundation.js

@@ -1,43 +0,0 @@
-'use strict';
-
-const { get, isFunction, isString } = require('@terascope/utils');
-const { makeContextLogger } = require('@terascope/job-components');
-const { safeDecode } = require('../../utils/encoding_utils');
-
-function generateWorkerId(context) {
-    const { hostname } = context.sysconfig.teraslice;
-    const clusterId = get(context, 'cluster.worker.id');
-    return `${hostname}__${clusterId}`;
-}
-
-function makeLogger(context, moduleName, extra = {}) {
-    if (!context || !context.apis) {
-        throw new Error('makeLogger expected terafoundation context as first arg');
-    }
-
-    if (!moduleName || !isString(moduleName)) {
-        throw new Error('makeLogger expected module name as second arg');
-    }
-
-    const exAPI = context.apis.executionContext;
-    if (exAPI && isFunction(exAPI.makeLogger)) {
-        return exAPI.makeLogger(moduleName, extra);
-    }
-
-    const defaultContext = {};
-    if (process.env.EX) {
-        const ex = safeDecode(process.env.EX);
-        const exId = get(ex, 'ex_id');
-        const jobId = get(ex, 'job_id');
-        if (exId) {
-            defaultContext.ex_id = exId;
-        }
-        if (jobId) {
-            defaultContext.job_id = jobId;
-        }
-    }
-
-    return makeContextLogger(context, moduleName, Object.assign(defaultContext, extra));
-}
-
-module.exports = { generateWorkerId, makeLogger };

package/lib/workers/helpers/worker-shutdown.js

@@ -1,187 +0,0 @@
-'use strict';
-
-const {
-    get, pDelay, pRaceWithTimeout, isError, logError
-} = require('@terascope/utils');
-const ms = require('ms');
-const { makeLogger } = require('./terafoundation');
-
-function waitForWorkerShutdown(context, eventName) {
-    const shutdownTimeout = get(context, 'sysconfig.teraslice.shutdown_timeout', 30000);
-    const events = context.apis.foundation.getSystemEvents();
-
-    return new Promise((resolve, reject) => {
-        const timeoutId = setTimeout(() => {
-            events.removeListener(eventName, handler);
-            reject(new Error('Timeout waiting for worker to shutdown'));
-        }, shutdownTimeout);
-
-        function handler(err) {
-            clearTimeout(timeoutId);
-            if (isError(err)) {
-                reject(err);
-            } else {
-                resolve();
-            }
-        }
-        events.once(eventName, handler);
-    });
-}
-
-/* istanbul ignore next */
-function shutdownHandler(context, shutdownFn) {
-    const assignment = context.assignment
-        || process.env.NODE_TYPE
-        || process.env.assignment
-        || 'unknown-assignment';
-
-    const isK8s = get(context, 'sysconfig.teraslice.cluster_manager_type') === 'kubernetes';
-    // this is native clustering only
-    const isProcessRestart = process.env.process_restart;
-    // everything but the k8s execution_controller should not be allowed be allowed to
-    // set a non-zero exit code (to avoid being restarted)
-    const allowNonZeroExitCode = !(isK8s && assignment === 'execution_controller');
-    const api = {
-        exiting: false,
-        exit
-    };
-
-    const shutdownTimeout = get(context, 'sysconfig.teraslice.shutdown_timeout', 20 * 1000);
-
-    const events = context.apis.foundation.getSystemEvents();
-    const logger = makeLogger(context, `${assignment}:shutdown_handler`);
-
-    if (assignment === 'execution_controller' && isProcessRestart) {
-        logger.fatal(
-            'Execution Controller runtime error led to a restart, terminating execution with failed status, please use the recover api to return slicer to a consistent state'
-        );
-        process.exit(0);
-    }
-
-    async function flushLogs() {
-        try {
-            await logger.flush();
-            await pDelay(1000);
-
-            const code = process.exitCode || 0;
-            logger.debug(`flushed logs successfully, will exit with code ${code}`);
-        } catch (err) {
-            logError(logger, err, 'flush error on shutdown');
-        }
-    }
-
-    let startTime;
-
-    function exitingIn() {
-        if (!api.exiting) {
-            return `exiting in ${ms(shutdownTimeout)}...`;
-        }
-
-        const elapsed = Date.now() - startTime;
-        return `already shutting down, remaining ${ms(shutdownTimeout - elapsed)}`;
-    }
-
-    async function callShutdownFn(event, err) {
-        // avoid failing before the promse is try / catched in pRaceWithTimeout
-        await pDelay(100);
-        await shutdownFn(event, err);
-    }
-
-    async function shutdownWithTimeout(event, err) {
-        const timeout = shutdownTimeout - 2000;
-        await pRaceWithTimeout(callShutdownFn(event, err), timeout, (timeoutErr) => {
-            logError(logger, timeoutErr, 'shutdown error after timeout');
-        });
-    }
-
-    async function exit(event, err) {
-        if (api.exiting) return;
-
-        api.exiting = true;
-        startTime = Date.now();
-
-        try {
-            await shutdownWithTimeout(event, err);
-        } catch (error) {
-            logError(logger, error, `${assignment} while shutting down`);
-        } finally {
-            await flushLogs();
-            if (allowNonZeroExitCode) {
-                const code = process.exitCode != null ? process.exitCode : 0;
-                logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with ${code} status code`);
-                process.exit();
-            } else {
-                logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with zero status code`);
-                process.exit(0);
-            }
-        }
-    }
-
-    function setStatusCode(code) {
-        if (api.exiting) return;
-        if (process.exitCode == null) {
-            process.exitCode = code;
-        }
-    }
-
-    process.on('SIGINT', () => {
-        logger.info(`${assignment} received process:SIGINT, ${exitingIn()}`);
-        setStatusCode(0);
-        exit('SIGINT');
-    });
-
-    process.on('SIGTERM', () => {
-        logger.info(`${assignment} received process:SIGTERM, ${exitingIn()}`);
-        setStatusCode(0);
-        exit('SIGTERM');
-    });
-
-    process.on('uncaughtException', (err) => {
-        logError(logger, err, `${assignment} received an uncaughtException, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('uncaughtException', err);
-    });
-
-    process.once('unhandledRejection', (err) => {
-        logError(logger, err, `${assignment} received an unhandledRejection, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('unhandledRejection', err);
-    });
-
-    // See https://github.com/trentm/node-bunyan/issues/246
-    function handleStdError(err) {
-        if (err.code === 'EPIPE' || err.code === 'ERR_STREAM_DESTROYED') {
-            // ignore
-        } else {
-            throw err;
-        }
-    }
-
-    process.stdout.on('error', handleStdError);
-    process.stderr.on('error', handleStdError);
-
-    // event is fired from terafoundation when an error occurs during instantiation of a client
-    // **DEPRECATED:** This handler should be removed on teraslice v1
-    events.once('client:initialization:error', (err) => {
-        logError(logger, err, `${assignment} received a client initialization error, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('client:initialization:error', err);
-    });
-
-    events.once('worker:shutdown:complete', (err) => {
-        setStatusCode(0);
-        if (err) {
-            logError(logger, err, `${assignment} shutdown error, ${exitingIn()}`);
-        } else {
-            logger.info(`${assignment} shutdown, ${exitingIn()}`);
-        }
-        exit('worker:shutdown:complete', err);
-    });
-
-    return api;
-}
-
-module.exports = {
-    shutdownHandler,
-    waitForWorkerShutdown
-};

package/lib/workers/metrics/index.js

@@ -1,139 +0,0 @@
-/* eslint-disable no-console */
-
-'use strict';
-
-const { EventEmitter } = require('events');
-const { pDelay, debugLogger, isTest } = require('@terascope/utils');
-
-const _logger = debugLogger('metrics');
-
-class Metrics extends EventEmitter {
-    constructor({ logger } = {}) {
-        super();
-
-        this._logger = logger
-            ? logger.child({
-                module: 'performance:metrics'
-            })
-            : _logger;
-
-        this.eventLoopInterval = isTest ? 100 : 5000;
-        this._intervals = [];
-        this._typesCollectedAt = {};
-
-        // never cause an unwanted error
-        try {
-            this.gcStats = require('gc-stats')();
-        } catch (err) {
-            this._logger.error(err, 'Failure to construct gc-stats');
-        }
-
-        try {
-            this.eventLoopStats = require('event-loop-stats');
-        } catch (err) {
-            this._logger.error(err, 'Failure to construct event-loop-stats');
-        }
-    }
-
-    async initialize() {
-        const gcEnabled = this.gcStats != null;
-        const loopEnabled = this.eventLoopStats != null;
-
-        this._logger.info('initializing performance metrics', {
-            gcEnabled,
-            loopEnabled
-        });
-
-        if (gcEnabled) {
-            // https://github.com/dainis/node-gcstats#property-insights
-            const typesToName = {
-                1: 'Scavenge',
-                2: 'MarkSweepCompact',
-                4: 'IncrementalMarking',
-                8: 'WeakPhantomCallbackProcessing',
-                15: 'All'
-            };
-
-            this.gcStats.on('stats', (metrics) => {
-                // never cause an unwanted error
-                if (!metrics) {
-                    this._logger.warn('invalid metrics received for gc stats', metrics);
-                    return;
-                }
-
-                const typeName = typesToName[metrics.gctype];
-                this._emitMetric('gc-stats', Object.assign({ typeName }, metrics));
-            });
-        }
-
-        if (loopEnabled) {
-            // https://github.com/bripkens/event-loop-stats#property-insights
-            this._intervals.push(
-                setInterval(() => {
-                    // never cause an unwanted error
-                    let metrics;
-                    try {
-                        metrics = this.eventLoopStats.sense();
-                    } catch (err) {
-                        this._logger.error(err, 'failure getting for event-loop-stats');
-                        return;
-                    }
-
-                    if (!metrics) {
-                        this._logger.warn('invalid metrics received for event-loop-stats', metrics);
-                        return;
-                    }
-
-                    this._emitMetric('event-loop-stats', metrics);
-                }, this.eventLoopInterval)
-            );
-        }
-    }
-
-    async shutdown() {
-        try {
-            this._intervals.forEach(clearInterval);
-            this._intervals = [];
-            this.removeAllListeners();
-            this._typesCollectedAt = {};
-        } catch (err) {
-            this.logger.debug(err, 'failure shutting down metrics');
-        }
-    }
-
-    _emitMetric(type, metrics) {
-        const lastCollectedAt = this._typesCollectedAt[type] || Date.now();
-        const msg = {
-            type,
-            timeSinceLast: Date.now() - lastCollectedAt,
-            metrics
-        };
-        this._typesCollectedAt[type] = Date.now();
-        this.emit('metric', msg);
-        this._logger.info(msg, `${type} performance metrics`);
-    }
-}
-
-if (require.main === module) {
-    (async () => {
-        const metrics = new Metrics();
-        metrics.on('metric', (metric) => {
-            console.dir(metric);
-        });
-
-        try {
-            await metrics.initialize();
-            console.log('staying alive for 30s...');
-            await pDelay(30000);
-            console.log('done, exiting...');
-        } catch (err) {
-            console.error(err);
-            process.exitCode = 1;
-        } finally {
-            await metrics.shutdown();
-            process.exit();
-        }
-    })();
-} else {
-    module.exports = Metrics;
-}