teraslice 2.10.0 → 2.12.0

package/dist/src/lib/workers/helpers/job.js

@@ -0,0 +1,80 @@
+import { makeISODate } from '@terascope/utils';
+import { JobValidator } from '@terascope/job-components';
+import { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
+// TODO: fix type here
+export async function validateJob(context, jobSpec) {
+    const jobValidator = new JobValidator(context);
+    try {
+        return await jobValidator.validateConfig(jobSpec);
+    }
+    catch (error) {
+        throw new Error(`validating job: ${error}`);
+    }
+}
+export async function initializeTestExecution({ context, config, stores = {}, isRecovery = false, cleanupType, createRecovery = true, shutdownStores = false, recoverySlices = [], lastStatus = 'failed' }) {
+    if (!stores.jobStore) {
+        const [jobStore, exStore, stateStore] = await Promise.all([
+            (async () => {
+                const store = new JobsStorage(context);
+                await store.initialize();
+                return store;
+            })(),
+            (async () => {
+                const store = new ExecutionStorage(context);
+                await store.initialize();
+                return store;
+            })(),
+            (async () => {
+                const store = new StateStorage(context);
+                await store.initialize();
+                return store;
+            })(),
+        ]);
+        stores.jobStore = jobStore;
+        stores.exStore = exStore;
+        stores.stateStore = stateStore;
+    }
+    const validJob = await validateJob(context, config);
+    const jobSpec = await stores.jobStore.create(config);
+    const job = Object.assign({}, jobSpec, validJob, {
+        job_id: jobSpec.job_id
+    });
+    // @ts-expect-error
+    const slicerHostname = job.slicer_hostname;
+    // @ts-expect-error
+    const slicerPort = job.slicer_port;
+    let ex;
+    if (isRecovery) {
+        ex = await stores.exStore.create(job, lastStatus);
+        if (recoverySlices.length) {
+            await Promise.all(recoverySlices
+                .map(({ slice, state }) => stores.stateStore.createState(ex.ex_id, slice, state, slice.error)));
+            await stores.stateStore.refresh();
+        }
+        if (createRecovery) {
+            ex = await stores.exStore.createRecoveredExecution(ex, cleanupType);
+        }
+    }
+    else {
+        ex = await stores.exStore.create(job);
+    }
+    if (slicerHostname && slicerPort) {
+        ex = await stores.exStore.updatePartial(ex.ex_id, async (existing) => Object.assign(existing, {
+            slicer_hostname: slicerHostname,
+            slicer_port: slicerPort,
+            _updated: makeISODate()
+        }));
+    }
+    if (shutdownStores) {
+        await Promise.all([
+            stores.exStore.shutdown(true),
+            stores.jobStore.shutdown(true),
+            stores.stateStore.shutdown(true)
+        ]);
+    }
+    return {
+        job,
+        ex,
+    };
+}
+//# sourceMappingURL=job.js.map

package/dist/src/lib/workers/helpers/op-analytics.js

@@ -0,0 +1,22 @@
+import { isString, isArray, truncate, } from '@terascope/utils';
+function formatVal(value) {
+    if (isString(value))
+        return `"${value}"`;
+    if (isArray(value))
+        return `[${value.join(', ')}]`;
+    return truncate(JSON.stringify(value), 30);
+}
+function format(input) {
+    const list = [];
+    for (const [key, value] of Object.entries(input)) {
+        list.push(`${key}: ${formatVal(value)}`);
+    }
+    return list.join(', ');
+}
+// TODO: fix type here
+export function logOpStats(logger, slice, analyticsData) {
+    const { request, ...record } = slice;
+    const obj = Object.assign({}, record, analyticsData);
+    logger.info(`analytics for slice: ${format(obj)}`);
+}
+//# sourceMappingURL=op-analytics.js.map

package/dist/src/lib/workers/helpers/terafoundation.js

@@ -0,0 +1,34 @@
+import { get, isFunction, isString } from '@terascope/utils';
+import { makeContextLogger } from '@terascope/job-components';
+import { safeDecode } from '../../utils/encoding_utils.js';
+export function generateWorkerId(context) {
+    const { hostname } = context.sysconfig.teraslice;
+    const clusterId = get(context, 'cluster.worker.id');
+    return `${hostname}__${clusterId}`;
+}
+export function makeLogger(context, moduleName, extra = {}) {
+    if (!context || !context.apis) {
+        throw new Error('makeLogger expected terafoundation context as first arg');
+    }
+    if (!moduleName || !isString(moduleName)) {
+        throw new Error('makeLogger expected module name as second arg');
+    }
+    const exAPI = context.apis.executionContext;
+    if (exAPI && isFunction(exAPI.makeLogger)) {
+        return exAPI.makeLogger(moduleName, extra);
+    }
+    const defaultContext = {};
+    if (process.env.EX) {
+        const ex = safeDecode(process.env.EX);
+        const exId = get(ex, 'ex_id');
+        const jobId = get(ex, 'job_id');
+        if (exId) {
+            defaultContext.ex_id = exId;
+        }
+        if (jobId) {
+            defaultContext.job_id = jobId;
+        }
+    }
+    return makeContextLogger(context, moduleName, Object.assign(defaultContext, extra));
+}
+//# sourceMappingURL=terafoundation.js.map

package/dist/src/lib/workers/helpers/worker-shutdown.js

@@ -0,0 +1,169 @@
+import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/utils';
+import ms from 'ms';
+import { makeLogger } from './terafoundation.js';
+export function waitForWorkerShutdown(context, eventName) {
+    const shutdownTimeout = get(context, 'sysconfig.teraslice.shutdown_timeout', 30000);
+    const events = context.apis.foundation.getSystemEvents();
+    return new Promise((resolve, reject) => {
+        const timeoutId = setTimeout(() => {
+            events.removeListener(eventName, handler);
+            reject(new Error('Timeout waiting for worker to shutdown'));
+        }, shutdownTimeout);
+        function handler(err) {
+            clearTimeout(timeoutId);
+            if (isError(err)) {
+                reject(err);
+            }
+            else {
+                resolve(true);
+            }
+        }
+        events.once(eventName, handler);
+    });
+}
+/* istanbul ignore next */
+export function shutdownHandler(context, shutdownFn) {
+    const assignment = context.assignment
+        || process.env.NODE_TYPE
+        || process.env.assignment
+        || 'unknown-assignment';
+    const clusteringType = get(context, 'sysconfig.teraslice.cluster_manager_type');
+    const isK8s = clusteringType === 'kubernetes' || clusteringType === 'kubernetesV2';
+    // this is native clustering only
+    const isProcessRestart = process.env.process_restart;
+    // everything but the k8s execution_controller should not be allowed be allowed to
+    // set a non-zero exit code (to avoid being restarted)
+    // This is overridden in V2 because it can restart
+    const allowNonZeroExitCode = !(isK8s
+        && assignment === 'execution_controller'
+        && context.sysconfig.teraslice.cluster_manager_type === 'kubernetes');
+    const api = {
+        exiting: false,
+        exit
+    };
+    const shutdownTimeout = get(context, 'sysconfig.teraslice.shutdown_timeout', 20 * 1000);
+    const events = context.apis.foundation.getSystemEvents();
+    const logger = makeLogger(context, `${assignment}:shutdown_handler`);
+    if (assignment === 'execution_controller' && isProcessRestart) {
+        logger.fatal('Execution Controller runtime error led to a restart, terminating execution with failed status, please use the recover api to return slicer to a consistent state');
+        process.exit(0);
+    }
+    async function flushLogs() {
+        try {
+            await logger.flush();
+            await pDelay(1000);
+            const code = process.exitCode || 0;
+            logger.debug(`flushed logs successfully, will exit with code ${code}`);
+        }
+        catch (err) {
+            logError(logger, err, 'flush error on shutdown');
+        }
+    }
+    let startTime;
+    function exitingIn() {
+        if (!api.exiting) {
+            return `exiting in ${ms(shutdownTimeout)}...`;
+        }
+        const elapsed = Date.now() - startTime;
+        return `already shutting down, remaining ${ms(shutdownTimeout - elapsed)}`;
+    }
+    async function callShutdownFn(event, err) {
+        // avoid failing before the promse is try / catched in pRaceWithTimeout
+        await pDelay(100);
+        await shutdownFn(event, err);
+    }
+    async function shutdownWithTimeout(event, err) {
+        const timeout = shutdownTimeout - 2000;
+        await pRaceWithTimeout(callShutdownFn(event, err), timeout, (timeoutErr) => {
+            logError(logger, timeoutErr, 'shutdown error after timeout');
+        });
+    }
+    async function exit(event, err) {
+        if (api.exiting)
+            return;
+        /// Potential logic for cluster_master and asset_service
+        if (err) {
+            if (err.name.includes('Error')) {
+                setStatusCode(1);
+            }
+        }
+        api.exiting = true;
+        startTime = Date.now();
+        try {
+            await shutdownWithTimeout(event, err);
+        }
+        catch (error) {
+            logError(logger, error, `${assignment} while shutting down`);
+        }
+        finally {
+            await flushLogs();
+            if (allowNonZeroExitCode) {
+                const code = process.exitCode != null ? process.exitCode : 0;
+                logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with ${code} status code`);
+                process.exit();
+            }
+            else {
+                logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with zero status code`);
+                process.exit(0);
+            }
+        }
+    }
+    function setStatusCode(code) {
+        if (api.exiting)
+            return;
+        if (process.exitCode == null) {
+            process.exitCode = code;
+        }
+    }
+    process.on('SIGINT', () => {
+        logger.info(`${assignment} received process:SIGINT, ${exitingIn()}`);
+        setStatusCode(0);
+        exit('SIGINT');
+    });
+    process.on('SIGTERM', () => {
+        logger.info(`${assignment} received process:SIGTERM, ${exitingIn()}`);
+        setStatusCode(0);
+        exit('SIGTERM');
+    });
+    process.on('uncaughtException', (err) => {
+        logError(logger, err, `${assignment} received an uncaughtException, ${exitingIn()}`);
+        setStatusCode(1);
+        exit('uncaughtException', err);
+    });
+    process.once('unhandledRejection', (err) => {
+        logError(logger, err, `${assignment} received an unhandledRejection, ${exitingIn()}`);
+        setStatusCode(1);
+        exit('unhandledRejection', err);
+    });
+    // See https://github.com/trentm/node-bunyan/issues/246
+    function handleStdError(err) {
+        // @ts-expect-error  TODO: fix error type
+        if (err.code === 'EPIPE' || err.code === 'ERR_STREAM_DESTROYED') {
+            // ignore
+        }
+        else {
+            throw err;
+        }
+    }
+    process.stdout.on('error', handleStdError);
+    process.stderr.on('error', handleStdError);
+    // event is fired from terafoundation when an error occurs during instantiation of a client
+    // **DEPRECATED:** This handler should be removed on teraslice v1
+    events.once('client:initialization:error', (err) => {
+        logError(logger, err, `${assignment} received a client initialization error, ${exitingIn()}`);
+        setStatusCode(1);
+        exit('client:initialization:error', err);
+    });
+    events.once('worker:shutdown:complete', (err) => {
+        setStatusCode(0);
+        if (err) {
+            logError(logger, err, `${assignment} shutdown error, ${exitingIn()}`);
+        }
+        else {
+            logger.info(`${assignment} shutdown, ${exitingIn()}`);
+        }
+        exit('worker:shutdown:complete', err);
+    });
+    return api;
+}
+//# sourceMappingURL=worker-shutdown.js.map

package/dist/src/lib/workers/metrics/index.js

@@ -0,0 +1,108 @@
+import { EventEmitter } from 'node:events';
+import { debugLogger, isTest } from '@terascope/utils';
+const defaultLogger = debugLogger('metrics');
+export class Metrics extends EventEmitter {
+    logger;
+    _intervals;
+    eventLoopInterval;
+    // TODO: fix types here
+    _typesCollectedAt;
+    gcStats;
+    eventLoopStats;
+    constructor(config) {
+        super();
+        const argLogger = config && config.logger;
+        this.logger = argLogger
+            ? argLogger.child({
+                module: 'performance:metrics'
+            })
+            : defaultLogger;
+        this.eventLoopInterval = isTest ? 100 : 5000;
+        this._intervals = [];
+        this._typesCollectedAt = {};
+        this.gcStats = null;
+    }
+    async initialize() {
+        // never cause an unwanted error
+        try {
+            const module = await import('gc-stats');
+            this.gcStats = module.default();
+        }
+        catch (err) {
+            this.logger.error(err, 'Failure to construct gc-stats');
+        }
+        try {
+            this.eventLoopStats = await import('event-loop-stats');
+        }
+        catch (err) {
+            this.logger.error(err, 'Failure to construct event-loop-stats');
+        }
+        const gcEnabled = this.gcStats != null;
+        const loopEnabled = this.eventLoopStats != null;
+        this.logger.info('initializing performance metrics', {
+            gcEnabled,
+            loopEnabled
+        });
+        if (this.gcStats !== null) {
+            // https://github.com/dainis/node-gcstats#property-insights
+            const typesToName = {
+                1: 'Scavenge',
+                2: 'MarkSweepCompact',
+                4: 'IncrementalMarking',
+                8: 'WeakPhantomCallbackProcessing',
+                15: 'All'
+            };
+            this.gcStats.on('stats', (metrics) => {
+                // never cause an unwanted error
+                if (!metrics) {
+                    this.logger.warn('invalid metrics received for gc stats', metrics);
+                    return;
+                }
+                const typeName = typesToName[metrics.gctype];
+                this._emitMetric('gc-stats', Object.assign({ typeName }, metrics));
+            });
+        }
+        if (loopEnabled) {
+            // https://github.com/bripkens/event-loop-stats#property-insights
+            this._intervals.push(setInterval(() => {
+                // never cause an unwanted error
+                let metrics;
+                try {
+                    metrics = this.eventLoopStats.sense();
+                }
+                catch (err) {
+                    this.logger.error(err, 'failure getting for event-loop-stats');
+                    return;
+                }
+                if (!metrics) {
+                    this.logger.warn('invalid metrics received for event-loop-stats', metrics);
+                    return;
+                }
+                this._emitMetric('event-loop-stats', metrics);
+            }, this.eventLoopInterval));
+        }
+    }
+    async shutdown() {
+        try {
+            this._intervals.forEach(clearInterval);
+            this._intervals = [];
+            this.removeAllListeners();
+            this._typesCollectedAt = {};
+        }
+        catch (err) {
+            this.logger.debug(err, 'failure shutting down metrics');
+        }
+    }
+    _emitMetric(type, metrics) {
+        const lastCollectedAt = this._typesCollectedAt[type] || Date.now();
+        const msg = {
+            type,
+            timeSinceLast: Date.now() - lastCollectedAt,
+            metrics
+        };
+        this._typesCollectedAt[type] = Date.now();
+        this.emit('metric', msg);
+        this.logger.info(msg, `${type} performance metrics`);
+    }
+}
+//# sourceMappingURL=index.js.map