teraslice 3.3.0 → 3.3.2

package/dist/src/lib/workers/helpers/job.js

@@ -1,80 +0,0 @@
-import { makeISODate } from '@terascope/core-utils';
-import { JobValidator } from '@terascope/job-components';
-import { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
-// TODO: fix type here
-export async function validateJob(context, jobSpec) {
-    const jobValidator = new JobValidator(context);
-    try {
-        return await jobValidator.validateConfig(jobSpec);
-    }
-    catch (error) {
-        throw new Error(`validating job: ${error}`);
-    }
-}
-export async function initializeTestExecution({ context, config, stores = {}, isRecovery = false, cleanupType, createRecovery = true, shutdownStores = false, recoverySlices = [], lastStatus = 'failed' }) {
-    if (!stores.jobStore) {
-        const [jobStore, exStore, stateStore] = await Promise.all([
-            (async () => {
-                const store = new JobsStorage(context);
-                await store.initialize();
-                return store;
-            })(),
-            (async () => {
-                const store = new ExecutionStorage(context);
-                await store.initialize();
-                return store;
-            })(),
-            (async () => {
-                const store = new StateStorage(context);
-                await store.initialize();
-                return store;
-            })(),
-        ]);
-        stores.jobStore = jobStore;
-        stores.exStore = exStore;
-        stores.stateStore = stateStore;
-    }
-    const validJob = await validateJob(context, config);
-    const jobSpec = await stores.jobStore.create(config);
-    const job = Object.assign({}, jobSpec, validJob, {
-        job_id: jobSpec.job_id
-    });
-    // @ts-expect-error
-    const slicerHostname = job.slicer_hostname;
-    // @ts-expect-error
-    const slicerPort = job.slicer_port;
-    let ex;
-    if (isRecovery) {
-        ex = await stores.exStore.create(job, lastStatus);
-        if (recoverySlices.length) {
-            await Promise.all(recoverySlices
-                .map(({ slice, state }) => stores.stateStore.createState(ex.ex_id, slice, state, slice.error)));
-            await stores.stateStore.refresh();
-        }
-        if (createRecovery) {
-            ex = await stores.exStore.createRecoveredExecution(ex, cleanupType);
-        }
-    }
-    else {
-        ex = await stores.exStore.create(job);
-    }
-    if (slicerHostname && slicerPort) {
-        ex = await stores.exStore.updatePartial(ex.ex_id, async (existing) => Object.assign(existing, {
-            slicer_hostname: slicerHostname,
-            slicer_port: slicerPort,
-            _updated: makeISODate()
-        }));
-    }
-    if (shutdownStores) {
-        await Promise.all([
-            stores.exStore.shutdown(true),
-            stores.jobStore.shutdown(true),
-            stores.stateStore.shutdown(true)
-        ]);
-    }
-    return {
-        job,
-        ex,
-    };
-}
-//# sourceMappingURL=job.js.map

package/dist/src/lib/workers/helpers/op-analytics.js

@@ -1,22 +0,0 @@
-import { isString, isArray, truncate, } from '@terascope/core-utils';
-function formatVal(value) {
-    if (isString(value))
-        return `"${value}"`;
-    if (isArray(value))
-        return `[${value.join(', ')}]`;
-    return truncate(JSON.stringify(value), 30);
-}
-function format(input) {
-    const list = [];
-    for (const [key, value] of Object.entries(input)) {
-        list.push(`${key}: ${formatVal(value)}`);
-    }
-    return list.join(', ');
-}
-// TODO: fix type here
-export function logOpStats(logger, slice, analyticsData) {
-    const { request, ...record } = slice;
-    const obj = Object.assign({}, record, analyticsData);
-    logger.info(`analytics for slice: ${format(obj)}`);
-}
-//# sourceMappingURL=op-analytics.js.map

package/dist/src/lib/workers/helpers/terafoundation.js

@@ -1,34 +0,0 @@
-import { get, isFunction, isString } from '@terascope/core-utils';
-import { makeContextLogger } from '@terascope/job-components';
-import { safeDecode } from '../../utils/encoding_utils.js';
-export function generateWorkerId(context) {
-    const { hostname } = context.sysconfig.teraslice;
-    const clusterId = get(context, 'cluster.worker.id');
-    return `${hostname}__${clusterId}`;
-}
-export function makeLogger(context, moduleName, extra = {}) {
-    if (!context || !context.apis) {
-        throw new Error('makeLogger expected terafoundation context as first arg');
-    }
-    if (!moduleName || !isString(moduleName)) {
-        throw new Error('makeLogger expected module name as second arg');
-    }
-    const exAPI = context.apis.executionContext;
-    if (exAPI && isFunction(exAPI.makeLogger)) {
-        return exAPI.makeLogger(moduleName, extra);
-    }
-    const defaultContext = {};
-    if (process.env.EX) {
-        const ex = safeDecode(process.env.EX);
-        const exId = get(ex, 'ex_id');
-        const jobId = get(ex, 'job_id');
-        if (exId) {
-            defaultContext.ex_id = exId;
-        }
-        if (jobId) {
-            defaultContext.job_id = jobId;
-        }
-    }
-    return makeContextLogger(context, moduleName, Object.assign(defaultContext, extra));
-}
-//# sourceMappingURL=terafoundation.js.map

package/dist/src/lib/workers/helpers/worker-shutdown.js

@@ -1,147 +0,0 @@
-import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/core-utils';
-import ms from 'ms';
-import { makeLogger } from './terafoundation.js';
-export function waitForWorkerShutdown(context, eventName) {
-    const shutdownTimeout = get(context, 'sysconfig.teraslice.shutdown_timeout', 30000);
-    const events = context.apis.foundation.getSystemEvents();
-    return new Promise((resolve, reject) => {
-        const timeoutId = setTimeout(() => {
-            events.removeListener(eventName, handler);
-            reject(new Error('Timeout waiting for worker to shutdown'));
-        }, shutdownTimeout);
-        function handler(err) {
-            clearTimeout(timeoutId);
-            if (isError(err)) {
-                reject(err);
-            }
-            else {
-                resolve(true);
-            }
-        }
-        events.once(eventName, handler);
-    });
-}
-/* istanbul ignore next */
-export function shutdownHandler(context, shutdownFn) {
-    const assignment = context.assignment
-        || process.env.NODE_TYPE
-        || process.env.assignment
-        || 'unknown-assignment';
-    // this is native clustering only
-    const isProcessRestart = process.env.__process_restart;
-    const api = {
-        exiting: false,
-        exit
-    };
-    const shutdownTimeout = get(context, 'sysconfig.teraslice.shutdown_timeout', 20 * 1000);
-    const events = context.apis.foundation.getSystemEvents();
-    const logger = makeLogger(context, `${assignment}:shutdown_handler`);
-    if (assignment === 'execution_controller' && isProcessRestart) {
-        logger.fatal('Execution Controller runtime error led to a restart, terminating execution with failed status, please use the recover api to return slicer to a consistent state');
-        process.exit(0);
-    }
-    async function flushLogs() {
-        try {
-            await logger.flush();
-            await pDelay(1000);
-            const code = process.exitCode || 0;
-            logger.debug(`flushed logs successfully, will exit with code ${code}`);
-        }
-        catch (err) {
-            logError(logger, err, 'flush error on shutdown');
-        }
-    }
-    let startTime;
-    function exitingIn() {
-        if (!api.exiting) {
-            return `exiting in ${ms(shutdownTimeout)}...`;
-        }
-        const elapsed = Date.now() - startTime;
-        return `already shutting down, remaining ${ms(shutdownTimeout - elapsed)}`;
-    }
-    async function callShutdownFn(event, err) {
-        // avoid failing before the promise is try / catched in pRaceWithTimeout
-        await pDelay(100);
-        await shutdownFn(event, err);
-    }
-    async function shutdownWithTimeout(event, err) {
-        const timeout = shutdownTimeout - 2000;
-        await pRaceWithTimeout(callShutdownFn(event, err), timeout, (timeoutErr) => {
-            logError(logger, timeoutErr, 'shutdown error after timeout');
-        });
-    }
-    async function exit(event, err) {
-        if (api.exiting)
-            return;
-        if (err) {
-            if (err.name.includes('Error')) {
-                setStatusCode(1);
-            }
-        }
-        api.exiting = true;
-        startTime = Date.now();
-        try {
-            await shutdownWithTimeout(event, err);
-        }
-        catch (error) {
-            logError(logger, error, `${assignment} while shutting down`);
-        }
-        finally {
-            await flushLogs();
-            const code = process.exitCode != null ? process.exitCode : 0;
-            logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with ${code} status code`);
-            process.exit();
-        }
-    }
-    function setStatusCode(code) {
-        if (api.exiting)
-            return;
-        if (process.exitCode == null) {
-            process.exitCode = code;
-        }
-    }
-    process.on('SIGINT', () => {
-        logger.info(`${assignment} received process:SIGINT, ${exitingIn()}`);
-        setStatusCode(0);
-        exit('SIGINT');
-    });
-    process.on('SIGTERM', () => {
-        logger.info(`${assignment} received process:SIGTERM, ${exitingIn()}`);
-        setStatusCode(0);
-        exit('SIGTERM');
-    });
-    process.on('uncaughtException', (err) => {
-        logError(logger, err, `${assignment} received an uncaughtException, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('uncaughtException', err);
-    });
-    process.once('unhandledRejection', (err) => {
-        logError(logger, err, `${assignment} received an unhandledRejection, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('unhandledRejection', err);
-    });
-    // See https://github.com/trentm/node-bunyan/issues/246
-    function handleStdError(err) {
-        // @ts-expect-error  TODO: fix error type
-        if (err.code === 'EPIPE' || err.code === 'ERR_STREAM_DESTROYED') {
-            // ignore
-        }
-        else {
-            throw err;
-        }
-    }
-    process.stdout.on('error', handleStdError);
-    process.stderr.on('error', handleStdError);
-    events.once('worker:shutdown:complete', (err) => {
-        setStatusCode(0);
-        if (err) {
-            logError(logger, err, `${assignment} shutdown error, ${exitingIn()}`);
-        }
-        else {
-            logger.info(`${assignment} shutdown, ${exitingIn()}`);
-        }
-        exit('worker:shutdown:complete', err);
-    });
-    return api;
-}
-//# sourceMappingURL=worker-shutdown.js.map

package/dist/src/lib/workers/metrics/index.js

@@ -1,108 +0,0 @@
-import { EventEmitter } from 'node:events';
-import { debugLogger, isTest } from '@terascope/core-utils';
-const defaultLogger = debugLogger('metrics');
-export class Metrics extends EventEmitter {
-    logger;
-    _intervals;
-    eventLoopInterval;
-    // TODO: fix types here
-    _typesCollectedAt;
-    gcStats;
-    eventLoopStats;
-    constructor(config) {
-        super();
-        const argLogger = config && config.logger;
-        this.logger = argLogger
-            ? argLogger.child({
-                module: 'performance:metrics'
-            })
-            : defaultLogger;
-        this.eventLoopInterval = isTest ? 100 : 5000;
-        this._intervals = [];
-        this._typesCollectedAt = {};
-        this.gcStats = null;
-    }
-    async initialize() {
-        // never cause an unwanted error
-        try {
-            const module = await import('gc-stats');
-            this.gcStats = module.default();
-        }
-        catch (err) {
-            this.logger.error(err, 'Failure to construct gc-stats');
-        }
-        try {
-            this.eventLoopStats = await import('event-loop-stats');
-        }
-        catch (err) {
-            this.logger.error(err, 'Failure to construct event-loop-stats');
-        }
-        const gcEnabled = this.gcStats != null;
-        const loopEnabled = this.eventLoopStats != null;
-        this.logger.info('initializing performance metrics', {
-            gcEnabled,
-            loopEnabled
-        });
-        if (this.gcStats !== null) {
-            // https://github.com/dainis/node-gcstats#property-insights
-            const typesToName = {
-                1: 'Scavenge',
-                2: 'MarkSweepCompact',
-                4: 'IncrementalMarking',
-                8: 'WeakPhantomCallbackProcessing',
-                15: 'All'
-            };
-            this.gcStats.on('stats', (metrics) => {
-                // never cause an unwanted error
-                if (!metrics) {
-                    this.logger.warn('invalid metrics received for gc stats', metrics);
-                    return;
-                }
-                const typeName = typesToName[metrics.gctype];
-                this._emitMetric('gc-stats', Object.assign({ typeName }, metrics));
-            });
-        }
-        if (loopEnabled) {
-            // https://github.com/bripkens/event-loop-stats#property-insights
-            this._intervals.push(setInterval(() => {
-                // never cause an unwanted error
-                let metrics;
-                try {
-                    metrics = this.eventLoopStats.sense();
-                }
-                catch (err) {
-                    this.logger.error(err, 'failure getting for event-loop-stats');
-                    return;
-                }
-                if (!metrics) {
-                    this.logger.warn('invalid metrics received for event-loop-stats', metrics);
-                    return;
-                }
-                this._emitMetric('event-loop-stats', metrics);
-            }, this.eventLoopInterval));
-        }
-    }
-    async shutdown() {
-        try {
-            this._intervals.forEach(clearInterval);
-            this._intervals = [];
-            this.removeAllListeners();
-            this._typesCollectedAt = {};
-        }
-        catch (err) {
-            this.logger.debug(err, 'failure shutting down metrics');
-        }
-    }
-    _emitMetric(type, metrics) {
-        const lastCollectedAt = this._typesCollectedAt[type] || Date.now();
-        const msg = {
-            type,
-            timeSinceLast: Date.now() - lastCollectedAt,
-            metrics
-        };
-        this._typesCollectedAt[type] = Date.now();
-        this.emit('metric', msg);
-        this.logger.info(msg, `${type} performance metrics`);
-    }
-}
-//# sourceMappingURL=index.js.map