teraslice 0.87.1 → 0.88.0

package/lib/storage/state.js

@@ -1,302 +0,0 @@
-'use strict';
-
-const {
-    RecoveryCleanupType
-} = require('@terascope/job-components');
-const {
-    TSError,
-    pRetry,
-    toString,
-    isRetryableError,
-    parseErrorInfo,
-    isTest,
-    times,
-    getFullErrorStack,
-} = require('@terascope/utils');
-const { timeseriesIndex } = require('../utils/date_utils');
-const { makeLogger } = require('../workers/helpers/terafoundation');
-const elasticsearchBackend = require('./backends/elasticsearch_store');
-
-const SliceState = Object.freeze({
-    pending: 'pending',
-    start: 'start',
-    error: 'error',
-    completed: 'completed',
-});
-
-// Module to manager job states in Elasticsearch.
-// All functions in this module return promises that must be resolved to
-// get the final result.
-async function stateStorage(context) {
-    const recordType = 'state';
-
-    const logger = makeLogger(context, 'state_storage');
-    const config = context.sysconfig.teraslice;
-    const _index = `${config.name}__state`;
-    // making this to pass down to backend for dynamic index searches
-    const indexName = `${_index}*`;
-    const timeseriesFormat = config.index_rollover_frequency.state;
-
-    const backendConfig = {
-        context,
-        indexName,
-        recordType,
-        idField: 'slice_id',
-        fullResponse: false,
-        logRecord: true,
-        forceRefresh: false,
-        storageName: 'state'
-    };
-
-    const backend = await elasticsearchBackend(backendConfig);
-
-    async function createState(exId, slice, state, error) {
-        await waitForClient();
-
-        const { record, index } = _createSliceRecord(exId, slice, state, error);
-
-        return backend.indexWithId(slice.slice_id, record, index);
-    }
-
-    async function createSlices(exId, slices) {
-        await waitForClient();
-
-        const bulkRequest = slices.map((slice) => {
-            const { record, index } = _createSliceRecord(exId, slice, SliceState.pending);
-            return {
-                action: {
-                    index: {
-                        _index: index,
-                        _type: recordType,
-                        _id: record.slice_id,
-                    },
-                },
-                data: record
-            };
-        });
-
-        return backend.bulkSend(bulkRequest);
-    }
-
-    function _createSliceRecord(exId, slice, state, error) {
-        if (!SliceState[state]) {
-            throw new Error(`Unknown slice state "${state}" on create`);
-        }
-        const { index } = timeseriesIndex(timeseriesFormat, _index, slice._created);
-        const record = {
-            slice_id: slice.slice_id,
-            slicer_id: slice.slicer_id,
-            slicer_order: slice.slicer_order,
-            request: JSON.stringify(slice.request),
-            state,
-            ex_id: exId,
-            _created: slice._created,
-            _updated: slice._created,
-        };
-
-        if (error) {
-            record.error = toString(error);
-        }
-        return { record, index };
-    }
-
-    async function updateState(slice, state, error) {
-        if (!SliceState[state]) {
-            throw new Error(`Unknown slice state "${state}" on update`);
-        }
-
-        const indexData = timeseriesIndex(timeseriesFormat, _index, slice._created);
-        const record = {
-            _updated: indexData.timestamp,
-            state
-        };
-
-        // it will usaully just be error
-        if (state === SliceState.error || error) {
-            if (error) {
-                record.error = getFullErrorStack(error);
-            } else {
-                record.error = new Error('Unkown slice error').stack;
-            }
-        }
-
-        let notFoundErrCount = 0;
-
-        async function update() {
-            await waitForClient();
-
-            try {
-                return await backend.update(slice.slice_id, record, indexData.index);
-            } catch (_err) {
-                const { statusCode, message } = parseErrorInfo(_err);
-                let retryable = isRetryableError(_err);
-                if (statusCode === 404) {
-                    notFoundErrCount++;
-                    retryable = notFoundErrCount < 3;
-                } else if (message.includes('Request Timeout')) {
-                    retryable = true;
-                }
-
-                throw new TSError(_err, {
-                    retryable,
-                    reason: `Failure to update ${state} state`
-                });
-            }
-        }
-
-        return pRetry(update, {
-            retries: 10000,
-            delay: isTest ? 100 : 1000,
-            backoff: 5,
-            endWithFatal: true,
-        });
-    }
-
-    /**
-     * Get the starting position for the slicer
-     *
-     * @param {string} exId
-     * @param {number} slicerId
-     * @returns {Promise<import('@terascope/job-components').SlicerRecoveryData>}
-    */
-    async function _getSlicerStartingPoint(exId, slicerId) {
-        const startQuery = `ex_id:"${exId}" AND slicer_id:"${slicerId}" AND state:${SliceState.completed}`;
-        await waitForClient();
-
-        try {
-            const [slice] = await search(startQuery, 0, 1, 'slicer_order:desc');
-            const recoveryData = {
-                slicer_id: slicerId,
-                lastSlice: undefined
-            };
-
-            if (slice) {
-                recoveryData.lastSlice = JSON.parse(slice.request);
-                logger.info(`last slice process for slicer_id ${slicerId}, ex_id: ${exId} is`, slice.lastSlice);
-            }
-
-            return recoveryData;
-        } catch (err) {
-            throw new TSError(err, {
-                reason: 'Failure getting the newest record'
-            });
-        }
-    }
-
-    /**
-     * Get the starting positions for all of the slicers
-     *
-     * @param {string} exId
-     * @param {number} slicer
-     * @returns {Promise<import('@terascope/job-components').SlicerRecoveryData[]>}
-    */
-    async function getStartingPoints(exId, slicers) {
-        const recoveredSlices = times(slicers, (i) => _getSlicerStartingPoint(exId, i));
-        return Promise.all(recoveredSlices);
-    }
-
-    /**
-     * @private
-     * @param {string} exId
-     * @param {number} slicerId
-     * @param {import('@terascope/job-components').RecoveryCleanupType} [cleanupType]
-     * @returns {string}
-    */
-    function _getRecoverSlicesQuery(exId, slicerId, cleanupType) {
-        let query = `ex_id:"${exId}"`;
-        if (slicerId !== -1) {
-            query = `${query} AND slicer_id:"${slicerId}"`;
-        }
-
-        if (cleanupType && cleanupType === RecoveryCleanupType.errors) {
-            query = `${query} AND state:"${SliceState.error}"`;
-        } else if (cleanupType && cleanupType === RecoveryCleanupType.pending) {
-            query = `${query} AND state:"${SliceState.pending}"`;
-        } else {
-            query = `${query} AND NOT state:"${SliceState.completed}"`;
-        }
-        logger.debug('recovery slices query:', query);
-        return query;
-    }
-
-    /**
-     * @param {string} exId
-     * @param {number} slicerId
-     * @param {import('@terascope/job-components').RecoveryCleanupType} [cleanupType]
-     * @returns {Promise<import('@terascope/job-components').Slice[]>}
-    */
-    async function recoverSlices(exId, slicerId, cleanupType) {
-        const query = _getRecoverSlicesQuery(exId, slicerId, cleanupType);
-        // Look for all slices that haven't been completed so they can be retried.
-        try {
-            await waitForClient();
-            await backend.refresh(indexName);
-
-            const results = await search(query, 0, 5000, 'slicer_order:desc');
-            return results.map((doc) => ({
-                slice_id: doc.slice_id,
-                slicer_id: doc.slicer_id,
-                request: JSON.parse(doc.request),
-                _created: doc._created
-            }));
-        } catch (err) {
-            throw new TSError(err, {
-                reason: 'Failure to get recovered slices'
-            });
-        }
-    }
-
-    async function search(query, from, size, sort, fields) {
-        return backend.search(query, from, size, sort || '_updated:desc', fields);
-    }
-
-    async function count(query, from = 0, sort = '_updated:desc') {
-        return backend.count(query, from, sort);
-    }
-
-    async function countByState(exId, state) {
-        if (!SliceState[state]) {
-            throw new Error(`Unknown slice state "${state}" on update`);
-        }
-        const query = `ex_id:"${exId}" AND state:${state}`;
-        return count(query);
-    }
-
-    async function shutdown(forceShutdown) {
-        logger.info('shutting down');
-        return backend.shutdown(forceShutdown);
-    }
-
-    async function refresh() {
-        const { index } = timeseriesIndex(timeseriesFormat, _index);
-        return backend.refresh(index);
-    }
-
-    function verifyClient() {
-        return backend.verifyClient();
-    }
-
-    async function waitForClient() {
-        return backend.waitForClient();
-    }
-
-    logger.info('state storage initialized');
-    return {
-        search,
-        createState,
-        createSlices,
-        updateState,
-        recoverSlices,
-        getStartingPoints,
-        count,
-        countByState,
-        waitForClient,
-        verifyClient,
-        shutdown,
-        refresh,
-    };
-}
-
-stateStorage.SliceState = SliceState;
-
-module.exports = stateStorage;

package/lib/utils/api_utils.js

@@ -1,173 +0,0 @@
-'use strict';
-
-const Table = require('easy-table');
-const {
-    parseErrorInfo,
-    parseList,
-    logError,
-    isString,
-    get,
-    toInteger,
-} = require('@terascope/utils');
-
-function makeTable(req, defaults, data, mappingFn) {
-    const query = fieldsQuery(req.query, defaults);
-    let emptyChar = 'N/A';
-
-    // used to create an empty table if there are no jobs
-    if (data.length === 0) {
-        emptyChar = '';
-        data.push({});
-    }
-
-    return Table.print(data, (item, cell) => {
-        const fn = mappingFn ? mappingFn(item) : (field) => get(item, field, emptyChar);
-        query.forEach((field) => {
-            cell(field, fn(field));
-        });
-    }, (table) => {
-        if (('headers' in req.query) && req.query.headers === 'false') {
-            return table.print();
-        }
-        return table.toString();
-    });
-}
-
-function fieldsQuery(query, defaults) {
-    if (!query.fields) {
-        return defaults || [];
-    }
-
-    const results = parseList(query.fields);
-
-    if (results.length === 0) {
-        return defaults;
-    }
-
-    return results;
-}
-
-function handleRequest(req, res, defaultErrorMsg = 'Failure to process request', { errorCode = 500, successCode = 200 } = {}) {
-    logRequest(req);
-    return async (fn) => {
-        try {
-            const result = await fn();
-            if (isString(result)) {
-                res.status(successCode).send(result);
-            } else {
-                res.status(successCode).json(result);
-            }
-        } catch (err) {
-            const { statusCode, message } = parseErrorInfo(err, {
-                defaultErrorMsg,
-                defaultStatusCode: errorCode,
-            });
-
-            if (statusCode >= 500) {
-                logError(req.logger, err);
-            } else {
-                logError(req.logger, message);
-            }
-
-            sendError(res, statusCode, message, req.logger);
-        }
-    };
-}
-
-function sendError(res, code, message, logger) {
-    if (res.headersSent) {
-        const error = new Error(message);
-        error.statusCode = code;
-        if (logger) {
-            logger.error(error, 'request send error after headers sent');
-        } else {
-            console.error(error, 'request send error after headers sent');
-        }
-        return;
-    }
-    res.status(code).json({
-        error: code,
-        message
-    });
-}
-
-// NOTE: This only works for counters, if you're trying to extend this, you
-// should probably switch to using prom-client.
-function makePrometheus(stats, defaultLabels = {}) {
-    const metricMapping = {
-        processed: 'teraslice_slices_processed',
-        failed: 'teraslice_slices_failed',
-        queued: 'teraslice_slices_queued',
-        job_duration: '', // this isn't really useful, omitting
-        workers_joined: 'teraslice_workers_joined',
-        workers_disconnected: 'teraslice_workers_disconnected',
-        workers_reconnected: 'teraslice_workers_reconnected'
-    };
-
-    let returnString = '';
-    Object.entries(stats.controllers).forEach(([key, value]) => {
-        const name = metricMapping[key];
-        if (name !== '') {
-            returnString += `# TYPE ${name} counter\n`;
-            const labels = makePrometheusLabels(defaultLabels);
-            returnString += `${name}${labels} ${value}\n`;
-        }
-    });
-    return returnString;
-}
-
-function makePrometheusLabels(defaults, custom) {
-    const labels = Object.assign({}, defaults, custom);
-    const keys = Object.keys(labels);
-    if (!keys.length) return '';
-
-    const labelsStr = keys.map((key) => {
-        const val = labels[key];
-        return `${key}="${val}"`;
-    }).join(',');
-
-    return `{${labelsStr}}`;
-}
-
-function isPrometheusRequest(req) {
-    const acceptHeader = get(req, 'headers.accept', '');
-    return acceptHeader && acceptHeader.indexOf('application/openmetrics-text;') > -1;
-}
-
-/**
- * @returns {number}
-*/
-function parseQueryInt(req, prop, defaultVal) {
-    const val = req.query[prop];
-    if (val == null || val === '') return defaultVal;
-    const parsed = toInteger(val);
-    // allow the invalid prop to be passed through
-    // (because an error should thrown downstream)
-    if (parsed === false) return req.query[prop];
-    return parsed;
-}
-
-function getSearchOptions(req, defaultSort = '_updated:desc') {
-    const sort = req.query.sort || defaultSort;
-    const size = parseQueryInt(req, 'size', 100);
-    const from = parseQueryInt(req, 'from', 0);
-    return { size, from, sort };
-}
-
-function logRequest(req) {
-    const queryInfo = Object.entries(req.query)
-        .map(([key, val]) => `${key}: ${val}`)
-        .join(', ');
-    const { method, path } = req;
-    req.logger.trace(`${method.toUpperCase()} ${path} endpoint has been called, ${queryInfo}`);
-}
-
-module.exports = {
-    isPrometheusRequest,
-    makePrometheus,
-    makeTable,
-    logRequest,
-    getSearchOptions,
-    handleRequest,
-    sendError
-};

package/lib/utils/asset_utils.js

@@ -1,117 +0,0 @@
-'use strict';
-
-const {
-    isInteger, trimStart, trim, getFirst, joinList
-} = require('@terascope/utils');
-const semver = require('semver');
-
-function findMatchingAsset(records, name, version) {
-    const range = toSemverRange(version);
-    const assets = records
-        .filter(_isCompatibleAsset(name, range, false))
-        .sort((a, b) => semver.rcompare(a.version, b.version));
-
-    return getFirst(assets);
-}
-
-function findSimilarAssets(records, name, version) {
-    const range = toSemverRange(version);
-    const assets = records
-        .filter(_isCompatibleAsset(name, range, true))
-        .sort((a, b) => semver.rcompare(a.version, b.version));
-
-    return assets;
-}
-
-function getInCompatibilityReason(assets, prefix) {
-    if (!assets || !assets.length) return '';
-
-    const reasons = [];
-
-    assets.slice(0, 3).forEach((asset) => {
-        if (!isCompatibleNodeVersion(asset.node_version)) {
-            reasons.push('node_version');
-        }
-        if (asset.platform != null && asset.platform !== process.platform) {
-            reasons.push('platform');
-        }
-        if (asset.arch != null && asset.arch !== process.arch) {
-            reasons.push('arch');
-        }
-    });
-
-    if (!reasons.length) return '';
-
-    return `${prefix ? `${trim(prefix)} ` : ''}${joinList(reasons, ',', 'or')} mismatch`;
-}
-
-function getMajorVersion(version) {
-    if (version == null) return version;
-    if (isInteger(version)) return version;
-    return semver.major(version);
-}
-
-const SYSTEM_NODE_VERSION = getMajorVersion(process.version);
-/**
- * This just compares the major version
-*/
-function isCompatibleNodeVersion(version) {
-    if (version == null) return true;
-
-    // anything less than or equal to current node version
-    return getMajorVersion(version) <= SYSTEM_NODE_VERSION;
-}
-
-function _isCompatibleAsset(name, range, skipRestrictions = false) {
-    return (record) => {
-        if (record.name !== name) return false;
-        if (!semver.satisfies(record.version, range)) {
-            return false;
-        }
-        if (skipRestrictions) return true;
-
-        if (!isCompatibleNodeVersion(record.node_version)) {
-            return false;
-        }
-        if (record.arch != null && record.arch !== process.arch) {
-            return false;
-        }
-        if (record.platform != null && record.platform !== process.platform) {
-            return false;
-        }
-        return true;
-    };
-}
-
-function toSemverRange(version) {
-    if (!version || version === 'latest') return '*';
-    if (semver.validRange(version)) {
-        return trimStart(trim(version), 'v');
-    }
-
-    throw new Error(`Version "${version}" is not a valid semver range`);
-}
-
-function toVersionQuery(_version) {
-    const version = trimStart(trim(_version));
-
-    if (!version || version === 'latest' || version === '*') {
-        return 'version:*';
-    }
-
-    // if there a number and * next to each other that is not valid
-    // so lets just return the query: 12.34*.55
-    if (/\d\*/.test(version)) return `version:${version}`;
-
-    const range = new semver.Range(version);
-    return `version:${range.range.split(' ').join(' AND version:')}`;
-}
-
-module.exports = {
-    findSimilarAssets,
-    getInCompatibilityReason,
-    getMajorVersion,
-    findMatchingAsset,
-    toSemverRange,
-    toVersionQuery,
-};

package/lib/utils/date_utils.js

@@ -1,58 +0,0 @@
-'use strict';
-
-const { makeISODate } = require('@terascope/utils');
-
-function dateOptions(value) {
-    const options = {
-        year: 'y',
-        years: 'y',
-        y: 'y',
-        months: 'M',
-        month: 'M',
-        mo: 'M',
-        mos: 'M',
-        M: 'M',
-        weeks: 'w',
-        week: 'w',
-        wks: 'w',
-        wk: 'w',
-        w: 'w',
-        days: 'd',
-        day: 'd',
-        d: 'd',
-        hours: 'h',
-        hour: 'h',
-        hr: 'h',
-        hrs: 'h',
-        h: 'h',
-        minutes: 'm',
-        minute: 'm',
-        min: 'm',
-        mins: 'm',
-        m: 'm',
-        seconds: 's',
-        second: 's',
-        s: 's',
-        milliseconds: 'ms',
-        millisecond: 'ms',
-        ms: 'ms'
-    };
-
-    if (options[value]) {
-        return options[value];
-    }
-
-    throw new Error(`the time descriptor of "${value}" for the interval is malformed`);
-}
-
-function timeseriesIndex(timeseriesFormat, index, dateStr) {
-    const timestamp = makeISODate();
-    const formatter = { daily: 10, monthly: 7, yearly: 4 };
-    const dateString = dateStr || timestamp;
-    return { index: `${index}-${dateString.slice(0, formatter[timeseriesFormat]).replace(/-/g, '.')}`, timestamp };
-}
-
-module.exports = {
-    dateOptions,
-    timeseriesIndex
-};

package/lib/utils/encoding_utils.js

@@ -1,29 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-
-function safeEncode(obj) {
-    let str;
-    if (_.isString(obj)) {
-        str = obj;
-    } else if (_.isObjectLike(obj)) {
-        str = JSON.stringify(obj);
-    }
-    return Buffer.from(str).toString('base64');
-}
-
-function safeDecode(str) {
-    if (!_.isString(str) && _.isObjectLike(str)) {
-        return str;
-    }
-    try {
-        return JSON.parse(Buffer.from(str, 'base64').toString('utf-8'));
-    } catch (err) {
-        throw new Error(`Unable to decode ${str}`);
-    }
-}
-
-module.exports = {
-    safeEncode,
-    safeDecode
-};

package/lib/utils/events.js

@@ -1,7 +0,0 @@
-'use strict';
-
-const events = require('events');
-
-const eventEmitter = new events.EventEmitter();
-
-module.exports = eventEmitter;