teraslice 3.3.0 → 3.3.2

package/dist/src/lib/storage/state.js

@@ -1,254 +0,0 @@
-import { RecoveryCleanupType } from '@terascope/job-components';
-import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/core-utils';
-import { timeseriesIndex } from '../utils/date_utils.js';
-import { makeLogger } from '../workers/helpers/terafoundation.js';
-import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
-export const SliceState = Object.freeze({
-    pending: 'pending',
-    start: 'start',
-    error: 'error',
-    completed: 'completed',
-});
-export class StateStorage {
-    backend;
-    logger;
-    recordType;
-    timeseriesFormat;
-    baseIndex;
-    index;
-    constructor(context) {
-        const recordType = 'state';
-        const logger = makeLogger(context, 'state_storage');
-        const config = context.sysconfig.teraslice;
-        const _index = `${config.name}__state`;
-        // making this to pass down to backend for dynamic index searches
-        const indexName = `${_index}*`;
-        const timeseriesFormat = config.index_rollover_frequency.state;
-        const backendConfig = {
-            context,
-            indexName,
-            recordType,
-            idField: 'slice_id',
-            fullResponse: false,
-            logRecord: true,
-            forceRefresh: false,
-            storageName: 'state',
-            logger
-        };
-        this.backend = new TerasliceElasticsearchStorage(backendConfig);
-        this.logger = logger;
-        this.recordType = recordType;
-        this.timeseriesFormat = timeseriesFormat;
-        this.baseIndex = _index;
-        this.index = indexName;
-    }
-    async initialize() {
-        await this.backend.initialize();
-        this.logger.info('state storage initialized');
-    }
-    async createState(exId, slice, state, error) {
-        const { record, index } = this._createSliceRecord(exId, slice, state, error);
-        return this.backend.indexWithId(slice.slice_id, record, index);
-    }
-    async createSlices(exId, slices) {
-        const bulkRequest = slices.map((slice) => {
-            const { record, index } = this._createSliceRecord(exId, slice, SliceState.pending);
-            return {
-                action: {
-                    index: {
-                        _index: index,
-                        _id: record.slice_id,
-                    },
-                },
-                data: record
-            };
-        });
-        return this.backend.bulkSend(bulkRequest);
-    }
-    _createSliceRecord(exId, slice, state, error) {
-        if (!isKey(SliceState, state)) {
-            throw new Error(`Unknown slice state "${state}" on create`);
-        }
-        const { index } = timeseriesIndex(this.timeseriesFormat, this.baseIndex, slice._created);
-        // TODO: type this better
-        const record = {
-            slice_id: slice.slice_id,
-            slicer_id: slice.slicer_id,
-            slicer_order: slice.slicer_order,
-            request: JSON.stringify(slice.request),
-            state,
-            ex_id: exId,
-            _created: slice._created,
-            _updated: slice._created,
-        };
-        if (error) {
-            record.error = toString(error);
-        }
-        return { record, index };
-    }
-    // TODO: type this better
-    async updateState(slice, state, error) {
-        if (!isKey(SliceState, state)) {
-            throw new Error(`Unknown slice state "${state}" on update`);
-        }
-        const indexData = timeseriesIndex(this.timeseriesFormat, this.baseIndex, slice._created);
-        // TODO: type this better
-        const record = {
-            _updated: indexData.timestamp,
-            state
-        };
-        // it will usaully just be error
-        if (state === SliceState.error || error) {
-            if (error) {
-                record.error = getFullErrorStack(error);
-            }
-            else {
-                record.error = new Error('Unknown slice error').stack;
-            }
-        }
-        let notFoundErrCount = 0;
-        const updateFn = this.backend.update.bind(this.backend);
-        async function update() {
-            try {
-                return await updateFn(slice.slice_id, record, indexData.index);
-            }
-            catch (_err) {
-                const { statusCode, message } = parseErrorInfo(_err);
-                let retryable = isRetryableError(_err);
-                if (statusCode === 404) {
-                    notFoundErrCount++;
-                    retryable = notFoundErrCount < 3;
-                }
-                else if (message.includes('Request Timeout')) {
-                    retryable = true;
-                }
-                throw new TSError(_err, {
-                    retryable,
-                    reason: `Failure to update ${state} state`
-                });
-            }
-        }
-        return pRetry(update, {
-            retries: 10000,
-            delay: isTest ? 100 : 1000,
-            backoff: 5,
-            endWithFatal: true,
-        });
-    }
-    /**
-     * Get the starting position for the slicer
-     *
-     * @param {string} exId
-     * @param {number} slicerId
-     * @returns {Promise<import('@terascope/job-components').SlicerRecoveryData>}
-    */
-    async _getSlicerStartingPoint(exId, slicerId) {
-        const startQuery = `ex_id:"${exId}" AND slicer_id:"${slicerId}" AND state:${SliceState.completed}`;
-        try {
-            const [slice] = await this.search(startQuery, 0, 1, 'slicer_order:desc');
-            const recoveryData = {
-                slicer_id: slicerId,
-                lastSlice: undefined
-            };
-            if (slice) {
-                recoveryData.lastSlice = JSON.parse(slice.request);
-                this.logger.info(`last slice process for slicer_id ${slicerId}, ex_id: ${exId} is`, slice.lastSlice);
-            }
-            return recoveryData;
-        }
-        catch (err) {
-            throw new TSError(err, {
-                reason: 'Failure getting the newest record'
-            });
-        }
-    }
-    /**
-     * Get the starting positions for all of the slicers
-     *
-     * @param {string} exId
-     * @param {number} slicer
-     * @returns {Promise<import('@terascope/job-components').SlicerRecoveryData[]>}
-    */
-    async getStartingPoints(exId, slicers) {
-        const recoveredSlices = times(slicers, (i) => this._getSlicerStartingPoint(exId, i));
-        return Promise.all(recoveredSlices);
-    }
-    // TODO: fix types
-    /**
-     * @private
-     * @param {string} exId
-     * @param {number} slicerId
-     * @param {import('@terascope/job-components').RecoveryCleanupType} [cleanupType]
-     * @returns {string}
-    */
-    _getRecoverSlicesQuery(exId, slicerId, cleanupType) {
-        let query = `ex_id:"${exId}"`;
-        if (slicerId !== -1) {
-            query = `${query} AND slicer_id:"${slicerId}"`;
-        }
-        if (cleanupType && cleanupType === RecoveryCleanupType.errors) {
-            query = `${query} AND state:"${SliceState.error}"`;
-        }
-        else if (cleanupType && cleanupType === RecoveryCleanupType.pending) {
-            query = `${query} AND state:"${SliceState.pending}"`;
-        }
-        else {
-            query = `${query} AND NOT state:"${SliceState.completed}"`;
-        }
-        this.logger.debug('recovery slices query:', query);
-        return query;
-    }
-    /**
-     * @param {string} exId
-     * @param {number} slicerId
-     * @param {import('@terascope/job-components').RecoveryCleanupType} [cleanupType]
-     * @returns {Promise<import('@terascope/job-components').Slice[]>}
-    */
-    async recoverSlices(exId, slicerId, cleanupType) {
-        const query = this._getRecoverSlicesQuery(exId, slicerId, cleanupType);
-        // Look for all slices that haven't been completed so they can be retried.
-        try {
-            await this.backend.refresh(this.index);
-            const results = await this.search(query, 0, 5000, 'slicer_order:desc');
-            return results.map((doc) => ({
-                slice_id: doc.slice_id,
-                slicer_id: doc.slicer_id,
-                request: JSON.parse(doc.request),
-                _created: doc._created
-            }));
-        }
-        catch (err) {
-            throw new TSError(err, {
-                reason: 'Failure to get recovered slices'
-            });
-        }
-    }
-    async search(query, from, size, sort, fields) {
-        return this.backend.search(query, from, size, sort || '_updated:desc', fields);
-    }
-    async count(query, from = 0, sort = '_updated:desc') {
-        return this.backend.count(query, from, sort);
-    }
-    async countByState(exId, state) {
-        if (!isKey(SliceState, state)) {
-            throw new Error(`Unknown slice state "${state}" on update`);
-        }
-        const query = `ex_id:"${exId}" AND state:${state}`;
-        return this.count(query);
-    }
-    async shutdown(forceShutdown) {
-        this.logger.info('shutting down');
-        return this.backend.shutdown(forceShutdown);
-    }
-    async refresh() {
-        const { index } = timeseriesIndex(this.timeseriesFormat, this.baseIndex);
-        return this.backend.refresh(index);
-    }
-    verifyClient() {
-        return this.backend.verifyClient();
-    }
-    async waitForClient() {
-        return this.backend.waitForClient();
-    }
-}
-//# sourceMappingURL=state.js.map

package/dist/src/lib/utils/api_utils.js

@@ -1,128 +0,0 @@
-import Table from 'easy-table';
-import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/core-utils';
-export function makeTable(req, defaults, data, mappingFn) {
-    const query = fieldsQuery(req.query, defaults);
-    let emptyChar = 'N/A';
-    // used to create an empty table if there are no jobs
-    if (data.length === 0) {
-        emptyChar = '';
-        data.push({});
-    }
-    return Table.print(data, (item, cell) => {
-        const fn = mappingFn ? mappingFn(item) : (field) => get(item, field, emptyChar);
-        query.forEach((field) => {
-            cell(field, fn(field));
-        });
-    }, (table) => {
-        if (('headers' in req.query) && req.query.headers === 'false') {
-            return table.print();
-        }
-        return table.toString();
-    });
-}
-function fieldsQuery(query, defaults) {
-    if (!query.fields) {
-        return defaults || [];
-    }
-    const results = parseList(query.fields);
-    if (results.length === 0) {
-        return defaults;
-    }
-    return results;
-}
-export function handleTerasliceRequest(req, res, defaultErrorMsg = 'Failure to process TerasliceRequest', { errorCode = 500, successCode = 200 } = {}) {
-    logTerasliceRequest(req);
-    return async (fn) => {
-        try {
-            const result = await fn();
-            if (isString(result)) {
-                res.status(successCode).send(result);
-            }
-            else {
-                res.status(successCode).json(result);
-            }
-        }
-        catch (err) {
-            const { statusCode, message } = parseErrorInfo(err, {
-                defaultErrorMsg,
-                defaultStatusCode: errorCode,
-            });
-            if (statusCode >= 500) {
-                logError(req.logger, err);
-            }
-            else {
-                logError(req.logger, message);
-            }
-            sendError(res, statusCode, message, req.logger);
-        }
-    };
-}
-export function sendError(res, code, message, logger) {
-    if (res.headersSent) {
-        const error = new TSError(message);
-        error.statusCode = code;
-        if (logger) {
-            logger.error(error, 'TerasliceRequest send error after headers sent');
-        }
-        else {
-            console.error(error, 'TerasliceRequest send error after headers sent');
-        }
-        return;
-    }
-    res.status(code).json({
-        error: code,
-        message
-    });
-}
-function parseQueryInt(req, key, defaultVal) {
-    const val = req.query[key];
-    if (val == null || val === '')
-        return defaultVal;
-    const parsed = toInteger(val);
-    // allow the invalid prop to be passed through
-    // (because an error should thrown downstream)
-    if (parsed === false) {
-        return req.query[key];
-    }
-    return parsed;
-}
-export function getSearchOptions(req, defaultSort = '_updated:desc') {
-    const sort = req.query.sort || defaultSort;
-    const size = parseQueryInt(req, 'size', 100);
-    const from = parseQueryInt(req, 'from', 0);
-    const filter = req.query.filter || '';
-    return { size, from, sort, filter };
-}
-export function logTerasliceRequest(req) {
-    const queryInfo = Object.entries(req.query)
-        .map(([key, val]) => `${key}: ${val}`)
-        .join(', ');
-    const { method, path } = req;
-    req.logger.trace(`${method.toUpperCase()} ${path} endpoint has been called, ${queryInfo}`);
-}
-export function createJobActiveQuery(active) {
-    if (active === 'true') {
-        return 'job_id:* AND !active:false';
-    }
-    if (active === 'false') {
-        return 'job_id:* AND active:false';
-    }
-    return 'job_id:*';
-}
-export function addDeletedToQuery(deleted, query) {
-    if (deleted === 'false') {
-        return `${query} AND (_deleted:false OR (* AND -_deleted:*))`;
-    }
-    return `${query} AND _deleted:true`;
-}
-/**
- * Combines a base query from an endpoint with an optional filter using AND from lucene.
- * @param query - The base Lucene query string
- * @param filter - Optional filter query to append. getSearchOptions() will return an empty string
- * if no filter is present.
- * @returns The combined query, or original query if filter is empty
- */
-export function addFilterToQuery(query, filter) {
-    return filter ? `(${query}) AND (${filter})` : query;
-}
-//# sourceMappingURL=api_utils.js.map

package/dist/src/lib/utils/asset_utils.js

@@ -1,94 +0,0 @@
-import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/core-utils';
-import * as semver from 'semver';
-export function findMatchingAsset(records, name, version) {
-    const range = toSemverRange(version);
-    const assets = records
-        .filter(_isCompatibleAsset(name, range, false))
-        .sort((a, b) => semver.rcompare(a.version, b.version));
-    return getFirst(assets);
-}
-export function findSimilarAssets(records, name, version) {
-    const range = toSemverRange(version);
-    const assets = records
-        .filter(_isCompatibleAsset(name, range, true))
-        .sort((a, b) => semver.rcompare(a.version, b.version));
-    return assets;
-}
-export function getInCompatibilityReason(assets, prefix) {
-    if (!assets || !assets.length)
-        return '';
-    const reasons = [];
-    assets.slice(0, 3).forEach((asset) => {
-        if (!isCompatibleNodeVersion(asset.node_version)) {
-            reasons.push('node_version');
-        }
-        if (asset.platform != null && asset.platform !== process.platform) {
-            reasons.push('platform');
-        }
-        if (asset.arch != null && asset.arch !== process.arch) {
-            reasons.push('arch');
-        }
-    });
-    if (!reasons.length)
-        return '';
-    return `${prefix ? `${trim(prefix)} ` : ''}${joinList(reasons, ',', 'or')} mismatch`;
-}
-export function getMajorVersion(version) {
-    if (version == null)
-        return version;
-    if (isInteger(version))
-        return version;
-    return semver.major(version);
-}
-const SYSTEM_NODE_VERSION = getMajorVersion(process.version);
-/**
- * This just compares the major version
-*/
-function isCompatibleNodeVersion(version) {
-    if (version == null)
-        return true;
-    // anything less than or equal to current node version
-    return getMajorVersion(version) <= SYSTEM_NODE_VERSION;
-}
-function _isCompatibleAsset(name, range, skipRestrictions = false) {
-    return function checkIfCompatibleAsset(record) {
-        if (record.name !== name)
-            return false;
-        if (!semver.satisfies(record.version, range, { includePrerelease: true })) {
-            return false;
-        }
-        if (skipRestrictions)
-            return true;
-        if (!isCompatibleNodeVersion(record.node_version)) {
-            return false;
-        }
-        if (record.arch != null && record.arch !== process.arch) {
-            return false;
-        }
-        if (record.platform != null && record.platform !== process.platform) {
-            return false;
-        }
-        return true;
-    };
-}
-export function toSemverRange(version) {
-    if (!version || version === 'latest')
-        return '*';
-    if (semver.validRange(version)) {
-        return trimStart(trim(version), 'v');
-    }
-    throw new Error(`Version "${version}" is not a valid semver range`);
-}
-export function toVersionQuery(_version) {
-    const version = trimStart(trim(_version));
-    if (!version || version === 'latest' || version === '*') {
-        return 'version:*';
-    }
-    // if there a number and * next to each other that is not valid
-    // so lets just return the query: 12.34*.55
-    if (/\d\*/.test(version))
-        return `version:${version}`;
-    const range = new semver.Range(version);
-    return `version:${range.range.split(' ').join(' AND version:')}`;
-}
-//# sourceMappingURL=asset_utils.js.map

package/dist/src/lib/utils/date_utils.js

@@ -1,52 +0,0 @@
-import { isKey, makeISODate } from '@terascope/core-utils';
-const options = {
-    year: 'y',
-    years: 'y',
-    y: 'y',
-    months: 'M',
-    month: 'M',
-    mo: 'M',
-    mos: 'M',
-    M: 'M',
-    weeks: 'w',
-    week: 'w',
-    wks: 'w',
-    wk: 'w',
-    w: 'w',
-    days: 'd',
-    day: 'd',
-    d: 'd',
-    hours: 'h',
-    hour: 'h',
-    hr: 'h',
-    hrs: 'h',
-    h: 'h',
-    minutes: 'm',
-    minute: 'm',
-    min: 'm',
-    mins: 'm',
-    m: 'm',
-    seconds: 's',
-    second: 's',
-    s: 's',
-    milliseconds: 'ms',
-    millisecond: 'ms',
-    ms: 'ms'
-};
-const formatter = {
-    daily: 10,
-    monthly: 7,
-    yearly: 4
-};
-export function dateOptions(value) {
-    if (value && isKey(options, value)) {
-        return options[value];
-    }
-    throw new Error(`the time descriptor of "${value}" for the interval is malformed`);
-}
-export function timeseriesIndex(timeseriesFormat, index, dateStr) {
-    const timestamp = makeISODate();
-    const dateString = dateStr || timestamp;
-    return { index: `${index}-${dateString.slice(0, formatter[timeseriesFormat]).replace(/-/g, '.')}`, timestamp };
-}
-//# sourceMappingURL=date_utils.js.map

package/dist/src/lib/utils/encoding_utils.js

@@ -1,27 +0,0 @@
-import { isString, isObjectEntity, getTypeOf, isArrayLike } from '@terascope/core-utils';
-export function safeEncode(obj) {
-    let str;
-    if (isString(obj)) {
-        str = obj;
-    }
-    else if (isObjectEntity(obj) || isArrayLike(obj)) {
-        str = JSON.stringify(obj);
-    }
-    else {
-        throw new Error(`unprocessable entity to encode: ${getTypeOf(obj)}`);
-    }
-    return Buffer.from(str).toString('base64');
-}
-// TODO: proper types for this
-export function safeDecode(str) {
-    if (!isString(str) && isObjectEntity(str)) {
-        return str;
-    }
-    try {
-        return JSON.parse(Buffer.from(str, 'base64').toString('utf-8'));
-    }
-    catch (err) {
-        throw new Error(`Unable to decode ${str}`);
-    }
-}
-//# sourceMappingURL=encoding_utils.js.map

package/dist/src/lib/utils/events.js

@@ -1,4 +0,0 @@
-import events from 'node:events';
-const eventEmitter = new events.EventEmitter();
-export { eventEmitter };
-//# sourceMappingURL=events.js.map

package/dist/src/lib/utils/file_utils.js

@@ -1,124 +0,0 @@
-import { accessSync, readFileSync } from 'node:fs';
-import path from 'node:path';
-import fse from 'fs-extra';
-import semver from 'semver';
-import { Mutex } from 'async-mutex';
-import { TSError } from '@terascope/core-utils';
-import decompress from 'decompress';
-import { getMajorVersion } from './asset_utils.js';
-const mutex = new Mutex();
-const packagePath = path.join(process.cwd(), './package.json');
-let _packageJSON;
-export function getPackageJSON() {
-    if (_packageJSON)
-        return _packageJSON;
-    const file = readFileSync(packagePath, { encoding: 'utf8' });
-    const results = JSON.parse(file);
-    _packageJSON = results;
-    return results;
-}
-export function existsSync(filename) {
-    try {
-        accessSync(filename);
-        return true;
-    }
-    catch (ex) {
-        return false;
-    }
-}
-export function deleteDir(dirPath) {
-    return fse.remove(dirPath);
-}
-export async function verifyAssetJSON(id, newPath) {
-    const hasAssetJSONTopLevel = await fse.pathExists(path.join(newPath, 'asset.json'));
-    if (!hasAssetJSONTopLevel) {
-        const err = new TSError('asset.json was not found in root directory of asset bundle', { statusCode: 422 });
-        throw err;
-    }
-    let packageData;
-    try {
-        packageData = await fse.readJson(path.join(newPath, 'asset.json'));
-    }
-    catch (_err) {
-        const err = new TSError(_err, {
-            message: 'Failure parsing asset.json, please ensure that\'s it formatted correctly',
-            statusCode: 422
-        });
-        throw err;
-    }
-    const metadata = Object.assign({ id }, packageData);
-    if (!metadata.name) {
-        throw new Error('Missing name');
-    }
-    metadata.version = semver.clean(metadata.version);
-    if (!semver.valid(metadata.version)) {
-        throw new Error(`Invalid version "${metadata.version}"`);
-    }
-    /**
-         * If node_version, platform, or arch is set to a falsey
-         * value we should delete it so it is considered a wildcard.
-         *
-         * This is useful for making an asset bundle that isn't
-         * locked down.
-        */
-    if (metadata.node_version) {
-        metadata.node_version = getMajorVersion(metadata.node_version);
-    }
-    else {
-        delete metadata.node_version;
-    }
-    if (!metadata.platform) {
-        delete metadata.platform;
-    }
-    if (!metadata.arch) {
-        delete metadata.arch;
-    }
-    if (metadata.minimum_teraslice_version) {
-        const terasliceVersion = getPackageJSON().version;
-        if (semver.gt(metadata.minimum_teraslice_version, terasliceVersion)) {
-            throw new Error(`Asset requires teraslice version ${metadata.minimum_teraslice_version} or greater.`);
-        }
-    }
-    return metadata;
-}
-async function _saveAsset(logger, assetsPath, id, binaryData, metaCheck) {
-    const newPath = path.join(assetsPath, id);
-    try {
-        if (await fse.pathExists(newPath)) {
-            await fse.emptyDir(newPath);
-        }
-        else {
-            await fse.mkdir(newPath);
-        }
-        logger.info(`decompressing and saving asset ${id} to ${newPath}`);
-        await decompress(binaryData, newPath);
-        logger.info(`decompressed asset ${id} to ${newPath}`);
-        const metadata = await verifyAssetJSON(id, newPath);
-        logger.info(`asset ${id} saved to file ${newPath}`, metadata);
-        // storage/assets save fn needs to check the return metadata for uniqueness
-        if (metaCheck) {
-            return await metaCheck(metadata);
-        }
-        return metadata;
-    }
-    catch (err) {
-        await deleteDir(newPath);
-        throw err;
-    }
-}
-export async function saveAsset(logger, assetsPath, id, binaryData, metaCheck) {
-    return mutex.runExclusive(() => _saveAsset(logger, assetsPath, id, binaryData, metaCheck));
-}
-/**
- * Check if a buffer contains a zip file.
- * Note that file extension and mime type are not checked.
- * Any file type that is actually a zip (.xlsx, .jar, .apk,
- * .epub) will return true.
- * @param {Buffer} buffer A buffer containing a file file
- * @returns {boolean}
- */
-export function isZipFile(buffer) {
-    const zipSignature = [0x50, 0x4B, 0x03, 0x04];
-    return zipSignature.every((byte, index) => buffer[index] === byte);
-}
-//# sourceMappingURL=file_utils.js.map