teraslice 3.3.0 → 3.3.2

package/dist/src/lib/storage/backends/elasticsearch_store.js

@@ -1,496 +0,0 @@
-import ms from 'ms';
-import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/core-utils';
-import elasticsearchApi from '@terascope/elasticsearch-api';
-import { getClient } from '@terascope/job-components';
-import { DataType } from '@terascope/data-types';
-import { makeLogger } from '../../workers/helpers/terafoundation.js';
-import { timeseriesIndex } from '../../utils/date_utils.js';
-import { analyticsDataTypeConfig, analyticsTemplate } from './mappings/analytics.js';
-import { assetDataTypeConfig, assetMappingOverrides } from './mappings/asset.js';
-import { executionDataTypeConfig } from './mappings/ex.js';
-import { jobDataTypeConfig } from './mappings/job.js';
-import { stateDataTypeConfig, stateTemplate } from './mappings/state.js';
-function validateId(recordId, recordType) {
-    if (!recordId || !isString(recordId)) {
-        throw new TSError(`Invalid ${recordType} id given ${getTypeOf(recordId)}`, {
-            statusCode: 422
-        });
-    }
-}
-function validateIdAndRecord(recordId, record, recordType, idField) {
-    validateId(recordId, recordType);
-    const id = record[idField];
-    if (id && id !== recordId) {
-        throw new TSError(`${recordType}.${idField} doesn't match request id`, {
-            statusCode: 406
-        });
-    }
-}
-// TODO: fix types here
-function _getTimeout(timeout) {
-    if (isInteger(timeout)) {
-        // don't allow a timeout of less than 1 second
-        if (timeout <= 1000)
-            return undefined;
-        return ms(timeout);
-    }
-    if (isString(timeout)) {
-        return timeout;
-    }
-    return undefined;
-}
-// TODO: type this better
-export class TerasliceElasticsearchStorage {
-    context;
-    logger;
-    recordType;
-    // TODO: should this be here?
-    defaultIndexName;
-    storageName;
-    flushInterval;
-    isShuttingDown = false;
-    // flag to know if already in a bulk call
-    savingBulk = false;
-    // Buffer to build up bulk requests. TODO: fix type
-    bulkQueue = [];
-    idField;
-    options;
-    mapping;
-    api;
-    indexSettings;
-    constructor(backendConfig) {
-        const { context, indexName, recordType, idField, storageName, bulkSize = 1000, fullResponse = false, logRecord = true, forceRefresh = true, logger } = backendConfig;
-        this.context = context;
-        this.storageName = storageName;
-        this.logger = logger ?? makeLogger(context, 'elasticsearch_backend', { storageName });
-        this.recordType = recordType;
-        // Validate recordType
-        const validRecordTypes = ['analytics', 'asset', 'ex', 'job', 'state'];
-        if (!validRecordTypes.includes(recordType)) {
-            throw new Error(`Could not find mapping for recordType: ${recordType}`);
-        }
-        const config = this.context.sysconfig.teraslice;
-        this.indexSettings = get(config, ['index_settings', this.storageName], {
-            number_of_shards: 5,
-            number_of_replicas: 1,
-        });
-        this.defaultIndexName = indexName;
-        this.idField = idField;
-        this.options = {
-            bulkSize,
-            fullResponse,
-            logRecord,
-            forceRefresh
-        };
-    }
-    async initialize() {
-        const config = this.context.sysconfig.teraslice;
-        const isMultiIndex = this.defaultIndexName[this.defaultIndexName.length - 1] === '*';
-        let newIndex = this.defaultIndexName;
-        if (isMultiIndex) {
-            // @ts-expect-error TODO: fix this
-            const storeType = this.defaultIndexName.match(/__(.*)\*/)[1];
-            const timeseriesFormat = config.index_rollover_frequency[storeType];
-            const nameSize = this.defaultIndexName.length - 1;
-            newIndex = timeseriesIndex(timeseriesFormat, this.defaultIndexName.slice(0, nameSize)).index;
-        }
-        const clientName = JSON.stringify({
-            connection: config.state.connection,
-            index: this.defaultIndexName,
-        });
-        const connectionConfig = Object.assign({}, config.state);
-        if (connectionConfig.connection_cache == null) {
-            connectionConfig.connection_cache = true;
-        }
-        const { connection } = config.state;
-        const options = {
-            full_response: !!this.options.fullResponse,
-            connection,
-        };
-        await pWhile(async () => {
-            try {
-                const client = await getClient(this.context, connectionConfig, 'elasticsearch-next');
-                this.api = elasticsearchApi(client, this.logger, options);
-                this._generateMapping();
-                await this._createIndex(newIndex);
-                await this.api.isAvailable(newIndex);
-                return true;
-            }
-            catch (err) {
-                const error = new TSError(err, {
-                    reason: `Failure initializing ${this.recordType} index: ${this.defaultIndexName}`,
-                });
-                if (error.statusCode >= 400 && error.statusCode < 500) {
-                    throw error;
-                }
-                logError(this.logger, error, `Failed attempt connecting to elasticsearch: ${clientName} (will retry)`);
-                await pDelay(isTest ? 0 : random(2000, 4000));
-                return false;
-            }
-        });
-        // Periodically flush the bulkQueue so we don't end up with cached data lingering.
-        this.flushInterval = setInterval(() => {
-            // TODO: this might be undefined
-            this._flush().catch((err) => {
-                logError(this.logger, err, 'background flush failure');
-                return null;
-            });
-            // stager the interval to avoid collisions
-        }, random(9000, 11000));
-    }
-    async get(recordId, index = this.defaultIndexName, fields) {
-        validateId(recordId, this.recordType);
-        this.logger.trace(`getting record id: ${recordId}`);
-        const query = {
-            index,
-            id: recordId,
-        };
-        if (fields) {
-            query._source_includes = fields;
-        }
-        return this.api.get(query);
-    }
-    async search(query, from = 0, size = 10000, sort, fields, index = this.defaultIndexName) {
-        if (from != null && !isInteger(from)) {
-            throw new Error(`from parameter must be a integer, got ${from}`);
-        }
-        if (size != null && !isInteger(size)) {
-            throw new Error(`size parameter must be a integer, got ${size}`);
-        }
-        if (sort != null && !isString(sort)) {
-            throw new Error(`sort parameter must be a string, got ${sort}`);
-        }
-        const esQuery = {
-            index,
-            from,
-            size,
-            sort,
-        };
-        if (typeof query === 'string') {
-            esQuery.q = query;
-        }
-        else {
-            esQuery.body = query;
-        }
-        if (fields) {
-            esQuery._source_includes = fields;
-        }
-        return this.api.search(esQuery);
-    }
-    /*
-     * index saves a record to elasticsearch allowing automatic
-     * ID creation
-     */
-    async index(record, indexArg = this.defaultIndexName) {
-        this.logger.trace('indexing record', this.options.logRecord ? record : undefined);
-        const query = {
-            index: indexArg,
-            body: record,
-            refresh: this.options.forceRefresh,
-        };
-        return this.api.index(query);
-    }
-    /*
-     * index saves a record to opensearch with a specified ID.
-     * If the document is already there it will be replaced.
-     */
-    async indexWithId(recordId, record, index = this.defaultIndexName, timeout) {
-        validateIdAndRecord(recordId, record, this.recordType, this.idField);
-        this.logger.trace(`indexWithId call with id: ${recordId}, record`, this.options.logRecord ? record : null);
-        const query = {
-            index,
-            id: recordId,
-            body: record,
-            refresh: this.options.forceRefresh,
-            timeout: _getTimeout(timeout)
-        };
-        return this.api.indexWithId(query);
-    }
-    /*
-     * Create saves a record to opensearch under the provided id.
-     * If the record already exists it will not be inserted.
-     */
-    async create(record, index = this.defaultIndexName) {
-        this.logger.trace('creating record', this.options.logRecord ? record : null);
-        const query = {
-            index,
-            id: record[this.idField],
-            body: record,
-            refresh: this.options.forceRefresh,
-        };
-        return this.api.create(query);
-    }
-    async count(query, from, sort, index = this.defaultIndexName) {
-        if (from != null && !isInteger(from)) {
-            throw new Error(`from parameter must be a integer, got ${from}`);
-        }
-        if (sort != null && !isString(sort)) {
-            throw new Error(`sort parameter must be a string, got ${sort}`);
-        }
-        // TODO: check from
-        const esQuery = {
-            index,
-            // @ts-expect-error
-            from,
-            sort,
-        };
-        if (isString(query)) {
-            esQuery.q = query;
-        }
-        else {
-            esQuery.body = query;
-        }
-        return this.api.count(esQuery);
-    }
-    async update(recordId, updateSpec, index = this.defaultIndexName) {
-        validateIdAndRecord(recordId, updateSpec, this.recordType, this.idField);
-        this.logger.trace(`updating record ${recordId}, `, this.options.logRecord ? updateSpec : null);
-        const query = {
-            index,
-            id: recordId,
-            body: {
-                doc: updateSpec,
-            },
-            refresh: this.options.forceRefresh,
-            // TODO: check this retry
-            retry_on_conflict: 3,
-        };
-        return this.api.update(query);
-    }
-    async updatePartial(recordId, applyChanges, index = this.defaultIndexName) {
-        if (typeof applyChanges !== 'function') {
-            throw new Error('Update Partial expected a applyChanges function');
-        }
-        validateId(recordId, this.recordType);
-        const getParams = {
-            index,
-            id: recordId,
-        };
-        const existing = await pRetry(() => this.api.get(getParams, true), {
-            matches: ['no_shard_available_action_exception'],
-            delay: 1000,
-            retries: 10,
-            backoff: 5
-        });
-        const doc = await applyChanges(Object.assign({}, existing._source));
-        this.logger.trace(`updating partial record ${recordId}, `, this.options.logRecord ? doc : null);
-        validateIdAndRecord(recordId, doc, this.recordType, this.idField);
-        const query = {
-            index,
-            id: recordId,
-            body: doc,
-            refresh: this.options.forceRefresh,
-        };
-        query.if_seq_no = existing._seq_no;
-        query.if_primary_term = existing._primary_term;
-        try {
-            await this.api.indexWithId(query);
-            return doc;
-        }
-        catch (err) {
-            // if there is a version conflict
-            if (err.statusCode === 409 && err.message.includes('version conflict')) {
-                this.logger.debug({ error: err }, `version conflict when updating "${recordId}" (${this.recordType})`);
-                return this.updatePartial(recordId, applyChanges, index);
-            }
-            throw new TSError(err);
-        }
-    }
-    async remove(recordId, index = this.defaultIndexName) {
-        validateId(recordId, this.recordType);
-        this.logger.trace(`removing record ${recordId}`);
-        const query = {
-            index,
-            id: recordId,
-            refresh: this.options.forceRefresh,
-        };
-        return this.api.remove(query);
-    }
-    async bulk(record, type = 'index', index = this.defaultIndexName) {
-        if (this.isShuttingDown) {
-            throw new TSError('Unable to send bulk record after shutdown', {
-                context: {
-                    recordType: this.recordType,
-                    record,
-                },
-            });
-        }
-        const action = {
-            [type]: { _index: index }
-        };
-        this.bulkQueue.push({
-            action,
-            data: type === 'delete' ? undefined : record
-        });
-        // We only flush once enough records have accumulated for it to make sense.
-        if (this.bulkQueue.length >= this.options.bulkSize) {
-            this.logger.trace(`flushing bulk queue ${this.bulkQueue.length}`);
-            return this._flush();
-        }
-        // Bulk saving is a background operation so we don't have
-        // anything meaningful to return.
-        return Promise.resolve(true);
-    }
-    _destroy(err) {
-        this.bulkQueue = [];
-        this.isShuttingDown = true;
-        if (err) {
-            throw err;
-        }
-        return true;
-    }
-    async shutdown(forceShutdown = false) {
-        const startTime = Date.now();
-        this.logger.trace(`shutdown store, took ${ms(Date.now() - startTime)}`);
-        clearInterval(this.flushInterval);
-        if (forceShutdown !== true) {
-            return this._flush(true);
-        }
-        const destroy = this._destroy.bind(this);
-        const timeout = this.context.sysconfig.teraslice.shutdown_timeout;
-        this.logger.trace(`attempting to shutdown, will destroy in ${timeout}`);
-        setTimeout(destroy, timeout).unref();
-        try {
-            await this._flush(true);
-            this._destroy();
-        }
-        catch (err) {
-            this._destroy(err);
-        }
-    }
-    async bulkSend(bulkRequest) {
-        return this.api.bulkSend(bulkRequest);
-    }
-    async _flush(shuttingDown = false) {
-        if (!this.bulkQueue.length)
-            return;
-        if (!shuttingDown && this.savingBulk)
-            return;
-        this.savingBulk = true;
-        const bulkRequest = this.bulkQueue.slice();
-        this.bulkQueue = [];
-        try {
-            const recordCount = await this.bulkSend(bulkRequest);
-            const extraMsg = shuttingDown ? ', on shutdown' : '';
-            this.logger.debug(`flushed ${recordCount}${extraMsg} records to index ${this.defaultIndexName}`);
-        }
-        finally {
-            this.savingBulk = false;
-        }
-    }
-    async sendTemplate(mapping) {
-        if (mapping.template) {
-            const clusterName = this.context.sysconfig.teraslice.name;
-            const name = `${clusterName}_${this.recordType}_template`;
-            // setting template name to reflect current teraslice instance name to help prevent
-            // conflicts with differing versions of teraslice with same elastic db
-            if (mapping.template) {
-                if (!mapping.template.match(clusterName)) {
-                    mapping.template = `${clusterName}${mapping.template}`;
-                }
-            }
-            return this.putTemplate(mapping, name);
-        }
-        return Promise.resolve(true);
-    }
-    _getDataTypeConfig() {
-        switch (this.recordType) {
-            case 'analytics':
-                return {
-                    config: analyticsDataTypeConfig,
-                    template: analyticsTemplate
-                };
-            case 'asset':
-                return {
-                    config: assetDataTypeConfig,
-                    overrides: assetMappingOverrides
-                };
-            case 'ex':
-                return { config: executionDataTypeConfig };
-            case 'job':
-                return { config: jobDataTypeConfig };
-            case 'state':
-                return {
-                    config: stateDataTypeConfig,
-                    template: stateTemplate
-                };
-            default:
-                throw new Error(`Could not find DataType config for recordType: ${this.recordType}`);
-        }
-    }
-    _generateMapping() {
-        const clientMetadata = this.api.getClientMetadata();
-        const { config, overrides, template } = this._getDataTypeConfig();
-        const dataType = new DataType(config);
-        // Generate the mapping using DataType with client metadata for version-specific mappings
-        this.mapping = dataType.toESMapping({
-            ...clientMetadata,
-            overrides: {
-                ...overrides,
-                settings: {
-                    'index.number_of_shards': this.indexSettings.number_of_shards,
-                    'index.number_of_replicas': this.indexSettings.number_of_replicas,
-                }
-            }
-        });
-        // Add template pattern if this is an index that
-        // adds time at the end of the index (analytics/state)
-        if (template) {
-            this.mapping.template = template;
-        }
-    }
-    async _createIndex(index = this.defaultIndexName) {
-        // @ts-expect-error TODO: check type missing id
-        const existQuery = { index };
-        const exists = await this.api.indexExists(existQuery);
-        if (!exists) {
-            // Make sure the index exists before we do anything else.
-            const createQuery = {
-                index,
-                body: this.mapping,
-            };
-            try {
-                await this.sendTemplate(this.mapping);
-                return await this.api.indexCreate(createQuery);
-            }
-            catch (err) {
-                // It's not really an error if it's just that the index is already there
-                if (parseError(err).includes('already_exists_exception')) {
-                    return true;
-                }
-                const error = new TSError(err, {
-                    reason: `Could not create index: ${index}`,
-                });
-                throw error;
-            }
-        }
-        // Index already exists. nothing to do.
-        return true;
-    }
-    async refresh(index = this.defaultIndexName) {
-        const query = { index };
-        return this.api.indexRefresh(query);
-    }
-    // TODO: fix type here
-    async putTemplate(template, name) {
-        return this.api.putTemplate(template, name);
-    }
-    verifyClient() {
-        if (this.isShuttingDown)
-            return false;
-        return this.api.verifyClient();
-    }
-    async waitForClient() {
-        if (this.api.verifyClient())
-            return;
-        await pWhile(async () => {
-            if (this.isShuttingDown)
-                throw new Error('Elasticsearch store is shutdown');
-            if (this.api.verifyClient())
-                return true;
-            await pDelay(100);
-            return false;
-        });
-    }
-}
-//# sourceMappingURL=elasticsearch_store.js.map

package/dist/src/lib/storage/backends/mappings/analytics.js

@@ -1,20 +0,0 @@
-import { FieldType } from '@terascope/types';
-export const analyticsDataTypeConfig = {
-    version: 1,
-    fields: {
-        ex_id: { type: FieldType.Keyword },
-        job_id: { type: FieldType.Keyword },
-        worker_id: { type: FieldType.Keyword },
-        slice_id: { type: FieldType.Keyword },
-        slicer_id: { type: FieldType.Keyword },
-        op: { type: FieldType.Keyword },
-        order: { type: FieldType.Integer },
-        count: { type: FieldType.Integer },
-        state: { type: FieldType.Keyword },
-        time: { type: FieldType.Integer },
-        memory: { type: FieldType.Long },
-        '@timestamp': { type: FieldType.Date }
-    }
-};
-export const analyticsTemplate = '__analytics*';
-//# sourceMappingURL=analytics.js.map

package/dist/src/lib/storage/backends/mappings/asset.js

@@ -1,32 +0,0 @@
-import { FieldType } from '@terascope/types';
-export const assetDataTypeConfig = {
-    version: 1,
-    fields: {
-        name: { type: FieldType.Keyword },
-        version: { type: FieldType.Keyword },
-        id: { type: FieldType.Keyword },
-        description: { type: FieldType.Keyword },
-        arch: { type: FieldType.Keyword },
-        platform: { type: FieldType.Keyword },
-        node_version: { type: FieldType.Integer },
-        _created: { type: FieldType.Date }
-    }
-};
-/**
- * Override for the blob field which uses ES binary type.
- * I could not find the datatype equivalent
- * so we must add this field manually via mapping overrides.
- * Once the issue below is resolved we can remove this.
- * https://github.com/terascope/teraslice/issues/4296
- */
-export const assetMappingOverrides = {
-    mappings: {
-        properties: {
-            blob: {
-                type: 'binary',
-                doc_values: false
-            }
-        }
-    }
-};
-//# sourceMappingURL=asset.js.map

package/dist/src/lib/storage/backends/mappings/ex.js

@@ -1,53 +0,0 @@
-import { FieldType } from '@terascope/types';
-export const executionDataTypeConfig = {
-    version: 1,
-    fields: {
-        // ExecutionConfig fields
-        job_id: { type: FieldType.Keyword },
-        ex_id: { type: FieldType.Keyword },
-        _context: { type: FieldType.Keyword },
-        _created: { type: FieldType.Date },
-        _updated: { type: FieldType.Date },
-        _deleted: { type: FieldType.Boolean },
-        _deleted_on: { type: FieldType.Date },
-        _status: { type: FieldType.Keyword },
-        _has_errors: { type: FieldType.Boolean },
-        _slicer_stats: { type: FieldType.Object },
-        _failureReason: { type: FieldType.Text },
-        metadata: { type: FieldType.Object, indexed: false },
-        recovered_execution: { type: FieldType.Keyword },
-        recovered_slice_type: { type: FieldType.Keyword },
-        slicer_port: { type: FieldType.Integer },
-        slicer_hostname: { type: FieldType.Keyword },
-        teraslice_version: { type: FieldType.Keyword },
-        // ValidatedJobConfig fields
-        active: { type: FieldType.Boolean },
-        analytics: { type: FieldType.Boolean },
-        assets: { type: FieldType.Keyword },
-        autorecover: { type: FieldType.Boolean },
-        lifecycle: { type: FieldType.Keyword },
-        max_retries: { type: FieldType.Integer },
-        name: { type: FieldType.Text },
-        probation_window: { type: FieldType.Integer },
-        performance_metrics: { type: FieldType.Boolean },
-        log_level: { type: FieldType.Keyword },
-        slicers: { type: FieldType.Integer },
-        workers: { type: FieldType.Integer },
-        stateful: { type: FieldType.Boolean },
-        // k8s field
-        cpu: { type: FieldType.Float },
-        cpu_execution_controller: { type: FieldType.Float },
-        ephemeral_storage: { type: FieldType.Boolean },
-        memory: { type: FieldType.Integer },
-        memory_execution_controller: { type: FieldType.Integer },
-        resources_requests_cpu: { type: FieldType.Float },
-        resources_requests_memory: { type: FieldType.Integer },
-        resources_limits_cpu: { type: FieldType.Float },
-        resources_limits_memory: { type: FieldType.Integer },
-        kubernetes_image: { type: FieldType.Keyword },
-        prom_metrics_enabled: { type: FieldType.Boolean },
-        prom_metrics_port: { type: FieldType.Integer },
-        prom_metrics_add_default: { type: FieldType.Boolean }
-    }
-};
-//# sourceMappingURL=ex.js.map

package/dist/src/lib/storage/backends/mappings/job.js

@@ -1,42 +0,0 @@
-import { FieldType } from '@terascope/types';
-export const jobDataTypeConfig = {
-    version: 1,
-    fields: {
-        // JobConfig fields
-        job_id: { type: FieldType.Keyword },
-        _context: { type: FieldType.Keyword },
-        _created: { type: FieldType.Date },
-        _updated: { type: FieldType.Date },
-        _deleted: { type: FieldType.Boolean },
-        _deleted_on: { type: FieldType.Date },
-        // ValidatedJobConfig fields
-        active: { type: FieldType.Boolean },
-        analytics: { type: FieldType.Boolean },
-        assets: { type: FieldType.Keyword },
-        autorecover: { type: FieldType.Boolean },
-        lifecycle: { type: FieldType.Keyword },
-        max_retries: { type: FieldType.Integer },
-        name: { type: FieldType.Text },
-        probation_window: { type: FieldType.Integer },
-        performance_metrics: { type: FieldType.Boolean },
-        log_level: { type: FieldType.Keyword },
-        slicers: { type: FieldType.Integer },
-        workers: { type: FieldType.Integer },
-        stateful: { type: FieldType.Boolean },
-        // K8s fields
-        cpu: { type: FieldType.Float },
-        cpu_execution_controller: { type: FieldType.Float },
-        ephemeral_storage: { type: FieldType.Boolean },
-        memory: { type: FieldType.Integer },
-        memory_execution_controller: { type: FieldType.Integer },
-        resources_requests_cpu: { type: FieldType.Float },
-        resources_requests_memory: { type: FieldType.Integer },
-        resources_limits_cpu: { type: FieldType.Float },
-        resources_limits_memory: { type: FieldType.Integer },
-        kubernetes_image: { type: FieldType.Keyword },
-        prom_metrics_enabled: { type: FieldType.Boolean },
-        prom_metrics_port: { type: FieldType.Integer },
-        prom_metrics_add_default: { type: FieldType.Boolean }
-    }
-};
-//# sourceMappingURL=job.js.map

package/dist/src/lib/storage/backends/mappings/state.js

@@ -1,16 +0,0 @@
-import { FieldType } from '@terascope/types';
-export const stateDataTypeConfig = {
-    version: 1,
-    fields: {
-        ex_id: { type: FieldType.Keyword },
-        slice_id: { type: FieldType.Keyword },
-        slicer_id: { type: FieldType.Keyword },
-        slicer_order: { type: FieldType.Integer },
-        state: { type: FieldType.Keyword },
-        _created: { type: FieldType.Date },
-        _updated: { type: FieldType.Date },
-        error: { type: FieldType.Keyword }
-    }
-};
-export const stateTemplate = '__state*';
-//# sourceMappingURL=state.js.map