teraslice 2.11.0 → 2.12.1

package/dist/src/lib/storage/backends/elasticsearch_store.js

@@ -0,0 +1,494 @@
+import ms from 'ms';
+import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/utils';
+import elasticsearchApi from '@terascope/elasticsearch-api';
+import { getClient } from '@terascope/job-components';
+import { makeLogger } from '../../workers/helpers/terafoundation.js';
+import { timeseriesIndex } from '../../utils/date_utils.js';
+import analyticsSchema from './mappings/analytics.js';
+import assetSchema from './mappings/asset.js';
+import executionSchema from './mappings/ex.js';
+import jobsSchema from './mappings/job.js';
+import stateSchema from './mappings/state.js';
+function validateId(recordId, recordType) {
+    if (!recordId || !isString(recordId)) {
+        throw new TSError(`Invalid ${recordType} id given ${getTypeOf(recordId)}`, {
+            statusCode: 422
+        });
+    }
+}
+function validateIdAndRecord(recordId, record, recordType, idField) {
+    validateId(recordId, recordType);
+    const id = record[idField];
+    if (id && id !== recordId) {
+        throw new TSError(`${recordType}.${idField} doesn't match request id`, {
+            statusCode: 406
+        });
+    }
+}
+// TODO: fix types here
+function _getTimeout(timeout) {
+    if (isInteger(timeout)) {
+        // don't allow a timeout of less than 1 second
+        if (timeout <= 1000)
+            return undefined;
+        return ms(timeout);
+    }
+    if (isString(timeout)) {
+        return timeout;
+    }
+    return undefined;
+}
+// TODO: type this better
+export class TerasliceElasticsearchStorage {
+    context;
+    logger;
+    recordType;
+    // TODO: should this be here?
+    defaultIndexName;
+    storageName;
+    flushInterval;
+    isShuttingDown = false;
+    // flag to know if already in a bulk call
+    savingBulk = false;
+    // Buffer to build up bulk requests. TODO: fix type
+    bulkQueue = [];
+    idField;
+    options;
+    mapping;
+    api;
+    constructor(backendConfig) {
+        const { context, indexName, recordType, idField, storageName, bulkSize = 1000, fullResponse = false, logRecord = true, forceRefresh = true, logger } = backendConfig;
+        this.context = context;
+        this.storageName = storageName;
+        this.logger = logger ?? makeLogger(context, 'elasticsearch_backend', { storageName });
+        this.recordType = recordType;
+        if (recordType === 'analytics') {
+            this.mapping = analyticsSchema;
+        }
+        else if (recordType === 'asset') {
+            this.mapping = assetSchema;
+        }
+        else if (recordType === 'ex') {
+            this.mapping = executionSchema;
+        }
+        else if (recordType === 'job') {
+            this.mapping = jobsSchema;
+        }
+        else if (recordType === 'state') {
+            this.mapping = stateSchema;
+        }
+        else {
+            throw new Error(`Could not find mapping for recordType: ${recordType}`);
+        }
+        const config = this.context.sysconfig.teraslice;
+        const indexSettings = get(config, ['index_settings', this.storageName], {
+            number_of_shards: 5,
+            number_of_replicas: 1,
+        });
+        this.mapping.settings = {
+            'index.number_of_shards': indexSettings.number_of_shards,
+            'index.number_of_replicas': indexSettings.number_of_replicas,
+        };
+        this.defaultIndexName = indexName;
+        this.idField = idField;
+        this.options = {
+            bulkSize,
+            fullResponse,
+            logRecord,
+            forceRefresh
+        };
+    }
+    async initialize() {
+        const config = this.context.sysconfig.teraslice;
+        const isMultiIndex = this.defaultIndexName[this.defaultIndexName.length - 1] === '*';
+        let newIndex = this.defaultIndexName;
+        if (isMultiIndex) {
+            // @ts-expect-error TODO: fix this
+            const storeType = this.defaultIndexName.match(/__(.*)\*/)[1];
+            const timeseriesFormat = config.index_rollover_frequency[storeType];
+            const nameSize = this.defaultIndexName.length - 1;
+            newIndex = timeseriesIndex(timeseriesFormat, this.defaultIndexName.slice(0, nameSize)).index;
+        }
+        const clientName = JSON.stringify({
+            connection: config.state.connection,
+            index: this.defaultIndexName,
+        });
+        const connectionConfig = Object.assign({}, config.state);
+        if (connectionConfig.connection_cache == null) {
+            connectionConfig.connection_cache = true;
+        }
+        const { connection } = config.state;
+        const options = {
+            full_response: !!this.options.fullResponse,
+            connection,
+        };
+        await pWhile(async () => {
+            try {
+                const client = await getClient(this.context, connectionConfig, 'elasticsearch-next');
+                this.api = elasticsearchApi(client, this.logger, options);
+                await this._createIndex(newIndex);
+                await this.api.isAvailable(newIndex, this.recordType);
+                return true;
+            }
+            catch (err) {
+                const error = new TSError(err, {
+                    reason: `Failure initializing ${this.recordType} index: ${this.defaultIndexName}`,
+                });
+                if (error.statusCode >= 400 && error.statusCode < 500) {
+                    throw error;
+                }
+                logError(this.logger, error, `Failed attempt connecting to elasticsearch: ${clientName} (will retry)`);
+                await pDelay(isTest ? 0 : random(2000, 4000));
+                return false;
+            }
+        });
+        // Periodically flush the bulkQueue so we don't end up with cached data lingering.
+        this.flushInterval = setInterval(() => {
+            // TODO: this might be undefined
+            this._flush().catch((err) => {
+                logError(this.logger, err, 'background flush failure');
+                return null;
+            });
+            // stager the interval to avoid collisions
+        }, random(9000, 11000));
+    }
+    async get(recordId, index = this.defaultIndexName, fields) {
+        validateId(recordId, this.recordType);
+        this.logger.trace(`getting record id: ${recordId}`);
+        const query = {
+            index,
+            type: this.recordType,
+            id: recordId,
+        };
+        if (fields) {
+            if (!this.api.isElasticsearch6()) {
+                query._source_includes = fields;
+            }
+            else {
+                // @ts-expect-error backwards compatible
+                query._sourceInclude = fields;
+            }
+        }
+        return this.api.get(query);
+    }
+    async search(query, from = 0, size = 10000, sort, fields, index = this.defaultIndexName) {
+        if (from != null && !isInteger(from)) {
+            throw new Error(`from parameter must be a integer, got ${from}`);
+        }
+        if (size != null && !isInteger(size)) {
+            throw new Error(`size parameter must be a integer, got ${size}`);
+        }
+        if (sort != null && !isString(sort)) {
+            throw new Error(`sort parameter must be a string, got ${sort}`);
+        }
+        const esQuery = {
+            index,
+            from,
+            size,
+            type: this.recordType,
+            sort,
+        };
+        if (typeof query === 'string') {
+            esQuery.q = query;
+        }
+        else {
+            esQuery.body = query;
+        }
+        if (fields) {
+            if (!this.api.isElasticsearch6()) {
+                esQuery._source_includes = fields;
+            }
+            else {
+                // @ts-expect-error backwards compatible
+                esQuery._sourceInclude = fields;
+            }
+        }
+        return this.api.search(esQuery);
+    }
+    /*
+     * index saves a record to elasticsearch allowing automatic
+     * ID creation
+     */
+    async index(record, indexArg = this.defaultIndexName) {
+        this.logger.trace('indexing record', this.options.logRecord ? record : undefined);
+        const query = {
+            index: indexArg,
+            type: this.recordType,
+            body: record,
+            refresh: this.options.forceRefresh,
+        };
+        return this.api.index(query);
+    }
+    /*
+     * index saves a record to elasticsearch with a specified ID.
+     * If the document is already there it will be replaced.
+     */
+    async indexWithId(recordId, record, index = this.defaultIndexName, timeout) {
+        validateIdAndRecord(recordId, record, this.recordType, this.idField);
+        this.logger.trace(`indexWithId call with id: ${recordId}, record`, this.options.logRecord ? record : null);
+        const query = {
+            index,
+            type: this.recordType,
+            id: recordId,
+            body: record,
+            refresh: this.options.forceRefresh,
+            timeout: _getTimeout(timeout)
+        };
+        return this.api.indexWithId(query);
+    }
+    /*
+     * Create saves a record to elasticsearch under the provided id.
+     * If the record already exists it will not be inserted.
+     */
+    async create(record, index = this.defaultIndexName) {
+        this.logger.trace('creating record', this.options.logRecord ? record : null);
+        const query = {
+            index,
+            type: this.recordType,
+            id: record[this.idField],
+            body: record,
+            refresh: this.options.forceRefresh,
+        };
+        return this.api.create(query);
+    }
+    async count(query, from, sort, index = this.defaultIndexName) {
+        if (from != null && !isInteger(from)) {
+            throw new Error(`from parameter must be a integer, got ${from}`);
+        }
+        if (sort != null && !isString(sort)) {
+            throw new Error(`sort parameter must be a string, got ${sort}`);
+        }
+        // TODO: check from
+        const esQuery = {
+            index,
+            type: this.recordType,
+            // @ts-expect-error
+            from,
+            sort,
+        };
+        if (isString(query)) {
+            esQuery.q = query;
+        }
+        else {
+            esQuery.body = query;
+        }
+        return this.api.count(esQuery);
+    }
+    async update(recordId, updateSpec, index = this.defaultIndexName) {
+        validateIdAndRecord(recordId, updateSpec, this.recordType, this.idField);
+        this.logger.trace(`updating record ${recordId}, `, this.options.logRecord ? updateSpec : null);
+        const query = {
+            index,
+            type: this.recordType,
+            id: recordId,
+            body: {
+                doc: updateSpec,
+            },
+            refresh: this.options.forceRefresh,
+            // TODO: check this retry
+            retry_on_conflict: 3,
+        };
+        return this.api.update(query);
+    }
+    async updatePartial(recordId, applyChanges, index = this.defaultIndexName) {
+        if (typeof applyChanges !== 'function') {
+            throw new Error('Update Partial expected a applyChanges function');
+        }
+        validateId(recordId, this.recordType);
+        const getParams = {
+            index,
+            type: this.recordType,
+            id: recordId,
+        };
+        const existing = await pRetry(() => this.api.get(getParams, true), {
+            matches: ['no_shard_available_action_exception'],
+            delay: 1000,
+            retries: 10,
+            backoff: 5
+        });
+        const doc = await applyChanges(Object.assign({}, existing._source));
+        this.logger.trace(`updating partial record ${recordId}, `, this.options.logRecord ? doc : null);
+        validateIdAndRecord(recordId, doc, this.recordType, this.idField);
+        const query = {
+            index,
+            type: this.recordType,
+            id: recordId,
+            body: doc,
+            refresh: this.options.forceRefresh,
+        };
+        if (!this.api.isElasticsearch6()) {
+            query.if_seq_no = existing._seq_no;
+            query.if_primary_term = existing._primary_term;
+        }
+        else {
+            query.version = existing._version;
+        }
+        try {
+            await this.api.indexWithId(query);
+            return doc;
+        }
+        catch (err) {
+            // if there is a version conflict
+            if (err.statusCode === 409 && err.message.includes('version conflict')) {
+                this.logger.debug({ error: err }, `version conflict when updating "${recordId}" (${this.recordType})`);
+                return this.updatePartial(recordId, applyChanges, index);
+            }
+            throw new TSError(err);
+        }
+    }
+    async remove(recordId, index = this.defaultIndexName) {
+        validateId(recordId, this.recordType);
+        this.logger.trace(`removing record ${recordId}`);
+        const query = {
+            index,
+            type: this.recordType,
+            id: recordId,
+            refresh: this.options.forceRefresh,
+        };
+        return this.api.remove(query);
+    }
+    async bulk(record, type = 'index', index = this.defaultIndexName) {
+        if (this.isShuttingDown) {
+            throw new TSError('Unable to send bulk record after shutdown', {
+                context: {
+                    recordType: this.recordType,
+                    record,
+                },
+            });
+        }
+        const action = {
+            [type]: {
+                _index: index,
+                _type: this.recordType,
+            }
+        };
+        this.bulkQueue.push({
+            action,
+            data: type === 'delete' ? undefined : record
+        });
+        // We only flush once enough records have accumulated for it to make sense.
+        if (this.bulkQueue.length >= this.options.bulkSize) {
+            this.logger.trace(`flushing bulk queue ${this.bulkQueue.length}`);
+            return this._flush();
+        }
+        // Bulk saving is a background operation so we don't have
+        // anything meaningful to return.
+        return Promise.resolve(true);
+    }
+    _destroy(err) {
+        this.bulkQueue = [];
+        this.isShuttingDown = true;
+        if (err) {
+            throw err;
+        }
+        return true;
+    }
+    async shutdown(forceShutdown = false) {
+        const startTime = Date.now();
+        this.logger.trace(`shutdown store, took ${ms(Date.now() - startTime)}`);
+        clearInterval(this.flushInterval);
+        if (forceShutdown !== true) {
+            return this._flush(true);
+        }
+        const destroy = this._destroy.bind(this);
+        const timeout = this.context.sysconfig.teraslice.shutdown_timeout;
+        this.logger.trace(`attempting to shutdown, will destroy in ${timeout}`);
+        setTimeout(destroy, timeout).unref();
+        try {
+            await this._flush(true);
+            this._destroy();
+        }
+        catch (err) {
+            this._destroy(err);
+        }
+    }
+    async bulkSend(bulkRequest) {
+        return this.api.bulkSend(bulkRequest);
+    }
+    async _flush(shuttingDown = false) {
+        if (!this.bulkQueue.length)
+            return;
+        if (!shuttingDown && this.savingBulk)
+            return;
+        this.savingBulk = true;
+        const bulkRequest = this.bulkQueue.slice();
+        this.bulkQueue = [];
+        try {
+            const recordCount = await this.bulkSend(bulkRequest);
+            const extraMsg = shuttingDown ? ', on shutdown' : '';
+            this.logger.debug(`flushed ${recordCount}${extraMsg} records to index ${this.defaultIndexName}`);
+        }
+        finally {
+            this.savingBulk = false;
+        }
+    }
+    async sendTemplate(mapping) {
+        if (mapping.template) {
+            const clusterName = this.context.sysconfig.teraslice.name;
+            const name = `${clusterName}_${this.recordType}_template`;
+            // setting template name to reflect current teraslice instance name to help prevent
+            // conflicts with differing versions of teraslice with same elastic db
+            if (mapping.template) {
+                if (!mapping.template.match(clusterName)) {
+                    mapping.template = `${clusterName}${mapping.template}`;
+                }
+            }
+            return this.putTemplate(mapping, name);
+        }
+        return Promise.resolve(true);
+    }
+    async _createIndex(index = this.defaultIndexName) {
+        // @ts-expect-error TODO: check type missing id
+        const existQuery = { index };
+        const exists = await this.api.indexExists(existQuery);
+        if (!exists) {
+            // Make sure the index exists before we do anything else.
+            const createQuery = {
+                index,
+                body: this.mapping,
+            };
+            try {
+                await this.sendTemplate(this.mapping);
+                return await this.api.indexCreate(createQuery);
+            }
+            catch (err) {
+                // It's not really an error if it's just that the index is already there
+                if (parseError(err).includes('already_exists_exception')) {
+                    return true;
+                }
+                const error = new TSError(err, {
+                    reason: `Could not create index: ${index}`,
+                });
+                throw error;
+            }
+        }
+        // Index already exists. nothing to do.
+        return true;
+    }
+    async refresh(index = this.defaultIndexName) {
+        const query = { index };
+        return this.api.indexRefresh(query);
+    }
+    // TODO: fix type here
+    async putTemplate(template, name) {
+        return this.api.putTemplate(template, name);
+    }
+    verifyClient() {
+        if (this.isShuttingDown)
+            return false;
+        return this.api.verifyClient();
+    }
+    async waitForClient() {
+        if (this.api.verifyClient())
+            return;
+        await pWhile(async () => {
+            if (this.isShuttingDown)
+                throw new Error('Elasticsearch store is shutdown');
+            if (this.api.verifyClient())
+                return true;
+            await pDelay(100);
+            return false;
+        });
+    }
+}
+//# sourceMappingURL=elasticsearch_store.js.map

package/dist/src/lib/storage/backends/mappings/analytics.js

@@ -0,0 +1,50 @@
+export default {
+    template: '__analytics*',
+    mappings: {
+        analytics: {
+            _all: {
+                enabled: false
+            },
+            dynamic: false,
+            properties: {
+                ex_id: {
+                    type: 'keyword'
+                },
+                job_id: {
+                    type: 'keyword'
+                },
+                worker_id: {
+                    type: 'keyword'
+                },
+                slice_id: {
+                    type: 'keyword'
+                },
+                slicer_id: {
+                    type: 'keyword'
+                },
+                op: {
+                    type: 'keyword'
+                },
+                order: {
+                    type: 'integer'
+                },
+                count: {
+                    type: 'integer'
+                },
+                state: {
+                    type: 'keyword'
+                },
+                time: {
+                    type: 'integer'
+                },
+                memory: {
+                    type: 'long'
+                },
+                '@timestamp': {
+                    type: 'date'
+                }
+            }
+        }
+    }
+};
+//# sourceMappingURL=analytics.js.map

package/dist/src/lib/storage/backends/mappings/asset.js

@@ -0,0 +1,41 @@
+export default {
+    mappings: {
+        asset: {
+            _all: {
+                enabled: false
+            },
+            dynamic: false,
+            properties: {
+                blob: {
+                    type: 'binary',
+                    doc_values: false
+                },
+                name: {
+                    type: 'keyword'
+                },
+                version: {
+                    type: 'keyword'
+                },
+                id: {
+                    type: 'keyword'
+                },
+                description: {
+                    type: 'keyword'
+                },
+                arch: {
+                    type: 'keyword'
+                },
+                platform: {
+                    type: 'keyword'
+                },
+                node_version: {
+                    type: 'integer'
+                },
+                _created: {
+                    type: 'date'
+                }
+            }
+        }
+    }
+};
+//# sourceMappingURL=asset.js.map

package/dist/src/lib/storage/backends/mappings/ex.js

@@ -0,0 +1,62 @@
+export default {
+    mappings: {
+        ex: {
+            _all: {
+                enabled: false
+            },
+            dynamic: false,
+            properties: {
+                active: {
+                    type: 'boolean'
+                },
+                job_id: {
+                    type: 'keyword'
+                },
+                ex_id: {
+                    type: 'keyword'
+                },
+                _context: {
+                    type: 'keyword'
+                },
+                _status: {
+                    type: 'keyword'
+                },
+                _has_errors: {
+                    type: 'keyword'
+                },
+                slicer_hostname: {
+                    type: 'keyword'
+                },
+                slicer_port: {
+                    type: 'keyword'
+                },
+                recovered_execution: {
+                    type: 'keyword'
+                },
+                recovered_slice_type: {
+                    type: 'keyword'
+                },
+                metadata: {
+                    type: 'object',
+                    enabled: false
+                },
+                _slicer_stats: {
+                    type: 'object'
+                },
+                _created: {
+                    type: 'date'
+                },
+                _updated: {
+                    type: 'date'
+                },
+                _deleted: {
+                    type: 'boolean'
+                },
+                _deleted_on: {
+                    type: 'date'
+                }
+            }
+        }
+    }
+};
+//# sourceMappingURL=ex.js.map

package/dist/src/lib/storage/backends/mappings/job.js

@@ -0,0 +1,38 @@
+export default {
+    settings: {
+        'index.number_of_shards': 5,
+        'index.number_of_replicas': 1
+    },
+    mappings: {
+        job: {
+            _all: {
+                enabled: false
+            },
+            dynamic: false,
+            properties: {
+                active: {
+                    type: 'boolean'
+                },
+                job_id: {
+                    type: 'keyword'
+                },
+                _context: {
+                    type: 'keyword'
+                },
+                _created: {
+                    type: 'date'
+                },
+                _updated: {
+                    type: 'date'
+                },
+                _deleted: {
+                    type: 'boolean'
+                },
+                _deleted_on: {
+                    type: 'date'
+                }
+            }
+        }
+    }
+};
+//# sourceMappingURL=job.js.map