teraslice 2.11.0 → 2.12.0

package/dist/src/lib/storage/backends/mappings/state.js

@@ -0,0 +1,38 @@
+export default {
+    template: '__state*',
+    mappings: {
+        state: {
+            _all: {
+                enabled: false
+            },
+            dynamic: false,
+            properties: {
+                ex_id: {
+                    type: 'keyword'
+                },
+                slice_id: {
+                    type: 'keyword'
+                },
+                slicer_id: {
+                    type: 'keyword'
+                },
+                slicer_order: {
+                    type: 'integer'
+                },
+                state: {
+                    type: 'keyword'
+                },
+                _created: {
+                    type: 'date'
+                },
+                _updated: {
+                    type: 'date'
+                },
+                error: {
+                    type: 'keyword'
+                }
+            }
+        }
+    }
+};
+//# sourceMappingURL=state.js.map

package/dist/src/lib/storage/backends/s3_store.js

@@ -0,0 +1,237 @@
+import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/utils';
+import { S3ClientResponse, createS3Bucket, deleteS3Object, doesBucketExist, getS3Object, listS3Objects, putS3Object, s3RequestWithRetry } from '@terascope/file-asset-apis';
+import { makeLogger } from '../../workers/helpers/terafoundation.js';
+export class S3Store {
+    bucket;
+    connection;
+    terafoundation;
+    isShuttingDown;
+    logger;
+    api;
+    context;
+    constructor(backendConfig) {
+        const { context, terafoundation, connection, bucket, logger } = backendConfig;
+        this.context = context;
+        this.bucket = bucket || this.createDefaultBucketName();
+        this.connection = connection;
+        this.isShuttingDown = false;
+        this.logger = logger ?? makeLogger(context, 's3_backend', { storageName: this.bucket });
+        this.terafoundation = terafoundation;
+    }
+    async initialize() {
+        const { client } = await this.context.apis.foundation
+            .createClient({ type: 's3', cached: true, endpoint: this.connection });
+        this.api = client;
+        await pWhile(async () => {
+            try {
+                const exists = await doesBucketExist(client, { Bucket: this.bucket });
+                if (!exists) {
+                    await createS3Bucket(client, { Bucket: this.bucket });
+                }
+                const isReady = await this.verifyClient();
+                return isReady;
+            }
+            catch (err) {
+                if (err.Code === 'InvalidAccessKeyId') {
+                    throw new TSError(`accessKeyId ${this.terafoundation.connectors.s3[this.connection].accessKeyId} specified in S3 ${this.connection} does not exit: ${err.message}`);
+                }
+                if (err.Code === 'SignatureDoesNotMatch') {
+                    throw new TSError(`secretAccessKey specified in S3 ${this.connection} does not match accessKeyId: ${err.message}`);
+                }
+                if (err.Code === 'InvalidBucketName') {
+                    throw new TSError(`Bucket name does not follow S3 naming rules: ${err.message}`);
+                }
+                if (err instanceof S3ClientResponse.BucketAlreadyExists) {
+                    throw new TSError(`Bucket name ${this.bucket} not available. accessKeyId ${this.terafoundation.connectors.s3[this.connection].accessKeyId} does not own this bucket. ${err.message}`);
+                }
+                logError(this.logger, err, `Failed attempt connecting to S3 ${this.connection} connection, ${this.bucket} bucket (will retry)`);
+                await pDelay(isTest ? 0 : random(2000, 4000));
+                return false;
+            }
+        });
+    }
+    // TODO: if we want to use the S3 store more generically we can't
+    // assume the key will have a '.zip' extension
+    async get(recordId) {
+        const command = {
+            Bucket: this.bucket,
+            Key: `${recordId}.zip`
+        };
+        try {
+            this.logger.debug(`getting record with id: ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket.`);
+            const client = this.api;
+            const bufferArray = [];
+            let triggerReturn = false;
+            const response = await s3RequestWithRetry({
+                client,
+                func: getS3Object,
+                params: command
+            });
+            /// Convert the response body to a Node read stream
+            const s3Stream = response.Body;
+            /// Store the data coming into s3 into a buffer array
+            s3Stream.on('data', (chunk) => {
+                bufferArray.push(chunk);
+            });
+            s3Stream.on('end', () => {
+                triggerReturn = true;
+            });
+            s3Stream.on('error', (err) => {
+                throw new TSError(`Unable to get recordId ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket.
+                Reason: ${err.message}`);
+            });
+            await pWhile(async () => triggerReturn);
+            return Buffer.concat(bufferArray);
+        }
+        catch (err) {
+            if (err instanceof S3ClientResponse.NoSuchKey) {
+                throw new TSError(`recordId ${recordId} does not exist in s3 ${this.connection} connection, ${this.bucket} bucket.`, {
+                    statusCode: 404
+                });
+            }
+            throw new TSError(`Retrieval of recordId ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket failed: `, err);
+        }
+    }
+    async save(recordId, data, timeout) {
+        try {
+            this.logger.debug(`saving recordId ${recordId} to s3 ${this.connection} connection, ${this.bucket} bucket.`);
+            const command = {
+                Bucket: this.bucket,
+                Key: `${recordId}.zip`,
+                Body: data
+            };
+            const timeoutID = setTimeout(() => {
+                throw new TSError(`Timeout saving recordId ${recordId}`);
+            }, timeout);
+            try {
+                const client = this.api;
+                await s3RequestWithRetry({
+                    client,
+                    func: putS3Object,
+                    params: command
+                });
+            }
+            catch (err) {
+                throw new TSError(`Failure saving recordId ${recordId} to S3: ${err}`);
+            }
+            finally {
+                clearTimeout(timeoutID);
+            }
+        }
+        catch (err) {
+            throw new TSError(`Error saving recordId ${recordId} to S3 ${this.connection} connection, ${this.bucket} bucket: ${err}`);
+        }
+    }
+    async remove(recordId) {
+        try {
+            this.logger.debug(`removing record ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket.`);
+            const command = {
+                Bucket: this.bucket,
+                Key: `${recordId}.zip`
+            };
+            const client = this.api;
+            await s3RequestWithRetry({
+                client,
+                func: deleteS3Object,
+                params: command
+            });
+        }
+        catch (err) {
+            throw new TSError(`Error deleting recordId ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket: ${err}`);
+        }
+    }
+    async list() {
+        /// list all keys in bucket
+        const objectList = [];
+        let nextContinuationToken;
+        let continuePagination = true;
+        try {
+            do {
+                const command = {
+                    Bucket: this.bucket,
+                    ContinuationToken: nextContinuationToken || undefined,
+                    // MaxKeys: 1000  // Default is 1000
+                };
+                const client = this.api;
+                const response = await s3RequestWithRetry({
+                    client,
+                    func: listS3Objects,
+                    params: command
+                });
+                response.Contents?.forEach((c) => {
+                    const s3Record = {
+                        File: c.Key,
+                        Size: c.Size,
+                        // Created: c.LastModified
+                    };
+                    objectList.push(s3Record);
+                });
+                if (!response.IsTruncated) {
+                    continuePagination = false;
+                    nextContinuationToken = undefined;
+                }
+                else {
+                    nextContinuationToken = response.NextContinuationToken;
+                }
+                const recordsReceived = response.Contents?.length || 0;
+                this.logger.debug(`Received ${recordsReceived} records from s3 ${this.connection} connection, ${this.bucket} bucket.`);
+            } while (continuePagination);
+            this.logger.info(`Found ${objectList.length} records in s3 ${this.connection} connection, ${this.bucket} bucket.`);
+        }
+        catch (err) {
+            throw new TSError(`Error listing records from s3 ${this.connection} connection, ${this.bucket} bucket: ${err}`);
+        }
+        return objectList;
+    }
+    /*
+    * The S3 client has no built in functionality to determine if the client is connected.
+    * If we can make a request to ListS3Objects then we know that the bucket exists and
+    * credentials are valid.
+    */
+    async verifyClient() {
+        if (this.isShuttingDown)
+            return false;
+        const command = {
+            Bucket: this.bucket
+        };
+        const config = this.terafoundation.connectors.s3[this.connection];
+        try {
+            const client = this.api;
+            await s3RequestWithRetry({
+                client,
+                func: listS3Objects,
+                params: command
+            });
+            this.logger.debug(`S3 Client verification succeeded. Connection: ${this.connection}, endpoint: ${config.endpoint}`);
+            return true;
+        }
+        catch (err) {
+            this.logger.debug(`S3 Client verification failed. Connection: ${this.connection}, endpoint: ${config.endpoint}: `, err);
+            return false;
+        }
+    }
+    async waitForClient() {
+        this.logger.debug('waiting for s3 client');
+        if (await this.verifyClient())
+            return;
+        await pWhile(async () => {
+            if (this.isShuttingDown)
+                throw new Error('S3 store is shutdown');
+            if (await this.verifyClient())
+                return true;
+            await pDelay(100);
+            return false;
+        });
+    }
+    async shutdown() {
+        this.isShuttingDown = true;
+        this.api.destroy();
+    }
+    // TODO: Use more generic bucket prefix (not related to assets)
+    // or pass in prefix from calling class
+    createDefaultBucketName() {
+        const safeName = this.context.sysconfig.teraslice.name.replaceAll('_', '-');
+        return `ts-assets-${safeName}`;
+    }
+}
+//# sourceMappingURL=s3_store.js.map

package/dist/src/lib/storage/execution.js

@@ -0,0 +1,300 @@
+import { TSError, includes, getTypeOf, makeISODate } from '@terascope/utils';
+import { RecoveryCleanupType } from '@terascope/job-components';
+import { v4 as uuid } from 'uuid';
+import { makeLogger } from '../workers/helpers/terafoundation.js';
+import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
+const INIT_STATUS = ['pending', 'scheduling', 'initializing'];
+const RUNNING_STATUS = ['recovering', 'running', 'failing', 'paused', 'stopping'];
+const TERMINAL_STATUS = ['completed', 'stopped', 'rejected', 'failed', 'terminated'];
+const VALID_STATUS = INIT_STATUS.concat(RUNNING_STATUS).concat(TERMINAL_STATUS);
+// Module to manager job states in Elasticsearch.
+// All functions in this module return promises that must be resolved to
+// get the final result.
+export class ExecutionStorage {
+    backend;
+    jobType;
+    logger;
+    constructor(context) {
+        const logger = makeLogger(context, 'ex_storage');
+        const config = context.sysconfig.teraslice;
+        const jobType = 'ex';
+        const indexName = `${config.name}__ex`;
+        const backendConfig = {
+            context,
+            indexName,
+            recordType: 'ex',
+            idField: 'ex_id',
+            fullResponse: false,
+            logRecord: false,
+            storageName: 'execution',
+            logger
+        };
+        this.jobType = jobType;
+        this.logger = logger;
+        this.backend = new TerasliceElasticsearchStorage(backendConfig);
+        if (context.apis.executionContext) {
+            context.apis.executionContext.registerMetadataFns({ get: this.getMetadata.bind(this), update: this.updateMetadata.bind(this) });
+        }
+    }
+    async initialize() {
+        await this.backend.initialize();
+        this.logger.info('execution storage initialized');
+    }
+    async get(exId) {
+        const results = await this.backend.get(exId);
+        return results;
+    }
+    // encompasses all executions in either initialization or running statuses
+    async getActiveExecution(exId) {
+        const str = this.getTerminalStatuses().map((state) => ` _status:${state} `)
+            .join('OR');
+        const query = `ex_id:"${exId}" NOT (${str.trim()})`;
+        const executions = await this.backend.search(query, undefined, 1, '_created:desc');
+        if (!executions.length) {
+            throw new TSError(`no active execution context was found for ex_id: ${exId}`, {
+                statusCode: 404
+            });
+        }
+        return executions[0];
+    }
+    async search(query, from, size, sort, fields) {
+        const results = await this.backend.search(query, from, size, sort, fields);
+        return results;
+    }
+    async create(record, status = 'pending') {
+        if (!this._isValidStatus(status)) {
+            throw new Error(`Unknown status "${status}" on execution create`);
+        }
+        if (!record.job_id) {
+            throw new Error('Missing job_id on execution create');
+        }
+        const date = makeISODate();
+        const doc = Object.assign({}, record, {
+            ex_id: uuid(),
+            metadata: {},
+            _status: status,
+            _context: this.jobType,
+            _created: date,
+            _updated: date,
+            _has_errors: false,
+            _slicer_stats: {},
+            _failureReason: ''
+        });
+        // @ts-expect-error
+        delete doc.slicer_port;
+        // @ts-expect-error
+        delete doc.slicer_hostname;
+        try {
+            await this.backend.create(doc);
+        }
+        catch (err) {
+            throw new TSError(err, {
+                reason: 'Failure to create execution context'
+            });
+        }
+        return doc;
+    }
+    async updatePartial(exId, applyChanges) {
+        return this.backend.updatePartial(exId, applyChanges);
+    }
+    /**
+     * @typedef ExErrorMetadata
+     * @property _has_errors {boolean}
+     * @property _failureReason {string}
+     * @property _slicer_stats {import(
+     *  '../workers/execution-controller/execution-analytics.js'
+     * ).ExecutionStats}
+     */
+    /**
+     * Format the execution error stats, primarly used for updating the
+     * status.
+     *
+     * If no error message is passed, it will reset the _has_errors and _failureReason.
+     * If execution stats is provided it will set the _slicer_stats
+     *
+     * @param stats {import(
+     *  '../workers/execution-controller/execution-analytics.js'
+     * ).ExecutionStats=}
+     * @param errMsg {string=}
+     * @return {ExErrorMetadata}
+    */
+    // TODO: type out stats
+    executionMetaData(stats, errMsg) {
+        const errMetadata = {
+            _has_errors: false,
+            _failureReason: ''
+        };
+        const statsMetadata = {};
+        if (errMsg) {
+            errMetadata._has_errors = true;
+            errMetadata._failureReason = errMsg;
+        }
+        if (stats) {
+            statsMetadata._slicer_stats = Object.assign({}, stats);
+        }
+        return Object.assign({}, errMetadata, statsMetadata);
+    }
+    async getMetadata(exId) {
+        const ex = await this.get(exId);
+        return ex.metadata ?? {};
+    }
+    // TODO: type this
+    async updateMetadata(exId, metadata = {}) {
+        await this.backend.update(exId, {
+            metadata,
+            _updated: makeISODate()
+        });
+    }
+    // TODO: put a type of return
+    async getStatus(exId) {
+        try {
+            const result = await this.get(exId);
+            return result._status;
+        }
+        catch (err) {
+            throw new TSError(err, {
+                reason: `Cannot get execution status ${exId}`
+            });
+        }
+    }
+    // TODO: type this
+    // verify the current status to make sure it can be updated to the desired status
+    async verifyStatusUpdate(exId, desiredStatus) {
+        if (!desiredStatus || !this._isValidStatus(desiredStatus)) {
+            throw new TSError(`Invalid Job status: "${desiredStatus}"`, {
+                statusCode: 422
+            });
+        }
+        const status = await this.getStatus(exId);
+        this._verifyStatus(status, desiredStatus);
+    }
+    _verifyStatus(status, desiredStatus) {
+        // when setting the same status to shouldn't throw an error
+        if (desiredStatus === status) {
+            return status;
+        }
+        // when the current status is running it cannot be set to an init status
+        if (this._isRunningStatus(status) && this._isInitStatus(desiredStatus)) {
+            throw new TSError(`Cannot update running job status of "${status}" to init status of "${desiredStatus}"`, {
+                statusCode: 422
+            });
+        }
+        // if it is set to stop but the execution finishes before it can stop
+        // it is okay to set it to completed
+        if (status === 'stopped' && desiredStatus === 'completed') {
+            return status;
+        }
+        // when the status is a terminal status, it cannot be set to again
+        if (this._isTerminalStatus(status)) {
+            throw new TSError(`Cannot update terminal job status of "${status}" to "${desiredStatus}"`, {
+                statusCode: 422
+            });
+        }
+        // otherwise allow the update
+        return status;
+    }
+    // TODO: type this
+    /**
+     * Set the status
+     *
+     * @param {string} exId
+     * @param {string} status
+     * @param {Partial<import('@terascope/job-components').ExecutionConfig>} body
+     * @returns {Promise<import('@terascope/job-components').ExecutionConfig>}
+    */
+    async setStatus(exId, status, body) {
+        try {
+            return await this.updatePartial(exId, async (existing) => {
+                this._verifyStatus(existing._status, status);
+                return Object.assign(existing, body, {
+                    _status: status,
+                    _updated: makeISODate()
+                });
+            });
+        }
+        catch (err) {
+            throw new TSError(err, {
+                statusCode: 422,
+                reason: `Unable to set execution ${exId} status code to ${status}`
+            });
+        }
+    }
+    async softDelete(exId) {
+        try {
+            const date = makeISODate();
+            return await this.updatePartial(exId, async (existing) => Object.assign(existing, {
+                _deleted: true,
+                _deleted_on: date,
+                _updated: date
+            }));
+        }
+        catch (err) {
+            throw new TSError(err, {
+                statusCode: 422,
+                reason: `Unable to delete execution ${exId}`
+            });
+        }
+    }
+    async remove(exId) {
+        return this.backend.remove(exId);
+    }
+    async shutdown(forceShutdown) {
+        this.logger.info('shutting down.');
+        return this.backend.shutdown(forceShutdown);
+    }
+    verifyClient() {
+        return this.backend.verifyClient();
+    }
+    async waitForClient() {
+        return this.backend.waitForClient();
+    }
+    getTerminalStatuses() {
+        return TERMINAL_STATUS.slice();
+    }
+    getRunningStatuses() {
+        return RUNNING_STATUS.slice();
+    }
+    getLivingStatuses() {
+        return INIT_STATUS.concat(RUNNING_STATUS);
+    }
+    _isValidStatus(status) {
+        return includes(VALID_STATUS, status);
+    }
+    _isRunningStatus(status) {
+        return includes(RUNNING_STATUS, status);
+    }
+    _isTerminalStatus(status) {
+        return includes(TERMINAL_STATUS, status);
+    }
+    _isInitStatus(status) {
+        return includes(INIT_STATUS, status);
+    }
+    // TODO: fix types
+    /**
+     * @param {import('@terascope/job-components').ExecutionConfig} recoverFrom
+     * @param {RecoveryCleanupType} [cleanupType]
+     * @returns {Promise<import('@terascope/job-components').ExecutionConfig>}
+    */
+    async createRecoveredExecution(recoverFrom, cleanupType) {
+        if (!recoverFrom) {
+            throw new Error(`Invalid execution given, got ${getTypeOf(recoverFrom)}`);
+        }
+        if (!recoverFrom.ex_id) {
+            throw new Error('Unable to recover execution with missing ex_id');
+        }
+        const recoverFromId = recoverFrom.ex_id;
+        const ex = Object.assign({}, recoverFrom);
+        if (cleanupType && !RecoveryCleanupType[cleanupType]) {
+            throw new Error(`Unknown cleanup type "${cleanupType}" to recover`);
+        }
+        ex.recovered_execution = recoverFromId;
+        if (cleanupType) {
+            ex.recovered_slice_type = cleanupType;
+        }
+        else if (ex.autorecover) {
+            ex.recovered_slice_type = RecoveryCleanupType.pending;
+        }
+        return this.create(ex);
+    }
+}
+//# sourceMappingURL=execution.js.map

package/dist/src/lib/storage/index.js

@@ -0,0 +1,7 @@
+import { AnalyticsStorage } from './analytics.js';
+import { AssetsStorage } from './assets.js';
+import { ExecutionStorage } from './execution.js';
+import { JobsStorage } from './jobs.js';
+import { StateStorage, SliceState } from './state.js';
+export { SliceState, AnalyticsStorage, AssetsStorage, ExecutionStorage, JobsStorage, StateStorage, };
+//# sourceMappingURL=index.js.map

package/dist/src/lib/storage/jobs.js

@@ -0,0 +1,81 @@
+import { v4 as uuid } from 'uuid';
+import { TSError, makeISODate } from '@terascope/utils';
+import { makeLogger } from '../workers/helpers/terafoundation.js';
+import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
+export class JobsStorage {
+    backend;
+    jobType;
+    logger;
+    constructor(context) {
+        const logger = makeLogger(context, 'job_storage');
+        const config = context.sysconfig.teraslice;
+        const jobType = 'job';
+        const indexName = `${config.name}__jobs`;
+        const backendConfig = {
+            context,
+            indexName,
+            recordType: 'job',
+            idField: 'job_id',
+            fullResponse: false,
+            logRecord: false,
+            storageName: 'jobs',
+            logger
+        };
+        this.logger = logger;
+        this.jobType = jobType;
+        this.backend = new TerasliceElasticsearchStorage(backendConfig);
+    }
+    async initialize() {
+        await this.backend.initialize();
+        this.logger.info('job storage initialized');
+    }
+    async get(jobId) {
+        const doc = await this.backend.get(jobId);
+        return doc;
+    }
+    async search(query, from, size, sort, fields) {
+        const results = await this.backend.search(query, from, size, sort, fields);
+        return results;
+    }
+    async create(record) {
+        const date = makeISODate();
+        const doc = Object.assign({}, record, {
+            job_id: uuid(),
+            _context: this.jobType,
+            _created: date,
+            _updated: date
+        });
+        try {
+            await this.backend.create(doc);
+        }
+        catch (err) {
+            throw new TSError(err, {
+                reason: 'Failure to create job'
+            });
+        }
+        return doc;
+    }
+    async update(jobId, updateSpec) {
+        // We want to save the whole job as it is posted, update api does partial doc updates
+        const results = await this.backend.indexWithId(jobId, Object.assign({}, updateSpec, {
+            job_id: jobId,
+            _context: this.jobType,
+            _updated: makeISODate()
+        }));
+        return results;
+    }
+    async remove(jobId) {
+        return this.backend.remove(jobId);
+    }
+    async shutdown(forceShutdown) {
+        this.logger.info('shutting down.');
+        return this.backend.shutdown(forceShutdown);
+    }
+    verifyClient() {
+        return this.backend.verifyClient();
+    }
+    async waitForClient() {
+        return this.backend.waitForClient();
+    }
+}
+//# sourceMappingURL=jobs.js.map