teraslice 3.3.0 → 3.3.2

package/dist/src/lib/storage/backends/s3_store.js

@@ -1,237 +0,0 @@
-import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/core-utils';
-import { S3ClientResponse, createS3Bucket, deleteS3Object, doesBucketExist, getS3Object, listS3Objects, putS3Object, s3RequestWithRetry } from '@terascope/file-asset-apis';
-import { makeLogger } from '../../workers/helpers/terafoundation.js';
-export class S3Store {
-    bucket;
-    connection;
-    terafoundation;
-    isShuttingDown;
-    logger;
-    api;
-    context;
-    constructor(backendConfig) {
-        const { context, terafoundation, connection, bucket, logger } = backendConfig;
-        this.context = context;
-        this.bucket = bucket || this.createDefaultBucketName();
-        this.connection = connection;
-        this.isShuttingDown = false;
-        this.logger = logger ?? makeLogger(context, 's3_backend', { storageName: this.bucket });
-        this.terafoundation = terafoundation;
-    }
-    async initialize() {
-        const { client } = await this.context.apis.foundation
-            .createClient({ type: 's3', cached: true, endpoint: this.connection });
-        this.api = client;
-        await pWhile(async () => {
-            try {
-                const exists = await doesBucketExist(client, { Bucket: this.bucket });
-                if (!exists) {
-                    await createS3Bucket(client, { Bucket: this.bucket });
-                }
-                const isReady = await this.verifyClient();
-                return isReady;
-            }
-            catch (err) {
-                if (err.Code === 'InvalidAccessKeyId') {
-                    throw new TSError(`accessKeyId ${this.terafoundation.connectors.s3[this.connection].accessKeyId} specified in S3 ${this.connection} does not exit: ${err.message}`);
-                }
-                if (err.Code === 'SignatureDoesNotMatch') {
-                    throw new TSError(`secretAccessKey specified in S3 ${this.connection} does not match accessKeyId: ${err.message}`);
-                }
-                if (err.Code === 'InvalidBucketName') {
-                    throw new TSError(`Bucket name does not follow S3 naming rules: ${err.message}`);
-                }
-                if (err instanceof S3ClientResponse.BucketAlreadyExists) {
-                    throw new TSError(`Bucket name ${this.bucket} not available. accessKeyId ${this.terafoundation.connectors.s3[this.connection].accessKeyId} does not own this bucket. ${err.message}`);
-                }
-                logError(this.logger, err, `Failed attempt connecting to S3 ${this.connection} connection, ${this.bucket} bucket (will retry)`);
-                await pDelay(isTest ? 0 : random(2000, 4000));
-                return false;
-            }
-        });
-    }
-    // TODO: if we want to use the S3 store more generically we can't
-    // assume the key will have a '.zip' extension
-    async get(recordId) {
-        const command = {
-            Bucket: this.bucket,
-            Key: `${recordId}.zip`
-        };
-        try {
-            this.logger.debug(`getting record with id: ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket.`);
-            const client = this.api;
-            const bufferArray = [];
-            let triggerReturn = false;
-            const response = await s3RequestWithRetry({
-                client,
-                func: getS3Object,
-                params: command
-            });
-            /// Convert the response body to a Node read stream
-            const s3Stream = response.Body;
-            /// Store the data coming into s3 into a buffer array
-            s3Stream.on('data', (chunk) => {
-                bufferArray.push(chunk);
-            });
-            s3Stream.on('end', () => {
-                triggerReturn = true;
-            });
-            s3Stream.on('error', (err) => {
-                throw new TSError(`Unable to get recordId ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket.
-                Reason: ${err.message}`);
-            });
-            await pWhile(async () => triggerReturn);
-            return Buffer.concat(bufferArray);
-        }
-        catch (err) {
-            if (err instanceof S3ClientResponse.NoSuchKey) {
-                throw new TSError(`recordId ${recordId} does not exist in s3 ${this.connection} connection, ${this.bucket} bucket.`, {
-                    statusCode: 404
-                });
-            }
-            throw new TSError(`Retrieval of recordId ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket failed: `, err);
-        }
-    }
-    async save(recordId, data, timeout) {
-        try {
-            this.logger.debug(`saving recordId ${recordId} to s3 ${this.connection} connection, ${this.bucket} bucket.`);
-            const command = {
-                Bucket: this.bucket,
-                Key: `${recordId}.zip`,
-                Body: data
-            };
-            const timeoutID = setTimeout(() => {
-                throw new TSError(`Timeout saving recordId ${recordId}`);
-            }, timeout);
-            try {
-                const client = this.api;
-                await s3RequestWithRetry({
-                    client,
-                    func: putS3Object,
-                    params: command
-                });
-            }
-            catch (err) {
-                throw new TSError(`Failure saving recordId ${recordId} to S3: ${err}`);
-            }
-            finally {
-                clearTimeout(timeoutID);
-            }
-        }
-        catch (err) {
-            throw new TSError(`Error saving recordId ${recordId} to S3 ${this.connection} connection, ${this.bucket} bucket: ${err}`);
-        }
-    }
-    async remove(recordId) {
-        try {
-            this.logger.debug(`removing record ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket.`);
-            const command = {
-                Bucket: this.bucket,
-                Key: `${recordId}.zip`
-            };
-            const client = this.api;
-            await s3RequestWithRetry({
-                client,
-                func: deleteS3Object,
-                params: command
-            });
-        }
-        catch (err) {
-            throw new TSError(`Error deleting recordId ${recordId} from s3 ${this.connection} connection, ${this.bucket} bucket: ${err}`);
-        }
-    }
-    async list() {
-        /// list all keys in bucket
-        const objectList = [];
-        let nextContinuationToken;
-        let continuePagination = true;
-        try {
-            do {
-                const command = {
-                    Bucket: this.bucket,
-                    ContinuationToken: nextContinuationToken || undefined,
-                    // MaxKeys: 1000  // Default is 1000
-                };
-                const client = this.api;
-                const response = await s3RequestWithRetry({
-                    client,
-                    func: listS3Objects,
-                    params: command
-                });
-                response.Contents?.forEach((c) => {
-                    const s3Record = {
-                        File: c.Key,
-                        Size: c.Size,
-                        // Created: c.LastModified
-                    };
-                    objectList.push(s3Record);
-                });
-                if (!response.IsTruncated) {
-                    continuePagination = false;
-                    nextContinuationToken = undefined;
-                }
-                else {
-                    nextContinuationToken = response.NextContinuationToken;
-                }
-                const recordsReceived = response.Contents?.length || 0;
-                this.logger.debug(`Received ${recordsReceived} records from s3 ${this.connection} connection, ${this.bucket} bucket.`);
-            } while (continuePagination);
-            this.logger.info(`Found ${objectList.length} records in s3 ${this.connection} connection, ${this.bucket} bucket.`);
-        }
-        catch (err) {
-            throw new TSError(`Error listing records from s3 ${this.connection} connection, ${this.bucket} bucket: ${err}`);
-        }
-        return objectList;
-    }
-    /*
-    * The S3 client has no built in functionality to determine if the client is connected.
-    * If we can make a request to ListS3Objects then we know that the bucket exists and
-    * credentials are valid.
-    */
-    async verifyClient() {
-        if (this.isShuttingDown)
-            return false;
-        const command = {
-            Bucket: this.bucket
-        };
-        const config = this.terafoundation.connectors.s3[this.connection];
-        try {
-            const client = this.api;
-            await s3RequestWithRetry({
-                client,
-                func: listS3Objects,
-                params: command
-            });
-            this.logger.debug(`S3 Client verification succeeded. Connection: ${this.connection}, endpoint: ${config.endpoint}`);
-            return true;
-        }
-        catch (err) {
-            this.logger.debug(`S3 Client verification failed. Connection: ${this.connection}, endpoint: ${config.endpoint}: `, err);
-            return false;
-        }
-    }
-    async waitForClient() {
-        this.logger.debug('waiting for s3 client');
-        if (await this.verifyClient())
-            return;
-        await pWhile(async () => {
-            if (this.isShuttingDown)
-                throw new Error('S3 store is shutdown');
-            if (await this.verifyClient())
-                return true;
-            await pDelay(100);
-            return false;
-        });
-    }
-    async shutdown() {
-        this.isShuttingDown = true;
-        this.api.destroy();
-    }
-    // TODO: Use more generic bucket prefix (not related to assets)
-    // or pass in prefix from calling class
-    createDefaultBucketName() {
-        const safeName = this.context.sysconfig.teraslice.name.replaceAll('_', '-');
-        return `ts-assets-${safeName}`;
-    }
-}
-//# sourceMappingURL=s3_store.js.map

package/dist/src/lib/storage/execution.js

@@ -1,302 +0,0 @@
-import { TSError, includes, getTypeOf, makeISODate } from '@terascope/core-utils';
-import { RecoveryCleanupType } from '@terascope/job-components';
-import { v4 as uuid } from 'uuid';
-import { makeLogger } from '../workers/helpers/terafoundation.js';
-import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
-import { getPackageJSON } from '../utils/file_utils.js';
-const INIT_STATUS = ['pending', 'scheduling', 'initializing'];
-const RUNNING_STATUS = ['recovering', 'running', 'failing', 'paused', 'stopping'];
-const TERMINAL_STATUS = ['completed', 'stopped', 'rejected', 'failed', 'terminated'];
-const VALID_STATUS = INIT_STATUS.concat(RUNNING_STATUS).concat(TERMINAL_STATUS);
-// Module to manager job states in Elasticsearch.
-// All functions in this module return promises that must be resolved to
-// get the final result.
-export class ExecutionStorage {
-    backend;
-    jobType;
-    logger;
-    constructor(context) {
-        const logger = makeLogger(context, 'ex_storage');
-        const config = context.sysconfig.teraslice;
-        const jobType = 'ex';
-        const indexName = `${config.name}__ex`;
-        const backendConfig = {
-            context,
-            indexName,
-            recordType: 'ex',
-            idField: 'ex_id',
-            fullResponse: false,
-            logRecord: false,
-            storageName: 'execution',
-            logger
-        };
-        this.jobType = jobType;
-        this.logger = logger;
-        this.backend = new TerasliceElasticsearchStorage(backendConfig);
-        if (context.apis.executionContext) {
-            context.apis.executionContext.registerMetadataFns({ get: this.getMetadata.bind(this), update: this.updateMetadata.bind(this) });
-        }
-    }
-    async initialize() {
-        await this.backend.initialize();
-        this.logger.info('execution storage initialized');
-    }
-    async get(exId) {
-        const results = await this.backend.get(exId);
-        return results;
-    }
-    // encompasses all executions in either initialization or running statuses
-    async getActiveExecution(exId) {
-        const str = this.getTerminalStatuses().map((state) => ` _status:${state} `)
-            .join('OR');
-        const query = `ex_id:"${exId}" NOT (${str.trim()})`;
-        const executions = await this.backend.search(query, undefined, 1, '_created:desc');
-        if (!executions.length) {
-            throw new TSError(`no active execution context was found for ex_id: ${exId}`, {
-                statusCode: 404
-            });
-        }
-        return executions[0];
-    }
-    async search(query, from, size, sort, fields) {
-        const results = await this.backend.search(query, from, size, sort, fields);
-        return results;
-    }
-    async create(record, status = 'pending') {
-        if (!this._isValidStatus(status)) {
-            throw new Error(`Unknown status "${status}" on execution create`);
-        }
-        if (!record.job_id) {
-            throw new Error('Missing job_id on execution create');
-        }
-        const date = makeISODate();
-        const doc = Object.assign({}, record, {
-            ex_id: uuid(),
-            metadata: {},
-            _status: status,
-            _context: this.jobType,
-            _created: date,
-            _updated: date,
-            _has_errors: false,
-            _slicer_stats: {},
-            _failureReason: '',
-            teraslice_version: `v${getPackageJSON().version}`
-        });
-        // @ts-expect-error
-        delete doc.slicer_port;
-        // @ts-expect-error
-        delete doc.slicer_hostname;
-        try {
-            await this.backend.create(doc);
-        }
-        catch (err) {
-            throw new TSError(err, {
-                reason: 'Failure to create execution context'
-            });
-        }
-        return doc;
-    }
-    async updatePartial(exId, applyChanges) {
-        return this.backend.updatePartial(exId, applyChanges);
-    }
-    /**
-     * @typedef ExErrorMetadata
-     * @property _has_errors {boolean}
-     * @property _failureReason {string}
-     * @property _slicer_stats {import(
-     *  '../workers/execution-controller/execution-analytics.js'
-     * ).ExecutionStats}
-     */
-    /**
-     * Format the execution error stats, primarly used for updating the
-     * status.
-     *
-     * If no error message is passed, it will reset the _has_errors and _failureReason.
-     * If execution stats is provided it will set the _slicer_stats
-     *
-     * @param stats {import(
-     *  '../workers/execution-controller/execution-analytics.js'
-     * ).ExecutionStats=}
-     * @param errMsg {string=}
-     * @return {ExErrorMetadata}
-    */
-    // TODO: type out stats
-    executionMetaData(stats, errMsg) {
-        const errMetadata = {
-            _has_errors: false,
-            _failureReason: ''
-        };
-        const statsMetadata = {};
-        if (errMsg) {
-            errMetadata._has_errors = true;
-            errMetadata._failureReason = errMsg;
-        }
-        if (stats) {
-            statsMetadata._slicer_stats = Object.assign({}, stats);
-        }
-        return Object.assign({}, errMetadata, statsMetadata);
-    }
-    async getMetadata(exId) {
-        const ex = await this.get(exId);
-        return ex.metadata ?? {};
-    }
-    // TODO: type this
-    async updateMetadata(exId, metadata = {}) {
-        await this.backend.update(exId, {
-            metadata,
-            _updated: makeISODate()
-        });
-    }
-    // TODO: put a type of return
-    async getStatus(exId) {
-        try {
-            const result = await this.get(exId);
-            return result._status;
-        }
-        catch (err) {
-            throw new TSError(err, {
-                reason: `Cannot get execution status ${exId}`
-            });
-        }
-    }
-    // TODO: type this
-    // verify the current status to make sure it can be updated to the desired status
-    async verifyStatusUpdate(exId, desiredStatus) {
-        if (!desiredStatus || !this._isValidStatus(desiredStatus)) {
-            throw new TSError(`Invalid Job status: "${desiredStatus}"`, {
-                statusCode: 422
-            });
-        }
-        const status = await this.getStatus(exId);
-        this._verifyStatus(status, desiredStatus);
-    }
-    _verifyStatus(status, desiredStatus) {
-        // when setting the same status to shouldn't throw an error
-        if (desiredStatus === status) {
-            return status;
-        }
-        // when the current status is running it cannot be set to an init status
-        if (this._isRunningStatus(status) && this._isInitStatus(desiredStatus)) {
-            throw new TSError(`Cannot update running job status of "${status}" to init status of "${desiredStatus}"`, {
-                statusCode: 422
-            });
-        }
-        // if it is set to stop but the execution finishes before it can stop
-        // it is okay to set it to completed
-        if (status === 'stopped' && desiredStatus === 'completed') {
-            return status;
-        }
-        // when the status is a terminal status, it cannot be set to again
-        if (this._isTerminalStatus(status)) {
-            throw new TSError(`Cannot update terminal job status of "${status}" to "${desiredStatus}"`, {
-                statusCode: 422
-            });
-        }
-        // otherwise allow the update
-        return status;
-    }
-    // TODO: type this
-    /**
-     * Set the status
-     *
-     * @param {string} exId
-     * @param {string} status
-     * @param {Partial<import('@terascope/job-components').ExecutionConfig>} body
-     * @returns {Promise<import('@terascope/job-components').ExecutionConfig>}
-    */
-    async setStatus(exId, status, body) {
-        try {
-            return await this.updatePartial(exId, async (existing) => {
-                this._verifyStatus(existing._status, status);
-                return Object.assign(existing, body, {
-                    _status: status,
-                    _updated: makeISODate()
-                });
-            });
-        }
-        catch (err) {
-            throw new TSError(err, {
-                statusCode: 422,
-                reason: `Unable to set execution ${exId} status code to ${status}`
-            });
-        }
-    }
-    async softDelete(exId) {
-        try {
-            const date = makeISODate();
-            return await this.updatePartial(exId, async (existing) => Object.assign(existing, {
-                _deleted: true,
-                _deleted_on: date,
-                _updated: date
-            }));
-        }
-        catch (err) {
-            throw new TSError(err, {
-                statusCode: 422,
-                reason: `Unable to delete execution ${exId}`
-            });
-        }
-    }
-    async remove(exId) {
-        return this.backend.remove(exId);
-    }
-    async shutdown(forceShutdown) {
-        this.logger.info('shutting down.');
-        return this.backend.shutdown(forceShutdown);
-    }
-    verifyClient() {
-        return this.backend.verifyClient();
-    }
-    async waitForClient() {
-        return this.backend.waitForClient();
-    }
-    getTerminalStatuses() {
-        return TERMINAL_STATUS.slice();
-    }
-    getRunningStatuses() {
-        return RUNNING_STATUS.slice();
-    }
-    getLivingStatuses() {
-        return INIT_STATUS.concat(RUNNING_STATUS);
-    }
-    _isValidStatus(status) {
-        return includes(VALID_STATUS, status);
-    }
-    _isRunningStatus(status) {
-        return includes(RUNNING_STATUS, status);
-    }
-    _isTerminalStatus(status) {
-        return includes(TERMINAL_STATUS, status);
-    }
-    _isInitStatus(status) {
-        return includes(INIT_STATUS, status);
-    }
-    // TODO: fix types
-    /**
-     * @param {import('@terascope/job-components').ExecutionConfig} recoverFrom
-     * @param {RecoveryCleanupType} [cleanupType]
-     * @returns {Promise<import('@terascope/job-components').ExecutionConfig>}
-    */
-    async createRecoveredExecution(recoverFrom, cleanupType) {
-        if (!recoverFrom) {
-            throw new Error(`Invalid execution given, got ${getTypeOf(recoverFrom)}`);
-        }
-        if (!recoverFrom.ex_id) {
-            throw new Error('Unable to recover execution with missing ex_id');
-        }
-        const recoverFromId = recoverFrom.ex_id;
-        const ex = Object.assign({}, recoverFrom);
-        if (cleanupType && !RecoveryCleanupType[cleanupType]) {
-            throw new Error(`Unknown cleanup type "${cleanupType}" to recover`);
-        }
-        ex.recovered_execution = recoverFromId;
-        if (cleanupType) {
-            ex.recovered_slice_type = cleanupType;
-        }
-        else if (ex.autorecover) {
-            ex.recovered_slice_type = RecoveryCleanupType.pending;
-        }
-        return this.create(ex);
-    }
-}
-//# sourceMappingURL=execution.js.map

package/dist/src/lib/storage/index.js

@@ -1,7 +0,0 @@
-import { AnalyticsStorage } from './analytics.js';
-import { AssetsStorage } from './assets.js';
-import { ExecutionStorage } from './execution.js';
-import { JobsStorage } from './jobs.js';
-import { StateStorage, SliceState } from './state.js';
-export { SliceState, AnalyticsStorage, AssetsStorage, ExecutionStorage, JobsStorage, StateStorage, };
-//# sourceMappingURL=index.js.map

package/dist/src/lib/storage/jobs.js

@@ -1,81 +0,0 @@
-import { v4 as uuid } from 'uuid';
-import { TSError, makeISODate } from '@terascope/core-utils';
-import { makeLogger } from '../workers/helpers/terafoundation.js';
-import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
-export class JobsStorage {
-    backend;
-    jobType;
-    logger;
-    constructor(context) {
-        const logger = makeLogger(context, 'job_storage');
-        const config = context.sysconfig.teraslice;
-        const jobType = 'job';
-        const indexName = `${config.name}__jobs`;
-        const backendConfig = {
-            context,
-            indexName,
-            recordType: 'job',
-            idField: 'job_id',
-            fullResponse: false,
-            logRecord: false,
-            storageName: 'jobs',
-            logger
-        };
-        this.logger = logger;
-        this.jobType = jobType;
-        this.backend = new TerasliceElasticsearchStorage(backendConfig);
-    }
-    async initialize() {
-        await this.backend.initialize();
-        this.logger.info('job storage initialized');
-    }
-    async get(jobId) {
-        const doc = await this.backend.get(jobId);
-        return doc;
-    }
-    async search(query, from, size, sort, fields) {
-        const results = await this.backend.search(query, from, size, sort, fields);
-        return results;
-    }
-    async create(record) {
-        const date = makeISODate();
-        const doc = Object.assign({}, record, {
-            job_id: uuid(),
-            _context: this.jobType,
-            _created: date,
-            _updated: date
-        });
-        try {
-            await this.backend.create(doc);
-        }
-        catch (err) {
-            throw new TSError(err, {
-                reason: 'Failure to create job'
-            });
-        }
-        return doc;
-    }
-    async update(jobId, updateSpec) {
-        // We want to save the whole job as it is posted, update api does partial doc updates
-        const results = await this.backend.indexWithId(jobId, Object.assign({}, updateSpec, {
-            job_id: jobId,
-            _context: this.jobType,
-            _updated: makeISODate()
-        }));
-        return results;
-    }
-    async remove(jobId) {
-        return this.backend.remove(jobId);
-    }
-    async shutdown(forceShutdown) {
-        this.logger.info('shutting down.');
-        return this.backend.shutdown(forceShutdown);
-    }
-    verifyClient() {
-        return this.backend.verifyClient();
-    }
-    async waitForClient() {
-        return this.backend.waitForClient();
-    }
-}
-//# sourceMappingURL=jobs.js.map