teraslice 3.3.0 → 3.3.1

package/dist/src/lib/cluster/services/assets.js

@@ -1,224 +0,0 @@
-import express from 'express';
-import { TSError, parseErrorInfo, logError, toBoolean } from '@terascope/core-utils';
-import { makeLogger } from '../../workers/helpers/terafoundation.js';
-import { AssetsStorage } from '../../storage/index.js';
-import { getBackendConfig } from '../../storage/assets.js';
-import { makeTable, handleTerasliceRequest, getSearchOptions, sendError, } from '../../utils/api_utils.js';
-export class AssetsService {
-    context;
-    assetsStorage;
-    logger;
-    port;
-    app;
-    running = false;
-    constructor(context) {
-        this.context = context;
-        this.logger = makeLogger(context, 'assets_service');
-        this.app = express();
-        const { port } = process.env;
-        this.port = port;
-        this.app.set('json spaces', 4);
-        this.app.use((req, _res, next) => {
-            // @ts-expect-error
-            req.logger = this.logger;
-            next();
-        });
-    }
-    async initialize() {
-        try {
-            this.assetsStorage = new AssetsStorage(this.context);
-            await this.assetsStorage.initialize();
-            this.app.get('/status', (req, res) => {
-                const requestHandler = handleTerasliceRequest(req, res);
-                requestHandler(async () => ({ available: this.running }));
-            });
-            this.app.post('/assets', (req, res) => {
-                const blocking = toBoolean(req.query.blocking);
-                this.logger.debug('loading an asset', { blocking });
-                const results = [];
-                req.on('data', (buff) => {
-                    results.push(buff);
-                });
-                req.on('end', async () => {
-                    try {
-                        const data = Buffer.concat(results);
-                        const { assetId, created } = await this.assetsStorage.save(data, blocking);
-                        const code = created ? 201 : 200;
-                        const assetResponse = {
-                            asset_id: assetId,
-                        };
-                        res.status(code).json(assetResponse);
-                    }
-                    catch (err) {
-                        const { statusCode, message } = parseErrorInfo(err);
-                        logError(this.logger, err, 'failure saving assets via proxy request');
-                        sendError(res, statusCode, message, this.logger);
-                    }
-                });
-                req.on('error', (err) => {
-                    const { statusCode, message } = parseErrorInfo(err);
-                    logError(this.logger, err, 'failure writing asset');
-                    res.status(statusCode).send(message);
-                });
-            });
-            this.app.delete('/assets/:assetId', (req, res) => {
-                const { assetId } = req.params;
-                // @ts-expect-error
-                const requestHandler = handleTerasliceRequest(req, res, `Could not delete asset ${assetId}`);
-                if (assetId.length !== 40) {
-                    res.status(400).json({
-                        error: `asset ${assetId} is not formatted correctly, please provide the full asset_id`
-                    });
-                }
-                else {
-                    const assetResponse = {
-                        asset_id: assetId,
-                    };
-                    requestHandler(async () => {
-                        await this.assetsStorage.remove(assetId);
-                        return assetResponse;
-                    });
-                }
-            });
-            this.app.get('/txt/assets', (req, res) => {
-                const query = 'id:*';
-                this.createAssetTable(query, req, res);
-            });
-            this.app.get('/txt/assets/:name', (req, res) => {
-                const query = `id:* AND name:"${req.params.name}"`;
-                this.createAssetTable(query, req, res);
-            });
-            this.app.get('/txt/assets/:name/:version', (req, res) => {
-                const query = `id:* AND name:"${req.params.name}" AND version:"${req.params.version}"`;
-                this.createAssetTable(query, req, res);
-            });
-            this.app.get('/assets', (req, res) => {
-                const query = 'id:*';
-                this.assetsSearch(query, req, res);
-            });
-            this.app.get('/assets/:name', (req, res) => {
-                const query = `id:* AND name:"${req.params.name}"`;
-                this.assetsSearch(query, req, res);
-            });
-            this.app.get('/assets/:name/:version', (req, res) => {
-                const query = `id:* AND name:"${req.params.name}" AND version:"${req.params.version}"`;
-                this.assetsSearch(query, req, res);
-            });
-            await new Promise((resolve, reject) => {
-                // @ts-expect-error
-                this.app.listen(this.port, (err) => {
-                    if (err) {
-                        reject(err);
-                        return;
-                    }
-                    this.logger.info(`assets_service is listening on port ${this.port}`);
-                    resolve(true);
-                });
-                // @ts-expect-error TODO: verify this
-                this.app.timeout = this.context.sysconfig.teraslice.api_response_timeout;
-            });
-            await this.assetsStorage.autoload();
-            this.running = true;
-        }
-        catch (err) {
-            this.running = false;
-            throw new TSError(err, {
-                reason: 'Failure while creating assets_service'
-            });
-        }
-    }
-    getS3AssetStatus(s3List, esList) {
-        const result = [...esList];
-        for (const esRecord of result) {
-            esRecord.external_storage = 'missing';
-            for (const s3Record of s3List) {
-                /// s3AssetId is just the file name without the .zip
-                const s3AssetId = s3Record.File.slice(0, -4);
-                if (s3AssetId === esRecord.id) {
-                    esRecord.external_storage = 'available';
-                    break;
-                }
-            }
-        }
-        return result;
-    }
-    createAssetTable(query, req, res) {
-        const { size, from, sort } = getSearchOptions(req, '_created:desc');
-        const defaults = [
-            'name',
-            'version',
-            'id',
-            '_created',
-            'description',
-            'node_version',
-            'platform',
-            'arch'
-        ];
-        const s3Defaults = [...defaults, 'external_storage'];
-        function mapping(item) {
-            return (field) => {
-                if (field === 'description') {
-                    return item[field] ? item[field].slice(0, 30) : item[field];
-                }
-                return item[field];
-            };
-        }
-        const requestHandler = handleTerasliceRequest(req, res, 'Could not get assets');
-        requestHandler(async () => {
-            const results = await this.assetsStorage.search(query, from, size, sort, defaults);
-            const assets = results.hits.hits.map((asset) => {
-                const record = asset._source;
-                record.id = asset._id;
-                return record;
-            });
-            const { assetConnectionType } = getBackendConfig(this.context, this.logger);
-            if (assetConnectionType === 's3') {
-                const s3Assets = await this.assetsStorage.grabS3Info();
-                const updatedAssets = this.getS3AssetStatus(s3Assets, assets);
-                return makeTable(req, s3Defaults, updatedAssets, mapping);
-            }
-            return makeTable(req, defaults, assets, mapping);
-        });
-    }
-    assetsSearch(query, req, res) {
-        const { size, from, sort } = getSearchOptions(req, '_created:desc');
-        const requestHandler = handleTerasliceRequest(req, res, 'Could not get assets');
-        requestHandler(async () => {
-            const fields = ['_created', 'name', 'version', 'description', 'node_version', 'platform', 'arch'];
-            const results = await this.assetsStorage.search(query, from, size, sort, fields);
-            const mappedRecords = results.hits.hits.map((asset) => {
-                const record = asset._source;
-                record.id = asset._id;
-                return record;
-            });
-            const { assetConnectionType } = getBackendConfig(this.context, this.logger);
-            if (assetConnectionType === 's3') {
-                const s3Assets = await this.assetsStorage.grabS3Info();
-                const updatedAssets = this.getS3AssetStatus(s3Assets, mappedRecords);
-                return updatedAssets;
-            }
-            return mappedRecords;
-        });
-    }
-    run() {
-        return new Promise((resolve) => {
-            if (!this.running) {
-                resolve(true);
-                return;
-            }
-            const runningInterval = setInterval(() => {
-                if (!this.running) {
-                    clearInterval(runningInterval);
-                    resolve(true);
-                }
-            }, 1000);
-        });
-    }
-    async shutdown() {
-        this.running = false;
-        if (!this.assetsStorage)
-            return;
-        await this.assetsStorage.shutdown(true);
-    }
-}
-//# sourceMappingURL=assets.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js

@@ -1,192 +0,0 @@
-import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/core-utils';
-import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
-import { gen } from './k8sState.js';
-import { K8s } from './k8s.js';
-import { getRetryConfig } from './utils.js';
-import { K8sJobResource } from './k8sJobResource.js';
-import { K8sServiceResource } from './k8sServiceResource.js';
-import { K8sDeploymentResource } from './k8sDeploymentResource.js';
-/*
- Execution Life Cycle for _status
- pending -> scheduling -> running -> [ paused -> running ] -> [ stopped | completed ]
- Exceptions
- rejected - when a job is rejected prior to scheduling
- failed - when there is an error while the job is running
- aborted - when a job was running at the point when the cluster shuts down
- */
-export class KubernetesClusterBackendV2 {
-    context;
-    k8s;
-    logger;
-    clusterStateInterval;
-    clusterState = {};
-    clusterNameLabel;
-    constructor(context, clusterMasterServer) {
-        const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default');
-        const clusterName = get(context, 'sysconfig.teraslice.name');
-        this.context = context;
-        this.logger = makeLogger(context, 'kubernetesV2_cluster_service');
-        this.clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
-        this.clusterState = {};
-        this.clusterStateInterval = undefined;
-        this.k8s = new K8s(this.logger, null, kubernetesNamespace, context.sysconfig.teraslice.kubernetes_api_poll_delay, context.sysconfig.teraslice.shutdown_timeout);
-        clusterMasterServer.onClientOnline((exId) => {
-            this.logger.info(`execution ${exId} is connected`);
-        });
-    }
-    /**
-     * getClusterState returns a copy of the clusterState object
-     * @return {Object} a copy of the clusterState object
-     */
-    getClusterState() {
-        return cloneDeep(this.clusterState);
-    }
-    /**
-     * Creates clusterState by iterating over all k8s pods matching both labels
-     *   app.kubernetes.io/name=teraslice
-     *   app.kubernetes.io/instance=${clusterNameLabel}
-     * @constructor
-     * @return      {Promise} void
-     */
-    async _getClusterState() {
-        return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods')
-            .then((tsPodList) => gen(tsPodList, this.clusterState))
-            .catch((err) => {
-            // TODO: We might need to do more here.  I think it's OK to just
-            // log though.  This only gets used to show slicer info through
-            // the API.  We wouldn't want to disrupt the cluster master
-            // for rare failures to reach the k8s API.
-            logError(this.logger, err, 'Error listing teraslice pods in k8s');
-        });
-    }
-    /**
-     * Return value indicates whether the cluster has enough workers to start
-     * an execution.  It must be able to allocate a slicer and at least one
-     * worker.
-     * @return {boolean} Ok to create job?
-     */
-    readyForAllocation() {
-        // return _availableWorkers() >= 2;
-        // TODO: This will be addressed in the future, see:
-        //   https://github.com/terascope/teraslice/issues/744
-        return true;
-    }
-    /**
-     * Creates k8s Service and Job for the Teraslice Execution Controller
-     * (formerly slicer).  This currently works by creating a service with a
-     * hostname that contains the exId in it listening on a well known port.
-     * The hostname and port are used later by the workers to contact this
-     * Execution Controller.
-     * @param  {Object} execution        Object containing execution details
-     * @return {Promise}                 [description]
-     */
-    async allocateSlicer(ex) {
-        const execution = cloneDeep(ex);
-        execution.slicer_port = 45680;
-        const exJobResource = new K8sJobResource(this.context.sysconfig.teraslice, execution, this.logger);
-        const exJob = exJobResource.resource;
-        this.logger.debug(exJob, 'execution allocating slicer');
-        const jobResult = await this.k8s.post(exJob);
-        if (!jobResult.metadata.uid) {
-            throw new Error('Required field uid missing from jobResult.metadata');
-        }
-        const exServiceResource = new K8sServiceResource(this.context.sysconfig.teraslice, execution, this.logger, 
-        // Needed to create the deployment and service resource ownerReferences
-        jobResult.metadata.name, jobResult.metadata.uid);
-        const exService = exServiceResource.resource;
-        const serviceResult = await this.k8s.post(exService);
-        this.logger.debug(jobResult, 'k8s slicer job submitted');
-        const exServiceName = serviceResult.metadata.name;
-        const exServiceHostName = `${exServiceName}.${this.k8s.defaultNamespace}`;
-        this.logger.debug(`Slicer is using host name: ${exServiceHostName}`);
-        execution.slicer_hostname = `${exServiceHostName}`;
-        return execution;
-    }
-    /**
-     * Creates k8s deployment that executes Teraslice workers for specified
-     * Execution.
-     * @param  {Object} execution  Object that contains information of Execution
-     * @return {Promise}           [description]
-     */
-    async allocateWorkers(execution) {
-        // NOTE: I tried to set these on the execution inside allocateSlicer
-        // but these properties were gone by the time this was called, perhaps
-        // because they are not on the schema.  So I do this k8s API call
-        // instead.
-        const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`;
-        const jobs = await pRetry(() => this.k8s.nonEmptyJobList(selector), getRetryConfig());
-        if (!jobs.items[0].metadata.uid) {
-            throw new Error('Required field uid missing from kubernetes job metadata');
-        }
-        const kr = new K8sDeploymentResource(this.context.sysconfig.teraslice, execution, this.logger, jobs.items[0].metadata.name, jobs.items[0].metadata.uid);
-        const workerDeployment = kr.resource;
-        this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`);
-        return this.k8s.post(workerDeployment)
-            .then((result) => this.logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`))
-            .catch((err) => {
-            const error = new TSError(err, {
-                reason: 'Error submitting k8s worker deployment'
-            });
-            return Promise.reject(error);
-        });
-    }
-    // FIXME: These functions should probably do something with the response
-    // NOTE: I find is strange that the expected return value here is
-    //        effectively the same as the function inputs
-    async addWorkers(executionContext, numWorkers) {
-        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'add');
-        return { action: 'add', ex_id: executionContext.ex_id, workerNum: numWorkers };
-    }
-    // NOTE: This is passed exId instead of executionContext like addWorkers and
-    // removeWorkers.  I don't know why, just dealing with it.
-    async removeWorkers(exId, numWorkers) {
-        await this.k8s.scaleExecution(exId, numWorkers, 'remove');
-        return { action: 'remove', ex_id: exId, workerNum: numWorkers };
-    }
-    // TODO: fix types here
-    async setWorkers(executionContext, numWorkers) {
-        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'set');
-        return { action: 'set', ex_id: executionContext.ex_id, workerNum: numWorkers };
-    }
-    /**
-     * Stops all workers for exId
-     * @param  {String}                 exId      The execution ID of the Execution to stop
-     * @param  {StopExecutionOptions} options     force, timeout, and excludeNode
-     *                                    force: stop all related pod, deployment, and job resources
-     *                                    timeout and excludeNode are not used in k8s clustering.
-     * @return {Promise}
-     */
-    async stopExecution(exId, options) {
-        return this.k8s.deleteExecution(exId, options?.force);
-    }
-    async shutdown() {
-        clearInterval(this.clusterStateInterval);
-    }
-    /**
-     * Returns a list of all k8s resources associated with a job ID
-     * @param {string}         jobId   The job ID of the job to list associated resources
-     * @returns {Array<TSPod[] | TSDeployment[] | TSService[]
-     *  | TSJob[] | TSReplicaSet[]>}
-     */
-    async listResourcesForJobId(jobId) {
-        const resources = [];
-        const resourceTypes = ['pods', 'deployments', 'services', 'jobs', 'replicasets'];
-        for (const type of resourceTypes) {
-            const list = await this.k8s.list(`teraslice.terascope.io/jobId=${jobId}`, type);
-            if (list.items.length > 0) {
-                resources.push(list.items);
-            }
-        }
-        return resources;
-    }
-    async initialize() {
-        this.logger.info('kubernetesV2 clustering initializing');
-        // Periodically update cluster state, update period controlled by:
-        //  context.sysconfig.teraslice.node_state_interval
-        this.clusterStateInterval = setInterval(() => {
-            this.logger.trace('cluster_master requesting cluster state update.');
-            this._getClusterState();
-        }, this.context.sysconfig.teraslice.node_state_interval);
-    }
-}
-//# sourceMappingURL=index.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=interfaces.js.map