teraslice 2.10.0 → 2.12.0

package/dist/src/lib/cluster/services/assets.js

@@ -0,0 +1,226 @@
+import express from 'express';
+import { TSError, parseErrorInfo, logError, toBoolean } from '@terascope/utils';
+import { makeLogger } from '../../workers/helpers/terafoundation.js';
+import { AssetsStorage } from '../../storage/index.js';
+import { getBackendConfig } from '../../storage/assets.js';
+import { makeTable, handleTerasliceRequest, getSearchOptions, sendError, } from '../../utils/api_utils.js';
+export class AssetsService {
+    context;
+    assetsStorage;
+    logger;
+    port;
+    app;
+    running = false;
+    constructor(context) {
+        this.context = context;
+        this.logger = makeLogger(context, 'assets_service');
+        this.app = express();
+        const { port } = process.env;
+        this.port = port;
+        this.app.set('json spaces', 4);
+        this.app.use((req, _res, next) => {
+            // @ts-expect-error
+            req.logger = this.logger;
+            next();
+        });
+    }
+    async initialize() {
+        try {
+            this.assetsStorage = new AssetsStorage(this.context);
+            await this.assetsStorage.initialize();
+            this.app.get('/status', (req, res) => {
+                const requestHandler = handleTerasliceRequest(req, res);
+                requestHandler(async () => ({ available: this.running }));
+            });
+            this.app.post('/assets', (req, res) => {
+                const blocking = toBoolean(req.query.blocking);
+                this.logger.debug('loading an asset', { blocking });
+                const results = [];
+                req.on('data', (buff) => {
+                    results.push(buff);
+                });
+                req.on('end', async () => {
+                    try {
+                        const data = Buffer.concat(results);
+                        const { assetId, created } = await this.assetsStorage.save(data, blocking);
+                        const code = created ? 201 : 200;
+                        const assetResponse = {
+                            asset_id: assetId,
+                            _id: assetId
+                        };
+                        res.status(code).json(assetResponse);
+                    }
+                    catch (err) {
+                        const { statusCode, message } = parseErrorInfo(err);
+                        logError(this.logger, err, 'failure saving assets via proxy request');
+                        sendError(res, statusCode, message, this.logger);
+                    }
+                });
+                req.on('error', (err) => {
+                    const { statusCode, message } = parseErrorInfo(err);
+                    logError(this.logger, err, 'failure writing asset');
+                    res.status(statusCode).send(message);
+                });
+            });
+            this.app.delete('/assets/:assetId', (req, res) => {
+                const { assetId } = req.params;
+                // @ts-expect-error
+                const requestHandler = handleTerasliceRequest(req, res, `Could not delete asset ${assetId}`);
+                if (assetId.length !== 40) {
+                    res.status(400).json({
+                        error: `asset ${assetId} is not formatted correctly, please provide the full asset_id`
+                    });
+                }
+                else {
+                    const assetResponse = {
+                        asset_id: assetId,
+                        _id: assetId
+                    };
+                    requestHandler(async () => {
+                        await this.assetsStorage.remove(assetId);
+                        return assetResponse;
+                    });
+                }
+            });
+            this.app.get('/txt/assets', (req, res) => {
+                const query = 'id:*';
+                this.createAssetTable(query, req, res);
+            });
+            this.app.get('/txt/assets/:name', (req, res) => {
+                const query = `id:* AND name:"${req.params.name}"`;
+                this.createAssetTable(query, req, res);
+            });
+            this.app.get('/txt/assets/:name/:version', (req, res) => {
+                const query = `id:* AND name:"${req.params.name}" AND version:"${req.params.version}"`;
+                this.createAssetTable(query, req, res);
+            });
+            this.app.get('/assets', (req, res) => {
+                const query = 'id:*';
+                this.assetsSearch(query, req, res);
+            });
+            this.app.get('/assets/:name', (req, res) => {
+                const query = `id:* AND name:"${req.params.name}"`;
+                this.assetsSearch(query, req, res);
+            });
+            this.app.get('/assets/:name/:version', (req, res) => {
+                const query = `id:* AND name:"${req.params.name}" AND version:"${req.params.version}"`;
+                this.assetsSearch(query, req, res);
+            });
+            await new Promise((resolve, reject) => {
+                // @ts-expect-error
+                this.app.listen(this.port, (err) => {
+                    if (err) {
+                        reject(err);
+                        return;
+                    }
+                    this.logger.info(`assets_service is listening on port ${this.port}`);
+                    resolve(true);
+                });
+                // @ts-expect-error TODO: verify this
+                this.app.timeout = this.context.sysconfig.teraslice.api_response_timeout;
+            });
+            await this.assetsStorage.autoload();
+            this.running = true;
+        }
+        catch (err) {
+            this.running = false;
+            throw new TSError(err, {
+                reason: 'Failure while creating assets_service'
+            });
+        }
+    }
+    getS3AssetStatus(s3List, esList) {
+        const result = [...esList];
+        for (const esRecord of result) {
+            esRecord.external_storage = 'missing';
+            for (const s3Record of s3List) {
+                /// s3AssetId is just the file name without the .zip
+                const s3AssetId = s3Record.File.slice(0, -4);
+                if (s3AssetId === esRecord.id) {
+                    esRecord.external_storage = 'available';
+                    break;
+                }
+            }
+        }
+        return result;
+    }
+    createAssetTable(query, req, res) {
+        const { size, from, sort } = getSearchOptions(req, '_created:desc');
+        const defaults = [
+            'name',
+            'version',
+            'id',
+            '_created',
+            'description',
+            'node_version',
+            'platform',
+            'arch'
+        ];
+        const s3Defaults = [...defaults, 'external_storage'];
+        function mapping(item) {
+            return (field) => {
+                if (field === 'description') {
+                    return item[field] ? item[field].slice(0, 30) : item[field];
+                }
+                return item[field];
+            };
+        }
+        const requestHandler = handleTerasliceRequest(req, res, 'Could not get assets');
+        requestHandler(async () => {
+            const results = await this.assetsStorage.search(query, from, size, sort, defaults);
+            const assets = results.hits.hits.map((asset) => {
+                const record = asset._source;
+                record.id = asset._id;
+                return record;
+            });
+            const { assetConnectionType } = getBackendConfig(this.context, this.logger);
+            if (assetConnectionType === 's3') {
+                const s3Assets = await this.assetsStorage.grabS3Info();
+                const updatedAssets = this.getS3AssetStatus(s3Assets, assets);
+                return makeTable(req, s3Defaults, updatedAssets, mapping);
+            }
+            return makeTable(req, defaults, assets, mapping);
+        });
+    }
+    assetsSearch(query, req, res) {
+        const { size, from, sort } = getSearchOptions(req, '_created:desc');
+        const requestHandler = handleTerasliceRequest(req, res, 'Could not get assets');
+        requestHandler(async () => {
+            const fields = ['_created', 'name', 'version', 'description', 'node_version', 'platform', 'arch'];
+            const results = await this.assetsStorage.search(query, from, size, sort, fields);
+            const mappedRecords = results.hits.hits.map((asset) => {
+                const record = asset._source;
+                record.id = asset._id;
+                return record;
+            });
+            const { assetConnectionType } = getBackendConfig(this.context, this.logger);
+            if (assetConnectionType === 's3') {
+                const s3Assets = await this.assetsStorage.grabS3Info();
+                const updatedAssets = this.getS3AssetStatus(s3Assets, mappedRecords);
+                return updatedAssets;
+            }
+            return mappedRecords;
+        });
+    }
+    run() {
+        return new Promise((resolve) => {
+            if (!this.running) {
+                resolve(true);
+                return;
+            }
+            const runningInterval = setInterval(() => {
+                if (!this.running) {
+                    clearInterval(runningInterval);
+                    resolve(true);
+                }
+            }, 1000);
+        });
+    }
+    async shutdown() {
+        this.running = false;
+        if (!this.assetsStorage)
+            return;
+        await this.assetsStorage.shutdown(true);
+    }
+}
+//# sourceMappingURL=assets.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetes/index.js

@@ -0,0 +1,192 @@
+import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/utils';
+import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
+import { K8sResource } from './k8sResource.js';
+import { gen } from './k8sState.js';
+import { K8s } from './k8s.js';
+import { getRetryConfig } from './utils.js';
+/*
+ Execution Life Cycle for _status
+ pending -> scheduling -> running -> [ paused -> running ] -> [ stopped | completed ]
+ Exceptions
+ rejected - when a job is rejected prior to scheduling
+ failed - when there is an error while the job is running
+ aborted - when a job was running at the point when the cluster shutsdown
+ */
+export class KubernetesClusterBackend {
+    context;
+    k8s;
+    logger;
+    clusterStateInterval;
+    clusterState = {};
+    clusterNameLabel;
+    constructor(context, clusterMasterServer) {
+        const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default');
+        const clusterName = get(context, 'sysconfig.teraslice.name');
+        this.context = context;
+        this.logger = makeLogger(context, 'kubernetes_cluster_service');
+        this.clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
+        this.clusterState = {};
+        this.clusterStateInterval = undefined;
+        this.k8s = new K8s(this.logger, null, kubernetesNamespace, context.sysconfig.teraslice.kubernetes_api_poll_delay, context.sysconfig.teraslice.shutdown_timeout);
+        clusterMasterServer.onClientOnline((exId) => {
+            this.logger.info(`execution ${exId} is connected`);
+        });
+    }
+    /**
+     * getClusterState returns a copy of the clusterState object
+     * @return {Object} a copy of the clusterState object
+     */
+    getClusterState() {
+        return cloneDeep(this.clusterState);
+    }
+    /**
+     * Creates clusterState by iterating over all k8s pods matching both labels
+     *   app.kubernetes.io/name=teraslice
+     *   app.kubernetes.io/instance=${clusterNameLabel}
+     * @constructor
+     * @return      {Promise} [description]
+     */
+    async _getClusterState() {
+        return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods')
+            .then((k8sPods) => gen(k8sPods, this.clusterState))
+            .catch((err) => {
+            // TODO: We might need to do more here.  I think it's OK to just
+            // log though.  This only gets used to show slicer info through
+            // the API.  We wouldn't want to disrupt the cluster master
+            // for rare failures to reach the k8s API.
+            logError(this.logger, err, 'Error listing teraslice pods in k8s');
+        });
+    }
+    /**
+     * Return value indicates whether the cluster has enough workers to start
+     * an execution.  It must be able to allocate a slicer and at least one
+     * worker.
+     * @return {boolean} Ok to create job?
+     */
+    readyForAllocation() {
+        // return _availableWorkers() >= 2;
+        // TODO: This will be addressed in the future, see:
+        //   https://github.com/terascope/teraslice/issues/744
+        return true;
+    }
+    /**
+     * Creates k8s Service and Job for the Teraslice Execution Controller
+     * (formerly slicer).  This currently works by creating a service with a
+     * hostname that contains the exId in it listening on a well known port.
+     * The hostname and port are used later by the workers to contact this
+     * Execution Controller.
+     * @param  {Object} execution        Object containing execution details
+     * @return {Promise}                 [description]
+     */
+    async allocateSlicer(ex) {
+        const execution = cloneDeep(ex);
+        execution.slicer_port = 45680;
+        const exJobResource = new K8sResource('jobs', 'execution_controller', this.context.sysconfig.teraslice, execution, this.logger);
+        const exJob = exJobResource.resource;
+        this.logger.debug(exJob, 'execution allocating slicer');
+        const jobResult = await this.k8s.post(exJob, 'job');
+        this.logger.debug(jobResult, 'k8s slicer job submitted');
+        let controllerLabel;
+        if (jobResult.spec.selector.matchLabels['controller-uid']) {
+            /// If running on kubernetes < v1.27.0
+            controllerLabel = 'controller-uid';
+        }
+        else {
+            /// If running on kubernetes v1.27.0 or later
+            controllerLabel = 'batch.kubernetes.io/controller-uid';
+        }
+        const controllerUid = jobResult.spec.selector.matchLabels[controllerLabel];
+        const pod = await this.k8s.waitForSelectedPod(`${controllerLabel}=${controllerUid}`, undefined, this.context.sysconfig.teraslice.slicer_timeout);
+        this.logger.debug(`Slicer is using IP: ${pod.status.podIP}`);
+        execution.slicer_hostname = `${pod.status.podIP}`;
+        return execution;
+    }
+    /**
+     * Creates k8s deployment that executes Teraslice workers for specified
+     * Execution.
+     * @param  {Object} execution  Object that contains information of Execution
+     * @return {Promise}           [description]
+     */
+    async allocateWorkers(execution) {
+        // NOTE: I tried to set these on the execution inside allocateSlicer
+        // but these properties were gone by the time this was called, perhaps
+        // because they are not on the schema.  So I do this k8s API call
+        // instead.
+        const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`;
+        const jobs = await pRetry(() => this.k8s.nonEmptyList(selector, 'jobs'), getRetryConfig());
+        // @ts-expect-error
+        execution.k8sName = jobs.items[0].metadata.name;
+        // @ts-expect-error
+        execution.k8sUid = jobs.items[0].metadata.uid;
+        const kr = new K8sResource('deployments', 'worker', this.context.sysconfig.teraslice, execution, this.logger);
+        const workerDeployment = kr.resource;
+        this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`);
+        return this.k8s.post(workerDeployment, 'deployment')
+            .then((result) => this.logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`))
+            .catch((err) => {
+            const error = new TSError(err, {
+                reason: 'Error submitting k8s worker deployment'
+            });
+            return Promise.reject(error);
+        });
+    }
+    // FIXME: These functions should probably do something with the response
+    // NOTE: I find is strange that the expected return value here is
+    //        effectively the same as the function inputs
+    async addWorkers(executionContext, numWorkers) {
+        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'add');
+        return { action: 'add', ex_id: executionContext.ex_id, workerNum: numWorkers };
+    }
+    // NOTE: This is passed exId instead of executionContext like addWorkers and
+    // removeWorkers.  I don't know why, just dealing with it.
+    async removeWorkers(exId, numWorkers) {
+        await this.k8s.scaleExecution(exId, numWorkers, 'remove');
+        return { action: 'remove', ex_id: exId, workerNum: numWorkers };
+    }
+    // TODO: fix types here
+    async setWorkers(executionContext, numWorkers) {
+        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'set');
+        return { action: 'set', ex_id: executionContext.ex_id, workerNum: numWorkers };
+    }
+    /**
+     * Stops all workers for exId
+     * @param  {String}                 exId      The execution ID of the Execution to stop
+     * @param  {StopExecutionOptions} options     force, timeout, and excludeNode
+     *                                    force: stop all related pod, deployment, and job resources
+     *                                    timeout and excludeNode are not used in k8s clustering.
+     * @return {Promise}
+     */
+    async stopExecution(exId, options) {
+        return this.k8s.deleteExecution(exId, options?.force);
+    }
+    async shutdown() {
+        clearInterval(this.clusterStateInterval);
+    }
+    /**
+     * Returns a list of all k8s resources associated with a job ID
+     * @param {string}         jobId   The job ID of the job to list associated resources
+     * @returns {Array<any>}
+     */
+    async listResourcesForJobId(jobId) {
+        const resources = [];
+        const resourceTypes = ['pods', 'deployments', 'services', 'jobs', 'replicasets'];
+        for (const type of resourceTypes) {
+            const list = await this.k8s.list(`teraslice.terascope.io/jobId=${jobId}`, type);
+            if (list.items.length > 0) {
+                resources.push(list.items);
+            }
+        }
+        return resources;
+    }
+    async initialize() {
+        this.logger.info('kubernetes clustering initializing');
+        // Periodically update cluster state, update period controlled by:
+        //  context.sysconfig.teraslice.node_state_interval
+        this.clusterStateInterval = setInterval(() => {
+            this.logger.trace('cluster_master requesting cluster state update.');
+            this._getClusterState();
+        }, this.context.sysconfig.teraslice.node_state_interval);
+        await this.k8s.init();
+    }
+}
+//# sourceMappingURL=index.js.map