teraslice 0.87.1 → 0.88.0

package/lib/cluster/services/assets.js

@@ -1,211 +0,0 @@
-'use strict';
-
-const express = require('express');
-const {
-    TSError, parseErrorInfo, logError, toBoolean
-} = require('@terascope/utils');
-const { makeLogger } = require('../../workers/helpers/terafoundation');
-const makeAssetsStore = require('../../storage/assets');
-const {
-    makeTable,
-    handleRequest,
-    getSearchOptions,
-    sendError,
-} = require('../../utils/api_utils');
-
-module.exports = function assetsService(context) {
-    const logger = makeLogger(context, 'assets_service');
-    const app = express();
-
-    let assetsStore;
-    let running = false;
-
-    app.set('json spaces', 4);
-
-    app.use((req, res, next) => {
-        req.logger = logger;
-        next();
-    });
-
-    app.get('/status', (req, res) => {
-        const requestHandler = handleRequest(req, res);
-        requestHandler(() => ({ available: running }));
-    });
-
-    app.post('/assets', (req, res) => {
-        const blocking = toBoolean(req.query.blocking);
-        logger.debug('loading an asset', { blocking });
-
-        const results = [];
-
-        req.on('data', (buff) => {
-            results.push(buff);
-        });
-
-        req.on('end', () => {
-            const data = Buffer.concat(results);
-            assetsStore.save(data, blocking)
-                .then(({ assetId, created }) => {
-                    const code = created ? 201 : 200;
-                    res.status(code).json({
-                        _id: assetId
-                    });
-                })
-                .catch((err) => {
-                    const { statusCode, message } = parseErrorInfo(err);
-                    logError(logger, err, 'failure saving assets via proxy request');
-                    sendError(res, statusCode, message);
-                });
-        });
-
-        req.on('error', (err) => {
-            const { statusCode, message } = parseErrorInfo(err);
-            logError(logger, err, 'failure writing asset');
-            res.status(statusCode).send(message);
-        });
-    });
-
-    app.delete('/assets/:assetId', (req, res) => {
-        const { assetId } = req.params;
-        const requestHandler = handleRequest(req, res, `Could not delete asset ${assetId}`);
-
-        if (assetId.length !== 40) {
-            res.status(400).json({
-                error: `asset ${assetId} is not formatted correctly, please provide the full asset_id`
-            });
-        } else {
-            requestHandler(async () => {
-                await assetsStore.remove(assetId);
-                return { _id: assetId };
-            });
-        }
-    });
-
-    app.get('/txt/assets', (req, res) => {
-        const query = 'id:*';
-        createAssetTable(query, req, res);
-    });
-
-    app.get('/txt/assets/:name', (req, res) => {
-        const query = `id:* AND name:"${req.params.name}"`;
-        createAssetTable(query, req, res);
-    });
-
-    app.get('/txt/assets/:name/:version', (req, res) => {
-        const query = `id:* AND name:"${req.params.name}" AND version:"${req.params.version}"`;
-        createAssetTable(query, req, res);
-    });
-
-    app.get('/assets', (req, res) => {
-        const query = 'id:*';
-        assetsSearch(query, req, res);
-    });
-
-    app.get('/assets/:name', (req, res) => {
-        const query = `id:* AND name:"${req.params.name}"`;
-        assetsSearch(query, req, res);
-    });
-
-    app.get('/assets/:name/:version', (req, res) => {
-        const query = `id:* AND name:"${req.params.name}" AND version:"${req.params.version}"`;
-        assetsSearch(query, req, res);
-    });
-
-    function createAssetTable(query, req, res) {
-        const { size, from, sort } = getSearchOptions(req, '_created:desc');
-
-        const defaults = [
-            'name',
-            'version',
-            'id',
-            '_created',
-            'description',
-            'node_version',
-            'platform',
-            'arch'
-        ];
-
-        function mapping(item) {
-            return (field) => {
-                if (field === 'description') {
-                    return item[field] ? item[field].slice(0, 30) : item[field];
-                }
-                return item[field];
-            };
-        }
-
-        const requestHandler = handleRequest(req, res, 'Could not get assets');
-        requestHandler(async () => {
-            const results = await assetsStore.search(query, from, size, sort, defaults);
-            const assets = results.hits.hits.map((asset) => {
-                const record = asset._source;
-                record.id = asset._id;
-                return record;
-            });
-            return makeTable(req, defaults, assets, mapping);
-        });
-    }
-
-    function assetsSearch(query, req, res) {
-        const { size, from, sort } = getSearchOptions(req, '_created:desc');
-
-        const requestHandler = handleRequest(req, res, 'Could not get assets');
-        requestHandler(async () => {
-            const fields = ['_created', 'name', 'version', 'description', 'node_version', 'platform', 'arch'];
-            const results = await assetsStore.search(query, from, size, sort, fields);
-            return results.hits.hits.map((asset) => {
-                const record = asset._source;
-                record.id = asset._id;
-                return record;
-            });
-        });
-    }
-
-    const { port } = process.env;
-    return {
-        async initialize() {
-            try {
-                assetsStore = await makeAssetsStore(context);
-
-                await new Promise((resolve, reject) => {
-                    app.listen(port, (err) => {
-                        if (err) {
-                            reject(err);
-                            return;
-                        }
-                        logger.info(`assets_service is listening on port ${port}`);
-                        resolve();
-                    });
-                    app.timeout = context.sysconfig.teraslice.api_response_timeout;
-                });
-
-                await assetsStore.autoload();
-                running = true;
-            } catch (err) {
-                running = false;
-                throw new TSError(err, {
-                    reason: 'Failure while creating assets_service'
-                });
-            }
-        },
-        run() {
-            return new Promise((resolve) => {
-                if (!running) {
-                    resolve();
-                    return;
-                }
-                const runningInterval = setInterval(() => {
-                    if (!running) {
-                        clearInterval(runningInterval);
-                        resolve();
-                    }
-                }, 1000);
-            });
-        },
-        async shutdown() {
-            running = false;
-            if (!assetsStore) return;
-            await assetsStore.shutdown(true);
-        }
-    };
-};

package/lib/cluster/services/cluster/backends/kubernetes/deployments/worker.hbs

@@ -1,86 +0,0 @@
-{
-    "apiVersion": "apps/v1",
-    "kind": "Deployment",
-    "metadata": {
-        "labels": {
-            "app.kubernetes.io/name": "teraslice",
-            "app.kubernetes.io/component": "{{nodeType}}",
-            "teraslice.terascope.io/exId": "{{exId}}",
-            "teraslice.terascope.io/jobId": "{{jobId}}",
-            "teraslice.terascope.io/jobName": "{{jobNameLabel}}",
-            "app.kubernetes.io/instance": "{{clusterNameLabel}}"
-        },
-        "name": "{{name}}",
-        "namespace": "{{namespace}}",
-        "ownerReferences": [
-            {
-                "apiVersion": "batch/v1",
-                "controller": false,
-                "blockOwnerDeletion": false,
-                "kind": "Job",
-                "name": "{{exName}}",
-                "uid": "{{exUid}}"
-            }
-        ]
-    },
-    "spec": {
-        "replicas": {{replicas}},
-        "template": {
-            "metadata": {
-                "labels": {
-                    "app.kubernetes.io/name": "teraslice",
-                    "app.kubernetes.io/component": "{{nodeType}}",
-                    "teraslice.terascope.io/exId": "{{exId}}",
-                    "teraslice.terascope.io/jobId": "{{jobId}}",
-                    "teraslice.terascope.io/jobName": "{{jobNameLabel}}",
-                    "app.kubernetes.io/instance": "{{clusterNameLabel}}"
-                }
-            },
-            "spec": {
-                "containers": [{
-                    "image": "{{dockerImage}}",
-                    "name": "{{name}}",
-                    "ports": [{ "containerPort": 45680 }],
-                    "volumeMounts": [{
-                        "mountPath": "/app/config",
-                        "name": "config"
-                    }],
-                    "env": [{
-                        "name": "NODE_TYPE",
-                        "value": "{{nodeType}}"
-                    },
-                    {
-                        "name": "EX",
-                        "value": "{{execution}}"
-                    },
-                    {
-                        "name": "POD_IP",
-                        "valueFrom": {
-                            "fieldRef": {
-                                "fieldPath": "status.podIP"
-                            }
-                        }
-                    }]
-                }],
-                "volumes": [{
-                    "name": "config",
-                    "configMap": {
-                        "name": "{{configMapName}}",
-                        "items": [{
-                            "key": "teraslice.yaml",
-                            "path": "teraslice.yaml"
-                        }]
-                    }
-                }],
-                "terminationGracePeriodSeconds": {{shutdownTimeout}}
-            }
-        },
-        "selector": {
-            "matchLabels": {
-                "app.kubernetes.io/name": "teraslice",
-                "app.kubernetes.io/component": "{{nodeType}}",
-                "teraslice.terascope.io/exId": "{{exId}}"
-            }
-        }
-    }
-}

package/lib/cluster/services/cluster/backends/kubernetes/index.js

@@ -1,225 +0,0 @@
-'use strict';
-
-const {
-    TSError, logError, get, cloneDeep, pRetry
-} = require('@terascope/utils');
-const { makeLogger } = require('../../../../../workers/helpers/terafoundation');
-const K8sResource = require('./k8sResource');
-const k8sState = require('./k8sState');
-const K8s = require('./k8s');
-const { getRetryConfig } = require('./utils');
-
-/*
- Execution Life Cycle for _status
- pending -> scheduling -> running -> [ paused -> running ] -> [ stopped | completed ]
- Exceptions
- rejected - when a job is rejected prior to scheduling
- failed - when there is an error while the job is running
- aborted - when a job was running at the point when the cluster shutsdown
- */
-
-module.exports = function kubernetesClusterBackend(context, clusterMasterServer) {
-    const logger = makeLogger(context, 'kubernetes_cluster_service');
-
-    const clusterName = get(context, 'sysconfig.teraslice.name');
-    const clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
-    const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default');
-
-    const clusterState = {};
-    let clusterStateInterval = null;
-
-    const k8s = new K8s(
-        logger,
-        null,
-        kubernetesNamespace,
-        context.sysconfig.teraslice.kubernetes_api_poll_delay,
-        context.sysconfig.teraslice.shutdown_timeout
-    );
-
-    clusterMasterServer.onClientOnline((exId) => {
-        logger.info(`execution ${exId} is connected`);
-    });
-
-    /**
-     * getClusterState returns a copy of the clusterState object
-     * @return {Object} a copy of the clusterState object
-     */
-    function getClusterState() {
-        return cloneDeep(clusterState);
-    }
-
-    /**
-     * Creates clusterState by iterating over all k8s pods matching both labels
-     *   app.kubernetes.io/name=teraslice
-     *   app.kubernetes.io/instance=${clusterNameLabel}
-     * @constructor
-     * @return      {Promise} [description]
-     */
-    function _getClusterState() {
-        return k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${clusterNameLabel}`, 'pods')
-            .then((k8sPods) => k8sState.gen(k8sPods, clusterState, clusterNameLabel))
-            .catch((err) => {
-                // TODO: We might need to do more here.  I think it's OK to just
-                // log though.  This only gets used to show slicer info through
-                // the API.  We wouldn't want to disrupt the cluster master
-                // for rare failures to reach the k8s API.
-                logError(logger, err, 'Error listing teraslice pods in k8s');
-            });
-    }
-
-    /**
-     * Return value indicates whether the cluster has enough workers to start
-     * an execution.  It must be able to allocate a slicer and at least one
-     * worker.
-     * @return {boolean} Ok to create job?
-     */
-    function readyForAllocation() {
-        // return _availableWorkers() >= 2;
-        // TODO: This will be addressed in the future, see:
-        //   https://github.com/terascope/teraslice/issues/744
-        return true;
-    }
-
-    /**
-     * Creates k8s Service and Job for the Teraslice Execution Controller
-     * (formerly slicer).  This currently works by creating a service with a
-     * hostname that contains the exId in it listening on a well known port.
-     * The hostname and port are used later by the workers to contact this
-     * Execution Controller.
-     * @param  {Object} execution        Object containing execution details
-     * @return {Promise}                 [description]
-     */
-    async function allocateSlicer(ex) {
-        const execution = cloneDeep(ex);
-
-        execution.slicer_port = 45680;
-        const exJobResource = new K8sResource(
-            'jobs', 'execution_controller', context.sysconfig.teraslice, execution, logger
-        );
-        const exJob = exJobResource.resource;
-
-        logger.debug(exJob, 'execution allocating slicer');
-
-        const jobResult = await k8s.post(exJob, 'job');
-        logger.debug(jobResult, 'k8s slicer job submitted');
-
-        let controllerLabel;
-        if (jobResult.spec.selector.matchLabels['controller-uid']) {
-            /// If running on kubernetes < v1.27.0
-            controllerLabel = 'controller-uid';
-        } else {
-            /// If running on kubernetes v1.27.0 or later
-            controllerLabel = 'batch.kubernetes.io/controller-uid';
-        }
-
-        const controllerUid = jobResult.spec.selector.matchLabels[controllerLabel];
-
-        const pod = await k8s.waitForSelectedPod(
-            `${controllerLabel}=${controllerUid}`,
-            null,
-            context.sysconfig.teraslice.slicer_timeout
-        );
-
-        logger.debug(`Slicer is using IP: ${pod.status.podIP}`);
-        execution.slicer_hostname = `${pod.status.podIP}`;
-
-        return execution;
-    }
-
-    /**
-     * Creates k8s deployment that executes Teraslice workers for specified
-     * Execution.
-     * @param  {Object} execution  Object that contains information of Execution
-     * @return {Promise}           [description]
-     */
-    async function allocateWorkers(execution) {
-        // NOTE: I tried to set these on the execution inside allocateSlicer
-        // but these properties were gone by the time this was called, perhaps
-        // because they are not on the schema.  So I do this k8s API call
-        // instead.
-        const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`;
-        const jobs = await pRetry(
-            () => k8s.nonEmptyList(selector, 'jobs'), getRetryConfig
-        );
-        execution.k8sName = jobs.items[0].metadata.name;
-        execution.k8sUid = jobs.items[0].metadata.uid;
-
-        const kr = new K8sResource(
-            'deployments', 'worker', context.sysconfig.teraslice, execution, logger
-        );
-
-        const workerDeployment = kr.resource;
-
-        logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`);
-
-        return k8s.post(workerDeployment, 'deployment')
-            .then((result) => logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`))
-            .catch((err) => {
-                const error = new TSError(err, {
-                    reason: 'Error submitting k8s worker deployment'
-                });
-                return Promise.reject(error);
-            });
-    }
-
-    // FIXME: These functions should probably do something with the response
-    // NOTE: I find is strange that the expected return value here is
-    //        effectively the same as the function inputs
-    async function addWorkers(executionContext, numWorkers) {
-        await k8s.scaleExecution(executionContext.ex_id, numWorkers, 'add');
-        return { action: 'add', ex_id: executionContext.ex_id, workerNum: numWorkers };
-    }
-
-    // NOTE: This is passed exId instead of executionContext like addWorkers and
-    // removeWorkers.  I don't know why, just dealing with it.
-    async function removeWorkers(exId, numWorkers) {
-        await k8s.scaleExecution(exId, numWorkers, 'remove');
-        return { action: 'remove', ex_id: exId, workerNum: numWorkers };
-    }
-
-    async function setWorkers(executionContext, numWorkers) {
-        await k8s.scaleExecution(executionContext.ex_id, numWorkers, 'set');
-        return { action: 'set', ex_id: executionContext.ex_id, workerNum: numWorkers };
-    }
-
-    /**
-     * Stops all workers for exId
-     * @param  {string} exId    The execution ID of the Execution to stop
-     * @return {Promise}
-     */
-
-    async function stopExecution(exId) {
-        return k8s.deleteExecution(exId);
-    }
-
-    async function shutdown() {
-        clearInterval(clusterStateInterval);
-    }
-
-    async function initialize() {
-        logger.info('kubernetes clustering initializing');
-
-        // Periodically update cluster state, update period controlled by:
-        //  context.sysconfig.teraslice.node_state_interval
-        clusterStateInterval = setInterval(() => {
-            logger.trace('cluster_master requesting cluster state update.');
-            _getClusterState();
-        }, context.sysconfig.teraslice.node_state_interval);
-
-        await k8s.init();
-    }
-
-    return {
-        getClusterState,
-        allocateWorkers,
-        allocateSlicer,
-        initialize,
-        shutdown,
-        stopExecution,
-        removeWorkers,
-        addWorkers,
-        setWorkers,
-        readyForAllocation,
-        // clusterAvailable TODO: return false if k8s API unavailable, not in use
-    };
-};

package/lib/cluster/services/cluster/backends/kubernetes/jobs/execution_controller.hbs

@@ -1,69 +0,0 @@
-{
-    "apiVersion": "batch/v1",
-    "kind": "Job",
-    "metadata": {
-        "labels": {
-            "app.kubernetes.io/name": "teraslice",
-            "app.kubernetes.io/component": "{{nodeType}}",
-            "teraslice.terascope.io/exId": "{{exId}}",
-            "teraslice.terascope.io/jobId": "{{jobId}}",
-            "teraslice.terascope.io/jobName": "{{jobNameLabel}}",
-            "app.kubernetes.io/instance": "{{clusterNameLabel}}"
-        },
-        "name": "{{name}}",
-        "namespace": "{{namespace}}"
-    },
-    "spec": {
-        "template": {
-            "metadata": {
-                "labels": {
-                    "app.kubernetes.io/name": "teraslice",
-                    "app.kubernetes.io/component": "{{nodeType}}",
-                    "teraslice.terascope.io/exId": "{{exId}}",
-                    "teraslice.terascope.io/jobId": "{{jobId}}",
-                    "teraslice.terascope.io/jobName": "{{jobNameLabel}}",
-                    "app.kubernetes.io/instance": "{{clusterNameLabel}}"
-                }
-            },
-            "spec": {
-                "containers": [{
-                    "image": "{{dockerImage}}",
-                    "name": "{{name}}",
-                    "ports": [{ "containerPort": 45680 }],
-                    "volumeMounts": [{
-                        "mountPath": "/app/config",
-                        "name": "config"
-                    }],
-                    "env": [{
-                        "name": "NODE_TYPE",
-                        "value": "{{nodeType}}"
-                    },
-                    {
-                        "name": "EX",
-                        "value": "{{execution}}"
-                    },
-                    {
-                        "name": "POD_IP",
-                        "valueFrom": {
-                            "fieldRef": {
-                                "fieldPath": "status.podIP"
-                            }
-                        }
-                    }]
-                }],
-                "volumes": [{
-                    "name": "config",
-                    "configMap": {
-                        "name": "{{configMapName}}",
-                        "items": [{
-                            "key": "teraslice.yaml",
-                            "path": "teraslice.yaml"
-                        }]
-                    }
-                }],
-                "terminationGracePeriodSeconds": {{shutdownTimeout}},
-                "restartPolicy": "Never"
-            }
-        }
-    }
-}