teraslice 0.87.1 → 0.88.0

package/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -1,443 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-
-const barbe = require('barbe');
-const _ = require('lodash');
-
-const { isNumber } = require('@terascope/utils');
-
-const { safeEncode } = require('../../../../../utils/encoding_utils');
-const { setMaxOldSpaceViaEnv } = require('./utils');
-
-class K8sResource {
-    /**
-     * K8sResource allows the generation of k8s resources based on templates.
-     * After creating the object, the k8s resource is accessible on the objects
-     * .resource property.
-     *
-     * @param {String} resourceType - jobs/services/deployments
-     * @param {String} resourceName - worker/execution_controller
-     * @param {Object} terasliceConfig - teraslice cluster config from context
-     * @param {Object} execution - teraslice execution
-     */
-    constructor(resourceType, resourceName, terasliceConfig, execution, logger) {
-        this.execution = execution;
-        this.jobLabelPrefix = 'job.teraslice.terascope.io';
-        this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
-        this.logger = logger;
-        this.nodeType = resourceName;
-        this.terasliceConfig = terasliceConfig;
-
-        if (resourceName === 'worker') {
-            this.nameInfix = 'wkr';
-        } else if (resourceName === 'execution_controller') {
-            this.nameInfix = 'exc';
-        } else {
-            throw new Error(`Unsupported resourceName: ${resourceName}`);
-        }
-
-        this.templateGenerator = this._makeTemplate(resourceType, resourceName);
-        this.templateConfig = this._makeConfig();
-        this.resource = this.templateGenerator(this.templateConfig);
-
-        this._setJobLabels();
-
-        // Apply job `targets` setting as k8s nodeAffinity
-        // We assume that multiple targets require both to match ...
-        // NOTE: If you specify multiple `matchExpressions` associated with
-        // `nodeSelectorTerms`, then the pod can be scheduled onto a node
-        // only if *all* `matchExpressions` can be satisfied.
-        this._setTargets();
-        this._setResources();
-        this._setVolumes();
-        this._setAssetsVolume();
-        this._setImagePullSecret();
-        this._setEphemeralStorage();
-        this._setExternalPorts();
-        this._setPriorityClassName();
-
-        if (resourceName === 'worker') {
-            this._setWorkerAntiAffinity();
-        }
-
-        // Execution controller targets are required nodeAffinities, if
-        // required job targets are also supplied, then *all* of the matches
-        // will have to be satisfied for the job to be scheduled.  This also
-        // adds tolerations for any specified targets
-        if (resourceName === 'execution_controller') {
-            this._setExecutionControllerTargets();
-        }
-
-        if (this.terasliceConfig.kubernetes_overrides_enabled) {
-            this._mergePodSpecOverlay();
-        }
-    }
-
-    _makeConfig() {
-        const clusterName = _.get(this.terasliceConfig, 'name');
-        const clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
-        const configMapName = _.get(
-            this.terasliceConfig,
-            'kubernetes_config_map_name',
-            `${this.terasliceConfig.name}-worker`
-        );
-        const dockerImage = this.execution.kubernetes_image
-            || this.terasliceConfig.kubernetes_image;
-        // name needs to be a valid DNS name since it is used in the svc name,
-        // so we can only permit alphanumeric and - characters.  _ is forbidden.
-        // -> regex used for validation is '[a-z]([-a-z0-9]*[a-z0-9])?'
-        const jobNameLabel = this.execution.name
-            .toLowerCase()
-            .replace(/[^a-zA-Z0-9\-.]/g, '-')
-            .replace(/^[^a-z]/, 'a')
-            .replace(/[^a-z0-9]$/, '0')
-            .substring(0, 63);
-        const name = `ts-${this.nameInfix}-${jobNameLabel.substring(0, 35)}-${this.execution.job_id.substring(0, 13)}`;
-        const shutdownTimeoutMs = _.get(this.terasliceConfig, 'shutdown_timeout', 60000);
-        const shutdownTimeoutSeconds = Math.round(shutdownTimeoutMs / 1000);
-
-        const config = {
-            // assetsDirectory: _.get(this.terasliceConfig, 'assets_directory', ''),
-            // assetsVolume: _.get(this.terasliceConfig, 'assets_volume', ''),
-            clusterName,
-            clusterNameLabel,
-            configMapName,
-            dockerImage,
-            execution: safeEncode(this.execution),
-            exId: this.execution.ex_id,
-            exName: this.execution.k8sName,
-            exUid: this.execution.k8sUid,
-            jobId: this.execution.job_id,
-            jobNameLabel,
-            name,
-            namespace: _.get(this.terasliceConfig, 'kubernetes_namespace', 'default'),
-            nodeType: this.nodeType,
-            replicas: this.execution.workers,
-            shutdownTimeout: shutdownTimeoutSeconds
-        };
-        return config;
-    }
-
-    _makeTemplate(folder, fileName) {
-        const filePath = path.join(__dirname, folder, `${fileName}.hbs`);
-        const templateData = fs.readFileSync(filePath, 'utf-8');
-        const templateKeys = ['{{', '}}'];
-
-        return (config) => {
-            const templated = barbe(templateData, templateKeys, config);
-            return JSON.parse(templated);
-        };
-    }
-
-    _setWorkerAntiAffinity() {
-        if (this.terasliceConfig.kubernetes_worker_antiaffinity) {
-            const targetKey = 'spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution';
-            if (!_.has(this.resource, targetKey)) {
-                _.set(this.resource, targetKey, []);
-            }
-
-            // eslint-disable-next-line max-len
-            this.resource.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.push(
-                {
-                    weight: 1,
-                    podAffinityTerm: {
-                        labelSelector: {
-                            matchExpressions: [
-                                {
-                                    key: 'app.kubernetes.io/name',
-                                    operator: 'In',
-                                    values: [
-                                        'teraslice'
-                                    ]
-                                },
-                                {
-                                    key: 'app.kubernetes.io/instance',
-                                    operator: 'In',
-                                    values: [
-                                        this.templateConfig.clusterNameLabel
-                                    ]
-                                }
-                            ]
-                        },
-                        topologyKey: 'kubernetes.io/hostname'
-                    }
-                }
-            );
-        }
-    }
-
-    /**
-     * Execution Controllers get tolerations and required affinities
-     *
-     * NOTE: We considered changing `execution_controller_targets` to be an
-     * object but the inconsistency with `targets` made this awkward.  See the
-     * `teraslice config with execution_controller_targets and job targets set`
-     * test for an example.  If the syntax for this were to change, we should
-     * also consider changing `execution.targets`, which is a change on the job.
-     */
-    _setExecutionControllerTargets() {
-        if (this.terasliceConfig.execution_controller_targets) {
-            _.forEach(this.terasliceConfig.execution_controller_targets, (target) => {
-                this._setTargetRequired(target);
-                this._setTargetAccepted(target);
-            });
-        }
-    }
-
-    _setEphemeralStorage() {
-        if (this.execution.ephemeral_storage) {
-            this.resource.spec.template.spec.containers[0].volumeMounts.push({
-                name: 'ephemeral-volume',
-                mountPath: '/ephemeral0'
-            });
-            this.resource.spec.template.spec.volumes.push({
-                name: 'ephemeral-volume',
-                emptyDir: {}
-            });
-        }
-    }
-
-    _setExternalPorts() {
-        if (this.execution.external_ports) {
-            _.forEach(this.execution.external_ports, (portValue) => {
-                if (isNumber(portValue)) {
-                    this.resource.spec.template.spec.containers[0].ports
-                        .push({ containerPort: portValue });
-                } else {
-                    this.resource.spec.template.spec.containers[0].ports
-                        .push(
-                            {
-                                name: portValue.name,
-                                containerPort: portValue.port
-                            }
-                        );
-                }
-            });
-        }
-    }
-
-    _setImagePullSecret() {
-        if (this.terasliceConfig.kubernetes_image_pull_secret) {
-            this.resource.spec.template.spec.imagePullSecrets = [
-                { name: this.terasliceConfig.kubernetes_image_pull_secret }
-            ];
-        }
-    }
-
-    _setPriorityClassName() {
-        if (this.terasliceConfig.kubernetes_priority_class_name) {
-            if (this.nodeType === 'execution_controller') {
-                // eslint-disable-next-line max-len
-                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
-                if (this.execution.stateful) {
-                    // eslint-disable-next-line max-len
-                    this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
-                }
-            }
-            if (this.nodeType === 'worker' && this.execution.stateful) {
-                // eslint-disable-next-line max-len
-                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
-                // eslint-disable-next-line max-len
-                this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
-            }
-        }
-    }
-
-    _setAssetsVolume() {
-        if (this.terasliceConfig.assets_directory && this.terasliceConfig.assets_volume) {
-            this.resource.spec.template.spec.volumes.push({
-                name: this.terasliceConfig.assets_volume,
-                persistentVolumeClaim: { claimName: this.terasliceConfig.assets_volume }
-            });
-            this.resource.spec.template.spec.containers[0].volumeMounts.push({
-                name: this.terasliceConfig.assets_volume,
-                mountPath: this.terasliceConfig.assets_directory
-            });
-        }
-    }
-
-    _setJobLabels() {
-        if (this.execution.labels != null) {
-            Object.entries(this.execution.labels).forEach(([k, v]) => {
-                const key = `${this.jobLabelPrefix}/${_.replace(k, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63)}`;
-                const value = _.replace(v, /[^a-zA-Z0-9\-._]/g, '-').substring(0, 63);
-                this.resource.metadata.labels[key] = value;
-
-                if (this.resource.kind !== 'Service') {
-                    // Services don't have templates, so if it's a service,
-                    // don't add this
-                    this.resource.spec.template.metadata.labels[key] = value;
-                }
-            });
-        }
-    }
-
-    _setVolumes() {
-        if (this.execution.volumes != null) {
-            _.forEach(this.execution.volumes, (volume) => {
-                this.resource.spec.template.spec.volumes.push({
-                    name: volume.name,
-                    persistentVolumeClaim: { claimName: volume.name }
-                });
-                this.resource.spec.template.spec.containers[0].volumeMounts.push({
-                    name: volume.name,
-                    mountPath: volume.path
-                });
-            });
-        }
-    }
-
-    _setResources() {
-        let cpu;
-        let memory;
-        let maxMemory;
-
-        const container = this.resource.spec.template.spec.containers[0];
-
-        // use teraslice config as defaults and execution config will override it
-        const envVars = Object.assign({}, this.terasliceConfig.env_vars, this.execution.env_vars);
-
-        if (this.nodeType === 'worker') {
-            if (this.execution.resources_requests_cpu
-                || this.execution.resources_limits_cpu) {
-                if (this.execution.resources_requests_cpu) {
-                    _.set(container, 'resources.requests.cpu', this.execution.resources_requests_cpu);
-                }
-                if (this.execution.resources_limits_cpu) {
-                    _.set(container, 'resources.limits.cpu', this.execution.resources_limits_cpu);
-                }
-            } else if (this.execution.cpu || this.terasliceConfig.cpu) {
-                // The settings on the executions override the cluster configs
-                cpu = this.execution.cpu || this.terasliceConfig.cpu || -1;
-                _.set(container, 'resources.requests.cpu', cpu);
-                _.set(container, 'resources.limits.cpu', cpu);
-            }
-            if (this.execution.resources_requests_memory
-                || this.execution.resources_limits_memory) {
-                _.set(container, 'resources.requests.memory', this.execution.resources_requests_memory);
-                _.set(container, 'resources.limits.memory', this.execution.resources_limits_memory);
-                maxMemory = this.execution.resources_limits_memory;
-            } else if (this.execution.memory || this.terasliceConfig.memory) {
-                // The settings on the executions override the cluster configs
-                memory = this.execution.memory || this.terasliceConfig.memory || -1;
-                _.set(container, 'resources.requests.memory', memory);
-                _.set(container, 'resources.limits.memory', memory);
-                maxMemory = memory;
-            }
-        }
-
-        if (this.nodeType === 'execution_controller') {
-            // The settings on the executions override the cluster configs
-            cpu = this.execution.cpu_execution_controller
-                || this.terasliceConfig.cpu_execution_controller || -1;
-            memory = this.execution.memory_execution_controller
-                || this.terasliceConfig.memory_execution_controller || -1;
-            _.set(container, 'resources.requests.cpu', cpu);
-            _.set(container, 'resources.limits.cpu', cpu);
-            _.set(container, 'resources.requests.memory', memory);
-            _.set(container, 'resources.limits.memory', memory);
-            maxMemory = memory;
-        }
-
-        // NOTE: This sucks, this manages the memory env var but it ALSO is
-        // responsible for doing the config and execution env var merge, which
-        // should NOT be in this function
-        setMaxOldSpaceViaEnv(container.env, envVars, maxMemory);
-    }
-
-    _setTargets() {
-        if (_.has(this.execution, 'targets') && (!_.isEmpty(this.execution.targets))) {
-            _.forEach(this.execution.targets, (target) => {
-                // `required` is the default if no `constraint` is provided for
-                // backwards compatibility and as the most likely case
-                if (target.constraint === 'required' || !_.has(target, 'constraint')) {
-                    this._setTargetRequired(target);
-                }
-
-                if (target.constraint === 'preferred') {
-                    this._setTargetPreferred(target);
-                }
-
-                if (target.constraint === 'accepted') {
-                    this._setTargetAccepted(target);
-                }
-            });
-        }
-    }
-
-    _setTargetRequired(target) {
-        const targetKey = 'spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution';
-        if (!_.has(this.resource, targetKey)) {
-            const nodeSelectorObj = {
-                nodeSelectorTerms: [{ matchExpressions: [] }]
-            };
-            _.set(this.resource, targetKey, nodeSelectorObj);
-        }
-
-        this.resource.spec.template.spec.affinity.nodeAffinity
-            .requiredDuringSchedulingIgnoredDuringExecution
-            .nodeSelectorTerms[0].matchExpressions.push({
-                key: target.key,
-                operator: 'In',
-                values: [target.value]
-            });
-    }
-
-    _setTargetPreferred(target) {
-        const targetKey = 'spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution';
-        if (!_.has(this.resource, targetKey)) {
-            _.set(this.resource, targetKey, []);
-        }
-
-        this.resource.spec.template.spec.affinity.nodeAffinity
-            .preferredDuringSchedulingIgnoredDuringExecution.push({
-                weight: 1,
-                preference: {
-                    matchExpressions: [{
-                        key: target.key,
-                        operator: 'In',
-                        values: [target.value]
-                    }]
-                }
-            });
-    }
-
-    _setTargetAccepted(target) {
-        const targetKey = 'spec.template.spec.tolerations';
-        if (!_.has(this.resource, targetKey)) {
-            _.set(this.resource, targetKey, []);
-        }
-
-        this.resource.spec.template.spec.tolerations.push({
-            key: target.key,
-            operator: 'Equal',
-            value: target.value,
-            effect: 'NoSchedule'
-        });
-    }
-
-    /**
-     * _mergePodSpecOverlay - allows the author of the job to override anything
-     * in the pod .spec for both the execution controller and the worker pods
-     * created in Kubernetes.  This can be useful in many ways including these:
-     *
-     *   * add `initContainers` to the pods
-     *   * add `hostAliases` to the pods
-     *
-     * Note that this happens at the end of the process, so anything added by
-     * this overlay will overwrite any other setting set on the job or by the
-     * config.
-     *
-     * Job setting: `pod_spec_override`
-     */
-    _mergePodSpecOverlay() {
-        this.resource.spec.template.spec = _.merge(
-            this.resource.spec.template.spec,
-            this.execution.pod_spec_override
-        );
-    }
-}
-
-module.exports = K8sResource;

package/lib/cluster/services/cluster/backends/kubernetes/k8sState.js

@@ -1,67 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-
-/**
- * Given the k8s Pods API output generates the appropriate Teraslice cluster
- * state.  NOTE: This assumes the pods have already been filtered to ensure they
- * are teraslice pods and match the cluster in question.
- * @param  {Object} k8sPods          k8s pods API object (k8s v1.10+)
- * @param  {Object} clusterState     Teraslice Cluster State
- * @param  {String} clusterNameLabel k8s label containing clusterName
- */
-function gen(k8sPods, clusterState) {
-    // Make sure we clean up the old
-    const hostIPs = _.uniq(_.map(k8sPods.items, 'status.hostIP'));
-    const oldHostIps = _.difference(_.keys(clusterState), hostIPs);
-    _.forEach(oldHostIps, (ip) => {
-        delete clusterState[ip];
-    });
-
-    // Loop over the nodes in clusterState and set active = [] so we can append
-    // later
-    Object.keys(clusterState).forEach((nodeId) => {
-        clusterState[nodeId].active = [];
-    });
-
-    // add a worker for each pod
-    k8sPods.items.forEach((pod) => {
-        if (!_.has(clusterState, pod.status.hostIP)) {
-            // If the node isn't in clusterState, add it
-            clusterState[pod.status.hostIP] = {
-                node_id: pod.status.hostIP,
-                hostname: pod.status.hostIP,
-                pid: 'N/A',
-                node_version: 'N/A',
-                teraslice_version: 'N/A',
-                total: 'N/A',
-                state: 'connected',
-                available: 'N/A',
-                active: []
-            };
-        }
-
-        const worker = {
-            assets: [],
-            assignment: pod.metadata.labels['app.kubernetes.io/component'],
-            ex_id: pod.metadata.labels['teraslice.terascope.io/exId'],
-            // WARNING: This makes the assumption that the first container
-            // in the pod is the teraslice container.  Currently it is the
-            // only container, so this assumption is safe for now.
-            image: pod.spec.containers[0].image,
-            job_id: pod.metadata.labels['teraslice.terascope.io/jobId'],
-            pod_name: pod.metadata.name,
-            pod_ip: pod.status.podIP,
-            worker_id: pod.metadata.name,
-        };
-
-        // k8s pods can have status.phase = `Pending`, `Running`, `Succeeded`,
-        // `Failed`, `Unknown`.  We will only add `Running` pods to the
-        // Teraslice cluster state.
-        if (pod.status.phase === 'Running') {
-            clusterState[pod.status.hostIP].active.push(worker);
-        }
-    });
-}
-
-exports.gen = gen;

package/lib/cluster/services/cluster/backends/kubernetes/utils.js

@@ -1,58 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const barbe = require('barbe');
-
-const { isTest } = require('@terascope/utils');
-
-function makeTemplate(folder, fileName) {
-    const filePath = path.join(__dirname, folder, `${fileName}.hbs`);
-    const templateData = fs.readFileSync(filePath, 'utf-8');
-    const templateKeys = ['{{', '}}'];
-
-    return (config) => {
-        const templated = barbe(templateData, templateKeys, config);
-        return JSON.parse(templated);
-    };
-}
-
-// Convert bytes to MB and reduce by 10%
-function getMaxOldSpace(memory) {
-    return Math.round(0.9 * (memory / 1024 / 1024));
-}
-
-function setMaxOldSpaceViaEnv(envArr, jobEnv, memory) {
-    const envObj = {};
-    if (memory && memory > -1) {
-        // Set NODE_OPTIONS to override max-old-space-size
-        const maxOldSpace = getMaxOldSpace(memory);
-        envObj.NODE_OPTIONS = `--max-old-space-size=${maxOldSpace}`;
-    }
-
-    Object.assign(envObj, jobEnv);
-
-    Object.entries(envObj).forEach(([name, value]) => {
-        envArr.push({
-            name,
-            value
-        });
-    });
-}
-
-const MAX_RETRIES = isTest ? 2 : 3;
-const RETRY_DELAY = isTest ? 50 : 1000; // time in ms
-
-function getRetryConfig() {
-    return {
-        retries: MAX_RETRIES,
-        delay: RETRY_DELAY
-    };
-}
-
-module.exports = {
-    getMaxOldSpace,
-    getRetryConfig,
-    makeTemplate,
-    setMaxOldSpaceViaEnv
-};