teraslice 2.11.0 → 2.12.0

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js

@@ -0,0 +1,192 @@
+import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/utils';
+import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
+import { gen } from './k8sState.js';
+import { K8s } from './k8s.js';
+import { getRetryConfig } from './utils.js';
+import { K8sJobResource } from './k8sJobResource.js';
+import { K8sServiceResource } from './k8sServiceResource.js';
+import { K8sDeploymentResource } from './k8sDeploymentResource.js';
+/*
+ Execution Life Cycle for _status
+ pending -> scheduling -> running -> [ paused -> running ] -> [ stopped | completed ]
+ Exceptions
+ rejected - when a job is rejected prior to scheduling
+ failed - when there is an error while the job is running
+ aborted - when a job was running at the point when the cluster shutsdown
+ */
+export class KubernetesClusterBackendV2 {
+    context;
+    k8s;
+    logger;
+    clusterStateInterval;
+    clusterState = {};
+    clusterNameLabel;
+    constructor(context, clusterMasterServer) {
+        const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default');
+        const clusterName = get(context, 'sysconfig.teraslice.name');
+        this.context = context;
+        this.logger = makeLogger(context, 'kubernetesV2_cluster_service');
+        this.clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
+        this.clusterState = {};
+        this.clusterStateInterval = undefined;
+        this.k8s = new K8s(this.logger, null, kubernetesNamespace, context.sysconfig.teraslice.kubernetes_api_poll_delay, context.sysconfig.teraslice.shutdown_timeout);
+        clusterMasterServer.onClientOnline((exId) => {
+            this.logger.info(`execution ${exId} is connected`);
+        });
+    }
+    /**
+     * getClusterState returns a copy of the clusterState object
+     * @return {Object} a copy of the clusterState object
+     */
+    getClusterState() {
+        return cloneDeep(this.clusterState);
+    }
+    /**
+     * Creates clusterState by iterating over all k8s pods matching both labels
+     *   app.kubernetes.io/name=teraslice
+     *   app.kubernetes.io/instance=${clusterNameLabel}
+     * @constructor
+     * @return      {Promise} void
+     */
+    async _getClusterState() {
+        return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods')
+            .then((tsPodList) => gen(tsPodList, this.clusterState))
+            .catch((err) => {
+            // TODO: We might need to do more here.  I think it's OK to just
+            // log though.  This only gets used to show slicer info through
+            // the API.  We wouldn't want to disrupt the cluster master
+            // for rare failures to reach the k8s API.
+            logError(this.logger, err, 'Error listing teraslice pods in k8s');
+        });
+    }
+    /**
+     * Return value indicates whether the cluster has enough workers to start
+     * an execution.  It must be able to allocate a slicer and at least one
+     * worker.
+     * @return {boolean} Ok to create job?
+     */
+    readyForAllocation() {
+        // return _availableWorkers() >= 2;
+        // TODO: This will be addressed in the future, see:
+        //   https://github.com/terascope/teraslice/issues/744
+        return true;
+    }
+    /**
+     * Creates k8s Service and Job for the Teraslice Execution Controller
+     * (formerly slicer).  This currently works by creating a service with a
+     * hostname that contains the exId in it listening on a well known port.
+     * The hostname and port are used later by the workers to contact this
+     * Execution Controller.
+     * @param  {Object} execution        Object containing execution details
+     * @return {Promise}                 [description]
+     */
+    async allocateSlicer(ex) {
+        const execution = cloneDeep(ex);
+        execution.slicer_port = 45680;
+        const exJobResource = new K8sJobResource(this.context.sysconfig.teraslice, execution, this.logger);
+        const exJob = exJobResource.resource;
+        this.logger.debug(exJob, 'execution allocating slicer');
+        const jobResult = await this.k8s.post(exJob);
+        if (!jobResult.metadata.uid) {
+            throw new Error('Required field uid missing from jobResult.metadata');
+        }
+        const exServiceResource = new K8sServiceResource(this.context.sysconfig.teraslice, execution, this.logger, 
+        // Needed to create the deployment and service resource ownerReferences
+        jobResult.metadata.name, jobResult.metadata.uid);
+        const exService = exServiceResource.resource;
+        const serviceResult = await this.k8s.post(exService);
+        this.logger.debug(jobResult, 'k8s slicer job submitted');
+        const exServiceName = serviceResult.metadata.name;
+        const exServiceHostName = `${exServiceName}.${this.k8s.defaultNamespace}`;
+        this.logger.debug(`Slicer is using host name: ${exServiceHostName}`);
+        execution.slicer_hostname = `${exServiceHostName}`;
+        return execution;
+    }
+    /**
+     * Creates k8s deployment that executes Teraslice workers for specified
+     * Execution.
+     * @param  {Object} execution  Object that contains information of Execution
+     * @return {Promise}           [description]
+     */
+    async allocateWorkers(execution) {
+        // NOTE: I tried to set these on the execution inside allocateSlicer
+        // but these properties were gone by the time this was called, perhaps
+        // because they are not on the schema.  So I do this k8s API call
+        // instead.
+        const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`;
+        const jobs = await pRetry(() => this.k8s.nonEmptyJobList(selector), getRetryConfig());
+        if (!jobs.items[0].metadata.uid) {
+            throw new Error('Required field uid missing from kubernetes job metadata');
+        }
+        const kr = new K8sDeploymentResource(this.context.sysconfig.teraslice, execution, this.logger, jobs.items[0].metadata.name, jobs.items[0].metadata.uid);
+        const workerDeployment = kr.resource;
+        this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`);
+        return this.k8s.post(workerDeployment)
+            .then((result) => this.logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`))
+            .catch((err) => {
+            const error = new TSError(err, {
+                reason: 'Error submitting k8s worker deployment'
+            });
+            return Promise.reject(error);
+        });
+    }
+    // FIXME: These functions should probably do something with the response
+    // NOTE: I find is strange that the expected return value here is
+    //        effectively the same as the function inputs
+    async addWorkers(executionContext, numWorkers) {
+        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'add');
+        return { action: 'add', ex_id: executionContext.ex_id, workerNum: numWorkers };
+    }
+    // NOTE: This is passed exId instead of executionContext like addWorkers and
+    // removeWorkers.  I don't know why, just dealing with it.
+    async removeWorkers(exId, numWorkers) {
+        await this.k8s.scaleExecution(exId, numWorkers, 'remove');
+        return { action: 'remove', ex_id: exId, workerNum: numWorkers };
+    }
+    // TODO: fix types here
+    async setWorkers(executionContext, numWorkers) {
+        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'set');
+        return { action: 'set', ex_id: executionContext.ex_id, workerNum: numWorkers };
+    }
+    /**
+     * Stops all workers for exId
+     * @param  {String}                 exId      The execution ID of the Execution to stop
+     * @param  {StopExecutionOptions} options     force, timeout, and excludeNode
+     *                                    force: stop all related pod, deployment, and job resources
+     *                                    timeout and excludeNode are not used in k8s clustering.
+     * @return {Promise}
+     */
+    async stopExecution(exId, options) {
+        return this.k8s.deleteExecution(exId, options?.force);
+    }
+    async shutdown() {
+        clearInterval(this.clusterStateInterval);
+    }
+    /**
+     * Returns a list of all k8s resources associated with a job ID
+     * @param {string}         jobId   The job ID of the job to list associated resources
+     * @returns {Array<TSPod[] | TSDeployment[] | TSService[]
+     *  | TSJob[] | TSReplicaSet[]>}
+     */
+    async listResourcesForJobId(jobId) {
+        const resources = [];
+        const resourceTypes = ['pods', 'deployments', 'services', 'jobs', 'replicasets'];
+        for (const type of resourceTypes) {
+            const list = await this.k8s.list(`teraslice.terascope.io/jobId=${jobId}`, type);
+            if (list.items.length > 0) {
+                resources.push(list.items);
+            }
+        }
+        return resources;
+    }
+    async initialize() {
+        this.logger.info('kubernetesV2 clustering initializing');
+        // Periodically update cluster state, update period controlled by:
+        //  context.sysconfig.teraslice.node_state_interval
+        this.clusterStateInterval = setInterval(() => {
+            this.logger.trace('cluster_master requesting cluster state update.');
+            this._getClusterState();
+        }, this.context.sysconfig.teraslice.node_state_interval);
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=interfaces.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js

@@ -0,0 +1,423 @@
+import { TSError, get, isEmpty, pDelay, pRetry } from '@terascope/utils';
+import * as k8s from '@kubernetes/client-node';
+import { convertToTSResource, convertToTSResourceList, getRetryConfig, isDeployment, isJob, isPod, isReplicaSet, isService, isTSPod } from './utils.js';
+export class K8s {
+    logger;
+    apiPollDelay;
+    defaultNamespace;
+    shutdownTimeout;
+    kc;
+    k8sAppsV1Api;
+    k8sBatchV1Api;
+    k8sCoreV1Api;
+    constructor(logger, clientConfig, defaultNamespace, apiPollDelay, shutdownTimeout) {
+        this.apiPollDelay = apiPollDelay;
+        this.defaultNamespace = defaultNamespace || 'default';
+        this.logger = logger;
+        this.shutdownTimeout = shutdownTimeout; // this is in milliseconds
+        this.kc = new k8s.KubeConfig();
+        if (clientConfig) {
+            this.kc.loadFromOptions(clientConfig);
+        }
+        else if (process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT) {
+            this.kc.loadFromCluster();
+        }
+        else {
+            this.kc.loadFromDefault();
+        }
+        this.k8sAppsV1Api = this.kc.makeApiClient(k8s.AppsV1Api);
+        this.k8sBatchV1Api = this.kc.makeApiClient(k8s.BatchV1Api);
+        this.k8sCoreV1Api = this.kc.makeApiClient(k8s.CoreV1Api);
+    }
+    /**
+     * Returns the k8s NamespaceList object
+     * @return {Promise} [description]
+     */
+    async getNamespaces() {
+        let namespaces;
+        try {
+            namespaces = await pRetry(() => this.k8sCoreV1Api.listNamespace(), getRetryConfig());
+        }
+        catch (err) {
+            const error = new TSError(err, {
+                reason: 'Failure getting in namespaces'
+            });
+            throw error;
+        }
+        return namespaces.body;
+    }
+    /**
+     * Returns the first pod matching the provided selector after it has
+     * entered the `Running` state.
+     *
+     * TODO: Make more generic to search for different statuses
+     *
+     * NOTE: If your selector will return multiple pods, this method probably
+     * won't work for you.
+     * @param {String} selector kubernetes selector, like 'controller-uid=XXX'
+     * @param {String} ns       namespace to search, this will override the default
+     * @param {Number} timeout  time, in ms, to wait for pod to start
+     * @return {k8s.V1Pod}      pod object
+     *
+     * TODO: Should this use the cluster state that gets polled periodically,
+     * rather than making it's own k8s API calls
+     */
+    async waitForSelectedPod(selector, statusType, ns, timeout = 10000) {
+        const namespace = ns || this.defaultNamespace;
+        let now = Date.now();
+        const end = now + timeout;
+        while (true) {
+            const result = await pRetry(() => this.k8sCoreV1Api
+                .listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector), getRetryConfig());
+            // NOTE: This assumes the first pod returned.
+            const pod = get(result, 'body.items[0]');
+            if (pod && isTSPod(pod)) {
+                if (statusType === 'readiness-probe') {
+                    if (pod.status.conditions) {
+                        for (const condition of pod.status.conditions) {
+                            if (condition.type === 'ContainersReady'
+                                && condition.status === 'True') {
+                                return pod;
+                            }
+                        }
+                    }
+                }
+                else if (statusType === 'pod-status') {
+                    if (get(pod, 'status.phase') === 'Running')
+                        return pod;
+                }
+            }
+            if (now > end)
+                throw new Error(`Timeout waiting for pod matching: ${selector}`);
+            this.logger.debug(`waiting for pod matching: ${selector}`);
+            await pDelay(this.apiPollDelay);
+            now = Date.now();
+        }
+    }
+    /**
+     * Waits for the number of pods to equal number.
+     * @param {Number} number   Number of pods to wait for, e.g.: 0, 10
+     * @param {String} selector kubernetes selector, like 'controller-uid=XXX'
+     * @param {String} ns       namespace to search, this will override the default
+     * @param {Number} timeout  time, in ms, to wait for pod to start
+     * @return {k8s.V1Pod[]}    Array of pod objects
+     *
+     * TODO: Should this use the cluster state that gets polled periodically,
+     * rather than making it's own k8s API calls?
+     */
+    async waitForNumPods(number, selector, ns, timeout = 10000) {
+        const namespace = ns || this.defaultNamespace;
+        let now = Date.now();
+        const end = now + timeout;
+        while (true) {
+            const result = await pRetry(() => this.k8sCoreV1Api
+                .listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector), getRetryConfig());
+            const podList = get(result, 'body.items');
+            if (podList.length === number)
+                return podList;
+            const msg = `Waiting: pods matching ${selector} is ${podList.length}/${number}`;
+            if (now > end)
+                throw new Error(`Timeout ${msg}`);
+            this.logger.debug(msg);
+            await pDelay(this.apiPollDelay);
+            now = Date.now();
+        }
+    }
+    async list(selector, objType, ns) {
+        const namespace = ns || this.defaultNamespace;
+        let responseObj;
+        const params = [
+            namespace,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            selector
+        ];
+        try {
+            if (objType === 'deployments') {
+                responseObj = await pRetry(() => this.k8sAppsV1Api.listNamespacedDeployment(...params), getRetryConfig());
+            }
+            else if (objType === 'jobs') {
+                responseObj = await pRetry(() => this.k8sBatchV1Api.listNamespacedJob(...params), getRetryConfig());
+            }
+            else if (objType === 'pods') {
+                responseObj = await pRetry(() => this.k8sCoreV1Api.listNamespacedPod(...params), getRetryConfig());
+            }
+            else if (objType === 'replicasets') {
+                responseObj = await pRetry(() => this.k8sAppsV1Api.listNamespacedReplicaSet(...params), getRetryConfig());
+            }
+            else if (objType === 'services') {
+                responseObj = await pRetry(() => this.k8sCoreV1Api.listNamespacedService(...params), getRetryConfig());
+            }
+            else {
+                const error = new Error(`Invalid objType provided to get: ${objType}`);
+                this.logger.error(error);
+                return Promise.reject(error);
+            }
+            return convertToTSResourceList(responseObj.body);
+        }
+        catch (e) {
+            const err = new Error(`Request k8s.list of ${objType} with selector ${selector} failed: ${e}`);
+            this.logger.error(err);
+            return Promise.reject(err);
+        }
+    }
+    async nonEmptyJobList(selector) {
+        const jobs = await this.list(selector, 'jobs');
+        if (jobs.items.length === 1) {
+            return jobs;
+        }
+        else if (jobs.items.length === 0) {
+            const msg = `Teraslice job matching the following selector was not found: ${selector} (retriable)`;
+            this.logger.warn(msg);
+            throw new TSError(msg, { retryable: true });
+        }
+        else {
+            throw new TSError(`Unexpected number of Teraslice jobs matching the following selector: ${selector}`, {
+                retryable: true
+            });
+        }
+    }
+    async post(manifest) {
+        let responseObj;
+        try {
+            if (isDeployment(manifest)) {
+                responseObj = await this.k8sAppsV1Api
+                    .createNamespacedDeployment(this.defaultNamespace, manifest);
+            }
+            else if (isJob(manifest)) {
+                responseObj = await this.k8sBatchV1Api
+                    .createNamespacedJob(this.defaultNamespace, manifest);
+            }
+            else if (isPod(manifest)) {
+                responseObj = await this.k8sCoreV1Api
+                    .createNamespacedPod(this.defaultNamespace, manifest);
+            }
+            else if (isReplicaSet(manifest)) {
+                responseObj = await this.k8sAppsV1Api
+                    .createNamespacedReplicaSet(this.defaultNamespace, manifest);
+            }
+            else if (isService(manifest)) {
+                responseObj = await this.k8sCoreV1Api
+                    .createNamespacedService(this.defaultNamespace, manifest);
+            }
+            else {
+                const error = new Error('Invalid manifest type');
+                return Promise.reject(error);
+            }
+            return convertToTSResource(responseObj.body);
+        }
+        catch (e) {
+            const err = new Error(`Request k8s.post of ${manifest.kind} with body ${JSON.stringify(manifest)} failed: ${e}`);
+            return Promise.reject(err);
+        }
+    }
+    /**
+     * Patches specified k8s deployment with the provided record
+     * @param  {String} record record, like 'app=teraslice'
+     * @param  {String} name   Name of the deployment to patch
+     * @return {Object}        body of k8s patch response.
+     */
+    // TODO: I renamed this from patchDeployment to just patch because this is
+    // the low level k8s api method, I expect to eventually change the interface
+    // on this to require `objType` to support patching other things
+    async patch(record, name) {
+        let responseObj;
+        try {
+            const options = { headers: { 'Content-type': k8s.PatchUtils.PATCH_FORMAT_JSON_PATCH } };
+            responseObj = await pRetry(() => this.k8sAppsV1Api
+                .patchNamespacedDeployment(name, this.defaultNamespace, record, undefined, undefined, undefined, undefined, undefined, options), getRetryConfig());
+            return responseObj.body;
+        }
+        catch (e) {
+            const err = new Error(`Request k8s.patch with name: ${name} failed with: ${e}`);
+            this.logger.error(err);
+            return Promise.reject(err);
+        }
+    }
+    async delete(name, objType, force) {
+        if (name === undefined || name.trim() === '') {
+            throw new Error(`Name of resource to delete must be specified. Received: "${name}".`);
+        }
+        let responseObj;
+        // To get a Job to remove the associated pods you have to
+        // include a body like the one below with the delete request.
+        // To force: setting gracePeriodSeconds to 1 will send a SIGKILL command to the resource
+        const deleteOptions = {
+            apiVersion: 'v1',
+            kind: 'DeleteOptions',
+            propagationPolicy: 'Background'
+        };
+        if (force) {
+            deleteOptions.gracePeriodSeconds = 1;
+        }
+        const params = [
+            name,
+            this.defaultNamespace,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            deleteOptions
+        ];
+        const deleteWithErrorHandling = async (deleteFn) => {
+            try {
+                const res = await deleteFn();
+                return res;
+            }
+            catch (e) {
+                if (e.statusCode) {
+                    // 404 should be an acceptable response to a delete request, not an error
+                    if (e.statusCode === 404) {
+                        this.logger.info(`No ${objType} with name ${name} found while attempting to delete.`);
+                        return e;
+                    }
+                }
+                throw e;
+            }
+        };
+        try {
+            if (objType === 'services') {
+                responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api
+                    .deleteNamespacedService(...params)), getRetryConfig());
+            }
+            else if (objType === 'deployments') {
+                responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api
+                    .deleteNamespacedDeployment(...params)), getRetryConfig());
+            }
+            else if (objType === 'jobs') {
+                responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sBatchV1Api
+                    .deleteNamespacedJob(...params)), getRetryConfig());
+            }
+            else if (objType === 'pods') {
+                responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sCoreV1Api
+                    .deleteNamespacedPod(...params)), getRetryConfig());
+            }
+            else if (objType === 'replicasets') {
+                responseObj = await pRetry(() => deleteWithErrorHandling(() => this.k8sAppsV1Api
+                    .deleteNamespacedReplicaSet(...params)), getRetryConfig());
+            }
+            else {
+                throw new Error(`Invalid objType: ${objType}`);
+            }
+            return responseObj.body;
+        }
+        catch (e) {
+            const err = new Error(`Request k8s.delete with name: ${name} failed with: ${e}`);
+            this.logger.error(err);
+            return Promise.reject(err);
+        }
+    }
+    /**
+     * Delete all of Kubernetes resources related to the specified exId
+     * @param  {String}   exId    ID of the execution
+     * @param  {Boolean}  force   Forcefully stop all pod, deployment,
+     *                            service, replicaset and job resources
+     * @return {Promise}
+     */
+    async deleteExecution(exId, force = false) {
+        if (!exId) {
+            throw new Error('deleteExecution requires an executionId');
+        }
+        if (force) {
+            // Order matters. If we delete a parent resource before its children it
+            // will be marked for background deletion and then can't be force deleted.
+            await this._deleteObjByExId(exId, 'worker', 'pods', force);
+            await this._deleteObjByExId(exId, 'worker', 'replicasets', force);
+            await this._deleteObjByExId(exId, 'worker', 'deployments', force);
+            await this._deleteObjByExId(exId, 'execution_controller', 'pods', force);
+            await this._deleteObjByExId(exId, 'execution_controller', 'services', force);
+        }
+        await this._deleteObjByExId(exId, 'execution_controller', 'jobs', force);
+    }
+    async _deleteObjByExId(exId, nodeType, objType, force) {
+        let objList;
+        const deleteResponses = [];
+        try {
+            objList = await this.list(`app.kubernetes.io/component=${nodeType},teraslice.terascope.io/exId=${exId}`, objType);
+        }
+        catch (e) {
+            const err = new Error(`Request ${objType} list in _deleteObjByExId with app.kubernetes.io/component: ${nodeType} and exId: ${exId} failed with: ${e}`);
+            this.logger.error(err);
+            return Promise.reject(err);
+        }
+        if (isEmpty(objList.items)) {
+            this.logger.info(`k8s._deleteObjByExId: ${exId} ${nodeType} ${objType} has already been deleted`);
+            return Promise.resolve();
+        }
+        for (const obj of objList.items) {
+            const name = obj.metadata.name;
+            const deletionTimestamp = obj.metadata.deletionTimestamp;
+            // If deletionTimestamp is present then the resource is already terminating.
+            // K8s will not change the grace period in this case, so force deletion is not possible
+            if (force && deletionTimestamp) {
+                this.logger.warn(`Cannot force delete ${name} for ExId: ${exId}. It will finish deleting gracefully by ${deletionTimestamp}`);
+                return Promise.resolve();
+            }
+            this.logger.info(`k8s._deleteObjByExId: ${exId} ${nodeType} ${objType} ${force ? 'force' : ''} deleting: ${name}`);
+            try {
+                deleteResponses.push(await this.delete(name, objType, force));
+            }
+            catch (e) {
+                const err = new Error(`Request k8s.delete in _deleteObjByExId with name: ${name} failed with: ${e}`);
+                this.logger.error(err);
+                return Promise.reject(err);
+            }
+        }
+        return deleteResponses;
+    }
+    /**
+     * Scales the k8s deployment for the specified exId to the desired number
+     * of workers.
+     * @param  {String} exId                 exId of execution to scale
+     * @param  {number} numWorkers           number of workers to scale by
+     * @param  {ScaleOp} op                  Scale operation: `set`, `add`, `remove`
+     * @return {Promise<k8s.V1Deployment>}   Body of patch response.
+     */
+    async scaleExecution(exId, numWorkers, op) {
+        let newScale;
+        const selector = `app.kubernetes.io/component=worker,teraslice.terascope.io/exId=${exId}`;
+        this.logger.info(`Scaling exId: ${exId}, op: ${op}, numWorkers: ${numWorkers}`);
+        const listResponse = await this.list(selector, 'deployments');
+        this.logger.debug(`k8s worker query listResponse: ${JSON.stringify(listResponse)}`);
+        // the selector provided to list above should always result in a single
+        // deployment in the response.
+        if (listResponse.items.length === 0) {
+            const msg = `Teraslice deployment matching the following selector was not found: ${selector}`;
+            this.logger.warn(msg);
+            throw new TSError(msg);
+        }
+        else if (listResponse.items.length > 1) {
+            throw new TSError(`Unexpected number of Teraslice deployments matching the following selector: ${selector}`);
+        }
+        const workerDeployment = listResponse.items[0];
+        this.logger.info(`Current Scale for exId=${exId}: ${workerDeployment.spec.replicas}`);
+        if (op === 'set') {
+            newScale = numWorkers;
+        }
+        else if (op === 'add') {
+            newScale = workerDeployment.spec.replicas + numWorkers;
+        }
+        else if (op === 'remove') {
+            newScale = workerDeployment.spec.replicas - numWorkers;
+        }
+        else {
+            throw new Error('scaleExecution only accepts the following operations: add, remove, set');
+        }
+        this.logger.info(`New Scale for exId=${exId}: ${newScale}`);
+        const scalePatch = [
+            {
+                op: 'replace',
+                path: '/spec/replicas',
+                value: newScale
+            }
+        ];
+        const patchResponseBody = await this
+            .patch(scalePatch, workerDeployment.metadata.name);
+        this.logger.debug(`k8s.scaleExecution patchResponseBody: ${JSON.stringify(patchResponseBody)}`);
+        return patchResponseBody;
+    }
+}
+//# sourceMappingURL=k8s.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js

@@ -0,0 +1,60 @@
+import { V1Deployment } from '@kubernetes/client-node';
+import { convertToTSResource, makeTemplate } from './utils.js';
+import { K8sResource } from './k8sResource.js';
+export class K8sDeploymentResource extends K8sResource {
+    nodeType = 'worker';
+    nameInfix = 'wkr';
+    templateGenerator;
+    templateConfig;
+    resource;
+    exName;
+    exUid;
+    /**
+     * K8sDeploymentResource allows the generation of a k8s deployment based on a template.
+     * After creating the object, the k8s deployment is accessible on the objects
+     * .resource property.
+     *
+     * @param {Object} terasliceConfig - teraslice cluster config from context
+     * @param {Object} execution - teraslice execution
+     * @param {Logger} logger - teraslice logger
+     * @param {String} exName - name from execution resource
+     * @param {String} exUid - uid from execution resource
+     */
+    constructor(terasliceConfig, execution, logger, exName, exUid) {
+        super(terasliceConfig, execution, logger);
+        this.execution = execution;
+        this.logger = logger;
+        this.terasliceConfig = terasliceConfig;
+        this.exName = exName;
+        this.exUid = exUid;
+        this.templateGenerator = makeTemplate('deployments', this.nodeType);
+        this.templateConfig = this._makeConfig(this.nameInfix, exName, exUid);
+        const k8sDeployment = new V1Deployment();
+        Object.assign(k8sDeployment, this.templateGenerator(this.templateConfig));
+        this.resource = convertToTSResource(k8sDeployment);
+        this._setJobLabels(this.resource);
+        // Apply job `targets` setting as k8s nodeAffinity
+        // We assume that multiple targets require both to match ...
+        // NOTE: If you specify multiple `matchExpressions` associated with
+        // `nodeSelectorTerms`, then the pod can be scheduled onto a node
+        // only if *all* `matchExpressions` can be satisfied.
+        this._setTargets(this.resource);
+        this._setResources(this.resource);
+        this._setVolumes(this.resource);
+        if (process.env.MOUNT_LOCAL_TERASLICE !== undefined) {
+            this._mountLocalTeraslice(this.resource);
+        }
+        this._setEnvVariables();
+        this._setAssetsVolume(this.resource);
+        this._setImagePullSecret(this.resource);
+        this._setEphemeralStorage(this.resource);
+        this._setExternalPorts(this.resource);
+        this._setPriorityClassName(this.resource);
+        this._setWorkerAntiAffinity(this.resource);
+        // override must happen last
+        if (this.terasliceConfig.kubernetes_overrides_enabled) {
+            this._mergePodSpecOverlay(this.resource);
+        }
+    }
+}
+//# sourceMappingURL=k8sDeploymentResource.js.map