teraslice 2.17.4 → 3.0.0-dev.1

package/dist/src/lib/cluster/services/api.js

@@ -394,7 +394,7 @@ export class ApiService {
             const requestHandler = handleTerasliceRequest(req, res, 'Could not get cluster statistics');
             requestHandler(async () => {
                 const stats = executionService.getClusterAnalytics();
-                // for backwards compatability
+                // for backwards compatibility
                 // @ts-expect-error
                 stats.slicer = stats.controllers;
                 return stats;
@@ -416,7 +416,7 @@ export class ApiService {
             if (this.clusterType === 'native') {
                 defaults = ['assignment', 'job_id', 'ex_id', 'node_id', 'pid'];
             }
-            if (this.clusterType === 'kubernetes' || this.clusterType === 'kubernetesV2') {
+            if (this.clusterType === 'kubernetesV2') {
                 defaults = ['assignment', 'job_id', 'ex_id', 'node_id', 'pod_name', 'image'];
             }
             const requestHandler = handleTerasliceRequest(req, res, 'Could not get all workers');
@@ -624,6 +624,7 @@ export class ApiService {
                                 }
                             }
                         }
+                        // TODO: removing native clustering will remove the need for any here
                         const clusterState = this.clusterService.getClusterState();
                         /// Filter out information about kubernetes ex pods
                         const filteredExecutions = {};

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js

@@ -12,7 +12,7 @@ import { K8sDeploymentResource } from './k8sDeploymentResource.js';
  Exceptions
  rejected - when a job is rejected prior to scheduling
  failed - when there is an error while the job is running
- aborted - when a job was running at the point when the cluster shutsdown
+ aborted - when a job was running at the point when the cluster shuts down
  */
 export class KubernetesClusterBackendV2 {
     context;

package/dist/src/lib/cluster/services/cluster/index.js

@@ -1,14 +1,10 @@
 import { NativeClustering } from './backends/native/index.js';
-import { KubernetesClusterBackend } from './backends/kubernetes/index.js';
 import { KubernetesClusterBackendV2 } from './backends/kubernetesV2/index.js';
 export function makeClustering(context, { clusterMasterServer }) {
     const clusterType = context.sysconfig.teraslice.cluster_manager_type;
     if (clusterType === 'native') {
         return new NativeClustering(context, clusterMasterServer);
     }
-    if (clusterType === 'kubernetes') {
-        return new KubernetesClusterBackend(context, clusterMasterServer);
-    }
     if (clusterType === 'kubernetesV2') {
         return new KubernetesClusterBackendV2(context, clusterMasterServer);
     }

package/dist/src/lib/cluster/services/execution.js

@@ -386,7 +386,7 @@ export class ExecutionService {
                     }));
                 }
                 const clusteringType = this.context.sysconfig.teraslice.cluster_manager_type;
-                if (clusteringType === 'kubernetes' || clusteringType === 'kubernetesV2') {
+                if (clusteringType === 'kubernetesV2') {
                     // Since this condition is only hit in cases where the pods
                     // are never scheduled, all this call to stopExecution
                     // accomplishes is to delete the k8s resources, which is

package/dist/src/lib/cluster/services/jobs.js

@@ -114,12 +114,14 @@ export class JobsService {
                 statusCode: 409
             });
         }
-        let currentResources = await this.executionService.listResourcesForJobId(jobId);
+        const currentResources = await this.executionService.listResourcesForJobId(jobId);
         if (currentResources.length > 0) {
-            currentResources = currentResources.flat();
+            const flattenedResources = currentResources.flat();
             const exIdsSet = new Set();
-            for (const resource of currentResources) {
-                exIdsSet.add(resource.metadata.labels['teraslice.terascope.io/exId']);
+            for (const resource of flattenedResources) {
+                if (resource.metadata.labels) {
+                    exIdsSet.add(resource.metadata.labels['teraslice.terascope.io/exId']);
+                }
             }
             const exIdsArr = Array.from(exIdsSet);
             const exIdsString = exIdsArr.join(', ');
@@ -202,12 +204,14 @@ export class JobsService {
         }
         // This will return any orphaned resources in k8s clustering
         // or an empty array in native clustering
-        let currentResources = await this.executionService.listResourcesForJobId(jobId);
+        const currentResources = await this.executionService.listResourcesForJobId(jobId);
         if (currentResources.length > 0) {
-            currentResources = currentResources.flat();
+            const flattenedResources = currentResources.flat();
             const exIdsSet = new Set();
-            for (const resource of currentResources) {
-                exIdsSet.add(resource.metadata.labels['teraslice.terascope.io/exId']);
+            for (const resource of flattenedResources) {
+                if (resource.metadata.labels) {
+                    exIdsSet.add(resource.metadata.labels['teraslice.terascope.io/exId']);
+                }
             }
             const exIdsArr = Array.from(exIdsSet);
             const exIdsString = exIdsArr.join(', ');

package/dist/src/lib/config/schemas/system.js

@@ -209,7 +209,7 @@ export const schema = {
     cluster_manager_type: {
         doc: 'determines which cluster system should be used',
         default: 'native',
-        format: ['native', 'kubernetes', 'kubernetesV2']
+        format: ['native', 'kubernetesV2']
     },
     cpu: {
         doc: 'number of cpus to reserve per teraslice worker in kubernetes',

package/dist/src/lib/workers/helpers/worker-shutdown.js

@@ -27,16 +27,8 @@ export function shutdownHandler(context, shutdownFn) {
         || process.env.NODE_TYPE
         || process.env.assignment
         || 'unknown-assignment';
-    const clusteringType = get(context, 'sysconfig.teraslice.cluster_manager_type');
-    const isK8s = clusteringType === 'kubernetes' || clusteringType === 'kubernetesV2';
     // this is native clustering only
     const isProcessRestart = process.env.process_restart;
-    // everything but the k8s execution_controller should not be allowed be allowed to
-    // set a non-zero exit code (to avoid being restarted)
-    // This is overridden in V2 because it can restart
-    const allowNonZeroExitCode = !(isK8s
-        && assignment === 'execution_controller'
-        && context.sysconfig.teraslice.cluster_manager_type === 'kubernetes');
     const api = {
         exiting: false,
         exit
@@ -68,7 +60,7 @@ export function shutdownHandler(context, shutdownFn) {
         return `already shutting down, remaining ${ms(shutdownTimeout - elapsed)}`;
     }
     async function callShutdownFn(event, err) {
-        // avoid failing before the promse is try / catched in pRaceWithTimeout
+        // avoid failing before the promise is try / catched in pRaceWithTimeout
         await pDelay(100);
         await shutdownFn(event, err);
     }
@@ -81,7 +73,6 @@ export function shutdownHandler(context, shutdownFn) {
     async function exit(event, err) {
         if (api.exiting)
             return;
-        /// Potential logic for cluster_master and asset_service
         if (err) {
             if (err.name.includes('Error')) {
                 setStatusCode(1);
@@ -97,15 +88,9 @@ export function shutdownHandler(context, shutdownFn) {
         }
         finally {
             await flushLogs();
-            if (allowNonZeroExitCode) {
-                const code = process.exitCode != null ? process.exitCode : 0;
-                logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with ${code} status code`);
-                process.exit();
-            }
-            else {
-                logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with zero status code`);
-                process.exit(0);
-            }
+            const code = process.exitCode != null ? process.exitCode : 0;
+            logger.info(`${assignment} shutdown took ${ms(Date.now() - startTime)}, exit with ${code} status code`);
+            process.exit();
         }
     }
     function setStatusCode(code) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "teraslice",
   "displayName": "Teraslice",
-  "version": "2.17.4",
+  "version": "3.0.0-dev.1",
   "description": "Distributed computing platform for processing JSON data",
   "homepage": "https://github.com/terascope/teraslice#readme",
   "bugs": {
@@ -40,11 +40,11 @@
   },
   "dependencies": {
     "@kubernetes/client-node": "~1.3.0",
-    "@terascope/elasticsearch-api": "~4.13.1",
-    "@terascope/job-components": "~1.12.4",
-    "@terascope/teraslice-messaging": "~1.13.4",
-    "@terascope/types": "~1.4.4",
-    "@terascope/utils": "~1.10.4",
+    "@terascope/elasticsearch-api": "~5.0.0-dev.1",
+    "@terascope/job-components": "~2.0.0-dev.1",
+    "@terascope/teraslice-messaging": "~2.0.0-dev.1",
+    "@terascope/types": "~2.0.0-dev.1",
+    "@terascope/utils": "~2.0.0-dev.1",
     "async-mutex": "~0.5.0",
     "barbe": "~3.0.17",
     "body-parser": "~2.2.0",
@@ -57,17 +57,16 @@
     "get-port": "~7.1.0",
     "got": "~14.4.8",
     "ip": "~2.0.1",
-    "kubernetes-client": "~9.0.0",
     "ms": "~2.1.3",
     "nanoid": "~5.1.5",
     "semver": "~7.7.2",
     "socket.io": "~1.7.4",
     "socket.io-client": "~1.7.4",
-    "terafoundation": "~1.15.4",
+    "terafoundation": "~2.0.0-dev.1",
     "uuid": "~13.0.0"
   },
   "devDependencies": {
-    "@terascope/opensearch-client": "~1.1.4",
+    "@terascope/opensearch-client": "~2.0.0-dev.1",
     "@types/archiver": "~6.0.3",
     "@types/express": "~5.0.3",
     "@types/gc-stats": "~1.4.3",

package/dist/src/lib/cluster/services/cluster/backends/kubernetes/index.js

@@ -1,192 +0,0 @@
-import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/utils';
-import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
-import { K8sResource } from './k8sResource.js';
-import { gen } from './k8sState.js';
-import { K8s } from './k8s.js';
-import { getRetryConfig } from './utils.js';
-/*
- Execution Life Cycle for _status
- pending -> scheduling -> running -> [ paused -> running ] -> [ stopped | completed ]
- Exceptions
- rejected - when a job is rejected prior to scheduling
- failed - when there is an error while the job is running
- aborted - when a job was running at the point when the cluster shutsdown
- */
-export class KubernetesClusterBackend {
-    context;
-    k8s;
-    logger;
-    clusterStateInterval;
-    clusterState = {};
-    clusterNameLabel;
-    constructor(context, clusterMasterServer) {
-        const kubernetesNamespace = get(context, 'sysconfig.teraslice.kubernetes_namespace', 'default');
-        const clusterName = get(context, 'sysconfig.teraslice.name');
-        this.context = context;
-        this.logger = makeLogger(context, 'kubernetes_cluster_service');
-        this.clusterNameLabel = clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63);
-        this.clusterState = {};
-        this.clusterStateInterval = undefined;
-        this.k8s = new K8s(this.logger, null, kubernetesNamespace, context.sysconfig.teraslice.kubernetes_api_poll_delay, context.sysconfig.teraslice.shutdown_timeout);
-        clusterMasterServer.onClientOnline((exId) => {
-            this.logger.info(`execution ${exId} is connected`);
-        });
-    }
-    /**
-     * getClusterState returns a copy of the clusterState object
-     * @return {Object} a copy of the clusterState object
-     */
-    getClusterState() {
-        return cloneDeep(this.clusterState);
-    }
-    /**
-     * Creates clusterState by iterating over all k8s pods matching both labels
-     *   app.kubernetes.io/name=teraslice
-     *   app.kubernetes.io/instance=${clusterNameLabel}
-     * @constructor
-     * @return      {Promise} [description]
-     */
-    async _getClusterState() {
-        return this.k8s.list(`app.kubernetes.io/name=teraslice,app.kubernetes.io/instance=${this.clusterNameLabel}`, 'pods')
-            .then((k8sPods) => gen(k8sPods, this.clusterState))
-            .catch((err) => {
-            // TODO: We might need to do more here.  I think it's OK to just
-            // log though.  This only gets used to show slicer info through
-            // the API.  We wouldn't want to disrupt the cluster master
-            // for rare failures to reach the k8s API.
-            logError(this.logger, err, 'Error listing teraslice pods in k8s');
-        });
-    }
-    /**
-     * Return value indicates whether the cluster has enough workers to start
-     * an execution.  It must be able to allocate a slicer and at least one
-     * worker.
-     * @return {boolean} Ok to create job?
-     */
-    readyForAllocation() {
-        // return _availableWorkers() >= 2;
-        // TODO: This will be addressed in the future, see:
-        //   https://github.com/terascope/teraslice/issues/744
-        return true;
-    }
-    /**
-     * Creates k8s Service and Job for the Teraslice Execution Controller
-     * (formerly slicer).  This currently works by creating a service with a
-     * hostname that contains the exId in it listening on a well known port.
-     * The hostname and port are used later by the workers to contact this
-     * Execution Controller.
-     * @param  {Object} execution        Object containing execution details
-     * @return {Promise}                 [description]
-     */
-    async allocateSlicer(ex) {
-        const execution = cloneDeep(ex);
-        execution.slicer_port = 45680;
-        const exJobResource = new K8sResource('jobs', 'execution_controller', this.context.sysconfig.teraslice, execution, this.logger);
-        const exJob = exJobResource.resource;
-        this.logger.debug(exJob, 'execution allocating slicer');
-        const jobResult = await this.k8s.post(exJob, 'job');
-        this.logger.debug(jobResult, 'k8s slicer job submitted');
-        let controllerLabel;
-        if (jobResult.spec.selector.matchLabels['controller-uid']) {
-            /// If running on kubernetes < v1.27.0
-            controllerLabel = 'controller-uid';
-        }
-        else {
-            /// If running on kubernetes v1.27.0 or later
-            controllerLabel = 'batch.kubernetes.io/controller-uid';
-        }
-        const controllerUid = jobResult.spec.selector.matchLabels[controllerLabel];
-        const pod = await this.k8s.waitForSelectedPod(`${controllerLabel}=${controllerUid}`, undefined, this.context.sysconfig.teraslice.slicer_timeout);
-        this.logger.debug(`Slicer is using IP: ${pod.status.podIP}`);
-        execution.slicer_hostname = `${pod.status.podIP}`;
-        return execution;
-    }
-    /**
-     * Creates k8s deployment that executes Teraslice workers for specified
-     * Execution.
-     * @param  {Object} execution  Object that contains information of Execution
-     * @return {Promise}           [description]
-     */
-    async allocateWorkers(execution) {
-        // NOTE: I tried to set these on the execution inside allocateSlicer
-        // but these properties were gone by the time this was called, perhaps
-        // because they are not on the schema.  So I do this k8s API call
-        // instead.
-        const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`;
-        const jobs = await pRetry(() => this.k8s.nonEmptyList(selector, 'jobs'), getRetryConfig());
-        // @ts-expect-error
-        execution.k8sName = jobs.items[0].metadata.name;
-        // @ts-expect-error
-        execution.k8sUid = jobs.items[0].metadata.uid;
-        const kr = new K8sResource('deployments', 'worker', this.context.sysconfig.teraslice, execution, this.logger);
-        const workerDeployment = kr.resource;
-        this.logger.debug(`workerDeployment:\n\n${JSON.stringify(workerDeployment, null, 2)}`);
-        return this.k8s.post(workerDeployment, 'deployment')
-            .then((result) => this.logger.debug(`k8s worker deployment submitted: ${JSON.stringify(result)}`))
-            .catch((err) => {
-            const error = new TSError(err, {
-                reason: 'Error submitting k8s worker deployment'
-            });
-            return Promise.reject(error);
-        });
-    }
-    // FIXME: These functions should probably do something with the response
-    // NOTE: I find is strange that the expected return value here is
-    //        effectively the same as the function inputs
-    async addWorkers(executionContext, numWorkers) {
-        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'add');
-        return { action: 'add', ex_id: executionContext.ex_id, workerNum: numWorkers };
-    }
-    // NOTE: This is passed exId instead of executionContext like addWorkers and
-    // removeWorkers.  I don't know why, just dealing with it.
-    async removeWorkers(exId, numWorkers) {
-        await this.k8s.scaleExecution(exId, numWorkers, 'remove');
-        return { action: 'remove', ex_id: exId, workerNum: numWorkers };
-    }
-    // TODO: fix types here
-    async setWorkers(executionContext, numWorkers) {
-        await this.k8s.scaleExecution(executionContext.ex_id, numWorkers, 'set');
-        return { action: 'set', ex_id: executionContext.ex_id, workerNum: numWorkers };
-    }
-    /**
-     * Stops all workers for exId
-     * @param  {String}                 exId      The execution ID of the Execution to stop
-     * @param  {StopExecutionOptions} options     force, timeout, and excludeNode
-     *                                    force: stop all related pod, deployment, and job resources
-     *                                    timeout and excludeNode are not used in k8s clustering.
-     * @return {Promise}
-     */
-    async stopExecution(exId, options) {
-        return this.k8s.deleteExecution(exId, options?.force);
-    }
-    async shutdown() {
-        clearInterval(this.clusterStateInterval);
-    }
-    /**
-     * Returns a list of all k8s resources associated with a job ID
-     * @param {string}         jobId   The job ID of the job to list associated resources
-     * @returns {Array<any>}
-     */
-    async listResourcesForJobId(jobId) {
-        const resources = [];
-        const resourceTypes = ['pods', 'deployments', 'services', 'jobs', 'replicasets'];
-        for (const type of resourceTypes) {
-            const list = await this.k8s.list(`teraslice.terascope.io/jobId=${jobId}`, type);
-            if (list.items.length > 0) {
-                resources.push(list.items);
-            }
-        }
-        return resources;
-    }
-    async initialize() {
-        this.logger.info('kubernetes clustering initializing');
-        // Periodically update cluster state, update period controlled by:
-        //  context.sysconfig.teraslice.node_state_interval
-        this.clusterStateInterval = setInterval(() => {
-            this.logger.trace('cluster_master requesting cluster state update.');
-            this._getClusterState();
-        }, this.context.sysconfig.teraslice.node_state_interval);
-        await this.k8s.init();
-    }
-}
-//# sourceMappingURL=index.js.map