teraslice 2.10.0 → 2.12.0

package/dist/src/lib/cluster/services/execution.js

@@ -0,0 +1,435 @@
+import { Queue, TSError, getFullErrorStack, logError, get, withoutNil, isEmpty, multiFieldSort, isString, flatten, includes, cloneDeep } from '@terascope/utils';
+import { makeLogger } from '../../workers/helpers/terafoundation.js';
+/**
+ * New execution result
+ * @typedef NewExecutionResult
+ * @property {string} job_id
+ * @property {string} ex_id
+ */
+/*
+ Execution Life Cycle for _status
+ pending -> scheduling -> running -> [ paused -> running ] -> [ stopped | completed ]
+ Exceptions
+ rejected - when a execution is rejected prior to scheduling
+ failed - when there is an error while the execution is running
+ aborted - when a execution was running at the point when the cluster shutsdown
+ */
+export class ExecutionService {
+    logger;
+    pendingExecutionQueue = new Queue();
+    isNative;
+    context;
+    clusterMasterServer;
+    executionStorage;
+    stateStorage;
+    clusterService;
+    allocateInterval;
+    reapInterval;
+    constructor(context, { clusterMasterServer }) {
+        this.context = context;
+        this.logger = makeLogger(context, 'execution_service');
+        this.isNative = context.sysconfig.teraslice.cluster_manager_type === 'native';
+        this.clusterMasterServer = clusterMasterServer;
+    }
+    async initialize() {
+        const { executionStorage, stateStorage } = this.context.stores;
+        if (executionStorage == null || stateStorage == null) {
+            throw new Error('Missing required stores');
+        }
+        const { clusterService } = this.context.services;
+        if (clusterService == null) {
+            throw new Error('Missing required services');
+        }
+        this.executionStorage = executionStorage;
+        this.stateStorage = stateStorage;
+        this.clusterService = clusterService;
+        this.logger.info('execution service is initializing...');
+        // listen for an execution finished events
+        // TODO: look closer at the types of the callback
+        this.clusterMasterServer.onExecutionFinished(this._finishExecution.bind(this));
+        // lets call this before calling it
+        // in the background
+        await this.reapExecutions();
+        const pending = await executionStorage.search('_status:pending', undefined, 10000, '_created:asc');
+        for (const execution of pending) {
+            this.logger.info(`enqueuing ${execution._status} execution: ${execution.ex_id}`);
+            this.enqueue(execution);
+        }
+        const queueSize = this.pendingExecutionQueue.size();
+        if (queueSize > 0) {
+            this.logger.info(`execution queue initialization complete, ${this.pendingExecutionQueue.size()} pending executions have been enqueued`);
+        }
+        else {
+            this.logger.debug('execution queue initialization complete');
+        }
+        const executionAllocator = this._executionAllocator().bind(this);
+        this.allocateInterval = setInterval(executionAllocator, 1000);
+        this.reapInterval = setInterval(this.reapExecutions.bind(this), this.context.sysconfig.teraslice.shutdown_timeout || 30000);
+    }
+    enqueue(ex) {
+        const size = this.pendingExecutionQueue.size();
+        this.logger.debug(ex, `enqueueing execution to be processed (queue size ${size})`);
+        this.pendingExecutionQueue.enqueue(cloneDeep(ex));
+    }
+    getClusterAnalytics() {
+        return this.clusterMasterServer.getClusterAnalytics();
+    }
+    async waitForExecutionStatus(exId, _status) {
+        const status = _status || 'stopped';
+        return new Promise((resolve) => {
+            const checkCluster = async () => {
+                const state = this.clusterService.getClusterState();
+                const dict = Object.create(null);
+                Object.values(state).forEach((node) => node.active.forEach((worker) => {
+                    if (worker.ex_id) {
+                        dict[worker.ex_id] = true;
+                    }
+                }));
+                // if found, do not resolve
+                if (dict[exId]) {
+                    setTimeout(checkCluster, 3000);
+                    return;
+                }
+                try {
+                    await this.executionStorage.verifyStatusUpdate(exId, status);
+                    await this.executionStorage.setStatus(exId, status);
+                }
+                catch (err) {
+                    logError(this.logger, err, `failure setting execution, ${exId}, to ${status}`);
+                }
+                finally {
+                    resolve(true);
+                }
+            };
+            checkCluster();
+        });
+    }
+    async shutdown() {
+        this.logger.info('shutting down');
+        clearInterval(this.allocateInterval);
+        clearInterval(this.reapInterval);
+        this.allocateInterval = undefined;
+        this.reapInterval = undefined;
+        const query = this.executionStorage.getLivingStatuses().map((str) => `_status:${str}`)
+            .join(' OR ');
+        const executions = await this.executionStorage.search(query);
+        await Promise.all(executions.map(async (execution) => {
+            if (!this.isNative)
+                return;
+            this.logger.warn(`marking execution ex_id: ${execution.ex_id}, job_id: ${execution.job_id} as terminated`);
+            const exId = execution.ex_id;
+            const { hostname } = this.context.sysconfig.teraslice;
+            // need to exclude sending a stop to cluster master host, the shutdown event
+            // has already been propagated this can cause a condition of it waiting for
+            // stop to return but it already has which pauses this service shutdown
+            await this.stopExecution(exId, { excludeNode: hostname });
+            await this.waitForExecutionStatus(exId, 'terminated');
+        }));
+    }
+    findAllWorkers() {
+        return flatten(Object.values(this.clusterService.getClusterState())
+            .filter((node) => node.state === 'connected')
+            .map((node) => {
+            const workers = node.active.filter(Boolean);
+            return workers.map((worker) => {
+                worker.node_id = node.node_id;
+                worker.hostname = node.hostname;
+                return worker;
+            });
+        }))
+            .filter(Boolean);
+    }
+    async addWorkers(exId, workerNum) {
+        return this.executionStorage.getActiveExecution(exId)
+            .then((execution) => this.clusterService.addWorkers(execution, workerNum));
+    }
+    async setWorkers(exId, workerNum) {
+        return this.executionStorage.getActiveExecution(exId)
+            .then((execution) => this.clusterService.setWorkers(execution, workerNum));
+    }
+    async removeWorkers(exId, workerNum) {
+        return this.executionStorage.getActiveExecution(exId)
+            .then((execution) => this.clusterService.removeWorkers(execution.ex_id, workerNum));
+    }
+    /**
+     * Check if the execution is in a terminal status
+     *
+     * @param {import('@terascope/job-components').ExecutionConfig} execution
+     * @returns {boolean}
+    */
+    isExecutionTerminal(execution) {
+        const terminalList = this.executionStorage.getTerminalStatuses();
+        return terminalList.find((tStat) => tStat === execution._status) != null;
+    }
+    // safely stop the execution without setting the ex status to stopping or stopped
+    async _finishExecution(exId, err) {
+        if (err) {
+            const error = new TSError(err, {
+                reason: `terminal error for execution: ${exId}, shutting down execution`,
+                context: {
+                    ex_id: exId,
+                }
+            });
+            this.logger.error(error);
+        }
+        const execution = await this.getExecutionContext(exId);
+        if (!execution) {
+            throw new Error(`Execution: ${exId} was not found to finish execution`);
+        }
+        const status = execution._status;
+        if (['stopping', 'stopped'].includes(status)) {
+            this.logger.debug(`execution ${exId} is already stopping which means there is no need to stop the execution`);
+            return;
+        }
+        const runningStatuses = this.executionStorage.getRunningStatuses();
+        if (runningStatuses.includes(status)) {
+            // This should never happen. If we get here with a running status
+            // something has gone wrong. Mark execution as failed before shutdown.
+            this.logger.warn(`Cluster_master is changing status of execution ${exId} from ${status} to failed`);
+            await this.executionStorage.setStatus(exId, 'failed', this.executionStorage.executionMetaData(null, getFullErrorStack(err)));
+        }
+        this.logger.debug(`execution ${exId} finished, shutting down execution`);
+        try {
+            await this.clusterService.stopExecution(exId);
+        }
+        catch (stopErr) {
+            const stopError = new TSError(stopErr, {
+                reason: 'error finishing the execution',
+                context: {
+                    ex_id: exId,
+                }
+            });
+            logError(this.logger, stopError);
+        }
+    }
+    async stopExecution(exId, options) {
+        const execution = await this.getExecutionContext(exId);
+        if (!execution) {
+            throw new Error(`Execution: ${exId} was not found`);
+        }
+        const isTerminal = this.isExecutionTerminal(execution);
+        if (!options.force) {
+            if (isTerminal) {
+                this.logger.info(`execution ${exId} is in terminal status "${execution._status}", it cannot be stopped`);
+                return;
+            }
+            if (execution._status === 'stopping') {
+                this.logger.info('execution is already stopping...');
+                // we are kicking this off in the background, not part of the promise chain
+                this.waitForExecutionStatus(exId);
+                return;
+            }
+            this.logger.debug(`stopping execution ${exId}...`, withoutNil(options));
+            await this.executionStorage.setStatus(exId, 'stopping');
+        }
+        else {
+            this.logger.info(`force stopping execution ${exId}...`, withoutNil(options));
+        }
+        await this.clusterService.stopExecution(exId, options);
+        // we are kicking this off in the background, not part of the promise chain
+        this.waitForExecutionStatus(exId);
+    }
+    async pauseExecution(exId) {
+        const status = 'paused';
+        const execution = await this.executionStorage.getActiveExecution(exId);
+        if (!this.clusterMasterServer.isClientReady(execution.ex_id)) {
+            throw new Error(`Execution ${execution.ex_id} is not available to pause`);
+        }
+        await this.clusterMasterServer.sendExecutionPause(exId);
+        await this.executionStorage.setStatus(exId, status);
+        return { status };
+    }
+    async resumeExecution(exId) {
+        const status = 'running';
+        const execution = await this.executionStorage.getActiveExecution(exId);
+        if (!this.clusterMasterServer.isClientReady(execution.ex_id)) {
+            throw new Error(`Execution ${execution.ex_id} is not available to resume`);
+        }
+        await this.clusterMasterServer.sendExecutionResume(execution.ex_id);
+        await this.executionStorage.setStatus(execution.ex_id, status);
+        return { status };
+    }
+    async getControllerStats(exId) {
+        // if no exId is provided it returns all running executions
+        const specificId = exId ?? false;
+        const exIds = await this.getRunningExecutions(exId);
+        const clients = this.clusterMasterServer.onlineClients.filter(({ clientId }) => {
+            if (specificId && clientId === specificId)
+                return true;
+            return includes(exIds, clientId);
+        });
+        function formatResponse(msg) {
+            const payload = get(msg, 'payload', {});
+            const identifiers = {
+                ex_id: payload.ex_id,
+                job_id: payload.job_id,
+                name: payload.name
+            };
+            return Object.assign(identifiers, payload.stats);
+        }
+        if (isEmpty(clients)) {
+            if (specificId) {
+                throw new TSError(`Could not find active slicer for ex_id: ${specificId}`, {
+                    statusCode: 404
+                });
+            }
+            return [];
+        }
+        const promises = clients.map((client) => {
+            const { clientId } = client;
+            return this.clusterMasterServer
+                .sendExecutionAnalyticsRequest(clientId)
+                .then(formatResponse);
+        });
+        const results = await Promise.all(promises);
+        return multiFieldSort(results, ['name', 'started']).reverse();
+    }
+    /**
+     * Create a new execution context
+     *
+     * @param {string|import('@terascope/job-components').JobConfigParams} job
+     * @return {Promise<NewExecutionResult>}
+    */
+    async createExecutionContext(job) {
+        const ex = await this.executionStorage.create(job);
+        this.enqueue(ex);
+        return { job_id: ex.job_id, ex_id: ex.ex_id };
+    }
+    async getExecutionContext(exId) {
+        try {
+            const record = this.executionStorage.get(exId);
+            if (!record) {
+                throw new Error(`Execution ${exId} was not found`);
+            }
+            return record;
+        }
+        catch (err) {
+            logError(this.logger, err, `error getting execution context for ex: ${exId}`);
+            throw err;
+        }
+    }
+    async softDeleteExecutionContext(exId) {
+        const exIds = await this.getRunningExecutions(exId);
+        if (exIds.length > 0) {
+            throw new TSError(`Execution ${exId} is currently running, cannot delete a running execution.`, {
+                statusCode: 409
+            });
+        }
+        return this.executionStorage.softDelete(exId);
+    }
+    async getRunningExecutions(exId) {
+        let query = this.executionStorage.getRunningStatuses().map((state) => ` _status:${state} `)
+            .join('OR');
+        if (exId) {
+            query = `ex_id:"${exId}" AND (${query.trim()})`;
+        }
+        const exs = await this.executionStorage.search(query, undefined, undefined, '_created:desc');
+        return exs.map((ex) => ex.ex_id);
+    }
+    /**
+     * Recover the execution
+     *
+     * @param {string|import('@terascope/job-components').ExecutionConfig} exIdOrEx
+     * @param {import('@terascope/job-components').RecoveryCleanupType} [cleanupType]
+     * @return {Promise<NewExecutionResult>}
+    */
+    async recoverExecution(exIdOrEx, cleanupType) {
+        const recoverFromEx = isString(exIdOrEx)
+            ? await this.getExecutionContext(exIdOrEx)
+            : cloneDeep(exIdOrEx);
+        if (!recoverFromEx) {
+            throw new Error(`Could not find execution: ${exIdOrEx} to recover from`);
+        }
+        const ex = await this.executionStorage.createRecoveredExecution(recoverFromEx, cleanupType);
+        this.enqueue(ex);
+        return { job_id: ex.job_id, ex_id: ex.ex_id };
+    }
+    _executionAllocator() {
+        let allocatingExecution = false;
+        const allocator = async () => {
+            const canAllocate = !allocatingExecution
+                && this.pendingExecutionQueue.size() > 0
+                && this.clusterService.readyForAllocation();
+            if (!canAllocate)
+                return;
+            allocatingExecution = true;
+            let execution = this.pendingExecutionQueue.dequeue();
+            this.logger.info(`Scheduling execution: ${execution.ex_id}`);
+            try {
+                execution = await this.executionStorage.setStatus(execution.ex_id, 'scheduling');
+                execution = await this.clusterService.allocateSlicer(execution);
+                execution = await this.executionStorage.setStatus(execution.ex_id, 'initializing', {
+                    slicer_port: execution.slicer_port,
+                    slicer_hostname: execution.slicer_hostname
+                });
+                try {
+                    await this.clusterService.allocateWorkers(execution, execution.workers);
+                }
+                catch (err) {
+                    throw new TSError(err, {
+                        reason: `Failure to allocateWorkers ${execution.ex_id}`
+                    });
+                }
+            }
+            catch (err) {
+                const msg = `Failed to provision execution ${execution.ex_id}`;
+                const error = new TSError(err, {
+                    reason: msg
+                });
+                this.logger.warn(msg);
+                try {
+                    await this.executionStorage.setStatus(execution.ex_id, 'failed', this.executionStorage.executionMetaData(null, getFullErrorStack(error)));
+                }
+                catch (failedErr) {
+                    this.logger.error(new TSError(err, {
+                        reason: 'Failure to set execution status to failed after provision failed'
+                    }));
+                }
+                const clusteringType = this.context.sysconfig.teraslice.cluster_manager_type;
+                if (clusteringType === 'kubernetes' || clusteringType === 'kubernetesV2') {
+                    // Since this condition is only hit in cases where the pods
+                    // are never scheduled, all this call to stopExecution
+                    // accomplishes is to delete the k8s resources, which is
+                    // probably just the k8s job for the execution controller.
+                    // Calling delete on the worker deployment that doesn't
+                    // exist is OK.
+                    this.logger.warn(`Calling stopExecution on execution: ${execution.ex_id} to clean up k8s resources.`);
+                    await this.clusterService.stopExecution(execution.ex_id);
+                }
+            }
+            finally {
+                allocatingExecution = false;
+                allocator();
+            }
+        };
+        return allocator;
+    }
+    async reapExecutions() {
+        // make sure to capture the error avoid throwing an
+        // unhandled rejection
+        try {
+            // sometimes in development an execution gets stuck in stopping
+            // status since the process gets force killed in before it
+            // can be updated to stopped.
+            const stopping = await this.executionStorage.search('_status:stopping');
+            for (const execution of stopping) {
+                const updatedAt = new Date(execution._updated).getTime();
+                const timeout = this.context.sysconfig.teraslice.shutdown_timeout;
+                const updatedWithTimeout = updatedAt + timeout;
+                // Since we don't want to break executions that actually are "stopping"
+                // we need to verify that the job has exceeded the shutdown timeout
+                if (Date.now() > updatedWithTimeout) {
+                    this.logger.info(`stopping stuck executing ${execution._status} execution: ${execution.ex_id}`);
+                    await this.executionStorage.setStatus(execution.ex_id, 'stopped');
+                }
+            }
+        }
+        catch (err) {
+            this.logger.error(err, 'failure reaping executions');
+        }
+    }
+    async listResourcesForJobId(jobId) {
+        return this.clusterService.listResourcesForJobId(jobId);
+    }
+}
+//# sourceMappingURL=execution.js.map

package/dist/src/lib/cluster/services/index.js

@@ -0,0 +1,6 @@
+export * from './cluster/index.js';
+export * from './api.js';
+export * from './assets.js';
+export * from './execution.js';
+export * from './jobs.js';
+//# sourceMappingURL=index.js.map

package/dist/src/lib/cluster/services/interfaces.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=interfaces.js.map