teraslice 3.17.2 → 3.18.0

package/dist/src/lib/cluster/node_master.js

@@ -0,0 +1,355 @@
+import ms from 'ms';
+import { Mutex } from 'async-mutex';
+import { getFullErrorStack, debounce, isEmpty, has } from '@terascope/core-utils';
+import { makeLogger } from '../workers/helpers/terafoundation.js';
+import { Messaging } from './services/cluster/backends/native/messaging.js';
+import { spawnAssetLoader } from '../workers/assets/spawn.js';
+import { safeEncode } from '../utils/encoding_utils.js';
+import { findPort, getPorts } from '../utils/port_utils.js';
+import { getPackageJSON } from '../utils/file_utils.js';
+const nodeVersion = process.version;
+const terasliceVersion = getPackageJSON().version;
+export async function nodeMaster(context) {
+    const logger = makeLogger(context, 'node_master');
+    const configWorkerLimit = context.sysconfig.teraslice.workers;
+    const config = context.sysconfig.teraslice;
+    const events = context.apis.foundation.getSystemEvents();
+    const mutex = new Mutex();
+    const messaging = new Messaging(context, logger);
+    const host = messaging.getHostUrl();
+    const isShuttingDown = false;
+    const ports = getPorts(context);
+    logger.info(`node ${context.sysconfig._nodeName} is attempting to connect to cluster_master: ${host}`);
+    function sendNodeStateNow() {
+        if (isShuttingDown)
+            return;
+        const state = getNodeState();
+        messaging.send({
+            to: 'cluster_master',
+            message: 'node:state',
+            node_id: state.node_id,
+            payload: state
+        });
+    }
+    const sendNodeState = debounce(sendNodeStateNow, 500, { leading: false, trailing: true });
+    let pendingAllocations = 0;
+    function allocateWorkers(count, exConfig, fn) {
+        const startTime = Date.now();
+        pendingAllocations += count;
+        sendNodeStateNow();
+        const locked = mutex.isLocked() ? ' (locked)' : '';
+        logger.info(`allocating ${count} workers...${locked}`);
+        return mutex.runExclusive(async () => {
+            try {
+                await loadAssetsIfNeeded(exConfig.job, exConfig.ex_id);
+            }
+            catch (err) {
+                logger.error(`Failure to allocated assets for execution ${exConfig.ex_id}`);
+                throw err;
+            }
+            finally {
+                pendingAllocations -= count;
+            }
+            try {
+                const workers = await fn();
+                const elapsed = Date.now() - startTime;
+                if (workers.length === count) {
+                    logger.info(`allocated ${workers.length} workers, took ${ms(elapsed)}`);
+                }
+                else {
+                    logger.info(`allocated ${workers.length} out of the requested ${count} workers, took ${ms(elapsed)}`);
+                }
+                return workers.length;
+            }
+            catch (err) {
+                logger.error(`Failure to allocate workers for execution ${exConfig.ex_id}`);
+                throw err;
+            }
+        });
+    }
+    function canAllocateWorkers(requestedWorkers) {
+        const numOfCurrentWorkers = Object.keys(context.cluster.workers).length;
+        // if there is an over allocation, send back rest to be enqueued
+        if (configWorkerLimit < numOfCurrentWorkers + requestedWorkers) {
+            return configWorkerLimit - numOfCurrentWorkers > 0;
+        }
+        return true;
+    }
+    messaging.registerChildOnlineHook(sendNodeState);
+    messaging.register({
+        event: 'network:connect',
+        callback: () => {
+            logger.info(`node has successfully connected to: ${host}`);
+            const state = getNodeState();
+            messaging.send({
+                to: 'cluster_master', message: 'node:online', node_id: state.node_id, payload: state
+            });
+        }
+    });
+    messaging.register({
+        event: 'network:disconnect',
+        callback: () => logger.info(`node has disconnected from: ${host}`)
+    });
+    messaging.register({
+        event: 'network:error',
+        callback: (err) => logger.warn(err, `Attempting to connect to cluster_master: ${host}`)
+    });
+    messaging.register({
+        event: 'cluster:execution_controller:create',
+        // TODO: type this
+        callback: (createSlicerRequest) => {
+            const createSlicerMsg = createSlicerRequest.payload;
+            logger.info(`starting execution_controller for execution ${createSlicerMsg.ex_id}...`);
+            allocateWorkers(1, createSlicerMsg, async () => {
+                const controllerContext = {
+                    assignment: 'execution_controller',
+                    NODE_TYPE: 'execution_controller',
+                    EX: safeEncode(createSlicerMsg.job),
+                    job: createSlicerMsg.job,
+                    node_id: context.sysconfig._nodeName,
+                    ex_id: createSlicerMsg.ex_id,
+                    job_id: createSlicerMsg.job_id,
+                    slicer_port: createSlicerMsg.slicer_port
+                };
+                logger.trace('starting a execution controller', controllerContext);
+                return context.apis.foundation.startWorkers(1, controllerContext);
+            })
+                .then(() => messaging.respond(createSlicerRequest))
+                .catch((error) => {
+                messaging.respond(createSlicerRequest, {
+                    error: getFullErrorStack(error),
+                });
+            });
+        }
+    });
+    messaging.register({
+        event: 'cluster:workers:create',
+        callback: (createWorkerRequest) => {
+            const createWorkerMsg = createWorkerRequest.payload;
+            const requestedWorkers = createWorkerMsg.workers;
+            logger.info(`starting ${requestedWorkers} workers for execution ${createWorkerMsg.ex_id}...`);
+            if (!canAllocateWorkers(requestedWorkers)) {
+                logger.warn(`worker is overallocated, maximum number of workers of ${configWorkerLimit}`);
+                messaging.respond(createWorkerRequest, {
+                    payload: {
+                        createdWorkers: 0,
+                    }
+                });
+                return;
+            }
+            allocateWorkers(requestedWorkers, createWorkerMsg, async () => {
+                let newWorkers = requestedWorkers;
+                const numOfCurrentWorkers = Object.keys(context.cluster.workers).length;
+                // if there is an over allocation, send back rest to be enqueued
+                if (configWorkerLimit < numOfCurrentWorkers + requestedWorkers) {
+                    newWorkers = configWorkerLimit - numOfCurrentWorkers;
+                    logger.warn(`worker allocation request would exceed maximum number of workers of ${configWorkerLimit}`);
+                    logger.warn(`reducing allocation to ${newWorkers} workers.`);
+                }
+                let workers = [];
+                if (newWorkers > 0) {
+                    logger.trace(`starting ${newWorkers} workers`, createWorkerMsg.ex_id);
+                    workers = context.apis.foundation.startWorkers(newWorkers, {
+                        NODE_TYPE: 'worker',
+                        EX: safeEncode(createWorkerMsg.job),
+                        assignment: 'worker',
+                        node_id: context.sysconfig._nodeName,
+                        job: createWorkerMsg.job,
+                        ex_id: createWorkerMsg.ex_id,
+                        job_id: createWorkerMsg.job_id
+                    });
+                }
+                return workers;
+            })
+                .then((createdWorkers) => messaging.respond(createWorkerRequest, {
+                payload: {
+                    createdWorkers,
+                }
+            }))
+                .catch(() => messaging.respond(createWorkerRequest, {
+                payload: {
+                    createdWorkers: 0,
+                }
+            }));
+        }
+    });
+    messaging.register({ event: 'cluster:node:state', callback: () => sendNodeState() });
+    // this fires when entire server will be shutdown
+    events.once('terafoundation:shutdown', () => {
+        logger.debug('received shutdown notice from terafoundation');
+        const filterFn = () => context.cluster.workers;
+        const isActionCompleteFn = () => isEmpty(getNodeState().active);
+        shutdownProcesses({}, filterFn, isActionCompleteFn, true);
+    });
+    messaging.register({
+        event: 'cluster:execution:stop',
+        callback: (networkMsg) => {
+            const exId = networkMsg.ex_id;
+            logger.debug(`received cluster execution stop for execution ${exId}`);
+            const filterFn = () => {
+                return Object.values(context.cluster.workers)
+                    .filter((worker) => {
+                    return worker.ex_id === exId;
+                });
+            };
+            function actionCompleteFn() {
+                const children = getNodeState().active;
+                const workers = children.filter((worker) => worker.ex_id === exId);
+                logger.debug(`waiting for ${workers.length} to stop for ex: ${exId}`);
+                return workers.length === 0;
+            }
+            shutdownProcesses(networkMsg, filterFn, actionCompleteFn);
+        }
+    });
+    messaging.register({
+        event: 'cluster:workers:remove',
+        callback: (networkMsg) => {
+            const numberToRemove = networkMsg.payload.workers;
+            const children = getNodeState().active;
+            const startingWorkerCount = children.filter((worker) => worker.ex_id === networkMsg.ex_id && worker.assignment === 'worker').length;
+            const filterFn = () => children.filter((worker) => worker.ex_id === networkMsg.ex_id && worker.assignment === 'worker').slice(0, numberToRemove);
+            function actionCompleteFn() {
+                const childWorkers = getNodeState().active;
+                const currentWorkersForJob = childWorkers.filter((worker) => worker.ex_id === networkMsg.ex_id && worker.assignment === 'worker').length;
+                return currentWorkersForJob + numberToRemove <= startingWorkerCount;
+            }
+            shutdownProcesses(networkMsg, filterFn, actionCompleteFn);
+        }
+    });
+    // used to find an open port for slicer
+    messaging.register({
+        event: 'cluster:node:get_port',
+        callback: async (msg) => {
+            const port = await findPort(ports);
+            logger.debug(`assigning port ${port} for new job`);
+            messaging.respond(msg, { port });
+        }
+    });
+    messaging.register({
+        event: 'cluster:error:terminal',
+        callback: () => {
+            logger.error('terminal error in cluster_master, flushing logs and shutting down');
+            logger.flush()
+                .then(() => process.exit(0));
+        }
+    });
+    messaging.register({
+        event: 'child:exit',
+        callback: () => sendNodeState()
+    });
+    function getAssetsFromJob(jobStr) {
+        const job = typeof jobStr === 'string' ? JSON.parse(jobStr) : jobStr;
+        return job.assets || [];
+    }
+    async function loadAssetsIfNeeded(job, exId) {
+        const assets = getAssetsFromJob(job);
+        if (!assets.length)
+            return;
+        logger.info(`node ${context.sysconfig._nodeName} is checking assets for job, exId: ${exId}`);
+        await spawnAssetLoader(assets, context);
+    }
+    function shutdownWorkers(signal, filterFn) {
+        const allWorkersForJob = filterFn();
+        allWorkersForJob.forEach((worker) => {
+            const workerID = worker.worker_id || worker.id;
+            if (has(context.cluster.workers, workerID)) {
+                const clusterWorker = context.cluster.workers[workerID];
+                const processId = clusterWorker.process.pid;
+                if (clusterWorker.isDead())
+                    return;
+                // if the worker has already been sent a SIGTERM signal it should send a SIGKILL
+                logger.warn(`sending ${signal} to process ${processId}, assignment: ${worker.assignment}, ex_id: ${worker.ex_id}`);
+                clusterWorker.kill(signal);
+            }
+        });
+    }
+    function shutdownProcesses(message, filterFn, isActionCompleteFn, onlySigKill = false) {
+        const intervalTime = 200;
+        const needsResponse = message.response && message.to;
+        // give a little extra time to finish shutting down
+        let stopTime = config.shutdown_timeout + 3000;
+        if (!onlySigKill) {
+            shutdownWorkers('SIGTERM', filterFn);
+        }
+        const stop = setInterval(() => {
+            if (isActionCompleteFn()) {
+                clearInterval(stop);
+                if (needsResponse)
+                    messaging.respond(message);
+            }
+            if (stopTime <= 0) {
+                clearInterval(stop);
+                shutdownWorkers('SIGKILL', filterFn);
+                if (needsResponse)
+                    messaging.respond(message);
+            }
+            stopTime -= intervalTime;
+        }, intervalTime);
+    }
+    function getNodeState() {
+        const nodeId = context.sysconfig._nodeName;
+        const state = {
+            node_id: nodeId,
+            hostname: context.sysconfig.teraslice.hostname,
+            pid: process.pid,
+            node_version: nodeVersion,
+            teraslice_version: terasliceVersion,
+            total: context.sysconfig.teraslice.workers,
+            state: 'connected'
+        };
+        const clusterWorkers = context.cluster.workers;
+        const active = [];
+        Object.values(clusterWorkers).forEach((worker) => {
+            const child = {
+                worker_id: worker.id,
+                assignment: worker.assignment,
+                pid: worker.process.pid
+            };
+            if (worker.ex_id) {
+                child.ex_id = worker.ex_id;
+            }
+            if (worker.job_id) {
+                child.job_id = worker.job_id;
+            }
+            if (worker.assets) {
+                child.assets = worker.assets.map((asset) => asset.id);
+            }
+            active.push(child);
+        });
+        const total = state.total;
+        state.available = total - active.length - pendingAllocations;
+        state.active = active;
+        return state;
+    }
+    messaging.listen({
+        query: {
+            node_id: context.sysconfig._nodeName
+        }
+    });
+    if (context.sysconfig.teraslice.master) {
+        logger.debug(`node ${context.sysconfig._nodeName} is creating the cluster_master`);
+        const [clusterMaster] = context.apis.foundation.startWorkers(1, {
+            assignment: 'cluster_master',
+            assets_port: ports.assetsPort,
+            node_id: context.sysconfig._nodeName
+        });
+        clusterMaster.on('exit', (code) => {
+            if (code !== 0) {
+                throw Error(`Cluster master has shutdown with exit code ${code}!`);
+            }
+        });
+        logger.debug(`node ${context.sysconfig._nodeName} is creating assets endpoint on port ${ports.assetsPort}`);
+        const [assetService] = context.apis.foundation.startWorkers(1, {
+            assignment: 'assets_service',
+            // key needs to be called port to bypass cluster port sharing
+            port: ports.assetsPort,
+            node_id: context.sysconfig._nodeName
+        });
+        assetService.on('exit', (code) => {
+            if (code !== 0) {
+                throw Error(`Asset Service has shutdown with exit code ${code}!`);
+            }
+        });
+    }
+}
+//# sourceMappingURL=node_master.js.map

package/dist/src/lib/cluster/node_master.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"node_master.js","sourceRoot":"","sources":["../../../../src/lib/cluster/node_master.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EACH,iBAAiB,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAC5C,MAAM,uBAAuB,CAAC;AAK/B,OAAO,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,iDAAiD,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAGxD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;AACpC,MAAM,gBAAgB,GAAG,cAAc,EAAE,CAAC,OAAO,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA6B;IAC1D,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAClD,MAAM,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;IACzD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAE1B,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,EAAE,CAAC;IACpC,MAAM,cAAc,GAAG,KAAK,CAAC;IAC7B,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEhC,MAAM,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,SAAS,CAAC,SAAS,gDAAgD,IAAI,EAAE,CAAC,CAAC;IAEvG,SAAS,gBAAgB;QACrB,IAAI,cAAc;YAAE,OAAO;QAC3B,MAAM,KAAK,GAAG,YAAY,EAAE,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,gBAAgB;YACpB,OAAO,EAAE,YAAY;YACrB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK;SACjB,CAAC,CAAC;IACP,CAAC;IAED,MAAM,aAAa,GAAG,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1F,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,SAAS,eAAe,CACpB,KAAa,EACb,QAA6B,EAC7B,EAAsB;QAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,kBAAkB,IAAI,KAAK,CAAC;QAC5B,gBAAgB,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,cAAc,KAAK,cAAc,MAAM,EAAE,CAAC,CAAC;QAEvD,OAAO,KAAK,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC;gBACD,MAAM,kBAAkB,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC3D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,6CAA6C,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,MAAM,GAAG,CAAC;YACd,CAAC;oBAAS,CAAC;gBACP,kBAAkB,IAAI,KAAK,CAAC;YAChC,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,EAAE,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBACvC,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,MAAM,kBAAkB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC5E,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,MAAM,yBAAyB,KAAK,kBAAkB,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC1G,CAAC;gBACD,OAAO,OAAO,CAAC,MAAM,CAAC;YAC1B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,6CAA6C,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,MAAM,GAAG,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,SAAS,kBAAkB,CAAC,gBAAwB;QAChD,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACxE,gEAAgE;QAChE,IAAI,iBAAiB,GAAG,mBAAmB,GAAG,gBAAgB,EAAE,CAAC;YAC7D,OAAO,iBAAiB,GAAG,mBAAmB,GAAG,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,SAAS,CAAC,uBAAuB,CAAC,aAAa,CAAC,CAAC;IAEjD,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,iBAAiB;QACxB,QAAQ,EAAE,GAAG,EAAE;YACX,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAI,EAAE,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,YAAY,EAAE,CAAC;YAC7B,SAAS,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK;aACvF,CAAC,CAAC;QACP,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,oBAAoB;QAC3B,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,IAAI,EAAE,CAAC;KACrE,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,4CAA4C,IAAI,EAAE,CAAC;KACjG,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,qCAAqC;QAC5C,kBAAkB;QAClB,QAAQ,EAAE,CAAC,mBAAwC,EAAE,EAAE;YACnD,MAAM,eAAe,GAAG,mBAAmB,CAAC,OAAO,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,+CAA+C,eAAe,CAAC,KAAK,KAAK,CAAC,CAAC;YAEvF,eAAe,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,IAAI,EAAE;gBAC3C,MAAM,iBAAiB,GAAG;oBACtB,UAAU,EAAE,sBAAsB;oBAClC,SAAS,EAAE,sBAAsB;oBACjC,EAAE,EAAE,UAAU,CAAC,eAAe,CAAC,GAAG,CAAC;oBACnC,GAAG,EAAE,eAAe,CAAC,GAAG;oBACxB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS;oBACpC,KAAK,EAAE,eAAe,CAAC,KAAK;oBAC5B,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,WAAW,EAAE,eAAe,CAAC,WAAW;iBACR,CAAC;gBACrC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,iBAAiB,CAAC,CAAC;gBAEnE,OAAO,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;YACtE,CAAC,CAAC;iBACG,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;iBAClD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACb,SAAS,CAAC,OAAO,CAAC,mBAAmB,EAAE;oBACnC,KAAK,EAAE,iBAAiB,CAAC,KAAK,CAAC;iBAClC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACX,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,wBAAwB;QAC/B,QAAQ,EAAE,CAAC,mBAAwC,EAAE,EAAE;YACnD,MAAM,eAAe,GAAG,mBAAmB,CAAC,OAAO,CAAC;YACpD,MAAM,gBAAgB,GAAG,eAAe,CAAC,OAAO,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,YAAY,gBAAgB,0BAA0B,eAAe,CAAC,KAAK,KAAK,CAAC,CAAC;YAE9F,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,yDAAyD,iBAAiB,EAAE,CAAC,CAAC;gBAC1F,SAAS,CAAC,OAAO,CAAC,mBAAmB,EAAE;oBACnC,OAAO,EAAE;wBACL,cAAc,EAAE,CAAC;qBACpB;iBACJ,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YAED,eAAe,CAAC,gBAAgB,EAAE,eAAe,EAAE,KAAK,IAAI,EAAE;gBAC1D,IAAI,UAAU,GAAG,gBAAgB,CAAC;gBAElC,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;gBACxE,gEAAgE;gBAChE,IAAI,iBAAiB,GAAG,mBAAmB,GAAG,gBAAgB,EAAE,CAAC;oBAC7D,UAAU,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;oBACrD,MAAM,CAAC,IAAI,CAAC,uEAAuE,iBAAiB,EAAE,CAAC,CAAC;oBACxG,MAAM,CAAC,IAAI,CAAC,0BAA0B,UAAU,WAAW,CAAC,CAAC;gBACjE,CAAC;gBAED,IAAI,OAAO,GAAsC,EAAE,CAAC;gBACpD,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACjB,MAAM,CAAC,KAAK,CAAC,YAAY,UAAU,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC;oBAEtE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,UAAU,EAAE;wBACvD,SAAS,EAAE,QAAQ;wBACnB,EAAE,EAAE,UAAU,CAAC,eAAe,CAAC,GAAG,CAAC;wBACnC,UAAU,EAAE,QAAQ;wBACpB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS;wBACpC,GAAG,EAAE,eAAe,CAAC,GAAG;wBACxB,KAAK,EAAE,eAAe,CAAC,KAAK;wBAC5B,MAAM,EAAE,eAAe,CAAC,MAAM;qBACX,CAAC,CAAC;gBAC7B,CAAC;gBAED,OAAO,OAAO,CAAC;YACnB,CAAC,CAAC;iBACG,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,mBAAmB,EAAE;gBAC7D,OAAO,EAAE;oBACL,cAAc;iBACjB;aACJ,CAAC,CAAC;iBACF,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,mBAAmB,EAAE;gBAChD,OAAO,EAAE;oBACL,cAAc,EAAE,CAAC;iBACpB;aACJ,CAAC,CAAC,CAAC;QACZ,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,oBAAoB,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAErF,iDAAiD;IACjD,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACxC,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QAC/C,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC;QAChE,iBAAiB,CAAC,EAAE,EAAE,QAAQ,EAAE,kBAAkB,EAAE,IAAI,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,wBAAwB;QAC/B,QAAQ,EAAE,CAAC,UAA+B,EAAE,EAAE;YAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,iDAAiD,IAAI,EAAE,CAAC,CAAC;YAEtE,MAAM,QAAQ,GAAG,GAAG,EAAE;gBAClB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;qBACxC,MAAM,CAAC,CAAC,MAA2B,EAAE,EAAE;oBACpC,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC;gBACjC,CAAC,CAAC,CAAC;YACX,CAAC,CAAC;YAEF,SAAS,gBAAgB;gBACrB,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC,MAAM,CAAC;gBACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAC3B,CAAC,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CACzD,CAAC;gBAEF,MAAM,CAAC,KAAK,CAAC,eAAe,OAAO,CAAC,MAAM,oBAAoB,IAAI,EAAE,CAAC,CAAC;gBACtE,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;YAChC,CAAC;YAED,iBAAiB,CAAC,UAAU,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAC9D,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,wBAAwB;QAC/B,QAAQ,EAAE,CAAC,UAA+B,EAAE,EAAE;YAC1C,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC;YAClD,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC,MAAM,CAAC;YAEvC,MAAM,mBAAmB,GAAG,QAAQ,CAAC,MAAM,CACvC,CAAC,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,CACvG,CAAC,MAAM,CAAC;YAET,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAClC,CAAC,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,CACvG,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YAE3B,SAAS,gBAAgB;gBACrB,MAAM,YAAY,GAAG,YAAY,EAAE,CAAC,MAAM,CAAC;gBAC3C,MAAM,oBAAoB,GAAG,YAAY,CAAC,MAAM,CAC5C,CAAC,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,KAAK,QAAQ,CACvG,CAAC,MAAM,CAAC;gBAET,OAAO,oBAAoB,GAAG,cAAc,IAAI,mBAAmB,CAAC;YACxE,CAAC;YAED,iBAAiB,CAAC,UAAU,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QAC9D,CAAC;KACJ,CAAC,CAAC;IAEH,uCAAuC;IACvC,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,uBAAuB;QAC9B,QAAQ,EAAE,KAAK,EAAE,GAAwB,EAAE,EAAE;YACzC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,KAAK,CAAC,kBAAkB,IAAI,cAAc,CAAC,CAAC;YACnD,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,wBAAwB;QAC/B,QAAQ,EAAE,GAAG,EAAE;YACX,MAAM,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;YAClF,MAAM,CAAC,KAAK,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC;KACJ,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC;QACf,KAAK,EAAE,YAAY;QACnB,QAAQ,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE;KAClC,CAAC,CAAC;IAEH,SAAS,gBAAgB,CAAC,MAAoC;QAC1D,MAAM,GAAG,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACrE,OAAO,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,UAAU,kBAAkB,CAAC,GAAwB,EAAE,IAAY;QACpE,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO;QAE3B,MAAM,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,SAAS,CAAC,SAAS,sCAAsC,IAAI,EAAE,CAAC,CAAC;QAC7F,MAAM,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,SAAS,eAAe,CAAC,MAAc,EAAE,QAAa;QAClD,MAAM,gBAAgB,GAAG,QAAQ,EAAE,CAAC;QAEpC,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAA2B,EAAE,EAAE;YACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACzC,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACxD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC;gBAE5C,IAAI,aAAa,CAAC,MAAM,EAAE;oBAAE,OAAO;gBACnC,gFAAgF;gBAChF,MAAM,CAAC,IAAI,CAAC,WAAW,MAAM,eAAe,SAAS,iBAAiB,MAAM,CAAC,UAAU,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBACnH,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,SAAS,iBAAiB,CACtB,OAA4B,EAC5B,QAAa,EACb,kBAAuB,EACvB,WAAW,GAAG,KAAK;QACnB,MAAM,YAAY,GAAG,GAAG,CAAC;QACzB,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,EAAE,CAAC;QAErD,mDAAmD;QACnD,IAAI,QAAQ,GAAG,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE9C,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1B,IAAI,kBAAkB,EAAE,EAAE,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,CAAC;gBACpB,IAAI,aAAa;oBAAE,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC;YACD,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAChB,aAAa,CAAC,IAAI,CAAC,CAAC;gBACpB,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACrC,IAAI,aAAa;oBAAE,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC;YAED,QAAQ,IAAI,YAAY,CAAC;QAC7B,CAAC,EAAE,YAAY,CAAC,CAAC;IACrB,CAAC;IAED,SAAS,YAAY;QACjB,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;QAE3C,MAAM,KAAK,GAAG;YACV,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ;YAC9C,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,YAAY,EAAE,WAAW;YACzB,iBAAiB,EAAE,gBAAgB;YACnC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO;YAC1C,KAAK,EAAE,WAAW;SACC,CAAC;QAExB,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QAC/C,MAAM,MAAM,GAAiB,EAAE,CAAC;QAEhC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,MAA2B,EAAE,EAAE;YAClE,MAAM,KAAK,GAAG;gBACV,SAAS,EAAE,MAAM,CAAC,EAAE;gBACpB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG;aACH,CAAC;YAEzB,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC/B,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAA0B,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC/E,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,KAAmB,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,KAAK,CAAC,KAAe,CAAC;QACpC,KAAK,CAAC,SAAS,GAAG,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,kBAAkB,CAAC;QAC7D,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QAEtB,OAAO,KAAkB,CAAC;IAC9B,CAAC;IAED,SAAS,CAAC,MAAM,CAAC;QACb,KAAK,EAAE;YACH,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS;SACvC;KACJ,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,SAAS,CAAC,SAAS,iCAAiC,CAAC,CAAC;QAEnF,MAAM,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,EAAE;YAC5D,UAAU,EAAE,gBAAgB;YAC5B,WAAW,EAAE,KAAK,CAAC,UAAU;YAC7B,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS;SACZ,CAAC,CAAC;QAE9B,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;YACnC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACb,MAAM,KAAK,CAAC,8CAA8C,IAAI,GAAG,CAAC,CAAC;YACvE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,CAAC,SAAS,CAAC,SAAS,wCAAwC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QAE5G,MAAM,CAAC,YAAY,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,EAAE;YAC3D,UAAU,EAAE,gBAAgB;YAC5B,6DAA6D;YAC7D,IAAI,EAAE,KAAK,CAAC,UAAU;YACtB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS;SACZ,CAAC,CAAC;QAE9B,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;YAClC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACb,MAAM,KAAK,CAAC,6CAA6C,IAAI,GAAG,CAAC,CAAC;YACtE,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;AACL,CAAC"}

package/dist/src/lib/cluster/services/api.d.ts

@@ -0,0 +1,42 @@
+import { Express } from 'express';
+import { TerasliceConfig } from '@terascope/job-components';
+import { Logger } from '@terascope/core-utils';
+import { ClusterMasterContext } from '../../../interfaces.js';
+import { ExecutionService, JobsService, ClusterServiceType } from '../services/index.js';
+import type { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
+export declare class ApiService {
+    context: ClusterMasterContext;
+    logger: Logger;
+    jobsStorage: JobsStorage;
+    executionStorage: ExecutionStorage;
+    stateStorage: StateStorage;
+    executionService: ExecutionService;
+    jobsService: JobsService;
+    clusterService: ClusterServiceType;
+    available: boolean;
+    clusterType: string;
+    assetsUrl: string;
+    terasliceConfig: TerasliceConfig;
+    app: Express;
+    constructor(context: ClusterMasterContext, { assetsUrl, app }: {
+        assetsUrl: string;
+        app: Express;
+    });
+    private _changeWorkers;
+    private _getExIdFromRequest;
+    private _assetRedirect;
+    private _controllerStats;
+    shutdown(): Promise<void>;
+    initialize(): Promise<void>;
+    private _waitForStop;
+    /**
+     * Starts an interval if prom metrics is enabled to periodically grab
+     * cluster state/info and set metrics.
+     * @async
+     * @private
+     * @function _updatePromMetrics
+     * @return {Promise<void>}
+     */
+    private _updatePromMetrics;
+}
+//# sourceMappingURL=api.d.ts.map

package/dist/src/lib/cluster/services/api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../../../../src/lib/cluster/services/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,OAAO,EAAE,MAAM,SAAS,CAAC;AAI1C,OAAO,EAAuB,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACjF,OAAO,EAEkB,MAAM,EAE9B,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,oBAAoB,EAAuC,MAAM,wBAAwB,CAAC;AAEnG,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AA2B1F,qBAAa,UAAU;IACnB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAG,WAAW,CAAC;IAC1B,gBAAgB,EAAG,gBAAgB,CAAC;IACpC,YAAY,EAAG,YAAY,CAAC;IAC5B,gBAAgB,EAAG,gBAAgB,CAAC;IACpC,WAAW,EAAG,WAAW,CAAC;IAC1B,cAAc,EAAG,kBAAkB,CAAC;IACpC,SAAS,UAAS;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,eAAe,CAAC;IACjC,GAAG,EAAE,OAAO,CAAC;gBAGT,OAAO,EAAE,oBAAoB,EAC7B,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,OAAO,CAAA;KAAE;YAU7C,cAAc;YAqCd,mBAAmB;YAiCnB,cAAc;YAkCd,gBAAgB;IAIxB,QAAQ;IAIR,UAAU;YAodF,YAAY;IAwB1B;;;;;;;OAOG;YACW,kBAAkB;CA4PnC"}