teraslice 3.17.2 → 3.18.0

package/dist/src/lib/cluster/services/cluster/backends/native/index.js

@@ -0,0 +1,526 @@
+/* eslint-disable @typescript-eslint/prefer-for-of */
+import { TSError, getFullErrorStack, debounce, pDelay, cloneDeep, pMap, orderBy, isInteger, get, Queue } from '@terascope/core-utils';
+import { Dispatch } from './dispatch.js';
+import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
+import { findWorkersByExecutionID } from '../state-utils.js';
+import { Messaging } from './messaging.js';
+export class NativeClustering {
+    context;
+    logger;
+    events;
+    executionStore;
+    pendingWorkerRequests = new Queue();
+    nodeStateInterval;
+    slicerAllocationAttempts;
+    clusterState = {};
+    clusterStateInterval;
+    messaging;
+    droppedNodes = {};
+    clusterMasterServer;
+    constructor(context, clusterMasterServer) {
+        this.context = context;
+        this.events = context.apis.foundation.getSystemEvents();
+        this.logger = makeLogger(context, 'native_cluster_service');
+        const nodeDisconnectTimeout = context.sysconfig.teraslice.node_disconnect_timeout;
+        this.nodeStateInterval = context.sysconfig.teraslice.node_state_interval;
+        this.slicerAllocationAttempts = context.sysconfig.teraslice.slicer_allocation_attempts;
+        this.messaging = new Messaging(context, this.logger);
+        this.clusterMasterServer = clusterMasterServer;
+        // temporary holding spot used to attach nodes that are non responsive or
+        // disconnect before final cleanup
+        this.messaging.register({
+            event: 'node:online',
+            identifier: 'node_id',
+            callback: (data, nodeId) => {
+                this.logger.info(`node ${nodeId} has connected`);
+                // if a reconnect happens stop timer
+                if (this.droppedNodes[nodeId]) {
+                    clearTimeout(this.droppedNodes[nodeId]);
+                    delete this.droppedNodes[nodeId];
+                }
+                this.logger.trace(`node ${nodeId} has state:`, data.payload);
+                this.clusterState[nodeId] = data.payload;
+                // if new node comes online, check if jobs need more workers
+                this.events.emit('cluster:available_workers');
+            }
+        });
+        this.messaging.register({
+            event: 'node:state',
+            callback: (stateMsg) => {
+                const data = stateMsg.payload;
+                this.clusterState[data.node_id] = data;
+                this.logger.trace(`node ${data.node_id} state is being updated`, data);
+                // check to see if we can provision any additional workers
+                this.events.emit('cluster:available_workers');
+            }
+        });
+        this.messaging.register({
+            event: 'network:error',
+            callback: (err) => this.logger.error(err, 'cluster_master had an error with one of its connections')
+        });
+        this.messaging.register({
+            event: 'network:disconnect',
+            identifier: 'node_id',
+            callback: (msg, nodeId) => {
+                if (!this.clusterState[nodeId])
+                    return;
+                if (this.clusterState[nodeId].active.length === 0) {
+                    this.logger.warn(`node ${nodeId} has disconnected`);
+                    delete this.clusterState[nodeId];
+                }
+                else {
+                    this.clusterState[nodeId].state = 'disconnected';
+                    const timer = setTimeout(async () => {
+                        await this._cleanUpNode(nodeId);
+                    }, nodeDisconnectTimeout);
+                    this.droppedNodes[nodeId] = timer;
+                }
+            }
+        });
+        // TODO: should this be in initialize?
+        const schedulePendingRequests = debounce(() => {
+            if (this.pendingWorkerRequests.size() && this._availableWorkers(false, true) >= 1) {
+                const requestedWorker = this.pendingWorkerRequests.dequeue();
+                const job = JSON.parse(requestedWorker.job);
+                this.allocateWorkers(job, requestedWorker.workers)
+                    .catch((err) => {
+                    const error = new TSError(err, {
+                        reason: 'Error processing pending requests'
+                    });
+                    this.logger.error(error);
+                });
+            }
+        }, 500, { leading: false, trailing: true });
+        this.events.on('cluster:available_workers', schedulePendingRequests);
+    }
+    async _cleanUpNode(nodeId) {
+        // check workers and slicers
+        const node = this._checkNode(this.clusterState[nodeId]);
+        // if disconnected node had a slicer, we stop the execution of each slicer on it
+        // and mark it as failed
+        if (node.hasSlicer) {
+            await pMap(Object.values(node.slicerExecutions), async (exId) => {
+                const errMsg = `node ${nodeId} has been disconnected from cluster_master past the allowed timeout, it has an active slicer for execution: ${exId} which will be marked as terminated and shut down`;
+                this.logger.error(errMsg);
+                const metaData = this.executionStore.executionMetaData(null, errMsg);
+                this.pendingWorkerRequests.remove(exId, 'ex_id');
+                try {
+                    await this.executionStore.setStatus(exId, 'terminated', metaData);
+                }
+                catch (err) {
+                    this.logger.error(err, `failure to set execution ${exId} status to terminated`);
+                }
+                finally {
+                    this.messaging.broadcast('cluster:execution:stop', { ex_id: exId });
+                }
+            });
+        }
+        // for any other worker not part of what is being shutdown, we attempt to reallocate
+        await pMap(Object.keys(node.workerExecutions), async (exId) => {
+            // looking for unique ex_id's not in slicerJobID
+            if (!node.slicerExecutions[exId]) {
+                const activeWorkers = this.clusterState[nodeId].active;
+                const numOfWorkers = activeWorkers.filter((worker) => worker.ex_id === exId).length;
+                try {
+                    const execution = await this.executionStore.getActiveExecution(exId);
+                    this.addWorkers(execution, numOfWorkers);
+                }
+                catch (err) {
+                    this.logger.error(err, `failure to add workers to execution ${exId}`);
+                }
+            }
+        });
+        // cleanup key so we don't get ever growing obj
+        delete this.droppedNodes[nodeId];
+        delete this.clusterState[nodeId];
+    }
+    async initialize() {
+        this.logger.info('native clustering initializing');
+        this.executionStore = this.context.stores.executionStorage;
+        if (!this.executionStore) {
+            throw new Error('Missing required stores');
+        }
+        const server = this.clusterMasterServer.httpServer;
+        await this.messaging.listen({ server });
+        this.clusterStateInterval = setInterval(() => {
+            this.logger.trace('cluster_master requesting state update for all nodes');
+            this.messaging.broadcast('cluster:node:state');
+        }, this.nodeStateInterval);
+    }
+    getClusterState() {
+        return cloneDeep(this.clusterState);
+    }
+    _checkNode(node) {
+        const obj = {
+            hasSlicer: false,
+            numOfSlicers: 0,
+            slicerExecutions: {},
+            workerExecutions: {},
+            numOfWorkers: 0,
+            available: node.available
+        };
+        return node.active.reduce((prev, curr) => {
+            if (curr.assignment === 'execution_controller') {
+                prev.hasSlicer = true;
+                prev.numOfSlicers += 1;
+                prev.slicerExecutions[curr.ex_id] = curr.ex_id;
+            }
+            if (curr.assignment === 'worker') {
+                prev.numOfWorkers += 1;
+                // if not resgistered, set it to one, if so then increment it
+                if (!prev.workerExecutions[curr.ex_id]) {
+                    prev.workerExecutions[curr.ex_id] = 1;
+                }
+                else {
+                    prev.workerExecutions[curr.ex_id] += 1;
+                }
+            }
+            return prev;
+        }, obj);
+    }
+    _findNodeForSlicer(stateArray, errorNodes) {
+        let slicerNode = null;
+        for (let i = 0; i < stateArray.length; i += 1) {
+            if (stateArray[i].state === 'connected' && stateArray[i].available > 0 && !errorNodes[stateArray[i].node_id]) {
+                const node = this._checkNode(stateArray[i]);
+                if (!node.hasSlicer) {
+                    slicerNode = stateArray[i].node_id;
+                    break;
+                }
+            }
+        }
+        // if all nodes have a slicer
+        if (!slicerNode) {
+            // list is already sorted by num available since stateArray is sorted
+            slicerNode = stateArray[0].node_id;
+        }
+        return slicerNode;
+    }
+    _findNodesForExecution(exId, slicerOnly) {
+        const nodes = [];
+        for (const [, node] of Object.entries(this.clusterState)) {
+            if (node.state !== 'disconnected') {
+                const hasJob = node.active.filter((worker) => {
+                    if (slicerOnly) {
+                        return worker.ex_id === exId && worker.assignment === 'execution_controller';
+                    }
+                    return worker.ex_id === exId;
+                });
+                if (hasJob.length >= 1) {
+                    nodes.push({
+                        node_id: node.node_id,
+                        ex_id: exId,
+                        hostname: node.hostname,
+                        workers: hasJob
+                    });
+                }
+            }
+        }
+        return nodes;
+    }
+    _availableWorkers(all, forceCheck) {
+        let num = 0;
+        // determine which key to search for in cluster state
+        if (this.pendingWorkerRequests.size() === 0 || forceCheck) {
+            const key = all ? 'total' : 'available';
+            for (const [, node] of Object.entries(this.clusterState)) {
+                if (node.state === 'connected') {
+                    num += node[key];
+                }
+            }
+        }
+        return num;
+    }
+    _findPort(nodeId) {
+        return this.messaging.send({
+            to: 'node_master',
+            address: nodeId,
+            message: 'cluster:node:get_port',
+            response: true
+        });
+    }
+    // designed to allocate additional workers, not any future slicers
+    async allocateWorkers(execution, numOfWorkersRequested) {
+        const exId = execution.ex_id;
+        const jobId = execution.job_id;
+        const jobStr = JSON.stringify(execution);
+        const sortedNodes = orderBy(this.clusterState, 'available', 'desc');
+        let workersRequested = numOfWorkersRequested;
+        let availWorkers = this._availableWorkers(false, true);
+        const dispatch = new Dispatch();
+        while (workersRequested > 0 && availWorkers > 0) {
+            for (let i = 0; i < sortedNodes.length; i += 1) {
+                // each iteration check if it can allocate
+                if (workersRequested > 0 && availWorkers > 0) {
+                    if (sortedNodes[i].available >= 1) {
+                        dispatch.set(sortedNodes[i].node_id, 1);
+                        availWorkers -= 1;
+                        workersRequested -= 1;
+                    }
+                }
+                else {
+                    break;
+                }
+            }
+        }
+        // if left over worker requests, enqueue them, queue works based off of id
+        // so it redundantly references ex_id
+        const workerData = {
+            job: jobStr,
+            id: exId,
+            ex_id: exId,
+            job_id: jobId,
+            workers: 1,
+            assignment: 'worker'
+        };
+        while (workersRequested > 0) {
+            this.logger.trace(`adding worker to pending queue for ex: ${exId}`);
+            this.pendingWorkerRequests.enqueue(workerData);
+            workersRequested -= 1;
+        }
+        const results = [];
+        for (const [nodeId, workerCount] of Object.entries(dispatch.getDispatch())) {
+            const requestedWorkersData = {
+                job: jobStr,
+                id: exId,
+                ex_id: exId,
+                job_id: jobId,
+                workers: workerCount,
+                assignment: 'worker'
+            };
+            const createRequest = this.messaging.send({
+                to: 'node_master',
+                address: nodeId,
+                message: 'cluster:workers:create',
+                payload: requestedWorkersData,
+                response: true
+            }).then((msg) => {
+                const createdWorkers = get(msg, 'payload.createdWorkers');
+                if (!isInteger(createdWorkers)) {
+                    this.logger.error(`malformed response from create workers request to node ${nodeId}`, msg);
+                    return;
+                }
+                if (createdWorkers < workerCount) {
+                    this.logger.warn(`node ${nodeId} was only able to allocate ${createdWorkers} the request worker count of ${workerCount}, enqueing the remainder`);
+                    const newWorkersRequest = cloneDeep(requestedWorkersData);
+                    newWorkersRequest.workers = workerCount - createdWorkers;
+                    this.pendingWorkerRequests.enqueue(newWorkersRequest);
+                }
+                else {
+                    this.logger.debug(`node ${nodeId} allocated ${createdWorkers}`);
+                }
+            })
+                .catch((err) => {
+                this.logger.error(err, `An error has occurred in allocating : ${workerCount} workers to node : ${nodeId}, the worker request has been enqueued`);
+                this.pendingWorkerRequests.enqueue(requestedWorkersData);
+            });
+            results.push(createRequest);
+        }
+        // this will resolve successfully if one worker was actually allocated
+        return Promise.all(results);
+    }
+    async _createSlicer(ex, errorNodes) {
+        const execution = cloneDeep(ex);
+        const sortedNodes = orderBy(this.clusterState, 'available', 'desc');
+        const slicerNodeID = this._findNodeForSlicer(sortedNodes, errorNodes);
+        // need to mutate job so that workers will know the specific port and
+        // hostname of the created slicer
+        const portObj = await this._findPort(slicerNodeID);
+        execution.slicer_port = portObj.port;
+        execution.slicer_hostname = this.clusterState[slicerNodeID].hostname;
+        this.logger.debug(`node ${this.clusterState[slicerNodeID].hostname} has been elected for slicer, listening on port: ${portObj.port}`);
+        const exId = execution.ex_id;
+        const jobId = execution.job_id;
+        const jobStr = JSON.stringify(execution);
+        const data = {
+            job: jobStr,
+            ex_id: exId,
+            job_id: jobId,
+            workers: 1,
+            slicer_port: portObj.port,
+            node_id: slicerNodeID,
+            assignment: 'execution_controller'
+        };
+        try {
+            await this.messaging.send({
+                to: 'node_master',
+                address: slicerNodeID,
+                message: 'cluster:execution_controller:create',
+                payload: data,
+                response: true
+            });
+            return execution;
+        }
+        catch (err) {
+            const error = new TSError(err, {
+                reason: `failed to allocate execution_controller to ${slicerNodeID}`
+            });
+            this.logger.error(error);
+            errorNodes[slicerNodeID] = getFullErrorStack(error);
+            throw err;
+        }
+    }
+    async allocateSlicer(ex) {
+        let retryCount = 0;
+        const errorNodes = {};
+        const _allocateSlicer = async () => {
+            try {
+                return await this._createSlicer(ex, errorNodes);
+            }
+            catch (err) {
+                retryCount += 1;
+                if (retryCount >= this.slicerAllocationAttempts) {
+                    throw new Error(`Failed to allocate execution_controller to nodes: ${JSON.stringify(errorNodes)}`);
+                }
+                else {
+                    await pDelay(100);
+                    return _allocateSlicer();
+                }
+            }
+        };
+        return _allocateSlicer();
+    }
+    addWorkers(execution, workerNum) {
+        const workerData = {
+            job: JSON.stringify(execution),
+            id: execution.ex_id,
+            ex_id: execution.ex_id,
+            job_id: execution.job_id,
+            workers: workerNum,
+            assignment: 'worker'
+        };
+        this.pendingWorkerRequests.enqueue(workerData);
+        return { action: 'enqueued', ex_id: execution.ex_id, workerNum };
+    }
+    setWorkers(execution, workerNum) {
+        const totalWorker = findWorkersByExecutionID(this.clusterState, execution.ex_id).length;
+        if (totalWorker > workerNum) {
+            const removedWorkersCount = totalWorker - workerNum;
+            return this.removeWorkers(execution.ex_id, removedWorkersCount);
+        }
+        if (totalWorker < workerNum) {
+            const addWorkersCount = workerNum - totalWorker;
+            return this.addWorkers(execution, addWorkersCount);
+        }
+        // if they are equal then no work needs to be done
+        return { action: 'set', ex_id: execution.ex_id, workerNum };
+    }
+    removeWorkers(exId, workerNum) {
+        const dispatch = new Dispatch();
+        const workers = findWorkersByExecutionID(this.clusterState, exId);
+        let workerCount = workerNum;
+        const workersData = workers.reduce((prev, curr) => {
+            if (!prev[curr.node_id]) {
+                prev[curr.node_id] = 1;
+            }
+            else {
+                prev[curr.node_id] += 1;
+            }
+            prev._total += 1;
+            return prev;
+        }, { _total: 0 });
+        if (workerNum >= workersData._total) {
+            const errMsg = `workers to be removed: ${workerNum} cannot be >= to current workers: ${workersData._total}`;
+            const error = new TSError(errMsg, {
+                statusCode: 400,
+            });
+            this.logger.error(error);
+            return Promise.reject(error);
+        }
+        while (workerCount) {
+            for (const [key] of Object.entries(workersData)) {
+                if (key !== '_total') {
+                    if (workersData[key] >= 1 && workerCount > 0) {
+                        dispatch.set(key, 1);
+                        workersData[key] -= 1;
+                        workerCount -= 1;
+                    }
+                }
+            }
+        }
+        const nodes = dispatch.getDispatch();
+        const messagesSent = [];
+        for (const [key, val] of Object.entries(nodes)) {
+            messagesSent.push(this.messaging.send({
+                to: 'node_master',
+                address: key,
+                message: 'cluster:workers:remove',
+                ex_id: exId,
+                payload: { workers: val },
+                response: true
+            }));
+        }
+        return Promise.all(messagesSent)
+            .then(() => ({ action: 'remove', ex_id: exId, workerNum }))
+            .catch((err) => {
+            const error = new TSError(err, {
+                reason: `Error while releasing workers from job ${exId}`
+            });
+            this.logger.error(error);
+            return Promise.reject(error);
+        });
+    }
+    _notifyNodesWithExecution(exId, messageData, excludeNode) {
+        return new Promise((resolve, reject) => {
+            let nodes = this._findNodesForExecution(exId);
+            if (excludeNode) {
+                nodes = nodes.filter((node) => node.hostname !== excludeNode);
+            }
+            else if (messageData.message !== 'cluster:execution:stop' && nodes.length === 0) {
+                // exclude node is only in regards to a shutdown on the cluster_master, which
+                // already receives the shutdown notice so it can be empty, in all other
+                // circumstances if the node list length is zero then reject
+                const error = new TSError(`Could not find active execution processes for ex_id: ${exId}`);
+                error.statusCode = 404;
+                reject(error);
+                return;
+            }
+            const promises = nodes.map((node) => {
+                const sendingMsg = Object.assign(messageData, {
+                    to: 'node_master',
+                    address: node.node_id,
+                    ex_id: exId,
+                    response: false
+                });
+                this.logger.trace(`notifying node ${node.node_id} to stop execution ${exId}`, sendingMsg);
+                return this.messaging.send(sendingMsg);
+            });
+            Promise.all(promises)
+                .then(() => {
+                resolve(true);
+            })
+                .catch((err) => {
+                const error = new Error(`Failure to notify node with execution ${exId}, caused by ${err.message}`);
+                this.logger.error(error);
+                reject(error);
+            });
+        });
+    }
+    readyForAllocation() {
+        return this._availableWorkers() >= 2;
+    }
+    clusterAvailable() { }
+    async stopExecution(exId, options) {
+        // we are allowing stopExecution to be non blocking, we block at api level
+        this.pendingWorkerRequests.remove(exId, 'ex_id');
+        const sendingMessage = { message: 'cluster:execution:stop' };
+        if (options?.timeout) {
+            sendingMessage.timeout = options.timeout;
+        }
+        return this._notifyNodesWithExecution(exId, sendingMessage, options?.excludeNode);
+    }
+    async shutdown() {
+        clearInterval(this.clusterStateInterval);
+        this.logger.info('native clustering shutting down');
+        if (this.messaging) {
+            await this.messaging.shutdown();
+        }
+        else {
+            await pDelay(100);
+        }
+    }
+    async listResourcesForJobId() {
+        return [];
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/src/lib/cluster/services/cluster/backends/native/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/native/index.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,OAAO,EACH,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EACpC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAC7B,KAAK,EACR,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,MAAM,kDAAkD,CAAC;AAC9E,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AA+B3C,MAAM,OAAO,gBAAgB;IACzB,OAAO,CAAuB;IAC9B,MAAM,CAAS;IACf,MAAM,CAAe;IACrB,cAAc,CAAoB;IAClC,qBAAqB,GAAG,IAAI,KAAK,EAAO,CAAC;IACzC,iBAAiB,CAAS;IAC1B,wBAAwB,CAAS;IACjC,YAAY,GAA8B,EAAE,CAAC;IAC7C,oBAAoB,CAA8B;IAClD,SAAS,CAAY;IACrB,YAAY,GAAwB,EAAE,CAAC;IACvC,mBAAmB,CAAM;IAEzB,YAAY,OAA6B,EAAE,mBAAwB;QAC/D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;QAC5D,MAAM,qBAAqB,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,uBAAuB,CAAC;QAClF,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,mBAAmB,CAAC;QACzE,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,0BAA0B,CAAC;QACvF,IAAI,CAAC,SAAS,GAAG,IAAI,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,yEAAyE;QACzE,kCAAkC;QAElC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACpB,KAAK,EAAE,aAAa;YACpB,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,CAAC,IAAa,EAAE,MAAc,EAAE,EAAE;gBACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,gBAAgB,CAAC,CAAC;gBACjD,oCAAoC;gBACpC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC5B,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;oBACxC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC7D,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAoB,CAAC;gBACtD,4DAA4D;gBAC5D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAClD,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACpB,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE,CAAC,QAAiB,EAAE,EAAE;gBAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAC9B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAiB,CAAC;gBACpD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,OAAO,yBAAyB,EAAE,IAAI,CAAC,CAAC;gBACvE,0DAA0D;gBAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAClD,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACpB,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,yDAAyD,CAAC;SAC9G,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;YACpB,KAAK,EAAE,oBAAoB;YAC3B,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,CAAC,GAAiB,EAAE,MAAc,EAAE,EAAE;gBAC5C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;oBAAE,OAAO;gBAEvC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,mBAAmB,CAAC,CAAC;oBACpD,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrC,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,cAAc,CAAC;oBACjD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;wBAChC,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACpC,CAAC,EAAE,qBAAqB,CAAC,CAAC;oBAE1B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;gBACtC,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QACH,sCAAsC;QACtC,MAAM,uBAAuB,GAAG,QAAQ,CAAC,GAAG,EAAE;YAC1C,IAAI,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChF,MAAM,eAAe,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBAE5C,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,eAAe,CAAC,OAAO,CAAC;qBAC7C,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE;wBAC3B,MAAM,EAAE,mCAAmC;qBAC9C,CAAC,CAAC;oBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC,CAAC,CAAC;YACX,CAAC;QACL,CAAC,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,2BAA2B,EAAE,uBAAuB,CAAC,CAAC;IACzE,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAc;QACrC,4BAA4B;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,gFAAgF;QAChF,wBAAwB;QACxB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC5D,MAAM,MAAM,GAAG,QAAQ,MAAM,+GAA+G,IAAI,mDAAmD,CAAC;gBACpM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACrE,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAEjD,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;gBACtE,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,4BAA4B,IAAI,uBAAuB,CAAC,CAAC;gBACpF,CAAC;wBAAS,CAAC;oBACP,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxE,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QACD,oFAAoF;QACpF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1D,gDAAgD;YAChD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;gBAEvD,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CACrC,CAAC,MAAW,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CACzC,CAAC,MAAM,CAAC;gBAET,IAAI,CAAC;oBACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;oBACrE,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;gBAC7C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,uCAAuC,IAAI,EAAE,CAAC,CAAC;gBAC1E,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,+CAA+C;QAC/C,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;QAE3D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC;QAEnD,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAExC,IAAI,CAAC,oBAAoB,GAAG,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;YAC1E,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QACnD,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED,eAAe;QACX,OAAO,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxC,CAAC;IAEO,UAAU,CAAC,IAAe;QAC9B,MAAM,GAAG,GAAmB;YACxB,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,CAAC;YACf,gBAAgB,EAAE,EAAE;YACpB,gBAAgB,EAAE,EAAE;YACpB,YAAY,EAAE,CAAC;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;YACrC,IAAI,IAAI,CAAC,UAAU,KAAK,sBAAsB,EAAE,CAAC;gBAC7C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;gBACvB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YACnD,CAAC;YAED,IAAI,IAAI,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC/B,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC;gBACvB,6DAA6D;gBAC7D,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACrC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3C,CAAC;YACL,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC,EAAE,GAAG,CAAC,CAAC;IACZ,CAAC;IAEO,kBAAkB,CAAC,UAAuB,EAAE,UAA+B;QAC/E,IAAI,UAAU,GAAG,IAAI,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3G,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE5C,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClB,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;oBACnC,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,qEAAqE;YACrE,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACvC,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAEO,sBAAsB,CAAC,IAAY,EAAE,UAAoB;QAC7D,MAAM,KAAK,GAAU,EAAE,CAAC;QAExB,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YACvD,IAAI,IAAI,CAAC,KAAK,KAAK,cAAc,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAW,EAAE,EAAE;oBAC9C,IAAI,UAAU,EAAE,CAAC;wBACb,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,UAAU,KAAK,sBAAsB,CAAC;oBACjF,CAAC;oBAED,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC;gBACjC,CAAC,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBACrB,KAAK,CAAC,IAAI,CAAC;wBACP,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,KAAK,EAAE,IAAI;wBACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,OAAO,EAAE,MAAM;qBAClB,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,iBAAiB,CAAC,GAAa,EAAE,UAAoB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,qDAAqD;QACrD,IAAI,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;YACxD,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC;YAExC,KAAK,MAAM,CAAC,EAAC,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtD,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;oBAC7B,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,SAAS,CAAC,MAAc;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YACvB,EAAE,EAAE,aAAa;YACjB,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,uBAAuB;YAChC,QAAQ,EAAE,IAAI;SACjB,CAAC,CAAC;IACP,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,eAAe,CAAC,SAA0B,EAAE,qBAA6B;QAC3E,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACpE,IAAI,gBAAgB,GAAG,qBAAqB,CAAC;QAC7C,IAAI,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAEvD,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAEhC,OAAO,gBAAgB,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,0CAA0C;gBAC1C,IAAI,gBAAgB,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;oBAC3C,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;wBAChC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;wBACxC,YAAY,IAAI,CAAC,CAAC;wBAClB,gBAAgB,IAAI,CAAC,CAAC;oBAC1B,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC;QACD,0EAA0E;QAC1E,qCAAqC;QAErC,MAAM,UAAU,GAAG;YACf,GAAG,EAAE,MAAM;YACX,EAAE,EAAE,IAAI;YACR,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,CAAC;YACV,UAAU,EAAE,QAAQ;SACvB,CAAC;QAEF,OAAO,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,IAAI,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC/C,gBAAgB,IAAI,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,OAAO,GAAU,EAAE,CAAC;QAE1B,KAAK,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzE,MAAM,oBAAoB,GAAG;gBACzB,GAAG,EAAE,MAAM;gBACX,EAAE,EAAE,IAAI;gBACR,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,WAAW;gBACpB,UAAU,EAAE,QAAQ;aACvB,CAAC;YAEF,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBACtC,EAAE,EAAE,aAAa;gBACjB,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,wBAAwB;gBACjC,OAAO,EAAE,oBAAoB;gBAC7B,QAAQ,EAAE,IAAI;aACjB,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBACZ,MAAM,cAAc,GAAG,GAAG,CAAC,GAAG,EAAE,wBAAwB,CAAsB,CAAC;gBAC/E,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0DAA0D,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;oBAC3F,OAAO;gBACX,CAAC;gBACD,IAAI,cAAc,GAAG,WAAW,EAAE,CAAC;oBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,8BAA8B,cAAc,gCAAgC,WAAW,0BAA0B,CAAC,CAAC;oBAClJ,MAAM,iBAAiB,GAAG,SAAS,CAAC,oBAAoB,CAAC,CAAC;oBAC1D,iBAAiB,CAAC,OAAO,GAAG,WAAW,GAAG,cAAc,CAAC;oBACzD,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;gBAC1D,CAAC;qBAAM,CAAC;oBACJ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,cAAc,cAAc,EAAE,CAAC,CAAC;gBACpE,CAAC;YACL,CAAC,CAAC;iBACG,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,yCAAyC,WAAW,sBAAsB,MAAM,wCAAwC,CAAC,CAAC;gBACjJ,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;YAEP,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChC,CAAC;QAED,sEAAsE;QACtE,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,EAAmB,EAAE,UAA+B;QAC5E,MAAM,SAAS,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAEtE,qEAAqE;QACrE,iCAAiC;QACjC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAQ,CAAC;QAC1D,SAAS,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;QACrC,SAAS,CAAC,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC;QAErE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,QAAQ,oDAAoD,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAEtI,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,IAAI,GAAG;YACT,GAAG,EAAE,MAAM;YACX,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,OAAO,EAAE,YAAY;YACrB,UAAU,EAAE,sBAAsB;SACrC,CAAC;QAEF,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBACtB,EAAE,EAAE,aAAa;gBACjB,OAAO,EAAE,YAAY;gBACrB,OAAO,EAAE,qCAAqC;gBAC9C,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,IAAI;aACjB,CAAC,CAAC;YACH,OAAO,SAAS,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE;gBAC3B,MAAM,EAAE,8CAA8C,YAAY,EAAE;aACvE,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,UAAU,CAAC,YAAY,CAAC,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAmB;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,UAAU,GAAG,EAAE,CAAC;QAEtB,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YAC/B,IAAI,CAAC;gBACD,OAAO,MAAM,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,UAAU,IAAI,CAAC,CAAC;gBAChB,IAAI,UAAU,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;oBAC9C,MAAM,IAAI,KAAK,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBACvG,CAAC;qBAAM,CAAC;oBACJ,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;oBAClB,OAAO,eAAe,EAAE,CAAC;gBAC7B,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QACF,OAAO,eAAe,EAAE,CAAC;IAC7B,CAAC;IAED,UAAU,CAAC,SAAc,EAAE,SAAiB;QACxC,MAAM,UAAU,GAAG;YACf,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;YAC9B,EAAE,EAAE,SAAS,CAAC,KAAK;YACnB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,OAAO,EAAE,SAAS;YAClB,UAAU,EAAE,QAAQ;SACvB,CAAC;QACF,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IACrE,CAAC;IAED,UAAU,CAAC,SAAc,EAAE,SAAiB;QACxC,MAAM,WAAW,GAAG,wBAAwB,CACxC,IAAI,CAAC,YAAY,EACjB,SAAS,CAAC,KAAK,CAClB,CAAC,MAAM,CAAC;QACT,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,mBAAmB,GAAG,WAAW,GAAG,SAAS,CAAC;YACpD,OAAO,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,CAAC;YAChD,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;QACvD,CAAC;QACD,kDAAkD;QAClD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;IAChE,CAAC;IAED,aAAa,CAAC,IAAY,EAAE,SAAiB;QACzC,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,wBAAwB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAClE,IAAI,WAAW,GAAG,SAAS,CAAC;QAE5B,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAyB,EAAE,IAAI,EAAE,EAAE;YACnE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;YACD,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YAEjB,OAAO,IAAI,CAAC;QAChB,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QAElB,IAAI,SAAS,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,0BAA0B,SAAS,qCAAqC,WAAW,CAAC,MAAM,EAAE,CAAC;YAC5G,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE;gBAC9B,UAAU,EAAE,GAAG;aAClB,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,WAAW,EAAE,CAAC;YACjB,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9C,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACnB,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;wBAC3C,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;wBACrB,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;wBACtB,WAAW,IAAI,CAAC,CAAC;oBACrB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,EAAE,CAAC;QAExB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,YAAY,CAAC,IAAI,CACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAChB,EAAE,EAAE,aAAa;gBACjB,OAAO,EAAE,GAAG;gBACZ,OAAO,EAAE,wBAAwB;gBACjC,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE;gBACzB,QAAQ,EAAE,IAAI;aACjB,CAAC,CACL,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;aAC3B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;aAC1D,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACX,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE;gBAC3B,MAAM,EAAE,0CAA0C,IAAI,EAAE;aAC3D,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACX,CAAC;IAEO,yBAAyB,CAAC,IAAY,EAAE,WAAgB,EAAE,WAAoB;QAClF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAE9C,IAAI,WAAW,EAAE,CAAC;gBACd,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,WAAW,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,WAAW,CAAC,OAAO,KAAK,wBAAwB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChF,6EAA6E;gBAC7E,wEAAwE;gBACxE,4DAA4D;gBAC5D,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,wDAAwD,IAAI,EAAE,CAAC,CAAC;gBAC1F,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC;gBACvB,MAAM,CAAC,KAAK,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAChC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE;oBAC1C,EAAE,EAAE,aAAa;oBACjB,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,KAAK,EAAE,IAAI;oBACX,QAAQ,EAAE,KAAK;iBAClB,CAAC,CAAC;gBAEH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,OAAO,sBAAsB,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;gBAE1F,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;iBAChB,IAAI,CAAC,GAAG,EAAE;gBACP,OAAO,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,yCAAyC,IAAI,eAAe,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACnG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACP,CAAC;IAED,kBAAkB;QACd,OAAO,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC;IAED,gBAAgB,KAAI,CAAC;IAErB,KAAK,CAAC,aAAa,CAAC,IAAY,EAAE,OAA8B;QAC5D,0EAA0E;QAC1E,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,cAAc,GAAG,EAAE,OAAO,EAAE,wBAAwB,EAAyB,CAAC;QAEpF,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YACnB,cAAc,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC7C,CAAC;QACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;IACtF,CAAC;IAED,KAAK,CAAC,QAAQ;QACV,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,qBAAqB;QACvB,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}

package/dist/src/lib/cluster/services/cluster/backends/native/messaging.d.ts

@@ -0,0 +1,70 @@
+import type { EventEmitter } from 'node:events';
+import type { Server as HttpServer } from 'node:http';
+import type { Server as HttpsServer } from 'node:https';
+import { Logger, Queue } from '@terascope/core-utils';
+import { Context } from '@terascope/job-components';
+export declare const routing: Readonly<{
+    cluster_master: {
+        node_master: string;
+    };
+    node_master: {
+        cluster_process: string;
+        cluster_master: string;
+    };
+    assets_service: {
+        cluster_master: string;
+    };
+}>;
+type HookFN = () => void | null;
+type ListenOptions = {
+    server?: HttpServer | HttpsServer;
+    port?: string | number;
+    query?: {
+        node_id: string;
+    };
+};
+export declare class Messaging {
+    context: Context;
+    logger: Logger;
+    events: EventEmitter;
+    configTimeout: number;
+    networkLatencyBuffer: number;
+    hostURL: string;
+    messsagingOnline: boolean;
+    messagingQueue: Queue<any>;
+    childHookFn: HookFN;
+    config: any;
+    self: string;
+    selfMessages: Record<string, any>;
+    processContext: Record<string, any>;
+    functionMapping: Record<string, any>;
+    io: Record<string, any>;
+    constructor(context: Context, logger: Logger, __io?: any);
+    private _handleResponse;
+    respond(incoming: any, outgoing?: any): Promise<unknown>;
+    private _findAndSend;
+    private _sendToProcesses;
+    register(eventConfig: Record<string, any>): void;
+    private _registerFns;
+    private _determinePathForMessage;
+    private _attachRoomsSocketIO;
+    private _createIOServer;
+    listen(options?: ListenOptions): void;
+    broadcast(eventName: string, payload?: Record<string, any>): void;
+    private _forwardMessage;
+    send(messageSent: any): Promise<unknown>;
+    private _makeConfigurations;
+    private _makeHostName;
+    private _getMessages;
+    getHostUrl(): string | null;
+    getClientCounts(): any;
+    listRooms(): string[];
+    registerChildOnlineHook(fn: () => void): void;
+    private _isExecutionStateQuery;
+    private _emitIpcMessage;
+    private _handleIpcMessages;
+    private _handleWorkerMessage;
+    shutdown(): Promise<void>;
+}
+export {};
+//# sourceMappingURL=messaging.d.ts.map

package/dist/src/lib/cluster/services/cluster/backends/native/messaging.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"messaging.d.ts","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/native/messaging.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,KAAK,EAAE,MAAM,IAAI,WAAW,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EACK,MAAM,EAEd,KAAK,EACR,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AAwDpD,eAAO,MAAM,OAAO;;;;;;;;;;;EAWlB,CAAC;AAEH,KAAK,MAAM,GAAG,MAAM,IAAI,GAAG,IAAI,CAAC;AAEhC,KAAK,aAAa,GAAG;IACjB,MAAM,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/B,CAAC;AAEF,qBAAa,SAAS;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,YAAY,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,UAAS;IACzB,cAAc,aAAoB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,GAAG,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAClC,cAAc,EAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACrC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACrC,EAAE,EAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAEb,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG;IA+CxD,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,GAAG;IAWrC,OAAO,CAAC,YAAY;IAqBpB,OAAO,CAAC,gBAAgB;IAiBxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAiCzC,OAAO,CAAC,YAAY;IAsCpB,OAAO,CAAC,wBAAwB;IAoBhC,OAAO,CAAC,oBAAoB;IAkB5B,OAAO,CAAC,eAAe;IAwBvB,MAAM,CAAC,OAAO,GAAE,aAAkB;IAyClC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAQ9D,OAAO,CAAC,eAAe;IAsBvB,IAAI,CAAC,WAAW,EAAE,GAAG;IAoCrB,OAAO,CAAC,mBAAmB;IA0C3B,OAAO,CAAC,aAAa;IAuBrB,OAAO,CAAC,YAAY;IAOpB,UAAU;IAQV,eAAe;IAQf,SAAS,IAAI,MAAM,EAAE;IAUrB,uBAAuB,CAAC,EAAE,EAAE,MAAM,IAAI;IAItC,OAAO,CAAC,sBAAsB;IAU9B,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,oBAAoB;IAYtB,QAAQ;CAMjB"}