teraslice 3.2.1 → 3.3.1

package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js

@@ -1,548 +0,0 @@
-import { nanoid } from 'nanoid';
-import { pDelay, isFunction, isEmpty, get, toNumber, isKey, Queue } from '@terascope/core-utils';
-import { io as socketIOClient } from 'socket.io-client';
-import { Server as socketIOServer } from 'socket.io';
-import { isProcessAssignment, ProcessAssignment } from '../../../../../../interfaces.js';
-// messages send to cluster_master
-const clusterMasterMessages = {
-    ipc: {
-        'process:SIGTERM': 'SIGTERM',
-        'process:SIGINT': 'SIGINT',
-    },
-    intraProcess: {
-        'network:disconnect': 'disconnect',
-        'network:error': 'error',
-    },
-    network: {
-        'node:online': 'node:online',
-        'node:state': 'node:state',
-        'execution:recovery:failed': 'execution:recovery:failed',
-        'execution:finished': 'execution:finished',
-        'cluster:analytics': 'cluster:analytics',
-        'execution:error:terminal': 'execution:error:terminal',
-        'assets:preloaded': 'assets:preloaded',
-        'assets:service:available': 'assets:service:available'
-    }
-};
-const nodeMasterMessages = {
-    ipc: {
-        'cluster:error:terminal': 'cluster:error:terminal',
-        'child:exit': 'exit'
-    },
-    intraProcess: {
-        'network:connect': 'connect',
-        'network:disconnect': 'disconnect',
-        'network:error': 'error',
-    },
-    network: {
-        'cluster:execution_controller:create': 'cluster:execution_controller:create',
-        'cluster:workers:create': 'cluster:workers:create',
-        'cluster:workers:remove': 'cluster:workers:remove',
-        'cluster:node:state': 'cluster:node:state',
-        'cluster:execution:stop': 'cluster:execution:stop',
-        'cluster:node:get_port': 'cluster:node:get_port',
-    }
-};
-const assetServiceMessages = {
-    ipc: {
-        'process:SIGTERM': 'SIGTERM',
-        'process:SIGINT': 'SIGINT',
-    },
-    network: {}
-};
-// messaging destination relative towards each process type
-export const routing = Object.freeze({
-    cluster_master: {
-        node_master: 'network',
-    },
-    node_master: {
-        cluster_process: 'ipc',
-        cluster_master: 'network',
-    },
-    assets_service: {
-        cluster_master: 'ipc'
-    }
-});
-export class Messaging {
-    context;
-    logger;
-    events;
-    configTimeout;
-    networkLatencyBuffer;
-    hostURL;
-    messsagingOnline = false;
-    messagingQueue = new Queue();
-    childHookFn;
-    config;
-    self;
-    selfMessages;
-    processContext;
-    functionMapping;
-    io;
-    constructor(context, logger, __io) {
-        if (__io) {
-            this.io = __io;
-        }
-        this.context = context;
-        this.functionMapping = {};
-        // processContext is set in _makeConfigurations
-        this.config = this._makeConfigurations();
-        const { hostURL } = this.config;
-        this.hostURL = hostURL;
-        this.configTimeout = context.sysconfig.teraslice.action_timeout;
-        this.networkLatencyBuffer = context.sysconfig.teraslice.network_latency_buffer;
-        this.self = this.config.assignment;
-        this.events = context.apis.foundation.getSystemEvents();
-        this.selfMessages = this._getMessages(this.self);
-        this.logger = logger;
-        // @ts-expect-error TODO: fixme
-        this.childHookFn = null;
-        this.logger.debug(`messaging service configuration for assignment ${this.config.assignment}`);
-        // set a default listener the is used for forwarding/completing responses
-        this.functionMapping['messaging:response'] = this._handleResponse.bind(this);
-        this.processContext.on('messaging:response', this._handleResponse.bind(this));
-        // all child processes need to set up a process listener on the 'message' event
-        if (this.config.clients.ipcClient) {
-            process.on('message', this._handleIpcMessages().bind(this));
-        }
-        else {
-            this.processContext.on('online', (worker) => {
-                this.logger.debug('worker process has come online');
-                if (this.childHookFn) {
-                    this.childHookFn();
-                }
-                // @ts-expect-error
-                const contextWorker = this.processContext.workers[worker.id];
-                // don't double subscribe
-                contextWorker.removeListener('message', this._handleWorkerMessage.bind(this));
-                // set up a message handler on each child created, if a child is talking to cluster
-                // then pass it on, else invoke process event handler on node_master
-                contextWorker.on('message', this._handleWorkerMessage.bind(this));
-            });
-        }
-    }
-    _handleResponse(msgResponse) {
-        // if msg has returned to source then emit it else pass it along
-        if (msgResponse.__source === this.config.assignment) {
-            this.logger.trace(`node message ${msgResponse.__msgId} has been processed`);
-            // we are in the right spot, emit to complete the promise from send
-            this.events.emit(msgResponse.__msgId, msgResponse);
-        }
-        else {
-            this._forwardMessage(msgResponse);
-        }
-    }
-    respond(incoming, outgoing) {
-        const outgoingResponse = (outgoing && typeof outgoing === 'object') ? outgoing : {};
-        if (incoming.__msgId) {
-            outgoingResponse.__msgId = incoming.__msgId;
-        }
-        outgoingResponse.__source = incoming.__source;
-        outgoingResponse.message = 'messaging:response';
-        outgoingResponse.to = incoming.__source;
-        return this.send(outgoingResponse);
-    }
-    _findAndSend(filterFn, msg, msgHookFn) {
-        const childProcesses = this.context.cluster.workers;
-        const children = Object.values(childProcesses).filter(filterFn);
-        if (children.length === 0 && msg.response) {
-            // if there are no child processes found and it needs a response, answer back so
-            // that it does not hold for a long time
-            this.respond(msg);
-        }
-        children.forEach((childProcess) => {
-            if (msgHookFn)
-                msgHookFn(childProcess);
-            // @ts-expect-error
-            if (childProcess.connected) {
-                childProcess.send(msg);
-            }
-            else {
-                this.logger.warn('cannot send message to process', msg);
-            }
-        });
-    }
-    _sendToProcesses(msg) {
-        const msgExId = msg.ex_id || get(msg, 'payload.ex_id');
-        if (msgExId) {
-            // all processes that have the same assignment and exId
-            const filterFn = (process) => {
-                if (process.assignment !== msg.to)
-                    return false;
-                if (process.ex_id !== msgExId)
-                    return false;
-                return true;
-            };
-            this._findAndSend(filterFn, msg);
-        }
-        else {
-            // all processes that have the same assignment
-            const filterFn = (process) => process.assignment === msg.to;
-            this._findAndSend(filterFn, msg);
-        }
-    }
-    register(eventConfig) {
-        const eventName = eventConfig.event;
-        const { callback, identifier } = eventConfig;
-        const selfHasEvent = Object.values(this.selfMessages)
-            .some((type) => type[eventName] != null);
-        if (!selfHasEvent) {
-            throw new Error(`"${self}" cannot register for event, "${eventName}", in messaging module`);
-        }
-        if (this.selfMessages.ipc[eventName]) {
-            const realKey = this.selfMessages.ipc[eventName];
-            // this needs to be directly allocated so that IPC messaging can happen if
-            // network was not instantiated
-            this.processContext.on(realKey, callback);
-        }
-        else {
-            // we attach events etc later when the connection is made, this is async
-            // while others are sync registration
-            let trueEventName = this.selfMessages.network[eventName];
-            if (!trueEventName) {
-                trueEventName = this.selfMessages.intraProcess[eventName];
-            }
-            if (identifier) {
-                callback.__socketIdentifier = identifier;
-            }
-            this.functionMapping[trueEventName] = callback;
-        }
-    }
-    _registerFns(socket) {
-        for (const [key, func] of Object.entries(this.functionMapping)) {
-            if (func.__socketIdentifier) {
-                const wrappedFunc = (msg = {}) => {
-                    const identifier = func.__socketIdentifier;
-                    let id = msg[identifier];
-                    // if already set, extract value else set it on socket
-                    if (socket[identifier]) {
-                        id = socket[identifier];
-                    }
-                    else {
-                        socket[identifier] = id;
-                    }
-                    // if network host (slicer, cluster_master) and connection
-                    // or retry event, join room
-                    if (key === 'node:online') {
-                        const rooms = Object.keys(socket.rooms);
-                        const hasRoom = rooms.some((r) => r === id);
-                        if (!hasRoom) {
-                            this.logger.info(`joining room ${id}`);
-                            socket.join(id);
-                        }
-                    }
-                    // not all events have messages, if so then pass it, else just pass identifier
-                    if (msg) {
-                        func(msg, id, identifier);
-                    }
-                    else {
-                        func(id);
-                    }
-                };
-                socket.on(key, wrappedFunc);
-            }
-            else {
-                this.logger.trace(`setting listener key ${key}`);
-                socket.on(key, func);
-            }
-        }
-    }
-    _determinePathForMessage(messageSent) {
-        const { to } = messageSent;
-        let destinationType = undefined;
-        if (isKey(routing, this.self) && isKey(routing[this.self], to)) {
-            destinationType = routing[this.self][to];
-        }
-        // cluster_master has two types of connections to node_master, if it does not have a
-        // address then its talking to its own node_master through ipc
-        // TODO: reference self message, remove cluster_master specific code
-        if (this.self === 'cluster_master' && !messageSent.address && (messageSent.message in clusterMasterMessages.ipc)) {
-            destinationType = 'ipc';
-        }
-        if (destinationType === undefined) {
-            throw new Error(`could not determine how to pass on message to: ${JSON.stringify(messageSent)}`);
-        }
-        return destinationType;
-    }
-    // join rooms before on connect to avoid race conditions
-    _attachRoomsSocketIO() {
-        if (!this.io)
-            return;
-        // middleware
-        this.io.use((socket, next) => {
-            const { node_id: nodeId, } = socket.handshake.query;
-            if (nodeId) {
-                this.logger.info(`node ${nodeId} joining room on connect`);
-                socket.join(nodeId);
-            }
-            return next();
-        });
-    }
-    _createIOServer(options) {
-        const { server, port } = options;
-        const opts = {
-            path: '/native-clustering',
-            pingTimeout: this.configTimeout,
-            pingInterval: this.configTimeout + this.networkLatencyBuffer,
-            perMessageDeflate: false,
-            serveClient: false,
-        };
-        if (server) {
-            this.io = new socketIOServer(server, opts);
-        }
-        else if (port) {
-            this.io = new socketIOServer(opts);
-            this.io.listen(port);
-        }
-        this._attachRoomsSocketIO();
-        this.io.on('connection', (socket) => {
-            this.logger.debug('a connection to cluster_master has been made');
-            this._registerFns(socket);
-        });
-    }
-    listen(options = {}) {
-        const { query } = options;
-        this.messsagingOnline = true;
-        if (this.config.clients.networkClient) {
-            // node_master, worker
-            this.io = socketIOClient(this.hostURL, {
-                forceNew: true,
-                path: '/native-clustering',
-                query
-            });
-            this._registerFns(this.io);
-            if (this.self === 'node_master') {
-                this.io.on('networkMessage', (networkMsg) => {
-                    const { message } = networkMsg;
-                    const func = this.functionMapping[message];
-                    if (func) {
-                        func(networkMsg);
-                    }
-                    else {
-                        // if no function is registered, it is meant to by passed along to child
-                        this._sendToProcesses(networkMsg);
-                    }
-                });
-            }
-            this.logger.debug('client network connection is online');
-        }
-        else {
-            // cluster_master and test processes
-            this._createIOServer(options);
-        }
-        // TODO: message queuing will be used until formal process lifecycles are implemented
-        while (this.messagingQueue.size() > 0) {
-            const cachedMessages = this.messagingQueue.dequeue();
-            // they are put in as a tuple, [realMsg, ipcMessage]
-            this.processContext.emit(cachedMessages[0], cachedMessages[1]);
-        }
-    }
-    broadcast(eventName, payload = {}) {
-        this.logger.trace('broadcasting a network message', { eventName, payload });
-        if (!payload.message) {
-            payload.message = eventName;
-        }
-        this.io.emit('networkMessage', payload);
-    }
-    _forwardMessage(messageSent) {
-        const messageType = this._determinePathForMessage(messageSent);
-        if (messageType === 'network') {
-            // worker and node_master communicate through broadcast to slicer/cluster_master
-            if (this.self === 'node_master') {
-                this.io.emit(messageSent.message, messageSent);
-            }
-            else if (this.self === 'cluster_master') {
-                this.io.sockets.in(messageSent.address).emit('networkMessage', messageSent);
-            }
-            else {
-                this.io.sockets.in(messageSent.address).emit(messageSent.message, messageSent);
-            }
-        }
-        else if (this.self === 'node_master') {
-            this._sendToProcesses(messageSent);
-        }
-        else if (this.processContext) {
-            if (this.processContext.connected) {
-                this.processContext.send(messageSent);
-            }
-            else {
-                this.logger.warn('cannot send to process because it is not connected', messageSent);
-            }
-        }
-    }
-    send(messageSent) {
-        if (!messageSent.__source) {
-            messageSent.__source = this.self;
-        }
-        const needsReply = messageSent.response;
-        if (!needsReply) {
-            this._forwardMessage(messageSent);
-            return Promise.resolve(true);
-        }
-        return new Promise((resolve, reject) => {
-            let timer;
-            const msgID = nanoid(8);
-            const actionTimeout = messageSent.timeout || this.configTimeout;
-            const messageTimeout = toNumber(actionTimeout) + this.networkLatencyBuffer;
-            messageSent.__msgId = msgID;
-            this.events.once(msgID, (nodeMasterData) => {
-                clearTimeout(timer);
-                if (nodeMasterData.error) {
-                    reject(new Error(`${nodeMasterData.error} occurred on node: ${nodeMasterData.__source}`));
-                }
-                else {
-                    resolve(nodeMasterData);
-                }
-            });
-            this._forwardMessage(messageSent);
-            timer = setTimeout(() => {
-                // remove listener to prevent memory leaks
-                this.events.removeAllListeners(msgID);
-                reject(new Error(`timeout error while communicating with ${messageSent.to}, msg: ${messageSent.message}, data: ${JSON.stringify(messageSent)}`));
-            }, messageTimeout);
-        });
-    }
-    _makeConfigurations() {
-        let host;
-        let port;
-        const options = {
-            node_master: { networkClient: true, ipcClient: false },
-            cluster_master: { networkClient: false, ipcClient: true },
-            execution_controller: { networkClient: false, ipcClient: true },
-            worker: { networkClient: true, ipcClient: true },
-            assets_service: { networkClient: true, ipcClient: true }
-        };
-        // @ts-expect-error
-        const env = this.context.__testingModule ? this.context.__testingModule.env : process.env;
-        const processConfig = {};
-        // @ts-expect-error
-        const testProcess = this.context.__testingModule;
-        const { assignment } = env;
-        if (!isProcessAssignment(assignment)) {
-            throw new Error(`assignment must be on of: ${Object.values(ProcessAssignment).toString()}. Received ${assignment}`);
-        }
-        processConfig.clients = options[assignment];
-        if (processConfig.clients.ipcClient) {
-            // all children of node_master
-            this.processContext = testProcess ? testProcess : process;
-        }
-        else {
-            // node_master
-            this.processContext = this.context.cluster;
-        }
-        if (processConfig.clients.networkClient) {
-            if (assignment === 'node_master' || assignment === 'assets_service') {
-                host = this.context.sysconfig.teraslice.master_hostname;
-                ({ port } = this.context.sysconfig.teraslice);
-            }
-            processConfig.hostURL = this._makeHostName(host, port);
-        }
-        processConfig.assignment = assignment;
-        return processConfig;
-    }
-    _makeHostName(host, port, nameSpace) {
-        let name;
-        let hostname = host;
-        if (!hostname.match(/http/)) {
-            hostname = `http://${hostname}`;
-        }
-        const lastChar = hostname[hostname.length - 1];
-        if (lastChar !== ':') {
-            name = `${hostname}:${port}`;
-        }
-        else {
-            name = hostname + port;
-        }
-        if (nameSpace) {
-            return `${name}/${nameSpace}`;
-        }
-        return name;
-    }
-    _getMessages(type) {
-        if (type === 'cluster_master')
-            return clusterMasterMessages;
-        if (type === 'node_master')
-            return nodeMasterMessages;
-        if (type === 'assets_service')
-            return assetServiceMessages;
-        return new Error(`could not find message model for type: ${type}`);
-    }
-    getHostUrl() {
-        if (this.hostURL) {
-            return this.hostURL;
-        }
-        return null;
-    }
-    getClientCounts() {
-        if (this.io) {
-            return this.io.eio.clientsCount;
-        }
-        // there are no connected clients because the network is not instantiated
-        return 0;
-    }
-    listRooms() {
-        const connected = get(this.io, 'sockets.connected', {});
-        if (isEmpty(connected))
-            return [];
-        return Object.values(connected).flatMap((meta) => Object.keys(meta.rooms));
-    }
-    registerChildOnlineHook(fn) {
-        this.childHookFn = fn;
-    }
-    _isExecutionStateQuery(msg) {
-        const stateQuery = {
-            'cluster:execution:pause': 'cluster:execution:pause',
-            'cluster:execution:resume': 'cluster:execution:resume',
-            'cluster:slicer:analytics': 'cluster:slicer:analytics',
-        };
-        return (isKey(stateQuery, msg) && stateQuery[msg] !== undefined);
-    }
-    _emitIpcMessage(fn) {
-        return (ipcMessage) => {
-            const msg = ipcMessage.message;
-            const realMsg = get(this.selfMessages, `ipc.${msg}`, null);
-            if (realMsg) {
-                fn(realMsg, ipcMessage);
-            }
-            else {
-                this.logger.error(`process: ${self} has received a message: ${msg}, which is not registered in the messaging module`);
-            }
-        };
-    }
-    _handleIpcMessages() {
-        if (this.self === 'execution_controller') {
-            const checkAndEmit = (realMsg, ipcMessage) => {
-                if (this.messsagingOnline || !this._isExecutionStateQuery(realMsg)) {
-                    this.processContext.emit(realMsg, ipcMessage);
-                }
-                else {
-                    this.messagingQueue.enqueue([realMsg, ipcMessage]);
-                }
-            };
-            return this._emitIpcMessage(checkAndEmit);
-        }
-        // for everything else just emit the message
-        const emitFn = (realMsg, ipcMessage) => this.processContext.emit(realMsg, ipcMessage);
-        return this._emitIpcMessage(emitFn);
-    }
-    _handleWorkerMessage(ipcMessage) {
-        if (ipcMessage.to === 'cluster_master') {
-            this.logger.trace('network passing process message', ipcMessage.message, ipcMessage);
-            this.send(ipcMessage);
-        }
-        else if (ipcMessage.to === 'node_master') {
-            this.logger.trace('process message', ipcMessage.message, ipcMessage);
-            this.processContext.emit(ipcMessage.message, ipcMessage);
-        }
-        else {
-            this._sendToProcesses(ipcMessage);
-        }
-    }
-    async shutdown() {
-        if (this.io && isFunction(this.io.close)) {
-            this.io.close();
-            await pDelay(100);
-        }
-    }
-}
-//# sourceMappingURL=messaging.js.map

package/dist/src/lib/cluster/services/cluster/backends/state-utils.js

@@ -1,26 +0,0 @@
-import { cloneDeep } from '@terascope/core-utils';
-export function iterateState(clusterState, cb) {
-    // I clone here, because the code below accidentally modifies clusterState.
-    // Not sure if this is the best choice.
-    const state = cloneDeep(clusterState);
-    return Object.values(state)
-        .filter((node) => node.state === 'connected')
-        .flatMap((node) => {
-        const workers = node.active.filter(cb);
-        return workers.map((worker) => {
-            worker.node_id = node.node_id;
-            worker.hostname = node.hostname;
-            return worker;
-        });
-    });
-}
-export function findAllSlicers(clusterState) {
-    return iterateState(clusterState, (worker) => worker.assignment === 'execution_controller');
-}
-export function findWorkersByExecutionID(clusterState, exId) {
-    return iterateState(clusterState, (worker) => worker.assignment === 'worker' && worker.ex_id === exId);
-}
-export function findSlicersByExecutionID(clusterState, exIdDict) {
-    return iterateState(clusterState, (worker) => worker.assignment === 'execution_controller' && exIdDict[worker.ex_id]);
-}
-//# sourceMappingURL=state-utils.js.map

package/dist/src/lib/cluster/services/cluster/index.js

@@ -1,13 +0,0 @@
-import { NativeClustering } from './backends/native/index.js';
-import { KubernetesClusterBackendV2 } from './backends/kubernetesV2/index.js';
-export function makeClustering(context, { clusterMasterServer }) {
-    const clusterType = context.sysconfig.teraslice.cluster_manager_type;
-    if (clusterType === 'native') {
-        return new NativeClustering(context, clusterMasterServer);
-    }
-    if (clusterType === 'kubernetesV2') {
-        return new KubernetesClusterBackendV2(context, clusterMasterServer);
-    }
-    throw new Error(`unknown cluster service ${clusterType}, cannot find cluster module`);
-}
-//# sourceMappingURL=index.js.map