teraslice 2.11.0 → 2.12.0

package/dist/src/interfaces.js

@@ -0,0 +1,12 @@
+export var ProcessAssignment;
+(function (ProcessAssignment) {
+    ProcessAssignment["node_master"] = "node_master";
+    ProcessAssignment["cluster_master"] = "cluster_master";
+    ProcessAssignment["assets_service"] = "assets_service";
+    ProcessAssignment["execution_controller"] = "execution_controller";
+    ProcessAssignment["worker"] = "worker";
+})(ProcessAssignment || (ProcessAssignment = {}));
+export function isProcessAssignment(value) {
+    return value in ProcessAssignment;
+}
+//# sourceMappingURL=interfaces.js.map

package/dist/src/lib/cluster/cluster_master.js

@@ -0,0 +1,246 @@
+import express from 'express';
+import got from 'got';
+import { pDelay, logError, get, parseError } from '@terascope/utils';
+import { ClusterMaster as ClusterMasterMessaging } from '@terascope/teraslice-messaging';
+import { isPromAvailable } from '@terascope/job-components';
+import { makeLogger } from '../workers/helpers/terafoundation.js';
+import { ExecutionService, ApiService, JobsService, makeClustering } from './services/index.js';
+import { JobsStorage, ExecutionStorage, StateStorage } from '../storage/index.js';
+export class ClusterMaster {
+    context;
+    logger;
+    running = false;
+    assetsUrl;
+    messagingServer;
+    constructor(context) {
+        this.context = context;
+        this.logger = makeLogger(context, 'cluster_master');
+        const assetsPort = process.env.assets_port;
+        this.assetsUrl = `http://127.0.0.1:${assetsPort}`;
+    }
+    async isAssetServiceUp() {
+        try {
+            const response = await got.get('status', {
+                prefixUrl: this.assetsUrl,
+                responseType: 'json',
+                throwHttpErrors: true,
+                timeout: {
+                    request: 900
+                },
+                retry: {
+                    limit: 0
+                }
+            });
+            return get(response, 'body.available', false);
+        }
+        catch (err) {
+            this.logger.debug(`asset service not up yet, error: ${parseError(err)}`);
+            return false;
+        }
+    }
+    async waitForAssetsService(timeoutAt) {
+        if (Date.now() > timeoutAt) {
+            return Promise.reject(new Error('Timeout waiting for asset service to come online'));
+        }
+        const isUp = await this.isAssetServiceUp();
+        if (isUp) {
+            return true;
+        }
+        await pDelay(1000);
+        return this.waitForAssetsService(timeoutAt);
+    }
+    async initialize() {
+        const clusterConfig = this.context.sysconfig.teraslice;
+        const { logger } = this;
+        try {
+            // Initialize the HTTP service for handling incoming requests.
+            const app = express();
+            this.messagingServer = new ClusterMasterMessaging.Server({
+                port: clusterConfig.port,
+                nodeDisconnectTimeout: clusterConfig.node_disconnect_timeout,
+                // setting request timeout to 5 minutes
+                serverTimeout: clusterConfig.api_response_timeout,
+                // we do this to override express final response handler
+                requestListener(req, res) {
+                    // @ts-expect-error
+                    app(req, res, (err) => {
+                        if (err) {
+                            logger.warn(err, 'unexpected server error');
+                        }
+                        res.setHeader('Content-Type', 'application/json');
+                        res.statusCode = 500;
+                        res.end(JSON.stringify({ error: 'api is not available' }));
+                    });
+                },
+                networkLatencyBuffer: clusterConfig.network_latency_buffer,
+                actionTimeout: clusterConfig.action_timeout,
+                logger: this.logger,
+            });
+            const serviceOptions = {
+                assetsUrl: this.assetsUrl,
+                app,
+                clusterMasterServer: this.messagingServer
+            };
+            const executionService = new ExecutionService(this.context, serviceOptions);
+            const jobsService = new JobsService(this.context);
+            const clusterService = makeClustering(this.context, serviceOptions);
+            const apiService = new ApiService(this.context, serviceOptions);
+            const services = Object.freeze({
+                executionService,
+                jobsService,
+                clusterService,
+                apiService,
+            });
+            this.context.services = services;
+            await this.messagingServer.start();
+            this.logger.info(`cluster master listening on port ${clusterConfig.port}`);
+            const executionStorage = new ExecutionStorage(this.context);
+            const stateStorage = new StateStorage(this.context);
+            const jobsStorage = new JobsStorage(this.context);
+            await Promise.all([
+                executionStorage.initialize(),
+                stateStorage.initialize(),
+                jobsStorage.initialize()
+            ]);
+            this.context.stores = {
+                executionStorage,
+                stateStorage,
+                jobsStorage,
+            };
+            // order matters
+            await services.clusterService.initialize();
+            await services.executionService.initialize();
+            await services.jobsService.initialize();
+            this.logger.debug('services has been initialized');
+            // give the assets service a bit to come up
+            const fiveMinutes = 5 * 60 * 1000;
+            await this.waitForAssetsService(Date.now() + fiveMinutes);
+            // this needs to be last
+            await services.apiService.initialize();
+            /// initialize promClient
+            if (this.context.sysconfig.teraslice.cluster_manager_type === 'native') {
+                this.logger.warn('Skipping PromMetricsAPI initialization: incompatible with native clustering.');
+            }
+            else {
+                const { terafoundation } = this.context.sysconfig;
+                await this.context.apis.foundation.promMetrics.init({
+                    terasliceName: this.context.sysconfig.teraslice.name,
+                    tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default,
+                    tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled,
+                    tf_prom_metrics_port: terafoundation.prom_metrics_port,
+                    logger: this.logger,
+                    assignment: 'master',
+                    prefix: 'teraslice_',
+                    prom_metrics_display_url: terafoundation.prom_metrics_display_url
+                });
+                await this.setupPromMetrics();
+            }
+            this.logger.info('cluster master is ready!');
+            this.running = true;
+        }
+        catch (err) {
+            logError(this.logger, err, 'error during service initialization');
+            this.running = false;
+            throw err;
+        }
+    }
+    async run() {
+        return new Promise((resolve) => {
+            if (!this.running) {
+                resolve(true);
+                return;
+            }
+            const runningInterval = setInterval(() => {
+                if (!this.running) {
+                    clearInterval(runningInterval);
+                    resolve(true);
+                }
+            }, 1000);
+        });
+    }
+    async shutdown() {
+        this.running = false;
+        this.logger.info('cluster_master is shutting down');
+        this.messagingServer.isShuttingDown = true;
+        await Promise.all(Object.entries(this.context.services)
+            .map(async ([name, service]) => {
+            try {
+                await service.shutdown();
+            }
+            catch (err) {
+                logError(this.logger, err, `Failure to shutdown service ${name}`);
+            }
+        }));
+        await Promise.all(Object.entries(this.context.stores)
+            .map(async ([name, store]) => {
+            try {
+                await store.shutdown();
+            }
+            catch (err) {
+                logError(this.logger, err, `Failure to shutdown store ${name}`);
+            }
+        }));
+        await this.messagingServer.shutdown();
+        if (isPromAvailable(this.context)) {
+            await this.context.apis.foundation.promMetrics.shutdown();
+        }
+    }
+    /**
+     * Adds all prom metrics specific to the cluster_master.
+     *
+     * If trying to add a new metric for the cluster_master, it belongs here.
+     * @async
+     * @function setupPromMetrics
+     * @return {Promise<void>}
+     * @link https://terascope.github.io/teraslice/docs/development/k8s#prometheus-metrics-api
+     */
+    async setupPromMetrics() {
+        if (isPromAvailable(this.context)) {
+            this.logger.info(`adding ${this.context.assignment} prom metrics...`);
+            /*
+                TODO: After reviewing these metrics, I've conluded that all of these
+                can be handled by th execution controller. We might move these into the execution
+                controller metrics down the line. The master can maybe keep track of how many ex
+                controllers there are? Some sort of overview of everything and leave the specifics
+                to each ex.
+
+            */
+            await Promise.all([
+                this.context.apis.foundation.promMetrics.addGauge('master_info', 'Information about Teraslice cluster master', ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version']),
+                this.context.apis.foundation.promMetrics.addGauge('slices_processed', 'Total slices processed across the cluster', []),
+                this.context.apis.foundation.promMetrics.addGauge('slices_failed', 'Total slices failed across the cluster', []),
+                this.context.apis.foundation.promMetrics.addGauge('slices_queued', 'Total slices queued across the cluster', []),
+                this.context.apis.foundation.promMetrics.addGauge('workers_joined', 'Total workers joined across the cluster', []),
+                this.context.apis.foundation.promMetrics.addGauge('workers_disconnected', 'Total workers disconnected across the cluster', []),
+                this.context.apis.foundation.promMetrics.addGauge('workers_reconnected', 'Total workers reconnected across the cluster', []),
+                this.context.apis.foundation.promMetrics.addGauge('controller_workers_active', 'Number of Teraslice workers actively processing slices.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_workers_available', 'Number of Teraslice workers running and waiting for work.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_workers_joined', 'Total number of Teraslice workers that have joined the execution controller for this job.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_workers_reconnected', 'Total number of Teraslice workers that have reconnected to the execution controller for this job.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_workers_disconnected', 'Total number of Teraslice workers that have disconnected from execution controller for this job.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_info', 'Information about Teraslice execution.', ['ex_id', 'job_id', 'image', 'version']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_slicers_count', 'Number of execution controllers (slicers) running for this execution.', ['ex_id', 'job_id', 'job_name']),
+                // Execution Related Metrics
+                this.context.apis.foundation.promMetrics.addGauge('execution_cpu_limit', 'CPU core limit for a Teraslice worker container.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_cpu_request', 'Requested number of CPU cores for a Teraslice worker container.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_memory_limit', 'Memory limit for Teraslice a worker container.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_memory_request', 'Requested amount of memory for a Teraslice worker container.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_status', 'Current status of the Teraslice execution.', ['ex_id', 'job_id', 'job_name', 'status']),
+                /*
+                    TODO: The following gauges should be Counters. This was not done because
+                    teraslice master already provided the count total for most of these metrics.
+                    So setting the gauge is the only real way to gather the metrics in master.
+                    Solution to convert would be setting the count in the ex process.
+                */
+                this.context.apis.foundation.promMetrics.addGauge('controller_slices_processed', 'Number of slices processed.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_slices_failed', 'Number of slices failed.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('controller_slices_queued', 'Number of slices queued for processing.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_created_timestamp_seconds', 'Execution creation time.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_updated_timestamp_seconds', 'Execution update time.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_slicers', 'Number of slicers defined on the execution.', ['ex_id', 'job_id', 'job_name']),
+                this.context.apis.foundation.promMetrics.addGauge('execution_workers', 'Number of workers defined on the execution.  Note that the number of actual workers can differ from this value.', ['ex_id', 'job_id', 'job_name']),
+            ]);
+        }
+    }
+}
+//# sourceMappingURL=cluster_master.js.map

package/dist/src/lib/cluster/node_master.js

@@ -0,0 +1,355 @@
+import ms from 'ms';
+import { Mutex } from 'async-mutex';
+import { getFullErrorStack, debounce, isEmpty, has } from '@terascope/utils';
+import { makeLogger } from '../workers/helpers/terafoundation.js';
+import { Messaging } from './services/cluster/backends/native/messaging.js';
+import { spawnAssetLoader } from '../workers/assets/spawn.js';
+import { safeEncode } from '../utils/encoding_utils.js';
+import { findPort, getPorts } from '../utils/port_utils.js';
+import { getPackageJSON } from '../utils/file_utils.js';
+const nodeVersion = process.version;
+const terasliceVersion = getPackageJSON().version;
+export async function nodeMaster(context) {
+    const logger = makeLogger(context, 'node_master');
+    const configWorkerLimit = context.sysconfig.teraslice.workers;
+    const config = context.sysconfig.teraslice;
+    const events = context.apis.foundation.getSystemEvents();
+    const mutex = new Mutex();
+    const messaging = new Messaging(context, logger);
+    const host = messaging.getHostUrl();
+    const isShuttingDown = false;
+    const ports = getPorts(context);
+    logger.info(`node ${context.sysconfig._nodeName} is attempting to connect to cluster_master: ${host}`);
+    function sendNodeStateNow() {
+        if (isShuttingDown)
+            return;
+        const state = getNodeState();
+        messaging.send({
+            to: 'cluster_master',
+            message: 'node:state',
+            node_id: state.node_id,
+            payload: state
+        });
+    }
+    const sendNodeState = debounce(sendNodeStateNow, 500, { leading: false, trailing: true });
+    let pendingAllocations = 0;
+    function allocateWorkers(count, exConfig, fn) {
+        const startTime = Date.now();
+        pendingAllocations += count;
+        sendNodeStateNow();
+        const locked = mutex.isLocked() ? ' (locked)' : '';
+        logger.info(`allocating ${count} workers...${locked}`);
+        return mutex.runExclusive(async () => {
+            try {
+                await loadAssetsIfNeeded(exConfig.job, exConfig.ex_id);
+            }
+            catch (err) {
+                logger.error(`Failure to allocated assets for execution ${exConfig.ex_id}`);
+                throw err;
+            }
+            finally {
+                pendingAllocations -= count;
+            }
+            try {
+                const workers = await fn();
+                const elapsed = Date.now() - startTime;
+                if (workers.length === count) {
+                    logger.info(`allocated ${workers.length} workers, took ${ms(elapsed)}`);
+                }
+                else {
+                    logger.info(`allocated ${workers.length} out of the requested ${count} workers, took ${ms(elapsed)}`);
+                }
+                return workers.length;
+            }
+            catch (err) {
+                logger.error(`Failure to allocate workers for execution ${exConfig.ex_id}`);
+                throw err;
+            }
+        });
+    }
+    function canAllocateWorkers(requestedWorkers) {
+        const numOfCurrentWorkers = Object.keys(context.cluster.workers).length;
+        // if there is an over allocation, send back rest to be enqueued
+        if (configWorkerLimit < numOfCurrentWorkers + requestedWorkers) {
+            return configWorkerLimit - numOfCurrentWorkers > 0;
+        }
+        return true;
+    }
+    messaging.registerChildOnlineHook(sendNodeState);
+    messaging.register({
+        event: 'network:connect',
+        callback: () => {
+            logger.info(`node has successfully connected to: ${host}`);
+            const state = getNodeState();
+            messaging.send({
+                to: 'cluster_master', message: 'node:online', node_id: state.node_id, payload: state
+            });
+        }
+    });
+    messaging.register({
+        event: 'network:disconnect',
+        callback: () => logger.info(`node has disconnected from: ${host}`)
+    });
+    messaging.register({
+        event: 'network:error',
+        callback: (err) => logger.warn(err, `Attempting to connect to cluster_master: ${host}`)
+    });
+    messaging.register({
+        event: 'cluster:execution_controller:create',
+        // TODO: type this
+        callback: (createSlicerRequest) => {
+            const createSlicerMsg = createSlicerRequest.payload;
+            logger.info(`starting execution_controller for execution ${createSlicerMsg.ex_id}...`);
+            allocateWorkers(1, createSlicerMsg, async () => {
+                const controllerContext = {
+                    assignment: 'execution_controller',
+                    NODE_TYPE: 'execution_controller',
+                    EX: safeEncode(createSlicerMsg.job),
+                    job: createSlicerMsg.job,
+                    node_id: context.sysconfig._nodeName,
+                    ex_id: createSlicerMsg.ex_id,
+                    job_id: createSlicerMsg.job_id,
+                    slicer_port: createSlicerMsg.slicer_port
+                };
+                logger.trace('starting a execution controller', controllerContext);
+                return context.apis.foundation.startWorkers(1, controllerContext);
+            })
+                .then(() => messaging.respond(createSlicerRequest))
+                .catch((error) => {
+                messaging.respond(createSlicerRequest, {
+                    error: getFullErrorStack(error),
+                });
+            });
+        }
+    });
+    messaging.register({
+        event: 'cluster:workers:create',
+        callback: (createWorkerRequest) => {
+            const createWorkerMsg = createWorkerRequest.payload;
+            const requestedWorkers = createWorkerMsg.workers;
+            logger.info(`starting ${requestedWorkers} workers for execution ${createWorkerMsg.ex_id}...`);
+            if (!canAllocateWorkers(requestedWorkers)) {
+                logger.warn(`worker is overallocated, maximum number of workers of ${configWorkerLimit}`);
+                messaging.respond(createWorkerRequest, {
+                    payload: {
+                        createdWorkers: 0,
+                    }
+                });
+                return;
+            }
+            allocateWorkers(requestedWorkers, createWorkerMsg, async () => {
+                let newWorkers = requestedWorkers;
+                const numOfCurrentWorkers = Object.keys(context.cluster.workers).length;
+                // if there is an over allocation, send back rest to be enqueued
+                if (configWorkerLimit < numOfCurrentWorkers + requestedWorkers) {
+                    newWorkers = configWorkerLimit - numOfCurrentWorkers;
+                    logger.warn(`worker allocation request would exceed maximum number of workers of ${configWorkerLimit}`);
+                    logger.warn(`reducing allocation to ${newWorkers} workers.`);
+                }
+                let workers = [];
+                if (newWorkers > 0) {
+                    logger.trace(`starting ${newWorkers} workers`, createWorkerMsg.ex_id);
+                    workers = context.apis.foundation.startWorkers(newWorkers, {
+                        NODE_TYPE: 'worker',
+                        EX: safeEncode(createWorkerMsg.job),
+                        assignment: 'worker',
+                        node_id: context.sysconfig._nodeName,
+                        job: createWorkerMsg.job,
+                        ex_id: createWorkerMsg.ex_id,
+                        job_id: createWorkerMsg.job_id
+                    });
+                }
+                return workers;
+            })
+                .then((createdWorkers) => messaging.respond(createWorkerRequest, {
+                payload: {
+                    createdWorkers,
+                }
+            }))
+                .catch(() => messaging.respond(createWorkerRequest, {
+                payload: {
+                    createdWorkers: 0,
+                }
+            }));
+        }
+    });
+    messaging.register({ event: 'cluster:node:state', callback: () => sendNodeState() });
+    // this fires when entire server will be shutdown
+    events.once('terafoundation:shutdown', () => {
+        logger.debug('received shutdown notice from terafoundation');
+        const filterFn = () => context.cluster.workers;
+        const isActionCompleteFn = () => isEmpty(getNodeState().active);
+        shutdownProcesses({}, filterFn, isActionCompleteFn, true);
+    });
+    messaging.register({
+        event: 'cluster:execution:stop',
+        callback: (networkMsg) => {
+            const exId = networkMsg.ex_id;
+            logger.debug(`received cluster execution stop for execution ${exId}`);
+            const filterFn = () => {
+                return Object.values(context.cluster.workers)
+                    .filter((worker) => {
+                    return worker.ex_id === exId;
+                });
+            };
+            function actionCompleteFn() {
+                const children = getNodeState().active;
+                const workers = children.filter((worker) => worker.ex_id === exId);
+                logger.debug(`waiting for ${workers.length} to stop for ex: ${exId}`);
+                return workers.length === 0;
+            }
+            shutdownProcesses(networkMsg, filterFn, actionCompleteFn);
+        }
+    });
+    messaging.register({
+        event: 'cluster:workers:remove',
+        callback: (networkMsg) => {
+            const numberToRemove = networkMsg.payload.workers;
+            const children = getNodeState().active;
+            const startingWorkerCount = children.filter((worker) => worker.ex_id === networkMsg.ex_id && worker.assignment === 'worker').length;
+            const filterFn = () => children.filter((worker) => worker.ex_id === networkMsg.ex_id && worker.assignment === 'worker').slice(0, numberToRemove);
+            function actionCompleteFn() {
+                const childWorkers = getNodeState().active;
+                const currentWorkersForJob = childWorkers.filter((worker) => worker.ex_id === networkMsg.ex_id && worker.assignment === 'worker').length;
+                return currentWorkersForJob + numberToRemove <= startingWorkerCount;
+            }
+            shutdownProcesses(networkMsg, filterFn, actionCompleteFn);
+        }
+    });
+    // used to find an open port for slicer
+    messaging.register({
+        event: 'cluster:node:get_port',
+        callback: async (msg) => {
+            const port = await findPort(ports);
+            logger.debug(`assigning port ${port} for new job`);
+            messaging.respond(msg, { port });
+        }
+    });
+    messaging.register({
+        event: 'cluster:error:terminal',
+        callback: () => {
+            logger.error('terminal error in cluster_master, flushing logs and shutting down');
+            logger.flush()
+                .then(() => process.exit(0));
+        }
+    });
+    messaging.register({
+        event: 'child:exit',
+        callback: () => sendNodeState()
+    });
+    function getAssetsFromJob(jobStr) {
+        const job = typeof jobStr === 'string' ? JSON.parse(jobStr) : jobStr;
+        return job.assets || [];
+    }
+    async function loadAssetsIfNeeded(job, exId) {
+        const assets = getAssetsFromJob(job);
+        if (!assets.length)
+            return;
+        logger.info(`node ${context.sysconfig._nodeName} is checking assets for job, exId: ${exId}`);
+        await spawnAssetLoader(assets, context);
+    }
+    function shutdownWorkers(signal, filterFn) {
+        const allWorkersForJob = filterFn();
+        allWorkersForJob.forEach((worker) => {
+            const workerID = worker.worker_id || worker.id;
+            if (has(context.cluster.workers, workerID)) {
+                const clusterWorker = context.cluster.workers[workerID];
+                const processId = clusterWorker.process.pid;
+                if (clusterWorker.isDead())
+                    return;
+                // if the worker has already been sent a SIGTERM signal it should send a SIGKILL
+                logger.warn(`sending ${signal} to process ${processId}, assignment: ${worker.assignment}, ex_id: ${worker.ex_id}`);
+                clusterWorker.kill(signal);
+            }
+        });
+    }
+    function shutdownProcesses(message, filterFn, isActionCompleteFn, onlySigKill = false) {
+        const intervalTime = 200;
+        const needsResponse = message.response && message.to;
+        // give a little extra time to finish shutting down
+        let stopTime = config.shutdown_timeout + 3000;
+        if (!onlySigKill) {
+            shutdownWorkers('SIGTERM', filterFn);
+        }
+        const stop = setInterval(() => {
+            if (isActionCompleteFn()) {
+                clearInterval(stop);
+                if (needsResponse)
+                    messaging.respond(message);
+            }
+            if (stopTime <= 0) {
+                clearInterval(stop);
+                shutdownWorkers('SIGKILL', filterFn);
+                if (needsResponse)
+                    messaging.respond(message);
+            }
+            stopTime -= intervalTime;
+        }, intervalTime);
+    }
+    function getNodeState() {
+        const nodeId = context.sysconfig._nodeName;
+        const state = {
+            node_id: nodeId,
+            hostname: context.sysconfig.teraslice.hostname,
+            pid: process.pid,
+            node_version: nodeVersion,
+            teraslice_version: terasliceVersion,
+            total: context.sysconfig.teraslice.workers,
+            state: 'connected'
+        };
+        const clusterWorkers = context.cluster.workers;
+        const active = [];
+        Object.values(clusterWorkers).forEach((worker) => {
+            const child = {
+                worker_id: worker.id,
+                assignment: worker.assignment,
+                pid: worker.process.pid
+            };
+            if (worker.ex_id) {
+                child.ex_id = worker.ex_id;
+            }
+            if (worker.job_id) {
+                child.job_id = worker.job_id;
+            }
+            if (worker.assets) {
+                child.assets = worker.assets.map((asset) => asset.id);
+            }
+            active.push(child);
+        });
+        const total = state.total;
+        state.available = total - active.length - pendingAllocations;
+        state.active = active;
+        return state;
+    }
+    messaging.listen({
+        query: {
+            node_id: context.sysconfig._nodeName
+        }
+    });
+    if (context.sysconfig.teraslice.master) {
+        logger.debug(`node ${context.sysconfig._nodeName} is creating the cluster_master`);
+        const [clusterMaster] = context.apis.foundation.startWorkers(1, {
+            assignment: 'cluster_master',
+            assets_port: ports.assetsPort,
+            node_id: context.sysconfig._nodeName
+        });
+        clusterMaster.on('exit', (code) => {
+            if (code !== 0) {
+                throw Error(`Cluster master has shutdown with exit code ${code}!`);
+            }
+        });
+        logger.debug(`node ${context.sysconfig._nodeName} is creating assets endpoint on port ${ports.assetsPort}`);
+        const [assetService] = context.apis.foundation.startWorkers(1, {
+            assignment: 'assets_service',
+            // key needs to be called port to bypass cluster port sharing
+            port: ports.assetsPort,
+            node_id: context.sysconfig._nodeName
+        });
+        assetService.on('exit', (code) => {
+            if (code !== 0) {
+                throw Error(`Asset Service has shutdown with exit code ${code}!`);
+            }
+        });
+    }
+}
+//# sourceMappingURL=node_master.js.map