teraslice 0.87.1 → 0.89.0

package/cluster-service.js

@@ -1,10 +1,10 @@
 'use strict';
 
 const { get, logError } = require('@terascope/utils');
-const { shutdownHandler } = require('./lib/workers/helpers/worker-shutdown');
-const makeTerafoundationContext = require('./lib/workers/context/terafoundation-context');
-const makeClusterMaster = require('./lib/cluster/cluster_master');
-const makeAssetService = require('./lib/cluster/services/assets');
+const { shutdownHandler } = require('./dist/src/lib/workers/helpers/worker-shutdown');
+const { makeTerafoundationContext } = require('./dist/src/lib/workers/context/terafoundation-context');
+const { ClusterMaster } = require('./dist/src/lib/cluster/cluster_master');
+const { AssetsService } = require('./dist/src/lib/cluster/services');
 
 class Service {
     constructor(context) {
@@ -19,10 +19,10 @@ class Service {
 
         if (assignment === 'cluster_master') {
             // require this here so node doesn't have load extra code into memory
-            this.instance = makeClusterMaster(this.context);
+            this.instance = new ClusterMaster(this.context);
         } else if (assignment === 'assets_service') {
             // require this here so node doesn't have load extra code into memory
-            this.instance = makeAssetService(this.context);
+            this.instance = new AssetsService(this.context);
         }
 
         await this.instance.initialize();
@@ -42,16 +42,22 @@ class Service {
     }
 }
 
-const context = makeTerafoundationContext();
-const cmd = new Service(context);
-
-cmd.shutdownHandler = shutdownHandler(context, () => {
-    if (!cmd.instance) return Promise.resolve();
-    return cmd.instance.shutdown();
-});
+async function main() {
+    const context = makeTerafoundationContext();
+    const cmd = new Service(context);
+
+    cmd.shutdownHandler = shutdownHandler(context, () => {
+        if (!cmd.instance) return Promise.resolve();
+        return cmd.instance.shutdown();
+    });
+
+    try {
+        await cmd.initialize();
+        await cmd.run();
+        await cmd.shutdown();
+    } catch (err) {
+        cmd.shutdown(err);
+    }
+}
 
-Promise.resolve()
-    .then(() => cmd.initialize())
-    .then(() => cmd.run())
-    .then(() => cmd.shutdown())
-    .catch((err) => cmd.shutdown(err));
+main();

package/dist/src/index.js

@@ -0,0 +1,42 @@
+"use strict";
+/**
+ * Export of internal components and functions of teraslice.
+ *
+ * WARNING:
+ *      Since these are internal components, breaking changes may occur if using them.
+ *      For best results teraslice with an exact semver match, i.e "0.38.0".
+*/
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stores = exports.config = exports.makeTerafoundationContext = exports.initializeTestExecution = void 0;
+const config = __importStar(require("./lib/config"));
+exports.config = config;
+const stores = __importStar(require("./lib/storage"));
+exports.stores = stores;
+const terafoundation_context_1 = require("./lib/workers/context/terafoundation-context");
+Object.defineProperty(exports, "makeTerafoundationContext", { enumerable: true, get: function () { return terafoundation_context_1.makeTerafoundationContext; } });
+const job_1 = require("./lib/workers/helpers/job");
+Object.defineProperty(exports, "initializeTestExecution", { enumerable: true, get: function () { return job_1.initializeTestExecution; } });
+//# sourceMappingURL=index.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.87.1",
+    "version": "0.89.0",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -13,14 +13,8 @@
     },
     "license": "Apache-2.0",
     "author": "Terascope, LLC <info@terascope.io>",
-    "main": "index.js",
-    "bin": "service.js",
-    "directories": {
-        "lib": "lib",
-        "bin": "service.js",
-        "test": "test",
-        "main": "index.js"
-    },
+    "main": "dist/src/index.js",
+    "typings": "dist/src/index.d.ts",
     "files": [
         "*.js",
         "lib/**/*",
@@ -41,10 +35,11 @@
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^3.11.0",
-        "@terascope/job-components": "^0.64.0",
-        "@terascope/teraslice-messaging": "^0.34.1",
-        "@terascope/utils": "^0.51.0",
+        "@terascope/elasticsearch-api": "^3.12.0",
+        "@terascope/job-components": "^0.65.0",
+        "@terascope/teraslice-messaging": "^0.35.0",
+        "@terascope/types": "^0.12.0",
+        "@terascope/utils": "^0.52.0",
         "async-mutex": "^0.4.0",
         "barbe": "^3.0.16",
         "body-parser": "^1.20.2",
@@ -66,11 +61,12 @@
         "semver": "^7.3.8",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.52.0",
+        "terafoundation": "^0.53.0",
         "uuid": "^9.0.0"
     },
     "devDependencies": {
         "@terascope/teraslice-op-test-harness": "^1.24.1",
+        "@types/express": "^4.17.20",
         "archiver": "^5.3.1",
         "bufferstreams": "^3.0.0",
         "chance": "^1.1.11",

package/service.js

@@ -3,8 +3,10 @@
 'use strict';
 
 const util = require('util');
-
-const { safeEncode, safeDecode } = require('./lib/utils/encoding_utils');
+const { ClusterContext } = require('terafoundation');
+const { safeEncode, safeDecode } = require('./dist/src/lib/utils/encoding_utils');
+const { nodeMaster } = require('./dist/src/lib/cluster/node_master');
+const { getTerasliceConfig } = require('./dist/src/lib/config');
 
 const assignment = process.env.assignment || process.env.NODE_TYPE;
 
@@ -28,10 +30,6 @@ if (['execution_controller', 'worker'].includes(assignment)) {
     process.env.assignment = 'node_master';
     process.env.NODE_TYPE = 'node_master';
 
-    const { ClusterContext } = require('terafoundation');
-    const nodeMaster = require('./lib/cluster/node_master');
-    const { getTerasliceConfig } = require('./lib/config');
-
     const terasliceConfig = getTerasliceConfig({
         master: nodeMaster,
         cluster_master: deprecatedUseOf('cluster_master'),

package/worker-service.js

@@ -3,12 +3,12 @@
 const {
     logError, get, isEmpty, isPlainObject
 } = require('@terascope/utils');
-const { shutdownHandler } = require('./lib/workers/helpers/worker-shutdown');
-const { safeDecode } = require('./lib/utils/encoding_utils');
-const makeExecutionContext = require('./lib/workers/context/execution-context');
-const makeTerafoundationContext = require('./lib/workers/context/terafoundation-context');
-const ExecutionController = require('./lib/workers/execution-controller');
-const Worker = require('./lib/workers/worker');
+const { shutdownHandler } = require('./dist/src/lib/workers/helpers/worker-shutdown');
+const { safeDecode } = require('./dist/src/lib/utils/encoding_utils');
+const { makeExecutionContext } = require('./dist/src/lib/workers/context/execution-context');
+const { makeTerafoundationContext } = require('./dist/src/lib/workers/context/terafoundation-context');
+const { ExecutionController } = require('./dist/src/lib/workers/execution-controller');
+const { Worker } = require('./dist/src/lib/workers/worker');
 
 class Service {
     constructor(context) {

package/index.js

@@ -1,21 +0,0 @@
-'use strict';
-
-/**
- * Export of internal components and functions of teraslice.
- *
- * WARNING:
- *      Since these are internal components, breaking changes may occur if using them.
- *      For best results teraslice with an exact semver match, i.e "0.38.0".
-*/
-
-const config = require('./lib/config');
-const stores = require('./lib/storage');
-const makeTerafoundationContext = require('./lib/workers/context/terafoundation-context');
-const { initializeTestExecution } = require('./lib/workers/helpers/job');
-
-module.exports = {
-    initializeTestExecution,
-    makeTerafoundationContext,
-    config,
-    stores,
-};

package/lib/cluster/cluster_master.js

@@ -1,164 +0,0 @@
-'use strict';
-
-const express = require('express');
-const got = require('got');
-const {
-    pDelay, logError, get, parseError
-} = require('@terascope/utils');
-const { ClusterMaster } = require('@terascope/teraslice-messaging');
-const { makeLogger } = require('../workers/helpers/terafoundation');
-const makeExecutionService = require('./services/execution');
-const makeApiService = require('./services/api');
-const makeJobsService = require('./services/jobs');
-const makeClusterService = require('./services/cluster');
-const makeJobStore = require('../storage/jobs');
-const makeExStore = require('../storage/execution');
-const makeStateStore = require('../storage/state');
-
-module.exports = function _clusterMaster(context) {
-    const logger = makeLogger(context, 'cluster_master');
-    const clusterConfig = context.sysconfig.teraslice;
-    const assetsPort = process.env.assets_port;
-    const assetsUrl = `http://127.0.0.1:${assetsPort}`;
-    let running = false;
-
-    // Initialize the HTTP service for handling incoming requests.
-    const app = express();
-
-    const clusterMasterServer = new ClusterMaster.Server({
-        port: clusterConfig.port,
-        nodeDisconnectTimeout: clusterConfig.node_disconnect_timeout,
-        // setting request timeout to 5 minutes
-        serverTimeout: clusterConfig.api_response_timeout,
-        // we do this to override express final response handler
-        requestListener(req, res) {
-            app(req, res, (err) => {
-                if (err) logger.warn(err, 'unexpected server error');
-                res.setHeader('Content-Type', 'application/json');
-                res.statusCode = 500;
-                res.end(JSON.stringify({ error: 'api is not available' }));
-            });
-        },
-        networkLatencyBuffer: clusterConfig.network_latency_buffer,
-        actionTimeout: clusterConfig.action_timeout,
-        logger,
-    });
-
-    async function isAssetServiceUp() {
-        try {
-            const response = await got.get('status', {
-                prefixUrl: assetsUrl,
-                responseType: 'json',
-                throwHttpErrors: true,
-                timeout: 900,
-                retry: 0,
-            });
-            return get(response, 'body.available', false);
-        } catch (err) {
-            logger.debug(`asset service not up yet, error: ${parseError(err)}`);
-            return false;
-        }
-    }
-
-    function waitForAssetsService(timeoutAt) {
-        if (Date.now() > timeoutAt) {
-            return Promise.reject(new Error('Timeout waiting for asset service to come online'));
-        }
-        return isAssetServiceUp().then((isUp) => {
-            if (isUp) return Promise.resolve();
-            return pDelay(1000).then(() => waitForAssetsService(timeoutAt));
-        });
-    }
-
-    const serviceOptions = { assetsUrl, app, clusterMasterServer };
-    const services = Object.freeze({
-        execution: makeExecutionService(context, serviceOptions),
-        jobs: makeJobsService(context, serviceOptions),
-        cluster: makeClusterService(context, serviceOptions),
-        api: makeApiService(context, serviceOptions),
-    });
-
-    context.services = services;
-
-    return {
-        async initialize() {
-            try {
-                await clusterMasterServer.start();
-                logger.info(`cluster master listening on port ${clusterConfig.port}`);
-
-                const [exStore, stateStore, jobStore] = await Promise.all([
-                    makeExStore(context),
-                    makeStateStore(context),
-                    makeJobStore(context)
-                ]);
-
-                context.stores = {
-                    execution: exStore,
-                    state: stateStore,
-                    jobs: jobStore,
-                };
-
-                // order matters
-                await services.cluster.initialize();
-                await services.execution.initialize();
-                await services.jobs.initialize();
-
-                logger.debug('services has been initialized');
-
-                // give the assets service a bit to come up
-                const fiveMinutes = 5 * 60 * 1000;
-                await waitForAssetsService(Date.now() + fiveMinutes);
-
-                // this needs to be last
-                await services.api.initialize();
-
-                logger.info('cluster master is ready!');
-                running = true;
-            } catch (err) {
-                logError(logger, err, 'error during service initialization');
-                running = false;
-                throw err;
-            }
-        },
-        run() {
-            return new Promise((resolve) => {
-                if (!running) {
-                    resolve();
-                    return;
-                }
-                const runningInterval = setInterval(() => {
-                    if (!running) {
-                        clearInterval(runningInterval);
-                        resolve();
-                    }
-                }, 1000);
-            });
-        },
-        async shutdown() {
-            running = false;
-
-            logger.info('cluster_master is shutting down');
-            clusterMasterServer.isShuttingDown = true;
-
-            await Promise.all(Object.entries(context.services)
-                .map(async ([name, service]) => {
-                    try {
-                        await service.shutdown();
-                    } catch (err) {
-                        logError(logger, err, `Failure to shutdown service ${name}`);
-                    }
-                }));
-
-            await Promise.all(Object.entries(context.stores)
-                .map(async ([name, store]) => {
-                    try {
-                        await store.shutdown();
-                    } catch (err) {
-                        logError(logger, err, `Failure to shutdown store ${name}`);
-                    }
-                }));
-
-            await clusterMasterServer.shutdown();
-        },
-    };
-};