teraslice 3.0.0-dev.11 → 3.0.0-dev.13

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js

@@ -55,8 +55,6 @@ export class K8sResource {
         const shutdownTimeoutMs = get(this.terasliceConfig, 'shutdown_timeout', 60000);
         const shutdownTimeoutSeconds = Math.round(shutdownTimeoutMs / 1000);
         const config = {
-            // assetsDirectory: get(this.terasliceConfig, 'assets_directory', ''),
-            // assetsVolume: get(this.terasliceConfig, 'assets_volume', ''),
             clusterName,
             clusterNameLabel,
             configMapName,

package/dist/src/lib/config/index.js

@@ -1,7 +1,6 @@
 import path from 'node:path';
 import { get } from '@terascope/core-utils';
 import { fileURLToPath } from 'node:url';
-import { formats } from '@terascope/job-components';
 import { configSchema } from './schemas/system.js';
 const filePath = fileURLToPath(new URL('.', import.meta.url));
 export function clusterName(configFile) {
@@ -12,11 +11,10 @@ export function getTerasliceConfig(sysconfig) {
         name: 'teraslice',
         default_config_file: path.join(filePath, 'default-sysconfig.js'),
         config_schema: configSchema,
-        schema_formats: formats,
         cluster_name: clusterName,
         shutdownMessaging: false,
         start_workers: false,
     }, sysconfig);
 }
-export { formats, configSchema, };
+export { configSchema, };
 //# sourceMappingURL=index.js.map

package/dist/src/lib/config/schemas/system.js

@@ -32,18 +32,18 @@ export const schema = {
     },
     assets_volume: {
         doc: 'name of shared asset volume (k8s)',
-        default: '',
-        format: 'optional_String'
+        default: undefined,
+        format: 'optional_string'
     },
     autoload_directory: {
         doc: 'directory to look for assets to auto deploy when teraslice boots up',
         default: path.join(process.cwd(), './autoload'),
-        format: 'optional_String'
+        format: 'optional_string'
     },
     hostname: {
         doc: 'IP or hostname for server',
         default: ip.address(),
-        format: 'required_String'
+        format: 'required_string'
     },
     workers: {
         doc: 'Number of workers per server',
@@ -58,7 +58,7 @@ export const schema = {
     master_hostname: {
         doc: 'hostname where the cluster_master resides, used to notify all node_masters where to connect',
         default: 'localhost',
-        format: 'required_String'
+        format: 'required_string'
     },
     port: {
         doc: 'port for the cluster_master to listen on',
@@ -68,7 +68,7 @@ export const schema = {
     name: {
         doc: 'Name for the cluster itself, its used for naming log files/indices',
         default: 'teracluster',
-        format: 'elasticsearch_Name'
+        format: 'elasticsearch_name'
     },
     state: {
         doc: 'Elasticsearch cluster where job state, analytics and logs are stored',
@@ -273,12 +273,12 @@ export const schema = {
     kubernetes_image: {
         doc: 'Specify a custom image name for kubernetes, this only applies to kubernetes systems',
         default: 'terascope/teraslice',
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_namespace: {
         doc: 'Specify a custom kubernetes namespace, this only applies to kubernetes systems',
         default: 'default',
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_overrides_enabled: {
         doc: '',
@@ -288,17 +288,17 @@ export const schema = {
     kubernetes_priority_class_name: {
         doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
         default: undefined,
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_config_map_name: {
         doc: 'Specify the name of the Kubernetes ConfigMap used to configure worker pods',
         default: 'teraslice-worker',
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_image_pull_secret: {
         doc: 'Name of Kubernetes secret used to pull docker images from private repository',
-        default: '',
-        format: 'optional_String'
+        default: undefined,
+        format: 'optional_string'
     },
     kubernetes_worker_antiaffinity: {
         doc: 'Enable Teraslice woker pod AntiAffinity in Kubernetes',
@@ -318,7 +318,7 @@ export const schema = {
     asset_storage_bucket: {
         doc: 'Name of S3 bucket used to store assets. Can only be used if "asset_storage_connection_type" is "s3".',
         default: undefined,
-        format: String
+        format: 'optional_string'
     },
 };
 export function configSchema() {

package/dist/src/lib/workers/helpers/worker-shutdown.js

@@ -28,7 +28,7 @@ export function shutdownHandler(context, shutdownFn) {
         || process.env.assignment
         || 'unknown-assignment';
     // this is native clustering only
-    const isProcessRestart = process.env.process_restart;
+    const isProcessRestart = process.env.__process_restart;
     const api = {
         exiting: false,
         exit
@@ -132,13 +132,6 @@ export function shutdownHandler(context, shutdownFn) {
     }
     process.stdout.on('error', handleStdError);
     process.stderr.on('error', handleStdError);
-    // event is fired from terafoundation when an error occurs during instantiation of a client
-    // **DEPRECATED:** This handler should be removed on teraslice v1
-    events.once('client:initialization:error', (err) => {
-        logError(logger, err, `${assignment} received a client initialization error, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('client:initialization:error', err);
-    });
     events.once('worker:shutdown:complete', (err) => {
         setStatusCode(0);
         if (err) {

package/dist/test/config/schemas/system_schema-spec.js

@@ -1,22 +1,11 @@
-import convict from 'convict';
-// @ts-expect-error no types
-import convict_format_with_validator from 'convict-format-with-validator';
-// @ts-expect-error no types
-import convict_format_with_moment from 'convict-format-with-moment';
-import { formats } from '@terascope/job-components';
+import { SchemaValidator } from '@terascope/core-utils';
 import { config_schema } from '../../../src/lib/config/schemas/system.js';
-// load any convict schema
-convict.addFormats(convict_format_with_validator);
-convict.addFormats(convict_format_with_moment);
-formats.forEach((format) => convict.addFormat(format));
 describe('system_schema', () => {
     const schema = config_schema().schema.teraslice;
     function checkValidation(config) {
         try {
-            const validator = convict(schema);
-            validator.load(config);
-            validator.validate();
-            return validator.getProperties();
+            const validator = new SchemaValidator(schema, 'teraslice');
+            return validator.validate(config);
         }
         catch (err) {
             return err.message;
@@ -31,7 +20,7 @@ describe('system_schema', () => {
         expect(schema.state.default).toEqual({ connection: 'default' });
     });
     it('assets_directory is optional but requires a string', () => {
-        expect(checkValidation({ assets_directory: 234 })).toEqual('assets_directory: Invalid parameter assets_directory, it must either be a string or an array of strings: value was 234');
+        expect(checkValidation({ assets_directory: 234 })).toContain('Invalid parameter assets_directory, it must either be a string or an array of strings');
     });
 });
 //# sourceMappingURL=system_schema-spec.js.map

package/dist/test/services/assets-spec.js

@@ -4,7 +4,7 @@ import got from 'got';
 import { createClient } from '@terascope/opensearch-client';
 import { createS3Client } from '@terascope/file-asset-apis';
 import { AssetsService } from '../../src/lib/cluster/services/assets';
-import { TEST_INDEX_PREFIX } from '../test.config';
+import { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } from '../test.config';
 import { findPort } from '../../src/lib/utils/port_utils.js';
 describe('Assets Service', () => {
     const contextOptions = {
@@ -48,7 +48,7 @@ describe('Assets Service', () => {
             },
             'elasticsearch-next': {
                 default: {
-                    node: [process.env.SEARCH_TEST_HOST]
+                    node: [SEARCH_TEST_HOST]
                 },
             },
         }

package/dist/test/storage/assets_storage-spec.js

@@ -3,7 +3,7 @@ import { TestContext } from '@terascope/job-components';
 import { createClient } from '@terascope/opensearch-client';
 import { createS3Client } from '@terascope/file-asset-apis';
 import { AssetsStorage } from '../../src/lib/storage';
-import { TEST_INDEX_PREFIX } from '../test.config';
+import { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } from '../test.config';
 describe('AssetsStorage using S3 backend', () => {
     let storage;
     const options = {
@@ -32,7 +32,7 @@ describe('AssetsStorage using S3 backend', () => {
         connectors: {
             'elasticsearch-next': {
                 default: {
-                    node: [process.env.SEARCH_TEST_HOST]
+                    node: [SEARCH_TEST_HOST]
                 }
             },
             s3: {

package/dist/test/test.config.js

@@ -1,8 +1,23 @@
+import { SchemaValidator } from '@terascope/core-utils';
 import { newId } from '../src/lib/utils/id_utils.js';
-const { TEST_INDEX_PREFIX = 'teratest_', ELASTICSEARCH_HOST = 'http://localhost:9200', TERASLICE_CLUSTER_NAME = newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2), ELASTICSEARCH_VERSION = '6.8', SEARCH_TEST_HOST = ELASTICSEARCH_HOST } = process.env;
-process.env.TERASLICE_CLUSTER_NAME = TERASLICE_CLUSTER_NAME;
-process.env.ELASTICSEARCH_HOST = ELASTICSEARCH_HOST;
-process.env.ELASTICSEARCH_VERSION = ELASTICSEARCH_VERSION;
-process.env.SEARCH_TEST_HOST = SEARCH_TEST_HOST;
-export { TEST_INDEX_PREFIX, SEARCH_TEST_HOST, TERASLICE_CLUSTER_NAME, ELASTICSEARCH_VERSION, };
+const terasliceTestEnvSchema = {
+    SEARCH_TEST_HOST: {
+        default: undefined,
+        format: String,
+    },
+    TERASLICE_CLUSTER_NAME: {
+        default: null,
+        format: 'optional_string',
+    },
+    TEST_INDEX_PREFIX: {
+        default: undefined,
+        format: String,
+    },
+};
+const validator = new SchemaValidator(terasliceTestEnvSchema, 'terasliceTestEnvSchema', undefined, 'allow');
+const envConfig = validator.validate(process.env);
+const { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } = envConfig;
+const TERASLICE_CLUSTER_NAME = envConfig.TERASLICE_CLUSTER_NAME
+    || newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2);
+export { TEST_INDEX_PREFIX, SEARCH_TEST_HOST, TERASLICE_CLUSTER_NAME, };
 //# sourceMappingURL=test.config.js.map

package/dist/test/workers/helpers/configs.js

@@ -4,7 +4,7 @@ import { withoutNil } from '@terascope/core-utils';
 // @ts-expect-error
 import Chance from 'chance';
 import { newId } from '../../../src/lib/utils/id_utils.js';
-const { SEARCH_TEST_HOST } = process.env;
+import { SEARCH_TEST_HOST } from '../../test.config.js';
 const dirname = path.dirname(fileURLToPath(import.meta.url));
 const opsPath = path.join(dirname, '..', 'fixtures', 'ops');
 const chance = new Chance();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "teraslice",
   "displayName": "Teraslice",
-  "version": "3.0.0-dev.11",
+  "version": "3.0.0-dev.13",
   "description": "Distributed computing platform for processing JSON data",
   "homepage": "https://github.com/terascope/teraslice#readme",
   "bugs": {
@@ -39,11 +39,11 @@
   },
   "dependencies": {
     "@kubernetes/client-node": "~1.3.0",
-    "@terascope/core-utils": "~2.0.0-dev.8",
-    "@terascope/elasticsearch-api": "~5.0.0-dev.9",
-    "@terascope/job-components": "~2.0.0-dev.9",
-    "@terascope/teraslice-messaging": "~2.0.0-dev.9",
-    "@terascope/types": "~2.0.0-dev.7",
+    "@terascope/core-utils": "~2.0.0-dev.10",
+    "@terascope/elasticsearch-api": "~5.0.0-dev.11",
+    "@terascope/job-components": "~2.0.0-dev.11",
+    "@terascope/teraslice-messaging": "~2.0.0-dev.11",
+    "@terascope/types": "~2.0.0-dev.8",
     "async-mutex": "~0.5.0",
     "barbe": "~3.0.17",
     "body-parser": "~2.2.0",
@@ -61,20 +61,23 @@
     "semver": "~7.7.2",
     "socket.io": "~4.8.1",
     "socket.io-client": "~4.8.1",
-    "terafoundation": "~2.0.0-dev.10",
+    "terafoundation": "~2.0.0-dev.12",
     "uuid": "~13.0.0"
   },
   "devDependencies": {
-    "@terascope/opensearch-client": "~2.0.0-dev.8",
+    "@terascope/opensearch-client": "~2.0.0-dev.10",
     "@types/archiver": "~6.0.3",
+    "@types/body-parser": "^1.19.6",
+    "@types/decompress": "^4.2.7",
     "@types/express": "~5.0.3",
     "@types/gc-stats": "~1.4.3",
+    "@types/ip": "^1.1.3",
+    "@types/js-yaml": "^4.0.9",
+    "@types/ms": "~0.7.34",
+    "@types/semver": "^7.7.1",
     "archiver": "~7.0.1",
     "bufferstreams": "~3.0.0",
     "chance": "~1.1.13",
-    "convict": "~6.2.4",
-    "convict-format-with-moment": "~6.2.0",
-    "convict-format-with-validator": "~6.2.0",
     "jest-fixtures": "~0.6.0",
     "js-yaml": "~4.1.0",
     "nock": "~13.5.6"