teraslice 3.0.0-dev.8 → 3.0.0-rc.0

package/dist/src/lib/cluster/services/api.js

@@ -558,7 +558,7 @@ export class ApiService {
                             name,
                             node_version: process.version,
                             platform: this.context.platform,
-                            teraslice_version: `v${getPackageJSON().version}`
+                            teraslice_version: `v${terasliceVersion}`
                         }, 1);
                         this.context.apis.foundation.promMetrics.set('slices_processed', {}, stats.controllers.processed);
                         this.context.apis.foundation.promMetrics.set('slices_failed', {}, stats.controllers.failed);

package/dist/src/lib/cluster/services/assets.js

@@ -46,7 +46,6 @@ export class AssetsService {
                         const code = created ? 201 : 200;
                         const assetResponse = {
                             asset_id: assetId,
-                            _id: assetId
                         };
                         res.status(code).json(assetResponse);
                     }
@@ -74,7 +73,6 @@ export class AssetsService {
                 else {
                     const assetResponse = {
                         asset_id: assetId,
-                        _id: assetId
                     };
                     requestHandler(async () => {
                         await this.assetsStorage.remove(assetId);

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js

@@ -55,8 +55,6 @@ export class K8sResource {
         const shutdownTimeoutMs = get(this.terasliceConfig, 'shutdown_timeout', 60000);
         const shutdownTimeoutSeconds = Math.round(shutdownTimeoutMs / 1000);
         const config = {
-            // assetsDirectory: get(this.terasliceConfig, 'assets_directory', ''),
-            // assetsVolume: get(this.terasliceConfig, 'assets_volume', ''),
             clusterName,
             clusterNameLabel,
             configMapName,

package/dist/src/lib/cluster/services/jobs.js

@@ -382,17 +382,17 @@ export class JobsService {
     }
     adjustNamesByAsset(jobConfig, dict) {
         jobConfig.operations = jobConfig.operations.map((op) => {
-            if (op.api_name?.includes('@')) {
-                const { name, assetIdentifier, tag } = parseName(op.api_name);
+            if (op._api_name?.includes('@')) {
+                const { name, assetIdentifier, tag } = parseName(op._api_name);
                 const hashId = dict.get(assetIdentifier);
                 if (!hashId) {
-                    throw new Error(`Invalid operation api_name for _op: ${name}, could not find the hashID for asset identifier ${assetIdentifier}`);
+                    throw new Error(`Invalid operation _api_name for _op: ${name}, could not find the hashID for asset identifier ${assetIdentifier}`);
                 }
                 let hashedName = `${name}@${hashId}`;
                 if (tag) {
                     hashedName = `${hashedName}:${tag}`;
                 }
-                op.api_name = hashedName;
+                op._api_name = hashedName;
             }
             if (op._op.includes('@')) {
                 const { name, assetIdentifier } = parseName(op._op);

package/dist/src/lib/config/index.js

@@ -1,7 +1,6 @@
 import path from 'node:path';
 import { get } from '@terascope/core-utils';
 import { fileURLToPath } from 'node:url';
-import { formats } from '@terascope/job-components';
 import { configSchema } from './schemas/system.js';
 const filePath = fileURLToPath(new URL('.', import.meta.url));
 export function clusterName(configFile) {
@@ -12,11 +11,10 @@ export function getTerasliceConfig(sysconfig) {
         name: 'teraslice',
         default_config_file: path.join(filePath, 'default-sysconfig.js'),
         config_schema: configSchema,
-        schema_formats: formats,
         cluster_name: clusterName,
         shutdownMessaging: false,
         start_workers: false,
     }, sysconfig);
 }
-export { formats, configSchema, };
+export { configSchema, };
 //# sourceMappingURL=index.js.map

package/dist/src/lib/config/schemas/system.js

@@ -32,18 +32,18 @@ export const schema = {
     },
     assets_volume: {
         doc: 'name of shared asset volume (k8s)',
-        default: '',
-        format: 'optional_String'
+        default: undefined,
+        format: 'optional_string'
     },
     autoload_directory: {
         doc: 'directory to look for assets to auto deploy when teraslice boots up',
         default: path.join(process.cwd(), './autoload'),
-        format: 'optional_String'
+        format: 'optional_string'
     },
     hostname: {
         doc: 'IP or hostname for server',
         default: ip.address(),
-        format: 'required_String'
+        format: 'required_string'
     },
     workers: {
         doc: 'Number of workers per server',
@@ -58,7 +58,7 @@ export const schema = {
     master_hostname: {
         doc: 'hostname where the cluster_master resides, used to notify all node_masters where to connect',
         default: 'localhost',
-        format: 'required_String'
+        format: 'required_string'
     },
     port: {
         doc: 'port for the cluster_master to listen on',
@@ -68,7 +68,7 @@ export const schema = {
     name: {
         doc: 'Name for the cluster itself, its used for naming log files/indices',
         default: 'teracluster',
-        format: 'elasticsearch_Name'
+        format: 'elasticsearch_name'
     },
     state: {
         doc: 'Elasticsearch cluster where job state, analytics and logs are stored',
@@ -273,12 +273,12 @@ export const schema = {
     kubernetes_image: {
         doc: 'Specify a custom image name for kubernetes, this only applies to kubernetes systems',
         default: 'terascope/teraslice',
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_namespace: {
         doc: 'Specify a custom kubernetes namespace, this only applies to kubernetes systems',
         default: 'default',
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_overrides_enabled: {
         doc: '',
@@ -288,17 +288,17 @@ export const schema = {
     kubernetes_priority_class_name: {
         doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
         default: undefined,
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_config_map_name: {
         doc: 'Specify the name of the Kubernetes ConfigMap used to configure worker pods',
         default: 'teraslice-worker',
-        format: 'optional_String'
+        format: 'optional_string'
     },
     kubernetes_image_pull_secret: {
         doc: 'Name of Kubernetes secret used to pull docker images from private repository',
-        default: '',
-        format: 'optional_String'
+        default: undefined,
+        format: 'optional_string'
     },
     kubernetes_worker_antiaffinity: {
         doc: 'Enable Teraslice woker pod AntiAffinity in Kubernetes',
@@ -318,7 +318,7 @@ export const schema = {
     asset_storage_bucket: {
         doc: 'Name of S3 bucket used to store assets. Can only be used if "asset_storage_connection_type" is "s3".',
         default: undefined,
-        format: String
+        format: 'optional_string'
     },
 };
 export function configSchema() {

package/dist/src/lib/storage/backends/mappings/ex.js

@@ -50,6 +50,9 @@ export default {
             },
             _deleted_on: {
                 type: 'date'
+            },
+            teraslice_version: {
+                type: 'keyword'
             }
         }
     }

package/dist/src/lib/storage/execution.js

@@ -3,6 +3,7 @@ import { RecoveryCleanupType } from '@terascope/job-components';
 import { v4 as uuid } from 'uuid';
 import { makeLogger } from '../workers/helpers/terafoundation.js';
 import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
+import { getPackageJSON } from '../utils/file_utils.js';
 const INIT_STATUS = ['pending', 'scheduling', 'initializing'];
 const RUNNING_STATUS = ['recovering', 'running', 'failing', 'paused', 'stopping'];
 const TERMINAL_STATUS = ['completed', 'stopped', 'rejected', 'failed', 'terminated'];
@@ -78,7 +79,8 @@ export class ExecutionStorage {
             _updated: date,
             _has_errors: false,
             _slicer_stats: {},
-            _failureReason: ''
+            _failureReason: '',
+            teraslice_version: `v${getPackageJSON().version}`
         });
         // @ts-expect-error
         delete doc.slicer_port;

package/dist/src/lib/utils/asset_utils.js

@@ -54,7 +54,7 @@ function _isCompatibleAsset(name, range, skipRestrictions = false) {
     return function checkIfCompatibleAsset(record) {
         if (record.name !== name)
             return false;
-        if (!semver.satisfies(record.version, range)) {
+        if (!semver.satisfies(record.version, range, { includePrerelease: true })) {
             return false;
         }
         if (skipRestrictions)

package/dist/src/lib/workers/helpers/worker-shutdown.js

@@ -28,7 +28,7 @@ export function shutdownHandler(context, shutdownFn) {
         || process.env.assignment
         || 'unknown-assignment';
     // this is native clustering only
-    const isProcessRestart = process.env.process_restart;
+    const isProcessRestart = process.env.__process_restart;
     const api = {
         exiting: false,
         exit
@@ -132,13 +132,6 @@ export function shutdownHandler(context, shutdownFn) {
     }
     process.stdout.on('error', handleStdError);
     process.stderr.on('error', handleStdError);
-    // event is fired from terafoundation when an error occurs during instantiation of a client
-    // **DEPRECATED:** This handler should be removed on teraslice v1
-    events.once('client:initialization:error', (err) => {
-        logError(logger, err, `${assignment} received a client initialization error, ${exitingIn()}`);
-        setStatusCode(1);
-        exit('client:initialization:error', err);
-    });
     events.once('worker:shutdown:complete', (err) => {
         setStatusCode(0);
         if (err) {

package/dist/test/config/schemas/system_schema-spec.js

@@ -1,22 +1,11 @@
-import convict from 'convict';
-// @ts-expect-error no types
-import convict_format_with_validator from 'convict-format-with-validator';
-// @ts-expect-error no types
-import convict_format_with_moment from 'convict-format-with-moment';
-import { formats } from '@terascope/job-components';
+import { SchemaValidator } from '@terascope/core-utils';
 import { config_schema } from '../../../src/lib/config/schemas/system.js';
-// load any convict schema
-convict.addFormats(convict_format_with_validator);
-convict.addFormats(convict_format_with_moment);
-formats.forEach((format) => convict.addFormat(format));
 describe('system_schema', () => {
     const schema = config_schema().schema.teraslice;
     function checkValidation(config) {
         try {
-            const validator = convict(schema);
-            validator.load(config);
-            validator.validate();
-            return validator.getProperties();
+            const validator = new SchemaValidator(schema, 'teraslice');
+            return validator.validate(config);
         }
         catch (err) {
             return err.message;
@@ -31,7 +20,7 @@ describe('system_schema', () => {
         expect(schema.state.default).toEqual({ connection: 'default' });
     });
     it('assets_directory is optional but requires a string', () => {
-        expect(checkValidation({ assets_directory: 234 })).toEqual('assets_directory: Invalid parameter assets_directory, it must either be a string or an array of strings: value was 234');
+        expect(checkValidation({ assets_directory: 234 })).toContain('Invalid parameter assets_directory, it must either be a string or an array of strings');
     });
 });
 //# sourceMappingURL=system_schema-spec.js.map

package/dist/test/node_master-spec.js

@@ -11,6 +11,14 @@ describe('Node master', () => {
         lifecycle: 'persistent',
         workers: 1,
         assets: ['elasticsearch', 'standard'],
+        apis: [
+            {
+                _name: 'elasticsearch_sender_api',
+                index: 'example-logs',
+                type: 'events',
+                size: 5000
+            }
+        ],
         operations: [
             {
                 _op: 'data_generator',
@@ -18,9 +26,7 @@ describe('Node master', () => {
             },
             {
                 _op: 'elasticsearch_bulk',
-                index: 'example-logs',
-                type: 'events',
-                size: 5000
+                _api_name: 'elasticsearch_sender_api'
             }
         ]
     };

package/dist/test/services/assets-spec.js

@@ -4,7 +4,7 @@ import got from 'got';
 import { createClient } from '@terascope/opensearch-client';
 import { createS3Client } from '@terascope/file-asset-apis';
 import { AssetsService } from '../../src/lib/cluster/services/assets';
-import { TEST_INDEX_PREFIX } from '../test.config';
+import { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } from '../test.config';
 import { findPort } from '../../src/lib/utils/port_utils.js';
 describe('Assets Service', () => {
     const contextOptions = {
@@ -48,7 +48,7 @@ describe('Assets Service', () => {
             },
             'elasticsearch-next': {
                 default: {
-                    node: [process.env.SEARCH_TEST_HOST]
+                    node: [SEARCH_TEST_HOST]
                 },
             },
         }

package/dist/test/storage/assets_storage-spec.js

@@ -3,7 +3,7 @@ import { TestContext } from '@terascope/job-components';
 import { createClient } from '@terascope/opensearch-client';
 import { createS3Client } from '@terascope/file-asset-apis';
 import { AssetsStorage } from '../../src/lib/storage';
-import { TEST_INDEX_PREFIX } from '../test.config';
+import { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } from '../test.config';
 describe('AssetsStorage using S3 backend', () => {
     let storage;
     const options = {
@@ -32,7 +32,7 @@ describe('AssetsStorage using S3 backend', () => {
         connectors: {
             'elasticsearch-next': {
                 default: {
-                    node: [process.env.SEARCH_TEST_HOST]
+                    node: [SEARCH_TEST_HOST]
                 }
             },
             s3: {

package/dist/test/test.config.js

@@ -1,8 +1,23 @@
+import { SchemaValidator } from '@terascope/core-utils';
 import { newId } from '../src/lib/utils/id_utils.js';
-const { TEST_INDEX_PREFIX = 'teratest_', ELASTICSEARCH_HOST = 'http://localhost:9200', TERASLICE_CLUSTER_NAME = newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2), ELASTICSEARCH_VERSION = '6.8', SEARCH_TEST_HOST = ELASTICSEARCH_HOST } = process.env;
-process.env.TERASLICE_CLUSTER_NAME = TERASLICE_CLUSTER_NAME;
-process.env.ELASTICSEARCH_HOST = ELASTICSEARCH_HOST;
-process.env.ELASTICSEARCH_VERSION = ELASTICSEARCH_VERSION;
-process.env.SEARCH_TEST_HOST = SEARCH_TEST_HOST;
-export { TEST_INDEX_PREFIX, SEARCH_TEST_HOST, TERASLICE_CLUSTER_NAME, ELASTICSEARCH_VERSION, };
+const terasliceTestEnvSchema = {
+    SEARCH_TEST_HOST: {
+        default: undefined,
+        format: String,
+    },
+    TERASLICE_CLUSTER_NAME: {
+        default: null,
+        format: 'optional_string',
+    },
+    TEST_INDEX_PREFIX: {
+        default: undefined,
+        format: String,
+    },
+};
+const validator = new SchemaValidator(terasliceTestEnvSchema, 'terasliceTestEnvSchema', undefined, 'allow');
+const envConfig = validator.validate(process.env);
+const { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } = envConfig;
+const TERASLICE_CLUSTER_NAME = envConfig.TERASLICE_CLUSTER_NAME
+    || newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2);
+export { TEST_INDEX_PREFIX, SEARCH_TEST_HOST, TERASLICE_CLUSTER_NAME, };
 //# sourceMappingURL=test.config.js.map

package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js

@@ -8,7 +8,6 @@ import { ExecutionController } from '../../../src/lib/workers/execution-controll
 import { findPort } from '../../../src/lib/utils/port_utils.js';
 import { newId } from '../../../src/lib/utils/id_utils.js';
 const ExecutionControllerClient = ExController.Client;
-process.env.BLUEBIRD_LONG_STACK_TRACES = '1';
 describe('ExecutionController Special Tests', () => {
     // [ message, config ]
     const testCases = [

package/dist/test/workers/execution-controller/execution-test-cases-spec.js

@@ -4,6 +4,7 @@ import { TestContext } from '../helpers/index.js';
 import { getTestCases } from '../helpers/execution-controller-helper.js';
 import { findPort } from '../../../src/lib/utils/port_utils.js';
 import { newId } from '../../../src/lib/utils/id_utils.js';
+import { getPackageJSON } from '../../../src/lib/utils/file_utils.js';
 import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
 const ExecutionControllerClient = ExController.Client;
 describe('ExecutionController Test Cases', () => {
@@ -333,6 +334,10 @@ describe('ExecutionController Test Cases', () => {
             const metadata = updateMetadata ? { slice_calls: count + 1 } : {};
             expect(executionRecord).toHaveProperty('metadata', metadata);
         });
+        it('should have the correct teraslice version', () => {
+            const terasliceVersion = `v${getPackageJSON().version}`;
+            expect(executionRecord.teraslice_version).toBe(terasliceVersion);
+        });
     });
 });
 //# sourceMappingURL=execution-test-cases-spec.js.map

package/dist/test/workers/helpers/configs.js

@@ -4,7 +4,7 @@ import { withoutNil } from '@terascope/core-utils';
 // @ts-expect-error
 import Chance from 'chance';
 import { newId } from '../../../src/lib/utils/id_utils.js';
-const { SEARCH_TEST_HOST } = process.env;
+import { SEARCH_TEST_HOST } from '../../test.config.js';
 const dirname = path.dirname(fileURLToPath(import.meta.url));
 const opsPath = path.join(dirname, '..', 'fixtures', 'ops');
 const chance = new Chance();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "teraslice",
   "displayName": "Teraslice",
-  "version": "3.0.0-dev.8",
+  "version": "3.0.0-rc.0",
   "description": "Distributed computing platform for processing JSON data",
   "homepage": "https://github.com/terascope/teraslice#readme",
   "bugs": {
@@ -39,11 +39,11 @@
   },
   "dependencies": {
     "@kubernetes/client-node": "~1.3.0",
-    "@terascope/core-utils": "~2.0.0-dev.7",
-    "@terascope/elasticsearch-api": "~5.0.0-dev.7",
-    "@terascope/job-components": "~2.0.0-dev.7",
-    "@terascope/teraslice-messaging": "~2.0.0-dev.8",
-    "@terascope/types": "~2.0.0-dev.6",
+    "@terascope/core-utils": "~2.0.0-rc.0",
+    "@terascope/elasticsearch-api": "~5.0.0-rc.0",
+    "@terascope/job-components": "~2.0.0-rc.0",
+    "@terascope/teraslice-messaging": "~2.0.0-rc.0",
+    "@terascope/types": "~2.0.0-rc.0",
     "async-mutex": "~0.5.0",
     "barbe": "~3.0.17",
     "body-parser": "~2.2.0",
@@ -61,20 +61,23 @@
     "semver": "~7.7.2",
     "socket.io": "~4.8.1",
     "socket.io-client": "~4.8.1",
-    "terafoundation": "~2.0.0-dev.7",
+    "terafoundation": "~2.0.0-rc.0",
     "uuid": "~13.0.0"
   },
   "devDependencies": {
-    "@terascope/opensearch-client": "~2.0.0-dev.7",
+    "@terascope/opensearch-client": "~2.0.0-rc.0",
     "@types/archiver": "~6.0.3",
+    "@types/body-parser": "^1.19.6",
+    "@types/decompress": "^4.2.7",
     "@types/express": "~5.0.3",
     "@types/gc-stats": "~1.4.3",
+    "@types/ip": "^1.1.3",
+    "@types/js-yaml": "^4.0.9",
+    "@types/ms": "~0.7.34",
+    "@types/semver": "^7.7.1",
     "archiver": "~7.0.1",
     "bufferstreams": "~3.0.0",
     "chance": "~1.1.13",
-    "convict": "~6.2.4",
-    "convict-format-with-moment": "~6.2.0",
-    "convict-format-with-validator": "~6.2.0",
     "jest-fixtures": "~0.6.0",
     "js-yaml": "~4.1.0",
     "nock": "~13.5.6"