teraslice 3.1.1 → 3.2.1

package/dist/src/lib/storage/backends/elasticsearch_store.js

@@ -2,13 +2,14 @@ import ms from 'ms';
 import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/core-utils';
 import elasticsearchApi from '@terascope/elasticsearch-api';
 import { getClient } from '@terascope/job-components';
+import { DataType } from '@terascope/data-types';
 import { makeLogger } from '../../workers/helpers/terafoundation.js';
 import { timeseriesIndex } from '../../utils/date_utils.js';
-import analyticsSchema from './mappings/analytics.js';
-import assetSchema from './mappings/asset.js';
-import executionSchema from './mappings/ex.js';
-import jobsSchema from './mappings/job.js';
-import stateSchema from './mappings/state.js';
+import { analyticsDataTypeConfig, analyticsTemplate } from './mappings/analytics.js';
+import { assetDataTypeConfig, assetMappingOverrides } from './mappings/asset.js';
+import { executionDataTypeConfig } from './mappings/ex.js';
+import { jobDataTypeConfig } from './mappings/job.js';
+import { stateDataTypeConfig, stateTemplate } from './mappings/state.js';
 function validateId(recordId, recordType) {
     if (!recordId || !isString(recordId)) {
         throw new TSError(`Invalid ${recordType} id given ${getTypeOf(recordId)}`, {
@@ -56,39 +57,23 @@ export class TerasliceElasticsearchStorage {
     options;
     mapping;
     api;
+    indexSettings;
     constructor(backendConfig) {
         const { context, indexName, recordType, idField, storageName, bulkSize = 1000, fullResponse = false, logRecord = true, forceRefresh = true, logger } = backendConfig;
         this.context = context;
         this.storageName = storageName;
         this.logger = logger ?? makeLogger(context, 'elasticsearch_backend', { storageName });
         this.recordType = recordType;
-        if (recordType === 'analytics') {
-            this.mapping = analyticsSchema;
-        }
-        else if (recordType === 'asset') {
-            this.mapping = assetSchema;
-        }
-        else if (recordType === 'ex') {
-            this.mapping = executionSchema;
-        }
-        else if (recordType === 'job') {
-            this.mapping = jobsSchema;
-        }
-        else if (recordType === 'state') {
-            this.mapping = stateSchema;
-        }
-        else {
+        // Validate recordType
+        const validRecordTypes = ['analytics', 'asset', 'ex', 'job', 'state'];
+        if (!validRecordTypes.includes(recordType)) {
             throw new Error(`Could not find mapping for recordType: ${recordType}`);
         }
         const config = this.context.sysconfig.teraslice;
-        const indexSettings = get(config, ['index_settings', this.storageName], {
+        this.indexSettings = get(config, ['index_settings', this.storageName], {
             number_of_shards: 5,
             number_of_replicas: 1,
         });
-        this.mapping.settings = {
-            'index.number_of_shards': indexSettings.number_of_shards,
-            'index.number_of_replicas': indexSettings.number_of_replicas,
-        };
         this.defaultIndexName = indexName;
         this.idField = idField;
         this.options = {
@@ -126,6 +111,7 @@ export class TerasliceElasticsearchStorage {
             try {
                 const client = await getClient(this.context, connectionConfig, 'elasticsearch-next');
                 this.api = elasticsearchApi(client, this.logger, options);
+                this._generateMapping();
                 await this._createIndex(newIndex);
                 await this.api.isAvailable(newIndex);
                 return true;
@@ -407,6 +393,52 @@ export class TerasliceElasticsearchStorage {
         }
         return Promise.resolve(true);
     }
+    _getDataTypeConfig() {
+        switch (this.recordType) {
+            case 'analytics':
+                return {
+                    config: analyticsDataTypeConfig,
+                    template: analyticsTemplate
+                };
+            case 'asset':
+                return {
+                    config: assetDataTypeConfig,
+                    overrides: assetMappingOverrides
+                };
+            case 'ex':
+                return { config: executionDataTypeConfig };
+            case 'job':
+                return { config: jobDataTypeConfig };
+            case 'state':
+                return {
+                    config: stateDataTypeConfig,
+                    template: stateTemplate
+                };
+            default:
+                throw new Error(`Could not find DataType config for recordType: ${this.recordType}`);
+        }
+    }
+    _generateMapping() {
+        const clientMetadata = this.api.getClientMetadata();
+        const { config, overrides, template } = this._getDataTypeConfig();
+        const dataType = new DataType(config);
+        // Generate the mapping using DataType with client metadata for version-specific mappings
+        this.mapping = dataType.toESMapping({
+            ...clientMetadata,
+            overrides: {
+                ...overrides,
+                settings: {
+                    'index.number_of_shards': this.indexSettings.number_of_shards,
+                    'index.number_of_replicas': this.indexSettings.number_of_replicas,
+                }
+            }
+        });
+        // Add template pattern if this is an index that
+        // adds time at the end of the index (analytics/state)
+        if (template) {
+            this.mapping.template = template;
+        }
+    }
     async _createIndex(index = this.defaultIndexName) {
         // @ts-expect-error TODO: check type missing id
         const existQuery = { index };

package/dist/src/lib/storage/backends/mappings/analytics.js

@@ -1,45 +1,20 @@
-export default {
-    template: '__analytics*',
-    mappings: {
-        dynamic: false,
-        properties: {
-            ex_id: {
-                type: 'keyword'
-            },
-            job_id: {
-                type: 'keyword'
-            },
-            worker_id: {
-                type: 'keyword'
-            },
-            slice_id: {
-                type: 'keyword'
-            },
-            slicer_id: {
-                type: 'keyword'
-            },
-            op: {
-                type: 'keyword'
-            },
-            order: {
-                type: 'integer'
-            },
-            count: {
-                type: 'integer'
-            },
-            state: {
-                type: 'keyword'
-            },
-            time: {
-                type: 'integer'
-            },
-            memory: {
-                type: 'long'
-            },
-            '@timestamp': {
-                type: 'date'
-            }
-        }
+import { FieldType } from '@terascope/types';
+export const analyticsDataTypeConfig = {
+    version: 1,
+    fields: {
+        ex_id: { type: FieldType.Keyword },
+        job_id: { type: FieldType.Keyword },
+        worker_id: { type: FieldType.Keyword },
+        slice_id: { type: FieldType.Keyword },
+        slicer_id: { type: FieldType.Keyword },
+        op: { type: FieldType.Keyword },
+        order: { type: FieldType.Integer },
+        count: { type: FieldType.Integer },
+        state: { type: FieldType.Keyword },
+        time: { type: FieldType.Integer },
+        memory: { type: FieldType.Long },
+        '@timestamp': { type: FieldType.Date }
     }
 };
+export const analyticsTemplate = '__analytics*';
 //# sourceMappingURL=analytics.js.map

package/dist/src/lib/storage/backends/mappings/asset.js

@@ -1,34 +1,30 @@
-export default {
+import { FieldType } from '@terascope/types';
+export const assetDataTypeConfig = {
+    version: 1,
+    fields: {
+        name: { type: FieldType.Keyword },
+        version: { type: FieldType.Keyword },
+        id: { type: FieldType.Keyword },
+        description: { type: FieldType.Keyword },
+        arch: { type: FieldType.Keyword },
+        platform: { type: FieldType.Keyword },
+        node_version: { type: FieldType.Integer },
+        _created: { type: FieldType.Date }
+    }
+};
+/**
+ * Override for the blob field which uses ES binary type.
+ * I could not find the datatype equivalent
+ * so we must add this field manually via mapping overrides.
+ * Once the issue below is resolved we can remove this.
+ * https://github.com/terascope/teraslice/issues/4296
+ */
+export const assetMappingOverrides = {
     mappings: {
-        dynamic: false,
         properties: {
             blob: {
                 type: 'binary',
                 doc_values: false
-            },
-            name: {
-                type: 'keyword'
-            },
-            version: {
-                type: 'keyword'
-            },
-            id: {
-                type: 'keyword'
-            },
-            description: {
-                type: 'keyword'
-            },
-            arch: {
-                type: 'keyword'
-            },
-            platform: {
-                type: 'keyword'
-            },
-            node_version: {
-                type: 'integer'
-            },
-            _created: {
-                type: 'date'
             }
         }
     }

package/dist/src/lib/storage/backends/mappings/ex.js

@@ -1,60 +1,53 @@
-export default {
-    mappings: {
-        dynamic: false,
-        properties: {
-            active: {
-                type: 'boolean'
-            },
-            job_id: {
-                type: 'keyword'
-            },
-            ex_id: {
-                type: 'keyword'
-            },
-            _context: {
-                type: 'keyword'
-            },
-            _status: {
-                type: 'keyword'
-            },
-            _has_errors: {
-                type: 'keyword'
-            },
-            slicer_hostname: {
-                type: 'keyword'
-            },
-            slicer_port: {
-                type: 'keyword'
-            },
-            recovered_execution: {
-                type: 'keyword'
-            },
-            recovered_slice_type: {
-                type: 'keyword'
-            },
-            metadata: {
-                type: 'object',
-                enabled: false
-            },
-            _slicer_stats: {
-                type: 'object'
-            },
-            _created: {
-                type: 'date'
-            },
-            _updated: {
-                type: 'date'
-            },
-            _deleted: {
-                type: 'boolean'
-            },
-            _deleted_on: {
-                type: 'date'
-            },
-            teraslice_version: {
-                type: 'keyword'
-            }
-        }
+import { FieldType } from '@terascope/types';
+export const executionDataTypeConfig = {
+    version: 1,
+    fields: {
+        // ExecutionConfig fields
+        job_id: { type: FieldType.Keyword },
+        ex_id: { type: FieldType.Keyword },
+        _context: { type: FieldType.Keyword },
+        _created: { type: FieldType.Date },
+        _updated: { type: FieldType.Date },
+        _deleted: { type: FieldType.Boolean },
+        _deleted_on: { type: FieldType.Date },
+        _status: { type: FieldType.Keyword },
+        _has_errors: { type: FieldType.Boolean },
+        _slicer_stats: { type: FieldType.Object },
+        _failureReason: { type: FieldType.Text },
+        metadata: { type: FieldType.Object, indexed: false },
+        recovered_execution: { type: FieldType.Keyword },
+        recovered_slice_type: { type: FieldType.Keyword },
+        slicer_port: { type: FieldType.Integer },
+        slicer_hostname: { type: FieldType.Keyword },
+        teraslice_version: { type: FieldType.Keyword },
+        // ValidatedJobConfig fields
+        active: { type: FieldType.Boolean },
+        analytics: { type: FieldType.Boolean },
+        assets: { type: FieldType.Keyword },
+        autorecover: { type: FieldType.Boolean },
+        lifecycle: { type: FieldType.Keyword },
+        max_retries: { type: FieldType.Integer },
+        name: { type: FieldType.Text },
+        probation_window: { type: FieldType.Integer },
+        performance_metrics: { type: FieldType.Boolean },
+        log_level: { type: FieldType.Keyword },
+        slicers: { type: FieldType.Integer },
+        workers: { type: FieldType.Integer },
+        stateful: { type: FieldType.Boolean },
+        // k8s field
+        cpu: { type: FieldType.Float },
+        cpu_execution_controller: { type: FieldType.Float },
+        ephemeral_storage: { type: FieldType.Boolean },
+        memory: { type: FieldType.Integer },
+        memory_execution_controller: { type: FieldType.Integer },
+        resources_requests_cpu: { type: FieldType.Float },
+        resources_requests_memory: { type: FieldType.Integer },
+        resources_limits_cpu: { type: FieldType.Float },
+        resources_limits_memory: { type: FieldType.Integer },
+        kubernetes_image: { type: FieldType.Keyword },
+        prom_metrics_enabled: { type: FieldType.Boolean },
+        prom_metrics_port: { type: FieldType.Integer },
+        prom_metrics_add_default: { type: FieldType.Boolean }
     }
 };
 //# sourceMappingURL=ex.js.map

package/dist/src/lib/storage/backends/mappings/job.js

@@ -1,33 +1,42 @@
-export default {
-    settings: {
-        'index.number_of_shards': 5,
-        'index.number_of_replicas': 1
-    },
-    mappings: {
-        dynamic: false,
-        properties: {
-            active: {
-                type: 'boolean'
-            },
-            job_id: {
-                type: 'keyword'
-            },
-            _context: {
-                type: 'keyword'
-            },
-            _created: {
-                type: 'date'
-            },
-            _updated: {
-                type: 'date'
-            },
-            _deleted: {
-                type: 'boolean'
-            },
-            _deleted_on: {
-                type: 'date'
-            }
-        }
+import { FieldType } from '@terascope/types';
+export const jobDataTypeConfig = {
+    version: 1,
+    fields: {
+        // JobConfig fields
+        job_id: { type: FieldType.Keyword },
+        _context: { type: FieldType.Keyword },
+        _created: { type: FieldType.Date },
+        _updated: { type: FieldType.Date },
+        _deleted: { type: FieldType.Boolean },
+        _deleted_on: { type: FieldType.Date },
+        // ValidatedJobConfig fields
+        active: { type: FieldType.Boolean },
+        analytics: { type: FieldType.Boolean },
+        assets: { type: FieldType.Keyword },
+        autorecover: { type: FieldType.Boolean },
+        lifecycle: { type: FieldType.Keyword },
+        max_retries: { type: FieldType.Integer },
+        name: { type: FieldType.Text },
+        probation_window: { type: FieldType.Integer },
+        performance_metrics: { type: FieldType.Boolean },
+        log_level: { type: FieldType.Keyword },
+        slicers: { type: FieldType.Integer },
+        workers: { type: FieldType.Integer },
+        stateful: { type: FieldType.Boolean },
+        // K8s fields
+        cpu: { type: FieldType.Float },
+        cpu_execution_controller: { type: FieldType.Float },
+        ephemeral_storage: { type: FieldType.Boolean },
+        memory: { type: FieldType.Integer },
+        memory_execution_controller: { type: FieldType.Integer },
+        resources_requests_cpu: { type: FieldType.Float },
+        resources_requests_memory: { type: FieldType.Integer },
+        resources_limits_cpu: { type: FieldType.Float },
+        resources_limits_memory: { type: FieldType.Integer },
+        kubernetes_image: { type: FieldType.Keyword },
+        prom_metrics_enabled: { type: FieldType.Boolean },
+        prom_metrics_port: { type: FieldType.Integer },
+        prom_metrics_add_default: { type: FieldType.Boolean }
     }
 };
 //# sourceMappingURL=job.js.map

package/dist/src/lib/storage/backends/mappings/state.js

@@ -1,33 +1,16 @@
-export default {
-    template: '__state*',
-    mappings: {
-        dynamic: false,
-        properties: {
-            ex_id: {
-                type: 'keyword'
-            },
-            slice_id: {
-                type: 'keyword'
-            },
-            slicer_id: {
-                type: 'keyword'
-            },
-            slicer_order: {
-                type: 'integer'
-            },
-            state: {
-                type: 'keyword'
-            },
-            _created: {
-                type: 'date'
-            },
-            _updated: {
-                type: 'date'
-            },
-            error: {
-                type: 'keyword'
-            }
-        }
+import { FieldType } from '@terascope/types';
+export const stateDataTypeConfig = {
+    version: 1,
+    fields: {
+        ex_id: { type: FieldType.Keyword },
+        slice_id: { type: FieldType.Keyword },
+        slicer_id: { type: FieldType.Keyword },
+        slicer_order: { type: FieldType.Integer },
+        state: { type: FieldType.Keyword },
+        _created: { type: FieldType.Date },
+        _updated: { type: FieldType.Date },
+        error: { type: FieldType.Keyword }
     }
 };
+export const stateTemplate = '__state*';
 //# sourceMappingURL=state.js.map

package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js

@@ -282,7 +282,7 @@ describe('k8s', () => {
         it('can patch a deployment by name', async () => {
             nock(_url, { encodedQueryParams: true })
                 .patch('/apis/apps/v1/namespaces/default/deployments/test1')
-                .reply(204, {});
+                .reply(200, {});
             const response = await k8s.patch({ name: 'testName' }, 'test1');
             expect(response).toEqual({});
         });

package/dist/test/storage/s3_store-spec.js

@@ -41,7 +41,7 @@ describe('S3 backend test', () => {
                 connection: 'default',
                 bucket: 'Invalid-Bucket-Name@'
             });
-            await expect(s3Backend.initialize()).rejects.toThrow('Bucket name does not follow S3 naming rules: The specified bucket is not valid.');
+            await expect(s3Backend.initialize()).rejects.toThrow('Bucket name does not follow S3 naming rules: InvalidBucketName');
         });
     });
     describe('->verifyClient', () => {

package/dist/test/workers/helpers/configs.js

@@ -80,8 +80,7 @@ const newSysConfig = (options) => {
                 'elasticsearch-next': {
                     default: {
                         node: [SEARCH_TEST_HOST],
-                        requestTimeout: timeout,
-                        deadTimeout: timeout
+                        requestTimeout: timeout
                     }
                 }
             }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "teraslice",
   "displayName": "Teraslice",
-  "version": "3.1.1",
+  "version": "3.2.1",
   "description": "Distributed computing platform for processing JSON data",
   "homepage": "https://github.com/terascope/teraslice#readme",
   "bugs": {
@@ -39,12 +39,12 @@
   },
   "dependencies": {
     "@kubernetes/client-node": "~1.4.0",
-    "@terascope/core-utils": "~2.0.1",
-    "@terascope/elasticsearch-api": "~5.0.1",
-    "@terascope/file-asset-apis": "~2.0.0",
-    "@terascope/job-components": "~2.0.1",
-    "@terascope/teraslice-messaging": "~2.0.1",
-    "@terascope/types": "~2.0.0",
+    "@terascope/core-utils": "~2.1.0",
+    "@terascope/elasticsearch-api": "~5.1.0",
+    "@terascope/file-asset-apis": "~2.0.1",
+    "@terascope/job-components": "~2.1.0",
+    "@terascope/teraslice-messaging": "~2.1.0",
+    "@terascope/types": "~2.1.0",
     "async-mutex": "~0.5.0",
     "barbe": "~3.0.17",
     "body-parser": "~2.2.2",
@@ -62,12 +62,12 @@
     "semver": "~7.7.3",
     "socket.io": "~4.8.3",
     "socket.io-client": "~4.8.3",
-    "terafoundation": "~2.0.2",
-    "terafoundation_kafka_connector": "~2.0.0",
+    "terafoundation": "~2.1.0",
+    "terafoundation_kafka_connector": "~2.0.1",
     "uuid": "~13.0.0"
   },
   "devDependencies": {
-    "@terascope/opensearch-client": "~2.0.1",
+    "@terascope/opensearch-client": "~2.1.0",
     "@types/archiver": "~7.0.0",
     "@types/body-parser": "~1.19.6",
     "@types/decompress": "~4.2.7",
@@ -82,7 +82,7 @@
     "chance": "~1.1.13",
     "jest-fixtures": "~0.6.0",
     "js-yaml": "~4.1.1",
-    "nock": "~13.5.6"
+    "nock": "~14.0.10"
   },
   "engines": {
     "node": ">=22.0.0",