teraslice 3.0.0-dev.1 → 3.0.0-dev.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/cluster-service.js +1 -1
  2. package/dist/src/lib/cluster/cluster_master.js +1 -1
  3. package/dist/src/lib/cluster/node_master.js +1 -1
  4. package/dist/src/lib/cluster/services/api.js +2 -2
  5. package/dist/src/lib/cluster/services/assets.js +1 -1
  6. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js +1 -1
  7. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js +1 -1
  8. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js +1 -1
  9. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js +1 -1
  10. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js +1 -1
  11. package/dist/src/lib/cluster/services/cluster/backends/native/index.js +1 -1
  12. package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js +6 -5
  13. package/dist/src/lib/cluster/services/cluster/backends/state-utils.js +1 -1
  14. package/dist/src/lib/cluster/services/execution.js +1 -1
  15. package/dist/src/lib/cluster/services/jobs.js +1 -1
  16. package/dist/src/lib/config/default-sysconfig.js +0 -1
  17. package/dist/src/lib/config/index.js +1 -1
  18. package/dist/src/lib/config/schemas/system.js +2 -2
  19. package/dist/src/lib/storage/analytics.js +1 -1
  20. package/dist/src/lib/storage/assets.js +1 -1
  21. package/dist/src/lib/storage/backends/elasticsearch_store.js +7 -37
  22. package/dist/src/lib/storage/backends/mappings/analytics.js +37 -42
  23. package/dist/src/lib/storage/backends/mappings/asset.js +29 -34
  24. package/dist/src/lib/storage/backends/mappings/ex.js +51 -53
  25. package/dist/src/lib/storage/backends/mappings/job.js +21 -26
  26. package/dist/src/lib/storage/backends/mappings/state.js +24 -29
  27. package/dist/src/lib/storage/backends/s3_store.js +1 -1
  28. package/dist/src/lib/storage/execution.js +4 -2
  29. package/dist/src/lib/storage/jobs.js +1 -1
  30. package/dist/src/lib/storage/state.js +1 -2
  31. package/dist/src/lib/utils/api_utils.js +1 -1
  32. package/dist/src/lib/utils/asset_utils.js +2 -2
  33. package/dist/src/lib/utils/date_utils.js +1 -1
  34. package/dist/src/lib/utils/encoding_utils.js +1 -1
  35. package/dist/src/lib/utils/file_utils.js +1 -1
  36. package/dist/src/lib/utils/id_utils.js +1 -1
  37. package/dist/src/lib/utils/port_utils.js +1 -1
  38. package/dist/src/lib/workers/assets/loader-executable.js +1 -1
  39. package/dist/src/lib/workers/assets/loader.js +1 -1
  40. package/dist/src/lib/workers/assets/spawn.js +1 -1
  41. package/dist/src/lib/workers/execution-controller/execution-analytics.js +1 -1
  42. package/dist/src/lib/workers/execution-controller/index.js +1 -1
  43. package/dist/src/lib/workers/execution-controller/recovery.js +1 -1
  44. package/dist/src/lib/workers/execution-controller/scheduler.js +1 -1
  45. package/dist/src/lib/workers/execution-controller/slice-analytics.js +1 -1
  46. package/dist/src/lib/workers/helpers/job.js +1 -1
  47. package/dist/src/lib/workers/helpers/op-analytics.js +1 -1
  48. package/dist/src/lib/workers/helpers/terafoundation.js +1 -1
  49. package/dist/src/lib/workers/helpers/worker-shutdown.js +1 -1
  50. package/dist/src/lib/workers/metrics/index.js +1 -1
  51. package/dist/src/lib/workers/worker/index.js +1 -1
  52. package/dist/src/lib/workers/worker/slice.js +1 -1
  53. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js +1 -1
  54. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js +1 -1
  55. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js +1 -1
  56. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js +1 -1
  57. package/dist/test/node_master-spec.js +3 -3
  58. package/dist/test/services/messaging-spec.js +1 -1
  59. package/dist/test/workers/execution-controller/execution-controller-spec.js +1 -1
  60. package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js +1 -2
  61. package/dist/test/workers/execution-controller/execution-test-cases-spec.js +6 -1
  62. package/dist/test/workers/execution-controller/recovery-spec.js +1 -1
  63. package/dist/test/workers/execution-controller/scheduler-spec.js +1 -1
  64. package/dist/test/workers/fixtures/ops/example-reader/slicer.js +1 -1
  65. package/dist/test/workers/fixtures/ops/new-reader/slicer.js +1 -1
  66. package/dist/test/workers/helpers/configs.js +1 -2
  67. package/dist/test/workers/helpers/execution-controller-helper.js +1 -1
  68. package/dist/test/workers/helpers/test-context.js +1 -1
  69. package/dist/test/workers/helpers/zip-directory.js +1 -1
  70. package/dist/test/workers/worker/slice-spec.js +1 -1
  71. package/dist/test/workers/worker/worker-spec.js +1 -1
  72. package/package.json +11 -12
  73. package/worker-service.js +1 -1
@@ -4,33 +4,28 @@ export default {
4
4
  'index.number_of_replicas': 1
5
5
  },
6
6
  mappings: {
7
- job: {
8
- _all: {
9
- enabled: false
7
+ dynamic: false,
8
+ properties: {
9
+ active: {
10
+ type: 'boolean'
10
11
  },
11
- dynamic: false,
12
- properties: {
13
- active: {
14
- type: 'boolean'
15
- },
16
- job_id: {
17
- type: 'keyword'
18
- },
19
- _context: {
20
- type: 'keyword'
21
- },
22
- _created: {
23
- type: 'date'
24
- },
25
- _updated: {
26
- type: 'date'
27
- },
28
- _deleted: {
29
- type: 'boolean'
30
- },
31
- _deleted_on: {
32
- type: 'date'
33
- }
12
+ job_id: {
13
+ type: 'keyword'
14
+ },
15
+ _context: {
16
+ type: 'keyword'
17
+ },
18
+ _created: {
19
+ type: 'date'
20
+ },
21
+ _updated: {
22
+ type: 'date'
23
+ },
24
+ _deleted: {
25
+ type: 'boolean'
26
+ },
27
+ _deleted_on: {
28
+ type: 'date'
34
29
  }
35
30
  }
36
31
  }
@@ -1,36 +1,31 @@
1
1
  export default {
2
2
  template: '__state*',
3
3
  mappings: {
4
- state: {
5
- _all: {
6
- enabled: false
4
+ dynamic: false,
5
+ properties: {
6
+ ex_id: {
7
+ type: 'keyword'
7
8
  },
8
- dynamic: false,
9
- properties: {
10
- ex_id: {
11
- type: 'keyword'
12
- },
13
- slice_id: {
14
- type: 'keyword'
15
- },
16
- slicer_id: {
17
- type: 'keyword'
18
- },
19
- slicer_order: {
20
- type: 'integer'
21
- },
22
- state: {
23
- type: 'keyword'
24
- },
25
- _created: {
26
- type: 'date'
27
- },
28
- _updated: {
29
- type: 'date'
30
- },
31
- error: {
32
- type: 'keyword'
33
- }
9
+ slice_id: {
10
+ type: 'keyword'
11
+ },
12
+ slicer_id: {
13
+ type: 'keyword'
14
+ },
15
+ slicer_order: {
16
+ type: 'integer'
17
+ },
18
+ state: {
19
+ type: 'keyword'
20
+ },
21
+ _created: {
22
+ type: 'date'
23
+ },
24
+ _updated: {
25
+ type: 'date'
26
+ },
27
+ error: {
28
+ type: 'keyword'
34
29
  }
35
30
  }
36
31
  }
@@ -1,4 +1,4 @@
1
- import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/utils';
1
+ import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/core-utils';
2
2
  import { S3ClientResponse, createS3Bucket, deleteS3Object, doesBucketExist, getS3Object, listS3Objects, putS3Object, s3RequestWithRetry } from '@terascope/file-asset-apis';
3
3
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
4
4
  export class S3Store {
@@ -1,8 +1,9 @@
1
- import { TSError, includes, getTypeOf, makeISODate } from '@terascope/utils';
1
+ import { TSError, includes, getTypeOf, makeISODate } from '@terascope/core-utils';
2
2
  import { RecoveryCleanupType } from '@terascope/job-components';
3
3
  import { v4 as uuid } from 'uuid';
4
4
  import { makeLogger } from '../workers/helpers/terafoundation.js';
5
5
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
6
+ import { getPackageJSON } from '../utils/file_utils.js';
6
7
  const INIT_STATUS = ['pending', 'scheduling', 'initializing'];
7
8
  const RUNNING_STATUS = ['recovering', 'running', 'failing', 'paused', 'stopping'];
8
9
  const TERMINAL_STATUS = ['completed', 'stopped', 'rejected', 'failed', 'terminated'];
@@ -78,7 +79,8 @@ export class ExecutionStorage {
78
79
  _updated: date,
79
80
  _has_errors: false,
80
81
  _slicer_stats: {},
81
- _failureReason: ''
82
+ _failureReason: '',
83
+ teraslice_version: `v${getPackageJSON().version}`
82
84
  });
83
85
  // @ts-expect-error
84
86
  delete doc.slicer_port;
@@ -1,5 +1,5 @@
1
1
  import { v4 as uuid } from 'uuid';
2
- import { TSError, makeISODate } from '@terascope/utils';
2
+ import { TSError, makeISODate } from '@terascope/core-utils';
3
3
  import { makeLogger } from '../workers/helpers/terafoundation.js';
4
4
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
5
5
  export class JobsStorage {
@@ -1,5 +1,5 @@
1
1
  import { RecoveryCleanupType } from '@terascope/job-components';
2
- import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/utils';
2
+ import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/core-utils';
3
3
  import { timeseriesIndex } from '../utils/date_utils.js';
4
4
  import { makeLogger } from '../workers/helpers/terafoundation.js';
5
5
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
@@ -57,7 +57,6 @@ export class StateStorage {
57
57
  action: {
58
58
  index: {
59
59
  _index: index,
60
- _type: this.recordType,
61
60
  _id: record.slice_id,
62
61
  },
63
62
  },
@@ -1,5 +1,5 @@
1
1
  import Table from 'easy-table';
2
- import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/utils';
2
+ import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/core-utils';
3
3
  export function makeTable(req, defaults, data, mappingFn) {
4
4
  const query = fieldsQuery(req.query, defaults);
5
5
  let emptyChar = 'N/A';
@@ -1,4 +1,4 @@
1
- import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/utils';
1
+ import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/core-utils';
2
2
  import * as semver from 'semver';
3
3
  export function findMatchingAsset(records, name, version) {
4
4
  const range = toSemverRange(version);
@@ -54,7 +54,7 @@ function _isCompatibleAsset(name, range, skipRestrictions = false) {
54
54
  return function checkIfCompatibleAsset(record) {
55
55
  if (record.name !== name)
56
56
  return false;
57
- if (!semver.satisfies(record.version, range)) {
57
+ if (!semver.satisfies(record.version, range, { includePrerelease: true })) {
58
58
  return false;
59
59
  }
60
60
  if (skipRestrictions)
@@ -1,4 +1,4 @@
1
- import { isKey, makeISODate } from '@terascope/utils';
1
+ import { isKey, makeISODate } from '@terascope/core-utils';
2
2
  const options = {
3
3
  year: 'y',
4
4
  years: 'y',
@@ -1,4 +1,4 @@
1
- import { isString, isObjectEntity, getTypeOf, isArrayLike } from '@terascope/utils';
1
+ import { isString, isObjectEntity, getTypeOf, isArrayLike } from '@terascope/core-utils';
2
2
  export function safeEncode(obj) {
3
3
  let str;
4
4
  if (isString(obj)) {
@@ -3,7 +3,7 @@ import path from 'node:path';
3
3
  import fse from 'fs-extra';
4
4
  import semver from 'semver';
5
5
  import { Mutex } from 'async-mutex';
6
- import { TSError } from '@terascope/utils';
6
+ import { TSError } from '@terascope/core-utils';
7
7
  import decompress from 'decompress';
8
8
  import { getMajorVersion } from './asset_utils.js';
9
9
  const mutex = new Mutex();
@@ -1,4 +1,4 @@
1
- import { trim, padEnd } from '@terascope/utils';
1
+ import { trim, padEnd } from '@terascope/core-utils';
2
2
  import { customAlphabet } from 'nanoid';
3
3
  export function newId(prefix, lowerCase = false, length = 15) {
4
4
  let characters = '-0123456789abcdefghijklmnopqrstuvwxyz';
@@ -1,4 +1,4 @@
1
- import { get, toIntegerOrThrow, shuffle, range } from '@terascope/utils';
1
+ import { get, toIntegerOrThrow, shuffle, range } from '@terascope/core-utils';
2
2
  import getPort from 'get-port';
3
3
  const usedPorts = [];
4
4
  /**
@@ -1,4 +1,4 @@
1
- import { getFullErrorStack, pDelay } from '@terascope/utils';
1
+ import { getFullErrorStack, pDelay } from '@terascope/core-utils';
2
2
  import { fileURLToPath } from 'node:url';
3
3
  import { AssetLoader } from './loader.js';
4
4
  import { makeTerafoundationContext } from '../context/terafoundation-context.js';
@@ -1,6 +1,6 @@
1
1
  import path from 'node:path';
2
2
  import fs from 'node:fs';
3
- import { get, isEmpty } from '@terascope/utils';
3
+ import { get, isEmpty } from '@terascope/core-utils';
4
4
  import { AssetsStorage } from '../../storage/index.js';
5
5
  import { makeLogger } from '../helpers/terafoundation.js';
6
6
  import { saveAsset } from '../../utils/file_utils.js';
@@ -2,7 +2,7 @@ import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
4
  import { fork } from 'node:child_process';
5
- import { isEmpty, get, has } from '@terascope/utils';
5
+ import { isEmpty, get, has } from '@terascope/core-utils';
6
6
  import { makeLogger } from '../helpers/terafoundation.js';
7
7
  import { safeEncode } from '../../utils/encoding_utils.js';
8
8
  const filePath = fileURLToPath(new URL('.', import.meta.url));
@@ -1,4 +1,4 @@
1
- import { makeISODate, get, has } from '@terascope/utils';
1
+ import { get, has, makeISODate } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../helpers/terafoundation.js';
3
3
  /**
4
4
  * @typedef ExecutionStats
@@ -1,6 +1,6 @@
1
1
  import ms from 'ms';
2
2
  import { formatURL, ExecutionController as ExController, ClusterMaster } from '@terascope/teraslice-messaging';
3
- import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile, makeISODate, debounce, throttle } from '@terascope/utils';
3
+ import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile, debounce, throttle, makeISODate } from '@terascope/core-utils';
4
4
  import { isPromAvailable } from '@terascope/job-components';
5
5
  import { waitForWorkerShutdown } from '../helpers/worker-shutdown.js';
6
6
  import { StateStorage, ExecutionStorage, SliceState } from '../../storage/index.js';
@@ -1,4 +1,4 @@
1
- import { pRaceWithTimeout, logError, Queue } from '@terascope/utils';
1
+ import { pRaceWithTimeout, logError, Queue } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../helpers/terafoundation.js';
3
3
  export class RecoveryModule {
4
4
  logger;
@@ -1,4 +1,4 @@
1
- import { Queue, noop, pDelay, get, toString, makeISODate, logError, pWhile } from '@terascope/utils';
1
+ import { noop, pDelay, get, toString, logError, pWhile, Queue, makeISODate } from '@terascope/core-utils';
2
2
  import { RecoveryModule } from './recovery.js';
3
3
  import { makeLogger } from '../helpers/terafoundation.js';
4
4
  export class Scheduler {
@@ -1,4 +1,4 @@
1
- import { has, isInteger, toNumber } from '@terascope/utils';
1
+ import { has, isInteger, toNumber } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../helpers/terafoundation.js';
3
3
  // TODO: more types
4
4
  export class SliceAnalytics {
@@ -1,4 +1,4 @@
1
- import { makeISODate } from '@terascope/utils';
1
+ import { makeISODate } from '@terascope/core-utils';
2
2
  import { JobValidator } from '@terascope/job-components';
3
3
  import { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
4
4
  // TODO: fix type here
@@ -1,4 +1,4 @@
1
- import { isString, isArray, truncate, } from '@terascope/utils';
1
+ import { isString, isArray, truncate, } from '@terascope/core-utils';
2
2
  function formatVal(value) {
3
3
  if (isString(value))
4
4
  return `"${value}"`;
@@ -1,4 +1,4 @@
1
- import { get, isFunction, isString } from '@terascope/utils';
1
+ import { get, isFunction, isString } from '@terascope/core-utils';
2
2
  import { makeContextLogger } from '@terascope/job-components';
3
3
  import { safeDecode } from '../../utils/encoding_utils.js';
4
4
  export function generateWorkerId(context) {
@@ -1,4 +1,4 @@
1
- import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/utils';
1
+ import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/core-utils';
2
2
  import ms from 'ms';
3
3
  import { makeLogger } from './terafoundation.js';
4
4
  export function waitForWorkerShutdown(context, eventName) {
@@ -1,5 +1,5 @@
1
1
  import { EventEmitter } from 'node:events';
2
- import { debugLogger, isTest } from '@terascope/utils';
2
+ import { debugLogger, isTest } from '@terascope/core-utils';
3
3
  const defaultLogger = debugLogger('metrics');
4
4
  export class Metrics extends EventEmitter {
5
5
  logger;
@@ -1,5 +1,5 @@
1
1
  /* eslint-disable prefer-const */
2
- import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/utils';
2
+ import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/core-utils';
3
3
  import { ExecutionController, formatURL } from '@terascope/teraslice-messaging';
4
4
  import { isPromAvailable } from '@terascope/job-components';
5
5
  import { StateStorage, AnalyticsStorage } from '../../storage/index.js';
@@ -1,4 +1,4 @@
1
- import { TSError, getTypeOf, logError } from '@terascope/utils';
1
+ import { TSError, getTypeOf, logError } from '@terascope/core-utils';
2
2
  import { SliceState } from '../../storage/index.js';
3
3
  import { makeLogger } from '../helpers/terafoundation.js';
4
4
  import { logOpStats } from '../helpers/op-analytics.js';
@@ -1,6 +1,6 @@
1
1
  import { V1Deployment, V1Job, V1Pod, V1ReplicaSet, V1Service } from '@kubernetes/client-node';
2
2
  import nock from 'nock';
3
- import { debugLogger } from '@terascope/job-components';
3
+ import { debugLogger } from '@terascope/core-utils';
4
4
  import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js';
5
5
  const logger = debugLogger('k8s-v2-spec');
6
6
  const _url = 'http://mock.kube.api';
@@ -1,5 +1,5 @@
1
1
  import yaml from 'js-yaml';
2
- import { debugLogger } from '@terascope/utils';
2
+ import { debugLogger } from '@terascope/core-utils';
3
3
  import { K8sDeploymentResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js';
4
4
  import { K8sJobResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js';
5
5
  import { K8sServiceResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js';
@@ -1,4 +1,4 @@
1
- import { cloneDeep } from '@terascope/utils';
1
+ import { cloneDeep } from '@terascope/core-utils';
2
2
  import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json';
3
3
  import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
4
4
  describe('k8sState with pods from multiple clusters', () => {
@@ -1,4 +1,4 @@
1
- import { cloneDeep } from '@terascope/utils';
1
+ import { cloneDeep } from '@terascope/core-utils';
2
2
  import _podsJobRunning from '../files/job-running-v1-k8s-pods.json';
3
3
  import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
4
4
  describe('k8sState', () => {
@@ -1,6 +1,6 @@
1
1
  import { EventEmitter } from 'node:events';
2
- import { debugLogger } from '@terascope/utils';
3
- import Socket from 'socket.io';
2
+ import { debugLogger } from '@terascope/core-utils';
3
+ import { Server as SocketServer } from 'socket.io';
4
4
  import { nodeMaster } from '../src/lib/cluster/node_master.js';
5
5
  process.env.assignment = 'node_master';
6
6
  describe('Node master', () => {
@@ -49,7 +49,7 @@ describe('Node master', () => {
49
49
  __test_job: JSON.stringify(jobJson),
50
50
  __test_assignment: 'worker'
51
51
  };
52
- const fakeClusterMaster = Socket({
52
+ const fakeClusterMaster = new SocketServer({
53
53
  path: '/native-clustering'
54
54
  });
55
55
  fakeClusterMaster.on('connection', (socket) => {
@@ -1,6 +1,6 @@
1
1
  import { jest } from '@jest/globals';
2
2
  import events from 'node:events';
3
- import { debugLogger } from '@terascope/utils';
3
+ import { debugLogger } from '@terascope/core-utils';
4
4
  import { Messaging, routing } from '../../src/lib/cluster/services/cluster/backends/native/messaging.js';
5
5
  describe('messaging module', () => {
6
6
  const logger = debugLogger('messaging');
@@ -1,5 +1,5 @@
1
1
  import { jest } from '@jest/globals';
2
- import { pDelay, get, set } from '@terascope/utils';
2
+ import { pDelay, get, set } from '@terascope/core-utils';
3
3
  import { TestContext } from '../helpers/index.js';
4
4
  import { findPort } from '../../../src/lib/utils/port_utils.js';
5
5
  import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
@@ -1,5 +1,5 @@
1
1
  import { v4 as uuidv4 } from 'uuid';
2
- import { pDelay, times, random } from '@terascope/utils';
2
+ import { pDelay, times, random } from '@terascope/core-utils';
3
3
  import { RecoveryCleanupType } from '@terascope/job-components';
4
4
  import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
5
5
  import { TestContext } from '../helpers/index.js';
@@ -8,7 +8,6 @@ import { ExecutionController } from '../../../src/lib/workers/execution-controll
8
8
  import { findPort } from '../../../src/lib/utils/port_utils.js';
9
9
  import { newId } from '../../../src/lib/utils/id_utils.js';
10
10
  const ExecutionControllerClient = ExController.Client;
11
- process.env.BLUEBIRD_LONG_STACK_TRACES = '1';
12
11
  describe('ExecutionController Special Tests', () => {
13
12
  // [ message, config ]
14
13
  const testCases = [
@@ -1,9 +1,10 @@
1
- import { pDelay, times, random, isNotNil } from '@terascope/utils';
1
+ import { pDelay, times, random, isNotNil } from '@terascope/core-utils';
2
2
  import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
3
3
  import { TestContext } from '../helpers/index.js';
4
4
  import { getTestCases } from '../helpers/execution-controller-helper.js';
5
5
  import { findPort } from '../../../src/lib/utils/port_utils.js';
6
6
  import { newId } from '../../../src/lib/utils/id_utils.js';
7
+ import { getPackageJSON } from '../../../src/lib/utils/file_utils.js';
7
8
  import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
8
9
  const ExecutionControllerClient = ExController.Client;
9
10
  describe('ExecutionController Test Cases', () => {
@@ -333,6 +334,10 @@ describe('ExecutionController Test Cases', () => {
333
334
  const metadata = updateMetadata ? { slice_calls: count + 1 } : {};
334
335
  expect(executionRecord).toHaveProperty('metadata', metadata);
335
336
  });
337
+ it('should have the correct teraslice version', () => {
338
+ const terasliceVersion = `v${getPackageJSON().version}`;
339
+ expect(executionRecord.teraslice_version).toBe(terasliceVersion);
340
+ });
336
341
  });
337
342
  });
338
343
  //# sourceMappingURL=execution-test-cases-spec.js.map
@@ -1,4 +1,4 @@
1
- import { pDelay, pWhile, debugLogger } from '@terascope/utils';
1
+ import { pDelay, pWhile, debugLogger } from '@terascope/core-utils';
2
2
  import { EventEmitter } from 'node:events';
3
3
  import { RecoveryModule } from '../../../src/lib/workers/execution-controller/recovery.js';
4
4
  const eventEmitter = new EventEmitter();
@@ -1,6 +1,6 @@
1
1
  import { jest } from '@jest/globals';
2
2
  import { v4 as uuidv4 } from 'uuid';
3
- import { pDelay, once, times } from '@terascope/utils';
3
+ import { pDelay, once, times } from '@terascope/core-utils';
4
4
  import { TestContext } from '../helpers/test-context.js';
5
5
  import { Scheduler } from '../../../src/lib/workers/execution-controller/scheduler.js';
6
6
  describe('Scheduler', () => {
@@ -1,5 +1,5 @@
1
1
  import { ParallelSlicer } from '@terascope/job-components';
2
- import { get } from '@terascope/utils';
2
+ import { get } from '@terascope/core-utils';
3
3
  const defaultSlicerResults = [{ howdy: true }, null];
4
4
  export default class ExampleSlicer extends ParallelSlicer {
5
5
  async newSlicer(_id) {
@@ -1,5 +1,5 @@
1
1
  import { ParallelSlicer } from '@terascope/job-components';
2
- import { pDelay, times } from '@terascope/utils';
2
+ import { pDelay, times } from '@terascope/core-utils';
3
3
  export default class ExampleSlicer extends ParallelSlicer {
4
4
  async newSlicer(id) {
5
5
  const { countPerSlicer } = this.opConfig;
@@ -1,6 +1,6 @@
1
1
  import path from 'node:path';
2
2
  import { fileURLToPath } from 'node:url';
3
- import { withoutNil } from '@terascope/utils';
3
+ import { withoutNil } from '@terascope/core-utils';
4
4
  // @ts-expect-error
5
5
  import Chance from 'chance';
6
6
  import { newId } from '../../../src/lib/utils/id_utils.js';
@@ -75,7 +75,6 @@ const newSysConfig = (options) => {
75
75
  const { clusterName = 'test-teraslice-cluster', timeout = 3000, actionTimeout = 2000, shutdownTimeout = 4000, assetDir, clusterMasterPort, log_level_terafoundation = 'info', } = options;
76
76
  return {
77
77
  terafoundation: {
78
- environment: 'development',
79
78
  log_level: log_level_terafoundation,
80
79
  connectors: {
81
80
  'elasticsearch-next': {
@@ -1,4 +1,4 @@
1
- import { pDelay, pDefer } from '@terascope/utils';
1
+ import { pDelay, pDefer } from '@terascope/core-utils';
2
2
  export function makeShutdownEarlyFn({ exController, enabled = false }) {
3
3
  let shutdownErr = {
4
4
  message: 'Shutdown never triggered'
@@ -2,7 +2,7 @@ import fs from 'fs-extra';
2
2
  import path from 'node:path';
3
3
  import { createTempDirSync, cleanupTempDirs } from 'jest-fixtures';
4
4
  import { newTestSlice } from '@terascope/job-components';
5
- import { get, pWhile } from '@terascope/utils';
5
+ import { get, pWhile } from '@terascope/core-utils';
6
6
  import { ClusterMaster } from '@terascope/teraslice-messaging';
7
7
  import { AssetsStorage, StateStorage, AnalyticsStorage, ExecutionStorage, JobsStorage } from '../../../src/lib/storage/index.js';
8
8
  import { initializeTestExecution } from '../../../src/lib/workers/helpers/job.js';
@@ -1,5 +1,5 @@
1
1
  import path from 'node:path';
2
- import { random } from '@terascope/utils';
2
+ import { random } from '@terascope/core-utils';
3
3
  // @ts-expect-error
4
4
  import BufferStreams from 'bufferstreams';
5
5
  import archiver from 'archiver';
@@ -1,5 +1,5 @@
1
1
  import { jest } from '@jest/globals';
2
- import { times } from '@terascope/utils';
2
+ import { times } from '@terascope/core-utils';
3
3
  import { SliceExecution } from '../../../src/lib/workers/worker/slice.js';
4
4
  import { TestContext } from '../helpers/index.js';
5
5
  describe('Slice', () => {
@@ -1,6 +1,6 @@
1
1
  import { jest } from '@jest/globals';
2
2
  import 'jest-extended';
3
- import { pDelay } from '@terascope/utils';
3
+ import { pDelay } from '@terascope/core-utils';
4
4
  import { ExecutionController } from '@terascope/teraslice-messaging';
5
5
  import { findPort } from '../../../src/lib/utils/port_utils.js';
6
6
  import { Worker } from '../../../src/lib/workers/worker/index.js';
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "teraslice",
3
3
  "displayName": "Teraslice",
4
- "version": "3.0.0-dev.1",
4
+ "version": "3.0.0-dev.10",
5
5
  "description": "Distributed computing platform for processing JSON data",
6
6
  "homepage": "https://github.com/terascope/teraslice#readme",
7
7
  "bugs": {
@@ -27,9 +27,8 @@
27
27
  "start": "node service.js",
28
28
  "test": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
29
29
  "test:debug": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --debug ../teraslice --",
30
- "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='6.8.6' node ../scripts/bin/ts-scripts test ../teraslice --",
31
30
  "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' node ../scripts/bin/ts-scripts test ../teraslice --",
32
- "test:opensearch1": "TEST_OPENSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
31
+ "test:opensearch1": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='1.3.11' node ../scripts/bin/ts-scripts test ../teraslice --",
33
32
  "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' node ../scripts/bin/ts-scripts test ../teraslice --",
34
33
  "test:opensearch3": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='3.1.0' node ../scripts/bin/ts-scripts test ../teraslice --",
35
34
  "test:watch": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --watch ../teraslice --"
@@ -40,11 +39,11 @@
40
39
  },
41
40
  "dependencies": {
42
41
  "@kubernetes/client-node": "~1.3.0",
43
- "@terascope/elasticsearch-api": "~5.0.0-dev.1",
44
- "@terascope/job-components": "~2.0.0-dev.1",
45
- "@terascope/teraslice-messaging": "~2.0.0-dev.1",
46
- "@terascope/types": "~2.0.0-dev.1",
47
- "@terascope/utils": "~2.0.0-dev.1",
42
+ "@terascope/core-utils": "~2.0.0-dev.8",
43
+ "@terascope/elasticsearch-api": "~5.0.0-dev.9",
44
+ "@terascope/job-components": "~2.0.0-dev.9",
45
+ "@terascope/teraslice-messaging": "~2.0.0-dev.9",
46
+ "@terascope/types": "~2.0.0-dev.7",
48
47
  "async-mutex": "~0.5.0",
49
48
  "barbe": "~3.0.17",
50
49
  "body-parser": "~2.2.0",
@@ -60,13 +59,13 @@
60
59
  "ms": "~2.1.3",
61
60
  "nanoid": "~5.1.5",
62
61
  "semver": "~7.7.2",
63
- "socket.io": "~1.7.4",
64
- "socket.io-client": "~1.7.4",
65
- "terafoundation": "~2.0.0-dev.1",
62
+ "socket.io": "~4.8.1",
63
+ "socket.io-client": "~4.8.1",
64
+ "terafoundation": "~2.0.0-dev.9",
66
65
  "uuid": "~13.0.0"
67
66
  },
68
67
  "devDependencies": {
69
- "@terascope/opensearch-client": "~2.0.0-dev.1",
68
+ "@terascope/opensearch-client": "~2.0.0-dev.8",
70
69
  "@types/archiver": "~6.0.3",
71
70
  "@types/express": "~5.0.3",
72
71
  "@types/gc-stats": "~1.4.3",