teraslice 3.0.0-dev.7 → 3.0.0-dev.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/cluster-service.js +1 -1
  2. package/dist/src/lib/cluster/cluster_master.js +1 -1
  3. package/dist/src/lib/cluster/node_master.js +1 -1
  4. package/dist/src/lib/cluster/services/api.js +2 -2
  5. package/dist/src/lib/cluster/services/assets.js +1 -1
  6. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js +1 -1
  7. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js +1 -1
  8. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js +1 -1
  9. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js +1 -1
  10. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js +1 -1
  11. package/dist/src/lib/cluster/services/cluster/backends/native/index.js +1 -1
  12. package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js +1 -1
  13. package/dist/src/lib/cluster/services/cluster/backends/state-utils.js +1 -1
  14. package/dist/src/lib/cluster/services/execution.js +1 -1
  15. package/dist/src/lib/cluster/services/jobs.js +1 -1
  16. package/dist/src/lib/config/index.js +1 -1
  17. package/dist/src/lib/config/schemas/system.js +1 -1
  18. package/dist/src/lib/storage/analytics.js +1 -1
  19. package/dist/src/lib/storage/assets.js +1 -1
  20. package/dist/src/lib/storage/backends/elasticsearch_store.js +1 -1
  21. package/dist/src/lib/storage/backends/mappings/ex.js +3 -0
  22. package/dist/src/lib/storage/backends/s3_store.js +1 -1
  23. package/dist/src/lib/storage/execution.js +4 -2
  24. package/dist/src/lib/storage/jobs.js +1 -1
  25. package/dist/src/lib/storage/state.js +1 -1
  26. package/dist/src/lib/utils/api_utils.js +1 -1
  27. package/dist/src/lib/utils/asset_utils.js +1 -1
  28. package/dist/src/lib/utils/date_utils.js +1 -1
  29. package/dist/src/lib/utils/encoding_utils.js +1 -1
  30. package/dist/src/lib/utils/file_utils.js +1 -1
  31. package/dist/src/lib/utils/id_utils.js +1 -1
  32. package/dist/src/lib/utils/port_utils.js +1 -1
  33. package/dist/src/lib/workers/assets/loader-executable.js +1 -1
  34. package/dist/src/lib/workers/assets/loader.js +1 -1
  35. package/dist/src/lib/workers/assets/spawn.js +1 -1
  36. package/dist/src/lib/workers/execution-controller/execution-analytics.js +1 -1
  37. package/dist/src/lib/workers/execution-controller/index.js +1 -1
  38. package/dist/src/lib/workers/execution-controller/recovery.js +1 -1
  39. package/dist/src/lib/workers/execution-controller/scheduler.js +1 -1
  40. package/dist/src/lib/workers/execution-controller/slice-analytics.js +1 -1
  41. package/dist/src/lib/workers/helpers/job.js +1 -1
  42. package/dist/src/lib/workers/helpers/op-analytics.js +1 -1
  43. package/dist/src/lib/workers/helpers/terafoundation.js +1 -1
  44. package/dist/src/lib/workers/helpers/worker-shutdown.js +1 -1
  45. package/dist/src/lib/workers/metrics/index.js +1 -1
  46. package/dist/src/lib/workers/worker/index.js +1 -1
  47. package/dist/src/lib/workers/worker/slice.js +1 -1
  48. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js +1 -1
  49. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js +1 -1
  50. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js +1 -1
  51. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js +1 -1
  52. package/dist/test/node_master-spec.js +1 -1
  53. package/dist/test/services/messaging-spec.js +1 -1
  54. package/dist/test/workers/execution-controller/execution-controller-spec.js +1 -1
  55. package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js +1 -1
  56. package/dist/test/workers/execution-controller/execution-test-cases-spec.js +6 -1
  57. package/dist/test/workers/execution-controller/recovery-spec.js +1 -1
  58. package/dist/test/workers/execution-controller/scheduler-spec.js +1 -1
  59. package/dist/test/workers/fixtures/ops/example-reader/slicer.js +1 -1
  60. package/dist/test/workers/fixtures/ops/new-reader/slicer.js +1 -1
  61. package/dist/test/workers/helpers/configs.js +1 -1
  62. package/dist/test/workers/helpers/execution-controller-helper.js +1 -1
  63. package/dist/test/workers/helpers/test-context.js +1 -1
  64. package/dist/test/workers/helpers/zip-directory.js +1 -1
  65. package/dist/test/workers/worker/slice-spec.js +1 -1
  66. package/dist/test/workers/worker/worker-spec.js +1 -1
  67. package/package.json +8 -8
  68. package/worker-service.js +1 -1
@@ -1,4 +1,4 @@
1
- import { get, logError } from '@terascope/utils';
1
+ import { get, logError } from '@terascope/core-utils';
2
2
  import { shutdownHandler } from './dist/src/lib/workers/helpers/worker-shutdown.js';
3
3
  import { makeTerafoundationContext } from './dist/src/lib/workers/context/terafoundation-context.js';
4
4
  import { ClusterMaster } from './dist/src/lib/cluster/cluster_master.js';
@@ -1,6 +1,6 @@
1
1
  import express from 'express';
2
2
  import got from 'got';
3
- import { pDelay, logError, get, parseError } from '@terascope/utils';
3
+ import { pDelay, logError, get, parseError } from '@terascope/core-utils';
4
4
  import { ClusterMaster as ClusterMasterMessaging } from '@terascope/teraslice-messaging';
5
5
  import { isPromAvailable } from '@terascope/job-components';
6
6
  import { makeLogger } from '../workers/helpers/terafoundation.js';
@@ -1,6 +1,6 @@
1
1
  import ms from 'ms';
2
2
  import { Mutex } from 'async-mutex';
3
- import { getFullErrorStack, debounce, isEmpty, has } from '@terascope/utils';
3
+ import { getFullErrorStack, debounce, isEmpty, has } from '@terascope/core-utils';
4
4
  import { makeLogger } from '../workers/helpers/terafoundation.js';
5
5
  import { Messaging } from './services/cluster/backends/native/messaging.js';
6
6
  import { spawnAssetLoader } from '../workers/assets/spawn.js';
@@ -3,7 +3,7 @@ import bodyParser from 'body-parser';
3
3
  import { pipeline as streamPipeline } from 'node:stream/promises';
4
4
  import got from 'got';
5
5
  import { RecoveryCleanupType } from '@terascope/job-components';
6
- import { parseErrorInfo, parseList, logError, TSError, startsWith, pWhile, isKey } from '@terascope/utils';
6
+ import { parseErrorInfo, parseList, logError, TSError, startsWith, pWhile, isKey } from '@terascope/core-utils';
7
7
  import { ExecutionStatusEnum } from '@terascope/types';
8
8
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
9
9
  import { makeTable, sendError, handleTerasliceRequest, getSearchOptions, createJobActiveQuery, addDeletedToQuery } from '../../utils/api_utils.js';
@@ -558,7 +558,7 @@ export class ApiService {
558
558
  name,
559
559
  node_version: process.version,
560
560
  platform: this.context.platform,
561
- teraslice_version: `v${getPackageJSON().version}`
561
+ teraslice_version: `v${terasliceVersion}`
562
562
  }, 1);
563
563
  this.context.apis.foundation.promMetrics.set('slices_processed', {}, stats.controllers.processed);
564
564
  this.context.apis.foundation.promMetrics.set('slices_failed', {}, stats.controllers.failed);
@@ -1,5 +1,5 @@
1
1
  import express from 'express';
2
- import { TSError, parseErrorInfo, logError, toBoolean } from '@terascope/utils';
2
+ import { TSError, parseErrorInfo, logError, toBoolean } from '@terascope/core-utils';
3
3
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
4
4
  import { AssetsStorage } from '../../storage/index.js';
5
5
  import { getBackendConfig } from '../../storage/assets.js';
@@ -1,4 +1,4 @@
1
- import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/utils';
1
+ import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
3
3
  import { gen } from './k8sState.js';
4
4
  import { K8s } from './k8s.js';
@@ -1,4 +1,4 @@
1
- import { TSError, get, isEmpty, pDelay, pRetry } from '@terascope/utils';
1
+ import { TSError, get, isEmpty, pDelay, pRetry } from '@terascope/core-utils';
2
2
  import * as k8s from '@kubernetes/client-node';
3
3
  import { convertToTSResource, convertToTSResourceList, getRetryConfig, isDeployment, isJob, isPod, isReplicaSet, isService, isTSPod } from './utils.js';
4
4
  export class K8s {
@@ -1,4 +1,4 @@
1
- import { isNumber, get, set, has, isEmpty, merge } from '@terascope/utils';
1
+ import { isNumber, get, set, has, isEmpty, merge } from '@terascope/core-utils';
2
2
  import { safeEncode } from '../../../../../utils/encoding_utils.js';
3
3
  import { setMaxOldSpaceViaEnv } from './utils.js';
4
4
  export class K8sResource {
@@ -1,4 +1,4 @@
1
- import { get, has, uniq, difference } from '@terascope/utils';
1
+ import { get, has, uniq, difference } from '@terascope/core-utils';
2
2
  /**
3
3
  * Given the k8s Pods API output generates the appropriate Teraslice cluster
4
4
  * state. NOTE: This assumes the pods have already been filtered to ensure they
@@ -3,7 +3,7 @@ import path from 'node:path';
3
3
  // @ts-expect-error
4
4
  import barbe from 'barbe';
5
5
  import * as k8s from '@kubernetes/client-node';
6
- import { isTest } from '@terascope/utils';
6
+ import { isTest } from '@terascope/core-utils';
7
7
  const MAX_RETRIES = isTest ? 2 : 3;
8
8
  const RETRY_DELAY = isTest ? 50 : 1000; // time in ms
9
9
  const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/');
@@ -1,5 +1,5 @@
1
1
  /* eslint-disable @typescript-eslint/prefer-for-of */
2
- import { Queue, TSError, getFullErrorStack, pDelay, cloneDeep, debounce, pMap, orderBy, isInteger, get } from '@terascope/utils';
2
+ import { TSError, getFullErrorStack, debounce, pDelay, cloneDeep, pMap, orderBy, isInteger, get, Queue } from '@terascope/core-utils';
3
3
  import { Dispatch } from './dispatch.js';
4
4
  import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
5
5
  import { findWorkersByExecutionID } from '../state-utils.js';
@@ -1,5 +1,5 @@
1
1
  import { nanoid } from 'nanoid';
2
- import { pDelay, Queue, isFunction, isEmpty, get, toNumber, isKey } from '@terascope/utils';
2
+ import { pDelay, isFunction, isEmpty, get, toNumber, isKey, Queue } from '@terascope/core-utils';
3
3
  import { io as socketIOClient } from 'socket.io-client';
4
4
  import { Server as socketIOServer } from 'socket.io';
5
5
  import { isProcessAssignment, ProcessAssignment } from '../../../../../../interfaces.js';
@@ -1,4 +1,4 @@
1
- import { cloneDeep } from '@terascope/utils';
1
+ import { cloneDeep } from '@terascope/core-utils';
2
2
  export function iterateState(clusterState, cb) {
3
3
  // I clone here, because the code below accidentally modifies clusterState.
4
4
  // Not sure if this is the best choice.
@@ -1,4 +1,4 @@
1
- import { Queue, TSError, getFullErrorStack, logError, get, withoutNil, isEmpty, multiFieldSort, isString, flatten, includes, cloneDeep } from '@terascope/utils';
1
+ import { TSError, getFullErrorStack, get, logError, withoutNil, isEmpty, multiFieldSort, isString, flatten, includes, cloneDeep, Queue } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
3
3
  /**
4
4
  * New execution result
@@ -1,4 +1,4 @@
1
- import { TSError, uniq, cloneDeep, isEmpty, getTypeOf, isString, makeISODate, defaultsDeep } from '@terascope/utils';
1
+ import { TSError, uniq, cloneDeep, isEmpty, getTypeOf, isString, defaultsDeep, makeISODate } from '@terascope/core-utils';
2
2
  import { JobValidator, parseName } from '@terascope/job-components';
3
3
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
4
4
  import { spawnAssetLoader } from '../../workers/assets/spawn.js';
@@ -1,5 +1,5 @@
1
1
  import path from 'node:path';
2
- import { get } from '@terascope/utils';
2
+ import { get } from '@terascope/core-utils';
3
3
  import { fileURLToPath } from 'node:url';
4
4
  import { formats } from '@terascope/job-components';
5
5
  import { configSchema } from './schemas/system.js';
@@ -1,6 +1,6 @@
1
1
  import ip from 'ip';
2
2
  import path from 'node:path';
3
- import { isPlainObject, isString, isArray, isInteger } from '@terascope/utils';
3
+ import { isPlainObject, isString, isArray, isInteger } from '@terascope/core-utils';
4
4
  import { cpus } from 'node:os';
5
5
  const workerCount = cpus().length;
6
6
  const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next';
@@ -1,4 +1,4 @@
1
- import { pMap } from '@terascope/utils';
1
+ import { pMap } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../workers/helpers/terafoundation.js';
3
3
  import { timeseriesIndex } from '../utils/date_utils.js';
4
4
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
@@ -1,7 +1,7 @@
1
1
  import path from 'node:path';
2
2
  import fse from 'fs-extra';
3
3
  import crypto from 'node:crypto';
4
- import { TSError, uniq, isString, toString, filterObject, pDelay } from '@terascope/utils';
4
+ import { TSError, uniq, isString, toString, filterObject, pDelay } from '@terascope/core-utils';
5
5
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
6
6
  import { S3Store } from './backends/s3_store.js';
7
7
  import { makeLogger } from '../workers/helpers/terafoundation.js';
@@ -1,5 +1,5 @@
1
1
  import ms from 'ms';
2
- import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/utils';
2
+ import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/core-utils';
3
3
  import elasticsearchApi from '@terascope/elasticsearch-api';
4
4
  import { getClient } from '@terascope/job-components';
5
5
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
@@ -50,6 +50,9 @@ export default {
50
50
  },
51
51
  _deleted_on: {
52
52
  type: 'date'
53
+ },
54
+ teraslice_version: {
55
+ type: 'keyword'
53
56
  }
54
57
  }
55
58
  }
@@ -1,4 +1,4 @@
1
- import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/utils';
1
+ import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/core-utils';
2
2
  import { S3ClientResponse, createS3Bucket, deleteS3Object, doesBucketExist, getS3Object, listS3Objects, putS3Object, s3RequestWithRetry } from '@terascope/file-asset-apis';
3
3
  import { makeLogger } from '../../workers/helpers/terafoundation.js';
4
4
  export class S3Store {
@@ -1,8 +1,9 @@
1
- import { TSError, includes, getTypeOf, makeISODate } from '@terascope/utils';
1
+ import { TSError, includes, getTypeOf, makeISODate } from '@terascope/core-utils';
2
2
  import { RecoveryCleanupType } from '@terascope/job-components';
3
3
  import { v4 as uuid } from 'uuid';
4
4
  import { makeLogger } from '../workers/helpers/terafoundation.js';
5
5
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
6
+ import { getPackageJSON } from '../utils/file_utils.js';
6
7
  const INIT_STATUS = ['pending', 'scheduling', 'initializing'];
7
8
  const RUNNING_STATUS = ['recovering', 'running', 'failing', 'paused', 'stopping'];
8
9
  const TERMINAL_STATUS = ['completed', 'stopped', 'rejected', 'failed', 'terminated'];
@@ -78,7 +79,8 @@ export class ExecutionStorage {
78
79
  _updated: date,
79
80
  _has_errors: false,
80
81
  _slicer_stats: {},
81
- _failureReason: ''
82
+ _failureReason: '',
83
+ teraslice_version: `v${getPackageJSON().version}`
82
84
  });
83
85
  // @ts-expect-error
84
86
  delete doc.slicer_port;
@@ -1,5 +1,5 @@
1
1
  import { v4 as uuid } from 'uuid';
2
- import { TSError, makeISODate } from '@terascope/utils';
2
+ import { TSError, makeISODate } from '@terascope/core-utils';
3
3
  import { makeLogger } from '../workers/helpers/terafoundation.js';
4
4
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
5
5
  export class JobsStorage {
@@ -1,5 +1,5 @@
1
1
  import { RecoveryCleanupType } from '@terascope/job-components';
2
- import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/utils';
2
+ import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/core-utils';
3
3
  import { timeseriesIndex } from '../utils/date_utils.js';
4
4
  import { makeLogger } from '../workers/helpers/terafoundation.js';
5
5
  import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
@@ -1,5 +1,5 @@
1
1
  import Table from 'easy-table';
2
- import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/utils';
2
+ import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/core-utils';
3
3
  export function makeTable(req, defaults, data, mappingFn) {
4
4
  const query = fieldsQuery(req.query, defaults);
5
5
  let emptyChar = 'N/A';
@@ -1,4 +1,4 @@
1
- import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/utils';
1
+ import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/core-utils';
2
2
  import * as semver from 'semver';
3
3
  export function findMatchingAsset(records, name, version) {
4
4
  const range = toSemverRange(version);
@@ -1,4 +1,4 @@
1
- import { isKey, makeISODate } from '@terascope/utils';
1
+ import { isKey, makeISODate } from '@terascope/core-utils';
2
2
  const options = {
3
3
  year: 'y',
4
4
  years: 'y',
@@ -1,4 +1,4 @@
1
- import { isString, isObjectEntity, getTypeOf, isArrayLike } from '@terascope/utils';
1
+ import { isString, isObjectEntity, getTypeOf, isArrayLike } from '@terascope/core-utils';
2
2
  export function safeEncode(obj) {
3
3
  let str;
4
4
  if (isString(obj)) {
@@ -3,7 +3,7 @@ import path from 'node:path';
3
3
  import fse from 'fs-extra';
4
4
  import semver from 'semver';
5
5
  import { Mutex } from 'async-mutex';
6
- import { TSError } from '@terascope/utils';
6
+ import { TSError } from '@terascope/core-utils';
7
7
  import decompress from 'decompress';
8
8
  import { getMajorVersion } from './asset_utils.js';
9
9
  const mutex = new Mutex();
@@ -1,4 +1,4 @@
1
- import { trim, padEnd } from '@terascope/utils';
1
+ import { trim, padEnd } from '@terascope/core-utils';
2
2
  import { customAlphabet } from 'nanoid';
3
3
  export function newId(prefix, lowerCase = false, length = 15) {
4
4
  let characters = '-0123456789abcdefghijklmnopqrstuvwxyz';
@@ -1,4 +1,4 @@
1
- import { get, toIntegerOrThrow, shuffle, range } from '@terascope/utils';
1
+ import { get, toIntegerOrThrow, shuffle, range } from '@terascope/core-utils';
2
2
  import getPort from 'get-port';
3
3
  const usedPorts = [];
4
4
  /**
@@ -1,4 +1,4 @@
1
- import { getFullErrorStack, pDelay } from '@terascope/utils';
1
+ import { getFullErrorStack, pDelay } from '@terascope/core-utils';
2
2
  import { fileURLToPath } from 'node:url';
3
3
  import { AssetLoader } from './loader.js';
4
4
  import { makeTerafoundationContext } from '../context/terafoundation-context.js';
@@ -1,6 +1,6 @@
1
1
  import path from 'node:path';
2
2
  import fs from 'node:fs';
3
- import { get, isEmpty } from '@terascope/utils';
3
+ import { get, isEmpty } from '@terascope/core-utils';
4
4
  import { AssetsStorage } from '../../storage/index.js';
5
5
  import { makeLogger } from '../helpers/terafoundation.js';
6
6
  import { saveAsset } from '../../utils/file_utils.js';
@@ -2,7 +2,7 @@ import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
4
  import { fork } from 'node:child_process';
5
- import { isEmpty, get, has } from '@terascope/utils';
5
+ import { isEmpty, get, has } from '@terascope/core-utils';
6
6
  import { makeLogger } from '../helpers/terafoundation.js';
7
7
  import { safeEncode } from '../../utils/encoding_utils.js';
8
8
  const filePath = fileURLToPath(new URL('.', import.meta.url));
@@ -1,4 +1,4 @@
1
- import { makeISODate, get, has } from '@terascope/utils';
1
+ import { get, has, makeISODate } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../helpers/terafoundation.js';
3
3
  /**
4
4
  * @typedef ExecutionStats
@@ -1,6 +1,6 @@
1
1
  import ms from 'ms';
2
2
  import { formatURL, ExecutionController as ExController, ClusterMaster } from '@terascope/teraslice-messaging';
3
- import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile, makeISODate, debounce, throttle } from '@terascope/utils';
3
+ import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile, debounce, throttle, makeISODate } from '@terascope/core-utils';
4
4
  import { isPromAvailable } from '@terascope/job-components';
5
5
  import { waitForWorkerShutdown } from '../helpers/worker-shutdown.js';
6
6
  import { StateStorage, ExecutionStorage, SliceState } from '../../storage/index.js';
@@ -1,4 +1,4 @@
1
- import { pRaceWithTimeout, logError, Queue } from '@terascope/utils';
1
+ import { pRaceWithTimeout, logError, Queue } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../helpers/terafoundation.js';
3
3
  export class RecoveryModule {
4
4
  logger;
@@ -1,4 +1,4 @@
1
- import { Queue, noop, pDelay, get, toString, makeISODate, logError, pWhile } from '@terascope/utils';
1
+ import { noop, pDelay, get, toString, logError, pWhile, Queue, makeISODate } from '@terascope/core-utils';
2
2
  import { RecoveryModule } from './recovery.js';
3
3
  import { makeLogger } from '../helpers/terafoundation.js';
4
4
  export class Scheduler {
@@ -1,4 +1,4 @@
1
- import { has, isInteger, toNumber } from '@terascope/utils';
1
+ import { has, isInteger, toNumber } from '@terascope/core-utils';
2
2
  import { makeLogger } from '../helpers/terafoundation.js';
3
3
  // TODO: more types
4
4
  export class SliceAnalytics {
@@ -1,4 +1,4 @@
1
- import { makeISODate } from '@terascope/utils';
1
+ import { makeISODate } from '@terascope/core-utils';
2
2
  import { JobValidator } from '@terascope/job-components';
3
3
  import { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
4
4
  // TODO: fix type here
@@ -1,4 +1,4 @@
1
- import { isString, isArray, truncate, } from '@terascope/utils';
1
+ import { isString, isArray, truncate, } from '@terascope/core-utils';
2
2
  function formatVal(value) {
3
3
  if (isString(value))
4
4
  return `"${value}"`;
@@ -1,4 +1,4 @@
1
- import { get, isFunction, isString } from '@terascope/utils';
1
+ import { get, isFunction, isString } from '@terascope/core-utils';
2
2
  import { makeContextLogger } from '@terascope/job-components';
3
3
  import { safeDecode } from '../../utils/encoding_utils.js';
4
4
  export function generateWorkerId(context) {
@@ -1,4 +1,4 @@
1
- import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/utils';
1
+ import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/core-utils';
2
2
  import ms from 'ms';
3
3
  import { makeLogger } from './terafoundation.js';
4
4
  export function waitForWorkerShutdown(context, eventName) {
@@ -1,5 +1,5 @@
1
1
  import { EventEmitter } from 'node:events';
2
- import { debugLogger, isTest } from '@terascope/utils';
2
+ import { debugLogger, isTest } from '@terascope/core-utils';
3
3
  const defaultLogger = debugLogger('metrics');
4
4
  export class Metrics extends EventEmitter {
5
5
  logger;
@@ -1,5 +1,5 @@
1
1
  /* eslint-disable prefer-const */
2
- import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/utils';
2
+ import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/core-utils';
3
3
  import { ExecutionController, formatURL } from '@terascope/teraslice-messaging';
4
4
  import { isPromAvailable } from '@terascope/job-components';
5
5
  import { StateStorage, AnalyticsStorage } from '../../storage/index.js';
@@ -1,4 +1,4 @@
1
- import { TSError, getTypeOf, logError } from '@terascope/utils';
1
+ import { TSError, getTypeOf, logError } from '@terascope/core-utils';
2
2
  import { SliceState } from '../../storage/index.js';
3
3
  import { makeLogger } from '../helpers/terafoundation.js';
4
4
  import { logOpStats } from '../helpers/op-analytics.js';
@@ -1,6 +1,6 @@
1
1
  import { V1Deployment, V1Job, V1Pod, V1ReplicaSet, V1Service } from '@kubernetes/client-node';
2
2
  import nock from 'nock';
3
- import { debugLogger } from '@terascope/job-components';
3
+ import { debugLogger } from '@terascope/core-utils';
4
4
  import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js';
5
5
  const logger = debugLogger('k8s-v2-spec');
6
6
  const _url = 'http://mock.kube.api';
@@ -1,5 +1,5 @@
1
1
  import yaml from 'js-yaml';
2
- import { debugLogger } from '@terascope/utils';
2
+ import { debugLogger } from '@terascope/core-utils';
3
3
  import { K8sDeploymentResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js';
4
4
  import { K8sJobResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js';
5
5
  import { K8sServiceResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js';
@@ -1,4 +1,4 @@
1
- import { cloneDeep } from '@terascope/utils';
1
+ import { cloneDeep } from '@terascope/core-utils';
2
2
  import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json';
3
3
  import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
4
4
  describe('k8sState with pods from multiple clusters', () => {
@@ -1,4 +1,4 @@
1
- import { cloneDeep } from '@terascope/utils';
1
+ import { cloneDeep } from '@terascope/core-utils';
2
2
  import _podsJobRunning from '../files/job-running-v1-k8s-pods.json';
3
3
  import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
4
4
  describe('k8sState', () => {
@@ -1,5 +1,5 @@
1
1
  import { EventEmitter } from 'node:events';
2
- import { debugLogger } from '@terascope/utils';
2
+ import { debugLogger } from '@terascope/core-utils';
3
3
  import { Server as SocketServer } from 'socket.io';
4
4
  import { nodeMaster } from '../src/lib/cluster/node_master.js';
5
5
  process.env.assignment = 'node_master';
@@ -1,6 +1,6 @@
1
1
  import { jest } from '@jest/globals';
2
2
  import events from 'node:events';
3
- import { debugLogger } from '@terascope/utils';
3
+ import { debugLogger } from '@terascope/core-utils';
4
4
  import { Messaging, routing } from '../../src/lib/cluster/services/cluster/backends/native/messaging.js';
5
5
  describe('messaging module', () => {
6
6
  const logger = debugLogger('messaging');
@@ -1,5 +1,5 @@
1
1
  import { jest } from '@jest/globals';
2
- import { pDelay, get, set } from '@terascope/utils';
2
+ import { pDelay, get, set } from '@terascope/core-utils';
3
3
  import { TestContext } from '../helpers/index.js';
4
4
  import { findPort } from '../../../src/lib/utils/port_utils.js';
5
5
  import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
@@ -1,5 +1,5 @@
1
1
  import { v4 as uuidv4 } from 'uuid';
2
- import { pDelay, times, random } from '@terascope/utils';
2
+ import { pDelay, times, random } from '@terascope/core-utils';
3
3
  import { RecoveryCleanupType } from '@terascope/job-components';
4
4
  import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
5
5
  import { TestContext } from '../helpers/index.js';
@@ -1,9 +1,10 @@
1
- import { pDelay, times, random, isNotNil } from '@terascope/utils';
1
+ import { pDelay, times, random, isNotNil } from '@terascope/core-utils';
2
2
  import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
3
3
  import { TestContext } from '../helpers/index.js';
4
4
  import { getTestCases } from '../helpers/execution-controller-helper.js';
5
5
  import { findPort } from '../../../src/lib/utils/port_utils.js';
6
6
  import { newId } from '../../../src/lib/utils/id_utils.js';
7
+ import { getPackageJSON } from '../../../src/lib/utils/file_utils.js';
7
8
  import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
8
9
  const ExecutionControllerClient = ExController.Client;
9
10
  describe('ExecutionController Test Cases', () => {
@@ -333,6 +334,10 @@ describe('ExecutionController Test Cases', () => {
333
334
  const metadata = updateMetadata ? { slice_calls: count + 1 } : {};
334
335
  expect(executionRecord).toHaveProperty('metadata', metadata);
335
336
  });
337
+ it('should have the correct teraslice version', () => {
338
+ const terasliceVersion = `v${getPackageJSON().version}`;
339
+ expect(executionRecord.teraslice_version).toBe(terasliceVersion);
340
+ });
336
341
  });
337
342
  });
338
343
  //# sourceMappingURL=execution-test-cases-spec.js.map
@@ -1,4 +1,4 @@
1
- import { pDelay, pWhile, debugLogger } from '@terascope/utils';
1
+ import { pDelay, pWhile, debugLogger } from '@terascope/core-utils';
2
2
  import { EventEmitter } from 'node:events';
3
3
  import { RecoveryModule } from '../../../src/lib/workers/execution-controller/recovery.js';
4
4
  const eventEmitter = new EventEmitter();
@@ -1,6 +1,6 @@
1
1
  import { jest } from '@jest/globals';
2
2
  import { v4 as uuidv4 } from 'uuid';
3
- import { pDelay, once, times } from '@terascope/utils';
3
+ import { pDelay, once, times } from '@terascope/core-utils';
4
4
  import { TestContext } from '../helpers/test-context.js';
5
5
  import { Scheduler } from '../../../src/lib/workers/execution-controller/scheduler.js';
6
6
  describe('Scheduler', () => {
@@ -1,5 +1,5 @@
1
1
  import { ParallelSlicer } from '@terascope/job-components';
2
- import { get } from '@terascope/utils';
2
+ import { get } from '@terascope/core-utils';
3
3
  const defaultSlicerResults = [{ howdy: true }, null];
4
4
  export default class ExampleSlicer extends ParallelSlicer {
5
5
  async newSlicer(_id) {
@@ -1,5 +1,5 @@
1
1
  import { ParallelSlicer } from '@terascope/job-components';
2
- import { pDelay, times } from '@terascope/utils';
2
+ import { pDelay, times } from '@terascope/core-utils';
3
3
  export default class ExampleSlicer extends ParallelSlicer {
4
4
  async newSlicer(id) {
5
5
  const { countPerSlicer } = this.opConfig;
@@ -1,6 +1,6 @@
1
1
  import path from 'node:path';
2
2
  import { fileURLToPath } from 'node:url';
3
- import { withoutNil } from '@terascope/utils';
3
+ import { withoutNil } from '@terascope/core-utils';
4
4
  // @ts-expect-error
5
5
  import Chance from 'chance';
6
6
  import { newId } from '../../../src/lib/utils/id_utils.js';
@@ -1,4 +1,4 @@
1
- import { pDelay, pDefer } from '@terascope/utils';
1
+ import { pDelay, pDefer } from '@terascope/core-utils';
2
2
  export function makeShutdownEarlyFn({ exController, enabled = false }) {
3
3
  let shutdownErr = {
4
4
  message: 'Shutdown never triggered'
@@ -2,7 +2,7 @@ import fs from 'fs-extra';
2
2
  import path from 'node:path';
3
3
  import { createTempDirSync, cleanupTempDirs } from 'jest-fixtures';
4
4
  import { newTestSlice } from '@terascope/job-components';
5
- import { get, pWhile } from '@terascope/utils';
5
+ import { get, pWhile } from '@terascope/core-utils';
6
6
  import { ClusterMaster } from '@terascope/teraslice-messaging';
7
7
  import { AssetsStorage, StateStorage, AnalyticsStorage, ExecutionStorage, JobsStorage } from '../../../src/lib/storage/index.js';
8
8
  import { initializeTestExecution } from '../../../src/lib/workers/helpers/job.js';
@@ -1,5 +1,5 @@
1
1
  import path from 'node:path';
2
- import { random } from '@terascope/utils';
2
+ import { random } from '@terascope/core-utils';
3
3
  // @ts-expect-error
4
4
  import BufferStreams from 'bufferstreams';
5
5
  import archiver from 'archiver';
@@ -1,5 +1,5 @@
1
1
  import { jest } from '@jest/globals';
2
- import { times } from '@terascope/utils';
2
+ import { times } from '@terascope/core-utils';
3
3
  import { SliceExecution } from '../../../src/lib/workers/worker/slice.js';
4
4
  import { TestContext } from '../helpers/index.js';
5
5
  describe('Slice', () => {
@@ -1,6 +1,6 @@
1
1
  import { jest } from '@jest/globals';
2
2
  import 'jest-extended';
3
- import { pDelay } from '@terascope/utils';
3
+ import { pDelay } from '@terascope/core-utils';
4
4
  import { ExecutionController } from '@terascope/teraslice-messaging';
5
5
  import { findPort } from '../../../src/lib/utils/port_utils.js';
6
6
  import { Worker } from '../../../src/lib/workers/worker/index.js';
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "teraslice",
3
3
  "displayName": "Teraslice",
4
- "version": "3.0.0-dev.7",
4
+ "version": "3.0.0-dev.9",
5
5
  "description": "Distributed computing platform for processing JSON data",
6
6
  "homepage": "https://github.com/terascope/teraslice#readme",
7
7
  "bugs": {
@@ -39,11 +39,11 @@
39
39
  },
40
40
  "dependencies": {
41
41
  "@kubernetes/client-node": "~1.3.0",
42
- "@terascope/elasticsearch-api": "~5.0.0-dev.6",
43
- "@terascope/job-components": "~2.0.0-dev.6",
44
- "@terascope/teraslice-messaging": "~2.0.0-dev.7",
45
- "@terascope/types": "~2.0.0-dev.6",
46
- "@terascope/utils": "~2.0.0-dev.6",
42
+ "@terascope/core-utils": "~2.0.0-dev.8",
43
+ "@terascope/elasticsearch-api": "~5.0.0-dev.8",
44
+ "@terascope/job-components": "~2.0.0-dev.8",
45
+ "@terascope/teraslice-messaging": "~2.0.0-dev.9",
46
+ "@terascope/types": "~2.0.0-dev.7",
47
47
  "async-mutex": "~0.5.0",
48
48
  "barbe": "~3.0.17",
49
49
  "body-parser": "~2.2.0",
@@ -61,11 +61,11 @@
61
61
  "semver": "~7.7.2",
62
62
  "socket.io": "~4.8.1",
63
63
  "socket.io-client": "~4.8.1",
64
- "terafoundation": "~2.0.0-dev.6",
64
+ "terafoundation": "~2.0.0-dev.8",
65
65
  "uuid": "~13.0.0"
66
66
  },
67
67
  "devDependencies": {
68
- "@terascope/opensearch-client": "~2.0.0-dev.6",
68
+ "@terascope/opensearch-client": "~2.0.0-dev.8",
69
69
  "@types/archiver": "~6.0.3",
70
70
  "@types/express": "~5.0.3",
71
71
  "@types/gc-stats": "~1.4.3",
package/worker-service.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  logError, get, isEmpty, isPlainObject
3
- } from '@terascope/utils';
3
+ } from '@terascope/core-utils';
4
4
  import { shutdownHandler } from './dist/src/lib/workers/helpers/worker-shutdown.js';
5
5
  import { safeDecode } from './dist/src/lib/utils/encoding_utils.js';
6
6
  import { makeExecutionContext } from './dist/src/lib/workers/context/execution-context.js';