teraslice 3.0.0-dev.1 → 3.0.0-dev.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cluster-service.js +1 -1
- package/dist/src/lib/cluster/cluster_master.js +1 -1
- package/dist/src/lib/cluster/node_master.js +1 -1
- package/dist/src/lib/cluster/services/api.js +2 -2
- package/dist/src/lib/cluster/services/assets.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/native/index.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js +6 -5
- package/dist/src/lib/cluster/services/cluster/backends/state-utils.js +1 -1
- package/dist/src/lib/cluster/services/execution.js +1 -1
- package/dist/src/lib/cluster/services/jobs.js +1 -1
- package/dist/src/lib/config/default-sysconfig.js +0 -1
- package/dist/src/lib/config/index.js +1 -1
- package/dist/src/lib/config/schemas/system.js +2 -2
- package/dist/src/lib/storage/analytics.js +1 -1
- package/dist/src/lib/storage/assets.js +1 -1
- package/dist/src/lib/storage/backends/elasticsearch_store.js +7 -37
- package/dist/src/lib/storage/backends/mappings/analytics.js +37 -42
- package/dist/src/lib/storage/backends/mappings/asset.js +29 -34
- package/dist/src/lib/storage/backends/mappings/ex.js +51 -53
- package/dist/src/lib/storage/backends/mappings/job.js +21 -26
- package/dist/src/lib/storage/backends/mappings/state.js +24 -29
- package/dist/src/lib/storage/backends/s3_store.js +1 -1
- package/dist/src/lib/storage/execution.js +4 -2
- package/dist/src/lib/storage/jobs.js +1 -1
- package/dist/src/lib/storage/state.js +1 -2
- package/dist/src/lib/utils/api_utils.js +1 -1
- package/dist/src/lib/utils/asset_utils.js +2 -2
- package/dist/src/lib/utils/date_utils.js +1 -1
- package/dist/src/lib/utils/encoding_utils.js +1 -1
- package/dist/src/lib/utils/file_utils.js +1 -1
- package/dist/src/lib/utils/id_utils.js +1 -1
- package/dist/src/lib/utils/port_utils.js +1 -1
- package/dist/src/lib/workers/assets/loader-executable.js +1 -1
- package/dist/src/lib/workers/assets/loader.js +1 -1
- package/dist/src/lib/workers/assets/spawn.js +1 -1
- package/dist/src/lib/workers/execution-controller/execution-analytics.js +1 -1
- package/dist/src/lib/workers/execution-controller/index.js +1 -1
- package/dist/src/lib/workers/execution-controller/recovery.js +1 -1
- package/dist/src/lib/workers/execution-controller/scheduler.js +1 -1
- package/dist/src/lib/workers/execution-controller/slice-analytics.js +1 -1
- package/dist/src/lib/workers/helpers/job.js +1 -1
- package/dist/src/lib/workers/helpers/op-analytics.js +1 -1
- package/dist/src/lib/workers/helpers/terafoundation.js +1 -1
- package/dist/src/lib/workers/helpers/worker-shutdown.js +1 -1
- package/dist/src/lib/workers/metrics/index.js +1 -1
- package/dist/src/lib/workers/worker/index.js +1 -1
- package/dist/src/lib/workers/worker/slice.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js +1 -1
- package/dist/test/node_master-spec.js +3 -3
- package/dist/test/services/messaging-spec.js +1 -1
- package/dist/test/workers/execution-controller/execution-controller-spec.js +1 -1
- package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js +1 -2
- package/dist/test/workers/execution-controller/execution-test-cases-spec.js +6 -1
- package/dist/test/workers/execution-controller/recovery-spec.js +1 -1
- package/dist/test/workers/execution-controller/scheduler-spec.js +1 -1
- package/dist/test/workers/fixtures/ops/example-reader/slicer.js +1 -1
- package/dist/test/workers/fixtures/ops/new-reader/slicer.js +1 -1
- package/dist/test/workers/helpers/configs.js +1 -2
- package/dist/test/workers/helpers/execution-controller-helper.js +1 -1
- package/dist/test/workers/helpers/test-context.js +1 -1
- package/dist/test/workers/helpers/zip-directory.js +1 -1
- package/dist/test/workers/worker/slice-spec.js +1 -1
- package/dist/test/workers/worker/worker-spec.js +1 -1
- package/package.json +11 -12
- package/worker-service.js +1 -1
|
@@ -4,33 +4,28 @@ export default {
|
|
|
4
4
|
'index.number_of_replicas': 1
|
|
5
5
|
},
|
|
6
6
|
mappings: {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
dynamic: false,
|
|
8
|
+
properties: {
|
|
9
|
+
active: {
|
|
10
|
+
type: 'boolean'
|
|
10
11
|
},
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
_deleted: {
|
|
29
|
-
type: 'boolean'
|
|
30
|
-
},
|
|
31
|
-
_deleted_on: {
|
|
32
|
-
type: 'date'
|
|
33
|
-
}
|
|
12
|
+
job_id: {
|
|
13
|
+
type: 'keyword'
|
|
14
|
+
},
|
|
15
|
+
_context: {
|
|
16
|
+
type: 'keyword'
|
|
17
|
+
},
|
|
18
|
+
_created: {
|
|
19
|
+
type: 'date'
|
|
20
|
+
},
|
|
21
|
+
_updated: {
|
|
22
|
+
type: 'date'
|
|
23
|
+
},
|
|
24
|
+
_deleted: {
|
|
25
|
+
type: 'boolean'
|
|
26
|
+
},
|
|
27
|
+
_deleted_on: {
|
|
28
|
+
type: 'date'
|
|
34
29
|
}
|
|
35
30
|
}
|
|
36
31
|
}
|
|
@@ -1,36 +1,31 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
template: '__state*',
|
|
3
3
|
mappings: {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
dynamic: false,
|
|
5
|
+
properties: {
|
|
6
|
+
ex_id: {
|
|
7
|
+
type: 'keyword'
|
|
7
8
|
},
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
_updated: {
|
|
29
|
-
type: 'date'
|
|
30
|
-
},
|
|
31
|
-
error: {
|
|
32
|
-
type: 'keyword'
|
|
33
|
-
}
|
|
9
|
+
slice_id: {
|
|
10
|
+
type: 'keyword'
|
|
11
|
+
},
|
|
12
|
+
slicer_id: {
|
|
13
|
+
type: 'keyword'
|
|
14
|
+
},
|
|
15
|
+
slicer_order: {
|
|
16
|
+
type: 'integer'
|
|
17
|
+
},
|
|
18
|
+
state: {
|
|
19
|
+
type: 'keyword'
|
|
20
|
+
},
|
|
21
|
+
_created: {
|
|
22
|
+
type: 'date'
|
|
23
|
+
},
|
|
24
|
+
_updated: {
|
|
25
|
+
type: 'date'
|
|
26
|
+
},
|
|
27
|
+
error: {
|
|
28
|
+
type: 'keyword'
|
|
34
29
|
}
|
|
35
30
|
}
|
|
36
31
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/utils';
|
|
1
|
+
import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/core-utils';
|
|
2
2
|
import { S3ClientResponse, createS3Bucket, deleteS3Object, doesBucketExist, getS3Object, listS3Objects, putS3Object, s3RequestWithRetry } from '@terascope/file-asset-apis';
|
|
3
3
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
4
4
|
export class S3Store {
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { TSError, includes, getTypeOf, makeISODate } from '@terascope/utils';
|
|
1
|
+
import { TSError, includes, getTypeOf, makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
3
3
|
import { v4 as uuid } from 'uuid';
|
|
4
4
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
5
5
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
6
|
+
import { getPackageJSON } from '../utils/file_utils.js';
|
|
6
7
|
const INIT_STATUS = ['pending', 'scheduling', 'initializing'];
|
|
7
8
|
const RUNNING_STATUS = ['recovering', 'running', 'failing', 'paused', 'stopping'];
|
|
8
9
|
const TERMINAL_STATUS = ['completed', 'stopped', 'rejected', 'failed', 'terminated'];
|
|
@@ -78,7 +79,8 @@ export class ExecutionStorage {
|
|
|
78
79
|
_updated: date,
|
|
79
80
|
_has_errors: false,
|
|
80
81
|
_slicer_stats: {},
|
|
81
|
-
_failureReason: ''
|
|
82
|
+
_failureReason: '',
|
|
83
|
+
teraslice_version: `v${getPackageJSON().version}`
|
|
82
84
|
});
|
|
83
85
|
// @ts-expect-error
|
|
84
86
|
delete doc.slicer_port;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { v4 as uuid } from 'uuid';
|
|
2
|
-
import { TSError, makeISODate } from '@terascope/utils';
|
|
2
|
+
import { TSError, makeISODate } from '@terascope/core-utils';
|
|
3
3
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
4
4
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
5
5
|
export class JobsStorage {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
2
|
-
import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/utils';
|
|
2
|
+
import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/core-utils';
|
|
3
3
|
import { timeseriesIndex } from '../utils/date_utils.js';
|
|
4
4
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
5
5
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
@@ -57,7 +57,6 @@ export class StateStorage {
|
|
|
57
57
|
action: {
|
|
58
58
|
index: {
|
|
59
59
|
_index: index,
|
|
60
|
-
_type: this.recordType,
|
|
61
60
|
_id: record.slice_id,
|
|
62
61
|
},
|
|
63
62
|
},
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import Table from 'easy-table';
|
|
2
|
-
import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/utils';
|
|
2
|
+
import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/core-utils';
|
|
3
3
|
export function makeTable(req, defaults, data, mappingFn) {
|
|
4
4
|
const query = fieldsQuery(req.query, defaults);
|
|
5
5
|
let emptyChar = 'N/A';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/utils';
|
|
1
|
+
import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/core-utils';
|
|
2
2
|
import * as semver from 'semver';
|
|
3
3
|
export function findMatchingAsset(records, name, version) {
|
|
4
4
|
const range = toSemverRange(version);
|
|
@@ -54,7 +54,7 @@ function _isCompatibleAsset(name, range, skipRestrictions = false) {
|
|
|
54
54
|
return function checkIfCompatibleAsset(record) {
|
|
55
55
|
if (record.name !== name)
|
|
56
56
|
return false;
|
|
57
|
-
if (!semver.satisfies(record.version, range)) {
|
|
57
|
+
if (!semver.satisfies(record.version, range, { includePrerelease: true })) {
|
|
58
58
|
return false;
|
|
59
59
|
}
|
|
60
60
|
if (skipRestrictions)
|
|
@@ -3,7 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import fse from 'fs-extra';
|
|
4
4
|
import semver from 'semver';
|
|
5
5
|
import { Mutex } from 'async-mutex';
|
|
6
|
-
import { TSError } from '@terascope/utils';
|
|
6
|
+
import { TSError } from '@terascope/core-utils';
|
|
7
7
|
import decompress from 'decompress';
|
|
8
8
|
import { getMajorVersion } from './asset_utils.js';
|
|
9
9
|
const mutex = new Mutex();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { trim, padEnd } from '@terascope/utils';
|
|
1
|
+
import { trim, padEnd } from '@terascope/core-utils';
|
|
2
2
|
import { customAlphabet } from 'nanoid';
|
|
3
3
|
export function newId(prefix, lowerCase = false, length = 15) {
|
|
4
4
|
let characters = '-0123456789abcdefghijklmnopqrstuvwxyz';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getFullErrorStack, pDelay } from '@terascope/utils';
|
|
1
|
+
import { getFullErrorStack, pDelay } from '@terascope/core-utils';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { AssetLoader } from './loader.js';
|
|
4
4
|
import { makeTerafoundationContext } from '../context/terafoundation-context.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import fs from 'node:fs';
|
|
3
|
-
import { get, isEmpty } from '@terascope/utils';
|
|
3
|
+
import { get, isEmpty } from '@terascope/core-utils';
|
|
4
4
|
import { AssetsStorage } from '../../storage/index.js';
|
|
5
5
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
6
6
|
import { saveAsset } from '../../utils/file_utils.js';
|
|
@@ -2,7 +2,7 @@ import fs from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { fork } from 'node:child_process';
|
|
5
|
-
import { isEmpty, get, has } from '@terascope/utils';
|
|
5
|
+
import { isEmpty, get, has } from '@terascope/core-utils';
|
|
6
6
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
7
7
|
import { safeEncode } from '../../utils/encoding_utils.js';
|
|
8
8
|
const filePath = fileURLToPath(new URL('.', import.meta.url));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import ms from 'ms';
|
|
2
2
|
import { formatURL, ExecutionController as ExController, ClusterMaster } from '@terascope/teraslice-messaging';
|
|
3
|
-
import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile,
|
|
3
|
+
import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile, debounce, throttle, makeISODate } from '@terascope/core-utils';
|
|
4
4
|
import { isPromAvailable } from '@terascope/job-components';
|
|
5
5
|
import { waitForWorkerShutdown } from '../helpers/worker-shutdown.js';
|
|
6
6
|
import { StateStorage, ExecutionStorage, SliceState } from '../../storage/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { noop, pDelay, get, toString, logError, pWhile, Queue, makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { RecoveryModule } from './recovery.js';
|
|
3
3
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
4
4
|
export class Scheduler {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { makeISODate } from '@terascope/utils';
|
|
1
|
+
import { makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { JobValidator } from '@terascope/job-components';
|
|
3
3
|
import { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
|
|
4
4
|
// TODO: fix type here
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { get, isFunction, isString } from '@terascope/utils';
|
|
1
|
+
import { get, isFunction, isString } from '@terascope/core-utils';
|
|
2
2
|
import { makeContextLogger } from '@terascope/job-components';
|
|
3
3
|
import { safeDecode } from '../../utils/encoding_utils.js';
|
|
4
4
|
export function generateWorkerId(context) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/utils';
|
|
1
|
+
import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/core-utils';
|
|
2
2
|
import ms from 'ms';
|
|
3
3
|
import { makeLogger } from './terafoundation.js';
|
|
4
4
|
export function waitForWorkerShutdown(context, eventName) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/* eslint-disable prefer-const */
|
|
2
|
-
import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/utils';
|
|
2
|
+
import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/core-utils';
|
|
3
3
|
import { ExecutionController, formatURL } from '@terascope/teraslice-messaging';
|
|
4
4
|
import { isPromAvailable } from '@terascope/job-components';
|
|
5
5
|
import { StateStorage, AnalyticsStorage } from '../../storage/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, getTypeOf, logError } from '@terascope/utils';
|
|
1
|
+
import { TSError, getTypeOf, logError } from '@terascope/core-utils';
|
|
2
2
|
import { SliceState } from '../../storage/index.js';
|
|
3
3
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
4
4
|
import { logOpStats } from '../helpers/op-analytics.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { V1Deployment, V1Job, V1Pod, V1ReplicaSet, V1Service } from '@kubernetes/client-node';
|
|
2
2
|
import nock from 'nock';
|
|
3
|
-
import { debugLogger } from '@terascope/
|
|
3
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
4
4
|
import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js';
|
|
5
5
|
const logger = debugLogger('k8s-v2-spec');
|
|
6
6
|
const _url = 'http://mock.kube.api';
|
package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import yaml from 'js-yaml';
|
|
2
|
-
import { debugLogger } from '@terascope/utils';
|
|
2
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
3
3
|
import { K8sDeploymentResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js';
|
|
4
4
|
import { K8sJobResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js';
|
|
5
5
|
import { K8sServiceResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cloneDeep } from '@terascope/utils';
|
|
1
|
+
import { cloneDeep } from '@terascope/core-utils';
|
|
2
2
|
import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json';
|
|
3
3
|
import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
|
|
4
4
|
describe('k8sState with pods from multiple clusters', () => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cloneDeep } from '@terascope/utils';
|
|
1
|
+
import { cloneDeep } from '@terascope/core-utils';
|
|
2
2
|
import _podsJobRunning from '../files/job-running-v1-k8s-pods.json';
|
|
3
3
|
import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
|
|
4
4
|
describe('k8sState', () => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import { debugLogger } from '@terascope/utils';
|
|
3
|
-
import
|
|
2
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
3
|
+
import { Server as SocketServer } from 'socket.io';
|
|
4
4
|
import { nodeMaster } from '../src/lib/cluster/node_master.js';
|
|
5
5
|
process.env.assignment = 'node_master';
|
|
6
6
|
describe('Node master', () => {
|
|
@@ -49,7 +49,7 @@ describe('Node master', () => {
|
|
|
49
49
|
__test_job: JSON.stringify(jobJson),
|
|
50
50
|
__test_assignment: 'worker'
|
|
51
51
|
};
|
|
52
|
-
const fakeClusterMaster =
|
|
52
|
+
const fakeClusterMaster = new SocketServer({
|
|
53
53
|
path: '/native-clustering'
|
|
54
54
|
});
|
|
55
55
|
fakeClusterMaster.on('connection', (socket) => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
import events from 'node:events';
|
|
3
|
-
import { debugLogger } from '@terascope/utils';
|
|
3
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
4
4
|
import { Messaging, routing } from '../../src/lib/cluster/services/cluster/backends/native/messaging.js';
|
|
5
5
|
describe('messaging module', () => {
|
|
6
6
|
const logger = debugLogger('messaging');
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
|
-
import { pDelay, get, set } from '@terascope/utils';
|
|
2
|
+
import { pDelay, get, set } from '@terascope/core-utils';
|
|
3
3
|
import { TestContext } from '../helpers/index.js';
|
|
4
4
|
import { findPort } from '../../../src/lib/utils/port_utils.js';
|
|
5
5
|
import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { v4 as uuidv4 } from 'uuid';
|
|
2
|
-
import { pDelay, times, random } from '@terascope/utils';
|
|
2
|
+
import { pDelay, times, random } from '@terascope/core-utils';
|
|
3
3
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
4
4
|
import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
|
|
5
5
|
import { TestContext } from '../helpers/index.js';
|
|
@@ -8,7 +8,6 @@ import { ExecutionController } from '../../../src/lib/workers/execution-controll
|
|
|
8
8
|
import { findPort } from '../../../src/lib/utils/port_utils.js';
|
|
9
9
|
import { newId } from '../../../src/lib/utils/id_utils.js';
|
|
10
10
|
const ExecutionControllerClient = ExController.Client;
|
|
11
|
-
process.env.BLUEBIRD_LONG_STACK_TRACES = '1';
|
|
12
11
|
describe('ExecutionController Special Tests', () => {
|
|
13
12
|
// [ message, config ]
|
|
14
13
|
const testCases = [
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { pDelay, times, random, isNotNil } from '@terascope/utils';
|
|
1
|
+
import { pDelay, times, random, isNotNil } from '@terascope/core-utils';
|
|
2
2
|
import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
|
|
3
3
|
import { TestContext } from '../helpers/index.js';
|
|
4
4
|
import { getTestCases } from '../helpers/execution-controller-helper.js';
|
|
5
5
|
import { findPort } from '../../../src/lib/utils/port_utils.js';
|
|
6
6
|
import { newId } from '../../../src/lib/utils/id_utils.js';
|
|
7
|
+
import { getPackageJSON } from '../../../src/lib/utils/file_utils.js';
|
|
7
8
|
import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
|
|
8
9
|
const ExecutionControllerClient = ExController.Client;
|
|
9
10
|
describe('ExecutionController Test Cases', () => {
|
|
@@ -333,6 +334,10 @@ describe('ExecutionController Test Cases', () => {
|
|
|
333
334
|
const metadata = updateMetadata ? { slice_calls: count + 1 } : {};
|
|
334
335
|
expect(executionRecord).toHaveProperty('metadata', metadata);
|
|
335
336
|
});
|
|
337
|
+
it('should have the correct teraslice version', () => {
|
|
338
|
+
const terasliceVersion = `v${getPackageJSON().version}`;
|
|
339
|
+
expect(executionRecord.teraslice_version).toBe(terasliceVersion);
|
|
340
|
+
});
|
|
336
341
|
});
|
|
337
342
|
});
|
|
338
343
|
//# sourceMappingURL=execution-test-cases-spec.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { pDelay, pWhile, debugLogger } from '@terascope/utils';
|
|
1
|
+
import { pDelay, pWhile, debugLogger } from '@terascope/core-utils';
|
|
2
2
|
import { EventEmitter } from 'node:events';
|
|
3
3
|
import { RecoveryModule } from '../../../src/lib/workers/execution-controller/recovery.js';
|
|
4
4
|
const eventEmitter = new EventEmitter();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
|
-
import { pDelay, once, times } from '@terascope/utils';
|
|
3
|
+
import { pDelay, once, times } from '@terascope/core-utils';
|
|
4
4
|
import { TestContext } from '../helpers/test-context.js';
|
|
5
5
|
import { Scheduler } from '../../../src/lib/workers/execution-controller/scheduler.js';
|
|
6
6
|
describe('Scheduler', () => {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ParallelSlicer } from '@terascope/job-components';
|
|
2
|
-
import { get } from '@terascope/utils';
|
|
2
|
+
import { get } from '@terascope/core-utils';
|
|
3
3
|
const defaultSlicerResults = [{ howdy: true }, null];
|
|
4
4
|
export default class ExampleSlicer extends ParallelSlicer {
|
|
5
5
|
async newSlicer(_id) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ParallelSlicer } from '@terascope/job-components';
|
|
2
|
-
import { pDelay, times } from '@terascope/utils';
|
|
2
|
+
import { pDelay, times } from '@terascope/core-utils';
|
|
3
3
|
export default class ExampleSlicer extends ParallelSlicer {
|
|
4
4
|
async newSlicer(id) {
|
|
5
5
|
const { countPerSlicer } = this.opConfig;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
|
-
import { withoutNil } from '@terascope/utils';
|
|
3
|
+
import { withoutNil } from '@terascope/core-utils';
|
|
4
4
|
// @ts-expect-error
|
|
5
5
|
import Chance from 'chance';
|
|
6
6
|
import { newId } from '../../../src/lib/utils/id_utils.js';
|
|
@@ -75,7 +75,6 @@ const newSysConfig = (options) => {
|
|
|
75
75
|
const { clusterName = 'test-teraslice-cluster', timeout = 3000, actionTimeout = 2000, shutdownTimeout = 4000, assetDir, clusterMasterPort, log_level_terafoundation = 'info', } = options;
|
|
76
76
|
return {
|
|
77
77
|
terafoundation: {
|
|
78
|
-
environment: 'development',
|
|
79
78
|
log_level: log_level_terafoundation,
|
|
80
79
|
connectors: {
|
|
81
80
|
'elasticsearch-next': {
|
|
@@ -2,7 +2,7 @@ import fs from 'fs-extra';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { createTempDirSync, cleanupTempDirs } from 'jest-fixtures';
|
|
4
4
|
import { newTestSlice } from '@terascope/job-components';
|
|
5
|
-
import { get, pWhile } from '@terascope/utils';
|
|
5
|
+
import { get, pWhile } from '@terascope/core-utils';
|
|
6
6
|
import { ClusterMaster } from '@terascope/teraslice-messaging';
|
|
7
7
|
import { AssetsStorage, StateStorage, AnalyticsStorage, ExecutionStorage, JobsStorage } from '../../../src/lib/storage/index.js';
|
|
8
8
|
import { initializeTestExecution } from '../../../src/lib/workers/helpers/job.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
|
-
import { times } from '@terascope/utils';
|
|
2
|
+
import { times } from '@terascope/core-utils';
|
|
3
3
|
import { SliceExecution } from '../../../src/lib/workers/worker/slice.js';
|
|
4
4
|
import { TestContext } from '../helpers/index.js';
|
|
5
5
|
describe('Slice', () => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
import 'jest-extended';
|
|
3
|
-
import { pDelay } from '@terascope/utils';
|
|
3
|
+
import { pDelay } from '@terascope/core-utils';
|
|
4
4
|
import { ExecutionController } from '@terascope/teraslice-messaging';
|
|
5
5
|
import { findPort } from '../../../src/lib/utils/port_utils.js';
|
|
6
6
|
import { Worker } from '../../../src/lib/workers/worker/index.js';
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "teraslice",
|
|
3
3
|
"displayName": "Teraslice",
|
|
4
|
-
"version": "3.0.0-dev.
|
|
4
|
+
"version": "3.0.0-dev.11",
|
|
5
5
|
"description": "Distributed computing platform for processing JSON data",
|
|
6
6
|
"homepage": "https://github.com/terascope/teraslice#readme",
|
|
7
7
|
"bugs": {
|
|
@@ -27,9 +27,8 @@
|
|
|
27
27
|
"start": "node service.js",
|
|
28
28
|
"test": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
29
29
|
"test:debug": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --debug ../teraslice --",
|
|
30
|
-
"test:elasticsearch6": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='6.8.6' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
31
30
|
"test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
32
|
-
"test:opensearch1": "TEST_OPENSEARCH='true' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
31
|
+
"test:opensearch1": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='1.3.11' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
33
32
|
"test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
34
33
|
"test:opensearch3": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='3.1.0' node ../scripts/bin/ts-scripts test ../teraslice --",
|
|
35
34
|
"test:watch": "TEST_ELASTICSEARCH='true' node ../scripts/bin/ts-scripts test --watch ../teraslice --"
|
|
@@ -40,11 +39,11 @@
|
|
|
40
39
|
},
|
|
41
40
|
"dependencies": {
|
|
42
41
|
"@kubernetes/client-node": "~1.3.0",
|
|
43
|
-
"@terascope/
|
|
44
|
-
"@terascope/
|
|
45
|
-
"@terascope/
|
|
46
|
-
"@terascope/
|
|
47
|
-
"@terascope/
|
|
42
|
+
"@terascope/core-utils": "~2.0.0-dev.8",
|
|
43
|
+
"@terascope/elasticsearch-api": "~5.0.0-dev.9",
|
|
44
|
+
"@terascope/job-components": "~2.0.0-dev.9",
|
|
45
|
+
"@terascope/teraslice-messaging": "~2.0.0-dev.9",
|
|
46
|
+
"@terascope/types": "~2.0.0-dev.7",
|
|
48
47
|
"async-mutex": "~0.5.0",
|
|
49
48
|
"barbe": "~3.0.17",
|
|
50
49
|
"body-parser": "~2.2.0",
|
|
@@ -60,13 +59,13 @@
|
|
|
60
59
|
"ms": "~2.1.3",
|
|
61
60
|
"nanoid": "~5.1.5",
|
|
62
61
|
"semver": "~7.7.2",
|
|
63
|
-
"socket.io": "~
|
|
64
|
-
"socket.io-client": "~
|
|
65
|
-
"terafoundation": "~2.0.0-dev.
|
|
62
|
+
"socket.io": "~4.8.1",
|
|
63
|
+
"socket.io-client": "~4.8.1",
|
|
64
|
+
"terafoundation": "~2.0.0-dev.10",
|
|
66
65
|
"uuid": "~13.0.0"
|
|
67
66
|
},
|
|
68
67
|
"devDependencies": {
|
|
69
|
-
"@terascope/opensearch-client": "~2.0.0-dev.
|
|
68
|
+
"@terascope/opensearch-client": "~2.0.0-dev.8",
|
|
70
69
|
"@types/archiver": "~6.0.3",
|
|
71
70
|
"@types/express": "~5.0.3",
|
|
72
71
|
"@types/gc-stats": "~1.4.3",
|