teraslice 3.0.0-dev.7 → 3.0.0-dev.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cluster-service.js +1 -1
- package/dist/src/lib/cluster/cluster_master.js +1 -1
- package/dist/src/lib/cluster/node_master.js +1 -1
- package/dist/src/lib/cluster/services/api.js +1 -1
- package/dist/src/lib/cluster/services/assets.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/native/index.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js +1 -1
- package/dist/src/lib/cluster/services/cluster/backends/state-utils.js +1 -1
- package/dist/src/lib/cluster/services/execution.js +1 -1
- package/dist/src/lib/cluster/services/jobs.js +1 -1
- package/dist/src/lib/config/index.js +1 -1
- package/dist/src/lib/config/schemas/system.js +1 -1
- package/dist/src/lib/storage/analytics.js +1 -1
- package/dist/src/lib/storage/assets.js +1 -1
- package/dist/src/lib/storage/backends/elasticsearch_store.js +1 -1
- package/dist/src/lib/storage/backends/s3_store.js +1 -1
- package/dist/src/lib/storage/execution.js +1 -1
- package/dist/src/lib/storage/jobs.js +1 -1
- package/dist/src/lib/storage/state.js +1 -1
- package/dist/src/lib/utils/api_utils.js +1 -1
- package/dist/src/lib/utils/asset_utils.js +1 -1
- package/dist/src/lib/utils/date_utils.js +1 -1
- package/dist/src/lib/utils/encoding_utils.js +1 -1
- package/dist/src/lib/utils/file_utils.js +1 -1
- package/dist/src/lib/utils/id_utils.js +1 -1
- package/dist/src/lib/utils/port_utils.js +1 -1
- package/dist/src/lib/workers/assets/loader-executable.js +1 -1
- package/dist/src/lib/workers/assets/loader.js +1 -1
- package/dist/src/lib/workers/assets/spawn.js +1 -1
- package/dist/src/lib/workers/execution-controller/execution-analytics.js +1 -1
- package/dist/src/lib/workers/execution-controller/index.js +1 -1
- package/dist/src/lib/workers/execution-controller/recovery.js +1 -1
- package/dist/src/lib/workers/execution-controller/scheduler.js +1 -1
- package/dist/src/lib/workers/execution-controller/slice-analytics.js +1 -1
- package/dist/src/lib/workers/helpers/job.js +1 -1
- package/dist/src/lib/workers/helpers/op-analytics.js +1 -1
- package/dist/src/lib/workers/helpers/terafoundation.js +1 -1
- package/dist/src/lib/workers/helpers/worker-shutdown.js +1 -1
- package/dist/src/lib/workers/metrics/index.js +1 -1
- package/dist/src/lib/workers/worker/index.js +1 -1
- package/dist/src/lib/workers/worker/slice.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js +1 -1
- package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js +1 -1
- package/dist/test/node_master-spec.js +1 -1
- package/dist/test/services/messaging-spec.js +1 -1
- package/dist/test/workers/execution-controller/execution-controller-spec.js +1 -1
- package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js +1 -1
- package/dist/test/workers/execution-controller/execution-test-cases-spec.js +1 -1
- package/dist/test/workers/execution-controller/recovery-spec.js +1 -1
- package/dist/test/workers/execution-controller/scheduler-spec.js +1 -1
- package/dist/test/workers/fixtures/ops/example-reader/slicer.js +1 -1
- package/dist/test/workers/fixtures/ops/new-reader/slicer.js +1 -1
- package/dist/test/workers/helpers/configs.js +1 -1
- package/dist/test/workers/helpers/execution-controller-helper.js +1 -1
- package/dist/test/workers/helpers/test-context.js +1 -1
- package/dist/test/workers/helpers/zip-directory.js +1 -1
- package/dist/test/workers/worker/slice-spec.js +1 -1
- package/dist/test/workers/worker/worker-spec.js +1 -1
- package/package.json +7 -7
- package/worker-service.js +1 -1
package/cluster-service.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { get, logError } from '@terascope/utils';
|
|
1
|
+
import { get, logError } from '@terascope/core-utils';
|
|
2
2
|
import { shutdownHandler } from './dist/src/lib/workers/helpers/worker-shutdown.js';
|
|
3
3
|
import { makeTerafoundationContext } from './dist/src/lib/workers/context/terafoundation-context.js';
|
|
4
4
|
import { ClusterMaster } from './dist/src/lib/cluster/cluster_master.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import express from 'express';
|
|
2
2
|
import got from 'got';
|
|
3
|
-
import { pDelay, logError, get, parseError } from '@terascope/utils';
|
|
3
|
+
import { pDelay, logError, get, parseError } from '@terascope/core-utils';
|
|
4
4
|
import { ClusterMaster as ClusterMasterMessaging } from '@terascope/teraslice-messaging';
|
|
5
5
|
import { isPromAvailable } from '@terascope/job-components';
|
|
6
6
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import ms from 'ms';
|
|
2
2
|
import { Mutex } from 'async-mutex';
|
|
3
|
-
import { getFullErrorStack, debounce, isEmpty, has } from '@terascope/utils';
|
|
3
|
+
import { getFullErrorStack, debounce, isEmpty, has } from '@terascope/core-utils';
|
|
4
4
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
5
5
|
import { Messaging } from './services/cluster/backends/native/messaging.js';
|
|
6
6
|
import { spawnAssetLoader } from '../workers/assets/spawn.js';
|
|
@@ -3,7 +3,7 @@ import bodyParser from 'body-parser';
|
|
|
3
3
|
import { pipeline as streamPipeline } from 'node:stream/promises';
|
|
4
4
|
import got from 'got';
|
|
5
5
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
6
|
-
import { parseErrorInfo, parseList, logError, TSError, startsWith, pWhile, isKey } from '@terascope/utils';
|
|
6
|
+
import { parseErrorInfo, parseList, logError, TSError, startsWith, pWhile, isKey } from '@terascope/core-utils';
|
|
7
7
|
import { ExecutionStatusEnum } from '@terascope/types';
|
|
8
8
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
9
9
|
import { makeTable, sendError, handleTerasliceRequest, getSearchOptions, createJobActiveQuery, addDeletedToQuery } from '../../utils/api_utils.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import express from 'express';
|
|
2
|
-
import { TSError, parseErrorInfo, logError, toBoolean } from '@terascope/utils';
|
|
2
|
+
import { TSError, parseErrorInfo, logError, toBoolean } from '@terascope/core-utils';
|
|
3
3
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
4
4
|
import { AssetsStorage } from '../../storage/index.js';
|
|
5
5
|
import { getBackendConfig } from '../../storage/assets.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/utils';
|
|
1
|
+
import { TSError, logError, get, cloneDeep, pRetry } from '@terascope/core-utils';
|
|
2
2
|
import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
|
|
3
3
|
import { gen } from './k8sState.js';
|
|
4
4
|
import { K8s } from './k8s.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, get, isEmpty, pDelay, pRetry } from '@terascope/utils';
|
|
1
|
+
import { TSError, get, isEmpty, pDelay, pRetry } from '@terascope/core-utils';
|
|
2
2
|
import * as k8s from '@kubernetes/client-node';
|
|
3
3
|
import { convertToTSResource, convertToTSResourceList, getRetryConfig, isDeployment, isJob, isPod, isReplicaSet, isService, isTSPod } from './utils.js';
|
|
4
4
|
export class K8s {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isNumber, get, set, has, isEmpty, merge } from '@terascope/utils';
|
|
1
|
+
import { isNumber, get, set, has, isEmpty, merge } from '@terascope/core-utils';
|
|
2
2
|
import { safeEncode } from '../../../../../utils/encoding_utils.js';
|
|
3
3
|
import { setMaxOldSpaceViaEnv } from './utils.js';
|
|
4
4
|
export class K8sResource {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { get, has, uniq, difference } from '@terascope/utils';
|
|
1
|
+
import { get, has, uniq, difference } from '@terascope/core-utils';
|
|
2
2
|
/**
|
|
3
3
|
* Given the k8s Pods API output generates the appropriate Teraslice cluster
|
|
4
4
|
* state. NOTE: This assumes the pods have already been filtered to ensure they
|
|
@@ -3,7 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
// @ts-expect-error
|
|
4
4
|
import barbe from 'barbe';
|
|
5
5
|
import * as k8s from '@kubernetes/client-node';
|
|
6
|
-
import { isTest } from '@terascope/utils';
|
|
6
|
+
import { isTest } from '@terascope/core-utils';
|
|
7
7
|
const MAX_RETRIES = isTest ? 2 : 3;
|
|
8
8
|
const RETRY_DELAY = isTest ? 50 : 1000; // time in ms
|
|
9
9
|
const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/');
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/prefer-for-of */
|
|
2
|
-
import {
|
|
2
|
+
import { TSError, getFullErrorStack, debounce, pDelay, cloneDeep, pMap, orderBy, isInteger, get, Queue } from '@terascope/core-utils';
|
|
3
3
|
import { Dispatch } from './dispatch.js';
|
|
4
4
|
import { makeLogger } from '../../../../../workers/helpers/terafoundation.js';
|
|
5
5
|
import { findWorkersByExecutionID } from '../state-utils.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { nanoid } from 'nanoid';
|
|
2
|
-
import { pDelay,
|
|
2
|
+
import { pDelay, isFunction, isEmpty, get, toNumber, isKey, Queue } from '@terascope/core-utils';
|
|
3
3
|
import { io as socketIOClient } from 'socket.io-client';
|
|
4
4
|
import { Server as socketIOServer } from 'socket.io';
|
|
5
5
|
import { isProcessAssignment, ProcessAssignment } from '../../../../../../interfaces.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TSError, getFullErrorStack, get, logError, withoutNil, isEmpty, multiFieldSort, isString, flatten, includes, cloneDeep, Queue } from '@terascope/core-utils';
|
|
2
2
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
3
3
|
/**
|
|
4
4
|
* New execution result
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, uniq, cloneDeep, isEmpty, getTypeOf, isString,
|
|
1
|
+
import { TSError, uniq, cloneDeep, isEmpty, getTypeOf, isString, defaultsDeep, makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { JobValidator, parseName } from '@terascope/job-components';
|
|
3
3
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
4
4
|
import { spawnAssetLoader } from '../../workers/assets/spawn.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import { get } from '@terascope/utils';
|
|
2
|
+
import { get } from '@terascope/core-utils';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { formats } from '@terascope/job-components';
|
|
5
5
|
import { configSchema } from './schemas/system.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import ip from 'ip';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { isPlainObject, isString, isArray, isInteger } from '@terascope/utils';
|
|
3
|
+
import { isPlainObject, isString, isArray, isInteger } from '@terascope/core-utils';
|
|
4
4
|
import { cpus } from 'node:os';
|
|
5
5
|
const workerCount = cpus().length;
|
|
6
6
|
const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { pMap } from '@terascope/utils';
|
|
1
|
+
import { pMap } from '@terascope/core-utils';
|
|
2
2
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
3
3
|
import { timeseriesIndex } from '../utils/date_utils.js';
|
|
4
4
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import fse from 'fs-extra';
|
|
3
3
|
import crypto from 'node:crypto';
|
|
4
|
-
import { TSError, uniq, isString, toString, filterObject, pDelay } from '@terascope/utils';
|
|
4
|
+
import { TSError, uniq, isString, toString, filterObject, pDelay } from '@terascope/core-utils';
|
|
5
5
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
6
6
|
import { S3Store } from './backends/s3_store.js';
|
|
7
7
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import ms from 'ms';
|
|
2
|
-
import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/utils';
|
|
2
|
+
import { TSError, parseError, isTest, pDelay, pRetry, logError, pWhile, isString, getTypeOf, get, random, isInteger } from '@terascope/core-utils';
|
|
3
3
|
import elasticsearchApi from '@terascope/elasticsearch-api';
|
|
4
4
|
import { getClient } from '@terascope/job-components';
|
|
5
5
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/utils';
|
|
1
|
+
import { TSError, isTest, logError, pDelay, pWhile, random } from '@terascope/core-utils';
|
|
2
2
|
import { S3ClientResponse, createS3Bucket, deleteS3Object, doesBucketExist, getS3Object, listS3Objects, putS3Object, s3RequestWithRetry } from '@terascope/file-asset-apis';
|
|
3
3
|
import { makeLogger } from '../../workers/helpers/terafoundation.js';
|
|
4
4
|
export class S3Store {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, includes, getTypeOf, makeISODate } from '@terascope/utils';
|
|
1
|
+
import { TSError, includes, getTypeOf, makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
3
3
|
import { v4 as uuid } from 'uuid';
|
|
4
4
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { v4 as uuid } from 'uuid';
|
|
2
|
-
import { TSError, makeISODate } from '@terascope/utils';
|
|
2
|
+
import { TSError, makeISODate } from '@terascope/core-utils';
|
|
3
3
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
4
4
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
5
5
|
export class JobsStorage {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
2
|
-
import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/utils';
|
|
2
|
+
import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, times, getFullErrorStack, isKey } from '@terascope/core-utils';
|
|
3
3
|
import { timeseriesIndex } from '../utils/date_utils.js';
|
|
4
4
|
import { makeLogger } from '../workers/helpers/terafoundation.js';
|
|
5
5
|
import { TerasliceElasticsearchStorage } from './backends/elasticsearch_store.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import Table from 'easy-table';
|
|
2
|
-
import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/utils';
|
|
2
|
+
import { parseErrorInfo, parseList, logError, isString, get, toInteger, TSError } from '@terascope/core-utils';
|
|
3
3
|
export function makeTable(req, defaults, data, mappingFn) {
|
|
4
4
|
const query = fieldsQuery(req.query, defaults);
|
|
5
5
|
let emptyChar = 'N/A';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/utils';
|
|
1
|
+
import { isInteger, trimStart, trim, getFirst, joinList } from '@terascope/core-utils';
|
|
2
2
|
import * as semver from 'semver';
|
|
3
3
|
export function findMatchingAsset(records, name, version) {
|
|
4
4
|
const range = toSemverRange(version);
|
|
@@ -3,7 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import fse from 'fs-extra';
|
|
4
4
|
import semver from 'semver';
|
|
5
5
|
import { Mutex } from 'async-mutex';
|
|
6
|
-
import { TSError } from '@terascope/utils';
|
|
6
|
+
import { TSError } from '@terascope/core-utils';
|
|
7
7
|
import decompress from 'decompress';
|
|
8
8
|
import { getMajorVersion } from './asset_utils.js';
|
|
9
9
|
const mutex = new Mutex();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { trim, padEnd } from '@terascope/utils';
|
|
1
|
+
import { trim, padEnd } from '@terascope/core-utils';
|
|
2
2
|
import { customAlphabet } from 'nanoid';
|
|
3
3
|
export function newId(prefix, lowerCase = false, length = 15) {
|
|
4
4
|
let characters = '-0123456789abcdefghijklmnopqrstuvwxyz';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getFullErrorStack, pDelay } from '@terascope/utils';
|
|
1
|
+
import { getFullErrorStack, pDelay } from '@terascope/core-utils';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { AssetLoader } from './loader.js';
|
|
4
4
|
import { makeTerafoundationContext } from '../context/terafoundation-context.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import fs from 'node:fs';
|
|
3
|
-
import { get, isEmpty } from '@terascope/utils';
|
|
3
|
+
import { get, isEmpty } from '@terascope/core-utils';
|
|
4
4
|
import { AssetsStorage } from '../../storage/index.js';
|
|
5
5
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
6
6
|
import { saveAsset } from '../../utils/file_utils.js';
|
|
@@ -2,7 +2,7 @@ import fs from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { fork } from 'node:child_process';
|
|
5
|
-
import { isEmpty, get, has } from '@terascope/utils';
|
|
5
|
+
import { isEmpty, get, has } from '@terascope/core-utils';
|
|
6
6
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
7
7
|
import { safeEncode } from '../../utils/encoding_utils.js';
|
|
8
8
|
const filePath = fileURLToPath(new URL('.', import.meta.url));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import ms from 'ms';
|
|
2
2
|
import { formatURL, ExecutionController as ExController, ClusterMaster } from '@terascope/teraslice-messaging';
|
|
3
|
-
import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile,
|
|
3
|
+
import { TSError, includes, get, pDelay, getFullErrorStack, logError, pWhile, debounce, throttle, makeISODate } from '@terascope/core-utils';
|
|
4
4
|
import { isPromAvailable } from '@terascope/job-components';
|
|
5
5
|
import { waitForWorkerShutdown } from '../helpers/worker-shutdown.js';
|
|
6
6
|
import { StateStorage, ExecutionStorage, SliceState } from '../../storage/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { noop, pDelay, get, toString, logError, pWhile, Queue, makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { RecoveryModule } from './recovery.js';
|
|
3
3
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
4
4
|
export class Scheduler {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { makeISODate } from '@terascope/utils';
|
|
1
|
+
import { makeISODate } from '@terascope/core-utils';
|
|
2
2
|
import { JobValidator } from '@terascope/job-components';
|
|
3
3
|
import { JobsStorage, ExecutionStorage, StateStorage } from '../../storage/index.js';
|
|
4
4
|
// TODO: fix type here
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { get, isFunction, isString } from '@terascope/utils';
|
|
1
|
+
import { get, isFunction, isString } from '@terascope/core-utils';
|
|
2
2
|
import { makeContextLogger } from '@terascope/job-components';
|
|
3
3
|
import { safeDecode } from '../../utils/encoding_utils.js';
|
|
4
4
|
export function generateWorkerId(context) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/utils';
|
|
1
|
+
import { get, pDelay, pRaceWithTimeout, isError, logError } from '@terascope/core-utils';
|
|
2
2
|
import ms from 'ms';
|
|
3
3
|
import { makeLogger } from './terafoundation.js';
|
|
4
4
|
export function waitForWorkerShutdown(context, eventName) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/* eslint-disable prefer-const */
|
|
2
|
-
import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/utils';
|
|
2
|
+
import { get, getFullErrorStack, isFatalError, logError, pWhile } from '@terascope/core-utils';
|
|
3
3
|
import { ExecutionController, formatURL } from '@terascope/teraslice-messaging';
|
|
4
4
|
import { isPromAvailable } from '@terascope/job-components';
|
|
5
5
|
import { StateStorage, AnalyticsStorage } from '../../storage/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TSError, getTypeOf, logError } from '@terascope/utils';
|
|
1
|
+
import { TSError, getTypeOf, logError } from '@terascope/core-utils';
|
|
2
2
|
import { SliceState } from '../../storage/index.js';
|
|
3
3
|
import { makeLogger } from '../helpers/terafoundation.js';
|
|
4
4
|
import { logOpStats } from '../helpers/op-analytics.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { V1Deployment, V1Job, V1Pod, V1ReplicaSet, V1Service } from '@kubernetes/client-node';
|
|
2
2
|
import nock from 'nock';
|
|
3
|
-
import { debugLogger } from '@terascope/
|
|
3
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
4
4
|
import { K8s } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js';
|
|
5
5
|
const logger = debugLogger('k8s-v2-spec');
|
|
6
6
|
const _url = 'http://mock.kube.api';
|
package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import yaml from 'js-yaml';
|
|
2
|
-
import { debugLogger } from '@terascope/utils';
|
|
2
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
3
3
|
import { K8sDeploymentResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js';
|
|
4
4
|
import { K8sJobResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js';
|
|
5
5
|
import { K8sServiceResource } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cloneDeep } from '@terascope/utils';
|
|
1
|
+
import { cloneDeep } from '@terascope/core-utils';
|
|
2
2
|
import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json';
|
|
3
3
|
import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
|
|
4
4
|
describe('k8sState with pods from multiple clusters', () => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { cloneDeep } from '@terascope/utils';
|
|
1
|
+
import { cloneDeep } from '@terascope/core-utils';
|
|
2
2
|
import _podsJobRunning from '../files/job-running-v1-k8s-pods.json';
|
|
3
3
|
import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js';
|
|
4
4
|
describe('k8sState', () => {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import { debugLogger } from '@terascope/utils';
|
|
2
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
3
3
|
import { Server as SocketServer } from 'socket.io';
|
|
4
4
|
import { nodeMaster } from '../src/lib/cluster/node_master.js';
|
|
5
5
|
process.env.assignment = 'node_master';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
import events from 'node:events';
|
|
3
|
-
import { debugLogger } from '@terascope/utils';
|
|
3
|
+
import { debugLogger } from '@terascope/core-utils';
|
|
4
4
|
import { Messaging, routing } from '../../src/lib/cluster/services/cluster/backends/native/messaging.js';
|
|
5
5
|
describe('messaging module', () => {
|
|
6
6
|
const logger = debugLogger('messaging');
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
|
-
import { pDelay, get, set } from '@terascope/utils';
|
|
2
|
+
import { pDelay, get, set } from '@terascope/core-utils';
|
|
3
3
|
import { TestContext } from '../helpers/index.js';
|
|
4
4
|
import { findPort } from '../../../src/lib/utils/port_utils.js';
|
|
5
5
|
import { ExecutionController } from '../../../src/lib/workers/execution-controller/index.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { v4 as uuidv4 } from 'uuid';
|
|
2
|
-
import { pDelay, times, random } from '@terascope/utils';
|
|
2
|
+
import { pDelay, times, random } from '@terascope/core-utils';
|
|
3
3
|
import { RecoveryCleanupType } from '@terascope/job-components';
|
|
4
4
|
import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
|
|
5
5
|
import { TestContext } from '../helpers/index.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { pDelay, times, random, isNotNil } from '@terascope/utils';
|
|
1
|
+
import { pDelay, times, random, isNotNil } from '@terascope/core-utils';
|
|
2
2
|
import { ExecutionController as ExController } from '@terascope/teraslice-messaging';
|
|
3
3
|
import { TestContext } from '../helpers/index.js';
|
|
4
4
|
import { getTestCases } from '../helpers/execution-controller-helper.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { pDelay, pWhile, debugLogger } from '@terascope/utils';
|
|
1
|
+
import { pDelay, pWhile, debugLogger } from '@terascope/core-utils';
|
|
2
2
|
import { EventEmitter } from 'node:events';
|
|
3
3
|
import { RecoveryModule } from '../../../src/lib/workers/execution-controller/recovery.js';
|
|
4
4
|
const eventEmitter = new EventEmitter();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
import { v4 as uuidv4 } from 'uuid';
|
|
3
|
-
import { pDelay, once, times } from '@terascope/utils';
|
|
3
|
+
import { pDelay, once, times } from '@terascope/core-utils';
|
|
4
4
|
import { TestContext } from '../helpers/test-context.js';
|
|
5
5
|
import { Scheduler } from '../../../src/lib/workers/execution-controller/scheduler.js';
|
|
6
6
|
describe('Scheduler', () => {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ParallelSlicer } from '@terascope/job-components';
|
|
2
|
-
import { get } from '@terascope/utils';
|
|
2
|
+
import { get } from '@terascope/core-utils';
|
|
3
3
|
const defaultSlicerResults = [{ howdy: true }, null];
|
|
4
4
|
export default class ExampleSlicer extends ParallelSlicer {
|
|
5
5
|
async newSlicer(_id) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ParallelSlicer } from '@terascope/job-components';
|
|
2
|
-
import { pDelay, times } from '@terascope/utils';
|
|
2
|
+
import { pDelay, times } from '@terascope/core-utils';
|
|
3
3
|
export default class ExampleSlicer extends ParallelSlicer {
|
|
4
4
|
async newSlicer(id) {
|
|
5
5
|
const { countPerSlicer } = this.opConfig;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
|
-
import { withoutNil } from '@terascope/utils';
|
|
3
|
+
import { withoutNil } from '@terascope/core-utils';
|
|
4
4
|
// @ts-expect-error
|
|
5
5
|
import Chance from 'chance';
|
|
6
6
|
import { newId } from '../../../src/lib/utils/id_utils.js';
|
|
@@ -2,7 +2,7 @@ import fs from 'fs-extra';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { createTempDirSync, cleanupTempDirs } from 'jest-fixtures';
|
|
4
4
|
import { newTestSlice } from '@terascope/job-components';
|
|
5
|
-
import { get, pWhile } from '@terascope/utils';
|
|
5
|
+
import { get, pWhile } from '@terascope/core-utils';
|
|
6
6
|
import { ClusterMaster } from '@terascope/teraslice-messaging';
|
|
7
7
|
import { AssetsStorage, StateStorage, AnalyticsStorage, ExecutionStorage, JobsStorage } from '../../../src/lib/storage/index.js';
|
|
8
8
|
import { initializeTestExecution } from '../../../src/lib/workers/helpers/job.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
|
-
import { times } from '@terascope/utils';
|
|
2
|
+
import { times } from '@terascope/core-utils';
|
|
3
3
|
import { SliceExecution } from '../../../src/lib/workers/worker/slice.js';
|
|
4
4
|
import { TestContext } from '../helpers/index.js';
|
|
5
5
|
describe('Slice', () => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
import 'jest-extended';
|
|
3
|
-
import { pDelay } from '@terascope/utils';
|
|
3
|
+
import { pDelay } from '@terascope/core-utils';
|
|
4
4
|
import { ExecutionController } from '@terascope/teraslice-messaging';
|
|
5
5
|
import { findPort } from '../../../src/lib/utils/port_utils.js';
|
|
6
6
|
import { Worker } from '../../../src/lib/workers/worker/index.js';
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "teraslice",
|
|
3
3
|
"displayName": "Teraslice",
|
|
4
|
-
"version": "3.0.0-dev.
|
|
4
|
+
"version": "3.0.0-dev.8",
|
|
5
5
|
"description": "Distributed computing platform for processing JSON data",
|
|
6
6
|
"homepage": "https://github.com/terascope/teraslice#readme",
|
|
7
7
|
"bugs": {
|
|
@@ -39,11 +39,11 @@
|
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
41
|
"@kubernetes/client-node": "~1.3.0",
|
|
42
|
-
"@terascope/
|
|
43
|
-
"@terascope/
|
|
44
|
-
"@terascope/
|
|
42
|
+
"@terascope/core-utils": "~2.0.0-dev.7",
|
|
43
|
+
"@terascope/elasticsearch-api": "~5.0.0-dev.7",
|
|
44
|
+
"@terascope/job-components": "~2.0.0-dev.7",
|
|
45
|
+
"@terascope/teraslice-messaging": "~2.0.0-dev.8",
|
|
45
46
|
"@terascope/types": "~2.0.0-dev.6",
|
|
46
|
-
"@terascope/utils": "~2.0.0-dev.6",
|
|
47
47
|
"async-mutex": "~0.5.0",
|
|
48
48
|
"barbe": "~3.0.17",
|
|
49
49
|
"body-parser": "~2.2.0",
|
|
@@ -61,11 +61,11 @@
|
|
|
61
61
|
"semver": "~7.7.2",
|
|
62
62
|
"socket.io": "~4.8.1",
|
|
63
63
|
"socket.io-client": "~4.8.1",
|
|
64
|
-
"terafoundation": "~2.0.0-dev.
|
|
64
|
+
"terafoundation": "~2.0.0-dev.7",
|
|
65
65
|
"uuid": "~13.0.0"
|
|
66
66
|
},
|
|
67
67
|
"devDependencies": {
|
|
68
|
-
"@terascope/opensearch-client": "~2.0.0-dev.
|
|
68
|
+
"@terascope/opensearch-client": "~2.0.0-dev.7",
|
|
69
69
|
"@types/archiver": "~6.0.3",
|
|
70
70
|
"@types/express": "~5.0.3",
|
|
71
71
|
"@types/gc-stats": "~1.4.3",
|
package/worker-service.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
logError, get, isEmpty, isPlainObject
|
|
3
|
-
} from '@terascope/utils';
|
|
3
|
+
} from '@terascope/core-utils';
|
|
4
4
|
import { shutdownHandler } from './dist/src/lib/workers/helpers/worker-shutdown.js';
|
|
5
5
|
import { safeDecode } from './dist/src/lib/utils/encoding_utils.js';
|
|
6
6
|
import { makeExecutionContext } from './dist/src/lib/workers/context/execution-context.js';
|