teraslice 3.3.0 → 3.3.2

package/dist/test/workers/helpers/configs.js

@@ -1,128 +0,0 @@
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { withoutNil } from '@terascope/core-utils';
-// @ts-expect-error
-import Chance from 'chance';
-import { newId } from '../../../src/lib/utils/id_utils.js';
-import { SEARCH_TEST_HOST } from '../../test.config.js';
-const dirname = path.dirname(fileURLToPath(import.meta.url));
-const opsPath = path.join(dirname, '..', 'fixtures', 'ops');
-const chance = new Chance();
-const newConfig = (options = {}) => {
-    const { newOps } = options;
-    let { operations } = options;
-    if (operations == null) {
-        if (newOps) {
-            operations = [
-                withoutNil({
-                    _op: path.join(opsPath, 'new-reader'),
-                    countPerSlicer: options.countPerSlicer
-                }),
-                withoutNil({
-                    _op: path.join(opsPath, 'new-op'),
-                    failOnSliceRetry: options.failOnSliceRetry || false
-                }),
-                {
-                    _op: 'noop'
-                }
-            ];
-        }
-        else {
-            operations = [
-                withoutNil({
-                    _op: path.join(opsPath, 'example-reader'),
-                    exampleProp: 321,
-                    updateMetadata: options.updateMetadata,
-                    errorAt: options.readerErrorAt,
-                    results: options.readerResults,
-                    slicerResults: options.slicerResults,
-                    slicerErrorAt: options.slicerErrorAt,
-                    slicerQueueLength: options.slicerQueueLength
-                }),
-                withoutNil({
-                    _op: path.join(opsPath, 'example-op'),
-                    exampleProp: 123,
-                    errorAt: options.opErrorAt,
-                    results: options.opResults
-                })
-            ];
-        }
-    }
-    const { analytics = false, maxRetries = 0, slicerPort = 0, lifecycle = 'once', assets = [], workers = 1, slicers = 1, autorecover = false, recoveredExecution, recoveredSliceType, probationWindow = 5000, log_level } = options;
-    return {
-        name: chance.name({ middle: true }),
-        slicers,
-        workers,
-        assets,
-        analytics,
-        lifecycle,
-        max_retries: maxRetries,
-        operations,
-        autorecover,
-        performance_metrics: false,
-        recovered_execution: recoveredExecution,
-        recovered_slice_type: recoveredSliceType,
-        ex_id: newId('ex-id', true),
-        job_id: newId('job-id', true),
-        node_id: newId('node-id', true),
-        slicer_port: slicerPort,
-        slicer_hostname: 'localhost',
-        probation_window: probationWindow,
-        log_level
-    };
-};
-const newSysConfig = (options) => {
-    const { clusterName = 'test-teraslice-cluster', timeout = 3000, actionTimeout = 2000, shutdownTimeout = 4000, assetDir, clusterMasterPort, log_level_terafoundation = 'info', } = options;
-    return {
-        terafoundation: {
-            log_level: log_level_terafoundation,
-            connectors: {
-                'elasticsearch-next': {
-                    default: {
-                        node: [SEARCH_TEST_HOST],
-                        requestTimeout: timeout
-                    }
-                }
-            }
-        },
-        teraslice: {
-            assets_directory: assetDir,
-            shutdown_timeout: shutdownTimeout,
-            action_timeout: actionTimeout,
-            network_latency_buffer: 100,
-            slicer_timeout: timeout,
-            slicer_allocation_attempts: 3,
-            node_state_interval: timeout,
-            node_disconnect_timeout: timeout,
-            worker_disconnect_timeout: timeout,
-            analytics_rate: 100,
-            name: clusterName,
-            master_hostname: 'localhost',
-            port: clusterMasterPort,
-            index_settings: {
-                analytics: {
-                    number_of_shards: 1,
-                    number_of_replicas: 0
-                },
-                assets: {
-                    number_of_shards: 1,
-                    number_of_replicas: 0
-                },
-                jobs: {
-                    number_of_shards: 1,
-                    number_of_replicas: 0
-                },
-                execution: {
-                    number_of_shards: 1,
-                    number_of_replicas: 0
-                },
-                state: {
-                    number_of_shards: 1,
-                    number_of_replicas: 0
-                }
-            }
-        }
-    };
-};
-export { opsPath, newConfig, newSysConfig };
-//# sourceMappingURL=configs.js.map

package/dist/test/workers/helpers/execution-controller-helper.js

@@ -1,49 +0,0 @@
-import { pDelay, pDefer } from '@terascope/core-utils';
-export function makeShutdownEarlyFn({ exController, enabled = false }) {
-    let shutdownErr = {
-        message: 'Shutdown never triggered'
-    };
-    let alreadyCalled = false;
-    const deferred = pDefer();
-    const catchShutdown = async () => {
-        if (alreadyCalled)
-            return;
-        alreadyCalled = true;
-        shutdownErr.message = 'Shutdown trigger but did not error';
-        try {
-            await exController.shutdown();
-        }
-        catch (err) {
-            shutdownErr = err;
-        }
-        deferred.resolve();
-    };
-    return {
-        error: () => shutdownErr,
-        wait: async () => {
-            if (!enabled)
-                return;
-            await deferred.promise;
-        },
-        shutdown: async () => {
-            if (!enabled)
-                return;
-            await pDelay(100);
-            catchShutdown();
-            await pDelay(100);
-        }
-    };
-}
-export function getTestCases(testCases) {
-    const onlyCases = testCases.filter((ts) => ts[1].only);
-    if (onlyCases.length > 0) {
-        console.warn('[WARNING]: test cases includes a "only" property, make sure to remove this before committing');
-        return onlyCases;
-    }
-    const cases = testCases.filter((ts) => !ts[1].only);
-    if (cases.length !== testCases.length) {
-        console.warn('[WARNING]: test cases includes a "skip" property, make sure to remove this before committing');
-    }
-    return cases;
-}
-//# sourceMappingURL=execution-controller-helper.js.map

package/dist/test/workers/helpers/index.js

@@ -1,5 +0,0 @@
-import { TestContext } from './test-context';
-import { newId } from '../../../src/lib/utils/id_utils.js';
-import { newConfig, newSysConfig, opsPath } from './configs';
-export { newConfig, opsPath, newId, newSysConfig, TestContext };
-//# sourceMappingURL=index.js.map

package/dist/test/workers/helpers/test-context.js

@@ -1,210 +0,0 @@
-import fs from 'fs-extra';
-import path from 'node:path';
-import { createTempDirSync, cleanupTempDirs } from 'jest-fixtures';
-import { newTestSlice } from '@terascope/job-components';
-import { get, pWhile } from '@terascope/core-utils';
-import { ClusterMaster } from '@terascope/teraslice-messaging';
-import { AssetsStorage, StateStorage, AnalyticsStorage, ExecutionStorage, JobsStorage } from '../../../src/lib/storage/index.js';
-import { initializeTestExecution } from '../../../src/lib/workers/helpers/job.js';
-import { makeTerafoundationContext } from '../../../src/lib/workers/context/terafoundation-context.js';
-import { makeExecutionContext } from '../../../src/lib/workers/context/execution-context.js';
-import { newId } from '../../../src/lib/utils/id_utils.js';
-import { findPort } from '../../../src/lib/utils/port_utils.js';
-import { newConfig, newSysConfig } from './configs.js';
-import { zipDirectory } from './zip-directory.js';
-import { TERASLICE_CLUSTER_NAME } from '../../test.config.js';
-const tmpAssetDir = createTempDirSync();
-const clusterName = `${TERASLICE_CLUSTER_NAME}`;
-export class TestContext {
-    setupId;
-    assetDir;
-    sysconfig;
-    config;
-    context;
-    clean = false;
-    nodeId;
-    exId;
-    jobId;
-    events;
-    _cleanupFns = [];
-    executionContext;
-    assignment;
-    clusterMaster;
-    _stores = {};
-    cleanups = {};
-    constructor(options = {}) {
-        const { clusterMasterPort, shutdownTimeout, actionTimeout, timeout, log_level_terafoundation } = options;
-        this.setupId = newId('setup', true);
-        this.assetDir = tmpAssetDir;
-        this.sysconfig = newSysConfig({
-            clusterName,
-            assetDir: this.assetDir,
-            clusterMasterPort,
-            actionTimeout,
-            timeout,
-            shutdownTimeout,
-            log_level_terafoundation
-        });
-        this.config = newConfig(options);
-        this.cleanups[this.setupId] = () => this.cleanup();
-        this.assignment = options.assignment || 'worker';
-    }
-    // make sure we cleanup if any test fails to cleanup properly
-    async cleanupAll(withEs = false) {
-        const count = Object.keys(this.cleanups).length;
-        if (!count)
-            return;
-        const fns = Object.keys(this.cleanups).map(async (name) => {
-            const fn = this.cleanups[name];
-            try {
-                await fn();
-            }
-            catch (err) {
-                console.error(`Failed to cleanup ${name}`, err);
-            }
-            delete this.cleanups[name];
-        });
-        await Promise.all(fns);
-        try {
-            cleanupTempDirs();
-        }
-        catch (err) {
-            console.error(err);
-        }
-        if (withEs && Object.keys(this._stores).length) {
-            try {
-                await Promise.all(Object.values(this._stores).map((store) => store.shutdown(true)));
-            }
-            catch (err) {
-                console.error(err);
-            }
-        }
-    }
-    async waitForCleanup() {
-        return pWhile(async () => !Object.keys(this.cleanups).length, {
-            name: 'Test Context',
-            timeoutMs: 3000
-        });
-    }
-    async initialize(makeItReal = false, initOptions = {}) {
-        this.context = await makeTerafoundationContext({ sysconfig: this.sysconfig });
-        this.context.assignment = this.assignment;
-        this.events = this.context.apis.foundation.getSystemEvents();
-        if (makeItReal) {
-            await Promise.all([
-                this.addJobStore(),
-                this.addExStore(),
-                this.addStateStore()
-            ]);
-            const { ex } = await initializeTestExecution(Object.assign({
-                context: this.context,
-                config: this.config,
-                stores: this._stores,
-            }, initOptions));
-            this.config = ex;
-        }
-        this.executionContext = await makeExecutionContext(this.context, this.config);
-        // @ts-expect-error
-        this.nodeId = this.executionContext.config.node_id;
-        this.exId = this.executionContext.config.ex_id;
-        this.jobId = this.executionContext.config.job_id;
-    }
-    get stores() {
-        return this._stores;
-    }
-    async addClusterMaster() {
-        if (this.clusterMaster)
-            return this.clusterMaster;
-        const port = await findPort();
-        const networkLatencyBuffer = get(this.context, 'sysconfig.teraslice.network_latency_buffer');
-        const actionTimeout = get(this.context, 'sysconfig.teraslice.action_timeout');
-        const nodeDisconnectTimeout = get(this.context, 'sysconfig.teraslice.node_disconnect_timeout');
-        this.clusterMaster = new ClusterMaster.Server({
-            port,
-            networkLatencyBuffer,
-            actionTimeout,
-            nodeDisconnectTimeout
-        });
-        await this.clusterMaster.start();
-        this.context.sysconfig.teraslice.port = port;
-        this.attachCleanup(() => this.clusterMaster.shutdown());
-        return this.clusterMaster;
-    }
-    attachCleanup(fn) {
-        this._cleanupFns.push(fn);
-    }
-    async saveAsset(assetDir, cleanup) {
-        await this.addAssetStore();
-        const exists = await fs.pathExists(assetDir);
-        if (!exists) {
-            const err = new Error(`Asset Directory ${assetDir} does not exist`);
-            throw err;
-        }
-        const assetZip = await zipDirectory(assetDir);
-        if (!this._stores.assetsStorage) {
-            throw new Error('Assets storage is not setup');
-        }
-        const { assetId } = await this._stores.assetsStorage.save(assetZip);
-        if (cleanup) {
-            await fs.remove(path.join(this.assetDir, assetId));
-        }
-        return assetId;
-    }
-    async newSlice() {
-        const sliceConfig = newTestSlice({ request: { example: 'slice-data' } });
-        await this.addStateStore();
-        await this._stores.stateStorage.createState(this.exId, sliceConfig, 'start');
-        return sliceConfig;
-    }
-    async addAssetStore() {
-        if (this._stores.assetsStorage) {
-            return this._stores.assetsStorage;
-        }
-        this._stores.assetsStorage = new AssetsStorage(this.context);
-        await this._stores.assetsStorage.initialize();
-        delete this.context.apis.assets;
-        return this._stores.assetsStorage;
-    }
-    async addStateStore() {
-        if (this._stores.stateStorage) {
-            return this._stores.stateStorage;
-        }
-        this._stores.stateStorage = new StateStorage(this.context);
-        await this._stores.stateStorage.initialize();
-        return this._stores.stateStorage;
-    }
-    async addAnalyticsStore() {
-        if (this._stores.analyticsStorage) {
-            return this._stores.analyticsStorage;
-        }
-        this._stores.analyticsStorage = new AnalyticsStorage(this.context);
-        await this._stores.analyticsStorage.initialize();
-        return this._stores.analyticsStorage;
-    }
-    async addJobStore() {
-        if (this._stores.jobsStorage) {
-            return this._stores.jobsStorage;
-        }
-        this._stores.jobsStorage = new JobsStorage(this.context);
-        await this._stores.jobsStorage.initialize();
-        return this._stores.jobsStorage;
-    }
-    async addExStore() {
-        if (this._stores.executionStorage) {
-            return this._stores.executionStorage;
-        }
-        this._stores.executionStorage = new ExecutionStorage(this.context);
-        await this._stores.executionStorage.initialize();
-        return this._stores.executionStorage;
-    }
-    async cleanup() {
-        if (this.clean)
-            return;
-        await Promise.all(this._cleanupFns.map((fn) => fn()));
-        this._cleanupFns.length = 0;
-        this.events.removeAllListeners();
-        delete this.cleanups[this.setupId];
-        this.clean = true;
-    }
-}
-//# sourceMappingURL=test-context.js.map

package/dist/test/workers/helpers/zip-directory.js

@@ -1,25 +0,0 @@
-import path from 'node:path';
-import { random } from '@terascope/core-utils';
-// @ts-expect-error
-import BufferStreams from 'bufferstreams';
-import archiver from 'archiver';
-import { newId } from '../../../src/lib/utils/id_utils.js';
-export function zipDirectory(dir) {
-    return new Promise((resolve, reject) => {
-        const archive = archiver('zip', { zlib: { level: 9 } });
-        archive.append(JSON.stringify({
-            name: path.basename(dir),
-            version: `${random(0, 100)}.${random(0, 100)}.${random(0, 100)}`,
-            someProp: newId()
-        }, null, 4), { name: 'asset.json' });
-        archive.pipe(new BufferStreams((err, buf) => {
-            if (err) {
-                reject(err);
-                return;
-            }
-            resolve(buf);
-        }));
-        archive.directory(dir, 'asset').finalize();
-    });
-}
-//# sourceMappingURL=zip-directory.js.map