teraslice 2.10.0 → 2.12.0

package/dist/test/workers/helpers/configs.js

@@ -0,0 +1,130 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { withoutNil } from '@terascope/utils';
+// @ts-expect-error
+import Chance from 'chance';
+import { newId } from '../../../src/lib/utils/id_utils.js';
+const { SEARCH_TEST_HOST } = process.env;
+const dirname = path.dirname(fileURLToPath(import.meta.url));
+const opsPath = path.join(dirname, '..', 'fixtures', 'ops');
+const chance = new Chance();
+const newConfig = (options = {}) => {
+    const { newOps } = options;
+    let { operations } = options;
+    if (operations == null) {
+        if (newOps) {
+            operations = [
+                withoutNil({
+                    _op: path.join(opsPath, 'new-reader'),
+                    countPerSlicer: options.countPerSlicer
+                }),
+                withoutNil({
+                    _op: path.join(opsPath, 'new-op'),
+                    failOnSliceRetry: options.failOnSliceRetry || false
+                }),
+                {
+                    _op: 'noop'
+                }
+            ];
+        }
+        else {
+            operations = [
+                withoutNil({
+                    _op: path.join(opsPath, 'example-reader'),
+                    exampleProp: 321,
+                    updateMetadata: options.updateMetadata,
+                    errorAt: options.readerErrorAt,
+                    results: options.readerResults,
+                    slicerResults: options.slicerResults,
+                    slicerErrorAt: options.slicerErrorAt,
+                    slicerQueueLength: options.slicerQueueLength
+                }),
+                withoutNil({
+                    _op: path.join(opsPath, 'example-op'),
+                    exampleProp: 123,
+                    errorAt: options.opErrorAt,
+                    results: options.opResults
+                })
+            ];
+        }
+    }
+    const { analytics = false, maxRetries = 0, slicerPort = 0, lifecycle = 'once', assets = [], workers = 1, slicers = 1, autorecover = false, recoveredExecution, recoveredSliceType, probationWindow = 5000, log_level } = options;
+    return {
+        name: chance.name({ middle: true }),
+        slicers,
+        workers,
+        assets,
+        analytics,
+        lifecycle,
+        max_retries: maxRetries,
+        operations,
+        autorecover,
+        performance_metrics: false,
+        recovered_execution: recoveredExecution,
+        recovered_slice_type: recoveredSliceType,
+        ex_id: newId('ex-id', true),
+        job_id: newId('job-id', true),
+        node_id: newId('node-id', true),
+        slicer_port: slicerPort,
+        slicer_hostname: 'localhost',
+        probation_window: probationWindow,
+        log_level
+    };
+};
+const newSysConfig = (options) => {
+    const { clusterName = 'test-teraslice-cluster', timeout = 3000, actionTimeout = 2000, shutdownTimeout = 4000, assetDir, clusterMasterPort, log_level_terafoundation = 'info', } = options;
+    return {
+        terafoundation: {
+            environment: 'development',
+            log_level: log_level_terafoundation,
+            connectors: {
+                'elasticsearch-next': {
+                    default: {
+                        node: [SEARCH_TEST_HOST],
+                        requestTimeout: timeout,
+                        deadTimeout: timeout
+                    }
+                }
+            }
+        },
+        teraslice: {
+            assets_directory: assetDir,
+            shutdown_timeout: shutdownTimeout,
+            action_timeout: actionTimeout,
+            network_latency_buffer: 100,
+            slicer_timeout: timeout,
+            slicer_allocation_attempts: 3,
+            node_state_interval: timeout,
+            node_disconnect_timeout: timeout,
+            worker_disconnect_timeout: timeout,
+            analytics_rate: 100,
+            name: clusterName,
+            master_hostname: 'localhost',
+            port: clusterMasterPort,
+            index_settings: {
+                analytics: {
+                    number_of_shards: 1,
+                    number_of_replicas: 0
+                },
+                assets: {
+                    number_of_shards: 1,
+                    number_of_replicas: 0
+                },
+                jobs: {
+                    number_of_shards: 1,
+                    number_of_replicas: 0
+                },
+                execution: {
+                    number_of_shards: 1,
+                    number_of_replicas: 0
+                },
+                state: {
+                    number_of_shards: 1,
+                    number_of_replicas: 0
+                }
+            }
+        }
+    };
+};
+export { opsPath, newConfig, newSysConfig };
+//# sourceMappingURL=configs.js.map

package/dist/test/workers/helpers/execution-controller-helper.js

@@ -0,0 +1,49 @@
+import { pDelay, pDefer } from '@terascope/utils';
+export function makeShutdownEarlyFn({ exController, enabled = false }) {
+    let shutdownErr = {
+        message: 'Shutdown never triggered'
+    };
+    let alreadyCalled = false;
+    const deferred = pDefer();
+    const catchShutdown = async () => {
+        if (alreadyCalled)
+            return;
+        alreadyCalled = true;
+        shutdownErr.message = 'Shutdown trigger but did not error';
+        try {
+            await exController.shutdown();
+        }
+        catch (err) {
+            shutdownErr = err;
+        }
+        deferred.resolve();
+    };
+    return {
+        error: () => shutdownErr,
+        wait: async () => {
+            if (!enabled)
+                return;
+            await deferred.promise;
+        },
+        shutdown: async () => {
+            if (!enabled)
+                return;
+            await pDelay(100);
+            catchShutdown();
+            await pDelay(100);
+        }
+    };
+}
+export function getTestCases(testCases) {
+    const onlyCases = testCases.filter((ts) => ts[1].only);
+    if (onlyCases.length > 0) {
+        console.warn('[WARNING]: test cases includes a "only" property, make sure to remove this before committing');
+        return onlyCases;
+    }
+    const cases = testCases.filter((ts) => !ts[1].only);
+    if (cases.length !== testCases.length) {
+        console.warn('[WARNING]: test cases includes a "skip" property, make sure to remove this before committing');
+    }
+    return cases;
+}
+//# sourceMappingURL=execution-controller-helper.js.map

package/dist/test/workers/helpers/index.js

@@ -0,0 +1,5 @@
+import { TestContext } from './test-context';
+import { newId } from '../../../src/lib/utils/id_utils.js';
+import { newConfig, newSysConfig, opsPath } from './configs';
+export { newConfig, opsPath, newId, newSysConfig, TestContext };
+//# sourceMappingURL=index.js.map

package/dist/test/workers/helpers/test-context.js

@@ -0,0 +1,210 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+import { createTempDirSync, cleanupTempDirs } from 'jest-fixtures';
+import { newTestSlice } from '@terascope/job-components';
+import { get, pWhile } from '@terascope/utils';
+import { ClusterMaster } from '@terascope/teraslice-messaging';
+import { AssetsStorage, StateStorage, AnalyticsStorage, ExecutionStorage, JobsStorage } from '../../../src/lib/storage/index.js';
+import { initializeTestExecution } from '../../../src/lib/workers/helpers/job.js';
+import { makeTerafoundationContext } from '../../../src/lib/workers/context/terafoundation-context.js';
+import { makeExecutionContext } from '../../../src/lib/workers/context/execution-context.js';
+import { newId } from '../../../src/lib/utils/id_utils.js';
+import { findPort } from '../../../src/lib/utils/port_utils.js';
+import { newConfig, newSysConfig } from './configs.js';
+import { zipDirectory } from './zip-directory.js';
+import { TERASLICE_CLUSTER_NAME } from '../../test.config.js';
+const tmpAssetDir = createTempDirSync();
+const clusterName = `${TERASLICE_CLUSTER_NAME}`;
+export class TestContext {
+    setupId;
+    assetDir;
+    sysconfig;
+    config;
+    context;
+    clean = false;
+    nodeId;
+    exId;
+    jobId;
+    events;
+    _cleanupFns = [];
+    executionContext;
+    assignment;
+    clusterMaster;
+    _stores = {};
+    cleanups = {};
+    constructor(options = {}) {
+        const { clusterMasterPort, shutdownTimeout, actionTimeout, timeout, log_level_terafoundation } = options;
+        this.setupId = newId('setup', true);
+        this.assetDir = tmpAssetDir;
+        this.sysconfig = newSysConfig({
+            clusterName,
+            assetDir: this.assetDir,
+            clusterMasterPort,
+            actionTimeout,
+            timeout,
+            shutdownTimeout,
+            log_level_terafoundation
+        });
+        this.config = newConfig(options);
+        this.cleanups[this.setupId] = () => this.cleanup();
+        this.assignment = options.assignment || 'worker';
+    }
+    // make sure we cleanup if any test fails to cleanup properly
+    async cleanupAll(withEs = false) {
+        const count = Object.keys(this.cleanups).length;
+        if (!count)
+            return;
+        const fns = Object.keys(this.cleanups).map(async (name) => {
+            const fn = this.cleanups[name];
+            try {
+                await fn();
+            }
+            catch (err) {
+                console.error(`Failed to cleanup ${name}`, err);
+            }
+            delete this.cleanups[name];
+        });
+        await Promise.all(fns);
+        try {
+            cleanupTempDirs();
+        }
+        catch (err) {
+            console.error(err);
+        }
+        if (withEs && Object.keys(this._stores).length) {
+            try {
+                await Promise.all(Object.values(this._stores).map((store) => store.shutdown(true)));
+            }
+            catch (err) {
+                console.error(err);
+            }
+        }
+    }
+    async waitForCleanup() {
+        return pWhile(async () => !Object.keys(this.cleanups).length, {
+            name: 'Test Context',
+            timeoutMs: 3000
+        });
+    }
+    async initialize(makeItReal = false, initOptions = {}) {
+        this.context = await makeTerafoundationContext({ sysconfig: this.sysconfig });
+        this.context.assignment = this.assignment;
+        this.events = this.context.apis.foundation.getSystemEvents();
+        if (makeItReal) {
+            await Promise.all([
+                this.addJobStore(),
+                this.addExStore(),
+                this.addStateStore()
+            ]);
+            const { ex } = await initializeTestExecution(Object.assign({
+                context: this.context,
+                config: this.config,
+                stores: this._stores,
+            }, initOptions));
+            this.config = ex;
+        }
+        this.executionContext = await makeExecutionContext(this.context, this.config);
+        // @ts-expect-error
+        this.nodeId = this.executionContext.config.node_id;
+        this.exId = this.executionContext.config.ex_id;
+        this.jobId = this.executionContext.config.job_id;
+    }
+    get stores() {
+        return this._stores;
+    }
+    async addClusterMaster() {
+        if (this.clusterMaster)
+            return this.clusterMaster;
+        const port = await findPort();
+        const networkLatencyBuffer = get(this.context, 'sysconfig.teraslice.network_latency_buffer');
+        const actionTimeout = get(this.context, 'sysconfig.teraslice.action_timeout');
+        const nodeDisconnectTimeout = get(this.context, 'sysconfig.teraslice.node_disconnect_timeout');
+        this.clusterMaster = new ClusterMaster.Server({
+            port,
+            networkLatencyBuffer,
+            actionTimeout,
+            nodeDisconnectTimeout
+        });
+        await this.clusterMaster.start();
+        this.context.sysconfig.teraslice.port = port;
+        this.attachCleanup(() => this.clusterMaster.shutdown());
+        return this.clusterMaster;
+    }
+    attachCleanup(fn) {
+        this._cleanupFns.push(fn);
+    }
+    async saveAsset(assetDir, cleanup) {
+        await this.addAssetStore();
+        const exists = await fs.pathExists(assetDir);
+        if (!exists) {
+            const err = new Error(`Asset Directory ${assetDir} does not exist`);
+            throw err;
+        }
+        const assetZip = await zipDirectory(assetDir);
+        if (!this._stores.assetsStorage) {
+            throw new Error('Assets storage is not setup');
+        }
+        const { assetId } = await this._stores.assetsStorage.save(assetZip);
+        if (cleanup) {
+            await fs.remove(path.join(this.assetDir, assetId));
+        }
+        return assetId;
+    }
+    async newSlice() {
+        const sliceConfig = newTestSlice({ request: { example: 'slice-data' } });
+        await this.addStateStore();
+        await this._stores.stateStorage.createState(this.exId, sliceConfig, 'start');
+        return sliceConfig;
+    }
+    async addAssetStore() {
+        if (this._stores.assetsStorage) {
+            return this._stores.assetsStorage;
+        }
+        this._stores.assetsStorage = new AssetsStorage(this.context);
+        await this._stores.assetsStorage.initialize();
+        delete this.context.apis.assets;
+        return this._stores.assetsStorage;
+    }
+    async addStateStore() {
+        if (this._stores.stateStorage) {
+            return this._stores.stateStorage;
+        }
+        this._stores.stateStorage = new StateStorage(this.context);
+        await this._stores.stateStorage.initialize();
+        return this._stores.stateStorage;
+    }
+    async addAnalyticsStore() {
+        if (this._stores.analyticsStorage) {
+            return this._stores.analyticsStorage;
+        }
+        this._stores.analyticsStorage = new AnalyticsStorage(this.context);
+        await this._stores.analyticsStorage.initialize();
+        return this._stores.analyticsStorage;
+    }
+    async addJobStore() {
+        if (this._stores.jobsStorage) {
+            return this._stores.jobsStorage;
+        }
+        this._stores.jobsStorage = new JobsStorage(this.context);
+        await this._stores.jobsStorage.initialize();
+        return this._stores.jobsStorage;
+    }
+    async addExStore() {
+        if (this._stores.executionStorage) {
+            return this._stores.executionStorage;
+        }
+        this._stores.executionStorage = new ExecutionStorage(this.context);
+        await this._stores.executionStorage.initialize();
+        return this._stores.executionStorage;
+    }
+    async cleanup() {
+        if (this.clean)
+            return;
+        await Promise.all(this._cleanupFns.map((fn) => fn()));
+        this._cleanupFns.length = 0;
+        this.events.removeAllListeners();
+        delete this.cleanups[this.setupId];
+        this.clean = true;
+    }
+}
+//# sourceMappingURL=test-context.js.map

package/dist/test/workers/helpers/zip-directory.js

@@ -0,0 +1,25 @@
+import path from 'node:path';
+import { random } from '@terascope/utils';
+// @ts-expect-error
+import BufferStreams from 'bufferstreams';
+import archiver from 'archiver';
+import { newId } from '../../../src/lib/utils/id_utils.js';
+export function zipDirectory(dir) {
+    return new Promise((resolve, reject) => {
+        const archive = archiver('zip', { zlib: { level: 9 } });
+        archive.append(JSON.stringify({
+            name: path.basename(dir),
+            version: `${random(0, 100)}.${random(0, 100)}.${random(0, 100)}`,
+            someProp: newId()
+        }, null, 4), { name: 'asset.json' });
+        archive.pipe(new BufferStreams((err, buf) => {
+            if (err) {
+                reject(err);
+                return;
+            }
+            resolve(buf);
+        }));
+        archive.directory(dir, 'asset').finalize();
+    });
+}
+//# sourceMappingURL=zip-directory.js.map