teraslice 3.3.0 → 3.3.2

package/dist/test/node_master-spec.js

@@ -1,194 +0,0 @@
-import { EventEmitter } from 'node:events';
-import { debugLogger } from '@terascope/core-utils';
-import { Server as SocketServer } from 'socket.io';
-import { nodeMaster } from '../src/lib/cluster/node_master.js';
-process.env.assignment = 'node_master';
-describe('Node master', () => {
-    let eventEmitter;
-    const logger = debugLogger('node-master-spec');
-    const jobJson = {
-        name: 'Data Generator',
-        lifecycle: 'persistent',
-        workers: 1,
-        assets: ['elasticsearch', 'standard'],
-        apis: [
-            {
-                _name: 'elasticsearch_sender_api',
-                index: 'example-logs',
-                type: 'events',
-                size: 5000
-            }
-        ],
-        operations: [
-            {
-                _op: 'data_generator',
-                size: 5000
-            },
-            {
-                _op: 'elasticsearch_bulk',
-                _api_name: 'elasticsearch_sender_api'
-            }
-        ]
-    };
-    const context = {
-        sysconfig: {
-            teraslice: {
-                master_hostname: 'localhost',
-                port: 9999,
-                shutdown_timeout: 400,
-                workers: 10,
-                assets_directory: '',
-                slicer_port_range: '5678:5700',
-                action_timeout: 500
-            },
-            _nodeName: 'testNodeName'
-        },
-        apis: {
-            foundation: {
-                makeLogger: () => logger,
-                getSystemEvents: () => eventEmitter
-            },
-            registerAPI: () => { }
-        },
-        cluster: {},
-        logger,
-        __test_job: JSON.stringify(jobJson),
-        __test_assignment: 'worker'
-    };
-    const fakeClusterMaster = new SocketServer({
-        path: '/native-clustering'
-    });
-    fakeClusterMaster.on('connection', (socket) => {
-        socket.on('node:state', (data) => {
-            eventEmitter.emit('node:state', data);
-        });
-        socket.on('node:online', (data) => {
-            eventEmitter.emit('node:online', data);
-        });
-        socket.on('messaging:response', (data) => {
-            eventEmitter.emit('messaging:response', data);
-        });
-    });
-    fakeClusterMaster.listen(9999);
-    function waitForEvent(eventName, fn) {
-        return new Promise((resolve) => {
-            eventEmitter.once(eventName, (data) => {
-                resolve(data);
-            });
-            if (fn) {
-                fn();
-            }
-        });
-    }
-    // act like the pid
-    let processCounter = 0;
-    let delayRemoval = false;
-    class ProcessWorker {
-        _isDead = false;
-        connected = true;
-        process;
-        id;
-        constructor(envConfig) {
-            processCounter += 1;
-            for (const [key, value] of Object.entries(envConfig)) {
-                this[key] = value;
-            }
-            this.process = {
-                connected: this.connected,
-                pid: processCounter,
-                _msgSent: null,
-                kill: (signal) => {
-                    this._isDead = true;
-                    this.connected = false;
-                    if (delayRemoval && signal === 'SIGTERM') {
-                        setTimeout(() => eventEmitter.emit('deleteWorker', this.id), 500);
-                    }
-                    else {
-                        eventEmitter.emit('deleteWorker', this.id);
-                    }
-                }
-            };
-            this.connected = true;
-        }
-        kill(signal) {
-            this.process.kill(signal);
-        }
-        isDead() {
-            return this._isDead;
-        }
-    }
-    function makeWorker(config) {
-        return new ProcessWorker(config);
-    }
-    function setUpNodeMaster() {
-        const newEmitter = new EventEmitter();
-        eventEmitter = newEmitter;
-        const startingWorkers = {
-            workers: {
-                worker1: makeWorker({
-                    id: 'worker1',
-                    ex_id: '1234',
-                    job_id: '5678',
-                    assignment: 'worker'
-                }),
-                worker2: makeWorker({
-                    id: 'worker2',
-                    ex_id: '1234',
-                    job_id: '5678',
-                    assignment: 'slicer'
-                }),
-                worker3: makeWorker({
-                    id: 'worker3',
-                    ex_id: '1234',
-                    job_id: '5678',
-                    assignment: 'worker'
-                }),
-                worker4: makeWorker({
-                    id: 'worker4',
-                    ex_id: '1212',
-                    job_id: '2323',
-                    assets: [{ id: 1234567890 }],
-                    assignment: 'worker'
-                })
-            }
-        };
-        context.apis.foundation.getSystemEvents = () => newEmitter;
-        context.cluster = Object.assign(Object.create(newEmitter), startingWorkers);
-        eventEmitter.on('deleteWorker', (id) => delete context.cluster.workers[id]);
-        return nodeMaster(context);
-    }
-    function waitFor(timeout) {
-        return new Promise((resolve) => {
-            setTimeout(() => {
-                resolve(true);
-            }, timeout);
-        });
-    }
-    function sendMsg(data) {
-        fakeClusterMaster.emit('networkMessage', data);
-    }
-    beforeEach(() => {
-        delayRemoval = false;
-    });
-    afterAll(() => fakeClusterMaster.close());
-    it('can load without throwing', () => {
-        expect(() => setUpNodeMaster()).not.toThrow();
-    });
-    it('can remove workers', async () => {
-        setUpNodeMaster();
-        const networkMsg = {
-            ex_id: '1234',
-            response: true,
-            to: 'node_master',
-            message: 'cluster:workers:remove',
-            __source: 'cluster_master',
-            payload: { workers: 1 }
-        };
-        delayRemoval = true;
-        await waitForEvent('node:online');
-        await Promise.all([sendMsg(networkMsg), waitFor(700)]);
-        const processesAlive = Object.keys(context.cluster.workers).length;
-        expect(processesAlive).toEqual(3);
-    });
-});
-//# sourceMappingURL=node_master-spec.js.map

package/dist/test/services/api-spec.js

@@ -1,79 +0,0 @@
-import got from 'got';
-import express from 'express';
-import { TestContext } from '@terascope/job-components';
-import { findPort } from '../../src/lib/utils/port_utils.js';
-import { ApiService } from '../../src/lib/cluster/services/api.js';
-import { getPackageJSON } from '../../src/lib/utils/file_utils.js';
-describe('HTTP API', () => {
-    const { version } = getPackageJSON();
-    const app = express();
-    const assetsUrl = 'http://example.asset:1234';
-    const context = new TestContext('http-api');
-    context.stores = {
-        stateStorage: {},
-        executionStorage: {},
-        jobsStorage: {},
-    };
-    context.services = {
-        clusterService: {},
-        executionService: {},
-        jobsService: {},
-    };
-    let api;
-    let port;
-    let baseUrl;
-    let server;
-    beforeAll(async () => {
-        port = await findPort();
-        baseUrl = `http://localhost:${port}`;
-        api = new ApiService(context, { assetsUrl, app });
-        await api.initialize();
-        await new Promise((resolve, reject) => {
-            server = app.listen(port, (err) => {
-                if (err)
-                    reject(err);
-                else
-                    resolve(true);
-            });
-        });
-        await api.initialize();
-    });
-    afterAll(async () => {
-        if (api) {
-            await api.shutdown();
-        }
-        if (server) {
-            await new Promise((resolve, reject) => {
-                server.close((err) => {
-                    if (err)
-                        reject(err);
-                    else
-                        resolve(true);
-                });
-            });
-        }
-    });
-    describe('GET /', () => {
-        it('should the correct response', async () => {
-            let response;
-            try {
-                response = await got(baseUrl, {
-                    responseType: 'json',
-                    throwHttpErrors: true
-                });
-                expect(response.body).toMatchObject({
-                    arch: context.arch,
-                    clustering_type: context.sysconfig.teraslice.cluster_manager_type,
-                    name: context.sysconfig.teraslice.name,
-                    node_version: process.version,
-                    platform: context.platform,
-                    teraslice_version: `v${version}`
-                });
-            }
-            catch (err) {
-                expect(err.stack).toBeNil();
-            }
-        });
-    });
-});
-//# sourceMappingURL=api-spec.js.map

package/dist/test/services/assets-spec.js

@@ -1,158 +0,0 @@
-import { TestContext } from '@terascope/job-components';
-import fs from 'node:fs';
-import got from 'got';
-import { createClient } from '@terascope/opensearch-client';
-import { createS3Client } from '@terascope/file-asset-apis';
-import { AssetsService } from '../../src/lib/cluster/services/assets';
-import { SEARCH_TEST_HOST, TEST_INDEX_PREFIX } from '../test.config';
-import { findPort } from '../../src/lib/utils/port_utils.js';
-describe('Assets Service', () => {
-    const contextOptions = {
-        assignment: 'assets_service',
-        clients: [
-            {
-                type: 'elasticsearch-next',
-                async createClient(customConfig, logger) {
-                    const { client } = await createClient(customConfig, logger);
-                    return { client, logger };
-                },
-                endpoint: 'default'
-            },
-            {
-                type: 's3',
-                createClient: async (customConfig, logger) => {
-                    const client = await createS3Client(customConfig, logger);
-                    return { client, logger };
-                },
-                endpoint: 'default'
-            }
-        ]
-    };
-    /// It's important to keep the test context name unique
-    /// This is so we don't share indices and buckets with other test suites
-    const context = new TestContext(`${TEST_INDEX_PREFIX}assets-spec-test`, contextOptions);
-    context.sysconfig.terafoundation = Object.assign(context.sysconfig.terafoundation, {
-        prom_metrics_enabled: false,
-        prom_metrics_port: 3333,
-        prom_metrics_add_default: true,
-        connectors: {
-            s3: {
-                default: {
-                    endpoint: process.env.MINIO_HOST,
-                    accessKeyId: process.env.MINIO_ACCESS_KEY,
-                    secretAccessKey: process.env.MINIO_SECRET_KEY,
-                    forcePathStyle: true,
-                    sslEnabled: false,
-                    region: 'test-region'
-                },
-            },
-            'elasticsearch-next': {
-                default: {
-                    node: [SEARCH_TEST_HOST]
-                },
-            },
-        }
-    });
-    context.sysconfig.teraslice.asset_storage_connection_type = 's3';
-    context.sysconfig.teraslice.asset_storage_connection = 'default';
-    context.sysconfig.teraslice.asset_storage_bucket = 'assets-spec-test-bucket';
-    context.sysconfig.teraslice.api_response_timeout = 30000;
-    let service;
-    beforeAll(async () => {
-        /// Setting port for the asset service
-        process.env.port = (await findPort()).toString();
-        service = new AssetsService(context);
-        await service.initialize();
-    });
-    afterAll(async () => {
-        // Clean up asset process
-        await service.shutdown();
-    });
-    describe('getS3AssetStatus function', () => {
-        it('Should return correct info if given valid lists', async () => {
-            const s3List = [
-                { File: '123.zip', Size: '12345' },
-                { File: 'abc.zip', Size: '56873' },
-                { File: 'foo.zip', Size: '1' },
-            ];
-            const esList = [
-                {
-                    _created: '500BC',
-                    id: '123',
-                    name: 'asset-123',
-                    version: '1.0.0',
-                    description: 'A description',
-                    node_version: 19,
-                    platform: '',
-                    arch: 'arm64'
-                },
-                {
-                    _created: '2020-12-20T17:16:44.036Z',
-                    id: 'abc',
-                    name: 'asset-abc',
-                    version: '1.0.0',
-                    description: 'A description',
-                    node_version: 19,
-                    platform: '',
-                    arch: ''
-                },
-                {
-                    _created: '2024-03-20T17:16:05.037Z',
-                    id: 'foo',
-                    name: 'foo-asset',
-                    version: '6.7.1',
-                    description: 'test',
-                    node_version: 18,
-                    platform: '',
-                    arch: ''
-                }
-            ];
-            // Do this because getS3AssetStatus is a private function
-            // @ts-expect-error
-            const result = service.getS3AssetStatus(s3List, esList);
-            expect(result[0].external_storage).toEqual('available');
-            expect(result[1].external_storage).toEqual('available');
-            expect(result[2].external_storage).toEqual('available');
-        });
-        it('Should return "missing" if id is not available in s3List', async () => {
-            const s3List = [
-                { File: '123.zip', Size: '12345' },
-                { File: 'abc.zip', Size: '56873' },
-                { File: 'foo.zip', Size: '1' },
-            ];
-            const esList = [
-                {
-                    _created: '500BC',
-                    id: '50000',
-                    name: 'asset-123',
-                    version: '1.0.0',
-                    description: 'A description',
-                    node_version: 19,
-                    platform: '',
-                    arch: 'arm64'
-                }
-            ];
-            // Do this because getS3AssetStatus is a private function
-            // @ts-expect-error
-            const result = service.getS3AssetStatus(s3List, esList);
-            expect(result[0].external_storage).toEqual('missing');
-        });
-    });
-    describe('Testing /txt/assets enpoint', () => {
-        it('Should return proper table with specific format', async () => {
-            const filePathOne = 'packages/teraslice/test/fixtures/assets/asset-with-long-description.zip';
-            const filePathOneStream = fs.readFileSync(filePathOne);
-            // Upload file to storage
-            await service.assetsStorage.save(filePathOneStream);
-            const resultTable = await got.get(`http://localhost:${process.env.port}/txt/assets`, {
-                searchParams: {
-                    fields: 'name,version,id,description,node_version,platform,arch'
-                }
-            }).text();
-            expect(resultTable).toEqual('name                         version  id                                        description                     node_version  platform  arch\n'
-                + '---------------------------  -------  ----------------------------------------  ------------------------------  ------------  --------  ----\n'
-                + 'asset-with-long-description  3.6.3    ba53aa515e20e0c7da93bd2a373b84819fba7b2a  This description is longer tha  18                          \n');
-        });
-    });
-});
-//# sourceMappingURL=assets-spec.js.map