teraslice 3.3.0 → 3.3.2

package/dist/src/lib/utils/id_utils.js

@@ -1,15 +0,0 @@
-import { trim, padEnd } from '@terascope/core-utils';
-import { customAlphabet } from 'nanoid';
-export function newId(prefix, lowerCase = false, length = 15) {
-    let characters = '-0123456789abcdefghijklmnopqrstuvwxyz';
-    if (!lowerCase) {
-        characters += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
-    }
-    let id = trim(customAlphabet(characters, length)(), '-');
-    id = padEnd(id, length, 'abcdefghijklmnopqrstuvwxyz');
-    if (prefix) {
-        return `${prefix}-${id}`;
-    }
-    return id;
-}
-//# sourceMappingURL=id_utils.js.map

package/dist/src/lib/utils/port_utils.js

@@ -1,32 +0,0 @@
-import { get, toIntegerOrThrow, shuffle, range } from '@terascope/core-utils';
-import getPort from 'get-port';
-const usedPorts = [];
-/**
- * Return a random open port between start(inclusive) and end(exclusive), excluding usedPorts.
- * If all ports in that range are being used a random port between 1024 and 65535
- * will be returned instead.
- * @param {PortOptions} options Start, end, and assetsPort to exclude
- * @returns {number}
- */
-export async function findPort(options = {}) {
-    const { start = 1024, end = 65536, assetsPort = 8003 } = options;
-    usedPorts.push(assetsPort);
-    const port = await getPort({
-        port: shuffle(range(start, end)),
-        exclude: usedPorts
-    });
-    usedPorts.push(port);
-    if (port)
-        return port;
-    throw new Error(`No available port between ${start}-${end}`);
-}
-export function getPorts(context) {
-    const portConfig = get(context, 'sysconfig.teraslice.slicer_port_range');
-    const dataArray = portConfig.split(':', 2);
-    const assetsPort = toIntegerOrThrow(dataArray[0]);
-    const start = assetsPort + 1;
-    // range end is exclusive, so we need to add one
-    const end = toIntegerOrThrow(dataArray[1]) + 1;
-    return { assetsPort, start, end };
-}
-//# sourceMappingURL=port_utils.js.map

package/dist/src/lib/workers/assets/index.js

@@ -1,3 +0,0 @@
-export * from './loader.js';
-export * from './spawn.js';
-//# sourceMappingURL=index.js.map

package/dist/src/lib/workers/assets/loader-executable.js

@@ -1,40 +0,0 @@
-import { getFullErrorStack, pDelay } from '@terascope/core-utils';
-import { fileURLToPath } from 'node:url';
-import { AssetLoader } from './loader.js';
-import { makeTerafoundationContext } from '../context/terafoundation-context.js';
-import { safeDecode } from '../../utils/encoding_utils.js';
-async function loadAssets(context, assets) {
-    const assetLoader = new AssetLoader(context, assets);
-    return assetLoader.load();
-}
-if (import.meta.url.startsWith('file:')) {
-    const modulePath = fileURLToPath(import.meta.url);
-    const executePath = process.argv[1];
-    if (executePath === modulePath) {
-        const context = await makeTerafoundationContext();
-        const assets = safeDecode(process.env.ASSETS);
-        (async () => {
-            try {
-                const assetIds = await loadAssets(context, assets);
-                // @ts-expect-error
-                process.send({
-                    assetIds: assetIds || [],
-                    success: true
-                });
-            }
-            catch (err) {
-                // @ts-expect-error
-                process.send({
-                    error: getFullErrorStack(err),
-                    success: false
-                });
-                process.exitCode = 1;
-            }
-            finally {
-                await pDelay(500);
-                process.exit();
-            }
-        })();
-    }
-}
-//# sourceMappingURL=loader-executable.js.map

package/dist/src/lib/workers/assets/loader.js

@@ -1,73 +0,0 @@
-import path from 'node:path';
-import fs from 'node:fs';
-import { get, isEmpty } from '@terascope/core-utils';
-import { AssetsStorage } from '../../storage/index.js';
-import { makeLogger } from '../helpers/terafoundation.js';
-import { saveAsset } from '../../utils/file_utils.js';
-import { getBackendConfig } from '../../storage/assets.js';
-export class AssetLoader {
-    context;
-    logger;
-    assets;
-    assetsDirectory;
-    assetsStorage;
-    isShuttingDown = false;
-    constructor(context, assets = []) {
-        this.context = context;
-        this.logger = makeLogger(context, 'asset_loader');
-        this.assets = assets;
-        // TODO: need guard here
-        this.assetsDirectory = get(context, 'sysconfig.teraslice.assets_directory');
-        this.assetsStorage = new AssetsStorage(context);
-    }
-    async load() {
-        // no need to load assets
-        if (isEmpty(this.assets))
-            return [];
-        const { assignment } = this.context;
-        const isWorker = assignment && assignment !== 'cluster_master';
-        // A worker should already have the assets loaded,
-        // so we should only log at info level if it gets here
-        // If it is not worker this message isn't important so it
-        // should be a debug log message
-        if (isWorker) {
-            this.logger.info('Loading assets...', this.assets);
-        }
-        else {
-            this.logger.debug('Loading assets...', this.assets);
-        }
-        await this.assetsStorage.initialize();
-        const idArray = await this.assetsStorage.parseAssetsArray(this.assets);
-        const actualIds = await Promise.all(idArray.map(async (assetIdentifier) => {
-            const assetDir = path.join(this.assetsDirectory, assetIdentifier);
-            const downloaded = fs.existsSync(assetDir);
-            // need to return the id to the assets array sent back
-            if (downloaded)
-                return assetIdentifier;
-            const assetRecord = await this.assetsStorage.get(assetIdentifier);
-            this.logger.info(`loading assets: ${assetIdentifier}`);
-            let buff;
-            const { context, logger } = this;
-            const connectionType = getBackendConfig(context, logger).assetConnectionType;
-            if (connectionType === 's3') {
-                buff = assetRecord.blob;
-            }
-            else {
-                if (!assetRecord.blob) {
-                    throw new Error(`No asset blob found in opensearch index for asset identifier: ${assetIdentifier}.\n`
-                        + `Confirm that "teraslice.ASSET_STORAGE_CONNECTION_TYPE" should be ${connectionType}.\n`
-                        + 'Then try deleting and redeploying the asset.');
-                }
-                buff = Buffer.from(assetRecord.blob, 'base64');
-            }
-            const saveResult = await saveAsset(this.logger, this.assetsDirectory, assetIdentifier, buff);
-            return saveResult.id;
-        }));
-        const matches = JSON.stringify(actualIds) === JSON.stringify(this.assets);
-        if (!matches && isWorker) {
-            this.logger.warn(`asset loader expected any array of the asset ids but got ${JSON.stringify(this.assets)}`);
-        }
-        return idArray;
-    }
-}
-//# sourceMappingURL=loader.js.map

package/dist/src/lib/workers/assets/spawn.js

@@ -1,55 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { fork } from 'node:child_process';
-import { isEmpty, get, has } from '@terascope/core-utils';
-import { makeLogger } from '../helpers/terafoundation.js';
-import { safeEncode } from '../../utils/encoding_utils.js';
-const filePath = fileURLToPath(new URL('.', import.meta.url));
-const loaderPath = path.join(filePath, './loader-executable.js');
-export async function spawnAssetLoader(assets, context) {
-    // if assets is empty return early
-    if (isEmpty(assets)) {
-        return [];
-    }
-    // if the assets are ids and are already loaded, return early
-    if (context) {
-        const assetDir = get(context, 'sysconfig.teraslice.assets_directory');
-        const alreadyExists = assets.every((id) => {
-            const assetPath = path.join(assetDir, id);
-            return fs.existsSync(assetPath);
-        });
-        if (alreadyExists) {
-            const logger = makeLogger(context, 'asset_loader');
-            logger.debug('assets already loaded...');
-            return assets;
-        }
-    }
-    return new Promise((resolve, reject) => {
-        let message;
-        const child = fork(loaderPath, process.argv, {
-            stdio: 'inherit',
-            env: Object.assign({}, process.env, {
-                ASSETS: safeEncode(assets)
-            })
-        });
-        child.on('message', (msg) => {
-            if (has(msg, 'success')) {
-                message = msg;
-            }
-        });
-        child.on('close', (code) => {
-            const isSuccess = get(message, 'success', false) && code === 0;
-            if (!isSuccess) {
-                const errMsg = get(message, 'error', `exit code ${code}`);
-                const errOOM = 'If running out of memory, try consider increasing the memory allocation for the process by adding/modifying the "memory_execution_controller" or "resources_limits_memory" (for workers) field in the job file.';
-                const error = new Error(`Failure to get assets, caused by ${errMsg}\n${errOOM}`);
-                reject(error);
-            }
-            else {
-                resolve(get(message, 'assetIds', []));
-            }
-        });
-    });
-}
-//# sourceMappingURL=spawn.js.map

package/dist/src/lib/workers/context/execution-context.js

@@ -1,12 +0,0 @@
-import { makeExecutionContext as makeEx } from '@terascope/job-components';
-import { spawnAssetLoader } from '../assets/spawn.js';
-export async function makeExecutionContext(context, executionConfig) {
-    const assetIds = await spawnAssetLoader(executionConfig.assets, context);
-    const config = {
-        context,
-        executionConfig,
-        assetIds,
-    };
-    return makeEx(config);
-}
-//# sourceMappingURL=execution-context.js.map

package/dist/src/lib/workers/context/terafoundation-context.js

@@ -1,8 +0,0 @@
-import { ProcessContext } from 'terafoundation';
-import { getTerasliceConfig } from '../../config/index.js';
-export async function makeTerafoundationContext({ sysconfig } = {}) {
-    return ProcessContext.createContext(getTerasliceConfig(), sysconfig
-        ? { configfile: sysconfig }
-        : undefined);
-}
-//# sourceMappingURL=terafoundation-context.js.map

package/dist/src/lib/workers/execution-controller/execution-analytics.js

@@ -1,188 +0,0 @@
-import { get, has, makeISODate } from '@terascope/core-utils';
-import { makeLogger } from '../helpers/terafoundation.js';
-/**
- * @typedef ExecutionStats
- * @property workers_available {number}
- * @property workers_active {number}
- * @property workers_joined {number}
- * @property workers_reconnected {number}
- * @property workers_disconnected {number}
- * @property job_duration {number}
- * @property failed {number}
- * @property subslices {number}
- * @property queued {number}
- * @property slice_range_expansion {number}
- * @property processed {number}
- * @property slicers {number}
- * @property subslice_by_key {number}
- * @property started {String} a date string
- * @property queuing_complete {String} a date string
-*/
-export class ExecutionAnalytics {
-    logger;
-    events;
-    executionContext;
-    client;
-    analyticsRate;
-    _handlers;
-    _pushing;
-    _started;
-    isShutdown;
-    executionAnalytics;
-    pushedAnalytics;
-    analyticsInterval;
-    constructor(context, executionContext, client) {
-        this.logger = makeLogger(context, 'execution_analytics');
-        this.events = context.apis.foundation.getSystemEvents();
-        this.executionContext = executionContext;
-        this.client = client;
-        this.analyticsRate = get(context, 'sysconfig.teraslice.analytics_rate');
-        this._handlers = {};
-        this._pushing = false;
-        /** @property {ExecutionStats} */
-        this.executionAnalytics = {
-            workers_available: 0,
-            workers_active: 0,
-            workers_joined: 0,
-            workers_reconnected: 0,
-            workers_disconnected: 0,
-            job_duration: 0,
-            failed: 0,
-            subslices: 0,
-            queued: 0,
-            slice_range_expansion: 0,
-            processed: 0,
-            slicers: 0,
-            subslice_by_key: 0,
-            started: undefined,
-            queuing_complete: undefined
-        };
-        this.pushedAnalytics = {
-            processed: 0,
-            failed: 0,
-            queued: 0,
-            job_duration: 0,
-            workers_joined: 0,
-            workers_disconnected: 0,
-            workers_reconnected: 0
-        };
-        this._registerHandlers();
-        this._started = false;
-        this.isShutdown = false;
-    }
-    /**
-     * Used to determine if the analytics should be sent
-    */
-    get isRunning() {
-        if (this.isShutdown)
-            return false;
-        if (!this._started)
-            return false;
-        return this.client.ready;
-    }
-    start() {
-        const { exId, jobId } = this.executionContext;
-        const { name } = this.executionContext.config;
-        this.set('started', makeISODate());
-        this.client.onExecutionAnalytics(() => ({
-            name,
-            ex_id: exId,
-            job_id: jobId,
-            stats: this.getAnalytics()
-        }));
-        this._started = true;
-        this.analyticsInterval = setInterval(() => {
-            if (!this.isRunning)
-                return;
-            this._pushAnalytics();
-        }, this.analyticsRate);
-    }
-    set(key, value) {
-        this.executionAnalytics[key] = value;
-    }
-    increment(key) {
-        if (!has(this.executionAnalytics, key)) {
-            this.logger.warn(`"${key}" is not a valid analytics property`);
-            return;
-        }
-        this.executionAnalytics[key] += 1;
-    }
-    get(key) {
-        if (key === 'started' || key === 'queuing_complete') {
-            return this.executionAnalytics[key];
-        }
-        if (key === undefined) {
-            return this.executionAnalytics;
-        }
-        return this.executionAnalytics[key];
-    }
-    getAnalytics() {
-        return Object.assign({}, this.executionAnalytics);
-    }
-    async shutdown(timeout) {
-        this.isShutdown = true;
-        clearInterval(this.analyticsInterval);
-        Object.entries(this._handlers).forEach(([event, handler]) => {
-            this.events.removeListener(event, handler);
-            // @ts-expect-error
-            this._handlers[event] = null;
-        });
-        await this._pushAnalytics(timeout);
-    }
-    async _pushAnalytics(timeout = Math.round(this.analyticsRate / 2)) {
-        if (this._pushing)
-            return;
-        this._pushing = true;
-        const analytics = this.getAnalytics();
-        // save a copy of what we push so we can emit diffs
-        const diffs = {};
-        const copy = {};
-        Object.entries(this.pushedAnalytics).forEach(([field, value]) => {
-            diffs[field] = analytics[field] - value;
-            copy[field] = analytics[field];
-        });
-        const response = await this.client.sendClusterAnalytics(diffs, timeout);
-        const recorded = get(response, 'payload.recorded', false);
-        this._pushing = false;
-        if (!recorded && this.isRunning) {
-            this.logger.warn('cluster master did not record the cluster analytics');
-            return;
-        }
-        this.pushedAnalytics = copy;
-    }
-    _registerHandlers() {
-        const { exId } = this.executionContext;
-        this._handlers['slicer:slice:recursion'] = () => {
-            this.logger.trace('id subslicing has occurred');
-            this.increment('subslices');
-        };
-        this._handlers['slicer:slice:range_expansion'] = () => {
-            this.logger.trace('a slice range expansion has occurred');
-            this.increment('slice_range_expansion');
-        };
-        this._handlers['slicer:subslice'] = () => {
-            this.logger.warn(`slicer for execution: ${exId} is subslicing by key`);
-            this.increment('subslice_by_key');
-        };
-        this._handlers['slicers:queued'] = (queueSize) => {
-            this.set('queued', queueSize);
-        };
-        this._handlers['slicers:registered'] = (count) => {
-            this.set('slicers', count);
-        };
-        this._handlers['slice:success'] = () => {
-            this.increment('processed');
-        };
-        this._handlers['slice:failure'] = () => {
-            this.increment('processed');
-            this.increment('failed');
-        };
-        this._handlers['slicers:finished'] = () => {
-            this.set('queuing_complete', makeISODate());
-        };
-        Object.entries(this._handlers).forEach(([event, handler]) => {
-            this.events.on(event, handler);
-        });
-    }
-}
-//# sourceMappingURL=execution-analytics.js.map