teraslice 2.11.0 → 2.12.0

package/dist/src/lib/utils/file_utils.js

@@ -0,0 +1,124 @@
+import { accessSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import fse from 'fs-extra';
+import semver from 'semver';
+import { Mutex } from 'async-mutex';
+import { TSError } from '@terascope/utils';
+import decompress from 'decompress';
+import { getMajorVersion } from './asset_utils.js';
+const mutex = new Mutex();
+const packagePath = path.join(process.cwd(), './package.json');
+let _packageJSON;
+export function getPackageJSON() {
+    if (_packageJSON)
+        return _packageJSON;
+    const file = readFileSync(packagePath, { encoding: 'utf8' });
+    const results = JSON.parse(file);
+    _packageJSON = results;
+    return results;
+}
+export function existsSync(filename) {
+    try {
+        accessSync(filename);
+        return true;
+    }
+    catch (ex) {
+        return false;
+    }
+}
+export function deleteDir(dirPath) {
+    return fse.remove(dirPath);
+}
+export async function verifyAssetJSON(id, newPath) {
+    const hasAssetJSONTopLevel = await fse.pathExists(path.join(newPath, 'asset.json'));
+    if (!hasAssetJSONTopLevel) {
+        const err = new TSError('asset.json was not found in root directory of asset bundle', { statusCode: 422 });
+        throw err;
+    }
+    let packageData;
+    try {
+        packageData = await fse.readJson(path.join(newPath, 'asset.json'));
+    }
+    catch (_err) {
+        const err = new TSError(_err, {
+            message: 'Failure parsing asset.json, please ensure that\'s it formatted correctly',
+            statusCode: 422
+        });
+        throw err;
+    }
+    const metadata = Object.assign({ id }, packageData);
+    if (!metadata.name) {
+        throw new Error('Missing name');
+    }
+    metadata.version = semver.clean(metadata.version);
+    if (!semver.valid(metadata.version)) {
+        throw new Error(`Invalid version "${metadata.version}"`);
+    }
+    /**
+         * If node_version, platform, or arch is set to a falsey
+         * value we should delete it so it is considered a wildcard.
+         *
+         * This is useful for making an asset bundle that isn't
+         * locked down.
+        */
+    if (metadata.node_version) {
+        metadata.node_version = getMajorVersion(metadata.node_version);
+    }
+    else {
+        delete metadata.node_version;
+    }
+    if (!metadata.platform) {
+        delete metadata.platform;
+    }
+    if (!metadata.arch) {
+        delete metadata.arch;
+    }
+    if (metadata.minimum_teraslice_version) {
+        const terasliceVersion = getPackageJSON().version;
+        if (semver.gt(metadata.minimum_teraslice_version, terasliceVersion)) {
+            throw new Error(`Asset requires teraslice version ${metadata.minimum_teraslice_version} or greater.`);
+        }
+    }
+    return metadata;
+}
+async function _saveAsset(logger, assetsPath, id, binaryData, metaCheck) {
+    const newPath = path.join(assetsPath, id);
+    try {
+        if (await fse.pathExists(newPath)) {
+            await fse.emptyDir(newPath);
+        }
+        else {
+            await fse.mkdir(newPath);
+        }
+        logger.info(`decompressing and saving asset ${id} to ${newPath}`);
+        await decompress(binaryData, newPath);
+        logger.info(`decompressed asset ${id} to ${newPath}`);
+        const metadata = await verifyAssetJSON(id, newPath);
+        logger.info(`asset ${id} saved to file ${newPath}`, metadata);
+        // storage/assets save fn needs to check the return metadata for uniqueness
+        if (metaCheck) {
+            return await metaCheck(metadata);
+        }
+        return metadata;
+    }
+    catch (err) {
+        await deleteDir(newPath);
+        throw err;
+    }
+}
+export async function saveAsset(logger, assetsPath, id, binaryData, metaCheck) {
+    return mutex.runExclusive(() => _saveAsset(logger, assetsPath, id, binaryData, metaCheck));
+}
+/**
+ * Check if a buffer contains a zip file.
+ * Note that file extension and mime type are not checked.
+ * Any file type that is actually a zip (.xlsx, .jar, .apk,
+ * .epub) will return true.
+ * @param {Buffer} buffer A buffer containing a file file
+ * @returns {boolean}
+ */
+export function isZipFile(buffer) {
+    const zipSignature = [0x50, 0x4B, 0x03, 0x04];
+    return zipSignature.every((byte, index) => buffer[index] === byte);
+}
+//# sourceMappingURL=file_utils.js.map

package/dist/src/lib/utils/id_utils.js

@@ -0,0 +1,15 @@
+import { trim, padEnd } from '@terascope/utils';
+import { customAlphabet } from 'nanoid';
+export function newId(prefix, lowerCase = false, length = 15) {
+    let characters = '-0123456789abcdefghijklmnopqrstuvwxyz';
+    if (!lowerCase) {
+        characters += 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+    }
+    let id = trim(customAlphabet(characters, length)(), '-');
+    id = padEnd(id, length, 'abcdefghijklmnopqrstuvwxyz');
+    if (prefix) {
+        return `${prefix}-${id}`;
+    }
+    return id;
+}
+//# sourceMappingURL=id_utils.js.map

package/dist/src/lib/utils/port_utils.js

@@ -0,0 +1,32 @@
+import { get, toIntegerOrThrow, shuffle, range } from '@terascope/utils';
+import getPort from 'get-port';
+const usedPorts = [];
+/**
+ * Return a random open port between start(inclusive) and end(exclusive), excluding usedPorts.
+ * If all ports in that range are being used a random port between 1024 and 65535
+ * will be returned instead.
+ * @param {PortOptions} options Start, end, and assetsPort to exclude
+ * @returns {number}
+ */
+export async function findPort(options = {}) {
+    const { start = 1024, end = 65536, assetsPort = 8003 } = options;
+    usedPorts.push(assetsPort);
+    const port = await getPort({
+        port: shuffle(range(start, end)),
+        exclude: usedPorts
+    });
+    usedPorts.push(port);
+    if (port)
+        return port;
+    throw new Error(`No available port between ${start}-${end}`);
+}
+export function getPorts(context) {
+    const portConfig = get(context, 'sysconfig.teraslice.slicer_port_range');
+    const dataArray = portConfig.split(':', 2);
+    const assetsPort = toIntegerOrThrow(dataArray[0]);
+    const start = assetsPort + 1;
+    // range end is exclusive, so we need to add one
+    const end = toIntegerOrThrow(dataArray[1]) + 1;
+    return { assetsPort, start, end };
+}
+//# sourceMappingURL=port_utils.js.map

package/dist/src/lib/workers/assets/index.js

@@ -0,0 +1,3 @@
+export * from './loader.js';
+export * from './spawn.js';
+//# sourceMappingURL=index.js.map

package/dist/src/lib/workers/assets/loader-executable.js

@@ -0,0 +1,40 @@
+import { getFullErrorStack, pDelay } from '@terascope/utils';
+import { fileURLToPath } from 'node:url';
+import { AssetLoader } from './loader.js';
+import { makeTerafoundationContext } from '../context/terafoundation-context.js';
+import { safeDecode } from '../../utils/encoding_utils.js';
+async function loadAssets(context, assets) {
+    const assetLoader = new AssetLoader(context, assets);
+    return assetLoader.load();
+}
+if (import.meta.url.startsWith('file:')) {
+    const modulePath = fileURLToPath(import.meta.url);
+    const executePath = process.argv[1];
+    if (executePath === modulePath) {
+        const context = await makeTerafoundationContext();
+        const assets = safeDecode(process.env.ASSETS);
+        (async () => {
+            try {
+                const assetIds = await loadAssets(context, assets);
+                // @ts-expect-error
+                process.send({
+                    assetIds: assetIds || [],
+                    success: true
+                });
+            }
+            catch (err) {
+                // @ts-expect-error
+                process.send({
+                    error: getFullErrorStack(err),
+                    success: false
+                });
+                process.exitCode = 1;
+            }
+            finally {
+                await pDelay(500);
+                process.exit();
+            }
+        })();
+    }
+}
+//# sourceMappingURL=loader-executable.js.map

package/dist/src/lib/workers/assets/loader.js

@@ -0,0 +1,73 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { get, isEmpty } from '@terascope/utils';
+import { AssetsStorage } from '../../storage/index.js';
+import { makeLogger } from '../helpers/terafoundation.js';
+import { saveAsset } from '../../utils/file_utils.js';
+import { getBackendConfig } from '../../storage/assets.js';
+export class AssetLoader {
+    context;
+    logger;
+    assets;
+    assetsDirectory;
+    assetsStorage;
+    isShuttingDown = false;
+    constructor(context, assets = []) {
+        this.context = context;
+        this.logger = makeLogger(context, 'asset_loader');
+        this.assets = assets;
+        // TODO: need guard here
+        this.assetsDirectory = get(context, 'sysconfig.teraslice.assets_directory');
+        this.assetsStorage = new AssetsStorage(context);
+    }
+    async load() {
+        // no need to load assets
+        if (isEmpty(this.assets))
+            return [];
+        const { assignment } = this.context;
+        const isWorker = assignment && assignment !== 'cluster_master';
+        // A worker should already have the assets loaded,
+        // so we should only log at info level if it gets here
+        // If it is not worker this message isn't important so it
+        // should be a debug log message
+        if (isWorker) {
+            this.logger.info('Loading assets...', this.assets);
+        }
+        else {
+            this.logger.debug('Loading assets...', this.assets);
+        }
+        await this.assetsStorage.initialize();
+        const idArray = await this.assetsStorage.parseAssetsArray(this.assets);
+        const actualIds = await Promise.all(idArray.map(async (assetIdentifier) => {
+            const assetDir = path.join(this.assetsDirectory, assetIdentifier);
+            const downloaded = fs.existsSync(assetDir);
+            // need to return the id to the assets array sent back
+            if (downloaded)
+                return assetIdentifier;
+            const assetRecord = await this.assetsStorage.get(assetIdentifier);
+            this.logger.info(`loading assets: ${assetIdentifier}`);
+            let buff;
+            const { context, logger } = this;
+            const connectionType = getBackendConfig(context, logger).assetConnectionType;
+            if (connectionType === 's3') {
+                buff = assetRecord.blob;
+            }
+            else {
+                if (!assetRecord.blob) {
+                    throw new Error(`No asset blob found in elasticsearch index for asset identifier: ${assetIdentifier}.\n`
+                        + `Confirm that "teraslice.ASSET_STORAGE_CONNECTION_TYPE" should be ${connectionType}.\n`
+                        + 'Then try deleting and redeploying the asset.');
+                }
+                buff = Buffer.from(assetRecord.blob, 'base64');
+            }
+            const saveResult = await saveAsset(this.logger, this.assetsDirectory, assetIdentifier, buff);
+            return saveResult.id;
+        }));
+        const matches = JSON.stringify(actualIds) === JSON.stringify(this.assets);
+        if (!matches && isWorker) {
+            this.logger.warn(`asset loader expected any array of the asset ids but got ${JSON.stringify(this.assets)}`);
+        }
+        return idArray;
+    }
+}
+//# sourceMappingURL=loader.js.map

package/dist/src/lib/workers/assets/spawn.js

@@ -0,0 +1,55 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { fork } from 'node:child_process';
+import { isEmpty, get, has } from '@terascope/utils';
+import { makeLogger } from '../helpers/terafoundation.js';
+import { safeEncode } from '../../utils/encoding_utils.js';
+const filePath = fileURLToPath(new URL('.', import.meta.url));
+const loaderPath = path.join(filePath, './loader-executable.js');
+export async function spawnAssetLoader(assets, context) {
+    // if assets is empty return early
+    if (isEmpty(assets)) {
+        return [];
+    }
+    // if the assets are ids and are already loaded, return early
+    if (context) {
+        const assetDir = get(context, 'sysconfig.teraslice.assets_directory');
+        const alreadyExists = assets.every((id) => {
+            const assetPath = path.join(assetDir, id);
+            return fs.existsSync(assetPath);
+        });
+        if (alreadyExists) {
+            const logger = makeLogger(context, 'asset_loader');
+            logger.debug('assets already loaded...');
+            return assets;
+        }
+    }
+    return new Promise((resolve, reject) => {
+        let message;
+        const child = fork(loaderPath, process.argv, {
+            stdio: 'inherit',
+            env: Object.assign({}, process.env, {
+                ASSETS: safeEncode(assets)
+            })
+        });
+        child.on('message', (msg) => {
+            if (has(msg, 'success')) {
+                message = msg;
+            }
+        });
+        child.on('close', (code) => {
+            const isSuccess = get(message, 'success', false) && code === 0;
+            if (!isSuccess) {
+                const errMsg = get(message, 'error', `exit code ${code}`);
+                const errOOM = 'If running out of memory, try consider increasing the memory allocation for the process by adding/modifying the "memory_execution_controller" or "resources_limits_memory" (for workers) field in the job file.';
+                const error = new Error(`Failure to get assets, caused by ${errMsg}\n${errOOM}`);
+                reject(error);
+            }
+            else {
+                resolve(get(message, 'assetIds', []));
+            }
+        });
+    });
+}
+//# sourceMappingURL=spawn.js.map

package/dist/src/lib/workers/context/execution-context.js

@@ -0,0 +1,12 @@
+import { makeExecutionContext as makeEx } from '@terascope/job-components';
+import { spawnAssetLoader } from '../assets/spawn.js';
+export async function makeExecutionContext(context, executionConfig) {
+    const assetIds = await spawnAssetLoader(executionConfig.assets, context);
+    const config = {
+        context,
+        executionConfig,
+        assetIds,
+    };
+    return makeEx(config);
+}
+//# sourceMappingURL=execution-context.js.map

package/dist/src/lib/workers/context/terafoundation-context.js

@@ -0,0 +1,8 @@
+import { ProcessContext } from 'terafoundation';
+import { getTerasliceConfig } from '../../config/index.js';
+export async function makeTerafoundationContext({ sysconfig } = {}) {
+    return ProcessContext.createContext(getTerasliceConfig(), sysconfig
+        ? { configfile: sysconfig }
+        : undefined);
+}
+//# sourceMappingURL=terafoundation-context.js.map

package/dist/src/lib/workers/execution-controller/execution-analytics.js

@@ -0,0 +1,188 @@
+import { makeISODate, get, has } from '@terascope/utils';
+import { makeLogger } from '../helpers/terafoundation.js';
+/**
+ * @typedef ExecutionStats
+ * @property workers_available {number}
+ * @property workers_active {number}
+ * @property workers_joined {number}
+ * @property workers_reconnected {number}
+ * @property workers_disconnected {number}
+ * @property job_duration {number}
+ * @property failed {number}
+ * @property subslices {number}
+ * @property queued {number}
+ * @property slice_range_expansion {number}
+ * @property processed {number}
+ * @property slicers {number}
+ * @property subslice_by_key {number}
+ * @property started {String} a date string
+ * @property queuing_complete {String} a date string
+*/
+export class ExecutionAnalytics {
+    logger;
+    events;
+    executionContext;
+    client;
+    analyticsRate;
+    _handlers;
+    _pushing;
+    _started;
+    isShutdown;
+    executionAnalytics;
+    pushedAnalytics;
+    analyticsInterval;
+    constructor(context, executionContext, client) {
+        this.logger = makeLogger(context, 'execution_analytics');
+        this.events = context.apis.foundation.getSystemEvents();
+        this.executionContext = executionContext;
+        this.client = client;
+        this.analyticsRate = get(context, 'sysconfig.teraslice.analytics_rate');
+        this._handlers = {};
+        this._pushing = false;
+        /** @property {ExecutionStats} */
+        this.executionAnalytics = {
+            workers_available: 0,
+            workers_active: 0,
+            workers_joined: 0,
+            workers_reconnected: 0,
+            workers_disconnected: 0,
+            job_duration: 0,
+            failed: 0,
+            subslices: 0,
+            queued: 0,
+            slice_range_expansion: 0,
+            processed: 0,
+            slicers: 0,
+            subslice_by_key: 0,
+            started: undefined,
+            queuing_complete: undefined
+        };
+        this.pushedAnalytics = {
+            processed: 0,
+            failed: 0,
+            queued: 0,
+            job_duration: 0,
+            workers_joined: 0,
+            workers_disconnected: 0,
+            workers_reconnected: 0
+        };
+        this._registerHandlers();
+        this._started = false;
+        this.isShutdown = false;
+    }
+    /**
+     * Used to determine if the analytics should be sent
+    */
+    get isRunning() {
+        if (this.isShutdown)
+            return false;
+        if (!this._started)
+            return false;
+        return this.client.ready;
+    }
+    start() {
+        const { exId, jobId } = this.executionContext;
+        const { name } = this.executionContext.config;
+        this.set('started', makeISODate());
+        this.client.onExecutionAnalytics(() => ({
+            name,
+            ex_id: exId,
+            job_id: jobId,
+            stats: this.getAnalytics()
+        }));
+        this._started = true;
+        this.analyticsInterval = setInterval(() => {
+            if (!this.isRunning)
+                return;
+            this._pushAnalytics();
+        }, this.analyticsRate);
+    }
+    set(key, value) {
+        this.executionAnalytics[key] = value;
+    }
+    increment(key) {
+        if (!has(this.executionAnalytics, key)) {
+            this.logger.warn(`"${key}" is not a valid analytics property`);
+            return;
+        }
+        this.executionAnalytics[key] += 1;
+    }
+    get(key) {
+        if (key === 'started' || key === 'queuing_complete') {
+            return this.executionAnalytics[key];
+        }
+        if (key === undefined) {
+            return this.executionAnalytics;
+        }
+        return this.executionAnalytics[key];
+    }
+    getAnalytics() {
+        return Object.assign({}, this.executionAnalytics);
+    }
+    async shutdown(timeout) {
+        this.isShutdown = true;
+        clearInterval(this.analyticsInterval);
+        Object.entries(this._handlers).forEach(([event, handler]) => {
+            this.events.removeListener(event, handler);
+            // @ts-expect-error
+            this._handlers[event] = null;
+        });
+        await this._pushAnalytics(timeout);
+    }
+    async _pushAnalytics(timeout = Math.round(this.analyticsRate / 2)) {
+        if (this._pushing)
+            return;
+        this._pushing = true;
+        const analytics = this.getAnalytics();
+        // save a copy of what we push so we can emit diffs
+        const diffs = {};
+        const copy = {};
+        Object.entries(this.pushedAnalytics).forEach(([field, value]) => {
+            diffs[field] = analytics[field] - value;
+            copy[field] = analytics[field];
+        });
+        const response = await this.client.sendClusterAnalytics(diffs, timeout);
+        const recorded = get(response, 'payload.recorded', false);
+        this._pushing = false;
+        if (!recorded && this.isRunning) {
+            this.logger.warn('cluster master did not record the cluster analytics');
+            return;
+        }
+        this.pushedAnalytics = copy;
+    }
+    _registerHandlers() {
+        const { exId } = this.executionContext;
+        this._handlers['slicer:slice:recursion'] = () => {
+            this.logger.trace('id subslicing has occurred');
+            this.increment('subslices');
+        };
+        this._handlers['slicer:slice:range_expansion'] = () => {
+            this.logger.trace('a slice range expansion has occurred');
+            this.increment('slice_range_expansion');
+        };
+        this._handlers['slicer:subslice'] = () => {
+            this.logger.warn(`slicer for execution: ${exId} is subslicing by key`);
+            this.increment('subslice_by_key');
+        };
+        this._handlers['slicers:queued'] = (queueSize) => {
+            this.set('queued', queueSize);
+        };
+        this._handlers['slicers:registered'] = (count) => {
+            this.set('slicers', count);
+        };
+        this._handlers['slice:success'] = () => {
+            this.increment('processed');
+        };
+        this._handlers['slice:failure'] = () => {
+            this.increment('processed');
+            this.increment('failed');
+        };
+        this._handlers['slicers:finished'] = () => {
+            this.set('queuing_complete', makeISODate());
+        };
+        Object.entries(this._handlers).forEach(([event, handler]) => {
+            this.events.on(event, handler);
+        });
+    }
+}
+//# sourceMappingURL=execution-analytics.js.map