teraslice 0.87.1 → 0.89.0

package/lib/config/schemas/system.js

@@ -1,333 +0,0 @@
-'use strict';
-
-const ip = require('ip');
-const path = require('path');
-const {
-    isPlainObject, isString, isArray, isInteger
-} = require('@terascope/utils');
-
-const workerCount = require('os').cpus().length;
-
-/**
- * This schema object is for the Teraslice configuration settings coming from
- * its configuration file.
- */
-const schema = {
-    api_response_timeout: {
-        doc: 'HTTP response timeout for the Teraslice API server',
-        default: '5 minutes',
-        format: 'duration'
-    },
-    assets_directory: {
-        doc: 'directory to look for assets',
-        default: path.join(process.cwd(), './assets'),
-        format: (val) => {
-            if (val) {
-                if (isArray(val)) {
-                    const containStrings = val.every(isString);
-                    if (!containStrings) throw new Error('Invalid parameter assets_directory, if specified as an array, it must contain an array of strings');
-                    return;
-                }
-                if (!isString(val)) throw new Error('Invalid parameter assets_directory, it must either be a string or an array of strings');
-            }
-        }
-    },
-    assets_volume: {
-        doc: 'name of shared asset volume (k8s)',
-        default: '',
-        format: 'optional_String'
-    },
-    autoload_directory: {
-        doc: 'directory to look for assets to auto deploy when teraslice boots up',
-        default: path.join(process.cwd(), './autoload'),
-        format: 'optional_String'
-    },
-    hostname: {
-        doc: 'IP or hostname for server',
-        default: ip.address(),
-        format: 'required_String'
-    },
-    workers: {
-        doc: 'Number of workers per server',
-        default: workerCount,
-        format: 'nat'
-    },
-    master: {
-        doc: 'boolean for determining if cluster_master should live on this node',
-        default: false,
-        format: Boolean
-    },
-    master_hostname: {
-        doc:
-            'hostname where the cluster_master resides, used to notify all node_masters where to connect',
-        default: 'localhost',
-        format: 'required_String'
-    },
-    port: {
-        doc: 'port for the cluster_master to listen on',
-        default: 5678,
-        format: 'port'
-    },
-    name: {
-        doc: 'Name for the cluster itself, its used for naming log files/indices',
-        default: 'teracluster',
-        format: 'elasticsearch_Name'
-    },
-    state: {
-        doc: 'Elasticsearch cluster where job state, analytics and logs are stored',
-        default: { connection: 'default' },
-        format(val) {
-            if (!val.connection) {
-                throw new Error('state parameter must be an object with a key named "connection"');
-            }
-            if (typeof val.connection !== 'string') {
-                throw new Error(
-                    'state parameter object with a key "connection" must be of type String as the value'
-                );
-            }
-        }
-    },
-    index_settings: {
-        analytics: {
-            number_of_shards: {
-                doc: 'The number of shards for the analytics index',
-                default: 5
-            },
-            number_of_replicas: {
-                doc: 'The number of replicas for the analytics index',
-                default: 1
-            }
-        },
-        assets: {
-            number_of_shards: {
-                doc: 'The number of shards for the assets index',
-                default: 5
-            },
-            number_of_replicas: {
-                doc: 'The number of replicas for the assets index',
-                default: 1
-            }
-        },
-        jobs: {
-            number_of_shards: {
-                doc: 'The number of shards for the jobs index',
-                default: 5
-            },
-            number_of_replicas: {
-                doc: 'The number of replicas for the jobs index',
-                default: 1
-            }
-        },
-        execution: {
-            number_of_shards: {
-                doc: 'The number of shards for the execution index',
-                default: 5
-            },
-            number_of_replicas: {
-                doc: 'The number of replicas for the execution index',
-                default: 1
-            }
-        },
-        state: {
-            number_of_shards: {
-                doc: 'The number of shards for the state index',
-                default: 5
-            },
-            number_of_replicas: {
-                doc: 'The number of replicas for the state index',
-                default: 1
-            }
-        }
-    },
-    shutdown_timeout: {
-        doc:
-            'time in milliseconds for workers and slicers to finish operations before forcefully shutting down',
-        default: '1 minute',
-        format: 'duration'
-    },
-    node_disconnect_timeout: {
-        doc:
-            'time in milliseconds that the cluster  will wait untill it drops that node from state and attempts to provision the lost workers',
-        default: '5 minutes',
-        format: 'duration'
-    },
-    worker_disconnect_timeout: {
-        doc:
-            'time in milliseconds that the slicer will wait after all workers have disconnected before terminating the job',
-        default: '5 minutes',
-        format: 'duration'
-    },
-    slicer_timeout: {
-        doc:
-            'time in milliseconds that the slicer will wait for worker connection before terminating the job',
-        default: '3 minutes',
-        format: 'duration'
-    },
-    action_timeout: {
-        doc:
-            'time in milliseconds for waiting for a network message (pause/stop job, etc) to complete before throwing an error',
-        default: '2 minutes',
-        format: 'duration'
-    },
-    network_latency_buffer: {
-        doc:
-            'time in milliseconds buffer which is combined with action_timeout to determine how long a network message will wait till it throws an error',
-        default: '15 seconds',
-        format: 'duration'
-    },
-    node_state_interval: {
-        doc:
-            'time in milliseconds that indicates when the cluster master will ping nodes for their state',
-        default: '5 seconds',
-        format: 'duration'
-    },
-    analytics_rate: {
-        doc: 'time in milliseconds in which to push analytics to cluster master',
-        default: '1 minute',
-        format: 'duration'
-    },
-    slicer_allocation_attempts: {
-        doc: 'The number of times a slicer will try to be allocated before failing',
-        default: 3,
-        format: 'nat', // integer >=0 (natural number)
-    },
-    slicer_port_range: {
-        doc: 'range of ports that slicers will use per node',
-        default: '45679:46678',
-        format(val) {
-            const arr = val.split(':');
-            if (arr.length !== 2) {
-                throw new Error('slicer_port_range is formatted incorrectly');
-            }
-            arr.forEach((value) => {
-                if (isInteger(value) !== false) {
-                    throw new Error(
-                        'values specified in slicer_port_range must be a number specified as a string'
-                    );
-                }
-            });
-        }
-    },
-    index_rollover_frequency: {
-        state: {
-            doc: 'How frequently the teraslice state indices are created',
-            default: 'monthly',
-            format: ['daily', 'monthly', 'yearly']
-        },
-        analytics: {
-            doc: 'How frequently the analytics indices are created',
-            default: 'monthly',
-            format: ['daily', 'monthly', 'yearly']
-        }
-    },
-    cluster_manager_type: {
-        doc: 'determines which cluster system should be used',
-        default: 'native',
-        format: ['native', 'kubernetes']
-    },
-    cpu: {
-        doc: 'number of cpus to reserve per teraslice worker in kubernetes',
-        default: undefined,
-        format: 'Number'
-    },
-    cpu_execution_controller: {
-        doc: 'number of cpus to reserve per teraslice execution controller in kubernetes',
-        default: 0.5,
-        format: 'Number'
-    },
-    ephemeral_storage: {
-        doc: 'Add ephemeral storage volume to worker and execution controller pods',
-        default: false,
-        format: Boolean
-    },
-    memory: {
-        doc: 'memory, in bytes, to reserve per teraslice worker in kubernetes',
-        default: undefined,
-        format: 'Number'
-    },
-    memory_execution_controller: {
-        doc: 'memory, in bytes, to reserve per teraslice execution controller in kubernetes',
-        default: 512000000,
-        format: 'Number'
-    },
-    env_vars: {
-        default: {},
-        doc: 'default environment variables to set on each the teraslice worker, in the format, { "EXAMPLE": "test" }',
-        format(obj) {
-            if (!isPlainObject(obj)) {
-                throw new Error('must be object');
-            }
-            Object.entries(obj).forEach(([key, val]) => {
-                if (key == null || key === '') {
-                    throw new Error('key must be not empty');
-                }
-
-                if (val == null || val === '') {
-                    throw new Error(`value for key "${key}" must be not empty`);
-                }
-            });
-        },
-    },
-    execution_controller_targets: {
-        default: null,
-        doc: 'Specify an array of {"key": ..., "value": ...} targets for execution controllers',
-        format(arr) {
-            if (arr != null) {
-                if (!Array.isArray(arr)) {
-                    throw new Error('labels is required to be an array');
-                    // FIXME: improve input and error handling
-                }
-            }
-        }
-    },
-    kubernetes_api_poll_delay: {
-        doc: 'Specify the delay between attempts to poll the kubernetes API',
-        default: '1 second',
-        format: 'duration'
-    },
-    kubernetes_image: {
-        doc: 'Specify a custom image name for kubernetes, this only applies to kubernetes systems',
-        default: 'terascope/teraslice',
-        format: 'optional_String'
-    },
-    kubernetes_namespace: {
-        doc: 'Specify a custom kubernetes namespace, this only applies to kubernetes systems',
-        default: 'default',
-        format: 'optional_String'
-    },
-    kubernetes_overrides_enabled: {
-        doc: '',
-        default: false,
-        format: Boolean
-    },
-    kubernetes_priority_class_name: {
-        doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
-        default: undefined,
-        format: 'optional_String'
-    },
-    kubernetes_config_map_name: {
-        doc: 'Specify the name of the Kubernetes ConfigMap used to configure worker pods',
-        default: 'teraslice-worker',
-        format: 'optional_String'
-    },
-    kubernetes_image_pull_secret: {
-        doc: 'Name of Kubernetes secret used to pull docker images from private repository',
-        default: '',
-        format: 'optional_String'
-    },
-    kubernetes_worker_antiaffinity: {
-        doc: 'Enable Teraslice woker pod AntiAffinity in Kubernetes',
-        default: false,
-        format: Boolean
-    }
-};
-
-function configSchema() {
-    return { teraslice: schema };
-}
-
-module.exports = {
-    configSchema,
-    config_schema: configSchema,
-    schema
-};

package/lib/processors/save_file/index.js

@@ -1,9 +0,0 @@
-'use strict';
-
-const { legacyProcessorShim } = require('@terascope/job-components');
-const Processor = require('./processor');
-const Schema = require('./schema');
-
-// This file for backwards compatibility and functionality will be limited
-// but it should allow you to write processors using the new way today
-module.exports = legacyProcessorShim(Processor, Schema);

package/lib/processors/save_file/processor.js

@@ -1,17 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const { EachProcessor } = require('@terascope/job-components');
-
-class SaveFile extends EachProcessor {
-    constructor(...args) {
-        super(...args);
-        this.filePath = this.opConfig.file_path;
-    }
-
-    async forEach(record) {
-        fs.appendFileSync(this.filePath, `${JSON.stringify(record)}\n`);
-    }
-}
-
-module.exports = SaveFile;

package/lib/processors/save_file/schema.js

@@ -1,17 +0,0 @@
-'use strict';
-
-const { ConvictSchema } = require('@terascope/job-components');
-
-class Schema extends ConvictSchema {
-    build() {
-        return {
-            file_path: {
-                doc: 'Specify a number > 0 to limit the number of results printed to the console log.'
-                + 'This prints results from the beginning of the result set.',
-                default: __dirname
-            }
-        };
-    }
-}
-
-module.exports = Schema;

package/lib/processors/script.js

@@ -1,130 +0,0 @@
-'use strict';
-
-const { spawn } = require('child_process');
-const path = require('path');
-const { TSError } = require('@terascope/utils');
-
-function newProcessor(context, opConfig) {
-    const { args, options } = opConfig;
-
-    let command = '';
-
-    return new Promise(((resolve, reject) => {
-        if (opConfig.asset === undefined || opConfig.asset === '' || opConfig.asset === 'echo') {
-            // this would be used when a path is defined to the asset in the job
-            ({ command } = opConfig);
-            resolve(procData);
-        } else {
-            context.apis.assets.getPath(opConfig.asset)
-                .then((apath) => {
-                    command = path.join(apath, opConfig.command);
-                    resolve(procData);
-                })
-                .catch((err) => {
-                    reject(new TSError(err, {
-                        reason: 'asset not in specified path'
-                    }));
-                });
-        }
-    }));
-
-    function procData(data) {
-        return new Promise(((resolve, reject) => {
-            let inData = '';
-            try {
-                inData = JSON.stringify(data);
-            } catch (err) {
-                reject(new TSError(err, {
-                    reason: 'failed to convert input data to string',
-                }));
-                return;
-            }
-
-            let outErrors = '';
-            let outData = '';
-            let childProcess;
-
-            try {
-                childProcess = spawn(command, args, options);
-            } catch (err) {
-                reject(new TSError(err, {
-                    reason: 'when trying to run command'
-                }));
-                return;
-            }
-
-            childProcess.stdin.setEncoding('utf-8');
-            childProcess.stdin.write(`${inData}\n`);
-            childProcess.stdin.end();
-
-            childProcess.on('error', (err) => {
-                reject(err);
-            });
-
-            childProcess.stdout.on('data', (outDataItem) => {
-                outData += outDataItem;
-            });
-
-            childProcess.stdout.on('end', () => {
-                if (outErrors) {
-                    reject(outErrors);
-                } else {
-                    try {
-                        const final = JSON.parse(outData);
-                        resolve(final);
-                    } catch (err) {
-                        reject(new TSError(err, {
-                            reason: 'processing script stdout pipe'
-                        }));
-                    }
-                }
-            });
-
-            childProcess.stderr.on('data', (outError) => {
-                outErrors += outError;
-            });
-
-            childProcess.on('close', (code) => {
-                if (code === 0) return;
-                reject(new Error('child process non-zero exit'));
-            });
-
-            childProcess.on('error', (err) => {
-                reject(err);
-            });
-        }));
-    }
-}
-
-function schema() {
-    return {
-        command: {
-            doc: 'what command to run',
-            default: 'echo',
-            format: 'required_String'
-        },
-        args: {
-            doc: 'arguments to pass along with the command',
-            default: [],
-            format(val) {
-                if (!Array.isArray(val)) {
-                    throw new Error('args for script must be an array');
-                }
-            }
-        },
-        options: {
-            doc: 'Obj containing options to pass into the process env',
-            default: {}
-        },
-        asset: {
-            doc: 'name of asset to use for op',
-            default: 'echo',
-            format: 'optional_String'
-        }
-    };
-}
-
-module.exports = {
-    newProcessor,
-    schema
-};

package/lib/processors/stdout/index.js

@@ -1,9 +0,0 @@
-'use strict';
-
-const { legacyProcessorShim } = require('@terascope/job-components');
-const Processor = require('./processor');
-const Schema = require('./schema');
-
-// This file for backwards compatibility and functionality will be limited
-// but it should allow you to write processors using the new way today
-module.exports = legacyProcessorShim(Processor, Schema);

package/lib/processors/stdout/processor.js

@@ -1,19 +0,0 @@
-'use strict';
-
-/* eslint-disable no-console */
-
-const _ = require('lodash');
-const { BatchProcessor } = require('@terascope/job-components');
-
-class Stdout extends BatchProcessor {
-    async onBatch(data) {
-        if (this.opConfig.limit === 0) {
-            console.log(data);
-        } else {
-            console.log(_.take(data, this.opConfig.limit));
-        }
-        return data;
-    }
-}
-
-module.exports = Stdout;

package/lib/processors/stdout/schema.js

@@ -1,18 +0,0 @@
-'use strict';
-
-const { ConvictSchema } = require('@terascope/job-components');
-
-class Schema extends ConvictSchema {
-    build() {
-        return {
-            limit: {
-                doc: 'Specify a number > 0 to limit the number of results printed to the console log.'
-                    + 'This prints results from the beginning of the result set.',
-                default: 0,
-                format: 'nat'
-            }
-        };
-    }
-}
-
-module.exports = Schema;

package/lib/storage/analytics.js

@@ -1,106 +0,0 @@
-'use strict';
-
-const { makeLogger } = require('../workers/helpers/terafoundation');
-const { timeseriesIndex } = require('../utils/date_utils');
-const elasticsearchBackend = require('./backends/elasticsearch_store');
-
-// Module to manager job states in Elasticsearch.
-// All functions in this module return promises that must be resolved to
-// get the final result.
-module.exports = async function analyticsService(context) {
-    const logger = makeLogger(context, 'analytics_storage');
-    const config = context.sysconfig.teraslice;
-    const workerId = `${context.sysconfig.teraslice.hostname}__${context.cluster.worker.id}`;
-    const _index = `${config.name}__analytics`;
-    // making this to pass down to backend for dynamic index searches
-    const indexName = `${_index}*`;
-    const timeseriesFormat = config.index_rollover_frequency.analytics;
-
-    const backendConfig = {
-        context,
-        indexName,
-        recordType: 'analytics',
-        idField: '_id',
-        fullResponse: false,
-        logRecord: false,
-        forceRefresh: false,
-        storageName: 'analytics',
-    };
-
-    const backend = await elasticsearchBackend(backendConfig);
-
-    async function log(job, sliceInfo, stats, state = 'completed') {
-        const indexData = timeseriesIndex(timeseriesFormat, _index);
-        const esIndex = indexData.index;
-        const { timestamp } = indexData;
-
-        // These records are uniquely identified by ex_id + slice_id
-        const results = job.config.operations.map((op, index) => backend.bulk(
-            {
-                '@timestamp': timestamp,
-                ex_id: job.config.ex_id,
-                job_id: job.config.job_id,
-                worker_id: workerId,
-                slice_id: sliceInfo.slice_id,
-                slicer_id: sliceInfo.slicer_id,
-                op: op._op,
-                state,
-                order: index,
-                count: stats.size[index],
-                time: stats.time[index],
-                memory: stats.memory[index],
-            },
-            'index',
-            esIndex
-        ));
-
-        return Promise.all(results);
-    }
-
-    async function getRecord(recordId, index) {
-        return backend.get(recordId, index);
-    }
-
-    async function search(query, from, size, sort, fields) {
-        return backend.search(query, from, size, sort, fields);
-    }
-
-    async function update(recordId, updateSpec, index) {
-        return backend.update(recordId, updateSpec, index);
-    }
-
-    async function remove(recordId, index) {
-        return backend.remove(recordId, index);
-    }
-
-    async function shutdown(forceShutdown) {
-        logger.info('shutting down.');
-        return backend.shutdown(forceShutdown);
-    }
-
-    async function refresh() {
-        const { index } = timeseriesIndex(timeseriesFormat, _index);
-        return backend.refresh(index);
-    }
-
-    function verifyClient() {
-        return backend.verifyClient();
-    }
-
-    async function waitForClient() {
-        return backend.waitForClient();
-    }
-
-    logger.info('analytics storage initialized');
-    return {
-        log,
-        get: getRecord,
-        search,
-        update,
-        remove,
-        shutdown,
-        refresh,
-        waitForClient,
-        verifyClient,
-    };
-};