teraslice-client-js 2.16.2 → 2.17.0

package/examples/cluster.js

@@ -1,13 +0,0 @@
-/* eslint-disable no-console */
-
-import { TerasliceClient } from 'teraslice-client-js';
-
-const client = new TerasliceClient({
-    host: 'http://localhost:5678'
-});
-
-const state = await client.cluster.state();
-
-const controllers = await client.cluster.controllers();
-
-console.dir({ state, controllers }, { depth: 40 });

package/examples/list.js

@@ -1,16 +0,0 @@
-/* eslint-disable no-console */
-
-import { TerasliceClient } from 'teraslice-client-js';
-
-const client = new TerasliceClient({
-    host: 'http://localhost:5678'
-});
-
-const jobs = await client.jobs.list();
-const executionsCompleted = await client.executions.list('completed');
-const executionsRunning = await client.executions.list('running');
-const executionsPending = await client.executions.list('pending');
-
-console.dir({
-    jobs, executionsCompleted, executionsRunning, executionsPending
-}, { depth: 40 });

package/examples/submit.js

@@ -1,48 +0,0 @@
-/* eslint-disable no-console */
-import { TerasliceClient } from 'teraslice-client-js';
-
-const client = new TerasliceClient({
-    host: 'http://localhost:5678'
-});
-
-const exampleJob = {
-    name: 'Data Generator',
-    lifecycle: 'once',
-    workers: 1,
-    assets: ['elasticsearch', 'standard'],
-    apis: [
-        {
-            _name: 'elasticsearch_sender_api',
-            index: 'client-test-logs',
-            type: 'events',
-            size: 50
-        }
-    ],
-    operations: [
-        {
-            _op: 'data_generator',
-            size: 1
-        },
-        {
-            _op: 'elasticsearch_bulk',
-            _api_name: 'elasticsearch_sender_api'
-        }
-    ]
-};
-
-async function main() {
-    const job = await client.jobs.submit(exampleJob);
-
-    console.log(job.id());
-
-    const job2 = client.jobs.wrap(job.id());
-    const status = await job2.status();
-
-    console.log(status);
-
-    console.log('Waiting for the job to finish.');
-    await job2.waitForStatus('completed');
-    console.log('Job completed');
-}
-
-main();

package/src/assets.ts

@@ -1,99 +0,0 @@
-import { TSError, isEmpty, isString } from '@terascope/core-utils';
-import { Teraslice } from '@terascope/types';
-import path from 'node:path';
-import autoBind from 'auto-bind';
-import Client from './client.js';
-import { PostData, RequestOptions, ClientConfig } from './interfaces.js';
-
-export default class Assets extends Client {
-    constructor(config: ClientConfig) {
-        super(config);
-        autoBind(this);
-    }
-
-    async upload(
-        data: PostData,
-        query: Teraslice.AssetUploadQuery = {}
-    ): Promise<Teraslice.AssetIDResponse> {
-        if (isEmpty(data)) {
-            throw new TSError('Asset stream must not be empty', {
-                statusCode: 400
-            });
-        }
-
-        const results = await this.post('/assets', data, {
-            searchParams: query as Record<string, any>
-        });
-
-        return this.parse(results);
-    }
-
-    async remove(
-        id: string,
-        searchOptions: RequestOptions = {}
-    ): Promise<Teraslice.AssetIDResponse> {
-        if (isEmpty(id)) {
-            throw new TSError('Asset delete requires a ID', {
-                statusCode: 400
-            });
-        }
-
-        const results = await this.delete<Teraslice.AssetIDResponse>(`/assets/${id}`, searchOptions);
-
-        return this.parse(results);
-    }
-
-    async list(
-        query: Teraslice.SearchQuery = {},
-        searchOptions: RequestOptions = {}
-    ): Promise<Teraslice.AssetRecord[]> {
-        const options = { ...searchOptions, searchParams: query };
-        return this.get('/assets', options);
-    }
-
-    async getAsset(
-        name: string,
-        version = '',
-        searchOptions: RequestOptions = {}
-    ): Promise<Teraslice.AssetRecord[]> {
-        if (!name || !isString(name)) {
-            throw new TSError('Name is required, and must be of type string', {
-                statusCode: 400
-            });
-        }
-        if (version && !isString(version)) {
-            throw new TSError('Version if provided must be of type string', {
-                statusCode: 400
-            });
-        }
-
-        const assetUrl = path.join('/assets', name, version);
-
-        return this.get(assetUrl, searchOptions);
-    }
-
-    async txt(
-        name = '',
-        version = '',
-        query: Teraslice.TxtSearchParams = {},
-        searchOptions: RequestOptions = {}
-    ): Promise<string> {
-        if (name && !isString(name)) {
-            throw new TSError('Name must be of type string', {
-                statusCode: 400
-            });
-        }
-        if (version && !isString(version)) {
-            throw new TSError('Version must be of type string', {
-                statusCode: 400
-            });
-        }
-
-        const options = Object.assign({ responseType: 'text' }, searchOptions, {
-            searchParams: query
-        });
-        const pathing = path.join('/txt/assets', name, version);
-
-        return super.get(pathing, options);
-    }
-}

package/src/client.ts

@@ -1,197 +0,0 @@
-import {
-    isString, TSError, isPlainObject,
-    isTest, trimStart, tryParseJSON,
-    withoutNil,
-} from '@terascope/core-utils';
-import { STATUS_CODES } from 'node:http';
-import { URL } from 'node:url';
-import got, { type Got, HTTPError } from 'got';
-import { ClientConfig, RequestOptions } from './interfaces.js';
-
-export default class Client {
-    private readonly _apiVersion: string;
-    private readonly _request: Got;
-    protected readonly _config: ClientConfig;
-
-    constructor(config: ClientConfig = {}) {
-        const configUrl = new URL(config.host || config.baseUrl || 'http://localhost:5678');
-        configUrl.pathname = '';
-        configUrl.hash = '';
-        const prefixUrl = configUrl.toString();
-        const { apiVersion = '/v1' } = config;
-
-        this._config = config;
-        this._apiVersion = apiVersion;
-        this._request = got.extend({
-            prefixUrl,
-            headers: {
-                'User-Agent': 'Teraslice Client',
-                Accept: 'application/json',
-            },
-            retry: {
-                limit: isTest ? 0 : 3,
-                maxRetryAfter: 15000 // 15 seconds
-            },
-            timeout: {
-                request: config.timeout
-            },
-            responseType: 'json'
-        });
-    }
-
-    async get<T = any>(endpoint: string, options?: RequestOptions): Promise<T> {
-        return this._makeRequest<T>('get', endpoint, options);
-    }
-
-    async post<T = any>(endpoint: string, data?: unknown, options?: RequestOptions): Promise<T> {
-        return this._makeRequest<T>('post', endpoint, options, data);
-    }
-
-    async put<T = any>(endpoint: string, data: unknown, options?: RequestOptions): Promise<T> {
-        return this._makeRequest<T>('put', endpoint, options, data);
-    }
-
-    async delete<T = any>(endpoint: string, options?: RequestOptions): Promise<T> {
-        return this._makeRequest<T>('delete', endpoint, options);
-    }
-
-    private async _makeRequest<T = any>(
-        method: 'get' | 'post' | 'put' | 'delete',
-        endpoint: string,
-        searchOptions?: RequestOptions,
-        data?: any
-    ): Promise<T> {
-        const errorMsg = validateRequestOptions(endpoint, searchOptions);
-        if (errorMsg) {
-            return Promise.reject(new TSError(errorMsg, {
-                statusCode: 422
-            }));
-        }
-
-        let options: RequestOptions;
-        if (data != null && method.toLowerCase() !== 'get') {
-            options = getRequestOptionsWithData(data, searchOptions ?? {});
-        } else {
-            options = { ...searchOptions };
-        }
-
-        // migrate to using searchParams
-        if ((options as any).query) {
-            options.searchParams = (options as any).query;
-            delete (options as any).query;
-        }
-
-        // got doesn't like `undefined` values in the search params
-        // so we need to remove them
-        if (isPlainObject(options.searchParams)) {
-            options.searchParams = withoutNil(options.searchParams as any);
-        }
-
-        const newEndpoint = getAPIEndpoint(endpoint, this._apiVersion);
-        try {
-            return await this._request[method](newEndpoint, {
-                resolveBodyOnly: true,
-                throwHttpErrors: true,
-                ...options,
-            } as any) as any as T;
-        } catch (err) {
-            if (err instanceof HTTPError) {
-                throw makeErrorFromResponse(err.response);
-            }
-
-            throw err;
-        }
-    }
-
-    protected parse(results: unknown): any {
-        return tryParseJSON(results);
-    }
-
-    protected makeOptions(
-        searchParams: Record<string, any> | undefined, options: RequestOptions
-    ): RequestOptions {
-        return { ...options, searchParams };
-    }
-}
-
-function getAPIEndpoint(uri: string, apiVersion: string) {
-    // remove start and ending slash
-    const endpoint = uri ? uri.trim().replace(/(^\/)|(\/$)/, '') : '';
-    if (!apiVersion) return endpoint;
-
-    const txtIndex = endpoint.indexOf('txt');
-    const isTxt = txtIndex < 2 && txtIndex > -1;
-    if (isTxt) return endpoint;
-
-    const versionPrefix = trimStart(apiVersion, '/');
-    if (endpoint.startsWith(versionPrefix)) return endpoint;
-    return `${versionPrefix}/${endpoint}`;
-}
-
-function getErrorFromResponse(response: any) {
-    let { body } = response;
-
-    if (body && isString(body)) {
-        try {
-            body = JSON.parse(body);
-        } catch (err) {
-            return { message: body };
-        }
-    }
-
-    if (body && isPlainObject(body)) {
-        if (isString(body.error)) {
-            return {
-                message: body.error
-            };
-        }
-        return {
-            message: body.message,
-            code: body.error
-        };
-    }
-    return {};
-}
-
-interface OldErrorOutput extends TSError {
-    error: number;
-    code: string;
-    statusCode: number;
-}
-
-function makeErrorFromResponse(response: any): OldErrorOutput {
-    const { statusCode } = response;
-    const parsedError = getErrorFromResponse(response);
-    const {
-        message = STATUS_CODES[statusCode],
-        code = statusCode
-    } = parsedError;
-
-    const error: Partial<OldErrorOutput> = new Error(message);
-
-    error.error = code; // for legacy support
-    error.code = code;
-    error.statusCode = code;
-
-    return error as OldErrorOutput;
-}
-
-// TODO: do more validations
-function validateRequestOptions(endpoint: string, _options?: RequestOptions) {
-    if (!endpoint) {
-        return 'endpoint must not be empty';
-    }
-    if (!isString(endpoint)) {
-        return 'endpoint must be a string';
-    }
-    return null;
-}
-
-function getRequestOptionsWithData(
-    data: any, options: RequestOptions
-): RequestOptions {
-    if (isPlainObject(data) || Array.isArray(data)) {
-        return { ...options, json: data };
-    }
-    return { ...options, body: data };
-}

package/src/cluster.ts

@@ -1,48 +0,0 @@
-import { startsWith } from '@terascope/core-utils';
-import { Teraslice } from '@terascope/types';
-import util from 'node:util';
-import autoBind from 'auto-bind';
-import Client from './client.js';
-
-function _deprecateSlicerName(fn: () => Promise<Teraslice.ExecutionList>) {
-    const msg = 'api endpoints with /slicers are being deprecated in favor of the semantically correct term of /controllers';
-    return util.deprecate(fn, msg);
-}
-
-export default class Cluster extends Client {
-    constructor(config: Record<string, any>) {
-        super(config);
-        this.slicers = _deprecateSlicerName(this.slicers);
-        autoBind(this);
-    }
-
-    async info(): Promise<Teraslice.ApiRootResponse> {
-        return this.get('/');
-    }
-
-    async state(): Promise<Teraslice.ClusterState> {
-        return this.get('/cluster/state');
-    }
-
-    async stats(): Promise<Teraslice.ClusterStats> {
-        return this.get('/cluster/stats');
-    }
-
-    async slicers(): Promise<Teraslice.ExecutionList> {
-        return this.get('/cluster/slicers');
-    }
-
-    async controllers(): Promise<Teraslice.ExecutionList> {
-        return this.get('/cluster/controllers');
-    }
-
-    async txt(type: string): Promise<string> {
-        const validTypes = ['assets', 'slicers', 'ex', 'jobs', 'nodes', 'workers'];
-        const isValid = validTypes.some((validType) => startsWith(type, validType));
-        if (!isValid) {
-            const error = new Error(`"${type}" is not a valid type. Must be one of ${JSON.stringify(validTypes)}`);
-            return Promise.reject(error);
-        }
-        return this.get(`/txt/${type}`, { responseType: 'text' });
-    }
-}

package/src/ex.ts

@@ -1,223 +0,0 @@
-import {
-    isString, TSError, toString,
-    pDelay, toHumanTime
-} from '@terascope/core-utils';
-import { Teraslice } from '@terascope/types';
-import autoBind from 'auto-bind';
-import Client from './client.js';
-import { ClientConfig, RequestOptions } from './interfaces.js';
-
-export default class Ex extends Client {
-    private readonly _exId: string;
-
-    constructor(config: ClientConfig, exId: string) {
-        validateExId(exId);
-
-        super(config);
-        autoBind(this);
-
-        this._exId = exId;
-    }
-
-    id(): string {
-        return this._exId;
-    }
-
-    async stop(
-        query?: Teraslice.StopQuery,
-        searchOptions: RequestOptions = {}
-    ): Promise<Teraslice.ApiStoppedResponse> {
-        const options = this.makeOptions(query, searchOptions);
-        return this.post(`/ex/${this._exId}/_stop`, null, options);
-    }
-
-    async pause(
-        query?: Teraslice.SearchQuery,
-        searchOptions: RequestOptions = {}
-    ): Promise<Teraslice.ApiPausedResponse> {
-        const options = this.makeOptions(query, searchOptions);
-        return this.post(`/ex/${this._exId}/_pause`, null, options);
-    }
-
-    async resume(
-        query?: Teraslice.SearchQuery,
-        searchOptions: RequestOptions = {}
-    ): Promise<Teraslice.ApiResumeResponse> {
-        const options = this.makeOptions(query, searchOptions);
-        return this.post(`/ex/${this._exId}/_resume`, null, options);
-    }
-
-    async recover(
-        query: Teraslice.RecoverQuery = {},
-        searchOptions: RequestOptions = {}
-    ): Promise<Ex> {
-        const options = this.makeOptions(query, searchOptions);
-        const result: Teraslice.ApiJobCreateResponse = await this.post(`/ex/${this._exId}/_recover`, null, options);
-
-        // support older version of teraslice
-        if (!result.ex_id) {
-            const { ex_id: exId } = await this.get(`/jobs/${result.job_id}/ex`);
-            return new Ex(this._config, exId);
-        }
-
-        return new Ex(this._config, result.ex_id);
-    }
-
-    async status(requestOptions?: RequestOptions): Promise<Teraslice.ExecutionStatus> {
-        const { _status: status } = await this.config(requestOptions);
-        return status;
-    }
-
-    async slicer(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionList> {
-        return this.get(`/ex/${this._exId}/slicer`, requestOptions);
-    }
-
-    async controller(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionList> {
-        return this.get(`/ex/${this._exId}/controller`, requestOptions);
-    }
-
-    async config(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionConfig> {
-        return this.get(`/ex/${this._exId}`, requestOptions);
-    }
-
-    async workers(requestOptions: RequestOptions = {}): Promise<Teraslice.WorkerNode[]> {
-        const state: Teraslice.ClusterState = await this.get('/cluster/state', requestOptions);
-        return filterProcesses<Teraslice.WorkerNode>(state, this._exId, 'worker');
-    }
-
-    async errors(options?: Teraslice.SearchQuery): Promise<Teraslice.ErrorRecord[]> {
-        return this.get(`/ex/${this._exId}/errors`, {
-            searchParams: options,
-        });
-    }
-
-    async changeWorkers(
-        action: Teraslice.ChangeWorkerQueryParams,
-        workerNum: number,
-        requestOptions: RequestOptions = {}
-    ): Promise<Teraslice.ChangeWorkerResponse | string> {
-        if (action == null || workerNum == null) {
-            throw new TSError('Change workers requires action and count', {
-                statusCode: 400
-            });
-        }
-        if (!['add', 'remove', 'total'].includes(action)) {
-            throw new TSError('Change workers requires action to be one of add, remove, or total', {
-                statusCode: 400
-            });
-        }
-
-        const query = { [action]: workerNum };
-        requestOptions.responseType = 'text';
-        const options = this.makeOptions(query, requestOptions);
-
-        const response = await this.post(`/ex/${this._exId}/_workers`, null, options);
-        // for backwards compatibility
-        if (typeof response === 'string') {
-            try {
-                return this.parse(response);
-            } catch (err) {
-                // do nothing
-            }
-        }
-
-        return response;
-    }
-
-    async waitForStatus(
-        target: Teraslice.ExecutionStatus,
-        intervalMs = 1000,
-        timeoutMs = 0,
-        requestOptions: RequestOptions = {}
-    ): Promise<Teraslice.ExecutionStatus> {
-        const terminal = {
-            [Teraslice.ExecutionStatusEnum.terminated]: true,
-            [Teraslice.ExecutionStatusEnum.failed]: true,
-            [Teraslice.ExecutionStatusEnum.rejected]: true,
-            [Teraslice.ExecutionStatusEnum.completed]: true,
-            [Teraslice.ExecutionStatusEnum.stopped]: true,
-        };
-
-        const startTime = Date.now();
-        const options = Object.assign({}, {
-            responseType: 'json',
-            timeout: {
-                request: intervalMs < 1000 ? 1000 : intervalMs
-            },
-        }, requestOptions);
-
-        const checkStatus = async (): Promise<Teraslice.ExecutionStatus> => {
-            let result;
-            try {
-                result = await this.status(options);
-            } catch (err) {
-                if (/(timeout|timedout)/i.test(toString(err))) {
-                    await pDelay(intervalMs);
-                    return checkStatus();
-                }
-                throw err;
-            }
-
-            if (result === target) {
-                return result;
-            }
-
-            // These are terminal states for a job so if we're not explicitly
-            // watching for these then we need to stop waiting as the job
-            // status won't change further.
-            if (result in terminal) {
-                throw new TSError(
-                    `Execution cannot reach the target status, "${target}", because it is in the terminal state, "${result}"`,
-                    { context: { lastStatus: result } }
-                );
-            }
-
-            const elapsed = Date.now() - startTime;
-            if (timeoutMs > 0 && elapsed >= timeoutMs) {
-                throw new TSError(
-                    `Execution status failed to change from status "${result}" to "${target}" within ${toHumanTime(timeoutMs)}`,
-                    { context: { lastStatus: result } }
-                );
-            }
-
-            await pDelay(intervalMs);
-            return checkStatus();
-        };
-
-        return checkStatus();
-    }
-}
-
-function validateExId(exId?: string) {
-    if (!exId) {
-        throw new TSError('Ex requires exId', {
-            statusCode: 400
-        });
-    }
-    if (!isString(exId)) {
-        throw new TSError('Ex requires exId to be a string', {
-            statusCode: 400
-        });
-    }
-}
-
-function filterProcesses<T>(
-    state: Teraslice.ClusterState,
-    exId: string,
-    type: Teraslice.Assignment
-) {
-    const results: T[] = [];
-
-    for (const node of Object.values(state)) {
-        // TODO: fixme
-        node.active.forEach((child: any) => {
-            const { assignment, ex_id: procExId } = child;
-            if ((assignment && assignment === type) && (procExId && procExId === exId)) {
-                const jobProcess = Object.assign({}, child, { node_id: node.node_id });
-                results.push(jobProcess as any);
-            }
-        });
-    }
-
-    return results;
-}