teraslice-client-js 0.55.1 → 0.57.0

package/src/interfaces.ts

@@ -1,13 +1,4 @@
-import {
-    ClusterManagerType,
-    Assignment,
-    ValidatedJobConfig,
-    ExecutionConfig,
-    SliceRequest,
-    Omit,
-    Overwrite,
-    RecoveryCleanupType
-} from '@terascope/job-components';
+import { Omit } from '@terascope/utils';
 import * as got from 'got';
 
 export interface ClientConfig {
@@ -16,333 +7,9 @@ export interface ClientConfig {
     apiVersion?: string;
     timeout?: number;
 }
-// TODO: lockdown query parameter types
-
-export interface APISearchParams {
-    size?: number;
-    from?: number;
-    sort?: string;
-}
-
-export interface TxtSearchParams extends APISearchParams {
-    fields?: string | string[];
-}
-
-export type SearchJobStatus = '*' | ExecutionStatus;
-
-export type JobListStatusQuery = SearchJobStatus | JobSearchParams;
-
-export interface JobSearchParams extends APISearchParams {
-    status: SearchJobStatus;
-}
-
-export type SearchQuery = APISearchParams & Record<string, any>;
 
 export type RequestOptions = got.Options;
 
 export type SearchOptions = Omit<got.Options, 'searchParams'>;
 
 export type PostData = string | NodeJS.ReadableStream | Buffer;
-
-export type TxtType = 'assets' | 'slicers' | 'ex' | 'jobs' | 'nodes' | 'workers';
-
-export interface RootResponse {
-    arch: string;
-    clustering_type: ClusterManagerType;
-    name: string;
-    node_version: string;
-    platform: string;
-    teraslice_version: string;
-}
-
-export interface ErrorResponse {
-    error: number;
-    message: string;
-}
-
-export interface JobNativeProcess extends NativeProcess {
-    node_id: string;
-}
-
-export interface JobKubernetesProcess extends KubernetesProcess {
-    node_id: string;
-}
-
-export type JobProcesses = JobNativeProcess | JobKubernetesProcess;
-
-export type WorkerJobProcesses = Overwrite<JobProcesses, { assignment: 'worker' }>;
-
-/*
-    ASSETS
-*/
-export interface Asset {
-    _created: string;
-    version: string;
-    id: string;
-    name: string;
-    description?: string;
-    blob?: string;
-}
-
-export interface AssetStatusResponse {
-    available: boolean;
-}
-
-export type AssetIDResponse = {
-    _id: string;
-}
-
-/*
-    Cluster State Native
-*/
-
-export interface NativeProcess {
-    worker_id: string;
-    assignment: Assignment;
-    pid: number;
-    ex_id?: string;
-    job_id?: string;
-}
-
-export interface BaseClusterState {
-    node_id: string;
-    hostname: string;
-    pid: number|'N/A';
-    node_version: string;
-    teraslice_version: string;
-    total: number|'N/A';
-    state: string;
-    available: number|'N/A';
-    active: any[];
-}
-
-export interface ClusterStateNodeNative extends BaseClusterState {
-    node_id: string;
-    hostname: string;
-    pid: number;
-    node_version: string;
-    teraslice_version: string;
-    total: number;
-    state: string;
-    available: number;
-    active: NativeProcess[];
-}
-
-export interface ClusterStateNative {
-    [key: string]: ClusterStateNodeNative;
-}
-
-/*
-    Cluster State Kubernetes
-*/
-
-export interface KubernetesProcess {
-    worker_id: string;
-    assignment: Assignment;
-    pod_name: string;
-    ex_id: string;
-    job_id: string;
-    pod_ip: string;
-    assets: string[];
-    image: string;
-}
-
-export interface ClusterStateNodeKubernetes extends BaseClusterState {
-    node_id: string;
-    hostname: string;
-    pid: 'N/A';
-    node_version: 'N/A';
-    teraslice_version: 'N/A';
-    total: 'N/A';
-    state: 'connected';
-    available: 'N/A';
-    active: KubernetesProcess[];
-}
-
-export interface ClusterStateKubernetes {
-    [key: string]: ClusterStateNodeKubernetes;
-}
-
-export type ClusterState = ClusterStateNative | ClusterStateKubernetes;
-export type ClusterProcess = NativeProcess | KubernetesProcess;
-
-/*
-    Jobs
-*/
-
-export interface JobConfiguration extends ValidatedJobConfig {
-    job_id: string;
-    _context: 'job';
-    _created: string;
-    _updated: string;
-}
-
-export interface JobIDResponse {
-    job_id: string;
-    ex_id?: string;
-}
-
-/*
-    Cluster Stats
-*/
-
-export interface SliceAccumulationStats {
-    processed: number;
-    failed: number;
-    queued: number;
-    job_duration: number;
-    workers_joined: number;
-    workers_disconnected: number;
-    workers_reconnected: number;
-}
-
-export interface ClusterStats {
-    controllers: SliceAccumulationStats;
-    slicer: SliceAccumulationStats;
-}
-
-/*
-    Execution Context
-*/
-
-export enum ExecutionStatus {
-    pending = 'pending',
-    scheduling = 'scheduling',
-    initializing = 'initializing',
-
-    running = 'running',
-    recovering = 'recovering',
-    failing = 'failing',
-    paused = 'paused',
-    stopping = 'stopping',
-
-    completed = 'completed',
-    stopped = 'stopped',
-    rejected = 'rejected',
-    failed = 'failed',
-    terminated = 'terminated'
-}
-
-export type ExecutionInitStatus =
-    ExecutionStatus.pending |
-    ExecutionStatus.scheduling |
-    ExecutionStatus.recovering;
-
-export type ExecutionRunningStatus =
-    ExecutionStatus.recovering |
-    ExecutionStatus.running |
-    ExecutionStatus.failing |
-    ExecutionStatus.paused |
-    ExecutionStatus.stopping;
-
-export type ExecutionTerminalStatus =
-    ExecutionStatus.completed |
-    ExecutionStatus.stopped |
-    ExecutionStatus.rejected |
-    ExecutionStatus.failed |
-    ExecutionStatus.terminated;
-
-export interface SlicerAnalytics extends SliceAccumulationStats {
-    workers_available: number;
-    workers_active: number;
-    subslices: number;
-    slice_range_expansion: number;
-    slicers: number;
-    subslice_by_key: number;
-    started?: string;
-    queuing_complete?: string;
-}
-
-export interface Execution extends ExecutionConfig {
-    _context: 'ex';
-    _created: string;
-    _updated: string;
-    _status: ExecutionStatus;
-    _has_errors?: boolean;
-    _failureReason?: string;
-    _slicer_stats?: SlicerAnalytics;
-}
-
-// Native Cluster => terminated, stats null
-// failing, hasErrors, failureReason, slice stats
-// failed, hasErrors, failureReason, slice stats
-// teraminated, hasErrors, failureReason, slice stats
-// completed, slice stats
-// stopping/stopped, slice stats
-
-export interface ExecutionIDResponse {
-    ex_id: string;
-}
-
-/*
-*    Lifecyle Response
-*/
-
-/**
- * Recover Job / Execution Options
-*/
-export interface RecoverQuery {
-    /** @deprecated use `cleanup_type` */
-    cleanup?: RecoveryCleanupType;
-    cleanup_type?: RecoveryCleanupType;
-}
-
-export interface PausedResponse {
-    status: ExecutionStatus.paused;
-}
-
-export interface ResumeResponse {
-    status: ExecutionStatus.running;
-}
-
-export interface StoppedResponse {
-    status: ExecutionStatus.stopped | ExecutionStatus.stopping;
-}
-
-export interface StopQuery {
-    timeout?: number;
-    blocking?: boolean;
-}
-
-export interface AssetUploadQuery {
-    blocking?: boolean;
-}
-
-/*
-    Worker changes Response
-*/
-
-export type ChangeWorkerQueryParams = 'add' | 'remove' | 'total';
-
-export interface ChangeWorkerResponse {
-    message: string;
-}
-
-/*
-    Slicer/Controller Response
-*/
-
-export interface SlicerStats extends SlicerAnalytics {
-    ex_id: string;
-    job_id: string;
-}
-
-export type ControllerState = SlicerStats[];
-
-/*
-    Error Response
-*/
-
-export interface ErrorStateRecord {
-    ex_id: string;
-    slice_id: string;
-    slicer_id: string;
-    slicer_order: number;
-    request: SliceRequest;
-    state: 'error';
-    error: string;
-    _created: string;
-    _updated: string;
-}
-
-export type StateErrors = ErrorStateRecord[];

package/src/job.ts

@@ -1,36 +1,14 @@
 import util from 'util';
 import autoBind from 'auto-bind';
 import {
-    pDelay,
-    isString,
-    toString,
-    TSError,
-    Assignment,
-    toHumanTime,
-    JobConfig,
-} from '@terascope/job-components';
+    pDelay, isString, toString,
+    TSError, toHumanTime,
+} from '@terascope/utils';
+import { Teraslice } from '@terascope/types';
 import {
     ClientConfig,
-    SearchQuery,
-    ExecutionStatus,
-    ChangeWorkerQueryParams,
-    ClusterState,
-    ClusterProcess,
-    ControllerState,
-    StateErrors,
-    WorkerJobProcesses,
     SearchOptions,
-    ChangeWorkerResponse,
-    RequestOptions,
-    ExecutionIDResponse,
-    RecoverQuery,
-    ResumeResponse,
-    StoppedResponse,
-    PausedResponse,
-    JobIDResponse,
-    StopQuery,
-    Execution,
-    JobConfiguration
+    RequestOptions
 } from './interfaces';
 import Client from './client';
 
@@ -41,7 +19,7 @@ import Client from './client';
  * Depending on how usage of this API develops we may want to reconsider this.
  */
 
-function _deprecateSlicerName(fn: () => Promise<ControllerState>) {
+function _deprecateSlicerName(fn: () => Promise<Teraslice.ExecutionList>) {
     const msg = 'api endpoints with /slicers are being deprecated in favor of the semantically correct term of /controllers';
     return util.deprecate(fn, msg);
 }
@@ -63,78 +41,92 @@ export default class Job extends Client {
 
     id(): string { return this._jobId; }
 
-    async slicer(requestOptions: RequestOptions = {}): Promise<ControllerState> {
+    async slicer(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionList> {
         return this.get(`/jobs/${this._jobId}/slicer`, requestOptions);
     }
 
-    async controller(requestOptions: RequestOptions = {}): Promise<ControllerState> {
+    async controller(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionList> {
         return this.get(`/jobs/${this._jobId}/controller`, requestOptions);
     }
 
-    async start(query?: SearchQuery, searchOptions: SearchOptions = {}): Promise<JobIDResponse> {
+    async start(
+        query?: Teraslice.SearchQuery,
+        searchOptions: SearchOptions = {}
+    ): Promise<Teraslice.ApiJobCreateResponse> {
         const options = this.makeOptions(query, searchOptions);
         return this.post(`/jobs/${this._jobId}/_start`, null, options);
     }
 
-    async stop(query?: StopQuery, searchOptions: SearchOptions = {}): Promise<StoppedResponse> {
+    async stop(
+        query?: Teraslice.StopQuery,
+        searchOptions: SearchOptions = {}
+    ): Promise<Teraslice.ApiStoppedResponse> {
         const options = this.makeOptions(query, searchOptions);
         return this.post(`/jobs/${this._jobId}/_stop`, null, options);
     }
 
-    async pause(query?: SearchQuery, searchOptions: SearchOptions = {}): Promise<PausedResponse> {
+    async pause(
+        query?: Teraslice.SearchQuery,
+        searchOptions: SearchOptions = {}
+    ): Promise<Teraslice.ApiPausedResponse> {
         const options = this.makeOptions(query, searchOptions);
         return this.post(`/jobs/${this._jobId}/_pause`, null, options);
     }
 
-    async resume(query?: SearchQuery, searchOptions: SearchOptions = {}): Promise<ResumeResponse> {
+    async resume(
+        query?: Teraslice.SearchQuery,
+        searchOptions: SearchOptions = {}
+    ): Promise<Teraslice.ApiResumeResponse> {
         const options = this.makeOptions(query, searchOptions);
         return this.post(`/jobs/${this._jobId}/_resume`, null, options);
     }
 
     async recover(
-        query: RecoverQuery = {},
+        query: Teraslice.RecoverQuery = {},
         searchOptions: SearchOptions = {}
-    ): Promise<JobIDResponse> {
+    ): Promise<Teraslice.ApiJobCreateResponse> {
         const options = this.makeOptions(query, searchOptions);
         return this.post(`/jobs/${this._jobId}/_recover`, null, options);
     }
 
-    async update(jobSpec: JobConfiguration): Promise<JobConfiguration> {
+    async update(jobSpec: Teraslice.JobRecord): Promise<Teraslice.JobRecord> {
         return this.put(`/jobs/${this._jobId}`, jobSpec);
     }
 
-    async updatePartial(jobSpec: Partial<JobConfiguration|JobConfig>): Promise<JobConfiguration> {
+    async updatePartial(
+        jobSpec: Partial<Teraslice.JobRecord>
+    ): Promise<Teraslice.JobRecord> {
         const current = await this.config();
-        const body: JobConfiguration = Object.assign({}, current, jobSpec);
+        const body: Teraslice.JobRecord = Object.assign({}, current, jobSpec);
         return this.update(body);
     }
 
-    async execution(requestOptions: RequestOptions = {}): Promise<Execution> {
+    async execution(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionRecord> {
         return this.get(`/jobs/${this._jobId}/ex`, requestOptions);
     }
 
-    async exId(requestOptions: RequestOptions = {}): Promise<ExecutionIDResponse> {
+    async exId(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionIDResponse> {
         const { ex_id: exId } = await this.get(`/jobs/${this._jobId}/ex`, requestOptions);
         return exId;
     }
 
-    async status(requestOptions: RequestOptions = {}): Promise<ExecutionStatus> {
+    async status(requestOptions: RequestOptions = {}): Promise<Teraslice.ExecutionStatus> {
         const { _status: status } = await this.get(`/jobs/${this._jobId}/ex`, requestOptions);
         return status;
     }
 
     async waitForStatus(
-        target: ExecutionStatus[] | ExecutionStatus,
+        target: Teraslice.ExecutionStatus[] | Teraslice.ExecutionStatus,
         intervalMs = 1000,
         timeoutMs = 0,
         requestOptions: RequestOptions = {}
-    ): Promise<ExecutionStatus> {
+    ): Promise<Teraslice.ExecutionStatus> {
         const terminal = {
-            [ExecutionStatus.terminated]: true,
-            [ExecutionStatus.failed]: true,
-            [ExecutionStatus.rejected]: true,
-            [ExecutionStatus.completed]: true,
-            [ExecutionStatus.stopped]: true,
+            [Teraslice.ExecutionStatusEnum.terminated]: true,
+            [Teraslice.ExecutionStatusEnum.failed]: true,
+            [Teraslice.ExecutionStatusEnum.rejected]: true,
+            [Teraslice.ExecutionStatusEnum.completed]: true,
+            [Teraslice.ExecutionStatusEnum.stopped]: true,
         };
 
         const startTime = Date.now();
@@ -144,7 +136,7 @@ export default class Job extends Client {
         }, requestOptions);
         let exId: string;
 
-        const checkStatus = async (): Promise<ExecutionStatus> => {
+        const checkStatus = async (): Promise<Teraslice.ExecutionStatus> => {
             let result;
             try {
                 const ex = await this.get(`/jobs/${this._jobId}/ex`, options);
@@ -190,24 +182,27 @@ export default class Job extends Client {
         return checkStatus();
     }
 
-    async config(requestOptions: RequestOptions = {}): Promise<JobConfiguration> {
+    async config(requestOptions: RequestOptions = {}): Promise<Teraslice.JobRecord> {
         return this.get(`/jobs/${this._jobId}`, requestOptions);
     }
 
-    async errors(query: SearchQuery = {}, searchOptions: SearchOptions = {}): Promise<StateErrors> {
+    async errors(
+        query: Teraslice.SearchQuery = {},
+        searchOptions: SearchOptions = {}
+    ): Promise<Teraslice.ErrorRecord[]> {
         return this.get(`/jobs/${this._jobId}/errors`, this.makeOptions(query, searchOptions));
     }
 
-    async workers(requestOptions: RequestOptions = {}): Promise<WorkerJobProcesses[]> {
-        const state: ClusterState = await this.get('/cluster/state', requestOptions);
-        return filterProcesses<WorkerJobProcesses>(state, this._jobId, 'worker');
+    async workers(requestOptions: RequestOptions = {}): Promise<Teraslice.WorkerNode[]> {
+        const state: Teraslice.ClusterState = await this.get('/cluster/state', requestOptions);
+        return filterProcesses<Teraslice.WorkerNode>(state, this._jobId, 'worker');
     }
 
     async changeWorkers(
-        action: ChangeWorkerQueryParams,
+        action: Teraslice.ChangeWorkerQueryParams,
         workerNum: number,
         requestOptions: RequestOptions = {}
-    ): Promise<ChangeWorkerResponse | string> {
+    ): Promise<Teraslice.ChangeWorkerResponse | string> {
         if (action == null || workerNum == null) {
             throw new TSError('Change workers requires action and count', {
                 statusCode: 400
@@ -237,16 +232,22 @@ export default class Job extends Client {
     }
 }
 
-function filterProcesses<T>(state: ClusterState, jobId: string, type: Assignment) {
+function filterProcesses<T>(
+    state: Teraslice.ClusterState,
+    jobId: string,
+    type: Teraslice.Assignment
+) {
     const results: T[] = [];
 
     for (const [, node] of Object.entries(state)) {
-        node.active.forEach((child: ClusterProcess) => {
-            const { assignment, job_id: procJobId } = child;
-            if ((assignment && assignment === type) && (procJobId && procJobId === jobId)) {
-                const jobProcess = Object.assign({}, child, { node_id: node.node_id });
-                // @ts-expect-error
-                results.push(jobProcess);
+        // TODO: fix this
+        node.active.forEach((child: any) => {
+            if (child.job_id) {
+                const { assignment, job_id: procJobId } = child;
+                if ((assignment && assignment === type) && (procJobId && procJobId === jobId)) {
+                    const jobProcess = Object.assign({}, child, { node_id: node.node_id });
+                    results.push(jobProcess);
+                }
             }
         });
     }

package/src/jobs.ts

@@ -1,15 +1,9 @@
-import { isString, JobConfig, TSError } from '@terascope/job-components';
+import { isString, TSError } from '@terascope/utils';
+import { Teraslice } from '@terascope/types';
 import autoBind from 'auto-bind';
 import Client from './client';
 import Job from './job';
-import {
-    ClientConfig,
-    JobConfiguration,
-    SearchOptions,
-    JobSearchParams,
-    JobListStatusQuery,
-    JobIDResponse
-} from './interfaces';
+import { SearchOptions, ClientConfig } from './interfaces';
 
 export default class Jobs extends Client {
     constructor(config: ClientConfig) {
@@ -17,22 +11,24 @@ export default class Jobs extends Client {
         autoBind(this);
     }
 
-    async submit(jobSpec: JobConfig, shouldNotStart?: boolean): Promise<Job> {
+    async submit(jobSpec: Teraslice.JobConfig, shouldNotStart?: boolean): Promise<Job> {
         if (!jobSpec) {
             throw new TSError('Submit requires a jobSpec', {
                 statusCode: 400
             });
         }
-        const job: JobIDResponse = await this.post('/jobs', jobSpec, {
+
+        const job: Teraslice.ApiJobCreateResponse = await this.post('/jobs', jobSpec, {
             searchParams: { start: !shouldNotStart }
         });
+
         return this.wrap(job.job_id);
     }
 
     async list(
-        status?: JobListStatusQuery,
+        status?: Teraslice.JobListStatusQuery,
         searchOptions: SearchOptions = {}
-    ): Promise<JobConfiguration[]> {
+    ): Promise<Teraslice.JobRecord[]> {
         const query = _parseListOptions(status);
         return this.get('/jobs', this.makeOptions(query, searchOptions));
     }
@@ -46,7 +42,7 @@ export default class Jobs extends Client {
     }
 }
 
-function _parseListOptions(options?: JobListStatusQuery): JobSearchParams {
+function _parseListOptions(options?: Teraslice.JobListStatusQuery): Teraslice.JobSearchParams {
     // support legacy
     if (!options) return { status: '*' };
     if (isString(options)) return { status: options };

package/test/assets-spec.ts

@@ -1,8 +1,8 @@
 import nock from 'nock';
 import path from 'path';
 import fs from 'fs';
+import { AssetRecord, AssetIDResponse } from '@terascope/types';
 import Assets from '../src/assets';
-import { AssetIDResponse, Asset } from '../src/interfaces';
 
 describe('Teraslice Assets', () => {
     let assets: Assets;
@@ -22,17 +22,19 @@ describe('Teraslice Assets', () => {
 
     const date = new Date();
 
-    const assetVersion1: Asset = {
+    const assetVersion1: AssetRecord = {
         id: 'someId',
         version: '1.0',
         name: 'iAmAnAsset',
+        blob: '',
         _created: date.toISOString()
     };
 
-    const assetVersion2: Asset = {
+    const assetVersion2: AssetRecord = {
         id: 'someId',
         version: '2.0',
         name: 'iAmAnAsset',
+        blob: '',
         _created: new Date(date.getTime() + 500000).toISOString()
     };
 
@@ -64,7 +66,11 @@ describe('Teraslice Assets', () => {
             });
         });
 
-        describe('when called with a stream', () => {
+        // This test hangs in node version 20.10.0 and above.
+        // See issue: https://github.com/nock/nock/issues/2595
+        // teraslice/e2e/test/cases/assets/simple-spec.ts covers this test case
+        // eslint-disable-next-line jest/no-disabled-tests
+        xdescribe('when called with a stream', () => {
             const testFilePath = path.join(__dirname, 'fixtures', 'test.txt');
             const contents = fs.readFileSync(testFilePath, 'utf-8');
             const idResponse: AssetIDResponse = { _id: 'some-asset-id' };
@@ -83,7 +89,7 @@ describe('Teraslice Assets', () => {
     });
 
     describe('->list', () => {
-        const assetList: Asset[] = [
+        const assetList: AssetRecord[] = [
             assetVersion1,
             assetVersion2
         ];