teraslice 3.17.2 → 3.18.0

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.d.ts

@@ -0,0 +1,42 @@
+import * as k8s from '@kubernetes/client-node';
+import * as i from './interfaces.js';
+export declare function makeTemplate<T = k8s.V1Deployment | k8s.V1Job | k8s.V1Service>(folder: 'deployments' | 'jobs' | 'services', fileName: i.NodeType): (config: i.K8sConfig) => T;
+export declare function getMaxOldSpace(memory: number): number;
+export declare function setMaxOldSpaceViaEnv(envArr: k8s.V1EnvVar[], jobEnv: Record<string, any>, memory: number): void;
+export declare function getRetryConfig(): {
+    retries: number;
+    delay: number;
+};
+export declare function isDeployment(resource: i.K8sResource): resource is k8s.V1Deployment;
+export declare function isJob(resource: i.K8sResource): resource is k8s.V1Job;
+export declare function isPod(resource: i.K8sResource): resource is k8s.V1Pod;
+export declare function isReplicaSet(resource: i.K8sResource): resource is k8s.V1ReplicaSet;
+export declare function isService(resource: i.K8sResource): resource is k8s.V1Service;
+export declare function isTSDeployment(manifest: k8s.V1Deployment): manifest is i.TSDeployment;
+export declare function isTSJob(manifest: k8s.V1Job): manifest is i.TSJob;
+export declare function isTSPod(manifest: i.K8sResource): manifest is i.TSPod;
+export declare function isTSReplicaSet(manifest: k8s.V1ReplicaSet): manifest is i.TSReplicaSet;
+export declare function isTSService(manifest: k8s.V1Service): manifest is i.TSService;
+export declare function convertToTSResource(resource: k8s.V1Deployment): i.TSDeployment;
+export declare function convertToTSResource(resource: k8s.V1Job): i.TSJob;
+export declare function convertToTSResource(resource: k8s.V1Pod): i.TSPod;
+export declare function convertToTSResource(resource: k8s.V1ReplicaSet): i.TSReplicaSet;
+export declare function convertToTSResource(resource: k8s.V1Service): i.TSService;
+export declare function convertToTSResource(resource: i.K8sResource): i.TSResource;
+export declare function isDeploymentList(manifest: i.K8sResourceList): manifest is k8s.V1DeploymentList;
+export declare function isJobList(manifest: i.K8sResourceList): manifest is k8s.V1JobList;
+export declare function isPodList(manifest: i.K8sResourceList): manifest is k8s.V1PodList;
+export declare function isReplicaSetList(manifest: i.K8sResourceList): manifest is k8s.V1ReplicaSetList;
+export declare function isServiceList(manifest: i.K8sResourceList): manifest is k8s.V1ServiceList;
+export declare function isTSDeploymentList(manifest: k8s.V1DeploymentList): manifest is i.TSDeploymentList;
+export declare function isTSJobList(manifest: k8s.V1JobList): manifest is i.TSJobList;
+export declare function isTSPodList(manifest: k8s.V1PodList): manifest is i.TSPodList;
+export declare function isTSReplicaSetList(manifest: k8s.V1ReplicaSetList): manifest is i.TSReplicaSetList;
+export declare function isTSServiceList(manifest: k8s.V1ServiceList): manifest is i.TSServiceList;
+export declare function convertToTSResourceList(resourceList: k8s.V1DeploymentList): i.TSDeploymentList;
+export declare function convertToTSResourceList(resourceList: k8s.V1JobList): i.TSJobList;
+export declare function convertToTSResourceList(resourceList: k8s.V1PodList): i.TSPodList;
+export declare function convertToTSResourceList(resourceList: k8s.V1ReplicaSetList): i.TSReplicaSetList;
+export declare function convertToTSResourceList(resourceList: k8s.V1ServiceList): i.TSServiceList;
+export declare function convertToTSResourceList(resourceList: i.K8sResourceList): i.TSResourceList;
+//# sourceMappingURL=utils.d.ts.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,GAAG,MAAM,yBAAyB,CAAC;AAE/C,OAAO,KAAK,CAAC,MAAM,iBAAiB,CAAC;AAMrC,wBAAgB,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,SAAS,EACzE,MAAM,EAAE,aAAa,GAAG,MAAM,GAAG,UAAU,EAC3C,QAAQ,EAAE,CAAC,CAAC,QAAQ,GACrB,CAAC,MAAM,EAAE,CAAC,CAAC,SAAS,KAAK,CAAC,CAiB5B;AAGD,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,UAE5C;AAED,wBAAgB,oBAAoB,CAChC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,EACtB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,MAAM,EAAE,MAAM,QAiBjB;AAED,wBAAgB,cAAc;;;EAK7B;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,QAAQ,IAAI,GAAG,CAAC,YAAY,CAElF;AAED,wBAAgB,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,QAAQ,IAAI,GAAG,CAAC,KAAK,CAEpE;AAED,wBAAgB,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,QAAQ,IAAI,GAAG,CAAC,KAAK,CAEpE;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,QAAQ,IAAI,GAAG,CAAC,YAAY,CAElF;AAED,wBAAgB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,QAAQ,IAAI,GAAG,CAAC,SAAS,CAE5E;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,GAAG,QAAQ,IAAI,CAAC,CAAC,YAAY,CAQrF;AAED,wBAAgB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,IAAI,CAAC,CAAC,KAAK,CAOhE;AAED,wBAAgB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,QAAQ,IAAI,CAAC,CAAC,KAAK,CAIpE;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,GAAG,QAAQ,IAAI,CAAC,CAAC,YAAY,CAIrF;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,GAAG,QAAQ,IAAI,CAAC,CAAC,SAAS,CAK5E;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;AAChF,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;AAClE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;AAClE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;AAChF,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;AAC1E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC;AAqB3E,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,GAAG,QAAQ,IAAI,GAAG,CAAC,gBAAgB,CAE9F;AAED,wBAAgB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,GAAG,QAAQ,IAAI,GAAG,CAAC,SAAS,CAEhF;AAED,wBAAgB,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,GAAG,QAAQ,IAAI,GAAG,CAAC,SAAS,CAEhF;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,GAAG,QAAQ,IAAI,GAAG,CAAC,gBAAgB,CAE9F;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,GAAG,QAAQ,IAAI,GAAG,CAAC,aAAa,CAExF;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,gBAAgB,GAAG,QAAQ,IAAI,CAAC,CAAC,gBAAgB,CAGjG;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,GAAG,QAAQ,IAAI,CAAC,CAAC,SAAS,CAG5E;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,GAAG,QAAQ,IAAI,CAAC,CAAC,SAAS,CAG5E;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,gBAAgB,GAAG,QAAQ,IAAI,CAAC,CAAC,gBAAgB,CAGjG;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,GAAG,CAAC,aAAa,GAAG,QAAQ,IAAI,CAAC,CAAC,aAAa,CAGxF;AAED,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,gBAAgB,CAAC;AAChG,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;AAClF,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;AAClF,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,gBAAgB,CAAC;AAChG,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC;AAC1F,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,cAAc,CAAC"}

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js

@@ -0,0 +1,170 @@
+import fs from 'node:fs';
+import path from 'node:path';
+// @ts-expect-error
+import barbe from 'barbe';
+import * as k8s from '@kubernetes/client-node';
+import { isTest } from '@terascope/core-utils';
+const MAX_RETRIES = isTest ? 2 : 3;
+const RETRY_DELAY = isTest ? 50 : 1000; // time in ms
+const resourcePath = path.join(process.cwd(), './packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/');
+export function makeTemplate(folder, fileName) {
+    const filePath = path.join(resourcePath, folder, `${fileName}.hbs`);
+    const templateData = fs.readFileSync(filePath, 'utf-8');
+    const templateKeys = ['{{', '}}'];
+    return (config) => {
+        if (folder !== 'jobs' && (config.exName === undefined || config.exUid === undefined)) {
+            throw new Error(`K8s config requires ${config.exName === undefined ? 'exName' : 'exUid'} to create a ${folder} template`);
+        }
+        if (folder !== 'services' && config.dockerImage === undefined) {
+            throw new Error(`K8s config requires a dockerImage to create a ${folder} template`);
+        }
+        const templated = barbe(templateData, templateKeys, config);
+        return JSON.parse(templated);
+    };
+}
+// Convert bytes to MB and reduce by 10%
+export function getMaxOldSpace(memory) {
+    return Math.round(0.9 * (memory / 1024 / 1024));
+}
+export function setMaxOldSpaceViaEnv(envArr, jobEnv, memory) {
+    const envObj = {};
+    if (memory && memory > -1) {
+        // Set NODE_OPTIONS to override max-old-space-size
+        const maxOldSpace = getMaxOldSpace(memory);
+        envObj.NODE_OPTIONS = `--max-old-space-size=${maxOldSpace}`;
+    }
+    Object.assign(envObj, jobEnv);
+    Object.entries(envObj).forEach(([name, value]) => {
+        envArr.push({
+            name,
+            value
+        });
+    });
+}
+export function getRetryConfig() {
+    return {
+        retries: MAX_RETRIES,
+        delay: RETRY_DELAY
+    };
+}
+export function isDeployment(resource) {
+    return resource instanceof k8s.V1Deployment;
+}
+export function isJob(resource) {
+    return resource instanceof k8s.V1Job;
+}
+export function isPod(resource) {
+    return resource instanceof k8s.V1Pod;
+}
+export function isReplicaSet(resource) {
+    return resource instanceof k8s.V1ReplicaSet;
+}
+export function isService(resource) {
+    return resource instanceof k8s.V1Service;
+}
+export function isTSDeployment(manifest) {
+    return manifest instanceof k8s.V1Deployment
+        && manifest.metadata?.labels !== undefined
+        && manifest.metadata.name !== undefined
+        && manifest.spec?.replicas !== undefined
+        && manifest.spec.template.metadata?.labels !== undefined
+        && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined
+        && manifest.spec.template.spec.volumes !== undefined;
+}
+export function isTSJob(manifest) {
+    return manifest instanceof k8s.V1Job
+        && manifest.metadata?.labels !== undefined
+        && manifest.metadata.name !== undefined
+        && manifest.spec?.template.metadata?.labels !== undefined
+        && manifest.spec.template.spec?.containers[0].volumeMounts !== undefined
+        && manifest.spec.template.spec.volumes !== undefined;
+}
+export function isTSPod(manifest) {
+    return manifest instanceof k8s.V1Pod
+        && manifest.metadata?.name !== undefined
+        && manifest.status !== undefined;
+}
+export function isTSReplicaSet(manifest) {
+    return manifest instanceof k8s.V1ReplicaSet
+        && manifest.metadata?.name !== undefined
+        && manifest.status !== undefined;
+}
+export function isTSService(manifest) {
+    return manifest instanceof k8s.V1Service
+        && manifest.metadata?.name !== undefined
+        && manifest.spec?.selector !== undefined
+        && manifest.spec.ports !== undefined;
+}
+export function convertToTSResource(resource) {
+    if (isDeployment(resource) && isTSDeployment(resource)) {
+        return resource;
+    }
+    if (isJob(resource) && isTSJob(resource)) {
+        return resource;
+    }
+    if (isPod(resource) && isTSPod(resource)) {
+        return resource;
+    }
+    if (isReplicaSet(resource) && isTSReplicaSet(resource)) {
+        return resource;
+    }
+    if (isService(resource) && isTSService(resource)) {
+        return resource;
+    }
+    throw new Error('K8sResource missing required field(s) to be converted to TSResource.');
+}
+export function isDeploymentList(manifest) {
+    return manifest.kind === 'DeploymentList';
+}
+export function isJobList(manifest) {
+    return manifest.kind === 'JobList';
+}
+export function isPodList(manifest) {
+    return manifest.kind === 'PodList';
+}
+export function isReplicaSetList(manifest) {
+    return manifest.kind === 'ReplicaSetList';
+}
+export function isServiceList(manifest) {
+    return manifest.kind === 'ServiceList';
+}
+export function isTSDeploymentList(manifest) {
+    return manifest.kind === 'DeploymentList'
+        && (manifest.items[0] ? isTSDeployment(manifest.items[0]) : true);
+}
+export function isTSJobList(manifest) {
+    return manifest.kind === 'JobList'
+        && (manifest.items[0] ? isTSJob(manifest.items[0]) : true);
+}
+export function isTSPodList(manifest) {
+    return manifest.kind === 'PodList'
+        && (manifest.items[0] ? isTSPod(manifest.items[0]) : true);
+}
+export function isTSReplicaSetList(manifest) {
+    return manifest.kind === 'ReplicaSetList'
+        && (manifest.items[0] ? isTSReplicaSet(manifest.items[0]) : true);
+}
+export function isTSServiceList(manifest) {
+    return manifest.kind === 'ServiceList'
+        && (manifest.items[0] ? isTSService(manifest.items[0]) : true);
+}
+export function convertToTSResourceList(resourceList) {
+    resourceList.items.map((resource) => convertToTSResource(resource));
+    if (isDeploymentList(resourceList) && isTSDeploymentList(resourceList)) {
+        return resourceList;
+    }
+    if (isJobList(resourceList) && isTSJobList(resourceList)) {
+        return resourceList;
+    }
+    if (isPodList(resourceList) && isTSPodList(resourceList)) {
+        return resourceList;
+    }
+    if (isReplicaSetList(resourceList) && isTSReplicaSetList(resourceList)) {
+        return resourceList;
+    }
+    if (isServiceList(resourceList) && isTSServiceList(resourceList)) {
+        return resourceList;
+    }
+    throw new Error('K8sResource missing required field(s) to be converted to TSResourceList.');
+}
+//# sourceMappingURL=utils.js.map

package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,mBAAmB;AACnB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,GAAG,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnC,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa;AACrD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,8EAA8E,CAAC,CAAC;AAE9H,MAAM,UAAU,YAAY,CACxB,MAA2C,EAC3C,QAAoB;IAEpB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,QAAQ,MAAM,CAAC,CAAC;IACpE,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAElC,OAAO,CAAC,MAAmB,EAAE,EAAE;QAC3B,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,gBAAgB,MAAM,WAAW,CAAC,CAAC;QAC9H,CAAC;QAED,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,iDAAiD,MAAM,WAAW,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC,CAAC;AACN,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,cAAc,CAAC,MAAc;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAChC,MAAsB,EACtB,MAA2B,EAC3B,MAAc;IAEd,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,IAAI,MAAM,IAAI,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACxB,kDAAkD;QAClD,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,CAAC,YAAY,GAAG,wBAAwB,WAAW,EAAE,CAAC;IAChE,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE9B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE;QAC7C,MAAM,CAAC,IAAI,CAAC;YACR,IAAI;YACJ,KAAK;SACR,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,cAAc;IAC1B,OAAO;QACH,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,WAAW;KACrB,CAAC;AACN,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAuB;IAChD,OAAO,QAAQ,YAAY,GAAG,CAAC,YAAY,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,QAAuB;IACzC,OAAO,QAAQ,YAAY,GAAG,CAAC,KAAK,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,QAAuB;IACzC,OAAO,QAAQ,YAAY,GAAG,CAAC,KAAK,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAuB;IAChD,OAAO,QAAQ,YAAY,GAAG,CAAC,YAAY,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,QAAuB;IAC7C,OAAO,QAAQ,YAAY,GAAG,CAAC,SAAS,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAA0B;IACrD,OAAO,QAAQ,YAAY,GAAG,CAAC,YAAY;WACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,KAAK,SAAS;WACvC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS;WACpC,QAAQ,CAAC,IAAI,EAAE,QAAQ,KAAK,SAAS;WACrC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,KAAK,SAAS;WACrD,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS;WACrE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,QAAmB;IACvC,OAAO,QAAQ,YAAY,GAAG,CAAC,KAAK;WAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,KAAK,SAAS;WACvC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS;WACpC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,KAAK,SAAS;WACtD,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS;WACrE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,QAAuB;IAC3C,OAAO,QAAQ,YAAY,GAAG,CAAC,KAAK;WAC7B,QAAQ,CAAC,QAAQ,EAAE,IAAI,KAAK,SAAS;WACrC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAA0B;IACrD,OAAO,QAAQ,YAAY,GAAG,CAAC,YAAY;WACpC,QAAQ,CAAC,QAAQ,EAAE,IAAI,KAAK,SAAS;WACrC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAuB;IAC/C,OAAO,QAAQ,YAAY,GAAG,CAAC,SAAS;WACjC,QAAQ,CAAC,QAAQ,EAAE,IAAI,KAAK,SAAS;WACrC,QAAQ,CAAC,IAAI,EAAE,QAAQ,KAAK,SAAS;WACrC,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC;AAC7C,CAAC;AAQD,MAAM,UAAU,mBAAmB,CAAC,QAAuB;IACvD,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrD,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrD,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;AAC5F,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAA2B;IACxD,OAAO,QAAQ,CAAC,IAAI,KAAK,gBAAgB,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,QAA2B;IACjD,OAAO,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,QAA2B;IACjD,OAAO,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAA2B;IACxD,OAAO,QAAQ,CAAC,IAAI,KAAK,gBAAgB,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAA2B;IACrD,OAAO,QAAQ,CAAC,IAAI,KAAK,aAAa,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,QAA8B;IAC7D,OAAO,QAAQ,CAAC,IAAI,KAAK,gBAAgB;WAClC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAuB;IAC/C,OAAO,QAAQ,CAAC,IAAI,KAAK,SAAS;WAC3B,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAuB;IAC/C,OAAO,QAAQ,CAAC,IAAI,KAAK,SAAS;WAC3B,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,QAA8B;IAC7D,OAAO,QAAQ,CAAC,IAAI,KAAK,gBAAgB;WAClC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAA2B;IACvD,OAAO,QAAQ,CAAC,IAAI,KAAK,aAAa;WAC/B,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACvE,CAAC;AAQD,MAAM,UAAU,uBAAuB,CAAC,YAA+B;IACnE,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEpE,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,kBAAkB,CAAC,YAAY,CAAC,EAAE,CAAC;QACrE,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,IAAI,SAAS,CAAC,YAAY,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC;QACvD,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,IAAI,SAAS,CAAC,YAAY,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC;QACvD,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,kBAAkB,CAAC,YAAY,CAAC,EAAE,CAAC;QACrE,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,IAAI,aAAa,CAAC,YAAY,CAAC,IAAI,eAAe,CAAC,YAAY,CAAC,EAAE,CAAC;QAC/D,OAAO,YAAY,CAAC;IACxB,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;AAChG,CAAC"}

package/dist/src/lib/cluster/services/cluster/backends/native/dispatch.d.ts

@@ -0,0 +1,6 @@
+export declare class Dispatch {
+    dispatch: Record<string, number>;
+    set(nodeId: string, numOfWorkers: number): void;
+    getDispatch: () => Record<string, number>;
+}
+//# sourceMappingURL=dispatch.d.ts.map

package/dist/src/lib/cluster/services/cluster/backends/native/dispatch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/native/dispatch.ts"],"names":[],"mappings":"AAAA,qBAAa,QAAQ;IACjB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IAEtC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;IAQxC,WAAW,+BAAuB;CACrC"}

package/dist/src/lib/cluster/services/cluster/backends/native/dispatch.js

@@ -0,0 +1,13 @@
+export class Dispatch {
+    dispatch = {};
+    set(nodeId, numOfWorkers) {
+        if (this.dispatch[nodeId]) {
+            this.dispatch[nodeId] += numOfWorkers;
+        }
+        else {
+            this.dispatch[nodeId] = numOfWorkers;
+        }
+    }
+    getDispatch = () => this.dispatch;
+}
+//# sourceMappingURL=dispatch.js.map

package/dist/src/lib/cluster/services/cluster/backends/native/dispatch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dispatch.js","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/native/dispatch.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,QAAQ;IACjB,QAAQ,GAA2B,EAAE,CAAC;IAEtC,GAAG,CAAC,MAAc,EAAE,YAAoB;QACpC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC;QAC1C,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,YAAY,CAAC;QACzC,CAAC;IACL,CAAC;IAED,WAAW,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;CACrC"}

package/dist/src/lib/cluster/services/cluster/backends/native/index.d.ts

@@ -0,0 +1,59 @@
+import { Logger, Queue } from '@terascope/core-utils';
+import type { EventEmitter } from 'node:events';
+import { ExecutionConfig } from '@terascope/types';
+import type { ClusterMasterContext, NodeState } from '../../../../../../interfaces.js';
+import { Messaging } from './messaging.js';
+import { ExecutionStorage } from '../../../../../storage/index.js';
+import { StopExecutionOptions } from '../../../interfaces.js';
+export declare class NativeClustering {
+    context: ClusterMasterContext;
+    logger: Logger;
+    events: EventEmitter;
+    executionStore: ExecutionStorage;
+    pendingWorkerRequests: Queue<any>;
+    nodeStateInterval: number;
+    slicerAllocationAttempts: number;
+    clusterState: Record<string, NodeState>;
+    clusterStateInterval: NodeJS.Timeout | undefined;
+    messaging: Messaging;
+    droppedNodes: Record<string, any>;
+    clusterMasterServer: any;
+    constructor(context: ClusterMasterContext, clusterMasterServer: any);
+    private _cleanUpNode;
+    initialize(): Promise<void>;
+    getClusterState(): Record<string, NodeState>;
+    private _checkNode;
+    private _findNodeForSlicer;
+    private _findNodesForExecution;
+    private _availableWorkers;
+    private _findPort;
+    allocateWorkers(execution: ExecutionConfig, numOfWorkersRequested: number): Promise<any[]>;
+    private _createSlicer;
+    allocateSlicer(ex: ExecutionConfig): Promise<ExecutionConfig>;
+    addWorkers(execution: any, workerNum: number): {
+        action: string;
+        ex_id: any;
+        workerNum: number;
+    };
+    setWorkers(execution: any, workerNum: number): {
+        action: string;
+        ex_id: any;
+        workerNum: number;
+    } | Promise<{
+        action: string;
+        ex_id: string;
+        workerNum: number;
+    }>;
+    removeWorkers(exId: string, workerNum: number): Promise<{
+        action: string;
+        ex_id: string;
+        workerNum: number;
+    }>;
+    private _notifyNodesWithExecution;
+    readyForAllocation(): boolean;
+    clusterAvailable(): void;
+    stopExecution(exId: string, options?: StopExecutionOptions): Promise<unknown>;
+    shutdown(): Promise<void>;
+    listResourcesForJobId(): Promise<never[]>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/src/lib/cluster/services/cluster/backends/native/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../../../src/lib/cluster/services/cluster/backends/native/index.ts"],"names":[],"mappings":"AACA,OAAO,EAEgB,MAAM,EAEzB,KAAK,EACR,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAGvF,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AA6B9D,qBAAa,gBAAgB;IACzB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,YAAY,CAAC;IACrB,cAAc,EAAG,gBAAgB,CAAC;IAClC,qBAAqB,aAAoB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAM;IAC7C,oBAAoB,EAAG,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC;IAClD,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM;IACvC,mBAAmB,EAAE,GAAG,CAAC;gBAEb,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,GAAG;YAoFrD,YAAY;IA6CpB,UAAU;IAiBhB,eAAe;IAIf,OAAO,CAAC,UAAU;IA+BlB,OAAO,CAAC,kBAAkB;IAuB1B,OAAO,CAAC,sBAAsB;IA2B9B,OAAO,CAAC,iBAAiB;IAgBzB,OAAO,CAAC,SAAS;IAUX,eAAe,CAAC,SAAS,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM;YAsFjE,aAAa;IA8CrB,cAAc,CAAC,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAoBnE,UAAU,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM;;;;;IAa5C,UAAU,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM;;;;;;;;;IAiB5C,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;;;;;IAgE7C,OAAO,CAAC,yBAAyB;IAyCjC,kBAAkB;IAIlB,gBAAgB;IAEV,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB;IAW1D,QAAQ;IAWR,qBAAqB;CAG9B"}