terafoundation 0.64.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/api/index.d.ts +2 -2
- package/dist/src/api/index.d.ts.map +1 -1
- package/dist/src/api/index.js +17 -108
- package/dist/src/api/index.js.map +1 -1
- package/dist/src/api/prom-metrics/exporter.js +10 -14
- package/dist/src/api/prom-metrics/exporter.js.map +1 -1
- package/dist/src/api/prom-metrics/prom-metrics-api.d.ts.map +1 -1
- package/dist/src/api/prom-metrics/prom-metrics-api.js +25 -21
- package/dist/src/api/prom-metrics/prom-metrics-api.js.map +1 -1
- package/dist/src/api/utils.d.ts +3 -3
- package/dist/src/api/utils.d.ts.map +1 -1
- package/dist/src/api/utils.js +13 -43
- package/dist/src/api/utils.js.map +1 -1
- package/dist/src/cluster-context.d.ts +4 -3
- package/dist/src/cluster-context.d.ts.map +1 -1
- package/dist/src/cluster-context.js +19 -46
- package/dist/src/cluster-context.js.map +1 -1
- package/dist/src/connector-utils.d.ts +4 -5
- package/dist/src/connector-utils.d.ts.map +1 -1
- package/dist/src/connector-utils.js +39 -45
- package/dist/src/connector-utils.js.map +1 -1
- package/dist/src/connectors/elasticsearch-next.d.ts +3 -10
- package/dist/src/connectors/elasticsearch-next.d.ts.map +1 -1
- package/dist/src/connectors/elasticsearch-next.js +5 -8
- package/dist/src/connectors/elasticsearch-next.js.map +1 -1
- package/dist/src/connectors/hdfs.d.ts +3 -9
- package/dist/src/connectors/hdfs.d.ts.map +1 -1
- package/dist/src/connectors/hdfs.js +13 -13
- package/dist/src/connectors/hdfs.js.map +1 -1
- package/dist/src/connectors/hdfs_ha.d.ts +3 -9
- package/dist/src/connectors/hdfs_ha.d.ts.map +1 -1
- package/dist/src/connectors/hdfs_ha.js +25 -24
- package/dist/src/connectors/hdfs_ha.js.map +1 -1
- package/dist/src/connectors/s3.d.ts +3 -13
- package/dist/src/connectors/s3.d.ts.map +1 -1
- package/dist/src/connectors/s3.js +9 -14
- package/dist/src/connectors/s3.js.map +1 -1
- package/dist/src/core-context.d.ts +8 -9
- package/dist/src/core-context.d.ts.map +1 -1
- package/dist/src/core-context.js +16 -40
- package/dist/src/core-context.js.map +1 -1
- package/dist/src/index.d.ts +5 -5
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -21
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces.d.ts +2 -107
- package/dist/src/interfaces.d.ts.map +1 -1
- package/dist/src/interfaces.js +1 -2
- package/dist/src/master.d.ts +2 -2
- package/dist/src/master.d.ts.map +1 -1
- package/dist/src/master.js +7 -10
- package/dist/src/master.js.map +1 -1
- package/dist/src/process-context.d.ts +4 -3
- package/dist/src/process-context.d.ts.map +1 -1
- package/dist/src/process-context.js +10 -17
- package/dist/src/process-context.js.map +1 -1
- package/dist/src/schema.d.ts +0 -2
- package/dist/src/schema.d.ts.map +1 -1
- package/dist/src/schema.js +3 -56
- package/dist/src/schema.js.map +1 -1
- package/dist/src/sysconfig.d.ts +3 -3
- package/dist/src/sysconfig.d.ts.map +1 -1
- package/dist/src/sysconfig.js +28 -34
- package/dist/src/sysconfig.js.map +1 -1
- package/dist/src/test-context.d.ts +7 -7
- package/dist/src/test-context.d.ts.map +1 -1
- package/dist/src/test-context.js +33 -80
- package/dist/src/test-context.js.map +1 -1
- package/dist/src/validate-configs.d.ts +2 -2
- package/dist/src/validate-configs.d.ts.map +1 -1
- package/dist/src/validate-configs.js +31 -60
- package/dist/src/validate-configs.js.map +1 -1
- package/dist/src/worker.d.ts +2 -2
- package/dist/src/worker.d.ts.map +1 -1
- package/dist/src/worker.js +1 -4
- package/dist/src/worker.js.map +1 -1
- package/package.json +6 -5
|
@@ -1,51 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.ClusterContext = void 0;
|
|
30
|
-
const cluster_1 = __importDefault(require("cluster"));
|
|
31
|
-
const ts = __importStar(require("@terascope/utils"));
|
|
32
|
-
const sysconfig_1 = require("./sysconfig");
|
|
33
|
-
const validate_configs_1 = __importDefault(require("./validate-configs"));
|
|
34
|
-
const core_context_1 = require("./core-context");
|
|
35
|
-
const master_1 = __importDefault(require("./master"));
|
|
36
|
-
const cluster = cluster_1.default;
|
|
1
|
+
import _cluster from 'node:cluster';
|
|
2
|
+
import { get, isFunction, getFullErrorStack } from '@terascope/utils';
|
|
3
|
+
import { getArgs } from './sysconfig.js';
|
|
4
|
+
import validateConfigs from './validate-configs.js';
|
|
5
|
+
import { CoreContext, handleStdStreams } from './core-context.js';
|
|
6
|
+
import master from './master.js';
|
|
7
|
+
const cluster = _cluster;
|
|
37
8
|
// this module is not really testable
|
|
38
9
|
/* istanbul ignore next */
|
|
39
10
|
/**
|
|
40
11
|
* A Cluster Context (with worker processes), useful for scaling
|
|
41
12
|
*/
|
|
42
|
-
class ClusterContext extends
|
|
43
|
-
constructor(config) {
|
|
44
|
-
const parsedArgs = (0, sysconfig_1.getArgs)(config.default_config_file);
|
|
45
|
-
const sysconfig = (0, validate_configs_1.default)(cluster, config, parsedArgs.configfile);
|
|
13
|
+
export class ClusterContext extends CoreContext {
|
|
14
|
+
constructor(config, sysconfig) {
|
|
46
15
|
super(config, cluster, sysconfig);
|
|
47
16
|
this._errorHandler = this._errorHandler.bind(this);
|
|
48
|
-
|
|
17
|
+
handleStdStreams();
|
|
49
18
|
process.on('uncaughtException', this._errorHandler);
|
|
50
19
|
process.on('unhandledRejection', this._errorHandler);
|
|
51
20
|
if (config.script) {
|
|
@@ -58,8 +27,8 @@ class ClusterContext extends core_context_1.CoreContext {
|
|
|
58
27
|
/**
|
|
59
28
|
* Use cluster to start multiple workers
|
|
60
29
|
*/
|
|
61
|
-
(
|
|
62
|
-
if (
|
|
30
|
+
master(this, config);
|
|
31
|
+
if (isFunction(config.master)) {
|
|
63
32
|
config.master(this, config);
|
|
64
33
|
}
|
|
65
34
|
}
|
|
@@ -91,16 +60,20 @@ class ClusterContext extends core_context_1.CoreContext {
|
|
|
91
60
|
? this.logger.error.bind(this.logger)
|
|
92
61
|
: console.error;
|
|
93
62
|
if (cluster.isMaster) {
|
|
94
|
-
logErr(
|
|
63
|
+
logErr(getFullErrorStack(err), `Error in master with pid: ${process.pid}`);
|
|
95
64
|
}
|
|
96
65
|
else {
|
|
97
|
-
logErr(
|
|
66
|
+
logErr(getFullErrorStack(err), `Error in worker: ${get(this.cluster, 'worker.id')} pid: ${process.pid}`);
|
|
98
67
|
}
|
|
99
68
|
// log saving to disk is async, using hack to give time to flush
|
|
100
69
|
setTimeout(() => {
|
|
101
70
|
process.exit(-1);
|
|
102
71
|
}, 600);
|
|
103
72
|
}
|
|
73
|
+
static async createContext(config) {
|
|
74
|
+
const parsedArgs = await getArgs(config.default_config_file);
|
|
75
|
+
const sysconfig = await validateConfigs(cluster, config, parsedArgs.configfile);
|
|
76
|
+
return new ClusterContext(config, sysconfig);
|
|
77
|
+
}
|
|
104
78
|
}
|
|
105
|
-
exports.ClusterContext = ClusterContext;
|
|
106
79
|
//# sourceMappingURL=cluster-context.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cluster-context.js","sourceRoot":"","sources":["../../src/cluster-context.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cluster-context.js","sourceRoot":"","sources":["../../src/cluster-context.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEtE,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,eAAe,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,OAAO,GAA4B,QAAgB,CAAC;AAE1D,qCAAqC;AACrC,0BAA0B;AAC1B;;EAEE;AACF,MAAM,OAAO,cAIX,SAAQ,WAAoB;IAC1B,YACI,MAAsC,EACtC,SAAsC;QAEtC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEnD,gBAAgB,EAAE,CAAC;QACnB,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACpD,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAErD,IAAI,MAAM,CAAC,MAAM,EAAE;YACf;;eAEG;YACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SACvB;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE;YAC9B;;eAEG;YACH,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACrB,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;aAC/B;SACJ;aAAM;YACH;;eAEG;YACH,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,MAAM,CAAC,WAAW,EAAE;gBACpB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC5C,IAAI,IAAI,CAAC,UAAU,KAAK,GAAG,EAAE;wBACzB,QAAQ,GAAG,IAAI,CAAC;wBAChB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;qBACrB;gBACL,CAAC,CAAC,CAAC;gBACH,sDAAsD;gBACtD,IAAI,CAAC,QAAQ,EAAE;oBACX,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,CAAC;iBACxB;aACJ;iBAAM;gBACH,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,CAAC;aACxB;SACJ;IACL,CAAC;IAEO,aAAa,CAAC,GAAQ;QAC1B,sCAAsC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YACrC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QAEpB,IAAI,OAAO,CAAC,QAAQ,EAAE;YAClB,MAAM,CACF,iBAAiB,CAAC,GAAG,CAAC,EACtB,6BAA6B,OAAO,CAAC,GAAG,EAAE,CAC7C,CAAC;SACL;aAAM;YACH,MAAM,CACF,iBAAiB,CAAC,GAAG,CAAC,EACtB,oBAAoB,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,SAAS,OAAO,CAAC,GAAG,EAAE,CAC3E,CAAC;SACL;QAED,gEAAgE;QAChE,UAAU,CAAC,GAAG,EAAE;YACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,EAAE,GAAG,CAAC,CAAC;IACZ,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CAKtB,MAAsC;QAEtC,MAAM,UAAU,GAAG,MAAM,OAAO,CAC5B,MAAM,CAAC,mBAAmB,CAC7B,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,eAAe,CACnC,OAAO,EACP,MAAM,EACN,UAAU,CAAC,UAAU,CACxB,CAAC;QAEF,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACjD,CAAC;CACJ"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Logger } from '@terascope/utils';
|
|
2
|
-
import { Terafoundation } from '@terascope/types';
|
|
3
|
-
export declare function getConnectorModule(name: string, reason: string):
|
|
4
|
-
export declare function getConnectorSchemaAndValFn<S>(name: string): Terafoundation.Initializers<S
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function createClient(name: string, moduleConfig: Record<string, any>, logger: Logger, options: Record<string, any>): Promise<any>;
|
|
2
|
+
import type { Terafoundation } from '@terascope/types';
|
|
3
|
+
export declare function getConnectorModule<S = Record<string, any>>(name: string, reason: string): Promise<Terafoundation.Connector<S> | null>;
|
|
4
|
+
export declare function getConnectorSchemaAndValFn<S>(name: string): Promise<Terafoundation.Initializers<S>>;
|
|
5
|
+
export declare function createClient(name: string, moduleConfig: Record<string, any>, logger: Logger, options: Record<string, any>): Promise<Terafoundation.ConnectorOutput>;
|
|
7
6
|
//# sourceMappingURL=connector-utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connector-utils.d.ts","sourceRoot":"","sources":["../../src/connector-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"connector-utils.d.ts","sourceRoot":"","sources":["../../src/connector-utils.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAqFvD,wBAAsB,kBAAkB,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5D,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CA2C7C;AAED,wBAAsB,0BAA0B,CAAC,CAAC,EAC9C,IAAI,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAWzC;AAED,wBAAsB,YAAY,CAC9B,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,2CAUhG"}
|
|
@@ -1,17 +1,25 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
function requireConnector(filePath, errors) {
|
|
10
|
-
let mod = require(filePath);
|
|
11
|
-
if (mod && mod.default) {
|
|
12
|
-
mod = mod.default;
|
|
13
|
-
}
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { TSError, parseError } from '@terascope/utils';
|
|
4
|
+
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
5
|
+
const pathFragments = dirname.split('/');
|
|
6
|
+
const terafoundationPathIndex = pathFragments.findIndex((name) => name === 'terafoundation') + 1;
|
|
7
|
+
const builtinConnectorPath = pathFragments.slice(0, terafoundationPathIndex).concat(['dist', 'src', 'connectors']).join('/');
|
|
8
|
+
async function requireConnector(filePath, errors) {
|
|
14
9
|
let valid = true;
|
|
10
|
+
let mod;
|
|
11
|
+
try {
|
|
12
|
+
mod = await import(filePath);
|
|
13
|
+
if (mod && mod.default) {
|
|
14
|
+
mod = mod.default;
|
|
15
|
+
if (mod.default) {
|
|
16
|
+
mod = mod.default;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
valid = false;
|
|
22
|
+
}
|
|
15
23
|
if (typeof mod !== 'object') {
|
|
16
24
|
valid = false;
|
|
17
25
|
}
|
|
@@ -29,60 +37,58 @@ function requireConnector(filePath, errors) {
|
|
|
29
37
|
});
|
|
30
38
|
valid = false;
|
|
31
39
|
}
|
|
32
|
-
if (mod && typeof mod.
|
|
40
|
+
if (mod && typeof mod.createClient !== 'function') {
|
|
33
41
|
errors.push({
|
|
34
42
|
filePath,
|
|
35
|
-
message: `Connector ${filePath} missing required
|
|
43
|
+
message: `Connector ${filePath} missing required createClient function`
|
|
36
44
|
});
|
|
37
45
|
valid = false;
|
|
38
46
|
}
|
|
39
|
-
if (valid)
|
|
47
|
+
if (valid) {
|
|
40
48
|
return mod;
|
|
49
|
+
}
|
|
41
50
|
return null;
|
|
42
51
|
}
|
|
43
|
-
function guardedRequire(filePath, errors) {
|
|
52
|
+
async function guardedRequire(filePath, errors) {
|
|
44
53
|
try {
|
|
45
54
|
return requireConnector(filePath, errors);
|
|
46
55
|
}
|
|
47
56
|
catch (error) {
|
|
48
|
-
if (error.code === 'MODULE_NOT_FOUND') {
|
|
49
|
-
return false;
|
|
50
|
-
}
|
|
51
57
|
errors.push({
|
|
52
58
|
filePath,
|
|
53
|
-
message:
|
|
59
|
+
message: parseError(error, true),
|
|
54
60
|
});
|
|
55
61
|
return null;
|
|
56
62
|
}
|
|
57
63
|
}
|
|
58
|
-
function getConnectorModule(name, reason) {
|
|
64
|
+
export async function getConnectorModule(name, reason) {
|
|
59
65
|
let mod;
|
|
60
66
|
// collect the errors
|
|
61
67
|
const errors = [];
|
|
62
|
-
const localPath =
|
|
63
|
-
mod = guardedRequire(localPath, errors);
|
|
68
|
+
const localPath = `${path.join(builtinConnectorPath, name)}.js`;
|
|
69
|
+
mod = await guardedRequire(localPath, errors);
|
|
64
70
|
// check if its a node module
|
|
65
71
|
if (!mod) {
|
|
66
|
-
mod = guardedRequire(name, errors);
|
|
72
|
+
mod = await guardedRequire(name, errors);
|
|
67
73
|
}
|
|
68
74
|
// Still not found check for a connector with underscores
|
|
69
75
|
if (!mod) {
|
|
70
|
-
mod = guardedRequire(`terafoundation_${name}_connector`, errors);
|
|
76
|
+
mod = await guardedRequire(`terafoundation_${name}_connector`, errors);
|
|
71
77
|
}
|
|
72
78
|
// Still not found check for a connector with dashes
|
|
73
79
|
if (!mod) {
|
|
74
|
-
mod = guardedRequire(`terafoundation-${name}-connector`, errors);
|
|
80
|
+
mod = await guardedRequire(`terafoundation-${name}-connector`, errors);
|
|
75
81
|
}
|
|
76
82
|
// Stil not found check for the @terascope namespace
|
|
77
83
|
if (!mod) {
|
|
78
|
-
mod = guardedRequire(`@terascope/${name}`, errors);
|
|
84
|
+
mod = await guardedRequire(`@terascope/${name}`, errors);
|
|
79
85
|
}
|
|
80
86
|
if (mod)
|
|
81
87
|
return mod;
|
|
82
88
|
if (errors.length) {
|
|
83
89
|
const filePaths = errors.map(({ filePath }) => filePath);
|
|
84
90
|
const messages = errors.map(({ message }) => message);
|
|
85
|
-
throw new
|
|
91
|
+
throw new TSError(messages.join(', caused by '), {
|
|
86
92
|
reason,
|
|
87
93
|
context: {
|
|
88
94
|
filePaths,
|
|
@@ -91,33 +97,21 @@ function getConnectorModule(name, reason) {
|
|
|
91
97
|
}
|
|
92
98
|
return null;
|
|
93
99
|
}
|
|
94
|
-
|
|
95
|
-
function getConnectorSchemaAndValFn(name) {
|
|
100
|
+
export async function getConnectorSchemaAndValFn(name) {
|
|
96
101
|
const reason = `Could not retrieve schema code for: ${name}\n`;
|
|
97
|
-
const mod = getConnectorModule(name, reason);
|
|
102
|
+
const mod = await getConnectorModule(name, reason);
|
|
98
103
|
if (!mod) {
|
|
99
104
|
console.warn(`[WARNING] ${reason}`);
|
|
100
105
|
return { schema: {} };
|
|
101
106
|
}
|
|
102
107
|
return { schema: mod.config_schema(), validatorFn: mod.validate_config };
|
|
103
108
|
}
|
|
104
|
-
|
|
105
|
-
function createConnection(name, moduleConfig, logger, options) {
|
|
106
|
-
const reason = `Could not find connector implementation for: ${name}\n`;
|
|
107
|
-
const mod = getConnectorModule(name, reason);
|
|
108
|
-
if (!mod) {
|
|
109
|
-
throw new Error(reason);
|
|
110
|
-
}
|
|
111
|
-
return mod.create(moduleConfig, logger, options);
|
|
112
|
-
}
|
|
113
|
-
exports.createConnection = createConnection;
|
|
114
|
-
async function createClient(name, moduleConfig, logger, options) {
|
|
109
|
+
export async function createClient(name, moduleConfig, logger, options) {
|
|
115
110
|
const reason = `Could not find connector implementation for: ${name}\n`;
|
|
116
|
-
const mod = getConnectorModule(name, reason);
|
|
111
|
+
const mod = await getConnectorModule(name, reason);
|
|
117
112
|
if (!mod) {
|
|
118
113
|
throw new Error(reason);
|
|
119
114
|
}
|
|
120
115
|
return mod.createClient(moduleConfig, logger, options);
|
|
121
116
|
}
|
|
122
|
-
exports.createClient = createClient;
|
|
123
117
|
//# sourceMappingURL=connector-utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connector-utils.js","sourceRoot":"","sources":["../../src/connector-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"connector-utils.js","sourceRoot":"","sources":["../../src/connector-utils.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAU,MAAM,kBAAkB,CAAC;AAG/D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7D,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzC,MAAM,uBAAuB,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAEjG,MAAM,oBAAoB,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAO7H,KAAK,UAAU,gBAAgB,CAC3B,QAAgB,EAChB,MAAqB;IAErB,IAAI,KAAK,GAAG,IAAI,CAAC;IACjB,IAAI,GAAQ,CAAC;IAEb,IAAI;QACA,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE7B,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE;YACpB,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC;YAClB,IAAI,GAAG,CAAC,OAAO,EAAE;gBACb,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC;aACrB;SACJ;KACJ;IAAC,OAAO,GAAG,EAAE;QACV,KAAK,GAAG,KAAK,CAAC;KACjB;IAED,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;QACzB,KAAK,GAAG,KAAK,CAAC;KACjB;IAED,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,UAAU,EAAE;QAChD,MAAM,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,OAAO,EAAE,aAAa,QAAQ,0CAA0C;SAC3E,CAAC,CAAC;QACH,KAAK,GAAG,KAAK,CAAC;KACjB;IAED,IAAI,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,OAAO,GAAG,CAAC,eAAe,KAAK,UAAU,EAAE;QACzE,MAAM,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,OAAO,EAAE,aAAa,QAAQ,qCAAqC;SACtE,CAAC,CAAC;QACH,KAAK,GAAG,KAAK,CAAC;KACjB;IAED,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE;QAC/C,MAAM,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,OAAO,EAAE,aAAa,QAAQ,yCAAyC;SAC1E,CAAC,CAAC;QAEH,KAAK,GAAG,KAAK,CAAC;KACjB;IAED,IAAI,KAAK,EAAE;QACP,OAAO,GAAG,CAAC;KACd;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CACzB,QAAgB,EAChB,MAAqB;IAErB,IAAI;QACA,OAAO,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;KAC7C;IAAC,OAAO,KAAK,EAAE;QACZ,MAAM,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC;SACnC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;KACf;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,IAAY,EACZ,MAAc;IAEd,IAAI,GAAuC,CAAC;IAE5C,qBAAqB;IACrB,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,MAAM,SAAS,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC;IAChE,GAAG,GAAG,MAAM,cAAc,CAAI,SAAS,EAAE,MAAM,CAAC,CAAC;IAEjD,6BAA6B;IAC7B,IAAI,CAAC,GAAG,EAAE;QACN,GAAG,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;KAC5C;IAED,yDAAyD;IACzD,IAAI,CAAC,GAAG,EAAE;QACN,GAAG,GAAG,MAAM,cAAc,CAAC,kBAAkB,IAAI,YAAY,EAAE,MAAM,CAAC,CAAC;KAC1E;IAED,oDAAoD;IACpD,IAAI,CAAC,GAAG,EAAE;QACN,GAAG,GAAG,MAAM,cAAc,CAAC,kBAAkB,IAAI,YAAY,EAAE,MAAM,CAAC,CAAC;KAC1E;IAED,oDAAoD;IACpD,IAAI,CAAC,GAAG,EAAE;QACN,GAAG,GAAG,MAAM,cAAc,CAAC,cAAc,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;KAC5D;IAED,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,IAAI,MAAM,CAAC,MAAM,EAAE;QACf,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;YAC7C,MAAM;YACN,OAAO,EAAE;gBACL,SAAS;aACZ;SACJ,CAAC,CAAC;KACN;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC5C,IAAY;IAEZ,MAAM,MAAM,GAAG,uCAAuC,IAAI,IAAI,CAAC;IAE/D,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAI,IAAI,EAAE,MAAM,CAAC,CAAC;IAEtD,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,CAAC,IAAI,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;QACpC,OAAO,EAAE,MAAM,EAAE,EAA8B,EAAE,CAAC;KACrD;IAED,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,eAAe,EAAE,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,IAAY,EAAE,YAAiC,EAAE,MAAc,EAAE,OAA4B;IAE7F,MAAM,MAAM,GAAG,gDAAgD,IAAI,IAAI,CAAC;IAExE,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,GAAG,EAAE;QACN,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;KAC3B;IAED,OAAO,GAAG,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -1,11 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
declare const
|
|
3
|
-
|
|
4
|
-
createClient(customConfig: Record<string, any>, logger: Logger): Promise<{
|
|
5
|
-
log: () => Logger;
|
|
6
|
-
client: import("elasticsearch-store").Client;
|
|
7
|
-
}>;
|
|
8
|
-
config_schema(): Record<string, any>;
|
|
9
|
-
};
|
|
10
|
-
export default _default;
|
|
1
|
+
import type { Terafoundation } from '@terascope/types';
|
|
2
|
+
declare const connector: Terafoundation.Connector;
|
|
3
|
+
export default connector;
|
|
11
4
|
//# sourceMappingURL=elasticsearch-next.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"elasticsearch-next.d.ts","sourceRoot":"","sources":["../../../src/connectors/elasticsearch-next.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"elasticsearch-next.d.ts","sourceRoot":"","sources":["../../../src/connectors/elasticsearch-next.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGvD,QAAA,MAAM,SAAS,EAAE,cAAc,CAAC,SA8B/B,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
const elasticsearch_store_1 = require("elasticsearch-store");
|
|
4
|
-
exports.default = {
|
|
5
|
-
create() {
|
|
6
|
-
throw new Error('elasticsearch-next does not support the deprecated "create" method');
|
|
7
|
-
},
|
|
1
|
+
import { createClient as esCreateClient } from 'elasticsearch-store';
|
|
2
|
+
const connector = {
|
|
8
3
|
async createClient(customConfig, logger) {
|
|
9
|
-
|
|
4
|
+
const { client, log } = await esCreateClient(customConfig, logger);
|
|
5
|
+
return { client, logger: log() };
|
|
10
6
|
},
|
|
11
7
|
config_schema() {
|
|
12
8
|
return {
|
|
@@ -34,4 +30,5 @@ exports.default = {
|
|
|
34
30
|
};
|
|
35
31
|
}
|
|
36
32
|
};
|
|
33
|
+
export default connector;
|
|
37
34
|
//# sourceMappingURL=elasticsearch-next.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"elasticsearch-next.js","sourceRoot":"","sources":["../../../src/connectors/elasticsearch-next.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"elasticsearch-next.js","sourceRoot":"","sources":["../../../src/connectors/elasticsearch-next.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAAiC,EAAE,MAAc;QAChE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACnE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;IACrC,CAAC;IACD,aAAa;QACT,OAAO;YACH,IAAI,EAAE;gBACF,GAAG,EAAE,+BAA+B;gBACpC,OAAO,EAAE,CAAC,uBAAuB,CAAC;aACrC;YACD,YAAY,EAAE;gBACV,GAAG,EAAE,yBAAyB;gBAC9B,OAAO,EAAE,KAAK;aACjB;YACD,sBAAsB,EAAE;gBACpB,GAAG,EAAE,mCAAmC;gBACxC,OAAO,EAAE,KAAK;aACjB;YACD,cAAc,EAAE;gBACZ,GAAG,EAAE,iBAAiB;gBACtB,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,UAAU;aACrB;YACD,UAAU,EAAE;gBACR,GAAG,EAAE,sCAAsC;gBAC3C,OAAO,EAAE,CAAC;aACb;SACJ,CAAC;IACN,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
declare
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
declare const _default: {
|
|
6
|
-
create: typeof create;
|
|
7
|
-
config_schema(): Record<string, any>;
|
|
8
|
-
};
|
|
9
|
-
export default _default;
|
|
1
|
+
import type { Terafoundation } from '@terascope/types';
|
|
2
|
+
declare const connector: Terafoundation.Connector;
|
|
3
|
+
export default connector;
|
|
10
4
|
//# sourceMappingURL=hdfs.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs.d.ts","sourceRoot":"","sources":["../../../src/connectors/hdfs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hdfs.d.ts","sourceRoot":"","sources":["../../../src/connectors/hdfs.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIvD,QAAA,MAAM,SAAS,EAAE,cAAc,CAAC,SAgC/B,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
create,
|
|
1
|
+
// @ts-expect-error
|
|
2
|
+
import { WebHDFSClient } from 'node-webhdfs';
|
|
3
|
+
const connector = {
|
|
4
|
+
async createClient(customConfig, logger) {
|
|
5
|
+
logger.info(`Using hdfs hosts: ${customConfig.host}`);
|
|
6
|
+
// TODO: there's no error handling here at all???
|
|
7
|
+
const client = new WebHDFSClient(customConfig);
|
|
8
|
+
return {
|
|
9
|
+
client,
|
|
10
|
+
logger
|
|
11
|
+
};
|
|
12
|
+
},
|
|
14
13
|
config_schema() {
|
|
15
14
|
return {
|
|
16
15
|
user: {
|
|
@@ -32,4 +31,5 @@ exports.default = {
|
|
|
32
31
|
};
|
|
33
32
|
}
|
|
34
33
|
};
|
|
34
|
+
export default connector;
|
|
35
35
|
//# sourceMappingURL=hdfs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs.js","sourceRoot":"","sources":["../../../src/connectors/hdfs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hdfs.js","sourceRoot":"","sources":["../../../src/connectors/hdfs.ts"],"names":[],"mappings":"AAEA,mBAAmB;AACnB,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAAiC,EAAE,MAAc;QAChE,MAAM,CAAC,IAAI,CAAC,qBAAqB,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QAEtD,iDAAiD;QACjD,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,YAAY,CAAC,CAAC;QAE/C,OAAO;YACH,MAAM;YACN,MAAM;SACT,CAAC;IACN,CAAC;IACD,aAAa;QACT,OAAO;YACH,IAAI,EAAE;gBACF,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,SAAS;aACrB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,KAAK;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,WAAW;aACvB;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,aAAa;aACzB;SACJ,CAAC;IACN,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
declare
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
declare const _default: {
|
|
6
|
-
create: typeof create;
|
|
7
|
-
config_schema(): Record<string, any>;
|
|
8
|
-
};
|
|
9
|
-
export default _default;
|
|
1
|
+
import type { Terafoundation } from '@terascope/types';
|
|
2
|
+
declare const connector: Terafoundation.Connector;
|
|
3
|
+
export default connector;
|
|
10
4
|
//# sourceMappingURL=hdfs_ha.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs_ha.d.ts","sourceRoot":"","sources":["../../../src/connectors/hdfs_ha.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"hdfs_ha.d.ts","sourceRoot":"","sources":["../../../src/connectors/hdfs_ha.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAOvD,QAAA,MAAM,SAAS,EAAE,cAAc,CAAC,SAwD/B,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
(
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
1
|
+
import pkg from 'bluebird';
|
|
2
|
+
// @ts-expect-error
|
|
3
|
+
import { WebHDFSClient } from 'node-webhdfs';
|
|
4
|
+
const { promisifyAll } = pkg;
|
|
5
|
+
const connector = {
|
|
6
|
+
async createClient(customConfig, logger) {
|
|
7
|
+
let highAvailibility = false;
|
|
8
|
+
let currentNameNode;
|
|
9
|
+
if (Array.isArray(customConfig.namenode_host)) {
|
|
10
|
+
([currentNameNode] = customConfig.namenode_host);
|
|
11
|
+
customConfig.namenode_list = customConfig.namenode_host;
|
|
12
|
+
highAvailibility = true;
|
|
13
|
+
}
|
|
14
|
+
else {
|
|
15
|
+
currentNameNode = customConfig.namenode_host;
|
|
16
|
+
}
|
|
17
|
+
const config = Object.assign({}, customConfig, { namenode_host: currentNameNode });
|
|
18
|
+
const client = new WebHDFSClient(config);
|
|
19
|
+
logger.info(`Using hdfs hosts: ${currentNameNode}, high-availability: ${highAvailibility}`);
|
|
20
|
+
return {
|
|
21
|
+
client: promisifyAll(client),
|
|
22
|
+
logger
|
|
23
|
+
};
|
|
24
|
+
},
|
|
25
25
|
config_schema() {
|
|
26
26
|
return {
|
|
27
27
|
user: {
|
|
@@ -56,4 +56,5 @@ exports.default = {
|
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
58
|
};
|
|
59
|
+
export default connector;
|
|
59
60
|
//# sourceMappingURL=hdfs_ha.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs_ha.js","sourceRoot":"","sources":["../../../src/connectors/hdfs_ha.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hdfs_ha.js","sourceRoot":"","sources":["../../../src/connectors/hdfs_ha.ts"],"names":[],"mappings":"AAEA,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,mBAAmB;AACnB,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;AAE7B,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAAiC,EAAE,MAAc;QAChE,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,eAAe,CAAC;QAEpB,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE;YAC3C,CAAC,CAAC,eAAe,CAAC,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;YACjD,YAAY,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAC;YACxD,gBAAgB,GAAG,IAAI,CAAC;SAC3B;aAAM;YACH,eAAe,GAAG,YAAY,CAAC,aAAa,CAAC;SAChD;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC,CAAC;QACnF,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,CAAC,IAAI,CAAC,qBAAqB,eAAe,wBAAwB,gBAAgB,EAAE,CAAC,CAAC;QAE5F,OAAO;YACH,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC;YAC5B,MAAM;SACT,CAAC;IACN,CAAC;IACD,aAAa;QACT,OAAO;YACH,IAAI,EAAE;gBACF,GAAG,EAAE,6BAA6B;gBAClC,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,MAAM;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,KAAK;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,qDAAqD;gBAC1D,OAAO,EAAE,IAAI;gBACb,MAAM,CAAC,GAAQ;oBACX,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;wBACzB,OAAO;qBACV;oBACD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;wBACpB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;4BAChB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;yBACzF;wBACD,OAAO;qBACV;oBACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;gBAC7G,CAAC;aACJ;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,iCAAiC;gBACtC,OAAO,EAAE,aAAa;aACzB;SACJ,CAAC;IACN,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,14 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
declare const _default: {
|
|
5
|
-
create(): never;
|
|
6
|
-
createClient(customConfig: S3ClientConfig, logger: Logger): Promise<{
|
|
7
|
-
client: import("@terascope/file-asset-apis").S3Client;
|
|
8
|
-
logger: Logger;
|
|
9
|
-
}>;
|
|
10
|
-
config_schema(): Record<string, any>;
|
|
11
|
-
validate_config<S>(subconfig: S3ClientConfig, _sysconfig: Terafoundation.SysConfig<S>): void;
|
|
12
|
-
};
|
|
13
|
-
export default _default;
|
|
1
|
+
import type { Terafoundation } from '@terascope/types';
|
|
2
|
+
declare const connector: Terafoundation.Connector;
|
|
3
|
+
export default connector;
|
|
14
4
|
//# sourceMappingURL=s3.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../src/connectors/s3.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../src/connectors/s3.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEvD,QAAA,MAAM,SAAS,EAAE,cAAc,CAAC,SA2E/B,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,12 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
const file_asset_apis_1 = require("@terascope/file-asset-apis");
|
|
4
|
-
exports.default = {
|
|
5
|
-
create() {
|
|
6
|
-
throw new Error('s3 does not support the deprecated "create" method, please use file-assets >= v2.4.0');
|
|
7
|
-
},
|
|
1
|
+
import { createS3Client } from '@terascope/file-asset-apis';
|
|
2
|
+
const connector = {
|
|
8
3
|
async createClient(customConfig, logger) {
|
|
9
|
-
const client = await
|
|
4
|
+
const client = await createS3Client(customConfig, logger);
|
|
10
5
|
return { client, logger };
|
|
11
6
|
},
|
|
12
7
|
config_schema() {
|
|
@@ -62,19 +57,19 @@ exports.default = {
|
|
|
62
57
|
}
|
|
63
58
|
};
|
|
64
59
|
},
|
|
65
|
-
validate_config(
|
|
66
|
-
|
|
67
|
-
const
|
|
68
|
-
const certLocationExists = (((_b = subconfig.certLocation) === null || _b === void 0 ? void 0 : _b.length) !== 0);
|
|
60
|
+
validate_config(config, _sysconfig) {
|
|
61
|
+
const caCertExists = (config.caCertificate?.length !== 0);
|
|
62
|
+
const certLocationExists = (config.certLocation?.length !== 0);
|
|
69
63
|
if (caCertExists && certLocationExists) {
|
|
70
64
|
throw new Error('"caCertificate" and "certLocation" contradict.\n'
|
|
71
65
|
+ ' Use only one or the other.');
|
|
72
66
|
}
|
|
73
|
-
else if ((caCertExists && !
|
|
74
|
-
|| (certLocationExists && !
|
|
67
|
+
else if ((caCertExists && !config.sslEnabled)
|
|
68
|
+
|| (certLocationExists && !config.sslEnabled)) {
|
|
75
69
|
throw new Error('A certificate is provided but sslEnabled is set to "false".\n'
|
|
76
70
|
+ ' Set sslEnabled to "true" or don\'t provide a certificate.');
|
|
77
71
|
}
|
|
78
72
|
}
|
|
79
73
|
};
|
|
74
|
+
export default connector;
|
|
80
75
|
//# sourceMappingURL=s3.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../../../src/connectors/s3.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../../../src/connectors/s3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAkB,MAAM,4BAA4B,CAAC;AAG5E,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAA4B,EAAE,MAAc;QAC3D,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC9B,CAAC;IACD,aAAa;QACT,OAAO;YACH,QAAQ,EAAE;gBACN,GAAG,EAAE,oBAAoB;gBACzB,OAAO,EAAE,cAAc;aAC1B;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM;aACjB;YACD,eAAe,EAAE;gBACb,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM;aACjB;YACD,MAAM,EAAE;gBACJ,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,MAAM;aACjB;YACD,UAAU,EAAE;gBACR,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,MAAM;aACjB;YACD,UAAU,EAAE;gBACR,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;aAClB;YACD,YAAY,EAAE;gBACV,GAAG,EAAE,uDAAuD;gBAC5D,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,MAAM;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,0DAA0D;gBAC/D,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,MAAM;aACjB;YACD,cAAc,EAAE;gBACZ,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,OAAO;aAClB;YACD,cAAc,EAAE;gBACZ,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,OAAO;aAClB;SACJ,CAAC;IACN,CAAC;IACD,eAAe,CACX,MAAsB,EACtB,UAAuC;QAEvC,MAAM,YAAY,GAAY,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAY,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC;QACxE,IAAI,YAAY,IAAI,kBAAkB,EAAE;YACpC,MAAM,IAAI,KAAK,CAAC,kDAAkD;kBAChE,8BAA8B,CAAC,CAAC;SACrC;aAAM,IACH,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;eACjC,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAC/C;YACE,MAAM,IAAI,KAAK,CAAC,+DAA+D;kBACzE,6DAA6D,CAAC,CAAC;SACxE;IACL,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|