terafoundation 2.0.0-dev.9 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/api/index.d.ts.map +1 -1
- package/dist/src/api/index.js.map +1 -1
- package/dist/src/connectors/kafka.d.ts +3 -0
- package/dist/src/connectors/kafka.d.ts.map +1 -0
- package/dist/src/connectors/kafka.js +3 -0
- package/dist/src/connectors/kafka.js.map +1 -0
- package/dist/src/connectors/s3.js +2 -2
- package/dist/src/connectors/s3.js.map +1 -1
- package/dist/src/schema.d.ts +2 -2
- package/dist/src/schema.d.ts.map +1 -1
- package/dist/src/schema.js.map +1 -1
- package/dist/src/validate-configs.d.ts.map +1 -1
- package/dist/src/validate-configs.js +7 -28
- package/dist/src/validate-configs.js.map +1 -1
- package/package.json +7 -9
- package/dist/src/connectors/hdfs.d.ts +0 -4
- package/dist/src/connectors/hdfs.d.ts.map +0 -1
- package/dist/src/connectors/hdfs.js +0 -35
- package/dist/src/connectors/hdfs.js.map +0 -1
- package/dist/src/connectors/hdfs_ha.d.ts +0 -4
- package/dist/src/connectors/hdfs_ha.d.ts.map +0 -1
- package/dist/src/connectors/hdfs_ha.js +0 -68
- package/dist/src/connectors/hdfs_ha.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/api/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/api/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAqB,MAAM,kBAAkB,CAAC;AAQ1E,MAAM,CAAC,OAAO,UAAU,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC,OAAO,GAAG,IAAI,CAoI1E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/api/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAEjE;;GAEG;AACH,MAAM,CAAC,OAAO,UAAU,YAAY,CAAC,OAA+B;IAChE,MAAM,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IAClC,OAAO,CAAC,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,mBAAmB;IACnB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,cAAc,GAAkC;QAClD,KAAK,CAAC,YAAY,CAAC,OAAO;YACtB,MAAM,EAAE,IAAI,EAAE,QAAQ,GAAG,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;YAEvD,oEAAoE;YACpE,wBAAwB;YACxB,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAC;YAElC,8DAA8D;YAC9D,MAAM,EAAE,UAAU,EAAE,GAAG,gBAAgB,CAAC;YAExC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACnE,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC;gBACzD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;gBAEvD,IAAI,YAAY,GAAG,EAAE,CAAC;gBAEtB,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;oBACnE,YAAY,GAAG,MAAM,CAAC,MAAM,CACxB,EAAE,EACF,gBAAgB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAC9C,CAAC;oBACF,mEAAmE;gBACvE,CAAC;qBAAM,IAAI,QAAQ,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,qCAAqC,QAAQ,EAAE,CAAC,CAAC;gBAC/E,CAAC;gBAED,MAAM,UAAU,GAAG,MAAM,cAAc,CACnC,IAAI,EACJ,YAAY,EACZ,OAAO,CAAC,MAAM,EACd,OAAO,CACV,CAAC;gBAEF,IAAI,MAAM,EAAE,CAAC;oBACT,WAAW,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAClC,CAAC;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,kEAAkE;QAClE,6EAA6E;QAC7E,oCAAoC;QACpC,KAAK,CAAC,iBAAiB,CAAC,OAAO;YAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QACD,UAAU,CAAC,GAAG,IAAW;YACrB,6DAA6D;YAC7D,sCAAsC;YAEtC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3D,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACT,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACf,iCAAiC;YACjC,WAAW,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAEzC,OAAO,WAAW,CAAC;QACvB,CAAC;QACD,eAAe;YACX,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,YAAY,CAAC,GAAW,EAAE,UAAkC;YACxD,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;YAC5B,sCAAsC;YACtC,kEAAkE;YAClE,qCAAqC;YACrC,MAAM,GAAG,GAAG;gBACR,UAAU,EAAE,QAAQ;gBACpB,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/api/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAE7D,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAEjE;;GAEG;AACH,MAAM,CAAC,OAAO,UAAU,YAAY,CAAC,OAA+B;IAChE,MAAM,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;IAClC,OAAO,CAAC,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,mBAAmB;IACnB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,cAAc,GAAkC;QAClD,KAAK,CAAC,YAAY,CAAC,OAAO;YACtB,MAAM,EAAE,IAAI,EAAE,QAAQ,GAAG,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;YAEvD,oEAAoE;YACpE,wBAAwB;YACxB,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAC;YAElC,8DAA8D;YAC9D,MAAM,EAAE,UAAU,EAAE,GAAG,gBAAgB,CAAC;YAExC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACnE,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YAED,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC;gBACzD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;gBAEvD,IAAI,YAAY,GAAG,EAAE,CAAC;gBAEtB,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;oBACnE,YAAY,GAAG,MAAM,CAAC,MAAM,CACxB,EAAE,EACF,gBAAgB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAC9C,CAAC;oBACF,mEAAmE;gBACvE,CAAC;qBAAM,IAAI,QAAQ,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,qCAAqC,QAAQ,EAAE,CAAC,CAAC;gBAC/E,CAAC;gBAED,MAAM,UAAU,GAAG,MAAM,cAAc,CACnC,IAAI,EACJ,YAAY,EACZ,OAAO,CAAC,MAAM,EACd,OAAO,CACV,CAAC;gBAEF,IAAI,MAAM,EAAE,CAAC;oBACT,WAAW,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAClC,CAAC;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,kEAAkE;QAClE,6EAA6E;QAC7E,oCAAoC;QACpC,KAAK,CAAC,iBAAiB,CAAC,OAAO;YAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QACD,UAAU,CAAC,GAAG,IAAW;YACrB,6DAA6D;YAC7D,sCAAsC;YAEtC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC3D,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACT,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACf,iCAAiC;YACjC,WAAW,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;YAEzC,OAAO,WAAW,CAAC;QACvB,CAAC;QACD,eAAe;YACX,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,YAAY,CAAC,GAAW,EAAE,UAAkC;YACxD,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;YAC5B,sCAAsC;YACtC,kEAAkE;YAClE,qCAAqC;YACrC,MAAM,GAAG,GAAG;gBACR,UAAU,EAAE,QAAQ;gBACpB,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;aAChC,CAAC;YAE9B,IAAI,UAAU,EAAE,CAAC;gBACb,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC/B,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,OAAO,GAAsC,EAAE,CAAC;YACtD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;gBACzD,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE;oBACZ,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAEjC,wDAAwD;oBACxD,+CAA+C;oBAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;oBAE3B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzB,CAAC,CAAC,CAAC;YACP,CAAC;YAED,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,WAAW,EAAE,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;KAC/C,CAAC;IACF,SAAS,uBAAuB;QAC5B,WAAW,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;OAKG;IACH,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ;QACvC,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,sBAAsB,CAAC,CAAC;QAC5F,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,mEAAmE;IACnE,OAAO,CAAC,IAAI,GAAG;QACX,WAAW;KACP,CAAC;IAET,uBAAuB,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/connectors/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,cAAc,MAAM,gCAAgC,CAAC;AAE5D,eAAe,cAAc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/connectors/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,cAAc,MAAM,gCAAgC,CAAC;AAE5D,eAAe,cAAc,CAAC"}
|
|
@@ -13,12 +13,12 @@ const connector = {
|
|
|
13
13
|
accessKeyId: {
|
|
14
14
|
doc: '',
|
|
15
15
|
default: null,
|
|
16
|
-
format:
|
|
16
|
+
format: 'optional_string'
|
|
17
17
|
},
|
|
18
18
|
secretAccessKey: {
|
|
19
19
|
doc: '',
|
|
20
20
|
default: null,
|
|
21
|
-
format:
|
|
21
|
+
format: 'optional_string'
|
|
22
22
|
},
|
|
23
23
|
region: {
|
|
24
24
|
doc: '',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../../../src/connectors/s3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAkB,MAAM,4BAA4B,CAAC;AAG5E,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAA4B,EAAE,MAAc;QAC3D,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC9B,CAAC;IACD,aAAa;QACT,OAAO;YACH,QAAQ,EAAE;gBACN,GAAG,EAAE,oBAAoB;gBACzB,OAAO,EAAE,cAAc;aAC1B;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../../../src/connectors/s3.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAkB,MAAM,4BAA4B,CAAC;AAG5E,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAA4B,EAAE,MAAc;QAC3D,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC9B,CAAC;IACD,aAAa;QACT,OAAO;YACH,QAAQ,EAAE;gBACN,GAAG,EAAE,oBAAoB;gBACzB,OAAO,EAAE,cAAc;aAC1B;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,iBAAiB;aAC5B;YACD,eAAe,EAAE;gBACb,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,iBAAiB;aAC5B;YACD,MAAM,EAAE;gBACJ,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,MAAM;aACjB;YACD,UAAU,EAAE;gBACR,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,MAAM;aACjB;YACD,UAAU,EAAE;gBACR,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;aAClB;YACD,YAAY,EAAE;gBACV,GAAG,EAAE,uDAAuD;gBAC5D,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,MAAM;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,0DAA0D;gBAC/D,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,MAAM;aACjB;YACD,cAAc,EAAE;gBACZ,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,OAAO;aAClB;YACD,cAAc,EAAE;gBACZ,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,OAAO;aAClB;SACJ,CAAC;IACN,CAAC;IACD,eAAe,CACX,MAAsB,EACtB,UAAuC;QAEvC,MAAM,YAAY,GAAY,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAY,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC;QACxE,IAAI,YAAY,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,kDAAkD;kBAC5D,8BAA8B,CAAC,CAAC;QAC1C,CAAC;aAAM,IACH,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;eACjC,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAC/C,CAAC;YACC,MAAM,IAAI,KAAK,CAAC,+DAA+D;kBACzE,6DAA6D,CAAC,CAAC;QACzE,CAAC;IACL,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
package/dist/src/schema.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import
|
|
2
|
-
export declare function foundationSchema():
|
|
1
|
+
import { Terafoundation } from '@terascope/types';
|
|
2
|
+
export declare function foundationSchema(): Terafoundation.Schema<Record<string, any>>;
|
|
3
3
|
//# sourceMappingURL=schema.d.ts.map
|
package/dist/src/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/schema.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIlD,wBAAgB,gBAAgB,+CAoF/B"}
|
package/dist/src/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/schema.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG/B,MAAM,WAAW,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC;AAElC,MAAM,UAAU,gBAAgB;IAC5B,MAAM,MAAM,GAA+C;QACvD,QAAQ,EAAE;YACN,GAAG,EAAE,qEAAqE;YAC1E,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE;YACtB,MAAM,EAAE,MAAM;SACjB;QACD,OAAO,EAAE;YACL,GAAG,EAAE,2DAA2D;YAChE,OAAO,EAAE,CAAC,SAAS,CAAC;YACpB,MAAM,CAAC,MAAe;gBAClB,MAAM,MAAM,GAA4B,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;gBAChF,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;wBAChB,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,2CAA2C,CAAC,CAAC;oBAC/E,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;SACJ;QACD,SAAS,EAAE;YACP,GAAG,EAAE,wBAAwB;YAC7B,OAAO,EAAE,MAAM;YACf,MAAM,CAAC,GAAY;gBACf,MAAM,KAAK,GAA4B;oBACnC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI;iBAC7E,CAAC;gBACF,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;wBACd,MAAM,IAAI,KAAK,CAAC,0EAA0E,GAAG,EAAE,CAAC,CAAC;oBACrG,CAAC;gBACL,CAAC;qBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,0EAA0E;oBAC1E,MAAM,OAAO,GAA4B,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;oBACvE,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;wBAC5C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;4BAC9B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gCAChB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACpB,CAAC;4BACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gCACpB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACpB,CAAC;wBACL,CAAC,CAAC,CAAC;wBACH,OAAO,IAAI,CAAC;oBAChB,CAAC,EAAE,EAAE,CAAC,CAAC;oBAEP,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC3B,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;oBAChG,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,MAAM,IAAI,KAAK,CAAC,0GAA0G,CAAC,CAAC;gBAChI,CAAC;YACL,CAAC;SACJ;QACD,OAAO,EAAE;YACL,GAAG,EAAE,8BAA8B;YACnC,OAAO,EAAE,WAAW;SACvB;QACD,oBAAoB,EAAE;YAClB,GAAG,EAAE,6BAA6B;YAClC,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,OAAO;SAClB;QACD,iBAAiB,EAAE;YACf,GAAG,EAAE,oCAAoC;YACzC,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,MAAM;SACjB;QACD,wBAAwB,EAAE;YACtB,GAAG,EAAE,+CAA+C;YACpD,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,OAAO;SAClB;QACD,wBAAwB,EAAE;YACtB,GAAG,EAAE,sDAAsD;YAC3D,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,MAAM;SACjB;KACJ,CAAC;IAEF,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-configs.d.ts","sourceRoot":"","sources":["../../src/validate-configs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"validate-configs.d.ts","sourceRoot":"","sources":["../../src/validate-configs.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAuDpE;;;;EAIE;AACF,wBAA8B,eAAe,CACzC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvB,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvB,CAAC,SAAS,MAAM,GAAG,MAAM,EAEzB,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EACtC,SAAS,EAAE,WAAW,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GACpD,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAqGtC"}
|
|
@@ -1,28 +1,12 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
|
-
import
|
|
3
|
-
import { TSError, isFunction, isPlainObject, isEmpty, concat, pMap, cloneDeep } from '@terascope/core-utils';
|
|
4
|
-
// @ts-expect-error no types
|
|
5
|
-
import convict_format_with_validator from 'convict-format-with-validator';
|
|
6
|
-
// @ts-expect-error no types
|
|
7
|
-
import convict_format_with_moment from 'convict-format-with-moment';
|
|
2
|
+
import { TSError, isFunction, isPlainObject, isEmpty, concat, pMap, cloneDeep, SchemaValidator } from '@terascope/core-utils';
|
|
8
3
|
import { getConnectorSchemaAndValFn } from './connector-utils.js';
|
|
9
4
|
import { foundationSchema } from './schema.js';
|
|
10
|
-
|
|
11
|
-
convict.addFormats(convict_format_with_moment);
|
|
12
|
-
function validateConfig(cluster, schema, namespaceConfig) {
|
|
5
|
+
function validateConfig(cluster, schema, namespaceConfig, schemaKey, extraFormats) {
|
|
13
6
|
try {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
config.validate({
|
|
18
|
-
// IMPORTANT: changing this will break things
|
|
19
|
-
// FIXME
|
|
20
|
-
// false is deprecated and will be removed in ^5.0.0
|
|
21
|
-
// must be warn or strict
|
|
22
|
-
allowed: true,
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
return config.getProperties();
|
|
7
|
+
const checkKeys = cluster.isMaster ? 'allow' : 'warn';
|
|
8
|
+
const validator = new SchemaValidator(schema, schemaKey, extraFormats, checkKeys);
|
|
9
|
+
return validator.validate(namespaceConfig);
|
|
26
10
|
}
|
|
27
11
|
catch (err) {
|
|
28
12
|
throw new TSError(err, { reason: 'Error validating configuration' });
|
|
@@ -69,16 +53,11 @@ export default async function validateConfigs(cluster, config, sysconfig) {
|
|
|
69
53
|
const schema = await extractSchema(config.config_schema, sysconfig);
|
|
70
54
|
schema.terafoundation = foundationSchema();
|
|
71
55
|
const result = {};
|
|
72
|
-
if (config.schema_formats) {
|
|
73
|
-
config.schema_formats.forEach((format) => {
|
|
74
|
-
convict.addFormat(format);
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
56
|
const schemaKeys = concat(Object.keys(schema), Object.keys(sysconfig));
|
|
78
57
|
for (const schemaKey of schemaKeys) {
|
|
79
58
|
const subSchema = schema[schemaKey] || {};
|
|
80
59
|
const subConfig = sysconfig[schemaKey] || {};
|
|
81
|
-
const validatedConfig = validateConfig(cluster, subSchema, subConfig);
|
|
60
|
+
const validatedConfig = validateConfig(cluster, subSchema, subConfig, schemaKey, config.schema_formats);
|
|
82
61
|
result[schemaKey] = validatedConfig;
|
|
83
62
|
if (schemaKey === 'terafoundation') {
|
|
84
63
|
result[schemaKey].connectors = {};
|
|
@@ -88,7 +67,7 @@ export default async function validateConfigs(cluster, config, sysconfig) {
|
|
|
88
67
|
const { schema: connectorSchema, validatorFn: connValidatorFn } = await getConnectorSchemaAndValFn(connector);
|
|
89
68
|
result[schemaKey].connectors[connector] = {};
|
|
90
69
|
for (const [connection, connectionConfig] of Object.entries(connectorConfig)) {
|
|
91
|
-
const validatedConnConfig = validateConfig(cluster, connectorSchema, connectionConfig);
|
|
70
|
+
const validatedConnConfig = validateConfig(cluster, connectorSchema, connectionConfig, `${connector}.${connection}`, config.schema_formats);
|
|
92
71
|
result[schemaKey].connectors[connector][connection] = validatedConnConfig;
|
|
93
72
|
listOfValidations[`${connector}:${connection}`] = {
|
|
94
73
|
validatorFn: connValidatorFn,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-configs.js","sourceRoot":"","sources":["../../src/validate-configs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,
|
|
1
|
+
{"version":3,"file":"validate-configs.js","sourceRoot":"","sources":["../../src/validate-configs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EACH,OAAO,EAAE,UAAU,EAAE,aAAa,EAClC,OAAO,EAAE,MAAM,EAAE,IAAI,EACrB,SAAS,EAAE,eAAe,EAC7B,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,SAAS,cAAc,CACnB,OAA8B,EAC9B,MAAkC,EAClC,eAAoB,EACpB,SAAiB,EACjB,YAAsC;IAEtC,IAAI,CAAC;QACD,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QACtD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAM,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QACvF,OAAO,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,gCAAgC,EAAE,CAAC,CAAC;IACzE,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CACxB,EAAO,EACP,SAAmD;IAEnD,IAAI,UAAU,CAAC,EAAE,CAAC,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;QACnC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,MAAM,CAAC,MAAM,CAAC;QACzB,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,IAAI,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,EAAE,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACvB,EAAO,EACP,SAAmD;IAEnD,IAAI,UAAU,CAAC,EAAE,CAAC,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,MAAM,CAAC,WAAW,CAAC;QAC9B,CAAC;IACL,CAAC;IACD,IAAI,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;EAIE;AACF,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,eAAe,CAKzC,OAA+B,EAC/B,MAAsC,EACtC,SAAmD;IAEnD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IACjF,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,iBAAiB,GAAoD,EAAE,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAEpE,MAAM,CAAC,cAAc,GAAG,gBAAgB,EAAE,CAAC;IAE3C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEvE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,SAAS,GACT,SAAS,CAAC,SAA8C,CAAC,IAAI,EAAE,CAAC;QAEtE,MAAM,eAAe,GAAG,cAAc,CAClC,OAAO,EACP,SAAS,EACT,SAAS,EACT,SAAS,EACT,MAAM,CAAC,cAAc,CACxB,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC;QAEpC,IAAI,SAAS,KAAK,gBAAgB,EAAE,CAAC;YACjC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,GAAG,EAAE,CAAC;YAElC,MAAM,UAAU,GAAwB,SAAS,CAAC,UAAU,IAAI,EAAE,CAAC;YACnE,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEjD,MAAM,IAAI,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,EAAE;gBAC7D,MAAM,EACF,MAAM,EAAE,eAAe,EACvB,WAAW,EAAE,eAAe,EAC/B,GAAG,MAAM,0BAA0B,CAAC,SAAS,CAAC,CAAC;gBAEhD,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;gBAE7C,KAAK,MAAM,CAAC,UAAU,EAAE,gBAAgB,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;oBAC3E,MAAM,mBAAmB,GAAG,cAAc,CACtC,OAAO,EACP,eAAe,EACf,gBAAuB,EACvB,GAAG,SAAS,IAAI,UAAU,EAAE,EAC5B,MAAM,CAAC,cAAc,CACxB,CAAC;oBAEF,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,GAAG,mBAAmB,CAAC;oBAE1E,iBAAiB,CAAC,GAAG,SAAS,IAAI,UAAU,EAAE,CAAC,GAAG;wBAC9C,WAAW,EAAE,eAAe;wBAC5B,MAAM,EAAE,mBAAmB;wBAC3B,SAAS,EAAE,IAAI;qBAClB,CAAC;gBACN,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,iBAAiB,CAAC,SAAS,CAAC,GAAG;gBAC3B,MAAM,EAAE,eAAe;aAC1B,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,iBAAiB,CAAC,SAAS,CAAC,GAAG;gBAC3B,WAAW,EAAE,kBAAkB,CAAI,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC;gBACnE,MAAM,EAAE,eAAe;aAC1B,CAAC;QACN,CAAC;IACL,CAAC;IAED,iEAAiE;IACjE,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE/B,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IAC1C,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,CAAC,SAAS,GAAG,GAAG,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;IAC1D,CAAC;SAAM,CAAC;QACJ,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC;IAChC,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,GAAG,KAAK,CAAC;QAEnC,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,OAAO,CAAC,qCAAqC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;YAC1H,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "terafoundation",
|
|
3
3
|
"displayName": "Terafoundation",
|
|
4
|
-
"version": "2.0.0
|
|
4
|
+
"version": "2.0.0",
|
|
5
5
|
"description": "A Clustering and Foundation tool for Terascope Tools",
|
|
6
6
|
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
|
|
7
7
|
"bugs": {
|
|
@@ -28,20 +28,17 @@
|
|
|
28
28
|
"test:watch": "node ../scripts/bin/ts-scripts test --watch ../terafoundation --"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@terascope/core-utils": "~2.0.0
|
|
32
|
-
"@terascope/file-asset-apis": "~
|
|
33
|
-
"@terascope/opensearch-client": "~2.0.0
|
|
34
|
-
"@terascope/types": "~2.0.0
|
|
31
|
+
"@terascope/core-utils": "~2.0.0",
|
|
32
|
+
"@terascope/file-asset-apis": "~2.0.0",
|
|
33
|
+
"@terascope/opensearch-client": "~2.0.0",
|
|
34
|
+
"@terascope/types": "~2.0.0",
|
|
35
35
|
"bunyan": "~1.8.15",
|
|
36
|
-
"convict": "~6.2.4",
|
|
37
|
-
"convict-format-with-moment": "~6.2.0",
|
|
38
|
-
"convict-format-with-validator": "~6.2.0",
|
|
39
36
|
"elasticsearch": "~15.4.1",
|
|
40
37
|
"express": "~5.1.0",
|
|
41
38
|
"js-yaml": "~4.1.0",
|
|
42
39
|
"nanoid": "~5.1.5",
|
|
43
|
-
"node-webhdfs": "~1.0.2",
|
|
44
40
|
"prom-client": "~15.1.3",
|
|
41
|
+
"terafoundation_kafka_connector": "~2.0.0",
|
|
45
42
|
"yargs": "~18.0.0"
|
|
46
43
|
},
|
|
47
44
|
"devDependencies": {
|
|
@@ -49,6 +46,7 @@
|
|
|
49
46
|
"@types/elasticsearch": "~5.0.43",
|
|
50
47
|
"@types/express": "~5.0.3",
|
|
51
48
|
"@types/js-yaml": "~4.0.9",
|
|
49
|
+
"@types/yargs": "~17.0.33",
|
|
52
50
|
"got": "~14.4.8"
|
|
53
51
|
},
|
|
54
52
|
"engines": {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs.d.ts","sourceRoot":"","sources":["../../../src/connectors/hdfs.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAIvD,QAAA,MAAM,SAAS,EAAE,cAAc,CAAC,SAgC/B,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
// @ts-expect-error
|
|
2
|
-
import { WebHDFSClient } from 'node-webhdfs';
|
|
3
|
-
const connector = {
|
|
4
|
-
async createClient(customConfig, logger) {
|
|
5
|
-
logger.info(`Using hdfs hosts: ${customConfig.host}`);
|
|
6
|
-
// TODO: there's no error handling here at all???
|
|
7
|
-
const client = new WebHDFSClient(customConfig);
|
|
8
|
-
return {
|
|
9
|
-
client,
|
|
10
|
-
logger
|
|
11
|
-
};
|
|
12
|
-
},
|
|
13
|
-
config_schema() {
|
|
14
|
-
return {
|
|
15
|
-
user: {
|
|
16
|
-
doc: '',
|
|
17
|
-
default: 'webuser'
|
|
18
|
-
},
|
|
19
|
-
namenode_port: {
|
|
20
|
-
doc: '',
|
|
21
|
-
default: 50070
|
|
22
|
-
},
|
|
23
|
-
namenode_host: {
|
|
24
|
-
doc: '',
|
|
25
|
-
default: 'localhost'
|
|
26
|
-
},
|
|
27
|
-
path_prefix: {
|
|
28
|
-
doc: '',
|
|
29
|
-
default: '/webhdfs/v1'
|
|
30
|
-
}
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
export default connector;
|
|
35
|
-
//# sourceMappingURL=hdfs.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs.js","sourceRoot":"","sources":["../../../src/connectors/hdfs.ts"],"names":[],"mappings":"AAEA,mBAAmB;AACnB,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAAiC,EAAE,MAAc;QAChE,MAAM,CAAC,IAAI,CAAC,qBAAqB,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QAEtD,iDAAiD;QACjD,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,YAAY,CAAC,CAAC;QAE/C,OAAO;YACH,MAAM;YACN,MAAM;SACT,CAAC;IACN,CAAC;IACD,aAAa;QACT,OAAO;YACH,IAAI,EAAE;gBACF,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,SAAS;aACrB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,KAAK;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,WAAW;aACvB;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,EAAE;gBACP,OAAO,EAAE,aAAa;aACzB;SACJ,CAAC;IACN,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs_ha.d.ts","sourceRoot":"","sources":["../../../src/connectors/hdfs_ha.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAa,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAclE,QAAA,MAAM,SAAS,EAAE,cAAc,CAAC,SAwD/B,CAAC;AAEF,eAAe,SAAS,CAAC"}
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
import util from 'node:util';
|
|
2
|
-
// @ts-expect-error
|
|
3
|
-
import { WebHDFSClient } from 'node-webhdfs';
|
|
4
|
-
function promisifyAll(obj) {
|
|
5
|
-
const promisified = {};
|
|
6
|
-
for (const key of Object.keys(obj)) {
|
|
7
|
-
if (typeof obj[key] === 'function') {
|
|
8
|
-
promisified[key] = util.promisify(obj[key]);
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
return promisified;
|
|
12
|
-
}
|
|
13
|
-
const connector = {
|
|
14
|
-
async createClient(customConfig, logger) {
|
|
15
|
-
let highAvailibility = false;
|
|
16
|
-
let currentNameNode;
|
|
17
|
-
if (Array.isArray(customConfig.namenode_host)) {
|
|
18
|
-
([currentNameNode] = customConfig.namenode_host);
|
|
19
|
-
customConfig.namenode_list = customConfig.namenode_host;
|
|
20
|
-
highAvailibility = true;
|
|
21
|
-
}
|
|
22
|
-
else {
|
|
23
|
-
currentNameNode = customConfig.namenode_host;
|
|
24
|
-
}
|
|
25
|
-
const config = Object.assign({}, customConfig, { namenode_host: currentNameNode });
|
|
26
|
-
const client = new WebHDFSClient(config);
|
|
27
|
-
logger.info(`Using hdfs hosts: ${currentNameNode}, high-availability: ${highAvailibility}`);
|
|
28
|
-
return {
|
|
29
|
-
client: promisifyAll(client),
|
|
30
|
-
logger
|
|
31
|
-
};
|
|
32
|
-
},
|
|
33
|
-
config_schema() {
|
|
34
|
-
return {
|
|
35
|
-
user: {
|
|
36
|
-
doc: 'user type for hdfs requests',
|
|
37
|
-
default: 'hdfs',
|
|
38
|
-
format: String
|
|
39
|
-
},
|
|
40
|
-
namenode_port: {
|
|
41
|
-
doc: 'port of hdfs',
|
|
42
|
-
default: 50070
|
|
43
|
-
},
|
|
44
|
-
namenode_host: {
|
|
45
|
-
doc: 'a single host, or multiple hosts listed in an array',
|
|
46
|
-
default: null,
|
|
47
|
-
format(val) {
|
|
48
|
-
if (typeof val === 'string') {
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
if (Array.isArray(val)) {
|
|
52
|
-
if (val.length < 2) {
|
|
53
|
-
throw new Error('namenode_host must have at least two namenodes listed in the array');
|
|
54
|
-
}
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
throw new Error('namenode_host configuration must be set to an array for high availability or a string');
|
|
58
|
-
}
|
|
59
|
-
},
|
|
60
|
-
path_prefix: {
|
|
61
|
-
doc: 'endpoint for hdfs web interface',
|
|
62
|
-
default: '/webhdfs/v1'
|
|
63
|
-
}
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
};
|
|
67
|
-
export default connector;
|
|
68
|
-
//# sourceMappingURL=hdfs_ha.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hdfs_ha.js","sourceRoot":"","sources":["../../../src/connectors/hdfs_ha.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,mBAAmB;AACnB,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,SAAS,YAAY,CAAC,GAAc;IAChC,MAAM,WAAW,GAAc,EAAE,CAAC;IAClC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,UAAU,EAAE,CAAC;YACjC,WAAW,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;IACL,CAAC;IACD,OAAO,WAAW,CAAC;AACvB,CAAC;AAED,MAAM,SAAS,GAA6B;IACxC,KAAK,CAAC,YAAY,CAAC,YAAiC,EAAE,MAAc;QAChE,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,eAAe,CAAC;QAEpB,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,CAAC;YAC5C,CAAC,CAAC,eAAe,CAAC,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;YACjD,YAAY,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAC;YACxD,gBAAgB,GAAG,IAAI,CAAC;QAC5B,CAAC;aAAM,CAAC;YACJ,eAAe,GAAG,YAAY,CAAC,aAAa,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC,CAAC;QACnF,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,CAAC,IAAI,CAAC,qBAAqB,eAAe,wBAAwB,gBAAgB,EAAE,CAAC,CAAC;QAE5F,OAAO;YACH,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC;YAC5B,MAAM;SACT,CAAC;IACN,CAAC;IACD,aAAa;QACT,OAAO;YACH,IAAI,EAAE;gBACF,GAAG,EAAE,6BAA6B;gBAClC,OAAO,EAAE,MAAM;gBACf,MAAM,EAAE,MAAM;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,KAAK;aACjB;YACD,aAAa,EAAE;gBACX,GAAG,EAAE,qDAAqD;gBAC1D,OAAO,EAAE,IAAI;gBACb,MAAM,CAAC,GAAQ;oBACX,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;wBAC1B,OAAO;oBACX,CAAC;oBACD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACrB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACjB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;wBAC1F,CAAC;wBACD,OAAO;oBACX,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,uFAAuF,CAAC,CAAC;gBAC7G,CAAC;aACJ;YACD,WAAW,EAAE;gBACT,GAAG,EAAE,iCAAiC;gBACtC,OAAO,EAAE,aAAa;aACzB;SACJ,CAAC;IACN,CAAC;CACJ,CAAC;AAEF,eAAe,SAAS,CAAC"}
|