teraslice 2.10.0 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/dist/src/interfaces.js +12 -0
  2. package/dist/src/lib/cluster/cluster_master.js +246 -0
  3. package/dist/src/lib/cluster/node_master.js +355 -0
  4. package/dist/src/lib/cluster/services/api.js +663 -0
  5. package/dist/src/lib/cluster/services/assets.js +226 -0
  6. package/dist/src/lib/cluster/services/cluster/backends/kubernetes/index.js +192 -0
  7. package/dist/src/lib/cluster/services/cluster/backends/kubernetes/k8s.js +481 -0
  8. package/dist/src/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js +414 -0
  9. package/dist/src/lib/cluster/services/cluster/backends/kubernetes/k8sState.js +59 -0
  10. package/dist/src/lib/cluster/services/cluster/backends/kubernetes/utils.js +43 -0
  11. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/index.js +192 -0
  12. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js +2 -0
  13. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8s.js +423 -0
  14. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sDeploymentResource.js +60 -0
  15. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sJobResource.js +55 -0
  16. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sResource.js +359 -0
  17. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sServiceResource.js +37 -0
  18. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js +60 -0
  19. package/dist/src/lib/cluster/services/cluster/backends/kubernetesV2/utils.js +170 -0
  20. package/dist/src/lib/cluster/services/cluster/backends/native/dispatch.js +13 -0
  21. package/dist/src/lib/cluster/services/cluster/backends/native/index.js +526 -0
  22. package/dist/src/lib/cluster/services/cluster/backends/native/messaging.js +547 -0
  23. package/dist/src/lib/cluster/services/cluster/backends/state-utils.js +26 -0
  24. package/dist/src/lib/cluster/services/cluster/index.js +17 -0
  25. package/dist/src/lib/cluster/services/execution.js +435 -0
  26. package/dist/src/lib/cluster/services/index.js +6 -0
  27. package/dist/src/lib/cluster/services/interfaces.js +2 -0
  28. package/dist/src/lib/cluster/services/jobs.js +454 -0
  29. package/dist/src/lib/config/default-sysconfig.js +26 -0
  30. package/dist/src/lib/config/index.js +22 -0
  31. package/dist/src/lib/config/schemas/system.js +360 -0
  32. package/dist/src/lib/storage/analytics.js +86 -0
  33. package/dist/src/lib/storage/assets.js +401 -0
  34. package/dist/src/lib/storage/backends/elasticsearch_store.js +494 -0
  35. package/dist/src/lib/storage/backends/mappings/analytics.js +50 -0
  36. package/dist/src/lib/storage/backends/mappings/asset.js +41 -0
  37. package/dist/src/lib/storage/backends/mappings/ex.js +62 -0
  38. package/dist/src/lib/storage/backends/mappings/job.js +38 -0
  39. package/dist/src/lib/storage/backends/mappings/state.js +38 -0
  40. package/dist/src/lib/storage/backends/s3_store.js +237 -0
  41. package/dist/src/lib/storage/execution.js +300 -0
  42. package/dist/src/lib/storage/index.js +7 -0
  43. package/dist/src/lib/storage/jobs.js +81 -0
  44. package/dist/src/lib/storage/state.js +255 -0
  45. package/dist/src/lib/utils/api_utils.js +157 -0
  46. package/dist/src/lib/utils/asset_utils.js +94 -0
  47. package/dist/src/lib/utils/date_utils.js +52 -0
  48. package/dist/src/lib/utils/encoding_utils.js +27 -0
  49. package/dist/src/lib/utils/events.js +4 -0
  50. package/dist/src/lib/utils/file_utils.js +124 -0
  51. package/dist/src/lib/utils/id_utils.js +15 -0
  52. package/dist/src/lib/utils/port_utils.js +32 -0
  53. package/dist/src/lib/workers/assets/index.js +3 -0
  54. package/dist/src/lib/workers/assets/loader-executable.js +40 -0
  55. package/dist/src/lib/workers/assets/loader.js +73 -0
  56. package/dist/src/lib/workers/assets/spawn.js +55 -0
  57. package/dist/src/lib/workers/context/execution-context.js +12 -0
  58. package/dist/src/lib/workers/context/terafoundation-context.js +8 -0
  59. package/dist/src/lib/workers/execution-controller/execution-analytics.js +188 -0
  60. package/dist/src/lib/workers/execution-controller/index.js +1024 -0
  61. package/dist/src/lib/workers/execution-controller/recovery.js +151 -0
  62. package/dist/src/lib/workers/execution-controller/scheduler.js +390 -0
  63. package/dist/src/lib/workers/execution-controller/slice-analytics.js +96 -0
  64. package/dist/src/lib/workers/helpers/job.js +80 -0
  65. package/dist/src/lib/workers/helpers/op-analytics.js +22 -0
  66. package/dist/src/lib/workers/helpers/terafoundation.js +34 -0
  67. package/dist/src/lib/workers/helpers/worker-shutdown.js +169 -0
  68. package/dist/src/lib/workers/metrics/index.js +108 -0
  69. package/dist/src/lib/workers/worker/index.js +378 -0
  70. package/dist/src/lib/workers/worker/slice.js +122 -0
  71. package/dist/test/config/schemas/system_schema-spec.js +37 -0
  72. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/k8s-spec.js +316 -0
  73. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js +795 -0
  74. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-multicluster-spec.js +67 -0
  75. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-spec.js +84 -0
  76. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/utils-spec.js +132 -0
  77. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8s-v2-spec.js +455 -0
  78. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sResource-v2-spec.js +818 -0
  79. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.js +67 -0
  80. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.js +84 -0
  81. package/dist/test/lib/cluster/services/cluster/backends/kubernetes/v2/utils-v2-spec.js +320 -0
  82. package/dist/test/lib/cluster/services/cluster/backends/state-utils-spec.js +37 -0
  83. package/dist/test/node_master-spec.js +188 -0
  84. package/dist/test/services/api-spec.js +80 -0
  85. package/dist/test/services/assets-spec.js +158 -0
  86. package/dist/test/services/messaging-spec.js +440 -0
  87. package/dist/test/storage/assets_storage-spec.js +95 -0
  88. package/dist/test/storage/s3_store-spec.js +138 -0
  89. package/dist/test/test.config.js +8 -0
  90. package/dist/test/test.setup.js +6 -0
  91. package/dist/test/utils/api_utils-spec.js +86 -0
  92. package/dist/test/utils/asset_utils-spec.js +141 -0
  93. package/dist/test/utils/elastic_utils-spec.js +25 -0
  94. package/dist/test/workers/execution-controller/execution-controller-spec.js +371 -0
  95. package/dist/test/workers/execution-controller/execution-special-test-cases-spec.js +520 -0
  96. package/dist/test/workers/execution-controller/execution-test-cases-spec.js +338 -0
  97. package/dist/test/workers/execution-controller/recovery-spec.js +160 -0
  98. package/dist/test/workers/execution-controller/scheduler-spec.js +249 -0
  99. package/dist/test/workers/execution-controller/slice-analytics-spec.js +121 -0
  100. package/dist/test/workers/fixtures/ops/example-op/processor.js +20 -0
  101. package/dist/test/workers/fixtures/ops/example-op/schema.js +19 -0
  102. package/dist/test/workers/fixtures/ops/example-reader/fetcher.js +20 -0
  103. package/dist/test/workers/fixtures/ops/example-reader/schema.js +41 -0
  104. package/dist/test/workers/fixtures/ops/example-reader/slicer.js +37 -0
  105. package/dist/test/workers/fixtures/ops/new-op/processor.js +29 -0
  106. package/dist/test/workers/fixtures/ops/new-op/schema.js +18 -0
  107. package/dist/test/workers/fixtures/ops/new-reader/fetcher.js +19 -0
  108. package/dist/test/workers/fixtures/ops/new-reader/schema.js +23 -0
  109. package/dist/test/workers/fixtures/ops/new-reader/slicer.js +13 -0
  110. package/dist/test/workers/helpers/configs.js +130 -0
  111. package/dist/test/workers/helpers/execution-controller-helper.js +49 -0
  112. package/dist/test/workers/helpers/index.js +5 -0
  113. package/dist/test/workers/helpers/test-context.js +210 -0
  114. package/dist/test/workers/helpers/zip-directory.js +25 -0
  115. package/dist/test/workers/worker/slice-spec.js +333 -0
  116. package/dist/test/workers/worker/worker-spec.js +356 -0
  117. package/package.json +94 -93
  118. package/service.js +0 -0
@@ -0,0 +1,440 @@
1
+ import { jest } from '@jest/globals';
2
+ import events from 'node:events';
3
+ import { debugLogger } from '@terascope/utils';
4
+ import { Messaging, routing } from '../../src/lib/cluster/services/cluster/backends/native/messaging.js';
5
+ describe('messaging module', () => {
6
+ const logger = debugLogger('messaging');
7
+ let connected = {};
8
+ const testExId = '7890';
9
+ let firstWorkerMsg;
10
+ let secondWorkerMsg;
11
+ let clusterFn = () => { };
12
+ function testMessaging(messaging, fn) {
13
+ return (...args) => messaging[fn](...args);
14
+ }
15
+ class MyCluster extends events.EventEmitter {
16
+ workers;
17
+ constructor() {
18
+ super();
19
+ this.workers = {
20
+ first: {
21
+ ex_id: testExId,
22
+ assignment: 'execution_controller',
23
+ connected: true,
24
+ send: (msg) => {
25
+ firstWorkerMsg = msg;
26
+ },
27
+ on: (key, fn) => {
28
+ clusterFn = fn;
29
+ },
30
+ removeListener: () => { }
31
+ },
32
+ second: {
33
+ ex_id: testExId,
34
+ assignment: 'worker',
35
+ connected: true,
36
+ send: (msg) => {
37
+ secondWorkerMsg = msg;
38
+ },
39
+ on: (key, fn) => {
40
+ clusterFn = fn;
41
+ },
42
+ removeListener: () => { }
43
+ },
44
+ third: {
45
+ ex_id: 'somethingElse',
46
+ assignment: 'worker',
47
+ connected: true,
48
+ send: () => {
49
+ },
50
+ on: (key, fn) => {
51
+ clusterFn = fn;
52
+ },
53
+ removeListener: () => { }
54
+ }
55
+ };
56
+ }
57
+ }
58
+ let emitMsg;
59
+ const io = {
60
+ emit: (msg, msgObj) => {
61
+ emitMsg = { message: msg, data: msgObj };
62
+ },
63
+ sockets: {
64
+ in: (address) => ({
65
+ emit: (msg, msgObj) => {
66
+ const socketMsg = { message: msg, data: msgObj, address };
67
+ logger.debug(socketMsg);
68
+ }
69
+ }),
70
+ connected
71
+ },
72
+ eio: {
73
+ clientsCount: 2
74
+ },
75
+ close() { }
76
+ };
77
+ function getContext(obj) {
78
+ const emitter = new events.EventEmitter();
79
+ emitter.setMaxListeners(40);
80
+ const systemEvents = new events.EventEmitter();
81
+ systemEvents.setMaxListeners(40);
82
+ const clusterEmitter = new MyCluster();
83
+ clusterEmitter.setMaxListeners(40);
84
+ const config = Object.assign(emitter, obj);
85
+ const cleanup = () => {
86
+ emitter.removeAllListeners();
87
+ systemEvents.removeAllListeners();
88
+ clusterEmitter.removeAllListeners();
89
+ };
90
+ return {
91
+ sysconfig: {
92
+ teraslice: {
93
+ master_hostname: '127.0.0.1',
94
+ port: 47898,
95
+ action_timeout: 5000,
96
+ network_latency_buffer: 10
97
+ }
98
+ },
99
+ apis: {
100
+ foundation: {
101
+ getSystemEvents: () => systemEvents
102
+ }
103
+ },
104
+ cluster: clusterEmitter,
105
+ __testingModule: config,
106
+ cleanup
107
+ };
108
+ }
109
+ beforeEach(() => {
110
+ firstWorkerMsg = null;
111
+ secondWorkerMsg = null;
112
+ connected = {};
113
+ io.sockets.connected = connected;
114
+ });
115
+ afterEach(() => {
116
+ process.removeAllListeners('message');
117
+ });
118
+ it('can make a hostname', () => {
119
+ const testContext = getContext({ env: { assignment: 'node_master' } });
120
+ const messaging = new Messaging(testContext, logger, io);
121
+ const test = testMessaging(messaging, '_makeHostName');
122
+ expect(test('127.0.0.1', 5647)).toEqual('http://127.0.0.1:5647');
123
+ expect(test('127.0.0.1:', 5647)).toEqual('http://127.0.0.1:5647');
124
+ expect(test('http://127.0.0.1', 5647)).toEqual('http://127.0.0.1:5647');
125
+ expect(test('https://127.0.0.1', 5647)).toEqual('https://127.0.0.1:5647');
126
+ expect(test('127.0.0.1', 5647, 'v1')).toEqual('http://127.0.0.1:5647/v1');
127
+ expect(messaging.getHostUrl()).toEqual('http://127.0.0.1:47898');
128
+ testContext.cleanup();
129
+ });
130
+ it('can detect itself and make configurations', () => {
131
+ function getConfig(type) {
132
+ const job = JSON.stringify({
133
+ slicer_hostname: '127.0.0.1',
134
+ slicer_port: 47898
135
+ });
136
+ const testContext = getContext({ env: { assignment: type, job } });
137
+ const testModule = new Messaging(testContext, logger, io);
138
+ testContext.cleanup();
139
+ // @ts-expect-error
140
+ return testModule._makeConfigurations();
141
+ }
142
+ expect(getConfig('node_master')).toEqual({
143
+ clients: { networkClient: true, ipcClient: false },
144
+ hostURL: 'http://127.0.0.1:47898',
145
+ assignment: 'node_master'
146
+ });
147
+ expect(getConfig('cluster_master')).toEqual({
148
+ clients: { networkClient: false, ipcClient: true },
149
+ assignment: 'cluster_master'
150
+ });
151
+ expect(getConfig('assets_service')).toEqual({
152
+ clients: { networkClient: true, ipcClient: true },
153
+ hostURL: 'http://127.0.0.1:47898',
154
+ assignment: 'assets_service'
155
+ });
156
+ });
157
+ it('can be called without errors', () => {
158
+ const testContext = getContext({ env: { assignment: 'node_master' } });
159
+ expect(() => new Messaging(testContext, logger)).not.toThrow();
160
+ testContext.cleanup();
161
+ });
162
+ it('has a router', () => {
163
+ const routingData = {
164
+ cluster_master: {
165
+ node_master: 'network'
166
+ },
167
+ node_master: {
168
+ cluster_process: 'ipc',
169
+ cluster_master: 'network'
170
+ },
171
+ assets_service: { cluster_master: 'ipc' }
172
+ };
173
+ expect(routing).toEqual(routingData);
174
+ });
175
+ it('can determie path for message', () => {
176
+ const testContext1 = getContext({ env: { assignment: 'node_master' } });
177
+ const messaging1 = new Messaging(testContext1, logger, io);
178
+ const test1 = testMessaging(messaging1, '_determinePathForMessage');
179
+ const testContext2 = getContext({ env: { assignment: 'cluster_master' } });
180
+ const messaging2 = new Messaging(testContext2, logger, io);
181
+ const test2 = testMessaging(messaging2, '_determinePathForMessage');
182
+ const testContext3 = getContext({ env: { assignment: 'assets_service' } });
183
+ const messaging3 = new Messaging(testContext3, logger, io);
184
+ const test3 = testMessaging(messaging3, '_determinePathForMessage');
185
+ const failureMessage1 = { to: 'theUnknown' };
186
+ expect(test1({ to: 'cluster_process' })).toEqual('ipc');
187
+ expect(test1({ to: 'cluster_master' })).toEqual('network');
188
+ expect(() => test1(failureMessage1)).toThrow();
189
+ expect(test2({ to: 'node_master' })).toEqual('network');
190
+ expect(test3({ to: 'cluster_master' })).toEqual('ipc');
191
+ testContext1.cleanup();
192
+ testContext2.cleanup();
193
+ testContext3.cleanup();
194
+ });
195
+ it('can get a list of rooms as the cluster_master', () => {
196
+ connected['some-socket-id'] = {
197
+ rooms: {
198
+ 'room-a': 'room-a',
199
+ 'room-b': 'room-b'
200
+ }
201
+ };
202
+ connected['another-socket-id'] = {
203
+ rooms: {
204
+ 'room-1': 'room-1'
205
+ }
206
+ };
207
+ const testContext = getContext({
208
+ env: {
209
+ assignment: 'cluster_master'
210
+ }
211
+ });
212
+ const messaging = new Messaging(testContext, logger, io);
213
+ const rooms = messaging.listRooms();
214
+ expect(rooms).toEqual(['room-a', 'room-b', 'room-1']);
215
+ testContext.cleanup();
216
+ });
217
+ it('can forward/broadcast messages', () => {
218
+ const testContext = getContext({ env: { assignment: 'node_master' } });
219
+ const messaging = new Messaging(testContext, logger, io);
220
+ // @ts-expect-error
221
+ messaging._forwardMessage({ to: 'cluster_master', message: 'someEvent' });
222
+ // should be a network emit msg
223
+ expect(emitMsg).toEqual({
224
+ message: 'someEvent',
225
+ data: {
226
+ to: 'cluster_master',
227
+ message: 'someEvent'
228
+ }
229
+ });
230
+ messaging.broadcast('someEvent', { some: 'data' });
231
+ expect(emitMsg).toEqual({
232
+ message: 'networkMessage',
233
+ data: {
234
+ some: 'data',
235
+ message: 'someEvent'
236
+ }
237
+ });
238
+ testContext.cleanup();
239
+ });
240
+ it('can work with messaging:response messages', () => {
241
+ const testContext = getContext({ env: { assignment: 'node_master' } });
242
+ const eventEmitter = testContext.apis.foundation.getSystemEvents();
243
+ const messaging = new Messaging(testContext, logger, io);
244
+ const msgId = 'someId';
245
+ const nodeMsg = { __source: 'node_master', __msgId: msgId, message: 'someMessage' };
246
+ let emittedData = null;
247
+ eventEmitter.once(msgId, (data) => {
248
+ emittedData = data;
249
+ });
250
+ // @ts-expect-error
251
+ messaging._handleResponse(nodeMsg);
252
+ expect(emittedData).toEqual(nodeMsg);
253
+ testContext.cleanup();
254
+ });
255
+ it('can getClientCounts', () => {
256
+ const testContext = getContext({ env: { assignment: 'node_master' } });
257
+ const messaging = new Messaging(testContext, logger, io);
258
+ expect(messaging.getClientCounts()).toEqual(2);
259
+ testContext.cleanup();
260
+ });
261
+ // This test is broken but it works.
262
+ // We may not need a test for this because this messaging service
263
+ // should be replaced with @terascope/teraslice-messaging.
264
+ // eslint-disable-next-line jest/no-disabled-tests
265
+ it.skip('can send transactional and non-transactional messages', async () => {
266
+ const testContext = getContext({ env: { assignment: 'cluster_master' } });
267
+ const eventEmitter = testContext.apis.foundation.getSystemEvents();
268
+ const messaging = new Messaging(testContext, logger, io);
269
+ const workerMsg = {
270
+ to: 'node_master',
271
+ node_id: 'someId',
272
+ message: 'cluster:execution:stop'
273
+ };
274
+ const transactionalMsg = Object.assign({}, workerMsg, { response: true });
275
+ const transactionalErrorMsg = Object.assign({}, workerMsg, {
276
+ response: true,
277
+ error: 'someError'
278
+ });
279
+ const transactionalTimeoutErrorMsg = Object.assign({}, workerMsg, {
280
+ response: true,
281
+ error: 'someError',
282
+ timeout: 30
283
+ });
284
+ function sendEvent(timer) {
285
+ return new Promise((resolve) => {
286
+ setTimeout(() => {
287
+ let msgId;
288
+ if (secondWorkerMsg) {
289
+ msgId = secondWorkerMsg.__msgId;
290
+ }
291
+ else {
292
+ msgId = '12345';
293
+ }
294
+ eventEmitter.emit(msgId, secondWorkerMsg);
295
+ resolve(true);
296
+ }, timer);
297
+ });
298
+ }
299
+ // this is technically a sync message, but a promise is returned for composability
300
+ // and to act similiar to its transactional counterpart
301
+ try {
302
+ const bool = await messaging.send(workerMsg);
303
+ expect(bool).toEqual(true);
304
+ expect(firstWorkerMsg).toEqual(workerMsg);
305
+ const [results] = await Promise.all([messaging.send(transactionalMsg), sendEvent(20)]);
306
+ expect(results).toEqual(secondWorkerMsg);
307
+ // testing error scenario
308
+ try {
309
+ await Promise.all([messaging.send(transactionalErrorMsg), sendEvent(20)]);
310
+ }
311
+ catch (err) {
312
+ expect(err).toEqual('Error: someError occurred on node: node_master');
313
+ try {
314
+ await Promise.all([
315
+ messaging.send(transactionalTimeoutErrorMsg),
316
+ sendEvent(100)
317
+ ]);
318
+ }
319
+ catch (error) {
320
+ const messageSent = secondWorkerMsg;
321
+ expect(error.message).toEqual(`timeout error while communicating with ${messageSent?.to}, msg: ${messageSent?.message}, data: ${JSON.stringify(messageSent)}`);
322
+ }
323
+ }
324
+ }
325
+ finally {
326
+ testContext.cleanup();
327
+ }
328
+ });
329
+ it('can register callbacks and attach them to socket/io', () => {
330
+ const testContext = getContext({ env: { assignment: 'node_master' } });
331
+ const messaging = new Messaging(testContext, logger, io);
332
+ let exitIsCalled = false;
333
+ const socketSettings = [];
334
+ messaging.register({
335
+ event: 'network:connect',
336
+ identifier: 'node_id',
337
+ callback: (msg, id, identifier) => {
338
+ socketSettings.push({ msg, id, identifier });
339
+ return true;
340
+ }
341
+ });
342
+ messaging.register({
343
+ event: 'cluster:node:get_port',
344
+ callback: () => 3452
345
+ });
346
+ messaging.register({
347
+ event: 'child:exit',
348
+ callback: () => {
349
+ exitIsCalled = true;
350
+ }
351
+ });
352
+ expect(() => messaging.register({
353
+ event: 'some:event',
354
+ callback: () => {
355
+ exitIsCalled = true;
356
+ }
357
+ })).toThrow();
358
+ const results = { ...messaging.functionMapping };
359
+ expect(results['cluster:node:get_port']).toBeDefined();
360
+ expect(typeof results['cluster:node:get_port']).toEqual('function');
361
+ expect(results['cluster:node:get_port']()).toEqual(3452);
362
+ // it internally maps 'network:connect' to connect
363
+ expect(results.connect).toBeDefined();
364
+ expect(typeof results.connect).toEqual('function');
365
+ expect(results.connect()).toEqual(true);
366
+ expect(results.connect.__socketIdentifier).toEqual('node_id');
367
+ // node master sets the event on the cluster obj
368
+ testContext.cluster.emit('exit');
369
+ expect(exitIsCalled).toEqual(true);
370
+ socketSettings.shift();
371
+ const socketList = {};
372
+ const joinList = {};
373
+ const socket = {
374
+ rooms: joinList,
375
+ on: (key, fn) => {
376
+ socketList[key] = fn;
377
+ },
378
+ join: (id) => {
379
+ joinList[id] = id;
380
+ },
381
+ removeListener: (key) => {
382
+ delete socketList[key];
383
+ },
384
+ removeListeners: (key) => {
385
+ delete socketList[key];
386
+ }
387
+ };
388
+ // @ts-expect-error
389
+ messaging._registerFns(socket);
390
+ socketList.connect({ node_id: 'someId' });
391
+ expect(socketSettings[0]).toEqual({
392
+ msg: { node_id: 'someId' },
393
+ id: 'someId',
394
+ identifier: 'node_id'
395
+ });
396
+ testContext.cleanup();
397
+ });
398
+ it('can fail if the registering an event for another assignment', () => {
399
+ const testContext = getContext({ env: { assignment: 'node_master' } });
400
+ const messaging = new Messaging(testContext, logger, io);
401
+ const socketSettings = [];
402
+ expect(() => {
403
+ messaging.register({
404
+ // this event is not available to node_master
405
+ event: 'node:online',
406
+ identifier: 'node_id',
407
+ callback: (msg, id, identifier) => {
408
+ socketSettings.push({ msg, id, identifier });
409
+ return true;
410
+ }
411
+ });
412
+ }).toThrow();
413
+ testContext.cleanup();
414
+ });
415
+ it('can listen and setup server', async () => {
416
+ const testContext1 = getContext({ env: { assignment: 'node_master' } });
417
+ const messaging1 = new Messaging(testContext1, logger);
418
+ const testContext2 = getContext({ env: { assignment: 'cluster_master' } });
419
+ const messaging2 = new Messaging(testContext2, logger);
420
+ expect(() => messaging1.listen()).not.toThrow();
421
+ expect(() => messaging2.listen({ port: 45645 })).not.toThrow();
422
+ await messaging1.shutdown();
423
+ await messaging2.shutdown();
424
+ testContext1.cleanup();
425
+ testContext2.cleanup();
426
+ });
427
+ it('sets up listeners', () => {
428
+ const testContext1 = getContext({ env: { assignment: 'node_master' } });
429
+ const messaging1 = new Messaging(testContext1, logger, io);
430
+ const spy = jest.fn();
431
+ messaging1.registerChildOnlineHook(spy);
432
+ testContext1.cluster.emit('online', { id: 'first' });
433
+ clusterFn({ to: 'cluster_master' });
434
+ clusterFn({ to: 'node_master' });
435
+ clusterFn({ to: 'worker' });
436
+ expect(spy).toHaveBeenCalled();
437
+ testContext1.cleanup();
438
+ });
439
+ });
440
+ //# sourceMappingURL=messaging-spec.js.map
@@ -0,0 +1,95 @@
1
+ import fs from 'node:fs';
2
+ import { TestContext } from '@terascope/job-components';
3
+ import { createClient } from 'elasticsearch-store';
4
+ import { createS3Client } from '@terascope/file-asset-apis';
5
+ import { AssetsStorage } from '../../src/lib/storage';
6
+ import { TEST_INDEX_PREFIX } from '../test.config';
7
+ describe('AssetsStorage using S3 backend', () => {
8
+ let storage;
9
+ const options = {
10
+ assignment: 'assets_service',
11
+ clients: [
12
+ {
13
+ type: 'elasticsearch-next',
14
+ async createClient(customConfig, logger) {
15
+ const { client } = await createClient(customConfig, logger);
16
+ return { client, logger };
17
+ },
18
+ endpoint: 'default'
19
+ },
20
+ {
21
+ type: 's3',
22
+ async createClient(customConfig, logger) {
23
+ const client = await createS3Client(customConfig, logger);
24
+ return { client, logger };
25
+ },
26
+ endpoint: 'default'
27
+ }
28
+ ]
29
+ };
30
+ const context = new TestContext(`${TEST_INDEX_PREFIX}assets-storage-test`, options);
31
+ context.sysconfig.terafoundation = {
32
+ connectors: {
33
+ 'elasticsearch-next': {
34
+ default: {
35
+ node: [process.env.SEARCH_TEST_HOST]
36
+ }
37
+ },
38
+ s3: {
39
+ default: {
40
+ endpoint: process.env.MINIO_HOST,
41
+ accessKeyId: process.env.MINIO_ACCESS_KEY,
42
+ secretAccessKey: process.env.MINIO_SECRET_KEY,
43
+ forcePathStyle: true,
44
+ sslEnabled: false,
45
+ region: 'test-region'
46
+ }
47
+ }
48
+ }
49
+ };
50
+ context.sysconfig.teraslice.asset_storage_connection_type = 's3';
51
+ context.sysconfig.teraslice.asset_storage_connection = 'default';
52
+ context.sysconfig.teraslice.api_response_timeout = 30000;
53
+ beforeAll(async () => {
54
+ storage = new AssetsStorage(context);
55
+ await storage.initialize();
56
+ }, 30000);
57
+ it('will reject an asset that isn\'t in zip format', async () => {
58
+ const filePath = 'e2e/test/fixtures/assets/fake_zip.zip';
59
+ const buffer = fs.readFileSync(filePath);
60
+ await expect(() => storage.save(buffer)).rejects.toThrow('Failed to save asset. File type not recognized as zip.');
61
+ });
62
+ it('will reject an asset if the minimum teraslice version is not met', async () => {
63
+ const filePath = 'e2e/test/fixtures/assets/test_asset_json.zip';
64
+ const buffer = fs.readFileSync(filePath);
65
+ await expect(() => storage.save(buffer)).rejects.toThrow('Asset requires teraslice version 999.9.9 or greater.');
66
+ expect(await storage.grabS3Info()).toEqual([]);
67
+ });
68
+ it('can save an asset to S3', async () => {
69
+ const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
70
+ const buffer = fs.readFileSync(filePath);
71
+ const result = await storage.save(buffer);
72
+ expect(result.assetId).toBe('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
73
+ });
74
+ it('can grab asset info from S3', async () => {
75
+ const list = await storage.grabS3Info();
76
+ expect(list).toEqual([{
77
+ File: 'caf0e5ce7cf1edc864f306b1d9edbad0f7060545.zip',
78
+ Size: 162711
79
+ }]);
80
+ });
81
+ it('can get an asset from S3', async () => {
82
+ /// create a buffer copy of example_asset_1.zip to test if it equals what s3 sends back
83
+ const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
84
+ const buffer = fs.readFileSync(filePath);
85
+ const assetRecord = await storage.get('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
86
+ expect(buffer.equals(assetRecord.blob)).toBe(true);
87
+ expect(assetRecord.name).toBe('ex1');
88
+ });
89
+ it('can delete an asset from S3', async () => {
90
+ await storage.remove('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
91
+ const list = await storage.grabS3Info();
92
+ expect(list).toBeEmpty();
93
+ });
94
+ });
95
+ //# sourceMappingURL=assets_storage-spec.js.map
@@ -0,0 +1,138 @@
1
+ import fse from 'fs-extra';
2
+ import { TestContext } from '@terascope/job-components';
3
+ import { createS3Client, deleteS3Bucket } from '@terascope/file-asset-apis';
4
+ import { S3Store } from '../../src/lib/storage/backends/s3_store';
5
+ import { TEST_INDEX_PREFIX } from '../test.config';
6
+ describe('S3 backend test', () => {
7
+ let s3Backend;
8
+ const contextOptions = {
9
+ // assignment: 'assets_service',
10
+ clients: [
11
+ {
12
+ type: 's3',
13
+ createClient: async (customConfig, logger) => {
14
+ const client = await createS3Client(customConfig, logger);
15
+ return { client, logger };
16
+ },
17
+ endpoint: 'default'
18
+ }
19
+ ]
20
+ };
21
+ const context = new TestContext(`${TEST_INDEX_PREFIX}s3-store-test`, contextOptions);
22
+ context.sysconfig.terafoundation = {
23
+ connectors: {
24
+ s3: {
25
+ default: {
26
+ endpoint: process.env.MINIO_HOST,
27
+ accessKeyId: process.env.MINIO_ACCESS_KEY,
28
+ secretAccessKey: process.env.MINIO_SECRET_KEY,
29
+ forcePathStyle: true,
30
+ sslEnabled: false,
31
+ region: 'test-region'
32
+ }
33
+ }
34
+ }
35
+ };
36
+ describe('->verifyClient', () => {
37
+ beforeEach(async () => {
38
+ s3Backend = new S3Store({
39
+ context,
40
+ terafoundation: context.sysconfig.terafoundation,
41
+ connection: 'default',
42
+ bucket: 'ts-assets'
43
+ });
44
+ await s3Backend.initialize();
45
+ });
46
+ afterEach(async () => {
47
+ await s3Backend.shutdown();
48
+ });
49
+ it('should throw error trying to verify client if bucket does not exist', async () => {
50
+ const command = {
51
+ Bucket: 'ts-assets'
52
+ };
53
+ await deleteS3Bucket(s3Backend.api, command);
54
+ const response = await s3Backend.verifyClient();
55
+ expect(response).toEqual(false);
56
+ });
57
+ it('should be able to verify the client is up', async () => {
58
+ const response = await s3Backend.verifyClient();
59
+ expect(response).toEqual(true);
60
+ const command = {
61
+ Bucket: 'ts-assets'
62
+ };
63
+ await deleteS3Bucket(s3Backend.api, command);
64
+ });
65
+ });
66
+ describe('save, get, and remove assets', () => {
67
+ beforeAll(async () => {
68
+ s3Backend = new S3Store({
69
+ context,
70
+ terafoundation: context.sysconfig.terafoundation,
71
+ connection: 'default',
72
+ bucket: 'ts-assets'
73
+ });
74
+ await s3Backend.initialize();
75
+ });
76
+ afterAll(async () => {
77
+ const command = {
78
+ Bucket: 'ts-assets'
79
+ };
80
+ await deleteS3Bucket(s3Backend.api, command);
81
+ await s3Backend.shutdown();
82
+ });
83
+ it('should be able to write in a zip file to ts_assets bucket', async () => {
84
+ const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
85
+ await s3Backend.save('ex1', fse.readFileSync(filePath), 30000);
86
+ const result = await s3Backend.list();
87
+ expect(result[0].File).toBe('ex1.zip');
88
+ });
89
+ it('should be able to delete zip file in ts_assets bucket', async () => {
90
+ await s3Backend.remove('ex1');
91
+ const result = await s3Backend.list();
92
+ expect(result).toBeEmpty();
93
+ });
94
+ it('should be able to download asset', async () => {
95
+ const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
96
+ const fileBuffer = fse.readFileSync(filePath);
97
+ await s3Backend.save('ex1', fileBuffer, 30000);
98
+ const result = await s3Backend.get('ex1');
99
+ expect(result.equals(fileBuffer)).toBe(true);
100
+ await s3Backend.remove('ex1');
101
+ });
102
+ });
103
+ describe('when bucket name is not defined', () => {
104
+ let bucketName;
105
+ afterAll(async () => {
106
+ const command = {
107
+ Bucket: bucketName
108
+ };
109
+ await deleteS3Bucket(s3Backend.api, command);
110
+ await s3Backend.shutdown();
111
+ });
112
+ it('should create a bucket name containing terafoundation.teraslice.name', async () => {
113
+ bucketName = `ts-assets-${TEST_INDEX_PREFIX}s3-store-test`.replaceAll('_', '-');
114
+ s3Backend = new S3Store({
115
+ context,
116
+ terafoundation: context.sysconfig.terafoundation,
117
+ connection: 'default',
118
+ bucket: undefined
119
+ });
120
+ await s3Backend.initialize();
121
+ expect(s3Backend.bucket).toBe(bucketName);
122
+ });
123
+ it('should create a bucket name where underscores in teraslice.name are replaced by dashes', async () => {
124
+ const contextWithUnderscoreName = new TestContext('s3_backend_underscores', contextOptions);
125
+ contextWithUnderscoreName.sysconfig.terafoundation = context.sysconfig.terafoundation;
126
+ s3Backend = new S3Store({
127
+ context: contextWithUnderscoreName,
128
+ terafoundation: contextWithUnderscoreName.sysconfig.terafoundation,
129
+ connection: 'default',
130
+ bucket: undefined
131
+ });
132
+ await s3Backend.initialize();
133
+ expect(s3Backend.bucket).toBe('ts-assets-s3-backend-underscores');
134
+ bucketName = 'ts-assets-s3-backend-underscores';
135
+ });
136
+ });
137
+ });
138
+ //# sourceMappingURL=s3_store-spec.js.map
@@ -0,0 +1,8 @@
1
+ import { newId } from '../src/lib/utils/id_utils.js';
2
+ const { TEST_INDEX_PREFIX = 'teratest_', ELASTICSEARCH_HOST = 'http://localhost:9200', TERASLICE_CLUSTER_NAME = newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2), ELASTICSEARCH_VERSION = '6.8', SEARCH_TEST_HOST = ELASTICSEARCH_HOST } = process.env;
3
+ process.env.TERASLICE_CLUSTER_NAME = TERASLICE_CLUSTER_NAME;
4
+ process.env.ELASTICSEARCH_HOST = ELASTICSEARCH_HOST;
5
+ process.env.ELASTICSEARCH_VERSION = ELASTICSEARCH_VERSION;
6
+ process.env.SEARCH_TEST_HOST = SEARCH_TEST_HOST;
7
+ export { TEST_INDEX_PREFIX, SEARCH_TEST_HOST, TERASLICE_CLUSTER_NAME, ELASTICSEARCH_VERSION, };
8
+ //# sourceMappingURL=test.config.js.map
@@ -0,0 +1,6 @@
1
+ import { jest } from '@jest/globals';
2
+ process.env.USE_DEBUG_LOGGER = 'true';
3
+ process.env.NODE_ENV = 'test';
4
+ // use a larger timeout
5
+ jest.setTimeout(60 * 1000);
6
+ //# sourceMappingURL=test.setup.js.map