queen-mq 0.2.0 → 0.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +165 -769
  2. package/{src → client-js}/benchmark/consumer.js +5 -3
  3. package/{src → client-js}/benchmark/producer.js +9 -4
  4. package/{src → client-js}/client/client.js +272 -6
  5. package/{src → client-js}/client/index.js +1 -4
  6. package/{src → client-js}/client/utils/http.js +3 -2
  7. package/{src → client-js}/client/utils/retry.js +7 -1
  8. package/{src → client-js}/test/core-tests.js +7 -2
  9. package/{src → client-js}/test/edge-case-tests.js +2 -1
  10. package/client-js/test/human.js +162 -0
  11. package/client-js/test/qos0-tests.js +334 -0
  12. package/{src → client-js}/test/test-new.js +42 -0
  13. package/package.json +8 -17
  14. package/init-db.js +0 -20
  15. package/src/cluster-server.js +0 -242
  16. package/src/config.js +0 -229
  17. package/src/database/connection.js +0 -129
  18. package/src/database/poolManager.js +0 -199
  19. package/src/database/schema-v2.sql +0 -278
  20. package/src/managers/eventManager.js +0 -59
  21. package/src/managers/queueManagerOptimized.js +0 -1634
  22. package/src/managers/resourceCache.js +0 -96
  23. package/src/managers/systemEventManager.js +0 -132
  24. package/src/routes/ack.js +0 -26
  25. package/src/routes/configure.js +0 -46
  26. package/src/routes/messages.js +0 -368
  27. package/src/routes/pop.js +0 -69
  28. package/src/routes/push.js +0 -28
  29. package/src/routes/resources.js +0 -330
  30. package/src/routes/status.js +0 -1037
  31. package/src/server.js +0 -1633
  32. package/src/test/MIGRATION_ISSUES.md +0 -174
  33. package/src/test/test.js +0 -4524
  34. package/src/utils/streaming.js +0 -231
  35. package/src/webapp-dist/assets/Analytics-DJrE4Rz2.css +0 -1
  36. package/src/webapp-dist/assets/Analytics-HEMXKmfJ.js +0 -1
  37. package/src/webapp-dist/assets/ConfirmDialog-YEoCdE7x.js +0 -1
  38. package/src/webapp-dist/assets/ConsumerGroups-CC5VJywp.css +0 -1
  39. package/src/webapp-dist/assets/ConsumerGroups-D8Z20HhZ.js +0 -1
  40. package/src/webapp-dist/assets/Dashboard-BRFOz6it.css +0 -1
  41. package/src/webapp-dist/assets/Dashboard-Bpx9I32h.js +0 -1
  42. package/src/webapp-dist/assets/LoadingSpinner-BPttXcbs.js +0 -1
  43. package/src/webapp-dist/assets/Messages-BOfVHKma.css +0 -1
  44. package/src/webapp-dist/assets/Messages-EKURCbLl.js +0 -1
  45. package/src/webapp-dist/assets/QueueDetail-BXXA7xoZ.css +0 -1
  46. package/src/webapp-dist/assets/QueueDetail-Bmxg0KiA.js +0 -1
  47. package/src/webapp-dist/assets/Queues-CvrJkrZT.css +0 -1
  48. package/src/webapp-dist/assets/Queues-yUx9_3-9.js +0 -1
  49. package/src/webapp-dist/assets/StatusBadge-C1bBtXHh.css +0 -1
  50. package/src/webapp-dist/assets/StatusBadge-C6N84XmO.js +0 -1
  51. package/src/webapp-dist/assets/analytics-BapTekLM.js +0 -1
  52. package/src/webapp-dist/assets/colors-5z6Q_kUr.js +0 -18
  53. package/src/webapp-dist/assets/index-B7w2bKT6.css +0 -1
  54. package/src/webapp-dist/assets/index-CUSnwYTH.js +0 -31
  55. package/src/webapp-dist/assets/messages-CRV6E-xU.js +0 -1
  56. package/src/webapp-dist/assets/queen-logo-blue.svg +0 -210
  57. package/src/webapp-dist/assets/queen-logo-cyan.svg +0 -210
  58. package/src/webapp-dist/assets/queen-logo-indigo.svg +0 -210
  59. package/src/webapp-dist/assets/queen-logo-orange.svg +0 -210
  60. package/src/webapp-dist/assets/queen-logo-pink.svg +0 -210
  61. package/src/webapp-dist/assets/queen-logo-purple.svg +0 -210
  62. package/src/webapp-dist/assets/queen-logo-rose.svg +0 -239
  63. package/src/webapp-dist/assets/queen-logo.svg +0 -263
  64. package/src/webapp-dist/assets/queues-BdUbxfBC.js +0 -1
  65. package/src/webapp-dist/assets/resources-Bs9U-ceF.js +0 -1
  66. package/src/webapp-dist/index.html +0 -15
  67. package/src/websocket/wsServer.js +0 -228
  68. /package/{src → client-js}/benchmark/consumer_multi.js +0 -0
  69. /package/{src → client-js}/benchmark/producer_multi.js +0 -0
  70. /package/{src → client-js}/client/utils/loadBalancer.js +0 -0
  71. /package/{src → client-js}/services/encryptionService.js +0 -0
  72. /package/{src → client-js}/services/evictionService.js +0 -0
  73. /package/{src → client-js}/services/retentionService.js +0 -0
  74. /package/{src → client-js}/services/startupSync.js +0 -0
  75. /package/{src → client-js}/test/README.md +0 -0
  76. /package/{src → client-js}/test/advanced-client-tests.js +0 -0
  77. /package/{src → client-js}/test/advanced-pattern-tests.js +0 -0
  78. /package/{src → client-js}/test/bus-mode-tests.js +0 -0
  79. /package/{src → client-js}/test/enterprise-tests.js +0 -0
  80. /package/{src → client-js}/test/partition-locking-tests.js +0 -0
  81. /package/{src → client-js}/test/utils.js +0 -0
  82. /package/{src → client-js}/test/window-buffer-test.js +0 -0
  83. /package/{src → client-js}/utils/logger.js +0 -0
  84. /package/{src → client-js}/utils/uuid.js +0 -0
package/src/server.js DELETED
@@ -1,1633 +0,0 @@
1
- import uWS from 'uWebSockets.js';
2
- import fs from 'fs';
3
- import path from 'path';
4
- import { fileURLToPath } from 'url';
5
- import config from './config.js';
6
- import { createPool, initDatabase } from './database/connection.js';
7
- import { createOptimizedQueueManager } from './managers/queueManagerOptimized.js';
8
- import { createResourceCache } from './managers/resourceCache.js';
9
- import { createEventManager } from './managers/eventManager.js';
10
- import { SystemEventManager, SYSTEM_QUEUE } from './managers/systemEventManager.js';
11
- import { syncSystemEvents } from './services/startupSync.js';
12
- //import { createQueenClient } from './client/queenClient.js';
13
- import { createPushRoute } from './routes/push.js';
14
- import { createPopRoute } from './routes/pop.js';
15
- import { createAckRoute } from './routes/ack.js';
16
- import { createConfigureRoute } from './routes/configure.js';
17
- import { createMessagesRoutes } from './routes/messages.js';
18
- import { createResourcesRoutes } from './routes/resources.js';
19
- import { createStatusRoutes } from './routes/status.js';
20
- import { createWebSocketServer } from './websocket/wsServer.js';
21
- import { initEncryption } from './services/encryptionService.js';
22
- import { startRetentionJob } from './services/retentionService.js';
23
- import { startEvictionJob } from './services/evictionService.js';
24
- import { log } from './utils/logger.js';
25
- import { streamJSON, sendJSON, sendError } from './utils/streaming.js';
26
- import { generateUUID } from './utils/uuid.js';
27
-
28
- const __dirname = path.dirname(fileURLToPath(import.meta.url));
29
-
30
- /**
31
- * Start Queen MQ Server
32
- * @param {Object} options - Server configuration options
33
- * @param {number} options.port - Server port (default: from config)
34
- * @param {string} options.host - Server host (default: from config)
35
- * @param {string} options.workerId - Worker ID for clustering (default: from config)
36
- * @param {boolean} options.serveWebapp - Serve the built frontend from /webapp/dist (default: false)
37
- * @param {string} options.webappPath - Path to webapp dist folder (default: '../webapp/dist')
38
- * @returns {Promise<Object>} Server instance with shutdown method
39
- */
40
- export async function QueenServer(options = {}) {
41
- const PORT = options.port || config.SERVER.PORT;
42
- const HOST = options.host || config.SERVER.HOST;
43
- const SERVE_WEBAPP = options.serveWebapp || false;
44
-
45
- // Default webapp path: try src/webapp-dist (npm package), fall back to ../webapp/dist (development)
46
- let defaultWebappPath = path.join(__dirname, 'webapp-dist');
47
- if (!fs.existsSync(defaultWebappPath)) {
48
- defaultWebappPath = path.join(__dirname, '..', 'webapp', 'dist');
49
- }
50
- const WEBAPP_PATH = options.webappPath || defaultWebappPath;
51
-
52
- // Use WORKER_ID from config as the server instance ID for consumer groups
53
- // This ensures the same server (identified by WORKER_ID) maintains its position in the event stream
54
- const SERVER_INSTANCE_ID = options.workerId || config.SERVER.WORKER_ID;
55
-
56
- const SKIP_DATABASE_MIGRATION = (options.skipDatabaseMigration || process.env.SKIP_DATABASE_MIGRATION) || false;
57
-
58
- // Performance monitoring
59
- let requestCount = 0;
60
- let messageCount = 0;
61
- const startTime = Date.now();
62
-
63
- // Shutdown flag to prevent background jobs from running during shutdown
64
- let isShuttingDown = false;
65
-
66
- // Initialize components
67
- const pool = createPool();
68
- const resourceCache = createResourceCache();
69
- const eventManager = createEventManager();
70
-
71
- // Initialize system event manager
72
- const systemEventManager = new SystemEventManager(pool, SERVER_INSTANCE_ID);
73
-
74
- // Register cache event handlers
75
- resourceCache.registerEventHandlers(systemEventManager);
76
-
77
- // Always use optimized queue manager - it's better for all scenarios
78
- const queueManager = createOptimizedQueueManager(pool, resourceCache, eventManager, systemEventManager);
79
- const messagesRoutes = createMessagesRoutes(pool, queueManager);
80
- const resourcesRoutes = createResourcesRoutes(pool, systemEventManager);
81
- const statusRoutes = createStatusRoutes(pool);
82
-
83
- // Initialize database with migrations
84
- const initDatabaseWithMigrations = async () => {
85
- // Run base schema
86
- await initDatabase(pool);
87
-
88
- // Run extension migrations
89
- const migrationPath = path.join(__dirname, '..', 'migrations', '001-extend-features.sql');
90
- if (fs.existsSync(migrationPath)) {
91
- const migration = fs.readFileSync(migrationPath, 'utf8');
92
- try {
93
- await pool.query(migration);
94
- log('✅ Extension features migration applied');
95
- } catch (error) {
96
- if (!error.message.includes('already exists')) {
97
- log('Migration error:', error);
98
- }
99
- }
100
- }
101
- };
102
-
103
- if (!SKIP_DATABASE_MIGRATION) {
104
- await initDatabaseWithMigrations();
105
- }
106
-
107
- // Initialize system queue
108
- async function initializeSystemQueue() {
109
- try {
110
- // Direct database operation to create system queue
111
- await pool.query(`
112
- INSERT INTO queen.queues (name, ttl, priority, max_queue_size)
113
- VALUES ($1, 300, 100, 10000)
114
- ON CONFLICT (name) DO UPDATE SET
115
- ttl = EXCLUDED.ttl,
116
- priority = EXCLUDED.priority,
117
- max_queue_size = EXCLUDED.max_queue_size
118
- `, [SYSTEM_QUEUE]);
119
-
120
- await pool.query(`
121
- INSERT INTO queen.partitions (queue_id, name)
122
- SELECT id, 'Default' FROM queen.queues WHERE name = $1
123
- ON CONFLICT (queue_id, name) DO NOTHING
124
- `, [SYSTEM_QUEUE]);
125
-
126
- log('✅ System event queue initialized');
127
- } catch (error) {
128
- console.error('Failed to initialize system queue:', error);
129
- throw error;
130
- }
131
- }
132
-
133
- await initializeSystemQueue();
134
-
135
- // Initialize extended features
136
- log('🚀 Initializing Queen Services...');
137
-
138
- // 1. Encryption service
139
- const encryptionEnabled = initEncryption();
140
-
141
- // 2. Retention service
142
- const stopRetention = startRetentionJob(pool);
143
-
144
- // 3. Eviction service
145
- const stopEviction = startEvictionJob(pool, eventManager);
146
-
147
- // Background job for lease reclamation
148
- const leaseReclaimInterval = setInterval(async () => {
149
- if (isShuttingDown) return;
150
- try {
151
- const reclaimed = await queueManager.reclaimExpiredLeases();
152
- if (reclaimed > 0) {
153
- log(`Reclaimed ${reclaimed} expired leases`);
154
- }
155
- } catch (error) {
156
- if (!isShuttingDown) {
157
- log('Error reclaiming leases:', error);
158
- }
159
- }
160
- }, config.JOBS.LEASE_RECLAIM_INTERVAL);
161
-
162
- // Create server
163
- const app = uWS.App();
164
-
165
- // Initialize WebSocket server
166
- const wsServer = createWebSocketServer(app, eventManager);
167
-
168
- // Background job for queue depth updates
169
- const queueDepthInterval = setInterval(async () => {
170
- if (isShuttingDown) return;
171
- try {
172
- await wsServer.updateQueueDepths(queueManager);
173
- } catch (error) {
174
- if (!isShuttingDown) {
175
- log('Error updating queue depths:', error);
176
- }
177
- }
178
- }, config.JOBS.QUEUE_DEPTH_UPDATE_INTERVAL);
179
-
180
- // Background job for system stats
181
- const systemStatsInterval = setInterval(async () => {
182
- if (isShuttingDown) return;
183
- try {
184
- await wsServer.sendSystemStats(pool);
185
- } catch (error) {
186
- if (!isShuttingDown) {
187
- log('Error sending system stats:', error);
188
- }
189
- }
190
- }, config.JOBS.SYSTEM_STATS_UPDATE_INTERVAL);
191
-
192
- // CORS headers helper
193
- const setCorsHeaders = (res) => {
194
- res.writeHeader('Access-Control-Allow-Origin', config.API.CORS_ALLOWED_ORIGINS);
195
- res.writeHeader('Access-Control-Allow-Methods', config.API.CORS_ALLOWED_METHODS);
196
- res.writeHeader('Access-Control-Allow-Headers', config.API.CORS_ALLOWED_HEADERS);
197
- res.writeHeader('Access-Control-Max-Age', config.API.CORS_MAX_AGE.toString());
198
- };
199
-
200
- // Helper to read JSON body
201
- const readJson = (res, cb, abortedRef) => {
202
- let buffer;
203
- const maxBodySize = config.API.MAX_BODY_SIZE; // 10MB max body size
204
-
205
- res.onData((ab, isLast) => {
206
- const chunk = Buffer.from(ab);
207
-
208
- // Check size limit
209
- const currentSize = (buffer ? buffer.length : 0) + chunk.length;
210
- if (currentSize > maxBodySize) {
211
- if (abortedRef && abortedRef.aborted) return;
212
- res.writeStatus(config.HTTP_STATUS.BAD_REQUEST.toString()).end(JSON.stringify({ error: 'Request body too large' }));
213
- return;
214
- }
215
-
216
- if (isLast) {
217
- let json;
218
- try {
219
- let fullData;
220
- if (buffer) {
221
- fullData = Buffer.concat([buffer, chunk]);
222
- } else {
223
- fullData = chunk;
224
- }
225
-
226
- const jsonString = fullData.toString('utf8');
227
-
228
- // Debug large payloads
229
- if (jsonString.length > 1000000) {
230
- log(`Parsing large JSON: ${jsonString.length} bytes, starts with: ${jsonString.substring(0, 100)}`);
231
- }
232
-
233
- json = JSON.parse(jsonString);
234
- } catch (e) {
235
- if (abortedRef && abortedRef.aborted) return;
236
- // Log the actual error for debugging
237
- log(`JSON parse error: ${e.message}`);
238
-
239
- // Log what we're trying to parse for debugging
240
- let sample = '';
241
- if (buffer) {
242
- const fullData = Buffer.concat([buffer, chunk]);
243
- sample = fullData.toString('utf8').substring(0, 200);
244
- } else {
245
- sample = chunk.toString('utf8').substring(0, 200);
246
- }
247
- log(`Failed to parse data starting with: ${sample}`);
248
-
249
- res.writeStatus(config.HTTP_STATUS.BAD_REQUEST.toString()).end(JSON.stringify({ error: 'Invalid JSON' }));
250
- return;
251
- }
252
- cb(json);
253
- } else {
254
- if (buffer) {
255
- buffer = Buffer.concat([buffer, chunk]);
256
- } else {
257
- buffer = Buffer.from(chunk); // Ensure it's a Buffer
258
- }
259
- }
260
- });
261
- };
262
-
263
- // Handle OPTIONS requests for CORS preflight
264
- app.options('/*', (res, req) => {
265
- res.onAborted(() => {
266
- log('OPTIONS request aborted');
267
- });
268
-
269
- res.cork(() => {
270
- setCorsHeaders(res);
271
- res.writeStatus(config.HTTP_STATUS.NO_CONTENT.toString()).end();
272
- });
273
- });
274
-
275
- // Configure route
276
- app.post('/api/v1/configure', (res, req) => {
277
- const abortedRef = { aborted: false };
278
- res.onAborted(() => {
279
- abortedRef.aborted = true;
280
- log('Configure request aborted');
281
- });
282
-
283
- readJson(res, async (body) => {
284
- if (abortedRef.aborted) return;
285
- try {
286
- const result = await createConfigureRoute(queueManager)(body);
287
- if (abortedRef.aborted) return;
288
- eventManager.emit('queue.created', result);
289
- res.cork(() => {
290
- setCorsHeaders(res);
291
- res.writeStatus(config.HTTP_STATUS.CREATED.toString()).end(JSON.stringify(result));
292
- });
293
- } catch (error) {
294
- if (abortedRef.aborted) return;
295
- log('Configure error:', error);
296
- res.cork(() => {
297
- setCorsHeaders(res);
298
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
299
- });
300
- }
301
- }, abortedRef);
302
- });
303
-
304
- // Push route
305
- app.post('/api/v1/push', (res, req) => {
306
- const abortedRef = { aborted: false };
307
- res.onAborted(() => {
308
- abortedRef.aborted = true;
309
- log('Push request aborted');
310
- });
311
-
312
- readJson(res, async (body) => {
313
- if (abortedRef.aborted) return;
314
- try {
315
- const result = await createPushRoute(queueManager)(body);
316
- messageCount += result.messages.length;
317
- requestCount++;
318
- if (abortedRef.aborted) return;
319
-
320
- // Emit events for pushed messages
321
- for (const msg of result.messages) {
322
- if (msg.status === 'queued') {
323
- const item = body.items.find(i =>
324
- i.transactionId === msg.transactionId || !i.transactionId
325
- );
326
- if (item) {
327
- const partition = item.partition ?? 'Default';
328
- const queuePath = `${item.queue}/${partition}`;
329
- eventManager.emit('message.pushed', {
330
- queue: item.queue,
331
- partition: partition,
332
- transactionId: msg.transactionId
333
- });
334
- eventManager.notifyMessageAvailable(queuePath);
335
- }
336
- }
337
- }
338
-
339
- res.cork(() => {
340
- setCorsHeaders(res);
341
- res.writeStatus(config.HTTP_STATUS.CREATED.toString()).end(JSON.stringify(result));
342
- });
343
- } catch (error) {
344
- if (abortedRef.aborted) return;
345
- log('Push error:', error.message || error);
346
- res.cork(() => {
347
- setCorsHeaders(res);
348
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
349
- });
350
- }
351
- }, abortedRef);
352
- });
353
-
354
- // Pop routes with new structure
355
- app.get('/api/v1/pop/queue/:queue/partition/:partition', (res, req) => {
356
- const queue = req.getParameter(0);
357
- const partition = req.getParameter(1);
358
- const query = new URLSearchParams(req.getQuery());
359
-
360
- let aborted = false;
361
- res.onAborted(() => {
362
- aborted = true;
363
- log('Pop request aborted');
364
- });
365
-
366
- createPopRoute(queueManager, eventManager)(
367
- {
368
- queue,
369
- partition,
370
- consumerGroup: query.get('consumerGroup') || query.get('consumer_group')
371
- },
372
- {
373
- wait: query.get('wait') === 'true',
374
- timeout: parseInt(query.get('timeout') || config.QUEUE.DEFAULT_TIMEOUT.toString()),
375
- batch: parseInt(query.get('batch') || config.QUEUE.DEFAULT_BATCH_SIZE.toString()),
376
- subscriptionMode: query.get('subscriptionMode'),
377
- subscriptionFrom: query.get('subscriptionFrom')
378
- }
379
- ).then(result => {
380
- if (aborted) return;
381
-
382
- // Emit events for processing messages
383
- for (const msg of result.messages) {
384
- eventManager.emit('message.processing', {
385
- queue: msg.queue,
386
- partition: msg.partition,
387
- transactionId: msg.transactionId,
388
- workerId: `worker-${process.pid}`
389
- });
390
- }
391
-
392
- // Use streaming with backpressure for large responses
393
- if (result.messages.length === 0) {
394
- res.cork(() => {
395
- setCorsHeaders(res);
396
- res.writeStatus(config.HTTP_STATUS.NO_CONTENT.toString()).end();
397
- });
398
- } else {
399
- res.cork(() => {
400
- setCorsHeaders(res);
401
- res.writeStatus(config.HTTP_STATUS.OK.toString());
402
- });
403
- streamJSON(res, result, { aborted });
404
- }
405
- }).catch(error => {
406
- if (aborted) return;
407
- log('Pop error:', error);
408
- res.cork(() => {
409
- setCorsHeaders(res);
410
- });
411
- sendError(res, error, config.HTTP_STATUS.INTERNAL_SERVER_ERROR);
412
- });
413
- });
414
-
415
- app.get('/api/v1/pop/queue/:queue', (res, req) => {
416
- const queue = req.getParameter(0);
417
- const query = new URLSearchParams(req.getQuery());
418
-
419
- let aborted = false;
420
- res.onAborted(() => {
421
- aborted = true;
422
- log('Pop request aborted');
423
- });
424
-
425
- createPopRoute(queueManager, eventManager)(
426
- {
427
- queue,
428
- consumerGroup: query.get('consumerGroup') || query.get('consumer_group')
429
- },
430
- {
431
- wait: query.get('wait') === 'true',
432
- timeout: parseInt(query.get('timeout') || config.QUEUE.DEFAULT_TIMEOUT.toString()),
433
- batch: parseInt(query.get('batch') || config.QUEUE.DEFAULT_BATCH_SIZE.toString()),
434
- subscriptionMode: query.get('subscriptionMode'),
435
- subscriptionFrom: query.get('subscriptionFrom')
436
- }
437
- ).then(result => {
438
- if (aborted) return;
439
-
440
- for (const msg of result.messages) {
441
- eventManager.emit('message.processing', {
442
- queue: msg.queue,
443
- partition: msg.partition,
444
- transactionId: msg.transactionId,
445
- workerId: `worker-${process.pid}`
446
- });
447
- }
448
-
449
- // Use streaming with backpressure for large responses
450
- if (result.messages.length === 0) {
451
- res.cork(() => {
452
- setCorsHeaders(res);
453
- res.writeStatus(config.HTTP_STATUS.NO_CONTENT.toString()).end();
454
- });
455
- } else {
456
- res.cork(() => {
457
- setCorsHeaders(res);
458
- res.writeStatus(config.HTTP_STATUS.OK.toString());
459
- });
460
- streamJSON(res, result, { aborted });
461
- }
462
- }).catch(error => {
463
- if (aborted) return;
464
- log('Pop error:', error);
465
- res.cork(() => {
466
- setCorsHeaders(res);
467
- });
468
- sendError(res, error, config.HTTP_STATUS.INTERNAL_SERVER_ERROR);
469
- });
470
- });
471
-
472
- // Pop with filters (namespace or task)
473
- app.get('/api/v1/pop', (res, req) => {
474
- const query = new URLSearchParams(req.getQuery());
475
- const namespace = query.get('namespace');
476
- const task = query.get('task');
477
-
478
- let aborted = false;
479
- res.onAborted(() => {
480
- aborted = true;
481
- log('Pop request aborted');
482
- });
483
-
484
- createPopRoute(queueManager, eventManager)(
485
- {
486
- namespace,
487
- task,
488
- consumerGroup: query.get('consumerGroup') || query.get('consumer_group')
489
- },
490
- {
491
- wait: query.get('wait') === 'true',
492
- timeout: parseInt(query.get('timeout') || config.QUEUE.DEFAULT_TIMEOUT.toString()),
493
- batch: parseInt(query.get('batch') || config.QUEUE.DEFAULT_BATCH_SIZE.toString()),
494
- subscriptionMode: query.get('subscriptionMode'),
495
- subscriptionFrom: query.get('subscriptionFrom')
496
- }
497
- ).then(result => {
498
- if (aborted) return;
499
-
500
- for (const msg of result.messages) {
501
- eventManager.emit('message.processing', {
502
- queue: msg.queue,
503
- partition: msg.partition,
504
- transactionId: msg.transactionId,
505
- workerId: `worker-${process.pid}`
506
- });
507
- }
508
-
509
- // Use streaming with backpressure for large responses
510
- if (result.messages.length === 0) {
511
- res.cork(() => {
512
- setCorsHeaders(res);
513
- res.writeStatus(config.HTTP_STATUS.NO_CONTENT.toString()).end();
514
- });
515
- } else {
516
- res.cork(() => {
517
- setCorsHeaders(res);
518
- res.writeStatus(config.HTTP_STATUS.OK.toString());
519
- });
520
- streamJSON(res, result, { aborted });
521
- }
522
- }).catch(error => {
523
- if (aborted) return;
524
- log('Pop error:', error);
525
- res.cork(() => {
526
- setCorsHeaders(res);
527
- });
528
- sendError(res, error, config.HTTP_STATUS.INTERNAL_SERVER_ERROR);
529
- });
530
- });
531
-
532
- // ACK route
533
- app.post('/api/v1/ack', (res, req) => {
534
- const abortedRef = { aborted: false };
535
- res.onAborted(() => {
536
- abortedRef.aborted = true;
537
- log('ACK request aborted');
538
- });
539
-
540
- readJson(res, async (body) => {
541
- if (abortedRef.aborted) return;
542
- try {
543
- const result = await createAckRoute(queueManager)(body);
544
- if (abortedRef.aborted) return;
545
-
546
- // Emit completion or failure event
547
- if (body.status === 'completed') {
548
- eventManager.emit('message.completed', {
549
- transactionId: body.transactionId
550
- });
551
- } else {
552
- eventManager.emit('message.failed', {
553
- transactionId: body.transactionId,
554
- error: body.error
555
- });
556
- }
557
-
558
- res.cork(() => {
559
- setCorsHeaders(res);
560
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
561
- });
562
- } catch (error) {
563
- if (abortedRef.aborted) return;
564
- log('ACK error:', error.message || error);
565
- res.cork(() => {
566
- res.writeStatus('500');
567
- setCorsHeaders(res);
568
- res.end(JSON.stringify({ error: error.message || 'Unknown error' }));
569
- });
570
- }
571
- }, abortedRef);
572
- });
573
-
574
- // Batch ACK route
575
- app.post('/api/v1/ack/batch', (res, req) => {
576
- const abortedRef = { aborted: false };
577
- res.onAborted(() => {
578
- abortedRef.aborted = true;
579
- log('Batch ACK request aborted');
580
- });
581
-
582
- readJson(res, async (body) => {
583
- if (abortedRef.aborted) return;
584
- try {
585
- const results = await queueManager.acknowledgeMessages(body.acknowledgments, body.consumerGroup);
586
- if (abortedRef.aborted) return;
587
-
588
- // Emit events for each acknowledgment
589
- for (const ack of body.acknowledgments) {
590
- if (ack.status === 'completed') {
591
- eventManager.emit('message.completed', {
592
- transactionId: ack.transactionId
593
- });
594
- } else {
595
- eventManager.emit('message.failed', {
596
- transactionId: ack.transactionId,
597
- error: ack.error
598
- });
599
- }
600
- }
601
-
602
- res.cork(() => {
603
- setCorsHeaders(res);
604
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({
605
- processed: results.length,
606
- results
607
- }));
608
- });
609
- } catch (error) {
610
- if (abortedRef.aborted) return;
611
- log('Batch ACK error:', error);
612
- res.cork(() => {
613
- setCorsHeaders(res);
614
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
615
- });
616
- }
617
- }, abortedRef);
618
- });
619
-
620
- // Lease extension endpoint
621
- app.post('/api/v1/lease/:leaseId/extend', (res, req) => {
622
- const abortedRef = { aborted: false };
623
- res.onAborted(() => {
624
- abortedRef.aborted = true;
625
- log('Lease extension request aborted');
626
- });
627
-
628
- const leaseId = req.getParameter(0); // In uWebSockets.js, parameters are accessed by index
629
-
630
- readJson(res, async (body) => {
631
- if (abortedRef.aborted) return;
632
-
633
- const { seconds = 60 } = body || {};
634
-
635
- try {
636
- const result = await pool.query(`
637
- UPDATE queen.partition_consumers
638
- SET lease_expires_at = GREATEST(lease_expires_at, NOW() + INTERVAL '1 second' * $1)
639
- WHERE worker_id = $2 AND lease_expires_at > NOW()
640
- RETURNING lease_expires_at, partition_id
641
- `, [seconds, leaseId]);
642
-
643
- if (result.rows.length === 0) {
644
- if (!abortedRef.aborted) {
645
- res.cork(() => {
646
- setCorsHeaders(res);
647
- res.writeStatus('404').end(JSON.stringify({ error: 'Lease not found or expired' }));
648
- });
649
- }
650
- return;
651
- }
652
-
653
- if (!abortedRef.aborted) {
654
- res.cork(() => {
655
- setCorsHeaders(res);
656
- res.writeStatus('200').end(JSON.stringify({
657
- leaseId,
658
- newExpiresAt: result.rows[0].lease_expires_at
659
- }));
660
- });
661
- }
662
- } catch (error) {
663
- if (!abortedRef.aborted) {
664
- log('Lease extension error:', error);
665
- res.cork(() => {
666
- setCorsHeaders(res);
667
- res.writeStatus('500').end(JSON.stringify({ error: error.message }));
668
- });
669
- }
670
- }
671
- }, abortedRef);
672
- });
673
-
674
- // Transaction endpoint
675
- app.post('/api/v1/transaction', (res, req) => {
676
- const abortedRef = { aborted: false };
677
- res.onAborted(() => {
678
- abortedRef.aborted = true;
679
- log('Transaction request aborted');
680
- });
681
-
682
- readJson(res, async (body) => {
683
- if (abortedRef.aborted) return;
684
-
685
- const { operations = [], requiredLeases = [] } = body;
686
-
687
- if (!Array.isArray(operations) || operations.length === 0) {
688
- if (!abortedRef.aborted) {
689
- res.cork(() => {
690
- setCorsHeaders(res);
691
- res.writeStatus('400').end(JSON.stringify({ error: 'Operations array required' }));
692
- });
693
- }
694
- return;
695
- }
696
-
697
- try {
698
- const results = await queueManager.executeTransaction(operations, requiredLeases);
699
-
700
- if (!abortedRef.aborted) {
701
- const transactionId = generateUUID();
702
- res.cork(() => {
703
- setCorsHeaders(res);
704
- res.writeStatus('200').end(JSON.stringify({
705
- success: true,
706
- results,
707
- transactionId
708
- }));
709
- });
710
- }
711
- } catch (error) {
712
- if (!abortedRef.aborted) {
713
- log('Transaction error:', error);
714
- res.cork(() => {
715
- setCorsHeaders(res);
716
- res.writeStatus('400').end(JSON.stringify({ error: error.message }));
717
- });
718
- }
719
- }
720
- }, abortedRef);
721
- });
722
-
723
- // Messages routes
724
- app.get('/api/v1/messages', (res, req) => {
725
- const query = new URLSearchParams(req.getQuery());
726
-
727
- let aborted = false;
728
- res.onAborted(() => {
729
- aborted = true;
730
- log('Messages list request aborted');
731
- });
732
-
733
- const filters = {
734
- ns: query.get('ns'),
735
- task: query.get('task'),
736
- queue: query.get('queue'),
737
- status: query.get('status'),
738
- limit: parseInt(query.get('limit') || config.API.DEFAULT_LIMIT.toString()),
739
- offset: parseInt(query.get('offset') || config.API.DEFAULT_OFFSET.toString())
740
- };
741
-
742
- messagesRoutes.listMessages(filters).then(result => {
743
- if (aborted) return;
744
- res.cork(() => {
745
- setCorsHeaders(res);
746
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({ messages: result }));
747
- });
748
- }).catch(error => {
749
- if (aborted) return;
750
- log('List messages error:', error);
751
- res.cork(() => {
752
- setCorsHeaders(res);
753
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
754
- });
755
- });
756
- });
757
-
758
- app.get('/api/v1/messages/:transactionId', (res, req) => {
759
- const transactionId = req.getParameter(0);
760
-
761
- let aborted = false;
762
- res.onAborted(() => {
763
- aborted = true;
764
- log('Get message request aborted');
765
- });
766
-
767
- messagesRoutes.getMessage(transactionId).then(result => {
768
- if (aborted) return;
769
- res.cork(() => {
770
- setCorsHeaders(res);
771
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
772
- });
773
- }).catch(error => {
774
- if (aborted) return;
775
- log('Get message error:', error);
776
- res.cork(() => {
777
- setCorsHeaders(res);
778
- res.writeStatus(error.message === 'Message not found' ? config.HTTP_STATUS.NOT_FOUND.toString() : config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString())
779
- .end(JSON.stringify({ error: error.message }));
780
- });
781
- });
782
- });
783
-
784
- app.del('/api/v1/messages/:transactionId', (res, req) => {
785
- const transactionId = req.getParameter(0);
786
-
787
- let aborted = false;
788
- res.onAborted(() => {
789
- aborted = true;
790
- log('Delete message request aborted');
791
- });
792
-
793
- messagesRoutes.deleteMessage(transactionId).then(result => {
794
- if (aborted) return;
795
- res.cork(() => {
796
- setCorsHeaders(res);
797
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
798
- });
799
- }).catch(error => {
800
- if (aborted) return;
801
- log('Delete message error:', error);
802
- res.cork(() => {
803
- setCorsHeaders(res);
804
- res.writeStatus(error.message === 'Message not found' ? config.HTTP_STATUS.NOT_FOUND.toString() : config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString())
805
- .end(JSON.stringify({ error: error.message }));
806
- });
807
- });
808
- });
809
-
810
- app.post('/api/v1/messages/:transactionId/retry', (res, req) => {
811
- const transactionId = req.getParameter(0);
812
-
813
- let aborted = false;
814
- res.onAborted(() => {
815
- aborted = true;
816
- log('Retry message request aborted');
817
- });
818
-
819
- messagesRoutes.retryMessage(transactionId).then(result => {
820
- if (aborted) return;
821
- res.cork(() => {
822
- setCorsHeaders(res);
823
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
824
- });
825
- }).catch(error => {
826
- if (aborted) return;
827
- log('Retry message error:', error);
828
- res.cork(() => {
829
- setCorsHeaders(res);
830
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
831
- });
832
- });
833
- });
834
-
835
- app.post('/api/v1/messages/:transactionId/dlq', (res, req) => {
836
- const transactionId = req.getParameter(0);
837
-
838
- let aborted = false;
839
- res.onAborted(() => {
840
- aborted = true;
841
- log('Move to DLQ request aborted');
842
- });
843
-
844
- messagesRoutes.moveToDLQ(transactionId).then(result => {
845
- if (aborted) return;
846
- res.cork(() => {
847
- setCorsHeaders(res);
848
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
849
- });
850
- }).catch(error => {
851
- if (aborted) return;
852
- log('Move to DLQ error:', error);
853
- res.cork(() => {
854
- setCorsHeaders(res);
855
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
856
- });
857
- });
858
- });
859
-
860
- app.get('/api/v1/messages/:transactionId/related', (res, req) => {
861
- const transactionId = req.getParameter(0);
862
-
863
- let aborted = false;
864
- res.onAborted(() => {
865
- aborted = true;
866
- log('Get related messages request aborted');
867
- });
868
-
869
- messagesRoutes.getRelatedMessages(transactionId).then(result => {
870
- if (aborted) return;
871
- res.cork(() => {
872
- setCorsHeaders(res);
873
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({ messages: result }));
874
- });
875
- }).catch(error => {
876
- if (aborted) return;
877
- log('Get related messages error:', error);
878
- res.cork(() => {
879
- setCorsHeaders(res);
880
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
881
- });
882
- });
883
- });
884
-
885
- app.del('/api/v1/queues/:queue/clear', (res, req) => {
886
- const queue = req.getParameter(0);
887
- const query = new URLSearchParams(req.getQuery());
888
- const partition = query.get('partition');
889
-
890
- let aborted = false;
891
- res.onAborted(() => {
892
- aborted = true;
893
- log('Clear queue request aborted');
894
- });
895
-
896
- messagesRoutes.clearQueue(queue, partition).then(result => {
897
- if (aborted) return;
898
- res.cork(() => {
899
- setCorsHeaders(res);
900
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
901
- });
902
- }).catch(error => {
903
- if (aborted) return;
904
- log('Clear queue error:', error);
905
- res.cork(() => {
906
- setCorsHeaders(res);
907
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
908
- });
909
- });
910
- });
911
-
912
- // Resources routes
913
- app.get('/api/v1/resources/queues', (res, req) => {
914
- const query = new URLSearchParams(req.getQuery());
915
- const namespace = query.get('namespace');
916
- const task = query.get('task');
917
-
918
- let aborted = false;
919
- res.onAborted(() => {
920
- aborted = true;
921
- log('Get queues request aborted');
922
- });
923
-
924
- resourcesRoutes.getQueues({ namespace, task }).then(result => {
925
- if (aborted) return;
926
- res.cork(() => {
927
- setCorsHeaders(res);
928
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({ queues: result }));
929
- });
930
- }).catch(error => {
931
- if (aborted) return;
932
- log('Get queues error:', error);
933
- res.cork(() => {
934
- setCorsHeaders(res);
935
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
936
- });
937
- });
938
- });
939
-
940
- app.get('/api/v1/resources/queues/:queue', (res, req) => {
941
- const queue = req.getParameter(0);
942
-
943
- let aborted = false;
944
- res.onAborted(() => {
945
- aborted = true;
946
- log('Get queue request aborted');
947
- });
948
-
949
- resourcesRoutes.getQueue(queue).then(result => {
950
- if (aborted) return;
951
- res.cork(() => {
952
- setCorsHeaders(res);
953
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
954
- });
955
- }).catch(error => {
956
- if (aborted) return;
957
- log('Get queue error:', error);
958
- res.cork(() => {
959
- setCorsHeaders(res);
960
- res.writeStatus(error.message === 'Queue not found' ? config.HTTP_STATUS.NOT_FOUND.toString() : config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString())
961
- .end(JSON.stringify({ error: error.message }));
962
- });
963
- });
964
- });
965
-
966
- app.get('/api/v1/resources/partitions', (res, req) => {
967
- const query = new URLSearchParams(req.getQuery());
968
- const queue = query.get('queue');
969
- const minDepth = query.get('minDepth');
970
-
971
- let aborted = false;
972
- res.onAborted(() => {
973
- aborted = true;
974
- log('Get partitions request aborted');
975
- });
976
-
977
- resourcesRoutes.getPartitions({ queue, minDepth: minDepth ? parseInt(minDepth) : null }).then(result => {
978
- if (aborted) return;
979
- res.cork(() => {
980
- setCorsHeaders(res);
981
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({ partitions: result }));
982
- });
983
- }).catch(error => {
984
- if (aborted) return;
985
- log('Get partitions error:', error);
986
- res.cork(() => {
987
- setCorsHeaders(res);
988
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
989
- });
990
- });
991
- });
992
-
993
- app.get('/api/v1/resources/namespaces', (res, req) => {
994
- let aborted = false;
995
- res.onAborted(() => {
996
- aborted = true;
997
- log('Get namespaces request aborted');
998
- });
999
-
1000
- resourcesRoutes.getNamespaces().then(result => {
1001
- if (aborted) return;
1002
- res.cork(() => {
1003
- setCorsHeaders(res);
1004
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({ namespaces: result }));
1005
- });
1006
- }).catch(error => {
1007
- if (aborted) return;
1008
- log('Get namespaces error:', error);
1009
- res.cork(() => {
1010
- setCorsHeaders(res);
1011
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1012
- });
1013
- });
1014
- });
1015
-
1016
- app.get('/api/v1/resources/tasks', (res, req) => {
1017
- let aborted = false;
1018
- res.onAborted(() => {
1019
- aborted = true;
1020
- log('Get tasks request aborted');
1021
- });
1022
-
1023
- resourcesRoutes.getTasks().then(result => {
1024
- if (aborted) return;
1025
- res.cork(() => {
1026
- setCorsHeaders(res);
1027
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({ tasks: result }));
1028
- });
1029
- }).catch(error => {
1030
- if (aborted) return;
1031
- log('Get tasks error:', error);
1032
- res.cork(() => {
1033
- setCorsHeaders(res);
1034
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1035
- });
1036
- });
1037
- });
1038
-
1039
- app.get('/api/v1/resources/overview', (res, req) => {
1040
- let aborted = false;
1041
- res.onAborted(() => {
1042
- aborted = true;
1043
- log('Get overview request aborted');
1044
- });
1045
-
1046
- resourcesRoutes.getSystemOverview().then(result => {
1047
- if (aborted) return;
1048
- res.cork(() => {
1049
- setCorsHeaders(res);
1050
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1051
- });
1052
- }).catch(error => {
1053
- if (aborted) return;
1054
- log('Get overview error:', error);
1055
- res.cork(() => {
1056
- setCorsHeaders(res);
1057
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1058
- });
1059
- });
1060
- });
1061
-
1062
- // Delete queue
1063
- app.del('/api/v1/resources/queues/:queue', (res, req) => {
1064
- const queue = req.getParameter(0);
1065
-
1066
- let aborted = false;
1067
- res.onAborted(() => {
1068
- aborted = true;
1069
- log('Delete queue request aborted');
1070
- });
1071
-
1072
- resourcesRoutes.deleteQueue(queue).then(result => {
1073
- if (aborted) return;
1074
- res.cork(() => {
1075
- setCorsHeaders(res);
1076
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1077
- });
1078
- }).catch(error => {
1079
- if (aborted) return;
1080
- log('Delete queue error:', error);
1081
- res.cork(() => {
1082
- setCorsHeaders(res);
1083
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1084
- });
1085
- });
1086
- });
1087
-
1088
- // ============================================================================
1089
- // Status/Dashboard Routes (New unified API)
1090
- // ============================================================================
1091
-
1092
- // 1. Dashboard Status - GET /api/v1/status
1093
- app.get('/api/v1/status', (res, req) => {
1094
- const query = new URLSearchParams(req.getQuery());
1095
-
1096
- let aborted = false;
1097
- res.onAborted(() => {
1098
- aborted = true;
1099
- log('Status request aborted');
1100
- });
1101
-
1102
- const filters = {
1103
- from: query.get('from'),
1104
- to: query.get('to'),
1105
- queue: query.get('queue'),
1106
- namespace: query.get('namespace'),
1107
- task: query.get('task')
1108
- };
1109
-
1110
- statusRoutes.getStatus(filters).then(result => {
1111
- if (aborted) return;
1112
- res.cork(() => {
1113
- setCorsHeaders(res);
1114
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1115
- });
1116
- }).catch(error => {
1117
- if (aborted) return;
1118
- log('Status error:', error);
1119
- res.cork(() => {
1120
- setCorsHeaders(res);
1121
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1122
- });
1123
- });
1124
- });
1125
-
1126
- // 2. Queues List - GET /api/v1/status/queues
1127
- app.get('/api/v1/status/queues', (res, req) => {
1128
- const query = new URLSearchParams(req.getQuery());
1129
-
1130
- let aborted = false;
1131
- res.onAborted(() => {
1132
- aborted = true;
1133
- log('Status queues request aborted');
1134
- });
1135
-
1136
- const filters = {
1137
- from: query.get('from'),
1138
- to: query.get('to'),
1139
- namespace: query.get('namespace'),
1140
- task: query.get('task'),
1141
- limit: query.get('limit'),
1142
- offset: query.get('offset')
1143
- };
1144
-
1145
- statusRoutes.getQueues(filters).then(result => {
1146
- if (aborted) return;
1147
- res.cork(() => {
1148
- setCorsHeaders(res);
1149
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1150
- });
1151
- }).catch(error => {
1152
- if (aborted) return;
1153
- log('Status queues error:', error);
1154
- res.cork(() => {
1155
- setCorsHeaders(res);
1156
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1157
- });
1158
- });
1159
- });
1160
-
1161
- // 3. Queue Detail - GET /api/v1/status/queues/:queueName
1162
- app.get('/api/v1/status/queues/:queue', (res, req) => {
1163
- const queueName = req.getParameter(0);
1164
- const query = new URLSearchParams(req.getQuery());
1165
-
1166
- let aborted = false;
1167
- res.onAborted(() => {
1168
- aborted = true;
1169
- log('Status queue detail request aborted');
1170
- });
1171
-
1172
- const filters = {
1173
- from: query.get('from'),
1174
- to: query.get('to')
1175
- };
1176
-
1177
- statusRoutes.getQueueDetail(queueName, filters).then(result => {
1178
- if (aborted) return;
1179
- res.cork(() => {
1180
- setCorsHeaders(res);
1181
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1182
- });
1183
- }).catch(error => {
1184
- if (aborted) return;
1185
- log('Status queue detail error:', error);
1186
- const statusCode = error.message === 'Queue not found'
1187
- ? config.HTTP_STATUS.NOT_FOUND
1188
- : config.HTTP_STATUS.INTERNAL_SERVER_ERROR;
1189
- res.cork(() => {
1190
- setCorsHeaders(res);
1191
- res.writeStatus(statusCode.toString()).end(JSON.stringify({ error: error.message }));
1192
- });
1193
- });
1194
- });
1195
-
1196
- // 4. Queue Messages - GET /api/v1/status/queues/:queueName/messages
1197
- app.get('/api/v1/status/queues/:queue/messages', (res, req) => {
1198
- const queueName = req.getParameter(0);
1199
- const query = new URLSearchParams(req.getQuery());
1200
-
1201
- let aborted = false;
1202
- res.onAborted(() => {
1203
- aborted = true;
1204
- log('Status queue messages request aborted');
1205
- });
1206
-
1207
- const filters = {
1208
- status: query.get('status'),
1209
- partition: query.get('partition'),
1210
- from: query.get('from'),
1211
- to: query.get('to'),
1212
- limit: query.get('limit'),
1213
- offset: query.get('offset')
1214
- };
1215
-
1216
- statusRoutes.getQueueMessages(queueName, filters).then(result => {
1217
- if (aborted) return;
1218
- res.cork(() => {
1219
- setCorsHeaders(res);
1220
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1221
- });
1222
- }).catch(error => {
1223
- if (aborted) return;
1224
- log('Status queue messages error:', error);
1225
- res.cork(() => {
1226
- setCorsHeaders(res);
1227
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1228
- });
1229
- });
1230
- });
1231
-
1232
- // 5. Analytics - GET /api/v1/status/analytics
1233
- app.get('/api/v1/status/analytics', (res, req) => {
1234
- const query = new URLSearchParams(req.getQuery());
1235
-
1236
- let aborted = false;
1237
- res.onAborted(() => {
1238
- aborted = true;
1239
- log('Status analytics request aborted');
1240
- });
1241
-
1242
- const filters = {
1243
- from: query.get('from'),
1244
- to: query.get('to'),
1245
- queue: query.get('queue'),
1246
- namespace: query.get('namespace'),
1247
- task: query.get('task'),
1248
- interval: query.get('interval') || 'hour'
1249
- };
1250
-
1251
- statusRoutes.getAnalytics(filters).then(result => {
1252
- if (aborted) return;
1253
- res.cork(() => {
1254
- setCorsHeaders(res);
1255
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(result));
1256
- });
1257
- }).catch(error => {
1258
- if (aborted) return;
1259
- log('Status analytics error:', error);
1260
- res.cork(() => {
1261
- setCorsHeaders(res);
1262
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end(JSON.stringify({ error: error.message }));
1263
- });
1264
- });
1265
- });
1266
-
1267
- // Health check with performance stats
1268
- app.get('/health', async (res, req) => {
1269
- let aborted = false;
1270
- res.onAborted(() => {
1271
- aborted = true;
1272
- log('Health check aborted');
1273
- });
1274
-
1275
- try {
1276
- await pool.query('SELECT 1');
1277
- if (aborted) return;
1278
-
1279
- const uptime = (Date.now() - startTime) / 1000;
1280
- res.cork(() => {
1281
- setCorsHeaders(res);
1282
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify({
1283
- status: 'healthy',
1284
- uptime: `${Math.floor(uptime)}s`,
1285
- connections: wsServer.getConnectionCount(),
1286
- stats: {
1287
- requests: requestCount,
1288
- messages: messageCount,
1289
- requestsPerSecond: (requestCount / uptime).toFixed(2),
1290
- messagesPerSecond: (messageCount / uptime).toFixed(2),
1291
- pool: {
1292
- total: pool.totalCount,
1293
- idle: pool.idleCount,
1294
- waiting: pool.waitingCount
1295
- }
1296
- }
1297
- }));
1298
- });
1299
- } catch (error) {
1300
- if (aborted) return;
1301
- res.cork(() => {
1302
- setCorsHeaders(res);
1303
- res.writeStatus(config.HTTP_STATUS.SERVICE_UNAVAILABLE.toString()).end(JSON.stringify({
1304
- status: 'unhealthy',
1305
- error: error.message
1306
- }));
1307
- });
1308
- }
1309
- });
1310
-
1311
- // Metrics endpoint for performance monitoring
1312
- app.get('/metrics', (res, req) => {
1313
- res.onAborted(() => {
1314
- log('Metrics request aborted');
1315
- });
1316
-
1317
- const uptime = (Date.now() - startTime) / 1000;
1318
- const metrics = {
1319
- uptime: uptime,
1320
- requests: {
1321
- total: requestCount,
1322
- rate: requestCount / uptime
1323
- },
1324
- messages: {
1325
- total: messageCount,
1326
- rate: messageCount / uptime
1327
- },
1328
- database: {
1329
- poolSize: pool.totalCount,
1330
- idleConnections: pool.idleCount,
1331
- waitingRequests: pool.waitingCount
1332
- },
1333
- memory: process.memoryUsage(),
1334
- cpu: process.cpuUsage()
1335
- };
1336
-
1337
- res.cork(() => {
1338
- setCorsHeaders(res);
1339
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(JSON.stringify(metrics));
1340
- });
1341
- });
1342
-
1343
- // ============================================================================
1344
- // Static File Serving (Optional - for serving built frontend)
1345
- // ============================================================================
1346
-
1347
- if (SERVE_WEBAPP) {
1348
- // Check if webapp directory exists
1349
- if (!fs.existsSync(WEBAPP_PATH)) {
1350
- log(`❌ ERROR: Webapp directory not found at: ${WEBAPP_PATH}`);
1351
- log(`💡 To enable frontend serving:`);
1352
- log(` 1. Build the webapp: cd webapp && npm install && npm run build`);
1353
- log(` 2. Or specify a custom path: QueenServer({ webappPath: '/path/to/dist' })`);
1354
- log(` 3. Or disable frontend: QueenServer({ serveWebapp: false })`);
1355
- throw new Error(`Webapp directory not found: ${WEBAPP_PATH}`);
1356
- }
1357
-
1358
- const indexPath = path.join(WEBAPP_PATH, 'index.html');
1359
- if (!fs.existsSync(indexPath)) {
1360
- log(`❌ ERROR: index.html not found in webapp directory: ${WEBAPP_PATH}`);
1361
- log(`💡 Make sure the webapp is built: cd webapp && npm run build`);
1362
- throw new Error(`index.html not found in: ${WEBAPP_PATH}`);
1363
- }
1364
-
1365
- log(`📁 Enabling static file serving from: ${WEBAPP_PATH}`);
1366
-
1367
- // MIME type mapping
1368
- const getMimeType = (filePath) => {
1369
- const ext = path.extname(filePath).toLowerCase();
1370
- const mimeTypes = {
1371
- '.html': 'text/html; charset=utf-8',
1372
- '.js': 'application/javascript; charset=utf-8',
1373
- '.css': 'text/css; charset=utf-8',
1374
- '.json': 'application/json; charset=utf-8',
1375
- '.png': 'image/png',
1376
- '.jpg': 'image/jpeg',
1377
- '.jpeg': 'image/jpeg',
1378
- '.gif': 'image/gif',
1379
- '.svg': 'image/svg+xml',
1380
- '.ico': 'image/x-icon',
1381
- '.woff': 'font/woff',
1382
- '.woff2': 'font/woff2',
1383
- '.ttf': 'font/ttf',
1384
- '.eot': 'application/vnd.ms-fontobject'
1385
- };
1386
- return mimeTypes[ext] || 'application/octet-stream';
1387
- };
1388
-
1389
- // Helper to serve static files
1390
- const serveStaticFile = async (res, filePath) => {
1391
- try {
1392
- // Security check - prevent directory traversal
1393
- const normalizedPath = path.normalize(filePath);
1394
- if (!normalizedPath.startsWith(WEBAPP_PATH)) {
1395
- res.writeStatus('403').end('Forbidden');
1396
- return;
1397
- }
1398
-
1399
- // Check if file exists
1400
- if (!fs.existsSync(filePath)) {
1401
- return false; // File not found, let caller handle it
1402
- }
1403
-
1404
- const stat = fs.statSync(filePath);
1405
- if (!stat.isFile()) {
1406
- return false;
1407
- }
1408
-
1409
- // Read file
1410
- const content = fs.readFileSync(filePath);
1411
- const mimeType = getMimeType(filePath);
1412
-
1413
- // Set cache headers for assets (fingerprinted files can be cached forever)
1414
- const isFingerprintedAsset = /\.[a-f0-9]{8}\.(js|css)$/.test(filePath);
1415
-
1416
- res.cork(() => {
1417
- setCorsHeaders(res);
1418
- res.writeHeader('Content-Type', mimeType);
1419
- res.writeHeader('Content-Length', content.length.toString());
1420
-
1421
- if (isFingerprintedAsset) {
1422
- res.writeHeader('Cache-Control', 'public, max-age=31536000, immutable');
1423
- } else if (filePath.endsWith('index.html')) {
1424
- res.writeHeader('Cache-Control', 'no-cache, no-store, must-revalidate');
1425
- } else {
1426
- res.writeHeader('Cache-Control', 'public, max-age=3600');
1427
- }
1428
-
1429
- res.writeStatus(config.HTTP_STATUS.OK.toString()).end(content);
1430
- });
1431
-
1432
- return true;
1433
- } catch (error) {
1434
- log('Error serving static file:', error);
1435
- res.writeStatus(config.HTTP_STATUS.INTERNAL_SERVER_ERROR.toString()).end('Internal Server Error');
1436
- return true;
1437
- }
1438
- };
1439
-
1440
- // Serve static assets (JS, CSS, images, fonts, etc.)
1441
- app.get('/assets/*', async (res, req) => {
1442
- let aborted = false;
1443
- res.onAborted(() => {
1444
- aborted = true;
1445
- });
1446
-
1447
- if (aborted) return;
1448
-
1449
- const url = req.getUrl();
1450
- const filePath = path.join(WEBAPP_PATH, url);
1451
-
1452
- const served = await serveStaticFile(res, filePath);
1453
- if (!served && !aborted) {
1454
- res.writeStatus('404').end('Not Found');
1455
- }
1456
- });
1457
-
1458
- // Serve root index.html
1459
- app.get('/', async (res, req) => {
1460
- let aborted = false;
1461
- res.onAborted(() => {
1462
- aborted = true;
1463
- });
1464
-
1465
- if (aborted) return;
1466
-
1467
- const indexPath = path.join(WEBAPP_PATH, 'index.html');
1468
- await serveStaticFile(res, indexPath);
1469
- });
1470
-
1471
- // SPA fallback - serve index.html for all non-API, non-asset routes
1472
- // This enables client-side routing to work
1473
- app.get('/*', async (res, req) => {
1474
- let aborted = false;
1475
- res.onAborted(() => {
1476
- aborted = true;
1477
- });
1478
-
1479
- if (aborted) return;
1480
-
1481
- const url = req.getUrl();
1482
-
1483
- // Don't serve index.html for API routes or special endpoints
1484
- if (url.startsWith('/api/') ||
1485
- url.startsWith('/health') ||
1486
- url.startsWith('/metrics') ||
1487
- url.startsWith('/ws/')) {
1488
- res.writeStatus('404').end('Not Found');
1489
- return;
1490
- }
1491
-
1492
- // First try to serve as a direct file (e.g., favicon.ico)
1493
- const directPath = path.join(WEBAPP_PATH, url);
1494
- const served = await serveStaticFile(res, directPath);
1495
-
1496
- // If file doesn't exist, serve index.html for SPA routing
1497
- if (!served && !aborted) {
1498
- const indexPath = path.join(WEBAPP_PATH, 'index.html');
1499
- await serveStaticFile(res, indexPath);
1500
- }
1501
- });
1502
-
1503
- log(`✅ Static file serving enabled at http://${HOST}:${PORT}/`);
1504
- }
1505
-
1506
- // Variable to hold system event consumer stop function
1507
- let stopSystemEventConsumer = null;
1508
-
1509
- // Start server
1510
- app.listen(HOST, PORT, async (token) => {
1511
- if (token) {
1512
- log(`✅ Queen Server running at http://${HOST}:${PORT}`);
1513
- log(`📊 Features enabled:`);
1514
- log(` - Encryption: ${encryptionEnabled ? '✅' : '❌ (set QUEEN_ENCRYPTION_KEY)'}`);
1515
- log(` - System Events: ${config.SYSTEM_EVENTS.ENABLED ? '✅' : '❌ (disabled via config)'}`);
1516
- log(` - Retention: ✅`);
1517
- log(` - Eviction: ✅`);
1518
- log(` - WebSocket Dashboard: ws://${HOST}:${PORT}/ws/dashboard`);
1519
- log(` - Server Instance ID: ${SERVER_INSTANCE_ID}`);
1520
- if (SERVE_WEBAPP) {
1521
- log(` - Frontend Dashboard: http://${HOST}:${PORT}/ ✅`);
1522
- }
1523
-
1524
- // Start system event synchronization and consumption (only if enabled)
1525
- if (config.SYSTEM_EVENTS.ENABLED) {
1526
- /*try { // TODO: Uncomment this when we have a way to synchronize system events
1527
- // Create temporary client for synchronization
1528
- const syncClient = createQueenClient({
1529
- baseUrl: `http://localhost:${PORT}`
1530
- });
1531
-
1532
- // Synchronize system events (catch up on missed events)
1533
- await syncSystemEvents(syncClient, systemEventManager, SERVER_INSTANCE_ID);
1534
-
1535
- // Start consuming system events
1536
- // Note: consume() returns the stop function directly
1537
- stopSystemEventConsumer = syncClient.consume({
1538
- queue: SYSTEM_QUEUE,
1539
- consumerGroup: SERVER_INSTANCE_ID,
1540
- handler: async (message) => {
1541
- await systemEventManager.processSystemEvent(message.data);
1542
- },
1543
- options: {
1544
- batch: 10,
1545
- wait: true
1546
- }
1547
- });
1548
-
1549
- log(`✅ System event consumer started`);
1550
- } catch (error) {
1551
- log(`⚠️ Failed to start system event consumer: ${error.message}`);
1552
- }*/
1553
- }
1554
-
1555
- log(`🎯 Ready to process messages with high performance!`);
1556
- } else {
1557
- log(`Failed to start server on port ${PORT}`);
1558
- process.exit(1);
1559
- }
1560
- });
1561
-
1562
- // Graceful shutdown
1563
- const shutdown = async (signal) => {
1564
- log(`\n🛑 Received ${signal}, shutting down gracefully...`);
1565
-
1566
- // Set shutdown flag immediately to prevent new operations
1567
- isShuttingDown = true;
1568
-
1569
- // Stop system event consumer
1570
- if (stopSystemEventConsumer) {
1571
- stopSystemEventConsumer();
1572
- log('✅ System event consumer stopped');
1573
- }
1574
-
1575
- // Stop background jobs
1576
- if (stopRetention) {
1577
- stopRetention();
1578
- log('✅ Retention job stopped');
1579
- }
1580
- if (stopEviction) {
1581
- stopEviction();
1582
- log('✅ Eviction job stopped');
1583
- }
1584
-
1585
- // Clear background intervals
1586
- if (leaseReclaimInterval) {
1587
- clearInterval(leaseReclaimInterval);
1588
- log('✅ Lease reclaim interval stopped');
1589
- }
1590
- if (queueDepthInterval) {
1591
- clearInterval(queueDepthInterval);
1592
- log('✅ Queue depth update interval stopped');
1593
- }
1594
- if (systemStatsInterval) {
1595
- clearInterval(systemStatsInterval);
1596
- log('✅ System stats interval stopped');
1597
- }
1598
-
1599
- // Wait a moment for any in-flight operations to complete
1600
- await new Promise(resolve => setTimeout(resolve, 100));
1601
-
1602
- const uptime = (Date.now() - startTime) / 1000;
1603
- if (messageCount > 0) {
1604
- log(`📊 Final stats: ${messageCount} messages processed in ${uptime.toFixed(1)}s`);
1605
- log(` Average: ${(messageCount / uptime).toFixed(2)} messages/second`);
1606
- }
1607
-
1608
- // Close database pool last
1609
- await pool.end();
1610
- log('✅ Database pool closed');
1611
-
1612
- process.exit(0);
1613
- };
1614
-
1615
- process.on('SIGTERM', () => shutdown('SIGTERM'));
1616
- process.on('SIGINT', () => shutdown('SIGINT'));
1617
-
1618
- // Return server instance with shutdown method
1619
- return {
1620
- port: PORT,
1621
- host: HOST,
1622
- shutdown,
1623
- app
1624
- };
1625
- }
1626
-
1627
- // Auto-start server if run directly (not imported)
1628
- if (import.meta.url === `file://${process.argv[1]}`) {
1629
- QueenServer().catch(error => {
1630
- console.error('Failed to start Queen server:', error);
1631
- process.exit(1);
1632
- });
1633
- }