queen-mq 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PROGRAMMATIC_SERVER.md +190 -0
- package/README.md +27 -0
- package/examples/batch-processing.js +2 -2
- package/examples/programmatic-server.js +58 -0
- package/examples/test-cache-multi-server.js +2 -2
- package/examples/test-connection-recovery.js +66 -0
- package/package.json +1 -1
- package/src/client/client.js +27 -1
- package/src/client/index.js +4 -3
- package/src/managers/queueManagerOptimized.js +64 -35
- package/src/routes/messages.js +8 -8
- package/src/routes/status.js +4 -4
- package/src/server.js +534 -504
- package/src/services/startupSync.js +1 -1
- package/src/test/advanced-pattern-tests.js +1 -1
- package/src/test/core-tests.js +110 -0
- package/src/test/test-new.js +3 -1
- package/src/test/test.js +1 -1
- package/src/test/window-buffer-test.js +114 -0
- package/webapp/src/components/messages/MessageFilters.vue +1 -14
- package/webapp/src/views/Analytics.vue +5 -5
- package/webapp/src/views/Messages.vue +4 -2
- package/src/client/queenClient.js +0 -518
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# Queen MQ - Programmatic Server API
|
|
2
|
+
|
|
3
|
+
As of version 0.1.2, Queen MQ can be run programmatically from your Node.js code, in addition to running it as a standalone process.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install queen-mq
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Basic Usage
|
|
12
|
+
|
|
13
|
+
### Starting the Server Programmatically
|
|
14
|
+
|
|
15
|
+
```javascript
|
|
16
|
+
import { QueenServer } from 'queen-mq';
|
|
17
|
+
|
|
18
|
+
// Start server with default configuration
|
|
19
|
+
const server = await QueenServer();
|
|
20
|
+
|
|
21
|
+
// Or with custom options
|
|
22
|
+
const server = await QueenServer({
|
|
23
|
+
port: 3000,
|
|
24
|
+
host: '127.0.0.1',
|
|
25
|
+
workerId: 'worker-1'
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
console.log(`Server running at http://${server.host}:${server.port}`);
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### Using Client and Server Together
|
|
32
|
+
|
|
33
|
+
```javascript
|
|
34
|
+
import { QueenServer, Queen } from 'queen-mq';
|
|
35
|
+
|
|
36
|
+
// Start the server
|
|
37
|
+
const server = await QueenServer({ port: 3000 });
|
|
38
|
+
|
|
39
|
+
// Create a client
|
|
40
|
+
const client = new Queen({
|
|
41
|
+
servers: [`http://localhost:${server.port}`]
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
// Use the queue
|
|
45
|
+
await client.push({
|
|
46
|
+
queue: 'my-queue',
|
|
47
|
+
items: [{ data: { message: 'Hello World' } }]
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const result = await client.pop({ queue: 'my-queue' });
|
|
51
|
+
console.log(result.messages);
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### Graceful Shutdown
|
|
55
|
+
|
|
56
|
+
```javascript
|
|
57
|
+
const server = await QueenServer();
|
|
58
|
+
|
|
59
|
+
// Shutdown programmatically
|
|
60
|
+
await server.shutdown('SIGTERM');
|
|
61
|
+
|
|
62
|
+
// Or handle process signals
|
|
63
|
+
process.on('SIGINT', async () => {
|
|
64
|
+
await server.shutdown('SIGINT');
|
|
65
|
+
});
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Server Options
|
|
69
|
+
|
|
70
|
+
The `QueenServer()` function accepts an options object:
|
|
71
|
+
|
|
72
|
+
```javascript
|
|
73
|
+
{
|
|
74
|
+
port: number, // Server port (default: from config, typically 3000)
|
|
75
|
+
host: string, // Server host (default: from config, typically '127.0.0.1')
|
|
76
|
+
workerId: string // Worker ID for clustering (default: from config)
|
|
77
|
+
}
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Return Value
|
|
81
|
+
|
|
82
|
+
The `QueenServer()` function returns a promise that resolves to a server instance with:
|
|
83
|
+
|
|
84
|
+
```javascript
|
|
85
|
+
{
|
|
86
|
+
port: number, // The port the server is listening on
|
|
87
|
+
host: string, // The host the server is bound to
|
|
88
|
+
shutdown: Function, // Async function to gracefully shutdown the server
|
|
89
|
+
app: Object // The underlying uWebSockets.js app instance
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Use Cases
|
|
94
|
+
|
|
95
|
+
### 1. Embedded Message Queue
|
|
96
|
+
|
|
97
|
+
Run Queen MQ embedded in your application instead of as a separate service:
|
|
98
|
+
|
|
99
|
+
```javascript
|
|
100
|
+
import { QueenServer, Queen } from 'queen-mq';
|
|
101
|
+
import express from 'express';
|
|
102
|
+
|
|
103
|
+
// Start Queen server
|
|
104
|
+
await QueenServer({ port: 3001 });
|
|
105
|
+
|
|
106
|
+
// Start your application server
|
|
107
|
+
const app = express();
|
|
108
|
+
const client = new Queen({ servers: ['http://localhost:3001'] });
|
|
109
|
+
|
|
110
|
+
app.post('/api/task', async (req, res) => {
|
|
111
|
+
await client.push({
|
|
112
|
+
queue: 'tasks',
|
|
113
|
+
items: [{ data: req.body }]
|
|
114
|
+
});
|
|
115
|
+
res.json({ status: 'queued' });
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
app.listen(3000);
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### 2. Testing
|
|
122
|
+
|
|
123
|
+
Easily spin up and tear down Queen servers in tests:
|
|
124
|
+
|
|
125
|
+
```javascript
|
|
126
|
+
import { QueenServer, Queen } from 'queen-mq';
|
|
127
|
+
import { describe, it, beforeAll, afterAll } from 'jest';
|
|
128
|
+
|
|
129
|
+
describe('Queue Tests', () => {
|
|
130
|
+
let server, client;
|
|
131
|
+
|
|
132
|
+
beforeAll(async () => {
|
|
133
|
+
server = await QueenServer({ port: 4000 });
|
|
134
|
+
client = new Queen({ servers: ['http://localhost:4000'] });
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
afterAll(async () => {
|
|
138
|
+
await server.shutdown('TEST');
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it('should process messages', async () => {
|
|
142
|
+
await client.push({
|
|
143
|
+
queue: 'test-queue',
|
|
144
|
+
items: [{ data: { test: true } }]
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
const result = await client.pop({ queue: 'test-queue' });
|
|
148
|
+
expect(result.messages).toHaveLength(1);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### 3. Multiple Servers
|
|
154
|
+
|
|
155
|
+
Run multiple Queen instances in the same process:
|
|
156
|
+
|
|
157
|
+
```javascript
|
|
158
|
+
import { QueenServer } from 'queen-mq';
|
|
159
|
+
|
|
160
|
+
// Start multiple servers for different purposes
|
|
161
|
+
const server1 = await QueenServer({ port: 3001, workerId: 'worker-1' });
|
|
162
|
+
const server2 = await QueenServer({ port: 3002, workerId: 'worker-2' });
|
|
163
|
+
|
|
164
|
+
console.log('Multi-server setup running');
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Traditional Usage
|
|
168
|
+
|
|
169
|
+
You can still run Queen as a standalone server:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
npm start
|
|
173
|
+
# or
|
|
174
|
+
node src/server.js
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
When `src/server.js` is run directly (not imported), it automatically starts the server with configuration from `src/config.js`.
|
|
178
|
+
|
|
179
|
+
## Requirements
|
|
180
|
+
|
|
181
|
+
- Node.js ≥ 22.0.0
|
|
182
|
+
- PostgreSQL database (configured via environment variables or config.js)
|
|
183
|
+
- Database must be initialized first (run `npm run init`)
|
|
184
|
+
|
|
185
|
+
## See Also
|
|
186
|
+
|
|
187
|
+
- [Main README](./README.md) - General Queen MQ documentation
|
|
188
|
+
- [API Documentation](./API.md) - HTTP API reference
|
|
189
|
+
- [Examples](./examples/) - More usage examples
|
|
190
|
+
|
package/README.md
CHANGED
|
@@ -134,6 +134,33 @@ npm install
|
|
|
134
134
|
node init-db.js
|
|
135
135
|
```
|
|
136
136
|
|
|
137
|
+
### Running the Server
|
|
138
|
+
|
|
139
|
+
**Option 1: Standalone Server (Traditional)**
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
npm start
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
**Option 2: Programmatic Server (New in v0.1.2)**
|
|
146
|
+
|
|
147
|
+
```javascript
|
|
148
|
+
import { QueenServer } from 'queen-mq';
|
|
149
|
+
|
|
150
|
+
// Start server with default config
|
|
151
|
+
const server = await QueenServer();
|
|
152
|
+
|
|
153
|
+
// Or with custom options
|
|
154
|
+
const server = await QueenServer({
|
|
155
|
+
port: 3000,
|
|
156
|
+
host: '127.0.0.1'
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
console.log(`Server running at http://${server.host}:${server.port}`);
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
This allows you to embed Queen MQ directly in your application, run multiple instances, or easily start/stop servers in tests. See [PROGRAMMATIC_SERVER.md](./PROGRAMMATIC_SERVER.md) for more details.
|
|
163
|
+
|
|
137
164
|
### Set Environment (Optional)
|
|
138
165
|
|
|
139
166
|
```bash
|
|
@@ -14,7 +14,7 @@ for await (const message of q.take('demo-queue', {
|
|
|
14
14
|
limit: 25, // Process 25 total
|
|
15
15
|
wait: false
|
|
16
16
|
})) {
|
|
17
|
-
console.log(`Message ${++count1}:`, message.
|
|
17
|
+
console.log(`Message ${++count1}:`, message.data);
|
|
18
18
|
await q.ack(message);
|
|
19
19
|
}
|
|
20
20
|
|
|
@@ -29,7 +29,7 @@ for await (const messages of q.takeBatch('demo-queue', {
|
|
|
29
29
|
|
|
30
30
|
// Process entire batch
|
|
31
31
|
for (const msg of messages) {
|
|
32
|
-
console.log(` - ${msg.
|
|
32
|
+
console.log(` - ${msg.data}`);
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
// Acknowledge entire batch at once
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Example: Starting Queen Server Programmatically
|
|
3
|
+
*
|
|
4
|
+
* This example shows how to import and run the Queen server
|
|
5
|
+
* directly from your code, rather than running it as a separate process.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { QueenServer, Queen } from 'queen-mq';
|
|
9
|
+
|
|
10
|
+
// Start the server programmatically
|
|
11
|
+
console.log('Starting Queen Server...');
|
|
12
|
+
|
|
13
|
+
const server = await QueenServer({
|
|
14
|
+
port: 3000,
|
|
15
|
+
host: '127.0.0.1'
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
console.log(`✅ Server started at http://${server.host}:${server.port}`);
|
|
19
|
+
|
|
20
|
+
// Now you can use the client in the same process
|
|
21
|
+
const client = new Queen({
|
|
22
|
+
servers: [`http://${server.host}:${server.port}`]
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// Create a queue
|
|
26
|
+
await client.push({
|
|
27
|
+
queue: 'example-queue',
|
|
28
|
+
items: [
|
|
29
|
+
{ data: { message: 'Hello from programmatic server!' } }
|
|
30
|
+
]
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
console.log('✅ Message pushed to queue');
|
|
34
|
+
|
|
35
|
+
// Pop a message
|
|
36
|
+
const result = await client.pop({
|
|
37
|
+
queue: 'example-queue'
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
console.log('✅ Message received:', result.messages[0]?.data);
|
|
41
|
+
|
|
42
|
+
// Acknowledge the message
|
|
43
|
+
if (result.messages.length > 0) {
|
|
44
|
+
await client.ack({
|
|
45
|
+
transactionId: result.messages[0].transactionId,
|
|
46
|
+
status: 'completed'
|
|
47
|
+
});
|
|
48
|
+
console.log('✅ Message acknowledged');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Graceful shutdown on SIGINT
|
|
52
|
+
process.on('SIGINT', async () => {
|
|
53
|
+
console.log('\nShutting down...');
|
|
54
|
+
await server.shutdown('SIGINT');
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
console.log('\n📝 Server is running. Press Ctrl+C to stop.');
|
|
58
|
+
|
|
@@ -120,7 +120,7 @@ async function testCacheConsistency() {
|
|
|
120
120
|
log(`Server 1 returned ${pop1.messages?.length || 0} messages`, servers[0]);
|
|
121
121
|
if (pop1.messages && pop1.messages.length > 0) {
|
|
122
122
|
for (const msg of pop1.messages) {
|
|
123
|
-
log(` - Message: ${JSON.stringify(msg.
|
|
123
|
+
log(` - Message: ${JSON.stringify(msg.data)}`, servers[0]);
|
|
124
124
|
await sessionClient1.ack(msg.transactionId, 'completed');
|
|
125
125
|
}
|
|
126
126
|
}
|
|
@@ -140,7 +140,7 @@ async function testCacheConsistency() {
|
|
|
140
140
|
log(`Server 2 returned ${pop2.messages?.length || 0} messages`, servers[1]);
|
|
141
141
|
if (pop2.messages && pop2.messages.length > 0) {
|
|
142
142
|
for (const msg of pop2.messages) {
|
|
143
|
-
log(` - Message: ${JSON.stringify(msg.
|
|
143
|
+
log(` - Message: ${JSON.stringify(msg.data)}`, servers[1]);
|
|
144
144
|
await sessionClient2.ack(msg.transactionId, 'completed');
|
|
145
145
|
}
|
|
146
146
|
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test connection recovery during take operations
|
|
3
|
+
* This demonstrates how the client handles server disconnections gracefully
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { Queen } from '../src/client/index.js';
|
|
7
|
+
|
|
8
|
+
async function testConnectionRecovery() {
|
|
9
|
+
const client = new Queen({
|
|
10
|
+
baseUrl: 'http://localhost:6632',
|
|
11
|
+
retryDelay: 1000, // 1 second base retry delay
|
|
12
|
+
retryAttempts: 3
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
try {
|
|
16
|
+
// Configure a test queue
|
|
17
|
+
await client.queue('test-recovery-queue');
|
|
18
|
+
console.log('Queue configured');
|
|
19
|
+
|
|
20
|
+
// Push some test messages
|
|
21
|
+
await client.push('test-recovery-queue', [
|
|
22
|
+
{ message: 'Test message 1' },
|
|
23
|
+
{ message: 'Test message 2' },
|
|
24
|
+
{ message: 'Test message 3' }
|
|
25
|
+
]);
|
|
26
|
+
console.log('Pushed 3 test messages');
|
|
27
|
+
|
|
28
|
+
console.log('\nStarting consumer...');
|
|
29
|
+
console.log('To test: Stop the server after receiving some messages');
|
|
30
|
+
console.log('Expected behavior: Client will retry with exponential backoff indefinitely (max 30s between retries)');
|
|
31
|
+
console.log(' When server comes back online, it will automatically reconnect\n');
|
|
32
|
+
|
|
33
|
+
let messageCount = 0;
|
|
34
|
+
|
|
35
|
+
// Consume messages with takeBatch
|
|
36
|
+
for await (const batch of client.takeBatch('test-recovery-queue', {
|
|
37
|
+
batch: 10,
|
|
38
|
+
wait: true,
|
|
39
|
+
timeout: 5000
|
|
40
|
+
})) {
|
|
41
|
+
console.log(`\nReceived batch of ${batch.length} messages:`);
|
|
42
|
+
|
|
43
|
+
for (const message of batch) {
|
|
44
|
+
messageCount++;
|
|
45
|
+
console.log(` ${messageCount}. ${JSON.stringify(message.payload)}`);
|
|
46
|
+
|
|
47
|
+
// Acknowledge the message
|
|
48
|
+
await client.ack(message, true);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
console.log('\nConsumer loop completed');
|
|
53
|
+
console.log(`Total messages processed: ${messageCount}`);
|
|
54
|
+
|
|
55
|
+
} catch (error) {
|
|
56
|
+
console.error('\nError in consumer:', error.message);
|
|
57
|
+
console.error('Stack:', error.stack);
|
|
58
|
+
} finally {
|
|
59
|
+
await client.close();
|
|
60
|
+
console.log('\nClient closed');
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
console.log('=== Connection Recovery Test ===\n');
|
|
65
|
+
testConnectionRecovery().catch(console.error);
|
|
66
|
+
|
package/package.json
CHANGED
package/src/client/client.js
CHANGED
|
@@ -234,6 +234,7 @@ export class Queen {
|
|
|
234
234
|
|
|
235
235
|
let totalCount = 0;
|
|
236
236
|
let consecutiveEmptyResponses = 0;
|
|
237
|
+
let consecutiveNetworkErrors = 0;
|
|
237
238
|
const maxConsecutiveEmpty = 3;
|
|
238
239
|
let lastMessageTime = idleTimeout ? Date.now() : null;
|
|
239
240
|
|
|
@@ -304,8 +305,9 @@ export class Queen {
|
|
|
304
305
|
}
|
|
305
306
|
}
|
|
306
307
|
|
|
307
|
-
// Reset
|
|
308
|
+
// Reset counters on successful fetch
|
|
308
309
|
consecutiveEmptyResponses = 0;
|
|
310
|
+
consecutiveNetworkErrors = 0;
|
|
309
311
|
|
|
310
312
|
// Update last message time if tracking idle timeout
|
|
311
313
|
if (idleTimeout) {
|
|
@@ -336,6 +338,30 @@ export class Queen {
|
|
|
336
338
|
continue;
|
|
337
339
|
}
|
|
338
340
|
|
|
341
|
+
// Check if this is a network error (connection refused, socket closed, etc.)
|
|
342
|
+
const isNetworkError = error.message?.includes('fetch failed') ||
|
|
343
|
+
error.message?.includes('ECONNREFUSED') ||
|
|
344
|
+
error.message?.includes('ECONNRESET') ||
|
|
345
|
+
error.message?.includes('closed') ||
|
|
346
|
+
error.cause?.code === 'UND_ERR_SOCKET' ||
|
|
347
|
+
error.code === 'ECONNREFUSED' ||
|
|
348
|
+
error.code === 'ECONNRESET';
|
|
349
|
+
|
|
350
|
+
if (isNetworkError) {
|
|
351
|
+
// Log the network error
|
|
352
|
+
console.warn(`Network error while polling queue: ${error.message}`);
|
|
353
|
+
|
|
354
|
+
// Retry with exponential backoff (capped at 30 seconds)
|
|
355
|
+
const retryDelay = Math.min(this.#config.retryDelay * Math.pow(2, Math.min(consecutiveNetworkErrors, 10)), 30000);
|
|
356
|
+
console.warn(`Retrying in ${retryDelay}ms... (attempt ${consecutiveNetworkErrors + 1})`);
|
|
357
|
+
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
|
358
|
+
|
|
359
|
+
consecutiveNetworkErrors++;
|
|
360
|
+
|
|
361
|
+
// Continue retrying indefinitely
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
|
|
339
365
|
// For other errors, throw
|
|
340
366
|
throw error;
|
|
341
367
|
}
|
package/src/client/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// Legacy exports (for backward compatibility)
|
|
2
|
-
export { createQueenClient, createConsumer, createProducer } from './queenClient.js';
|
|
3
1
|
export { LoadBalancingStrategy } from './utils/loadBalancer.js';
|
|
4
2
|
|
|
5
3
|
// New minimalist interface
|
|
6
|
-
export { Queen
|
|
4
|
+
export { Queen } from './client.js';
|
|
5
|
+
|
|
6
|
+
// Server
|
|
7
|
+
export { QueenServer } from '../server.js';
|
|
@@ -1029,6 +1029,11 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
|
|
|
1029
1029
|
FROM queen.queues q
|
|
1030
1030
|
JOIN queen.partitions p ON p.queue_id = q.id
|
|
1031
1031
|
WHERE q.name = $1 AND p.name = $2
|
|
1032
|
+
AND (q.window_buffer = 0 OR NOT EXISTS (
|
|
1033
|
+
SELECT 1 FROM queen.messages m
|
|
1034
|
+
WHERE m.partition_id = p.id
|
|
1035
|
+
AND m.created_at > NOW() - INTERVAL '1 second' * q.window_buffer
|
|
1036
|
+
))
|
|
1032
1037
|
`, [queue, partition]);
|
|
1033
1038
|
|
|
1034
1039
|
if (result.rows.length > 0) {
|
|
@@ -1037,7 +1042,7 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
|
|
|
1037
1042
|
}
|
|
1038
1043
|
|
|
1039
1044
|
} else if (accessMode === 'queue') {
|
|
1040
|
-
// CASE 2: Queue-level access - find unlocked partitions
|
|
1045
|
+
// CASE 2: Queue-level access - find unlocked partitions WITH unconsumed messages
|
|
1041
1046
|
const result = await client.query(`
|
|
1042
1047
|
SELECT
|
|
1043
1048
|
q.id as queue_id,
|
|
@@ -1052,13 +1057,27 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
|
|
|
1052
1057
|
p.name as partition_name
|
|
1053
1058
|
FROM queen.queues q
|
|
1054
1059
|
JOIN queen.partitions p ON p.queue_id = q.id
|
|
1060
|
+
LEFT JOIN queen.partition_consumers pc ON pc.partition_id = p.id
|
|
1061
|
+
AND pc.consumer_group = $1
|
|
1055
1062
|
WHERE q.name = $2
|
|
1056
1063
|
AND NOT EXISTS (
|
|
1057
|
-
SELECT 1 FROM queen.partition_consumers
|
|
1058
|
-
WHERE
|
|
1059
|
-
AND
|
|
1060
|
-
AND
|
|
1061
|
-
AND
|
|
1064
|
+
SELECT 1 FROM queen.partition_consumers pc2
|
|
1065
|
+
WHERE pc2.partition_id = p.id
|
|
1066
|
+
AND pc2.consumer_group = $1
|
|
1067
|
+
AND pc2.lease_expires_at IS NOT NULL
|
|
1068
|
+
AND pc2.lease_expires_at > NOW()
|
|
1069
|
+
)
|
|
1070
|
+
AND (q.window_buffer = 0 OR NOT EXISTS (
|
|
1071
|
+
SELECT 1 FROM queen.messages m
|
|
1072
|
+
WHERE m.partition_id = p.id
|
|
1073
|
+
AND m.created_at > NOW() - INTERVAL '1 second' * q.window_buffer
|
|
1074
|
+
))
|
|
1075
|
+
AND EXISTS (
|
|
1076
|
+
SELECT 1 FROM queen.messages m2
|
|
1077
|
+
WHERE m2.partition_id = p.id
|
|
1078
|
+
AND m2.id > COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid)
|
|
1079
|
+
AND (q.delayed_processing = 0 OR m2.created_at <= NOW() - INTERVAL '1 second' * q.delayed_processing)
|
|
1080
|
+
AND (q.max_wait_time_seconds = 0 OR m2.created_at > NOW() - INTERVAL '1 second' * q.max_wait_time_seconds)
|
|
1062
1081
|
)
|
|
1063
1082
|
ORDER BY RANDOM()
|
|
1064
1083
|
LIMIT $3
|
|
@@ -1071,29 +1090,23 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
|
|
|
1071
1090
|
|
|
1072
1091
|
} else {
|
|
1073
1092
|
// CASE 3: Filtered access - namespace/task
|
|
1093
|
+
// Use CTE to first filter partitions by windowBuffer, then find messages
|
|
1074
1094
|
let filterQuery = `
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
LEFT JOIN queen.partition_consumers pc ON p.id = pc.partition_id
|
|
1091
|
-
AND pc.consumer_group = $1
|
|
1092
|
-
WHERE m.id > COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid)
|
|
1093
|
-
AND (q.delayed_processing = 0 OR m.created_at <= NOW() - INTERVAL '1 second' * q.delayed_processing)
|
|
1094
|
-
AND (q.max_wait_time_seconds = 0 OR
|
|
1095
|
-
m.created_at > NOW() - INTERVAL '1 second' * q.max_wait_time_seconds)
|
|
1096
|
-
AND (q.window_buffer = 0 OR NOT EXISTS (
|
|
1095
|
+
WITH eligible_partitions AS (
|
|
1096
|
+
SELECT DISTINCT
|
|
1097
|
+
q.id as queue_id,
|
|
1098
|
+
q.name as queue_name,
|
|
1099
|
+
q.lease_time,
|
|
1100
|
+
q.retry_limit,
|
|
1101
|
+
q.delayed_processing,
|
|
1102
|
+
q.max_wait_time_seconds,
|
|
1103
|
+
q.window_buffer,
|
|
1104
|
+
q.priority,
|
|
1105
|
+
p.id as partition_id,
|
|
1106
|
+
p.name as partition_name
|
|
1107
|
+
FROM queen.queues q
|
|
1108
|
+
JOIN queen.partitions p ON p.queue_id = q.id
|
|
1109
|
+
WHERE (q.window_buffer = 0 OR NOT EXISTS (
|
|
1097
1110
|
SELECT 1 FROM queen.messages m2
|
|
1098
1111
|
WHERE m2.partition_id = p.id
|
|
1099
1112
|
AND m2.created_at > NOW() - INTERVAL '1 second' * q.window_buffer
|
|
@@ -1105,27 +1118,44 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
|
|
|
1105
1118
|
AND pc2.lease_expires_at IS NOT NULL
|
|
1106
1119
|
AND pc2.lease_expires_at > NOW()
|
|
1107
1120
|
)
|
|
1121
|
+
)
|
|
1122
|
+
SELECT
|
|
1123
|
+
ep.*,
|
|
1124
|
+
MIN(m.created_at) as oldest_message
|
|
1125
|
+
FROM queen.messages m
|
|
1126
|
+
JOIN eligible_partitions ep ON m.partition_id = ep.partition_id
|
|
1127
|
+
LEFT JOIN queen.partition_consumers pc ON ep.partition_id = pc.partition_id
|
|
1128
|
+
AND pc.consumer_group = $1
|
|
1129
|
+
WHERE m.id > COALESCE(pc.last_consumed_id, '00000000-0000-0000-0000-000000000000'::uuid)
|
|
1130
|
+
AND (ep.delayed_processing = 0 OR m.created_at <= NOW() - INTERVAL '1 second' * ep.delayed_processing)
|
|
1131
|
+
AND (ep.max_wait_time_seconds = 0 OR
|
|
1132
|
+
m.created_at > NOW() - INTERVAL '1 second' * ep.max_wait_time_seconds)
|
|
1108
1133
|
`;
|
|
1109
1134
|
|
|
1110
1135
|
const params = [actualConsumerGroup];
|
|
1111
1136
|
|
|
1137
|
+
//Build namespace/task filters for the CTE
|
|
1138
|
+
let cteFilters = '';
|
|
1112
1139
|
if (namespace) {
|
|
1113
1140
|
params.push(namespace);
|
|
1114
|
-
|
|
1141
|
+
cteFilters += ` AND q.namespace = $${params.length}`;
|
|
1115
1142
|
}
|
|
1116
1143
|
|
|
1117
1144
|
if (task) {
|
|
1118
1145
|
params.push(task);
|
|
1119
|
-
|
|
1146
|
+
cteFilters += ` AND q.task = $${params.length}`;
|
|
1120
1147
|
}
|
|
1121
1148
|
|
|
1149
|
+
// Insert CTE filters before the closing parenthesis
|
|
1150
|
+
filterQuery = filterQuery.replace(' )\n )', ` )${cteFilters}\n )`);
|
|
1151
|
+
|
|
1122
1152
|
params.push(config.QUEUE.MAX_PARTITION_CANDIDATES);
|
|
1123
1153
|
filterQuery += `
|
|
1124
|
-
GROUP BY
|
|
1125
|
-
|
|
1126
|
-
|
|
1154
|
+
GROUP BY ep.queue_id, ep.queue_name, ep.lease_time, ep.retry_limit, ep.delayed_processing,
|
|
1155
|
+
ep.max_wait_time_seconds, ep.window_buffer, ep.priority,
|
|
1156
|
+
ep.partition_id, ep.partition_name
|
|
1127
1157
|
HAVING COUNT(*) > 0
|
|
1128
|
-
ORDER BY
|
|
1158
|
+
ORDER BY ep.priority DESC, MIN(m.created_at) ASC, RANDOM()
|
|
1129
1159
|
LIMIT $${params.length}
|
|
1130
1160
|
`;
|
|
1131
1161
|
|
|
@@ -1344,7 +1374,6 @@ export const createOptimizedQueueManager = (pool, resourceCache, eventManager, s
|
|
|
1344
1374
|
queue: msg.queue_name,
|
|
1345
1375
|
partition: msg.partition_name,
|
|
1346
1376
|
data: payload,
|
|
1347
|
-
payload: payload,
|
|
1348
1377
|
retryCount: msg.retry_count || 0,
|
|
1349
1378
|
priority: msg.priority || 0,
|
|
1350
1379
|
createdAt: msg.created_at,
|
package/src/routes/messages.js
CHANGED
|
@@ -25,11 +25,11 @@ export const createMessagesRoutes = (pool, queueManager) => {
|
|
|
25
25
|
pc.last_consumed_id,
|
|
26
26
|
CASE
|
|
27
27
|
WHEN dlq.message_id IS NOT NULL THEN 'dead_letter'
|
|
28
|
-
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
29
28
|
WHEN pc.last_consumed_created_at IS NOT NULL AND (
|
|
30
29
|
m.created_at < pc.last_consumed_created_at OR
|
|
31
|
-
(m.created_at = pc.last_consumed_created_at AND m.id <= pc.last_consumed_id)
|
|
30
|
+
(DATE_TRUNC('milliseconds', m.created_at) = DATE_TRUNC('milliseconds', pc.last_consumed_created_at) AND m.id <= pc.last_consumed_id)
|
|
32
31
|
) THEN 'completed'
|
|
32
|
+
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
33
33
|
ELSE 'pending'
|
|
34
34
|
END as message_status
|
|
35
35
|
FROM queen.messages m
|
|
@@ -64,11 +64,11 @@ export const createMessagesRoutes = (pool, queueManager) => {
|
|
|
64
64
|
params.push(status);
|
|
65
65
|
query += ` AND CASE
|
|
66
66
|
WHEN dlq.message_id IS NOT NULL THEN 'dead_letter'
|
|
67
|
-
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
68
67
|
WHEN pc.last_consumed_created_at IS NOT NULL AND (
|
|
69
68
|
m.created_at < pc.last_consumed_created_at OR
|
|
70
|
-
(m.created_at = pc.last_consumed_created_at AND m.id <= pc.last_consumed_id)
|
|
69
|
+
(DATE_TRUNC('milliseconds', m.created_at) = DATE_TRUNC('milliseconds', pc.last_consumed_created_at) AND m.id <= pc.last_consumed_id)
|
|
71
70
|
) THEN 'completed'
|
|
71
|
+
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
72
72
|
ELSE 'pending'
|
|
73
73
|
END = $${++paramCount}`;
|
|
74
74
|
}
|
|
@@ -127,11 +127,11 @@ export const createMessagesRoutes = (pool, queueManager) => {
|
|
|
127
127
|
dlq.retry_count,
|
|
128
128
|
CASE
|
|
129
129
|
WHEN dlq.message_id IS NOT NULL THEN 'dead_letter'
|
|
130
|
-
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
131
130
|
WHEN pc.last_consumed_created_at IS NOT NULL AND (
|
|
132
131
|
m.created_at < pc.last_consumed_created_at OR
|
|
133
|
-
(m.created_at = pc.last_consumed_created_at AND m.id <= pc.last_consumed_id)
|
|
132
|
+
(DATE_TRUNC('milliseconds', m.created_at) = DATE_TRUNC('milliseconds', pc.last_consumed_created_at) AND m.id <= pc.last_consumed_id)
|
|
134
133
|
) THEN 'completed'
|
|
134
|
+
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
135
135
|
ELSE 'pending'
|
|
136
136
|
END as message_status
|
|
137
137
|
FROM queen.messages m
|
|
@@ -284,11 +284,11 @@ export const createMessagesRoutes = (pool, queueManager) => {
|
|
|
284
284
|
pc.last_consumed_id,
|
|
285
285
|
CASE
|
|
286
286
|
WHEN dlq.message_id IS NOT NULL THEN 'dead_letter'
|
|
287
|
-
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
288
287
|
WHEN pc.last_consumed_created_at IS NOT NULL AND (
|
|
289
288
|
m.created_at < pc.last_consumed_created_at OR
|
|
290
|
-
(m.created_at = pc.last_consumed_created_at AND m.id <= pc.last_consumed_id)
|
|
289
|
+
(DATE_TRUNC('milliseconds', m.created_at) = DATE_TRUNC('milliseconds', pc.last_consumed_created_at) AND m.id <= pc.last_consumed_id)
|
|
291
290
|
) THEN 'completed'
|
|
291
|
+
WHEN pc.lease_expires_at IS NOT NULL AND pc.lease_expires_at > NOW() THEN 'processing'
|
|
292
292
|
ELSE 'pending'
|
|
293
293
|
END as message_status
|
|
294
294
|
FROM queen.messages m
|