queen-mq 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API.md +1226 -0
- package/AUTH.md +2044 -0
- package/LICENSE.md +202 -0
- package/README.md +276 -53
- package/WEBAPP.md +1889 -0
- package/assets/dashboard-01.png +0 -0
- package/assets/queen-logo-blue.svg +210 -0
- package/assets/queen-logo-cyan.svg +210 -0
- package/assets/queen-logo-indigo.svg +210 -0
- package/assets/queen-logo-orange.svg +210 -0
- package/assets/queen-logo-pink.svg +210 -0
- package/assets/queen-logo-purple.svg +210 -0
- package/assets/queen-logo-rose.svg +239 -0
- package/assets/queen-logo.svg +263 -0
- package/examples/batch-processing.js +58 -0
- package/examples/test-dashboard-api.js +200 -0
- package/package.json +4 -2
- package/server.log +1 -0
- package/src/benchmark/consumer.js +207 -0
- package/src/benchmark/consumer_multi.js +216 -0
- package/src/benchmark/producer.js +75 -0
- package/src/benchmark/producer_multi.js +115 -0
- package/src/client/client.js +192 -16
- package/src/client/queenClient.js +5 -0
- package/src/cluster-server.js +242 -0
- package/src/config.js +19 -5
- package/src/database/connection.js +42 -16
- package/src/database/poolManager.js +7 -0
- package/src/database/schema-v2.sql +194 -130
- package/src/managers/queueManagerOptimized.js +823 -933
- package/src/managers/systemEventManager.js +8 -3
- package/src/routes/messages.js +127 -57
- package/src/routes/pop.js +27 -43
- package/src/routes/resources.js +61 -27
- package/src/routes/status.js +1037 -0
- package/src/server.js +308 -272
- package/src/services/evictionService.js +57 -28
- package/src/services/retentionService.js +44 -11
- package/src/test/advanced-pattern-tests.js +5 -5
- package/src/test/bus-mode-tests.js +24 -11
- package/src/test/edge-case-tests.js +12 -5
- package/src/test/enterprise-tests.js +48 -15
- package/src/test/utils.js +1 -1
- package/src/utils/streaming.js +231 -0
- package/src/utils/uuid.js +2 -2
- package/src/websocket/wsServer.js +10 -3
- package/test-keepalive-v2.sh +22 -0
- package/webapp/COLOR_GUIDE.md +118 -0
- package/webapp/README.md +143 -0
- package/webapp/index.html +14 -0
- package/webapp/package-lock.json +3184 -0
- package/webapp/package.json +25 -0
- package/webapp/postcss.config.js +7 -0
- package/webapp/public/assets/queen-logo-blue.svg +210 -0
- package/webapp/public/assets/queen-logo-cyan.svg +210 -0
- package/webapp/public/assets/queen-logo-indigo.svg +210 -0
- package/webapp/public/assets/queen-logo-orange.svg +210 -0
- package/webapp/public/assets/queen-logo-pink.svg +210 -0
- package/webapp/public/assets/queen-logo-purple.svg +210 -0
- package/webapp/public/assets/queen-logo-rose.svg +239 -0
- package/webapp/public/assets/queen-logo.svg +263 -0
- package/webapp/src/App.vue +19 -0
- package/webapp/src/api/analytics.js +10 -0
- package/webapp/src/api/client.js +29 -0
- package/webapp/src/api/consumers.js +52 -0
- package/webapp/src/api/health.js +7 -0
- package/webapp/src/api/messages.js +26 -0
- package/webapp/src/api/queues.js +14 -0
- package/webapp/src/api/resources.js +8 -0
- package/webapp/src/assets/styles/main.css +357 -0
- package/webapp/src/components/analytics/AnalyticsFilters.vue +87 -0
- package/webapp/src/components/analytics/AnalyticsMetrics.vue +57 -0
- package/webapp/src/components/analytics/MessageDistributionChart.vue +111 -0
- package/webapp/src/components/analytics/MessageFlowChart.vue +173 -0
- package/webapp/src/components/analytics/TimeRangeSelector.vue +27 -0
- package/webapp/src/components/analytics/TopQueuesChart.vue +132 -0
- package/webapp/src/components/common/ConfirmDialog.vue +56 -0
- package/webapp/src/components/common/LoadingSpinner.vue +6 -0
- package/webapp/src/components/common/MetricCard.vue +43 -0
- package/webapp/src/components/common/StatusBadge.vue +45 -0
- package/webapp/src/components/dashboard/MessageStatusCard.vue +50 -0
- package/webapp/src/components/dashboard/PerformanceCard.vue +38 -0
- package/webapp/src/components/dashboard/ThroughputChart.vue +182 -0
- package/webapp/src/components/dashboard/TopQueuesTable.vue +53 -0
- package/webapp/src/components/layout/AppLayout.vue +110 -0
- package/webapp/src/components/layout/AppSidebar.vue +304 -0
- package/webapp/src/components/messages/MessageDetailPanel.vue +242 -0
- package/webapp/src/components/messages/MessageFilters.vue +114 -0
- package/webapp/src/components/queue-detail/PartitionList.vue +79 -0
- package/webapp/src/components/queue-detail/PushMessageModal.vue +175 -0
- package/webapp/src/components/queue-detail/QueueConfig.vue +63 -0
- package/webapp/src/components/queue-detail/QueueDetailHeader.vue +53 -0
- package/webapp/src/components/queue-detail/RecentMessages.vue +76 -0
- package/webapp/src/components/queues/CreateQueueModal.vue +193 -0
- package/webapp/src/components/queues/QueueFilters.vue +90 -0
- package/webapp/src/composables/useApi.js +34 -0
- package/webapp/src/composables/useTheme.js +36 -0
- package/webapp/src/main.js +11 -0
- package/webapp/src/router/index.js +42 -0
- package/webapp/src/utils/colors.js +96 -0
- package/webapp/src/utils/formatters.js +49 -0
- package/webapp/src/views/Analytics.vue +377 -0
- package/webapp/src/views/ConsumerGroups.vue +433 -0
- package/webapp/src/views/Dashboard.vue +418 -0
- package/webapp/src/views/Messages.vue +361 -0
- package/webapp/src/views/QueueDetail.vue +582 -0
- package/webapp/src/views/Queues.vue +496 -0
- package/webapp/tailwind.config.js +25 -0
- package/webapp/vite.config.js +10 -0
- package/dashboard/.vscode/extensions.json +0 -3
- package/dashboard/README.md +0 -5
- package/dashboard/index.html +0 -14
- package/dashboard/package-lock.json +0 -1458
- package/dashboard/package.json +0 -25
- package/dashboard/public/vite.svg +0 -1
- package/dashboard/src/App.vue +0 -29
- package/dashboard/src/assets/styles/main.css +0 -908
- package/dashboard/src/assets/vue.svg +0 -1
- package/dashboard/src/components/cards/MetricCard.vue +0 -298
- package/dashboard/src/components/charts/QueueDepthChart.vue +0 -276
- package/dashboard/src/components/charts/QueueLagChart.vue +0 -436
- package/dashboard/src/components/charts/ThroughputChart.vue +0 -302
- package/dashboard/src/components/common/ActivityFeed.vue +0 -251
- package/dashboard/src/components/layout/AppHeader.vue +0 -208
- package/dashboard/src/components/layout/AppLayout.vue +0 -88
- package/dashboard/src/components/layout/AppSidebar.vue +0 -261
- package/dashboard/src/main.js +0 -44
- package/dashboard/src/router.js +0 -54
- package/dashboard/src/services/api.js +0 -187
- package/dashboard/src/services/websocket.js +0 -167
- package/dashboard/src/utils/constants.js +0 -56
- package/dashboard/src/utils/helpers.js +0 -118
- package/dashboard/src/views/Analytics.vue +0 -912
- package/dashboard/src/views/Dashboard.vue +0 -906
- package/dashboard/src/views/Messages.vue +0 -437
- package/dashboard/src/views/QueueDetail.vue +0 -501
- package/dashboard/src/views/Queues.vue +0 -333
- package/dashboard/vite.config.js +0 -30
- package/src/routes/analytics.js +0 -812
package/LICENSE.md
CHANGED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright 2023 Helium S.r.l
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -9,9 +9,11 @@
|
|
|
9
9
|
|
|
10
10
|
[Quick Start](#-quick-start) • [Client Examples](#-client-examples) • [Server Setup](#-server-setup) • [Core Concepts](#-core-concepts) • [API Reference](#-http-api-reference) • [Dashboard](#-dashboard)
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
<p align="center">
|
|
13
|
+
<img src="assets/queen-logo.svg" alt="Queen Logo" width="120" />
|
|
14
|
+
</p>
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
</div>
|
|
15
17
|
|
|
16
18
|
---
|
|
17
19
|
|
|
@@ -22,12 +24,14 @@
|
|
|
22
24
|
### Why Queen?
|
|
23
25
|
|
|
24
26
|
**🚀 Developer-First API**
|
|
25
|
-
- **4 methods, infinite patterns**: `queue()`, `push()`, `take()`, `ack()`
|
|
27
|
+
- **4 core methods, infinite patterns**: `queue()`, `push()`, `take()`, `ack()`
|
|
26
28
|
- **Async iteration**: Process messages with familiar `for await` syntax
|
|
29
|
+
- **Batch processing**: Use `takeBatch()` for 250k+ msg/sec throughput on millions of messages
|
|
27
30
|
- **Smart addressing**: `orders/urgent@workers` - queue, partition, and consumer group in one
|
|
28
31
|
|
|
29
32
|
**⚡ Production-Ready Performance**
|
|
30
|
-
- **
|
|
33
|
+
- **100,000+ msg/sec** throughput with cursor-based consumption
|
|
34
|
+
- **Constant-time batch operations** - O(batch_size) regardless of queue depth
|
|
31
35
|
- **Long polling** for event-driven, real-time message delivery
|
|
32
36
|
- **Partition locking** prevents duplicate processing across consumers
|
|
33
37
|
- **Connection pooling** and optimized batch operations
|
|
@@ -49,6 +53,7 @@
|
|
|
49
53
|
- **Rich Analytics**: Throughput, lag, queue depth metrics
|
|
50
54
|
- **Message Browser**: Search, inspect, and retry messages
|
|
51
55
|
- **System Health**: Database, memory, and performance metrics
|
|
56
|
+
- **Cursor Tracking**: Monitor consumption progress per consumer group
|
|
52
57
|
|
|
53
58
|
### Use Cases
|
|
54
59
|
|
|
@@ -67,6 +72,7 @@
|
|
|
67
72
|
- [Client Examples](#-client-examples)
|
|
68
73
|
- [Server Setup](#-server-setup)
|
|
69
74
|
- [Core Concepts](#-core-concepts)
|
|
75
|
+
- [Cursor-Based Consumption Strategy](#-cursor-based-consumption-strategy)
|
|
70
76
|
- [HTTP API Reference](#-http-api-reference)
|
|
71
77
|
- [Dashboard](#-dashboard)
|
|
72
78
|
- [Configuration](#-configuration)
|
|
@@ -252,6 +258,26 @@ for await (const message of client.take('orders/urgent')) {
|
|
|
252
258
|
await processUrgentOrder(message.data);
|
|
253
259
|
await client.ack(message);
|
|
254
260
|
}
|
|
261
|
+
|
|
262
|
+
// Use takeBatch to get arrays of messages (higher throughput)
|
|
263
|
+
for await (const messages of client.takeBatch('orders', {
|
|
264
|
+
batch: 1000, // Fetch 1000 at a time
|
|
265
|
+
wait: true
|
|
266
|
+
})) {
|
|
267
|
+
// messages is an array of up to 1000 messages
|
|
268
|
+
console.log(`Processing batch of ${messages.length} messages`);
|
|
269
|
+
|
|
270
|
+
try {
|
|
271
|
+
// Process entire batch
|
|
272
|
+
await processBatch(messages.map(m => m.data));
|
|
273
|
+
|
|
274
|
+
// Acknowledge entire batch at once (efficient!)
|
|
275
|
+
await client.ack(messages); // Pass array for batch ack
|
|
276
|
+
} catch (error) {
|
|
277
|
+
// Mark entire batch as failed
|
|
278
|
+
await client.ack(messages, false, { error: error.message });
|
|
279
|
+
}
|
|
280
|
+
}
|
|
255
281
|
```
|
|
256
282
|
|
|
257
283
|
#### 4. Acknowledge Messages
|
|
@@ -313,7 +339,7 @@ for await (const message of client.take('tasks', {
|
|
|
313
339
|
#### Batch Processing
|
|
314
340
|
|
|
315
341
|
```javascript
|
|
316
|
-
//
|
|
342
|
+
// Method 1: Manual batching with take()
|
|
317
343
|
const batch = [];
|
|
318
344
|
for await (const message of client.take('analytics', { batch: 100 })) {
|
|
319
345
|
batch.push(message);
|
|
@@ -328,6 +354,15 @@ for await (const message of client.take('analytics', { batch: 100 })) {
|
|
|
328
354
|
batch.length = 0;
|
|
329
355
|
}
|
|
330
356
|
}
|
|
357
|
+
|
|
358
|
+
// Method 2: Direct batch processing with takeBatch() (RECOMMENDED)
|
|
359
|
+
for await (const messages of client.takeBatch('analytics', { batch: 1000 })) {
|
|
360
|
+
// messages is already an array!
|
|
361
|
+
await processBatch(messages.map(m => m.data));
|
|
362
|
+
|
|
363
|
+
// Single batch acknowledgment (much faster!)
|
|
364
|
+
await client.ack(messages);
|
|
365
|
+
}
|
|
331
366
|
```
|
|
332
367
|
|
|
333
368
|
#### Parallel Processing with Partitions
|
|
@@ -881,6 +916,190 @@ await client.queue('time-sensitive', {
|
|
|
881
916
|
|
|
882
917
|
---
|
|
883
918
|
|
|
919
|
+
## 🚀 Cursor-Based Consumption Strategy
|
|
920
|
+
|
|
921
|
+
Queen uses a **cursor-based consumption model** for optimal performance at scale, providing O(batch_size) constant-time operations regardless of queue depth.
|
|
922
|
+
|
|
923
|
+
### How It Works
|
|
924
|
+
|
|
925
|
+
Traditional message queues scan through all messages to find pending ones, leading to performance degradation as messages accumulate. Queen's cursor-based approach maintains a position marker (cursor) for each consumer, allowing direct access to the next batch of messages.
|
|
926
|
+
|
|
927
|
+
**Partition Cursors:**
|
|
928
|
+
|
|
929
|
+
Each partition maintains a cursor position per consumer group:
|
|
930
|
+
- `last_consumed_created_at`: Timestamp of last consumed message
|
|
931
|
+
- `last_consumed_id`: UUID of last consumed message (tie-breaker for same timestamp)
|
|
932
|
+
- `total_messages_consumed`: Running count of consumed messages
|
|
933
|
+
|
|
934
|
+
The cursor always moves **forward** in time, ensuring strict FIFO ordering.
|
|
935
|
+
|
|
936
|
+
### The takeBatch Method
|
|
937
|
+
|
|
938
|
+
Queen provides two consumption methods:
|
|
939
|
+
|
|
940
|
+
**1. `take()` - Individual message iterator:**
|
|
941
|
+
```javascript
|
|
942
|
+
// Processes messages one at a time
|
|
943
|
+
for await (const message of client.take('orders', { batch: 1000 })) {
|
|
944
|
+
await processOrder(message.data);
|
|
945
|
+
await client.ack(message);
|
|
946
|
+
}
|
|
947
|
+
```
|
|
948
|
+
|
|
949
|
+
**2. `takeBatch()` - Array iterator (HIGH PERFORMANCE):**
|
|
950
|
+
```javascript
|
|
951
|
+
// Yields arrays of messages - achieves 100k+ msg/s throughput
|
|
952
|
+
for await (const messages of client.takeBatch('orders', { batch: 1000 })) {
|
|
953
|
+
// messages is an array of up to 1000 message objects
|
|
954
|
+
await processBatch(messages.map(m => m.data));
|
|
955
|
+
|
|
956
|
+
// Batch acknowledge - single DB transaction for all messages
|
|
957
|
+
await client.ack(messages);
|
|
958
|
+
}
|
|
959
|
+
```
|
|
960
|
+
|
|
961
|
+
**Under the hood**, both methods use cursor-based batch retrieval:
|
|
962
|
+
|
|
963
|
+
```sql
|
|
964
|
+
-- Cursor-based query (simplified)
|
|
965
|
+
SELECT * FROM messages
|
|
966
|
+
WHERE partition_id = $1
|
|
967
|
+
AND id > $2::uuid -- Start after last cursor position
|
|
968
|
+
ORDER BY created_at ASC, id ASC
|
|
969
|
+
LIMIT $3 -- Batch size
|
|
970
|
+
FOR UPDATE SKIP LOCKED
|
|
971
|
+
```
|
|
972
|
+
|
|
973
|
+
**Key characteristics:**
|
|
974
|
+
1. **Constant-time**: Performance stays consistent whether you've consumed 0% or 99% of messages
|
|
975
|
+
2. **FIFO guarantee**: Messages always returned in creation order
|
|
976
|
+
3. **Lock-free scanning**: `SKIP LOCKED` prevents contention between consumers
|
|
977
|
+
4. **Efficient**: No table scans - direct cursor-based access using UUIDv7 (time-ordered)
|
|
978
|
+
|
|
979
|
+
**Performance tip:** Use `takeBatch()` with large batch sizes (1,000-10,000) for maximum throughput. The server fetches messages in batches regardless, but `takeBatch()` gives you the array directly, allowing bulk processing and batch acknowledgment in a single operation.
|
|
980
|
+
|
|
981
|
+
### Batch Acknowledgment Semantics
|
|
982
|
+
|
|
983
|
+
Queen handles batch acknowledgments intelligently:
|
|
984
|
+
|
|
985
|
+
**Partial Success** (some messages succeed, some fail):
|
|
986
|
+
```javascript
|
|
987
|
+
// Batch: 10,000 messages
|
|
988
|
+
// Success: 9,999 messages
|
|
989
|
+
// Failed: 1 message
|
|
990
|
+
|
|
991
|
+
// Behavior:
|
|
992
|
+
// ✅ Cursor advances past all 10,000 messages
|
|
993
|
+
// ✅ Failed message moved to Dead Letter Queue
|
|
994
|
+
// ✅ Next take() starts from message 10,001
|
|
995
|
+
// ✅ FIFO maintained, no redelivery of successful messages
|
|
996
|
+
```
|
|
997
|
+
|
|
998
|
+
**Total Batch Failure** (all messages fail):
|
|
999
|
+
```javascript
|
|
1000
|
+
// Batch: 10,000 messages
|
|
1001
|
+
// Success: 0 messages
|
|
1002
|
+
// Failed: 10,000 messages
|
|
1003
|
+
|
|
1004
|
+
// Behavior:
|
|
1005
|
+
// ❌ Cursor DOES NOT advance
|
|
1006
|
+
// ❌ Messages NOT moved to DLQ
|
|
1007
|
+
// ✅ Lease released
|
|
1008
|
+
// ✅ Next take() gets SAME batch (retry)
|
|
1009
|
+
// ✅ Allows recovery from transient failures
|
|
1010
|
+
```
|
|
1011
|
+
|
|
1012
|
+
This design handles transient failures (network issues, service outages) gracefully while preventing poison messages from blocking the queue.
|
|
1013
|
+
|
|
1014
|
+
### Performance Comparison
|
|
1015
|
+
|
|
1016
|
+
| Operation | Traditional Approach | Cursor Approach | Improvement |
|
|
1017
|
+
|-----------|---------------------|-----------------|-------------|
|
|
1018
|
+
| Pop @ 0% consumed | O(partition_size) | O(batch_size) | Same |
|
|
1019
|
+
| Pop @ 50% consumed | O(partition_size) | O(batch_size) | **10-100x faster** |
|
|
1020
|
+
| Pop @ 99% consumed | O(partition_size) | O(batch_size) | **100-1000x faster** |
|
|
1021
|
+
|
|
1022
|
+
**Real-world benchmark** (1M messages):
|
|
1023
|
+
```
|
|
1024
|
+
Traditional:
|
|
1025
|
+
Early batches: 300ms per pop
|
|
1026
|
+
Late batches: 3500ms per pop (10x degradation)
|
|
1027
|
+
|
|
1028
|
+
Cursor-based:
|
|
1029
|
+
Early batches: 150ms per pop
|
|
1030
|
+
Late batches: 200ms per pop (constant!)
|
|
1031
|
+
```
|
|
1032
|
+
|
|
1033
|
+
### Dead Letter Queue
|
|
1034
|
+
|
|
1035
|
+
Individual message failures are moved to the Dead Letter Queue for inspection and manual intervention:
|
|
1036
|
+
|
|
1037
|
+
```javascript
|
|
1038
|
+
// Monitor DLQ
|
|
1039
|
+
const response = await fetch('http://localhost:6632/api/v1/analytics/dlq');
|
|
1040
|
+
const dlqMessages = await response.json();
|
|
1041
|
+
|
|
1042
|
+
// Inspect failed messages
|
|
1043
|
+
for (const msg of dlqMessages) {
|
|
1044
|
+
console.log(`Failed: ${msg.error_message}`);
|
|
1045
|
+
|
|
1046
|
+
// After fixing issue, can re-push if needed
|
|
1047
|
+
await client.push(msg.queue, fixedPayload);
|
|
1048
|
+
}
|
|
1049
|
+
```
|
|
1050
|
+
|
|
1051
|
+
**DLQ Query:**
|
|
1052
|
+
```sql
|
|
1053
|
+
SELECT * FROM queen.dead_letter_queue
|
|
1054
|
+
WHERE consumer_group = 'my-group'
|
|
1055
|
+
ORDER BY failed_at DESC
|
|
1056
|
+
LIMIT 100;
|
|
1057
|
+
```
|
|
1058
|
+
|
|
1059
|
+
### Batch Size Guidelines
|
|
1060
|
+
|
|
1061
|
+
Choose batch sizes based on your workload:
|
|
1062
|
+
|
|
1063
|
+
**Smaller batches (100-1,000):**
|
|
1064
|
+
- ✅ Faster individual batch processing
|
|
1065
|
+
- ✅ Less impact if entire batch fails
|
|
1066
|
+
- ✅ Lower memory footprint
|
|
1067
|
+
- ❌ More network round-trips
|
|
1068
|
+
|
|
1069
|
+
**Larger batches (5,000-10,000):**
|
|
1070
|
+
- ✅ Higher throughput (100,000+ msg/sec achievable)
|
|
1071
|
+
- ✅ Fewer network round-trips
|
|
1072
|
+
- ✅ Better database efficiency
|
|
1073
|
+
- ❌ More messages retry if entire batch fails
|
|
1074
|
+
- ❌ Higher memory usage
|
|
1075
|
+
|
|
1076
|
+
**Recommendation:** Start with 1,000-2,000 for balanced performance. Increase to 5,000-10,000 for maximum throughput with reliable processing.
|
|
1077
|
+
|
|
1078
|
+
### Monitoring Cursor Progress
|
|
1079
|
+
|
|
1080
|
+
Track consumption progress via SQL:
|
|
1081
|
+
|
|
1082
|
+
```sql
|
|
1083
|
+
-- View cursor positions
|
|
1084
|
+
SELECT
|
|
1085
|
+
p.name as partition,
|
|
1086
|
+
pc.consumer_group,
|
|
1087
|
+
pc.total_messages_consumed,
|
|
1088
|
+
pc.total_batches_consumed,
|
|
1089
|
+
pc.last_consumed_at,
|
|
1090
|
+
EXTRACT(EPOCH FROM (NOW() - pc.last_consumed_at)) as seconds_since_last_consume
|
|
1091
|
+
FROM queen.partition_cursors pc
|
|
1092
|
+
JOIN queen.partitions p ON p.id = pc.partition_id
|
|
1093
|
+
ORDER BY pc.last_consumed_at DESC;
|
|
1094
|
+
|
|
1095
|
+
-- Monitor DLQ
|
|
1096
|
+
SELECT COUNT(*) as failed_count, consumer_group
|
|
1097
|
+
FROM queen.dead_letter_queue
|
|
1098
|
+
GROUP BY consumer_group;
|
|
1099
|
+
```
|
|
1100
|
+
|
|
1101
|
+
---
|
|
1102
|
+
|
|
884
1103
|
## 🔌 HTTP API Reference
|
|
885
1104
|
|
|
886
1105
|
Base URL: `http://localhost:6632/api/v1`
|
|
@@ -1067,6 +1286,12 @@ POST /api/v1/messages/{transactionId}/dlq
|
|
|
1067
1286
|
DELETE /api/v1/queues/{queue}/clear
|
|
1068
1287
|
```
|
|
1069
1288
|
|
|
1289
|
+
**Delete queue:**
|
|
1290
|
+
```
|
|
1291
|
+
DELETE /api/v1/resources/queues/{queue}
|
|
1292
|
+
```
|
|
1293
|
+
_Note: Deletes the queue and all its partitions, messages, and related data._
|
|
1294
|
+
|
|
1070
1295
|
### System Health
|
|
1071
1296
|
|
|
1072
1297
|
**Health check:**
|
|
@@ -1108,10 +1333,12 @@ See [API.md](API.md) for complete API documentation.
|
|
|
1108
1333
|
|
|
1109
1334
|
Queen includes a comprehensive web dashboard for monitoring and management.
|
|
1110
1335
|
|
|
1336
|
+
[Dashboard](/assets/dashboard-01.png)
|
|
1337
|
+
|
|
1111
1338
|
### Access
|
|
1112
1339
|
|
|
1113
1340
|
1. Start the server: `npm start`
|
|
1114
|
-
2. Open browser: `http://localhost:
|
|
1341
|
+
2. Open browser: `http://localhost:4000`
|
|
1115
1342
|
3. WebSocket connection provides real-time updates
|
|
1116
1343
|
|
|
1117
1344
|
### Features
|
|
@@ -1496,7 +1723,7 @@ Promise.all([
|
|
|
1496
1723
|
await emitEvents();
|
|
1497
1724
|
```
|
|
1498
1725
|
|
|
1499
|
-
### Example 4: Batch Processing
|
|
1726
|
+
### Example 4: Batch Processing (High Throughput)
|
|
1500
1727
|
|
|
1501
1728
|
```javascript
|
|
1502
1729
|
import { Queen } from 'queen-mq';
|
|
@@ -1515,7 +1742,7 @@ await client.queue('data-processing', {
|
|
|
1515
1742
|
// Producer: Send data
|
|
1516
1743
|
async function sendData() {
|
|
1517
1744
|
const records = [];
|
|
1518
|
-
for (let i = 0; i <
|
|
1745
|
+
for (let i = 0; i < 100000; i++) {
|
|
1519
1746
|
records.push({ id: i, value: Math.random() });
|
|
1520
1747
|
}
|
|
1521
1748
|
|
|
@@ -1523,45 +1750,34 @@ async function sendData() {
|
|
|
1523
1750
|
await client.push('data-processing/analytics', records);
|
|
1524
1751
|
}
|
|
1525
1752
|
|
|
1526
|
-
// Consumer:
|
|
1753
|
+
// Consumer: HIGH PERFORMANCE batch processor using takeBatch()
|
|
1527
1754
|
async function batchProcessor() {
|
|
1528
|
-
const BATCH_SIZE =
|
|
1529
|
-
const batch = [];
|
|
1755
|
+
const BATCH_SIZE = 5000; // Large batches for 100k+ msg/s throughput
|
|
1530
1756
|
|
|
1531
|
-
|
|
1757
|
+
// takeBatch() yields arrays directly - no manual batching needed!
|
|
1758
|
+
for await (const messages of client.takeBatch('data-processing/analytics', {
|
|
1532
1759
|
batch: BATCH_SIZE,
|
|
1533
1760
|
wait: true,
|
|
1534
1761
|
timeout: 30000
|
|
1535
1762
|
})) {
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
console.log(`✓ Batch complete`);
|
|
1555
|
-
batch.length = 0;
|
|
1556
|
-
} catch (error) {
|
|
1557
|
-
console.error('Batch processing failed:', error);
|
|
1558
|
-
|
|
1559
|
-
// Mark all as failed
|
|
1560
|
-
for (const msg of batch) {
|
|
1561
|
-
await client.ack(msg, false);
|
|
1562
|
-
}
|
|
1563
|
-
batch.length = 0;
|
|
1564
|
-
}
|
|
1763
|
+
try {
|
|
1764
|
+
console.log(`Processing batch of ${messages.length} records`);
|
|
1765
|
+
|
|
1766
|
+
// Extract data
|
|
1767
|
+
const records = messages.map(m => m.data);
|
|
1768
|
+
|
|
1769
|
+
// Bulk process (single DB operation)
|
|
1770
|
+
await bulkInsertToDatabase(records);
|
|
1771
|
+
|
|
1772
|
+
// Batch acknowledge (single DB transaction!)
|
|
1773
|
+
await client.ack(messages);
|
|
1774
|
+
|
|
1775
|
+
console.log(`✓ Batch complete in single transaction`);
|
|
1776
|
+
} catch (error) {
|
|
1777
|
+
console.error('Batch processing failed:', error);
|
|
1778
|
+
|
|
1779
|
+
// Mark entire batch as failed (single transaction)
|
|
1780
|
+
await client.ack(messages, false, { error: error.message });
|
|
1565
1781
|
}
|
|
1566
1782
|
}
|
|
1567
1783
|
}
|
|
@@ -1569,6 +1785,12 @@ async function batchProcessor() {
|
|
|
1569
1785
|
// Run
|
|
1570
1786
|
await sendData();
|
|
1571
1787
|
await batchProcessor();
|
|
1788
|
+
|
|
1789
|
+
// Performance characteristics:
|
|
1790
|
+
// - Batch size 5000: ~100,000 messages/second
|
|
1791
|
+
// - Single DB transaction per batch (fetch + ack)
|
|
1792
|
+
// - Constant memory usage
|
|
1793
|
+
// - No performance degradation as queue grows
|
|
1572
1794
|
```
|
|
1573
1795
|
|
|
1574
1796
|
### Example 5: Scheduled Jobs
|
|
@@ -1886,33 +2108,34 @@ Apache License 2.0 - see [LICENSE.md](LICENSE.md) for details.
|
|
|
1886
2108
|
|
|
1887
2109
|
## 📈 Performance
|
|
1888
2110
|
|
|
1889
|
-
**Benchmarks** (PostgreSQL 16, Node.js 22):
|
|
1890
|
-
- **Throughput**:
|
|
2111
|
+
**Benchmarks** (PostgreSQL 16, Node.js 22, cursor-based consumption):
|
|
2112
|
+
- **Throughput**: 100,000+ messages/second with batch operations
|
|
1891
2113
|
- **Latency**: < 10ms for immediate pop operations
|
|
2114
|
+
- **Constant-time consumption**: O(batch_size) regardless of queue depth
|
|
1892
2115
|
- **Concurrent Connections**: 1,000+ long polling connections
|
|
1893
2116
|
- **Database**: Optimized with proper indexing and connection pooling
|
|
1894
2117
|
|
|
1895
|
-
**
|
|
2118
|
+
**Cursor-Based Architecture Benefits:**
|
|
2119
|
+
- **No performance degradation**: Consistent speed whether queue has 1K or 1B messages
|
|
2120
|
+
- **Predictable latency**: 150-200ms per batch throughout entire queue lifecycle
|
|
2121
|
+
- **Efficient batch processing**: Direct cursor access eliminates table scans
|
|
2122
|
+
- **Scalable to billions**: UUIDv7-based cursor positioning
|
|
2123
|
+
|
|
2124
|
+
**Additional Optimization Features:**
|
|
1896
2125
|
- Connection pooling with configurable size
|
|
1897
2126
|
- Resource caching for queue/partition lookups
|
|
1898
|
-
- Batch operations for bulk inserts/updates
|
|
2127
|
+
- Batch operations for bulk inserts/updates (up to 10,000 messages per batch)
|
|
1899
2128
|
- Optimized SQL queries with proper indexes
|
|
1900
2129
|
- Event-driven architecture for minimal polling overhead
|
|
1901
2130
|
- Long polling for real-time message delivery
|
|
2131
|
+
- SKIP LOCKED for lock-free concurrent consumption
|
|
1902
2132
|
|
|
1903
2133
|
---
|
|
1904
2134
|
|
|
1905
2135
|
## 🎯 Roadmap
|
|
1906
2136
|
|
|
1907
|
-
- [ ] **Horizontal Scaling**: Better support for multiple server instances
|
|
1908
2137
|
- [ ] **Message Scheduling**: Cron-like scheduling for recurring jobs
|
|
1909
|
-
- [ ] **Priority Lanes**: Dynamic priority adjustment based on load
|
|
1910
|
-
- [ ] **Metrics Export**: Prometheus/Grafana integration
|
|
1911
|
-
- [ ] **Admin API**: REST API for queue management
|
|
1912
2138
|
- [ ] **Client Libraries**: Python, Go, Java clients
|
|
1913
|
-
- [ ] **Message Tracing**: Distributed tracing integration
|
|
1914
|
-
- [ ] **Queue Templates**: Pre-configured queue patterns
|
|
1915
|
-
- [ ] **GraphQL API**: Alternative to REST API
|
|
1916
2139
|
- [ ] **Kubernetes Operator**: Native K8s support
|
|
1917
2140
|
|
|
1918
2141
|
---
|
|
@@ -1921,6 +2144,6 @@ Apache License 2.0 - see [LICENSE.md](LICENSE.md) for details.
|
|
|
1921
2144
|
|
|
1922
2145
|
**Queen Message Queue System** - Built for performance, reliability, and developer happiness 🚀
|
|
1923
2146
|
|
|
1924
|
-
Made with ❤️ by [
|
|
2147
|
+
Made with ❤️ by [Smartness](https://github.com/smartpricing)
|
|
1925
2148
|
|
|
1926
2149
|
</div>
|