tina4-nodejs 3.13.92 → 3.13.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +170 -28
- package/README.md +2 -2
- package/package.json +13 -9
- package/packages/cli/dist/bin.js +33126 -30055
- package/packages/cli/src/commands/metrics.ts +17 -11
- package/packages/cli/src/commands/serve.ts +10 -9
- package/packages/core/dist/index.js +33062 -29908
- package/packages/core/src/ai.ts +7 -1
- package/packages/core/src/auth.ts +191 -39
- package/packages/core/src/background.ts +19 -19
- package/packages/core/src/cache.ts +492 -49
- package/packages/core/src/devAdmin.ts +79 -32
- package/packages/core/src/devMailbox.ts +20 -44
- package/packages/core/src/dispatchPipeline.ts +285 -0
- package/packages/core/src/dotenv.ts +185 -40
- package/packages/core/src/index.ts +7 -6
- package/packages/core/src/logger.ts +257 -36
- package/packages/core/src/mcp.ts +1 -1
- package/packages/core/src/messenger.ts +81 -13
- package/packages/core/src/metrics.ts +199 -961
- package/packages/core/src/middleware.ts +390 -123
- package/packages/core/src/queue.ts +188 -32
- package/packages/core/src/queueBackends/kafkaBackend.ts +109 -13
- package/packages/core/src/queueBackends/liteBackend.ts +13 -0
- package/packages/core/src/queueBackends/mongoBackend.ts +101 -9
- package/packages/core/src/queueBackends/rabbitmqBackend.ts +22 -4
- package/packages/core/src/rateLimiter.ts +10 -5
- package/packages/core/src/request.ts +6 -9
- package/packages/core/src/response.ts +46 -1
- package/packages/core/src/router.ts +29 -4
- package/packages/core/src/server.ts +751 -414
- package/packages/core/src/session.ts +244 -27
- package/packages/core/src/sessionHandlers/childError.ts +72 -0
- package/packages/core/src/sessionHandlers/databaseHandler.ts +338 -48
- package/packages/core/src/sessionHandlers/memcachedHandler.ts +181 -0
- package/packages/core/src/sessionHandlers/mongoClient.ts +293 -202
- package/packages/core/src/sessionHandlers/mongoHandler.ts +88 -8
- package/packages/core/src/sessionHandlers/respClient.ts +16 -143
- package/packages/core/src/sessionHandlers/sqlClient.ts +290 -0
- package/packages/core/src/sessionHandlers/syncBridge.ts +190 -0
- package/packages/core/src/sessionHandlers/syncSocket.ts +236 -0
- package/packages/core/src/testClient.ts +18 -5
- package/packages/core/src/trustedProxy.ts +249 -0
- package/packages/core/src/types.ts +29 -5
- package/packages/core/src/websocket.ts +66 -0
- package/packages/frond/dist/index.js +74 -31
- package/packages/frond/src/engine.ts +99 -33
- package/packages/orm/dist/index.js +26554 -23400
- package/packages/orm/src/adapters/firebird.ts +183 -56
- package/packages/orm/src/adapters/mongodb.ts +25 -4
- package/packages/orm/src/adapters/mssql.ts +114 -29
- package/packages/orm/src/adapters/mysql.ts +103 -40
- package/packages/orm/src/adapters/odbc.ts +44 -21
- package/packages/orm/src/adapters/postgres.ts +118 -26
- package/packages/orm/src/adapters/sqlDialect.ts +120 -0
- package/packages/orm/src/adapters/sqlite.ts +64 -25
- package/packages/orm/src/baseModel.ts +135 -40
- package/packages/orm/src/cachedDatabase.ts +43 -19
- package/packages/orm/src/connectTimeout.ts +265 -0
- package/packages/orm/src/database.ts +338 -198
- package/packages/orm/src/databaseResult.ts +65 -13
- package/packages/orm/src/databaseUrl.ts +484 -0
- package/packages/orm/src/docstore.ts +386 -145
- package/packages/orm/src/index.ts +13 -3
- package/packages/orm/src/migration.ts +18 -3
- package/packages/orm/src/queryBuilder.ts +38 -4
- package/packages/orm/src/sqlTranslator.ts +310 -4
- package/packages/orm/src/types.ts +15 -4
- package/types/cli/src/bin.d.ts +92 -0
- package/types/cli/src/commands/build.d.ts +2 -0
- package/types/cli/src/commands/generate.d.ts +47 -0
- package/types/cli/src/commands/init.d.ts +1 -0
- package/types/cli/src/commands/metrics.d.ts +6 -0
- package/types/cli/src/commands/migrate.d.ts +1 -0
- package/types/cli/src/commands/migrateCreate.d.ts +1 -0
- package/types/cli/src/commands/migrateRollback.d.ts +1 -0
- package/types/cli/src/commands/migrateStatus.d.ts +1 -0
- package/types/cli/src/commands/queue.d.ts +20 -0
- package/types/cli/src/commands/routes.d.ts +1 -0
- package/types/cli/src/commands/seed.d.ts +1 -0
- package/types/cli/src/commands/serve.d.ts +6 -0
- package/types/cli/src/commands/test.d.ts +1 -0
- package/types/core/src/ai.d.ts +64 -0
- package/types/core/src/api.d.ts +262 -0
- package/types/core/src/auth.d.ts +177 -0
- package/types/core/src/authGate.d.ts +20 -0
- package/types/core/src/background.d.ts +34 -0
- package/types/core/src/cache.d.ts +163 -0
- package/types/core/src/constants.d.ts +38 -0
- package/types/core/src/container.d.ts +44 -0
- package/types/core/src/context/chunker.d.ts +31 -0
- package/types/core/src/context/index.d.ts +93 -0
- package/types/core/src/devAdmin.d.ts +179 -0
- package/types/core/src/devMailbox.d.ts +54 -0
- package/types/core/src/dispatchPipeline.d.ts +117 -0
- package/types/core/src/docs.d.ts +141 -0
- package/types/core/src/docsAutoDiscovery.d.ts +6 -0
- package/types/core/src/dotenv.d.ts +87 -0
- package/types/core/src/env.d.ts +28 -0
- package/types/core/src/errorOverlay.d.ts +36 -0
- package/types/core/src/events.d.ts +75 -0
- package/types/core/src/fakeData.d.ts +55 -0
- package/types/core/src/feedback.d.ts +90 -0
- package/types/core/src/graphql.d.ts +207 -0
- package/types/core/src/health.d.ts +22 -0
- package/types/core/src/htmlElement.d.ts +75 -0
- package/types/core/src/i18n.d.ts +37 -0
- package/types/core/src/index.d.ts +92 -0
- package/types/core/src/job.d.ts +39 -0
- package/types/core/src/logger.d.ts +200 -0
- package/types/core/src/mcp.d.ts +248 -0
- package/types/core/src/messenger.d.ts +191 -0
- package/types/core/src/metrics.d.ts +41 -0
- package/types/core/src/middleware.d.ts +330 -0
- package/types/core/src/mqtt.d.ts +257 -0
- package/types/core/src/mqttMessage.d.ts +67 -0
- package/types/core/src/plan.d.ts +96 -0
- package/types/core/src/projectIndex.d.ts +56 -0
- package/types/core/src/queue.d.ts +268 -0
- package/types/core/src/queueBackends/kafkaBackend.d.ts +117 -0
- package/types/core/src/queueBackends/liteBackend.d.ts +128 -0
- package/types/core/src/queueBackends/mongoBackend.d.ts +119 -0
- package/types/core/src/queueBackends/rabbitmqBackend.d.ts +55 -0
- package/types/core/src/rateLimiter.d.ts +49 -0
- package/types/core/src/request.d.ts +25 -0
- package/types/core/src/response.d.ts +28 -0
- package/types/core/src/routeDiscovery.d.ts +12 -0
- package/types/core/src/router.d.ts +366 -0
- package/types/core/src/scss.d.ts +19 -0
- package/types/core/src/server.d.ts +146 -0
- package/types/core/src/service.d.ts +115 -0
- package/types/core/src/session.d.ts +341 -0
- package/types/core/src/sessionHandlers/childError.d.ts +34 -0
- package/types/core/src/sessionHandlers/databaseHandler.d.ts +97 -0
- package/types/core/src/sessionHandlers/memcachedHandler.d.ts +60 -0
- package/types/core/src/sessionHandlers/mongoClient.d.ts +35 -0
- package/types/core/src/sessionHandlers/mongoHandler.d.ts +109 -0
- package/types/core/src/sessionHandlers/respClient.d.ts +22 -0
- package/types/core/src/sessionHandlers/sqlClient.d.ts +39 -0
- package/types/core/src/sessionHandlers/syncBridge.d.ts +91 -0
- package/types/core/src/sessionHandlers/syncSocket.d.ts +49 -0
- package/types/core/src/sessionHandlers/valkeyHandler.d.ts +65 -0
- package/types/core/src/static.d.ts +2 -0
- package/types/core/src/test.d.ts +94 -0
- package/types/core/src/testClient.d.ts +36 -0
- package/types/core/src/testing.d.ts +58 -0
- package/types/core/src/trustedProxy.d.ts +44 -0
- package/types/core/src/types.d.ts +242 -0
- package/types/core/src/validator.d.ts +52 -0
- package/types/core/src/websocket.d.ts +402 -0
- package/types/core/src/websocketBackplane.d.ts +166 -0
- package/types/core/src/websocketConnection.d.ts +54 -0
- package/types/core/src/wsdl.d.ts +101 -0
- package/types/frond/src/engine.d.ts +263 -0
- package/types/frond/src/index.d.ts +2 -0
- package/types/orm/src/adapters/firebird.d.ts +183 -0
- package/types/orm/src/adapters/mongodb.d.ts +81 -0
- package/types/orm/src/adapters/mssql.d.ts +77 -0
- package/types/orm/src/adapters/mysql.d.ts +67 -0
- package/types/orm/src/adapters/odbc.d.ts +94 -0
- package/types/orm/src/adapters/postgres.d.ts +86 -0
- package/types/orm/src/adapters/sqlDialect.d.ts +71 -0
- package/types/orm/src/adapters/sqlite.d.ts +68 -0
- package/types/orm/src/autoCrud.d.ts +73 -0
- package/types/orm/src/baseModel.d.ts +427 -0
- package/types/orm/src/cachedDatabase.d.ts +190 -0
- package/types/orm/src/connectTimeout.d.ts +100 -0
- package/types/orm/src/database.d.ts +655 -0
- package/types/orm/src/databaseResult.d.ts +109 -0
- package/types/orm/src/databaseUrl.d.ts +125 -0
- package/types/orm/src/docstore.d.ts +241 -0
- package/types/orm/src/fakeData.d.ts +22 -0
- package/types/orm/src/index.d.ts +43 -0
- package/types/orm/src/migration.d.ts +275 -0
- package/types/orm/src/model.d.ts +7 -0
- package/types/orm/src/query.d.ts +14 -0
- package/types/orm/src/queryBuilder.d.ts +193 -0
- package/types/orm/src/realtime/index.d.ts +7 -0
- package/types/orm/src/realtime/models/attachment.d.ts +43 -0
- package/types/orm/src/realtime/models/channel.d.ts +32 -0
- package/types/orm/src/realtime/models/channelMember.d.ts +32 -0
- package/types/orm/src/realtime/models/message.d.ts +36 -0
- package/types/orm/src/realtime/models/workspace.d.ts +26 -0
- package/types/orm/src/realtime/realtime.d.ts +24 -0
- package/types/orm/src/realtime/storage.d.ts +61 -0
- package/types/orm/src/seeder.d.ts +118 -0
- package/types/orm/src/sqlTranslator.d.ts +258 -0
- package/types/orm/src/types.d.ts +148 -0
- package/types/orm/src/validation.d.ts +6 -0
- package/types/swagger/src/generator.d.ts +46 -0
- package/types/swagger/src/index.d.ts +2 -0
- package/types/swagger/src/ui.d.ts +11 -0
- package/packages/core/src/sessionHandlers/redisHandler.ts +0 -206
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Tina4 Queue — Unified job queue with pluggable backends, zero dependencies.
|
|
3
3
|
*
|
|
4
|
-
* Switching from file to
|
|
4
|
+
* Switching from file to MongoDB is a .env change — no code change needed.
|
|
5
5
|
*
|
|
6
6
|
* Supported backends:
|
|
7
7
|
* - 'file' — JSON files on disk (default)
|
|
8
|
-
* - 'rabbitmq' — RabbitMQ via raw TCP (AMQP 0-9-1)
|
|
9
|
-
* - 'kafka' — Kafka via raw TCP
|
|
10
8
|
* - 'mongodb' — MongoDB via `mongodb` npm package (also 'mongo')
|
|
11
9
|
*
|
|
10
|
+
* REFUSED backends (ADR-0022):
|
|
11
|
+
* - 'rabbitmq' and 'kafka' THROW on construction. Node drives both through a
|
|
12
|
+
* child process per operation, so no connection survives between pop() and
|
|
13
|
+
* complete() and acknowledgement is impossible: RabbitMQ was at-most-once
|
|
14
|
+
* (a dead consumer lost the job) and Kafka could never drain a topic. They
|
|
15
|
+
* lost work silently, so they now refuse. tina4-python, tina4-php and
|
|
16
|
+
* tina4-ruby still offer both. See unsupportedBrokerMessage() below.
|
|
17
|
+
*
|
|
12
18
|
* Environment variables:
|
|
13
|
-
* TINA4_QUEUE_BACKEND — 'file'
|
|
14
|
-
* TINA4_QUEUE_URL — connection URL for
|
|
19
|
+
* TINA4_QUEUE_BACKEND — 'file' or 'mongodb'
|
|
20
|
+
* TINA4_QUEUE_URL — connection URL for mongodb
|
|
15
21
|
* TINA4_QUEUE_PATH — file backend storage path (default: data/queue)
|
|
16
22
|
*
|
|
17
23
|
* Usage:
|
|
@@ -22,14 +28,12 @@
|
|
|
22
28
|
* queue.push({ to: "alice@test.com", subject: "Hello" });
|
|
23
29
|
*
|
|
24
30
|
* // Explicit backend
|
|
25
|
-
* const queue = new Queue({ topic: "tasks", backend: "
|
|
31
|
+
* const queue = new Queue({ topic: "tasks", backend: "mongodb" });
|
|
26
32
|
*
|
|
27
33
|
* // Legacy usage (still works — uses file backend)
|
|
28
34
|
* const queue = new Queue();
|
|
29
35
|
* queue.push("emails", { to: "alice@test.com" });
|
|
30
36
|
*/
|
|
31
|
-
import { RabbitMQBackend } from "./queueBackends/rabbitmqBackend.js";
|
|
32
|
-
import { KafkaBackend } from "./queueBackends/kafkaBackend.js";
|
|
33
37
|
import { MongoBackend } from "./queueBackends/mongoBackend.js";
|
|
34
38
|
import { LiteBackend } from "./queueBackends/liteBackend.js";
|
|
35
39
|
import { type QueueJob, type JobData, createJob } from "./job.js";
|
|
@@ -58,13 +62,27 @@ export interface QueueConfig {
|
|
|
58
62
|
* re-enqueuing, or dead-lettering past maxRetries (at-least-once delivery).
|
|
59
63
|
* Falls back to TINA4_QUEUE_VISIBILITY_TIMEOUT, else 300 (5 min). <= 0
|
|
60
64
|
* disables the reclaim (a reservation then lasts until the consumer acks —
|
|
61
|
-
* the old at-most-once behaviour). File + MongoDB backends
|
|
62
|
-
*
|
|
65
|
+
* the old at-most-once behaviour). File + MongoDB backends, which are the
|
|
66
|
+
* only backends Node offers (ADR-0022). Parity with Python's
|
|
63
67
|
* visibility_timeout.
|
|
64
68
|
*/
|
|
65
69
|
visibilityTimeout?: number;
|
|
66
70
|
}
|
|
67
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Where the file-backed queue stores its jobs — `TINA4_QUEUE_PATH`, else
|
|
74
|
+
* `data/queue` (relative to the working directory).
|
|
75
|
+
*
|
|
76
|
+
* Exported because it is the ONE answer to "where do the queue files live",
|
|
77
|
+
* and anything that reads the store directly (the dev-admin queue panel) must
|
|
78
|
+
* ask here rather than re-deriving it. The dev admin hardcoded
|
|
79
|
+
* `cwd/data/queue/<topic>` and so listed a DIFFERENT directory from the one
|
|
80
|
+
* `Queue.size()` counted the moment `TINA4_QUEUE_PATH` was set.
|
|
81
|
+
*/
|
|
82
|
+
export function queueBasePath(): string {
|
|
83
|
+
return process.env.TINA4_QUEUE_PATH ?? "data/queue";
|
|
84
|
+
}
|
|
85
|
+
|
|
68
86
|
/**
|
|
69
87
|
* Reservation/visibility timeout in seconds, from env (default 300 = 5 min).
|
|
70
88
|
* Mirrors Python's _default_visibility_timeout().
|
|
@@ -99,15 +117,31 @@ export interface ConsumeOptions {
|
|
|
99
117
|
}
|
|
100
118
|
|
|
101
119
|
export interface QueueBackendInterface {
|
|
102
|
-
push(queue: string, payload: unknown, delay?: number): string;
|
|
120
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string;
|
|
103
121
|
pop(queue: string): QueueJob | null;
|
|
104
122
|
size(queue: string): number;
|
|
105
123
|
clear(queue: string): void;
|
|
124
|
+
/**
|
|
125
|
+
* Release whatever connection the backend holds, and be safe to call twice.
|
|
126
|
+
*
|
|
127
|
+
* REQUIRED, not optional, and deliberately so: it mirrors PHP's
|
|
128
|
+
* Tina4\Queue\QueueBackend, where close() has always been part of the
|
|
129
|
+
* interface. Optional would reintroduce exactly the bug this closes — a
|
|
130
|
+
* caller feature-detecting `backend.close?.()` silently skips the backend
|
|
131
|
+
* that forgot to implement it, which is how tina4-ruby's lite backend went
|
|
132
|
+
* un-closed by every `respond_to?(:close)` guard in its tree.
|
|
133
|
+
*/
|
|
134
|
+
close(): void;
|
|
106
135
|
// Optional full lifecycle. Reservation-based backends (MongoDB) implement
|
|
107
136
|
// these so complete()/fail() ack the ACTIVE store — without complete(), a
|
|
108
|
-
// reserved Mongo job is re-delivered after the visibility window.
|
|
109
|
-
//
|
|
110
|
-
//
|
|
137
|
+
// reserved Mongo job is re-delivered after the visibility window.
|
|
138
|
+
//
|
|
139
|
+
// Optional is a wart, not a design: a backend that omitted them silently fell
|
|
140
|
+
// through to the LOCAL FILE store, so a fail() on a broker-backed queue wrote
|
|
141
|
+
// JSON to disk while the broker still held the message. The only backends
|
|
142
|
+
// that did that were RabbitMQ and Kafka, and ADR-0022 now refuses both, so
|
|
143
|
+
// nothing in tree relies on the fallback. Make these required when the
|
|
144
|
+
// persistent-connection rewrite lands.
|
|
111
145
|
complete?(queue: string, id: string): void;
|
|
112
146
|
fail?(queue: string, id: string, error: string, maxRetries: number, retryBackoff: number): void;
|
|
113
147
|
retry?(queue: string, id: string, delaySeconds?: number): void;
|
|
@@ -117,6 +151,50 @@ export interface QueueBackendInterface {
|
|
|
117
151
|
purge?(queue: string, status?: string): number;
|
|
118
152
|
}
|
|
119
153
|
|
|
154
|
+
/**
|
|
155
|
+
* Why `backend: "rabbitmq"` and `backend: "kafka"` are refused in Node (ADR-0022).
|
|
156
|
+
*
|
|
157
|
+
* Both backends drive the wire protocol through `execFileSync`, spawning a fresh
|
|
158
|
+
* child process for EVERY operation. The child connects, performs one operation,
|
|
159
|
+
* destroys its socket and exits, so no connection, channel, consumer or session
|
|
160
|
+
* survives between push, pop and complete.
|
|
161
|
+
*
|
|
162
|
+
* That makes acknowledgement impossible, not merely unimplemented. An AMQP
|
|
163
|
+
* delivery tag identifies a delivery on a CHANNEL that is already closed by the
|
|
164
|
+
* time pop() returns, and a Kafka consumer-group offset belongs to a SESSION
|
|
165
|
+
* that never existed. So RabbitMQ ran Basic.Get with no-ack=true (at-most-once:
|
|
166
|
+
* a consumer that dies after pop() loses the job outright) and Kafka fetched
|
|
167
|
+
* from offset 0 every time (the topic could never drain).
|
|
168
|
+
*
|
|
169
|
+
* Refusing loudly follows the same call the session backend already made for
|
|
170
|
+
* `redis-npm`: a silent demotion looks like it is working while the operator
|
|
171
|
+
* believes otherwise, which is worse than an outage you can see. Anyone
|
|
172
|
+
* "successfully" running these today is already losing jobs and does not know it.
|
|
173
|
+
*
|
|
174
|
+
* This is a HOLDING POSITION, not the design. The fix is a persistent
|
|
175
|
+
* connection held on the backend instance, the way Python, PHP and Ruby already
|
|
176
|
+
* do it. There is deliberately no opt-in escape hatch: nobody has asked for
|
|
177
|
+
* fire-and-forget, and a knob for a hypothetical user is not worth its weight.
|
|
178
|
+
*/
|
|
179
|
+
function unsupportedBrokerMessage(backendName: string): string {
|
|
180
|
+
const broker = backendName === "kafka" ? "Kafka" : "RabbitMQ";
|
|
181
|
+
const lost = backendName === "kafka"
|
|
182
|
+
? "every pop() re-read offset 0, so the topic could never drain"
|
|
183
|
+
: "pop() destroyed the message with no acknowledgement, so a consumer that died lost the job";
|
|
184
|
+
return (
|
|
185
|
+
`Queue backend "${backendName}" is not available in tina4-nodejs.\n\n` +
|
|
186
|
+
`Reason: the ${broker} backend runs each operation in a separate child ` +
|
|
187
|
+
`process, so no connection survives between pop() and complete(). ` +
|
|
188
|
+
`Acknowledgement is therefore impossible, and ${lost}. It was losing work ` +
|
|
189
|
+
`silently, so it now refuses instead.\n\n` +
|
|
190
|
+
`Use backend "mongodb" (at-least-once, with a real reservation and ` +
|
|
191
|
+
`visibility timeout) or the default "file" backend. tina4-python, ` +
|
|
192
|
+
`tina4-php and tina4-ruby still offer ${broker}.\n\n` +
|
|
193
|
+
`Tracking: ADR-0022 in tina4-documentation/plan/v3/DECISIONS.md, ` +
|
|
194
|
+
`findings in plan/v3/features/048-queue-backends.md.`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
|
|
120
198
|
// ── Queue ────────────────────────────────────────────────────
|
|
121
199
|
|
|
122
200
|
export class Queue {
|
|
@@ -133,26 +211,28 @@ export class Queue {
|
|
|
133
211
|
* Unified Queue constructor.
|
|
134
212
|
*
|
|
135
213
|
* Accepts either:
|
|
136
|
-
* - new Queue({ topic: "tasks", backend: "
|
|
137
|
-
* - new Queue("
|
|
214
|
+
* - new Queue({ topic: "tasks", backend: "mongodb" })
|
|
215
|
+
* - new Queue("mongodb", { path: "data/queue" }) // legacy
|
|
138
216
|
* - new Queue() // file backend, default topic
|
|
217
|
+
*
|
|
218
|
+
* Throws on backend "rabbitmq" or "kafka" (ADR-0022).
|
|
139
219
|
*/
|
|
140
220
|
constructor(backendOrConfig?: string | QueueConfig, config?: QueueConfig) {
|
|
141
221
|
let resolvedConfig: QueueConfig = {};
|
|
142
222
|
|
|
143
223
|
if (typeof backendOrConfig === "string") {
|
|
144
|
-
// Legacy: new Queue("
|
|
224
|
+
// Legacy: new Queue("mongodb", { ... })
|
|
145
225
|
resolvedConfig = { ...(config ?? {}), backend: backendOrConfig };
|
|
146
226
|
} else if (typeof backendOrConfig === "object" && backendOrConfig !== null) {
|
|
147
227
|
resolvedConfig = backendOrConfig;
|
|
148
228
|
}
|
|
149
229
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
230
|
+
// Normalised (trimmed + lowercased) so " MongoDB " resolves, matching the
|
|
231
|
+
// Python master, Ruby and PHP. An unrecognised value THROWS below.
|
|
232
|
+
this.backendName = String(
|
|
233
|
+
resolvedConfig.backend ?? process.env.TINA4_QUEUE_BACKEND ?? "file",
|
|
234
|
+
).trim().toLowerCase();
|
|
235
|
+
this.basePath = resolvedConfig.path ?? queueBasePath();
|
|
156
236
|
this.topic = resolvedConfig.topic ?? "default";
|
|
157
237
|
this._maxRetries = resolvedConfig.maxRetries ?? 3;
|
|
158
238
|
this._retryBackoff = resolvedConfig.retryBackoff ?? 0;
|
|
@@ -160,15 +240,24 @@ export class Queue {
|
|
|
160
240
|
this.liteBackend = new LiteBackend(this.basePath, this._visibilityTimeout);
|
|
161
241
|
|
|
162
242
|
// Initialize external backends
|
|
163
|
-
if (this.backendName === "rabbitmq") {
|
|
164
|
-
|
|
165
|
-
// channel close) — the framework timeout is accepted but not used.
|
|
166
|
-
this.externalBackend = new RabbitMQBackend({ visibilityTimeout: this._visibilityTimeout });
|
|
167
|
-
} else if (this.backendName === "kafka") {
|
|
168
|
-
// Consumer-group offsets manage redelivery — framework timeout N/A.
|
|
169
|
-
this.externalBackend = new KafkaBackend({ visibilityTimeout: this._visibilityTimeout });
|
|
243
|
+
if (this.backendName === "rabbitmq" || this.backendName === "kafka") {
|
|
244
|
+
throw new Error(unsupportedBrokerMessage(this.backendName));
|
|
170
245
|
} else if (this.backendName === "mongodb" || this.backendName === "mongo") {
|
|
171
246
|
this.externalBackend = new MongoBackend({ visibilityTimeout: this._visibilityTimeout });
|
|
247
|
+
} else if (!["file", "default", "lite"].includes(this.backendName)) {
|
|
248
|
+
// An UNRECOGNISED backend name THROWS rather than falling through to the
|
|
249
|
+
// local file store.
|
|
250
|
+
//
|
|
251
|
+
// MEASURED 2026-08-03: a typo in TINA4_QUEUE_BACKEND produced a running
|
|
252
|
+
// app writing every job to local disk while the operator believed they
|
|
253
|
+
// were in MongoDB - jobs nothing consumes, on a container filesystem that
|
|
254
|
+
// vanishes on the next deploy, with no error at any point. Python and Ruby
|
|
255
|
+
// already raise here; this is the same rule the session backend adopted
|
|
256
|
+
// for the same reason.
|
|
257
|
+
throw new Error(
|
|
258
|
+
`Unknown queue backend: '${this.backendName}'. ` +
|
|
259
|
+
`Use 'file', 'rabbitmq', 'kafka', or 'mongodb'.`,
|
|
260
|
+
);
|
|
172
261
|
}
|
|
173
262
|
}
|
|
174
263
|
|
|
@@ -202,7 +291,10 @@ export class Queue {
|
|
|
202
291
|
*/
|
|
203
292
|
push(payload: unknown, delay?: number, priority: number = 0): string {
|
|
204
293
|
if (this.externalBackend) {
|
|
205
|
-
|
|
294
|
+
// priority goes to the external backend too. It used to be passed only to
|
|
295
|
+
// liteBackend, so switching to mongodb silently turned a prioritised
|
|
296
|
+
// queue into a FIFO one.
|
|
297
|
+
return this.externalBackend.push(this.topic, payload, delay, priority);
|
|
206
298
|
}
|
|
207
299
|
return this.liteBackend.push(this.topic, payload, delay, priority);
|
|
208
300
|
}
|
|
@@ -214,7 +306,15 @@ export class Queue {
|
|
|
214
306
|
const q = this.topic;
|
|
215
307
|
|
|
216
308
|
if (this.externalBackend) {
|
|
217
|
-
|
|
309
|
+
const raw = this.externalBackend.pop(q);
|
|
310
|
+
// Wrap it. An external backend returns PLAIN DATA with no lifecycle
|
|
311
|
+
// methods, so `queue.pop().fail("boom")` threw
|
|
312
|
+
// "TypeError: j.fail is not a function" on mongodb/rabbitmq/kafka while
|
|
313
|
+
// working on file — identical application code, different outcome, which
|
|
314
|
+
// is exactly what ADR-0024 forbids. createJob attaches
|
|
315
|
+
// complete()/fail()/reject()/retry(), and they route back through
|
|
316
|
+
// _completeJob/_failJob, which already dispatch to the external backend.
|
|
317
|
+
return raw ? createJob(raw as any, this) : null;
|
|
218
318
|
}
|
|
219
319
|
return this.liteBackend.pop(q, this);
|
|
220
320
|
}
|
|
@@ -223,6 +323,19 @@ export class Queue {
|
|
|
223
323
|
* Pop up to count jobs at once. Returns a partial batch if fewer available.
|
|
224
324
|
*/
|
|
225
325
|
popBatch(count: number): QueueJob[] {
|
|
326
|
+
// Route to the CONFIGURED backend. This used to read the LOCAL FILE STORE
|
|
327
|
+
// unconditionally, so a mongodb-backed queue always came back empty.
|
|
328
|
+
// Repeated pop() is the correct batch on an external backend: each claim is
|
|
329
|
+
// atomic, and a short batch simply means the queue drained.
|
|
330
|
+
if (this.externalBackend) {
|
|
331
|
+
const jobs: QueueJob[] = [];
|
|
332
|
+
for (let i = 0; i < count; i++) {
|
|
333
|
+
const job = this.pop();
|
|
334
|
+
if (!job) break;
|
|
335
|
+
jobs.push(job);
|
|
336
|
+
}
|
|
337
|
+
return jobs;
|
|
338
|
+
}
|
|
226
339
|
return this.liteBackend.popBatch(this.topic, this, count);
|
|
227
340
|
}
|
|
228
341
|
|
|
@@ -311,6 +424,34 @@ export class Queue {
|
|
|
311
424
|
return this.liteBackend.clear(q);
|
|
312
425
|
}
|
|
313
426
|
|
|
427
|
+
/**
|
|
428
|
+
* Release the backend's connection and free its resources.
|
|
429
|
+
*
|
|
430
|
+
* MEASURED 2026-08-04: close() was absent on the top-level Queue in ALL FOUR
|
|
431
|
+
* frameworks, and in Node it was absent on every backend class too — so an
|
|
432
|
+
* application had no way at all to hand a queue's client back. Same class of
|
|
433
|
+
* leak as ADR-0025 corollary 4 (client-lifecycle-is-bounded).
|
|
434
|
+
*
|
|
435
|
+
* Safe on EVERY backend: the file backend holds no connection and closes as a
|
|
436
|
+
* documented no-op, so a TINA4_QUEUE_BACKEND change never turns a working
|
|
437
|
+
* shutdown path into an error. Idempotent — each backend drops its handles on
|
|
438
|
+
* the first call, so a second call finds nothing to close and returns.
|
|
439
|
+
*
|
|
440
|
+
* HONEST CAVEAT specific to Node: neither backend it can reach holds a
|
|
441
|
+
* connection between calls today. The Mongo backend runs each operation in
|
|
442
|
+
* its own child process (ADR-0022), which closes its own client before it
|
|
443
|
+
* exits, and rabbitmq/kafka are refused outright at construction. So this
|
|
444
|
+
* releases nothing YET — it is here for the contract, and because the day the
|
|
445
|
+
* persistent-connection rewrite lands the client is released here with no
|
|
446
|
+
* change at any call site. Python, PHP and Ruby release a REAL client through
|
|
447
|
+
* the identically-named method.
|
|
448
|
+
*
|
|
449
|
+
* Treat the queue as spent afterwards and build a new one to keep working.
|
|
450
|
+
*/
|
|
451
|
+
close(): void {
|
|
452
|
+
(this.externalBackend ?? this.liteBackend).close();
|
|
453
|
+
}
|
|
454
|
+
|
|
314
455
|
/**
|
|
315
456
|
* Get jobs that failed at least once but are still being retried
|
|
316
457
|
* (0 < attempts < maxRetries). These live in the pending queue under the
|
|
@@ -497,6 +638,21 @@ export class Queue {
|
|
|
497
638
|
* Pop a specific job by ID from this queue's topic.
|
|
498
639
|
*/
|
|
499
640
|
popById(id: string): QueueJob | null {
|
|
641
|
+
// Same defect as popBatch: this read the local file store on every backend.
|
|
642
|
+
if (this.externalBackend) {
|
|
643
|
+
const claim = (this.externalBackend as any).popById;
|
|
644
|
+
if (typeof claim !== "function") {
|
|
645
|
+
throw new Error(
|
|
646
|
+
`The ${this.backendName} queue backend cannot perform popById(): it ` +
|
|
647
|
+
`cannot address a single message by id. Use the file or mongodb backend.`,
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
// Same shape as pop() — and, like pop(), PLAIN DATA with no lifecycle
|
|
651
|
+
// methods, so it must be wrapped or job.complete()/job.fail() is a
|
|
652
|
+
// TypeError on every external backend.
|
|
653
|
+
const raw = claim.call(this.externalBackend, this.topic, id);
|
|
654
|
+
return raw ? createJob(raw as any, this) : null;
|
|
655
|
+
}
|
|
500
656
|
return this.liteBackend.popById(this.topic, id);
|
|
501
657
|
}
|
|
502
658
|
|
|
@@ -92,7 +92,7 @@ export function kafkaSecurityConfig(
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
export interface QueueBackend {
|
|
95
|
-
push(queue: string, payload: unknown, delay?: number): string;
|
|
95
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string;
|
|
96
96
|
pop(queue: string): QueueJob | null;
|
|
97
97
|
size(queue: string): number;
|
|
98
98
|
clear(queue: string): void;
|
|
@@ -497,12 +497,15 @@ export class KafkaBackend implements QueueBackend {
|
|
|
497
497
|
if (errCode === 0) {
|
|
498
498
|
finish("__PUBLISHED__", 0);
|
|
499
499
|
} else {
|
|
500
|
+
// Report the CODE, not just "it failed" — the caller decides
|
|
501
|
+
// whether it is retriable (3/5, the async topic-creation race)
|
|
502
|
+
// or fatal (e.g. 29 TOPIC_AUTHORIZATION_FAILED).
|
|
500
503
|
process.stderr.write("Produce error code " + errCode);
|
|
501
|
-
finish("
|
|
504
|
+
finish("__PRODUCEERROR__" + errCode, 0);
|
|
502
505
|
}
|
|
503
506
|
} catch (e) {
|
|
504
507
|
process.stderr.write("produce parse: " + e.message);
|
|
505
|
-
finish("
|
|
508
|
+
finish("__PARSEERROR__produce: " + e.message, 0);
|
|
506
509
|
}
|
|
507
510
|
return;
|
|
508
511
|
} else if (operation === "get") {
|
|
@@ -516,6 +519,7 @@ export class KafkaBackend implements QueueBackend {
|
|
|
516
519
|
pos += 4; // throttleTimeMs (v1+)
|
|
517
520
|
const topicCount = buffer.readInt32BE(pos); pos += 4;
|
|
518
521
|
let out = "__EMPTY__";
|
|
522
|
+
let fatalCode = 0;
|
|
519
523
|
for (let t = 0; t < topicCount; t++) {
|
|
520
524
|
const tl = buffer.readInt16BE(pos); pos += 2 + tl;
|
|
521
525
|
const pc = buffer.readInt32BE(pos); pos += 4;
|
|
@@ -527,6 +531,14 @@ export class KafkaBackend implements QueueBackend {
|
|
|
527
531
|
const abortedCount = buffer.readInt32BE(pos); pos += 4;
|
|
528
532
|
if (abortedCount > 0) pos += abortedCount * 16; // (-1 => none, skip)
|
|
529
533
|
const recSetSize = buffer.readInt32BE(pos); pos += 4;
|
|
534
|
+
// 3 = UNKNOWN_TOPIC_OR_PARTITION, 5 = LEADER_NOT_AVAILABLE:
|
|
535
|
+
// "nothing to read here yet", which a consumer that starts
|
|
536
|
+
// before its producer hits on every cold start. Any OTHER code
|
|
537
|
+
// (29 TOPIC_AUTHORIZATION_FAILED, 13 STALE_CONTROLLER_EPOCH, …)
|
|
538
|
+
// is a real failure and must NOT be reported as an empty queue.
|
|
539
|
+
if (errCode !== 0 && errCode !== 3 && errCode !== 5) {
|
|
540
|
+
fatalCode = errCode;
|
|
541
|
+
}
|
|
530
542
|
if (errCode === 0 && recSetSize > 0) {
|
|
531
543
|
const val = firstRecordValue(buffer, pos, pos + recSetSize);
|
|
532
544
|
if (val !== null) out = val;
|
|
@@ -534,21 +546,33 @@ export class KafkaBackend implements QueueBackend {
|
|
|
534
546
|
pos += recSetSize > 0 ? recSetSize : 0;
|
|
535
547
|
}
|
|
536
548
|
}
|
|
549
|
+
if (fatalCode !== 0) {
|
|
550
|
+
process.stderr.write("Fetch error code " + fatalCode);
|
|
551
|
+
finish("__FETCHERROR__" + fatalCode, 0);
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
537
554
|
finish(out, 0);
|
|
538
555
|
} catch (e) {
|
|
556
|
+
// A parse failure is NOT an empty queue either — say so.
|
|
539
557
|
process.stderr.write("fetch parse: " + e.message);
|
|
540
|
-
finish("
|
|
558
|
+
finish("__PARSEERROR__fetch: " + e.message, 0);
|
|
541
559
|
}
|
|
542
560
|
return;
|
|
543
561
|
}
|
|
544
562
|
});
|
|
545
563
|
|
|
564
|
+
// Report the reason on STDOUT and exit 0. Writing it to stderr and
|
|
565
|
+
// exiting non-zero LOST it: stderr to a pipe is an async write and
|
|
566
|
+
// process.exit() truncates it, so the parent saw an empty stderr and fell
|
|
567
|
+
// back to execFileSync's message -- which embeds this entire script.
|
|
568
|
+
// stdout is flushed by finish()'s write callback, so it survives.
|
|
546
569
|
sock.on("error", (err) => {
|
|
547
|
-
|
|
548
|
-
finish("", 1);
|
|
570
|
+
finish("__TRANSPORTERROR__" + err.message, 0);
|
|
549
571
|
});
|
|
550
572
|
|
|
551
|
-
var timer = setTimeout(() => {
|
|
573
|
+
var timer = setTimeout(() => {
|
|
574
|
+
finish("__TRANSPORTERROR__timed out after 10s talking to " + host + ":" + port, 0);
|
|
575
|
+
}, 10000);
|
|
552
576
|
`;
|
|
553
577
|
|
|
554
578
|
try {
|
|
@@ -558,8 +582,59 @@ export class KafkaBackend implements QueueBackend {
|
|
|
558
582
|
stdio: ["pipe", "pipe", "pipe"],
|
|
559
583
|
});
|
|
560
584
|
return result;
|
|
561
|
-
} catch {
|
|
562
|
-
|
|
585
|
+
} catch (err) {
|
|
586
|
+
// Reached only when the child itself could not run (spawn failure, killed,
|
|
587
|
+
// the outer 15s timeout). The socket-level reasons come back through
|
|
588
|
+
// stdout as __TRANSPORTERROR__ instead. Swallowing this to "" made every
|
|
589
|
+
// failure indistinguishable from an empty queue.
|
|
590
|
+
//
|
|
591
|
+
// execFileSync's own message embeds the ENTIRE generated script, so it is
|
|
592
|
+
// truncated here -- a 20KB error that buries the cause is barely better
|
|
593
|
+
// than no error at all.
|
|
594
|
+
const e = err as { stderr?: Buffer | string; message?: string };
|
|
595
|
+
const reason = String(e.stderr ?? "").trim() || e.message || "unknown error";
|
|
596
|
+
const firstLine = reason.split("\n", 1)[0]!.slice(0, 200);
|
|
597
|
+
return "__TRANSPORTERROR__" + firstLine;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/**
|
|
602
|
+
* Sleep synchronously between produce retries.
|
|
603
|
+
*
|
|
604
|
+
* `push()` is synchronous (the whole backend drives its socket through a child
|
|
605
|
+
* process), so there is no event loop to await on. `Atomics.wait` on a
|
|
606
|
+
* SharedArrayBuffer is the stdlib way to block a thread for a fixed time --
|
|
607
|
+
* no dependency, no busy-wait burning CPU.
|
|
608
|
+
*/
|
|
609
|
+
private static sleepSync(ms: number): void {
|
|
610
|
+
const shared = new Int32Array(new SharedArrayBuffer(4));
|
|
611
|
+
Atomics.wait(shared, 0, 0, ms);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/**
|
|
615
|
+
* Turn a sentinel from the protocol child into a thrown error, or return.
|
|
616
|
+
*
|
|
617
|
+
* The wording matches the Python and PHP backends exactly -- the parity rule
|
|
618
|
+
* covers user-visible error messages, not just behaviour.
|
|
619
|
+
*/
|
|
620
|
+
private static assertNoError(result: string, operation: string, topic: string): void {
|
|
621
|
+
const fatal = /^__(PRODUCEERROR|FETCHERROR)__(\d+)/.exec(result);
|
|
622
|
+
if (fatal) {
|
|
623
|
+
throw new Error(
|
|
624
|
+
`Kafka rejected the ${operation} for topic ${topic}: error code ${fatal[2]}`,
|
|
625
|
+
);
|
|
626
|
+
}
|
|
627
|
+
if (result.startsWith("__TRANSPORTERROR__")) {
|
|
628
|
+
throw new Error(
|
|
629
|
+
`Kafka ${operation} for topic ${topic} failed: ` +
|
|
630
|
+
result.slice("__TRANSPORTERROR__".length),
|
|
631
|
+
);
|
|
632
|
+
}
|
|
633
|
+
if (result.startsWith("__PARSEERROR__")) {
|
|
634
|
+
throw new Error(
|
|
635
|
+
`Kafka ${operation} for topic ${topic} returned an unreadable response: ` +
|
|
636
|
+
result.slice("__PARSEERROR__".length),
|
|
637
|
+
);
|
|
563
638
|
}
|
|
564
639
|
}
|
|
565
640
|
|
|
@@ -576,15 +651,36 @@ export class KafkaBackend implements QueueBackend {
|
|
|
576
651
|
delayUntil: null,
|
|
577
652
|
};
|
|
578
653
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
654
|
+
// Topic auto-creation is ASYNCHRONOUS, so a brand-new topic answers
|
|
655
|
+
// UNKNOWN_TOPIC_OR_PARTITION (3) or LEADER_NOT_AVAILABLE (5) on the first
|
|
656
|
+
// attempt while the controller is still electing a leader. Retry those
|
|
657
|
+
// (same 10 attempts / 200ms as the Python and PHP backends) instead of
|
|
658
|
+
// failing a cold-start push; every other code throws immediately.
|
|
659
|
+
const body = JSON.stringify(job);
|
|
660
|
+
let result = "";
|
|
661
|
+
for (let attempt = 1; attempt <= 10; attempt++) {
|
|
662
|
+
result = this.execSync("publish", queue, body);
|
|
663
|
+
if (result.includes("__PUBLISHED__")) {
|
|
664
|
+
return id;
|
|
665
|
+
}
|
|
666
|
+
const retriable = /^__PRODUCEERROR__(3|5)\b/.test(result);
|
|
667
|
+
if (!retriable || attempt === 10) {
|
|
668
|
+
break;
|
|
669
|
+
}
|
|
670
|
+
KafkaBackend.sleepSync(200);
|
|
582
671
|
}
|
|
583
|
-
|
|
672
|
+
|
|
673
|
+
KafkaBackend.assertNoError(result, "produce", queue);
|
|
674
|
+
throw new Error(`Kafka publish failed for topic ${queue}: ${result || "no response"}`);
|
|
584
675
|
}
|
|
585
676
|
|
|
586
677
|
pop(queue: string): QueueJob | null {
|
|
587
678
|
const result = this.execSync("get", queue);
|
|
679
|
+
|
|
680
|
+
// A real failure must NOT read as an empty queue: a mis-permissioned
|
|
681
|
+
// consumer would otherwise poll an "idle" topic forever.
|
|
682
|
+
KafkaBackend.assertNoError(result, "fetch", queue);
|
|
683
|
+
|
|
588
684
|
if (!result || result === "__EMPTY__" || result === "__UNSUPPORTED__") return null;
|
|
589
685
|
|
|
590
686
|
try {
|
|
@@ -74,6 +74,19 @@ export class LiteBackend {
|
|
|
74
74
|
return `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* No-op: the file backend holds no connection to release.
|
|
79
|
+
*
|
|
80
|
+
* It exists so `Queue.close()` can call ONE method on every backend instead
|
|
81
|
+
* of testing for it, and so switching TINA4_QUEUE_BACKEND to "file" never
|
|
82
|
+
* turns a working close() into "backend.close is not a function". Idempotent
|
|
83
|
+
* by construction — there is nothing to drop.
|
|
84
|
+
*/
|
|
85
|
+
close(): void {
|
|
86
|
+
// Nothing held: every operation opens, reads/writes and closes its own file
|
|
87
|
+
// descriptor synchronously, so no handle survives a call.
|
|
88
|
+
}
|
|
89
|
+
|
|
77
90
|
push(queue: string, payload: unknown, delay?: number, priority?: number): string {
|
|
78
91
|
const dir = this.ensureDir(queue);
|
|
79
92
|
const id = randomUUID();
|