tina4-nodejs 3.13.94 → 3.13.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +157 -28
- package/README.md +1 -1
- package/package.json +2 -1
- package/packages/cli/dist/bin.js +32418 -29638
- package/packages/cli/src/commands/metrics.ts +17 -11
- package/packages/cli/src/commands/serve.ts +10 -9
- package/packages/core/dist/index.js +32364 -29501
- package/packages/core/src/ai.ts +7 -1
- package/packages/core/src/auth.ts +191 -39
- package/packages/core/src/background.ts +19 -19
- package/packages/core/src/cache.ts +492 -49
- package/packages/core/src/devAdmin.ts +79 -32
- package/packages/core/src/dispatchPipeline.ts +285 -0
- package/packages/core/src/dotenv.ts +185 -40
- package/packages/core/src/index.ts +5 -4
- package/packages/core/src/logger.ts +257 -36
- package/packages/core/src/mcp.ts +1 -1
- package/packages/core/src/messenger.ts +9 -13
- package/packages/core/src/metrics.ts +199 -961
- package/packages/core/src/middleware.ts +390 -123
- package/packages/core/src/queue.ts +188 -32
- package/packages/core/src/queueBackends/kafkaBackend.ts +1 -1
- package/packages/core/src/queueBackends/liteBackend.ts +13 -0
- package/packages/core/src/queueBackends/mongoBackend.ts +101 -9
- package/packages/core/src/queueBackends/rabbitmqBackend.ts +22 -4
- package/packages/core/src/rateLimiter.ts +10 -5
- package/packages/core/src/request.ts +6 -9
- package/packages/core/src/response.ts +46 -1
- package/packages/core/src/router.ts +29 -4
- package/packages/core/src/server.ts +751 -414
- package/packages/core/src/session.ts +244 -27
- package/packages/core/src/sessionHandlers/databaseHandler.ts +338 -48
- package/packages/core/src/sessionHandlers/memcachedHandler.ts +181 -0
- package/packages/core/src/sessionHandlers/mongoClient.ts +293 -208
- package/packages/core/src/sessionHandlers/mongoHandler.ts +88 -8
- package/packages/core/src/sessionHandlers/respClient.ts +16 -147
- package/packages/core/src/sessionHandlers/sqlClient.ts +290 -0
- package/packages/core/src/sessionHandlers/syncBridge.ts +190 -0
- package/packages/core/src/sessionHandlers/syncSocket.ts +236 -0
- package/packages/core/src/testClient.ts +18 -5
- package/packages/core/src/trustedProxy.ts +249 -0
- package/packages/core/src/types.ts +29 -5
- package/packages/core/src/websocket.ts +66 -0
- package/packages/orm/dist/index.js +22367 -19504
- package/packages/orm/src/adapters/firebird.ts +183 -56
- package/packages/orm/src/adapters/mongodb.ts +25 -4
- package/packages/orm/src/adapters/mssql.ts +114 -29
- package/packages/orm/src/adapters/mysql.ts +103 -40
- package/packages/orm/src/adapters/odbc.ts +44 -21
- package/packages/orm/src/adapters/postgres.ts +118 -26
- package/packages/orm/src/adapters/sqlDialect.ts +120 -0
- package/packages/orm/src/adapters/sqlite.ts +60 -24
- package/packages/orm/src/baseModel.ts +135 -40
- package/packages/orm/src/cachedDatabase.ts +43 -19
- package/packages/orm/src/connectTimeout.ts +265 -0
- package/packages/orm/src/database.ts +237 -197
- package/packages/orm/src/databaseResult.ts +65 -13
- package/packages/orm/src/databaseUrl.ts +484 -0
- package/packages/orm/src/docstore.ts +386 -145
- package/packages/orm/src/index.ts +13 -3
- package/packages/orm/src/migration.ts +18 -3
- package/packages/orm/src/queryBuilder.ts +38 -4
- package/packages/orm/src/sqlTranslator.ts +310 -4
- package/packages/orm/src/types.ts +15 -4
- package/types/core/src/ai.d.ts +1 -1
- package/types/core/src/auth.d.ts +28 -5
- package/types/core/src/background.d.ts +3 -3
- package/types/core/src/cache.d.ts +15 -12
- package/types/core/src/dispatchPipeline.d.ts +117 -0
- package/types/core/src/dotenv.d.ts +38 -16
- package/types/core/src/index.d.ts +5 -6
- package/types/core/src/logger.d.ts +93 -16
- package/types/core/src/messenger.d.ts +2 -2
- package/types/core/src/metrics.d.ts +25 -61
- package/types/core/src/middleware.d.ts +134 -11
- package/types/core/src/queue.d.ts +54 -5
- package/types/core/src/queueBackends/kafkaBackend.d.ts +1 -1
- package/types/core/src/queueBackends/liteBackend.d.ts +9 -0
- package/types/core/src/queueBackends/mongoBackend.d.ts +24 -2
- package/types/core/src/queueBackends/rabbitmqBackend.d.ts +3 -3
- package/types/core/src/router.d.ts +14 -3
- package/types/core/src/server.d.ts +15 -0
- package/types/core/src/session.d.ts +87 -2
- package/types/core/src/sessionHandlers/databaseHandler.d.ts +60 -5
- package/types/core/src/sessionHandlers/memcachedHandler.d.ts +60 -0
- package/types/core/src/sessionHandlers/mongoClient.d.ts +16 -5
- package/types/core/src/sessionHandlers/mongoHandler.d.ts +51 -3
- package/types/core/src/sessionHandlers/respClient.d.ts +2 -2
- package/types/core/src/sessionHandlers/sqlClient.d.ts +39 -0
- package/types/core/src/sessionHandlers/syncBridge.d.ts +91 -0
- package/types/core/src/sessionHandlers/syncSocket.d.ts +49 -0
- package/types/core/src/trustedProxy.d.ts +44 -0
- package/types/core/src/types.d.ts +28 -5
- package/types/core/src/websocket.d.ts +26 -0
- package/types/orm/src/adapters/firebird.d.ts +55 -10
- package/types/orm/src/adapters/mongodb.d.ts +2 -2
- package/types/orm/src/adapters/mssql.d.ts +18 -11
- package/types/orm/src/adapters/mysql.d.ts +11 -10
- package/types/orm/src/adapters/odbc.d.ts +9 -12
- package/types/orm/src/adapters/postgres.d.ts +11 -10
- package/types/orm/src/adapters/sqlDialect.d.ts +71 -0
- package/types/orm/src/adapters/sqlite.d.ts +15 -3
- package/types/orm/src/baseModel.d.ts +45 -9
- package/types/orm/src/cachedDatabase.d.ts +18 -5
- package/types/orm/src/connectTimeout.d.ts +100 -0
- package/types/orm/src/database.d.ts +72 -26
- package/types/orm/src/databaseResult.d.ts +24 -0
- package/types/orm/src/databaseUrl.d.ts +125 -0
- package/types/orm/src/docstore.d.ts +102 -43
- package/types/orm/src/index.d.ts +5 -2
- package/types/orm/src/queryBuilder.d.ts +23 -3
- package/types/orm/src/sqlTranslator.d.ts +126 -2
- package/types/orm/src/types.d.ts +14 -4
- package/packages/core/src/sessionHandlers/redisHandler.ts +0 -219
- package/types/core/src/sessionHandlers/redisHandler.d.ts +0 -60
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Tina4 Queue — Unified job queue with pluggable backends, zero dependencies.
|
|
3
3
|
*
|
|
4
|
-
* Switching from file to
|
|
4
|
+
* Switching from file to MongoDB is a .env change — no code change needed.
|
|
5
5
|
*
|
|
6
6
|
* Supported backends:
|
|
7
7
|
* - 'file' — JSON files on disk (default)
|
|
8
|
-
* - 'rabbitmq' — RabbitMQ via raw TCP (AMQP 0-9-1)
|
|
9
|
-
* - 'kafka' — Kafka via raw TCP
|
|
10
8
|
* - 'mongodb' — MongoDB via `mongodb` npm package (also 'mongo')
|
|
11
9
|
*
|
|
10
|
+
* REFUSED backends (ADR-0022):
|
|
11
|
+
* - 'rabbitmq' and 'kafka' THROW on construction. Node drives both through a
|
|
12
|
+
* child process per operation, so no connection survives between pop() and
|
|
13
|
+
* complete() and acknowledgement is impossible: RabbitMQ was at-most-once
|
|
14
|
+
* (a dead consumer lost the job) and Kafka could never drain a topic. They
|
|
15
|
+
* lost work silently, so they now refuse. tina4-python, tina4-php and
|
|
16
|
+
* tina4-ruby still offer both. See unsupportedBrokerMessage() below.
|
|
17
|
+
*
|
|
12
18
|
* Environment variables:
|
|
13
|
-
* TINA4_QUEUE_BACKEND — 'file'
|
|
14
|
-
* TINA4_QUEUE_URL — connection URL for
|
|
19
|
+
* TINA4_QUEUE_BACKEND — 'file' or 'mongodb'
|
|
20
|
+
* TINA4_QUEUE_URL — connection URL for mongodb
|
|
15
21
|
* TINA4_QUEUE_PATH — file backend storage path (default: data/queue)
|
|
16
22
|
*
|
|
17
23
|
* Usage:
|
|
@@ -22,14 +28,12 @@
|
|
|
22
28
|
* queue.push({ to: "alice@test.com", subject: "Hello" });
|
|
23
29
|
*
|
|
24
30
|
* // Explicit backend
|
|
25
|
-
* const queue = new Queue({ topic: "tasks", backend: "
|
|
31
|
+
* const queue = new Queue({ topic: "tasks", backend: "mongodb" });
|
|
26
32
|
*
|
|
27
33
|
* // Legacy usage (still works — uses file backend)
|
|
28
34
|
* const queue = new Queue();
|
|
29
35
|
* queue.push("emails", { to: "alice@test.com" });
|
|
30
36
|
*/
|
|
31
|
-
import { RabbitMQBackend } from "./queueBackends/rabbitmqBackend.js";
|
|
32
|
-
import { KafkaBackend } from "./queueBackends/kafkaBackend.js";
|
|
33
37
|
import { MongoBackend } from "./queueBackends/mongoBackend.js";
|
|
34
38
|
import { LiteBackend } from "./queueBackends/liteBackend.js";
|
|
35
39
|
import { type QueueJob, type JobData, createJob } from "./job.js";
|
|
@@ -58,13 +62,27 @@ export interface QueueConfig {
|
|
|
58
62
|
* re-enqueuing, or dead-lettering past maxRetries (at-least-once delivery).
|
|
59
63
|
* Falls back to TINA4_QUEUE_VISIBILITY_TIMEOUT, else 300 (5 min). <= 0
|
|
60
64
|
* disables the reclaim (a reservation then lasts until the consumer acks —
|
|
61
|
-
* the old at-most-once behaviour). File + MongoDB backends
|
|
62
|
-
*
|
|
65
|
+
* the old at-most-once behaviour). File + MongoDB backends, which are the
|
|
66
|
+
* only backends Node offers (ADR-0022). Parity with Python's
|
|
63
67
|
* visibility_timeout.
|
|
64
68
|
*/
|
|
65
69
|
visibilityTimeout?: number;
|
|
66
70
|
}
|
|
67
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Where the file-backed queue stores its jobs — `TINA4_QUEUE_PATH`, else
|
|
74
|
+
* `data/queue` (relative to the working directory).
|
|
75
|
+
*
|
|
76
|
+
* Exported because it is the ONE answer to "where do the queue files live",
|
|
77
|
+
* and anything that reads the store directly (the dev-admin queue panel) must
|
|
78
|
+
* ask here rather than re-deriving it. The dev admin hardcoded
|
|
79
|
+
* `cwd/data/queue/<topic>` and so listed a DIFFERENT directory from the one
|
|
80
|
+
* `Queue.size()` counted the moment `TINA4_QUEUE_PATH` was set.
|
|
81
|
+
*/
|
|
82
|
+
export function queueBasePath(): string {
|
|
83
|
+
return process.env.TINA4_QUEUE_PATH ?? "data/queue";
|
|
84
|
+
}
|
|
85
|
+
|
|
68
86
|
/**
|
|
69
87
|
* Reservation/visibility timeout in seconds, from env (default 300 = 5 min).
|
|
70
88
|
* Mirrors Python's _default_visibility_timeout().
|
|
@@ -99,15 +117,31 @@ export interface ConsumeOptions {
|
|
|
99
117
|
}
|
|
100
118
|
|
|
101
119
|
export interface QueueBackendInterface {
|
|
102
|
-
push(queue: string, payload: unknown, delay?: number): string;
|
|
120
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string;
|
|
103
121
|
pop(queue: string): QueueJob | null;
|
|
104
122
|
size(queue: string): number;
|
|
105
123
|
clear(queue: string): void;
|
|
124
|
+
/**
|
|
125
|
+
* Release whatever connection the backend holds, and be safe to call twice.
|
|
126
|
+
*
|
|
127
|
+
* REQUIRED, not optional, and deliberately so: it mirrors PHP's
|
|
128
|
+
* Tina4\Queue\QueueBackend, where close() has always been part of the
|
|
129
|
+
* interface. Optional would reintroduce exactly the bug this closes — a
|
|
130
|
+
* caller feature-detecting `backend.close?.()` silently skips the backend
|
|
131
|
+
* that forgot to implement it, which is how tina4-ruby's lite backend went
|
|
132
|
+
* un-closed by every `respond_to?(:close)` guard in its tree.
|
|
133
|
+
*/
|
|
134
|
+
close(): void;
|
|
106
135
|
// Optional full lifecycle. Reservation-based backends (MongoDB) implement
|
|
107
136
|
// these so complete()/fail() ack the ACTIVE store — without complete(), a
|
|
108
|
-
// reserved Mongo job is re-delivered after the visibility window.
|
|
109
|
-
//
|
|
110
|
-
//
|
|
137
|
+
// reserved Mongo job is re-delivered after the visibility window.
|
|
138
|
+
//
|
|
139
|
+
// Optional is a wart, not a design: a backend that omitted them silently fell
|
|
140
|
+
// through to the LOCAL FILE store, so a fail() on a broker-backed queue wrote
|
|
141
|
+
// JSON to disk while the broker still held the message. The only backends
|
|
142
|
+
// that did that were RabbitMQ and Kafka, and ADR-0022 now refuses both, so
|
|
143
|
+
// nothing in tree relies on the fallback. Make these required when the
|
|
144
|
+
// persistent-connection rewrite lands.
|
|
111
145
|
complete?(queue: string, id: string): void;
|
|
112
146
|
fail?(queue: string, id: string, error: string, maxRetries: number, retryBackoff: number): void;
|
|
113
147
|
retry?(queue: string, id: string, delaySeconds?: number): void;
|
|
@@ -117,6 +151,50 @@ export interface QueueBackendInterface {
|
|
|
117
151
|
purge?(queue: string, status?: string): number;
|
|
118
152
|
}
|
|
119
153
|
|
|
154
|
+
/**
|
|
155
|
+
* Why `backend: "rabbitmq"` and `backend: "kafka"` are refused in Node (ADR-0022).
|
|
156
|
+
*
|
|
157
|
+
* Both backends drive the wire protocol through `execFileSync`, spawning a fresh
|
|
158
|
+
* child process for EVERY operation. The child connects, performs one operation,
|
|
159
|
+
* destroys its socket and exits, so no connection, channel, consumer or session
|
|
160
|
+
* survives between push, pop and complete.
|
|
161
|
+
*
|
|
162
|
+
* That makes acknowledgement impossible, not merely unimplemented. An AMQP
|
|
163
|
+
* delivery tag identifies a delivery on a CHANNEL that is already closed by the
|
|
164
|
+
* time pop() returns, and a Kafka consumer-group offset belongs to a SESSION
|
|
165
|
+
* that never existed. So RabbitMQ ran Basic.Get with no-ack=true (at-most-once:
|
|
166
|
+
* a consumer that dies after pop() loses the job outright) and Kafka fetched
|
|
167
|
+
* from offset 0 every time (the topic could never drain).
|
|
168
|
+
*
|
|
169
|
+
* Refusing loudly follows the same call the session backend already made for
|
|
170
|
+
* `redis-npm`: a silent demotion looks like it is working while the operator
|
|
171
|
+
* believes otherwise, which is worse than an outage you can see. Anyone
|
|
172
|
+
* "successfully" running these today is already losing jobs and does not know it.
|
|
173
|
+
*
|
|
174
|
+
* This is a HOLDING POSITION, not the design. The fix is a persistent
|
|
175
|
+
* connection held on the backend instance, the way Python, PHP and Ruby already
|
|
176
|
+
* do it. There is deliberately no opt-in escape hatch: nobody has asked for
|
|
177
|
+
* fire-and-forget, and a knob for a hypothetical user is not worth its weight.
|
|
178
|
+
*/
|
|
179
|
+
function unsupportedBrokerMessage(backendName: string): string {
|
|
180
|
+
const broker = backendName === "kafka" ? "Kafka" : "RabbitMQ";
|
|
181
|
+
const lost = backendName === "kafka"
|
|
182
|
+
? "every pop() re-read offset 0, so the topic could never drain"
|
|
183
|
+
: "pop() destroyed the message with no acknowledgement, so a consumer that died lost the job";
|
|
184
|
+
return (
|
|
185
|
+
`Queue backend "${backendName}" is not available in tina4-nodejs.\n\n` +
|
|
186
|
+
`Reason: the ${broker} backend runs each operation in a separate child ` +
|
|
187
|
+
`process, so no connection survives between pop() and complete(). ` +
|
|
188
|
+
`Acknowledgement is therefore impossible, and ${lost}. It was losing work ` +
|
|
189
|
+
`silently, so it now refuses instead.\n\n` +
|
|
190
|
+
`Use backend "mongodb" (at-least-once, with a real reservation and ` +
|
|
191
|
+
`visibility timeout) or the default "file" backend. tina4-python, ` +
|
|
192
|
+
`tina4-php and tina4-ruby still offer ${broker}.\n\n` +
|
|
193
|
+
`Tracking: ADR-0022 in tina4-documentation/plan/v3/DECISIONS.md, ` +
|
|
194
|
+
`findings in plan/v3/features/048-queue-backends.md.`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
|
|
120
198
|
// ── Queue ────────────────────────────────────────────────────
|
|
121
199
|
|
|
122
200
|
export class Queue {
|
|
@@ -133,26 +211,28 @@ export class Queue {
|
|
|
133
211
|
* Unified Queue constructor.
|
|
134
212
|
*
|
|
135
213
|
* Accepts either:
|
|
136
|
-
* - new Queue({ topic: "tasks", backend: "
|
|
137
|
-
* - new Queue("
|
|
214
|
+
* - new Queue({ topic: "tasks", backend: "mongodb" })
|
|
215
|
+
* - new Queue("mongodb", { path: "data/queue" }) // legacy
|
|
138
216
|
* - new Queue() // file backend, default topic
|
|
217
|
+
*
|
|
218
|
+
* Throws on backend "rabbitmq" or "kafka" (ADR-0022).
|
|
139
219
|
*/
|
|
140
220
|
constructor(backendOrConfig?: string | QueueConfig, config?: QueueConfig) {
|
|
141
221
|
let resolvedConfig: QueueConfig = {};
|
|
142
222
|
|
|
143
223
|
if (typeof backendOrConfig === "string") {
|
|
144
|
-
// Legacy: new Queue("
|
|
224
|
+
// Legacy: new Queue("mongodb", { ... })
|
|
145
225
|
resolvedConfig = { ...(config ?? {}), backend: backendOrConfig };
|
|
146
226
|
} else if (typeof backendOrConfig === "object" && backendOrConfig !== null) {
|
|
147
227
|
resolvedConfig = backendOrConfig;
|
|
148
228
|
}
|
|
149
229
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
230
|
+
// Normalised (trimmed + lowercased) so " MongoDB " resolves, matching the
|
|
231
|
+
// Python master, Ruby and PHP. An unrecognised value THROWS below.
|
|
232
|
+
this.backendName = String(
|
|
233
|
+
resolvedConfig.backend ?? process.env.TINA4_QUEUE_BACKEND ?? "file",
|
|
234
|
+
).trim().toLowerCase();
|
|
235
|
+
this.basePath = resolvedConfig.path ?? queueBasePath();
|
|
156
236
|
this.topic = resolvedConfig.topic ?? "default";
|
|
157
237
|
this._maxRetries = resolvedConfig.maxRetries ?? 3;
|
|
158
238
|
this._retryBackoff = resolvedConfig.retryBackoff ?? 0;
|
|
@@ -160,15 +240,24 @@ export class Queue {
|
|
|
160
240
|
this.liteBackend = new LiteBackend(this.basePath, this._visibilityTimeout);
|
|
161
241
|
|
|
162
242
|
// Initialize external backends
|
|
163
|
-
if (this.backendName === "rabbitmq") {
|
|
164
|
-
|
|
165
|
-
// channel close) — the framework timeout is accepted but not used.
|
|
166
|
-
this.externalBackend = new RabbitMQBackend({ visibilityTimeout: this._visibilityTimeout });
|
|
167
|
-
} else if (this.backendName === "kafka") {
|
|
168
|
-
// Consumer-group offsets manage redelivery — framework timeout N/A.
|
|
169
|
-
this.externalBackend = new KafkaBackend({ visibilityTimeout: this._visibilityTimeout });
|
|
243
|
+
if (this.backendName === "rabbitmq" || this.backendName === "kafka") {
|
|
244
|
+
throw new Error(unsupportedBrokerMessage(this.backendName));
|
|
170
245
|
} else if (this.backendName === "mongodb" || this.backendName === "mongo") {
|
|
171
246
|
this.externalBackend = new MongoBackend({ visibilityTimeout: this._visibilityTimeout });
|
|
247
|
+
} else if (!["file", "default", "lite"].includes(this.backendName)) {
|
|
248
|
+
// An UNRECOGNISED backend name THROWS rather than falling through to the
|
|
249
|
+
// local file store.
|
|
250
|
+
//
|
|
251
|
+
// MEASURED 2026-08-03: a typo in TINA4_QUEUE_BACKEND produced a running
|
|
252
|
+
// app writing every job to local disk while the operator believed they
|
|
253
|
+
// were in MongoDB - jobs nothing consumes, on a container filesystem that
|
|
254
|
+
// vanishes on the next deploy, with no error at any point. Python and Ruby
|
|
255
|
+
// already raise here; this is the same rule the session backend adopted
|
|
256
|
+
// for the same reason.
|
|
257
|
+
throw new Error(
|
|
258
|
+
`Unknown queue backend: '${this.backendName}'. ` +
|
|
259
|
+
`Use 'file', 'rabbitmq', 'kafka', or 'mongodb'.`,
|
|
260
|
+
);
|
|
172
261
|
}
|
|
173
262
|
}
|
|
174
263
|
|
|
@@ -202,7 +291,10 @@ export class Queue {
|
|
|
202
291
|
*/
|
|
203
292
|
push(payload: unknown, delay?: number, priority: number = 0): string {
|
|
204
293
|
if (this.externalBackend) {
|
|
205
|
-
|
|
294
|
+
// priority goes to the external backend too. It used to be passed only to
|
|
295
|
+
// liteBackend, so switching to mongodb silently turned a prioritised
|
|
296
|
+
// queue into a FIFO one.
|
|
297
|
+
return this.externalBackend.push(this.topic, payload, delay, priority);
|
|
206
298
|
}
|
|
207
299
|
return this.liteBackend.push(this.topic, payload, delay, priority);
|
|
208
300
|
}
|
|
@@ -214,7 +306,15 @@ export class Queue {
|
|
|
214
306
|
const q = this.topic;
|
|
215
307
|
|
|
216
308
|
if (this.externalBackend) {
|
|
217
|
-
|
|
309
|
+
const raw = this.externalBackend.pop(q);
|
|
310
|
+
// Wrap it. An external backend returns PLAIN DATA with no lifecycle
|
|
311
|
+
// methods, so `queue.pop().fail("boom")` threw
|
|
312
|
+
// "TypeError: j.fail is not a function" on mongodb/rabbitmq/kafka while
|
|
313
|
+
// working on file — identical application code, different outcome, which
|
|
314
|
+
// is exactly what ADR-0024 forbids. createJob attaches
|
|
315
|
+
// complete()/fail()/reject()/retry(), and they route back through
|
|
316
|
+
// _completeJob/_failJob, which already dispatch to the external backend.
|
|
317
|
+
return raw ? createJob(raw as any, this) : null;
|
|
218
318
|
}
|
|
219
319
|
return this.liteBackend.pop(q, this);
|
|
220
320
|
}
|
|
@@ -223,6 +323,19 @@ export class Queue {
|
|
|
223
323
|
* Pop up to count jobs at once. Returns a partial batch if fewer available.
|
|
224
324
|
*/
|
|
225
325
|
popBatch(count: number): QueueJob[] {
|
|
326
|
+
// Route to the CONFIGURED backend. This used to read the LOCAL FILE STORE
|
|
327
|
+
// unconditionally, so a mongodb-backed queue always came back empty.
|
|
328
|
+
// Repeated pop() is the correct batch on an external backend: each claim is
|
|
329
|
+
// atomic, and a short batch simply means the queue drained.
|
|
330
|
+
if (this.externalBackend) {
|
|
331
|
+
const jobs: QueueJob[] = [];
|
|
332
|
+
for (let i = 0; i < count; i++) {
|
|
333
|
+
const job = this.pop();
|
|
334
|
+
if (!job) break;
|
|
335
|
+
jobs.push(job);
|
|
336
|
+
}
|
|
337
|
+
return jobs;
|
|
338
|
+
}
|
|
226
339
|
return this.liteBackend.popBatch(this.topic, this, count);
|
|
227
340
|
}
|
|
228
341
|
|
|
@@ -311,6 +424,34 @@ export class Queue {
|
|
|
311
424
|
return this.liteBackend.clear(q);
|
|
312
425
|
}
|
|
313
426
|
|
|
427
|
+
/**
|
|
428
|
+
* Release the backend's connection and free its resources.
|
|
429
|
+
*
|
|
430
|
+
* MEASURED 2026-08-04: close() was absent on the top-level Queue in ALL FOUR
|
|
431
|
+
* frameworks, and in Node it was absent on every backend class too — so an
|
|
432
|
+
* application had no way at all to hand a queue's client back. Same class of
|
|
433
|
+
* leak as ADR-0025 corollary 4 (client-lifecycle-is-bounded).
|
|
434
|
+
*
|
|
435
|
+
* Safe on EVERY backend: the file backend holds no connection and closes as a
|
|
436
|
+
* documented no-op, so a TINA4_QUEUE_BACKEND change never turns a working
|
|
437
|
+
* shutdown path into an error. Idempotent — each backend drops its handles on
|
|
438
|
+
* the first call, so a second call finds nothing to close and returns.
|
|
439
|
+
*
|
|
440
|
+
* HONEST CAVEAT specific to Node: neither backend it can reach holds a
|
|
441
|
+
* connection between calls today. The Mongo backend runs each operation in
|
|
442
|
+
* its own child process (ADR-0022), which closes its own client before it
|
|
443
|
+
* exits, and rabbitmq/kafka are refused outright at construction. So this
|
|
444
|
+
* releases nothing YET — it is here for the contract, and because the day the
|
|
445
|
+
* persistent-connection rewrite lands the client is released here with no
|
|
446
|
+
* change at any call site. Python, PHP and Ruby release a REAL client through
|
|
447
|
+
* the identically-named method.
|
|
448
|
+
*
|
|
449
|
+
* Treat the queue as spent afterwards and build a new one to keep working.
|
|
450
|
+
*/
|
|
451
|
+
close(): void {
|
|
452
|
+
(this.externalBackend ?? this.liteBackend).close();
|
|
453
|
+
}
|
|
454
|
+
|
|
314
455
|
/**
|
|
315
456
|
* Get jobs that failed at least once but are still being retried
|
|
316
457
|
* (0 < attempts < maxRetries). These live in the pending queue under the
|
|
@@ -497,6 +638,21 @@ export class Queue {
|
|
|
497
638
|
* Pop a specific job by ID from this queue's topic.
|
|
498
639
|
*/
|
|
499
640
|
popById(id: string): QueueJob | null {
|
|
641
|
+
// Same defect as popBatch: this read the local file store on every backend.
|
|
642
|
+
if (this.externalBackend) {
|
|
643
|
+
const claim = (this.externalBackend as any).popById;
|
|
644
|
+
if (typeof claim !== "function") {
|
|
645
|
+
throw new Error(
|
|
646
|
+
`The ${this.backendName} queue backend cannot perform popById(): it ` +
|
|
647
|
+
`cannot address a single message by id. Use the file or mongodb backend.`,
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
// Same shape as pop() — and, like pop(), PLAIN DATA with no lifecycle
|
|
651
|
+
// methods, so it must be wrapped or job.complete()/job.fail() is a
|
|
652
|
+
// TypeError on every external backend.
|
|
653
|
+
const raw = claim.call(this.externalBackend, this.topic, id);
|
|
654
|
+
return raw ? createJob(raw as any, this) : null;
|
|
655
|
+
}
|
|
500
656
|
return this.liteBackend.popById(this.topic, id);
|
|
501
657
|
}
|
|
502
658
|
|
|
@@ -92,7 +92,7 @@ export function kafkaSecurityConfig(
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
export interface QueueBackend {
|
|
95
|
-
push(queue: string, payload: unknown, delay?: number): string;
|
|
95
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string;
|
|
96
96
|
pop(queue: string): QueueJob | null;
|
|
97
97
|
size(queue: string): number;
|
|
98
98
|
clear(queue: string): void;
|
|
@@ -74,6 +74,19 @@ export class LiteBackend {
|
|
|
74
74
|
return `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* No-op: the file backend holds no connection to release.
|
|
79
|
+
*
|
|
80
|
+
* It exists so `Queue.close()` can call ONE method on every backend instead
|
|
81
|
+
* of testing for it, and so switching TINA4_QUEUE_BACKEND to "file" never
|
|
82
|
+
* turns a working close() into "backend.close is not a function". Idempotent
|
|
83
|
+
* by construction — there is nothing to drop.
|
|
84
|
+
*/
|
|
85
|
+
close(): void {
|
|
86
|
+
// Nothing held: every operation opens, reads/writes and closes its own file
|
|
87
|
+
// descriptor synchronously, so no handle survives a call.
|
|
88
|
+
}
|
|
89
|
+
|
|
77
90
|
push(queue: string, payload: unknown, delay?: number, priority?: number): string {
|
|
78
91
|
const dir = this.ensureDir(queue);
|
|
79
92
|
const id = randomUUID();
|
|
@@ -44,10 +44,12 @@ export interface MongoConfig {
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
export interface QueueBackend {
|
|
47
|
-
push(queue: string, payload: unknown, delay?: number): string;
|
|
47
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string;
|
|
48
48
|
pop(queue: string): QueueJob | null;
|
|
49
49
|
size(queue: string): number;
|
|
50
50
|
clear(queue: string): void;
|
|
51
|
+
/** Release whatever connection the backend holds. Must be idempotent. */
|
|
52
|
+
close(): void;
|
|
51
53
|
}
|
|
52
54
|
|
|
53
55
|
// ── MongoDB Backend ──────────────────────────────────────────
|
|
@@ -214,12 +216,31 @@ export class MongoBackend implements QueueBackend {
|
|
|
214
216
|
{
|
|
215
217
|
queue: queueName,
|
|
216
218
|
status: "pending",
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
219
|
+
// TWO INDEPENDENT GATES, both of which must pass: the
|
|
220
|
+
// reservation gate (availableAt) and the delay gate
|
|
221
|
+
// (delayUntil). These used to share ONE $or, which made them
|
|
222
|
+
// alternatives rather than requirements — a freshly pushed
|
|
223
|
+
// delayed job has no availableAt, matched
|
|
224
|
+
// { availableAt: { $exists: false } }, and was handed straight
|
|
225
|
+
// to a consumer. That is why push(payload, delay) fired
|
|
226
|
+
// immediately on Mongo and on time on the file backend.
|
|
227
|
+
// The $exists arms keep documents written before either field
|
|
228
|
+
// existed claimable, instead of stranding them forever.
|
|
229
|
+
$and: [
|
|
230
|
+
{
|
|
231
|
+
$or: [
|
|
232
|
+
{ availableAt: null },
|
|
233
|
+
{ availableAt: { $exists: false } },
|
|
234
|
+
{ availableAt: { $lte: now } },
|
|
235
|
+
],
|
|
236
|
+
},
|
|
237
|
+
{
|
|
238
|
+
$or: [
|
|
239
|
+
{ delayUntil: null },
|
|
240
|
+
{ delayUntil: { $exists: false } },
|
|
241
|
+
{ delayUntil: { $lte: now } },
|
|
242
|
+
],
|
|
243
|
+
},
|
|
223
244
|
],
|
|
224
245
|
},
|
|
225
246
|
{ $set: { status: "reserved", reservedAt: now, availableAt: future } },
|
|
@@ -250,6 +271,28 @@ export class MongoBackend implements QueueBackend {
|
|
|
250
271
|
process.stdout.write("__EMPTY__");
|
|
251
272
|
}
|
|
252
273
|
}
|
|
274
|
+
else if (operation === "popById") {
|
|
275
|
+
// Claim ONE specific job by id, the same way pop() claims the head.
|
|
276
|
+
// Queue.popById used to read the LOCAL FILE STORE regardless of the
|
|
277
|
+
// configured backend, so it never saw a mongodb job at all.
|
|
278
|
+
const now = new Date().toISOString();
|
|
279
|
+
const future = new Date(Date.now() + visibilityTimeout * 1000).toISOString();
|
|
280
|
+
const wanted = JSON.parse(data);
|
|
281
|
+
const result = await col.findOneAndUpdate(
|
|
282
|
+
{ queue: queueName, status: "pending", id: wanted.id },
|
|
283
|
+
{ $set: { status: "reserved", reservedAt: now, availableAt: future } },
|
|
284
|
+
{ returnDocument: "before" },
|
|
285
|
+
);
|
|
286
|
+
const doc = result && result.value ? result.value : (result && result._id ? { ...result } : null);
|
|
287
|
+
if (doc) {
|
|
288
|
+
doc.topic = queueName;
|
|
289
|
+
delete doc._id;
|
|
290
|
+
delete doc.queue;
|
|
291
|
+
process.stdout.write(JSON.stringify(doc));
|
|
292
|
+
} else {
|
|
293
|
+
process.stdout.write("__EMPTY__");
|
|
294
|
+
}
|
|
295
|
+
}
|
|
253
296
|
else if (operation === "size") {
|
|
254
297
|
const count = await col.countDocuments({
|
|
255
298
|
queue: queueName,
|
|
@@ -318,8 +361,20 @@ export class MongoBackend implements QueueBackend {
|
|
|
318
361
|
process.stdout.write(JSON.stringify(out));
|
|
319
362
|
}
|
|
320
363
|
else if (operation === "failed") {
|
|
364
|
+
// Found by the ATTEMPTS COUNTER, not by a "failed" status. The
|
|
365
|
+
// fail() branch above re-queues a still-retryable job as "pending"
|
|
366
|
+
// (that is what makes the next pop redeliver it) and dead-letters
|
|
367
|
+
// an exhausted one as "dead" - nothing ever writes "failed", so
|
|
368
|
+
// this query matched nothing and returned [] forever. An empty
|
|
369
|
+
// list is indistinguishable from "nothing has failed"
|
|
370
|
+
// (ADR-0022 decision 7). attempts > 0 is the real marker of a job
|
|
371
|
+
// that has already died at least once.
|
|
321
372
|
const docs = await col
|
|
322
|
-
.find({
|
|
373
|
+
.find({
|
|
374
|
+
queue: queueName,
|
|
375
|
+
status: "pending",
|
|
376
|
+
attempts: { $gt: 0, $lt: maxRetries },
|
|
377
|
+
})
|
|
323
378
|
.toArray();
|
|
324
379
|
const out = docs.map((d) => { delete d._id; delete d.queue; return d; });
|
|
325
380
|
process.stdout.write(JSON.stringify(out));
|
|
@@ -380,7 +435,17 @@ export class MongoBackend implements QueueBackend {
|
|
|
380
435
|
}
|
|
381
436
|
}
|
|
382
437
|
|
|
383
|
-
|
|
438
|
+
popById(queue: string, id: string): QueueJob | null {
|
|
439
|
+
const out = this.execSync("popById", queue, JSON.stringify({ id }));
|
|
440
|
+
if (!out || out === "__EMPTY__") return null;
|
|
441
|
+
try {
|
|
442
|
+
return JSON.parse(out) as QueueJob;
|
|
443
|
+
} catch {
|
|
444
|
+
return null;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string {
|
|
384
449
|
const id = randomUUID();
|
|
385
450
|
const now = new Date().toISOString();
|
|
386
451
|
|
|
@@ -391,6 +456,11 @@ export class MongoBackend implements QueueBackend {
|
|
|
391
456
|
createdAt: now,
|
|
392
457
|
attempts: 0,
|
|
393
458
|
delayUntil: delay ? new Date(Date.now() + delay * 1000).toISOString() : null,
|
|
459
|
+
// The pop sort has always been { priority: -1, createdAt: 1 }, but this
|
|
460
|
+
// field was never written, so every job scored undefined and the queue
|
|
461
|
+
// ran pure FIFO. Priority did not even reach here: the backend interface
|
|
462
|
+
// had no such parameter and Queue.push dropped it for external backends.
|
|
463
|
+
priority: priority ?? 0,
|
|
394
464
|
};
|
|
395
465
|
|
|
396
466
|
const result = this.execSync("push", queue, JSON.stringify(job));
|
|
@@ -471,4 +541,26 @@ export class MongoBackend implements QueueBackend {
|
|
|
471
541
|
const out = this.execSync("purge", queue, JSON.stringify({ status: status ?? "" }));
|
|
472
542
|
return parseInt(out, 10) || 0;
|
|
473
543
|
}
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Release the MongoDB connection. Idempotent — a second call is a no-op.
|
|
547
|
+
*
|
|
548
|
+
* HONEST CAVEAT, and it is the whole reason ADR-0022 exists: THIS backend
|
|
549
|
+
* holds no connection between calls to release. Every operation runs in its
|
|
550
|
+
* own child process (see execSync/buildScript), and that child's `finally`
|
|
551
|
+
* already does `await client.close()` before it exits — so the pool it opened
|
|
552
|
+
* is gone by the time the method returns. Unlike tina4-python, tina4-php and
|
|
553
|
+
* tina4-ruby, whose Mongo/broker backends hold a long-lived client that this
|
|
554
|
+
* method genuinely hands back, Node has nothing to give back.
|
|
555
|
+
*
|
|
556
|
+
* It is implemented anyway, and required by the QueueBackend interface,
|
|
557
|
+
* because the CONTRACT is what matters: `Queue.close()` must be callable on
|
|
558
|
+
* every backend in every framework, and the day the persistent-connection
|
|
559
|
+
* rewrite lands (ADR-0022's tracked fix) the client goes here with no change
|
|
560
|
+
* at any call site. A method that is a no-op today and correct forever beats
|
|
561
|
+
* a missing method the caller has to feature-detect.
|
|
562
|
+
*/
|
|
563
|
+
close(): void {
|
|
564
|
+
// Nothing held: the per-operation child process owns and closes its client.
|
|
565
|
+
}
|
|
474
566
|
}
|
|
@@ -40,8 +40,8 @@ export interface RabbitMQConfig {
|
|
|
40
40
|
* Parse an AMQP URL (amqp://[user:pass@]host[:port][/vhost]) into a partial
|
|
41
41
|
* RabbitMQConfig. Mirrors the Python/PHP/Ruby `parse_amqp_url` semantics:
|
|
42
42
|
* strips a leading amqp:// or amqps:// scheme, splits optional credentials,
|
|
43
|
-
* and
|
|
44
|
-
* in the URL are populated.
|
|
43
|
+
* and reads the path segment as the URL-decoded vhost name. Only fields
|
|
44
|
+
* present in the URL are populated.
|
|
45
45
|
*/
|
|
46
46
|
export function parseAmqpUrl(url: string): RabbitMQConfig {
|
|
47
47
|
const config: RabbitMQConfig = {};
|
|
@@ -65,8 +65,26 @@ export function parseAmqpUrl(url: string): RabbitMQConfig {
|
|
|
65
65
|
if (slashIndex !== -1) {
|
|
66
66
|
hostport = rest.slice(0, slashIndex);
|
|
67
67
|
const vhost = rest.slice(slashIndex + 1);
|
|
68
|
+
// THE VHOST IS THE PATH SEGMENT, URL-DECODED, WITH NO LEADING SLASH
|
|
69
|
+
// (RabbitMQ URI spec). This used to prepend "/", so
|
|
70
|
+
// amqp://guest:guest@rabbit:5672/orders asked for a vhost literally named
|
|
71
|
+
// "/orders". No broker has that one - it is named "orders" - so every
|
|
72
|
+
// publish failed against a named vhost, which is the ordinary multi-tenant
|
|
73
|
+
// setup and the form every RabbitMQ tutorial shows. MEASURED against a real
|
|
74
|
+
// broker: 4 of 5 URL shapes resolved to the wrong name, and the only one
|
|
75
|
+
// that worked carried no vhost at all, which is why four green suites never
|
|
76
|
+
// noticed.
|
|
77
|
+
//
|
|
78
|
+
// Decoding matters for the same reason: the DEFAULT vhost is named "/",
|
|
79
|
+
// which cannot appear literally in a path, so the spec spells it "%2f".
|
|
80
|
+
//
|
|
81
|
+
// DELIBERATE DEVIATION, one shape: the spec reads a bare trailing slash as
|
|
82
|
+
// the EMPTY vhost name. Tina4 treats it as "not specified" and keeps the
|
|
83
|
+
// caller's default - nobody writes a trailing slash intending a vhost named
|
|
84
|
+
// "", and reading it literally would break a working "amqp://host:5672/"
|
|
85
|
+
// for no benefit.
|
|
68
86
|
if (vhost) {
|
|
69
|
-
config.vhost = vhost
|
|
87
|
+
config.vhost = decodeURIComponent(vhost);
|
|
70
88
|
}
|
|
71
89
|
}
|
|
72
90
|
|
|
@@ -82,7 +100,7 @@ export function parseAmqpUrl(url: string): RabbitMQConfig {
|
|
|
82
100
|
}
|
|
83
101
|
|
|
84
102
|
export interface QueueBackend {
|
|
85
|
-
push(queue: string, payload: unknown, delay?: number): string;
|
|
103
|
+
push(queue: string, payload: unknown, delay?: number, priority?: number): string;
|
|
86
104
|
pop(queue: string): QueueJob | null;
|
|
87
105
|
size(queue: string): number;
|
|
88
106
|
clear(queue: string): void;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Middleware, Tina4Request, Tina4Response } from "./types.js";
|
|
2
|
+
import { resolveClientIp } from "./trustedProxy.js";
|
|
2
3
|
|
|
3
4
|
/** Per-IP sliding window entry */
|
|
4
5
|
interface RateLimitEntry {
|
|
@@ -59,11 +60,15 @@ export function rateLimiter(config?: RateLimiterConfig): Middleware {
|
|
|
59
60
|
const now = Date.now();
|
|
60
61
|
const cutoff = now - windowMs;
|
|
61
62
|
|
|
62
|
-
//
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
// Client key. X-Forwarded-For is honoured ONLY when the socket peer is a
|
|
64
|
+
// declared trusted proxy (TINA4_TRUSTED_PROXIES) - otherwise any client
|
|
65
|
+
// could pick its own bucket, and pick someone else's. ADR-0019.
|
|
66
|
+
//
|
|
67
|
+
// This derived the key itself rather than reading req.ip, and its
|
|
68
|
+
// `typeof forwarded === "string"` test meant a REPEATED header (which
|
|
69
|
+
// arrives as an array) silently fell through to the socket address -
|
|
70
|
+
// inconsistent with req.ip, which did read the array.
|
|
71
|
+
const ip = resolveClientIp(req.headers, req.socket?.remoteAddress ?? "") || "unknown";
|
|
67
72
|
|
|
68
73
|
// Get or create entry
|
|
69
74
|
let entry = store.get(ip);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { IncomingMessage, IncomingHttpHeaders } from "node:http";
|
|
2
2
|
import type { Tina4Request, UploadedFile } from "./types.js";
|
|
3
|
+
import { resolveClientIp } from "./trustedProxy.js";
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Wrap Node's `IncomingHttpHeaders` in a Proxy so mixed-case lookups
|
|
@@ -102,15 +103,11 @@ export function createRequest(req: IncomingMessage): Tina4Request {
|
|
|
102
103
|
}
|
|
103
104
|
tReq.cookies = cookies;
|
|
104
105
|
|
|
105
|
-
//
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
tReq.ip = forwarded[0].split(",")[0].trim();
|
|
111
|
-
} else {
|
|
112
|
-
tReq.ip = req.socket?.remoteAddress ?? "127.0.0.1";
|
|
113
|
-
}
|
|
106
|
+
// Raw socket peer — NEVER honours a forwarding header, so it can be trusted
|
|
107
|
+
// for security decisions. Resolved BEFORE .ip: the peer decides whether the
|
|
108
|
+
// forwarding headers may be believed at all (TINA4_TRUSTED_PROXIES, ADR-0019).
|
|
109
|
+
tReq.remoteIp = req.socket?.remoteAddress ?? "";
|
|
110
|
+
tReq.ip = resolveClientIp(req.headers, tReq.remoteIp) || "127.0.0.1";
|
|
114
111
|
|
|
115
112
|
// Add convenience methods
|
|
116
113
|
tReq.header = function (name: string): string | undefined {
|