tina4-nodejs 3.13.32 → 3.13.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +2 -2
- package/package.json +1 -1
- package/packages/cli/src/commands/init.ts +3 -3
- package/packages/core/src/index.ts +1 -1
- package/packages/core/src/job.ts +6 -1
- package/packages/core/src/queue.ts +20 -4
- package/packages/core/src/queueBackends/kafkaBackend.ts +21 -2
- package/packages/core/src/queueBackends/liteBackend.ts +186 -101
- package/packages/core/src/queueBackends/mongoBackend.ts +18 -4
- package/packages/core/src/queueBackends/rabbitmqBackend.ts +79 -10
- package/packages/core/src/server.ts +12 -20
package/CLAUDE.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# CLAUDE.md — AI Developer Guide for tina4-nodejs (v3.13.
|
|
1
|
+
# CLAUDE.md — AI Developer Guide for tina4-nodejs (v3.13.34)
|
|
2
2
|
|
|
3
3
|
> This file helps AI assistants (Claude, Copilot, Cursor, etc.) understand and work on this codebase effectively.
|
|
4
4
|
|
|
@@ -946,7 +946,7 @@ The `tina4` Rust CLI is the sole file watcher for the Tina4 stack — there is n
|
|
|
946
946
|
|
|
947
947
|
No configuration needed — set `TINA4_DEBUG=true` to enable. If you're running without the Rust CLI (e.g. Docker), there is no automatic reload; the production path is unaffected.
|
|
948
948
|
|
|
949
|
-
**AI dual-port mode:** when `TINA4_DEBUG=true` and `TINA4_NO_AI_PORT` is unset, the main port
|
|
949
|
+
**AI dual-port mode:** when `TINA4_DEBUG=true` and `TINA4_NO_AI_PORT` is unset, the **main port** provides the normal hot-reload experience (dev toolbar + `/__dev_reload` injected) for the human dev, and a second server on `port+1000` is the **stable AI port** — it suppresses reload/toolbar injection (and returns 404 for `/__dev_reload`) so an AI tool can drive it without its own edits triggering a refresh. The `tina4` client posts `/__dev/api/reload` to the **main port**. Matches Python (master), PHP, and Ruby.
|
|
950
950
|
|
|
951
951
|
## Conventions You Must Follow
|
|
952
952
|
|
package/package.json
CHANGED
|
@@ -63,11 +63,11 @@ export async function initProject(name: string): Promise<void> {
|
|
|
63
63
|
private: true,
|
|
64
64
|
type: "module",
|
|
65
65
|
scripts: {
|
|
66
|
-
dev: "
|
|
67
|
-
serve: "
|
|
66
|
+
dev: "npx tina4nodejs serve",
|
|
67
|
+
serve: "npx tina4nodejs serve",
|
|
68
68
|
},
|
|
69
69
|
dependencies: {
|
|
70
|
-
"tina4-nodejs": "^0.0
|
|
70
|
+
"tina4-nodejs": "^3.0.0",
|
|
71
71
|
},
|
|
72
72
|
devDependencies: {
|
|
73
73
|
typescript: "^5.7.0",
|
|
@@ -96,7 +96,7 @@ export { AI_TOOLS, isInstalled, showMenu, installSelected, installAll, generateC
|
|
|
96
96
|
export type { AiTool } from "./ai.js";
|
|
97
97
|
export type { ImapMessage, ImapFullMessage } from "./messenger.js";
|
|
98
98
|
export { LiteBackend } from "./queueBackends/liteBackend.js";
|
|
99
|
-
export { RabbitMQBackend } from "./queueBackends/rabbitmqBackend.js";
|
|
99
|
+
export { RabbitMQBackend, parseAmqpUrl } from "./queueBackends/rabbitmqBackend.js";
|
|
100
100
|
export type { RabbitMQConfig } from "./queueBackends/rabbitmqBackend.js";
|
|
101
101
|
export { KafkaBackend } from "./queueBackends/kafkaBackend.js";
|
|
102
102
|
export type { KafkaConfig } from "./queueBackends/kafkaBackend.js";
|
package/packages/core/src/job.ts
CHANGED
|
@@ -48,12 +48,17 @@ export function createJob(data: JobData, queue: JobQueueBridge): QueueJob {
|
|
|
48
48
|
const job: QueueJob = {
|
|
49
49
|
...data,
|
|
50
50
|
complete() {
|
|
51
|
+
// Terminal — the job was already removed from the queue on pop().
|
|
51
52
|
job.status = "completed";
|
|
52
53
|
},
|
|
53
54
|
fail(reason = "") {
|
|
55
|
+
// Record a failed attempt. `attempts` is incremented exactly once, inside
|
|
56
|
+
// the backend's failJob() — NOT here — so a persistently-failing job runs
|
|
57
|
+
// exactly maxRetries times before it is dead-lettered. The backend decides
|
|
58
|
+
// whether to re-enqueue (attempts < maxRetries) or dead-letter
|
|
59
|
+
// (attempts >= maxRetries).
|
|
54
60
|
job.status = "failed";
|
|
55
61
|
job.error = reason;
|
|
56
|
-
job.attempts = (job.attempts || 0) + 1;
|
|
57
62
|
queue._failJob(job.topic, job, reason, queue.getMaxRetries());
|
|
58
63
|
},
|
|
59
64
|
reject(reason = "") {
|
|
@@ -45,6 +45,12 @@ export interface QueueConfig {
|
|
|
45
45
|
path?: string;
|
|
46
46
|
topic?: string;
|
|
47
47
|
maxRetries?: number;
|
|
48
|
+
/**
|
|
49
|
+
* Seconds to delay a failed job's automatic re-enqueue. 0 (the default)
|
|
50
|
+
* means retry immediately — the next pop()/consume() iteration picks it up
|
|
51
|
+
* straight away. Parity with Python's retry_backoff.
|
|
52
|
+
*/
|
|
53
|
+
retryBackoff?: number;
|
|
48
54
|
}
|
|
49
55
|
|
|
50
56
|
export interface ProcessOptions {
|
|
@@ -75,6 +81,7 @@ export class Queue {
|
|
|
75
81
|
private basePath: string;
|
|
76
82
|
private topic: string;
|
|
77
83
|
private _maxRetries: number;
|
|
84
|
+
private _retryBackoff: number;
|
|
78
85
|
private externalBackend: QueueBackendInterface | null = null;
|
|
79
86
|
private liteBackend!: LiteBackend;
|
|
80
87
|
|
|
@@ -104,6 +111,7 @@ export class Queue {
|
|
|
104
111
|
?? "data/queue";
|
|
105
112
|
this.topic = resolvedConfig.topic ?? "default";
|
|
106
113
|
this._maxRetries = resolvedConfig.maxRetries ?? 3;
|
|
114
|
+
this._retryBackoff = resolvedConfig.retryBackoff ?? 0;
|
|
107
115
|
this.liteBackend = new LiteBackend(this.basePath);
|
|
108
116
|
|
|
109
117
|
// Initialize external backends
|
|
@@ -234,10 +242,12 @@ export class Queue {
|
|
|
234
242
|
}
|
|
235
243
|
|
|
236
244
|
/**
|
|
237
|
-
* Get
|
|
245
|
+
* Get jobs that failed at least once but are still being retried
|
|
246
|
+
* (0 < attempts < maxRetries). These live in the pending queue under the
|
|
247
|
+
* auto-retry lifecycle; dead-lettered jobs are returned by deadLetters().
|
|
238
248
|
*/
|
|
239
249
|
failed(): QueueJob[] {
|
|
240
|
-
return this.liteBackend.failed(this.topic);
|
|
250
|
+
return this.liteBackend.failed(this.topic, this._maxRetries);
|
|
241
251
|
}
|
|
242
252
|
|
|
243
253
|
/**
|
|
@@ -396,11 +406,17 @@ export class Queue {
|
|
|
396
406
|
return this._maxRetries;
|
|
397
407
|
}
|
|
398
408
|
|
|
409
|
+
getRetryBackoff(): number {
|
|
410
|
+
return this._retryBackoff;
|
|
411
|
+
}
|
|
412
|
+
|
|
399
413
|
/**
|
|
400
|
-
*
|
|
414
|
+
* Record a failed attempt for a job. The backend increments `attempts`
|
|
415
|
+
* exactly once and decides whether to re-enqueue (attempts < maxRetries,
|
|
416
|
+
* after retryBackoff seconds) or dead-letter (attempts >= maxRetries).
|
|
401
417
|
*/
|
|
402
418
|
_failJob(queue: string, job: QueueJob, error: string, maxRetries: number): void {
|
|
403
|
-
this.liteBackend.failJob(queue, job, error, maxRetries);
|
|
419
|
+
this.liteBackend.failJob(queue, job, error, maxRetries, this._retryBackoff);
|
|
404
420
|
}
|
|
405
421
|
|
|
406
422
|
/**
|
|
@@ -5,8 +5,12 @@
|
|
|
5
5
|
* for message storage and delivery.
|
|
6
6
|
*
|
|
7
7
|
* Configure via environment variables:
|
|
8
|
-
*
|
|
8
|
+
* TINA4_QUEUE_URL — broker list (strips a leading kafka:// if present)
|
|
9
|
+
* TINA4_KAFKA_BROKERS (override; default: "localhost:9092")
|
|
9
10
|
* TINA4_KAFKA_GROUP_ID (default: "tina4_consumer_group")
|
|
11
|
+
*
|
|
12
|
+
* Precedence for brokers: specific TINA4_KAFKA_BROKERS var (if set)
|
|
13
|
+
* > value derived from TINA4_QUEUE_URL > existing default.
|
|
10
14
|
*/
|
|
11
15
|
import net from "node:net";
|
|
12
16
|
import { execFileSync } from "node:child_process";
|
|
@@ -54,10 +58,25 @@ export class KafkaBackend implements QueueBackend {
|
|
|
54
58
|
private groupId: string;
|
|
55
59
|
|
|
56
60
|
constructor(config?: KafkaConfig) {
|
|
57
|
-
|
|
61
|
+
// Base layer: brokers derived from TINA4_QUEUE_URL (strip a leading
|
|
62
|
+
// kafka:// scheme if present; otherwise use the value as-is, e.g.
|
|
63
|
+
// "localhost:9092").
|
|
64
|
+
const url = process.env.TINA4_QUEUE_URL;
|
|
65
|
+
const fromUrl = url ? url.replace(/^kafka:\/\//, "") : undefined;
|
|
66
|
+
|
|
67
|
+
// Precedence: explicit config arg > specific TINA4_KAFKA_BROKERS var
|
|
68
|
+
// > value from TINA4_QUEUE_URL > existing default.
|
|
69
|
+
this.brokers = config?.brokers ?? process.env.TINA4_KAFKA_BROKERS ?? fromUrl ?? "localhost:9092";
|
|
58
70
|
this.groupId = config?.groupId ?? process.env.TINA4_KAFKA_GROUP_ID ?? "tina4_consumer_group";
|
|
59
71
|
}
|
|
60
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Resolved connection config — exposed for testing/introspection.
|
|
75
|
+
*/
|
|
76
|
+
getConfig(): Required<KafkaConfig> {
|
|
77
|
+
return { brokers: this.brokers, groupId: this.groupId };
|
|
78
|
+
}
|
|
79
|
+
|
|
61
80
|
/**
|
|
62
81
|
* Parse broker string into host:port.
|
|
63
82
|
*/
|
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* LiteBackend — file-based queue backend for Tina4 Queue.
|
|
3
3
|
* Stores jobs as JSON files on disk. Zero dependencies.
|
|
4
|
+
*
|
|
5
|
+
* Dequeue policy (parity with Python master): the highest-priority AVAILABLE
|
|
6
|
+
* job is returned first; ties are broken oldest-first by createdAt. The file
|
|
7
|
+
* *name* is no longer the ordering key — the stored `priority` and `createdAt`
|
|
8
|
+
* fields are. Delayed jobs (delayUntil in the future) are skipped until due.
|
|
9
|
+
*
|
|
10
|
+
* Failure lifecycle (parity with Python master): job.fail() records one failed
|
|
11
|
+
* attempt — `attempts` is incremented exactly once, here in failJob(). If the
|
|
12
|
+
* job still has retries left (attempts < maxRetries) it is automatically
|
|
13
|
+
* re-enqueued to the pending queue (immediately, or after retryBackoff seconds
|
|
14
|
+
* if configured) so the next pop()/consume() picks it up again. Once it has
|
|
15
|
+
* been attempted maxRetries times (attempts >= maxRetries) it is moved to the
|
|
16
|
+
* dead-letter (failed/) directory, where deadLetters() returns it.
|
|
4
17
|
*/
|
|
5
18
|
import { mkdirSync, readdirSync, readFileSync, writeFileSync, unlinkSync, existsSync } from "node:fs";
|
|
6
19
|
import { join } from "node:path";
|
|
@@ -28,11 +41,15 @@ export class LiteBackend {
|
|
|
28
41
|
return dir;
|
|
29
42
|
}
|
|
30
43
|
|
|
44
|
+
private nextPrefix(): string {
|
|
45
|
+
this.seq++;
|
|
46
|
+
return `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
|
|
47
|
+
}
|
|
48
|
+
|
|
31
49
|
push(queue: string, payload: unknown, delay?: number, priority?: number): string {
|
|
32
50
|
const dir = this.ensureDir(queue);
|
|
33
51
|
const id = randomUUID();
|
|
34
52
|
const now = new Date().toISOString();
|
|
35
|
-
this.seq++;
|
|
36
53
|
|
|
37
54
|
const job = {
|
|
38
55
|
id,
|
|
@@ -42,48 +59,74 @@ export class LiteBackend {
|
|
|
42
59
|
attempts: 0,
|
|
43
60
|
delayUntil: delay ? new Date(Date.now() + delay * 1000).toISOString() : null,
|
|
44
61
|
priority: priority ?? 0,
|
|
62
|
+
topic: queue,
|
|
63
|
+
error: undefined as string | undefined,
|
|
45
64
|
};
|
|
46
65
|
|
|
47
|
-
const prefix =
|
|
66
|
+
const prefix = this.nextPrefix();
|
|
48
67
|
writeFileSync(join(dir, `${prefix}_${id}.queue-data`), JSON.stringify(job, null, 2));
|
|
49
68
|
return id;
|
|
50
69
|
}
|
|
51
70
|
|
|
52
|
-
|
|
71
|
+
/**
|
|
72
|
+
* Return [filename, jobData] for every pending, non-delayed job, ordered by
|
|
73
|
+
* the dequeue policy: highest priority first, ties broken oldest-first by
|
|
74
|
+
* createdAt. createdAt is an ISO-8601 string, so lexicographic comparison ==
|
|
75
|
+
* chronological order.
|
|
76
|
+
*/
|
|
77
|
+
private availableCandidates(queue: string, now: string): Array<[string, any]> {
|
|
53
78
|
const dir = this.ensureDir(queue);
|
|
54
79
|
|
|
55
|
-
let
|
|
80
|
+
let filenames: string[];
|
|
56
81
|
try {
|
|
57
|
-
|
|
82
|
+
filenames = readdirSync(dir).filter(f => f.endsWith(".queue-data"));
|
|
58
83
|
} catch {
|
|
59
|
-
return
|
|
84
|
+
return [];
|
|
60
85
|
}
|
|
61
86
|
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
let job: QueueJob;
|
|
87
|
+
const candidates: Array<[string, any]> = [];
|
|
88
|
+
for (const filename of filenames) {
|
|
89
|
+
const filePath = join(dir, filename);
|
|
90
|
+
let job: any;
|
|
67
91
|
try {
|
|
68
92
|
job = JSON.parse(readFileSync(filePath, "utf-8"));
|
|
69
93
|
} catch {
|
|
70
94
|
continue;
|
|
71
95
|
}
|
|
72
|
-
|
|
73
96
|
if (job.status !== "pending") continue;
|
|
74
|
-
if (job.delayUntil && job.delayUntil > now) continue;
|
|
97
|
+
if (job.delayUntil && job.delayUntil > now) continue; // still delayed
|
|
98
|
+
candidates.push([filename, job]);
|
|
99
|
+
}
|
|
75
100
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
101
|
+
// priority DESC, then createdAt ASC (oldest first).
|
|
102
|
+
candidates.sort((a, b) => {
|
|
103
|
+
const pa = Number(a[1].priority ?? 0) || 0;
|
|
104
|
+
const pb = Number(b[1].priority ?? 0) || 0;
|
|
105
|
+
if (pb !== pa) return pb - pa;
|
|
106
|
+
const ca = (a[1].createdAt ?? "") as string;
|
|
107
|
+
const cb = (b[1].createdAt ?? "") as string;
|
|
108
|
+
return ca < cb ? -1 : ca > cb ? 1 : 0;
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
return candidates;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
pop(queue: string, bridge: JobQueueBridge): QueueJob | null {
|
|
115
|
+
const dir = this.ensureDir(queue);
|
|
116
|
+
const now = new Date().toISOString();
|
|
80
117
|
|
|
118
|
+
for (const [filename, job] of this.availableCandidates(queue, now)) {
|
|
119
|
+
const filePath = join(dir, filename);
|
|
120
|
+
// Claim the job by deleting the file.
|
|
81
121
|
try {
|
|
82
122
|
unlinkSync(filePath);
|
|
83
123
|
} catch {
|
|
84
|
-
// Already consumed by another worker
|
|
124
|
+
continue; // Already consumed by another worker
|
|
85
125
|
}
|
|
86
126
|
|
|
127
|
+
job.status = "reserved";
|
|
128
|
+
job.topic = queue;
|
|
129
|
+
job.priority = job.priority ?? 0;
|
|
87
130
|
return createJob(job as any, bridge);
|
|
88
131
|
}
|
|
89
132
|
|
|
@@ -92,71 +135,52 @@ export class LiteBackend {
|
|
|
92
135
|
|
|
93
136
|
popBatch(queue: string, bridge: JobQueueBridge, count: number): QueueJob[] {
|
|
94
137
|
const dir = this.ensureDir(queue);
|
|
95
|
-
|
|
96
|
-
let files: string[];
|
|
97
|
-
try {
|
|
98
|
-
files = readdirSync(dir).filter(f => f.endsWith(".queue-data")).sort();
|
|
99
|
-
} catch {
|
|
100
|
-
return [];
|
|
101
|
-
}
|
|
102
|
-
|
|
103
138
|
const now = new Date().toISOString();
|
|
104
139
|
const results: QueueJob[] = [];
|
|
105
140
|
|
|
106
|
-
for (const
|
|
141
|
+
for (const [filename, job] of this.availableCandidates(queue, now)) {
|
|
107
142
|
if (results.length >= count) break;
|
|
108
|
-
const filePath = join(dir,
|
|
109
|
-
let job: QueueJob;
|
|
143
|
+
const filePath = join(dir, filename);
|
|
110
144
|
try {
|
|
111
|
-
|
|
145
|
+
unlinkSync(filePath);
|
|
112
146
|
} catch {
|
|
113
|
-
continue;
|
|
147
|
+
continue; // Already consumed by another worker
|
|
114
148
|
}
|
|
115
149
|
|
|
116
|
-
if (job.status !== "pending") continue;
|
|
117
|
-
if (job.delayUntil && job.delayUntil > now) continue;
|
|
118
|
-
|
|
119
150
|
job.status = "reserved";
|
|
120
151
|
job.topic = queue;
|
|
121
152
|
job.priority = job.priority ?? 0;
|
|
122
|
-
writeFileSync(filePath, JSON.stringify(job, null, 2));
|
|
123
|
-
|
|
124
|
-
try {
|
|
125
|
-
unlinkSync(filePath);
|
|
126
|
-
} catch {
|
|
127
|
-
// Already consumed by another worker
|
|
128
|
-
}
|
|
129
|
-
|
|
130
153
|
results.push(createJob(job as any, bridge));
|
|
131
154
|
}
|
|
132
155
|
|
|
133
156
|
return results;
|
|
134
157
|
}
|
|
135
158
|
|
|
159
|
+
// Status aliases that live in the failed/ directory (dead-lettered jobs)
|
|
160
|
+
// rather than as pending files in the queue directory.
|
|
161
|
+
private static readonly DEAD_STATES = ["failed", "dead", "dead_letter"];
|
|
162
|
+
|
|
136
163
|
size(queue: string, status: string = "pending"): number {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
let files: string[];
|
|
140
|
-
try {
|
|
141
|
-
files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
|
|
142
|
-
} catch {
|
|
143
|
-
return 0;
|
|
144
|
-
}
|
|
145
|
-
return files.length;
|
|
146
|
-
}
|
|
164
|
+
const isDead = LiteBackend.DEAD_STATES.includes(status);
|
|
165
|
+
const scanDir = isDead ? this.ensureFailedDir(queue) : this.ensureDir(queue);
|
|
147
166
|
|
|
148
|
-
const dir = this.ensureDir(queue);
|
|
149
167
|
let files: string[];
|
|
150
168
|
try {
|
|
151
|
-
files = readdirSync(
|
|
169
|
+
files = readdirSync(scanDir).filter(f => f.endsWith(".queue-data"));
|
|
152
170
|
} catch {
|
|
153
171
|
return 0;
|
|
154
172
|
}
|
|
155
173
|
|
|
174
|
+
if (isDead) {
|
|
175
|
+
// Every file in failed/ is a dead-letter; count them all regardless of
|
|
176
|
+
// the exact stored status string.
|
|
177
|
+
return files.length;
|
|
178
|
+
}
|
|
179
|
+
|
|
156
180
|
let count = 0;
|
|
157
181
|
for (const file of files) {
|
|
158
182
|
try {
|
|
159
|
-
const job = JSON.parse(readFileSync(join(
|
|
183
|
+
const job = JSON.parse(readFileSync(join(scanDir, file), "utf-8"));
|
|
160
184
|
if (job.status === status) count++;
|
|
161
185
|
} catch {
|
|
162
186
|
// skip corrupt files
|
|
@@ -178,7 +202,7 @@ export class LiteBackend {
|
|
|
178
202
|
// directory might not exist
|
|
179
203
|
}
|
|
180
204
|
|
|
181
|
-
// Also clear
|
|
205
|
+
// Also clear dead-letter jobs.
|
|
182
206
|
const failedDir = join(dir, "failed");
|
|
183
207
|
try {
|
|
184
208
|
if (existsSync(failedDir)) {
|
|
@@ -194,16 +218,27 @@ export class LiteBackend {
|
|
|
194
218
|
return count;
|
|
195
219
|
}
|
|
196
220
|
|
|
197
|
-
|
|
198
|
-
|
|
221
|
+
/**
|
|
222
|
+
* Jobs that have failed at least once but are still being retried.
|
|
223
|
+
*
|
|
224
|
+
* Under the auto-retry lifecycle a failed-but-retryable job lives in the
|
|
225
|
+
* pending queue (not the dead-letter dir), so this scans the queue dir for
|
|
226
|
+
* pending jobs with attempts > 0 that have not yet exhausted their retries.
|
|
227
|
+
* Dead-lettered jobs are returned by deadLetters().
|
|
228
|
+
*/
|
|
229
|
+
failed(queue: string, maxRetries: number = 3): QueueJob[] {
|
|
230
|
+
const dir = this.ensureDir(queue);
|
|
199
231
|
const results: QueueJob[] = [];
|
|
200
232
|
|
|
201
233
|
try {
|
|
202
|
-
const files = readdirSync(
|
|
234
|
+
const files = readdirSync(dir).filter(f => f.endsWith(".queue-data")).sort();
|
|
203
235
|
for (const file of files) {
|
|
204
236
|
try {
|
|
205
|
-
const job: QueueJob = JSON.parse(readFileSync(join(
|
|
206
|
-
|
|
237
|
+
const job: QueueJob = JSON.parse(readFileSync(join(dir, file), "utf-8"));
|
|
238
|
+
const attempts = job.attempts || 0;
|
|
239
|
+
if (attempts > 0 && attempts < maxRetries) {
|
|
240
|
+
results.push(job);
|
|
241
|
+
}
|
|
207
242
|
} catch {
|
|
208
243
|
// skip corrupt files
|
|
209
244
|
}
|
|
@@ -215,6 +250,13 @@ export class LiteBackend {
|
|
|
215
250
|
return results;
|
|
216
251
|
}
|
|
217
252
|
|
|
253
|
+
/**
|
|
254
|
+
* Revive a specific dead-letter job by id back to the pending queue.
|
|
255
|
+
*
|
|
256
|
+
* Manual override (Queue.retry(jobId) / job.retry()) — always revives a
|
|
257
|
+
* dead-letter regardless of attempt count. Returns false only if no
|
|
258
|
+
* dead-letter with that id exists.
|
|
259
|
+
*/
|
|
218
260
|
retry(queue: string, jobId: string, delaySeconds?: number): boolean {
|
|
219
261
|
try {
|
|
220
262
|
const queues = readdirSync(this.basePath);
|
|
@@ -227,10 +269,10 @@ export class LiteBackend {
|
|
|
227
269
|
job.status = "pending";
|
|
228
270
|
job.attempts = (job.attempts || 0) + 1;
|
|
229
271
|
job.error = undefined;
|
|
272
|
+
job.createdAt = new Date().toISOString();
|
|
230
273
|
job.delayUntil = delaySeconds ? new Date(Date.now() + delaySeconds * 1000).toISOString() : null;
|
|
231
274
|
|
|
232
|
-
this.
|
|
233
|
-
const prefix = `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
|
|
275
|
+
const prefix = this.nextPrefix();
|
|
234
276
|
const queueDir = join(this.basePath, q);
|
|
235
277
|
writeFileSync(join(queueDir, `${prefix}_${jobId}.queue-data`), JSON.stringify(job, null, 2));
|
|
236
278
|
unlinkSync(filePath);
|
|
@@ -270,38 +312,19 @@ export class LiteBackend {
|
|
|
270
312
|
|
|
271
313
|
purge(queue: string, status: string, maxRetries: number = 3): number {
|
|
272
314
|
let count = 0;
|
|
315
|
+
const isDead = LiteBackend.DEAD_STATES.includes(status);
|
|
273
316
|
|
|
274
|
-
if (
|
|
275
|
-
|
|
276
|
-
try {
|
|
277
|
-
const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
|
|
278
|
-
for (const file of files) {
|
|
279
|
-
try {
|
|
280
|
-
const job: QueueJob = JSON.parse(readFileSync(join(failedDir, file), "utf-8"));
|
|
281
|
-
if ((job.attempts || 0) >= maxRetries) {
|
|
282
|
-
unlinkSync(join(failedDir, file));
|
|
283
|
-
count++;
|
|
284
|
-
}
|
|
285
|
-
} catch {
|
|
286
|
-
// skip corrupt files
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
} catch {
|
|
290
|
-
// directory might not exist
|
|
291
|
-
}
|
|
292
|
-
} else if (status === "failed") {
|
|
317
|
+
if (isDead) {
|
|
318
|
+
// Every file in failed/ is a dead-letter — purge them all.
|
|
293
319
|
const failedDir = this.ensureFailedDir(queue);
|
|
294
320
|
try {
|
|
295
321
|
const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
|
|
296
322
|
for (const file of files) {
|
|
297
323
|
try {
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
unlinkSync(join(failedDir, file));
|
|
301
|
-
count++;
|
|
302
|
-
}
|
|
324
|
+
unlinkSync(join(failedDir, file));
|
|
325
|
+
count++;
|
|
303
326
|
} catch {
|
|
304
|
-
//
|
|
327
|
+
// already removed
|
|
305
328
|
}
|
|
306
329
|
}
|
|
307
330
|
} catch {
|
|
@@ -330,6 +353,11 @@ export class LiteBackend {
|
|
|
330
353
|
return count;
|
|
331
354
|
}
|
|
332
355
|
|
|
356
|
+
/**
|
|
357
|
+
* Re-queue dead-letter jobs that are under the (possibly raised) limit back
|
|
358
|
+
* to pending. Mirrors Python retry_failed(): a job dead-lettered at the
|
|
359
|
+
* original maxRetries needs a raised limit to qualify again.
|
|
360
|
+
*/
|
|
333
361
|
retryFailed(queue: string, maxRetries: number = 3): number {
|
|
334
362
|
const failedDir = this.ensureFailedDir(queue);
|
|
335
363
|
const queueDir = this.ensureDir(queue);
|
|
@@ -348,9 +376,10 @@ export class LiteBackend {
|
|
|
348
376
|
|
|
349
377
|
job.status = "pending";
|
|
350
378
|
job.error = undefined;
|
|
379
|
+
job.createdAt = new Date().toISOString();
|
|
380
|
+
job.delayUntil = null;
|
|
351
381
|
|
|
352
|
-
this.
|
|
353
|
-
const prefix = `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
|
|
382
|
+
const prefix = this.nextPrefix();
|
|
354
383
|
writeFileSync(join(queueDir, `${prefix}_${job.id}.queue-data`), JSON.stringify(job, null, 2));
|
|
355
384
|
unlinkSync(filePath);
|
|
356
385
|
count++;
|
|
@@ -376,6 +405,7 @@ export class LiteBackend {
|
|
|
376
405
|
}
|
|
377
406
|
|
|
378
407
|
for (const file of files) {
|
|
408
|
+
if (!file.includes(id)) continue;
|
|
379
409
|
const filePath = join(dir, file);
|
|
380
410
|
let job: QueueJob;
|
|
381
411
|
try {
|
|
@@ -394,24 +424,79 @@ export class LiteBackend {
|
|
|
394
424
|
return null;
|
|
395
425
|
}
|
|
396
426
|
|
|
397
|
-
|
|
427
|
+
/**
|
|
428
|
+
* Write the job back to the pending queue (queue dir).
|
|
429
|
+
*
|
|
430
|
+
* Re-enqueued jobs get a fresh createdAt so that within a priority tier they
|
|
431
|
+
* sort behind jobs that have not yet been attempted. `attempts` already
|
|
432
|
+
* reflects the latest failure count. The job carries its prior error.
|
|
433
|
+
*/
|
|
434
|
+
private requeue(queue: string, job: QueueJob, delaySeconds: number = 0, error?: string): void {
|
|
435
|
+
const dir = this.ensureDir(queue);
|
|
436
|
+
const jobData = {
|
|
437
|
+
id: job.id,
|
|
438
|
+
payload: job.payload,
|
|
439
|
+
status: "pending" as const,
|
|
440
|
+
createdAt: new Date().toISOString(),
|
|
441
|
+
attempts: job.attempts ?? 0,
|
|
442
|
+
delayUntil: delaySeconds > 0 ? new Date(Date.now() + delaySeconds * 1000).toISOString() : null,
|
|
443
|
+
priority: job.priority ?? 0,
|
|
444
|
+
topic: job.topic,
|
|
445
|
+
error,
|
|
446
|
+
};
|
|
447
|
+
const prefix = this.nextPrefix();
|
|
448
|
+
writeFileSync(join(dir, `${prefix}_${job.id}.queue-data`), JSON.stringify(jobData, null, 2));
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* Move the job to the dead-letter (failed/) directory. Terminal until a
|
|
453
|
+
* manual retryFailed()/retry() revives it.
|
|
454
|
+
*/
|
|
455
|
+
private deadLetter(queue: string, job: QueueJob, error?: string): void {
|
|
398
456
|
const failedDir = this.ensureFailedDir(queue);
|
|
399
|
-
|
|
457
|
+
const jobData = {
|
|
458
|
+
id: job.id,
|
|
459
|
+
payload: job.payload,
|
|
460
|
+
status: "dead" as const,
|
|
461
|
+
createdAt: job.createdAt,
|
|
462
|
+
attempts: job.attempts ?? 0,
|
|
463
|
+
delayUntil: null,
|
|
464
|
+
priority: job.priority ?? 0,
|
|
465
|
+
topic: job.topic,
|
|
466
|
+
error,
|
|
467
|
+
failedAt: new Date().toISOString(),
|
|
468
|
+
};
|
|
469
|
+
writeFileSync(join(failedDir, `${job.id}.queue-data`), JSON.stringify(jobData, null, 2));
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Record a failed attempt.
|
|
474
|
+
*
|
|
475
|
+
* Increments `attempts` exactly once (the increment lives here, NOT in
|
|
476
|
+
* job.ts — see the double-increment fix). If the job still has retries left
|
|
477
|
+
* (attempts < maxRetries) it is automatically re-enqueued to pending, after
|
|
478
|
+
* an optional retryBackoff delay. Once it has been attempted maxRetries times
|
|
479
|
+
* (attempts >= maxRetries) it is moved to the dead-letter store.
|
|
480
|
+
*/
|
|
481
|
+
failJob(queue: string, job: QueueJob, error: string, maxRetries: number, retryBackoff: number = 0): void {
|
|
400
482
|
job.attempts = (job.attempts || 0) + 1;
|
|
401
483
|
job.error = error;
|
|
402
|
-
|
|
403
|
-
|
|
484
|
+
if (job.attempts < maxRetries) {
|
|
485
|
+
this.requeue(queue, job, retryBackoff, error);
|
|
486
|
+
} else {
|
|
487
|
+
this.deadLetter(queue, job, error);
|
|
488
|
+
}
|
|
404
489
|
}
|
|
405
490
|
|
|
491
|
+
/**
|
|
492
|
+
* Explicit re-queue requested by the caller (job.retry()).
|
|
493
|
+
*
|
|
494
|
+
* Always re-enqueues regardless of the retry limit — manual override,
|
|
495
|
+
* distinct from the automatic failJob() path.
|
|
496
|
+
*/
|
|
406
497
|
retryJob(queue: string, job: QueueJob, delaySeconds?: number): void {
|
|
407
|
-
const dir = this.ensureDir(queue);
|
|
408
|
-
job.status = "pending";
|
|
409
498
|
job.attempts = (job.attempts || 0) + 1;
|
|
410
499
|
job.error = undefined;
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
this.seq++;
|
|
414
|
-
const prefix = `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
|
|
415
|
-
writeFileSync(join(dir, `${prefix}_${job.id}.queue-data`), JSON.stringify(job, null, 2));
|
|
500
|
+
this.requeue(queue, job, delaySeconds ?? 0, undefined);
|
|
416
501
|
}
|
|
417
502
|
}
|
|
@@ -5,13 +5,18 @@
|
|
|
5
5
|
* for message storage and delivery. Atomic pop via findOneAndUpdate.
|
|
6
6
|
*
|
|
7
7
|
* Configure via environment variables:
|
|
8
|
+
* TINA4_QUEUE_URL — connection URI (mongodb://...)
|
|
9
|
+
* TINA4_MONGO_URI (override; wins over TINA4_QUEUE_URL)
|
|
8
10
|
* TINA4_MONGO_HOST (default: "localhost")
|
|
9
11
|
* TINA4_MONGO_PORT (default: 27017)
|
|
10
|
-
* TINA4_MONGO_URI (overrides host/port/username/password)
|
|
11
12
|
* TINA4_MONGO_USERNAME (optional)
|
|
12
13
|
* TINA4_MONGO_PASSWORD (optional)
|
|
13
14
|
* TINA4_MONGO_DB (default: "tina4")
|
|
14
15
|
* TINA4_MONGO_COLLECTION (default: "tina4_queue")
|
|
16
|
+
*
|
|
17
|
+
* Precedence for the connection URI: explicit config.uri
|
|
18
|
+
* > TINA4_MONGO_URI > TINA4_QUEUE_URL > a URI built from the
|
|
19
|
+
* TINA4_MONGO_HOST/PORT/USERNAME/PASSWORD field vars (existing defaults).
|
|
15
20
|
*/
|
|
16
21
|
import { randomUUID } from "node:crypto";
|
|
17
22
|
import { execFileSync } from "node:child_process";
|
|
@@ -63,9 +68,11 @@ export class MongoBackend implements QueueBackend {
|
|
|
63
68
|
this.database = config?.database ?? process.env.TINA4_MONGO_DB ?? "tina4";
|
|
64
69
|
this.collection = config?.collection ?? process.env.TINA4_MONGO_COLLECTION ?? "tina4_queue";
|
|
65
70
|
|
|
66
|
-
// URI
|
|
67
|
-
|
|
68
|
-
|
|
71
|
+
// Connection URI precedence: explicit config.uri > TINA4_MONGO_URI
|
|
72
|
+
// > TINA4_QUEUE_URL > a URI built from the host/port/auth field vars.
|
|
73
|
+
const explicitUri = config?.uri ?? process.env.TINA4_MONGO_URI ?? process.env.TINA4_QUEUE_URL;
|
|
74
|
+
if (explicitUri) {
|
|
75
|
+
this.uri = explicitUri;
|
|
69
76
|
} else {
|
|
70
77
|
const auth = this.username
|
|
71
78
|
? `${encodeURIComponent(this.username)}:${encodeURIComponent(this.password)}@`
|
|
@@ -74,6 +81,13 @@ export class MongoBackend implements QueueBackend {
|
|
|
74
81
|
}
|
|
75
82
|
}
|
|
76
83
|
|
|
84
|
+
/**
|
|
85
|
+
* Resolved connection config — exposed for testing/introspection.
|
|
86
|
+
*/
|
|
87
|
+
getConfig(): { uri: string; database: string; collection: string } {
|
|
88
|
+
return { uri: this.uri, database: this.database, collection: this.collection };
|
|
89
|
+
}
|
|
90
|
+
|
|
77
91
|
/**
|
|
78
92
|
* Execute a MongoDB operation synchronously via a child process.
|
|
79
93
|
*/
|
|
@@ -5,11 +5,15 @@
|
|
|
5
5
|
* for message storage and delivery.
|
|
6
6
|
*
|
|
7
7
|
* Configure via environment variables:
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
8
|
+
* TINA4_QUEUE_URL — AMQP URL (amqp://[user:pass@]host:port[/vhost])
|
|
9
|
+
* TINA4_RABBITMQ_HOST (override; default: "localhost")
|
|
10
|
+
* TINA4_RABBITMQ_PORT (override; default: 5672)
|
|
11
|
+
* TINA4_RABBITMQ_USERNAME (override; default: "guest")
|
|
12
|
+
* TINA4_RABBITMQ_PASSWORD (override; default: "guest")
|
|
13
|
+
* TINA4_RABBITMQ_VHOST (override; default: "/")
|
|
14
|
+
*
|
|
15
|
+
* Precedence per field: specific TINA4_RABBITMQ_* var (if set)
|
|
16
|
+
* > value derived from TINA4_QUEUE_URL > existing default.
|
|
13
17
|
*/
|
|
14
18
|
import net from "node:net";
|
|
15
19
|
import { execFileSync } from "node:child_process";
|
|
@@ -26,6 +30,51 @@ export interface RabbitMQConfig {
|
|
|
26
30
|
vhost?: string;
|
|
27
31
|
}
|
|
28
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Parse an AMQP URL (amqp://[user:pass@]host[:port][/vhost]) into a partial
|
|
35
|
+
* RabbitMQConfig. Mirrors the Python/PHP/Ruby `parse_amqp_url` semantics:
|
|
36
|
+
* strips a leading amqp:// or amqps:// scheme, splits optional credentials,
|
|
37
|
+
* and prepends a leading "/" to the vhost when missing. Only fields present
|
|
38
|
+
* in the URL are populated.
|
|
39
|
+
*/
|
|
40
|
+
export function parseAmqpUrl(url: string): RabbitMQConfig {
|
|
41
|
+
const config: RabbitMQConfig = {};
|
|
42
|
+
let rest = url.replace(/^amqps:\/\//, "").replace(/^amqp:\/\//, "");
|
|
43
|
+
|
|
44
|
+
const atIndex = rest.indexOf("@");
|
|
45
|
+
if (atIndex !== -1) {
|
|
46
|
+
const creds = rest.slice(0, atIndex);
|
|
47
|
+
rest = rest.slice(atIndex + 1);
|
|
48
|
+
const colonIndex = creds.indexOf(":");
|
|
49
|
+
if (colonIndex !== -1) {
|
|
50
|
+
config.username = creds.slice(0, colonIndex);
|
|
51
|
+
config.password = creds.slice(colonIndex + 1);
|
|
52
|
+
} else {
|
|
53
|
+
config.username = creds;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
let hostport = rest;
|
|
58
|
+
const slashIndex = rest.indexOf("/");
|
|
59
|
+
if (slashIndex !== -1) {
|
|
60
|
+
hostport = rest.slice(0, slashIndex);
|
|
61
|
+
const vhost = rest.slice(slashIndex + 1);
|
|
62
|
+
if (vhost) {
|
|
63
|
+
config.vhost = vhost.startsWith("/") ? vhost : "/" + vhost;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const portColon = hostport.indexOf(":");
|
|
68
|
+
if (portColon !== -1) {
|
|
69
|
+
config.host = hostport.slice(0, portColon);
|
|
70
|
+
config.port = parseInt(hostport.slice(portColon + 1), 10);
|
|
71
|
+
} else if (hostport) {
|
|
72
|
+
config.host = hostport;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return config;
|
|
76
|
+
}
|
|
77
|
+
|
|
29
78
|
export interface QueueBackend {
|
|
30
79
|
push(queue: string, payload: unknown, delay?: number): string;
|
|
31
80
|
pop(queue: string): QueueJob | null;
|
|
@@ -133,12 +182,32 @@ export class RabbitMQBackend implements QueueBackend {
|
|
|
133
182
|
private vhost: string;
|
|
134
183
|
|
|
135
184
|
constructor(config?: RabbitMQConfig) {
|
|
136
|
-
|
|
185
|
+
// Base layer: values derived from TINA4_QUEUE_URL (parsed as an AMQP URL).
|
|
186
|
+
const url = process.env.TINA4_QUEUE_URL;
|
|
187
|
+
const fromUrl = url ? parseAmqpUrl(url) : {};
|
|
188
|
+
|
|
189
|
+
// Precedence per field: explicit config arg > specific TINA4_RABBITMQ_* var
|
|
190
|
+
// > value from TINA4_QUEUE_URL > existing default.
|
|
191
|
+
this.host = config?.host ?? process.env.TINA4_RABBITMQ_HOST ?? fromUrl.host ?? "localhost";
|
|
137
192
|
this.port = config?.port
|
|
138
|
-
?? (process.env.TINA4_RABBITMQ_PORT ? parseInt(process.env.TINA4_RABBITMQ_PORT, 10) :
|
|
139
|
-
|
|
140
|
-
this.
|
|
141
|
-
this.
|
|
193
|
+
?? (process.env.TINA4_RABBITMQ_PORT ? parseInt(process.env.TINA4_RABBITMQ_PORT, 10) : undefined)
|
|
194
|
+
?? fromUrl.port ?? 5672;
|
|
195
|
+
this.username = config?.username ?? process.env.TINA4_RABBITMQ_USERNAME ?? fromUrl.username ?? "guest";
|
|
196
|
+
this.password = config?.password ?? process.env.TINA4_RABBITMQ_PASSWORD ?? fromUrl.password ?? "guest";
|
|
197
|
+
this.vhost = config?.vhost ?? process.env.TINA4_RABBITMQ_VHOST ?? fromUrl.vhost ?? "/";
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Resolved connection config — exposed for testing/introspection.
|
|
202
|
+
*/
|
|
203
|
+
getConfig(): Required<RabbitMQConfig> {
|
|
204
|
+
return {
|
|
205
|
+
host: this.host,
|
|
206
|
+
port: this.port,
|
|
207
|
+
username: this.username,
|
|
208
|
+
password: this.password,
|
|
209
|
+
vhost: this.vhost,
|
|
210
|
+
};
|
|
142
211
|
}
|
|
143
212
|
|
|
144
213
|
/**
|
|
@@ -1379,17 +1379,11 @@ ${reset}
|
|
|
1379
1379
|
// Assign to module-level so handle() can dispatch without a server reference
|
|
1380
1380
|
_dispatchFn = dispatch;
|
|
1381
1381
|
|
|
1382
|
-
//
|
|
1383
|
-
//
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
const
|
|
1387
|
-
? async (req: IncomingMessage, res: ServerResponse) => {
|
|
1388
|
-
(req as any)._tina4AiPort = true;
|
|
1389
|
-
await dispatch(req, res);
|
|
1390
|
-
}
|
|
1391
|
-
: dispatch;
|
|
1392
|
-
const server = createServer(mainPortDispatch);
|
|
1382
|
+
// Dual-port (debug + no TINA4_NO_AI_PORT): the MAIN port hot-reloads for the human
|
|
1383
|
+
// dev; the stable AI port (port+1000, created below) suppresses reload/toolbar so an
|
|
1384
|
+
// AI tool can drive it without its own edits triggering refreshes. The tina4 client
|
|
1385
|
+
// posts /__dev/api/reload to the MAIN port. Matches Python (master).
|
|
1386
|
+
const server = createServer(dispatch);
|
|
1393
1387
|
|
|
1394
1388
|
return new Promise((resolvePromise) => {
|
|
1395
1389
|
server.listen(port, host, () => {
|
|
@@ -1405,16 +1399,17 @@ ${reset}
|
|
|
1405
1399
|
// Determine server mode label
|
|
1406
1400
|
const serverMode = isDebug ? "single" : (cluster.isWorker ? "cluster-worker" : "single");
|
|
1407
1401
|
|
|
1408
|
-
// AI dual-port: main port =
|
|
1409
|
-
//
|
|
1410
|
-
//
|
|
1402
|
+
// AI dual-port: main port = hot-reload (human dev); port+1000 = stable AI port
|
|
1403
|
+
// (reload/toolbar suppressed) so an AI tool can drive it without its edits
|
|
1404
|
+
// triggering refreshes. The tina4 client fires reloads at the MAIN port. Matches Python.
|
|
1411
1405
|
const noAiPort = isTruthy(process.env.TINA4_NO_AI_PORT ?? "");
|
|
1412
1406
|
let aiServer: ReturnType<typeof createServer> | null = null;
|
|
1413
1407
|
let testPort = port + 1000;
|
|
1414
1408
|
|
|
1415
1409
|
if (isDebug && !noAiPort) {
|
|
1416
|
-
//
|
|
1410
|
+
// Stable AI port (port+1000): tag requests so /__dev_reload + toolbar are suppressed.
|
|
1417
1411
|
aiServer = createServer(async (req, res) => {
|
|
1412
|
+
(req as any)._tina4AiPort = true;
|
|
1418
1413
|
await dispatch(req, res);
|
|
1419
1414
|
});
|
|
1420
1415
|
|
|
@@ -1451,11 +1446,8 @@ ${reset}
|
|
|
1451
1446
|
}
|
|
1452
1447
|
const noBrowser = isTruthy(process.env.TINA4_NO_BROWSER);
|
|
1453
1448
|
if (!noBrowser) {
|
|
1454
|
-
// Open browser on
|
|
1455
|
-
|
|
1456
|
-
? `http://${displayHost}:${testPort}`
|
|
1457
|
-
: `http://${displayHost}:${port}`;
|
|
1458
|
-
openBrowser(browserTarget);
|
|
1449
|
+
// Open the browser on the MAIN port — that's the hot-reload port.
|
|
1450
|
+
openBrowser(`http://${displayHost}:${port}`);
|
|
1459
1451
|
}
|
|
1460
1452
|
resolvePromise({
|
|
1461
1453
|
close: () => {
|