tina4-nodejs 3.1.2 → 3.4.0

package/packages/core/src/queue.ts

@@ -7,9 +7,10 @@
  *   - 'file'     — JSON files on disk (default)
  *   - 'rabbitmq' — RabbitMQ via raw TCP (AMQP 0-9-1)
  *   - 'kafka'    — Kafka via raw TCP
+ *   - 'mongodb'  — MongoDB via `mongodb` npm package (also 'mongo')
  *
  * Environment variables:
- *   TINA4_QUEUE_BACKEND — 'file', 'rabbitmq', or 'kafka'
+ *   TINA4_QUEUE_BACKEND — 'file', 'rabbitmq', 'kafka', or 'mongodb'
  *   TINA4_QUEUE_URL     — connection URL for rabbitmq/kafka
  *   TINA4_QUEUE_PATH    — file backend storage path (default: data/queue)
  *
@@ -43,11 +44,37 @@ export interface QueueConfig {
 export interface QueueJob {
   id: string;
   payload: unknown;
-  status: "pending" | "reserved" | "failed" | "dead";
+  status: "pending" | "reserved" | "failed" | "dead" | "completed";
   createdAt: string;
   attempts: number;
   delayUntil: string | null;
   error?: string;
+  /** Mark this job as completed. */
+  complete(): void;
+  /** Mark this job as failed with a reason. */
+  fail(reason?: string): void;
+  /** Reject this job with a reason. Alias for fail(). */
+  reject(reason?: string): void;
+}
+
+/** Create a QueueJob with lifecycle methods bound to a Queue instance. */
+function createJob(data: Omit<QueueJob, "complete" | "fail" | "reject">, queue: Queue, topic: string): QueueJob {
+  const job: QueueJob = {
+    ...data,
+    complete() {
+      job.status = "completed";
+    },
+    fail(reason = "") {
+      job.status = "failed";
+      job.error = reason;
+      job.attempts = (job.attempts || 0) + 1;
+      queue._failJob(topic, job, reason, queue.getMaxRetries());
+    },
+    reject(reason = "") {
+      job.fail(reason);
+    },
+  };
+  return job;
 }
 
 export interface ProcessOptions {
@@ -69,7 +96,7 @@ export class Queue {
   private backendName: string;
   private basePath: string;
   private topic: string;
-  private maxRetries: number;
+  private _maxRetries: number;
   private seq: number = 0;
   private externalBackend: QueueBackendInterface | null = null;
 
@@ -98,7 +125,7 @@ export class Queue {
       ?? process.env.TINA4_QUEUE_PATH
       ?? "data/queue";
     this.topic = resolvedConfig.topic ?? "default";
-    this.maxRetries = resolvedConfig.maxRetries ?? 3;
+    this._maxRetries = resolvedConfig.maxRetries ?? 3;
 
     // Initialize external backends
     if (this.backendName === "rabbitmq") {
@@ -107,6 +134,9 @@ export class Queue {
     } else if (this.backendName === "kafka") {
       const { KafkaBackend } = require("./queueBackends/kafkaBackend.js");
       this.externalBackend = new KafkaBackend();
+    } else if (this.backendName === "mongodb" || this.backendName === "mongo") {
+      const { MongoBackend } = require("./queueBackends/mongoBackend.js");
+      this.externalBackend = new MongoBackend();
     }
   }
 
@@ -169,7 +199,7 @@ export class Queue {
     };
 
     const prefix = `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
-    writeFileSync(join(dir, `${prefix}_${id}.json`), JSON.stringify(job, null, 2));
+    writeFileSync(join(dir, `${prefix}_${id}.queue-data`), JSON.stringify(job, null, 2));
     return id;
   }
 
@@ -190,7 +220,7 @@ export class Queue {
 
     let files: string[];
     try {
-      files = readdirSync(dir).filter(f => f.endsWith(".json")).sort();
+      files = readdirSync(dir).filter(f => f.endsWith(".queue-data")).sort();
     } catch {
       return null;
     }
@@ -249,7 +279,7 @@ export class Queue {
     }
 
     const maxJobs = opts?.maxJobs ?? Infinity;
-    const maxRetries = opts?.maxRetries ?? this.maxRetries;
+    const maxRetries = opts?.maxRetries ?? this._maxRetries;
     let processed = 0;
 
     while (processed < maxJobs) {
@@ -283,7 +313,7 @@ export class Queue {
     const dir = this.ensureDir(q);
     let files: string[];
     try {
-      files = readdirSync(dir).filter(f => f.endsWith(".json"));
+      files = readdirSync(dir).filter(f => f.endsWith(".queue-data"));
     } catch {
       return 0;
     }
@@ -312,7 +342,7 @@ export class Queue {
     }
     const dir = this.ensureDir(q);
     try {
-      const files = readdirSync(dir).filter(f => f.endsWith(".json"));
+      const files = readdirSync(dir).filter(f => f.endsWith(".queue-data"));
       for (const file of files) {
         unlinkSync(join(dir, file));
       }
@@ -324,7 +354,7 @@ export class Queue {
     const failedDir = join(dir, "failed");
     try {
       if (existsSync(failedDir)) {
-        const files = readdirSync(failedDir).filter(f => f.endsWith(".json"));
+        const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
         for (const file of files) {
           unlinkSync(join(failedDir, file));
         }
@@ -343,7 +373,7 @@ export class Queue {
     const results: QueueJob[] = [];
 
     try {
-      const files = readdirSync(failedDir).filter(f => f.endsWith(".json")).sort();
+      const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data")).sort();
       for (const file of files) {
         try {
           const job: QueueJob = JSON.parse(readFileSync(join(failedDir, file), "utf-8"));
@@ -367,7 +397,7 @@ export class Queue {
       const queues = readdirSync(this.basePath);
       for (const queue of queues) {
         const failedDir = join(this.basePath, queue, "failed");
-        const filePath = join(failedDir, `${jobId}.json`);
+        const filePath = join(failedDir, `${jobId}.queue-data`);
 
         if (existsSync(filePath)) {
           const job: QueueJob = JSON.parse(readFileSync(filePath, "utf-8"));
@@ -378,7 +408,7 @@ export class Queue {
           this.seq++;
           const prefix = `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
           const queueDir = join(this.basePath, queue);
-          writeFileSync(join(queueDir, `${prefix}_${jobId}.json`), JSON.stringify(job, null, 2));
+          writeFileSync(join(queueDir, `${prefix}_${jobId}.queue-data`), JSON.stringify(job, null, 2));
           unlinkSync(filePath);
           return true;
         }
@@ -395,12 +425,12 @@ export class Queue {
    */
   deadLetters(queue?: string, maxRetries?: number): QueueJob[] {
     const q = queue ?? this.topic;
-    const mr = maxRetries ?? this.maxRetries;
+    const mr = maxRetries ?? this._maxRetries;
     const failedDir = this.ensureFailedDir(q);
     const results: QueueJob[] = [];
 
     try {
-      const files = readdirSync(failedDir).filter(f => f.endsWith(".json")).sort();
+      const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data")).sort();
       for (const file of files) {
         try {
           const job: QueueJob = JSON.parse(readFileSync(join(failedDir, file), "utf-8"));
@@ -431,12 +461,12 @@ export class Queue {
       // Legacy: purge("queueName", "status", maxRetries?)
       queue = statusOrQueue;
       status = statusOrMaxRetries;
-      mr = maxRetries ?? this.maxRetries;
+      mr = maxRetries ?? this._maxRetries;
     } else {
       // Unified: purge("status") or purge("status", maxRetries)
       queue = this.topic;
       status = statusOrQueue;
-      mr = typeof statusOrMaxRetries === "number" ? statusOrMaxRetries : (maxRetries ?? this.maxRetries);
+      mr = typeof statusOrMaxRetries === "number" ? statusOrMaxRetries : (maxRetries ?? this._maxRetries);
     }
 
     let count = 0;
@@ -444,7 +474,7 @@ export class Queue {
     if (status === "dead") {
       const failedDir = this.ensureFailedDir(queue);
       try {
-        const files = readdirSync(failedDir).filter(f => f.endsWith(".json"));
+        const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
         for (const file of files) {
           try {
             const job: QueueJob = JSON.parse(readFileSync(join(failedDir, file), "utf-8"));
@@ -462,7 +492,7 @@ export class Queue {
     } else if (status === "failed") {
       const failedDir = this.ensureFailedDir(queue);
       try {
-        const files = readdirSync(failedDir).filter(f => f.endsWith(".json"));
+        const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
         for (const file of files) {
           try {
             const job: QueueJob = JSON.parse(readFileSync(join(failedDir, file), "utf-8"));
@@ -480,7 +510,7 @@ export class Queue {
     } else {
       const dir = this.ensureDir(queue);
       try {
-        const files = readdirSync(dir).filter(f => f.endsWith(".json"));
+        const files = readdirSync(dir).filter(f => f.endsWith(".queue-data"));
         for (const file of files) {
           try {
             const job: QueueJob = JSON.parse(readFileSync(join(dir, file), "utf-8"));
@@ -505,13 +535,13 @@ export class Queue {
    */
   retryFailed(queue?: string, maxRetries?: number): number {
     const q = queue ?? this.topic;
-    const mr = maxRetries ?? this.maxRetries;
+    const mr = maxRetries ?? this._maxRetries;
     const failedDir = this.ensureFailedDir(q);
     const queueDir = this.ensureDir(q);
     let count = 0;
 
     try {
-      const files = readdirSync(failedDir).filter(f => f.endsWith(".json"));
+      const files = readdirSync(failedDir).filter(f => f.endsWith(".queue-data"));
       for (const file of files) {
         try {
           const filePath = join(failedDir, file);
@@ -526,7 +556,7 @@ export class Queue {
 
           this.seq++;
           const prefix = `${Date.now()}-${String(this.seq).padStart(6, "0")}`;
-          writeFileSync(join(queueDir, `${prefix}_${job.id}.json`), JSON.stringify(job, null, 2));
+          writeFileSync(join(queueDir, `${prefix}_${job.id}.queue-data`), JSON.stringify(job, null, 2));
           unlinkSync(filePath);
           count++;
         } catch {
@@ -540,6 +570,74 @@ export class Queue {
     return count;
   }
 
+  /**
+   * Produce a message onto a topic. Convenience wrapper around push().
+   */
+  produce(topic: string, payload: unknown, delay?: number): string {
+    return this.push(topic, payload, delay);
+  }
+
+  /**
+   * Consume jobs from a topic using a generator (yield pattern).
+   *
+   * Usage:
+   *   for (const job of queue.consume("emails")) {
+   *       processEmail(job);
+   *   }
+   *
+   *   // Consume a specific job by ID:
+   *   for (const job of queue.consume("emails", "job-id-123")) {
+   *       processEmail(job);
+   *   }
+   */
+  *consume(topic?: string, id?: string): Generator<QueueJob> {
+    const q = topic ?? this.topic;
+
+    if (id !== undefined) {
+      const raw = this.popById(q, id);
+      if (raw) yield createJob(raw as any, this, q);
+      return;
+    }
+
+    let raw: any;
+    while ((raw = this.pop(q)) !== null) {
+      yield createJob(raw, this, q);
+    }
+  }
+
+  /**
+   * Pop a specific job by ID from the queue.
+   */
+  popById(queue: string, id: string): QueueJob | null {
+    const q = queue ?? this.topic;
+    const dir = this.ensureDir(q);
+
+    let files: string[];
+    try {
+      files = readdirSync(dir).filter(f => f.endsWith(".queue-data"));
+    } catch {
+      return null;
+    }
+
+    for (const file of files) {
+      const filePath = join(dir, file);
+      let job: QueueJob;
+      try {
+        job = JSON.parse(readFileSync(filePath, "utf-8"));
+      } catch {
+        continue;
+      }
+
+      if (job.status !== "pending") continue;
+      if (job.id === id) {
+        try { unlinkSync(filePath); } catch { /* already consumed */ }
+        return job;
+      }
+    }
+
+    return null;
+  }
+
   /**
    * Get the configured topic name.
    */
@@ -547,15 +645,19 @@ export class Queue {
     return this.topic;
   }
 
+  getMaxRetries(): number {
+    return this._maxRetries;
+  }
+
   /**
    * Move a job to the failed directory.
    */
-  private _failJob(queue: string, job: QueueJob, error: string, maxRetries: number): void {
+  _failJob(queue: string, job: QueueJob, error: string, maxRetries: number): void {
     const failedDir = this.ensureFailedDir(queue);
     job.status = "failed";
     job.attempts = (job.attempts || 0) + 1;
     job.error = error;
 
-    writeFileSync(join(failedDir, `${job.id}.json`), JSON.stringify(job, null, 2));
+    writeFileSync(join(failedDir, `${job.id}.queue-data`), JSON.stringify(job, null, 2));
   }
 }

package/packages/core/src/queueBackends/mongoBackend.ts

@@ -0,0 +1,223 @@
+/**
+ * Tina4 MongoDB Queue Backend — uses `mongodb` npm package via dynamic import.
+ *
+ * Implements the same interface as the file-based queue but uses MongoDB
+ * for message storage and delivery. Atomic pop via findOneAndUpdate.
+ *
+ * Configure via environment variables:
+ *   TINA4_MONGO_HOST       (default: "localhost")
+ *   TINA4_MONGO_PORT       (default: 27017)
+ *   TINA4_MONGO_URI        (overrides host/port/username/password)
+ *   TINA4_MONGO_USERNAME   (optional)
+ *   TINA4_MONGO_PASSWORD   (optional)
+ *   TINA4_MONGO_DB         (default: "tina4")
+ *   TINA4_MONGO_COLLECTION (default: "tina4_queue")
+ */
+import { randomUUID } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import type { QueueJob } from "../queue.js";
+
+// ── Types ────────────────────────────────────────────────────
+
+export interface MongoConfig {
+  host?: string;
+  port?: number;
+  uri?: string;
+  username?: string;
+  password?: string;
+  database?: string;
+  collection?: string;
+}
+
+export interface QueueBackend {
+  push(queue: string, payload: unknown, delay?: number): string;
+  pop(queue: string): QueueJob | null;
+  size(queue: string): number;
+  clear(queue: string): void;
+}
+
+// ── MongoDB Backend ──────────────────────────────────────────
+
+/**
+ * MongoDB queue backend using the `mongodb` npm package.
+ *
+ * Uses synchronous-style communication by spawning a child process
+ * for each operation, similar to the RabbitMQ and Redis patterns.
+ * This keeps the interface synchronous as required by the Queue class.
+ */
+export class MongoBackend implements QueueBackend {
+  private host: string;
+  private port: number;
+  private uri: string;
+  private username: string;
+  private password: string;
+  private database: string;
+  private collection: string;
+
+  constructor(config?: MongoConfig) {
+    this.host = config?.host ?? process.env.TINA4_MONGO_HOST ?? "localhost";
+    this.port = config?.port
+      ?? (process.env.TINA4_MONGO_PORT ? parseInt(process.env.TINA4_MONGO_PORT, 10) : 27017);
+    this.username = config?.username ?? process.env.TINA4_MONGO_USERNAME ?? "";
+    this.password = config?.password ?? process.env.TINA4_MONGO_PASSWORD ?? "";
+    this.database = config?.database ?? process.env.TINA4_MONGO_DB ?? "tina4";
+    this.collection = config?.collection ?? process.env.TINA4_MONGO_COLLECTION ?? "tina4_queue";
+
+    // URI overrides individual host/port/auth settings
+    if (config?.uri ?? process.env.TINA4_MONGO_URI) {
+      this.uri = config?.uri ?? process.env.TINA4_MONGO_URI!;
+    } else {
+      const auth = this.username
+        ? `${encodeURIComponent(this.username)}:${encodeURIComponent(this.password)}@`
+        : "";
+      this.uri = `mongodb://${auth}${this.host}:${this.port}`;
+    }
+  }
+
+  /**
+   * Execute a MongoDB operation synchronously via a child process.
+   */
+  private execSync(operation: string, queue: string, data?: string): string {
+    const script = `
+      async function main() {
+        let mongodb;
+        try {
+          mongodb = await import("mongodb");
+        } catch {
+          process.stderr.write("mongodb package not installed — run: npm install mongodb");
+          process.exit(1);
+        }
+
+        const { MongoClient } = mongodb;
+        const uri = ${JSON.stringify(this.uri)};
+        const dbName = ${JSON.stringify(this.database)};
+        const collName = ${JSON.stringify(this.collection)};
+        const operation = ${JSON.stringify(operation)};
+        const queueName = ${JSON.stringify(queue)};
+        const data = ${JSON.stringify(data ?? "")};
+
+        const client = new MongoClient(uri, {
+          connectTimeoutMS: 5000,
+          serverSelectionTimeoutMS: 5000,
+        });
+
+        try {
+          await client.connect();
+          const db = client.db(dbName);
+          const col = db.collection(collName);
+
+          // Ensure indexes on first use
+          await col.createIndex({ queue: 1, status: 1, delayUntil: 1 });
+          await col.createIndex({ queue: 1, createdAt: 1 });
+
+          if (operation === "push") {
+            const job = JSON.parse(data);
+            await col.insertOne({ ...job, queue: queueName });
+            process.stdout.write("__PUSHED__");
+          }
+          else if (operation === "pop") {
+            const now = new Date().toISOString();
+            const result = await col.findOneAndUpdate(
+              {
+                queue: queueName,
+                status: "pending",
+                $or: [
+                  { delayUntil: null },
+                  { delayUntil: { $lte: now } },
+                ],
+              },
+              { $set: { status: "reserved" } },
+              { sort: { createdAt: 1 }, returnDocument: "before" },
+            );
+
+            if (result && result.value) {
+              // findOneAndUpdate returns { value: doc } in older drivers
+              const doc = result.value;
+              delete doc._id;
+              delete doc.queue;
+              process.stdout.write(JSON.stringify(doc));
+            } else if (result && result._id) {
+              // Some driver versions return the doc directly
+              const doc = { ...result };
+              delete doc._id;
+              delete doc.queue;
+              process.stdout.write(JSON.stringify(doc));
+            } else {
+              process.stdout.write("__EMPTY__");
+            }
+          }
+          else if (operation === "size") {
+            const count = await col.countDocuments({
+              queue: queueName,
+              status: "pending",
+            });
+            process.stdout.write(String(count));
+          }
+          else if (operation === "clear") {
+            await col.deleteMany({ queue: queueName });
+            process.stdout.write("__CLEARED__");
+          }
+        } catch (err) {
+          process.stderr.write(err.message || String(err));
+          process.exit(1);
+        } finally {
+          await client.close();
+        }
+      }
+
+      main();
+    `;
+
+    try {
+      const result = execFileSync(process.execPath, ["-e", script], {
+        encoding: "utf-8",
+        timeout: 15000,
+        stdio: ["pipe", "pipe", "pipe"],
+      });
+      return result;
+    } catch {
+      return "";
+    }
+  }
+
+  push(queue: string, payload: unknown, delay?: number): string {
+    const id = randomUUID();
+    const now = new Date().toISOString();
+
+    const job: QueueJob = {
+      id,
+      payload,
+      status: "pending",
+      createdAt: now,
+      attempts: 0,
+      delayUntil: delay ? new Date(Date.now() + delay * 1000).toISOString() : null,
+    };
+
+    const result = this.execSync("push", queue, JSON.stringify(job));
+    if (!result.includes("__PUSHED__")) {
+      throw new Error("MongoDB push failed");
+    }
+    return id;
+  }
+
+  pop(queue: string): QueueJob | null {
+    const result = this.execSync("pop", queue);
+    if (!result || result === "__EMPTY__") return null;
+
+    try {
+      return JSON.parse(result) as QueueJob;
+    } catch {
+      return null;
+    }
+  }
+
+  size(queue: string): number {
+    const result = this.execSync("size", queue);
+    const num = parseInt(result, 10);
+    return isNaN(num) ? 0 : num;
+  }
+
+  clear(queue: string): void {
+    this.execSync("clear", queue);
+  }
+}

package/packages/core/src/request.ts

@@ -129,12 +129,12 @@ function parseMultipart(
     const partContentType = parsePartContentType(headersStr);
 
     if (disposition.filename) {
-      // File upload
+      // File upload — standardised format: filename, type, content (raw bytes), size
       files.push({
         fieldName: disposition.name,
         filename: disposition.filename,
-        contentType: partContentType ?? "application/octet-stream",
-        data: Buffer.from(content),
+        type: partContentType ?? "application/octet-stream",
+        content: Buffer.from(content),
         size: content.length,
       });
     } else if (disposition.name) {

package/packages/core/src/routeDiscovery.ts

@@ -44,7 +44,8 @@ export async function discoverRoutes(routesDir: string): Promise<RouteDefinition
 
 function filePathToPattern(relativePath: string): string {
   // Remove the filename (get.ts, post.ts, etc.) to get the directory path
-  const parts = relativePath.split("/").slice(0, -1);
+  // Normalise backslashes for Windows compatibility
+  const parts = relativePath.replace(/\\/g, "/").split("/").slice(0, -1);
 
   // Convert directory segments to URL pattern
   // File system uses [id] notation, but URL patterns use {id} to match Python