queue-tool 1.0.0

package/README.md

@@ -0,0 +1,246 @@
+# ☄️ queue-tool (Node.js Adapter)
+
+The high-performance, cross-language, database-agnostic task queue for Node.js and TypeScript. Automatically supports binary MessagePack serialization and atomic claims across PostgreSQL, Redis, and MongoDB.
+
+---
+
+## 📦 Installation
+
+```bash
+npm install queue-tool
+# or
+pnpm add queue-tool
+# or
+yarn add queue-tool
+```
+
+Depending on which adapter you use, install the corresponding peer dependency:
+- **PostgreSQL**: `npm install pg @types/pg`
+- **Redis**: `npm install ioredis`
+- **MongoDB**: `npm install mongodb`
+
+---
+
+## 🚀 Basic Usage (Node.js / TypeScript)
+
+### 1. Initialize Adapter
+Use the `QueueFactory` to automatically instantiate the correct adapter based on your connection string:
+
+```typescript
+import { QueueFactory } from 'queue-tool';
+
+// Automatically instantiates PostgresQueueAdapter
+const adapter = await QueueFactory.create('postgresql://postgres:password@localhost:5432/queue_tool', {
+  completedJobTtlMs: 3600000, // Retain completed jobs for 1 hour
+  failedJobTtlMs: 86400000,    // Retain failed jobs for 24 hours
+  lockTimeoutMs: 300000,       // Automatically reclaim stuck jobs after 5 mins
+  pruneIntervalMs: 60000,      // Run background pruning check every 1 min
+});
+```
+
+### 2. Enqueue Jobs
+```typescript
+const job = await adapter.enqueue('image-processing', {
+  imageId: 'img-10293',
+  operations: ['resize', 'compress'],
+}, { 
+  priority: 10,       // Higher priority jobs are claimed first
+  maxAttempts: 3,     // Auto-retry up to 3 times before failing
+  delayMs: 5000       // Wait 5 seconds before making job available
+});
+```
+
+### 3. Claim and Complete Jobs
+```typescript
+const job = await adapter.claimJob('image-processing', 'worker-node-1');
+
+if (job) {
+  try {
+    console.log(`Processing payload:`, job.payload);
+    // ... do processing ...
+    
+    await adapter.completeJob(job.id);
+  } catch (error) {
+    // Moves back to pending (rescheduled in 5 seconds) or marks failed if attempts exhausted
+    await adapter.failJob(job.id, error.message);
+  }
+}
+```
+
+---
+
+## 🦅 NestJS Integration Guide
+
+To cleanly integrate `queue-tool` inside a **NestJS** application, you can create a custom dynamic module.
+
+### 1. Create the Queue Module
+
+Create a file named `queue.module.ts`:
+
+```typescript
+import { Module, DynamicModule, Global } from '@nestjs/common';
+import { QueueFactory } from 'queue-tool';
+import { QueueAdapterOptions } from 'queue-tool/dist/types';
+
+export const QUEUE_ADAPTER = 'QUEUE_ADAPTER';
+
+@Global()
+@Module({})
+export class QueueModule {
+  static register(connectionString: string, options?: QueueAdapterOptions): DynamicModule {
+    const provider = {
+      provide: QUEUE_ADAPTER,
+      useFactory: async () => {
+        const adapter = await QueueFactory.create(connectionString, options);
+        return adapter;
+      },
+    };
+
+    return {
+      module: QueueModule,
+      providers: [provider],
+      exports: [provider],
+    };
+  }
+}
+```
+
+### 2. Import Module in App Module
+
+```typescript
+import { Module } from '@nestjs/common';
+import { QueueModule } from './queue.module';
+
+@Module({
+  imports: [
+    QueueModule.register('redis://localhost:6379', {
+      lockTimeoutMs: 300000, // 5 minutes
+    }),
+  ],
+})
+export class AppModule {}
+```
+
+### 3. Inject and Use in a Service
+
+```typescript
+import { Injectable, Inject, OnModuleDestroy } from '@nestjs/common';
+import { QUEUE_ADAPTER } from './queue.module';
+import { IQueueAdapter } from 'queue-tool/dist/types';
+
+@Injectable()
+export class TaskService implements OnModuleDestroy {
+  constructor(
+    @Inject(QUEUE_ADAPTER) private readonly queueAdapter: IQueueAdapter,
+  ) {}
+
+  async createProcessTask(data: any) {
+    return this.queueAdapter.enqueue('my-task-queue', data, { priority: 5 });
+  }
+
+  async onModuleDestroy() {
+    await this.queueAdapter.close();
+  }
+}
+```
+
+### 4. Create a Background Worker with high-level Worker
+
+```typescript
+import { Injectable, Inject, OnModuleInit, OnModuleDestroy } from '@nestjs/common';
+import { QUEUE_ADAPTER } from './queue.module';
+import { IQueueAdapter, Worker } from 'queue-tool';
+
+@Injectable()
+export class WorkerService implements OnModuleInit, OnModuleDestroy {
+  private worker!: Worker;
+
+  constructor(
+    @Inject(QUEUE_ADAPTER) private readonly queueAdapter: IQueueAdapter,
+  ) {}
+
+  async onModuleInit() {
+    this.worker = new Worker(
+      'my-task-queue',
+      this.queueAdapter,
+      async (job) => {
+        console.log('Processing job payload:', job.payload);
+        // Your job execution logic here...
+      },
+      { concurrency: 2, pollIntervalMs: 500 }
+    );
+
+    await this.worker.start();
+  }
+
+  async onModuleDestroy() {
+    await this.worker.stop();
+  }
+}
+```
+
+---
+
+## 🚂 ExpressJS Integration Guide
+
+Create a background worker and API endpoints in your Express application:
+
+```typescript
+import express from 'express';
+import { QueueFactory, Worker } from 'queue-tool';
+
+const app = express();
+app.use(express.json());
+
+const startApp = async () => {
+  // 1. Initialize queue adapter
+  const adapter = await QueueFactory.create('redis://localhost:6379');
+
+  // 2. Start worker to process background tasks
+  const worker = new Worker('express-tasks', adapter, async (job) => {
+    console.log(`Processing background job ${job.id}:`, job.payload);
+  });
+  await worker.start();
+
+  // 3. Define routes
+  app.post('/enqueue', async (req, res) => {
+    const job = await adapter.enqueue('express-tasks', req.body);
+    res.json({ status: 'enqueued', jobId: job.id });
+  });
+
+  app.listen(3000, () => console.log('Server running on port 3000'));
+};
+
+startApp();
+```
+
+---
+
+## ☄️ Hono Integration Guide
+
+Hono works great with async background workers. Define workers at startup:
+
+```typescript
+import { Hono } from 'hono';
+import { QueueFactory, Worker } from 'queue-tool';
+
+const app = new Hono();
+
+// Instantiate adapter
+const adapter = await QueueFactory.create('mongodb://localhost:27017/queue_tool');
+
+// Start worker
+const worker = new Worker('hono-tasks', adapter, async (job) => {
+  console.log(`Hono task running: ${job.id}`);
+});
+await worker.start();
+
+app.post('/enqueue', async (c) => {
+  const body = await c.req.json();
+  const job = await adapter.enqueue('hono-tasks', body);
+  return c.json({ status: 'enqueued', jobId: job.id });
+});
+
+export default app;
+```
+

package/dist/adapters/mongodb.d.ts

@@ -0,0 +1,14 @@
+import { IQueueAdapter, Job, EnqueueOptions, QueueAdapterOptions } from '../types';
+export declare class MongoQueueAdapter implements IQueueAdapter {
+    private client;
+    private collection;
+    private options;
+    constructor(connectionString: string, dbName?: string, options?: QueueAdapterOptions);
+    init(): Promise<void>;
+    enqueue<T>(queueName: string, payload: T, options?: EnqueueOptions): Promise<Job<T>>;
+    claimJob(queueName: string, workerId: string): Promise<Job | null>;
+    completeJob(jobId: string): Promise<void>;
+    failJob(jobId: string, error: string): Promise<void>;
+    close(): Promise<void>;
+    private mapDocToJob;
+}

package/dist/adapters/mongodb.js

@@ -0,0 +1,173 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MongoQueueAdapter = void 0;
+const mongodb_1 = require("mongodb");
+const msgpack_1 = require("../utils/msgpack");
+const crypto_1 = require("crypto");
+const logger_1 = require("../utils/logger");
+class MongoQueueAdapter {
+    client;
+    collection;
+    options;
+    constructor(connectionString, dbName = 'queue_tool', options) {
+        this.client = new mongodb_1.MongoClient(connectionString);
+        this.collection = this.client.db(dbName).collection('jobs');
+        this.options = {
+            completedJobTtlMs: options?.completedJobTtlMs ?? 3600000, // 1 hour
+            failedJobTtlMs: options?.failedJobTtlMs ?? 86400000, // 24 hours
+            lockTimeoutMs: options?.lockTimeoutMs ?? 300000, // 5 minutes
+        };
+    }
+    async init() {
+        try {
+            await this.client.connect();
+            await this.client.db().command({ ping: 1 });
+            logger_1.logger.info('MongoDB connection verified successfully');
+        }
+        catch (e) {
+            logger_1.logger.error('MongoDB connection health check failed', { error: e.message });
+            throw new Error(`MongoDB connection health check failed: ${e.message}`);
+        }
+        // Enforce compound index for extremely fast claiming performance
+        await this.collection.createIndex({ queue: 1, status: 1, priority: -1, runAt: 1 }, { name: 'idx_jobs_claim' });
+        // Native TTL index to automatically purge expired documents
+        await this.collection.createIndex({ expiresAt: 1 }, { name: 'idx_jobs_ttl', expireAfterSeconds: 0 });
+    }
+    async enqueue(queueName, payload, options) {
+        const id = options?.id ?? (0, crypto_1.randomUUID)();
+        const serializedPayload = (0, msgpack_1.serialize)(payload);
+        const priority = options?.priority ?? 0;
+        const maxAttempts = options?.maxAttempts ?? 3;
+        if (priority < -1000 || priority > 1000) {
+            logger_1.logger.error('Invalid job priority', { priority, queue: queueName });
+            throw new Error('Priority must be in the range [-1000, 1000]');
+        }
+        let runAt = options?.runAt ?? new Date();
+        if (options?.delayMs) {
+            runAt = new Date(Date.now() + options.delayMs);
+        }
+        const existingDoc = await this.collection.findOne({ _id: id });
+        const existingJob = !!existingDoc;
+        const doc = {
+            _id: id,
+            queue: queueName,
+            payload: new mongodb_1.Binary(serializedPayload),
+            status: 'pending',
+            priority,
+            attempts: 0,
+            maxAttempts,
+            runAt,
+            lockedAt: null,
+            completedAt: null,
+            failedAt: null,
+            expiresAt: null,
+            error: null,
+            workerId: null,
+        };
+        const res = await this.collection.findOneAndUpdate({ _id: id }, { $set: doc }, { upsert: true, returnDocument: 'after' });
+        logger_1.logger.info(existingJob ? 'Job updated/retried successfully' : 'Job enqueued successfully', { queue: queueName, jobId: id, priority });
+        return this.mapDocToJob(res);
+    }
+    async claimJob(queueName, workerId) {
+        const now = new Date();
+        const lockTimeoutMs = this.options.lockTimeoutMs ?? 300000;
+        const lockTimeoutDate = new Date(now.getTime() - lockTimeoutMs);
+        const res = await this.collection.findOneAndUpdate({
+            queue: queueName,
+            $or: [
+                { status: 'pending', runAt: { $lte: now } },
+                { status: 'processing', lockedAt: { $lte: lockTimeoutDate } }
+            ]
+        }, {
+            $set: {
+                status: 'processing',
+                workerId: workerId,
+                lockedAt: now
+            },
+            $inc: {
+                attempts: 1
+            }
+        }, {
+            sort: { priority: -1, runAt: 1 },
+            returnDocument: 'after'
+        });
+        if (!res) {
+            return null;
+        }
+        const job = this.mapDocToJob(res);
+        logger_1.logger.info('Job claimed successfully', { queue: queueName, jobId: job.id, workerId });
+        return job;
+    }
+    async completeJob(jobId) {
+        const res = await this.collection.findOneAndUpdate({ _id: jobId }, {
+            $set: {
+                status: 'completed',
+                completedAt: new Date(),
+                expiresAt: null
+            }
+        }, { returnDocument: 'after' });
+        if (!res) {
+            logger_1.logger.warn('Tried to complete job that does not exist', { jobId });
+            return;
+        }
+        logger_1.logger.info('Job completed successfully', { queue: res.queue, jobId });
+    }
+    async failJob(jobId, error) {
+        const jobDoc = await this.collection.findOne({ _id: jobId });
+        if (!jobDoc) {
+            logger_1.logger.warn('Tried to fail job that does not exist', { jobId, error });
+            return;
+        }
+        const { queue, attempts, maxAttempts } = jobDoc;
+        if (attempts < maxAttempts) {
+            // Retry: Set status back to pending, delay retry by 5 seconds
+            await this.collection.updateOne({ _id: jobId }, {
+                $set: {
+                    status: 'pending',
+                    runAt: new Date(Date.now() + 5000),
+                    error,
+                    workerId: null,
+                    lockedAt: null
+                }
+            });
+            logger_1.logger.warn('Job execution failed, scheduling retry', { queue, jobId, error, attempt: attempts, maxAttempts });
+        }
+        else {
+            // Terminal Failure
+            const expiresAt = new Date(Date.now() + (this.options.failedJobTtlMs || 86400000));
+            await this.collection.updateOne({ _id: jobId }, {
+                $set: {
+                    status: 'failed',
+                    failedAt: new Date(),
+                    expiresAt: expiresAt,
+                    error
+                }
+            });
+            logger_1.logger.error('Job failed terminally', { queue, jobId, error, attempts });
+        }
+    }
+    async close() {
+        logger_1.logger.info('Closing MongoDB adapter client pool');
+        await this.client.close();
+    }
+    mapDocToJob(doc) {
+        const payloadBuffer = doc.payload.buffer ? doc.payload.buffer : Buffer.from(doc.payload);
+        return {
+            id: doc._id,
+            queue: doc.queue,
+            payload: (0, msgpack_1.deserialize)(payloadBuffer),
+            status: doc.status,
+            priority: doc.priority,
+            attempts: doc.attempts,
+            maxAttempts: doc.maxAttempts,
+            runAt: doc.runAt,
+            lockedAt: doc.lockedAt || undefined,
+            completedAt: doc.completedAt || undefined,
+            failedAt: doc.failedAt || undefined,
+            expiresAt: doc.expiresAt || undefined,
+            error: doc.error || undefined,
+            workerId: doc.workerId || undefined,
+        };
+    }
+}
+exports.MongoQueueAdapter = MongoQueueAdapter;

package/dist/adapters/mongodb.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/adapters/mongodb.test.js

@@ -0,0 +1,119 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const mongodb_1 = require("./mongodb");
+const mongodb_2 = require("mongodb");
+describe('MongoQueueAdapter Integration Tests', () => {
+    let adapter;
+    let mongoClient;
+    const queueName = 'test-mongo-integration-queue';
+    const connectionString = 'mongodb://localhost:27017';
+    const dbName = 'queue_tool';
+    let isAvailable = true;
+    beforeAll(async () => {
+        try {
+            mongoClient = new mongodb_2.MongoClient(connectionString);
+            await mongoClient.connect();
+            adapter = new mongodb_1.MongoQueueAdapter(connectionString, dbName);
+            await adapter.init();
+        }
+        catch (err) {
+            isAvailable = false;
+            console.warn('MongoDB is not running. Skipping MongoDB integration tests.');
+        }
+    });
+    afterAll(async () => {
+        if (!isAvailable)
+            return;
+        await adapter.close();
+        await mongoClient.close();
+    });
+    beforeEach(async () => {
+        if (!isAvailable)
+            return;
+        // Clean test queue jobs
+        await mongoClient.db(dbName).collection('jobs').deleteMany({ queue: queueName });
+    });
+    test('should enqueue a job successfully', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-mongo-enqueue', value: 100 };
+        const job = await adapter.enqueue(queueName, payload, { priority: 10 });
+        expect(job.id).toBeDefined();
+        expect(job.queue).toBe(queueName);
+        expect(job.payload).toEqual(payload);
+        expect(job.status).toBe('pending');
+        expect(job.priority).toBe(10);
+    });
+    test('should fail validation on invalid priority bounds', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-priority-bounds' };
+        await expect(adapter.enqueue(queueName, payload, { priority: 2000 })).rejects.toThrow('Priority must be in the range [-1000, 1000]');
+    });
+    test('should support deduplication and idempotency with custom job ID', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'dedup-mongo', step: 1 };
+        const customId = 'custom-mongo-job-id';
+        const job1 = await adapter.enqueue(queueName, payload, { id: customId, priority: 5 });
+        expect(job1.id).toBe(customId);
+        expect(job1.status).toBe('pending');
+        // Claim the job so it changes status to processing
+        const claimedJob = await adapter.claimJob(queueName, 'worker-mongo-1');
+        expect(claimedJob).not.toBeNull();
+        expect(claimedJob.id).toBe(customId);
+        expect(claimedJob.status).toBe('processing');
+        // Enqueue again with the same custom ID but updated payload
+        const updatedPayload = { task: 'dedup-mongo', step: 2 };
+        const job2 = await adapter.enqueue(queueName, updatedPayload, { id: customId, priority: 8 });
+        expect(job2.id).toBe(customId);
+        expect(job2.status).toBe('pending'); // Reset to pending
+        expect(job2.attempts).toBe(0); // Reset attempts
+        expect(job2.priority).toBe(8); // Updated priority
+        expect(job2.payload).toEqual(updatedPayload);
+        // Verify in db
+        const doc = await mongoClient.db(dbName).collection('jobs').findOne({ _id: customId });
+        expect(doc).not.toBeNull();
+        expect(doc.status).toBe('pending');
+        expect(doc.attempts).toBe(0);
+        expect(doc.priority).toBe(8);
+    });
+    test('should claim and complete a job successfully', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-mongo-claim', value: 200 };
+        const job = await adapter.enqueue(queueName, payload);
+        const claimedJob = await adapter.claimJob(queueName, 'worker-test-mongo-1');
+        expect(claimedJob).not.toBeNull();
+        expect(claimedJob.id).toBe(job.id);
+        expect(claimedJob.status).toBe('processing');
+        expect(claimedJob.attempts).toBe(1);
+        await adapter.completeJob(claimedJob.id);
+        const doc = await mongoClient.db(dbName).collection('jobs').findOne({ _id: job.id });
+        expect(doc).not.toBeNull();
+        expect(doc.status).toBe('completed');
+    });
+    test('should fail and retry job within maxAttempts limit', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-mongo-retry' };
+        const job = await adapter.enqueue(queueName, payload, { maxAttempts: 2 });
+        const claimedJob = await adapter.claimJob(queueName, 'worker-test-mongo-2');
+        expect(claimedJob).not.toBeNull();
+        await adapter.failJob(claimedJob.id, 'First test error mongo');
+        const doc1 = await mongoClient.db(dbName).collection('jobs').findOne({ _id: job.id });
+        expect(doc1.status).toBe('pending');
+        expect(doc1.error).toBe('First test error mongo');
+        // Reset runAt to the past so it can be claimed again immediately without waiting 5 seconds
+        await mongoClient.db(dbName).collection('jobs').updateOne({ _id: job.id }, { $set: { runAt: new Date(Date.now() - 60000) } });
+        // Claim again (2nd attempt)
+        const claimedJob2 = await adapter.claimJob(queueName, 'worker-test-mongo-2');
+        expect(claimedJob2).not.toBeNull();
+        expect(claimedJob2.attempts).toBe(2);
+        // Fail again -> terminal failure
+        await adapter.failJob(claimedJob2.id, 'Second test error mongo');
+        const doc2 = await mongoClient.db(dbName).collection('jobs').findOne({ _id: job.id });
+        expect(doc2.status).toBe('failed');
+        expect(doc2.error).toBe('Second test error mongo');
+    });
+});

package/dist/adapters/postgres.d.ts

@@ -0,0 +1,15 @@
+import { IQueueAdapter, Job, EnqueueOptions, QueueAdapterOptions } from '../types';
+export declare class PostgresQueueAdapter implements IQueueAdapter {
+    private pool;
+    private options;
+    private pruneTimer?;
+    constructor(connectionString: string, options?: QueueAdapterOptions);
+    init(): Promise<void>;
+    enqueue<T>(queueName: string, payload: T, options?: EnqueueOptions): Promise<Job<T>>;
+    claimJob(queueName: string, workerId: string): Promise<Job | null>;
+    completeJob(jobId: string): Promise<void>;
+    failJob(jobId: string, error: string): Promise<void>;
+    listenToQueue(queueName: string, callback: () => void): Promise<() => Promise<void>>;
+    close(): Promise<void>;
+    private mapRowToJob;
+}