queasy 0.2.0 → 0.3.1

package/test/{redis-functions.test.js → redis-functions.test.ts}

@@ -4,13 +4,13 @@ import { dirname, join } from 'node:path';
 import { after, afterEach, before, describe, it } from 'node:test';
 import { fileURLToPath } from 'node:url';
 import { createClient } from 'redis';
+import type { RedisClientType } from 'redis';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
 describe('Redis Lua functions', () => {
-    /** @type {import('redis').RedisClientType} */
-    let redis;
+    let redis: RedisClientType;
     const QUEUE_NAME = '{test-queue}';
 
     before(async () => {
@@ -281,12 +281,10 @@ describe('Redis Lua functions', () => {
 
             // Dequeue
             const expiryTime = now + 10000;
-            const result = /** @type {string[][]} */ (
-                await redis.fCall('queasy_dequeue', {
-                    keys: [QUEUE_NAME],
-                    arguments: [clientId, now.toString(), expiryTime.toString(), '10'],
-                })
-            );
+            const result = await redis.fCall('queasy_dequeue', {
+                keys: [QUEUE_NAME],
+                arguments: [clientId, now.toString(), expiryTime.toString(), '10'],
+            }) as string[][];
 
             assert.equal(result.length, 2);
             assert.deepEqual(result[0], ['id', jobId1]);
@@ -318,12 +316,10 @@ describe('Redis Lua functions', () => {
             await redis.hSet(`${QUEUE_NAME}:waiting_job:${jobId}`, 'id', jobId);
 
             // Try to dequeue
-            const result = /** @type {string[][]} */ (
-                await redis.fCall('queasy_dequeue', {
-                    keys: [QUEUE_NAME],
-                    arguments: [clientId, now.toString(), String(now + 10000), '10'],
-                })
-            );
+            const result = await redis.fCall('queasy_dequeue', {
+                keys: [QUEUE_NAME],
+                arguments: [clientId, now.toString(), String(now + 10000), '10'],
+            }) as string[][];
 
             assert.equal(result.length, 0);
         });
@@ -338,12 +334,10 @@ describe('Redis Lua functions', () => {
             await redis.hSet(`${QUEUE_NAME}:waiting_job:${jobId}`, 'id', jobId);
 
             // Try to dequeue
-            const result = /** @type {string[][]} */ (
-                await redis.fCall('queasy_dequeue', {
-                    keys: [QUEUE_NAME],
-                    arguments: [clientId, now.toString(), String(now + 10000), '10'],
-                })
-            );
+            const result = await redis.fCall('queasy_dequeue', {
+                keys: [QUEUE_NAME],
+                arguments: [clientId, now.toString(), String(now + 10000), '10'],
+            }) as string[][];
 
             assert.equal(result.length, 0);
         });

package/test/{utils.test.js → utils.test.ts}

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { describe, it } from 'node:test';
-import { compareSemver, parseVersion } from '../src/utils.js';
+import { compareSemver, parseVersion } from '../src/utils.ts';
 
 describe('parseVersion', () => {
     it('should parse a simple major.minor version', () => {

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+    "compilerOptions": {
+        "strict": true,
+        "target": "esnext",
+        "module": "nodenext",
+        "moduleResolution": "nodenext",
+        "rewriteRelativeImportExtensions": true,
+        "outDir": "dist",
+        "rootDir": "src",
+        "declaration": true,
+        "declarationMap": true,
+        "sourceMap": true,
+        "skipLibCheck": true,
+        "forceConsistentCasingInFileNames": true,
+        "erasableSyntaxOnly": true,
+        "resolveJsonModule": true,
+        "lib": ["ES2022"]
+    },
+    "include": ["src/**/*.ts"]
+}

package/jsconfig.json

@@ -1,17 +0,0 @@
-{
-    "compilerOptions": {
-        "checkJs": true,
-        "strict": true,
-        "target": "ES2022",
-        "module": "ESNext",
-        "moduleResolution": "node",
-        "allowSyntheticDefaultImports": true,
-        "esModuleInterop": true,
-        "skipLibCheck": true,
-        "forceConsistentCasingInFileNames": true,
-        "resolveJsonModule": true,
-        "noEmit": true,
-        "lib": ["ES2022"]
-    },
-    "include": ["src/**/*.js", "test/**/*.js"]
-}

package/src/client.js

@@ -1,258 +0,0 @@
-import EventEmitter from 'node:events';
-import { readFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { getEnvironmentData } from 'node:worker_threads';
-import { HEARTBEAT_INTERVAL, HEARTBEAT_TIMEOUT, LUA_FUNCTIONS_VERSION } from './constants.js';
-import { Manager } from './manager.js';
-import { Pool } from './pool.js';
-import { Queue } from './queue.js';
-import { compareSemver, generateId, parseVersion } from './utils.js';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const luaScript = readFileSync(join(__dirname, 'queasy.lua'), 'utf8').replace(
-    '__QUEASY_VERSION__',
-    LUA_FUNCTIONS_VERSION
-);
-
-/**
- * Check the installed version and load our Lua functions if needed.
- * Returns true if this client should be disconnected (newer major on server).
- * @param {RedisClient} redis
- * @returns {Promise<boolean>} Whether to disconnect.
- */
-async function installLuaFunctions(redis) {
-    const installedVersionString = /** @type {string?} */ (
-        await redis.fCall('queasy_version', { keys: [], arguments: [] }).catch(() => null)
-    );
-    const installedVersion = parseVersion(installedVersionString);
-    const availableVersion = parseVersion(LUA_FUNCTIONS_VERSION);
-
-    // No script installed or our version is later
-    if (compareSemver(availableVersion, installedVersion) > 0) {
-        await redis.sendCommand(['FUNCTION', 'LOAD', 'REPLACE', luaScript]);
-        return false;
-    }
-
-    // Keep the installed (newer) version. Return disconnect=true if the major versions disagree
-    return installedVersion[0] > availableVersion[0];
-}
-
-/** @typedef {import('redis').RedisClientType} RedisClient */
-/** @typedef {import('./types').Job} Job */
-
-/** @typedef {{ queue: Queue, bumpTimer?: NodeJS.Timeout }} QueueEntry */
-
-/**
- * Parse job data from Redis response
- * @param {string[]} jobArray - Flat array from HGETALL
- * @returns {Job | null}
- */
-export function parseJob(jobArray) {
-    if (!jobArray || jobArray.length === 0) return null;
-
-    /** @type {Record<string, string>} */
-    const job = {};
-    for (let i = 0; i < jobArray.length; i += 2) {
-        const key = jobArray[i];
-        const value = jobArray[i + 1];
-        job[key] = value;
-    }
-
-    return {
-        id: job.id,
-        data: job.data ? JSON.parse(job.data) : undefined,
-        runAt: job.run_at ? Number(job.run_at) : 0,
-        retryCount: Number(job.retry_count || 0),
-        stallCount: Number(job.stall_count || 0),
-    };
-}
-
-export class Client extends EventEmitter {
-    /**
-     * @param {RedisClient} redis - Redis client
-     * @param {number?} workerCount - Allow this client to dequeue jobs.
-     * @param {((client: Client) => any)} [callback] - Callback when client is ready
-     */
-    constructor(redis, workerCount, callback) {
-        super();
-        this.redis = redis;
-        this.clientId = generateId();
-
-        /** @type {Record<string, QueueEntry>} */
-        this.queues = {};
-        this.disconnected = false;
-
-        const inWorker = getEnvironmentData('queasy_worker_context');
-        this.pool = !inWorker && workerCount !== 0 ? new Pool(workerCount) : undefined;
-        if (this.pool) this.manager = new Manager(this.pool);
-
-        // Not awaited — the Lua script is read synchronously at module load,
-        // so Redis' single-threaded ordering ensures the FUNCTION LOAD completes
-        // before any subsequent fCalls from user code.
-        installLuaFunctions(this.redis).then((disconnect) => {
-            this.disconnected = disconnect;
-            if (disconnect) this.emit('disconnected', 'Redis has incompatible queasy version.');
-            else if (callback) callback(this);
-        });
-    }
-
-    /**
-     * Create a queue object for interacting with a named queue
-     * @param {string} name - Queue name (without braces - they will be added automatically)
-     * @returns {Queue} Queue object with dispatch, cancel, and listen methods
-     */
-    queue(name, isKey = false) {
-        if (this.disconnected) throw new Error('Can’t add queue: client disconnected');
-
-        const key = isKey ? name : `{${name}}`;
-        if (!this.queues[key]) {
-            this.queues[key] = /** @type {QueueEntry} */ ({
-                queue: new Queue(key, this, this.pool, this.manager),
-            });
-        }
-        return this.queues[key].queue;
-    }
-
-    /**
-     * Schedule the next bump timer
-     * @param {string} key
-     */
-    scheduleBump(key) {
-        const queueEntry = this.queues[key];
-        if (queueEntry.bumpTimer) clearTimeout(queueEntry.bumpTimer);
-        queueEntry.bumpTimer = setTimeout(() => this.bump(key), HEARTBEAT_INTERVAL);
-    }
-
-    /**
-     * @param {string} key
-     */
-    async bump(key) {
-        if (this.disconnected) return;
-        // Set up the next bump first, in case this
-        this.scheduleBump(key);
-        const now = Date.now();
-        const expiry = now + HEARTBEAT_TIMEOUT;
-        const bumped = await this.redis.fCall('queasy_bump', {
-            keys: [key],
-            arguments: [this.clientId, String(now), String(expiry)],
-        });
-
-        if (!bumped) {
-            // This client’s lock was lost and its jobs retried.
-            // We must stop processing jobs here to avoid duplication.
-            await this.close();
-            this.emit('disconnected', 'Lost locks, possible main thread freeze');
-        }
-    }
-
-    /**
-     * This marks this as disconnected.
-     */
-    async close() {
-        if (this.pool) await this.pool.close();
-        if (this.manager) await this.manager.close();
-        this.queues = {};
-        this.pool = undefined;
-        this.disconnected = true;
-    }
-
-    /**
-     * @param {string} key
-     * @param {string} id
-     * @param {number} runAt
-     * @param {any} data
-     * @param {boolean} updateData
-     * @param {boolean | string} updateRunAt
-     * @param {boolean} resetCounts
-     */
-    async dispatch(key, id, runAt, data, updateData, updateRunAt, resetCounts) {
-        await this.redis.fCall('queasy_dispatch', {
-            keys: [key],
-            arguments: [
-                id,
-                String(runAt),
-                JSON.stringify(data),
-                String(updateData),
-                String(updateRunAt),
-                String(resetCounts),
-            ],
-        });
-    }
-
-    /**
-     * @param {string} key
-     * @param {string} id
-     * @returns {Promise<boolean>}
-     */
-    async cancel(key, id) {
-        const result = await this.redis.fCall('queasy_cancel', {
-            keys: [key],
-            arguments: [id],
-        });
-        return result === 1;
-    }
-
-    /**
-     * @param {string} key
-     * @param {number} count
-     * @returns {Promise<Job[]>}
-     */
-    async dequeue(key, count) {
-        const now = Date.now();
-        const expiry = now + HEARTBEAT_TIMEOUT;
-        const result = /** @type {string[][]} */ (
-            await this.redis.fCall('queasy_dequeue', {
-                keys: [key],
-                arguments: [this.clientId, String(now), String(expiry), String(count)],
-            })
-        );
-
-        // Heartbeats should start with the first dequeue.
-        this.scheduleBump(key);
-
-        return /** @type Job[] */ (result.map((jobArray) => parseJob(jobArray)).filter(Boolean));
-    }
-
-    /**
-     * @param {string} key
-     * @param {string} jobId
-     */
-    async finish(key, jobId) {
-        await this.redis.fCall('queasy_finish', {
-            keys: [key],
-            arguments: [jobId, this.clientId, String(Date.now())],
-        });
-    }
-
-    /**
-     * @param {string} key
-     * @param {string} failkey
-     * @param {string} jobId
-     * @param {any} failJobData
-     */
-    async fail(key, failkey, jobId, failJobData) {
-        await this.redis.fCall('queasy_fail', {
-            keys: [key, failkey],
-            arguments: [
-                jobId,
-                this.clientId,
-                generateId(),
-                JSON.stringify(failJobData),
-                String(Date.now()),
-            ],
-        });
-    }
-
-    /**
-     * @param {string} key
-     * @param {string} jobId
-     * @param {number} retryAt
-     */
-    async retry(key, jobId, retryAt) {
-        await this.redis.fCall('queasy_retry', {
-            keys: [key],
-            arguments: [jobId, this.clientId, String(retryAt), String(Date.now())],
-        });
-    }
-}

package/src/index.js

@@ -1,2 +0,0 @@
-export { Client } from './client.js';
-export { PermanentError, StallError } from './errors.js';

package/src/manager.js

@@ -1,94 +0,0 @@
-/**
- * This class manages resource allocation between
- * different queues based on the size of the queue
- */
-
-import { DEQUEUE_INTERVAL } from './constants.js';
-
-/** @typedef {import('./pool').Pool} Pool */
-/** @typedef {import('./queue').ProcessingQueue} Queue */
-/** @typedef {{ queue: Queue, lastDequeuedAt: number, isBusy: boolean }} QueueEntry */
-
-export class Manager {
-    /** @param {Pool} pool */
-    constructor(pool) {
-        this.pool = pool;
-
-        /** @type {Array<QueueEntry>} */
-        this.queues = [];
-
-        /** @type {NodeJS.Timeout?} */
-        this.timer = null;
-
-        this.busyCount = 0;
-    }
-
-    /** @param {Queue} queue */
-    addQueue(queue) {
-        // Add this at the beginning so we dequeue it at the next available opportunity.
-        this.queues.unshift({ queue, lastDequeuedAt: 0, isBusy: false });
-        this.busyCount += 1;
-
-        // This delay is required for queue listen tests
-        // as they need to be able to control dequeueing
-        this.next();
-    }
-
-    async next() {
-        // If this function is called while the previous execution is in progress,
-        // we do not want both executions to use the same queue.
-        const entry = this.queues.shift();
-        if (!entry) return;
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-
-        const size = entry.queue.handlerOptions.size;
-        if (this.pool.capacity < size) return;
-
-        const batchSize = Math.max(1, Math.floor(this.pool.capacity / this.busyCount / size));
-        entry.lastDequeuedAt = Date.now(); // We store the time just before the call to dequeue.
-        const { count } = await entry.queue.dequeue(batchSize);
-
-        // Update
-        const nowBusy = count >= batchSize;
-        this.busyCount += Number(nowBusy) - Number(entry.isBusy);
-        entry.isBusy = nowBusy;
-
-        this.queues.push(entry);
-        this.queues.sort(compareQueueEntries);
-
-        if (!this.timer && this.queues.length) {
-            const { isBusy, lastDequeuedAt } = this.queues[0];
-            // If the current top queue is busy, retry now.
-            const delay = isBusy ? 0 : Math.max(0, lastDequeuedAt - Date.now() + DEQUEUE_INTERVAL);
-            this.timer = setTimeout(() => this.next(), delay);
-        }
-    }
-
-    close() {
-        if (this.timer) clearTimeout(this.timer);
-    }
-}
-
-/**
- * @param {QueueEntry} a
- * @param {QueueEntry} b
- * @returns -1 | 0 | 1
- */
-function compareQueueEntries(a, b) {
-    if (a.isBusy > b.isBusy) return -1; // a busy, b not -> a first
-    if (a.isBusy < b.isBusy) return 1; // a free, b busy -> b first
-
-    if (a.queue.handlerOptions.priority > b.queue.handlerOptions.priority) return 1; // a higher -> a first
-    if (a.queue.handlerOptions.priority < b.queue.handlerOptions.priority) return -1; // b higher -> b first
-
-    if (a.lastDequeuedAt > b.lastDequeuedAt) return -1; // a newer -> b first
-    if (a.lastDequeuedAt < b.lastDequeuedAt) return 1; // a older -> a first
-
-    if (a.queue.handlerOptions.size > b.queue.handlerOptions.size) return 1; // a larger -> a first
-    if (a.queue.handlerOptions.size < b.queue.handlerOptions.size) return -1; // b larger -> b first
-
-    return 0;
-}

package/src/queue.js

@@ -1,154 +0,0 @@
-import { DEFAULT_RETRY_OPTIONS, FAILJOB_RETRY_OPTIONS } from './constants.js';
-import { generateId } from './utils.js';
-
-// Import types:
-/** @typedef {import('redis').RedisClientType} RedisClient */
-/** @typedef {import('./types').HandlerOptions} HandlerOptions */
-/** @typedef {import('./types').ListenOptions} ListenOptions */
-/** @typedef {import('./types').JobOptions} JobOptions */
-/** @typedef {import('./types').Job} Job */
-/** @typedef {import('./types').DoneMessage} DoneMessage */
-/** @typedef {import('./client').Client} Client */
-/** @typedef {import('./pool').Pool} Pool */
-/** @typedef {import('./manager').Manager} Manager */
-
-/** @typedef {Required<Partial<Pick<Queue, keyof Queue>>>} ProcessingQueue */
-
-/**
- * Queue instance for managing a named job queue
- */
-export class Queue {
-    /**
-     * @param {string} key - Queue key
-     * @param {Client} client - Redis client wrapper
-     * @param {Pool | undefined} pool - Worker pool
-     * @param {Manager | undefined} manager - Capacity allocation manager
-     */
-    constructor(key, client, pool, manager) {
-        this.key = key;
-        this.client = client;
-        this.pool = pool;
-        this.manager = manager;
-
-        /** @type {NodeJS.Timeout | undefined} */
-        // this.dequeueInterval = undefined;
-
-        /** @type {Required<HandlerOptions> | undefined} */
-        this.handlerOptions = undefined;
-
-        /** @type {string | undefined} */
-        this.handlerPath = undefined;
-
-        /** @type {string | undefined} */
-        this.failKey = undefined;
-    }
-
-    /**
-     * Attach handlers to process jobs from this queue
-     * @param {string} handlerPath - Path to handler module
-     * @param {ListenOptions} [options] - Retry strategy options and failure handler
-     * @returns {Promise<void>}
-     */
-    async listen(handlerPath, { failHandler, failRetryOptions, ...retryOptions } = {}) {
-        if (this.client.disconnected) throw new Error('Can’t listen: client disconnected');
-        if (!this.pool || !this.manager) throw new Error('Can’t listen: non-processing client');
-
-        this.handlerPath = handlerPath;
-        this.handlerOptions = { ...DEFAULT_RETRY_OPTIONS, ...retryOptions };
-
-        // Initialize failure handler on all workers if provided
-        if (failHandler) {
-            this.failKey = `${this.key}-fail`;
-            const failQueue = this.client.queue(this.failKey, true);
-            failQueue.listen(failHandler, { ...FAILJOB_RETRY_OPTIONS, ...failRetryOptions });
-        }
-
-        this.manager.addQueue(/** @type {ProcessingQueue} */ (this));
-
-        // if (!this.dequeueInterval) {
-        //     this.dequeueInterval = setInterval(() => this.dequeue(), DEQUEUE_INTERVAL);
-        // }
-    }
-
-    /**
-     * Add a job to the queue
-     * @param {any} data - Job data (any JSON-serializable value)
-     * @param {JobOptions} [options] - Job options
-     * @returns {Promise<string>} Job ID
-     */
-    async dispatch(data, options = {}) {
-        if (this.client.disconnected) throw new Error('Can’t dispatch: client disconnected');
-        const {
-            id = generateId(),
-            runAt = 0,
-            updateData = false,
-            updateRunAt = false,
-            resetCounts = false,
-        } = options;
-
-        await this.client.dispatch(this.key, id, runAt, data, updateData, updateRunAt, resetCounts);
-        return id;
-    }
-
-    /**
-     * Cancel a waiting job
-     * @param {string} id - Job ID
-     * @returns {Promise<boolean>} True if job was cancelled
-     */
-    async cancel(id) {
-        if (this.client.disconnected) throw new Error('Can’t cancel: client disconnected');
-        return await this.client.cancel(this.key, id);
-    }
-
-    /**
-     * Picks jobs from the queue and processes them
-     * @param {number} count
-     * @returns {Promise<{count: number, promise: Promise<Array<unknown>>}>}
-     */
-
-    async dequeue(count) {
-        const { pool, handlerPath, handlerOptions } = /** @type {ProcessingQueue} */ (this);
-        const { maxRetries, maxStalls, maxBackoff, minBackoff, size, timeout } = handlerOptions;
-
-        // const capacity = pool.getCapacity(size);
-        // if (capacity <= 0) return;
-
-        const jobs = await this.client.dequeue(this.key, count);
-
-        const promise = Promise.all(
-            jobs.map(async (job) => {
-                // Check if job has exceeded stall limit
-                if (job.stallCount >= maxStalls) {
-                    // Job has stalled too many times - fail it permanently
-                    if (!this.failKey) return this.client.finish(this.key, job.id);
-
-                    const failJobData = [job.id, job.data, { message: 'Max stalls exceeded' }];
-                    return this.client.fail(this.key, this.failKey, job.id, failJobData);
-                }
-
-                try {
-                    await pool.process(handlerPath, job, size, timeout);
-                    await this.client.finish(this.key, job.id);
-                } catch (message) {
-                    const { error } = /** @type {Required<DoneMessage>} */ (message);
-                    const { retryAt = 0, kind } = error;
-
-                    if (kind === 'permanent' || job.retryCount >= maxRetries) {
-                        if (!this.failKey) return this.client.finish(this.key, job.id);
-
-                        const failJobData = [job.id, job.data, error];
-                        return this.client.fail(this.key, this.failKey, job.id, failJobData);
-                    }
-
-                    const backoffUntil =
-                        Date.now() + Math.min(maxBackoff, minBackoff * 2 ** job.retryCount);
-
-                    // Retriable error: call retry
-                    await this.client.retry(this.key, job.id, Math.max(retryAt, backoffUntil));
-                }
-            })
-        );
-
-        return { count: jobs.length, promise };
-    }
-}

package/test/fixtures/always-fail-handler.js

@@ -1,8 +0,0 @@
-/**
- * Handler that always fails with a retriable error
- * @param {any} _data - Job data
- * @param {import('../../src/types.js').Job} _job - Job metadata
- */
-export async function handle(_data, _job) {
-    throw new Error('Always fails');
-}

package/test/fixtures/data-logger-handler.js

@@ -1,19 +0,0 @@
-/**
- * Handler that logs received data to a global array (for testing)
- * @param {any} data - Job data
- * @param {import('../../src/types.js').Job} job - Job metadata
- */
-/** @type {{ data: any, job: import('../../src/types.js').Job }[]} */
-export const receivedJobs = [];
-
-/**
- * @param {any} data - Job data
- * @param {import('../../src/types.js').Job} job - Job metadata
- */
-export async function handle(data, job) {
-    receivedJobs.push({ data, job });
-}
-
-export function clearLogs() {
-    receivedJobs.length = 0;
-}

package/test/fixtures/failure-handler.js

@@ -1,9 +0,0 @@
-/**
- * Failure handler
- * @param {any} data - Failure data [originalJobId, originalData, error]
- * @param {import('../../src/types.js').Job} _job - Job metadata
- */
-export async function handle(data, _job) {
-    // Just succeed to process the failure
-    console.log('Failure handler called with:', data);
-}

package/test/fixtures/permanent-error-handler.js

@@ -1,10 +0,0 @@
-import { PermanentError } from '../../src/errors.js';
-
-/**
- * Handler that throws a PermanentError
- * @param {any} _data - Job data
- * @param {import('../../src/types.js').Job} _job - Job metadata
- */
-export async function handle(_data, _job) {
-    throw new PermanentError('Permanent failure');
-}