workmatic 1.0.0 → 1.0.2

package/dist/index.js

@@ -1,13 +1,689 @@
-/**
- * Workmatic - A persistent job queue for Node.js
- *
- * @packageDocumentation
- */
-// Main exports
-export { createDatabase } from './database.js';
-export { createClient } from './client.js';
-export { createWorker } from './worker.js';
-export { createDashboard, createDashboardMiddleware } from './dashboard.js';
-// Utility exports
-export { defaultBackoff, validatePayload } from './utils.js';
+// src/database.ts
+import Database from "better-sqlite3";
+import { Kysely, SqliteDialect } from "kysely";
+function createDatabase(options = {}) {
+  let sqliteDb;
+  if (options.db) {
+    sqliteDb = options.db;
+  } else {
+    const filename = options.filename ?? ":memory:";
+    sqliteDb = new Database(filename);
+  }
+  sqliteDb.pragma("journal_mode = WAL");
+  sqliteDb.pragma("synchronous = NORMAL");
+  sqliteDb.pragma("busy_timeout = 5000");
+  const db = new Kysely({
+    dialect: new SqliteDialect({
+      database: sqliteDb
+    })
+  });
+  createSchema(sqliteDb);
+  return db;
+}
+function createSchema(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS workmatic_jobs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      public_id TEXT UNIQUE NOT NULL,
+      queue TEXT NOT NULL,
+      payload TEXT NOT NULL,
+      status TEXT NOT NULL DEFAULT 'ready',
+      priority INTEGER NOT NULL DEFAULT 0,
+      run_at INTEGER NOT NULL,
+      attempts INTEGER NOT NULL DEFAULT 0,
+      max_attempts INTEGER NOT NULL DEFAULT 3,
+      lease_until INTEGER NOT NULL DEFAULT 0,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      last_error TEXT
+    )
+  `);
+  db.exec(`
+    CREATE INDEX IF NOT EXISTS idx_workmatic_jobs_claim 
+    ON workmatic_jobs (queue, status, run_at, priority, id)
+  `);
+  db.exec(`
+    CREATE INDEX IF NOT EXISTS idx_workmatic_jobs_lease 
+    ON workmatic_jobs (status, lease_until)
+  `);
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS workmatic_settings (
+      queue TEXT PRIMARY KEY,
+      paused INTEGER NOT NULL DEFAULT 0,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+}
+
+// src/client.ts
+import { nanoid } from "nanoid";
+import { sql } from "kysely";
+
+// src/utils.ts
+var defaultBackoff = (attempts) => {
+  return 1e3 * Math.pow(2, attempts);
+};
+function validatePayload(payload) {
+  try {
+    return JSON.stringify(payload);
+  } catch (error) {
+    throw new Error(
+      `Payload is not JSON-serializable: ${error instanceof Error ? error.message : "Unknown error"}`
+    );
+  }
+}
+function parsePayload(json) {
+  try {
+    return JSON.parse(json);
+  } catch (error) {
+    throw new Error(
+      `Invalid JSON payload: ${error instanceof Error ? error.message : "Unknown error"}`
+    );
+  }
+}
+function now() {
+  return Date.now();
+}
+
+// src/client.ts
+function createClient(options) {
+  const { db, queue = "default" } = options;
+  if (!db) {
+    throw new Error("Database instance is required");
+  }
+  return {
+    /**
+     * Add a job to the queue
+     */
+    async add(payload, opts = {}) {
+      const {
+        priority = 0,
+        delayMs = 0,
+        maxAttempts = 3
+      } = opts;
+      const payloadJson = validatePayload(payload);
+      const publicId = nanoid();
+      const timestamp = now();
+      const runAt = timestamp + delayMs;
+      await db.insertInto("workmatic_jobs").values({
+        public_id: publicId,
+        queue,
+        payload: payloadJson,
+        status: "ready",
+        priority,
+        run_at: runAt,
+        attempts: 0,
+        max_attempts: maxAttempts,
+        lease_until: 0,
+        created_at: timestamp,
+        updated_at: timestamp,
+        last_error: null
+      }).execute();
+      return { ok: true, id: publicId };
+    },
+    /**
+     * Get job statistics for the queue
+     */
+    async stats() {
+      const result = await db.selectFrom("workmatic_jobs").select([
+        "status",
+        sql`count(*)`.as("count")
+      ]).where("queue", "=", queue).groupBy("status").execute();
+      const stats = {
+        ready: 0,
+        running: 0,
+        done: 0,
+        failed: 0,
+        dead: 0,
+        total: 0
+      };
+      for (const row of result) {
+        const status = row.status;
+        const count = Number(row.count);
+        if (status in stats) {
+          stats[status] = count;
+        }
+        stats.total += count;
+      }
+      return stats;
+    }
+  };
+}
+
+// src/worker.ts
+import fastq from "fastq";
+import { sql as sql2 } from "kysely";
+function createWorker(options) {
+  const {
+    db,
+    queue = "default",
+    concurrency = 1,
+    leaseMs = 3e4,
+    pollMs = 1e3,
+    timeoutMs,
+    backoff = defaultBackoff
+  } = options;
+  if (!db) {
+    throw new Error("Database instance is required");
+  }
+  let running = false;
+  let paused = false;
+  let processor = null;
+  let pumpTimeout = null;
+  let fastqQueue = null;
+  async function requeueExpiredLeases() {
+    const timestamp = now();
+    const result = await db.updateTable("workmatic_jobs").set({
+      status: "ready",
+      lease_until: 0,
+      updated_at: timestamp
+    }).where("status", "=", "running").where("lease_until", "<", timestamp).where("lease_until", ">", 0).execute();
+    return Number(result[0]?.numUpdatedRows ?? 0);
+  }
+  async function claimBatch(limit) {
+    const timestamp = now();
+    const leaseUntil = timestamp + leaseMs;
+    return await db.transaction().execute(async (trx) => {
+      const jobs = await trx.selectFrom("workmatic_jobs").select(["id", "public_id", "queue", "payload", "attempts", "max_attempts"]).where("queue", "=", queue).where("status", "=", "ready").where("run_at", "<=", timestamp).orderBy("priority", "asc").orderBy("id", "asc").limit(limit).execute();
+      if (jobs.length === 0) {
+        return [];
+      }
+      const jobIds = jobs.map((j) => j.id);
+      await trx.updateTable("workmatic_jobs").set({
+        status: "running",
+        lease_until: leaseUntil,
+        updated_at: timestamp
+      }).where("id", "in", jobIds).execute();
+      return jobs;
+    });
+  }
+  async function markDone(jobId) {
+    await db.updateTable("workmatic_jobs").set({
+      status: "done",
+      lease_until: 0,
+      updated_at: now()
+    }).where("id", "=", jobId).execute();
+  }
+  async function markFailed(jobId, attempts, maxAttempts, error) {
+    const timestamp = now();
+    const newAttempts = attempts + 1;
+    const errorMessage = error.message || String(error);
+    if (newAttempts < maxAttempts) {
+      const runAt = timestamp + backoff(newAttempts);
+      await db.updateTable("workmatic_jobs").set({
+        status: "ready",
+        attempts: newAttempts,
+        run_at: runAt,
+        lease_until: 0,
+        last_error: errorMessage,
+        updated_at: timestamp
+      }).where("id", "=", jobId).execute();
+    } else {
+      await db.updateTable("workmatic_jobs").set({
+        status: "dead",
+        attempts: newAttempts,
+        lease_until: 0,
+        last_error: errorMessage,
+        updated_at: timestamp
+      }).where("id", "=", jobId).execute();
+    }
+  }
+  async function withTimeout(promise, ms, jobId) {
+    let timeoutId;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutId = setTimeout(() => {
+        reject(new Error(`Job ${jobId} timed out after ${ms}ms`));
+      }, ms);
+    });
+    try {
+      return await Promise.race([promise, timeoutPromise]);
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+  async function processJob(claimedJob) {
+    if (!processor) {
+      throw new Error("No processor set");
+    }
+    const payload = parsePayload(claimedJob.payload);
+    const job = {
+      id: claimedJob.public_id,
+      queue: claimedJob.queue,
+      payload,
+      status: "running",
+      priority: 0,
+      // Not needed for processing
+      attempts: claimedJob.attempts,
+      maxAttempts: claimedJob.max_attempts,
+      createdAt: 0,
+      // Not needed for processing
+      lastError: null
+    };
+    try {
+      if (timeoutMs) {
+        await withTimeout(processor(job), timeoutMs, job.id);
+      } else {
+        await processor(job);
+      }
+      await markDone(claimedJob.id);
+    } catch (error) {
+      await markFailed(
+        claimedJob.id,
+        claimedJob.attempts,
+        claimedJob.max_attempts,
+        error instanceof Error ? error : new Error(String(error))
+      );
+    }
+  }
+  async function isQueuePausedInDb() {
+    const setting = await db.selectFrom("workmatic_settings").select("paused").where("queue", "=", queue).executeTakeFirst();
+    return setting?.paused === 1;
+  }
+  async function pump() {
+    if (!running) {
+      return;
+    }
+    if (paused) {
+      pumpTimeout = setTimeout(pump, pollMs);
+      return;
+    }
+    try {
+      const dbPaused = await isQueuePausedInDb();
+      if (dbPaused) {
+        pumpTimeout = setTimeout(pump, pollMs);
+        return;
+      }
+      await requeueExpiredLeases();
+      const batchSize = concurrency * 2;
+      const jobs = await claimBatch(batchSize);
+      if (jobs.length > 0) {
+        for (const job of jobs) {
+          fastqQueue.push(job);
+        }
+        pumpTimeout = setTimeout(pump, 0);
+      } else {
+        pumpTimeout = setTimeout(pump, pollMs);
+      }
+    } catch (error) {
+      console.error("[workmatic] Pump error:", error);
+      pumpTimeout = setTimeout(pump, pollMs);
+    }
+  }
+  const worker = {
+    process(fn) {
+      processor = fn;
+    },
+    start() {
+      if (running) {
+        return;
+      }
+      if (!processor) {
+        throw new Error("No processor set. Call process() before start()");
+      }
+      running = true;
+      paused = false;
+      fastqQueue = fastq.promise(processJob, concurrency);
+      pump();
+    },
+    async stop() {
+      if (!running) {
+        return;
+      }
+      running = false;
+      if (pumpTimeout) {
+        clearTimeout(pumpTimeout);
+        pumpTimeout = null;
+      }
+      if (fastqQueue) {
+        await fastqQueue.drained();
+        fastqQueue = null;
+      }
+    },
+    pause() {
+      paused = true;
+    },
+    resume() {
+      paused = false;
+    },
+    async stats() {
+      const result = await db.selectFrom("workmatic_jobs").select([
+        "status",
+        sql2`count(*)`.as("count")
+      ]).where("queue", "=", queue).groupBy("status").execute();
+      const stats = {
+        ready: 0,
+        running: 0,
+        done: 0,
+        failed: 0,
+        dead: 0,
+        total: 0
+      };
+      for (const row of result) {
+        const status = row.status;
+        const count = Number(row.count);
+        if (status in stats) {
+          stats[status] = count;
+        }
+        stats.total += count;
+      }
+      return stats;
+    },
+    get isRunning() {
+      return running;
+    },
+    get isPaused() {
+      return paused;
+    },
+    get queue() {
+      return queue;
+    }
+  };
+  return worker;
+}
+
+// src/dashboard.ts
+import { createServer } from "http";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { readFile } from "fs/promises";
+import { sql as sql3 } from "kysely";
+var __filename2 = fileURLToPath(import.meta.url);
+var __dirname2 = dirname(__filename2);
+var CONTENT_TYPES = {
+  ".html": "text/html; charset=utf-8",
+  ".css": "text/css; charset=utf-8",
+  ".js": "application/javascript; charset=utf-8",
+  ".json": "application/json; charset=utf-8"
+};
+function createRequestHandler(db, workerMap, basePath = "") {
+  function sendJson(res, data, status = 200) {
+    res.writeHead(status, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(data));
+  }
+  function sendError(res, message, status = 500) {
+    sendJson(res, { error: message }, status);
+  }
+  function parseQuery(url) {
+    const queryIndex = url.indexOf("?");
+    if (queryIndex === -1) return new URLSearchParams();
+    return new URLSearchParams(url.slice(queryIndex + 1));
+  }
+  function getPath(url) {
+    const queryIndex = url.indexOf("?");
+    let path = queryIndex === -1 ? url : url.slice(0, queryIndex);
+    if (basePath && path.startsWith(basePath)) {
+      path = path.slice(basePath.length) || "/";
+    }
+    return path;
+  }
+  async function handleGetJobs(req, res) {
+    const query = parseQuery(req.url || "");
+    const queueFilter = query.get("queue");
+    const statusFilter = query.get("status");
+    const limit = Math.min(parseInt(query.get("limit") || "50", 10), 100);
+    const offset = parseInt(query.get("offset") || "0", 10);
+    let queryBuilder = db.selectFrom("workmatic_jobs").select([
+      "public_id",
+      "queue",
+      "payload",
+      "status",
+      "priority",
+      "attempts",
+      "max_attempts",
+      "run_at",
+      "created_at",
+      "updated_at",
+      "last_error"
+    ]).orderBy("created_at", "desc").limit(limit).offset(offset);
+    if (queueFilter) {
+      queryBuilder = queryBuilder.where("queue", "=", queueFilter);
+    }
+    if (statusFilter) {
+      queryBuilder = queryBuilder.where("status", "=", statusFilter);
+    }
+    const jobs = await queryBuilder.execute();
+    const apiJobs = jobs.map((job) => ({
+      id: job.public_id,
+      queue: job.queue,
+      payload: JSON.parse(job.payload),
+      status: job.status,
+      priority: job.priority,
+      attempts: job.attempts,
+      maxAttempts: job.max_attempts,
+      runAt: job.run_at,
+      createdAt: job.created_at,
+      updatedAt: job.updated_at,
+      lastError: job.last_error
+    }));
+    sendJson(res, { jobs: apiJobs, limit, offset });
+  }
+  async function handleGetStats(req, res) {
+    const query = parseQuery(req.url || "");
+    const queueFilter = query.get("queue");
+    let statsQuery = db.selectFrom("workmatic_jobs").select([
+      "status",
+      sql3`count(*)`.as("count")
+    ]).groupBy("status");
+    if (queueFilter) {
+      statsQuery = statsQuery.where("queue", "=", queueFilter);
+    }
+    const statusCounts = await statsQuery.execute();
+    const stats = {
+      ready: 0,
+      running: 0,
+      done: 0,
+      failed: 0,
+      dead: 0,
+      total: 0
+    };
+    for (const row of statusCounts) {
+      const status = row.status;
+      const count = Number(row.count);
+      if (status in stats) {
+        stats[status] = count;
+      }
+      stats.total += count;
+    }
+    const queuesResult = await db.selectFrom("workmatic_jobs").select("queue").groupBy("queue").execute();
+    const queues = queuesResult.map((q) => q.queue);
+    const workersStatus = Array.from(workerMap.entries()).map(([queue, worker]) => ({
+      queue,
+      running: worker.isRunning,
+      paused: worker.isPaused
+    }));
+    sendJson(res, { stats, queues, workers: workersStatus });
+  }
+  async function handleGetJob(req, res, publicId) {
+    const job = await db.selectFrom("workmatic_jobs").select([
+      "public_id",
+      "queue",
+      "payload",
+      "status",
+      "priority",
+      "attempts",
+      "max_attempts",
+      "run_at",
+      "lease_until",
+      "created_at",
+      "updated_at",
+      "last_error"
+    ]).where("public_id", "=", publicId).executeTakeFirst();
+    if (!job) {
+      return sendError(res, "Job not found", 404);
+    }
+    sendJson(res, {
+      id: job.public_id,
+      queue: job.queue,
+      payload: JSON.parse(job.payload),
+      status: job.status,
+      priority: job.priority,
+      attempts: job.attempts,
+      maxAttempts: job.max_attempts,
+      runAt: job.run_at,
+      leaseUntil: job.lease_until,
+      createdAt: job.created_at,
+      updatedAt: job.updated_at,
+      lastError: job.last_error
+    });
+  }
+  async function handlePauseWorker(res, queue) {
+    const worker = workerMap.get(queue);
+    if (!worker) {
+      return sendError(res, `Worker for queue '${queue}' not found`, 404);
+    }
+    worker.pause();
+    sendJson(res, { ok: true, queue, paused: true });
+  }
+  async function handleResumeWorker(res, queue) {
+    const worker = workerMap.get(queue);
+    if (!worker) {
+      return sendError(res, `Worker for queue '${queue}' not found`, 404);
+    }
+    worker.resume();
+    sendJson(res, { ok: true, queue, paused: false });
+  }
+  async function serveStatic(res, filePath) {
+    const dashboardDir = join(__dirname2, "..", "dashboard");
+    const fullPath = join(dashboardDir, filePath);
+    const ext = filePath.substring(filePath.lastIndexOf(".")) || ".html";
+    const contentType = CONTENT_TYPES[ext] || "application/octet-stream";
+    try {
+      const content = await readFile(fullPath, "utf-8");
+      res.writeHead(200, { "Content-Type": contentType });
+      res.end(content);
+    } catch (error) {
+      sendError(res, "Not found", 404);
+    }
+  }
+  return async function handleRequest(req, res, next) {
+    const path = getPath(req.url || "/");
+    const fullUrl = req.url || "/";
+    if (basePath && !fullUrl.startsWith(basePath)) {
+      if (next) next();
+      return false;
+    }
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+    res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+    if (req.method === "OPTIONS") {
+      res.writeHead(204);
+      res.end();
+      return true;
+    }
+    try {
+      if (path === "/api/jobs" && req.method === "GET") {
+        await handleGetJobs(req, res);
+        return true;
+      }
+      if (path === "/api/stats" && req.method === "GET") {
+        await handleGetStats(req, res);
+        return true;
+      }
+      const jobMatch = path.match(/^\/api\/jobs\/([^/]+)$/);
+      if (jobMatch && req.method === "GET") {
+        await handleGetJob(req, res, jobMatch[1]);
+        return true;
+      }
+      const pauseMatch = path.match(/^\/api\/workers\/([^/]+)\/pause$/);
+      if (pauseMatch && req.method === "POST") {
+        await handlePauseWorker(res, pauseMatch[1]);
+        return true;
+      }
+      const resumeMatch = path.match(/^\/api\/workers\/([^/]+)\/resume$/);
+      if (resumeMatch && req.method === "POST") {
+        await handleResumeWorker(res, resumeMatch[1]);
+        return true;
+      }
+      if (path === "/" || path === "/index.html") {
+        await serveStatic(res, "index.html");
+        return true;
+      }
+      if (path === "/style.css") {
+        await serveStatic(res, "style.css");
+        return true;
+      }
+      if (path === "/app.js") {
+        await serveStatic(res, "app.js");
+        return true;
+      }
+      sendError(res, "Not found", 404);
+      return true;
+    } catch (error) {
+      console.error("[workmatic] Dashboard error:", error);
+      sendError(res, "Internal server error", 500);
+      return true;
+    }
+  };
+}
+function createDashboard(options) {
+  const {
+    db,
+    port = 3e3,
+    workers = []
+  } = options;
+  if (!db) {
+    throw new Error("Database instance is required");
+  }
+  const workerMap = /* @__PURE__ */ new Map();
+  for (const worker of workers) {
+    workerMap.set(worker.queue, worker);
+  }
+  const handleRequest = createRequestHandler(db, workerMap);
+  const server = createServer((req, res) => {
+    handleRequest(req, res).catch((error) => {
+      console.error("[workmatic] Unhandled error:", error);
+      res.writeHead(500, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Internal server error" }));
+    });
+  });
+  server.listen(port);
+  return {
+    async close() {
+      return new Promise((resolve, reject) => {
+        server.close((err) => {
+          if (err) reject(err);
+          else resolve();
+        });
+      });
+    },
+    get port() {
+      return port;
+    }
+  };
+}
+function createDashboardMiddleware(options) {
+  const {
+    db,
+    workers = [],
+    basePath = ""
+  } = options;
+  if (!db) {
+    throw new Error("Database instance is required");
+  }
+  const workerMap = /* @__PURE__ */ new Map();
+  for (const worker of workers) {
+    workerMap.set(worker.queue, worker);
+  }
+  const handleRequest = createRequestHandler(db, workerMap, basePath);
+  return (req, res, next) => {
+    handleRequest(req, res, next).catch((error) => {
+      console.error("[workmatic] Unhandled error:", error);
+      if (next) {
+        next();
+      } else {
+        res.writeHead(500, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ error: "Internal server error" }));
+      }
+    });
+  };
+}
+export {
+  createClient,
+  createDashboard,
+  createDashboardMiddleware,
+  createDatabase,
+  createWorker,
+  defaultBackoff,
+  validatePayload
+};
 //# sourceMappingURL=index.js.map