workmatic 1.0.7 → 1.1.0

package/dist/index.cjs

@@ -35,6 +35,7 @@ __export(index_exports, {
   createDashboard: () => createDashboard,
   createDashboardMiddleware: () => createDashboardMiddleware,
   createDatabase: () => createDatabase,
+  createOrchestrator: () => createOrchestrator,
   createWorker: () => createWorker,
   defaultBackoff: () => defaultBackoff,
   getUnderlyingDb: () => getUnderlyingDb,
@@ -270,6 +271,29 @@ function createClient(options) {
 var import_fastq = __toESM(require("fastq"), 1);
 var import_kysely3 = require("kysely");
 var DEFAULT_WORKER_TIMEOUT_MS = 6e4;
+function parseClaimedRows(result) {
+  const rows = result.rows;
+  if (!rows || !Array.isArray(rows)) {
+    return [];
+  }
+  return rows.map((row) => ({
+    id: row.id,
+    public_id: row.public_id,
+    queue: row.queue,
+    payload: row.payload,
+    attempts: row.attempts,
+    max_attempts: row.max_attempts,
+    priority: row.priority,
+    created_at: row.created_at,
+    last_error: row.last_error
+  }));
+}
+function requireProcessor(processor) {
+  if (!processor) {
+    throw new Error("No processor set");
+  }
+  return processor;
+}
 function createWorker(options) {
   const {
     db,
@@ -354,21 +378,7 @@ function createWorker(options) {
         )
         RETURNING id, public_id, queue, payload, attempts, max_attempts, priority, created_at, last_error
       `.execute(trx);
-      const rows = result.rows;
-      if (!rows || !Array.isArray(rows)) {
-        return [];
-      }
-      return rows.map((row) => ({
-        id: row.id,
-        public_id: row.public_id,
-        queue: row.queue,
-        payload: row.payload,
-        attempts: row.attempts,
-        max_attempts: row.max_attempts,
-        priority: row.priority,
-        created_at: row.created_at,
-        last_error: row.last_error
-      }));
+      return parseClaimedRows(result);
     });
   }
   async function markDone(jobId) {
@@ -416,9 +426,7 @@ function createWorker(options) {
     }
   }
   async function processJob(claimedJob) {
-    if (!processor) {
-      throw new Error("No processor set");
-    }
+    const fn = requireProcessor(processor);
     const payload = parsePayload(claimedJob.payload);
     const job = {
       id: claimedJob.public_id,
@@ -433,9 +441,9 @@ function createWorker(options) {
     };
     try {
       if (timeoutMs) {
-        await withTimeout(processor(job), timeoutMs, job.id);
+        await withTimeout(fn(job), timeoutMs, job.id);
       } else {
-        await processor(job);
+        await fn(job);
       }
       await markDone(claimedJob.id);
     } catch (error) {
@@ -593,12 +601,211 @@ function createWorker(options) {
   return worker;
 }
 
+// src/orchestrator.ts
+var import_kysely4 = require("kysely");
+var DEFAULT_TRANSFER_STATUSES = ["ready", "dead"];
+var DEFAULT_TRANSFER_LIMIT = 1e4;
+function rowsUpdated(result) {
+  const row = result[0];
+  return Number(row?.numUpdatedRows ?? 0);
+}
+function normalizeStatuses(status) {
+  if (!status) return [...DEFAULT_TRANSFER_STATUSES];
+  return Array.isArray(status) ? status : [status];
+}
+function createOrchestrator(options) {
+  const { db } = options;
+  if (!db) {
+    throw new Error("Database instance is required");
+  }
+  const registry = /* @__PURE__ */ new Map();
+  function ensureEntry(queue) {
+    let entry = registry.get(queue);
+    if (!entry) {
+      entry = { client: createClient({ db, queue }) };
+      registry.set(queue, entry);
+    }
+    return entry;
+  }
+  async function setQueuePaused(queue, paused) {
+    const timestamp = now();
+    const value = paused ? 1 : 0;
+    await import_kysely4.sql`
+      INSERT INTO workmatic_settings (queue, paused, updated_at)
+      VALUES (${queue}, ${value}, ${timestamp})
+      ON CONFLICT(queue) DO UPDATE SET
+        paused = ${value},
+        updated_at = ${timestamp}
+    `.execute(db);
+  }
+  async function selectJobIds(from, status, limit) {
+    const rows = await db.selectFrom("workmatic_jobs").select("id").where("queue", "=", from).where("status", "=", status).orderBy("priority", "asc").orderBy("id", "asc").limit(limit).execute();
+    return rows.map((r) => r.id);
+  }
+  async function transferReady(from, to, limit) {
+    const ids = await selectJobIds(from, "ready", limit);
+    if (ids.length === 0) return 0;
+    const timestamp = now();
+    const result = await db.updateTable("workmatic_jobs").set({ queue: to, updated_at: timestamp }).where("id", "in", ids).execute();
+    return rowsUpdated(result);
+  }
+  async function transferDead(from, to, limit, resetForRetry) {
+    const ids = await selectJobIds(from, "dead", limit);
+    if (ids.length === 0) return 0;
+    const timestamp = now();
+    if (resetForRetry) {
+      const result2 = await db.updateTable("workmatic_jobs").set({
+        queue: to,
+        status: "ready",
+        attempts: 0,
+        lease_until: 0,
+        last_error: null,
+        updated_at: timestamp,
+        run_at: timestamp
+      }).where("id", "in", ids).execute();
+      return rowsUpdated(result2);
+    }
+    const result = await db.updateTable("workmatic_jobs").set({ queue: to, updated_at: timestamp }).where("id", "in", ids).execute();
+    return rowsUpdated(result);
+  }
+  async function transferOtherStatus(from, to, status, limit) {
+    const ids = await selectJobIds(from, status, limit);
+    if (ids.length === 0) return 0;
+    const timestamp = now();
+    const result = await db.updateTable("workmatic_jobs").set({ queue: to, updated_at: timestamp }).where("id", "in", ids).execute();
+    return rowsUpdated(result);
+  }
+  const orchestrator = {
+    register(queue, opts = {}) {
+      const client = createClient({ db, queue });
+      let worker;
+      if (opts.worker) {
+        worker = createWorker({ db, queue, ...opts.worker });
+      }
+      registry.set(queue, { client, worker });
+      return client;
+    },
+    client(queue) {
+      return ensureEntry(queue).client;
+    },
+    worker(queue) {
+      const entry = registry.get(queue);
+      if (!entry?.worker) {
+        throw new Error(`No worker registered for queue "${queue}"`);
+      }
+      return entry.worker;
+    },
+    workers() {
+      return [...registry.values()].map((e) => e.worker).filter((w) => w !== void 0);
+    },
+    async queues() {
+      const rows = await db.selectFrom("workmatic_jobs").select("queue").distinct().execute();
+      const names = new Set(rows.map((r) => r.queue));
+      for (const name of registry.keys()) {
+        names.add(name);
+      }
+      return [...names].sort();
+    },
+    process(queue, fn) {
+      this.worker(queue).process(fn);
+    },
+    startAll() {
+      for (const entry of registry.values()) {
+        entry.worker?.start();
+      }
+    },
+    async stopAll() {
+      const stops = [...registry.values()].map((e) => e.worker?.stop()).filter((p) => p !== void 0);
+      await Promise.all(stops);
+    },
+    async pause(queue) {
+      const entry = registry.get(queue);
+      entry?.worker?.pause();
+      await setQueuePaused(queue, true);
+    },
+    async resume(queue) {
+      const entry = registry.get(queue);
+      entry?.worker?.resume();
+      await setQueuePaused(queue, false);
+    },
+    async isPaused(queue) {
+      const setting = await db.selectFrom("workmatic_settings").select("paused").where("queue", "=", queue).executeTakeFirst();
+      return setting?.paused === 1;
+    },
+    async stats(queueName) {
+      const names = queueName ? [queueName] : await this.queues();
+      const result = {};
+      for (const name of names) {
+        result[name] = await ensureEntry(name).client.stats();
+      }
+      return result;
+    },
+    async transfer(opts) {
+      const { from, to, resetForRetry = false } = opts;
+      if (from === to) {
+        return { moved: 0 };
+      }
+      const statuses = normalizeStatuses(opts.status);
+      let remaining = opts.limit ?? DEFAULT_TRANSFER_LIMIT;
+      let moved = 0;
+      if (statuses.includes("ready") && remaining > 0) {
+        const n = await transferReady(from, to, remaining);
+        moved += n;
+        remaining -= n;
+      }
+      if (statuses.includes("dead") && remaining > 0) {
+        const n = await transferDead(from, to, remaining, resetForRetry);
+        moved += n;
+        remaining -= n;
+      }
+      for (const status of statuses) {
+        if (status === "ready" || status === "dead" || remaining <= 0) continue;
+        const n = await transferOtherStatus(from, to, status, remaining);
+        moved += n;
+        remaining -= n;
+      }
+      return { moved };
+    },
+    async moveJob(publicId, toQueue, opts = {}) {
+      const allowed = normalizeStatuses(opts.status);
+      const row = await db.selectFrom("workmatic_jobs").select(["id", "queue", "status"]).where("public_id", "=", publicId).executeTakeFirst();
+      if (!row) {
+        throw new Error(`Job not found: ${publicId}`);
+      }
+      if (row.queue === toQueue) {
+        return;
+      }
+      if (!allowed.includes(row.status)) {
+        throw new Error(
+          `Job ${publicId} has status "${row.status}" and cannot be moved (allowed: ${allowed.join(", ")})`
+        );
+      }
+      const timestamp = now();
+      const resetForRetry = opts.resetForRetry ?? false;
+      if (row.status === "dead" && resetForRetry) {
+        await db.updateTable("workmatic_jobs").set({
+          queue: toQueue,
+          status: "ready",
+          attempts: 0,
+          lease_until: 0,
+          last_error: null,
+          updated_at: timestamp,
+          run_at: timestamp
+        }).where("public_id", "=", publicId).execute();
+        return;
+      }
+      await db.updateTable("workmatic_jobs").set({ queue: toQueue, updated_at: timestamp }).where("public_id", "=", publicId).execute();
+    }
+  };
+  return orchestrator;
+}
+
 // src/dashboard.ts
 var import_http = require("http");
 var import_url = require("url");
 var import_path = require("path");
 var import_promises = require("fs/promises");
-var import_kysely4 = require("kysely");
+var import_kysely5 = require("kysely");
 var __filename2 = (0, import_url.fileURLToPath)(importMetaUrl);
 var __dirname = (0, import_path.dirname)(__filename2);
 var CONTENT_TYPES = {
@@ -607,6 +814,13 @@ var CONTENT_TYPES = {
   ".js": "application/javascript; charset=utf-8",
   ".json": "application/json; charset=utf-8"
 };
+function requestUrl(url) {
+  return url || "";
+}
+function staticContentTypeFor(filePath) {
+  const ext = filePath.substring(filePath.lastIndexOf(".")) || ".html";
+  return CONTENT_TYPES[ext] || "application/octet-stream";
+}
 function createRequestHandler(db, workerMap, basePath = "") {
   function sendJson(res, data, status = 200) {
     res.writeHead(status, { "Content-Type": "application/json" });
@@ -629,7 +843,7 @@ function createRequestHandler(db, workerMap, basePath = "") {
     return path;
   }
   async function handleGetJobs(req, res) {
-    const query = parseQuery(req.url || "");
+    const query = parseQuery(requestUrl(req.url));
     const queueFilter = query.get("queue");
     const statusFilter = query.get("status");
     const limit = Math.min(parseInt(query.get("limit") || "50", 10), 100);
@@ -670,11 +884,11 @@ function createRequestHandler(db, workerMap, basePath = "") {
     sendJson(res, { jobs: apiJobs, limit, offset });
   }
   async function handleGetStats(req, res) {
-    const query = parseQuery(req.url || "");
+    const query = parseQuery(requestUrl(req.url));
     const queueFilter = query.get("queue");
     let statsQuery = db.selectFrom("workmatic_jobs").select([
       "status",
-      import_kysely4.sql`count(*)`.as("count")
+      import_kysely5.sql`count(*)`.as("count")
     ]).groupBy("status");
     if (queueFilter) {
       statsQuery = statsQuery.where("queue", "=", queueFilter);
@@ -756,8 +970,7 @@ function createRequestHandler(db, workerMap, basePath = "") {
   async function serveStatic(res, filePath) {
     const dashboardDir = (0, import_path.join)(__dirname, "..", "dashboard");
     const fullPath = (0, import_path.join)(dashboardDir, filePath);
-    const ext = filePath.substring(filePath.lastIndexOf(".")) || ".html";
-    const contentType = CONTENT_TYPES[ext] || "application/octet-stream";
+    const contentType = staticContentTypeFor(filePath);
     try {
       const content = await (0, import_promises.readFile)(fullPath, "utf-8");
       res.writeHead(200, { "Content-Type": contentType });
@@ -767,21 +980,21 @@ function createRequestHandler(db, workerMap, basePath = "") {
     }
   }
   return async function handleRequest(req, res, next) {
-    const path = getPath(req.url || "/");
-    const fullUrl = req.url || "/";
-    if (basePath && !fullUrl.startsWith(basePath)) {
-      if (next) next();
-      return false;
-    }
-    res.setHeader("Access-Control-Allow-Origin", "*");
-    res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
-    res.setHeader("Access-Control-Allow-Headers", "Content-Type");
-    if (req.method === "OPTIONS") {
-      res.writeHead(204);
-      res.end();
-      return true;
-    }
     try {
+      const path = getPath(req.url || "/");
+      const fullUrl = req.url || "/";
+      if (basePath && !fullUrl.startsWith(basePath)) {
+        if (next) next();
+        return false;
+      }
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+      res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+      if (req.method === "OPTIONS") {
+        res.writeHead(204);
+        res.end();
+        return true;
+      }
       if (path === "/api/jobs" && req.method === "GET") {
         await handleGetJobs(req, res);
         return true;
@@ -841,11 +1054,7 @@ function createDashboard(options) {
   }
   const handleRequest = createRequestHandler(db, workerMap);
   const server = (0, import_http.createServer)((req, res) => {
-    handleRequest(req, res).catch((error) => {
-      console.error("[workmatic] Unhandled error:", error);
-      res.writeHead(500, { "Content-Type": "application/json" });
-      res.end(JSON.stringify({ error: "Internal server error" }));
-    });
+    void handleRequest(req, res);
   });
   server.listen(port);
   return {
@@ -877,15 +1086,7 @@ function createDashboardMiddleware(options) {
   }
   const handleRequest = createRequestHandler(db, workerMap, basePath);
   return (req, res, next) => {
-    handleRequest(req, res, next).catch((error) => {
-      console.error("[workmatic] Unhandled error:", error);
-      if (next) {
-        next();
-      } else {
-        res.writeHead(500, { "Content-Type": "application/json" });
-        res.end(JSON.stringify({ error: "Internal server error" }));
-      }
-    });
+    void handleRequest(req, res, next);
   };
 }
 // Annotate the CommonJS export names for ESM import in node:
@@ -895,6 +1096,7 @@ function createDashboardMiddleware(options) {
   createDashboard,
   createDashboardMiddleware,
   createDatabase,
+  createOrchestrator,
   createWorker,
   defaultBackoff,
   getUnderlyingDb,