workmatic 1.0.4 → 1.0.7

package/dist/index.d.cts

@@ -1,11 +1,12 @@
-import * as http from 'http';
 import * as better_sqlite3 from 'better-sqlite3';
+import better_sqlite3__default from 'better-sqlite3';
+import * as http from 'http';
 import { Kysely, Generated } from 'kysely';
 
 /**
  * Job status types
  */
-type JobStatus = 'ready' | 'running' | 'done' | 'failed' | 'dead';
+type JobStatus = 'ready' | 'running' | 'done' | 'dead';
 /**
  * Database table schema for workmatic_jobs
  */
@@ -63,7 +64,7 @@ interface Job<TPayload = unknown> {
     maxAttempts: number;
     /** When the job was created (unix ms) */
     createdAt: number;
-    /** Last error message if failed */
+    /** Last error from a previous attempt (e.g. before retry) */
     lastError: string | null;
 }
 /**
@@ -84,6 +85,13 @@ interface AddJobResult {
     ok: true;
     id: string;
 }
+/**
+ * Result of adding multiple jobs in one transaction
+ */
+interface AddManyResult {
+    ok: true;
+    ids: string[];
+}
 /**
  * Options for creating a database
  */
@@ -124,7 +132,7 @@ interface WorkerOptions {
     leaseMs?: number;
     /** Poll interval in ms when no jobs available. Default: 1000 */
     pollMs?: number;
-    /** Job execution timeout in ms. Default: undefined (no timeout) */
+    /** Job execution timeout in ms. Default: 60000 (1 min). Set to 0 to disable. */
     timeoutMs?: number;
     /** Backoff function for retries. Default: exponential */
     backoff?: BackoffFunction;
@@ -132,6 +140,18 @@ interface WorkerOptions {
     persistState?: boolean;
     /** Auto-restore worker state on creation. Default: true (only applies if persistState is true) */
     autoRestore?: boolean;
+    /**
+     * Minimum interval between database pause checks (CLI `pause`). Reduces round-trips while pumping.
+     * Default: 300 ms
+     */
+    pauseCheckIntervalMs?: number;
+    /**
+     * Minimum interval between lease requeue scans. Default: every pump (0).
+     * Set to e.g. 1000 to run expired-lease recovery at most once per second.
+     */
+    requeueExpiredIntervalMs?: number;
+    /** Called when the pump loop catches an error (after optional default logging) */
+    onPumpError?: (error: unknown) => void;
 }
 /**
  * Job processor function type
@@ -144,7 +164,6 @@ interface JobStats {
     ready: number;
     running: number;
     done: number;
-    failed: number;
     dead: number;
     total: number;
 }
@@ -154,8 +173,17 @@ interface JobStats {
 interface WorkmaticClient {
     /** Add a job to the queue */
     add<TPayload = unknown>(payload: TPayload, options?: AddJobOptions): Promise<AddJobResult>;
+    /**
+     * Add many jobs in a single transaction (shared priority, delay, maxAttempts).
+     * Faster than repeated `add()` when inserting large batches.
+     */
+    addMany<TPayload = unknown>(payloads: TPayload[], options?: AddJobOptions): Promise<AddManyResult>;
     /** Get job statistics */
     stats(): Promise<JobStats>;
+    /** Clear all jobs from the queue */
+    clear(options?: {
+        status?: JobStatus;
+    }): Promise<number>;
 }
 /**
  * Worker interface
@@ -175,6 +203,10 @@ interface WorkmaticWorker {
     stats(): Promise<JobStats>;
     /** Restore worker state from database (only when persistState is true) */
     restoreState(): Promise<WorkerState | null>;
+    /** Clear all jobs from the queue */
+    clear(options?: {
+        status?: JobStatus;
+    }): Promise<number>;
     /** Check if worker is running */
     readonly isRunning: boolean;
     /** Check if worker is paused */
@@ -227,6 +259,9 @@ interface ClaimedJob {
     payload: string;
     attempts: number;
     max_attempts: number;
+    priority: number;
+    created_at: number;
+    last_error: string | null;
 }
 
 /**
@@ -250,6 +285,12 @@ interface ClaimedJob {
  * ```
  */
 declare function createDatabase(options?: DatabaseOptions): WorkmaticDb;
+/**
+ * Get the underlying better-sqlite3 database instance from a Kysely instance
+ * created with {@link createDatabase}. For manually constructed `Kysely` instances,
+ * falls back to reading the dialect adapter (may break across Kysely versions).
+ */
+declare function getUnderlyingDb(db: WorkmaticDb): better_sqlite3__default.Database;
 
 /**
  * Create a job queue client for adding jobs
@@ -278,6 +319,8 @@ declare function createDatabase(options?: DatabaseOptions): WorkmaticDb;
  */
 declare function createClient(options: ClientOptions): WorkmaticClient;
 
+/** Default job execution timeout when `timeoutMs` is omitted (1 minute). Use `timeoutMs: 0` for no limit. */
+declare const DEFAULT_WORKER_TIMEOUT_MS = 60000;
 /**
  * Create a job queue worker for processing jobs
  *
@@ -375,4 +418,4 @@ declare const defaultBackoff: BackoffFunction;
  */
 declare function validatePayload(payload: unknown): string;
 
-export { type AddJobOptions, type AddJobResult, type BackoffFunction, type ClaimedJob, type ClientOptions, type DashboardMiddleware, type DashboardMiddlewareOptions, type DashboardOptions, type DatabaseOptions, type Job, type JobProcessor, type JobStats, type JobStatus, type WorkerOptions, type WorkerState, type WorkmaticClient, type WorkmaticDashboard, type WorkmaticDatabase, type WorkmaticDb, type WorkmaticJobsTable, type WorkmaticWorker, createClient, createDashboard, createDashboardMiddleware, createDatabase, createWorker, defaultBackoff, validatePayload };
+export { type AddJobOptions, type AddJobResult, type AddManyResult, type BackoffFunction, type ClaimedJob, type ClientOptions, DEFAULT_WORKER_TIMEOUT_MS, type DashboardMiddleware, type DashboardMiddlewareOptions, type DashboardOptions, type DatabaseOptions, type Job, type JobProcessor, type JobStats, type JobStatus, type WorkerOptions, type WorkerState, type WorkmaticClient, type WorkmaticDashboard, type WorkmaticDatabase, type WorkmaticDb, type WorkmaticJobsTable, type WorkmaticWorker, createClient, createDashboard, createDashboardMiddleware, createDatabase, createWorker, defaultBackoff, getUnderlyingDb, validatePayload };

package/dist/index.d.ts

@@ -1,11 +1,12 @@
-import * as http from 'http';
 import * as better_sqlite3 from 'better-sqlite3';
+import better_sqlite3__default from 'better-sqlite3';
+import * as http from 'http';
 import { Kysely, Generated } from 'kysely';
 
 /**
  * Job status types
  */
-type JobStatus = 'ready' | 'running' | 'done' | 'failed' | 'dead';
+type JobStatus = 'ready' | 'running' | 'done' | 'dead';
 /**
  * Database table schema for workmatic_jobs
  */
@@ -63,7 +64,7 @@ interface Job<TPayload = unknown> {
     maxAttempts: number;
     /** When the job was created (unix ms) */
     createdAt: number;
-    /** Last error message if failed */
+    /** Last error from a previous attempt (e.g. before retry) */
     lastError: string | null;
 }
 /**
@@ -84,6 +85,13 @@ interface AddJobResult {
     ok: true;
     id: string;
 }
+/**
+ * Result of adding multiple jobs in one transaction
+ */
+interface AddManyResult {
+    ok: true;
+    ids: string[];
+}
 /**
  * Options for creating a database
  */
@@ -124,7 +132,7 @@ interface WorkerOptions {
     leaseMs?: number;
     /** Poll interval in ms when no jobs available. Default: 1000 */
     pollMs?: number;
-    /** Job execution timeout in ms. Default: undefined (no timeout) */
+    /** Job execution timeout in ms. Default: 60000 (1 min). Set to 0 to disable. */
     timeoutMs?: number;
     /** Backoff function for retries. Default: exponential */
     backoff?: BackoffFunction;
@@ -132,6 +140,18 @@ interface WorkerOptions {
     persistState?: boolean;
     /** Auto-restore worker state on creation. Default: true (only applies if persistState is true) */
     autoRestore?: boolean;
+    /**
+     * Minimum interval between database pause checks (CLI `pause`). Reduces round-trips while pumping.
+     * Default: 300 ms
+     */
+    pauseCheckIntervalMs?: number;
+    /**
+     * Minimum interval between lease requeue scans. Default: every pump (0).
+     * Set to e.g. 1000 to run expired-lease recovery at most once per second.
+     */
+    requeueExpiredIntervalMs?: number;
+    /** Called when the pump loop catches an error (after optional default logging) */
+    onPumpError?: (error: unknown) => void;
 }
 /**
  * Job processor function type
@@ -144,7 +164,6 @@ interface JobStats {
     ready: number;
     running: number;
     done: number;
-    failed: number;
     dead: number;
     total: number;
 }
@@ -154,8 +173,17 @@ interface JobStats {
 interface WorkmaticClient {
     /** Add a job to the queue */
     add<TPayload = unknown>(payload: TPayload, options?: AddJobOptions): Promise<AddJobResult>;
+    /**
+     * Add many jobs in a single transaction (shared priority, delay, maxAttempts).
+     * Faster than repeated `add()` when inserting large batches.
+     */
+    addMany<TPayload = unknown>(payloads: TPayload[], options?: AddJobOptions): Promise<AddManyResult>;
     /** Get job statistics */
     stats(): Promise<JobStats>;
+    /** Clear all jobs from the queue */
+    clear(options?: {
+        status?: JobStatus;
+    }): Promise<number>;
 }
 /**
  * Worker interface
@@ -175,6 +203,10 @@ interface WorkmaticWorker {
     stats(): Promise<JobStats>;
     /** Restore worker state from database (only when persistState is true) */
     restoreState(): Promise<WorkerState | null>;
+    /** Clear all jobs from the queue */
+    clear(options?: {
+        status?: JobStatus;
+    }): Promise<number>;
     /** Check if worker is running */
     readonly isRunning: boolean;
     /** Check if worker is paused */
@@ -227,6 +259,9 @@ interface ClaimedJob {
     payload: string;
     attempts: number;
     max_attempts: number;
+    priority: number;
+    created_at: number;
+    last_error: string | null;
 }
 
 /**
@@ -250,6 +285,12 @@ interface ClaimedJob {
  * ```
  */
 declare function createDatabase(options?: DatabaseOptions): WorkmaticDb;
+/**
+ * Get the underlying better-sqlite3 database instance from a Kysely instance
+ * created with {@link createDatabase}. For manually constructed `Kysely` instances,
+ * falls back to reading the dialect adapter (may break across Kysely versions).
+ */
+declare function getUnderlyingDb(db: WorkmaticDb): better_sqlite3__default.Database;
 
 /**
  * Create a job queue client for adding jobs
@@ -278,6 +319,8 @@ declare function createDatabase(options?: DatabaseOptions): WorkmaticDb;
  */
 declare function createClient(options: ClientOptions): WorkmaticClient;
 
+/** Default job execution timeout when `timeoutMs` is omitted (1 minute). Use `timeoutMs: 0` for no limit. */
+declare const DEFAULT_WORKER_TIMEOUT_MS = 60000;
 /**
  * Create a job queue worker for processing jobs
  *
@@ -375,4 +418,4 @@ declare const defaultBackoff: BackoffFunction;
  */
 declare function validatePayload(payload: unknown): string;
 
-export { type AddJobOptions, type AddJobResult, type BackoffFunction, type ClaimedJob, type ClientOptions, type DashboardMiddleware, type DashboardMiddlewareOptions, type DashboardOptions, type DatabaseOptions, type Job, type JobProcessor, type JobStats, type JobStatus, type WorkerOptions, type WorkerState, type WorkmaticClient, type WorkmaticDashboard, type WorkmaticDatabase, type WorkmaticDb, type WorkmaticJobsTable, type WorkmaticWorker, createClient, createDashboard, createDashboardMiddleware, createDatabase, createWorker, defaultBackoff, validatePayload };
+export { type AddJobOptions, type AddJobResult, type AddManyResult, type BackoffFunction, type ClaimedJob, type ClientOptions, DEFAULT_WORKER_TIMEOUT_MS, type DashboardMiddleware, type DashboardMiddlewareOptions, type DashboardOptions, type DatabaseOptions, type Job, type JobProcessor, type JobStats, type JobStatus, type WorkerOptions, type WorkerState, type WorkmaticClient, type WorkmaticDashboard, type WorkmaticDatabase, type WorkmaticDb, type WorkmaticJobsTable, type WorkmaticWorker, createClient, createDashboard, createDashboardMiddleware, createDatabase, createWorker, defaultBackoff, getUnderlyingDb, validatePayload };

package/dist/index.js

@@ -1,6 +1,7 @@
 // src/database.ts
 import Database from "better-sqlite3";
 import { Kysely, SqliteDialect } from "kysely";
+var kyselyToSqlite = /* @__PURE__ */ new WeakMap();
 function createDatabase(options = {}) {
   let sqliteDb;
   if (options.db) {
@@ -18,6 +19,7 @@ function createDatabase(options = {}) {
     })
   });
   createSchema(sqliteDb);
+  kyselyToSqlite.set(db, sqliteDb);
   return db;
 }
 function createSchema(db) {
@@ -53,6 +55,26 @@ function createSchema(db) {
       updated_at INTEGER NOT NULL
     )
   `);
+  db.exec(`
+    UPDATE workmatic_jobs SET status = 'dead' WHERE status = 'failed'
+  `);
+}
+function getUnderlyingDb(db) {
+  const mapped = kyselyToSqlite.get(db);
+  if (mapped) {
+    return mapped;
+  }
+  try {
+    const ex = db.getExecutor?.();
+    const dialect = ex?.adapter?.db;
+    if (dialect) {
+      return dialect;
+    }
+  } catch {
+  }
+  throw new Error(
+    "getUnderlyingDb: could not resolve better-sqlite3 instance (use createDatabase() or pass db from it)"
+  );
 }
 
 // src/client.ts
@@ -121,6 +143,42 @@ function createClient(options) {
       }).execute();
       return { ok: true, id: publicId };
     },
+    async addMany(payloads, opts = {}) {
+      const {
+        priority = 0,
+        delayMs = 0,
+        maxAttempts = 3
+      } = opts;
+      if (payloads.length === 0) {
+        return { ok: true, ids: [] };
+      }
+      const timestamp = now();
+      const runAt = timestamp + delayMs;
+      return await db.transaction().execute(async (trx) => {
+        const ids = [];
+        const rows = payloads.map((payload) => {
+          const payloadJson = validatePayload(payload);
+          const publicId = nanoid();
+          ids.push(publicId);
+          return {
+            public_id: publicId,
+            queue,
+            payload: payloadJson,
+            status: "ready",
+            priority,
+            run_at: runAt,
+            attempts: 0,
+            max_attempts: maxAttempts,
+            lease_until: 0,
+            created_at: timestamp,
+            updated_at: timestamp,
+            last_error: null
+          };
+        });
+        await trx.insertInto("workmatic_jobs").values(rows).execute();
+        return { ok: true, ids };
+      });
+    },
     /**
      * Get job statistics for the queue
      */
@@ -133,7 +191,6 @@ function createClient(options) {
         ready: 0,
         running: 0,
         done: 0,
-        failed: 0,
         dead: 0,
         total: 0
       };
@@ -146,6 +203,17 @@ function createClient(options) {
         stats.total += count;
       }
       return stats;
+    },
+    /**
+     * Clear all jobs from the queue
+     */
+    async clear(options2 = {}) {
+      let query = db.deleteFrom("workmatic_jobs").where("queue", "=", queue);
+      if (options2.status) {
+        query = query.where("status", "=", options2.status);
+      }
+      const result = await query.execute();
+      return Number(result[0]?.numDeletedRows ?? 0);
     }
   };
 }
@@ -153,6 +221,7 @@ function createClient(options) {
 // src/worker.ts
 import fastq from "fastq";
 import { sql as sql2 } from "kysely";
+var DEFAULT_WORKER_TIMEOUT_MS = 6e4;
 function createWorker(options) {
   const {
     db,
@@ -160,10 +229,13 @@ function createWorker(options) {
     concurrency = 1,
     leaseMs = 3e4,
     pollMs = 1e3,
-    timeoutMs,
+    timeoutMs = DEFAULT_WORKER_TIMEOUT_MS,
     backoff = defaultBackoff,
     persistState = false,
-    autoRestore = true
+    autoRestore = true,
+    pauseCheckIntervalMs = 300,
+    requeueExpiredIntervalMs = 0,
+    onPumpError
   } = options;
   if (!db) {
     throw new Error("Database instance is required");
@@ -173,6 +245,13 @@ function createWorker(options) {
   let processor = null;
   let pumpTimeout = null;
   let fastqQueue = null;
+  let lastPauseCheckAt = 0;
+  let cachedDbPaused = false;
+  let lastRequeueAt = 0;
+  function notifyPumpError(error) {
+    console.error("[workmatic] Pump error:", error);
+    onPumpError?.(error);
+  }
   function getStateKey() {
     return `worker_state_${queue}`;
   }
@@ -180,8 +259,6 @@ function createWorker(options) {
     if (!persistState) return;
     const timestamp = now();
     const key = getStateKey();
-    await db.schema.createTable("workmatic_settings").ifNotExists().addColumn("queue", "text", (col) => col.primaryKey()).addColumn("paused", "integer", (col) => col.notNull().defaultTo(0)).addColumn("updated_at", "integer", (col) => col.notNull()).execute().catch(() => {
-    });
     await sql2`
       INSERT INTO workmatic_settings (queue, paused, updated_at)
       VALUES (${key}, ${state === "paused" ? 1 : state === "running" ? 2 : 0}, ${timestamp})
@@ -216,17 +293,34 @@ function createWorker(options) {
     const timestamp = now();
     const leaseUntil = timestamp + leaseMs;
     return await db.transaction().execute(async (trx) => {
-      const jobs = await trx.selectFrom("workmatic_jobs").select(["id", "public_id", "queue", "payload", "attempts", "max_attempts"]).where("queue", "=", queue).where("status", "=", "ready").where("run_at", "<=", timestamp).orderBy("priority", "asc").orderBy("id", "asc").limit(limit).execute();
-      if (jobs.length === 0) {
+      const result = await sql2`
+        UPDATE workmatic_jobs
+        SET status = 'running', lease_until = ${leaseUntil}, updated_at = ${timestamp}
+        WHERE rowid IN (
+          SELECT rowid FROM workmatic_jobs
+          WHERE queue = ${queue}
+            AND status = 'ready'
+            AND run_at <= ${timestamp}
+          ORDER BY priority ASC, id ASC
+          LIMIT ${limit}
+        )
+        RETURNING id, public_id, queue, payload, attempts, max_attempts, priority, created_at, last_error
+      `.execute(trx);
+      const rows = result.rows;
+      if (!rows || !Array.isArray(rows)) {
         return [];
       }
-      const jobIds = jobs.map((j) => j.id);
-      await trx.updateTable("workmatic_jobs").set({
-        status: "running",
-        lease_until: leaseUntil,
-        updated_at: timestamp
-      }).where("id", "in", jobIds).execute();
-      return jobs;
+      return rows.map((row) => ({
+        id: row.id,
+        public_id: row.public_id,
+        queue: row.queue,
+        payload: row.payload,
+        attempts: row.attempts,
+        max_attempts: row.max_attempts,
+        priority: row.priority,
+        created_at: row.created_at,
+        last_error: row.last_error
+      }));
     });
   }
   async function markDone(jobId) {
@@ -283,13 +377,11 @@ function createWorker(options) {
       queue: claimedJob.queue,
       payload,
       status: "running",
-      priority: 0,
-      // Not needed for processing
+      priority: claimedJob.priority,
       attempts: claimedJob.attempts,
       maxAttempts: claimedJob.max_attempts,
-      createdAt: 0,
-      // Not needed for processing
-      lastError: null
+      createdAt: claimedJob.created_at,
+      lastError: claimedJob.last_error
     };
     try {
       if (timeoutMs) {
@@ -320,12 +412,21 @@ function createWorker(options) {
       return;
     }
     try {
-      const dbPaused = await isQueuePausedInDb();
-      if (dbPaused) {
+      const t = now();
+      if (t - lastPauseCheckAt >= pauseCheckIntervalMs) {
+        lastPauseCheckAt = t;
+        cachedDbPaused = await isQueuePausedInDb();
+      }
+      if (cachedDbPaused) {
         pumpTimeout = setTimeout(pump, pollMs);
         return;
       }
-      await requeueExpiredLeases();
+      if (requeueExpiredIntervalMs <= 0 || t - lastRequeueAt >= requeueExpiredIntervalMs) {
+        if (requeueExpiredIntervalMs > 0) {
+          lastRequeueAt = t;
+        }
+        await requeueExpiredLeases();
+      }
       const batchSize = concurrency * 2;
       const jobs = await claimBatch(batchSize);
       if (jobs.length > 0) {
@@ -337,7 +438,7 @@ function createWorker(options) {
         pumpTimeout = setTimeout(pump, pollMs);
       }
     } catch (error) {
-      console.error("[workmatic] Pump error:", error);
+      notifyPumpError(error);
       pumpTimeout = setTimeout(pump, pollMs);
     }
   }
@@ -393,7 +494,6 @@ function createWorker(options) {
         ready: 0,
         running: 0,
         done: 0,
-        failed: 0,
         dead: 0,
         total: 0
       };
@@ -425,6 +525,14 @@ function createWorker(options) {
         this.pause();
       }
       return state;
+    },
+    async clear(options2 = {}) {
+      let query = db.deleteFrom("workmatic_jobs").where("queue", "=", queue);
+      if (options2.status) {
+        query = query.where("status", "=", options2.status);
+      }
+      const result = await query.execute();
+      return Number(result[0]?.numDeletedRows ?? 0);
     }
   };
   if (persistState && autoRestore) {
@@ -528,7 +636,6 @@ function createRequestHandler(db, workerMap, basePath = "") {
       ready: 0,
       running: 0,
       done: 0,
-      failed: 0,
       dead: 0,
       total: 0
     };
@@ -734,12 +841,14 @@ function createDashboardMiddleware(options) {
   };
 }
 export {
+  DEFAULT_WORKER_TIMEOUT_MS,
   createClient,
   createDashboard,
   createDashboardMiddleware,
   createDatabase,
   createWorker,
   defaultBackoff,
+  getUnderlyingDb,
   validatePayload
 };
 //# sourceMappingURL=index.js.map