workmatic 1.0.5 → 1.1.0

package/README.md

@@ -2,6 +2,8 @@
 
 A persistent job queue for Node.js using SQLite. Simple, reliable, and zero external dependencies beyond SQLite.
 
+**Full single-page reference (same material, browsable offline): [`docs/index.html`](docs/index.html)**.
+
 ## Why?
 
 I love [fastq](https://github.com/mcollina/fastq) - it's fast, simple, and has a great API. But it's in-memory only, which can be frustrating when you need jobs to survive process restarts or crashes.
@@ -61,13 +63,57 @@ worker.process(async (job) => {
 worker.start();
 ```
 
+## Multi-queue orchestration
+
+Use `createOrchestrator` to register several queues, control all workers together, and move jobs between queues:
+
+```typescript
+import { createDatabase, createOrchestrator } from 'workmatic';
+
+const db = createDatabase({ filename: './jobs.db' });
+const orch = createOrchestrator({ db });
+
+orch.register('emails', { worker: { concurrency: 4 } });
+orch.register('reports', { worker: { concurrency: 1 } });
+
+orch.process('emails', async (job) => { /* send email */ });
+orch.process('reports', async (job) => { /* generate PDF */ });
+
+await orch.client('emails').add({ to: 'user@example.com' });
+
+orch.startAll();
+
+// Move ready/dead jobs to another queue (e.g. retry pipeline)
+await orch.transfer({ from: 'emails', to: 'emails-retry', status: 'dead', resetForRetry: true });
+
+await orch.pause('reports');
+await orch.resume('reports');
+
+await orch.stopAll();
+```
+
+CLI transfer: `workmatic transfer ./jobs.db emails emails-retry --status=dead --retry`
+
+See [`examples/orchestrator.ts`](examples/orchestrator.ts) (transfer between queues) and [`examples/orchestrator-processors.ts`](examples/orchestrator-processors.ts) (different `process()` per queue).
+
+## Testing
+
+```bash
+npm test
+npm run test:coverage   # requires ≥90% line/branch/function coverage
+```
+
 ## API Reference
 
+Browsable copy with anchored sections: **[`docs/index.html`](docs/index.html)** (API from [`#api-database`](docs/index.html#api-database)).
+
 ### `createDatabase(options)`
 
 Initialize the database connection and schema.
 
 ```typescript
+import { createDatabase, getUnderlyingDb } from 'workmatic';
+
 const db = createDatabase({
   // Option 1: File path (creates or opens existing)
   filename: './jobs.db',
@@ -78,6 +124,9 @@ const db = createDatabase({
   // Option 3: Existing better-sqlite3 instance
   db: existingSqliteInstance,
 });
+
+// Underlying driver (only reliable for instances from createDatabase above)
+const sqlite = getUnderlyingDb(db);
 ```
 
 ### `createClient(options)`
@@ -112,7 +161,18 @@ Get job statistics for the queue.
 
 ```typescript
 const stats = await client.stats();
-// { ready: 5, running: 2, done: 100, failed: 0, dead: 1, total: 108 }
+// { ready: 5, running: 2, done: 100, dead: 1, total: 108 }
+```
+
+#### `client.addMany(payloads, options?)`
+
+Insert many jobs in a **single transaction**, sharing the same `priority`, `delayMs`, and `maxAttempts`.
+
+```typescript
+const { ok, ids } = await client.addMany(
+  [{ email: 'a@x.com' }, { email: 'b@x.com' }],
+  { priority: 1, delayMs: 0, maxAttempts: 3 }
+);
 ```
 
 #### `client.clear(options?)`
@@ -140,10 +200,13 @@ const worker = createWorker({
   concurrency: 1,         // Optional: Parallel job count (default: 1)
   leaseMs: 30000,         // Optional: Job lease duration in ms (default: 30000)
   pollMs: 1000,           // Optional: Poll interval when idle (default: 1000)
-  timeoutMs: 60000,       // Optional: Job execution timeout in ms (default: none)
+  timeoutMs: 60000,       // Optional: Job execution timeout in ms (default: 60000). Use 0 for no limit
   backoff: (n) => 1000 * Math.pow(2, n),  // Optional: Retry backoff function
   persistState: false,    // Optional: Persist worker state to database (default: false)
   autoRestore: true,      // Optional: Auto-restore state on creation (default: true)
+  pauseCheckIntervalMs: 300,       // Optional: Min ms between DB pause checks (CLI pause). Default: 300
+  requeueExpiredIntervalMs: 0,   // Optional: Min ms between lease requeue scans; 0 = every pump
+  onPumpError: (err) => { /* optional hook after default log */ },
 });
 ```
 
@@ -178,6 +241,26 @@ Stop processing and wait for current jobs to finish.
 await worker.stop();
 ```
 
+### `createOrchestrator(options)`
+
+Manage multiple queues with shared lifecycle and transfer helpers.
+
+```typescript
+const orch = createOrchestrator({ db });
+
+orch.register('emails', { worker: { concurrency: 4 } });
+const client = orch.client('emails'); // or use return value of register()
+
+orch.process('emails', async (job) => { /* ... */ });
+orch.startAll();
+await orch.stopAll();
+
+await orch.transfer({ from: 'emails', to: 'retry', status: 'dead', resetForRetry: true });
+await orch.moveJob(jobId, 'archive');
+await orch.pause('emails');
+await orch.resume('emails');
+```
+
 #### `worker.pause()` / `worker.resume()`
 
 Pause and resume job processing.
@@ -330,7 +413,7 @@ interface Job<TPayload> {
   id: string;           // Unique public ID (nanoid)
   queue: string;        // Queue name
   payload: TPayload;    // Your job data
-  status: JobStatus;    // 'ready' | 'running' | 'done' | 'failed' | 'dead'
+  status: JobStatus;    // 'ready' | 'running' | 'done' | 'dead' while running
   priority: number;     // Priority value
   attempts: number;     // Current attempt count (starts at 0)
   maxAttempts: number;  // Maximum attempts allowed
@@ -406,11 +489,14 @@ worker.process(async (job) => {
 | `concurrency` | `1` | Number of jobs to process in parallel |
 | `leaseMs` | `30000` | How long a job is "locked" during processing |
 | `pollMs` | `1000` | How often to check for new jobs when idle |
-| `timeoutMs` | `undefined` | Job execution timeout (fails job if exceeded) |
+| `timeoutMs` | `60000` | Job timeout in ms (`0` = no limit) |
 | `priority` | `0` | Job priority (lower = processed first) |
 | `delayMs` | `0` | Delay before job becomes available |
 | `maxAttempts` | `3` | Maximum processing attempts |
 | `backoff` | `2^n * 1000` | Function returning retry delay in ms |
+| `pauseCheckIntervalMs` | `300` | Throttle CLI/live pause checks from the pump loop |
+| `requeueExpiredIntervalMs` | `0` | Throttle expired-lease requeue (0 = run every pump tick) |
+| `onPumpError` | `undefined` | Callback after the default pump error log |
 
 ## Dashboard
 
@@ -450,6 +536,25 @@ npm run bench
 npm run bench -- --file
 ```
 
+### Micro benchmarks
+
+Short suite: **2,000 sequential `add()` calls** (same code path as the full insert benchmark, smaller batch) plus **1,000 `client.stats()` calls** on a queue that already holds jobs. Useful for quick regression checks without running the full workload.
+
+```bash
+npm run bench:micro
+# or: npm run bench -- --micro
+
+npm run bench:micro -- --file
+# or: npm run bench -- --micro --file
+```
+
+| Benchmark | In-Memory | File-based |
+|-----------|-----------|------------|
+| Micro Sequential Insert (2,000) | ~27,000/s | ~9,300/s |
+| Stats Query (×1,000) | ~9,200/s | ~5,500/s |
+
+Figures are rounded from a representative run; throughput changes with hardware, SQLite settings, and how “warm” the database is.
+
 ### Results Comparison
 
 | Benchmark | In-Memory | File-based |
@@ -461,6 +566,8 @@ npm run bench -- --file
 | Process (concurrency=8) | 10,000/s | 8,300/s |
 | Process (concurrency=16) | 18,000/s | 5,700/s |
 | Mixed Insert+Process | 7,500/s | 3,500/s |
+| Micro Sequential Insert (2,000) | ~27,000/s | ~9,300/s |
+| Stats Query (×1,000) | ~9,200/s | ~5,500/s |
 | Claim + Process Batch | 23,600/s | 11,800/s |
 
 **Note**: File-based performance degrades at high concurrency due to disk I/O. For file-based databases, `concurrency=8` is often optimal. Performance varies by hardware.
@@ -485,15 +592,9 @@ npx workmatic list ./jobs.db --status=dead --limit=10
 
 # Export jobs to CSV
 npx workmatic export ./jobs.db backup.csv
-npx workmatic export ./jobs.db --status=failed > failed.csv
-
-# Import jobs from CSV
-npx workmatic import ./jobs.db backup.csv
-
-# Delete jobs by status
-npx workmatic purge ./jobs.db --status=done
+npx workmatic export ./jobs.db --status=dead > dead-export.csv
 
-# Retry dead/failed jobs (reset to ready)
+# Retry dead jobs (reset to ready)
 npx workmatic retry ./jobs.db --status=dead
 ```
 
@@ -515,7 +616,7 @@ npx workmatic retry ./jobs.db --status=dead
 
 | Option | Description |
 |--------|-------------|
-| `--status=<status>` | Filter by status (ready/running/done/failed/dead) |
+| `--status=<status>` | Filter by status (ready/running/done/dead) |
 | `--queue=<queue>` | Filter by queue name |
 | `--limit=<n>` | Limit results (default: 100) |
 

package/dashboard/app.js

@@ -1,6 +1,6 @@
 // Dashboard state
 const state = {
-  stats: { ready: 0, running: 0, done: 0, failed: 0, dead: 0, total: 0 },
+  stats: { ready: 0, running: 0, done: 0, dead: 0, total: 0 },
   jobs: [],
   queues: [],
   workers: [],
@@ -18,7 +18,6 @@ const elements = {
   statReady: document.getElementById('stat-ready'),
   statRunning: document.getElementById('stat-running'),
   statDone: document.getElementById('stat-done'),
-  statFailed: document.getElementById('stat-failed'),
   statDead: document.getElementById('stat-dead'),
   workersSection: document.getElementById('workers-section'),
   workersGrid: document.getElementById('workers-grid'),
@@ -95,7 +94,6 @@ function updateStatsUI() {
   elements.statReady.textContent = state.stats.ready.toLocaleString();
   elements.statRunning.textContent = state.stats.running.toLocaleString();
   elements.statDone.textContent = state.stats.done.toLocaleString();
-  elements.statFailed.textContent = state.stats.failed.toLocaleString();
   elements.statDead.textContent = state.stats.dead.toLocaleString();
 }
 

package/dashboard/index.html

@@ -60,19 +60,6 @@
             <span class="stat-label">Done</span>
           </div>
         </div>
-        <div class="stat-card failed">
-          <div class="stat-icon">
-            <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
-              <circle cx="12" cy="12" r="10"/>
-              <line x1="12" y1="8" x2="12" y2="12"/>
-              <line x1="12" y1="16" x2="12.01" y2="16"/>
-            </svg>
-          </div>
-          <div class="stat-content">
-            <span class="stat-value" id="stat-failed">0</span>
-            <span class="stat-label">Failed</span>
-          </div>
-        </div>
         <div class="stat-card dead">
           <div class="stat-icon">
             <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
@@ -105,7 +92,6 @@
             <option value="ready">Ready</option>
             <option value="running">Running</option>
             <option value="done">Done</option>
-            <option value="failed">Failed</option>
             <option value="dead">Dead</option>
           </select>
         </div>