workerstack 0.1.0-beta.0

package/dist/jobs/worker.js

@@ -0,0 +1,605 @@
+// src/jobs/worker.ts — the queue worker. One `tick()` is a full cycle:
+// recover expired leases → reap old succeeded rows → claim and run queue jobs.
+import { and, desc, eq, inArray, isNotNull, lt, lte, sql } from 'drizzle-orm';
+import { workerstackJobs } from '../internal-tables.js';
+import { consoleLogger } from '../logging.js';
+import { validateStandardSchema } from '../standard-schema.js';
+import { parseCron } from './cron.js';
+import { backoffMs, DEFAULT_RETRIES, leaseDurationFor } from './define.js';
+import { enqueueJob } from './queue.js';
+import { CRON_PREFIX, floorSlot, nextCronSlot, slotsDue, SLOT_MS, } from './slots.js';
+const CLAIM_BATCH = 10;
+const SUCCEEDED_RETENTION_MS = 24 * 60 * 60 * 1000;
+const REAP_INTERVAL_MS = 60 * 60_000;
+class JobExecutionTimeoutError extends Error {
+    constructor(maxRuntime) {
+        super(`execution timed out after ${maxRuntime}ms`);
+        this.name = 'JobExecutionTimeoutError';
+    }
+}
+class JobLeaseLostError extends Error {
+    constructor() {
+        super('job lease ownership was lost');
+        this.name = 'JobLeaseLostError';
+    }
+}
+function jobEvent(event, row, fields = {}) {
+    return {
+        source: 'workerstack.jobs',
+        event,
+        jobId: row.id,
+        jobType: row.type,
+        attempt: Number(row.attempts),
+        ...fields,
+    };
+}
+function capacityFor(def) {
+    return def.kind === 'job' && def.concurrency !== undefined
+        ? def.concurrency
+        : CLAIM_BATCH;
+}
+function toError(err) {
+    return err instanceof Error ? err : new Error(String(err));
+}
+function maxAttempts(def) {
+    return 1 + (def.retries ?? DEFAULT_RETRIES);
+}
+/** Resolve a stored row type back to its definition. Cron rows carry the
+ *  reserved prefix; queue rows use the definition key verbatim. */
+function definitionFor(defs, type) {
+    if (type.startsWith(CRON_PREFIX)) {
+        const def = defs[type.slice(CRON_PREFIX.length)];
+        return def?.kind === 'cron' ? def : undefined;
+    }
+    const def = defs[type];
+    return def?.kind === 'job' ? def : undefined;
+}
+/** Queue jobs release terminal dedupe keys; cron slots retain durable ownership. */
+function terminalPatch(def) {
+    return def.kind === 'cron' ? {} : { dedupeKey: null };
+}
+/** `dedupeUntil: 'start'` releases the key at claim; everything else keeps it
+ *  until a terminal state, so the claim patch stays unchanged for them. */
+function claimPatch(def) {
+    return def.kind === 'job' && def.dedupeUntil === 'start'
+        ? { dedupeKey: null }
+        : {};
+}
+export function createJobRunner(deps) {
+    const { db, defs } = deps;
+    const logger = deps.logger ?? consoleLogger;
+    const t = workerstackJobs;
+    const ctx = { ...deps.ctx };
+    let lastReapAt = 0;
+    const active = new Map();
+    const cronCursors = new Map();
+    function ownershipPredicate(id, owner) {
+        return and(eq(t.id, id), eq(t.status, 'running'), eq(t.attempts, owner.attempt), eq(t.lockedUntil, owner.lockedUntil));
+    }
+    function startLeaseHeartbeat(args) {
+        const owner = args.owner;
+        const intervalMs = Math.max(10, Math.floor(leaseDurationFor(args.def) / 3));
+        let stopped = false;
+        let resolveLost;
+        const lost = new Promise((resolve) => {
+            resolveLost = resolve;
+        });
+        let chain = Promise.resolve();
+        const lose = () => {
+            if (stopped)
+                return;
+            stopped = true;
+            clearInterval(timer);
+            resolveLost();
+            args.onLost();
+        };
+        const renew = () => {
+            chain = chain
+                .then(async () => {
+                if (stopped)
+                    return;
+                const nextLockedUntil = Date.now() + leaseDurationFor(args.def);
+                const updated = await db
+                    .update(t)
+                    .set({ lockedUntil: nextLockedUntil })
+                    .where(ownershipPredicate(args.row.id, owner))
+                    .returning({ id: t.id });
+                if (!updated[0])
+                    return lose();
+                owner.lockedUntil = nextLockedUntil;
+            })
+                .catch(() => lose());
+        };
+        const timer = setInterval(renew, intervalMs);
+        return {
+            owner,
+            lost,
+            async stop() {
+                stopped = true;
+                clearInterval(timer);
+                await chain;
+            },
+        };
+    }
+    /** Cron rows carry no payload — their handler input is the slot itself. */
+    function resolveInput(def, row) {
+        if (def.kind === 'cron') {
+            return { scheduledFor: new Date(Number(row.runAt)) };
+        }
+        const raw = JSON.parse(row.payloadJson);
+        return def.input
+            ? validateStandardSchema(def.input, raw, 'job payload')
+            : undefined;
+    }
+    /** Hydrate each cron cursor once, then advance it in this worker's memory. */
+    async function cronCursor(type, now) {
+        const cached = cronCursors.get(type);
+        if (cached)
+            return cached;
+        const rows = await db
+            .select({ runAt: t.runAt })
+            .from(t)
+            .where(eq(t.type, type))
+            .orderBy(desc(t.runAt))
+            .limit(1);
+        const cursor = {
+            checkedThrough: rows[0]?.runAt == null
+                ? floorSlot(now) - SLOT_MS
+                : Number(rows[0].runAt),
+        };
+        cronCursors.set(type, cursor);
+        return cursor;
+    }
+    /** Enqueue a row per due slot. The unique(type, dedupeKey) constraint makes
+     *  this safe to run concurrently in any number of processes. */
+    async function materializeCronSlots(now) {
+        const through = floorSlot(now);
+        for (const [name, def] of Object.entries(defs)) {
+            if (def.kind !== 'cron')
+                continue;
+            const type = `${CRON_PREFIX}${name}`;
+            const cursor = await cronCursor(type, now);
+            const slots = slotsDue({
+                cron: parseCron(def.schedule),
+                from: cursor.checkedThrough,
+                to: through,
+                catchUp: def.catchUp,
+                catchUpWindowMs: def.catchUpWindow,
+            });
+            for (const slot of slots) {
+                await enqueueJob(db, defs, name, null, {
+                    runAt: slot,
+                    dedupeKey: String(slot),
+                });
+                cursor.checkedThrough = slot;
+            }
+            cursor.checkedThrough = Math.max(cursor.checkedThrough, through);
+        }
+    }
+    async function fireOnFailed(def, input, error) {
+        if (!def.onFailed)
+            return;
+        try {
+            await def.onFailed(input, error, ctx);
+        }
+        catch (hookErr) {
+            logger.error('[workerstack] onFailed hook threw:', hookErr);
+        }
+    }
+    /** running rows whose lease expired → pending (or failed when exhausted). */
+    async function recoverExpiredLeases(now) {
+        const expired = await db
+            .select({
+            id: t.id,
+            type: t.type,
+            payloadJson: t.payloadJson,
+            attempts: t.attempts,
+            lastError: t.lastError,
+            runAt: t.runAt,
+            lockedUntil: t.lockedUntil,
+        })
+            .from(t)
+            .where(and(eq(t.status, 'running'), isNotNull(t.lockedUntil), lt(t.lockedUntil, now)));
+        for (const row of expired) {
+            const owner = {
+                attempt: Number(row.attempts),
+                lockedUntil: Number(row.lockedUntil),
+            };
+            const def = definitionFor(defs, row.type);
+            const error = new Error('lease expired (worker crashed or timed out)');
+            if (!def) {
+                await db
+                    .update(t)
+                    .set({
+                    status: 'failed',
+                    finishedAt: now,
+                    lockedUntil: null,
+                    lastError: `unknown job type "${row.type}"`,
+                    dedupeKey: null,
+                })
+                    .where(ownershipPredicate(row.id, owner));
+                continue;
+            }
+            if (Number(row.attempts) >= maxAttempts(def)) {
+                const updated = await db
+                    .update(t)
+                    .set({
+                    status: 'failed',
+                    finishedAt: now,
+                    lockedUntil: null,
+                    lastError: error.message,
+                    ...terminalPatch(def),
+                })
+                    .where(ownershipPredicate(row.id, owner))
+                    .returning({ id: t.id });
+                if (updated[0])
+                    await fireOnFailed(def, resolveInput(def, row), error);
+            }
+            else {
+                await db
+                    .update(t)
+                    .set({
+                    status: 'pending',
+                    lockedUntil: null,
+                    runAt: now + backoffMs(def, Number(row.attempts)),
+                    lastError: error.message,
+                })
+                    .where(ownershipPredicate(row.id, owner));
+            }
+        }
+    }
+    async function reapSucceeded(now) {
+        await db
+            .delete(t)
+            .where(and(eq(t.status, 'succeeded'), lt(t.finishedAt, now - SUCCEEDED_RETENTION_MS)));
+    }
+    /** Atomically claim up to `limit` runnable jobs of one type. */
+    async function claim(type, def, limit, now, leaseUntil) {
+        const pendingIds = db
+            .select({ id: t.id })
+            .from(t)
+            .where(and(eq(t.type, type), eq(t.status, 'pending'), lte(t.runAt, now)))
+            .orderBy(t.runAt)
+            .limit(limit);
+        // SQLite's single-writer model makes the one-statement UPDATE atomic.
+        const sub = pendingIds;
+        const rows = await db
+            .update(t)
+            .set({
+            status: 'running',
+            lockedUntil: leaseUntil,
+            attempts: sql `${t.attempts} + 1`,
+            ...claimPatch(def),
+        })
+            .where(and(inArray(t.id, sub), eq(t.status, 'pending')))
+            .returning({
+            id: t.id,
+            type: t.type,
+            payloadJson: t.payloadJson,
+            attempts: t.attempts,
+            runAt: t.runAt,
+        });
+        return rows;
+    }
+    async function claimAvailable(type, def, now) {
+        const runningRows = await db
+            .select({ id: t.id })
+            .from(t)
+            .where(and(eq(t.type, type), eq(t.status, 'running')));
+        let available = capacityFor(def) - runningRows.length;
+        if (available <= 0)
+            return [];
+        const leaseUntil = now + leaseDurationFor(def);
+        const work = [];
+        while (available > 0) {
+            const limit = Math.min(CLAIM_BATCH, available);
+            const rows = await claim(type, def, limit, now, leaseUntil);
+            for (const row of rows) {
+                work.push({
+                    row,
+                    def,
+                    owner: { attempt: Number(row.attempts), lockedUntil: leaseUntil },
+                });
+            }
+            available -= rows.length;
+            if (rows.length < limit)
+                break;
+        }
+        return work;
+    }
+    // `now` is the tick's injected clock: retry runAt math uses it so tests can
+    // drive backoff deterministically. finishedAt uses the real clock (a handler
+    // may run long past the tick's start).
+    async function runJob(row, def, now, owner) {
+        const startedAt = Date.now();
+        let input;
+        try {
+            input = resolveInput(def, row);
+        }
+        catch (err) {
+            // Stored payload no longer parses (schema drift): retrying can't help.
+            const e = toError(err);
+            const updated = await db
+                .update(t)
+                .set({
+                status: 'failed',
+                finishedAt: Date.now(),
+                lockedUntil: null,
+                lastError: e.message,
+                ...terminalPatch(def),
+            })
+                .where(ownershipPredicate(row.id, owner))
+                .returning({ id: t.id });
+            if (!updated[0])
+                return 'lost';
+            await fireOnFailed(def, undefined, e);
+            return 'failed';
+        }
+        const controller = new AbortController();
+        const handlerCtx = { ...ctx, signal: controller.signal };
+        logger.info(JSON.stringify(jobEvent('job.claimed', row)));
+        let lost = false;
+        const heartbeat = startLeaseHeartbeat({
+            row,
+            def,
+            owner,
+            onLost: () => {
+                lost = true;
+                controller.abort(new JobLeaseLostError());
+                logger.warn(JSON.stringify(jobEvent('job.lease_lost', row, {
+                    durationMs: Date.now() - startedAt,
+                })));
+            },
+        });
+        const deadline = def.maxRuntime === undefined
+            ? undefined
+            : setTimeout(() => {
+                controller.abort(new JobExecutionTimeoutError(def.maxRuntime));
+                logger.warn(JSON.stringify(jobEvent('job.execution_timed_out', row, {
+                    durationMs: Date.now() - startedAt,
+                })));
+            }, def.maxRuntime);
+        try {
+            await def.handler(input, handlerCtx);
+            if (controller.signal.reason instanceof JobExecutionTimeoutError) {
+                throw controller.signal.reason;
+            }
+            if (deadline)
+                clearTimeout(deadline);
+            await heartbeat.stop();
+            if (lost)
+                return 'lost';
+            const updated = await db
+                .update(t)
+                .set({
+                status: 'succeeded',
+                finishedAt: Date.now(),
+                lockedUntil: null,
+                ...terminalPatch(def),
+            })
+                .where(ownershipPredicate(row.id, owner))
+                .returning({ id: t.id });
+            if (!updated[0])
+                return 'lost';
+            logger.info(JSON.stringify(jobEvent('job.completed', row, {
+                durationMs: Date.now() - startedAt,
+            })));
+            return 'ran';
+        }
+        catch (err) {
+            if (deadline)
+                clearTimeout(deadline);
+            const e = controller.signal.reason instanceof JobExecutionTimeoutError
+                ? controller.signal.reason
+                : toError(err);
+            await heartbeat.stop();
+            if (lost)
+                return 'lost';
+            if (Number(row.attempts) < maxAttempts(def)) {
+                const updated = await db
+                    .update(t)
+                    .set({
+                    status: 'pending',
+                    lockedUntil: null,
+                    runAt: now + backoffMs(def, Number(row.attempts)),
+                    lastError: e.message,
+                })
+                    .where(ownershipPredicate(row.id, owner))
+                    .returning({ id: t.id });
+                if (!updated[0])
+                    return 'lost';
+                logger.warn(JSON.stringify(jobEvent('job.retrying', row, {
+                    durationMs: Date.now() - startedAt,
+                    error: e.message,
+                })));
+            }
+            else {
+                const updated = await db
+                    .update(t)
+                    .set({
+                    status: 'failed',
+                    finishedAt: Date.now(),
+                    lockedUntil: null,
+                    lastError: e.message,
+                    ...terminalPatch(def),
+                })
+                    .where(ownershipPredicate(row.id, owner))
+                    .returning({ id: t.id });
+                if (!updated[0])
+                    return 'lost';
+                logger.error(JSON.stringify(jobEvent('job.failed', row, {
+                    durationMs: Date.now() - startedAt,
+                    error: e.message,
+                })));
+                await fireOnFailed(def, input, e);
+            }
+            return 'failed';
+        }
+    }
+    async function runClaimable(now) {
+        const claimedWork = [];
+        let totalClaimed = 0;
+        for (const [name, def] of Object.entries(defs)) {
+            const type = def.kind === 'cron' ? `${CRON_PREFIX}${name}` : name;
+            const claimed = await claimAvailable(type, def, now);
+            totalClaimed += claimed.length;
+            claimedWork.push(...claimed);
+        }
+        // Claim the whole tick snapshot before starting any handler. Work enqueued
+        // by a handler therefore belongs to the next tick, regardless of the
+        // declaration order of its target job type.
+        const outcomes = await Promise.all(claimedWork.map(({ row, def, owner }) => runJob(row, def, now, owner)));
+        let ran = 0;
+        let failed = 0;
+        for (const outcome of outcomes) {
+            if (outcome === 'ran')
+                ran++;
+            else if (outcome === 'failed')
+                failed++;
+        }
+        return { claimed: totalClaimed, ran, failed };
+    }
+    async function maintain(now) {
+        await materializeCronSlots(now);
+        await recoverExpiredLeases(now);
+        if (now - lastReapAt >= REAP_INTERVAL_MS) {
+            lastReapAt = now;
+            await reapSucceeded(now);
+        }
+    }
+    function startWork(type, work, claimedAt) {
+        let tasks = active.get(type);
+        if (!tasks) {
+            tasks = new Set();
+            active.set(type, tasks);
+        }
+        let task;
+        task = runJob(work.row, work.def, claimedAt, work.owner)
+            .then(() => undefined)
+            .catch((error) => {
+            logger.error('[workerstack] background job execution failed:', error);
+        })
+            .finally(() => {
+            tasks.delete(task);
+            if (tasks.size === 0)
+                active.delete(type);
+        });
+        tasks.add(task);
+        return task;
+    }
+    async function pump(now = Date.now()) {
+        await maintain(now);
+        for (const [name, def] of Object.entries(defs)) {
+            const type = def.kind === 'cron' ? `${CRON_PREFIX}${name}` : name;
+            const work = await claimAvailable(type, def, now);
+            for (const item of work)
+                startWork(type, item, now);
+        }
+        const snapshot = [...active.values()].flatMap((tasks) => [...tasks]);
+        return snapshot.length === 0
+            ? {}
+            : { wake: Promise.race(snapshot).then(() => undefined) };
+    }
+    async function drain() {
+        const snapshot = [...active.values()].flatMap((tasks) => [...tasks]);
+        await Promise.allSettled(snapshot);
+    }
+    return {
+        async tick(now = Date.now()) {
+            await maintain(now);
+            return runClaimable(now);
+        },
+        pump,
+        drain,
+        /**
+         * The earliest time that has work: a pending run_at, a running lease end,
+         * or a cron slot not yet materialized. Never earlier than `now`; null when
+         * nothing is due before `until`.
+         */
+        async nextDueAt(now, until) {
+            const candidates = [];
+            const [pending] = await db
+                .select({ at: sql `min(${t.runAt})` })
+                .from(t)
+                .where(eq(t.status, 'pending'));
+            if (pending?.at != null)
+                candidates.push(Number(pending.at));
+            const [running] = await db
+                .select({ at: sql `min(${t.lockedUntil})` })
+                .from(t)
+                .where(eq(t.status, 'running'));
+            if (running?.at != null)
+                candidates.push(Number(running.at));
+            for (const [name, def] of Object.entries(defs)) {
+                if (def.kind !== 'cron')
+                    continue;
+                const cursor = await cronCursor(`${CRON_PREFIX}${name}`, now);
+                const slot = nextCronSlot(parseCron(def.schedule), cursor.checkedThrough, until);
+                if (slot !== null)
+                    candidates.push(slot);
+            }
+            if (candidates.length === 0)
+                return null;
+            return Math.max(now, Math.min(...candidates));
+        },
+        async inspect(now) {
+            const runnableRows = await db
+                .select({ id: t.id })
+                .from(t)
+                .where(and(eq(t.status, 'pending'), lte(t.runAt, now)));
+            const failed = await db
+                .select({
+                id: t.id,
+                type: t.type,
+                attempts: t.attempts,
+                lastError: t.lastError,
+            })
+                .from(t)
+                .where(eq(t.status, 'failed'));
+            const rows = await db
+                .select({
+                id: t.id,
+                type: t.type,
+                status: t.status,
+                attempts: t.attempts,
+                runAt: t.runAt,
+                dedupeKey: t.dedupeKey,
+                lastError: t.lastError,
+                createdAt: t.createdAt,
+            })
+                .from(t);
+            return {
+                runnable: runnableRows.length,
+                failed: failed.map((row) => ({
+                    id: String(row.id),
+                    name: row.type.startsWith(CRON_PREFIX)
+                        ? row.type.slice(CRON_PREFIX.length)
+                        : row.type,
+                    attempts: Number(row.attempts),
+                    lastError: row.lastError,
+                })),
+                jobs: rows
+                    .sort((left, right) => Number(left.createdAt) - Number(right.createdAt) ||
+                    String(left.id).localeCompare(String(right.id)))
+                    .map((row) => ({
+                    id: String(row.id),
+                    name: row.type.startsWith(CRON_PREFIX)
+                        ? row.type.slice(CRON_PREFIX.length)
+                        : row.type,
+                    kind: row.type.startsWith(CRON_PREFIX)
+                        ? 'cron'
+                        : 'job',
+                    status: row.status,
+                    attempts: Number(row.attempts),
+                    runAt: Number(row.runAt),
+                    dedupeKey: row.dedupeKey,
+                    lastError: row.lastError,
+                })),
+            };
+        },
+        setJobsFacade(f) {
+            ctx.jobs = f;
+        },
+    };
+}
+//# sourceMappingURL=worker.js.map