pixivflow 2.18.0 → 2.19.0

package/dist/commands/SchedulerCommand.js

@@ -10,7 +10,9 @@ const config_1 = require("../config");
 const MultiScheduleManager_1 = require("../scheduler/MultiScheduleManager");
 const ScheduleTriggerServer_1 = require("../scheduler/ScheduleTriggerServer");
 const SlotCoordinator_1 = require("../scheduler/SlotCoordinator");
+const schedules_1 = require("../scheduler/schedules");
 const scheduler_runtime_1 = require("./scheduler-runtime");
+const SchedulerIdleLifecycle_1 = require("./SchedulerIdleLifecycle");
 /**
  * Decide whether the authenticated HTTP trigger server should be mounted. It is
  * always up in `external` mode (the external clock needs it); in `internal`
@@ -51,6 +53,7 @@ class SchedulerCommand extends Command_1.BaseCommand {
                 execute: runtime.runJob,
                 database: runtime.database,
                 onFailure: runtime.notifyScheduleFailure,
+                onAbandoned: (_config, _schedule, abandoned) => runtime.abandonActiveRun(abandoned.errorMessage ?? 'abandoned after timeout'),
                 telemetry: {
                     beginRun: () => runtime.database.getOverviewStats().totalDownloads,
                     endRun: () => runtime.database.getOverviewStats().totalDownloads,
@@ -65,19 +68,24 @@ class SchedulerCommand extends Command_1.BaseCommand {
             // Authenticated HTTP trigger. Mounted in external mode (the external clock
             // wakes the machine here) and, opt-in, in internal mode for manual/ops
             // triggers. Business logic lives in runJob/SlotCoordinator; this handler
-            // only authenticates, resolves the occurrence and delegates synchronously
-            // so the open request is the activity lease for the whole run.
+            // only authenticates, records the occurrence durably and hands it to the
+            // shared scheduler, then answers. It deliberately does not await the run:
+            // the open request is NOT an activity lease, and a 10-40 minute download
+            // would outlive the clock/proxy/client timeout carrying it.
             let triggerServer = null;
             if (triggerEnabled(runtime.config)) {
                 const rt = runtime.config.schedulerRuntime;
                 const coordinator = new SlotCoordinator_1.SlotCoordinator(runtime.database);
                 const resolveConfig = () => manager['activeConfig'] ?? runtime.config;
                 const findPlan = (cfg, scheduleId) => cfg.schedules?.find((s) => s.id === scheduleId && s.enabled !== false);
-                // In-process singleflight: coalesce concurrent triggers of the same
-                // occurrence on THIS process. Optimization only — the DB unique
-                // constraints + ledger are the real correctness guarantee across
-                // restarts/processes.
-                const inFlight = new Map();
+                // Durable dispatch. The trigger endpoint records the occurrence FIRST,
+                // then hands it to the shared scheduler and answers immediately. It must
+                // never await the download itself: a 10-40 minute run outlives the
+                // router/proxy/clock timeouts, and a request that dies mid-run used to
+                // take the occurrence with it. Idempotency comes from the slot ledger and
+                // the cross-process lease, so concurrent clocks (Cloudflare + watchdog +
+                // manual) converge on one worker instead of needing an in-process
+                // singleflight map.
                 triggerServer = new ScheduleTriggerServer_1.ScheduleTriggerServer(ScheduleTriggerServer_1.ScheduleTriggerServer.resolveToken(rt.trigger?.token), {
                     listSchedules: () => manager.scheduleIds(),
                     resolve: (scheduleId, source, at, label) => {
@@ -90,41 +98,61 @@ class SchedulerCommand extends Command_1.BaseCommand {
                             return { error: resolved.error ?? 'could not resolve occurrence', status: resolved.status ?? 400 };
                         return { context: resolved.context };
                     },
-                    run: (scheduleId, context) => {
-                        const key = context.slotId;
-                        const existing = inFlight.get(key);
-                        if (existing)
-                            return existing;
+                    run: async (scheduleId, context) => {
                         const cfg = resolveConfig();
                         const plan = findPlan(cfg, scheduleId);
-                        const promise = (async () => {
-                            try {
-                                if (!plan)
-                                    return { scheduleId, slotId: context.slotId, status: 'failed', alreadyCompleted: false };
-                                // Synchronous: awaited so the HTTP response does not return
-                                // (and release the activity lease) until the run finishes.
-                                await runtime.runJob(cfg, plan, { triggerSource: 'http', slot: context });
-                                const rec = runtime.database.slots.getSlot(context.slotId);
-                                const cells = runtime.database.slots.getCells(context.slotId).map((c) => ({
-                                    targetId: c.targetId,
-                                    status: c.status,
-                                    workId: c.workId,
-                                    error: c.lastError,
-                                }));
-                                return {
-                                    scheduleId,
-                                    slotId: context.slotId,
-                                    status: rec?.status ?? 'failed',
-                                    alreadyCompleted: rec?.status === 'success' || rec?.status === 'partial',
-                                    cells,
-                                };
-                            }
-                            finally {
-                                inFlight.delete(key);
-                            }
-                        })();
-                        inFlight.set(key, promise);
-                        return promise;
+                        if (!plan) {
+                            return { scheduleId, slotId: context.slotId, disposition: 'rejected', status: 'failed' };
+                        }
+                        const targets = (0, schedules_1.selectScheduleTargets)(cfg.targets, plan);
+                        if (targets.length === 0) {
+                            return { scheduleId, slotId: context.slotId, disposition: 'rejected', status: 'pending' };
+                        }
+                        // Step 1: make the occurrence durable BEFORE answering. A crash
+                        // between here and the claim leaves a `pending`, lease-less row that
+                        // the recovery loop re-dispatches, so an accepted trigger can never
+                        // be silently dropped.
+                        const prepared = coordinator.prepare(context, plan, targets);
+                        if (prepared.alreadyCompleted) {
+                            return {
+                                ...coordinator.completedSummary(context.slotId, plan),
+                                disposition: 'already_completed',
+                            };
+                        }
+                        // Step 2: admit to the scheduler. Fire-and-forget by contract — the
+                        // boolean only reports whether this process took the work.
+                        const admitted = manager.triggerSchedule(scheduleId, {
+                            triggerSource: 'http',
+                            slot: context,
+                        });
+                        const rec = runtime.database.slots.getSlot(context.slotId);
+                        if (admitted) {
+                            return {
+                                scheduleId,
+                                slotId: context.slotId,
+                                disposition: 'accepted',
+                                status: rec?.status ?? 'pending',
+                            };
+                        }
+                        // Not admitted locally. A live lease means another worker already
+                        // owns the occurrence; anything else is a refusal this process could
+                        // not take (busy/stopped/limit), which the clock may retry and the
+                        // recovery loop will also pick up.
+                        const lease = runtime.database.slots.getSlotLease(context.slotId);
+                        if (lease.owner && lease.until && lease.until > Date.now()) {
+                            return {
+                                scheduleId,
+                                slotId: context.slotId,
+                                disposition: 'already_running',
+                                status: 'running',
+                            };
+                        }
+                        return {
+                            scheduleId,
+                            slotId: context.slotId,
+                            disposition: 'rejected',
+                            status: rec?.status ?? 'pending',
+                        };
                     },
                     drainOutbox: () => runtime.drainOutbox(),
                     status: (scheduleId) => {
@@ -145,15 +173,47 @@ class SchedulerCommand extends Command_1.BaseCommand {
                     ? 'External scheduler mode: internal cron disabled, awaiting authenticated schedule triggers'
                     : 'Internal scheduler mode with authenticated HTTP trigger enabled', { port });
             }
-            const cleanup = () => {
-                context.logger.info('Shutting down PixivFlow');
+            // Run-to-completion lifecycle for the external clock. The machine was woken
+            // by a trigger and has to return to `stopped` by itself. Platform-side
+            // auto-stop cannot decide that: the trigger answers long before the run
+            // finishes, so the proxy sees an idle socket while downloads are still in
+            // flight. The worker therefore reads its OWN durable ledger (see
+            // SchedulerIdleLifecycle). Wired after the trigger server so the port is
+            // already being served when the first idle probe runs.
+            let idleLifecycle = null;
+            const shutdown = (reason, code = 0) => {
+                context.logger.info('Shutting down PixivFlow', { reason });
+                idleLifecycle?.stop();
                 triggerServer?.stop();
                 manager.stop();
                 runtime.close();
-                process.exit(0);
+                process.exit(code);
             };
-            process.on('SIGINT', cleanup);
-            process.on('SIGTERM', cleanup);
+            const scheduling = runtime.config.schedulerRuntime;
+            if (scheduling?.mode === 'external' && scheduling.exitWhenIdle) {
+                idleLifecycle = new SchedulerIdleLifecycle_1.SchedulerIdleLifecycle({
+                    snapshot: () => {
+                        const outbox = runtime.database.outbox.counts();
+                        return {
+                            activeSlots: runtime.database.slots.countActiveSlots(),
+                            processingOutbox: outbox.processing,
+                            // `pending` covers pending + retry_wait: a delivery still backing
+                            // off is active work. Backoff is bounded (rows flip to `dead` once
+                            // attempts are exhausted, which does not block exit), so this
+                            // cannot hold the machine open indefinitely.
+                            pendingOutbox: outbox.pending,
+                        };
+                    },
+                    idleGraceMs: scheduling.idleGraceMs,
+                    maxLifetimeMs: scheduling.maxLifetimeMs,
+                    onExit: (reason) => shutdown(reason === 'idle'
+                        ? 'external worker idle: slot ledger and outbox are both empty'
+                        : 'external worker exceeded the maxLifetimeMs backstop'),
+                });
+                idleLifecycle.start();
+            }
+            process.on('SIGINT', () => shutdown('SIGINT'));
+            process.on('SIGTERM', () => shutdown('SIGTERM'));
             process.on('SIGHUP', () => {
                 context.logger.info('Received SIGHUP; reloading scheduler configuration');
                 manager.reload();

package/dist/commands/SchedulerIdleLifecycle.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Run-to-completion lifecycle for the external-clock (autosleep) deployment.
+ *
+ * A PixivFlow machine can be woken by an authenticated HTTP trigger, execute a
+ * schedule, deliver through the durable outbox, and must then return to
+ * `stopped` on its own. Two things make that non-trivial:
+ *
+ *  - Platform-side auto-stop cannot be used. The trigger endpoint answers as
+ *    soon as the occurrence is durable (deliberately — a 10-40 minute run cannot
+ *    survive a router/proxy/client timeout), so from the proxy's point of view
+ *    the machine goes idle while the run is still in progress. Auto-stop would
+ *    kill a download mid-flight.
+ *  - "The request returned" and "nobody connected recently" are both wrong
+ *    signals. HTTP activity says nothing about whether a download is still
+ *    running, a Slot is still open, or an outbox row is still retrying.
+ *
+ * So this reads the worker's OWN durable ledger instead, and exits only when
+ * that ledger says nothing is left: no non-terminal Slot, no in-flight outbox
+ * row, no undelivered outbox row. It is deliberately the only thing it does —
+ * no second business state machine, no scheduler of its own, no queue.
+ *
+ * The idle grace is a merge window, not a timeout: two schedules ten minutes
+ * apart are normally served by a single wake-up, and a delivery retry that lands
+ * just after the run is drained without paying for a second cold start.
+ *
+ * `maxLifetimeMs` is a backstop, never the normal path. It exists so a worker
+ * that somehow never reaches idle (a wedged run, a Slot that recovery cannot
+ * re-dispatch) cannot bill indefinitely. Reaching it deletes nothing: the
+ * process closes its database cleanly and the next wake resumes from the same
+ * Slot/outbox rows.
+ */
+export type IdleExitReason = 'idle' | 'max-lifetime';
+/**
+ * A read-only view of the worker's own authoritative state. Every field must be
+ * derived from durable rows, never from in-memory scheduling or HTTP activity.
+ */
+export interface IdleSnapshot {
+    /** Non-terminal Slots (pending/running): work this ledger still owes. */
+    activeSlots: number;
+    /** Outbox rows currently claimed by this or another live worker. */
+    processingOutbox: number;
+    /** Undelivered outbox rows (pending, or waiting on a bounded retry backoff). */
+    pendingOutbox: number;
+}
+/** True only when the worker has no work of any kind left. */
+export declare function isIdle(snapshot: IdleSnapshot): boolean;
+/** Default idle grace: long enough to merge two schedules ~10 minutes apart. */
+export declare const DEFAULT_IDLE_GRACE_MS: number;
+/** Default backstop on awake time; the idle detector is the normal exit path. */
+export declare const DEFAULT_MAX_LIFETIME_MS: number;
+/** How often the durable ledger is re-read. */
+export declare const DEFAULT_IDLE_POLL_MS: number;
+export interface IdleLifecycleOptions {
+    /** Reads the worker's authoritative state. Must not mutate anything. */
+    snapshot(): IdleSnapshot;
+    /**
+     * Invoked at most once, either on real idleness or on the hard backstop. The
+     * caller owns the shutdown itself (stop serving, flush, exit).
+     */
+    onExit(reason: IdleExitReason): void;
+    idleGraceMs?: number;
+    maxLifetimeMs?: number;
+    pollMs?: number;
+}
+export declare class SchedulerIdleLifecycle {
+    private readonly options;
+    private readonly idleGraceMs;
+    private readonly maxLifetimeMs;
+    private readonly pollMs;
+    private pollTimer;
+    private lifetimeTimer;
+    private idleSince;
+    private exiting;
+    constructor(options: IdleLifecycleOptions);
+    /** Arm polling and the backstop. Idempotent. */
+    start(): void;
+    /** Disarm without exiting. Idempotent; used by the signal shutdown path. */
+    stop(): void;
+    private poll;
+    private exit;
+}
+//# sourceMappingURL=SchedulerIdleLifecycle.d.ts.map

package/dist/commands/SchedulerIdleLifecycle.js

@@ -0,0 +1,135 @@
+"use strict";
+/**
+ * Run-to-completion lifecycle for the external-clock (autosleep) deployment.
+ *
+ * A PixivFlow machine can be woken by an authenticated HTTP trigger, execute a
+ * schedule, deliver through the durable outbox, and must then return to
+ * `stopped` on its own. Two things make that non-trivial:
+ *
+ *  - Platform-side auto-stop cannot be used. The trigger endpoint answers as
+ *    soon as the occurrence is durable (deliberately — a 10-40 minute run cannot
+ *    survive a router/proxy/client timeout), so from the proxy's point of view
+ *    the machine goes idle while the run is still in progress. Auto-stop would
+ *    kill a download mid-flight.
+ *  - "The request returned" and "nobody connected recently" are both wrong
+ *    signals. HTTP activity says nothing about whether a download is still
+ *    running, a Slot is still open, or an outbox row is still retrying.
+ *
+ * So this reads the worker's OWN durable ledger instead, and exits only when
+ * that ledger says nothing is left: no non-terminal Slot, no in-flight outbox
+ * row, no undelivered outbox row. It is deliberately the only thing it does —
+ * no second business state machine, no scheduler of its own, no queue.
+ *
+ * The idle grace is a merge window, not a timeout: two schedules ten minutes
+ * apart are normally served by a single wake-up, and a delivery retry that lands
+ * just after the run is drained without paying for a second cold start.
+ *
+ * `maxLifetimeMs` is a backstop, never the normal path. It exists so a worker
+ * that somehow never reaches idle (a wedged run, a Slot that recovery cannot
+ * re-dispatch) cannot bill indefinitely. Reaching it deletes nothing: the
+ * process closes its database cleanly and the next wake resumes from the same
+ * Slot/outbox rows.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SchedulerIdleLifecycle = exports.DEFAULT_IDLE_POLL_MS = exports.DEFAULT_MAX_LIFETIME_MS = exports.DEFAULT_IDLE_GRACE_MS = void 0;
+exports.isIdle = isIdle;
+const logger_1 = require("../logger");
+/** True only when the worker has no work of any kind left. */
+function isIdle(snapshot) {
+    return (snapshot.activeSlots === 0 &&
+        snapshot.processingOutbox === 0 &&
+        snapshot.pendingOutbox === 0);
+}
+/** Default idle grace: long enough to merge two schedules ~10 minutes apart. */
+exports.DEFAULT_IDLE_GRACE_MS = 10 * 60 * 1000;
+/** Default backstop on awake time; the idle detector is the normal exit path. */
+exports.DEFAULT_MAX_LIFETIME_MS = 3 * 60 * 60 * 1000;
+/** How often the durable ledger is re-read. */
+exports.DEFAULT_IDLE_POLL_MS = 15 * 1000;
+class SchedulerIdleLifecycle {
+    options;
+    idleGraceMs;
+    maxLifetimeMs;
+    pollMs;
+    pollTimer = null;
+    lifetimeTimer = null;
+    idleSince = null;
+    exiting = false;
+    constructor(options) {
+        this.options = options;
+        this.idleGraceMs = options.idleGraceMs ?? exports.DEFAULT_IDLE_GRACE_MS;
+        this.maxLifetimeMs = options.maxLifetimeMs ?? exports.DEFAULT_MAX_LIFETIME_MS;
+        this.pollMs = options.pollMs ?? exports.DEFAULT_IDLE_POLL_MS;
+    }
+    /** Arm polling and the backstop. Idempotent. */
+    start() {
+        if (this.pollTimer || this.lifetimeTimer)
+            return;
+        logger_1.logger.info('External worker lifecycle armed; will exit once real idleness is confirmed', {
+            idleGraceMs: this.idleGraceMs,
+            maxLifetimeMs: this.maxLifetimeMs,
+            pollMs: this.pollMs,
+        });
+        this.lifetimeTimer = setTimeout(() => this.exit('max-lifetime'), this.maxLifetimeMs);
+        this.pollTimer = setInterval(() => this.poll(), this.pollMs);
+    }
+    /** Disarm without exiting. Idempotent; used by the signal shutdown path. */
+    stop() {
+        if (this.pollTimer) {
+            clearInterval(this.pollTimer);
+            this.pollTimer = null;
+        }
+        if (this.lifetimeTimer) {
+            clearTimeout(this.lifetimeTimer);
+            this.lifetimeTimer = null;
+        }
+    }
+    poll() {
+        if (this.exiting)
+            return;
+        let snapshot;
+        try {
+            snapshot = this.options.snapshot();
+        }
+        catch (error) {
+            // A failed read is NOT evidence of idleness. Stay awake and retry; the
+            // backstop still bounds how long a broken probe can hold the machine.
+            logger_1.logger.warn('Idle probe failed; staying awake and will retry', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return;
+        }
+        if (!isIdle(snapshot)) {
+            if (this.idleSince !== null) {
+                logger_1.logger.info('External worker picked up new work; idle grace window cancelled', { ...snapshot });
+            }
+            this.idleSince = null;
+            return;
+        }
+        if (this.idleGraceMs === 0) {
+            this.exit('idle');
+            return;
+        }
+        const now = Date.now();
+        if (this.idleSince === null) {
+            this.idleSince = now;
+            logger_1.logger.info('External worker idle; starting exit grace window', {
+                idleGraceMs: this.idleGraceMs,
+                ...snapshot,
+            });
+            return;
+        }
+        if (now - this.idleSince >= this.idleGraceMs) {
+            this.exit('idle');
+        }
+    }
+    exit(reason) {
+        if (this.exiting)
+            return;
+        this.exiting = true;
+        this.stop();
+        this.options.onExit(reason);
+    }
+}
+exports.SchedulerIdleLifecycle = SchedulerIdleLifecycle;
+//# sourceMappingURL=SchedulerIdleLifecycle.js.map

package/dist/commands/scheduler-runtime.d.ts

@@ -61,6 +61,12 @@ export interface SchedulerRuntime {
     runJob(snapshot: StandaloneConfig, schedule: ScheduleConfig, options?: RunJobOptions): Promise<void>;
     /** Cancel the in-flight download plan, if any. */
     cancelActive(reason: string): void;
+    /**
+     * The active run never settled after its timeout and drain window. Stop
+     * renewing its lease and take its Slot terminal so recovery cannot re-dispatch
+     * the same occurrence alongside the still-running job.
+     */
+    abandonActiveRun(reason: string): void;
     /** Notify the affected review group after a failed scheduled run. */
     notifyScheduleFailure(snapshot: StandaloneConfig, schedule: ScheduleConfig, failure: JobFailure): Promise<void>;
     /** Start the independent outbox pump (long-running daemon). */

package/dist/commands/scheduler-runtime.js

@@ -203,6 +203,12 @@ async function createSchedulerRuntime(configPathArg) {
         tokenMaintenance.start();
     }
     let activeDownloadManager = null;
+    /**
+     * Lease bookkeeping of the run that currently owns a Slot, exposed to
+     * cancelActive()/abandonActiveRun() (defined below, outside runJob's closure).
+     * Set while a lease is held and cleared as soon as it is released.
+     */
+    let activeLeaseHooks = null;
     // Independently-pumped durable outbox (content + notifications). Started in
     // the long-running scheduler daemon; run-once drains explicitly before exit.
     const deliveryDispatcher = new DeliveryDispatcher_1.DeliveryDispatcher(config.delivery, buildProxyUrl(config.network));
@@ -252,6 +258,12 @@ async function createSchedulerRuntime(configPathArg) {
         // resolve a canonical occurrence and converge on one durable Slot.
         let slotCtx = null;
         let varReleaseLease = null;
+        /**
+         * Set when the run was abandoned past its drain window. A wedged job can
+         * still return much later; it must not roll the slot up again (which would
+         * overwrite the terminal `failed` record the abandon path wrote).
+         */
+        let slotAbandoned = false;
         if (!adhoc) {
             if (providedSlot) {
                 slotCtx = providedSlot;
@@ -271,19 +283,20 @@ async function createSchedulerRuntime(configPathArg) {
             }
             const activeSlot = slotCtx;
             const runOwner = `run-${process.pid}-${(0, node_crypto_1.randomUUID)().slice(0, 8)}`;
-            const leaseMs = Math.max(60_000, (schedule.timeout ?? 30 * 60_000) + 60_000);
-            const begin = coordinator.begin(slotCtx, schedule, targets);
-            if (begin.alreadyCompleted && !onlyTarget) {
+            const prepared = coordinator.prepare(slotCtx, schedule, targets);
+            if (prepared.alreadyCompleted && !onlyTarget) {
                 logger_1.logger.info('Slot already terminal; nothing to do', {
                     slot: activeSlot.slotId,
-                    status: begin.slotRec.status,
+                    status: prepared.slotRec.status,
                 });
                 return;
             }
             // Cross-process lease: a concurrent Cloudflare + watchdog + manual
             // trigger on a SECOND process sees an active lease and converges instead
-            // of running the same targets in parallel.
-            const claimed = coordinator.claimRunLease(activeSlot.slotId, runOwner, leaseMs);
+            // of running the same targets in parallel. The lease TTL bounds how long a
+            // DEAD worker blocks its slot; it is intentionally independent of
+            // schedule.timeout (which bounds how long a live run may take).
+            const claimed = coordinator.claimRunLease(activeSlot.slotId, runOwner, SlotCoordinator_1.SLOT_LEASE_TTL_MS);
             if (!claimed) {
                 const lease = database.slots.getSlotLease(activeSlot.slotId);
                 logger_1.logger.info('Slot run already leased by another worker; converging', {
@@ -295,12 +308,48 @@ async function createSchedulerRuntime(configPathArg) {
                 await outboxWorker.drainOnce(1);
                 return;
             }
-            const heartbeat = setInterval(() => coordinator.heartbeatLease(activeSlot.slotId, runOwner, leaseMs), Math.floor(leaseMs / 3));
+            // Only the process that actually owns the lease may report the slot as
+            // running; the accepting adapter records it as pending instead.
+            coordinator.markRunning(activeSlot.slotId);
+            let cancelled = false;
+            const heartbeat = setInterval(() => {
+                // A cancelled/timed-out run must stop renewing its lease. An infinitely
+                // renewed lease is precisely what made a wedged run unrecoverable: the
+                // heartbeat outlived the scheduler timeout, so no other worker could ever
+                // claim the slot and the occurrence stayed stuck in `running` forever.
+                if (cancelled) {
+                    clearInterval(heartbeat);
+                    return;
+                }
+                coordinator.heartbeatLease(activeSlot.slotId, runOwner, SlotCoordinator_1.SLOT_LEASE_TTL_MS);
+            }, SlotCoordinator_1.SLOT_HEARTBEAT_MS);
             heartbeat.unref?.();
             varReleaseLease = () => {
                 clearInterval(heartbeat);
+                activeLeaseHooks = null;
                 coordinator.releaseRunLease(activeSlot.slotId, runOwner);
             };
+            activeLeaseHooks = {
+                stopHeartbeat: () => {
+                    cancelled = true;
+                    clearInterval(heartbeat);
+                },
+                /**
+                 * The run will never settle: stop heart-beating AND leave the ledger
+                 * terminal, so the recovery sweep (which only acts on `pending`/`running`
+                 * slots with no live lease) cannot re-dispatch this occurrence next to a
+                 * still-running job. Merely dropping the lease would be wrong here —
+                 * unlike a crash, this worker is alive.
+                 */
+                abandon: (reason) => {
+                    cancelled = true;
+                    slotAbandoned = true;
+                    clearInterval(heartbeat);
+                    database.slots.markSlotStatus(activeSlot.slotId, 'failed', `abandoned after scheduler timeout; no delivery (${reason})`);
+                    coordinator.releaseRunLease(activeSlot.slotId, runOwner);
+                    activeLeaseHooks = null;
+                },
+            };
         }
         // For a scheduled run, skip cells already in a terminal state (resume never
         // re-runs a finished cell — that is what prevents a second post). Membership
@@ -310,6 +359,10 @@ async function createSchedulerRuntime(configPathArg) {
         if (slotCtx && runTargets.length === 0) {
             logger_1.logger.info('All slot cells already complete', { slot: slotCtx.slotId });
             coordinator.finish(slotCtx, schedule, targets);
+            // Release LAST: the slot must stay owned until its aggregate state has
+            // been rolled up, otherwise a concurrent trigger could claim and re-run it
+            // against a half-finished ledger.
+            varReleaseLease?.();
             return;
         }
         // Ad-hoc refetch with a filter that matched no target (or all filtered out):
@@ -398,6 +451,10 @@ async function createSchedulerRuntime(configPathArg) {
         }
         catch (error) {
             if (!slotCtx || !(error instanceof Error) || !/^All \d+ target\(s\) failed\./.test(error.message)) {
+                // Abnormal abort: no roll-up is possible, so hand the slot back now
+                // instead of holding it until the lease TTL expires. Recovery sees a
+                // non-terminal slot with no live lease and resumes the SAME occurrence.
+                releaseLease?.();
                 throw error;
             }
             // Scheduled Slots treat terminal target failures (failed/no_candidate) as
@@ -408,10 +465,9 @@ async function createSchedulerRuntime(configPathArg) {
         finally {
             if (activeDownloadManager === downloadManager)
                 activeDownloadManager = null;
-            releaseLease?.();
         }
         const duration = Math.round((Date.now() - startTime) / 1000);
-        if (slotCtx) {
+        if (slotCtx && !slotAbandoned) {
             const summary = coordinator.finish(slotCtx, schedule, targets);
             notificationPolicy.sendSlotSummary(slotCtx, schedule, summary.cells.map((c) => {
                 const t = targets.find((x) => x.id === c.targetId);
@@ -437,6 +493,17 @@ async function createSchedulerRuntime(configPathArg) {
     };
     const cancelActive = (reason) => {
         activeDownloadManager?.cancel(reason);
+        // Scheduler timeout / process shutdown also stop the lease heartbeat. The run
+        // is on its way out (or about to be killed with the process), so it must not
+        // keep the slot locked while it unwinds. A shutdown deliberately leaves the
+        // slot NON-terminal: recovery resumes the same occurrence after restart.
+        activeLeaseHooks?.stopHeartbeat();
+    };
+    const abandonActiveRun = (reason) => {
+        // Cancellation did not take effect inside the drain window — a request that
+        // ignores the abort. This run will never settle, so finish the lease
+        // bookkeeping it cannot do itself.
+        activeLeaseHooks?.abandon(reason);
     };
     const close = () => {
         outboxWorker.stop();
@@ -454,6 +521,7 @@ async function createSchedulerRuntime(configPathArg) {
         tokenMaintenance,
         runJob,
         cancelActive,
+        abandonActiveRun,
         startOutboxWorker: () => outboxWorker.start(),
         drainOutbox: () => outboxWorker.drainOnce(),
         notifyScheduleFailure: (snapshot, schedule, failure) => notifyScheduleFailure(snapshot, database, schedule, failure),

package/dist/config/types.d.ts

@@ -426,6 +426,39 @@ export interface SchedulerRuntimeConfig {
      * One pending run per plan is retained; extra ticks are skipped.
      */
     queueLimit?: number;
+    /**
+     * External-mode run-to-completion lifecycle. When true, the daemon exits the
+     * process once its OWN durable ledger says there is nothing left to do, so a
+     * machine that was woken by an external clock returns to `stopped` by itself
+     * instead of idling on a fixed clock until the platform stops it.
+     *
+     * The platform's proxy-side auto-stop cannot make this call. The HTTP trigger
+     * answers as soon as the occurrence is durable (deliberately — a 10-40 minute
+     * run cannot survive a router/proxy/client timeout), so the proxy sees an idle
+     * socket while the real run is still going and would stop the machine
+     * mid-download. Only this process knows when its work is finished.
+     *
+     * Ignored in `internal` mode, where cron owns the process lifetime.
+     * Default: false.
+     */
+    exitWhenIdle?: boolean;
+    /**
+     * How long to stay awake after the last observed work, before exiting in
+     * `exitWhenIdle` mode. This is a merge window rather than a timeout: two
+     * schedules ten minutes apart are normally served by ONE wake-up, and a
+     * delivery retry that lands shortly after the run is drained without paying
+     * for a second cold start. Default: 600000ms (10 minutes).
+     */
+    idleGraceMs?: number;
+    /**
+     * Hard backstop on total awake time in `exitWhenIdle` mode. The native idle
+     * detector is the normal exit path and this should never fire; it exists so a
+     * worker that never reaches idle (a wedged run, a Slot recovery cannot
+     * re-dispatch) cannot bill indefinitely. On expiry the daemon logs the reason,
+     * closes its database cleanly and exits. Nothing is deleted — the next wake
+     * resumes from the same durable Slot/outbox rows. Default: 10800000ms (3h).
+     */
+    maxLifetimeMs?: number;
     /**
      * External-mode HTTP trigger settings. Ignored in internal mode.
      */