toiljs 0.0.60 → 0.0.62

package/src/devserver/daemon/host.ts

@@ -0,0 +1,224 @@
+/**
+ * Host-import surface for the dev daemon (cold) box, mirroring the production
+ * edge's `daemon.*` + `mstore.*` imports. DAEMON path only (streams are Phase 4).
+ *
+ * Per RECONCILIATION:
+ *   - Part 4 `daemon.*`: is_leader / current_epoch / yield / sleep_ms / task_count
+ *     / next_fire_ms / http_call / remote_call. In a single dev process the leader
+ *     stub is always true and the lease never expires (section 5.2). Fenced DB
+ *     writes are TRANSPARENT (no `daemon.db_write_fenced` import).
+ *   - Part 4 `mstore.*`: handleless, ttl_secs, inline drain (section 7.4).
+ *   - Part 3 error bridge: a u16 subsystem code `c` is returned as `-(0x10000 + c)`;
+ *     the buffer sentinels `-1` (TOO_SMALL) / `-2` (ABSENT) are unchanged.
+ *
+ * The cold box also imports the request-surface `env.*` MINUS the response/stream
+ * functions it must not have (no `set_status`/`set_header`/`respond_file`/
+ * `client_ip`/`ratelimit_check`); it keeps `@data`/crypto/env/`Date.now`/email so a
+ * daemon can read+write the DB and send mail. The two allow-lists live in
+ * `runtime/module.ts`.
+ */
+
+import { buildDatabaseImports, type DbDevState, freshDbState } from '../db/index.js';
+import { type DevMemoryStore } from '../mstore/store.js';
+import { buildCryptoImports, type CryptoState, freshCryptoState } from '../runtime/crypto.js';
+import {
+    buildEnvImports,
+    type MemoryRef,
+    readBytes,
+    writeBytesOut,
+} from '../runtime/host.js';
+
+/**
+ * Resolved daemon (L4) config the dev scheduler reads. Structurally identical to
+ * the compiler's `ResolvedDaemonConfig`; declared here so the devserver package has
+ * no source dependency on the compiler package (its tsconfig is isolated to
+ * `src/devserver`). The compiler's resolved config is passed in verbatim.
+ */
+export interface ResolvedDaemonConfig {
+    readonly region: string | null;
+    readonly standbyRegion: string | null;
+    readonly defaultIntervalMs: number;
+    readonly tickBudgetMs: number;
+    readonly gasTick: number;
+    readonly maxTasks: number;
+}
+
+/** RECONCILIATION Part 3 u16 error registry (the subset the daemon/mstore use). */
+export const enum AbiError {
+    MstoreNotFound = 0x0301,
+    MstoreNotANumber = 0x0306,
+    MstoreScanBusy = 0x0307,
+    MstoreConflict = 0x0304,
+    DaemonScheduleRejected = 0x0403,
+    DaemonCallFailed = 0x0405,
+}
+
+/** Encode a u16 subsystem error per the Part 3 negative-return bridge:
+ *  `code = (-v) - 0x10000`, so `v = -(0x10000 + code)`. */
+export function encodeAbiError(code: number): number {
+    return -(0x10000 + code);
+}
+
+/** The minimal scheduler/leader view the `daemon.*` imports read. Implemented by
+ *  the resident `DaemonHost`. */
+export interface DaemonRuntime {
+    isLeader(): boolean;
+    /** Monotonic fencing token; bumps on each (re)start. */
+    epoch(): bigint;
+    /** Number of registered `@scheduled` tasks. */
+    taskCount(): number;
+    /** Next computed fire time (epoch ms) for `taskId`, or `null` if unknown. */
+    nextFireMs(taskId: number): number | null;
+}
+
+/** A host-import map: each value is a function over i32 args (i64 params are typed
+ *  `number | bigint` individually, matching the existing db/crypto import maps). */
+type HostFnMap = Record<string, (...args: number[]) => number | bigint>;
+
+/** Build the `mstore.*` host imports, backed by `store`. Handleless; keys are read
+ *  from guest memory and auto-scoped to the single dev host/region. */
+export function buildMstoreImports(ref: MemoryRef, store: DevMemoryStore): HostFnMap {
+    const key = (p: number, l: number): string => readBytes(ref, p, l).toString('utf8');
+    return {
+        'mstore.get': (keyPtr: number, keyLen: number, outPtr: number, outCap: number): number => {
+            const v = store.get(key(keyPtr, keyLen));
+            if (v === null) return encodeAbiError(AbiError.MstoreNotFound);
+            return writeBytesOut(ref, v, outPtr, outCap);
+        },
+        'mstore.set': (
+            keyPtr: number,
+            keyLen: number,
+            valPtr: number,
+            valLen: number,
+            ttlSecs: number,
+        ): number => {
+            store.set(key(keyPtr, keyLen), readBytes(ref, valPtr, valLen), ttlSecs);
+            return 0;
+        },
+        'mstore.delete': (keyPtr: number, keyLen: number): number =>
+            store.delete(key(keyPtr, keyLen)) ? 0 : encodeAbiError(AbiError.MstoreNotFound),
+        'mstore.incr': (
+            keyPtr: number,
+            keyLen: number,
+            delta: number | bigint,
+            ttlSecs: number,
+        ): bigint => {
+            const d = typeof delta === 'bigint' ? delta : BigInt(delta);
+            const next = store.incr(key(keyPtr, keyLen), d, ttlSecs);
+            return next === null ? BigInt(encodeAbiError(AbiError.MstoreNotANumber)) : next;
+        },
+        'mstore.cas': (
+            keyPtr: number,
+            keyLen: number,
+            expectPtr: number,
+            expectLen: number,
+            newPtr: number,
+            newLen: number,
+            ttlSecs: number,
+        ): number => {
+            const expect = expectLen === 0 ? null : readBytes(ref, expectPtr, expectLen);
+            const ok = store.cas(
+                key(keyPtr, keyLen),
+                expect,
+                readBytes(ref, newPtr, newLen),
+                ttlSecs,
+            );
+            return ok ? 0 : encodeAbiError(AbiError.MstoreConflict);
+        },
+        'mstore.expire': (keyPtr: number, keyLen: number, ttlSecs: number): number =>
+            store.expire(key(keyPtr, keyLen), ttlSecs)
+                ? 0
+                : encodeAbiError(AbiError.MstoreNotFound),
+        'mstore.scan': (
+            prefixPtr: number,
+            prefixLen: number,
+            cursor: number | bigint,
+            outPtr: number,
+            outCap: number,
+        ): bigint => {
+            const cur = typeof cursor === 'bigint' ? cursor : BigInt(cursor);
+            const res = store.scan(key(prefixPtr, prefixLen), cur);
+            if (res === null) return BigInt(encodeAbiError(AbiError.MstoreScanBusy));
+            // Frame: u32 count, then per key (u32 len + bytes).
+            let total = 4;
+            for (const k of res.keys) total += 4 + Buffer.byteLength(k, 'utf8');
+            const frame = Buffer.alloc(total);
+            let o = frame.writeUInt32LE(res.keys.length, 0);
+            for (const k of res.keys) {
+                const kb = Buffer.from(k, 'utf8');
+                o = frame.writeUInt32LE(kb.length, o);
+                kb.copy(frame, o);
+                o += kb.length;
+            }
+            const len = writeBytesOut(ref, frame, outPtr, outCap);
+            if (len < 0) return BigInt(len); // TOO_SMALL sentinel
+            return (res.next << 32n) | BigInt(len);
+        },
+    };
+}
+
+/** Build the `daemon.*` host imports, closing over the resident `DaemonRuntime`.
+ *  These imports do not read guest memory (they answer from the resident scheduler
+ *  state), so they take no `MemoryRef`. */
+export function buildDaemonNamespace(rt: DaemonRuntime): HostFnMap {
+    return {
+        'daemon.is_leader': (): number => (rt.isLeader() ? 1 : 0),
+        'daemon.current_epoch': (): bigint => rt.epoch(),
+        // The dev lease never expires, so yield/sleep never report LEASE_LOST.
+        'daemon.yield': (): number => 0,
+        'daemon.sleep_ms': (_ms: number | bigint): number => 0,
+        'daemon.task_count': (): number => rt.taskCount(),
+        'daemon.next_fire_ms': (taskId: number): bigint => {
+            const at = rt.nextFireMs(taskId);
+            return at === null ? BigInt(encodeAbiError(AbiError.DaemonScheduleRejected)) : BigInt(at);
+        },
+        // Outbound call stubs: dev returns a "call failed" sentinel rather than
+        // performing real network I/O from a synchronous wasm import (section 5.4).
+        'daemon.http_call': (
+            _reqPtr: number,
+            _reqLen: number,
+            _outPtr: number,
+            _outCap: number,
+        ): bigint => BigInt(encodeAbiError(AbiError.DaemonCallFailed)),
+        'daemon.remote_call': (
+            _svcId: number,
+            _reqPtr: number,
+            _reqLen: number,
+            _outPtr: number,
+            _outCap: number,
+        ): bigint => BigInt(encodeAbiError(AbiError.DaemonCallFailed)),
+    };
+}
+
+/** Per-cold-box host state (DB + crypto scratch), analogous to `DispatchState`. */
+export interface DaemonState {
+    crypto: CryptoState;
+    db: DbDevState;
+}
+
+export function freshDaemonState(): DaemonState {
+    return { crypto: freshCryptoState(), db: freshDbState() };
+}
+
+/**
+ * The full `env` import object for the cold daemon box: the request-surface env
+ * MINUS the response/stream functions (built by `buildEnvImports`), PLUS the
+ * `daemon.*` and `mstore.*` namespaces. The cold box has no `handle` entry and no
+ * response surface.
+ */
+export function buildDaemonImports(
+    ref: MemoryRef,
+    state: DaemonState,
+    rt: DaemonRuntime,
+    store: DevMemoryStore,
+): WebAssembly.Imports {
+    return {
+        env: {
+            ...buildEnvImports(ref, state),
+            ...buildCryptoImports(ref, state.crypto),
+            ...buildDatabaseImports(ref, state.db),
+            ...buildDaemonNamespace(rt),
+            ...buildMstoreImports(ref, store),
+        },
+    };
+}

package/src/devserver/daemon/index.ts

@@ -0,0 +1,349 @@
+/**
+ * Dev DAEMON (L4) emulation. Loads `release-cold.wasm` ONCE into a single resident
+ * instance, calls the guest `daemon_start()` export (RECONCILIATION Part 2; runs
+ * once, `0` ok), registers the `@scheduled` tasks from `toildaemon.catalog`, and
+ * drives them: interval tasks via `setInterval`, cron tasks via a one-shot
+ * `setTimeout` armed at the next minute whose precomputed bitmasks all pass (F6;
+ * never a runtime cron-string parse).
+ *
+ * Single-process at-most-once: there is exactly one dev process, so the leader
+ * stub is always true and the lease never expires. The epoch is a fencing-token
+ * stub that bumps on each (re)start, so guest code that compares epochs behaves
+ * like the edge across a cold-artifact hot reload. A trapped tick does NOT tear
+ * down the long-lived daemon box (deliberate asymmetry with the stream box); the
+ * overlap guard (`ticking` set) prevents a slow tick from piling up
+ * (overlap_policy 0 = skip-if-running).
+ *
+ * DAEMON path only; stream + WebSocket dev emulation is Phase 4.
+ */
+
+import fs from 'node:fs';
+
+import pc from 'picocolors';
+
+import { type MemoryRef } from '../runtime/host.js';
+import { devMemoryStore } from '../mstore/store.js';
+import { parseSurface } from '../wasm/surface.js';
+import {
+    type CronMasks,
+    type DaemonCatalog,
+    parseDaemonCatalog,
+    type ScheduledTask,
+} from './catalog.js';
+import { cronMatches, cronNeverFires, nextCronFireMs } from './cron.js';
+import {
+    buildDaemonImports,
+    type DaemonRuntime,
+    type DaemonState,
+    freshDaemonState,
+    type ResolvedDaemonConfig,
+} from './host.js';
+
+interface ColdExports {
+    readonly memory: WebAssembly.Memory;
+    readonly daemon_start: () => number;
+    readonly scheduled_tick: (taskId: number) => bigint;
+    /** OPTIONAL: the host calls it before `daemon_start` if exported (Part 2). */
+    readonly init?: () => number;
+}
+
+/** RECONCILIATION Part 3 decode of a negative packed-i64 return, for logging. */
+function decodeAbiError(ret: bigint): string {
+    if (ret >= 0n) return 'ok';
+    if (ret === -1n) return 'STATUS_TOO_SMALL';
+    if (ret === -2n) return 'STATUS_ABSENT';
+    if (ret <= -0x10000n) return '0x' + ((-ret - 0x10000n) & 0xffffn).toString(16).padStart(4, '0');
+    if (ret <= -1000n) return 'DB(TDL ' + String(-ret - 1000n) + ')';
+    return String(ret);
+}
+
+/**
+ * Whether the daemon emulator should run for this dev process and artifact. Per
+ * doc 08 section 5.1: `nodeMode` is `daemon` or `all`, the cold artifact's
+ * `toil.surface` declares a daemon surface, AND `parseDaemonCatalog` returns
+ * non-null with a daemon present.
+ */
+export function daemonEmulationEnabled(nodeMode: string): boolean {
+    return nodeMode === 'daemon' || nodeMode === 'all';
+}
+
+export class DaemonHost implements DaemonRuntime {
+    private module: WebAssembly.Module | null = null;
+    private instance: WebAssembly.Instance | null = null;
+    private exports: ColdExports | null = null;
+    private state: DaemonState = freshDaemonState();
+    private catalog: DaemonCatalog | null = null;
+    private loadedMtimeMs = -1;
+    private running = false;
+    /** Fencing-token stub (00 D3); bumps on each (re)start. */
+    private epochValue = 0n;
+    /** task_index -> active interval/timeout handle. */
+    private timers = new Map<number, NodeJS.Timeout>();
+    /** task_index -> next computed fire time (epoch ms), for daemon.next_fire_ms. */
+    private nextFire = new Map<number, number>();
+    /** task_index of ticks currently executing (overlap guard). */
+    private ticking = new Set<number>();
+
+    constructor(
+        private readonly coldWasmPath: string,
+        private readonly cfg: ResolvedDaemonConfig,
+        private readonly nodeMode: string,
+        private readonly log: (s: string) => void = (s) => process.stdout.write(s),
+    ) {}
+
+    /** Whether the daemon box is currently resident and started. */
+    get active(): boolean {
+        return this.running;
+    }
+
+    get tasks(): readonly ScheduledTask[] {
+        return this.catalog?.tasks ?? [];
+    }
+
+    // --- DaemonRuntime (the daemon.* host imports read these) ---
+    isLeader(): boolean {
+        return true; // single dev process is always the leader
+    }
+    epoch(): bigint {
+        return this.epochValue;
+    }
+    taskCount(): number {
+        return this.catalog?.tasks.length ?? 0;
+    }
+    nextFireMs(taskId: number): number | null {
+        return this.nextFire.get(taskId) ?? null;
+    }
+
+    /**
+     * (Re)load on mtime change, mirroring `WasmServerModule.refresh`. A cold-artifact
+     * change PAUSES + RESTARTS the daemon with a bumped epoch (section 9.1). Returns
+     * `true` when a (re)load happened.
+     */
+    refresh(): boolean {
+        if (!daemonEmulationEnabled(this.nodeMode)) return false;
+        let mtimeMs: number;
+        try {
+            mtimeMs = fs.statSync(this.coldWasmPath).mtimeMs;
+        } catch {
+            // Cold artifact gone -> stop a running daemon, stay idle.
+            if (this.running) this.stop();
+            this.module = null;
+            this.loadedMtimeMs = -1;
+            return false;
+        }
+        if (this.module !== null && mtimeMs === this.loadedMtimeMs) return false;
+
+        const bytes = fs.readFileSync(this.coldWasmPath);
+
+        // The cold artifact must declare a daemon surface and a non-null catalog,
+        // else the emulator stays off (fail-closed; section 3.3 / 5.1).
+        const surface = parseSurface(bytes);
+        if (surface === 'invalid') {
+            this.log(pc.red('  ✗ cold artifact toil.surface is corrupt; daemon not started') + '\n');
+            if (this.running) this.stop();
+            this.loadedMtimeMs = mtimeMs;
+            return false;
+        }
+        if (surface !== 'absent' && surface.targetMode !== 'cold')
+            this.log(
+                pc.yellow('  ! ') +
+                    pc.dim('cold slot holds a hot-mode artifact; ignoring daemon emulator') +
+                    '\n',
+            );
+        const catalog = parseDaemonCatalog(bytes);
+        const declaresDaemon =
+            (surface === 'absent' ? false : surface.flags.daemon) || (catalog?.hasDaemon ?? false);
+
+        // A restart: stop the old box (timers + instance), bump epoch, start fresh.
+        if (this.running) this.stop();
+
+        if (!declaresDaemon || catalog === null || !catalog.hasDaemon) {
+            // No daemon in this artifact: load nothing, stay idle.
+            this.module = null;
+            this.catalog = null;
+            this.loadedMtimeMs = mtimeMs;
+            return false;
+        }
+
+        this.module = new WebAssembly.Module(bytes);
+        this.catalog = catalog;
+        this.loadedMtimeMs = mtimeMs;
+        this.epochValue += 1n; // fencing token bumps on each (re)start
+        this.start();
+        return true;
+    }
+
+    /** Instantiate the cold box, run daemon_start once, register the tasks. */
+    private start(): void {
+        if (this.module === null || this.catalog === null) return;
+        const ref: MemoryRef = { memory: null };
+        this.state = freshDaemonState();
+        const imports = buildDaemonImports(ref, this.state, this, devMemoryStore);
+
+        // Fail-closed up front, with names, when the cold box imports anything outside its allowed
+        // surface (the request env subset + crypto + @data + daemon.* + mstore.*; NO response/
+        // stream functions). Mirrors `WasmServerModule.assertImportSurface` (section 7.1).
+        const provided = new Set(Object.keys((imports as { env: Record<string, unknown> }).env));
+        const missing = WebAssembly.Module.imports(this.module)
+            .filter((i) => i.kind === 'function' && (i.module !== 'env' || !provided.has(i.name)))
+            .map((i) => `${i.module}.${i.name}`);
+        if (missing.length > 0) {
+            this.log(
+                pc.red(
+                    `  ✗ cold daemon wasm imports unsupported host functions: ${missing.join(', ')}`,
+                ) + '\n',
+            );
+            this.module = null;
+            return;
+        }
+
+        this.instance = new WebAssembly.Instance(this.module, imports);
+        this.exports = this.instance.exports as unknown as ColdExports;
+        ref.memory = this.exports.memory;
+
+        try {
+            if (typeof this.exports.init === 'function') this.exports.init();
+            const rc = this.exports.daemon_start();
+            if (rc !== 0) {
+                this.log(
+                    pc.red(`  ✗ daemon_start() returned ${String(rc)}; daemon not running`) + '\n',
+                );
+                this.instance = null;
+                this.exports = null;
+                return;
+            }
+        } catch (e) {
+            // A trap in daemon_start leaves the daemon stopped; surface it (do not
+            // retry-loop in dev, mirroring the request-path error handling).
+            this.log(pc.red(`  ✗ daemon_start() trapped: ${String(e)}`) + '\n');
+            this.instance = null;
+            this.exports = null;
+            return;
+        }
+
+        this.running = true;
+        const limited = this.catalog.tasks.slice(0, this.cfg.maxTasks);
+        for (const task of limited) this.registerTask(task);
+        this.log(
+            pc.green('  ⏱ ') +
+                pc.dim(
+                    `daemon started (epoch ${String(this.epochValue)}, ${String(limited.length)} task${
+                        limited.length === 1 ? '' : 's'
+                    })`,
+                ) +
+                '\n',
+        );
+    }
+
+    /** Clear timers, drop the resident instance. In-flight ticks finish on their own
+     *  (the overlap guard prevents a NEW tick; a running one completes). */
+    private stop(): void {
+        for (const t of this.timers.values()) clearTimeout(t);
+        this.timers.clear();
+        this.nextFire.clear();
+        // Best-effort guest stop hook, if the artifact exports one.
+        const stop = (this.exports as unknown as { daemon_stop?: () => void } | null)?.daemon_stop;
+        if (typeof stop === 'function') {
+            try {
+                stop();
+            } catch {
+                /* ignore a trap in the optional stop hook */
+            }
+        }
+        this.instance = null;
+        this.exports = null;
+        this.running = false;
+    }
+
+    private registerTask(task: ScheduledTask): void {
+        if (task.schedule.kind === 'interval') {
+            const ms = Math.max(1000, task.schedule.ms || this.cfg.defaultIntervalMs);
+            this.nextFire.set(task.taskIndex, Date.now() + ms);
+            const handle = setInterval(() => {
+                this.nextFire.set(task.taskIndex, Date.now() + ms);
+                this.runTick(task);
+            }, ms);
+            // Do not keep the event loop alive solely for the dev scheduler.
+            handle.unref?.();
+            this.timers.set(task.taskIndex, handle);
+        } else {
+            const masks = task.schedule.masks;
+            if (cronNeverFires(masks)) {
+                this.log(
+                    pc.yellow('  ! ') +
+                        pc.dim(
+                            `@scheduled ${task.name} has an unsatisfiable cron mask; skipping (DAEMON_SCHEDULE_REJECTED)`,
+                        ) +
+                        '\n',
+                );
+                return;
+            }
+            this.armCron(task, masks);
+        }
+    }
+
+    /** Arm a one-shot timer to the next cron fire time, re-arming after each tick. */
+    private armCron(task: ScheduledTask, masks: CronMasks): void {
+        const next = nextCronFireMs(masks, Date.now());
+        if (next === null) {
+            this.nextFire.delete(task.taskIndex);
+            return;
+        }
+        this.nextFire.set(task.taskIndex, next);
+        const delay = Math.max(0, next - Date.now());
+        const handle = setTimeout(() => {
+            // Guard against a coarse-timer early fire: only run when the masks
+            // actually match the current minute (they should by construction).
+            if (cronMatches(masks, new Date())) this.runTick(task);
+            if (this.running) this.armCron(task, masks);
+        }, delay);
+        handle.unref?.();
+        this.timers.set(task.taskIndex, handle);
+    }
+
+    /** Fire one `@scheduled` task via `scheduled_tick(task_id)` (Part 2). */
+    private runTick(task: ScheduledTask): void {
+        if (!this.running || this.exports === null) return;
+        if (this.ticking.has(task.taskIndex)) {
+            // overlap_policy 0 = skip-if-running: a slow tick must not pile up.
+            this.log(
+                pc.dim(`  ⏱ @scheduled ${task.name} overran its interval; skipping a tick`) + '\n',
+            );
+            return;
+        }
+        if (!this.isLeader()) return; // always true in dev; kept for parity
+        this.ticking.add(task.taskIndex);
+        const startedAt = Date.now();
+        try {
+            const ret = this.exports.scheduled_tick(task.taskIndex); // packed-i64
+            if (ret < 0n)
+                this.log(
+                    pc.yellow(
+                        `  ⏱ @scheduled ${task.name} returned error ${decodeAbiError(ret)}`,
+                    ) + '\n',
+                );
+        } catch (e) {
+            // A trapped tick does NOT tear down the long-lived daemon box (unlike a
+            // stream box); the next tick runs normally on the same memory.
+            this.log(pc.red(`  ✗ @scheduled ${task.name} trapped: ${String(e)}`) + '\n');
+        } finally {
+            const took = Date.now() - startedAt;
+            if (took > this.cfg.tickBudgetMs)
+                this.log(
+                    pc.yellow(
+                        `  ⏱ @scheduled ${task.name} took ${String(took)}ms (> tickBudgetMs ${String(
+                            this.cfg.tickBudgetMs,
+                        )})`,
+                    ) + '\n',
+                );
+            this.ticking.delete(task.taskIndex);
+        }
+    }
+
+    /** Tear the daemon down for good (dev-server shutdown). */
+    close(): void {
+        if (this.running) this.stop();
+        this.module = null;
+        this.catalog = null;
+    }
+}