toiljs 0.0.66 → 0.0.68

package/src/devserver/daemon/index.ts

@@ -139,20 +139,21 @@ export class DaemonHost implements DaemonRuntime {
         // else the emulator stays off (fail-closed; section 3.3 / 5.1).
         const surface = parseSurface(bytes);
         if (surface === 'invalid') {
-            this.log(pc.red('  ✗ cold artifact toil.surface is corrupt; daemon not started') + '\n');
+            this.log(
+                pc.red('  ✗ cold artifact toil.surface is corrupt; daemon not started') + '\n',
+            );
             if (this.running) this.stop();
             this.loadedMtimeMs = mtimeMs;
             return false;
         }
-        if (surface !== 'absent' && surface.targetMode !== 'cold')
+        if (surface.targetMode !== 'cold')
             this.log(
                 pc.yellow('  ! ') +
                     pc.dim('cold slot holds a hot-mode artifact; ignoring daemon emulator') +
                     '\n',
             );
         const catalog = parseDaemonCatalog(bytes);
-        const declaresDaemon =
-            (surface === 'absent' ? false : surface.flags.daemon) || (catalog?.hasDaemon ?? false);
+        const declaresDaemon = surface.flags.daemon || (catalog?.hasDaemon ?? false);
 
         // A restart: stop the old box (timers + instance), bump epoch, start fresh.
         if (this.running) this.stop();
@@ -318,9 +319,8 @@ export class DaemonHost implements DaemonRuntime {
             const ret = this.exports.scheduled_tick(task.taskIndex); // packed-i64
             if (ret < 0n)
                 this.log(
-                    pc.yellow(
-                        `  ⏱ @scheduled ${task.name} returned error ${decodeAbiError(ret)}`,
-                    ) + '\n',
+                    pc.yellow(`  ⏱ @scheduled ${task.name} returned error ${decodeAbiError(ret)}`) +
+                        '\n',
                 );
         } catch (e) {
             // A trapped tick does NOT tear down the long-lived daemon box (unlike a

package/src/devserver/daemon/runtime.ts

@@ -0,0 +1,48 @@
+import pc from 'picocolors';
+
+import { DaemonHost, daemonEmulationEnabled } from './index.js';
+import type { ResolvedDaemonConfig } from './host.js';
+
+export interface DaemonRuntimeOptions {
+    readonly coldWasmFile?: string;
+    readonly nodeMode?: string;
+    readonly daemon?: ResolvedDaemonConfig;
+    readonly pollMs?: number;
+}
+
+export interface RunningDaemonRuntime {
+    readonly host: DaemonHost;
+    close(): void;
+}
+
+/** Starts the shared cold-artifact daemon runtime used by both `dev` and `start`. */
+export function startDaemonRuntime(options: DaemonRuntimeOptions): RunningDaemonRuntime | null {
+    const nodeMode = options.nodeMode ?? 'all';
+    if (
+        options.coldWasmFile === undefined ||
+        !daemonEmulationEnabled(nodeMode) ||
+        options.daemon === undefined
+    ) {
+        return null;
+    }
+
+    const host = new DaemonHost(options.coldWasmFile, options.daemon, nodeMode);
+    const pollDaemon = (): void => {
+        try {
+            host.refresh();
+        } catch (e) {
+            process.stdout.write(pc.red(`  x daemon reload failed: ${String(e)}`) + '\n');
+        }
+    };
+    pollDaemon();
+    const timer = setInterval(pollDaemon, options.pollMs ?? 500);
+    timer.unref?.();
+
+    return {
+        host,
+        close: (): void => {
+            clearInterval(timer);
+            host.close();
+        },
+    };
+}

package/src/devserver/db/catalog.ts

@@ -48,7 +48,7 @@ export function parseCatalog(wasm: Buffer): DbCatalogState {
 
     const r = new DataReader(sec);
     const version = r.readU16();
-    if (!r.ok || (version !== 1 && version !== 2)) return { kind: 'malformed' };
+    if (!r.ok || version !== 1) return { kind: 'malformed' };
     const ndb = r.readU16();
     for (let d = 0; d < ndb && r.ok; d++) {
         const db = r.readString();
@@ -63,18 +63,14 @@ export function parseCatalog(wasm: Buffer): DbCatalogState {
             r.readU32(); // generation
             const replication = r.readU8(); // emitter order: replication then placement
             const placement = r.readU8();
-            let fillMaxWaitMs = DEFAULT_FILL_WAIT_MS;
-            let fillAllowStale = true;
-            if (version >= 2) {
-                fillMaxWaitMs = r.readU32();
-                const fillAllowStaleByte = r.readU8();
-                if (
-                    fillMaxWaitMs > MAX_FILL_WAIT_MS ||
-                    (fillAllowStaleByte !== 0 && fillAllowStaleByte !== 1)
-                )
-                    return { kind: 'malformed' };
-                fillAllowStale = fillAllowStaleByte === 1;
-            }
+            const fillMaxWaitMs = r.readU32();
+            const fillAllowStaleByte = r.readU8();
+            if (
+                fillMaxWaitMs > MAX_FILL_WAIT_MS ||
+                (fillAllowStaleByte !== 0 && fillAllowStaleByte !== 1)
+            )
+                return { kind: 'malformed' };
+            const fillAllowStale = fillAllowStaleByte === 1;
             const nFields = r.readU16();
             for (let f = 0; f < nFields; f++) {
                 r.readString(); // field name

package/src/devserver/db/database.ts

@@ -700,6 +700,7 @@ export class DevDatabase {
         if (outcome.kind === 'unit') {
             this.store.set(sk, value);
             this.stampVersion(coll, sk); // stamp the value type's current schema version
+            this.recordWrite(db, coll);
         }
         this.recordIdemFinish(coll, key, 'C', idem, requestHash, outcome);
         return this.replayRecordOutcome(db, outcome);
@@ -732,6 +733,7 @@ export class DevDatabase {
         if (outcome.kind === 'value') {
             this.store.set(sk, v);
             this.stampVersion(coll, sk); // a patch rewrites the row at the current version
+            this.recordWrite(db, coll);
         }
         this.recordIdemFinish(coll, key, 'P', idem, requestHash, outcome);
         return this.replayRecordOutcome(db, outcome);
@@ -1023,6 +1025,15 @@ export class DevDatabase {
 
     // `delta` is the wasm i64 (a BigInt across the boundary); `BigInt()`
     // normalizes the test's plain-number form too. Saturates like the edge.
+    /** Note a successful write to a SOURCE collection, so the runtime can re-run
+     *  the `@derive` materializers that depend on it after this dispatch. A
+     *  derive's OWN writes run under FunctionKind=Derive and must never
+     *  re-trigger a derive (which would loop), so they are never recorded. */
+    private recordWrite(db: DbDevState, coll: DevCollectionHandle): void {
+        if (db.functionKind === DbFunctionKind.Derive) return;
+        db.writtenCollections.add(coll.name);
+    }
+
     counterAdd(
         ref: MemoryRef,
         db: DbDevState,
@@ -1045,6 +1056,7 @@ export class DevDatabase {
         }
         const sk = storeKey(coll.name, key);
         this.counters.set(sk, satI64((this.counters.get(sk) ?? 0n) + d));
+        this.recordWrite(db, coll);
         return 0;
     }
 
@@ -1086,6 +1098,7 @@ export class DevDatabase {
             log.push(ev);
             (this.eventVersions.get(sk) ?? this.eventVersions.set(sk, []).get(sk)!).push(sv);
         }
+        this.recordWrite(db, coll);
         return 0;
     }
 
@@ -1124,6 +1137,7 @@ export class DevDatabase {
             (this.eventVersions.get(sk) ?? this.eventVersions.set(sk, []).get(sk)!).push(sv);
         }
         seen.add(evid);
+        this.recordWrite(db, coll);
         return 1;
     }
 

package/src/devserver/db/derives.ts

@@ -0,0 +1,121 @@
+import { customSection } from '../wasm/sections.js';
+
+/**
+ * Parses the `toildb.derives` wiring section emitted by toilscript for a
+ * `@database` class with `@derive` materializer methods (see the compiler's
+ * `buildToilDbDerives`). It maps each derive to its owning `@database` class, so
+ * the dev runtime (`runtime/module.ts`) can, after a dispatch writes a source
+ * collection, re-run that database's derives under FunctionKind=Derive. Fails
+ * closed: any malformed byte yields `[]` (no derive runs) rather than throwing.
+ *
+ * Section layout (LE), mirroring `buildToilDbDerives`:
+ *   u16 format_version = 1
+ *   u16 n_derives
+ *   per derive: u16 derive_id, str db_name, str method_name   (str = u32 len + bytes)
+ */
+export interface DeriveEntry {
+    readonly deriveId: number;
+    readonly dbName: string;
+    readonly methodName: string;
+}
+
+const SECTION = 'toildb.derives';
+const VERSION = 1;
+const MAX_SECTION_BYTES = 128 * 1024;
+const MAX_DERIVES = 1024;
+const MAX_NAME_BYTES = 1024;
+const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
+
+export function parseDerives(wasm: Buffer): readonly DeriveEntry[] {
+    let section: Buffer | null;
+    try {
+        section = customSection(wasm, SECTION);
+    } catch {
+        return [];
+    }
+    if (section === null) return [];
+    if (section.length > MAX_SECTION_BYTES) return [];
+
+    const r = new Reader(section);
+    const version = r.u16();
+    if (!r.ok || version !== VERSION) return [];
+    const count = r.u16();
+    if (!r.ok || count > MAX_DERIVES) return [];
+
+    const derives: DeriveEntry[] = [];
+    for (let i = 0; i < count && r.ok; i++) {
+        const deriveId = r.u16();
+        const dbName = r.string();
+        const methodName = r.string();
+        if (!r.ok || dbName.length === 0) return [];
+        derives.push({ deriveId, dbName, methodName });
+    }
+    if (!r.ok || r.remaining() !== 0) return [];
+    return derives;
+}
+
+/**
+ * The derives whose owning `@database` had at least one source collection
+ * written during this dispatch. `written` holds "Db/coll" store keys; the
+ * database is the prefix before the first `/`. Each affected derive appears at
+ * most once (coalescing: many writes to one database run its derives once).
+ */
+export function derivesForWrites(
+    derives: readonly DeriveEntry[],
+    written: ReadonlySet<string>,
+): readonly DeriveEntry[] {
+    if (derives.length === 0 || written.size === 0) return [];
+    const dbs = new Set<string>();
+    for (const key of written) {
+        const slash = key.indexOf('/');
+        dbs.add(slash >= 0 ? key.slice(0, slash) : key);
+    }
+    return derives.filter((d) => dbs.has(d.dbName));
+}
+
+class Reader {
+    private pos = 0;
+    ok = true;
+
+    constructor(private readonly bytes: Buffer) {}
+
+    remaining(): number {
+        return this.bytes.length - this.pos;
+    }
+
+    u16(): number {
+        if (!this.ok || this.pos + 2 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt16LE(this.pos);
+        this.pos += 2;
+        return out;
+    }
+
+    u32(): number {
+        if (!this.ok || this.pos + 4 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt32LE(this.pos);
+        this.pos += 4;
+        return out;
+    }
+
+    string(): string {
+        const len = this.u32();
+        if (!this.ok || len > MAX_NAME_BYTES || this.pos + len > this.bytes.length) {
+            this.ok = false;
+            return '';
+        }
+        try {
+            const out = UTF8_DECODER.decode(this.bytes.subarray(this.pos, this.pos + len));
+            this.pos += len;
+            return out;
+        } catch {
+            this.ok = false;
+            return '';
+        }
+    }
+}

package/src/devserver/db/index.ts

@@ -15,4 +15,5 @@ export {
     setDbCatalog,
 } from './database.js';
 export { parseCatalog } from './catalog.js';
+export { type DeriveEntry, derivesForWrites, parseDerives } from './derives.js';
 export { CollectionFamily, DbFunctionKind, type DbDevState, freshDbState } from './types.js';

package/src/devserver/db/types.ts

@@ -50,6 +50,11 @@ export interface DbDevState {
     lastResult: Buffer | null;
     lastResultVersion: number;
     functionKind: DbFunctionKind;
+    /** Names ("Db/coll") of source collections written during this dispatch, so
+     *  the runtime can re-run the affected `@derive` materializers afterward.
+     *  Only populated for non-Derive dispatches (a derive's own writes must not
+     *  re-trigger it - see `database.ts` `recordWrite`). */
+    writtenCollections: Set<string>;
 }
 
 export function freshDbState(): DbDevState {
@@ -58,6 +63,7 @@ export function freshDbState(): DbDevState {
         lastResult: null,
         lastResultVersion: -1,
         functionKind: DbFunctionKind.Job,
+        writtenCollections: new Set<string>(),
     };
 }
 

package/src/devserver/http/proxy.ts

@@ -114,45 +114,59 @@ export function wireWebsocketProxy(app: Server, target: ViteTarget): void {
         '/*',
         { message_type: 'Buffer', idle_timeout: 120, max_payload_length: 16 * 1024 * 1024 },
         (ws: Websocket) => {
-            const { url, protocol } = ws.context as { url: string; protocol: string };
-            const upstream = new WebSocket(
-                `ws://${target.host}:${String(target.port)}${url}`,
-                protocol ? protocol.split(',').map((p) => p.trim()) : [],
-            );
-            upstream.binaryType = 'arraybuffer';
-
-            const pending: (string | Uint8Array<ArrayBuffer>)[] = [];
-            let open = false;
-
-            upstream.onopen = (): void => {
-                open = true;
-                for (const m of pending) upstream.send(m);
-                pending.length = 0;
-            };
-            upstream.onmessage = (event: MessageEvent): void => {
-                if (typeof event.data === 'string') ws.send(event.data);
-                else ws.send(Buffer.from(event.data as ArrayBuffer), true);
-            };
-            upstream.onclose = (event: CloseEvent): void => {
-                ws.close(event.code, event.reason);
-            };
-            upstream.onerror = (): void => {
-                ws.close();
-            };
-
-            ws.on('message', (message: Buffer, isBinary: boolean) => {
-                const m = toUpstreamMessage(message, isBinary);
-                if (open) upstream.send(m);
-                else pending.push(m);
-            });
-            ws.on('close', () => {
-                if (
-                    upstream.readyState === WebSocket.OPEN ||
-                    upstream.readyState === WebSocket.CONNECTING
-                ) {
-                    upstream.close();
-                }
-            });
+            pipeToVite(ws, target, ws.context as { url: string; protocol: string });
         },
     );
 }
+
+/**
+ * Pipe ONE upgraded websocket to the internal Vite HMR server. The verbatim body extracted from
+ * {@link wireWebsocketProxy} so the dev STREAM router (doc 08 4.1 `wireStreams`) can reuse it for every
+ * NON-stream upgrade while it handles `@stream`-route upgrades itself - HMR stays byte-for-byte
+ * unchanged. `ctx` is the upgrade context (`{ url, protocol }`) the upgrade handler stamped.
+ */
+export function pipeToVite(
+    ws: Websocket,
+    target: ViteTarget,
+    ctx: { url: string; protocol: string },
+): void {
+    const { url, protocol } = ctx;
+    const upstream = new WebSocket(
+        `ws://${target.host}:${String(target.port)}${url}`,
+        protocol ? protocol.split(',').map((p) => p.trim()) : [],
+    );
+    upstream.binaryType = 'arraybuffer';
+
+    const pending: (string | Uint8Array<ArrayBuffer>)[] = [];
+    let open = false;
+
+    upstream.onopen = (): void => {
+        open = true;
+        for (const m of pending) upstream.send(m);
+        pending.length = 0;
+    };
+    upstream.onmessage = (event: MessageEvent): void => {
+        if (typeof event.data === 'string') ws.send(event.data);
+        else ws.send(Buffer.from(event.data as ArrayBuffer), true);
+    };
+    upstream.onclose = (event: CloseEvent): void => {
+        ws.close(event.code, event.reason);
+    };
+    upstream.onerror = (): void => {
+        ws.close();
+    };
+
+    ws.on('message', (message: Buffer, isBinary: boolean) => {
+        const m = toUpstreamMessage(message, isBinary);
+        if (open) upstream.send(m);
+        else pending.push(m);
+    });
+    ws.on('close', () => {
+        if (
+            upstream.readyState === WebSocket.OPEN ||
+            upstream.readyState === WebSocket.CONNECTING
+        ) {
+            upstream.close();
+        }
+    });
+}