toiljs 0.0.67 → 0.0.68

package/build/devserver/runtime/module.js

@@ -1,5 +1,5 @@
 import fs from 'node:fs';
-import { DbFunctionKind, persistDb, setDbCatalog } from '../db/index.js';
+import { DbFunctionKind, derivesForWrites, parseDerives, persistDb, setDbCatalog, } from '../db/index.js';
 import { parseRouteKinds, routeKindForRequest } from '../db/routeKinds.js';
 import { decodeResponseEnvelope, encodeRequestEnvelope, unpackHandleResult, } from '../http/envelope.js';
 import { buildHostImports, freshDispatchState } from './host.js';
@@ -87,6 +87,8 @@ export class WasmServerModule {
     module = null;
     loadedMtimeMs = -1;
     routeKinds = [];
+    derives = [];
+    derivesDirty = false;
     constructor(wasmPath) {
         this.wasmPath = wasmPath;
     }
@@ -101,6 +103,8 @@ export class WasmServerModule {
         catch {
             this.module = null;
             this.routeKinds = [];
+            this.derives = [];
+            this.derivesDirty = false;
             this.loadedMtimeMs = -1;
             return false;
         }
@@ -112,13 +116,16 @@ export class WasmServerModule {
         this.assertExportSurface(module);
         setDbCatalog(bytes);
         this.routeKinds = parseRouteKinds(bytes);
+        this.derives = parseDerives(bytes);
         this.module = module;
         this.loadedMtimeMs = mtimeMs;
+        this.derivesDirty = this.derives.length > 0;
         return true;
     }
     dispatch(req) {
         if (this.module === null)
             throw new Error(`server wasm not loaded (${this.wasmPath})`);
+        this.rebuildDerivedViewsIfStale();
         const envelope = encodeRequestEnvelope(req);
         const ref = { memory: null };
         const state = freshDispatchState();
@@ -139,6 +146,7 @@ export class WasmServerModule {
         const headers = [...resp.headers, ...state.headers];
         const status = state.status ?? resp.status;
         const unhandled = headers.some(([n]) => n.toLowerCase() === UNHANDLED_HEADER);
+        this.runAffectedDerives(state.db.writtenCollections);
         persistDb();
         return {
             status,
@@ -171,6 +179,44 @@ export class WasmServerModule {
             throw new Error(`guest returned out-of-bounds values: ptr=${String(ptr)} len=${String(len)}`);
         return new Uint8Array(exports.memory.buffer, ptr, len).slice();
     }
+    runAffectedDerives(written) {
+        if (this.module === null)
+            return;
+        for (const derive of derivesForWrites(this.derives, written)) {
+            try {
+                this.runDerive(derive.deriveId);
+            }
+            catch (err) {
+                console.error(`[toil] derive ${derive.dbName}#${derive.methodName} failed:`, err);
+            }
+        }
+    }
+    rebuildDerivedViewsIfStale() {
+        if (!this.derivesDirty)
+            return;
+        this.derivesDirty = false;
+        for (const derive of this.derives) {
+            try {
+                this.runDerive(derive.deriveId);
+            }
+            catch (err) {
+                console.error(`[toil] derive ${derive.dbName}#${derive.methodName} failed on load:`, err);
+            }
+        }
+    }
+    runDerive(deriveId) {
+        if (this.module === null)
+            return;
+        const ref = { memory: null };
+        const state = freshDispatchState();
+        state.db.functionKind = DbFunctionKind.Derive;
+        const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
+        const exports = instance.exports;
+        ref.memory = exports.memory;
+        if (typeof exports.derive_run !== 'function')
+            return;
+        exports.derive_run(deriveId);
+    }
     assertImportSurface(module) {
         const missing = WebAssembly.Module.imports(module)
             .filter((i) => i.kind === 'function' && (i.module !== 'env' || !PROVIDED_IMPORTS.has(i.name)))

package/build/devserver/server.d.ts

@@ -8,6 +8,7 @@ export interface DevServerOptions {
     readonly host?: string;
     readonly wasmFile: string;
     readonly coldWasmFile?: string;
+    readonly streamWasmFile?: string;
     readonly nodeMode?: string;
     readonly daemon?: ResolvedDaemonConfig;
     readonly vite: ViteTarget;

package/build/devserver/server.js

@@ -1,101 +1,19 @@
-import fs from 'node:fs';
 import path from 'node:path';
 import { Server } from '@dacely/hyper-express';
 import pc from 'picocolors';
-import { DaemonHost, daemonEmulationEnabled } from './daemon/index.js';
+import { startDaemonRuntime } from './daemon/runtime.js';
 import { configureDbPersistence } from './db/index.js';
 import { initEmailService } from './email/index.js';
-import { applyCacheRule, lookupCache } from './http/cache.js';
-import { METHOD_CODES } from './http/envelope.js';
 import { proxyToVite, wireWebsocketProxy } from './http/proxy.js';
+import { assembleRouteSsr, dispatchWasmRequest, installRuntimeErrorHandler, isDispatchableMethod, runtimeServerOptions, sendSsr, } from './http/runtime.js';
 import { WasmServerModule } from './runtime/module.js';
-import { assembleSsr, buildSsrRoutes, pathnameOf, } from './ssr.js';
-const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
+import { StreamRouter } from './stream/router.js';
+import { streamEmulationEnabled, wireStreams } from './stream/wire.js';
+import { buildSsrRoutes, pathnameOf } from './ssr.js';
 const VITE_PREFIXES = ['/@', '/node_modules/', '/__toil/'];
-const MIME = {
-    '.html': 'text/html; charset=utf-8',
-    '.js': 'text/javascript; charset=utf-8',
-    '.mjs': 'text/javascript; charset=utf-8',
-    '.css': 'text/css; charset=utf-8',
-    '.json': 'application/json; charset=utf-8',
-    '.txt': 'text/plain; charset=utf-8',
-    '.svg': 'image/svg+xml',
-    '.png': 'image/png',
-    '.jpg': 'image/jpeg',
-    '.jpeg': 'image/jpeg',
-    '.webp': 'image/webp',
-    '.avif': 'image/avif',
-    '.gif': 'image/gif',
-    '.ico': 'image/x-icon',
-    '.wasm': 'application/wasm',
-    '.woff2': 'font/woff2',
-};
 function isViteInternal(url) {
     return VITE_PREFIXES.some((p) => url.startsWith(p));
 }
-function resolveSendfile(root, file) {
-    const resolved = path.resolve(root, file);
-    if (resolved !== root && !resolved.startsWith(root + path.sep))
-        return null;
-    if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile())
-        return null;
-    return resolved;
-}
-async function toEnvelopeRequest(request) {
-    const hasBody = request.method !== 'GET' && request.method !== 'HEAD';
-    const body = hasBody ? new Uint8Array(await request.buffer()) : new Uint8Array(0);
-    const xff = request.headers['x-forwarded-for'];
-    const clientIp = typeof xff === 'string' && xff.length > 0 ? xff.split(',')[0].trim() : '127.0.0.1';
-    return {
-        method: request.method,
-        path: request.url,
-        headers: Object.entries(request.headers),
-        body,
-        clientIp,
-    };
-}
-function sendWasmResponse(response, root, result) {
-    response.status(result.status);
-    let hasContentType = false;
-    for (const [name, value] of result.headers) {
-        if (name.toLowerCase() === 'content-type')
-            hasContentType = true;
-        response.header(name, value);
-    }
-    response.header('server', 'toil-dev');
-    if (result.sendfile !== null) {
-        const file = resolveSendfile(root, result.sendfile);
-        if (file === null) {
-            response.status(404).send('not found\n');
-            return;
-        }
-        if (!hasContentType) {
-            response.header('content-type', MIME[path.extname(file).toLowerCase()] ?? 'application/octet-stream');
-        }
-        response.sendFile(file);
-        return;
-    }
-    if (!hasContentType)
-        response.header('content-type', 'text/plain; charset=utf-8');
-    response.send(Buffer.from(result.body.buffer, result.body.byteOffset, result.body.length));
-}
-function sendSsr(response, out, headOnly) {
-    response.status(out.status);
-    let hasContentType = false;
-    for (const [name, value] of out.headers) {
-        if (name.toLowerCase() === 'content-type')
-            hasContentType = true;
-        response.header(name, value);
-    }
-    if (!hasContentType)
-        response.header('content-type', 'text/html; charset=utf-8');
-    response.header('server', 'toil-dev');
-    if (headOnly) {
-        response.send('');
-        return;
-    }
-    response.send(Buffer.from(out.html.buffer, out.html.byteOffset, out.html.length));
-}
 export async function startDevServer(options) {
     const host = options.host ?? '127.0.0.1';
     const root = path.resolve(options.root);
@@ -132,74 +50,45 @@ export async function startDevServer(options) {
         }
     };
     refresh();
-    const app = new Server({
-        max_body_length: options.maxBodyLength ?? DEFAULT_MAX_BODY_LENGTH,
-        max_body_buffer: 1024 * 32,
-        fast_abort: true,
-    });
-    app.set_error_handler((_request, response, error) => {
-        if (response.completed)
-            return;
-        response.atomic(() => {
-            response.status(500).send(`internal error: ${error.message}\n`);
-        });
-    });
-    wireWebsocketProxy(app, options.vite);
+    const app = new Server(runtimeServerOptions(options));
+    installRuntimeErrorHandler(app);
     const nodeMode = options.nodeMode ?? 'all';
-    let daemonHost = null;
-    let daemonTimer = null;
-    if (options.coldWasmFile !== undefined && daemonEmulationEnabled(nodeMode) && options.daemon) {
-        daemonHost = new DaemonHost(options.coldWasmFile, options.daemon, nodeMode);
-        const pollDaemon = () => {
-            try {
-                daemonHost?.refresh();
-            }
-            catch (e) {
-                process.stdout.write(pc.red(`  ✗ daemon reload failed: ${String(e)}`) + '\n');
-            }
-        };
-        pollDaemon();
-        daemonTimer = setInterval(pollDaemon, 500);
-        daemonTimer.unref?.();
+    if (options.streamWasmFile !== undefined && streamEmulationEnabled(nodeMode)) {
+        wireStreams(app, options.vite, new StreamRouter(options.streamWasmFile));
+    }
+    else {
+        wireWebsocketProxy(app, options.vite);
     }
+    const daemon = startDaemonRuntime({
+        coldWasmFile: options.coldWasmFile,
+        nodeMode,
+        daemon: options.daemon,
+    });
     app.any('/*', async (request, response) => {
         response.removeHeader('uWebSockets');
-        const dispatchable = !isViteInternal(request.url) && METHOD_CODES[request.method] !== undefined;
+        const dispatchable = !isViteInternal(request.url) && isDispatchableMethod(request.method);
         if (dispatchable)
             refresh();
         if (dispatchable && module.available) {
-            const envelopeReq = await toEnvelopeRequest(request);
-            const cacheHost = request.headers.host ?? 'dev';
-            const hasAuth = request.headers.cookie !== undefined || request.headers.authorization !== undefined;
-            const cached = lookupCache(cacheHost, request.method, request.url, envelopeReq.body);
-            if (cached !== null) {
-                sendWasmResponse(response, root, cached);
-                return;
-            }
-            try {
-                const result = module.dispatch(envelopeReq);
-                if (!result.unhandled) {
-                    const finalized = applyCacheRule(cacheHost, request.method, request.url, envelopeReq.body, hasAuth, result);
-                    sendWasmResponse(response, root, finalized);
-                    return;
-                }
-            }
-            catch (e) {
-                process.stdout.write(pc.red(`  ✗ ${request.method} ${request.path} server error: ${String(e)}`) +
-                    '\n');
-                response.status(500).send('internal error\n');
+            const dispatch = await dispatchWasmRequest({
+                module,
+                request,
+                response,
+                root,
+                cacheHost: request.headers.host ?? 'dev',
+                serverHeader: 'toil-dev',
+                errorPrefix: '✗',
+            });
+            if (dispatch.handled) {
                 return;
             }
-            if ((request.method === 'GET' || request.method === 'HEAD') &&
-                ssrRoutes.length > 0) {
+            if ((request.method === 'GET' || request.method === 'HEAD') && ssrRoutes.length > 0) {
                 const route = ssrRoutes.find((r) => r.test(pathnameOf(request.url)));
                 if (route) {
                     try {
-                        const out = route.entries.length === 0
-                            ? { status: 200, headers: [], html: route.tmpl }
-                            : assembleSsr(route, module.dispatchRender(envelopeReq));
+                        const out = assembleRouteSsr(route, module, dispatch.envelopeReq);
                         if (out !== null) {
-                            sendSsr(response, out, request.method === 'HEAD');
+                            sendSsr(response, out, request.method === 'HEAD', 'toil-dev');
                             return;
                         }
                     }
@@ -216,9 +105,7 @@ export async function startDevServer(options) {
         port: options.port,
         host,
         close: async () => {
-            if (daemonTimer !== null)
-                clearInterval(daemonTimer);
-            daemonHost?.close();
+            daemon?.close();
             await app.shutdown();
         },
     };

package/build/devserver/ssr.d.ts

@@ -6,6 +6,7 @@ export interface DevSsrTemplate {
         id: number;
         offset: number;
     }[];
+    hash?: Uint8Array;
 }
 export interface SsrRoute {
     test: (pathname: string) => boolean;
@@ -14,6 +15,7 @@ export interface SsrRoute {
         id: number;
         offset: number;
     }[];
+    hash?: Uint8Array;
 }
 export declare function pathnameOf(url: string): string;
 export declare function buildSsrRoutes(templates: readonly DevSsrTemplate[]): SsrRoute[];

package/build/devserver/ssr.js

@@ -33,7 +33,12 @@ function patternToTest(pattern) {
     return (pathname) => compiled.test(norm(pathname));
 }
 export function buildSsrRoutes(templates) {
-    return templates.map((t) => ({ test: patternToTest(t.pattern), tmpl: t.tmpl, entries: t.entries }));
+    return templates.map((t) => ({
+        test: patternToTest(t.pattern),
+        tmpl: t.tmpl,
+        entries: t.entries,
+        hash: t.hash,
+    }));
 }
 function decodeValues(buf) {
     const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
@@ -44,6 +49,7 @@ function decodeValues(buf) {
             return null;
         const status = dv.getUint16(o, true);
         o += 2;
+        const hash = buf.subarray(o, o + 32);
         o += 32;
         const nHeaders = dv.getUint16(o, true);
         o += 2;
@@ -81,12 +87,21 @@ function decodeValues(buf) {
             values.set(id, buf.subarray(o, o + len));
             o += len;
         }
-        return { status, headers, values };
+        return { status, hash, headers, values };
     }
     catch {
         return null;
     }
 }
+function sameBytes(a, b) {
+    if (a.byteLength !== b.byteLength)
+        return false;
+    for (let i = 0; i < a.byteLength; i++) {
+        if (a[i] !== b[i])
+            return false;
+    }
+    return true;
+}
 function splice(tmpl, inserts) {
     const parts = [];
     let prev = 0;
@@ -107,6 +122,8 @@ export function assembleSsr(route, envelope) {
         return null;
     if (decoded.status >= 500 || decoded.values.size === 0)
         return null;
+    if (route.hash !== undefined && !sameBytes(decoded.hash, route.hash))
+        return null;
     const inserts = route.entries
         .map((e) => ({ offset: e.offset, value: decoded.values.get(e.id) ?? new Uint8Array(0) }))
         .sort((a, b) => a.offset - b.offset);

package/build/devserver/stream/catalog.d.ts

@@ -0,0 +1,20 @@
+export interface StreamDef {
+    readonly name: string;
+    readonly route: string;
+    readonly hooks: {
+        readonly connect: boolean;
+        readonly message: boolean;
+        readonly close: boolean;
+        readonly disconnect: boolean;
+    };
+    readonly scope: 'regional' | 'continental';
+    readonly messageMode: 'raw' | 'data';
+    readonly maxFrameBytes: number;
+    readonly ingressRingBytes: number;
+    readonly messageValueDataId: number;
+    readonly messageSchemaVersion: number;
+    readonly streamIndex: number;
+}
+export type StreamCatalog = Map<string, StreamDef>;
+export declare function parseStreamCatalog(wasm: Buffer): StreamCatalog;
+export declare function matchStreamRoute(catalog: StreamCatalog, path: string): StreamDef | null;

package/build/devserver/stream/catalog.js

@@ -0,0 +1,54 @@
+import { DataReader } from 'toiljs/io';
+import { customSection } from '../wasm/sections.js';
+export function parseStreamCatalog(wasm) {
+    const out = new Map();
+    let sec;
+    try {
+        sec = customSection(wasm, 'toilstream.catalog');
+    }
+    catch {
+        return out;
+    }
+    if (sec === null)
+        return out;
+    const r = new DataReader(sec);
+    r.readU16();
+    const n = r.readU16();
+    for (let i = 0; i < n && r.ok; i++) {
+        const name = r.readString();
+        const route = r.readString();
+        const bits = r.readU8();
+        const scope = r.readU8() === 1 ? 'continental' : 'regional';
+        const messageMode = r.readU8() === 1 ? 'data' : 'raw';
+        const maxFrameBytes = r.readU32();
+        const ingressRingBytes = r.readU32();
+        const messageValueDataId = r.readU32();
+        const messageSchemaVersion = r.readU32();
+        const streamIndex = r.readU16();
+        if (!r.ok)
+            break;
+        out.set(route, {
+            name,
+            route,
+            hooks: {
+                connect: (bits & 1) !== 0,
+                message: (bits & 2) !== 0,
+                close: (bits & 4) !== 0,
+                disconnect: (bits & 8) !== 0,
+            },
+            scope,
+            messageMode,
+            maxFrameBytes,
+            ingressRingBytes,
+            messageValueDataId,
+            messageSchemaVersion,
+            streamIndex,
+        });
+    }
+    return out;
+}
+export function matchStreamRoute(catalog, path) {
+    const q = path.indexOf('?');
+    const exact = q >= 0 ? path.slice(0, q) : path;
+    return catalog.get(exact) ?? null;
+}

package/build/devserver/stream/host.d.ts

@@ -0,0 +1,9 @@
+import { type DbDevState } from '../db/index.js';
+import { type CryptoState } from '../runtime/crypto.js';
+import { type MemoryRef } from '../runtime/host.js';
+export interface StreamBoxState {
+    crypto: CryptoState;
+    db: DbDevState;
+}
+export declare function freshStreamBoxState(): StreamBoxState;
+export declare function buildStreamImports(ref: MemoryRef, state: StreamBoxState): WebAssembly.Imports;

package/build/devserver/stream/host.js

@@ -0,0 +1,15 @@
+import { buildDatabaseImports, freshDbState } from '../db/index.js';
+import { buildCryptoImports, freshCryptoState } from '../runtime/crypto.js';
+import { buildEnvImports } from '../runtime/host.js';
+export function freshStreamBoxState() {
+    return { crypto: freshCryptoState(), db: freshDbState() };
+}
+export function buildStreamImports(ref, state) {
+    return {
+        env: {
+            ...buildEnvImports(ref, state),
+            ...buildCryptoImports(ref, state.crypto),
+            ...buildDatabaseImports(ref, state.db),
+        },
+    };
+}

package/build/devserver/stream/index.d.ts

@@ -0,0 +1,37 @@
+export declare function decodeRejectCode(rc: bigint): number;
+export type StreamMessageOutcome = {
+    readonly kind: 'reply';
+    readonly frames: Buffer[];
+} | {
+    readonly kind: 'reject';
+    readonly code: number;
+};
+export type StreamConnectOutcome = {
+    readonly kind: 'accept';
+} | {
+    readonly kind: 'reject';
+    readonly code: number;
+};
+export declare class DevStreamBox {
+    private readonly exports;
+    private readonly _state;
+    private readonly rings;
+    private readonly streamInfo;
+    private constructor();
+    static load(wasm: Buffer): DevStreamBox;
+    get hasRings(): boolean;
+    get hasConnectBridge(): boolean;
+    onConnect(streamId: bigint, authority: string, path: string): StreamConnectOutcome;
+    onClose(streamId: bigint): bigint;
+    onDisconnect(streamId: bigint): bigint;
+    onMessage(streamId: bigint, inbound: Buffer): StreamMessageOutcome;
+    private dispatch;
+    private static resolveRings;
+    private static resolveStreamInfo;
+    private stampRings;
+    private stampOne;
+    private writeConnectInfo;
+    private resetEgressRing;
+    private ingressWrite;
+    private egressDrain;
+}