toiljs 0.0.67 → 0.0.68

package/src/devserver/db/derives.ts

@@ -0,0 +1,121 @@
+import { customSection } from '../wasm/sections.js';
+
+/**
+ * Parses the `toildb.derives` wiring section emitted by toilscript for a
+ * `@database` class with `@derive` materializer methods (see the compiler's
+ * `buildToilDbDerives`). It maps each derive to its owning `@database` class, so
+ * the dev runtime (`runtime/module.ts`) can, after a dispatch writes a source
+ * collection, re-run that database's derives under FunctionKind=Derive. Fails
+ * closed: any malformed byte yields `[]` (no derive runs) rather than throwing.
+ *
+ * Section layout (LE), mirroring `buildToilDbDerives`:
+ *   u16 format_version = 1
+ *   u16 n_derives
+ *   per derive: u16 derive_id, str db_name, str method_name   (str = u32 len + bytes)
+ */
+export interface DeriveEntry {
+    readonly deriveId: number;
+    readonly dbName: string;
+    readonly methodName: string;
+}
+
+const SECTION = 'toildb.derives';
+const VERSION = 1;
+const MAX_SECTION_BYTES = 128 * 1024;
+const MAX_DERIVES = 1024;
+const MAX_NAME_BYTES = 1024;
+const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
+
+export function parseDerives(wasm: Buffer): readonly DeriveEntry[] {
+    let section: Buffer | null;
+    try {
+        section = customSection(wasm, SECTION);
+    } catch {
+        return [];
+    }
+    if (section === null) return [];
+    if (section.length > MAX_SECTION_BYTES) return [];
+
+    const r = new Reader(section);
+    const version = r.u16();
+    if (!r.ok || version !== VERSION) return [];
+    const count = r.u16();
+    if (!r.ok || count > MAX_DERIVES) return [];
+
+    const derives: DeriveEntry[] = [];
+    for (let i = 0; i < count && r.ok; i++) {
+        const deriveId = r.u16();
+        const dbName = r.string();
+        const methodName = r.string();
+        if (!r.ok || dbName.length === 0) return [];
+        derives.push({ deriveId, dbName, methodName });
+    }
+    if (!r.ok || r.remaining() !== 0) return [];
+    return derives;
+}
+
+/**
+ * The derives whose owning `@database` had at least one source collection
+ * written during this dispatch. `written` holds "Db/coll" store keys; the
+ * database is the prefix before the first `/`. Each affected derive appears at
+ * most once (coalescing: many writes to one database run its derives once).
+ */
+export function derivesForWrites(
+    derives: readonly DeriveEntry[],
+    written: ReadonlySet<string>,
+): readonly DeriveEntry[] {
+    if (derives.length === 0 || written.size === 0) return [];
+    const dbs = new Set<string>();
+    for (const key of written) {
+        const slash = key.indexOf('/');
+        dbs.add(slash >= 0 ? key.slice(0, slash) : key);
+    }
+    return derives.filter((d) => dbs.has(d.dbName));
+}
+
+class Reader {
+    private pos = 0;
+    ok = true;
+
+    constructor(private readonly bytes: Buffer) {}
+
+    remaining(): number {
+        return this.bytes.length - this.pos;
+    }
+
+    u16(): number {
+        if (!this.ok || this.pos + 2 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt16LE(this.pos);
+        this.pos += 2;
+        return out;
+    }
+
+    u32(): number {
+        if (!this.ok || this.pos + 4 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt32LE(this.pos);
+        this.pos += 4;
+        return out;
+    }
+
+    string(): string {
+        const len = this.u32();
+        if (!this.ok || len > MAX_NAME_BYTES || this.pos + len > this.bytes.length) {
+            this.ok = false;
+            return '';
+        }
+        try {
+            const out = UTF8_DECODER.decode(this.bytes.subarray(this.pos, this.pos + len));
+            this.pos += len;
+            return out;
+        } catch {
+            this.ok = false;
+            return '';
+        }
+    }
+}

package/src/devserver/db/index.ts

@@ -15,4 +15,5 @@ export {
     setDbCatalog,
 } from './database.js';
 export { parseCatalog } from './catalog.js';
+export { type DeriveEntry, derivesForWrites, parseDerives } from './derives.js';
 export { CollectionFamily, DbFunctionKind, type DbDevState, freshDbState } from './types.js';

package/src/devserver/db/types.ts

@@ -50,6 +50,11 @@ export interface DbDevState {
     lastResult: Buffer | null;
     lastResultVersion: number;
     functionKind: DbFunctionKind;
+    /** Names ("Db/coll") of source collections written during this dispatch, so
+     *  the runtime can re-run the affected `@derive` materializers afterward.
+     *  Only populated for non-Derive dispatches (a derive's own writes must not
+     *  re-trigger it - see `database.ts` `recordWrite`). */
+    writtenCollections: Set<string>;
 }
 
 export function freshDbState(): DbDevState {
@@ -58,6 +63,7 @@ export function freshDbState(): DbDevState {
         lastResult: null,
         lastResultVersion: -1,
         functionKind: DbFunctionKind.Job,
+        writtenCollections: new Set<string>(),
     };
 }
 

package/src/devserver/http/proxy.ts

@@ -114,45 +114,59 @@ export function wireWebsocketProxy(app: Server, target: ViteTarget): void {
         '/*',
         { message_type: 'Buffer', idle_timeout: 120, max_payload_length: 16 * 1024 * 1024 },
         (ws: Websocket) => {
-            const { url, protocol } = ws.context as { url: string; protocol: string };
-            const upstream = new WebSocket(
-                `ws://${target.host}:${String(target.port)}${url}`,
-                protocol ? protocol.split(',').map((p) => p.trim()) : [],
-            );
-            upstream.binaryType = 'arraybuffer';
-
-            const pending: (string | Uint8Array<ArrayBuffer>)[] = [];
-            let open = false;
-
-            upstream.onopen = (): void => {
-                open = true;
-                for (const m of pending) upstream.send(m);
-                pending.length = 0;
-            };
-            upstream.onmessage = (event: MessageEvent): void => {
-                if (typeof event.data === 'string') ws.send(event.data);
-                else ws.send(Buffer.from(event.data as ArrayBuffer), true);
-            };
-            upstream.onclose = (event: CloseEvent): void => {
-                ws.close(event.code, event.reason);
-            };
-            upstream.onerror = (): void => {
-                ws.close();
-            };
-
-            ws.on('message', (message: Buffer, isBinary: boolean) => {
-                const m = toUpstreamMessage(message, isBinary);
-                if (open) upstream.send(m);
-                else pending.push(m);
-            });
-            ws.on('close', () => {
-                if (
-                    upstream.readyState === WebSocket.OPEN ||
-                    upstream.readyState === WebSocket.CONNECTING
-                ) {
-                    upstream.close();
-                }
-            });
+            pipeToVite(ws, target, ws.context as { url: string; protocol: string });
         },
     );
 }
+
+/**
+ * Pipe ONE upgraded websocket to the internal Vite HMR server. The verbatim body extracted from
+ * {@link wireWebsocketProxy} so the dev STREAM router (doc 08 4.1 `wireStreams`) can reuse it for every
+ * NON-stream upgrade while it handles `@stream`-route upgrades itself - HMR stays byte-for-byte
+ * unchanged. `ctx` is the upgrade context (`{ url, protocol }`) the upgrade handler stamped.
+ */
+export function pipeToVite(
+    ws: Websocket,
+    target: ViteTarget,
+    ctx: { url: string; protocol: string },
+): void {
+    const { url, protocol } = ctx;
+    const upstream = new WebSocket(
+        `ws://${target.host}:${String(target.port)}${url}`,
+        protocol ? protocol.split(',').map((p) => p.trim()) : [],
+    );
+    upstream.binaryType = 'arraybuffer';
+
+    const pending: (string | Uint8Array<ArrayBuffer>)[] = [];
+    let open = false;
+
+    upstream.onopen = (): void => {
+        open = true;
+        for (const m of pending) upstream.send(m);
+        pending.length = 0;
+    };
+    upstream.onmessage = (event: MessageEvent): void => {
+        if (typeof event.data === 'string') ws.send(event.data);
+        else ws.send(Buffer.from(event.data as ArrayBuffer), true);
+    };
+    upstream.onclose = (event: CloseEvent): void => {
+        ws.close(event.code, event.reason);
+    };
+    upstream.onerror = (): void => {
+        ws.close();
+    };
+
+    ws.on('message', (message: Buffer, isBinary: boolean) => {
+        const m = toUpstreamMessage(message, isBinary);
+        if (open) upstream.send(m);
+        else pending.push(m);
+    });
+    ws.on('close', () => {
+        if (
+            upstream.readyState === WebSocket.OPEN ||
+            upstream.readyState === WebSocket.CONNECTING
+        ) {
+            upstream.close();
+        }
+    });
+}

package/src/devserver/http/runtime.ts

@@ -0,0 +1,287 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { type Request, type Response, type Server } from '@dacely/hyper-express';
+import pc from 'picocolors';
+
+import { type EnvelopeRequest, METHOD_CODES } from './envelope.js';
+import { applyCacheRule, lookupCache, type CacheableResult } from './cache.js';
+import { type WasmDispatchResult, WasmServerModule } from '../runtime/module.js';
+import { assembleSsr, type SsrResult, type SsrRoute } from '../ssr.js';
+
+export const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
+export const MAX_BODY_BUFFER = 1024 * 32;
+export const HTTP_IDLE_TIMEOUT = 60;
+export const HTTP_RESPONSE_TIMEOUT = 120;
+
+export const MIME: Readonly<Record<string, string>> = {
+    '.html': 'text/html; charset=utf-8',
+    '.js': 'text/javascript; charset=utf-8',
+    '.mjs': 'text/javascript; charset=utf-8',
+    '.css': 'text/css; charset=utf-8',
+    '.json': 'application/json; charset=utf-8',
+    '.txt': 'text/plain; charset=utf-8',
+    '.svg': 'image/svg+xml',
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.webp': 'image/webp',
+    '.avif': 'image/avif',
+    '.gif': 'image/gif',
+    '.ico': 'image/x-icon',
+    '.wasm': 'application/wasm',
+    '.woff2': 'font/woff2',
+};
+
+export interface RuntimeServerOptions {
+    readonly maxBodyLength?: number;
+}
+
+export function runtimeServerOptions(options: RuntimeServerOptions): {
+    max_body_length: number;
+    max_body_buffer: number;
+    fast_abort: true;
+    idle_timeout: number;
+    response_timeout: number;
+} {
+    return {
+        max_body_length: options.maxBodyLength ?? DEFAULT_MAX_BODY_LENGTH,
+        max_body_buffer: MAX_BODY_BUFFER,
+        fast_abort: true,
+        idle_timeout: HTTP_IDLE_TIMEOUT,
+        response_timeout: HTTP_RESPONSE_TIMEOUT,
+    };
+}
+
+export function installRuntimeErrorHandler(app: Server): void {
+    app.set_error_handler((_request: Request, response: Response, error: Error) => {
+        if (response.completed) return;
+        response.atomic(() => {
+            response.status(500).send(`internal error: ${error.message}\n`);
+        });
+    });
+}
+
+export function isDispatchableMethod(method: string): boolean {
+    return METHOD_CODES[method] !== undefined;
+}
+
+export function resolveFileInside(root: string, file: string): string | null {
+    const resolved = path.resolve(root, file);
+    if (resolved !== root && !resolved.startsWith(root + path.sep)) return null;
+    if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) return null;
+    return resolved;
+}
+
+export function resolveStaticFile(root: string, requestPath: string): string | null {
+    let decoded: string;
+    try {
+        decoded = decodeURIComponent(requestPath);
+    } catch {
+        return null;
+    }
+    if (decoded === '/' || decoded === '') return null;
+    return resolveFileInside(root, decoded.replace(/^\/+/, ''));
+}
+
+export interface PreparedHttpResponse {
+    readonly status: number;
+    readonly headers: readonly (readonly [string, string])[];
+    readonly body: Uint8Array;
+    readonly sendfile: string | null;
+}
+
+export async function toEnvelopeRequest(request: Request): Promise<EnvelopeRequest> {
+    const hasBody = request.method !== 'GET' && request.method !== 'HEAD';
+    const body = hasBody ? new Uint8Array(await request.buffer()) : new Uint8Array(0);
+    const xff = request.headers['x-forwarded-for'];
+    const clientIp =
+        typeof xff === 'string' && xff.length > 0 ? xff.split(',')[0]!.trim() : '127.0.0.1';
+    return {
+        method: request.method,
+        path: request.url,
+        headers: Object.entries(request.headers),
+        body,
+        clientIp,
+    };
+}
+
+export function prepareWasmResponse(
+    root: string,
+    result: Pick<CacheableResult, 'status' | 'headers' | 'body' | 'sendfile'>,
+    serverHeader: string,
+): PreparedHttpResponse {
+    const headers: (readonly [string, string])[] = [];
+    let hasContentType = false;
+    for (const [name, value] of result.headers) {
+        if (name.toLowerCase() === 'content-type') hasContentType = true;
+        headers.push([name, value]);
+    }
+    headers.push(['server', serverHeader]);
+
+    if (result.sendfile !== null) {
+        const file = resolveFileInside(root, result.sendfile);
+        if (file === null) {
+            return {
+                status: 404,
+                headers: [
+                    ['server', serverHeader],
+                    ['content-type', 'text/plain; charset=utf-8'],
+                ],
+                body: new TextEncoder().encode('not found\n'),
+                sendfile: null,
+            };
+        }
+        if (!hasContentType) {
+            headers.push([
+                'content-type',
+                MIME[path.extname(file).toLowerCase()] ?? 'application/octet-stream',
+            ]);
+        }
+        return { status: result.status, headers, body: new Uint8Array(0), sendfile: file };
+    }
+
+    if (!hasContentType) headers.push(['content-type', 'text/plain; charset=utf-8']);
+    return { status: result.status, headers, body: result.body, sendfile: null };
+}
+
+export function sendPreparedResponse(response: Response, out: PreparedHttpResponse): void {
+    response.status(out.status);
+    for (const [name, value] of out.headers) response.header(name, value);
+    if (out.sendfile !== null) {
+        response.sendFile(out.sendfile);
+        return;
+    }
+    response.send(Buffer.from(out.body.buffer, out.body.byteOffset, out.body.length));
+}
+
+export function sendWasmResponse(
+    response: Response,
+    root: string,
+    result: Pick<CacheableResult, 'status' | 'headers' | 'body' | 'sendfile'>,
+    serverHeader: string,
+): void {
+    sendPreparedResponse(response, prepareWasmResponse(root, result, serverHeader));
+}
+
+export function prepareSsrResponse(
+    out: SsrResult,
+    headOnly: boolean,
+    serverHeader: string,
+): PreparedHttpResponse {
+    const headers: (readonly [string, string])[] = [];
+    let hasContentType = false;
+    for (const [name, value] of out.headers) {
+        if (name.toLowerCase() === 'content-type') hasContentType = true;
+        headers.push([name, value]);
+    }
+    if (!hasContentType) headers.push(['content-type', 'text/html; charset=utf-8']);
+    headers.push(['server', serverHeader]);
+    return {
+        status: out.status,
+        headers,
+        body: headOnly ? new Uint8Array(0) : out.html,
+        sendfile: null,
+    };
+}
+
+export function sendSsr(
+    response: Response,
+    out: SsrResult,
+    headOnly: boolean,
+    serverHeader: string,
+): void {
+    sendPreparedResponse(response, prepareSsrResponse(out, headOnly, serverHeader));
+}
+
+export interface WasmDispatchOutcome {
+    readonly envelopeReq: EnvelopeRequest;
+    readonly handled: boolean;
+}
+
+export interface EnvelopeDispatchOutcome {
+    readonly result: CacheableResult | null;
+    readonly handled: boolean;
+}
+
+export function dispatchEnvelopeRequest(options: {
+    readonly module: WasmServerModule;
+    readonly envelopeReq: EnvelopeRequest;
+    readonly method: string;
+    readonly url: string;
+    readonly cacheHost: string;
+    readonly hasAuth: boolean;
+}): EnvelopeDispatchOutcome {
+    const cached = lookupCache(
+        options.cacheHost,
+        options.method,
+        options.url,
+        options.envelopeReq.body,
+    );
+    if (cached !== null) return { result: cached, handled: true };
+
+    const result = options.module.dispatch(options.envelopeReq);
+    if (result.unhandled) return { result: null, handled: false };
+
+    return {
+        result: applyCacheRule(
+            options.cacheHost,
+            options.method,
+            options.url,
+            options.envelopeReq.body,
+            options.hasAuth,
+            result,
+        ),
+        handled: true,
+    };
+}
+
+export async function dispatchWasmRequest(options: {
+    readonly module: WasmServerModule;
+    readonly request: Request;
+    readonly response: Response;
+    readonly root: string;
+    readonly cacheHost: string;
+    readonly serverHeader: string;
+    readonly errorPrefix: string;
+}): Promise<WasmDispatchOutcome> {
+    const envelopeReq = await toEnvelopeRequest(options.request);
+    const hasAuth =
+        options.request.headers.cookie !== undefined ||
+        options.request.headers.authorization !== undefined;
+
+    try {
+        const dispatch = dispatchEnvelopeRequest({
+            module: options.module,
+            envelopeReq,
+            method: options.request.method,
+            url: options.request.url,
+            cacheHost: options.cacheHost,
+            hasAuth,
+        });
+        if (dispatch.result !== null) {
+            sendWasmResponse(options.response, options.root, dispatch.result, options.serverHeader);
+            return { envelopeReq, handled: true };
+        }
+    } catch (e) {
+        process.stdout.write(
+            pc.red(
+                `  ${options.errorPrefix} ${options.request.method} ${options.request.path} server error: ${String(e)}`,
+            ) + '\n',
+        );
+        options.response.status(500).send('internal error\n');
+        return { envelopeReq, handled: true };
+    }
+
+    return { envelopeReq, handled: false };
+}
+
+export function assembleRouteSsr(
+    route: SsrRoute,
+    module: WasmServerModule | null,
+    envelopeReq: EnvelopeRequest,
+): SsrResult | null {
+    if (route.entries.length === 0) return { status: 200, headers: [], html: route.tmpl };
+    if (module === null || !module.available) return null;
+    return assembleSsr(route, module.dispatchRender(envelopeReq));
+}

package/src/devserver/index.ts

@@ -15,6 +15,8 @@
 
 export { startDevServer } from './server.js';
 export type { DevServerOptions, RunningDevServer } from './server.js';
+export { loadBuiltSsrTemplates, startBuiltServer } from './production.js';
+export type { BuiltServerOptions, RunningBuiltServer } from './production.js';
 
 export {
     METHOD_CODES,

package/src/devserver/production-ipc.ts

@@ -0,0 +1,63 @@
+export interface ThreadedRequest {
+    readonly id: number;
+    readonly method: string;
+    readonly url: string;
+    readonly path: string;
+    readonly headers: readonly (readonly [string, string])[];
+    readonly body: string;
+    readonly clientIp: string;
+}
+
+export interface ThreadedHttpResponse {
+    readonly kind: 'response';
+    readonly status: number;
+    readonly headers: readonly (readonly [string, string])[];
+    readonly body: string;
+    readonly sendfile: string | null;
+}
+
+export interface ThreadedFallbackResponse {
+    readonly kind: 'fallback';
+}
+
+export type ThreadedReply = ThreadedHttpResponse | ThreadedFallbackResponse;
+
+export type WorkerToPrimaryMessage =
+    | { readonly toil: 'ready'; readonly port: number; readonly workerId: number }
+    | { readonly toil: 'clientCount'; readonly count: number; readonly workerId: number }
+    | { readonly toil: 'request'; readonly request: ThreadedRequest };
+
+export type PrimaryToWorkerMessage =
+    | {
+          readonly toil: 'start';
+          readonly workerId: number;
+          readonly options: unknown;
+      }
+    | { readonly toil: 'reply'; readonly id: number; readonly reply: ThreadedReply }
+    | { readonly toil: 'broadcast'; readonly message: string }
+    | { readonly toil: 'shutdown' };
+
+export function encodeBody(body: Uint8Array): string {
+    return Buffer.from(body.buffer, body.byteOffset, body.length).toString('base64');
+}
+
+export function decodeBody(body: string): Uint8Array {
+    return Buffer.from(body, 'base64');
+}
+
+export function isWorkerToPrimaryMessage(value: unknown): value is WorkerToPrimaryMessage {
+    if (typeof value !== 'object' || value === null) return false;
+    const message = value as Record<string, unknown>;
+    return message.toil === 'ready' || message.toil === 'clientCount' || message.toil === 'request';
+}
+
+export function isPrimaryToWorkerMessage(value: unknown): value is PrimaryToWorkerMessage {
+    if (typeof value !== 'object' || value === null) return false;
+    const message = value as Record<string, unknown>;
+    return (
+        message.toil === 'start' ||
+        message.toil === 'reply' ||
+        message.toil === 'broadcast' ||
+        message.toil === 'shutdown'
+    );
+}

package/src/devserver/production-worker.ts

@@ -0,0 +1,83 @@
+import type { BuiltServerOptions, RunningBuiltServer } from './production.js';
+import { startBuiltServerWorker } from './production.js';
+import {
+    isPrimaryToWorkerMessage,
+    type PrimaryToWorkerMessage,
+    type ThreadedReply,
+    type ThreadedRequest,
+} from './production-ipc.js';
+
+let running: RunningBuiltServer | null = null;
+let workerId = 0;
+const pending = new Map<number, (reply: ThreadedReply) => void>();
+
+function send(message: object): void {
+    try {
+        process.send?.(message);
+    } catch {
+        // The parent is gone; normal process shutdown will follow.
+    }
+}
+
+function requestPrimary(request: ThreadedRequest): Promise<ThreadedReply> {
+    const id = request.id;
+    return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+            pending.delete(id);
+            reject(new Error('primary request timed out'));
+        }, 120_000);
+        timeout.unref();
+
+        pending.set(id, (reply) => {
+            clearTimeout(timeout);
+            resolve(reply);
+        });
+        send({ toil: 'request', request });
+    });
+}
+
+async function start(message: Extract<PrimaryToWorkerMessage, { toil: 'start' }>): Promise<void> {
+    if (running !== null) return;
+    workerId = message.workerId;
+    const options = message.options as BuiltServerOptions;
+    running = await startBuiltServerWorker(options, {
+        request: requestPrimary,
+        clientCount: (count) => send({ toil: 'clientCount', workerId, count }),
+    });
+    send({ toil: 'ready', workerId, port: running.port });
+}
+
+async function shutdown(): Promise<void> {
+    const server = running;
+    running = null;
+    if (server !== null) await server.close();
+}
+
+process.on('message', (value: unknown) => {
+    if (!isPrimaryToWorkerMessage(value)) return;
+    switch (value.toil) {
+        case 'start':
+            void start(value).catch((e: unknown) => {
+                process.stderr.write(`toiljs production worker failed: ${String(e)}\n`);
+                process.exit(1);
+            });
+            return;
+        case 'reply': {
+            const resolve = pending.get(value.id);
+            if (resolve === undefined) return;
+            pending.delete(value.id);
+            resolve(value.reply);
+            return;
+        }
+        case 'broadcast':
+            running?.broadcast(value.message);
+            return;
+        case 'shutdown':
+            void shutdown().finally(() => process.exit(0));
+            return;
+    }
+});
+
+process.once('disconnect', () => {
+    void shutdown().finally(() => process.exit(0));
+});