toiljs 0.0.66 → 0.0.68

package/build/devserver/production.d.ts

@@ -0,0 +1,35 @@
+import type { EmailBackendConfig } from 'toiljs/shared';
+import type { ResolvedDaemonConfig } from './daemon/host.js';
+import { type ThreadedReply, type ThreadedRequest } from './production-ipc.js';
+import { type DevSsrTemplate } from './ssr.js';
+export interface BuiltServerOptions {
+    readonly root: string;
+    readonly staticRoot: string;
+    readonly wasmFile?: string;
+    readonly coldWasmFile?: string;
+    readonly nodeMode?: string;
+    readonly daemon?: ResolvedDaemonConfig;
+    readonly port?: number;
+    readonly host?: string;
+    readonly allowedOrigins?: readonly string[];
+    readonly wsPath?: string;
+    readonly cors?: boolean;
+    readonly maxBodyLength?: number;
+    readonly email?: EmailBackendConfig;
+    readonly threads?: number | 'auto';
+}
+export interface RunningBuiltServer {
+    readonly port: number;
+    readonly host: string;
+    readonly wsPath: string;
+    broadcast(message: string): void;
+    clientCount(): number;
+    close(): Promise<void>;
+}
+export declare function loadBuiltSsrTemplates(staticRoot: string): DevSsrTemplate[];
+export interface BuiltServerWorkerController {
+    request(request: ThreadedRequest): Promise<ThreadedReply>;
+    clientCount(count: number): void;
+}
+export declare function startBuiltServerWorker(options: BuiltServerOptions, controller: BuiltServerWorkerController): Promise<RunningBuiltServer>;
+export declare function startBuiltServer(options: BuiltServerOptions): Promise<RunningBuiltServer>;

package/build/devserver/production.js

@@ -0,0 +1,502 @@
+import { fork } from 'node:child_process';
+import fs from 'node:fs';
+import { availableParallelism } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { Server, } from '@dacely/hyper-express';
+import pc from 'picocolors';
+import { startDaemonRuntime } from './daemon/runtime.js';
+import { configureDbPersistence } from './db/index.js';
+import { initEmailService } from './email/index.js';
+import { assembleRouteSsr, dispatchEnvelopeRequest, installRuntimeErrorHandler, isDispatchableMethod, prepareSsrResponse, prepareWasmResponse, resolveStaticFile, runtimeServerOptions, toEnvelopeRequest, } from './http/runtime.js';
+import { decodeBody, encodeBody, isWorkerToPrimaryMessage, } from './production-ipc.js';
+import { WasmServerModule } from './runtime/module.js';
+import { buildSsrRoutes, pathnameOf } from './ssr.js';
+const WS_MAX_PAYLOAD_LENGTH = 1024 * 1024;
+const WS_IDLE_TIMEOUT = 120;
+const WS_MAX_BACKPRESSURE = 1024 * 1024 * 2;
+const CORS_METHODS = 'GET, POST, OPTIONS, PUT, PATCH, DELETE';
+const CORS_HEADERS = 'X-Requested-With, content-type';
+function resolveBuiltServerPaths(options) {
+    const port = options.port ?? 3000;
+    const host = options.host ?? '127.0.0.1';
+    const wsPath = options.wsPath ?? '/_toil';
+    const staticRoot = path.resolve(options.staticRoot);
+    const indexHtml = path.join(staticRoot, 'index.html');
+    if (!fs.existsSync(indexHtml)) {
+        throw new Error(`No build found in ${staticRoot}. Run \`toiljs build\` first.`);
+    }
+    return {
+        port,
+        host,
+        wsPath,
+        cors: options.cors ?? true,
+        projectRoot: path.resolve(options.root),
+        staticRoot,
+        indexHtml,
+    };
+}
+function resolveThreadCount(threads) {
+    const raw = process.env.TOILJS_THREADS ?? threads;
+    if (raw === undefined || raw === 'auto')
+        return Math.max(1, availableParallelism());
+    const n = typeof raw === 'number' ? raw : Number.parseInt(raw, 10);
+    if (!Number.isFinite(n))
+        return Math.max(1, availableParallelism());
+    return Math.max(1, Math.min(128, Math.floor(n)));
+}
+function headerValue(headers, name) {
+    const lower = name.toLowerCase();
+    return headers.find(([k]) => k.toLowerCase() === lower)?.[1];
+}
+function textReply(status, body) {
+    return {
+        kind: 'response',
+        status,
+        headers: [['content-type', 'text/plain; charset=utf-8']],
+        body: encodeBody(new TextEncoder().encode(body)),
+        sendfile: null,
+    };
+}
+function preparedReply(out) {
+    return {
+        kind: 'response',
+        status: out.status,
+        headers: out.headers,
+        body: encodeBody(out.body),
+        sendfile: out.sendfile,
+    };
+}
+function sendThreadedReply(response, reply) {
+    if (reply.kind === 'fallback')
+        return false;
+    response.status(reply.status);
+    for (const [name, value] of reply.headers)
+        response.header(name, value);
+    if (reply.sendfile !== null) {
+        response.sendFile(reply.sendfile);
+        return true;
+    }
+    const body = decodeBody(reply.body);
+    response.send(Buffer.from(body.buffer, body.byteOffset, body.length));
+    return true;
+}
+function isWsOriginAllowed(origin, hostHeader, allowed) {
+    if (!origin)
+        return true;
+    if (allowed?.includes(origin))
+        return true;
+    try {
+        return new URL(origin).host === hostHeader;
+    }
+    catch {
+        return false;
+    }
+}
+function parseSlotsManifest(slotsFile) {
+    const bin = fs.readFileSync(slotsFile);
+    if (bin.length < 46)
+        return null;
+    if (bin.subarray(0, 4).toString('ascii') !== 'TSLT')
+        return null;
+    const n = bin.readUInt16LE(44);
+    if (bin.length < 46 + n * 8)
+        return null;
+    const entries = [];
+    let o = 46;
+    for (let i = 0; i < n; i++) {
+        entries.push({ offset: bin.readUInt32LE(o), id: bin.readUInt16LE(o + 4) });
+        o += 8;
+    }
+    return { entries, hash: Buffer.from(bin.subarray(12, 44)) };
+}
+function isTemplateIndexEntry(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const v = value;
+    return typeof v.route === 'string' && typeof v.name === 'string';
+}
+export function loadBuiltSsrTemplates(staticRoot) {
+    const ssrDir = path.join(staticRoot, '_ssr');
+    const indexFile = path.join(ssrDir, 'templates.json');
+    if (!fs.existsSync(indexFile))
+        return [];
+    let index;
+    try {
+        index = JSON.parse(fs.readFileSync(indexFile, 'utf8'));
+    }
+    catch {
+        return [];
+    }
+    if (!Array.isArray(index))
+        return [];
+    const out = [];
+    for (const item of index) {
+        if (!isTemplateIndexEntry(item))
+            continue;
+        if (!/^[A-Za-z0-9_]+$/.test(item.name))
+            continue;
+        const tmplFile = path.join(ssrDir, `${item.name}.tmpl`);
+        const slotsFile = path.join(ssrDir, `${item.name}.slots`);
+        if (!fs.existsSync(tmplFile) || !fs.existsSync(slotsFile))
+            continue;
+        const parsed = parseSlotsManifest(slotsFile);
+        if (parsed === null)
+            continue;
+        out.push({
+            pattern: item.route,
+            name: item.name,
+            tmpl: fs.readFileSync(tmplFile),
+            entries: parsed.entries,
+            hash: parsed.hash,
+        });
+    }
+    return out;
+}
+function createBuiltRuntime(options, paths) {
+    const emailInit = initEmailService(paths.projectRoot, options.email);
+    if (emailInit.service !== null) {
+        process.stdout.write(pc.dim(`  email enabled: ${emailInit.note}`) + '\n');
+    }
+    else if (emailInit.note !== null) {
+        process.stdout.write(pc.yellow('  ! ') + pc.dim(`email off: ${emailInit.note}`) + '\n');
+    }
+    const module = options.wasmFile !== undefined && fs.existsSync(options.wasmFile)
+        ? new WasmServerModule(options.wasmFile)
+        : null;
+    if (module !== null) {
+        try {
+            module.refresh();
+        }
+        catch (e) {
+            process.stdout.write(pc.red(`  x server wasm failed to load: ${String(e)}`) + '\n');
+        }
+    }
+    const ssrRoutes = buildSsrRoutes(loadBuiltSsrTemplates(paths.staticRoot));
+    if (ssrRoutes.length > 0) {
+        process.stdout.write(pc.dim(`  edge SSR: ${String(ssrRoutes.length)} route(s) served server-side`) + '\n');
+    }
+    configureDbPersistence(path.join(paths.projectRoot, '.toil', 'devdata.json'));
+    return { projectRoot: paths.projectRoot, module, ssrRoutes };
+}
+function startBuiltDaemon(options, paths) {
+    configureDbPersistence(path.join(paths.projectRoot, '.toil', 'devdata.json'));
+    return startDaemonRuntime({
+        coldWasmFile: options.coldWasmFile,
+        nodeMode: options.nodeMode,
+        daemon: options.daemon,
+    });
+}
+async function handleBuiltRuntimeRequest(runtime, request) {
+    const envelopeReq = {
+        method: request.method,
+        path: request.url,
+        headers: request.headers,
+        body: decodeBody(request.body),
+        clientIp: request.clientIp,
+    };
+    const dispatchable = isDispatchableMethod(request.method);
+    if (dispatchable && runtime.module !== null && runtime.module.available) {
+        const hasAuth = headerValue(request.headers, 'cookie') !== undefined ||
+            headerValue(request.headers, 'authorization') !== undefined;
+        try {
+            const dispatch = dispatchEnvelopeRequest({
+                module: runtime.module,
+                envelopeReq,
+                method: request.method,
+                url: request.url,
+                cacheHost: headerValue(request.headers, 'host') ?? 'self-host',
+                hasAuth,
+            });
+            if (dispatch.result !== null) {
+                return preparedReply(prepareWasmResponse(runtime.projectRoot, dispatch.result, 'toil'));
+            }
+        }
+        catch (e) {
+            process.stdout.write(pc.red(`  x ${request.method} ${request.path} server error: ${String(e)}`) + '\n');
+            return textReply(500, 'internal error\n');
+        }
+    }
+    if (request.method === 'GET' || request.method === 'HEAD') {
+        const route = runtime.ssrRoutes.find((r) => r.test(pathnameOf(request.url)));
+        if (route !== undefined) {
+            try {
+                const out = assembleRouteSsr(route, runtime.module, envelopeReq);
+                if (out !== null) {
+                    return preparedReply(prepareSsrResponse(out, request.method === 'HEAD', 'toil'));
+                }
+            }
+            catch (e) {
+                process.stdout.write(pc.red(`  x SSR ${request.path}: ${String(e)}`) + '\n');
+                return textReply(500, 'internal error\n');
+            }
+        }
+        return { kind: 'fallback' };
+    }
+    return textReply(404, 'not found\n');
+}
+async function startBuiltHttpServer(options, paths, dynamicHandler, runtimeOptions = {}) {
+    const cors = paths.cors;
+    const app = new Server(runtimeServerOptions(options));
+    const clients = new Set();
+    installRuntimeErrorHandler(app);
+    if (cors) {
+        app.use((request, response, next) => {
+            if (request.method !== 'OPTIONS') {
+                response.setHeader('Access-Control-Allow-Origin', '*');
+                response.setHeader('Access-Control-Allow-Methods', CORS_METHODS);
+                response.setHeader('Access-Control-Allow-Headers', CORS_HEADERS);
+            }
+            response.removeHeader('uWebSockets');
+            next();
+        });
+        app.options('/*', (_request, response) => {
+            response.setHeader('Access-Control-Allow-Origin', '*');
+            response.setHeader('Access-Control-Allow-Methods', CORS_METHODS);
+            response.setHeader('Access-Control-Allow-Headers', CORS_HEADERS);
+            response.setHeader('Access-Control-Max-Age', '86400');
+            response.status(204).send();
+        });
+    }
+    app.ws(paths.wsPath, {
+        message_type: 'String',
+        max_payload_length: WS_MAX_PAYLOAD_LENGTH,
+        idle_timeout: WS_IDLE_TIMEOUT,
+        max_backpressure: WS_MAX_BACKPRESSURE,
+    }, (ws) => {
+        clients.add(ws);
+        runtimeOptions.onClientCount?.(clients.size);
+        ws.send(JSON.stringify({ type: 'connected', clients: clients.size }));
+        ws.on('message', (message) => {
+            for (const client of clients)
+                client.send(message);
+        });
+        ws.on('drain', () => { });
+        ws.on('close', () => {
+            clients.delete(ws);
+            runtimeOptions.onClientCount?.(clients.size);
+        });
+    });
+    app.upgrade(paths.wsPath, (request, response) => {
+        if (!isWsOriginAllowed(request.headers.origin, request.headers.host, options.allowedOrigins)) {
+            response.status(403).send();
+            return;
+        }
+        response.upgrade({});
+    });
+    app.any('/*', async (request, response) => {
+        response.removeHeader('uWebSockets');
+        if (request.method === 'GET' || request.method === 'HEAD') {
+            const file = resolveStaticFile(paths.staticRoot, request.path);
+            if (file !== null) {
+                response.sendFile(file);
+                return;
+            }
+        }
+        if (await dynamicHandler(request, response))
+            return;
+        if (request.method === 'GET' || request.method === 'HEAD') {
+            response.sendFile(paths.indexHtml);
+            return;
+        }
+        response.status(404).send('not found\n');
+    });
+    await app.listen(paths.port, paths.host);
+    return {
+        port: paths.port,
+        host: paths.host,
+        wsPath: paths.wsPath,
+        broadcast: (message) => {
+            for (const client of clients)
+                client.send(message);
+        },
+        clientCount: () => clients.size,
+        close: async () => {
+            await app.shutdown();
+        },
+    };
+}
+async function toThreadedRequest(request, id) {
+    const envelopeReq = await toEnvelopeRequest(request);
+    return {
+        id,
+        method: request.method,
+        url: request.url,
+        path: request.path,
+        headers: envelopeReq.headers,
+        body: encodeBody(envelopeReq.body),
+        clientIp: envelopeReq.clientIp ?? '127.0.0.1',
+    };
+}
+async function startBuiltServerSingle(options) {
+    const paths = resolveBuiltServerPaths(options);
+    const runtime = createBuiltRuntime(options, paths);
+    const daemon = startBuiltDaemon(options, paths);
+    try {
+        const server = await startBuiltHttpServer(options, paths, async (request, response) => {
+            const reply = await handleBuiltRuntimeRequest(runtime, await toThreadedRequest(request, 0));
+            return sendThreadedReply(response, reply);
+        });
+        return {
+            ...server,
+            close: async () => {
+                daemon?.close();
+                await server.close();
+            },
+        };
+    }
+    catch (e) {
+        daemon?.close();
+        throw e;
+    }
+}
+export async function startBuiltServerWorker(options, controller) {
+    const paths = resolveBuiltServerPaths(options);
+    let nextRequestId = 1;
+    return startBuiltHttpServer({ ...options, threads: 1 }, paths, async (request, response) => {
+        const reply = await controller.request(await toThreadedRequest(request, nextRequestId++));
+        return sendThreadedReply(response, reply);
+    }, { onClientCount: (count) => controller.clientCount(count) });
+}
+function sendToWorker(worker, message) {
+    try {
+        worker.send?.(message);
+    }
+    catch {
+    }
+}
+function stopWorker(worker) {
+    return new Promise((resolve) => {
+        if (worker.exitCode !== null || worker.signalCode !== null) {
+            resolve();
+            return;
+        }
+        const hard = setTimeout(() => {
+            try {
+                worker.kill('SIGTERM');
+            }
+            catch {
+            }
+            resolve();
+        }, 1500);
+        hard.unref();
+        worker.once('exit', () => {
+            clearTimeout(hard);
+            resolve();
+        });
+        sendToWorker(worker, { toil: 'shutdown' });
+    });
+}
+async function startThreadedBuiltServer(options, threads) {
+    const paths = resolveBuiltServerPaths(options);
+    const runtime = createBuiltRuntime(options, paths);
+    const daemon = startBuiltDaemon(options, paths);
+    const workerScript = fileURLToPath(new URL('./production-worker.js', import.meta.url));
+    const workers = new Map();
+    const clientCounts = new Map();
+    let closing = false;
+    const handleMessage = (worker, message) => {
+        switch (message.toil) {
+            case 'clientCount':
+                clientCounts.set(message.workerId, message.count);
+                return;
+            case 'request':
+                void handleBuiltRuntimeRequest(runtime, message.request)
+                    .then((reply) => sendToWorker(worker, {
+                    toil: 'reply',
+                    id: message.request.id,
+                    reply,
+                }))
+                    .catch((e) => sendToWorker(worker, {
+                    toil: 'reply',
+                    id: message.request.id,
+                    reply: textReply(500, `internal error: ${String(e)}\n`),
+                }));
+                return;
+            case 'ready':
+                return;
+        }
+    };
+    const spawnWorker = (workerId, initial) => new Promise((resolve, reject) => {
+        const worker = fork(workerScript, [], {
+            stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+            env: process.env,
+        });
+        workers.set(workerId, worker);
+        clientCounts.set(workerId, 0);
+        let ready = false;
+        const failInitial = (error) => {
+            if (!initial || ready)
+                return;
+            reject(error);
+        };
+        worker.on('message', (value) => {
+            if (!isWorkerToPrimaryMessage(value))
+                return;
+            if (value.toil === 'ready') {
+                ready = true;
+                resolve();
+                return;
+            }
+            handleMessage(worker, value);
+        });
+        worker.once('error', (error) => {
+            failInitial(error);
+        });
+        worker.once('exit', (code, signal) => {
+            workers.delete(workerId);
+            clientCounts.delete(workerId);
+            if (closing)
+                return;
+            if (!ready) {
+                failInitial(new Error(`production worker ${String(workerId)} exited before listening (code ${String(code)}, signal ${String(signal)})`));
+                return;
+            }
+            void spawnWorker(workerId, false).catch((e) => {
+                process.stdout.write(pc.red(`  x production worker ${String(workerId)} restart failed: ${String(e)}`) + '\n');
+            });
+        });
+        sendToWorker(worker, {
+            toil: 'start',
+            workerId,
+            options: { ...options, threads: 1 },
+        });
+    });
+    try {
+        await Promise.all(Array.from({ length: threads }, (_, i) => spawnWorker(i + 1, true)));
+    }
+    catch (e) {
+        closing = true;
+        daemon?.close();
+        await Promise.all([...workers.values()].map((worker) => stopWorker(worker)));
+        throw e;
+    }
+    process.stdout.write(pc.dim(`  production threads: ${String(threads)} HTTP workers`) + '\n');
+    return {
+        port: paths.port,
+        host: paths.host,
+        wsPath: paths.wsPath,
+        broadcast: (message) => {
+            for (const worker of workers.values()) {
+                sendToWorker(worker, { toil: 'broadcast', message });
+            }
+        },
+        clientCount: () => {
+            let total = 0;
+            for (const count of clientCounts.values())
+                total += count;
+            return total;
+        },
+        close: async () => {
+            closing = true;
+            daemon?.close();
+            await Promise.all([...workers.values()].map((worker) => stopWorker(worker)));
+        },
+    };
+}
+export async function startBuiltServer(options) {
+    const threads = resolveThreadCount(options.threads);
+    if (threads <= 1)
+        return startBuiltServerSingle(options);
+    return startThreadedBuiltServer(options, threads);
+}

package/build/devserver/runtime/module.d.ts

@@ -16,11 +16,16 @@ export declare class WasmServerModule {
     private module;
     private loadedMtimeMs;
     private routeKinds;
+    private derives;
+    private derivesDirty;
     constructor(wasmPath: string);
     get available(): boolean;
     refresh(): boolean;
     dispatch(req: EnvelopeRequest): WasmDispatchResult;
     dispatchRender(req: EnvelopeRequest): Uint8Array;
+    private runAffectedDerives;
+    private rebuildDerivedViewsIfStale;
+    private runDerive;
     private assertImportSurface;
     private assertExportSurface;
 }

package/build/devserver/runtime/module.js

@@ -1,5 +1,5 @@
 import fs from 'node:fs';
-import { DbFunctionKind, persistDb, setDbCatalog } from '../db/index.js';
+import { DbFunctionKind, derivesForWrites, parseDerives, persistDb, setDbCatalog, } from '../db/index.js';
 import { parseRouteKinds, routeKindForRequest } from '../db/routeKinds.js';
 import { decodeResponseEnvelope, encodeRequestEnvelope, unpackHandleResult, } from '../http/envelope.js';
 import { buildHostImports, freshDispatchState } from './host.js';
@@ -87,6 +87,8 @@ export class WasmServerModule {
     module = null;
     loadedMtimeMs = -1;
     routeKinds = [];
+    derives = [];
+    derivesDirty = false;
     constructor(wasmPath) {
         this.wasmPath = wasmPath;
     }
@@ -101,6 +103,8 @@ export class WasmServerModule {
         catch {
             this.module = null;
             this.routeKinds = [];
+            this.derives = [];
+            this.derivesDirty = false;
             this.loadedMtimeMs = -1;
             return false;
         }
@@ -112,13 +116,16 @@ export class WasmServerModule {
         this.assertExportSurface(module);
         setDbCatalog(bytes);
         this.routeKinds = parseRouteKinds(bytes);
+        this.derives = parseDerives(bytes);
         this.module = module;
         this.loadedMtimeMs = mtimeMs;
+        this.derivesDirty = this.derives.length > 0;
         return true;
     }
     dispatch(req) {
         if (this.module === null)
             throw new Error(`server wasm not loaded (${this.wasmPath})`);
+        this.rebuildDerivedViewsIfStale();
         const envelope = encodeRequestEnvelope(req);
         const ref = { memory: null };
         const state = freshDispatchState();
@@ -139,6 +146,7 @@ export class WasmServerModule {
         const headers = [...resp.headers, ...state.headers];
         const status = state.status ?? resp.status;
         const unhandled = headers.some(([n]) => n.toLowerCase() === UNHANDLED_HEADER);
+        this.runAffectedDerives(state.db.writtenCollections);
         persistDb();
         return {
             status,
@@ -171,6 +179,44 @@ export class WasmServerModule {
             throw new Error(`guest returned out-of-bounds values: ptr=${String(ptr)} len=${String(len)}`);
         return new Uint8Array(exports.memory.buffer, ptr, len).slice();
     }
+    runAffectedDerives(written) {
+        if (this.module === null)
+            return;
+        for (const derive of derivesForWrites(this.derives, written)) {
+            try {
+                this.runDerive(derive.deriveId);
+            }
+            catch (err) {
+                console.error(`[toil] derive ${derive.dbName}#${derive.methodName} failed:`, err);
+            }
+        }
+    }
+    rebuildDerivedViewsIfStale() {
+        if (!this.derivesDirty)
+            return;
+        this.derivesDirty = false;
+        for (const derive of this.derives) {
+            try {
+                this.runDerive(derive.deriveId);
+            }
+            catch (err) {
+                console.error(`[toil] derive ${derive.dbName}#${derive.methodName} failed on load:`, err);
+            }
+        }
+    }
+    runDerive(deriveId) {
+        if (this.module === null)
+            return;
+        const ref = { memory: null };
+        const state = freshDispatchState();
+        state.db.functionKind = DbFunctionKind.Derive;
+        const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
+        const exports = instance.exports;
+        ref.memory = exports.memory;
+        if (typeof exports.derive_run !== 'function')
+            return;
+        exports.derive_run(deriveId);
+    }
     assertImportSurface(module) {
         const missing = WebAssembly.Module.imports(module)
             .filter((i) => i.kind === 'function' && (i.module !== 'env' || !PROVIDED_IMPORTS.has(i.name)))

package/build/devserver/server.d.ts

@@ -8,6 +8,7 @@ export interface DevServerOptions {
     readonly host?: string;
     readonly wasmFile: string;
     readonly coldWasmFile?: string;
+    readonly streamWasmFile?: string;
     readonly nodeMode?: string;
     readonly daemon?: ResolvedDaemonConfig;
     readonly vite: ViteTarget;