toiljs 0.0.66 → 0.0.68

package/build/devserver/db/derives.js

@@ -0,0 +1,94 @@
+import { customSection } from '../wasm/sections.js';
+const SECTION = 'toildb.derives';
+const VERSION = 1;
+const MAX_SECTION_BYTES = 128 * 1024;
+const MAX_DERIVES = 1024;
+const MAX_NAME_BYTES = 1024;
+const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
+export function parseDerives(wasm) {
+    let section;
+    try {
+        section = customSection(wasm, SECTION);
+    }
+    catch {
+        return [];
+    }
+    if (section === null)
+        return [];
+    if (section.length > MAX_SECTION_BYTES)
+        return [];
+    const r = new Reader(section);
+    const version = r.u16();
+    if (!r.ok || version !== VERSION)
+        return [];
+    const count = r.u16();
+    if (!r.ok || count > MAX_DERIVES)
+        return [];
+    const derives = [];
+    for (let i = 0; i < count && r.ok; i++) {
+        const deriveId = r.u16();
+        const dbName = r.string();
+        const methodName = r.string();
+        if (!r.ok || dbName.length === 0)
+            return [];
+        derives.push({ deriveId, dbName, methodName });
+    }
+    if (!r.ok || r.remaining() !== 0)
+        return [];
+    return derives;
+}
+export function derivesForWrites(derives, written) {
+    if (derives.length === 0 || written.size === 0)
+        return [];
+    const dbs = new Set();
+    for (const key of written) {
+        const slash = key.indexOf('/');
+        dbs.add(slash >= 0 ? key.slice(0, slash) : key);
+    }
+    return derives.filter((d) => dbs.has(d.dbName));
+}
+class Reader {
+    bytes;
+    pos = 0;
+    ok = true;
+    constructor(bytes) {
+        this.bytes = bytes;
+    }
+    remaining() {
+        return this.bytes.length - this.pos;
+    }
+    u16() {
+        if (!this.ok || this.pos + 2 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt16LE(this.pos);
+        this.pos += 2;
+        return out;
+    }
+    u32() {
+        if (!this.ok || this.pos + 4 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt32LE(this.pos);
+        this.pos += 4;
+        return out;
+    }
+    string() {
+        const len = this.u32();
+        if (!this.ok || len > MAX_NAME_BYTES || this.pos + len > this.bytes.length) {
+            this.ok = false;
+            return '';
+        }
+        try {
+            const out = UTF8_DECODER.decode(this.bytes.subarray(this.pos, this.pos + len));
+            this.pos += len;
+            return out;
+        }
+        catch {
+            this.ok = false;
+            return '';
+        }
+    }
+}

package/build/devserver/db/index.d.ts

@@ -1,3 +1,4 @@
 export { __resetDbForTests, __setDbCatalogForTests, buildDatabaseImports, configureDbPersistence, DevDatabase, devDb, persistDb, setDbCatalog, } from './database.js';
 export { parseCatalog } from './catalog.js';
+export { type DeriveEntry, derivesForWrites, parseDerives } from './derives.js';
 export { CollectionFamily, DbFunctionKind, type DbDevState, freshDbState } from './types.js';

package/build/devserver/db/index.js

@@ -1,3 +1,4 @@
 export { __resetDbForTests, __setDbCatalogForTests, buildDatabaseImports, configureDbPersistence, DevDatabase, devDb, persistDb, setDbCatalog, } from './database.js';
 export { parseCatalog } from './catalog.js';
+export { derivesForWrites, parseDerives } from './derives.js';
 export { CollectionFamily, DbFunctionKind, freshDbState } from './types.js';

package/build/devserver/db/types.d.ts

@@ -37,6 +37,7 @@ export interface DbDevState {
     lastResult: Buffer | null;
     lastResultVersion: number;
     functionKind: DbFunctionKind;
+    writtenCollections: Set<string>;
 }
 export declare function freshDbState(): DbDevState;
 export interface Reservation {

package/build/devserver/db/types.js

@@ -25,6 +25,7 @@ export function freshDbState() {
         lastResult: null,
         lastResultVersion: -1,
         functionKind: DbFunctionKind.Job,
+        writtenCollections: new Set(),
     };
 }
 export const MAX_RESERVATIONS = 4096;

package/build/devserver/http/proxy.d.ts

@@ -1,7 +1,11 @@
-import { type Request, type Response, type Server } from '@dacely/hyper-express';
+import { type Request, type Response, type Server, type Websocket } from '@dacely/hyper-express';
 export interface ViteTarget {
     readonly host: string;
     readonly port: number;
 }
 export declare function proxyToVite(request: Request, response: Response, target: ViteTarget): Promise<void>;
 export declare function wireWebsocketProxy(app: Server, target: ViteTarget): void;
+export declare function pipeToVite(ws: Websocket, target: ViteTarget, ctx: {
+    url: string;
+    protocol: string;
+}): void;

package/build/devserver/http/proxy.js

@@ -59,41 +59,44 @@ export function wireWebsocketProxy(app, target) {
         });
     });
     app.ws('/*', { message_type: 'Buffer', idle_timeout: 120, max_payload_length: 16 * 1024 * 1024 }, (ws) => {
-        const { url, protocol } = ws.context;
-        const upstream = new WebSocket(`ws://${target.host}:${String(target.port)}${url}`, protocol ? protocol.split(',').map((p) => p.trim()) : []);
-        upstream.binaryType = 'arraybuffer';
-        const pending = [];
-        let open = false;
-        upstream.onopen = () => {
-            open = true;
-            for (const m of pending)
-                upstream.send(m);
-            pending.length = 0;
-        };
-        upstream.onmessage = (event) => {
-            if (typeof event.data === 'string')
-                ws.send(event.data);
-            else
-                ws.send(Buffer.from(event.data), true);
-        };
-        upstream.onclose = (event) => {
-            ws.close(event.code, event.reason);
-        };
-        upstream.onerror = () => {
-            ws.close();
-        };
-        ws.on('message', (message, isBinary) => {
-            const m = toUpstreamMessage(message, isBinary);
-            if (open)
-                upstream.send(m);
-            else
-                pending.push(m);
-        });
-        ws.on('close', () => {
-            if (upstream.readyState === WebSocket.OPEN ||
-                upstream.readyState === WebSocket.CONNECTING) {
-                upstream.close();
-            }
-        });
+        pipeToVite(ws, target, ws.context);
+    });
+}
+export function pipeToVite(ws, target, ctx) {
+    const { url, protocol } = ctx;
+    const upstream = new WebSocket(`ws://${target.host}:${String(target.port)}${url}`, protocol ? protocol.split(',').map((p) => p.trim()) : []);
+    upstream.binaryType = 'arraybuffer';
+    const pending = [];
+    let open = false;
+    upstream.onopen = () => {
+        open = true;
+        for (const m of pending)
+            upstream.send(m);
+        pending.length = 0;
+    };
+    upstream.onmessage = (event) => {
+        if (typeof event.data === 'string')
+            ws.send(event.data);
+        else
+            ws.send(Buffer.from(event.data), true);
+    };
+    upstream.onclose = (event) => {
+        ws.close(event.code, event.reason);
+    };
+    upstream.onerror = () => {
+        ws.close();
+    };
+    ws.on('message', (message, isBinary) => {
+        const m = toUpstreamMessage(message, isBinary);
+        if (open)
+            upstream.send(m);
+        else
+            pending.push(m);
+    });
+    ws.on('close', () => {
+        if (upstream.readyState === WebSocket.OPEN ||
+            upstream.readyState === WebSocket.CONNECTING) {
+            upstream.close();
+        }
     });
 }

package/build/devserver/http/runtime.d.ts

@@ -0,0 +1,62 @@
+import { type Request, type Response, type Server } from '@dacely/hyper-express';
+import { type EnvelopeRequest } from './envelope.js';
+import { type CacheableResult } from './cache.js';
+import { WasmServerModule } from '../runtime/module.js';
+import { type SsrResult, type SsrRoute } from '../ssr.js';
+export declare const DEFAULT_MAX_BODY_LENGTH: number;
+export declare const MAX_BODY_BUFFER: number;
+export declare const HTTP_IDLE_TIMEOUT = 60;
+export declare const HTTP_RESPONSE_TIMEOUT = 120;
+export declare const MIME: Readonly<Record<string, string>>;
+export interface RuntimeServerOptions {
+    readonly maxBodyLength?: number;
+}
+export declare function runtimeServerOptions(options: RuntimeServerOptions): {
+    max_body_length: number;
+    max_body_buffer: number;
+    fast_abort: true;
+    idle_timeout: number;
+    response_timeout: number;
+};
+export declare function installRuntimeErrorHandler(app: Server): void;
+export declare function isDispatchableMethod(method: string): boolean;
+export declare function resolveFileInside(root: string, file: string): string | null;
+export declare function resolveStaticFile(root: string, requestPath: string): string | null;
+export interface PreparedHttpResponse {
+    readonly status: number;
+    readonly headers: readonly (readonly [string, string])[];
+    readonly body: Uint8Array;
+    readonly sendfile: string | null;
+}
+export declare function toEnvelopeRequest(request: Request): Promise<EnvelopeRequest>;
+export declare function prepareWasmResponse(root: string, result: Pick<CacheableResult, 'status' | 'headers' | 'body' | 'sendfile'>, serverHeader: string): PreparedHttpResponse;
+export declare function sendPreparedResponse(response: Response, out: PreparedHttpResponse): void;
+export declare function sendWasmResponse(response: Response, root: string, result: Pick<CacheableResult, 'status' | 'headers' | 'body' | 'sendfile'>, serverHeader: string): void;
+export declare function prepareSsrResponse(out: SsrResult, headOnly: boolean, serverHeader: string): PreparedHttpResponse;
+export declare function sendSsr(response: Response, out: SsrResult, headOnly: boolean, serverHeader: string): void;
+export interface WasmDispatchOutcome {
+    readonly envelopeReq: EnvelopeRequest;
+    readonly handled: boolean;
+}
+export interface EnvelopeDispatchOutcome {
+    readonly result: CacheableResult | null;
+    readonly handled: boolean;
+}
+export declare function dispatchEnvelopeRequest(options: {
+    readonly module: WasmServerModule;
+    readonly envelopeReq: EnvelopeRequest;
+    readonly method: string;
+    readonly url: string;
+    readonly cacheHost: string;
+    readonly hasAuth: boolean;
+}): EnvelopeDispatchOutcome;
+export declare function dispatchWasmRequest(options: {
+    readonly module: WasmServerModule;
+    readonly request: Request;
+    readonly response: Response;
+    readonly root: string;
+    readonly cacheHost: string;
+    readonly serverHeader: string;
+    readonly errorPrefix: string;
+}): Promise<WasmDispatchOutcome>;
+export declare function assembleRouteSsr(route: SsrRoute, module: WasmServerModule | null, envelopeReq: EnvelopeRequest): SsrResult | null;

package/build/devserver/http/runtime.js

@@ -0,0 +1,194 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import pc from 'picocolors';
+import { METHOD_CODES } from './envelope.js';
+import { applyCacheRule, lookupCache } from './cache.js';
+import { assembleSsr } from '../ssr.js';
+export const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
+export const MAX_BODY_BUFFER = 1024 * 32;
+export const HTTP_IDLE_TIMEOUT = 60;
+export const HTTP_RESPONSE_TIMEOUT = 120;
+export const MIME = {
+    '.html': 'text/html; charset=utf-8',
+    '.js': 'text/javascript; charset=utf-8',
+    '.mjs': 'text/javascript; charset=utf-8',
+    '.css': 'text/css; charset=utf-8',
+    '.json': 'application/json; charset=utf-8',
+    '.txt': 'text/plain; charset=utf-8',
+    '.svg': 'image/svg+xml',
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.webp': 'image/webp',
+    '.avif': 'image/avif',
+    '.gif': 'image/gif',
+    '.ico': 'image/x-icon',
+    '.wasm': 'application/wasm',
+    '.woff2': 'font/woff2',
+};
+export function runtimeServerOptions(options) {
+    return {
+        max_body_length: options.maxBodyLength ?? DEFAULT_MAX_BODY_LENGTH,
+        max_body_buffer: MAX_BODY_BUFFER,
+        fast_abort: true,
+        idle_timeout: HTTP_IDLE_TIMEOUT,
+        response_timeout: HTTP_RESPONSE_TIMEOUT,
+    };
+}
+export function installRuntimeErrorHandler(app) {
+    app.set_error_handler((_request, response, error) => {
+        if (response.completed)
+            return;
+        response.atomic(() => {
+            response.status(500).send(`internal error: ${error.message}\n`);
+        });
+    });
+}
+export function isDispatchableMethod(method) {
+    return METHOD_CODES[method] !== undefined;
+}
+export function resolveFileInside(root, file) {
+    const resolved = path.resolve(root, file);
+    if (resolved !== root && !resolved.startsWith(root + path.sep))
+        return null;
+    if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile())
+        return null;
+    return resolved;
+}
+export function resolveStaticFile(root, requestPath) {
+    let decoded;
+    try {
+        decoded = decodeURIComponent(requestPath);
+    }
+    catch {
+        return null;
+    }
+    if (decoded === '/' || decoded === '')
+        return null;
+    return resolveFileInside(root, decoded.replace(/^\/+/, ''));
+}
+export async function toEnvelopeRequest(request) {
+    const hasBody = request.method !== 'GET' && request.method !== 'HEAD';
+    const body = hasBody ? new Uint8Array(await request.buffer()) : new Uint8Array(0);
+    const xff = request.headers['x-forwarded-for'];
+    const clientIp = typeof xff === 'string' && xff.length > 0 ? xff.split(',')[0].trim() : '127.0.0.1';
+    return {
+        method: request.method,
+        path: request.url,
+        headers: Object.entries(request.headers),
+        body,
+        clientIp,
+    };
+}
+export function prepareWasmResponse(root, result, serverHeader) {
+    const headers = [];
+    let hasContentType = false;
+    for (const [name, value] of result.headers) {
+        if (name.toLowerCase() === 'content-type')
+            hasContentType = true;
+        headers.push([name, value]);
+    }
+    headers.push(['server', serverHeader]);
+    if (result.sendfile !== null) {
+        const file = resolveFileInside(root, result.sendfile);
+        if (file === null) {
+            return {
+                status: 404,
+                headers: [
+                    ['server', serverHeader],
+                    ['content-type', 'text/plain; charset=utf-8'],
+                ],
+                body: new TextEncoder().encode('not found\n'),
+                sendfile: null,
+            };
+        }
+        if (!hasContentType) {
+            headers.push([
+                'content-type',
+                MIME[path.extname(file).toLowerCase()] ?? 'application/octet-stream',
+            ]);
+        }
+        return { status: result.status, headers, body: new Uint8Array(0), sendfile: file };
+    }
+    if (!hasContentType)
+        headers.push(['content-type', 'text/plain; charset=utf-8']);
+    return { status: result.status, headers, body: result.body, sendfile: null };
+}
+export function sendPreparedResponse(response, out) {
+    response.status(out.status);
+    for (const [name, value] of out.headers)
+        response.header(name, value);
+    if (out.sendfile !== null) {
+        response.sendFile(out.sendfile);
+        return;
+    }
+    response.send(Buffer.from(out.body.buffer, out.body.byteOffset, out.body.length));
+}
+export function sendWasmResponse(response, root, result, serverHeader) {
+    sendPreparedResponse(response, prepareWasmResponse(root, result, serverHeader));
+}
+export function prepareSsrResponse(out, headOnly, serverHeader) {
+    const headers = [];
+    let hasContentType = false;
+    for (const [name, value] of out.headers) {
+        if (name.toLowerCase() === 'content-type')
+            hasContentType = true;
+        headers.push([name, value]);
+    }
+    if (!hasContentType)
+        headers.push(['content-type', 'text/html; charset=utf-8']);
+    headers.push(['server', serverHeader]);
+    return {
+        status: out.status,
+        headers,
+        body: headOnly ? new Uint8Array(0) : out.html,
+        sendfile: null,
+    };
+}
+export function sendSsr(response, out, headOnly, serverHeader) {
+    sendPreparedResponse(response, prepareSsrResponse(out, headOnly, serverHeader));
+}
+export function dispatchEnvelopeRequest(options) {
+    const cached = lookupCache(options.cacheHost, options.method, options.url, options.envelopeReq.body);
+    if (cached !== null)
+        return { result: cached, handled: true };
+    const result = options.module.dispatch(options.envelopeReq);
+    if (result.unhandled)
+        return { result: null, handled: false };
+    return {
+        result: applyCacheRule(options.cacheHost, options.method, options.url, options.envelopeReq.body, options.hasAuth, result),
+        handled: true,
+    };
+}
+export async function dispatchWasmRequest(options) {
+    const envelopeReq = await toEnvelopeRequest(options.request);
+    const hasAuth = options.request.headers.cookie !== undefined ||
+        options.request.headers.authorization !== undefined;
+    try {
+        const dispatch = dispatchEnvelopeRequest({
+            module: options.module,
+            envelopeReq,
+            method: options.request.method,
+            url: options.request.url,
+            cacheHost: options.cacheHost,
+            hasAuth,
+        });
+        if (dispatch.result !== null) {
+            sendWasmResponse(options.response, options.root, dispatch.result, options.serverHeader);
+            return { envelopeReq, handled: true };
+        }
+    }
+    catch (e) {
+        process.stdout.write(pc.red(`  ${options.errorPrefix} ${options.request.method} ${options.request.path} server error: ${String(e)}`) + '\n');
+        options.response.status(500).send('internal error\n');
+        return { envelopeReq, handled: true };
+    }
+    return { envelopeReq, handled: false };
+}
+export function assembleRouteSsr(route, module, envelopeReq) {
+    if (route.entries.length === 0)
+        return { status: 200, headers: [], html: route.tmpl };
+    if (module === null || !module.available)
+        return null;
+    return assembleSsr(route, module.dispatchRender(envelopeReq));
+}

package/build/devserver/index.d.ts

@@ -1,5 +1,7 @@
 export { startDevServer } from './server.js';
 export type { DevServerOptions, RunningDevServer } from './server.js';
+export { loadBuiltSsrTemplates, startBuiltServer } from './production.js';
+export type { BuiltServerOptions, RunningBuiltServer } from './production.js';
 export { METHOD_CODES, encodeRequestEnvelope, decodeResponseEnvelope, unpackHandleResult, } from './http/envelope.js';
 export type { EnvelopeRequest, EnvelopeResponse } from './http/envelope.js';
 export { WasmServerModule, WasmAbortError, UNHANDLED_HEADER } from './runtime/module.js';

package/build/devserver/index.js

@@ -1,4 +1,5 @@
 export { startDevServer } from './server.js';
+export { loadBuiltSsrTemplates, startBuiltServer } from './production.js';
 export { METHOD_CODES, encodeRequestEnvelope, decodeResponseEnvelope, unpackHandleResult, } from './http/envelope.js';
 export { WasmServerModule, WasmAbortError, UNHANDLED_HEADER } from './runtime/module.js';
 export { buildHostImports, freshDispatchState } from './runtime/host.js';

package/build/devserver/production-ipc.d.ts

@@ -0,0 +1,50 @@
+export interface ThreadedRequest {
+    readonly id: number;
+    readonly method: string;
+    readonly url: string;
+    readonly path: string;
+    readonly headers: readonly (readonly [string, string])[];
+    readonly body: string;
+    readonly clientIp: string;
+}
+export interface ThreadedHttpResponse {
+    readonly kind: 'response';
+    readonly status: number;
+    readonly headers: readonly (readonly [string, string])[];
+    readonly body: string;
+    readonly sendfile: string | null;
+}
+export interface ThreadedFallbackResponse {
+    readonly kind: 'fallback';
+}
+export type ThreadedReply = ThreadedHttpResponse | ThreadedFallbackResponse;
+export type WorkerToPrimaryMessage = {
+    readonly toil: 'ready';
+    readonly port: number;
+    readonly workerId: number;
+} | {
+    readonly toil: 'clientCount';
+    readonly count: number;
+    readonly workerId: number;
+} | {
+    readonly toil: 'request';
+    readonly request: ThreadedRequest;
+};
+export type PrimaryToWorkerMessage = {
+    readonly toil: 'start';
+    readonly workerId: number;
+    readonly options: unknown;
+} | {
+    readonly toil: 'reply';
+    readonly id: number;
+    readonly reply: ThreadedReply;
+} | {
+    readonly toil: 'broadcast';
+    readonly message: string;
+} | {
+    readonly toil: 'shutdown';
+};
+export declare function encodeBody(body: Uint8Array): string;
+export declare function decodeBody(body: string): Uint8Array;
+export declare function isWorkerToPrimaryMessage(value: unknown): value is WorkerToPrimaryMessage;
+export declare function isPrimaryToWorkerMessage(value: unknown): value is PrimaryToWorkerMessage;

package/build/devserver/production-ipc.js

@@ -0,0 +1,21 @@
+export function encodeBody(body) {
+    return Buffer.from(body.buffer, body.byteOffset, body.length).toString('base64');
+}
+export function decodeBody(body) {
+    return Buffer.from(body, 'base64');
+}
+export function isWorkerToPrimaryMessage(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const message = value;
+    return message.toil === 'ready' || message.toil === 'clientCount' || message.toil === 'request';
+}
+export function isPrimaryToWorkerMessage(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const message = value;
+    return (message.toil === 'start' ||
+        message.toil === 'reply' ||
+        message.toil === 'broadcast' ||
+        message.toil === 'shutdown');
+}

package/build/devserver/production-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/build/devserver/production-worker.js

@@ -0,0 +1,73 @@
+import { startBuiltServerWorker } from './production.js';
+import { isPrimaryToWorkerMessage, } from './production-ipc.js';
+let running = null;
+let workerId = 0;
+const pending = new Map();
+function send(message) {
+    try {
+        process.send?.(message);
+    }
+    catch {
+    }
+}
+function requestPrimary(request) {
+    const id = request.id;
+    return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+            pending.delete(id);
+            reject(new Error('primary request timed out'));
+        }, 120_000);
+        timeout.unref();
+        pending.set(id, (reply) => {
+            clearTimeout(timeout);
+            resolve(reply);
+        });
+        send({ toil: 'request', request });
+    });
+}
+async function start(message) {
+    if (running !== null)
+        return;
+    workerId = message.workerId;
+    const options = message.options;
+    running = await startBuiltServerWorker(options, {
+        request: requestPrimary,
+        clientCount: (count) => send({ toil: 'clientCount', workerId, count }),
+    });
+    send({ toil: 'ready', workerId, port: running.port });
+}
+async function shutdown() {
+    const server = running;
+    running = null;
+    if (server !== null)
+        await server.close();
+}
+process.on('message', (value) => {
+    if (!isPrimaryToWorkerMessage(value))
+        return;
+    switch (value.toil) {
+        case 'start':
+            void start(value).catch((e) => {
+                process.stderr.write(`toiljs production worker failed: ${String(e)}\n`);
+                process.exit(1);
+            });
+            return;
+        case 'reply': {
+            const resolve = pending.get(value.id);
+            if (resolve === undefined)
+                return;
+            pending.delete(value.id);
+            resolve(value.reply);
+            return;
+        }
+        case 'broadcast':
+            running?.broadcast(value.message);
+            return;
+        case 'shutdown':
+            void shutdown().finally(() => process.exit(0));
+            return;
+    }
+});
+process.once('disconnect', () => {
+    void shutdown().finally(() => process.exit(0));
+});