toiljs 0.0.124 → 0.0.125

package/build/devserver/db/limits.js

@@ -0,0 +1,118 @@
+import { DbFunctionKind } from './types.js';
+const counters = new WeakMap();
+const exempt = new Set([
+    'data.resolve_collection',
+    'data.take_result',
+    'data.result_schema_version',
+    'data.write_allowed',
+]);
+const reads = new Set([
+    'data.get',
+    'data.get_current',
+    'data.get_many',
+    'data.exists',
+    'data.unique_lookup',
+    'data.view_get',
+    'data.membership_contains',
+    'data.membership_list',
+    'data.membership_page',
+    'data.counter_get',
+    'data.events_get',
+    'data.events_reference',
+    'data.events_entries',
+    'data.events_last',
+    'data.latest',
+    'data.events_since',
+    'data.capacity_available',
+]);
+const results = new Set([
+    'data.get',
+    'data.get_current',
+    'data.get_many',
+    'data.get_delete',
+    'data.patch',
+    'data.unique_lookup',
+    'data.view_get',
+    'data.membership_list',
+    'data.membership_page',
+    'data.counter_get',
+    'data.events_get',
+    'data.events_reference',
+    'data.events_entries',
+    'data.events_last',
+    'data.latest',
+    'data.events_since',
+    'data.capacity_available',
+    'data.capacity_reserve',
+]);
+const writeLength = {
+    'data.create': 4,
+    'data.patch': 4,
+    'data.upsert': 4,
+    'data.enqueue': 4,
+    'data.compare_exchange': 6,
+    'data.compare_delete': 4,
+    'data.unique_claim': 4,
+    'data.view_publish': 4,
+    'data.membership_add': 4,
+    'data.append': 4,
+    'data.append_once': 6,
+};
+export function resetDbLimits(db) {
+    counters.delete(db);
+}
+export function checkDbResultSize(db, bytes) {
+    if (process.env.TOIL_DEV_CHECK_DB_LIMITS !== '1')
+        return;
+    const max = db.functionKind === DbFunctionKind.Query
+        ? 65536
+        : db.functionKind === DbFunctionKind.Action
+            ? 131072
+            : 1048576;
+    if (bytes > max)
+        throw new Error(`TDL021: ${db.functionKind} DB operation uses ${String(bytes)} bytes; maximum is ${String(max)}`);
+}
+export function withDbLimits(imports, db) {
+    return Object.fromEntries(Object.entries(imports).map(([name, operation]) => [
+        name,
+        (...args) => {
+            if (process.env.TOIL_DEV_CHECK_DB_LIMITS !== '1' || exempt.has(name))
+                return operation(...args);
+            const kind = db.functionKind;
+            let count = counters.get(db);
+            if (count === undefined || count.kind !== kind) {
+                count = { kind, ops: 0, writes: 0, bytes: 0 };
+                counters.set(db, count);
+            }
+            const maxOps = kind === DbFunctionKind.Query || kind === DbFunctionKind.Action ? 64 : 65535;
+            const maxWrites = kind === DbFunctionKind.Query ? 0 : kind === DbFunctionKind.Action ? 32 : 65535;
+            const maxBytes = kind === DbFunctionKind.Query
+                ? 131072
+                : kind === DbFunctionKind.Action
+                    ? 2097152
+                    : 16777216;
+            const writing = !reads.has(name);
+            if (++count.ops > maxOps || (writing && ++count.writes > maxWrites))
+                throw new Error(`TDL020: ${name} exceeds ${kind} DB budget (${String(count.ops)}/${String(maxOps)} operations, ${String(count.writes)}/${String(maxWrites)} writes)`);
+            const writeBytes = (name === 'data.events_redact' ? 16 : writeLength[name] === undefined ? 0 : args[writeLength[name]]) +
+                (name === 'data.compare_exchange' ? Math.max(0, args[4]) : 0);
+            checkDbResultSize(db, writeBytes);
+            if (count.bytes + writeBytes > maxBytes)
+                throw new Error(`TDL021: ${kind} cumulative DB byte budget exceeds ${String(maxBytes)}`);
+            count.bytes += writeBytes;
+            const status = operation(...args);
+            if (typeof status === 'number' && status >= 0) {
+                const bytes = results.has(name)
+                    ? (db.lastResult?.length ?? 0)
+                    : name === 'data.unique_claim' && status === 1
+                        ? (db.lastResult?.length ?? 0)
+                        : 0;
+                checkDbResultSize(db, bytes);
+                if (count.bytes + bytes > maxBytes)
+                    throw new Error(`TDL021: ${kind} cumulative DB byte budget exceeds ${String(maxBytes)}`);
+                count.bytes += bytes;
+            }
+            return status;
+        },
+    ]));
+}

package/build/devserver/db/materialization.d.ts

@@ -0,0 +1,4 @@
+export declare function registerMaterializer(key: string, runner: (written: ReadonlySet<string>) => void): void;
+export declare function unregisterMaterializer(key: string): void;
+export declare function queueMaterialization(collection: string): void;
+export declare function flushMaterializations(): void;

package/build/devserver/db/materialization.js

@@ -0,0 +1,25 @@
+const runners = new Map();
+const pending = new Set();
+let scheduled = false;
+export function registerMaterializer(key, runner) {
+    runners.set(key, runner);
+}
+export function unregisterMaterializer(key) {
+    runners.delete(key);
+}
+export function queueMaterialization(collection) {
+    pending.add(collection);
+    if (scheduled)
+        return;
+    scheduled = true;
+    queueMicrotask(flushMaterializations);
+}
+export function flushMaterializations() {
+    scheduled = false;
+    if (pending.size === 0)
+        return;
+    const written = new Set(pending);
+    pending.clear();
+    for (const runner of runners.values())
+        runner(written);
+}

package/build/devserver/db/types.d.ts

@@ -40,6 +40,7 @@ export type DbCatalogState = {
     collections: Map<string, DevCollectionHandle>;
 };
 export interface DbDevState {
+    logMessages: number;
     handles: DevCollectionHandle[];
     lastResult: Buffer | null;
     lastResultVersion: number;
@@ -83,7 +84,9 @@ export interface DbSnapshot {
     events: Record<string, {
         v: string;
         sv: number;
+        hlc?: string;
     }[]>;
+    erasedEvents?: Record<string, string[]>;
     eventDedup: Record<string, string[]>;
     eventPositions?: Record<string, [string, number][]>;
     deriveCheckpoints?: Record<string, number>;

package/build/devserver/db/types.js

@@ -21,6 +21,7 @@ export function isCollectionFamily(value) {
 }
 export function freshDbState() {
     return {
+        logMessages: 0,
         handles: [],
         lastResult: null,
         lastResultVersion: -1,

package/build/devserver/runtime/host.js

@@ -130,8 +130,24 @@ function devEmailSend(ref, reqPtr, reqLen) {
     });
     return EmailStatus.Sent;
 }
+function buildLogImports(ref, db) {
+    return {
+        server_log: (level, ptr, len) => {
+            if (!Number.isInteger(level) || level < 0 || level > 2 || len < 0 || len > 4096)
+                throw new Error('server_log: level must be 0..2 and message at most 4096 UTF-8 bytes');
+            const message = new TextDecoder('utf-8', { fatal: true }).decode(readBytes(ref, ptr, len));
+            if (db.logMessages >= 32)
+                return 0;
+            db.logMessages++;
+            const write = level === 0 ? console.info : level === 1 ? console.warn : console.error;
+            write('[toil guest]', message);
+            return 1;
+        },
+    };
+}
 export function buildEnvImports(ref, _state) {
     return {
+        ...buildLogImports(ref, _state.db),
         abort: (msgPtr, filePtr, line, col) => {
             throw new WasmAbortError(readGuestString(ref, msgPtr), readGuestString(ref, filePtr), line, col);
         },
@@ -146,6 +162,7 @@ export function buildEnvImports(ref, _state) {
 export function buildHostImports(ref, state) {
     return {
         env: {
+            ...buildLogImports(ref, state.db),
             abort: (msgPtr, filePtr, line, col) => {
                 throw new WasmAbortError(readGuestString(ref, msgPtr), readGuestString(ref, filePtr), line, col);
             },

package/build/devserver/runtime/module.d.ts

@@ -19,6 +19,8 @@ export declare class WasmServerModule {
     private rpcKinds;
     private derives;
     private derivesDirty;
+    private envelopeWindow;
+    private readonly pendingDerives;
     constructor(wasmPath: string);
     get available(): boolean;
     refresh(): boolean;
@@ -27,6 +29,8 @@ export declare class WasmServerModule {
     private runAffectedDerives;
     private rebuildDerivedViewsIfStale;
     private runDerive;
+    private requestEnvelope;
+    private writeRequestEnvelope;
     private assertImportSurface;
     private assertExportSurface;
 }

package/build/devserver/runtime/module.js

@@ -3,6 +3,8 @@ import { commitDeriveCheckpoints, DbFunctionKind, derivesForWrites, parseDerives
 import { parseRouteKinds, parseRpcKinds, routeKindForRequest, rpcKindForId, } from '../db/routeKinds.js';
 import { decodeResponseEnvelope, encodeRequestEnvelope, unpackHandleResult, } from '../http/envelope.js';
 import { buildHostImports, freshDispatchState } from './host.js';
+import { envelopeWindowOf } from '../wasm/envelope-window.js';
+import { registerMaterializer, unregisterMaterializer } from '../db/materialization.js';
 export { WasmAbortError } from './host.js';
 export const UNHANDLED_HEADER = 'x-toil-unhandled';
 const WASM_PAGE = 65536;
@@ -31,6 +33,7 @@ const PROVIDED_IMPORTS = new Set([
     'respond_file',
     'thread_spawn',
     'Date.now',
+    'server_log',
     'client_ip',
     'ratelimit_check',
     'email_send',
@@ -68,20 +71,26 @@ const PROVIDED_IMPORTS = new Set([
     'data.unique_release',
     'data.view_get',
     'data.view_publish',
+    'data.view_delete',
     'data.membership_contains',
     'data.membership_add',
     'data.membership_remove',
     'data.membership_list',
+    'data.membership_page',
     'data.counter_get',
     'data.counter_add',
     'data.append',
     'data.append_once',
     'data.enqueue',
     'data.compare_exchange',
+    'data.compare_delete',
     'data.get_current',
     'data.latest',
     'data.events_since',
     'data.events_get',
+    'data.events_reference',
+    'data.events_redact',
+    'data.events_entries',
     'data.events_last',
     'data.capacity_set_total',
     'data.capacity_available',
@@ -100,6 +109,8 @@ export class WasmServerModule {
     rpcKinds = [];
     derives = [];
     derivesDirty = false;
+    envelopeWindow = null;
+    pendingDerives = new Set();
     constructor(wasmPath) {
         this.wasmPath = wasmPath;
     }
@@ -112,6 +123,7 @@ export class WasmServerModule {
             mtimeMs = fs.statSync(this.wasmPath).mtimeMs;
         }
         catch {
+            unregisterMaterializer(this.wasmPath);
             this.module = null;
             this.routeKinds = [];
             this.rpcKinds = [];
@@ -123,6 +135,7 @@ export class WasmServerModule {
         if (this.module !== null && mtimeMs === this.loadedMtimeMs)
             return false;
         const bytes = fs.readFileSync(this.wasmPath);
+        this.envelopeWindow = envelopeWindowOf(bytes);
         const module = new WebAssembly.Module(bytes);
         this.assertImportSurface(module);
         this.assertExportSurface(module);
@@ -133,13 +146,21 @@ export class WasmServerModule {
         this.module = module;
         this.loadedMtimeMs = mtimeMs;
         this.derivesDirty = this.derives.length > 0;
+        registerMaterializer(this.wasmPath, (written) => {
+            if (!fs.existsSync(this.wasmPath)) {
+                unregisterMaterializer(this.wasmPath);
+                return;
+            }
+            this.runAffectedDerives(written);
+            persistDb();
+        });
         return true;
     }
     dispatch(req) {
         if (this.module === null)
             throw new Error(`server wasm not loaded (${this.wasmPath})`);
         this.rebuildDerivedViewsIfStale();
-        const envelope = encodeRequestEnvelope(req);
+        const envelope = this.requestEnvelope(req);
         const ref = { memory: null };
         const state = freshDispatchState();
         state.clientIp = req.clientIp ?? '';
@@ -163,9 +184,7 @@ export class WasmServerModule {
         const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
         const exports = instance.exports;
         ref.memory = exports.memory;
-        const reqOfs = exports.memory.buffer.byteLength;
-        exports.memory.grow(Math.ceil(envelope.length / WASM_PAGE) || 1);
-        Buffer.from(exports.memory.buffer).set(envelope, reqOfs);
+        const reqOfs = this.writeRequestEnvelope(exports.memory, envelope);
         const packed = exports.handle(reqOfs, envelope.length);
         const { ptr, len } = unpackHandleResult(packed);
         const memSize = exports.memory.buffer.byteLength;
@@ -188,7 +207,8 @@ export class WasmServerModule {
     dispatchRender(req) {
         if (this.module === null)
             throw new Error(`server wasm not loaded (${this.wasmPath})`);
-        const envelope = encodeRequestEnvelope(req);
+        this.rebuildDerivedViewsIfStale();
+        const envelope = this.requestEnvelope(req);
         const ref = { memory: null };
         const state = freshDispatchState();
         state.clientIp = req.clientIp ?? '';
@@ -198,9 +218,7 @@ export class WasmServerModule {
         if (typeof exports.render !== 'function')
             throw new Error(`guest wasm has no 'render' export (${this.wasmPath})`);
         ref.memory = exports.memory;
-        const reqOfs = exports.memory.buffer.byteLength;
-        exports.memory.grow(Math.ceil(envelope.length / WASM_PAGE) || 1);
-        Buffer.from(exports.memory.buffer).set(envelope, reqOfs);
+        const reqOfs = this.writeRequestEnvelope(exports.memory, envelope);
         const packed = exports.render(reqOfs, envelope.length);
         const { ptr, len } = unpackHandleResult(packed);
         const memSize = exports.memory.buffer.byteLength;
@@ -211,9 +229,13 @@ export class WasmServerModule {
     runAffectedDerives(written) {
         if (this.module === null)
             return;
-        for (const derive of derivesForWrites(this.derives, written)) {
+        const affected = derivesForWrites(this.derives, written);
+        for (const derive of affected)
+            this.pendingDerives.add(derive.deriveId);
+        for (const derive of this.derives.filter((d) => this.pendingDerives.has(d.deriveId))) {
             try {
                 this.runDerive(derive.deriveId);
+                this.pendingDerives.delete(derive.deriveId);
             }
             catch (err) {
                 console.error(`[toil] derive ${derive.dbName}#${derive.methodName} failed:`, err);
@@ -221,6 +243,7 @@ export class WasmServerModule {
         }
     }
     rebuildDerivedViewsIfStale() {
+        this.runAffectedDerives(new Set());
         if (!this.derivesDirty)
             return;
         this.derivesDirty = false;
@@ -248,6 +271,21 @@ export class WasmServerModule {
         exports.derive_run(deriveId);
         commitDeriveCheckpoints(state.db);
     }
+    requestEnvelope(req) {
+        const envelope = encodeRequestEnvelope(req);
+        if (envelope.length > 8 * 1024 * 1024)
+            throw new Error('request envelope exceeds the native 8 MiB request limit');
+        return envelope;
+    }
+    writeRequestEnvelope(memory, envelope) {
+        const fits = this.envelopeWindow === null || envelope.length <= this.envelopeWindow;
+        const offset = fits ? 0 : memory.buffer.byteLength;
+        const needed = offset + envelope.length;
+        if (needed > memory.buffer.byteLength)
+            memory.grow(Math.ceil((needed - memory.buffer.byteLength) / WASM_PAGE));
+        Buffer.from(memory.buffer).set(envelope, offset);
+        return offset;
+    }
     assertImportSurface(module) {
         const missing = WebAssembly.Module.imports(module)
             .filter((i) => i.kind === 'function' && (i.module !== 'env' || !PROVIDED_IMPORTS.has(i.name)))

package/build/devserver/wasm/envelope-window.d.ts

@@ -0,0 +1 @@
+export declare function envelopeWindowOf(wasm: Buffer): number | null;

package/build/devserver/wasm/envelope-window.js

@@ -0,0 +1,64 @@
+export function envelopeWindowOf(wasm) {
+    if (wasm.length < 8 || !wasm.subarray(0, 8).equals(Buffer.from([0, 97, 115, 109, 1, 0, 0, 0])))
+        throw new Error('invalid wasm header');
+    let pos = 8;
+    let end = wasm.length;
+    let window = null;
+    const byte = () => {
+        if (pos >= end)
+            throw new Error('truncated wasm data section');
+        return wasm[pos++];
+    };
+    const integer = (signed = false) => {
+        let value = 0;
+        let shift = 0;
+        for (let i = 0; i < 5; i++) {
+            const b = byte();
+            value += (b & 127) * 2 ** shift;
+            shift += 7;
+            if ((b & 128) === 0) {
+                if (signed && b & 64)
+                    value -= 2 ** shift;
+                if (value < (signed ? -0x80000000 : 0) ||
+                    value > (signed ? 0x7fffffff : 0xffffffff))
+                    throw new Error('invalid wasm data integer');
+                return value;
+            }
+        }
+        throw new Error('invalid wasm data integer');
+    };
+    while (pos < wasm.length) {
+        end = wasm.length;
+        const id = byte();
+        const size = integer();
+        end = pos + size;
+        if (end > wasm.length)
+            throw new Error('truncated wasm section');
+        if (id === 11) {
+            const count = integer();
+            for (let i = 0; i < count; i++) {
+                const flags = integer();
+                if (flags === 0 || flags === 2) {
+                    if (flags === 2 && integer() !== 0)
+                        throw new Error('unsupported data memory');
+                    if (byte() !== 0x41)
+                        throw new Error('data offset must be a constant i32');
+                    const offset = integer(true);
+                    if (offset < 0 || byte() !== 0x0b)
+                        throw new Error('invalid data offset');
+                    window = window === null ? offset : Math.min(window, offset);
+                }
+                else if (flags !== 1)
+                    throw new Error('unsupported data segment flags');
+                const length = integer();
+                pos += length;
+                if (pos > end)
+                    throw new Error('truncated wasm data segment');
+            }
+            if (pos !== end)
+                throw new Error('invalid wasm data section framing');
+        }
+        pos = end;
+    }
+    return window;
+}