toiljs 0.0.124 → 0.0.125

package/build/devserver/daemon/index.js

@@ -1,6 +1,9 @@
 import fs from 'node:fs';
 import { performance } from 'node:perf_hooks';
 import pc from 'picocolors';
+import { DbFunctionKind, persistDb, parseDerives, derivesForWrites, commitDeriveCheckpoints, setDbCatalog, } from '../db/index.js';
+import { flushMaterializations, queueMaterialization } from '../db/materialization.js';
+import { resetDbLimits } from '../db/limits.js';
 import { parseSurface } from '../wasm/surface.js';
 import { parseDaemonCatalog, } from './catalog.js';
 import { cronMatches, cronNeverFires, nextCronFireMs } from './cron.js';
@@ -35,8 +38,11 @@ export class DaemonHost {
     running = false;
     epochValue = 0n;
     timers = new Map();
+    deriveRetryTimer = null;
     nextFire = new Map();
     ticking = new Set();
+    derives = [];
+    pendingDerives = new Set();
     constructor(coldWasmPath, cfg, nodeMode, log = (s) => process.stdout.write(s)) {
         this.coldWasmPath = coldWasmPath;
         this.cfg = cfg;
@@ -115,6 +121,11 @@ export class DaemonHost {
             return false;
         }
         this.module = new WebAssembly.Module(bytes);
+        setDbCatalog(bytes, true);
+        this.derives = parseDerives(bytes);
+        this.pendingDerives.clear();
+        for (const derive of this.derives)
+            this.pendingDerives.add(derive.deriveId);
         this.catalog = catalog;
         this.loadedMtimeMs = mtimeMs;
         this.epochValue += 1n;
@@ -126,6 +137,7 @@ export class DaemonHost {
             return;
         const ref = { memory: null };
         this.state = freshDaemonState();
+        this.state.db.functionKind = DbFunctionKind.Action;
         const imports = buildDaemonImports(ref, this.state, this);
         const provided = imports;
         const missing = WebAssembly.Module.imports(this.module)
@@ -139,9 +151,24 @@ export class DaemonHost {
         this.instance = new WebAssembly.Instance(this.module, imports);
         this.exports = this.instance.exports;
         ref.memory = this.exports.memory;
+        if (this.derives.length > 0 && typeof this.exports.derive_run !== 'function') {
+            this.log(pc.red('  ✗ cold materializers require derive_run(i32) -> i64') + '\n');
+            this.instance = null;
+            this.exports = null;
+            return;
+        }
         try {
-            if (typeof this.exports.init === 'function')
-                this.exports.init();
+            if (typeof this.exports.init === 'function') {
+                const initRc = this.exports.init();
+                if (initRc !== 0) {
+                    this.log(pc.red(`  ✗ init() returned ${String(initRc)}; daemon not running`) + '\n');
+                    this.instance = null;
+                    this.exports = null;
+                    return;
+                }
+                this.state.db.logMessages = 0;
+                resetDbLimits(this.state.db);
+            }
             const rc = this.exports.daemon_start();
             if (rc !== 0) {
                 this.log(pc.red(`  ✗ daemon_start() returned ${String(rc)}; daemon not running`) + '\n');
@@ -156,7 +183,15 @@ export class DaemonHost {
             this.exports = null;
             return;
         }
+        finally {
+            this.materializeWrites();
+        }
         this.running = true;
+        this.deriveRetryTimer = setInterval(() => {
+            if (this.running && this.pendingDerives.size > 0 && this.isLeader())
+                this.materializeWrites();
+        }, 1000);
+        this.deriveRetryTimer.unref?.();
         const limited = this.catalog.tasks.slice(0, this.cfg.maxTasks);
         for (const task of limited)
             this.registerTask(task);
@@ -165,6 +200,9 @@ export class DaemonHost {
             '\n');
     }
     stop() {
+        if (this.deriveRetryTimer !== null)
+            clearInterval(this.deriveRetryTimer);
+        this.deriveRetryTimer = null;
         for (const t of this.timers.values())
             clearTimeout(t);
         this.timers.clear();
@@ -230,6 +268,10 @@ export class DaemonHost {
         if (!this.isLeader())
             return;
         this.ticking.add(task.taskIndex);
+        this.state.db.functionKind = DbFunctionKind.Action;
+        this.state.db.logMessages = 0;
+        resetDbLimits(this.state.db);
+        this.state.db.writtenCollections.clear();
         const startedAt = performance.now();
         try {
             const ret = this.exports.scheduled_tick(task.taskIndex);
@@ -241,12 +283,52 @@ export class DaemonHost {
             this.log(pc.red(`  ✗ @scheduled ${task.name} trapped: ${String(e)}`) + '\n');
         }
         finally {
+            this.materializeWrites();
             const took = performance.now() - startedAt;
             if (took > this.cfg.tickBudgetMs)
                 this.log(pc.yellow(`  ⏱ @scheduled ${task.name} took ${String(took)}ms (> tickBudgetMs ${String(this.cfg.tickBudgetMs)})`) + '\n');
             this.ticking.delete(task.taskIndex);
         }
     }
+    materializeWrites() {
+        if (this.derives.length > 0 && this.exports?.derive_run !== undefined) {
+            for (const derive of derivesForWrites(this.derives, this.state.db.writtenCollections))
+                this.pendingDerives.add(derive.deriveId);
+            const db = this.state.db;
+            const kind = db.functionKind;
+            try {
+                db.functionKind = DbFunctionKind.Derive;
+                for (const derive of this.derives.filter((d) => this.pendingDerives.has(d.deriveId))) {
+                    db.deriveId = derive.deriveId;
+                    db.logMessages = 0;
+                    resetDbLimits(db);
+                    db.pendingCheckpoints.clear();
+                    try {
+                        const rc = this.exports.derive_run(derive.deriveId);
+                        if (rc !== 0n)
+                            throw new Error(`derive_run returned ${String(rc)}`);
+                        commitDeriveCheckpoints(db);
+                        this.pendingDerives.delete(derive.deriveId);
+                    }
+                    catch (error) {
+                        this.log(pc.red(`  ✗ daemon derive ${derive.dbName}.${derive.methodName} failed; retry in one second: ${String(error)}`) + '\n');
+                        break;
+                    }
+                }
+            }
+            finally {
+                db.functionKind = kind;
+                db.pendingCheckpoints.clear();
+            }
+        }
+        else {
+            for (const collection of this.state.db.writtenCollections)
+                queueMaterialization(collection);
+            flushMaterializations();
+        }
+        persistDb();
+        this.state.db.writtenCollections.clear();
+    }
     close() {
         if (this.running)
             this.stop();

package/build/devserver/db/database.d.ts

@@ -23,10 +23,13 @@ export declare class DevDatabase {
     private readonly capacity;
     private readonly versions;
     private readonly eventVersions;
+    private readonly eventHlcs;
+    private readonly erasedEvents;
+    private eventClock;
     private readonly memberVersions;
     private catalog;
     private persistPath;
-    setCatalog(wasm: Buffer): void;
+    setCatalog(wasm: Buffer, merge?: boolean): void;
     private currentSchemaVersion;
     private stampVersion;
     private recordIdemStart;
@@ -57,8 +60,10 @@ export declare class DevDatabase {
     membershipAdd(ref: MemoryRef, db: DbDevState, handle: number, setPtr: number, setLen: number, memberPtr: number, memberLen: number): number;
     membershipRemove(ref: MemoryRef, db: DbDevState, handle: number, setPtr: number, setLen: number, memberPtr: number, memberLen: number): number;
     membershipList(ref: MemoryRef, db: DbDevState, handle: number, setPtr: number, setLen: number, limit: number): number;
+    membershipPage(ref: MemoryRef, db: DbDevState, handle: number, setPtr: number, setLen: number, cursorPtr: number, cursorLen: number, limit: number): number;
     viewGet(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number): number;
     viewPublish(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, valPtr: number, valLen: number): number;
+    viewDelete(ref: MemoryRef, db: DbDevState, handle: number, kp: number, kl: number): number;
     counterGet(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number): number;
     private recordWrite;
     counterAdd(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, delta: number | bigint, idemPtr: number): number;
@@ -67,7 +72,11 @@ export declare class DevDatabase {
     eventsGet(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, evidPtr: number, evidLen: number): number;
     eventsLast(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number): number;
     private eventResult;
+    eventsReference(ref: MemoryRef, db: DbDevState, handle: number, kp: number, kl: number, ip: number, il: number): number;
+    eventsRedact(ref: MemoryRef, db: DbDevState, handle: number, kp: number, kl: number, seq: bigint, hlc: bigint): number;
+    eventsEntries(ref: MemoryRef, db: DbDevState, handle: number, kp: number, kl: number, limit: number, after: bigint): number;
     compareExchange(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, expectedPtr: number, expectedLen: number, valuePtr: number, valueLen: number): number;
+    compareDelete(ref: MemoryRef, db: DbDevState, handle: number, kp: number, kl: number, ep: number, el: number): number;
     enqueue(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, valPtr: number, valLen: number, idemPtr: number): number;
     latest(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, limit: number): number;
     since(ref: MemoryRef, db: DbDevState, handle: number, keyPtr: number, keyLen: number, limit: number): number;
@@ -83,7 +92,7 @@ export declare class DevDatabase {
     setCatalogForTests(entries: Record<string, CatalogSeedEntry>): void;
 }
 export declare const devDb: DevDatabase;
-export declare function setDbCatalog(wasm: Buffer): void;
+export declare function setDbCatalog(wasm: Buffer, merge?: boolean): void;
 export declare function configureDbPersistence(filePath: string): void;
 export declare function persistDb(): void;
 export declare function commitDeriveCheckpoints(db: DbDevState): void;

package/build/devserver/db/database.js

@@ -3,7 +3,9 @@ import { createHash } from 'node:crypto';
 import path from 'node:path';
 import { DataReader, DataWriter } from 'toiljs/io';
 import { parseCatalog } from './catalog.js';
-import { ABSENT, ALREADY_EXISTS, CODEC_ERR, CollectionFamily, CONFLICT, DbFunctionKind, DEFAULT_FILL_WAIT_MS, INVALID_HANDLE, MAX_KEY, MAX_FILL_WAIT_MS, MAX_NAME, MAX_RESERVATION_TTL_MS, MAX_RESERVATIONS, MAX_VALUE, OP_NOT_ALLOWED_FOR_FAMILY, OP_NOT_ALLOWED_IN_KIND, satI64, SCHEMA_UNAVAILABLE, TOO_MANY_KEYS, TOO_SMALL, UNAVAILABLE, isCollectionFamily, } from './types.js';
+import { queueMaterialization } from './materialization.js';
+import { checkDbResultSize, withDbLimits } from './limits.js';
+import { ABSENT, ALREADY_EXISTS, CODEC_ERR, CollectionFamily, CONFLICT, DbFunctionKind, DEFAULT_FILL_WAIT_MS, INVALID_HANDLE, MAX_KEY, MAX_FILL_WAIT_MS, MAX_NAME, MAX_RESERVATION_TTL_MS, MAX_RESERVATIONS, MAX_VALUE, OP_NOT_ALLOWED_FOR_FAMILY, OP_NOT_ALLOWED_IN_KIND, satI64, SCHEMA_UNAVAILABLE, TOO_SMALL, UNAVAILABLE, isCollectionFamily, } from './types.js';
 function mem(ref) {
     if (!ref.memory)
         throw new Error('data host import called before memory was bound');
@@ -131,10 +133,14 @@ var DbOp;
     DbOp[DbOp["CapacityCancel"] = 24] = "CapacityCancel";
     DbOp[DbOp["ViewGet"] = 25] = "ViewGet";
     DbOp[DbOp["ViewPublish"] = 26] = "ViewPublish";
-    DbOp[DbOp["CapacitySetTotal"] = 27] = "CapacitySetTotal";
-    DbOp[DbOp["EventsSince"] = 28] = "EventsSince";
-    DbOp[DbOp["EventsGet"] = 29] = "EventsGet";
-    DbOp[DbOp["EventsLast"] = 30] = "EventsLast";
+    DbOp[DbOp["ViewDelete"] = 27] = "ViewDelete";
+    DbOp[DbOp["CapacitySetTotal"] = 28] = "CapacitySetTotal";
+    DbOp[DbOp["EventsSince"] = 29] = "EventsSince";
+    DbOp[DbOp["EventsGet"] = 30] = "EventsGet";
+    DbOp[DbOp["EventsLast"] = 31] = "EventsLast";
+    DbOp[DbOp["EventsEntries"] = 32] = "EventsEntries";
+    DbOp[DbOp["EventsReference"] = 33] = "EventsReference";
+    DbOp[DbOp["EventsRedact"] = 34] = "EventsRedact";
 })(DbOp || (DbOp = {}));
 var UpsertResult;
 (function (UpsertResult) {
@@ -155,10 +161,12 @@ function isReadOp(op) {
         op === DbOp.EventsSince ||
         op === DbOp.EventsGet ||
         op === DbOp.EventsLast ||
+        op === DbOp.EventsEntries ||
+        op === DbOp.EventsReference ||
         op === DbOp.CapacityAvailable);
 }
 function isScanOp(op) {
-    return op === DbOp.Latest || op === DbOp.MembershipList || op === DbOp.EventsSince;
+    return op === DbOp.Latest || op === DbOp.MembershipList || op === DbOp.EventsSince || op === DbOp.EventsEntries;
 }
 function kindAllows(kind, op) {
     if (kind === DbFunctionKind.Query)
@@ -169,10 +177,12 @@ function kindAllows(kind, op) {
             op === DbOp.Patch ||
             op === DbOp.Upsert ||
             op === DbOp.Delete ||
+            op === DbOp.ViewDelete ||
             op === DbOp.GetDelete ||
             op === DbOp.Enqueue ||
             op === DbOp.Append ||
             op === DbOp.AppendOnce ||
+            op === DbOp.EventsRedact ||
             op === DbOp.CounterAdd ||
             op === DbOp.MembershipAdd ||
             op === DbOp.MembershipRemove ||
@@ -183,7 +193,11 @@ function kindAllows(kind, op) {
             op === DbOp.CapacityCancel);
     }
     if (kind === DbFunctionKind.Derive)
-        return (isReadOp(op) || op === DbOp.ViewPublish || op === DbOp.Append || op === DbOp.CounterAdd);
+        return (isReadOp(op) ||
+            op === DbOp.ViewPublish ||
+            op === DbOp.ViewDelete ||
+            op === DbOp.Append ||
+            op === DbOp.CounterAdd);
     if (kind === DbFunctionKind.Job)
         return true;
     return false;
@@ -209,11 +223,20 @@ export class DevDatabase {
     capacity = new Map();
     versions = new Map();
     eventVersions = new Map();
+    eventHlcs = new Map();
+    erasedEvents = new Map();
+    eventClock = 0n;
     memberVersions = new Map();
     catalog = { kind: 'no-section' };
     persistPath = null;
-    setCatalog(wasm) {
-        this.catalog = parseCatalog(wasm);
+    setCatalog(wasm, merge = false) {
+        const incoming = parseCatalog(wasm);
+        if (merge && this.catalog.kind === 'present' && incoming.kind === 'present') {
+            for (const [name, collection] of incoming.collections)
+                this.catalog.collections.set(name, collection);
+        }
+        else if (!merge || incoming.kind !== 'no-section')
+            this.catalog = incoming;
     }
     currentSchemaVersion(coll) {
         if (this.catalog.kind !== 'present')
@@ -283,6 +306,7 @@ export class DevDatabase {
             events: {},
             eventDedup: {},
             eventPositions: {},
+            erasedEvents: {},
             deriveCheckpoints: {},
             capacity: {},
         };
@@ -311,10 +335,12 @@ export class DevDatabase {
             snap.counterIdem[k] = v.toString();
         for (const [k, log] of this.events) {
             const ver = this.eventVersions.get(k) ?? [];
-            snap.events[k] = log.map((b, i) => ({ v: b.toString('base64'), sv: ver[i] ?? 0 }));
+            snap.events[k] = log.map((b, i) => ({ v: b.toString('base64'), sv: ver[i] ?? 0, hlc: (this.eventHlcs.get(k)?.[i] ?? BigInt(i + 1)).toString() }));
         }
         for (const [k, positions] of this.eventPositions)
             snap.eventPositions[k] = [...positions];
+        for (const [k, erased] of this.erasedEvents)
+            snap.erasedEvents[k] = [...erased];
         for (const [k, s] of this.eventDedup)
             snap.eventDedup[k] = [...s];
         for (const [k, v] of this.deriveCheckpoints)
@@ -383,6 +409,21 @@ export class DevDatabase {
         for (const [k, log] of Object.entries(snap.events ?? {})) {
             this.events.set(k, log.map((e) => Buffer.from(e.v, 'base64')));
             this.eventVersions.set(k, log.map((e) => e.sv));
+            const hlcs = log.map((e, i) => BigInt(e.hlc ?? String(i + 1)));
+            this.eventHlcs.set(k, hlcs);
+            for (const hlc of hlcs)
+                if (hlc > this.eventClock)
+                    this.eventClock = hlc;
+        }
+        for (const [k, values] of Object.entries(snap.erasedEvents ?? {})) {
+            const erased = new Set(values);
+            this.erasedEvents.set(k, erased);
+            const log = this.events.get(k) ?? [];
+            const hlcs = this.eventHlcs.get(k) ?? [];
+            for (let i = 0; i < log.length; i++) {
+                if (erased.has(String(i + 1) + ':' + String(hlcs[i] ?? BigInt(i + 1))))
+                    log[i] = Buffer.alloc(0);
+            }
         }
         for (const [k, positions] of Object.entries(snap.eventPositions ?? {}))
             this.eventPositions.set(k, new Map(positions));
@@ -416,6 +457,9 @@ export class DevDatabase {
         this.counters.clear();
         this.counterIdem.clear();
         this.eventVersions.clear();
+        this.eventHlcs.clear();
+        this.erasedEvents.clear();
+        this.eventClock = 0n;
         this.events.clear();
         this.eventDedup.clear();
         this.eventPositions.clear();
@@ -505,15 +549,18 @@ export class DevDatabase {
                 ? 64
                 : 1024;
         if (count > maxKeys)
-            return TOO_MANY_KEYS;
+            throw new Error(`data.get_many failed: TDL020 batch-key budget exceeded; collection=${coll.name} kind=${db.functionKind} batch_keys=${String(count)} max_batch_keys=${String(maxKeys)}. Split the keys into smaller batches.`);
         const w = new DataWriter();
         w.writeU32(count);
+        let resultBytes = 4;
         for (let i = 0; i < count; i++) {
             const key = r.readBytes();
             if (key.length > MAX_KEY)
                 throw new Error('data: key too long');
             const sk = storeKey(coll.name, Buffer.from(key));
             const v = table.get(sk);
+            resultBytes += v === undefined ? 1 : 9 + v.length;
+            checkDbResultSize(db, resultBytes);
             if (v === undefined) {
                 w.writeU8(0);
             }
@@ -816,6 +863,32 @@ export class DevDatabase {
         db.lastResult = Buffer.from(w.toBytes());
         return db.lastResult.length;
     }
+    membershipPage(ref, db, handle, setPtr, setLen, cursorPtr, cursorLen, limit) {
+        if (cursorLen < -1 || limit < 1 || limit > 0xffff)
+            throw new Error('data.membership_page: invalid cursor length or limit');
+        checkValueSize(cursorLen, 'cursor');
+        const coll = collForOp(db, handle, DbOp.MembershipList, CollectionFamily.Membership);
+        if (typeof coll === 'number')
+            return coll;
+        if (limit > 1024)
+            throw new Error('data.membership_page: TDL020 scan limit exceeds 1024');
+        const cursor = cursorLen === -1 ? null : readCopy(ref, cursorPtr, cursorLen);
+        const sk = storeKey(coll.name, readKey(ref, setPtr, setLen));
+        const members = Array.from(this.members.get(sk)?.values() ?? [])
+            .sort(Buffer.compare)
+            .filter((m) => cursor === null || Buffer.compare(m, cursor) > 0)
+            .slice(0, limit);
+        const versions = this.memberVersions.get(sk);
+        const w = new DataWriter().writeU32(members.length);
+        for (const member of members)
+            w.writeU32(versions?.get(member.toString('latin1')) ?? 0).writeBytes(member);
+        const next = members.length === limit ? members[members.length - 1] : null;
+        w.writeU8(next === null ? 0 : 1);
+        if (next !== null)
+            w.writeBytes(next);
+        db.lastResult = Buffer.from(w.toBytes());
+        return db.lastResult.length;
+    }
     viewGet(ref, db, handle, keyPtr, keyLen) {
         const coll = collForOp(db, handle, DbOp.ViewGet, CollectionFamily.View);
         if (typeof coll === 'number')
@@ -838,6 +911,15 @@ export class DevDatabase {
         this.stampVersion(coll, sk);
         return 0;
     }
+    viewDelete(ref, db, handle, kp, kl) {
+        const coll = collForOp(db, handle, DbOp.ViewDelete, CollectionFamily.View);
+        if (typeof coll === 'number')
+            return coll;
+        const key = storeKey(coll.name, readKey(ref, kp, kl));
+        this.views.delete(key);
+        this.versions.delete(key);
+        return 0;
+    }
     counterGet(ref, db, handle, keyPtr, keyLen) {
         const coll = collForOp(db, handle, DbOp.CounterGet, CollectionFamily.Counter);
         if (typeof coll === 'number')
@@ -852,6 +934,8 @@ export class DevDatabase {
         if (db.functionKind === DbFunctionKind.Derive)
             return;
         db.writtenCollections.add(coll.name);
+        if (db.functionKind === DbFunctionKind.Job)
+            queueMaterialization(coll.name);
     }
     counterAdd(ref, db, handle, keyPtr, keyLen, delta, idemPtr) {
         const coll = collForOp(db, handle, DbOp.CounterAdd, CollectionFamily.Counter);
@@ -904,6 +988,11 @@ export class DevDatabase {
             versions.push(sv);
             this.eventVersions.set(sk, versions);
         }
+        const hlcs = this.eventHlcs.get(sk) ?? [];
+        const now = BigInt(Date.now()) << 16n;
+        this.eventClock = now > this.eventClock ? now : this.eventClock + 1n;
+        hlcs.push(this.eventClock);
+        this.eventHlcs.set(sk, hlcs);
         this.recordWrite(db, coll);
         return 0;
     }
@@ -942,6 +1031,11 @@ export class DevDatabase {
         }
         positions.set(evid, position);
         seen.add(evid);
+        const hlcs = this.eventHlcs.get(sk) ?? [];
+        const now = BigInt(Date.now()) << 16n;
+        this.eventClock = now > this.eventClock ? now : this.eventClock + 1n;
+        hlcs.push(this.eventClock);
+        this.eventHlcs.set(sk, hlcs);
         this.recordWrite(db, coll);
         return 1;
     }
@@ -963,10 +1057,15 @@ export class DevDatabase {
         if (typeof coll === 'number')
             return coll;
         const sk = storeKey(coll.name, readKey(ref, keyPtr, keyLen));
-        return this.eventResult(db, sk, (this.events.get(sk)?.length ?? 0) - 1);
+        const log = this.events.get(sk) ?? [];
+        let position = log.length - 1;
+        while (position >= 0 && log[position].length === 0)
+            position--;
+        return this.eventResult(db, sk, position);
     }
     eventResult(db, sk, position) {
-        const value = position === undefined ? undefined : this.events.get(sk)?.[position];
+        const stored = position === undefined ? undefined : this.events.get(sk)?.[position];
+        const value = stored?.length === 0 ? undefined : stored;
         db.lastResult = value === undefined ? null : Buffer.from(value);
         db.lastResultVersion =
             value === undefined || position === undefined
@@ -974,6 +1073,79 @@ export class DevDatabase {
                 : (this.eventVersions.get(sk)?.[position] ?? 0);
         return value === undefined ? ABSENT : value.length;
     }
+    eventsReference(ref, db, handle, kp, kl, ip, il) {
+        const coll = collForOp(db, handle, DbOp.EventsReference, CollectionFamily.Events);
+        if (typeof coll === 'number')
+            return coll;
+        if (il < 0 || il > MAX_KEY)
+            throw new Error('Invalid event ID length');
+        const sk = storeKey(coll.name, readKey(ref, kp, kl));
+        const id = readCopy(ref, ip, il).toString('latin1');
+        const position = this.eventPositions.get(sk)?.get(id);
+        if (position === undefined)
+            return this.eventDedup.get(sk)?.has(id) ? UNAVAILABLE : ABSENT;
+        const bytes = Buffer.alloc(16);
+        bytes.writeBigUInt64LE(BigInt(position + 1));
+        bytes.writeBigUInt64LE(this.eventHlcs.get(sk)?.[position] ?? BigInt(position + 1), 8);
+        db.lastResult = bytes;
+        db.lastResultVersion = -1;
+        return bytes.length;
+    }
+    eventsRedact(ref, db, handle, kp, kl, seq, hlc) {
+        const coll = collForOp(db, handle, DbOp.EventsRedact, CollectionFamily.Events);
+        if (typeof coll === 'number')
+            return coll;
+        if (seq <= 0n || hlc < 0n)
+            throw new Error('Invalid event reference');
+        const sk = storeKey(coll.name, readKey(ref, kp, kl));
+        let erased = this.erasedEvents.get(sk);
+        if (erased === undefined) {
+            erased = new Set();
+            this.erasedEvents.set(sk, erased);
+        }
+        erased.add(seq.toString() + ':' + hlc.toString());
+        const position = Number(seq - 1n);
+        const log = this.events.get(sk);
+        if (Number.isSafeInteger(position) && position >= 0 && log !== undefined && position < log.length &&
+            (this.eventHlcs.get(sk)?.[position] ?? seq) === hlc)
+            log[position] = Buffer.alloc(0);
+        for (const key of this.deriveCheckpoints.keys())
+            if (key.endsWith('\0' + sk))
+                this.deriveCheckpoints.delete(key);
+        for (const key of db.pendingCheckpoints.keys())
+            if (key.endsWith('\0' + sk))
+                db.pendingCheckpoints.delete(key);
+        this.recordWrite(db, coll);
+        return 1;
+    }
+    eventsEntries(ref, db, handle, kp, kl, limit, after) {
+        const coll = collForOp(db, handle, DbOp.EventsEntries, CollectionFamily.Events);
+        if (typeof coll === 'number')
+            return coll;
+        if (!Number.isInteger(limit) || limit < 1 || limit > 1024 || after < 0n || after > BigInt(Number.MAX_SAFE_INTEGER))
+            throw new Error('Invalid event page cursor or limit');
+        const sk = storeKey(coll.name, readKey(ref, kp, kl));
+        const log = this.events.get(sk) ?? [];
+        const positions = [];
+        let cursor = Number(after);
+        while (cursor < log.length && positions.length < limit) {
+            if (log[cursor].length > 0)
+                positions.push(cursor);
+            cursor++;
+        }
+        const writer = new DataWriter().writeU32(positions.length);
+        for (const position of positions)
+            writer.writeU64(BigInt(position + 1))
+                .writeU64(this.eventHlcs.get(sk)?.[position] ?? BigInt(position + 1))
+                .writeU32(this.eventVersions.get(sk)?.[position] ?? 0).writeBytes(log[position]);
+        const hasMore = positions.length === limit;
+        writer.writeU8(hasMore ? 1 : 0);
+        if (hasMore)
+            writer.writeU64(BigInt(cursor));
+        db.lastResult = Buffer.from(writer.toBytes());
+        db.lastResultVersion = -1;
+        return db.lastResult.length;
+    }
     compareExchange(ref, db, handle, keyPtr, keyLen, expectedPtr, expectedLen, valuePtr, valueLen) {
         const coll = collForOp(db, handle, DbOp.Patch, CollectionFamily.Record);
         if (typeof coll === 'number')
@@ -997,6 +1169,26 @@ export class DevDatabase {
         this.recordWrite(db, coll);
         return 1;
     }
+    compareDelete(ref, db, handle, kp, kl, ep, el) {
+        const coll = collForOp(db, handle, DbOp.Delete, CollectionFamily.Record);
+        if (typeof coll === 'number')
+            return coll;
+        checkKeyValueSize(kl, el, 'expected value');
+        checkValueSize(el, 'expected value');
+        if (el < 0)
+            throw new Error('data: conditional delete requires an expected value');
+        const expected = readCopy(ref, ep, el);
+        if (this.uniqueValuesOf(coll, expected).length > 0)
+            throw new Error('Conditional delete does not support @unique fields');
+        const sk = storeKey(coll.name, readKey(ref, kp, kl));
+        const prior = this.store.get(sk);
+        if (prior === undefined || !prior.equals(expected))
+            return 0;
+        this.store.delete(sk);
+        this.versions.delete(sk);
+        this.recordWrite(db, coll);
+        return 1;
+    }
     enqueue(ref, db, handle, keyPtr, keyLen, valPtr, valLen, idemPtr) {
         const coll = collForOp(db, handle, DbOp.Enqueue, CollectionFamily.Record);
         if (typeof coll === 'number')
@@ -1028,9 +1220,9 @@ export class DevDatabase {
         const log = this.events.get(sk) ?? [];
         const vers = this.eventVersions.get(sk) ?? [];
         const n = Math.max(0, Math.min(limit, 0xffff));
-        const start = Math.max(0, log.length - n);
-        const newest = log.slice(start).reverse();
-        const newestVers = vers.slice(start).reverse();
+        const positions = n === 0 ? [] : log.map((_, i) => i).filter(i => log[i].length > 0).slice(-n).reverse();
+        const newest = positions.map(i => log[i]);
+        const newestVers = positions.map(i => vers[i]);
         const w = new DataWriter();
         w.writeU32(newest.length);
         for (let i = 0; i < newest.length; i++) {
@@ -1049,9 +1241,15 @@ export class DevDatabase {
         const vers = this.eventVersions.get(sk) ?? [];
         const cursor = db.pendingCheckpoints.get(ckKey) ?? this.deriveCheckpoints.get(ckKey) ?? 0;
         const n = Math.max(0, Math.min(limit, 0xffff));
-        const end = Math.min(log.length, cursor + n);
-        const batch = log.slice(cursor, end);
-        const batchVers = vers.slice(cursor, end);
+        const positions = [];
+        let end = cursor;
+        while (end < log.length && positions.length < n) {
+            if (log[end].length > 0)
+                positions.push(end);
+            end++;
+        }
+        const batch = positions.map(i => log[i]);
+        const batchVers = positions.map(i => vers[i]);
         db.pendingCheckpoints.set(ckKey, end);
         const w = new DataWriter();
         w.writeU32(batch.length);
@@ -1204,8 +1402,8 @@ export class DevDatabase {
     }
 }
 export const devDb = new DevDatabase();
-export function setDbCatalog(wasm) {
-    devDb.setCatalog(wasm);
+export function setDbCatalog(wasm, merge = false) {
+    devDb.setCatalog(wasm, merge);
 }
 export function configureDbPersistence(filePath) {
     devDb.configurePersistence(filePath);
@@ -1217,7 +1415,7 @@ export function commitDeriveCheckpoints(db) {
     devDb.commitDeriveCheckpoints(db);
 }
 export function buildDatabaseImports(ref, db) {
-    return {
+    const imports = {
         'data.resolve_collection': (namePtr, nameLen, outHandlePtr) => devDb.resolveCollection(ref, db, namePtr, nameLen, outHandlePtr),
         'data.get': (handle, keyPtr, keyLen) => devDb.get(ref, db, handle, keyPtr, keyLen),
         'data.get_many': (handle, keysPtr, keysLen) => devDb.getMany(ref, db, handle, keysPtr, keysLen),
@@ -1233,7 +1431,9 @@ export function buildDatabaseImports(ref, db) {
         'data.membership_contains': (handle, setPtr, setLen, memberPtr, memberLen) => devDb.membershipContains(ref, db, handle, setPtr, setLen, memberPtr, memberLen),
         'data.membership_add': (handle, setPtr, setLen, memberPtr, memberLen, _idemPtr) => devDb.membershipAdd(ref, db, handle, setPtr, setLen, memberPtr, memberLen),
         'data.membership_remove': (handle, setPtr, setLen, memberPtr, memberLen, _idemPtr) => devDb.membershipRemove(ref, db, handle, setPtr, setLen, memberPtr, memberLen),
+        'data.membership_page': (h, sp, sl, cp, cl, limit) => devDb.membershipPage(ref, db, h, sp, sl, cp, cl, limit),
         'data.membership_list': (handle, setPtr, setLen, limit) => devDb.membershipList(ref, db, handle, setPtr, setLen, limit),
+        'data.view_delete': (h, kp, kl, _idem) => devDb.viewDelete(ref, db, h, kp, kl),
         'data.view_get': (handle, keyPtr, keyLen) => devDb.viewGet(ref, db, handle, keyPtr, keyLen),
         'data.view_publish': (handle, keyPtr, keyLen, valPtr, valLen, _idemPtr) => devDb.viewPublish(ref, db, handle, keyPtr, keyLen, valPtr, valLen),
         'data.counter_get': (handle, keyPtr, keyLen) => devDb.counterGet(ref, db, handle, keyPtr, keyLen),
@@ -1241,9 +1441,13 @@ export function buildDatabaseImports(ref, db) {
         'data.append': (handle, keyPtr, keyLen, evPtr, evLen, idemPtr) => devDb.append(ref, db, handle, keyPtr, keyLen, evPtr, evLen, idemPtr),
         'data.append_once': (handle, keyPtr, keyLen, evidPtr, evidLen, evPtr, evLen) => devDb.appendOnce(ref, db, handle, keyPtr, keyLen, evidPtr, evidLen, evPtr, evLen),
         'data.get_current': (handle, kp, kl) => devDb.get(ref, db, handle, kp, kl),
+        'data.compare_delete': (h, kp, kl, ep, el) => devDb.compareDelete(ref, db, h, kp, kl, ep, el),
         'data.compare_exchange': (handle, kp, kl, ep, el, vp, vl) => devDb.compareExchange(ref, db, handle, kp, kl, ep, el, vp, vl),
         'data.enqueue': (handle, keyPtr, keyLen, valPtr, valLen, idemPtr) => devDb.enqueue(ref, db, handle, keyPtr, keyLen, valPtr, valLen, idemPtr),
         'data.latest': (handle, keyPtr, keyLen, limit) => devDb.latest(ref, db, handle, keyPtr, keyLen, limit),
+        'data.events_reference': (h, kp, kl, ip, il) => devDb.eventsReference(ref, db, h, kp, kl, ip, il),
+        'data.events_redact': (h, kp, kl, seq, hlc) => devDb.eventsRedact(ref, db, h, kp, kl, BigInt(seq), BigInt(hlc)),
+        'data.events_entries': (h, kp, kl, limit, after) => devDb.eventsEntries(ref, db, h, kp, kl, limit, BigInt(after)),
         'data.events_get': (handle, keyPtr, keyLen, evidPtr, evidLen) => devDb.eventsGet(ref, db, handle, keyPtr, keyLen, evidPtr, evidLen),
         'data.events_last': (handle, keyPtr, keyLen) => devDb.eventsLast(ref, db, handle, keyPtr, keyLen),
         'data.events_since': (handle, keyPtr, keyLen, limit) => devDb.since(ref, db, handle, keyPtr, keyLen, limit),
@@ -1256,6 +1460,7 @@ export function buildDatabaseImports(ref, db) {
         'data.result_schema_version': () => devDb.resultSchemaVersion(db),
         'data.write_allowed': () => (kindAllows(db.functionKind, DbOp.Patch) ? 1 : 0),
     };
+    return withDbLimits(imports, db);
 }
 export function __resetDbForTests() {
     devDb.resetForTests();

package/build/devserver/db/limits.d.ts

@@ -0,0 +1,6 @@
+import { type DbDevState } from './types.js';
+type Imports = Record<string, (...args: number[]) => number | bigint>;
+export declare function resetDbLimits(db: DbDevState): void;
+export declare function checkDbResultSize(db: DbDevState, bytes: number): void;
+export declare function withDbLimits(imports: Imports, db: DbDevState): Imports;
+export {};