toiljs 0.0.59 → 0.0.61

package/src/devserver/mstore/store.ts

@@ -0,0 +1,121 @@
+/**
+ * Dev MemoryStore: a single in-memory `Map` with per-entry TTL, one per process,
+ * persisted nowhere (ephemeral by definition). Backs the `mstore.*` host imports
+ * (RECONCILIATION Part 4, F2: HANDLELESS, ttl in SECONDS, inline drain). Keys are
+ * auto-scoped to host+region; the dev process is one host/region, so the key is
+ * used verbatim. Shared by streams (Phase 4) AND the daemon (both reference the
+ * same `devMemoryStore` singleton), matching doc 06's "shared across
+ * streams/handlers on the same region".
+ *
+ * TTL is enforced LAZILY on read (no background sweep), mirroring the dev DB's
+ * no-background-thread design. The error space is RECONCILIATION Part 3's 0x03xx
+ * registry; the host-import layer (daemon/host.ts) maps these results onto the
+ * Part 3 negative-return bridge.
+ */
+
+interface MStoreEntry {
+    value: Buffer;
+    /** `0` means no TTL; otherwise the epoch-ms the entry expires at. */
+    expiresAtMs: number;
+}
+
+export class DevMemoryStore {
+    private readonly map = new Map<string, MStoreEntry>();
+
+    private now(): number {
+        return Date.now();
+    }
+
+    /** The live entry for `key`, collecting it lazily if its TTL has passed. */
+    private live(key: string): MStoreEntry | null {
+        const e = this.map.get(key);
+        if (e === undefined) return null;
+        if (e.expiresAtMs !== 0 && e.expiresAtMs <= this.now()) {
+            this.map.delete(key);
+            return null;
+        }
+        return e;
+    }
+
+    private exp(ttlSecs: number): number {
+        return ttlSecs > 0 ? this.now() + ttlSecs * 1000 : 0;
+    }
+
+    /** The value, or `null` (=> 0x0301 MSTORE_NOT_FOUND). */
+    get(key: string): Buffer | null {
+        const e = this.live(key);
+        return e ? e.value : null;
+    }
+
+    set(key: string, value: Buffer, ttlSecs: number): void {
+        this.map.set(key, { value: Buffer.from(value), expiresAtMs: this.exp(ttlSecs) });
+    }
+
+    delete(key: string): boolean {
+        return this.map.delete(key);
+    }
+
+    /** Add `delta` to the i64 stored at `key`; `null` => 0x0306 MSTORE_NOT_A_NUMBER. */
+    incr(key: string, delta: bigint, ttlSecs: number): bigint | null {
+        const e = this.live(key);
+        let cur = 0n;
+        if (e !== null) {
+            const s = e.value.toString('utf8').trim();
+            if (!/^-?\d+$/.test(s)) return null;
+            try {
+                cur = BigInt(s);
+            } catch {
+                return null;
+            }
+        }
+        const next = BigInt.asIntN(64, cur + delta);
+        this.map.set(key, {
+            value: Buffer.from(next.toString(), 'utf8'),
+            // An incr on an existing key keeps its TTL unless a new one is given.
+            expiresAtMs: ttlSecs > 0 ? this.exp(ttlSecs) : (e?.expiresAtMs ?? 0),
+        });
+        return next;
+    }
+
+    /** `expect === null` means expect-absent (the dev mapping of `expect_len == 0`).
+     *  Returns `false` => 0x0304 MSTORE_CONFLICT. */
+    cas(key: string, expect: Buffer | null, next: Buffer, ttlSecs: number): boolean {
+        const e = this.live(key);
+        if (expect === null) {
+            if (e !== null) return false; // expect-absent, but present
+        } else if (e === null || !e.value.equals(expect)) {
+            return false; // expect-match failed
+        }
+        this.map.set(key, { value: Buffer.from(next), expiresAtMs: this.exp(ttlSecs) });
+        return true;
+    }
+
+    /** Re-arm the TTL of a live key; `false` => key absent (0x0301). */
+    expire(key: string, ttlSecs: number): boolean {
+        const e = this.live(key);
+        if (!e) return false;
+        e.expiresAtMs = this.exp(ttlSecs);
+        return true;
+    }
+
+    /**
+     * Prefix walk. `cursor` is an opaque resume index; a stale cursor (one that
+     * points past the current live key set after deletions) returns `null`
+     * (=> 0x0307 MSTORE_SCAN_BUSY). Returns the next cursor + the matched keys.
+     */
+    scan(prefix: string, cursor: bigint): { next: bigint; keys: string[] } | null {
+        const live = [...this.map.keys()].filter((k) => this.live(k) !== null && k.startsWith(prefix));
+        live.sort();
+        const start = Number(cursor);
+        if (start < 0 || start > live.length) return null; // stale cursor
+        const batch = live.slice(start);
+        return { next: BigInt(live.length), keys: batch };
+    }
+
+    /** Test-only: drop all entries. */
+    __reset(): void {
+        this.map.clear();
+    }
+}
+
+export const devMemoryStore = new DevMemoryStore();

package/src/devserver/{host.ts → runtime/host.ts}

@@ -17,12 +17,12 @@
  * `WebAssembly.Instance`, so offering the full surface costs nothing.
  */
 
+import { devEnvGet, devEnvGetSecure } from '../config/env.js';
+import { ratelimitCheck } from '../config/ratelimit.js';
+import { buildDatabaseImports, type DbDevState, freshDbState } from '../db/index.js';
+import { EmailStatus, getEmailService } from '../email/index.js';
+import { parseEmailBlob } from '../email/wire.js';
 import { buildCryptoImports, type CryptoState, freshCryptoState } from './crypto.js';
-import { buildDatabaseImports, type DbDevState, freshDbState } from './database.js';
-import { EmailStatus, getEmailService } from './email/index.js';
-import { parseEmailBlob } from './email/wire.js';
-import { devEnvGet, devEnvGetSecure } from './env.js';
-import { ratelimitCheck } from './ratelimit.js';
 
 /** Limits identical to the edge's `set_header` / `respond_file` bounds. */
 const MAX_TOTAL_HEADERS_BYTES = 64 * 1024;
@@ -91,13 +91,33 @@ function mem(ref: MemoryRef): Buffer {
 }
 
 /** Bounds-checked byte read out of guest linear memory. */
-function readBytes(ref: MemoryRef, ptr: number, len: number): Buffer {
+export function readBytes(ref: MemoryRef, ptr: number, len: number): Buffer {
     const m = mem(ref);
     if (ptr < 0 || len < 0 || ptr + len > m.length)
         throw new Error(`host import read out of bounds: ptr=${String(ptr)} len=${String(len)}`);
     return m.subarray(ptr, ptr + len);
 }
 
+/**
+ * Bounds-checked write of a variable-length result into a guest out-buffer, with
+ * the edge's inline-drain return protocol: the byte length on success, or `-1`
+ * (STATUS_TOO_SMALL) when `outCap` is too small (the guest retries with a bigger
+ * buffer). Used by the handleless `mstore.*` imports (RECONCILIATION Part 4 F2).
+ */
+export function writeBytesOut(
+    ref: MemoryRef,
+    bytes: Buffer,
+    outPtr: number,
+    outCap: number,
+): number {
+    if (bytes.length > outCap) return -1; // TOO_SMALL
+    const m = mem(ref);
+    if (outPtr < 0 || outPtr + bytes.length > m.length)
+        throw new Error('host import write out of bounds');
+    bytes.copy(m, outPtr);
+    return bytes.length;
+}
+
 /**
  * Read a ToilScript string (UTF-16LE payload, byte length in the u32 at
  * `ptr - 4`). Used by `abort`, whose pointers reference string objects rather
@@ -169,6 +189,77 @@ function envLookup(
     return bytes.length;
 }
 
+/**
+ * The portion of the `env.*` request surface that is SHARED by the daemon (cold)
+ * box: panic hook, `Environment.get`/`getSecure`, `email_send`, `thread_spawn`,
+ * and `Date.now`. It deliberately EXCLUDES the response/stream functions a cold
+ * box must not have (`set_status`/`set_header`/`respond_file`/`client_ip`/
+ * `ratelimit_check`), which stay in {@link buildHostImports}. None of these read
+ * the per-dispatch response state, so they need only `ref`. The crypto and DB
+ * namespaces are spread on top by each box's loader (they carry their own state).
+ */
+export function buildEnvImports(
+    ref: MemoryRef,
+    _state: { crypto: CryptoState; db: DbDevState },
+): Record<string, (...a: never[]) => unknown> {
+    return {
+        abort: (msgPtr: number, filePtr: number, line: number, col: number): void => {
+            throw new WasmAbortError(
+                readGuestString(ref, msgPtr),
+                readGuestString(ref, filePtr),
+                line,
+                col,
+            );
+        },
+
+        // `Environment.get` / `getSecure`: copy one tenant env value into the
+        // guest buffer. Returns the byte length (0 = present-but-empty), -1 if
+        // the buffer is too small (the guest retries bigger), -2 if absent.
+        env_get: (keyPtr: number, keyLen: number, outPtr: number, outCap: number): number =>
+            envLookup(ref, keyPtr, keyLen, outPtr, outCap, false),
+        env_get_secure: (
+            keyPtr: number,
+            keyLen: number,
+            outPtr: number,
+            outCap: number,
+        ): number => envLookup(ref, keyPtr, keyLen, outPtr, outCap, true),
+
+        thread_spawn: (_startArg: number): number => -1,
+
+        // `Date.now()` -> wall-clock milliseconds, matching the edge host.
+        'Date.now': (): bigint => BigInt(Date.now()),
+
+        // `env::email_send`: the FULL email pipeline in dev. A daemon may send
+        // mail, so this stays in the shared subset (00 B2 / doc 08 AN-8).
+        email_send: (reqPtr: number, reqLen: number): number => {
+            const raw = readBytes(ref, reqPtr, reqLen);
+            const svc = getEmailService();
+            if (svc === null) {
+                const to = parseEmailBlob(raw)?.to ?? '<unparsed>';
+                process.stdout.write(`  ✉ dev email_send -> ${to} (no email config; not sent)\n`);
+                return EmailStatus.Sent;
+            }
+            const { status, parsed } = svc.prepare(raw);
+            if (parsed === null) {
+                process.stdout.write(`  ✉ dev email_send -> ${EmailStatus[status]}\n`);
+                return status;
+            }
+            void svc
+                .deliver(parsed)
+                .then((s) => {
+                    const label = s === EmailStatus.Sent ? 'sent' : EmailStatus[s];
+                    process.stdout.write(`  ✉ dev email_send -> ${parsed.to} (${label})\n`);
+                })
+                .catch((e: unknown) => {
+                    process.stdout.write(
+                        `  ✉ dev email_send -> ${parsed.to} (error: ${String(e)})\n`,
+                    );
+                });
+            return EmailStatus.Sent; // optimistic; sync wasm can't await the send
+        },
+    };
+}
+
 /**
  * Build the `env` import object for one instance. `state` collects what the
  * imperative imports produce during a dispatch; bind a fresh state per request.
@@ -307,7 +398,7 @@ export function buildHostImports(ref: MemoryRef, state: DispatchState): WebAssem
 
             // `Date.now()` -> wall-clock milliseconds, matching the edge host.
             // The guest divides by 1000 for Unix seconds (sessions, challenges).
-            'Date.now': (): number => Date.now(),
+            'Date.now': (): bigint => BigInt(Date.now()),
 
             // Web Crypto host functions (`env.crypto.*`), backed by Node's
             // `crypto`. The dev server skips metering, so these charge nothing.

package/src/devserver/{module.ts → runtime/module.ts}

@@ -13,12 +13,14 @@
 
 import fs from 'node:fs';
 
+import { DbFunctionKind, persistDb, setDbCatalog } from '../db/index.js';
+import { parseRouteKinds, routeKindForRequest, type RouteKindEntry } from '../db/routeKinds.js';
 import {
     decodeResponseEnvelope,
     encodeRequestEnvelope,
     type EnvelopeRequest,
     unpackHandleResult,
-} from './envelope.js';
+} from '../http/envelope.js';
 import { buildHostImports, freshDispatchState, type MemoryRef } from './host.js';
 
 export { WasmAbortError } from './host.js';
@@ -34,6 +36,21 @@ export const UNHANDLED_HEADER = 'x-toil-unhandled';
 
 const WASM_PAGE = 65536;
 
+function dbKindForHttpMethod(method: string): DbFunctionKind {
+    switch (method.toUpperCase()) {
+        case 'GET':
+        case 'HEAD':
+        case 'OPTIONS':
+            return DbFunctionKind.Query;
+        case 'POST':
+        case 'PUT':
+        case 'PATCH':
+        case 'DELETE':
+        default:
+            return DbFunctionKind.Action;
+    }
+}
+
 /** The shaped outcome of one guest dispatch. */
 export interface WasmDispatchResult {
     readonly status: number;
@@ -100,6 +117,8 @@ const PROVIDED_IMPORTS = new Set([
     'data.counter_get',
     'data.counter_add',
     'data.append',
+    'data.append_once',
+    'data.enqueue',
     'data.latest',
     'data.capacity_set_total',
     'data.capacity_available',
@@ -107,11 +126,14 @@ const PROVIDED_IMPORTS = new Set([
     'data.capacity_confirm',
     'data.capacity_cancel',
     'data.take_result',
+    'data.result_schema_version',
+    'data.write_allowed',
 ]);
 
 export class WasmServerModule {
     private module: WebAssembly.Module | null = null;
     private loadedMtimeMs = -1;
+    private routeKinds: readonly RouteKindEntry[] = [];
 
     constructor(private readonly wasmPath: string) {}
 
@@ -132,6 +154,7 @@ export class WasmServerModule {
             mtimeMs = fs.statSync(this.wasmPath).mtimeMs;
         } catch {
             this.module = null;
+            this.routeKinds = [];
             this.loadedMtimeMs = -1;
             return false;
         }
@@ -141,6 +164,10 @@ export class WasmServerModule {
         const module = new WebAssembly.Module(bytes);
         this.assertImportSurface(module);
         this.assertExportSurface(module);
+        // Refresh collection -> current schema_version so writes stamp the live layout;
+        // after a @data type evolves + rebuild, old on-disk rows now look out of date.
+        setDbCatalog(bytes);
+        this.routeKinds = parseRouteKinds(bytes);
         this.module = module;
         this.loadedMtimeMs = mtimeMs;
         return true;
@@ -159,6 +186,20 @@ export class WasmServerModule {
         const ref: MemoryRef = { memory: null };
         const state = freshDispatchState();
         state.clientIp = req.clientIp ?? '';
+        // Enforce per-route DB-kind gating ONLY when the guest declares its route
+        // kinds (the `toildb.route_kinds` custom section). A guest built with a
+        // toolchain that does not emit that section leaves `routeKinds` empty;
+        // inferring a kind from the HTTP method and enforcing it would wrongly
+        // reject legitimate bounded reads (e.g. a GET that reads `events.latest`,
+        // a scan-class op denied in `Query`). With no declarations we keep the
+        // unenforced default (`Job`, see `freshDbState`).
+        const routeKind = routeKindForRequest(this.routeKinds, req.method, req.path);
+        state.db.functionKind =
+            this.routeKinds.length === 0
+                ? DbFunctionKind.Job
+                : routeKind === DbFunctionKind.Query
+                  ? DbFunctionKind.Query
+                  : dbKindForHttpMethod(req.method);
         const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
         const exports = instance.exports as unknown as HandleExports;
         ref.memory = exports.memory;
@@ -193,6 +234,10 @@ export class WasmServerModule {
 
         const unhandled = headers.some(([n]) => n.toLowerCase() === UNHANDLED_HEADER);
 
+        // Flush any DB writes this request made to disk, so dev data survives a
+        // restart (and a crash never loses an already-served write).
+        persistDb();
+
         return {
             status,
             headers: headers.filter(([n]) => n.toLowerCase() !== UNHANDLED_HEADER),
@@ -216,6 +261,7 @@ export class WasmServerModule {
         const ref: MemoryRef = { memory: null };
         const state = freshDispatchState();
         state.clientIp = req.clientIp ?? '';
+        state.db.functionKind = DbFunctionKind.Query;
         const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
         const exports = instance.exports as unknown as HandleExports & {
             render?: (reqOfs: number, reqLen: number) => bigint;