toiljs 0.0.60 → 0.0.61

package/src/devserver/db/index.ts

@@ -15,4 +15,4 @@ export {
     setDbCatalog,
 } from './database.js';
 export { parseCatalog } from './catalog.js';
-export { type DbDevState, freshDbState } from './types.js';
+export { CollectionFamily, DbFunctionKind, type DbDevState, freshDbState } from './types.js';

package/src/devserver/db/routeKinds.ts

@@ -0,0 +1,147 @@
+import { customSection } from '../wasm/sections.js';
+import { DbFunctionKind } from './types.js';
+
+export interface RouteKindEntry {
+    readonly method: number;
+    readonly kind: DbFunctionKind;
+    readonly pattern: string;
+}
+
+const SECTION = 'toildb.route_kinds';
+const VERSION = 1;
+const MAX_SECTION_BYTES = 128 * 1024;
+const MAX_ROUTES = 2048;
+const MAX_PATTERN_BYTES = 2048;
+const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
+
+const METHOD_CODES: Readonly<Record<string, number>> = {
+    GET: 0,
+    POST: 1,
+    PUT: 2,
+    DELETE: 3,
+    PATCH: 4,
+    HEAD: 5,
+    OPTIONS: 6,
+};
+
+export function parseRouteKinds(wasm: Buffer): readonly RouteKindEntry[] {
+    let section: Buffer | null;
+    try {
+        section = customSection(wasm, SECTION);
+    } catch {
+        return [];
+    }
+    if (section === null) return [];
+    if (section.length > MAX_SECTION_BYTES) return [];
+
+    const r = new Reader(section);
+    const version = r.u16();
+    if (!r.ok || version !== VERSION) return [];
+    const count = r.u16();
+    if (!r.ok || count > MAX_ROUTES) return [];
+
+    const routes: RouteKindEntry[] = [];
+    for (let i = 0; i < count && r.ok; i++) {
+        const method = r.u8();
+        const kindByte = r.u8();
+        const pattern = r.string();
+        const kind =
+            kindByte === 0 ? DbFunctionKind.Query : kindByte === 1 ? DbFunctionKind.Action : null;
+        if (!r.ok || method < 0 || method > 6 || kind === null || !validPattern(pattern)) return [];
+        routes.push({ method, kind, pattern });
+    }
+    if (!r.ok || r.remaining() !== 0) return [];
+    return routes;
+}
+
+export function routeKindForRequest(
+    routes: readonly RouteKindEntry[],
+    method: string,
+    path: string,
+): DbFunctionKind | null {
+    const methodCode = METHOD_CODES[method.toUpperCase()];
+    if (methodCode === undefined) return null;
+    for (const route of routes) {
+        if (route.method === methodCode && routeMatches(route.pattern, path)) return route.kind;
+    }
+    return null;
+}
+
+function validPattern(pattern: string): boolean {
+    if (pattern.length === 0 || !pattern.startsWith('/')) return false;
+    for (let i = 0; i < pattern.length; i++) {
+        const c = pattern.charCodeAt(i);
+        if (c < 0x20 || c > 0x7e) return false;
+    }
+    return true;
+}
+
+function routeMatches(pattern: string, pathWithQuery: string): boolean {
+    const q = pathWithQuery.indexOf('?');
+    const path = q >= 0 ? pathWithQuery.slice(0, q) : pathWithQuery;
+    const patternSegs = pattern.split('/').filter(Boolean);
+    const pathSegs = path.split('/').filter(Boolean);
+    if (patternSegs.length !== pathSegs.length) return false;
+    for (let i = 0; i < patternSegs.length; i++) {
+        const p = patternSegs[i] ?? '';
+        const a = pathSegs[i] ?? '';
+        if (p.startsWith(':') && p.length > 1 && a.length > 0) continue;
+        if (p !== a) return false;
+    }
+    return true;
+}
+
+class Reader {
+    private pos = 0;
+    ok = true;
+
+    constructor(private readonly bytes: Buffer) {}
+
+    remaining(): number {
+        return this.bytes.length - this.pos;
+    }
+
+    u8(): number {
+        if (!this.ok || this.pos + 1 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        return this.bytes[this.pos++] ?? 0;
+    }
+
+    u16(): number {
+        if (!this.ok || this.pos + 2 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt16LE(this.pos);
+        this.pos += 2;
+        return out;
+    }
+
+    u32(): number {
+        if (!this.ok || this.pos + 4 > this.bytes.length) {
+            this.ok = false;
+            return 0;
+        }
+        const out = this.bytes.readUInt32LE(this.pos);
+        this.pos += 4;
+        return out;
+    }
+
+    string(): string {
+        const len = this.u32();
+        if (!this.ok || len > MAX_PATTERN_BYTES || this.pos + len > this.bytes.length) {
+            this.ok = false;
+            return '';
+        }
+        try {
+            const out = UTF8_DECODER.decode(this.bytes.subarray(this.pos, this.pos + len));
+            this.pos += len;
+            return out;
+        } catch {
+            this.ok = false;
+            return '';
+        }
+    }
+}

package/src/devserver/db/types.ts

@@ -6,16 +6,59 @@
  * `<= -1000` a typed error (`-(1000+TDLnnn)`).
  */
 
+export enum CollectionFamily {
+    Record = 0,
+    View = 1,
+    Events = 2,
+    Counter = 3,
+    Membership = 4,
+    Unique = 5,
+    Capacity = 6,
+}
+
+export enum DbFunctionKind {
+    Query = 'query',
+    Action = 'action',
+    Derive = 'derive',
+    Job = 'job',
+    Admin = 'admin',
+}
+
+export function isCollectionFamily(value: number): value is CollectionFamily {
+    return value >= CollectionFamily.Record && value <= CollectionFamily.Capacity;
+}
+
+export interface DevCollectionHandle {
+    name: string;
+    family: CollectionFamily;
+    schemaVersion: number;
+    replication: number;
+    placement: number;
+    fillMaxWaitMs: number;
+    fillAllowStale: boolean;
+}
+
+export type DbCatalogState =
+    | { kind: 'no-section' }
+    | { kind: 'malformed' }
+    | { kind: 'present'; collections: Map<string, DevCollectionHandle> };
+
 /** Per-request data state: resolved handles + the last variable-length result +
  *  the schema_version of that result's row (surfaced via result_schema_version). */
 export interface DbDevState {
-    handles: string[];
+    handles: DevCollectionHandle[];
     lastResult: Buffer | null;
     lastResultVersion: number;
+    functionKind: DbFunctionKind;
 }
 
 export function freshDbState(): DbDevState {
-    return { handles: [], lastResult: null, lastResultVersion: -1 };
+    return {
+        handles: [],
+        lastResult: null,
+        lastResultVersion: -1,
+        functionKind: DbFunctionKind.Job,
+    };
 }
 
 /** A finite-resource escrow: a ceiling + a set of reservations, each held (in
@@ -35,9 +78,15 @@ export interface CapLedger {
 /** The on-disk snapshot shape: dev data + its versions, JSON with base64 buffers. */
 export interface DbSnapshot {
     store: Record<string, { v: string; sv: number }>; // records + unique owners (+ schema_version)
+    recordIdem?: Record<
+        string,
+        { requestHash: string; state: 'pending' | 'done'; outcome?: RecordOutcomeSnapshot }
+    >;
+    uniqueIdem?: Record<string, string>;
     views: Record<string, { v: string; sv: number }>;
     members: Record<string, Record<string, { v: string; sv: number }>>;
     counters: Record<string, string>;
+    counterIdem?: Record<string, string>;
     events: Record<string, { v: string; sv: number }[]>;
     eventDedup: Record<string, string[]>;
     capacity: Record<
@@ -50,6 +99,14 @@ export interface DbSnapshot {
     >;
 }
 
+export type RecordOutcomeSnapshot =
+    | { kind: 'unit' }
+    | { kind: 'value'; v: string; sv: number }
+    | { kind: 'absent' }
+    | { kind: 'already_exists' }
+    | { kind: 'not_found' }
+    | { kind: 'conflict' };
+
 /** Edge caps (toildb::capacity::escrow): bound the reservation count + the hold TTL. */
 export const MAX_RESERVATIONS = 4096;
 export const MAX_RESERVATION_TTL_MS = 86_400_000; // 24h
@@ -57,6 +114,8 @@ export const MAX_RESERVATION_TTL_MS = 86_400_000; // 24h
 export const MAX_NAME = 512;
 export const MAX_KEY = 4096;
 export const MAX_VALUE = 256 * 1024;
+export const DEFAULT_FILL_WAIT_MS = 50;
+export const MAX_FILL_WAIT_MS = 60_000;
 
 // i64 saturation bounds (the edge `MemEngine`/`ScyllaEngine` counters are i64).
 const I64_MIN = -(2n ** 63n);
@@ -72,5 +131,9 @@ export const TOO_SMALL = -1;
 export const INVALID_HANDLE = -1001; // TDL001
 export const ALREADY_EXISTS = -1003; // TDL003 (create on an existing key)
 export const CONFLICT = -1004; // TDL004 (e.g. unique release by a non-owner)
+export const UNAVAILABLE = -1031; // TDL031 (retryable in-flight/uncertain op)
 export const CODEC_ERR = -1006; // TDL006 (e.g. a non-positive reserve amount)
+export const OP_NOT_ALLOWED_FOR_FAMILY = -1010; // TDL010
+export const OP_NOT_ALLOWED_IN_KIND = -1011; // TDL011
 export const TOO_MANY_KEYS = -1020; // TDL020 (get_many over the per-call cap)
+export const SCHEMA_UNAVAILABLE = -1070; // TDL070

package/src/devserver/index.ts

@@ -28,3 +28,15 @@ export type { WasmDispatchResult } from './runtime/module.js';
 export { buildHostImports, freshDispatchState } from './runtime/host.js';
 export type { DispatchState, MemoryRef } from './runtime/host.js';
 export type { ViteTarget } from './http/proxy.js';
+
+// Dev DAEMON (L4) emulation (RECONCILIATION Part 2/5; doc 08 section 5).
+export { DaemonHost, daemonEmulationEnabled } from './daemon/index.js';
+export { parseDaemonCatalog } from './daemon/catalog.js';
+export type { DaemonCatalog, ScheduledTask, CronMasks } from './daemon/catalog.js';
+export { buildDaemonImports, freshDaemonState } from './daemon/host.js';
+export type { DaemonState, DaemonRuntime, ResolvedDaemonConfig } from './daemon/host.js';
+export { cronMatches, cronNeverFires, nextCronFireMs } from './daemon/cron.js';
+export { parseSurface } from './wasm/surface.js';
+export type { Surface, SurfaceFlags } from './wasm/surface.js';
+export { customSection } from './wasm/sections.js';
+export { DevMemoryStore, devMemoryStore } from './mstore/store.js';

package/src/devserver/mstore/store.ts

@@ -0,0 +1,121 @@
+/**
+ * Dev MemoryStore: a single in-memory `Map` with per-entry TTL, one per process,
+ * persisted nowhere (ephemeral by definition). Backs the `mstore.*` host imports
+ * (RECONCILIATION Part 4, F2: HANDLELESS, ttl in SECONDS, inline drain). Keys are
+ * auto-scoped to host+region; the dev process is one host/region, so the key is
+ * used verbatim. Shared by streams (Phase 4) AND the daemon (both reference the
+ * same `devMemoryStore` singleton), matching doc 06's "shared across
+ * streams/handlers on the same region".
+ *
+ * TTL is enforced LAZILY on read (no background sweep), mirroring the dev DB's
+ * no-background-thread design. The error space is RECONCILIATION Part 3's 0x03xx
+ * registry; the host-import layer (daemon/host.ts) maps these results onto the
+ * Part 3 negative-return bridge.
+ */
+
+interface MStoreEntry {
+    value: Buffer;
+    /** `0` means no TTL; otherwise the epoch-ms the entry expires at. */
+    expiresAtMs: number;
+}
+
+export class DevMemoryStore {
+    private readonly map = new Map<string, MStoreEntry>();
+
+    private now(): number {
+        return Date.now();
+    }
+
+    /** The live entry for `key`, collecting it lazily if its TTL has passed. */
+    private live(key: string): MStoreEntry | null {
+        const e = this.map.get(key);
+        if (e === undefined) return null;
+        if (e.expiresAtMs !== 0 && e.expiresAtMs <= this.now()) {
+            this.map.delete(key);
+            return null;
+        }
+        return e;
+    }
+
+    private exp(ttlSecs: number): number {
+        return ttlSecs > 0 ? this.now() + ttlSecs * 1000 : 0;
+    }
+
+    /** The value, or `null` (=> 0x0301 MSTORE_NOT_FOUND). */
+    get(key: string): Buffer | null {
+        const e = this.live(key);
+        return e ? e.value : null;
+    }
+
+    set(key: string, value: Buffer, ttlSecs: number): void {
+        this.map.set(key, { value: Buffer.from(value), expiresAtMs: this.exp(ttlSecs) });
+    }
+
+    delete(key: string): boolean {
+        return this.map.delete(key);
+    }
+
+    /** Add `delta` to the i64 stored at `key`; `null` => 0x0306 MSTORE_NOT_A_NUMBER. */
+    incr(key: string, delta: bigint, ttlSecs: number): bigint | null {
+        const e = this.live(key);
+        let cur = 0n;
+        if (e !== null) {
+            const s = e.value.toString('utf8').trim();
+            if (!/^-?\d+$/.test(s)) return null;
+            try {
+                cur = BigInt(s);
+            } catch {
+                return null;
+            }
+        }
+        const next = BigInt.asIntN(64, cur + delta);
+        this.map.set(key, {
+            value: Buffer.from(next.toString(), 'utf8'),
+            // An incr on an existing key keeps its TTL unless a new one is given.
+            expiresAtMs: ttlSecs > 0 ? this.exp(ttlSecs) : (e?.expiresAtMs ?? 0),
+        });
+        return next;
+    }
+
+    /** `expect === null` means expect-absent (the dev mapping of `expect_len == 0`).
+     *  Returns `false` => 0x0304 MSTORE_CONFLICT. */
+    cas(key: string, expect: Buffer | null, next: Buffer, ttlSecs: number): boolean {
+        const e = this.live(key);
+        if (expect === null) {
+            if (e !== null) return false; // expect-absent, but present
+        } else if (e === null || !e.value.equals(expect)) {
+            return false; // expect-match failed
+        }
+        this.map.set(key, { value: Buffer.from(next), expiresAtMs: this.exp(ttlSecs) });
+        return true;
+    }
+
+    /** Re-arm the TTL of a live key; `false` => key absent (0x0301). */
+    expire(key: string, ttlSecs: number): boolean {
+        const e = this.live(key);
+        if (!e) return false;
+        e.expiresAtMs = this.exp(ttlSecs);
+        return true;
+    }
+
+    /**
+     * Prefix walk. `cursor` is an opaque resume index; a stale cursor (one that
+     * points past the current live key set after deletions) returns `null`
+     * (=> 0x0307 MSTORE_SCAN_BUSY). Returns the next cursor + the matched keys.
+     */
+    scan(prefix: string, cursor: bigint): { next: bigint; keys: string[] } | null {
+        const live = [...this.map.keys()].filter((k) => this.live(k) !== null && k.startsWith(prefix));
+        live.sort();
+        const start = Number(cursor);
+        if (start < 0 || start > live.length) return null; // stale cursor
+        const batch = live.slice(start);
+        return { next: BigInt(live.length), keys: batch };
+    }
+
+    /** Test-only: drop all entries. */
+    __reset(): void {
+        this.map.clear();
+    }
+}
+
+export const devMemoryStore = new DevMemoryStore();

package/src/devserver/runtime/host.ts

@@ -91,13 +91,33 @@ function mem(ref: MemoryRef): Buffer {
 }
 
 /** Bounds-checked byte read out of guest linear memory. */
-function readBytes(ref: MemoryRef, ptr: number, len: number): Buffer {
+export function readBytes(ref: MemoryRef, ptr: number, len: number): Buffer {
     const m = mem(ref);
     if (ptr < 0 || len < 0 || ptr + len > m.length)
         throw new Error(`host import read out of bounds: ptr=${String(ptr)} len=${String(len)}`);
     return m.subarray(ptr, ptr + len);
 }
 
+/**
+ * Bounds-checked write of a variable-length result into a guest out-buffer, with
+ * the edge's inline-drain return protocol: the byte length on success, or `-1`
+ * (STATUS_TOO_SMALL) when `outCap` is too small (the guest retries with a bigger
+ * buffer). Used by the handleless `mstore.*` imports (RECONCILIATION Part 4 F2).
+ */
+export function writeBytesOut(
+    ref: MemoryRef,
+    bytes: Buffer,
+    outPtr: number,
+    outCap: number,
+): number {
+    if (bytes.length > outCap) return -1; // TOO_SMALL
+    const m = mem(ref);
+    if (outPtr < 0 || outPtr + bytes.length > m.length)
+        throw new Error('host import write out of bounds');
+    bytes.copy(m, outPtr);
+    return bytes.length;
+}
+
 /**
  * Read a ToilScript string (UTF-16LE payload, byte length in the u32 at
  * `ptr - 4`). Used by `abort`, whose pointers reference string objects rather
@@ -169,6 +189,77 @@ function envLookup(
     return bytes.length;
 }
 
+/**
+ * The portion of the `env.*` request surface that is SHARED by the daemon (cold)
+ * box: panic hook, `Environment.get`/`getSecure`, `email_send`, `thread_spawn`,
+ * and `Date.now`. It deliberately EXCLUDES the response/stream functions a cold
+ * box must not have (`set_status`/`set_header`/`respond_file`/`client_ip`/
+ * `ratelimit_check`), which stay in {@link buildHostImports}. None of these read
+ * the per-dispatch response state, so they need only `ref`. The crypto and DB
+ * namespaces are spread on top by each box's loader (they carry their own state).
+ */
+export function buildEnvImports(
+    ref: MemoryRef,
+    _state: { crypto: CryptoState; db: DbDevState },
+): Record<string, (...a: never[]) => unknown> {
+    return {
+        abort: (msgPtr: number, filePtr: number, line: number, col: number): void => {
+            throw new WasmAbortError(
+                readGuestString(ref, msgPtr),
+                readGuestString(ref, filePtr),
+                line,
+                col,
+            );
+        },
+
+        // `Environment.get` / `getSecure`: copy one tenant env value into the
+        // guest buffer. Returns the byte length (0 = present-but-empty), -1 if
+        // the buffer is too small (the guest retries bigger), -2 if absent.
+        env_get: (keyPtr: number, keyLen: number, outPtr: number, outCap: number): number =>
+            envLookup(ref, keyPtr, keyLen, outPtr, outCap, false),
+        env_get_secure: (
+            keyPtr: number,
+            keyLen: number,
+            outPtr: number,
+            outCap: number,
+        ): number => envLookup(ref, keyPtr, keyLen, outPtr, outCap, true),
+
+        thread_spawn: (_startArg: number): number => -1,
+
+        // `Date.now()` -> wall-clock milliseconds, matching the edge host.
+        'Date.now': (): bigint => BigInt(Date.now()),
+
+        // `env::email_send`: the FULL email pipeline in dev. A daemon may send
+        // mail, so this stays in the shared subset (00 B2 / doc 08 AN-8).
+        email_send: (reqPtr: number, reqLen: number): number => {
+            const raw = readBytes(ref, reqPtr, reqLen);
+            const svc = getEmailService();
+            if (svc === null) {
+                const to = parseEmailBlob(raw)?.to ?? '<unparsed>';
+                process.stdout.write(`  ✉ dev email_send -> ${to} (no email config; not sent)\n`);
+                return EmailStatus.Sent;
+            }
+            const { status, parsed } = svc.prepare(raw);
+            if (parsed === null) {
+                process.stdout.write(`  ✉ dev email_send -> ${EmailStatus[status]}\n`);
+                return status;
+            }
+            void svc
+                .deliver(parsed)
+                .then((s) => {
+                    const label = s === EmailStatus.Sent ? 'sent' : EmailStatus[s];
+                    process.stdout.write(`  ✉ dev email_send -> ${parsed.to} (${label})\n`);
+                })
+                .catch((e: unknown) => {
+                    process.stdout.write(
+                        `  ✉ dev email_send -> ${parsed.to} (error: ${String(e)})\n`,
+                    );
+                });
+            return EmailStatus.Sent; // optimistic; sync wasm can't await the send
+        },
+    };
+}
+
 /**
  * Build the `env` import object for one instance. `state` collects what the
  * imperative imports produce during a dispatch; bind a fresh state per request.

package/src/devserver/runtime/module.ts

@@ -13,7 +13,8 @@
 
 import fs from 'node:fs';
 
-import { persistDb, setDbCatalog } from '../db/index.js';
+import { DbFunctionKind, persistDb, setDbCatalog } from '../db/index.js';
+import { parseRouteKinds, routeKindForRequest, type RouteKindEntry } from '../db/routeKinds.js';
 import {
     decodeResponseEnvelope,
     encodeRequestEnvelope,
@@ -35,6 +36,21 @@ export const UNHANDLED_HEADER = 'x-toil-unhandled';
 
 const WASM_PAGE = 65536;
 
+function dbKindForHttpMethod(method: string): DbFunctionKind {
+    switch (method.toUpperCase()) {
+        case 'GET':
+        case 'HEAD':
+        case 'OPTIONS':
+            return DbFunctionKind.Query;
+        case 'POST':
+        case 'PUT':
+        case 'PATCH':
+        case 'DELETE':
+        default:
+            return DbFunctionKind.Action;
+    }
+}
+
 /** The shaped outcome of one guest dispatch. */
 export interface WasmDispatchResult {
     readonly status: number;
@@ -117,6 +133,7 @@ const PROVIDED_IMPORTS = new Set([
 export class WasmServerModule {
     private module: WebAssembly.Module | null = null;
     private loadedMtimeMs = -1;
+    private routeKinds: readonly RouteKindEntry[] = [];
 
     constructor(private readonly wasmPath: string) {}
 
@@ -137,6 +154,7 @@ export class WasmServerModule {
             mtimeMs = fs.statSync(this.wasmPath).mtimeMs;
         } catch {
             this.module = null;
+            this.routeKinds = [];
             this.loadedMtimeMs = -1;
             return false;
         }
@@ -149,6 +167,7 @@ export class WasmServerModule {
         // Refresh collection -> current schema_version so writes stamp the live layout;
         // after a @data type evolves + rebuild, old on-disk rows now look out of date.
         setDbCatalog(bytes);
+        this.routeKinds = parseRouteKinds(bytes);
         this.module = module;
         this.loadedMtimeMs = mtimeMs;
         return true;
@@ -167,6 +186,20 @@ export class WasmServerModule {
         const ref: MemoryRef = { memory: null };
         const state = freshDispatchState();
         state.clientIp = req.clientIp ?? '';
+        // Enforce per-route DB-kind gating ONLY when the guest declares its route
+        // kinds (the `toildb.route_kinds` custom section). A guest built with a
+        // toolchain that does not emit that section leaves `routeKinds` empty;
+        // inferring a kind from the HTTP method and enforcing it would wrongly
+        // reject legitimate bounded reads (e.g. a GET that reads `events.latest`,
+        // a scan-class op denied in `Query`). With no declarations we keep the
+        // unenforced default (`Job`, see `freshDbState`).
+        const routeKind = routeKindForRequest(this.routeKinds, req.method, req.path);
+        state.db.functionKind =
+            this.routeKinds.length === 0
+                ? DbFunctionKind.Job
+                : routeKind === DbFunctionKind.Query
+                  ? DbFunctionKind.Query
+                  : dbKindForHttpMethod(req.method);
         const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
         const exports = instance.exports as unknown as HandleExports;
         ref.memory = exports.memory;
@@ -228,6 +261,7 @@ export class WasmServerModule {
         const ref: MemoryRef = { memory: null };
         const state = freshDispatchState();
         state.clientIp = req.clientIp ?? '';
+        state.db.functionKind = DbFunctionKind.Query;
         const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
         const exports = instance.exports as unknown as HandleExports & {
             render?: (reqOfs: number, reqLen: number) => bigint;