toiljs 0.0.58 → 0.0.60

package/src/devserver/db/index.ts

@@ -0,0 +1,18 @@
+/**
+ * The dev ToilDB subsystem: the {@link DevDatabase} class + its process
+ * singleton, the per-request state, the catalog parser, and the `data.*` host
+ * imports. The production edge backs the SAME imports with ScyllaDB.
+ */
+
+export {
+    __resetDbForTests,
+    __setDbCatalogForTests,
+    buildDatabaseImports,
+    configureDbPersistence,
+    DevDatabase,
+    devDb,
+    persistDb,
+    setDbCatalog,
+} from './database.js';
+export { parseCatalog } from './catalog.js';
+export { type DbDevState, freshDbState } from './types.js';

package/src/devserver/db/types.ts

@@ -0,0 +1,76 @@
+/**
+ * Shared types, limits, and ABI return codes for the dev ToilDB emulation
+ * (see `./database.ts`). The wire ABI mirrors the production edge byte-for-byte
+ * (toildb/ABI.md): every byte region is a (ptr,len) into guest memory; returns
+ * are `>= 0` success (a length/handle/flag), `-1` too-small, `-2` absent,
+ * `<= -1000` a typed error (`-(1000+TDLnnn)`).
+ */
+
+/** Per-request data state: resolved handles + the last variable-length result +
+ *  the schema_version of that result's row (surfaced via result_schema_version). */
+export interface DbDevState {
+    handles: string[];
+    lastResult: Buffer | null;
+    lastResultVersion: number;
+}
+
+export function freshDbState(): DbDevState {
+    return { handles: [], lastResult: null, lastResultVersion: -1 };
+}
+
+/** A finite-resource escrow: a ceiling + a set of reservations, each held (in
+ *  flight, TTL'd) or confirmed (a permanent consume). Both count against available;
+ *  a confirmed reservation never expires. Mirrors `toildb::capacity::Escrow`. */
+export interface Reservation {
+    amount: bigint;
+    expiresMs: number;
+    confirmed: boolean;
+}
+export interface CapLedger {
+    total: bigint;
+    reservations: Map<bigint, Reservation>;
+    nextId: bigint;
+}
+
+/** The on-disk snapshot shape: dev data + its versions, JSON with base64 buffers. */
+export interface DbSnapshot {
+    store: Record<string, { v: string; sv: number }>; // records + unique owners (+ schema_version)
+    views: Record<string, { v: string; sv: number }>;
+    members: Record<string, Record<string, { v: string; sv: number }>>;
+    counters: Record<string, string>;
+    events: Record<string, { v: string; sv: number }[]>;
+    eventDedup: Record<string, string[]>;
+    capacity: Record<
+        string,
+        {
+            total: string;
+            nextId: string;
+            reservations: [string, { amount: string; expiresMs: number; confirmed: boolean }][];
+        }
+    >;
+}
+
+/** Edge caps (toildb::capacity::escrow): bound the reservation count + the hold TTL. */
+export const MAX_RESERVATIONS = 4096;
+export const MAX_RESERVATION_TTL_MS = 86_400_000; // 24h
+
+export const MAX_NAME = 512;
+export const MAX_KEY = 4096;
+export const MAX_VALUE = 256 * 1024;
+
+// i64 saturation bounds (the edge `MemEngine`/`ScyllaEngine` counters are i64).
+const I64_MIN = -(2n ** 63n);
+const I64_MAX = 2n ** 63n - 1n;
+export function satI64(v: bigint): bigint {
+    return v < I64_MIN ? I64_MIN : v > I64_MAX ? I64_MAX : v;
+}
+
+// Return codes, mirroring the edge ABI (`toildb::observe::diagnostics`): a typed
+// error is `-(1000 + TDLnnn)`; a plain absence is ABSENT (-2), not a typed error.
+export const ABSENT = -2; // NotFound / absent
+export const TOO_SMALL = -1;
+export const INVALID_HANDLE = -1001; // TDL001
+export const ALREADY_EXISTS = -1003; // TDL003 (create on an existing key)
+export const CONFLICT = -1004; // TDL004 (e.g. unique release by a non-owner)
+export const CODEC_ERR = -1006; // TDL006 (e.g. a non-positive reserve amount)
+export const TOO_MANY_KEYS = -1020; // TDL020 (get_many over the per-call cap)

package/src/devserver/email/index.ts

@@ -9,7 +9,7 @@
  * optimistically; an async-capable host (a future self-host) can `await deliver()`
  * for the true status.
  */
-import { loadEnvFiles } from '../dotenv.js';
+import { loadEnvFiles } from '../config/dotenv.js';
 import { EmailCaps } from './caps.js';
 import { type ResolvedEmailConfig, resolveEmailConfig } from './config.js';
 import { type OutboundMessage, sendVia } from './providers.js';

package/src/devserver/index.ts

@@ -1,298 +1,30 @@
 /**
- * The toiljs WASM dev server: a uWebSockets.js front (via @dacely/hyper-express,
- * the same stack as `toiljs/backend`) that dispatches HTTP requests into the
- * ToilScript-compiled server wasm exactly like the production edge does, and
- * proxies everything the server does not claim to an internal Vite dev server,
- * so dev keeps 100% of Vite's behavior (HMR, transforms, toolbar endpoints,
- * public assets, SPA fallback).
+ * Public barrel for the toiljs WASM dev server. The folder is organized into:
  *
- * Request flow:
+ *   - `server.ts`  the uWebSockets.js front + `startDevServer` entrypoint
+ *   - `runtime/`   the wasm module loader, host import surface, and crypto shims
+ *   - `db/`        the in-process ToilDB emulation (the {@link DevDatabase} class)
+ *   - `http/`      the envelope codec, response cache, and Vite proxy
+ *   - `config/`    the dotenv loader, `Environment.get` source, and rate limiter
+ *   - `email/`     the dev / self-host email pipeline
  *
- *   browser ── uWS :port ──► wasm `handle()` (fresh instance, envelope ABI)
- *                 │                │
- *                 │                └─ "unhandled" marker (no route matched)
- *                 ▼                                   │
- *           Vite dev server (loopback) ◄──────────────┘
- *
- * Dev intentionally skips the edge's metering, gas, pooling and snapshot-reset
- * machinery; the ABI (envelope layout, `handle(ofs, len) -> i64`, host import
- * surface, trap isolation) is identical so a server that runs here runs there.
+ * The production edge backs the SAME wasm ABI (envelope layout, `handle(ofs,
+ * len) -> i64`, host import surface, trap isolation), so a server that runs here
+ * runs there. See `server.ts` for the request flow.
  */
 
-import fs from 'node:fs';
-import path from 'node:path';
-
-import { type Request, type Response, Server } from '@dacely/hyper-express';
-import pc from 'picocolors';
-
-import type { EmailBackendConfig } from 'toiljs/shared';
-
-import { applyCacheRule, lookupCache } from './cache.js';
-import { initEmailService } from './email/index.js';
-import { type EnvelopeRequest, METHOD_CODES } from './envelope.js';
-import { WasmServerModule } from './module.js';
-import { proxyToVite, type ViteTarget, wireWebsocketProxy } from './proxy.js';
+export { startDevServer } from './server.js';
+export type { DevServerOptions, RunningDevServer } from './server.js';
 
 export {
     METHOD_CODES,
     encodeRequestEnvelope,
     decodeResponseEnvelope,
     unpackHandleResult,
-} from './envelope.js';
-export type { EnvelopeRequest, EnvelopeResponse } from './envelope.js';
-export { WasmServerModule, WasmAbortError, UNHANDLED_HEADER } from './module.js';
-export type { WasmDispatchResult } from './module.js';
-export { buildHostImports, freshDispatchState } from './host.js';
-export type { DispatchState, MemoryRef } from './host.js';
-export type { ViteTarget } from './proxy.js';
-
-const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
-
-/**
- * Paths that are Vite's own by construction; skipping the wasm round-trip for
- * them keeps the hot path of module serving untouched. Everything else is
- * offered to the server first (it answers or yields via the unhandled marker).
- */
-const VITE_PREFIXES = ['/@', '/node_modules/', '/__toil/'];
-
-/** Minimal type map for `respond_file` bodies when the guest set no content-type. */
-const MIME: Readonly<Record<string, string>> = {
-    '.html': 'text/html; charset=utf-8',
-    '.js': 'text/javascript; charset=utf-8',
-    '.mjs': 'text/javascript; charset=utf-8',
-    '.css': 'text/css; charset=utf-8',
-    '.json': 'application/json; charset=utf-8',
-    '.txt': 'text/plain; charset=utf-8',
-    '.svg': 'image/svg+xml',
-    '.png': 'image/png',
-    '.jpg': 'image/jpeg',
-    '.jpeg': 'image/jpeg',
-    '.webp': 'image/webp',
-    '.avif': 'image/avif',
-    '.gif': 'image/gif',
-    '.ico': 'image/x-icon',
-    '.wasm': 'application/wasm',
-    '.woff2': 'font/woff2',
-};
-
-/** Options for {@link startDevServer}. */
-export interface DevServerOptions {
-    /** Project root; `respond_file` paths resolve against it (and may not escape it). */
-    readonly root: string;
-    /** Public listening port (the one the browser opens). */
-    readonly port: number;
-    /** Bind host. Default `127.0.0.1`. */
-    readonly host?: string;
-    /** Absolute path to the ToilScript server wasm (toilconfig `targets.release.outFile`). */
-    readonly wasmFile: string;
-    /** The internal Vite dev server to proxy unclaimed traffic to. */
-    readonly vite: ViteTarget;
-    /** Max request body bytes. Default 8 MB. */
-    readonly maxBodyLength?: number;
-    /**
-     * The `toil.config.ts` `server.email` section (non-secret). When set (and the
-     * API key is in `.env.secrets`), `EmailService.send` really sends in dev;
-     * otherwise it stays a log-only mock. See `./email`.
-     */
-    readonly email?: EmailBackendConfig;
-}
-
-/** A running dev server. */
-export interface RunningDevServer {
-    readonly port: number;
-    readonly host: string;
-    /** Gracefully shuts the front server down (the Vite server is owned by the caller). */
-    close(): Promise<void>;
-}
-
-/** True for requests that belong to Vite by construction (never offered to the wasm). */
-function isViteInternal(url: string): boolean {
-    return VITE_PREFIXES.some((p) => url.startsWith(p));
-}
-
-/** Resolves a guest `respond_file` path inside `root`, refusing traversal outside it. */
-function resolveSendfile(root: string, file: string): string | null {
-    const resolved = path.resolve(root, file);
-    if (resolved !== root && !resolved.startsWith(root + path.sep)) return null;
-    if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) return null;
-    return resolved;
-}
-
-/** Builds the envelope request for one incoming HTTP request. */
-async function toEnvelopeRequest(request: Request): Promise<EnvelopeRequest> {
-    const hasBody = request.method !== 'GET' && request.method !== 'HEAD';
-    const body = hasBody ? new Uint8Array(await request.buffer()) : new Uint8Array(0);
-    // Dev parity for `client_ip`: the edge keys on the unspoofable socket peer,
-    // but the dev server has no DPDK socket, so best-effort from a proxy's
-    // `x-forwarded-for`, else localhost, so `ctx.clientIp()` returns a value.
-    const xff = request.headers['x-forwarded-for'];
-    const clientIp =
-        typeof xff === 'string' && xff.length > 0 ? xff.split(',')[0]!.trim() : '127.0.0.1';
-    return {
-        method: request.method,
-        // `url` keeps the query string; the guest's RouteContext parses it off the path.
-        path: request.url,
-        headers: Object.entries(request.headers),
-        body,
-        clientIp,
-    };
-}
-
-/** Sends a shaped wasm response, mirroring the edge's response defaults. */
-function sendWasmResponse(
-    response: Response,
-    root: string,
-    result: {
-        status: number;
-        headers: readonly (readonly [string, string])[];
-        body: Uint8Array;
-        sendfile: string | null;
-    },
-): void {
-    response.status(result.status);
-    let hasContentType = false;
-    for (const [name, value] of result.headers) {
-        if (name.toLowerCase() === 'content-type') hasContentType = true;
-        response.header(name, value);
-    }
-    response.header('server', 'toil-dev');
-
-    if (result.sendfile !== null) {
-        const file = resolveSendfile(root, result.sendfile);
-        if (file === null) {
-            response.status(404).send('not found\n');
-            return;
-        }
-        if (!hasContentType) {
-            // The edge defaults file bodies to application/octet-stream; in dev we
-            // guess from the extension so a guest-served asset renders in the browser.
-            response.header(
-                'content-type',
-                MIME[path.extname(file).toLowerCase()] ?? 'application/octet-stream',
-            );
-        }
-        response.sendFile(file);
-        return;
-    }
-
-    if (!hasContentType) response.header('content-type', 'text/plain; charset=utf-8');
-    response.send(Buffer.from(result.body.buffer, result.body.byteOffset, result.body.length));
-}
-
-/**
- * Starts the front server. The caller owns the Vite dev server (start it on a
- * loopback port first) and the toilscript rebuild watcher; this watches only
- * the wasm artifact and hot-swaps the compiled module when it changes.
- */
-export async function startDevServer(options: DevServerOptions): Promise<RunningDevServer> {
-    const host = options.host ?? '127.0.0.1';
-    const root = path.resolve(options.root);
-
-    // Wire the email service from toil.config `server.email` + `.env.secrets`
-    // (TOIL_EMAIL_*). Configured -> real sends; otherwise the import stays a
-    // log-only mock. A partial-but-invalid config logs why it stayed off.
-    const emailInit = initEmailService(root, options.email);
-    if (emailInit.service !== null) {
-        process.stdout.write(pc.dim(`  ✉ email enabled: ${emailInit.note}`) + '\n');
-    } else if (emailInit.note !== null) {
-        process.stdout.write(pc.yellow('  ! ') + pc.dim(`email off: ${emailInit.note}`) + '\n');
-    }
-
-    const module = new WasmServerModule(options.wasmFile);
-
-    let warnedMissing = false;
-    let loadedOnce = false;
-    const refresh = (): void => {
-        try {
-            if (module.refresh() && loadedOnce) {
-                process.stdout.write(pc.green('  ✓ ') + pc.dim('server module reloaded') + '\n');
-            }
-            loadedOnce ||= module.available;
-        } catch (e) {
-            process.stdout.write(pc.red(`  ✗ server wasm failed to load: ${String(e)}`) + '\n');
-        }
-        if (!module.available && !warnedMissing) {
-            warnedMissing = true;
-            process.stdout.write(
-                pc.yellow('  ! ') +
-                    pc.dim(`server wasm not found at ${options.wasmFile}; serving client only`) +
-                    '\n',
-            );
-        }
-    };
-    refresh();
-
-    const app = new Server({
-        max_body_length: options.maxBodyLength ?? DEFAULT_MAX_BODY_LENGTH,
-        max_body_buffer: 1024 * 32,
-        fast_abort: true,
-    });
-
-    app.set_error_handler((_request: Request, response: Response, error: Error) => {
-        if (response.completed) return;
-        response.atomic(() => {
-            response.status(500).send(`internal error: ${error.message}\n`);
-        });
-    });
-
-    wireWebsocketProxy(app, options.vite);
-
-    app.any('/*', async (request: Request, response: Response) => {
-        response.removeHeader('uWebSockets');
-
-        const dispatchable =
-            !isViteInternal(request.url) && METHOD_CODES[request.method] !== undefined;
-        if (dispatchable) refresh();
-
-        if (dispatchable && module.available) {
-            const envelopeReq = await toEnvelopeRequest(request);
-            // Honor the tenant cache directive locally, same rules as the
-            // edge: serve an identical request from the per-process cache,
-            // else dispatch and apply/strip the directive on the response.
-            const cacheHost = request.headers.host ?? 'dev';
-            const hasAuth =
-                request.headers.cookie !== undefined || request.headers.authorization !== undefined;
-            const cached = lookupCache(cacheHost, request.method, request.url, envelopeReq.body);
-            if (cached !== null) {
-                sendWasmResponse(response, root, cached);
-                return;
-            }
-            try {
-                const result = module.dispatch(envelopeReq);
-                if (!result.unhandled) {
-                    const finalized = applyCacheRule(
-                        cacheHost,
-                        request.method,
-                        request.url,
-                        envelopeReq.body,
-                        hasAuth,
-                        result,
-                    );
-                    sendWasmResponse(response, root, finalized);
-                    return;
-                }
-            } catch (e) {
-                // A trap (ToilScript abort, OOB, malformed envelope) is isolated to
-                // this request, exactly like the edge poisoning one instance.
-                process.stdout.write(
-                    pc.red(`  ✗ ${request.method} ${request.path} server error: ${String(e)}`) +
-                        '\n',
-                );
-                response.status(500).send('internal error\n');
-                return;
-            }
-        }
-
-        await proxyToVite(request, response, options.vite);
-    });
-
-    await app.listen(options.port, host);
-
-    return {
-        port: options.port,
-        host,
-        close: async (): Promise<void> => {
-            await app.shutdown();
-        },
-    };
-}
+} from './http/envelope.js';
+export type { EnvelopeRequest, EnvelopeResponse } from './http/envelope.js';
+export { WasmServerModule, WasmAbortError, UNHANDLED_HEADER } from './runtime/module.js';
+export type { WasmDispatchResult } from './runtime/module.js';
+export { buildHostImports, freshDispatchState } from './runtime/host.js';
+export type { DispatchState, MemoryRef } from './runtime/host.js';
+export type { ViteTarget } from './http/proxy.js';

package/src/devserver/{host.ts → runtime/host.ts}

@@ -17,12 +17,12 @@
  * `WebAssembly.Instance`, so offering the full surface costs nothing.
  */
 
+import { devEnvGet, devEnvGetSecure } from '../config/env.js';
+import { ratelimitCheck } from '../config/ratelimit.js';
+import { buildDatabaseImports, type DbDevState, freshDbState } from '../db/index.js';
+import { EmailStatus, getEmailService } from '../email/index.js';
+import { parseEmailBlob } from '../email/wire.js';
 import { buildCryptoImports, type CryptoState, freshCryptoState } from './crypto.js';
-import { buildDatabaseImports, type DbDevState, freshDbState } from './database.js';
-import { EmailStatus, getEmailService } from './email/index.js';
-import { parseEmailBlob } from './email/wire.js';
-import { devEnvGet, devEnvGetSecure } from './env.js';
-import { ratelimitCheck } from './ratelimit.js';
 
 /** Limits identical to the edge's `set_header` / `respond_file` bounds. */
 const MAX_TOTAL_HEADERS_BYTES = 64 * 1024;
@@ -307,7 +307,7 @@ export function buildHostImports(ref: MemoryRef, state: DispatchState): WebAssem
 
             // `Date.now()` -> wall-clock milliseconds, matching the edge host.
             // The guest divides by 1000 for Unix seconds (sessions, challenges).
-            'Date.now': (): number => Date.now(),
+            'Date.now': (): bigint => BigInt(Date.now()),
 
             // Web Crypto host functions (`env.crypto.*`), backed by Node's
             // `crypto`. The dev server skips metering, so these charge nothing.

package/src/devserver/{module.ts → runtime/module.ts}

@@ -13,12 +13,13 @@
 
 import fs from 'node:fs';
 
+import { persistDb, setDbCatalog } from '../db/index.js';
 import {
     decodeResponseEnvelope,
     encodeRequestEnvelope,
     type EnvelopeRequest,
     unpackHandleResult,
-} from './envelope.js';
+} from '../http/envelope.js';
 import { buildHostImports, freshDispatchState, type MemoryRef } from './host.js';
 
 export { WasmAbortError } from './host.js';
@@ -100,6 +101,8 @@ const PROVIDED_IMPORTS = new Set([
     'data.counter_get',
     'data.counter_add',
     'data.append',
+    'data.append_once',
+    'data.enqueue',
     'data.latest',
     'data.capacity_set_total',
     'data.capacity_available',
@@ -107,6 +110,8 @@ const PROVIDED_IMPORTS = new Set([
     'data.capacity_confirm',
     'data.capacity_cancel',
     'data.take_result',
+    'data.result_schema_version',
+    'data.write_allowed',
 ]);
 
 export class WasmServerModule {
@@ -141,6 +146,9 @@ export class WasmServerModule {
         const module = new WebAssembly.Module(bytes);
         this.assertImportSurface(module);
         this.assertExportSurface(module);
+        // Refresh collection -> current schema_version so writes stamp the live layout;
+        // after a @data type evolves + rebuild, old on-disk rows now look out of date.
+        setDbCatalog(bytes);
         this.module = module;
         this.loadedMtimeMs = mtimeMs;
         return true;
@@ -193,6 +201,10 @@ export class WasmServerModule {
 
         const unhandled = headers.some(([n]) => n.toLowerCase() === UNHANDLED_HEADER);
 
+        // Flush any DB writes this request made to disk, so dev data survives a
+        // restart (and a crash never loses an already-served write).
+        persistDb();
+
         return {
             status,
             headers: headers.filter(([n]) => n.toLowerCase() !== UNHANDLED_HEADER),