toiljs 0.0.68 → 0.0.70

package/src/compiler/index.ts

@@ -149,6 +149,7 @@ export async function buildServer(root: string): Promise<void> {
     // SHARED into every pass). The request pass runs LAST because it (re)writes shared/server.ts via
     // --rpcModule, which the downstream client build imports.
     const split = splitSurfaceFiles(root, files);
+    assertNoStreamInRequestTier(root, split);
     if (split.hasDaemon || split.hasStream) {
         const artifacts = serverArtifacts(root);
         // DAEMON (cold) pass: --targetMode cold, no client RPC surface.
@@ -176,7 +177,16 @@ export async function buildServer(root: string): Promise<void> {
                 mode: 'hot',
                 outFile: serverWasmFile(root),
                 withRpc: true,
+                // Fold the @stream tier's classes into this pass's client surface so toilscript emits
+                // the typed `Server.Stream` (class names, message-type encoders, merged @rest type)
+                // WITHOUT compiling stream code into release.wasm.
+                rpcSurfaceFiles: split.hasStream ? split.stream : undefined,
             });
+        // The stream pass carries no client RPC surface (withRpc:false), so toilscript never emits the
+        // `Server.Stream` client into shared/server.ts. Append it here from the compiled stream
+        // artifact's `toilstream.catalog` (the origin stays runtime-resolved, so this is origin-agnostic).
+        if (split.hasStream && split.stream.length > 0)
+            emitStreamClientSurface(root, artifacts.stream, split.stream);
         return;
     }
 
@@ -215,6 +225,9 @@ interface SurfaceSplit {
     readonly stream: string[];
     /** Files for the REQUEST pass: `@rest`/`@service`/`@remote` surfaces + the request entry + shared helpers. */
     readonly request: string[];
+    /** The `@stream` / `*.stream.ts` modules (NOT the shared helpers). If a request-tier file imports one,
+     *  `stream_dispatch` + its ring buffers would compile into release.wasm (audit #17). */
+    readonly streamModules: string[];
 }
 
 /** A `@daemon`/`@scheduled` decorator at line start (the L4 cold/daemon surface). */
@@ -257,6 +270,7 @@ export function splitSurfaceFiles(root: string, files: string[]): SurfaceSplit {
     const cold: string[] = [];
     const stream: string[] = [];
     const request: string[] = [];
+    const streamModules: string[] = [];
     for (const rel of files) {
         let src = '';
         try {
@@ -274,14 +288,110 @@ export function splitSurfaceFiles(root: string, files: string[]): SurfaceSplit {
             REQUEST_DECORATOR.test(src) ||
             (RUNTIME_ENTRY.test(src) && !isStreamEntryFile(rel) && !isDaemonEntryFile(rel));
         if (isCold) hasDaemon ||= /^[ \t]*@daemon\b/m.test(src) || isDaemonEntryFile(rel);
-        if (isStream) hasStream = true;
+        if (isStream) {
+            hasStream = true;
+            streamModules.push(rel);
+        }
         // A file with no tier-specific surface is a SHARED helper, compiled into every pass.
         const shared = !isCold && !isStream && !isRequest;
         if (isCold || shared) cold.push(rel);
         if (isStream || shared) stream.push(rel);
         if (isRequest || shared) request.push(rel);
     }
-    return { hasDaemon, hasStream, cold, stream, request };
+    return { hasDaemon, hasStream, cold, stream, request, streamModules };
+}
+
+/** The module specifiers a source statically imports / re-exports / dynamically imports. `import type` /
+ *  `export type` are skipped: they are erased and never compile the target (so they cannot leak code). */
+function* importSpecifiers(rawSrc: string): Generator<string> {
+    // Strip comments first, so a commented-out `// import { X } from './streams/...'` cannot trip the
+    // guard (mirrors emitStreamClientSurface's comment strip). A string literal containing import-like
+    // text is a rarer case this does not cover; the realistic dev-time scenario is a commented import.
+    const src = rawSrc.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
+    const fromRe = /\b(import|export)(\s+type)?\b[^'";]*?\bfrom\s*['"]([^'"]+)['"]/g;
+    let m: RegExpExecArray | null;
+    while ((m = fromRe.exec(src)) !== null) {
+        if (m[2]) continue; // `import type` / `export type` - erased
+        yield m[3];
+    }
+    const bareRe = /\bimport\s+['"]([^'"]+)['"]/g; // bare side-effect `import '...'`
+    while ((m = bareRe.exec(src)) !== null) yield m[1];
+    const dynRe = /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g; // dynamic `import('...')`
+    while ((m = dynRe.exec(src)) !== null) yield m[1];
+}
+
+/** Resolve a RELATIVE import specifier to a real file under `root` (a repo-relative posix path), or null
+ *  for a bare/external/alias import or an unresolved path. Resolves against the FILESYSTEM (not just the
+ *  entry+surface file list) so the guard's import graph also traverses plain UNDECORATED helper modules:
+ *  a @stream reached transitively through one (main.ts -> util.ts -> Echo.ts) must still be caught (#6). */
+function resolveServerImport(root: string, fromRel: string, spec: string): string | null {
+    if (!spec.startsWith('.')) return null;
+    const base = path
+        .normalize(path.join(path.dirname(fromRel), spec))
+        .replace(/\\/g, '/')
+        .replace(/\.(js|mjs|jsx|ts|tsx)$/, '');
+    for (const cand of [
+        base,
+        `${base}.ts`,
+        `${base}.tsx`,
+        `${base}.js`,
+        `${base}.mjs`,
+        `${base}/index.ts`,
+        `${base}/index.tsx`,
+        `${base}/index.js`,
+    ]) {
+        if (fs.existsSync(path.join(root, cand))) return cand;
+    }
+    return null;
+}
+
+/**
+ * Fail closed (audit #17): a `@stream` class compiled into the REQUEST tier would bake `stream_dispatch`
+ * + its 128 KiB ring buffers into release.wasm. `splitSurfaceFiles` keeps `@stream` files out of the
+ * request SET, but a stray import from a request-tier file still pulls one into release.wasm's compile
+ * graph - the tier boundary is otherwise only structural. Reject any request-tier file that (transitively)
+ * reaches a `@stream`/`*.stream.ts` module. The `Server.Stream` TYPE surface arrives via `--rpcSurfaceFiles`,
+ * NOT an import, so legitimate stream typing is unaffected; `import type` is likewise erased and ignored.
+ */
+export function assertNoStreamInRequestTier(root: string, split: SurfaceSplit): void {
+    const streamSet = new Set(split.streamModules);
+    if (streamSet.size === 0) return;
+    const offenders = new Set<string>();
+    const seen = new Set<string>();
+    const queue = [...split.request];
+    while (queue.length > 0) {
+        const rel = queue.pop()!;
+        if (seen.has(rel)) continue;
+        seen.add(rel);
+        if (streamSet.has(rel)) {
+            // A @stream module sits directly in the request compile set (a file mixing @stream with
+            // request-tier code).
+            offenders.add(rel);
+            continue;
+        }
+        let src: string;
+        try {
+            src = fs.readFileSync(path.join(root, rel), 'utf8');
+        } catch {
+            continue;
+        }
+        for (const spec of importSpecifiers(src)) {
+            const target = resolveServerImport(root, rel, spec);
+            if (target === null) continue;
+            if (streamSet.has(target)) offenders.add(`${rel} -> ${target}`);
+            else if (!seen.has(target)) queue.push(target);
+        }
+    }
+    if (offenders.size > 0) {
+        throw new Error(
+            'toiljs: a @stream class would be compiled into the REQUEST tier (release.wasm). @stream ' +
+                'handlers (stream_dispatch + ring buffers) belong only in the stream tier ' +
+                '(release-stream.wasm), driven by server/main.stream.ts. A request-tier file reaches a ' +
+                '@stream module:\n  ' +
+                [...offenders].join('\n  ') +
+                '\nRemove the import, or move the code into a *.stream.ts module.',
+        );
+    }
 }
 
 interface PassOptions {
@@ -291,6 +401,10 @@ interface PassOptions {
     readonly outFile: string | null;
     /** Only the hot/default request pass carries `--rpcModule` (the cold artifact has no client surface). */
     readonly withRpc: boolean;
+    /** Files parsed for the client surface only (e.g. a sibling tier's `@stream` classes) - NOT compiled
+     *  into this artifact. Lets the request pass emit `Server.Stream` without pulling stream code into
+     *  release.wasm. */
+    readonly rpcSurfaceFiles?: readonly string[];
 }
 
 /** Run one toilscript pass. The toilscript CLI flag is `--targetMode` (camelCase). */
@@ -307,6 +421,8 @@ function runToilscriptPass(
     if (opts.mode !== null) args.push('--targetMode', opts.mode);
     if (opts.outFile !== null) args.push('--outFile', opts.outFile);
     if (opts.withRpc) args.push('--rpcModule', 'shared/server.ts');
+    if (opts.rpcSurfaceFiles)
+        for (const surfaceFile of opts.rpcSurfaceFiles) args.push('--rpcSurfaceFiles', surfaceFile);
     // Each pass is handed its OWN entry subset (the per-tier `files`); suppress the toilconfig
     // `entries` so toilscript does not ALSO append every project entry to every pass (which would
     // pull, e.g., a `@stream` class into the cold daemon pass). serverEntryFiles already folds
@@ -827,3 +943,237 @@ export type {
     DevtoolsAiConfig,
 } from './config.js';
 export type { RunningBackend, BackendOptions } from 'toiljs/backend';
+
+// --- @stream client-surface emission ---------------------------------------------------------------
+// The stream compile pass runs with `withRpc:false`, so toilscript never emits `Server.Stream` into
+// `shared/server.ts`. We append it after the request pass by reading the compiled stream artifact's
+// `toilstream.catalog`. Self-contained (the compiler tsconfig does not include the devserver walker).
+
+/** A LEB128 unsigned int from `buf` at `pos`; `[value, nextPos]`. Throws on overrun. */
+function lebU(buf: Buffer, pos: number): [number, number] {
+    let result = 0;
+    let shift = 0;
+    let p = pos;
+    for (;;) {
+        if (p >= buf.length) throw new RangeError('leb128 past end');
+        const b = buf[p++] as number;
+        result |= (b & 0x7f) << shift;
+        if ((b & 0x80) === 0) break;
+        shift += 7;
+        if (shift > 35) throw new RangeError('leb128 too long');
+    }
+    return [result >>> 0, p];
+}
+
+/** The bytes of the named wasm custom section, or `null` if absent/truncated. */
+function customSectionBytes(wasm: Buffer, want: string): Buffer | null {
+    if (wasm.length < 8 || wasm[0] !== 0x00 || wasm[1] !== 0x61 || wasm[2] !== 0x73 || wasm[3] !== 0x6d)
+        return null;
+    let pos = 8;
+    try {
+        while (pos < wasm.length) {
+            const id = wasm[pos++] as number;
+            let size: number;
+            [size, pos] = lebU(wasm, pos);
+            const end = pos + size;
+            if (end > wasm.length || end < pos) return null;
+            if (id === 0) {
+                const [nameLen, namePos] = lebU(wasm, pos);
+                if (
+                    namePos + nameLen <= end &&
+                    wasm.toString('latin1', namePos, namePos + nameLen) === want
+                )
+                    return wasm.subarray(namePos + nameLen, end);
+            }
+            pos = end;
+        }
+    } catch {
+        return null;
+    }
+    return null;
+}
+
+/** One `@stream` class from `toilstream.catalog`: the client key (class name) + its mount route. */
+interface CatalogStream {
+    name: string;
+    route: string;
+}
+
+/** Parse `toilstream.catalog` (doc 08 3.1; all little-endian) into `{ name, route }[]`, bounds-checked:
+ *  a short read mid-record yields the cleanly decoded prefix. */
+function readStreamCatalog(wasm: Buffer): CatalogStream[] {
+    const sec = customSectionBytes(wasm, 'toilstream.catalog');
+    if (sec === null) return [];
+    const out: CatalogStream[] = [];
+    let o = 0;
+    const need = (n: number): boolean => o + n <= sec.length;
+    const u8 = (): number => {
+        const v = sec.readUInt8(o);
+        o += 1;
+        return v;
+    };
+    const u16 = (): number => {
+        const v = sec.readUInt16LE(o);
+        o += 2;
+        return v;
+    };
+    const u32 = (): number => {
+        const v = sec.readUInt32LE(o);
+        o += 4;
+        return v;
+    };
+    const str = (): string => {
+        const len = u32();
+        if (!need(len)) {
+            o = sec.length + 1; // overrun: stop the loop on the next bounds check
+            return '';
+        }
+        const s = sec.toString('utf8', o, o + len);
+        o += len;
+        return s;
+    };
+    try {
+        if (!need(4)) return out;
+        u16(); // format_version
+        const n = u16();
+        for (let i = 0; i < n; i++) {
+            if (!need(8)) break;
+            const name = str();
+            const route = str();
+            if (o > sec.length || !need(21)) break; // the per-record tail is 3*u8 + 4*u32 + u16 = 21
+            u8(); // hook_presence_bitmask
+            u8(); // declared_scope
+            u8(); // message_mode
+            u32(); // max_frame_bytes
+            u32(); // ingress_ring_bytes
+            u32(); // message_value_data_id
+            u32(); // message_schema_version
+            u16(); // stream_index
+            if (name.length > 0 && route.length > 0) out.push({ name, route });
+        }
+    } catch {
+        /* truncated section: return the decoded prefix */
+    }
+    return out;
+}
+
+/** One `@stream` class from a source scan: its client key (the class name) and mount route. */
+interface SourceStream {
+    className: string;
+    route: string;
+}
+
+/** Scan the `@stream` tier source files for their classes. The catalog carries only the declared route
+ *  name; the typed client wants the CLASS name (`Server.Stream.Echo`), which only the source has.
+ *  Best-effort regex mirroring toilscript's streamRoute; a class the scan misses falls back to its
+ *  catalog declared name. */
+function scanStreamSource(root: string, files: string[]): SourceStream[] {
+    const out: SourceStream[] = [];
+    const re =
+        /@stream\b\s*(\([^)]*\))?\s*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)/g;
+    for (const rel of files) {
+        let src: string;
+        try {
+            src = fs.readFileSync(path.isAbsolute(rel) ? rel : path.join(root, rel), 'utf8');
+        } catch {
+            continue;
+        }
+        let m: RegExpExecArray | null;
+        while ((m = re.exec(src)) !== null) {
+            const className = m[2];
+            if (className === undefined) continue;
+            const args = m[1] ?? '';
+            // The declared name: `@stream('x')` / `@stream({ name: 'x' })`, else the class name.
+            let declared = className;
+            const strArg = /^\(\s*['"]([^'"]+)['"]/.exec(args);
+            const nameProp = /\bname\s*:\s*['"]([^'"]+)['"]/.exec(args);
+            if (strArg?.[1] !== undefined) declared = strArg[1];
+            else if (nameProp?.[1] !== undefined) declared = nameProp[1];
+            out.push({ className, route: '/' + declared });
+        }
+    }
+    return out;
+}
+
+/** Append the typed `Server.Stream` client surface to `shared/server.ts`. The compiled stream catalog
+ *  is authoritative for WHICH streams exist (their routes); a source scan supplies the class name (the
+ *  client key, `Server.Stream.Echo`). Emits the `__toilStream` runtime attach plus - unless
+ *  shared/server.ts already declares `Server` (a @rest surface toilscript owns) - the
+ *  `declare global { const Server }` ambient type. The origin stays runtime-resolved. Idempotent. */
+function emitStreamClientSurface(
+    root: string,
+    streamWasmPath: string | undefined,
+    streamFiles: string[],
+): void {
+    if (streamWasmPath === undefined) return;
+    let wasm: Buffer;
+    try {
+        const abs = path.isAbsolute(streamWasmPath)
+            ? streamWasmPath
+            : path.join(root, streamWasmPath);
+        wasm = fs.readFileSync(abs);
+    } catch {
+        return; // no stream artifact: nothing to wire
+    }
+    const catalog = readStreamCatalog(wasm);
+    if (catalog.length === 0) return;
+
+    const classByRoute = new Map(scanStreamSource(root, streamFiles).map((s) => [s.route, s.className]));
+    const streams = catalog.map((c) => ({ key: classByRoute.get(c.route) ?? c.name, route: c.route }));
+
+    const rpcModule = path.join(root, 'shared', 'server.ts');
+    let existing = '';
+    try {
+        existing = fs.readFileSync(rpcModule, 'utf8');
+    } catch {
+        /* absent (a stream-only project, or no @rest surface): create it */
+    }
+    if (existing.includes('__toilStream')) {
+        // toilscript already emitted the Server.Stream surface + ambient type (via --rpcSurfaceFiles),
+        // but imports only toiljs/io, so it never evaluates the client proxy. Prepend a bare side-effect
+        // import of toiljs/client so rpc.ts attaches `globalThis.Server`. (globalThis.Server is also set
+        // unconditionally by .toil/globals.ts; this is belt-and-suspenders, and a bare side-effect import
+        // is never tree-shaken, even under a future `sideEffects: false`.) Skip if it is already imported.
+        if (!existing.includes('toiljs/client')) {
+            fs.mkdirSync(path.dirname(rpcModule), { recursive: true });
+            fs.writeFileSync(rpcModule, 'import "toiljs/client";\n' + existing);
+        }
+        return;
+    }
+
+    const routes = streams
+        .map((s) => `        ${JSON.stringify(s.key)}: ${JSON.stringify(s.route)},`)
+        .join('\n');
+    const attach =
+        'if (typeof globalThis !== "undefined") {\n' +
+        `    (globalThis as Record<string, unknown>).__toilStream = __mkStream({\n${routes}\n    });\n` +
+        '}\n';
+
+    // The ambient `Server.Stream` type - only when toilscript has not already declared `Server` (a
+    // @rest surface). For a @rest project, teaching toilscript the @stream surface is the follow-up;
+    // the runtime attach above works regardless of the type.
+    // Strip comments first so a commented-out `declare global { const Server }` does not suppress the
+    // real type emit.
+    const uncommented = existing.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
+    const declareType = !/declare global[\s\S]*?const Server\b/.test(uncommented);
+    const typeBlock = declareType
+        ? 'declare global {\n' +
+          '    /** The client-callable server surface (generated from the @stream catalog). */\n' +
+          '    const Server: {\n' +
+          '        readonly Stream: {\n' +
+          streams
+              .map((s) => `            readonly ${s.key}: import('toiljs/client').StreamConnectable;`)
+              .join('\n') +
+          '\n        };\n    };\n}\n\nexport {};\n'
+        : '';
+
+    const out =
+        'import { makeStreamClient as __mkStream } from "toiljs/client";\n' +
+        existing +
+        '\n// --- @stream client surface (auto-generated from toilstream.catalog) ---\n' +
+        attach +
+        (typeBlock.length > 0 ? '\n' + typeBlock : '');
+
+    fs.mkdirSync(path.dirname(rpcModule), { recursive: true });
+    fs.writeFileSync(rpcModule, out);
+}