toiljs 0.0.69 → 0.0.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/build/cli/.tsbuildinfo +1 -1
  3. package/build/client/.tsbuildinfo +1 -1
  4. package/build/client/rpc.js +10 -4
  5. package/build/client/stream/client.js +108 -5
  6. package/build/compiler/.tsbuildinfo +1 -1
  7. package/build/compiler/index.d.ts +2 -0
  8. package/build/compiler/index.js +282 -2
  9. package/build/compiler/toil-docs.generated.js +1 -1
  10. package/build/compiler/vite.js +8 -0
  11. package/build/devserver/.tsbuildinfo +1 -1
  12. package/build/devserver/daemon/host.d.ts +1 -7
  13. package/build/devserver/daemon/host.js +5 -59
  14. package/build/devserver/daemon/index.d.ts +1 -0
  15. package/build/devserver/daemon/index.js +17 -4
  16. package/build/devserver/db/database.js +1 -1
  17. package/build/devserver/db/routeKinds.d.ts +6 -0
  18. package/build/devserver/db/routeKinds.js +40 -0
  19. package/build/devserver/index.d.ts +0 -1
  20. package/build/devserver/index.js +0 -1
  21. package/build/devserver/runtime/module.d.ts +1 -0
  22. package/build/devserver/runtime/module.js +18 -2
  23. package/build/devserver/stream/index.js +4 -3
  24. package/build/devserver/wasm/surface.d.ts +2 -0
  25. package/build/devserver/wasm/surface.js +35 -4
  26. package/docs/streams.md +3 -4
  27. package/examples/basic/server/services/Stats.ts +11 -3
  28. package/examples/basic/server/services/remotes.ts +8 -2
  29. package/package.json +3 -2
  30. package/server/runtime/exports/index.ts +8 -1
  31. package/server/runtime/index.ts +1 -0
  32. package/server/runtime/rpc/Rpc.ts +66 -0
  33. package/src/client/rpc.ts +21 -12
  34. package/src/client/stream/client.ts +133 -5
  35. package/src/compiler/index.ts +352 -2
  36. package/src/compiler/toil-docs.generated.ts +1 -1
  37. package/src/compiler/vite.ts +16 -0
  38. package/src/devserver/daemon/host.ts +10 -110
  39. package/src/devserver/daemon/index.ts +19 -6
  40. package/src/devserver/db/database.ts +1 -1
  41. package/src/devserver/db/routeKinds.ts +44 -0
  42. package/src/devserver/index.ts +0 -1
  43. package/src/devserver/runtime/host.ts +3 -7
  44. package/src/devserver/runtime/module.ts +30 -4
  45. package/src/devserver/stream/index.ts +8 -4
  46. package/src/devserver/wasm/surface.ts +33 -4
  47. package/test/daemon-build.test.ts +53 -0
  48. package/test/daemon-catalog.test.ts +78 -3
  49. package/test/daemon-emulation.test.ts +27 -29
  50. package/test/devserver-database.test.ts +93 -0
  51. package/test/fixtures/bignum-wire/spec.ts +3 -5
  52. package/test/fixtures/daemon-app.ts +25 -21
  53. package/test/rpc-dispatch.test.ts +132 -0
  54. package/test/rpc-kinds.test.ts +18 -0
  55. package/test/rpc.test.ts +20 -4
  56. package/build/devserver/mstore/store.d.ts +0 -18
  57. package/build/devserver/mstore/store.js +0 -82
  58. package/src/devserver/mstore/store.ts +0 -121
@@ -1,6 +1,7 @@
1
1
  /**
2
2
  * Client runtime for the typed `@stream` channel (doc 08 section 8.2). `Server.Stream.<Class>.connect(path?)`
3
- * opens a browser `WebSocket` to the class's `route` (same origin) and returns a channel:
3
+ * opens a browser `WebTransport` session to the class's `route` on the production edge (an `https://`
4
+ * stream-tier origin), or a `WebSocket` against the dev server (a `ws(s)://` origin), and returns a channel:
4
5
  * `onMessage` / `send` / `onClose` / `close`. A raw `@stream` channel sends `Uint8Array`; a typed
5
6
  * `@stream({ message: T })` channel sends the `@data` class and encodes it on send (the per-class
6
7
  * encoder the generated module passes). The inbound reply is always raw bytes. The generated
@@ -44,10 +45,13 @@ function connectStream<TSend = Uint8Array>(
44
45
  let opened = false;
45
46
  let messageCb: ((data: Uint8Array) => void) | undefined;
46
47
  let closeCb: ((code: number) => void) | undefined;
48
+ const pending: Uint8Array[] = [];
47
49
 
48
50
  const channel: StreamChannel<TSend> = {
49
51
  onMessage: (cb): void => {
50
52
  messageCb = cb;
53
+ for (const m of pending) cb(m);
54
+ pending.length = 0;
51
55
  },
52
56
  onClose: (cb): void => {
53
57
  closeCb = cb;
@@ -56,7 +60,11 @@ function connectStream<TSend = Uint8Array>(
56
60
  if (ws.readyState !== WebSocket.OPEN) return;
57
61
  // A typed channel encodes the @data message; a raw channel sends the bytes as-is.
58
62
  const bytes = encode ? encode(data as never) : (data as unknown as Uint8Array);
59
- ws.send(bytes as BufferSource);
63
+ try {
64
+ ws.send(bytes as BufferSource);
65
+ } catch {
66
+ /* socket transitioned OPEN -> CLOSING between the readyState check and send */
67
+ }
60
68
  },
61
69
  close: (): void => {
62
70
  ws.close();
@@ -68,7 +76,12 @@ function connectStream<TSend = Uint8Array>(
68
76
  resolve(channel);
69
77
  });
70
78
  ws.addEventListener('message', (event: MessageEvent) => {
71
- if (event.data instanceof ArrayBuffer) messageCb?.(new Uint8Array(event.data));
79
+ if (!(event.data instanceof ArrayBuffer)) return;
80
+ const bytes = new Uint8Array(event.data);
81
+ // Buffer a reply that arrives before onMessage() registers (mirror the WebTransport path),
82
+ // so an eager server reply is not silently dropped.
83
+ if (messageCb) messageCb(bytes);
84
+ else pending.push(bytes);
72
85
  });
73
86
  ws.addEventListener('close', (event: CloseEvent) => {
74
87
  if (!opened) reject(new Error(`stream connect failed (closed ${String(event.code)})`));
@@ -80,12 +93,125 @@ function connectStream<TSend = Uint8Array>(
80
93
  });
81
94
  }
82
95
 
96
+ /** Open a browser `WebTransport` session to `url` (an `https://` stream-tier origin) and resolve a
97
+ * channel once the session is ready. The browser drives the QUIC handshake, H3 Extended-CONNECT, and
98
+ * the RFC 9297 Quarter-Stream-ID datagram framing, so `send`/`onMessage` deal in raw bytes (no manual
99
+ * prefix). Rejects if the session fails to open (wrong node / unreachable / cert); a server close AFTER
100
+ * open surfaces via `onClose(code)`. This is the PRODUCTION transport - the L2/L3 edge is
101
+ * WebTransport-only; the dev server uses the `connectStream` WebSocket above. */
102
+ function connectStreamWT<TSend = Uint8Array>(
103
+ url: string,
104
+ encode?: (msg: never) => Uint8Array,
105
+ ): Promise<StreamChannel<TSend>> {
106
+ return new Promise((resolve, reject) => {
107
+ if (!('WebTransport' in globalThis)) {
108
+ reject(new Error('WebTransport is not available in this browser'));
109
+ return;
110
+ }
111
+ let transport: WebTransport;
112
+ try {
113
+ transport = new WebTransport(url);
114
+ } catch (e) {
115
+ reject(e instanceof Error ? e : new Error(String(e)));
116
+ return;
117
+ }
118
+ let messageCb: ((data: Uint8Array) => void) | undefined;
119
+ let closeCb: ((code: number) => void) | undefined;
120
+ let writer: WritableStreamDefaultWriter<Uint8Array> | undefined;
121
+ let opened = false;
122
+ const pending: Uint8Array[] = [];
123
+
124
+ const channel: StreamChannel<TSend> = {
125
+ onMessage: (cb): void => {
126
+ messageCb = cb;
127
+ for (const m of pending) cb(m);
128
+ pending.length = 0;
129
+ },
130
+ onClose: (cb): void => {
131
+ closeCb = cb;
132
+ },
133
+ send: (data): void => {
134
+ if (!writer) return;
135
+ const bytes = encode ? encode(data as never) : (data as unknown as Uint8Array);
136
+ // Fire-and-forget; a write that rejects (transport closing / backpressure) is surfaced
137
+ // via onClose, so swallow it here to avoid an unhandled promise rejection.
138
+ void writer.write(bytes).catch(() => {});
139
+ },
140
+ close: (): void => {
141
+ try {
142
+ transport.close();
143
+ } catch {
144
+ /* already closing */
145
+ }
146
+ },
147
+ };
148
+
149
+ // A close AFTER open (guest reject, idle sweep, server shutdown) reports through onClose; a
150
+ // failure BEFORE open rejects the connect() promise via the `ready` catch below.
151
+ transport.closed
152
+ .then((info) => {
153
+ if (opened) closeCb?.(info?.closeCode ?? 0);
154
+ })
155
+ .catch(() => {
156
+ if (opened) closeCb?.(1);
157
+ });
158
+
159
+ transport.ready
160
+ .then(() => {
161
+ opened = true;
162
+ writer = transport.datagrams.writable.getWriter();
163
+ const reader = transport.datagrams.readable.getReader();
164
+ void (async (): Promise<void> => {
165
+ try {
166
+ for (;;) {
167
+ const { value, done } = await reader.read();
168
+ if (done) break;
169
+ const bytes =
170
+ value instanceof Uint8Array
171
+ ? value
172
+ : new Uint8Array(value as ArrayBufferLike);
173
+ if (messageCb) messageCb(bytes);
174
+ else pending.push(bytes);
175
+ }
176
+ } catch {
177
+ /* read loop ended on session close */
178
+ }
179
+ })();
180
+ resolve(channel);
181
+ })
182
+ .catch((e) => reject(e instanceof Error ? e : new Error(String(e))));
183
+ });
184
+ }
185
+
186
+ /** Pick the transport by origin scheme: `https://` -> WebTransport (the production L2/L3 edge),
187
+ * `ws(s)://` -> WebSocket (the dev server). The generated `shared/server.ts` sets the scheme per build
188
+ * (edge: `https://wt.<tenant>`; dev: same-origin `wss://`). */
189
+ function openChannel<TSend = Uint8Array>(
190
+ url: string,
191
+ encode?: (msg: never) => Uint8Array,
192
+ ): Promise<StreamChannel<TSend>> {
193
+ return url.startsWith('https://')
194
+ ? connectStreamWT<TSend>(url, encode)
195
+ : connectStream<TSend>(url, encode);
196
+ }
197
+
83
198
  /** The same-origin WebSocket base (`ws://` / `wss://` per the page protocol). */
84
199
  function defaultOrigin(): string {
85
200
  const loc = globalThis.location;
86
201
  return `${loc.protocol === 'https:' ? 'wss:' : 'ws:'}//${loc.host}`;
87
202
  }
88
203
 
204
+ /** Resolve the stream-tier origin when the generated client passes none. A deploy/runtime override -
205
+ * `globalThis.__TOIL_STREAM_ORIGIN__` (e.g. `"https://wt.dacely.com"`, the production L2/L3 edge) -
206
+ * wins; otherwise fall back to the same-origin WebSocket base (the dev server). The edge override is
207
+ * `https://` so `openChannel` selects WebTransport; the dev fallback is `ws(s)://` so it selects
208
+ * WebSocket. This is how a deployed app points at its `wt.<tenant>` tier without the build knowing it. */
209
+ function resolveStreamOrigin(): string {
210
+ const override = (globalThis as { __TOIL_STREAM_ORIGIN__?: unknown }).__TOIL_STREAM_ORIGIN__;
211
+ if (typeof override === 'string' && override.length > 0) return override;
212
+ return defaultOrigin();
213
+ }
214
+
89
215
  /**
90
216
  * Build the `Server.Stream` client from the generated route map (`{ ClassName: route }`). `origin`
91
217
  * defaults to the page origin. `encoders` carries one `@data` encoder per typed `@stream({ message: T })`
@@ -97,12 +223,14 @@ export function makeStreamClient(
97
223
  origin?: string,
98
224
  encoders?: Record<string, (msg: never) => Uint8Array>,
99
225
  ): StreamClient {
100
- const base = origin ?? defaultOrigin();
101
226
  const client: StreamClient = {};
102
227
  for (const [name, route] of Object.entries(routes)) {
103
228
  const encode = encoders?.[name];
104
229
  client[name] = {
105
- connect: (path = ''): Promise<StreamChannel> => connectStream(`${base}${route}${path}`, encode),
230
+ // Resolve the origin LAZILY, per connect() - so a deploy/app that sets
231
+ // `__TOIL_STREAM_ORIGIN__` after this module loads is still honoured.
232
+ connect: (path = ''): Promise<StreamChannel> =>
233
+ openChannel(`${origin ?? resolveStreamOrigin()}${route}${path}`, encode),
106
234
  };
107
235
  }
108
236
  return client;
@@ -149,6 +149,7 @@ export async function buildServer(root: string): Promise<void> {
149
149
  // SHARED into every pass). The request pass runs LAST because it (re)writes shared/server.ts via
150
150
  // --rpcModule, which the downstream client build imports.
151
151
  const split = splitSurfaceFiles(root, files);
152
+ assertNoStreamInRequestTier(root, split);
152
153
  if (split.hasDaemon || split.hasStream) {
153
154
  const artifacts = serverArtifacts(root);
154
155
  // DAEMON (cold) pass: --targetMode cold, no client RPC surface.
@@ -176,7 +177,16 @@ export async function buildServer(root: string): Promise<void> {
176
177
  mode: 'hot',
177
178
  outFile: serverWasmFile(root),
178
179
  withRpc: true,
180
+ // Fold the @stream tier's classes into this pass's client surface so toilscript emits
181
+ // the typed `Server.Stream` (class names, message-type encoders, merged @rest type)
182
+ // WITHOUT compiling stream code into release.wasm.
183
+ rpcSurfaceFiles: split.hasStream ? split.stream : undefined,
179
184
  });
185
+ // The stream pass carries no client RPC surface (withRpc:false), so toilscript never emits the
186
+ // `Server.Stream` client into shared/server.ts. Append it here from the compiled stream
187
+ // artifact's `toilstream.catalog` (the origin stays runtime-resolved, so this is origin-agnostic).
188
+ if (split.hasStream && split.stream.length > 0)
189
+ emitStreamClientSurface(root, artifacts.stream, split.stream);
180
190
  return;
181
191
  }
182
192
 
@@ -215,6 +225,9 @@ interface SurfaceSplit {
215
225
  readonly stream: string[];
216
226
  /** Files for the REQUEST pass: `@rest`/`@service`/`@remote` surfaces + the request entry + shared helpers. */
217
227
  readonly request: string[];
228
+ /** The `@stream` / `*.stream.ts` modules (NOT the shared helpers). If a request-tier file imports one,
229
+ * `stream_dispatch` + its ring buffers would compile into release.wasm (audit #17). */
230
+ readonly streamModules: string[];
218
231
  }
219
232
 
220
233
  /** A `@daemon`/`@scheduled` decorator at line start (the L4 cold/daemon surface). */
@@ -257,6 +270,7 @@ export function splitSurfaceFiles(root: string, files: string[]): SurfaceSplit {
257
270
  const cold: string[] = [];
258
271
  const stream: string[] = [];
259
272
  const request: string[] = [];
273
+ const streamModules: string[] = [];
260
274
  for (const rel of files) {
261
275
  let src = '';
262
276
  try {
@@ -274,14 +288,110 @@ export function splitSurfaceFiles(root: string, files: string[]): SurfaceSplit {
274
288
  REQUEST_DECORATOR.test(src) ||
275
289
  (RUNTIME_ENTRY.test(src) && !isStreamEntryFile(rel) && !isDaemonEntryFile(rel));
276
290
  if (isCold) hasDaemon ||= /^[ \t]*@daemon\b/m.test(src) || isDaemonEntryFile(rel);
277
- if (isStream) hasStream = true;
291
+ if (isStream) {
292
+ hasStream = true;
293
+ streamModules.push(rel);
294
+ }
278
295
  // A file with no tier-specific surface is a SHARED helper, compiled into every pass.
279
296
  const shared = !isCold && !isStream && !isRequest;
280
297
  if (isCold || shared) cold.push(rel);
281
298
  if (isStream || shared) stream.push(rel);
282
299
  if (isRequest || shared) request.push(rel);
283
300
  }
284
- return { hasDaemon, hasStream, cold, stream, request };
301
+ return { hasDaemon, hasStream, cold, stream, request, streamModules };
302
+ }
303
+
304
+ /** The module specifiers a source statically imports / re-exports / dynamically imports. `import type` /
305
+ * `export type` are skipped: they are erased and never compile the target (so they cannot leak code). */
306
+ function* importSpecifiers(rawSrc: string): Generator<string> {
307
+ // Strip comments first, so a commented-out `// import { X } from './streams/...'` cannot trip the
308
+ // guard (mirrors emitStreamClientSurface's comment strip). A string literal containing import-like
309
+ // text is a rarer case this does not cover; the realistic dev-time scenario is a commented import.
310
+ const src = rawSrc.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
311
+ const fromRe = /\b(import|export)(\s+type)?\b[^'";]*?\bfrom\s*['"]([^'"]+)['"]/g;
312
+ let m: RegExpExecArray | null;
313
+ while ((m = fromRe.exec(src)) !== null) {
314
+ if (m[2]) continue; // `import type` / `export type` - erased
315
+ yield m[3];
316
+ }
317
+ const bareRe = /\bimport\s+['"]([^'"]+)['"]/g; // bare side-effect `import '...'`
318
+ while ((m = bareRe.exec(src)) !== null) yield m[1];
319
+ const dynRe = /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g; // dynamic `import('...')`
320
+ while ((m = dynRe.exec(src)) !== null) yield m[1];
321
+ }
322
+
323
+ /** Resolve a RELATIVE import specifier to a real file under `root` (a repo-relative posix path), or null
324
+ * for a bare/external/alias import or an unresolved path. Resolves against the FILESYSTEM (not just the
325
+ * entry+surface file list) so the guard's import graph also traverses plain UNDECORATED helper modules:
326
+ * a @stream reached transitively through one (main.ts -> util.ts -> Echo.ts) must still be caught (#6). */
327
+ function resolveServerImport(root: string, fromRel: string, spec: string): string | null {
328
+ if (!spec.startsWith('.')) return null;
329
+ const base = path
330
+ .normalize(path.join(path.dirname(fromRel), spec))
331
+ .replace(/\\/g, '/')
332
+ .replace(/\.(js|mjs|jsx|ts|tsx)$/, '');
333
+ for (const cand of [
334
+ base,
335
+ `${base}.ts`,
336
+ `${base}.tsx`,
337
+ `${base}.js`,
338
+ `${base}.mjs`,
339
+ `${base}/index.ts`,
340
+ `${base}/index.tsx`,
341
+ `${base}/index.js`,
342
+ ]) {
343
+ if (fs.existsSync(path.join(root, cand))) return cand;
344
+ }
345
+ return null;
346
+ }
347
+
348
+ /**
349
+ * Fail closed (audit #17): a `@stream` class compiled into the REQUEST tier would bake `stream_dispatch`
350
+ * + its 128 KiB ring buffers into release.wasm. `splitSurfaceFiles` keeps `@stream` files out of the
351
+ * request SET, but a stray import from a request-tier file still pulls one into release.wasm's compile
352
+ * graph - the tier boundary is otherwise only structural. Reject any request-tier file that (transitively)
353
+ * reaches a `@stream`/`*.stream.ts` module. The `Server.Stream` TYPE surface arrives via `--rpcSurfaceFiles`,
354
+ * NOT an import, so legitimate stream typing is unaffected; `import type` is likewise erased and ignored.
355
+ */
356
+ export function assertNoStreamInRequestTier(root: string, split: SurfaceSplit): void {
357
+ const streamSet = new Set(split.streamModules);
358
+ if (streamSet.size === 0) return;
359
+ const offenders = new Set<string>();
360
+ const seen = new Set<string>();
361
+ const queue = [...split.request];
362
+ while (queue.length > 0) {
363
+ const rel = queue.pop()!;
364
+ if (seen.has(rel)) continue;
365
+ seen.add(rel);
366
+ if (streamSet.has(rel)) {
367
+ // A @stream module sits directly in the request compile set (a file mixing @stream with
368
+ // request-tier code).
369
+ offenders.add(rel);
370
+ continue;
371
+ }
372
+ let src: string;
373
+ try {
374
+ src = fs.readFileSync(path.join(root, rel), 'utf8');
375
+ } catch {
376
+ continue;
377
+ }
378
+ for (const spec of importSpecifiers(src)) {
379
+ const target = resolveServerImport(root, rel, spec);
380
+ if (target === null) continue;
381
+ if (streamSet.has(target)) offenders.add(`${rel} -> ${target}`);
382
+ else if (!seen.has(target)) queue.push(target);
383
+ }
384
+ }
385
+ if (offenders.size > 0) {
386
+ throw new Error(
387
+ 'toiljs: a @stream class would be compiled into the REQUEST tier (release.wasm). @stream ' +
388
+ 'handlers (stream_dispatch + ring buffers) belong only in the stream tier ' +
389
+ '(release-stream.wasm), driven by server/main.stream.ts. A request-tier file reaches a ' +
390
+ '@stream module:\n ' +
391
+ [...offenders].join('\n ') +
392
+ '\nRemove the import, or move the code into a *.stream.ts module.',
393
+ );
394
+ }
285
395
  }
286
396
 
287
397
  interface PassOptions {
@@ -291,6 +401,10 @@ interface PassOptions {
291
401
  readonly outFile: string | null;
292
402
  /** Only the hot/default request pass carries `--rpcModule` (the cold artifact has no client surface). */
293
403
  readonly withRpc: boolean;
404
+ /** Files parsed for the client surface only (e.g. a sibling tier's `@stream` classes) - NOT compiled
405
+ * into this artifact. Lets the request pass emit `Server.Stream` without pulling stream code into
406
+ * release.wasm. */
407
+ readonly rpcSurfaceFiles?: readonly string[];
294
408
  }
295
409
 
296
410
  /** Run one toilscript pass. The toilscript CLI flag is `--targetMode` (camelCase). */
@@ -307,6 +421,8 @@ function runToilscriptPass(
307
421
  if (opts.mode !== null) args.push('--targetMode', opts.mode);
308
422
  if (opts.outFile !== null) args.push('--outFile', opts.outFile);
309
423
  if (opts.withRpc) args.push('--rpcModule', 'shared/server.ts');
424
+ if (opts.rpcSurfaceFiles)
425
+ for (const surfaceFile of opts.rpcSurfaceFiles) args.push('--rpcSurfaceFiles', surfaceFile);
310
426
  // Each pass is handed its OWN entry subset (the per-tier `files`); suppress the toilconfig
311
427
  // `entries` so toilscript does not ALSO append every project entry to every pass (which would
312
428
  // pull, e.g., a `@stream` class into the cold daemon pass). serverEntryFiles already folds
@@ -827,3 +943,237 @@ export type {
827
943
  DevtoolsAiConfig,
828
944
  } from './config.js';
829
945
  export type { RunningBackend, BackendOptions } from 'toiljs/backend';
946
+
947
+ // --- @stream client-surface emission ---------------------------------------------------------------
948
+ // The stream compile pass runs with `withRpc:false`, so toilscript never emits `Server.Stream` into
949
+ // `shared/server.ts`. We append it after the request pass by reading the compiled stream artifact's
950
+ // `toilstream.catalog`. Self-contained (the compiler tsconfig does not include the devserver walker).
951
+
952
+ /** A LEB128 unsigned int from `buf` at `pos`; `[value, nextPos]`. Throws on overrun. */
953
+ function lebU(buf: Buffer, pos: number): [number, number] {
954
+ let result = 0;
955
+ let shift = 0;
956
+ let p = pos;
957
+ for (;;) {
958
+ if (p >= buf.length) throw new RangeError('leb128 past end');
959
+ const b = buf[p++] as number;
960
+ result |= (b & 0x7f) << shift;
961
+ if ((b & 0x80) === 0) break;
962
+ shift += 7;
963
+ if (shift > 35) throw new RangeError('leb128 too long');
964
+ }
965
+ return [result >>> 0, p];
966
+ }
967
+
968
+ /** The bytes of the named wasm custom section, or `null` if absent/truncated. */
969
+ function customSectionBytes(wasm: Buffer, want: string): Buffer | null {
970
+ if (wasm.length < 8 || wasm[0] !== 0x00 || wasm[1] !== 0x61 || wasm[2] !== 0x73 || wasm[3] !== 0x6d)
971
+ return null;
972
+ let pos = 8;
973
+ try {
974
+ while (pos < wasm.length) {
975
+ const id = wasm[pos++] as number;
976
+ let size: number;
977
+ [size, pos] = lebU(wasm, pos);
978
+ const end = pos + size;
979
+ if (end > wasm.length || end < pos) return null;
980
+ if (id === 0) {
981
+ const [nameLen, namePos] = lebU(wasm, pos);
982
+ if (
983
+ namePos + nameLen <= end &&
984
+ wasm.toString('latin1', namePos, namePos + nameLen) === want
985
+ )
986
+ return wasm.subarray(namePos + nameLen, end);
987
+ }
988
+ pos = end;
989
+ }
990
+ } catch {
991
+ return null;
992
+ }
993
+ return null;
994
+ }
995
+
996
+ /** One `@stream` class from `toilstream.catalog`: the client key (class name) + its mount route. */
997
+ interface CatalogStream {
998
+ name: string;
999
+ route: string;
1000
+ }
1001
+
1002
+ /** Parse `toilstream.catalog` (doc 08 3.1; all little-endian) into `{ name, route }[]`, bounds-checked:
1003
+ * a short read mid-record yields the cleanly decoded prefix. */
1004
+ function readStreamCatalog(wasm: Buffer): CatalogStream[] {
1005
+ const sec = customSectionBytes(wasm, 'toilstream.catalog');
1006
+ if (sec === null) return [];
1007
+ const out: CatalogStream[] = [];
1008
+ let o = 0;
1009
+ const need = (n: number): boolean => o + n <= sec.length;
1010
+ const u8 = (): number => {
1011
+ const v = sec.readUInt8(o);
1012
+ o += 1;
1013
+ return v;
1014
+ };
1015
+ const u16 = (): number => {
1016
+ const v = sec.readUInt16LE(o);
1017
+ o += 2;
1018
+ return v;
1019
+ };
1020
+ const u32 = (): number => {
1021
+ const v = sec.readUInt32LE(o);
1022
+ o += 4;
1023
+ return v;
1024
+ };
1025
+ const str = (): string => {
1026
+ const len = u32();
1027
+ if (!need(len)) {
1028
+ o = sec.length + 1; // overrun: stop the loop on the next bounds check
1029
+ return '';
1030
+ }
1031
+ const s = sec.toString('utf8', o, o + len);
1032
+ o += len;
1033
+ return s;
1034
+ };
1035
+ try {
1036
+ if (!need(4)) return out;
1037
+ u16(); // format_version
1038
+ const n = u16();
1039
+ for (let i = 0; i < n; i++) {
1040
+ if (!need(8)) break;
1041
+ const name = str();
1042
+ const route = str();
1043
+ if (o > sec.length || !need(21)) break; // the per-record tail is 3*u8 + 4*u32 + u16 = 21
1044
+ u8(); // hook_presence_bitmask
1045
+ u8(); // declared_scope
1046
+ u8(); // message_mode
1047
+ u32(); // max_frame_bytes
1048
+ u32(); // ingress_ring_bytes
1049
+ u32(); // message_value_data_id
1050
+ u32(); // message_schema_version
1051
+ u16(); // stream_index
1052
+ if (name.length > 0 && route.length > 0) out.push({ name, route });
1053
+ }
1054
+ } catch {
1055
+ /* truncated section: return the decoded prefix */
1056
+ }
1057
+ return out;
1058
+ }
1059
+
1060
+ /** One `@stream` class from a source scan: its client key (the class name) and mount route. */
1061
+ interface SourceStream {
1062
+ className: string;
1063
+ route: string;
1064
+ }
1065
+
1066
+ /** Scan the `@stream` tier source files for their classes. The catalog carries only the declared route
1067
+ * name; the typed client wants the CLASS name (`Server.Stream.Echo`), which only the source has.
1068
+ * Best-effort regex mirroring toilscript's streamRoute; a class the scan misses falls back to its
1069
+ * catalog declared name. */
1070
+ function scanStreamSource(root: string, files: string[]): SourceStream[] {
1071
+ const out: SourceStream[] = [];
1072
+ const re =
1073
+ /@stream\b\s*(\([^)]*\))?\s*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)/g;
1074
+ for (const rel of files) {
1075
+ let src: string;
1076
+ try {
1077
+ src = fs.readFileSync(path.isAbsolute(rel) ? rel : path.join(root, rel), 'utf8');
1078
+ } catch {
1079
+ continue;
1080
+ }
1081
+ let m: RegExpExecArray | null;
1082
+ while ((m = re.exec(src)) !== null) {
1083
+ const className = m[2];
1084
+ if (className === undefined) continue;
1085
+ const args = m[1] ?? '';
1086
+ // The declared name: `@stream('x')` / `@stream({ name: 'x' })`, else the class name.
1087
+ let declared = className;
1088
+ const strArg = /^\(\s*['"]([^'"]+)['"]/.exec(args);
1089
+ const nameProp = /\bname\s*:\s*['"]([^'"]+)['"]/.exec(args);
1090
+ if (strArg?.[1] !== undefined) declared = strArg[1];
1091
+ else if (nameProp?.[1] !== undefined) declared = nameProp[1];
1092
+ out.push({ className, route: '/' + declared });
1093
+ }
1094
+ }
1095
+ return out;
1096
+ }
1097
+
1098
+ /** Append the typed `Server.Stream` client surface to `shared/server.ts`. The compiled stream catalog
1099
+ * is authoritative for WHICH streams exist (their routes); a source scan supplies the class name (the
1100
+ * client key, `Server.Stream.Echo`). Emits the `__toilStream` runtime attach plus - unless
1101
+ * shared/server.ts already declares `Server` (a @rest surface toilscript owns) - the
1102
+ * `declare global { const Server }` ambient type. The origin stays runtime-resolved. Idempotent. */
1103
+ function emitStreamClientSurface(
1104
+ root: string,
1105
+ streamWasmPath: string | undefined,
1106
+ streamFiles: string[],
1107
+ ): void {
1108
+ if (streamWasmPath === undefined) return;
1109
+ let wasm: Buffer;
1110
+ try {
1111
+ const abs = path.isAbsolute(streamWasmPath)
1112
+ ? streamWasmPath
1113
+ : path.join(root, streamWasmPath);
1114
+ wasm = fs.readFileSync(abs);
1115
+ } catch {
1116
+ return; // no stream artifact: nothing to wire
1117
+ }
1118
+ const catalog = readStreamCatalog(wasm);
1119
+ if (catalog.length === 0) return;
1120
+
1121
+ const classByRoute = new Map(scanStreamSource(root, streamFiles).map((s) => [s.route, s.className]));
1122
+ const streams = catalog.map((c) => ({ key: classByRoute.get(c.route) ?? c.name, route: c.route }));
1123
+
1124
+ const rpcModule = path.join(root, 'shared', 'server.ts');
1125
+ let existing = '';
1126
+ try {
1127
+ existing = fs.readFileSync(rpcModule, 'utf8');
1128
+ } catch {
1129
+ /* absent (a stream-only project, or no @rest surface): create it */
1130
+ }
1131
+ if (existing.includes('__toilStream')) {
1132
+ // toilscript already emitted the Server.Stream surface + ambient type (via --rpcSurfaceFiles),
1133
+ // but imports only toiljs/io, so it never evaluates the client proxy. Prepend a bare side-effect
1134
+ // import of toiljs/client so rpc.ts attaches `globalThis.Server`. (globalThis.Server is also set
1135
+ // unconditionally by .toil/globals.ts; this is belt-and-suspenders, and a bare side-effect import
1136
+ // is never tree-shaken, even under a future `sideEffects: false`.) Skip if it is already imported.
1137
+ if (!existing.includes('toiljs/client')) {
1138
+ fs.mkdirSync(path.dirname(rpcModule), { recursive: true });
1139
+ fs.writeFileSync(rpcModule, 'import "toiljs/client";\n' + existing);
1140
+ }
1141
+ return;
1142
+ }
1143
+
1144
+ const routes = streams
1145
+ .map((s) => ` ${JSON.stringify(s.key)}: ${JSON.stringify(s.route)},`)
1146
+ .join('\n');
1147
+ const attach =
1148
+ 'if (typeof globalThis !== "undefined") {\n' +
1149
+ ` (globalThis as Record<string, unknown>).__toilStream = __mkStream({\n${routes}\n });\n` +
1150
+ '}\n';
1151
+
1152
+ // The ambient `Server.Stream` type - only when toilscript has not already declared `Server` (a
1153
+ // @rest surface). For a @rest project, teaching toilscript the @stream surface is the follow-up;
1154
+ // the runtime attach above works regardless of the type.
1155
+ // Strip comments first so a commented-out `declare global { const Server }` does not suppress the
1156
+ // real type emit.
1157
+ const uncommented = existing.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
1158
+ const declareType = !/declare global[\s\S]*?const Server\b/.test(uncommented);
1159
+ const typeBlock = declareType
1160
+ ? 'declare global {\n' +
1161
+ ' /** The client-callable server surface (generated from the @stream catalog). */\n' +
1162
+ ' const Server: {\n' +
1163
+ ' readonly Stream: {\n' +
1164
+ streams
1165
+ .map((s) => ` readonly ${s.key}: import('toiljs/client').StreamConnectable;`)
1166
+ .join('\n') +
1167
+ '\n };\n };\n}\n\nexport {};\n'
1168
+ : '';
1169
+
1170
+ const out =
1171
+ 'import { makeStreamClient as __mkStream } from "toiljs/client";\n' +
1172
+ existing +
1173
+ '\n// --- @stream client surface (auto-generated from toilstream.catalog) ---\n' +
1174
+ attach +
1175
+ (typeBlock.length > 0 ? '\n' + typeBlock : '');
1176
+
1177
+ fs.mkdirSync(path.dirname(rpcModule), { recursive: true });
1178
+ fs.writeFileSync(rpcModule, out);
1179
+ }