toiljs 0.0.60 → 0.0.62

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/.github/workflows/ci.yml +31 -0
  2. package/CHANGELOG.md +17 -0
  3. package/build/cli/.tsbuildinfo +1 -1
  4. package/build/cli/index.js +2 -2
  5. package/build/client/.tsbuildinfo +1 -1
  6. package/build/client/index.d.ts +1 -1
  7. package/build/client/index.js +1 -1
  8. package/build/client/routing/mount.js +11 -26
  9. package/build/client/ssr/markers.d.ts +1 -0
  10. package/build/client/ssr/markers.js +9 -2
  11. package/build/compiler/.tsbuildinfo +1 -1
  12. package/build/compiler/config.d.ts +21 -0
  13. package/build/compiler/config.js +35 -0
  14. package/build/compiler/docs.d.ts +2 -1
  15. package/build/compiler/docs.js +33 -304
  16. package/build/compiler/index.d.ts +13 -0
  17. package/build/compiler/index.js +113 -21
  18. package/build/compiler/template-build.d.ts +23 -3
  19. package/build/compiler/template-build.js +120 -30
  20. package/build/compiler/toil-docs.generated.d.ts +1 -0
  21. package/build/compiler/toil-docs.generated.js +20 -0
  22. package/build/devserver/.tsbuildinfo +1 -1
  23. package/build/devserver/daemon/catalog.d.ts +26 -0
  24. package/build/devserver/daemon/catalog.js +48 -0
  25. package/build/devserver/daemon/cron.d.ts +4 -0
  26. package/build/devserver/daemon/cron.js +50 -0
  27. package/build/devserver/daemon/host.d.ts +37 -0
  28. package/build/devserver/daemon/host.js +94 -0
  29. package/build/devserver/daemon/index.d.ts +34 -0
  30. package/build/devserver/daemon/index.js +241 -0
  31. package/build/devserver/db/catalog.d.ts +2 -1
  32. package/build/devserver/db/catalog.js +44 -44
  33. package/build/devserver/db/database.d.ts +27 -11
  34. package/build/devserver/db/database.js +539 -169
  35. package/build/devserver/db/index.d.ts +1 -1
  36. package/build/devserver/db/index.js +1 -1
  37. package/build/devserver/db/routeKinds.d.ts +8 -0
  38. package/build/devserver/db/routeKinds.js +139 -0
  39. package/build/devserver/db/types.d.ts +64 -1
  40. package/build/devserver/db/types.js +33 -1
  41. package/build/devserver/index.d.ts +10 -0
  42. package/build/devserver/index.js +7 -0
  43. package/build/devserver/mstore/store.d.ts +18 -0
  44. package/build/devserver/mstore/store.js +82 -0
  45. package/build/devserver/runtime/host.d.ts +6 -0
  46. package/build/devserver/runtime/host.js +45 -1
  47. package/build/devserver/runtime/module.d.ts +1 -0
  48. package/build/devserver/runtime/module.js +27 -1
  49. package/build/devserver/server.d.ts +6 -0
  50. package/build/devserver/server.js +59 -0
  51. package/build/devserver/ssr.d.ts +25 -0
  52. package/build/devserver/ssr.js +114 -0
  53. package/build/devserver/wasm/sections.d.ts +2 -0
  54. package/build/devserver/wasm/sections.js +42 -0
  55. package/build/devserver/wasm/surface.d.ts +18 -0
  56. package/build/devserver/wasm/surface.js +41 -0
  57. package/docs/README.md +4 -4
  58. package/docs/auth-todo.md +6 -6
  59. package/docs/caching.md +5 -5
  60. package/docs/cli.md +15 -0
  61. package/docs/client.md +40 -0
  62. package/docs/crypto.md +4 -4
  63. package/docs/data.md +6 -6
  64. package/docs/email.md +28 -28
  65. package/docs/environment.md +10 -10
  66. package/docs/index.md +26 -0
  67. package/docs/ratelimit.md +10 -10
  68. package/docs/routing.md +2 -2
  69. package/docs/server.md +61 -0
  70. package/docs/ssr.md +561 -113
  71. package/docs/styling.md +22 -0
  72. package/docs/time.md +1 -1
  73. package/eslint.config.js +10 -1
  74. package/examples/basic/client/components/Header.tsx +3 -0
  75. package/examples/basic/client/routes/features/actions.tsx +0 -2
  76. package/examples/basic/client/routes/hello.tsx +89 -19
  77. package/examples/basic/client/styles/main.css +48 -0
  78. package/examples/basic/server/SsrHelloRender.ts +97 -0
  79. package/examples/basic/server/main.ts +5 -0
  80. package/examples/basic/server/streams/Echo.ts +49 -0
  81. package/package.json +12 -10
  82. package/scripts/gen-toil-docs.mjs +96 -0
  83. package/src/cli/create.ts +2 -2
  84. package/src/client/index.ts +1 -1
  85. package/src/client/routing/mount.tsx +19 -31
  86. package/src/client/ssr/markers.tsx +33 -4
  87. package/src/compiler/config.ts +88 -2
  88. package/src/compiler/docs.ts +47 -308
  89. package/src/compiler/index.ts +236 -32
  90. package/src/compiler/ssr-codegen.ts +1 -1
  91. package/src/compiler/template-build.ts +271 -53
  92. package/src/compiler/toil-docs.generated.ts +26 -0
  93. package/src/devserver/daemon/catalog.ts +120 -0
  94. package/src/devserver/daemon/cron.ts +87 -0
  95. package/src/devserver/daemon/host.ts +224 -0
  96. package/src/devserver/daemon/index.ts +349 -0
  97. package/src/devserver/db/catalog.ts +61 -53
  98. package/src/devserver/db/database.ts +613 -149
  99. package/src/devserver/db/index.ts +1 -1
  100. package/src/devserver/db/routeKinds.ts +147 -0
  101. package/src/devserver/db/types.ts +65 -2
  102. package/src/devserver/index.ts +12 -0
  103. package/src/devserver/mstore/store.ts +121 -0
  104. package/src/devserver/runtime/host.ts +92 -1
  105. package/src/devserver/runtime/module.ts +35 -1
  106. package/src/devserver/server.ts +101 -0
  107. package/src/devserver/ssr.ts +166 -0
  108. package/src/devserver/wasm/sections.ts +59 -0
  109. package/src/devserver/wasm/surface.ts +88 -0
  110. package/test/daemon-build.test.ts +198 -0
  111. package/test/daemon-catalog.test.ts +265 -0
  112. package/test/daemon-emulation.test.ts +216 -0
  113. package/test/devserver-database.test.ts +396 -5
  114. package/test/email-preview.test.ts +6 -1
  115. package/test/fixtures/daemon-app.ts +56 -0
  116. package/test/global-setup.ts +17 -0
  117. package/test/ssr-hydration.test.tsx +107 -0
  118. package/test/ssr-render.test.ts +96 -27
  119. package/test/ssr-template.test.tsx +47 -2
  120. package/vitest.config.ts +3 -0
@@ -27,12 +27,21 @@ import pc from 'picocolors';
27
27
 
28
28
  import type { EmailBackendConfig } from 'toiljs/shared';
29
29
 
30
+ import { DaemonHost, daemonEmulationEnabled } from './daemon/index.js';
31
+ import type { ResolvedDaemonConfig } from './daemon/host.js';
30
32
  import { configureDbPersistence } from './db/index.js';
31
33
  import { initEmailService } from './email/index.js';
32
34
  import { applyCacheRule, lookupCache } from './http/cache.js';
33
35
  import { type EnvelopeRequest, METHOD_CODES } from './http/envelope.js';
34
36
  import { proxyToVite, type ViteTarget, wireWebsocketProxy } from './http/proxy.js';
35
37
  import { WasmServerModule } from './runtime/module.js';
38
+ import {
39
+ assembleSsr,
40
+ buildSsrRoutes,
41
+ type DevSsrTemplate,
42
+ pathnameOf,
43
+ type SsrResult,
44
+ } from './ssr.js';
36
45
 
37
46
  const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
38
47
 
@@ -73,6 +82,16 @@ export interface DevServerOptions {
73
82
  readonly host?: string;
74
83
  /** Absolute path to the ToilScript server wasm (toilconfig `targets.release.outFile`). */
75
84
  readonly wasmFile: string;
85
+ /**
86
+ * Absolute path to the cold daemon artifact (`release-cold.wasm`). When present and the cold
87
+ * artifact declares a daemon surface, the dev daemon emulator drives its `@scheduled` tasks
88
+ * (per `nodeMode`). Omit for a project with no `@daemon` (the file is never built).
89
+ */
90
+ readonly coldWasmFile?: string;
91
+ /** Which layer the dev process emulates (gates the daemon emulator). Default `all`. */
92
+ readonly nodeMode?: string;
93
+ /** Daemon (L4) config mirror (drives the dev scheduler's budgets/caps). */
94
+ readonly daemon?: ResolvedDaemonConfig;
76
95
  /** The internal Vite dev server to proxy unclaimed traffic to. */
77
96
  readonly vite: ViteTarget;
78
97
  /** Max request body bytes. Default 8 MB. */
@@ -83,6 +102,13 @@ export interface DevServerOptions {
83
102
  * otherwise it stays a log-only mock. See `./email`.
84
103
  */
85
104
  readonly email?: EmailBackendConfig;
105
+ /**
106
+ * Edge-SSR templates (one per `ssr = true` route), extracted at dev startup
107
+ * against the live dev shell. When a GET/HEAD matches a route the dev server
108
+ * runs the guest `render`, splices the values, and serves the SSR HTML (same
109
+ * path as the prod edge). Omit / empty for a project with no SSR route.
110
+ */
111
+ readonly ssrTemplates?: readonly DevSsrTemplate[];
86
112
  }
87
113
 
88
114
  /** A running dev server. */
@@ -167,6 +193,23 @@ function sendWasmResponse(
167
193
  response.send(Buffer.from(result.body.buffer, result.body.byteOffset, result.body.length));
168
194
  }
169
195
 
196
+ /** Sends a spliced edge-SSR response (the full server-rendered HTML document). */
197
+ function sendSsr(response: Response, out: SsrResult, headOnly: boolean): void {
198
+ response.status(out.status);
199
+ let hasContentType = false;
200
+ for (const [name, value] of out.headers) {
201
+ if (name.toLowerCase() === 'content-type') hasContentType = true;
202
+ response.header(name, value);
203
+ }
204
+ if (!hasContentType) response.header('content-type', 'text/html; charset=utf-8');
205
+ response.header('server', 'toil-dev');
206
+ if (headOnly) {
207
+ response.send('');
208
+ return;
209
+ }
210
+ response.send(Buffer.from(out.html.buffer, out.html.byteOffset, out.html.length));
211
+ }
212
+
170
213
  /**
171
214
  * Starts the front server. The caller owns the Vite dev server (start it on a
172
215
  * loopback port first) and the toilscript rebuild watcher; this watches only
@@ -188,6 +231,16 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
188
231
 
189
232
  const module = new WasmServerModule(options.wasmFile);
190
233
 
234
+ // Edge-SSR routes (extracted against the live dev shell at startup). When a
235
+ // GET/HEAD matches one, the dev server runs the guest `render`, splices the
236
+ // values into the template, and serves the SSR HTML (prod-edge parity).
237
+ const ssrRoutes = buildSsrRoutes(options.ssrTemplates ?? []);
238
+ if (ssrRoutes.length > 0) {
239
+ process.stdout.write(
240
+ pc.dim(` edge SSR: ${String(ssrRoutes.length)} route(s) served server-side`) + '\n',
241
+ );
242
+ }
243
+
191
244
  // Persist dev DB data under the project's .toil/ so records, events, and their
192
245
  // schema_versions survive restarts (delete .toil/devdata.json to reset). Only
193
246
  // the running dev server persists; tests that construct WasmServerModule
@@ -231,6 +284,27 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
231
284
 
232
285
  wireWebsocketProxy(app, options.vite);
233
286
 
287
+ // Dev DAEMON (L4) emulation: load `release-cold.wasm` once, run `daemon_start()`, and drive
288
+ // its `@scheduled` tasks. Only when `nodeMode` is daemon/all and a cold artifact path is given;
289
+ // the host stays idle until the cold artifact appears (a `@daemon` build). It has no request to
290
+ // hang a refresh off, so it polls its own mtime-watch on a low-frequency timer (section 9.3).
291
+ const nodeMode = options.nodeMode ?? 'all';
292
+ let daemonHost: DaemonHost | null = null;
293
+ let daemonTimer: NodeJS.Timeout | null = null;
294
+ if (options.coldWasmFile !== undefined && daemonEmulationEnabled(nodeMode) && options.daemon) {
295
+ daemonHost = new DaemonHost(options.coldWasmFile, options.daemon, nodeMode);
296
+ const pollDaemon = (): void => {
297
+ try {
298
+ daemonHost?.refresh();
299
+ } catch (e) {
300
+ process.stdout.write(pc.red(` ✗ daemon reload failed: ${String(e)}`) + '\n');
301
+ }
302
+ };
303
+ pollDaemon();
304
+ daemonTimer = setInterval(pollDaemon, 500);
305
+ daemonTimer.unref?.();
306
+ }
307
+
234
308
  app.any('/*', async (request: Request, response: Response) => {
235
309
  response.removeHeader('uWebSockets');
236
310
 
@@ -275,6 +349,31 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
275
349
  response.status(500).send('internal error\n');
276
350
  return;
277
351
  }
352
+
353
+ // Edge SSR: handle() did not claim this path; if it matches an
354
+ // `ssr = true` route, run the guest `render`, splice the values, and
355
+ // serve the server-rendered HTML. A fail-safe envelope (no renderer
356
+ // matched / malformed) returns null, so we fall through to Vite (the
357
+ // route then client-renders, same as before).
358
+ if (
359
+ (request.method === 'GET' || request.method === 'HEAD') &&
360
+ ssrRoutes.length > 0
361
+ ) {
362
+ const route = ssrRoutes.find((r) => r.test(pathnameOf(request.url)));
363
+ if (route) {
364
+ try {
365
+ const out = assembleSsr(route, module.dispatchRender(envelopeReq));
366
+ if (out !== null) {
367
+ sendSsr(response, out, request.method === 'HEAD');
368
+ return;
369
+ }
370
+ } catch (e) {
371
+ process.stdout.write(
372
+ pc.red(` ✗ SSR ${request.path}: ${String(e)}`) + '\n',
373
+ );
374
+ }
375
+ }
376
+ }
278
377
  }
279
378
 
280
379
  await proxyToVite(request, response, options.vite);
@@ -286,6 +385,8 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
286
385
  port: options.port,
287
386
  host,
288
387
  close: async (): Promise<void> => {
388
+ if (daemonTimer !== null) clearInterval(daemonTimer);
389
+ daemonHost?.close();
289
390
  await app.shutdown();
290
391
  },
291
392
  };
@@ -0,0 +1,166 @@
1
+ /**
2
+ * Dev-server edge SSR: splice the guest `render` values into a route's
3
+ * template-with-holes and serve real server-rendered HTML, mirroring the
4
+ * production edge (`toil-backend/src/host/template/assemble.rs`).
5
+ *
6
+ * The templates are extracted once at dev startup against the LIVE (Vite-
7
+ * transformed) dev shell (see `compiler/template-build.ts extractDevSsrTemplates`)
8
+ * so the served markup boots the dev client and hydrates in place. At request
9
+ * time the dev server runs the real `render` export (`WasmServerModule.
10
+ * dispatchRender`), this module decodes the values envelope and splices each
11
+ * value at its manifest-fixed offset. The hash-coherence guard the prod edge
12
+ * enforces is skipped in dev: the guest and the template are built together
13
+ * here, so there is no deploy skew to catch — only fail-safe envelopes (status
14
+ * >= 500 or no slots) fall back to client rendering.
15
+ */
16
+
17
+ /** One SSR route's spliceable template + its slot insertion points. */
18
+ export interface DevSsrTemplate {
19
+ pattern: string;
20
+ name: string;
21
+ tmpl: Uint8Array;
22
+ entries: { id: number; offset: number }[];
23
+ }
24
+
25
+ /** A matchable SSR route. */
26
+ export interface SsrRoute {
27
+ /** Matches a request pathname (no query) to this route's template. */
28
+ test: (pathname: string) => boolean;
29
+ tmpl: Uint8Array;
30
+ entries: { id: number; offset: number }[];
31
+ }
32
+
33
+ /** The pathname of a request URL (strip the query string). */
34
+ export function pathnameOf(url: string): string {
35
+ const q = url.indexOf('?');
36
+ return q < 0 ? url : url.slice(0, q);
37
+ }
38
+
39
+ /** Compile a route pattern (`/hello`, `/u/:name`, `/blog/[id]`, `/files/[...path]`,
40
+ * `/*`) to a pathname matcher. Dynamic segments match one path segment; catch-all
41
+ * (`[...x]` / `*`) matches the rest. Trailing slashes are ignored. */
42
+ function patternToTest(pattern: string): (pathname: string) => boolean {
43
+ const norm = (p: string): string => (p.length > 1 && p.endsWith('/') ? p.slice(0, -1) : p);
44
+ let re = '';
45
+ // Tokenise into literal runs and dynamic/catch-all holes.
46
+ let i = 0;
47
+ while (i < pattern.length) {
48
+ const ch = pattern[i];
49
+ if (ch === ':') {
50
+ // `:name` — one segment.
51
+ i++;
52
+ while (i < pattern.length && /[A-Za-z0-9_]/.test(pattern[i])) i++;
53
+ re += '[^/]+';
54
+ } else if (ch === '*') {
55
+ re += '.*';
56
+ i++;
57
+ } else if (ch === '[') {
58
+ const end = pattern.indexOf(']', i);
59
+ const inner = end < 0 ? '' : pattern.slice(i + 1, end);
60
+ re += inner.startsWith('...') ? '.*' : '[^/]+';
61
+ i = end < 0 ? pattern.length : end + 1;
62
+ } else {
63
+ // Literal char, regex-escaped.
64
+ re += ch.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
65
+ i++;
66
+ }
67
+ }
68
+ const compiled = new RegExp(`^${re}$`);
69
+ return (pathname: string): boolean => compiled.test(norm(pathname));
70
+ }
71
+
72
+ /** Build matchable SSR routes from the extracted dev templates. */
73
+ export function buildSsrRoutes(templates: readonly DevSsrTemplate[]): SsrRoute[] {
74
+ return templates.map((t) => ({ test: patternToTest(t.pattern), tmpl: t.tmpl, entries: t.entries }));
75
+ }
76
+
77
+ /** A decoded guest values envelope. */
78
+ interface DecodedValues {
79
+ status: number;
80
+ headers: [string, string][];
81
+ /** Slot value bytes keyed by numeric slot id. */
82
+ values: Map<number, Uint8Array>;
83
+ }
84
+
85
+ /** Decode the guest values envelope (mirrors the prod host `decode_values`). All
86
+ * fields little-endian, no padding. Returns null on a malformed/short buffer. */
87
+ function decodeValues(buf: Uint8Array): DecodedValues | null {
88
+ const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
89
+ let o = 0;
90
+ const need = (n: number): boolean => o + n <= buf.byteLength;
91
+ try {
92
+ if (!need(2 + 32 + 2)) return null;
93
+ const status = dv.getUint16(o, true);
94
+ o += 2;
95
+ o += 32; // template hash (coherence is built together in dev; not checked)
96
+ const nHeaders = dv.getUint16(o, true);
97
+ o += 2;
98
+ const headers: [string, string][] = [];
99
+ const dec = new TextDecoder();
100
+ for (let i = 0; i < nHeaders; i++) {
101
+ if (!need(4)) return null;
102
+ const nameLen = dv.getUint16(o, true);
103
+ const valLen = dv.getUint16(o + 2, true);
104
+ o += 4;
105
+ if (!need(nameLen + valLen)) return null;
106
+ const name = dec.decode(buf.subarray(o, o + nameLen));
107
+ o += nameLen;
108
+ const val = dec.decode(buf.subarray(o, o + valLen));
109
+ o += valLen;
110
+ headers.push([name, val]);
111
+ }
112
+ if (!need(2)) return null;
113
+ const nSlots = dv.getUint16(o, true);
114
+ o += 2;
115
+ const values = new Map<number, Uint8Array>();
116
+ for (let i = 0; i < nSlots; i++) {
117
+ if (!need(2 + 1 + 4)) return null;
118
+ const id = dv.getUint16(o, true);
119
+ o += 2;
120
+ o += 1; // kind (the splice is kind-agnostic; the guest pre-escaped/stamped)
121
+ const len = dv.getUint32(o, true);
122
+ o += 4;
123
+ if (!need(len)) return null;
124
+ values.set(id, buf.subarray(o, o + len));
125
+ o += len;
126
+ }
127
+ return { status, headers, values };
128
+ } catch {
129
+ return null;
130
+ }
131
+ }
132
+
133
+ /** Splice ascending-offset inserts into the template (mirrors the host `assemble`). */
134
+ function splice(tmpl: Uint8Array, inserts: { offset: number; value: Uint8Array }[]): Uint8Array {
135
+ const parts: Uint8Array[] = [];
136
+ let prev = 0;
137
+ for (const ins of inserts) {
138
+ if (ins.offset > prev) parts.push(tmpl.subarray(prev, ins.offset));
139
+ if (ins.value.length > 0) parts.push(ins.value);
140
+ prev = ins.offset;
141
+ }
142
+ if (tmpl.length > prev) parts.push(tmpl.subarray(prev));
143
+ return Buffer.concat(parts.map((p) => Buffer.from(p.buffer, p.byteOffset, p.byteLength)));
144
+ }
145
+
146
+ /** A spliced SSR response. */
147
+ export interface SsrResult {
148
+ status: number;
149
+ headers: [string, string][];
150
+ html: Uint8Array;
151
+ }
152
+
153
+ /**
154
+ * Decode the guest envelope and splice it into `route`'s template. Returns null
155
+ * to fall back to client rendering: a fail-safe envelope (status >= 500, e.g. no
156
+ * renderer matched), no slots, or a decode error.
157
+ */
158
+ export function assembleSsr(route: SsrRoute, envelope: Uint8Array): SsrResult | null {
159
+ const decoded = decodeValues(envelope);
160
+ if (decoded === null) return null;
161
+ if (decoded.status >= 500 || decoded.values.size === 0) return null;
162
+ const inserts = route.entries
163
+ .map((e) => ({ offset: e.offset, value: decoded.values.get(e.id) ?? new Uint8Array(0) }))
164
+ .sort((a, b) => a.offset - b.offset);
165
+ return { status: decoded.status, headers: decoded.headers, html: splice(route.tmpl, inserts) };
166
+ }
@@ -0,0 +1,59 @@
1
+ /**
2
+ * Shared, bounds-checked wasm custom-section walker. Factored out of
3
+ * `db/catalog.ts` so the three Toil section parsers (`toildb.catalog`,
4
+ * `toilstream.catalog`, `toildaemon.catalog`, `toil.surface`) share ONE
5
+ * magic-skipping loop instead of drifting copies.
6
+ *
7
+ * Every input is a tenant-built, possibly mid-rebuild wasm, so the walker must
8
+ * never read past the buffer: a truncated/garbage section table returns `null`
9
+ * (treated by callers as "no section"). Mirrors the host-side walker
10
+ * (`toil-backend` custom-section reader) and the toilscript-side test walker.
11
+ */
12
+
13
+ /** Read a LEB128 from `buf` at `pos`; throws on overrun so a truncated module
14
+ * can never over-read (the caller catches and treats it as "no section"). */
15
+ export function leb(buf: Buffer, pos: number): [number, number] {
16
+ let result = 0;
17
+ let shift = 0;
18
+ let p = pos;
19
+ for (;;) {
20
+ if (p >= buf.length) throw new RangeError('leb128 past end of buffer');
21
+ const b = buf[p++];
22
+ result |= (b & 0x7f) << shift;
23
+ if ((b & 0x80) === 0) break;
24
+ shift += 7;
25
+ if (shift > 35) throw new RangeError('leb128 too long');
26
+ }
27
+ return [result >>> 0, p];
28
+ }
29
+
30
+ /** The bytes of the named wasm custom section, or `null` if absent. Bounds-checked
31
+ * so a truncated/garbage module can never read past the buffer. */
32
+ export function customSection(wasm: Buffer, want: string): Buffer | null {
33
+ if (
34
+ wasm.length < 8 ||
35
+ wasm[0] !== 0x00 ||
36
+ wasm[1] !== 0x61 ||
37
+ wasm[2] !== 0x73 ||
38
+ wasm[3] !== 0x6d
39
+ )
40
+ return null;
41
+ let pos = 8; // skip the 8-byte magic + version header
42
+ while (pos < wasm.length) {
43
+ const id = wasm[pos++];
44
+ let size: number;
45
+ [size, pos] = leb(wasm, pos);
46
+ const end = pos + size;
47
+ if (end > wasm.length || end < pos) return null; // truncated section table
48
+ if (id === 0) {
49
+ const [nameLen, namePos] = leb(wasm, pos);
50
+ if (
51
+ namePos + nameLen <= end &&
52
+ wasm.toString('latin1', namePos, namePos + nameLen) === want
53
+ )
54
+ return wasm.subarray(namePos + nameLen, end);
55
+ }
56
+ pos = end;
57
+ }
58
+ return null;
59
+ }
@@ -0,0 +1,88 @@
1
+ /**
2
+ * Parse a compiled artifact's `toil.surface` custom section. Emitted into EVERY
3
+ * Toil artifact by the toilscript `buildToilSurface` pass (hot AND cold). The dev
4
+ * server reads it to decide whether each artifact carries the daemon surface
5
+ * (start the daemon emulator) or the stream surface (Phase 4, deferred).
6
+ *
7
+ * Byte layout (RECONCILIATION Part 5, all little-endian; mirrors the toilscript
8
+ * `CatWriter` emitter byte-for-byte):
9
+ *
10
+ * u16 format_version = 1
11
+ * u8 target_mode (0 = hot, 1 = cold; there is no target_mode = 2)
12
+ * u8 reserved0
13
+ * u32 surface_flags (bit0 rest, bit1 stream, bit2 daemon,
14
+ * bit3 scheduled, bit4 database, bit5 render)
15
+ * u16 abi_version
16
+ * str build_id (u32 len + UTF-8)
17
+ * u32 fingerprint
18
+ * u32 data_coherence_hash
19
+ * u32 pair_coherence_hash (exactly THREE u32 after build_id, not four)
20
+ *
21
+ * Fail-closed per Part 5's host rule: an ABSENT section is "legacy single
22
+ * artifact, load as hot" (NOT a hard reject); a PRESENT-but-unparseable section is
23
+ * a corrupt artifact -> do not start that artifact's emulator.
24
+ */
25
+
26
+ import { DataReader } from 'toiljs/io';
27
+
28
+ import { customSection } from './sections.js';
29
+
30
+ export interface SurfaceFlags {
31
+ readonly rest: boolean;
32
+ readonly stream: boolean;
33
+ readonly daemon: boolean;
34
+ readonly scheduled: boolean;
35
+ readonly database: boolean;
36
+ readonly render: boolean;
37
+ }
38
+
39
+ export interface Surface {
40
+ /** 0 = hot, 1 = cold. */
41
+ readonly targetMode: 'hot' | 'cold';
42
+ readonly flags: SurfaceFlags;
43
+ readonly abiVersion: number;
44
+ readonly buildId: string;
45
+ readonly fingerprint: number;
46
+ readonly dataCoherenceHash: number;
47
+ readonly pairCoherenceHash: number;
48
+ }
49
+
50
+ /** `'absent'` => legacy single artifact (load as hot, no emulators).
51
+ * `'invalid'` => present but corrupt (fail closed). Otherwise the parsed surface. */
52
+ export function parseSurface(wasm: Buffer): Surface | 'absent' | 'invalid' {
53
+ let sec: Buffer | null;
54
+ try {
55
+ sec = customSection(wasm, 'toil.surface');
56
+ } catch {
57
+ return 'invalid'; // garbage section table
58
+ }
59
+ if (sec === null) return 'absent';
60
+
61
+ const r = new DataReader(sec);
62
+ r.readU16(); // format_version
63
+ const targetMode = r.readU8() === 1 ? 'cold' : 'hot';
64
+ r.readU8(); // reserved0
65
+ const f = r.readU32(); // surface_flags
66
+ const abiVersion = r.readU16();
67
+ const buildId = r.readString();
68
+ const fingerprint = r.readU32();
69
+ const dataCoherenceHash = r.readU32(); // exactly THREE u32 after build_id
70
+ const pairCoherenceHash = r.readU32();
71
+ if (!r.ok) return 'invalid'; // PRESENT but corrupt => fail closed
72
+ return {
73
+ targetMode,
74
+ flags: {
75
+ rest: !!(f & 1),
76
+ stream: !!(f & 2),
77
+ daemon: !!(f & 4),
78
+ scheduled: !!(f & 8),
79
+ database: !!(f & 16),
80
+ render: !!(f & 32),
81
+ },
82
+ abiVersion,
83
+ buildId,
84
+ fingerprint,
85
+ dataCoherenceHash,
86
+ pairCoherenceHash,
87
+ };
88
+ }
@@ -0,0 +1,198 @@
1
+ /**
2
+ * Two-pass build pipeline (doc 08 section 1). Asserts:
3
+ *
4
+ * - `serverArtifacts` derives the `-hot`/`-cold` paths from `outFile` when
5
+ * `hotFile`/`coldFile` are absent, and honors them when present.
6
+ * - `SURFACE_DECORATOR` matches `@stream`/`@daemon`/`@scheduled` at line start
7
+ * (not in a comment).
8
+ * - `buildServer` on a project that declares a `@daemon` runs TWO toilscript
9
+ * passes (one `--targetMode cold`, one `--targetMode hot`) and produces BOTH
10
+ * `release-hot.wasm` and `release-cold.wasm`; the cold artifact decodes to a
11
+ * daemon catalog and its `toil.surface` is target_mode = cold.
12
+ * - a project with only the legacy request surface keeps the single-artifact
13
+ * path (no cold pass, no cold artifact).
14
+ *
15
+ * The build invokes the LOCAL toilscript (branch feat/streams-phase0-compiler),
16
+ * which supports `--targetMode`; the test links it into the fixture project's
17
+ * `node_modules` the same way the dev build resolves it (`require.resolve`).
18
+ */
19
+
20
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
21
+ import { tmpdir } from 'node:os';
22
+ import { dirname, join } from 'node:path';
23
+ import { fileURLToPath } from 'node:url';
24
+
25
+ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
26
+
27
+ import {
28
+ buildServer,
29
+ serverArtifacts,
30
+ splitSurfaceFiles,
31
+ SURFACE_DECORATOR,
32
+ } from '../src/compiler/index.js';
33
+ import { parseDaemonCatalog } from '../src/devserver/daemon/catalog.js';
34
+ import { parseSurface } from '../src/devserver/wasm/surface.js';
35
+
36
+ const here = dirname(fileURLToPath(import.meta.url));
37
+ const LOCAL_TOILSCRIPT = join(here, '..', '..', 'toilscript');
38
+
39
+ let tmp: string;
40
+
41
+ beforeEach(() => {
42
+ tmp = mkdtempSync(join(tmpdir(), 'daemon-build-'));
43
+ });
44
+ afterEach(() => {
45
+ rmSync(tmp, { recursive: true, force: true });
46
+ });
47
+
48
+ /** Scaffold a minimal project at `tmp` with `server/main.ts` = `serverSrc`, the
49
+ * given toilconfig, and `node_modules/toilscript` symlinked to the local build. */
50
+ function scaffold(serverSrc: string, toilconfig: object): void {
51
+ writeFileSync(join(tmp, 'package.json'), JSON.stringify({ name: 'fixture', type: 'module' }));
52
+ writeFileSync(join(tmp, 'toilconfig.json'), JSON.stringify(toilconfig, null, 2));
53
+ mkdirSync(join(tmp, 'server'), { recursive: true });
54
+ writeFileSync(join(tmp, 'server', 'main.ts'), serverSrc);
55
+ mkdirSync(join(tmp, 'node_modules'), { recursive: true });
56
+ symlinkSync(LOCAL_TOILSCRIPT, join(tmp, 'node_modules', 'toilscript'), 'dir');
57
+ }
58
+
59
+ const BASE_TOILCONFIG = {
60
+ entries: ['server/main.ts'],
61
+ targets: { release: { outFile: 'build/server/release.wasm' } },
62
+ options: { runtime: 'stub', optimizeLevel: 0, shrinkLevel: 0 },
63
+ };
64
+
65
+ // A @daemon that declares its host imports directly (no toiljs globals lib needed).
66
+ const DAEMON_SRC = `@daemon
67
+ class Jobs {
68
+ @scheduled("2s") fast(): void {}
69
+ @scheduled("0 0 * * *") nightly(): void {}
70
+ }
71
+ export function probe(): i32 { return 1; }
72
+ `;
73
+
74
+ const LEGACY_SRC = `export function handle(ofs: i32, len: i32): i64 { return 0; }
75
+ export function probe(): i32 { return 1; }
76
+ `;
77
+
78
+ describe('serverArtifacts path derivation', () => {
79
+ it('derives -hot/-cold from outFile when hotFile/coldFile are absent', () => {
80
+ writeFileSync(
81
+ join(tmp, 'toilconfig.json'),
82
+ JSON.stringify({ targets: { release: { outFile: 'build/server/release.wasm' } } }),
83
+ );
84
+ const a = serverArtifacts(tmp);
85
+ expect(a.hot).toBe(join(tmp, 'build/server/release-hot.wasm'));
86
+ expect(a.cold).toBe(join(tmp, 'build/server/release-cold.wasm'));
87
+ });
88
+
89
+ it('honors explicit hotFile/coldFile when present', () => {
90
+ writeFileSync(
91
+ join(tmp, 'toilconfig.json'),
92
+ JSON.stringify({
93
+ targets: {
94
+ release: {
95
+ outFile: 'build/server/release.wasm',
96
+ hotFile: 'out/hot.wasm',
97
+ coldFile: 'out/cold.wasm',
98
+ },
99
+ },
100
+ }),
101
+ );
102
+ const a = serverArtifacts(tmp);
103
+ expect(a.hot).toBe(join(tmp, 'out/hot.wasm'));
104
+ expect(a.cold).toBe(join(tmp, 'out/cold.wasm'));
105
+ });
106
+ });
107
+
108
+ describe('SURFACE_DECORATOR', () => {
109
+ it('matches the streams/daemon decorators at line start', () => {
110
+ for (const deco of ['@stream', '@daemon', '@scheduled', '@rest', '@data']) {
111
+ expect(SURFACE_DECORATOR.test(`${deco} class X {}`)).toBe(true);
112
+ expect(SURFACE_DECORATOR.test(` ${deco}\nclass X {}`)).toBe(true);
113
+ }
114
+ });
115
+
116
+ it('does NOT match a decorator mentioned only in a comment', () => {
117
+ expect(SURFACE_DECORATOR.test('// the @daemon decorator marks a cold class')).toBe(false);
118
+ expect(SURFACE_DECORATOR.test('const s = "uses @scheduled internally";')).toBe(false);
119
+ });
120
+ });
121
+
122
+ describe('splitSurfaceFiles per-pass classification', () => {
123
+ /** Lay down `name -> contents` files under `tmp` and return their relative paths. */
124
+ function lay(files: Record<string, string>): string[] {
125
+ mkdirSync(join(tmp, 'server'), { recursive: true });
126
+ const rels: string[] = [];
127
+ for (const [name, src] of Object.entries(files)) {
128
+ writeFileSync(join(tmp, name), src);
129
+ rels.push(name);
130
+ }
131
+ return rels;
132
+ }
133
+
134
+ it('drops daemon-only files from the hot pass and hot-only files from the cold pass', () => {
135
+ const rels = lay({
136
+ 'server/jobs.ts': '@daemon\nclass J { @scheduled("1s") t(): void {} }\n',
137
+ 'server/api.ts': '@rest\nclass A {}\n',
138
+ 'server/model.ts': '@data\nclass M {}\n',
139
+ 'server/util.ts': 'export function helper(): i32 { return 1; }\n',
140
+ });
141
+ const split = splitSurfaceFiles(tmp, rels);
142
+ expect(split.hasDaemon).toBe(true);
143
+ // hot pass: everything except the daemon-only jobs.ts.
144
+ expect(split.hot.sort()).toEqual(
145
+ ['server/api.ts', 'server/model.ts', 'server/util.ts'].sort(),
146
+ );
147
+ // cold pass: everything except the hot-only api.ts.
148
+ expect(split.cold.sort()).toEqual(
149
+ ['server/jobs.ts', 'server/model.ts', 'server/util.ts'].sort(),
150
+ );
151
+ });
152
+
153
+ it('keeps a file that mixes both surfaces in both passes', () => {
154
+ const rels = lay({ 'server/both.ts': '@daemon\nclass J {}\n@rest\nclass A {}\n' });
155
+ const split = splitSurfaceFiles(tmp, rels);
156
+ expect(split.hot).toContain('server/both.ts');
157
+ expect(split.cold).toContain('server/both.ts');
158
+ });
159
+ });
160
+
161
+ // Needs the local toilscript dev build (with --targetMode) linked as a sibling
162
+ // repo; skip where it is absent (e.g. CI, which has only the published dep).
163
+ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT))('buildServer two-pass (daemon project)', () => {
164
+ it('runs the cold pass and produces the cold artifact with a daemon catalog', async () => {
165
+ scaffold(DAEMON_SRC, BASE_TOILCONFIG);
166
+ await buildServer(tmp);
167
+
168
+ const cold = join(tmp, 'build/server/release-cold.wasm');
169
+ expect(existsSync(cold), 'cold artifact missing').toBe(true);
170
+
171
+ // The cold artifact carries the daemon surface + catalog (decoded byte-for-byte).
172
+ const coldBytes = readFileSync(cold);
173
+ const surface = parseSurface(coldBytes);
174
+ expect(surface !== 'absent' && surface !== 'invalid' && surface.targetMode).toBe('cold');
175
+ expect(surface !== 'absent' && surface !== 'invalid' && surface.flags.daemon).toBe(true);
176
+
177
+ const catalog = parseDaemonCatalog(coldBytes);
178
+ expect(catalog).not.toBeNull();
179
+ expect(catalog!.hasDaemon).toBe(true);
180
+ expect(catalog!.tasks.map((t) => t.name)).toEqual(['fast', 'nightly']);
181
+ expect(catalog!.tasks[0].schedule.kind).toBe('interval');
182
+ expect(catalog!.tasks[1].schedule.kind).toBe('cron');
183
+
184
+ // A daemon-only project (no request/stream surface) has no hot files, so the hot pass is
185
+ // skipped (toilscript would HARD-ERROR a @daemon class under --targetMode hot). The legacy
186
+ // single-artifact `release.wasm` is therefore not produced for a pure background worker.
187
+ expect(existsSync(join(tmp, 'build/server/release.wasm'))).toBe(false);
188
+ }, 60_000);
189
+
190
+ it('keeps the single-artifact path for a legacy (no-daemon) project', async () => {
191
+ scaffold(LEGACY_SRC, BASE_TOILCONFIG);
192
+ await buildServer(tmp);
193
+
194
+ expect(existsSync(join(tmp, 'build/server/release.wasm'))).toBe(true);
195
+ // No @daemon -> no cold pass -> no cold artifact.
196
+ expect(existsSync(join(tmp, 'build/server/release-cold.wasm'))).toBe(false);
197
+ }, 60_000);
198
+ });