toiljs 0.0.60 → 0.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +31 -0
- package/CHANGELOG.md +17 -0
- package/build/cli/.tsbuildinfo +1 -1
- package/build/cli/index.js +2 -2
- package/build/client/.tsbuildinfo +1 -1
- package/build/client/index.d.ts +1 -1
- package/build/client/index.js +1 -1
- package/build/client/routing/mount.js +11 -26
- package/build/client/ssr/markers.d.ts +1 -0
- package/build/client/ssr/markers.js +9 -2
- package/build/compiler/.tsbuildinfo +1 -1
- package/build/compiler/config.d.ts +21 -0
- package/build/compiler/config.js +35 -0
- package/build/compiler/docs.d.ts +2 -1
- package/build/compiler/docs.js +33 -304
- package/build/compiler/index.d.ts +13 -0
- package/build/compiler/index.js +113 -21
- package/build/compiler/template-build.d.ts +23 -3
- package/build/compiler/template-build.js +120 -30
- package/build/compiler/toil-docs.generated.d.ts +1 -0
- package/build/compiler/toil-docs.generated.js +20 -0
- package/build/devserver/.tsbuildinfo +1 -1
- package/build/devserver/daemon/catalog.d.ts +26 -0
- package/build/devserver/daemon/catalog.js +48 -0
- package/build/devserver/daemon/cron.d.ts +4 -0
- package/build/devserver/daemon/cron.js +50 -0
- package/build/devserver/daemon/host.d.ts +37 -0
- package/build/devserver/daemon/host.js +94 -0
- package/build/devserver/daemon/index.d.ts +34 -0
- package/build/devserver/daemon/index.js +241 -0
- package/build/devserver/db/catalog.d.ts +2 -1
- package/build/devserver/db/catalog.js +44 -44
- package/build/devserver/db/database.d.ts +27 -11
- package/build/devserver/db/database.js +539 -169
- package/build/devserver/db/index.d.ts +1 -1
- package/build/devserver/db/index.js +1 -1
- package/build/devserver/db/routeKinds.d.ts +8 -0
- package/build/devserver/db/routeKinds.js +139 -0
- package/build/devserver/db/types.d.ts +64 -1
- package/build/devserver/db/types.js +33 -1
- package/build/devserver/index.d.ts +10 -0
- package/build/devserver/index.js +7 -0
- package/build/devserver/mstore/store.d.ts +18 -0
- package/build/devserver/mstore/store.js +82 -0
- package/build/devserver/runtime/host.d.ts +6 -0
- package/build/devserver/runtime/host.js +45 -1
- package/build/devserver/runtime/module.d.ts +1 -0
- package/build/devserver/runtime/module.js +27 -1
- package/build/devserver/server.d.ts +6 -0
- package/build/devserver/server.js +59 -0
- package/build/devserver/ssr.d.ts +25 -0
- package/build/devserver/ssr.js +114 -0
- package/build/devserver/wasm/sections.d.ts +2 -0
- package/build/devserver/wasm/sections.js +42 -0
- package/build/devserver/wasm/surface.d.ts +18 -0
- package/build/devserver/wasm/surface.js +41 -0
- package/docs/README.md +4 -4
- package/docs/auth-todo.md +6 -6
- package/docs/caching.md +5 -5
- package/docs/cli.md +15 -0
- package/docs/client.md +40 -0
- package/docs/crypto.md +4 -4
- package/docs/data.md +6 -6
- package/docs/email.md +28 -28
- package/docs/environment.md +10 -10
- package/docs/index.md +26 -0
- package/docs/ratelimit.md +10 -10
- package/docs/routing.md +2 -2
- package/docs/server.md +61 -0
- package/docs/ssr.md +561 -113
- package/docs/styling.md +22 -0
- package/docs/time.md +1 -1
- package/eslint.config.js +10 -1
- package/examples/basic/client/components/Header.tsx +3 -0
- package/examples/basic/client/routes/features/actions.tsx +0 -2
- package/examples/basic/client/routes/hello.tsx +89 -19
- package/examples/basic/client/styles/main.css +48 -0
- package/examples/basic/server/SsrHelloRender.ts +97 -0
- package/examples/basic/server/main.ts +5 -0
- package/examples/basic/server/streams/Echo.ts +49 -0
- package/package.json +12 -10
- package/scripts/gen-toil-docs.mjs +96 -0
- package/src/cli/create.ts +2 -2
- package/src/client/index.ts +1 -1
- package/src/client/routing/mount.tsx +19 -31
- package/src/client/ssr/markers.tsx +33 -4
- package/src/compiler/config.ts +88 -2
- package/src/compiler/docs.ts +47 -308
- package/src/compiler/index.ts +236 -32
- package/src/compiler/ssr-codegen.ts +1 -1
- package/src/compiler/template-build.ts +271 -53
- package/src/compiler/toil-docs.generated.ts +26 -0
- package/src/devserver/daemon/catalog.ts +120 -0
- package/src/devserver/daemon/cron.ts +87 -0
- package/src/devserver/daemon/host.ts +224 -0
- package/src/devserver/daemon/index.ts +349 -0
- package/src/devserver/db/catalog.ts +61 -53
- package/src/devserver/db/database.ts +613 -149
- package/src/devserver/db/index.ts +1 -1
- package/src/devserver/db/routeKinds.ts +147 -0
- package/src/devserver/db/types.ts +65 -2
- package/src/devserver/index.ts +12 -0
- package/src/devserver/mstore/store.ts +121 -0
- package/src/devserver/runtime/host.ts +92 -1
- package/src/devserver/runtime/module.ts +35 -1
- package/src/devserver/server.ts +101 -0
- package/src/devserver/ssr.ts +166 -0
- package/src/devserver/wasm/sections.ts +59 -0
- package/src/devserver/wasm/surface.ts +88 -0
- package/test/daemon-build.test.ts +198 -0
- package/test/daemon-catalog.test.ts +265 -0
- package/test/daemon-emulation.test.ts +216 -0
- package/test/devserver-database.test.ts +396 -5
- package/test/email-preview.test.ts +6 -1
- package/test/fixtures/daemon-app.ts +56 -0
- package/test/global-setup.ts +17 -0
- package/test/ssr-hydration.test.tsx +107 -0
- package/test/ssr-render.test.ts +96 -27
- package/test/ssr-template.test.tsx +47 -2
- package/vitest.config.ts +3 -0
package/src/devserver/server.ts
CHANGED
|
@@ -27,12 +27,21 @@ import pc from 'picocolors';
|
|
|
27
27
|
|
|
28
28
|
import type { EmailBackendConfig } from 'toiljs/shared';
|
|
29
29
|
|
|
30
|
+
import { DaemonHost, daemonEmulationEnabled } from './daemon/index.js';
|
|
31
|
+
import type { ResolvedDaemonConfig } from './daemon/host.js';
|
|
30
32
|
import { configureDbPersistence } from './db/index.js';
|
|
31
33
|
import { initEmailService } from './email/index.js';
|
|
32
34
|
import { applyCacheRule, lookupCache } from './http/cache.js';
|
|
33
35
|
import { type EnvelopeRequest, METHOD_CODES } from './http/envelope.js';
|
|
34
36
|
import { proxyToVite, type ViteTarget, wireWebsocketProxy } from './http/proxy.js';
|
|
35
37
|
import { WasmServerModule } from './runtime/module.js';
|
|
38
|
+
import {
|
|
39
|
+
assembleSsr,
|
|
40
|
+
buildSsrRoutes,
|
|
41
|
+
type DevSsrTemplate,
|
|
42
|
+
pathnameOf,
|
|
43
|
+
type SsrResult,
|
|
44
|
+
} from './ssr.js';
|
|
36
45
|
|
|
37
46
|
const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
|
|
38
47
|
|
|
@@ -73,6 +82,16 @@ export interface DevServerOptions {
|
|
|
73
82
|
readonly host?: string;
|
|
74
83
|
/** Absolute path to the ToilScript server wasm (toilconfig `targets.release.outFile`). */
|
|
75
84
|
readonly wasmFile: string;
|
|
85
|
+
/**
|
|
86
|
+
* Absolute path to the cold daemon artifact (`release-cold.wasm`). When present and the cold
|
|
87
|
+
* artifact declares a daemon surface, the dev daemon emulator drives its `@scheduled` tasks
|
|
88
|
+
* (per `nodeMode`). Omit for a project with no `@daemon` (the file is never built).
|
|
89
|
+
*/
|
|
90
|
+
readonly coldWasmFile?: string;
|
|
91
|
+
/** Which layer the dev process emulates (gates the daemon emulator). Default `all`. */
|
|
92
|
+
readonly nodeMode?: string;
|
|
93
|
+
/** Daemon (L4) config mirror (drives the dev scheduler's budgets/caps). */
|
|
94
|
+
readonly daemon?: ResolvedDaemonConfig;
|
|
76
95
|
/** The internal Vite dev server to proxy unclaimed traffic to. */
|
|
77
96
|
readonly vite: ViteTarget;
|
|
78
97
|
/** Max request body bytes. Default 8 MB. */
|
|
@@ -83,6 +102,13 @@ export interface DevServerOptions {
|
|
|
83
102
|
* otherwise it stays a log-only mock. See `./email`.
|
|
84
103
|
*/
|
|
85
104
|
readonly email?: EmailBackendConfig;
|
|
105
|
+
/**
|
|
106
|
+
* Edge-SSR templates (one per `ssr = true` route), extracted at dev startup
|
|
107
|
+
* against the live dev shell. When a GET/HEAD matches a route the dev server
|
|
108
|
+
* runs the guest `render`, splices the values, and serves the SSR HTML (same
|
|
109
|
+
* path as the prod edge). Omit / empty for a project with no SSR route.
|
|
110
|
+
*/
|
|
111
|
+
readonly ssrTemplates?: readonly DevSsrTemplate[];
|
|
86
112
|
}
|
|
87
113
|
|
|
88
114
|
/** A running dev server. */
|
|
@@ -167,6 +193,23 @@ function sendWasmResponse(
|
|
|
167
193
|
response.send(Buffer.from(result.body.buffer, result.body.byteOffset, result.body.length));
|
|
168
194
|
}
|
|
169
195
|
|
|
196
|
+
/** Sends a spliced edge-SSR response (the full server-rendered HTML document). */
|
|
197
|
+
function sendSsr(response: Response, out: SsrResult, headOnly: boolean): void {
|
|
198
|
+
response.status(out.status);
|
|
199
|
+
let hasContentType = false;
|
|
200
|
+
for (const [name, value] of out.headers) {
|
|
201
|
+
if (name.toLowerCase() === 'content-type') hasContentType = true;
|
|
202
|
+
response.header(name, value);
|
|
203
|
+
}
|
|
204
|
+
if (!hasContentType) response.header('content-type', 'text/html; charset=utf-8');
|
|
205
|
+
response.header('server', 'toil-dev');
|
|
206
|
+
if (headOnly) {
|
|
207
|
+
response.send('');
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
response.send(Buffer.from(out.html.buffer, out.html.byteOffset, out.html.length));
|
|
211
|
+
}
|
|
212
|
+
|
|
170
213
|
/**
|
|
171
214
|
* Starts the front server. The caller owns the Vite dev server (start it on a
|
|
172
215
|
* loopback port first) and the toilscript rebuild watcher; this watches only
|
|
@@ -188,6 +231,16 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
|
|
|
188
231
|
|
|
189
232
|
const module = new WasmServerModule(options.wasmFile);
|
|
190
233
|
|
|
234
|
+
// Edge-SSR routes (extracted against the live dev shell at startup). When a
|
|
235
|
+
// GET/HEAD matches one, the dev server runs the guest `render`, splices the
|
|
236
|
+
// values into the template, and serves the SSR HTML (prod-edge parity).
|
|
237
|
+
const ssrRoutes = buildSsrRoutes(options.ssrTemplates ?? []);
|
|
238
|
+
if (ssrRoutes.length > 0) {
|
|
239
|
+
process.stdout.write(
|
|
240
|
+
pc.dim(` edge SSR: ${String(ssrRoutes.length)} route(s) served server-side`) + '\n',
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
|
|
191
244
|
// Persist dev DB data under the project's .toil/ so records, events, and their
|
|
192
245
|
// schema_versions survive restarts (delete .toil/devdata.json to reset). Only
|
|
193
246
|
// the running dev server persists; tests that construct WasmServerModule
|
|
@@ -231,6 +284,27 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
|
|
|
231
284
|
|
|
232
285
|
wireWebsocketProxy(app, options.vite);
|
|
233
286
|
|
|
287
|
+
// Dev DAEMON (L4) emulation: load `release-cold.wasm` once, run `daemon_start()`, and drive
|
|
288
|
+
// its `@scheduled` tasks. Only when `nodeMode` is daemon/all and a cold artifact path is given;
|
|
289
|
+
// the host stays idle until the cold artifact appears (a `@daemon` build). It has no request to
|
|
290
|
+
// hang a refresh off, so it polls its own mtime-watch on a low-frequency timer (section 9.3).
|
|
291
|
+
const nodeMode = options.nodeMode ?? 'all';
|
|
292
|
+
let daemonHost: DaemonHost | null = null;
|
|
293
|
+
let daemonTimer: NodeJS.Timeout | null = null;
|
|
294
|
+
if (options.coldWasmFile !== undefined && daemonEmulationEnabled(nodeMode) && options.daemon) {
|
|
295
|
+
daemonHost = new DaemonHost(options.coldWasmFile, options.daemon, nodeMode);
|
|
296
|
+
const pollDaemon = (): void => {
|
|
297
|
+
try {
|
|
298
|
+
daemonHost?.refresh();
|
|
299
|
+
} catch (e) {
|
|
300
|
+
process.stdout.write(pc.red(` ✗ daemon reload failed: ${String(e)}`) + '\n');
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
pollDaemon();
|
|
304
|
+
daemonTimer = setInterval(pollDaemon, 500);
|
|
305
|
+
daemonTimer.unref?.();
|
|
306
|
+
}
|
|
307
|
+
|
|
234
308
|
app.any('/*', async (request: Request, response: Response) => {
|
|
235
309
|
response.removeHeader('uWebSockets');
|
|
236
310
|
|
|
@@ -275,6 +349,31 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
|
|
|
275
349
|
response.status(500).send('internal error\n');
|
|
276
350
|
return;
|
|
277
351
|
}
|
|
352
|
+
|
|
353
|
+
// Edge SSR: handle() did not claim this path; if it matches an
|
|
354
|
+
// `ssr = true` route, run the guest `render`, splice the values, and
|
|
355
|
+
// serve the server-rendered HTML. A fail-safe envelope (no renderer
|
|
356
|
+
// matched / malformed) returns null, so we fall through to Vite (the
|
|
357
|
+
// route then client-renders, same as before).
|
|
358
|
+
if (
|
|
359
|
+
(request.method === 'GET' || request.method === 'HEAD') &&
|
|
360
|
+
ssrRoutes.length > 0
|
|
361
|
+
) {
|
|
362
|
+
const route = ssrRoutes.find((r) => r.test(pathnameOf(request.url)));
|
|
363
|
+
if (route) {
|
|
364
|
+
try {
|
|
365
|
+
const out = assembleSsr(route, module.dispatchRender(envelopeReq));
|
|
366
|
+
if (out !== null) {
|
|
367
|
+
sendSsr(response, out, request.method === 'HEAD');
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
} catch (e) {
|
|
371
|
+
process.stdout.write(
|
|
372
|
+
pc.red(` ✗ SSR ${request.path}: ${String(e)}`) + '\n',
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
278
377
|
}
|
|
279
378
|
|
|
280
379
|
await proxyToVite(request, response, options.vite);
|
|
@@ -286,6 +385,8 @@ export async function startDevServer(options: DevServerOptions): Promise<Running
|
|
|
286
385
|
port: options.port,
|
|
287
386
|
host,
|
|
288
387
|
close: async (): Promise<void> => {
|
|
388
|
+
if (daemonTimer !== null) clearInterval(daemonTimer);
|
|
389
|
+
daemonHost?.close();
|
|
289
390
|
await app.shutdown();
|
|
290
391
|
},
|
|
291
392
|
};
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dev-server edge SSR: splice the guest `render` values into a route's
|
|
3
|
+
* template-with-holes and serve real server-rendered HTML, mirroring the
|
|
4
|
+
* production edge (`toil-backend/src/host/template/assemble.rs`).
|
|
5
|
+
*
|
|
6
|
+
* The templates are extracted once at dev startup against the LIVE (Vite-
|
|
7
|
+
* transformed) dev shell (see `compiler/template-build.ts extractDevSsrTemplates`)
|
|
8
|
+
* so the served markup boots the dev client and hydrates in place. At request
|
|
9
|
+
* time the dev server runs the real `render` export (`WasmServerModule.
|
|
10
|
+
* dispatchRender`), this module decodes the values envelope and splices each
|
|
11
|
+
* value at its manifest-fixed offset. The hash-coherence guard the prod edge
|
|
12
|
+
* enforces is skipped in dev: the guest and the template are built together
|
|
13
|
+
* here, so there is no deploy skew to catch — only fail-safe envelopes (status
|
|
14
|
+
* >= 500 or no slots) fall back to client rendering.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** One SSR route's spliceable template + its slot insertion points. */
|
|
18
|
+
export interface DevSsrTemplate {
|
|
19
|
+
pattern: string;
|
|
20
|
+
name: string;
|
|
21
|
+
tmpl: Uint8Array;
|
|
22
|
+
entries: { id: number; offset: number }[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** A matchable SSR route. */
|
|
26
|
+
export interface SsrRoute {
|
|
27
|
+
/** Matches a request pathname (no query) to this route's template. */
|
|
28
|
+
test: (pathname: string) => boolean;
|
|
29
|
+
tmpl: Uint8Array;
|
|
30
|
+
entries: { id: number; offset: number }[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The pathname of a request URL (strip the query string). */
|
|
34
|
+
export function pathnameOf(url: string): string {
|
|
35
|
+
const q = url.indexOf('?');
|
|
36
|
+
return q < 0 ? url : url.slice(0, q);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Compile a route pattern (`/hello`, `/u/:name`, `/blog/[id]`, `/files/[...path]`,
|
|
40
|
+
* `/*`) to a pathname matcher. Dynamic segments match one path segment; catch-all
|
|
41
|
+
* (`[...x]` / `*`) matches the rest. Trailing slashes are ignored. */
|
|
42
|
+
function patternToTest(pattern: string): (pathname: string) => boolean {
|
|
43
|
+
const norm = (p: string): string => (p.length > 1 && p.endsWith('/') ? p.slice(0, -1) : p);
|
|
44
|
+
let re = '';
|
|
45
|
+
// Tokenise into literal runs and dynamic/catch-all holes.
|
|
46
|
+
let i = 0;
|
|
47
|
+
while (i < pattern.length) {
|
|
48
|
+
const ch = pattern[i];
|
|
49
|
+
if (ch === ':') {
|
|
50
|
+
// `:name` — one segment.
|
|
51
|
+
i++;
|
|
52
|
+
while (i < pattern.length && /[A-Za-z0-9_]/.test(pattern[i])) i++;
|
|
53
|
+
re += '[^/]+';
|
|
54
|
+
} else if (ch === '*') {
|
|
55
|
+
re += '.*';
|
|
56
|
+
i++;
|
|
57
|
+
} else if (ch === '[') {
|
|
58
|
+
const end = pattern.indexOf(']', i);
|
|
59
|
+
const inner = end < 0 ? '' : pattern.slice(i + 1, end);
|
|
60
|
+
re += inner.startsWith('...') ? '.*' : '[^/]+';
|
|
61
|
+
i = end < 0 ? pattern.length : end + 1;
|
|
62
|
+
} else {
|
|
63
|
+
// Literal char, regex-escaped.
|
|
64
|
+
re += ch.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
65
|
+
i++;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const compiled = new RegExp(`^${re}$`);
|
|
69
|
+
return (pathname: string): boolean => compiled.test(norm(pathname));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Build matchable SSR routes from the extracted dev templates. */
|
|
73
|
+
export function buildSsrRoutes(templates: readonly DevSsrTemplate[]): SsrRoute[] {
|
|
74
|
+
return templates.map((t) => ({ test: patternToTest(t.pattern), tmpl: t.tmpl, entries: t.entries }));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** A decoded guest values envelope. */
|
|
78
|
+
interface DecodedValues {
|
|
79
|
+
status: number;
|
|
80
|
+
headers: [string, string][];
|
|
81
|
+
/** Slot value bytes keyed by numeric slot id. */
|
|
82
|
+
values: Map<number, Uint8Array>;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Decode the guest values envelope (mirrors the prod host `decode_values`). All
|
|
86
|
+
* fields little-endian, no padding. Returns null on a malformed/short buffer. */
|
|
87
|
+
function decodeValues(buf: Uint8Array): DecodedValues | null {
|
|
88
|
+
const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
89
|
+
let o = 0;
|
|
90
|
+
const need = (n: number): boolean => o + n <= buf.byteLength;
|
|
91
|
+
try {
|
|
92
|
+
if (!need(2 + 32 + 2)) return null;
|
|
93
|
+
const status = dv.getUint16(o, true);
|
|
94
|
+
o += 2;
|
|
95
|
+
o += 32; // template hash (coherence is built together in dev; not checked)
|
|
96
|
+
const nHeaders = dv.getUint16(o, true);
|
|
97
|
+
o += 2;
|
|
98
|
+
const headers: [string, string][] = [];
|
|
99
|
+
const dec = new TextDecoder();
|
|
100
|
+
for (let i = 0; i < nHeaders; i++) {
|
|
101
|
+
if (!need(4)) return null;
|
|
102
|
+
const nameLen = dv.getUint16(o, true);
|
|
103
|
+
const valLen = dv.getUint16(o + 2, true);
|
|
104
|
+
o += 4;
|
|
105
|
+
if (!need(nameLen + valLen)) return null;
|
|
106
|
+
const name = dec.decode(buf.subarray(o, o + nameLen));
|
|
107
|
+
o += nameLen;
|
|
108
|
+
const val = dec.decode(buf.subarray(o, o + valLen));
|
|
109
|
+
o += valLen;
|
|
110
|
+
headers.push([name, val]);
|
|
111
|
+
}
|
|
112
|
+
if (!need(2)) return null;
|
|
113
|
+
const nSlots = dv.getUint16(o, true);
|
|
114
|
+
o += 2;
|
|
115
|
+
const values = new Map<number, Uint8Array>();
|
|
116
|
+
for (let i = 0; i < nSlots; i++) {
|
|
117
|
+
if (!need(2 + 1 + 4)) return null;
|
|
118
|
+
const id = dv.getUint16(o, true);
|
|
119
|
+
o += 2;
|
|
120
|
+
o += 1; // kind (the splice is kind-agnostic; the guest pre-escaped/stamped)
|
|
121
|
+
const len = dv.getUint32(o, true);
|
|
122
|
+
o += 4;
|
|
123
|
+
if (!need(len)) return null;
|
|
124
|
+
values.set(id, buf.subarray(o, o + len));
|
|
125
|
+
o += len;
|
|
126
|
+
}
|
|
127
|
+
return { status, headers, values };
|
|
128
|
+
} catch {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Splice ascending-offset inserts into the template (mirrors the host `assemble`). */
|
|
134
|
+
function splice(tmpl: Uint8Array, inserts: { offset: number; value: Uint8Array }[]): Uint8Array {
|
|
135
|
+
const parts: Uint8Array[] = [];
|
|
136
|
+
let prev = 0;
|
|
137
|
+
for (const ins of inserts) {
|
|
138
|
+
if (ins.offset > prev) parts.push(tmpl.subarray(prev, ins.offset));
|
|
139
|
+
if (ins.value.length > 0) parts.push(ins.value);
|
|
140
|
+
prev = ins.offset;
|
|
141
|
+
}
|
|
142
|
+
if (tmpl.length > prev) parts.push(tmpl.subarray(prev));
|
|
143
|
+
return Buffer.concat(parts.map((p) => Buffer.from(p.buffer, p.byteOffset, p.byteLength)));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** A spliced SSR response. */
|
|
147
|
+
export interface SsrResult {
|
|
148
|
+
status: number;
|
|
149
|
+
headers: [string, string][];
|
|
150
|
+
html: Uint8Array;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Decode the guest envelope and splice it into `route`'s template. Returns null
|
|
155
|
+
* to fall back to client rendering: a fail-safe envelope (status >= 500, e.g. no
|
|
156
|
+
* renderer matched), no slots, or a decode error.
|
|
157
|
+
*/
|
|
158
|
+
export function assembleSsr(route: SsrRoute, envelope: Uint8Array): SsrResult | null {
|
|
159
|
+
const decoded = decodeValues(envelope);
|
|
160
|
+
if (decoded === null) return null;
|
|
161
|
+
if (decoded.status >= 500 || decoded.values.size === 0) return null;
|
|
162
|
+
const inserts = route.entries
|
|
163
|
+
.map((e) => ({ offset: e.offset, value: decoded.values.get(e.id) ?? new Uint8Array(0) }))
|
|
164
|
+
.sort((a, b) => a.offset - b.offset);
|
|
165
|
+
return { status: decoded.status, headers: decoded.headers, html: splice(route.tmpl, inserts) };
|
|
166
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared, bounds-checked wasm custom-section walker. Factored out of
|
|
3
|
+
* `db/catalog.ts` so the three Toil section parsers (`toildb.catalog`,
|
|
4
|
+
* `toilstream.catalog`, `toildaemon.catalog`, `toil.surface`) share ONE
|
|
5
|
+
* magic-skipping loop instead of drifting copies.
|
|
6
|
+
*
|
|
7
|
+
* Every input is a tenant-built, possibly mid-rebuild wasm, so the walker must
|
|
8
|
+
* never read past the buffer: a truncated/garbage section table returns `null`
|
|
9
|
+
* (treated by callers as "no section"). Mirrors the host-side walker
|
|
10
|
+
* (`toil-backend` custom-section reader) and the toilscript-side test walker.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** Read a LEB128 from `buf` at `pos`; throws on overrun so a truncated module
|
|
14
|
+
* can never over-read (the caller catches and treats it as "no section"). */
|
|
15
|
+
export function leb(buf: Buffer, pos: number): [number, number] {
|
|
16
|
+
let result = 0;
|
|
17
|
+
let shift = 0;
|
|
18
|
+
let p = pos;
|
|
19
|
+
for (;;) {
|
|
20
|
+
if (p >= buf.length) throw new RangeError('leb128 past end of buffer');
|
|
21
|
+
const b = buf[p++];
|
|
22
|
+
result |= (b & 0x7f) << shift;
|
|
23
|
+
if ((b & 0x80) === 0) break;
|
|
24
|
+
shift += 7;
|
|
25
|
+
if (shift > 35) throw new RangeError('leb128 too long');
|
|
26
|
+
}
|
|
27
|
+
return [result >>> 0, p];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The bytes of the named wasm custom section, or `null` if absent. Bounds-checked
|
|
31
|
+
* so a truncated/garbage module can never read past the buffer. */
|
|
32
|
+
export function customSection(wasm: Buffer, want: string): Buffer | null {
|
|
33
|
+
if (
|
|
34
|
+
wasm.length < 8 ||
|
|
35
|
+
wasm[0] !== 0x00 ||
|
|
36
|
+
wasm[1] !== 0x61 ||
|
|
37
|
+
wasm[2] !== 0x73 ||
|
|
38
|
+
wasm[3] !== 0x6d
|
|
39
|
+
)
|
|
40
|
+
return null;
|
|
41
|
+
let pos = 8; // skip the 8-byte magic + version header
|
|
42
|
+
while (pos < wasm.length) {
|
|
43
|
+
const id = wasm[pos++];
|
|
44
|
+
let size: number;
|
|
45
|
+
[size, pos] = leb(wasm, pos);
|
|
46
|
+
const end = pos + size;
|
|
47
|
+
if (end > wasm.length || end < pos) return null; // truncated section table
|
|
48
|
+
if (id === 0) {
|
|
49
|
+
const [nameLen, namePos] = leb(wasm, pos);
|
|
50
|
+
if (
|
|
51
|
+
namePos + nameLen <= end &&
|
|
52
|
+
wasm.toString('latin1', namePos, namePos + nameLen) === want
|
|
53
|
+
)
|
|
54
|
+
return wasm.subarray(namePos + nameLen, end);
|
|
55
|
+
}
|
|
56
|
+
pos = end;
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse a compiled artifact's `toil.surface` custom section. Emitted into EVERY
|
|
3
|
+
* Toil artifact by the toilscript `buildToilSurface` pass (hot AND cold). The dev
|
|
4
|
+
* server reads it to decide whether each artifact carries the daemon surface
|
|
5
|
+
* (start the daemon emulator) or the stream surface (Phase 4, deferred).
|
|
6
|
+
*
|
|
7
|
+
* Byte layout (RECONCILIATION Part 5, all little-endian; mirrors the toilscript
|
|
8
|
+
* `CatWriter` emitter byte-for-byte):
|
|
9
|
+
*
|
|
10
|
+
* u16 format_version = 1
|
|
11
|
+
* u8 target_mode (0 = hot, 1 = cold; there is no target_mode = 2)
|
|
12
|
+
* u8 reserved0
|
|
13
|
+
* u32 surface_flags (bit0 rest, bit1 stream, bit2 daemon,
|
|
14
|
+
* bit3 scheduled, bit4 database, bit5 render)
|
|
15
|
+
* u16 abi_version
|
|
16
|
+
* str build_id (u32 len + UTF-8)
|
|
17
|
+
* u32 fingerprint
|
|
18
|
+
* u32 data_coherence_hash
|
|
19
|
+
* u32 pair_coherence_hash (exactly THREE u32 after build_id, not four)
|
|
20
|
+
*
|
|
21
|
+
* Fail-closed per Part 5's host rule: an ABSENT section is "legacy single
|
|
22
|
+
* artifact, load as hot" (NOT a hard reject); a PRESENT-but-unparseable section is
|
|
23
|
+
* a corrupt artifact -> do not start that artifact's emulator.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { DataReader } from 'toiljs/io';
|
|
27
|
+
|
|
28
|
+
import { customSection } from './sections.js';
|
|
29
|
+
|
|
30
|
+
export interface SurfaceFlags {
|
|
31
|
+
readonly rest: boolean;
|
|
32
|
+
readonly stream: boolean;
|
|
33
|
+
readonly daemon: boolean;
|
|
34
|
+
readonly scheduled: boolean;
|
|
35
|
+
readonly database: boolean;
|
|
36
|
+
readonly render: boolean;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface Surface {
|
|
40
|
+
/** 0 = hot, 1 = cold. */
|
|
41
|
+
readonly targetMode: 'hot' | 'cold';
|
|
42
|
+
readonly flags: SurfaceFlags;
|
|
43
|
+
readonly abiVersion: number;
|
|
44
|
+
readonly buildId: string;
|
|
45
|
+
readonly fingerprint: number;
|
|
46
|
+
readonly dataCoherenceHash: number;
|
|
47
|
+
readonly pairCoherenceHash: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** `'absent'` => legacy single artifact (load as hot, no emulators).
|
|
51
|
+
* `'invalid'` => present but corrupt (fail closed). Otherwise the parsed surface. */
|
|
52
|
+
export function parseSurface(wasm: Buffer): Surface | 'absent' | 'invalid' {
|
|
53
|
+
let sec: Buffer | null;
|
|
54
|
+
try {
|
|
55
|
+
sec = customSection(wasm, 'toil.surface');
|
|
56
|
+
} catch {
|
|
57
|
+
return 'invalid'; // garbage section table
|
|
58
|
+
}
|
|
59
|
+
if (sec === null) return 'absent';
|
|
60
|
+
|
|
61
|
+
const r = new DataReader(sec);
|
|
62
|
+
r.readU16(); // format_version
|
|
63
|
+
const targetMode = r.readU8() === 1 ? 'cold' : 'hot';
|
|
64
|
+
r.readU8(); // reserved0
|
|
65
|
+
const f = r.readU32(); // surface_flags
|
|
66
|
+
const abiVersion = r.readU16();
|
|
67
|
+
const buildId = r.readString();
|
|
68
|
+
const fingerprint = r.readU32();
|
|
69
|
+
const dataCoherenceHash = r.readU32(); // exactly THREE u32 after build_id
|
|
70
|
+
const pairCoherenceHash = r.readU32();
|
|
71
|
+
if (!r.ok) return 'invalid'; // PRESENT but corrupt => fail closed
|
|
72
|
+
return {
|
|
73
|
+
targetMode,
|
|
74
|
+
flags: {
|
|
75
|
+
rest: !!(f & 1),
|
|
76
|
+
stream: !!(f & 2),
|
|
77
|
+
daemon: !!(f & 4),
|
|
78
|
+
scheduled: !!(f & 8),
|
|
79
|
+
database: !!(f & 16),
|
|
80
|
+
render: !!(f & 32),
|
|
81
|
+
},
|
|
82
|
+
abiVersion,
|
|
83
|
+
buildId,
|
|
84
|
+
fingerprint,
|
|
85
|
+
dataCoherenceHash,
|
|
86
|
+
pairCoherenceHash,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-pass build pipeline (doc 08 section 1). Asserts:
|
|
3
|
+
*
|
|
4
|
+
* - `serverArtifacts` derives the `-hot`/`-cold` paths from `outFile` when
|
|
5
|
+
* `hotFile`/`coldFile` are absent, and honors them when present.
|
|
6
|
+
* - `SURFACE_DECORATOR` matches `@stream`/`@daemon`/`@scheduled` at line start
|
|
7
|
+
* (not in a comment).
|
|
8
|
+
* - `buildServer` on a project that declares a `@daemon` runs TWO toilscript
|
|
9
|
+
* passes (one `--targetMode cold`, one `--targetMode hot`) and produces BOTH
|
|
10
|
+
* `release-hot.wasm` and `release-cold.wasm`; the cold artifact decodes to a
|
|
11
|
+
* daemon catalog and its `toil.surface` is target_mode = cold.
|
|
12
|
+
* - a project with only the legacy request surface keeps the single-artifact
|
|
13
|
+
* path (no cold pass, no cold artifact).
|
|
14
|
+
*
|
|
15
|
+
* The build invokes the LOCAL toilscript (branch feat/streams-phase0-compiler),
|
|
16
|
+
* which supports `--targetMode`; the test links it into the fixture project's
|
|
17
|
+
* `node_modules` the same way the dev build resolves it (`require.resolve`).
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
21
|
+
import { tmpdir } from 'node:os';
|
|
22
|
+
import { dirname, join } from 'node:path';
|
|
23
|
+
import { fileURLToPath } from 'node:url';
|
|
24
|
+
|
|
25
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
26
|
+
|
|
27
|
+
import {
|
|
28
|
+
buildServer,
|
|
29
|
+
serverArtifacts,
|
|
30
|
+
splitSurfaceFiles,
|
|
31
|
+
SURFACE_DECORATOR,
|
|
32
|
+
} from '../src/compiler/index.js';
|
|
33
|
+
import { parseDaemonCatalog } from '../src/devserver/daemon/catalog.js';
|
|
34
|
+
import { parseSurface } from '../src/devserver/wasm/surface.js';
|
|
35
|
+
|
|
36
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
37
|
+
const LOCAL_TOILSCRIPT = join(here, '..', '..', 'toilscript');
|
|
38
|
+
|
|
39
|
+
let tmp: string;
|
|
40
|
+
|
|
41
|
+
beforeEach(() => {
|
|
42
|
+
tmp = mkdtempSync(join(tmpdir(), 'daemon-build-'));
|
|
43
|
+
});
|
|
44
|
+
afterEach(() => {
|
|
45
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
/** Scaffold a minimal project at `tmp` with `server/main.ts` = `serverSrc`, the
|
|
49
|
+
* given toilconfig, and `node_modules/toilscript` symlinked to the local build. */
|
|
50
|
+
function scaffold(serverSrc: string, toilconfig: object): void {
|
|
51
|
+
writeFileSync(join(tmp, 'package.json'), JSON.stringify({ name: 'fixture', type: 'module' }));
|
|
52
|
+
writeFileSync(join(tmp, 'toilconfig.json'), JSON.stringify(toilconfig, null, 2));
|
|
53
|
+
mkdirSync(join(tmp, 'server'), { recursive: true });
|
|
54
|
+
writeFileSync(join(tmp, 'server', 'main.ts'), serverSrc);
|
|
55
|
+
mkdirSync(join(tmp, 'node_modules'), { recursive: true });
|
|
56
|
+
symlinkSync(LOCAL_TOILSCRIPT, join(tmp, 'node_modules', 'toilscript'), 'dir');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const BASE_TOILCONFIG = {
|
|
60
|
+
entries: ['server/main.ts'],
|
|
61
|
+
targets: { release: { outFile: 'build/server/release.wasm' } },
|
|
62
|
+
options: { runtime: 'stub', optimizeLevel: 0, shrinkLevel: 0 },
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
// A @daemon that declares its host imports directly (no toiljs globals lib needed).
|
|
66
|
+
const DAEMON_SRC = `@daemon
|
|
67
|
+
class Jobs {
|
|
68
|
+
@scheduled("2s") fast(): void {}
|
|
69
|
+
@scheduled("0 0 * * *") nightly(): void {}
|
|
70
|
+
}
|
|
71
|
+
export function probe(): i32 { return 1; }
|
|
72
|
+
`;
|
|
73
|
+
|
|
74
|
+
const LEGACY_SRC = `export function handle(ofs: i32, len: i32): i64 { return 0; }
|
|
75
|
+
export function probe(): i32 { return 1; }
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
describe('serverArtifacts path derivation', () => {
|
|
79
|
+
it('derives -hot/-cold from outFile when hotFile/coldFile are absent', () => {
|
|
80
|
+
writeFileSync(
|
|
81
|
+
join(tmp, 'toilconfig.json'),
|
|
82
|
+
JSON.stringify({ targets: { release: { outFile: 'build/server/release.wasm' } } }),
|
|
83
|
+
);
|
|
84
|
+
const a = serverArtifacts(tmp);
|
|
85
|
+
expect(a.hot).toBe(join(tmp, 'build/server/release-hot.wasm'));
|
|
86
|
+
expect(a.cold).toBe(join(tmp, 'build/server/release-cold.wasm'));
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('honors explicit hotFile/coldFile when present', () => {
|
|
90
|
+
writeFileSync(
|
|
91
|
+
join(tmp, 'toilconfig.json'),
|
|
92
|
+
JSON.stringify({
|
|
93
|
+
targets: {
|
|
94
|
+
release: {
|
|
95
|
+
outFile: 'build/server/release.wasm',
|
|
96
|
+
hotFile: 'out/hot.wasm',
|
|
97
|
+
coldFile: 'out/cold.wasm',
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
}),
|
|
101
|
+
);
|
|
102
|
+
const a = serverArtifacts(tmp);
|
|
103
|
+
expect(a.hot).toBe(join(tmp, 'out/hot.wasm'));
|
|
104
|
+
expect(a.cold).toBe(join(tmp, 'out/cold.wasm'));
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
describe('SURFACE_DECORATOR', () => {
|
|
109
|
+
it('matches the streams/daemon decorators at line start', () => {
|
|
110
|
+
for (const deco of ['@stream', '@daemon', '@scheduled', '@rest', '@data']) {
|
|
111
|
+
expect(SURFACE_DECORATOR.test(`${deco} class X {}`)).toBe(true);
|
|
112
|
+
expect(SURFACE_DECORATOR.test(` ${deco}\nclass X {}`)).toBe(true);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('does NOT match a decorator mentioned only in a comment', () => {
|
|
117
|
+
expect(SURFACE_DECORATOR.test('// the @daemon decorator marks a cold class')).toBe(false);
|
|
118
|
+
expect(SURFACE_DECORATOR.test('const s = "uses @scheduled internally";')).toBe(false);
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe('splitSurfaceFiles per-pass classification', () => {
|
|
123
|
+
/** Lay down `name -> contents` files under `tmp` and return their relative paths. */
|
|
124
|
+
function lay(files: Record<string, string>): string[] {
|
|
125
|
+
mkdirSync(join(tmp, 'server'), { recursive: true });
|
|
126
|
+
const rels: string[] = [];
|
|
127
|
+
for (const [name, src] of Object.entries(files)) {
|
|
128
|
+
writeFileSync(join(tmp, name), src);
|
|
129
|
+
rels.push(name);
|
|
130
|
+
}
|
|
131
|
+
return rels;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
it('drops daemon-only files from the hot pass and hot-only files from the cold pass', () => {
|
|
135
|
+
const rels = lay({
|
|
136
|
+
'server/jobs.ts': '@daemon\nclass J { @scheduled("1s") t(): void {} }\n',
|
|
137
|
+
'server/api.ts': '@rest\nclass A {}\n',
|
|
138
|
+
'server/model.ts': '@data\nclass M {}\n',
|
|
139
|
+
'server/util.ts': 'export function helper(): i32 { return 1; }\n',
|
|
140
|
+
});
|
|
141
|
+
const split = splitSurfaceFiles(tmp, rels);
|
|
142
|
+
expect(split.hasDaemon).toBe(true);
|
|
143
|
+
// hot pass: everything except the daemon-only jobs.ts.
|
|
144
|
+
expect(split.hot.sort()).toEqual(
|
|
145
|
+
['server/api.ts', 'server/model.ts', 'server/util.ts'].sort(),
|
|
146
|
+
);
|
|
147
|
+
// cold pass: everything except the hot-only api.ts.
|
|
148
|
+
expect(split.cold.sort()).toEqual(
|
|
149
|
+
['server/jobs.ts', 'server/model.ts', 'server/util.ts'].sort(),
|
|
150
|
+
);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('keeps a file that mixes both surfaces in both passes', () => {
|
|
154
|
+
const rels = lay({ 'server/both.ts': '@daemon\nclass J {}\n@rest\nclass A {}\n' });
|
|
155
|
+
const split = splitSurfaceFiles(tmp, rels);
|
|
156
|
+
expect(split.hot).toContain('server/both.ts');
|
|
157
|
+
expect(split.cold).toContain('server/both.ts');
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
// Needs the local toilscript dev build (with --targetMode) linked as a sibling
|
|
162
|
+
// repo; skip where it is absent (e.g. CI, which has only the published dep).
|
|
163
|
+
describe.skipIf(!existsSync(LOCAL_TOILSCRIPT))('buildServer two-pass (daemon project)', () => {
|
|
164
|
+
it('runs the cold pass and produces the cold artifact with a daemon catalog', async () => {
|
|
165
|
+
scaffold(DAEMON_SRC, BASE_TOILCONFIG);
|
|
166
|
+
await buildServer(tmp);
|
|
167
|
+
|
|
168
|
+
const cold = join(tmp, 'build/server/release-cold.wasm');
|
|
169
|
+
expect(existsSync(cold), 'cold artifact missing').toBe(true);
|
|
170
|
+
|
|
171
|
+
// The cold artifact carries the daemon surface + catalog (decoded byte-for-byte).
|
|
172
|
+
const coldBytes = readFileSync(cold);
|
|
173
|
+
const surface = parseSurface(coldBytes);
|
|
174
|
+
expect(surface !== 'absent' && surface !== 'invalid' && surface.targetMode).toBe('cold');
|
|
175
|
+
expect(surface !== 'absent' && surface !== 'invalid' && surface.flags.daemon).toBe(true);
|
|
176
|
+
|
|
177
|
+
const catalog = parseDaemonCatalog(coldBytes);
|
|
178
|
+
expect(catalog).not.toBeNull();
|
|
179
|
+
expect(catalog!.hasDaemon).toBe(true);
|
|
180
|
+
expect(catalog!.tasks.map((t) => t.name)).toEqual(['fast', 'nightly']);
|
|
181
|
+
expect(catalog!.tasks[0].schedule.kind).toBe('interval');
|
|
182
|
+
expect(catalog!.tasks[1].schedule.kind).toBe('cron');
|
|
183
|
+
|
|
184
|
+
// A daemon-only project (no request/stream surface) has no hot files, so the hot pass is
|
|
185
|
+
// skipped (toilscript would HARD-ERROR a @daemon class under --targetMode hot). The legacy
|
|
186
|
+
// single-artifact `release.wasm` is therefore not produced for a pure background worker.
|
|
187
|
+
expect(existsSync(join(tmp, 'build/server/release.wasm'))).toBe(false);
|
|
188
|
+
}, 60_000);
|
|
189
|
+
|
|
190
|
+
it('keeps the single-artifact path for a legacy (no-daemon) project', async () => {
|
|
191
|
+
scaffold(LEGACY_SRC, BASE_TOILCONFIG);
|
|
192
|
+
await buildServer(tmp);
|
|
193
|
+
|
|
194
|
+
expect(existsSync(join(tmp, 'build/server/release.wasm'))).toBe(true);
|
|
195
|
+
// No @daemon -> no cold pass -> no cold artifact.
|
|
196
|
+
expect(existsSync(join(tmp, 'build/server/release-cold.wasm'))).toBe(false);
|
|
197
|
+
}, 60_000);
|
|
198
|
+
});
|