toiljs 0.0.60 → 0.0.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +31 -0
- package/CHANGELOG.md +5 -0
- package/build/cli/.tsbuildinfo +1 -1
- package/build/cli/index.js +2 -2
- package/build/client/.tsbuildinfo +1 -1
- package/build/client/index.d.ts +1 -1
- package/build/client/index.js +1 -1
- package/build/client/routing/mount.js +12 -1
- package/build/client/ssr/markers.d.ts +1 -0
- package/build/client/ssr/markers.js +3 -0
- package/build/compiler/.tsbuildinfo +1 -1
- package/build/compiler/config.d.ts +21 -0
- package/build/compiler/config.js +35 -0
- package/build/compiler/docs.d.ts +2 -1
- package/build/compiler/docs.js +33 -304
- package/build/compiler/index.d.ts +13 -0
- package/build/compiler/index.js +113 -21
- package/build/compiler/template-build.d.ts +21 -1
- package/build/compiler/template-build.js +110 -26
- package/build/compiler/toil-docs.generated.d.ts +1 -0
- package/build/compiler/toil-docs.generated.js +20 -0
- package/build/devserver/.tsbuildinfo +1 -1
- package/build/devserver/daemon/catalog.d.ts +26 -0
- package/build/devserver/daemon/catalog.js +48 -0
- package/build/devserver/daemon/cron.d.ts +4 -0
- package/build/devserver/daemon/cron.js +50 -0
- package/build/devserver/daemon/host.d.ts +37 -0
- package/build/devserver/daemon/host.js +94 -0
- package/build/devserver/daemon/index.d.ts +34 -0
- package/build/devserver/daemon/index.js +241 -0
- package/build/devserver/db/catalog.d.ts +2 -1
- package/build/devserver/db/catalog.js +44 -44
- package/build/devserver/db/database.d.ts +27 -11
- package/build/devserver/db/database.js +539 -169
- package/build/devserver/db/index.d.ts +1 -1
- package/build/devserver/db/index.js +1 -1
- package/build/devserver/db/routeKinds.d.ts +8 -0
- package/build/devserver/db/routeKinds.js +139 -0
- package/build/devserver/db/types.d.ts +64 -1
- package/build/devserver/db/types.js +33 -1
- package/build/devserver/index.d.ts +10 -0
- package/build/devserver/index.js +7 -0
- package/build/devserver/mstore/store.d.ts +18 -0
- package/build/devserver/mstore/store.js +82 -0
- package/build/devserver/runtime/host.d.ts +6 -0
- package/build/devserver/runtime/host.js +45 -1
- package/build/devserver/runtime/module.d.ts +1 -0
- package/build/devserver/runtime/module.js +27 -1
- package/build/devserver/server.d.ts +6 -0
- package/build/devserver/server.js +59 -0
- package/build/devserver/ssr.d.ts +25 -0
- package/build/devserver/ssr.js +114 -0
- package/build/devserver/wasm/sections.d.ts +2 -0
- package/build/devserver/wasm/sections.js +42 -0
- package/build/devserver/wasm/surface.d.ts +18 -0
- package/build/devserver/wasm/surface.js +41 -0
- package/docs/README.md +4 -4
- package/docs/auth-todo.md +6 -6
- package/docs/caching.md +5 -5
- package/docs/cli.md +15 -0
- package/docs/client.md +40 -0
- package/docs/crypto.md +4 -4
- package/docs/data.md +6 -6
- package/docs/email.md +28 -28
- package/docs/environment.md +10 -10
- package/docs/index.md +26 -0
- package/docs/ratelimit.md +10 -10
- package/docs/routing.md +2 -2
- package/docs/server.md +61 -0
- package/docs/ssr.md +561 -113
- package/docs/styling.md +22 -0
- package/docs/time.md +1 -1
- package/eslint.config.js +10 -1
- package/examples/basic/client/components/Header.tsx +3 -0
- package/examples/basic/client/routes/features/actions.tsx +0 -2
- package/examples/basic/client/routes/hello.tsx +89 -19
- package/examples/basic/client/styles/main.css +48 -0
- package/examples/basic/server/SsrHelloRender.ts +97 -0
- package/examples/basic/server/main.ts +5 -0
- package/examples/basic/server/streams/Echo.ts +49 -0
- package/package.json +12 -10
- package/scripts/gen-toil-docs.mjs +96 -0
- package/src/cli/create.ts +2 -2
- package/src/client/index.ts +1 -1
- package/src/client/routing/mount.tsx +18 -2
- package/src/client/ssr/markers.tsx +22 -0
- package/src/compiler/config.ts +88 -2
- package/src/compiler/docs.ts +47 -308
- package/src/compiler/index.ts +236 -32
- package/src/compiler/ssr-codegen.ts +1 -1
- package/src/compiler/template-build.ts +247 -46
- package/src/compiler/toil-docs.generated.ts +26 -0
- package/src/devserver/daemon/catalog.ts +120 -0
- package/src/devserver/daemon/cron.ts +87 -0
- package/src/devserver/daemon/host.ts +224 -0
- package/src/devserver/daemon/index.ts +349 -0
- package/src/devserver/db/catalog.ts +61 -53
- package/src/devserver/db/database.ts +613 -149
- package/src/devserver/db/index.ts +1 -1
- package/src/devserver/db/routeKinds.ts +147 -0
- package/src/devserver/db/types.ts +65 -2
- package/src/devserver/index.ts +12 -0
- package/src/devserver/mstore/store.ts +121 -0
- package/src/devserver/runtime/host.ts +92 -1
- package/src/devserver/runtime/module.ts +35 -1
- package/src/devserver/server.ts +101 -0
- package/src/devserver/ssr.ts +166 -0
- package/src/devserver/wasm/sections.ts +59 -0
- package/src/devserver/wasm/surface.ts +88 -0
- package/test/daemon-build.test.ts +198 -0
- package/test/daemon-catalog.test.ts +265 -0
- package/test/daemon-emulation.test.ts +216 -0
- package/test/devserver-database.test.ts +396 -5
- package/test/email-preview.test.ts +6 -1
- package/test/fixtures/daemon-app.ts +56 -0
- package/test/global-setup.ts +17 -0
- package/test/ssr-render.test.ts +94 -27
- package/test/ssr-template.test.tsx +44 -1
- package/vitest.config.ts +3 -0
|
@@ -2,12 +2,14 @@ import fs from 'node:fs';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { Server } from '@dacely/hyper-express';
|
|
4
4
|
import pc from 'picocolors';
|
|
5
|
+
import { DaemonHost, daemonEmulationEnabled } from './daemon/index.js';
|
|
5
6
|
import { configureDbPersistence } from './db/index.js';
|
|
6
7
|
import { initEmailService } from './email/index.js';
|
|
7
8
|
import { applyCacheRule, lookupCache } from './http/cache.js';
|
|
8
9
|
import { METHOD_CODES } from './http/envelope.js';
|
|
9
10
|
import { proxyToVite, wireWebsocketProxy } from './http/proxy.js';
|
|
10
11
|
import { WasmServerModule } from './runtime/module.js';
|
|
12
|
+
import { assembleSsr, buildSsrRoutes, pathnameOf, } from './ssr.js';
|
|
11
13
|
const DEFAULT_MAX_BODY_LENGTH = 1024 * 1024 * 8;
|
|
12
14
|
const VITE_PREFIXES = ['/@', '/node_modules/', '/__toil/'];
|
|
13
15
|
const MIME = {
|
|
@@ -77,6 +79,23 @@ function sendWasmResponse(response, root, result) {
|
|
|
77
79
|
response.header('content-type', 'text/plain; charset=utf-8');
|
|
78
80
|
response.send(Buffer.from(result.body.buffer, result.body.byteOffset, result.body.length));
|
|
79
81
|
}
|
|
82
|
+
function sendSsr(response, out, headOnly) {
|
|
83
|
+
response.status(out.status);
|
|
84
|
+
let hasContentType = false;
|
|
85
|
+
for (const [name, value] of out.headers) {
|
|
86
|
+
if (name.toLowerCase() === 'content-type')
|
|
87
|
+
hasContentType = true;
|
|
88
|
+
response.header(name, value);
|
|
89
|
+
}
|
|
90
|
+
if (!hasContentType)
|
|
91
|
+
response.header('content-type', 'text/html; charset=utf-8');
|
|
92
|
+
response.header('server', 'toil-dev');
|
|
93
|
+
if (headOnly) {
|
|
94
|
+
response.send('');
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
response.send(Buffer.from(out.html.buffer, out.html.byteOffset, out.html.length));
|
|
98
|
+
}
|
|
80
99
|
export async function startDevServer(options) {
|
|
81
100
|
const host = options.host ?? '127.0.0.1';
|
|
82
101
|
const root = path.resolve(options.root);
|
|
@@ -88,6 +107,10 @@ export async function startDevServer(options) {
|
|
|
88
107
|
process.stdout.write(pc.yellow(' ! ') + pc.dim(`email off: ${emailInit.note}`) + '\n');
|
|
89
108
|
}
|
|
90
109
|
const module = new WasmServerModule(options.wasmFile);
|
|
110
|
+
const ssrRoutes = buildSsrRoutes(options.ssrTemplates ?? []);
|
|
111
|
+
if (ssrRoutes.length > 0) {
|
|
112
|
+
process.stdout.write(pc.dim(` edge SSR: ${String(ssrRoutes.length)} route(s) served server-side`) + '\n');
|
|
113
|
+
}
|
|
91
114
|
configureDbPersistence(path.join(root, '.toil', 'devdata.json'));
|
|
92
115
|
let warnedMissing = false;
|
|
93
116
|
let loadedOnce = false;
|
|
@@ -122,6 +145,23 @@ export async function startDevServer(options) {
|
|
|
122
145
|
});
|
|
123
146
|
});
|
|
124
147
|
wireWebsocketProxy(app, options.vite);
|
|
148
|
+
const nodeMode = options.nodeMode ?? 'all';
|
|
149
|
+
let daemonHost = null;
|
|
150
|
+
let daemonTimer = null;
|
|
151
|
+
if (options.coldWasmFile !== undefined && daemonEmulationEnabled(nodeMode) && options.daemon) {
|
|
152
|
+
daemonHost = new DaemonHost(options.coldWasmFile, options.daemon, nodeMode);
|
|
153
|
+
const pollDaemon = () => {
|
|
154
|
+
try {
|
|
155
|
+
daemonHost?.refresh();
|
|
156
|
+
}
|
|
157
|
+
catch (e) {
|
|
158
|
+
process.stdout.write(pc.red(` ✗ daemon reload failed: ${String(e)}`) + '\n');
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
pollDaemon();
|
|
162
|
+
daemonTimer = setInterval(pollDaemon, 500);
|
|
163
|
+
daemonTimer.unref?.();
|
|
164
|
+
}
|
|
125
165
|
app.any('/*', async (request, response) => {
|
|
126
166
|
response.removeHeader('uWebSockets');
|
|
127
167
|
const dispatchable = !isViteInternal(request.url) && METHOD_CODES[request.method] !== undefined;
|
|
@@ -150,6 +190,22 @@ export async function startDevServer(options) {
|
|
|
150
190
|
response.status(500).send('internal error\n');
|
|
151
191
|
return;
|
|
152
192
|
}
|
|
193
|
+
if ((request.method === 'GET' || request.method === 'HEAD') &&
|
|
194
|
+
ssrRoutes.length > 0) {
|
|
195
|
+
const route = ssrRoutes.find((r) => r.test(pathnameOf(request.url)));
|
|
196
|
+
if (route) {
|
|
197
|
+
try {
|
|
198
|
+
const out = assembleSsr(route, module.dispatchRender(envelopeReq));
|
|
199
|
+
if (out !== null) {
|
|
200
|
+
sendSsr(response, out, request.method === 'HEAD');
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
catch (e) {
|
|
205
|
+
process.stdout.write(pc.red(` ✗ SSR ${request.path}: ${String(e)}`) + '\n');
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
153
209
|
}
|
|
154
210
|
await proxyToVite(request, response, options.vite);
|
|
155
211
|
});
|
|
@@ -158,6 +214,9 @@ export async function startDevServer(options) {
|
|
|
158
214
|
port: options.port,
|
|
159
215
|
host,
|
|
160
216
|
close: async () => {
|
|
217
|
+
if (daemonTimer !== null)
|
|
218
|
+
clearInterval(daemonTimer);
|
|
219
|
+
daemonHost?.close();
|
|
161
220
|
await app.shutdown();
|
|
162
221
|
},
|
|
163
222
|
};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export interface DevSsrTemplate {
|
|
2
|
+
pattern: string;
|
|
3
|
+
name: string;
|
|
4
|
+
tmpl: Uint8Array;
|
|
5
|
+
entries: {
|
|
6
|
+
id: number;
|
|
7
|
+
offset: number;
|
|
8
|
+
}[];
|
|
9
|
+
}
|
|
10
|
+
export interface SsrRoute {
|
|
11
|
+
test: (pathname: string) => boolean;
|
|
12
|
+
tmpl: Uint8Array;
|
|
13
|
+
entries: {
|
|
14
|
+
id: number;
|
|
15
|
+
offset: number;
|
|
16
|
+
}[];
|
|
17
|
+
}
|
|
18
|
+
export declare function pathnameOf(url: string): string;
|
|
19
|
+
export declare function buildSsrRoutes(templates: readonly DevSsrTemplate[]): SsrRoute[];
|
|
20
|
+
export interface SsrResult {
|
|
21
|
+
status: number;
|
|
22
|
+
headers: [string, string][];
|
|
23
|
+
html: Uint8Array;
|
|
24
|
+
}
|
|
25
|
+
export declare function assembleSsr(route: SsrRoute, envelope: Uint8Array): SsrResult | null;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
export function pathnameOf(url) {
|
|
2
|
+
const q = url.indexOf('?');
|
|
3
|
+
return q < 0 ? url : url.slice(0, q);
|
|
4
|
+
}
|
|
5
|
+
function patternToTest(pattern) {
|
|
6
|
+
const norm = (p) => (p.length > 1 && p.endsWith('/') ? p.slice(0, -1) : p);
|
|
7
|
+
let re = '';
|
|
8
|
+
let i = 0;
|
|
9
|
+
while (i < pattern.length) {
|
|
10
|
+
const ch = pattern[i];
|
|
11
|
+
if (ch === ':') {
|
|
12
|
+
i++;
|
|
13
|
+
while (i < pattern.length && /[A-Za-z0-9_]/.test(pattern[i]))
|
|
14
|
+
i++;
|
|
15
|
+
re += '[^/]+';
|
|
16
|
+
}
|
|
17
|
+
else if (ch === '*') {
|
|
18
|
+
re += '.*';
|
|
19
|
+
i++;
|
|
20
|
+
}
|
|
21
|
+
else if (ch === '[') {
|
|
22
|
+
const end = pattern.indexOf(']', i);
|
|
23
|
+
const inner = end < 0 ? '' : pattern.slice(i + 1, end);
|
|
24
|
+
re += inner.startsWith('...') ? '.*' : '[^/]+';
|
|
25
|
+
i = end < 0 ? pattern.length : end + 1;
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
re += ch.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
29
|
+
i++;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const compiled = new RegExp(`^${re}$`);
|
|
33
|
+
return (pathname) => compiled.test(norm(pathname));
|
|
34
|
+
}
|
|
35
|
+
export function buildSsrRoutes(templates) {
|
|
36
|
+
return templates.map((t) => ({ test: patternToTest(t.pattern), tmpl: t.tmpl, entries: t.entries }));
|
|
37
|
+
}
|
|
38
|
+
function decodeValues(buf) {
|
|
39
|
+
const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
40
|
+
let o = 0;
|
|
41
|
+
const need = (n) => o + n <= buf.byteLength;
|
|
42
|
+
try {
|
|
43
|
+
if (!need(2 + 32 + 2))
|
|
44
|
+
return null;
|
|
45
|
+
const status = dv.getUint16(o, true);
|
|
46
|
+
o += 2;
|
|
47
|
+
o += 32;
|
|
48
|
+
const nHeaders = dv.getUint16(o, true);
|
|
49
|
+
o += 2;
|
|
50
|
+
const headers = [];
|
|
51
|
+
const dec = new TextDecoder();
|
|
52
|
+
for (let i = 0; i < nHeaders; i++) {
|
|
53
|
+
if (!need(4))
|
|
54
|
+
return null;
|
|
55
|
+
const nameLen = dv.getUint16(o, true);
|
|
56
|
+
const valLen = dv.getUint16(o + 2, true);
|
|
57
|
+
o += 4;
|
|
58
|
+
if (!need(nameLen + valLen))
|
|
59
|
+
return null;
|
|
60
|
+
const name = dec.decode(buf.subarray(o, o + nameLen));
|
|
61
|
+
o += nameLen;
|
|
62
|
+
const val = dec.decode(buf.subarray(o, o + valLen));
|
|
63
|
+
o += valLen;
|
|
64
|
+
headers.push([name, val]);
|
|
65
|
+
}
|
|
66
|
+
if (!need(2))
|
|
67
|
+
return null;
|
|
68
|
+
const nSlots = dv.getUint16(o, true);
|
|
69
|
+
o += 2;
|
|
70
|
+
const values = new Map();
|
|
71
|
+
for (let i = 0; i < nSlots; i++) {
|
|
72
|
+
if (!need(2 + 1 + 4))
|
|
73
|
+
return null;
|
|
74
|
+
const id = dv.getUint16(o, true);
|
|
75
|
+
o += 2;
|
|
76
|
+
o += 1;
|
|
77
|
+
const len = dv.getUint32(o, true);
|
|
78
|
+
o += 4;
|
|
79
|
+
if (!need(len))
|
|
80
|
+
return null;
|
|
81
|
+
values.set(id, buf.subarray(o, o + len));
|
|
82
|
+
o += len;
|
|
83
|
+
}
|
|
84
|
+
return { status, headers, values };
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
function splice(tmpl, inserts) {
|
|
91
|
+
const parts = [];
|
|
92
|
+
let prev = 0;
|
|
93
|
+
for (const ins of inserts) {
|
|
94
|
+
if (ins.offset > prev)
|
|
95
|
+
parts.push(tmpl.subarray(prev, ins.offset));
|
|
96
|
+
if (ins.value.length > 0)
|
|
97
|
+
parts.push(ins.value);
|
|
98
|
+
prev = ins.offset;
|
|
99
|
+
}
|
|
100
|
+
if (tmpl.length > prev)
|
|
101
|
+
parts.push(tmpl.subarray(prev));
|
|
102
|
+
return Buffer.concat(parts.map((p) => Buffer.from(p.buffer, p.byteOffset, p.byteLength)));
|
|
103
|
+
}
|
|
104
|
+
export function assembleSsr(route, envelope) {
|
|
105
|
+
const decoded = decodeValues(envelope);
|
|
106
|
+
if (decoded === null)
|
|
107
|
+
return null;
|
|
108
|
+
if (decoded.status >= 500 || decoded.values.size === 0)
|
|
109
|
+
return null;
|
|
110
|
+
const inserts = route.entries
|
|
111
|
+
.map((e) => ({ offset: e.offset, value: decoded.values.get(e.id) ?? new Uint8Array(0) }))
|
|
112
|
+
.sort((a, b) => a.offset - b.offset);
|
|
113
|
+
return { status: decoded.status, headers: decoded.headers, html: splice(route.tmpl, inserts) };
|
|
114
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export function leb(buf, pos) {
|
|
2
|
+
let result = 0;
|
|
3
|
+
let shift = 0;
|
|
4
|
+
let p = pos;
|
|
5
|
+
for (;;) {
|
|
6
|
+
if (p >= buf.length)
|
|
7
|
+
throw new RangeError('leb128 past end of buffer');
|
|
8
|
+
const b = buf[p++];
|
|
9
|
+
result |= (b & 0x7f) << shift;
|
|
10
|
+
if ((b & 0x80) === 0)
|
|
11
|
+
break;
|
|
12
|
+
shift += 7;
|
|
13
|
+
if (shift > 35)
|
|
14
|
+
throw new RangeError('leb128 too long');
|
|
15
|
+
}
|
|
16
|
+
return [result >>> 0, p];
|
|
17
|
+
}
|
|
18
|
+
export function customSection(wasm, want) {
|
|
19
|
+
if (wasm.length < 8 ||
|
|
20
|
+
wasm[0] !== 0x00 ||
|
|
21
|
+
wasm[1] !== 0x61 ||
|
|
22
|
+
wasm[2] !== 0x73 ||
|
|
23
|
+
wasm[3] !== 0x6d)
|
|
24
|
+
return null;
|
|
25
|
+
let pos = 8;
|
|
26
|
+
while (pos < wasm.length) {
|
|
27
|
+
const id = wasm[pos++];
|
|
28
|
+
let size;
|
|
29
|
+
[size, pos] = leb(wasm, pos);
|
|
30
|
+
const end = pos + size;
|
|
31
|
+
if (end > wasm.length || end < pos)
|
|
32
|
+
return null;
|
|
33
|
+
if (id === 0) {
|
|
34
|
+
const [nameLen, namePos] = leb(wasm, pos);
|
|
35
|
+
if (namePos + nameLen <= end &&
|
|
36
|
+
wasm.toString('latin1', namePos, namePos + nameLen) === want)
|
|
37
|
+
return wasm.subarray(namePos + nameLen, end);
|
|
38
|
+
}
|
|
39
|
+
pos = end;
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface SurfaceFlags {
|
|
2
|
+
readonly rest: boolean;
|
|
3
|
+
readonly stream: boolean;
|
|
4
|
+
readonly daemon: boolean;
|
|
5
|
+
readonly scheduled: boolean;
|
|
6
|
+
readonly database: boolean;
|
|
7
|
+
readonly render: boolean;
|
|
8
|
+
}
|
|
9
|
+
export interface Surface {
|
|
10
|
+
readonly targetMode: 'hot' | 'cold';
|
|
11
|
+
readonly flags: SurfaceFlags;
|
|
12
|
+
readonly abiVersion: number;
|
|
13
|
+
readonly buildId: string;
|
|
14
|
+
readonly fingerprint: number;
|
|
15
|
+
readonly dataCoherenceHash: number;
|
|
16
|
+
readonly pairCoherenceHash: number;
|
|
17
|
+
}
|
|
18
|
+
export declare function parseSurface(wasm: Buffer): Surface | 'absent' | 'invalid';
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { DataReader } from 'toiljs/io';
|
|
2
|
+
import { customSection } from './sections.js';
|
|
3
|
+
export function parseSurface(wasm) {
|
|
4
|
+
let sec;
|
|
5
|
+
try {
|
|
6
|
+
sec = customSection(wasm, 'toil.surface');
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return 'invalid';
|
|
10
|
+
}
|
|
11
|
+
if (sec === null)
|
|
12
|
+
return 'absent';
|
|
13
|
+
const r = new DataReader(sec);
|
|
14
|
+
r.readU16();
|
|
15
|
+
const targetMode = r.readU8() === 1 ? 'cold' : 'hot';
|
|
16
|
+
r.readU8();
|
|
17
|
+
const f = r.readU32();
|
|
18
|
+
const abiVersion = r.readU16();
|
|
19
|
+
const buildId = r.readString();
|
|
20
|
+
const fingerprint = r.readU32();
|
|
21
|
+
const dataCoherenceHash = r.readU32();
|
|
22
|
+
const pairCoherenceHash = r.readU32();
|
|
23
|
+
if (!r.ok)
|
|
24
|
+
return 'invalid';
|
|
25
|
+
return {
|
|
26
|
+
targetMode,
|
|
27
|
+
flags: {
|
|
28
|
+
rest: !!(f & 1),
|
|
29
|
+
stream: !!(f & 2),
|
|
30
|
+
daemon: !!(f & 4),
|
|
31
|
+
scheduled: !!(f & 8),
|
|
32
|
+
database: !!(f & 16),
|
|
33
|
+
render: !!(f & 32),
|
|
34
|
+
},
|
|
35
|
+
abiVersion,
|
|
36
|
+
buildId,
|
|
37
|
+
fingerprint,
|
|
38
|
+
dataCoherenceHash,
|
|
39
|
+
pairCoherenceHash,
|
|
40
|
+
};
|
|
41
|
+
}
|
package/docs/README.md
CHANGED
|
@@ -34,7 +34,7 @@ and as a named export from `toiljs/server/runtime`.
|
|
|
34
34
|
type, `AuthService` (post-quantum login, signed sessions, `getUser()`), and
|
|
35
35
|
the client half.
|
|
36
36
|
- [Environment variables & secrets](./environment.md): `Environment.get` /
|
|
37
|
-
`getSecure
|
|
37
|
+
`getSecure`, per-tenant config + secrets set out of band (GitHub-Actions
|
|
38
38
|
style), so the `.wasm` carries no credentials. Two disjoint buckets, read-only.
|
|
39
39
|
- [Email](./email.md): `EmailService`, `EmailTemplate`, the `emails/` React
|
|
40
40
|
template pipeline, the stateless `TwoFactor` codes, provider config
|
|
@@ -52,14 +52,14 @@ and as a named export from `toiljs/server/runtime`.
|
|
|
52
52
|
|
|
53
53
|
## Conventions
|
|
54
54
|
|
|
55
|
-
- **"Global, no import"
|
|
55
|
+
- **"Global, no import"**, a symbol marked `@global` in the runtime is in scope
|
|
56
56
|
everywhere in a tenant without an `import`, exactly like `crypto`. The
|
|
57
57
|
matching named export exists so editors resolve the type and so the module is
|
|
58
58
|
pulled into every build. Either form works.
|
|
59
|
-
- **Binary, not JSON, on the hot paths
|
|
59
|
+
- **Binary, not JSON, on the hot paths**, request/response bodies, sessions,
|
|
60
60
|
and cookies use the deterministic `DataWriter`/`DataReader` codec. JSON is
|
|
61
61
|
available for `@rest` routes but binary is the default for anything
|
|
62
62
|
performance- or security-sensitive.
|
|
63
|
-
- **One fresh instance per request
|
|
63
|
+
- **One fresh instance per request**, guest memory is wiped between requests,
|
|
64
64
|
so nothing persists in module globals across requests. Use a host-backed store
|
|
65
65
|
for anything that must outlive a single request.
|
package/docs/auth-todo.md
CHANGED
|
@@ -24,7 +24,7 @@ imports in `toil-backend` (`crypto.mlkem_decapsulate`, `crypto.voprf_evaluate`),
|
|
|
24
24
|
|
|
25
25
|
---
|
|
26
26
|
|
|
27
|
-
## Tier 1
|
|
27
|
+
## Tier 1, blocks production (cannot run on the edge yet)
|
|
28
28
|
|
|
29
29
|
### 1.1 Storage on toildb (THE blocker) -- "when building toildb, consider this"
|
|
30
30
|
The accounts + login challenges live in the **DEV-ONLY** `kv.*` Map
|
|
@@ -72,17 +72,17 @@ in-prod `mldsa_verify` import). Do before trusting in prod.
|
|
|
72
72
|
|
|
73
73
|
---
|
|
74
74
|
|
|
75
|
-
## Tier 2
|
|
75
|
+
## Tier 2, protocol hardening
|
|
76
76
|
|
|
77
|
-
### 2.6 A properly bound session key
|
|
77
|
+
### 2.6 A properly bound session key, DONE (on main, unreleased)
|
|
78
78
|
The session key is now `K = HMAC-SHA256(sharedSecret, SESSION_KEY_LABEL || H(M))` and the
|
|
79
79
|
mutual-auth tag is `HMAC-SHA256(K, SERVER_CONFIRM_LABEL || H(M))` (`AuthService.deriveSessionKey`
|
|
80
80
|
+ `serverConfirmTag`; client mirrors it with hash-wasm `createHMAC`). REMAINING: binding the
|
|
81
81
|
*session cookie* to the transport (so a stolen cookie is useless on another channel) needs
|
|
82
|
-
the TLS exporter, which the wasm guest cannot see
|
|
82
|
+
the TLS exporter, which the wasm guest cannot see, an edge/transport follow-up, not doable
|
|
83
83
|
purely in the guest.
|
|
84
84
|
|
|
85
|
-
### 2.7 Bind the KDF params + server key into the transcript
|
|
85
|
+
### 2.7 Bind the KDF params + server key into the transcript, DONE (on main, unreleased)
|
|
86
86
|
The single `buildLoginMessage` now binds the ML-KEM ciphertext, the Argon2id params
|
|
87
87
|
(mem/iters/par), and `serverKemKeyId = SHA-256(serverKemPublicKey)`. Closes
|
|
88
88
|
key-substitution and param-downgrade confusion. (There is ONE login message format, no
|
|
@@ -96,7 +96,7 @@ a reviewable artifact.
|
|
|
96
96
|
|
|
97
97
|
---
|
|
98
98
|
|
|
99
|
-
## Tier 3
|
|
99
|
+
## Tier 3, account lifecycle (missing today)
|
|
100
100
|
|
|
101
101
|
### 3.1 Password change / key rotation
|
|
102
102
|
Re-derive under a fresh salt while authenticated, re-register the new public key. None
|
package/docs/caching.md
CHANGED
|
@@ -59,14 +59,14 @@ return Response.bytes(blob).cacheFor(5);
|
|
|
59
59
|
|
|
60
60
|
The cache layer refuses to store anything unsafe, regardless of the directive:
|
|
61
61
|
|
|
62
|
-
- **5xx** responses are never cached
|
|
62
|
+
- **5xx** responses are never cached, a server error is transient, and `@cache`
|
|
63
63
|
wraps the whole route, so a `@cache`d route that hits a blip returns its 500
|
|
64
64
|
carrying the directive; caching it would serve the failure for the full TTL.
|
|
65
65
|
**2xx, 3xx, and 4xx are cacheable** (a redirect or a `404`/`410` is a
|
|
66
66
|
deterministic function of the request key);
|
|
67
67
|
- a response that sets a **`Set-Cookie`** is never cached;
|
|
68
68
|
- a response to an **authenticated** request is not cached unless you pass
|
|
69
|
-
`allowAuth = true
|
|
69
|
+
`allowAuth = true`, this prevents one user's personalized response from being
|
|
70
70
|
served to another;
|
|
71
71
|
- the edge TTL is **clamped to 24 hours**.
|
|
72
72
|
|
|
@@ -75,7 +75,7 @@ an unauthorized request is rejected with 401 before anything is cached, and a
|
|
|
75
75
|
cached entry is only ever produced from a handler that actually ran.
|
|
76
76
|
|
|
77
77
|
Caching is **always opt-in.** A response with no `Toil-Cache-Control` directive
|
|
78
|
-
(i.e. no `@cache` / `Response.cache(...)`) is never stored
|
|
78
|
+
(i.e. no `@cache` / `Response.cache(...)`) is never stored, there is no blind
|
|
79
79
|
"cache every GET" mode, because an automatic window cannot tell a personalized
|
|
80
80
|
response from a public one and would key it without a per-user component.
|
|
81
81
|
|
|
@@ -84,11 +84,11 @@ response from a public one and would key it without a per-user component.
|
|
|
84
84
|
The edge cache is per-core and hard-capped so it can never exhaust node memory.
|
|
85
85
|
It has two tiers:
|
|
86
86
|
|
|
87
|
-
- **RAM tier
|
|
87
|
+
- **RAM tier**, small, short-TTL responses. Bounded by a per-core byte budget
|
|
88
88
|
(each core holds at most ~128 MB) plus an entry-count cap; an insert that would
|
|
89
89
|
exceed the budget drops expired entries first, then evicts the soonest-to-expire
|
|
90
90
|
ones. A response over ~256 KB does not go in the RAM tier.
|
|
91
|
-
- **Disk tier (spill)
|
|
91
|
+
- **Disk tier (spill)**, when the operator enables `--spill-dir`, a **big**
|
|
92
92
|
(over the ~256 KB RAM cap) or **long-TTL** (≥ 10 min) cacheable response is
|
|
93
93
|
written to disk instead and served back zero-RAM via a memory map, the same way
|
|
94
94
|
static files are served. This keeps the RAM tier for the hot working set while
|
package/docs/cli.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# CLI
|
|
2
|
+
|
|
3
|
+
- `toiljs create [name]`, scaffold a project. Flags: `--template app|minimal`,
|
|
4
|
+
`--style css|sass|less|stylus`, `--tailwind`, `--no-ai`, `-y`/`--yes`.
|
|
5
|
+
- `toiljs dev`, dev server with HMR (`--port`, `--root`). With a `toilconfig.json` it builds
|
|
6
|
+
the server first, then rebuilds it whenever a `server/` file changes (regenerating
|
|
7
|
+
`shared/server.ts`, which Vite HMRs into the client); client-only edits just HMR the client.
|
|
8
|
+
- `toiljs build`, production build. With a `toilconfig.json` it builds the server (toilscript,
|
|
9
|
+
regenerating `shared/server.ts`) first, then the client → `build/client`. `--server` builds
|
|
10
|
+
only the server. Every `server/` file declaring a surface (`@data`/`@rest`/...) is compiled.
|
|
11
|
+
- `toiljs start`, self-host the built app (hyper-express) with a `/_toil` WebSocket channel.
|
|
12
|
+
- `toiljs configure`, toggle styling features on an existing project (see [styling.md](./styling.md)).
|
|
13
|
+
- `toiljs doctor`, diagnose project setup (`--json` for CI). `--fix` auto-wires the typed-RPC
|
|
14
|
+
setup (build scripts, tsconfig `shared` + alias, `.gitignore`, toilscript version, and the
|
|
15
|
+
toilscript prettier plugin) so an existing project upgrades in one command.
|
package/docs/client.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Client runtime
|
|
2
|
+
|
|
3
|
+
Everything is on the `Toil` global, no imports needed in route files.
|
|
4
|
+
|
|
5
|
+
## Entry
|
|
6
|
+
|
|
7
|
+
`client/toil.tsx` imports the route table + global styles and mounts the app:
|
|
8
|
+
|
|
9
|
+
```tsx
|
|
10
|
+
import { routes, layout, notFound } from "toiljs/routes";
|
|
11
|
+
import "./styles/main.css";
|
|
12
|
+
Toil.mount(routes, layout, notFound);
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## API (on `Toil`)
|
|
16
|
+
|
|
17
|
+
- Components: `Link`, `NavLink`, `Head`
|
|
18
|
+
- Navigation: `navigate`, `useRouter`, `useNavigate`
|
|
19
|
+
- Location: `usePathname`, `useSearchParams`, `useParams`, `useNavigationPending`
|
|
20
|
+
- Data: `useLoaderData` (see [routing.md](./routing.md))
|
|
21
|
+
- Head: `useHead`, `useTitle`, `<Head>`, set the `<title>` / meta per route
|
|
22
|
+
- Realtime: `useChannel`, `connectChannel` (WebSocket to the backend at `/_toil`)
|
|
23
|
+
- IO globals (no `Toil.` prefix): `FastMap`, `FastSet`, `DataWriter`, `DataReader`
|
|
24
|
+
- `parseError(err)` global: message from an unknown caught value (handy in `catch`)
|
|
25
|
+
- `Server` global: the typed RPC surface generated from the server (see [server.md](./server.md))
|
|
26
|
+
- `Server.REST.<controller>.<route>(args)`: a working, typed `fetch` client for your
|
|
27
|
+
`@rest` controllers, e.g. `await Server.REST.todos.getTodo({ params: { id } })` or
|
|
28
|
+
`await Server.REST.todos.add({ body: new AddTodo("milk") })`. `args` is
|
|
29
|
+
`{ params?, body?, query?, headers? }`; returns are typed (`@data` classes are parsed for
|
|
30
|
+
you). The REST client attaches when you import from `shared/server`.
|
|
31
|
+
|
|
32
|
+
## Head example
|
|
33
|
+
|
|
34
|
+
```tsx
|
|
35
|
+
Toil.useHead({
|
|
36
|
+
title: "Blog",
|
|
37
|
+
titleTemplate: "%s, MyApp",
|
|
38
|
+
meta: [{ name: "description", content: "..." }],
|
|
39
|
+
});
|
|
40
|
+
```
|
package/docs/crypto.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The guest gets a synchronous Web Crypto surface through the ambient `crypto`
|
|
4
4
|
global, backed by host functions. It mirrors the browser `crypto` /
|
|
5
|
-
`crypto.subtle` API but **without Promises
|
|
5
|
+
`crypto.subtle` API but **without Promises**, ToilScript has no `async`, so
|
|
6
6
|
every call returns its result directly. Keys are opaque per-request handles in a
|
|
7
7
|
host keystore; a `CryptoKey` is valid only for the request that created it.
|
|
8
8
|
|
|
@@ -84,7 +84,7 @@ Each algorithm has a small params class you pass to `importKey`/`sign`/etc.:
|
|
|
84
84
|
- **Key formats:** `FMT_RAW`, `FMT_PKCS8`, `FMT_SPKI` (`FMT_JWK` is rejected).
|
|
85
85
|
- **Usages (bitmask):** `USAGE_ENCRYPT`, `USAGE_DECRYPT`, `USAGE_SIGN`,
|
|
86
86
|
`USAGE_VERIFY`, `USAGE_DERIVE_KEY`, `USAGE_DERIVE_BITS`, `USAGE_WRAP_KEY`,
|
|
87
|
-
`USAGE_UNWRAP_KEY
|
|
87
|
+
`USAGE_UNWRAP_KEY`, OR them together.
|
|
88
88
|
- **Named curves:** `CURVE_P256`, `CURVE_P384` (`CURVE_P521` is not supported).
|
|
89
89
|
|
|
90
90
|
### `CryptoKey`
|
|
@@ -116,7 +116,7 @@ const ct = crypto.subtle.encrypt(new AesGcmParams(iv, aad, 128), k, plaintext);
|
|
|
116
116
|
## Post-quantum verify
|
|
117
117
|
|
|
118
118
|
The host also exposes ML-DSA-44 (FIPS 204) signature verification as
|
|
119
|
-
`crypto.mldsa_verify`. It is verify-only
|
|
119
|
+
`crypto.mldsa_verify`. It is verify-only, the host never holds a secret key, and
|
|
120
120
|
underpins the [auth primitive](./auth.md). Most code reaches it through
|
|
121
121
|
`AuthService.verifyLogin(publicKey, message, signature)` rather than calling the
|
|
122
122
|
import directly. Public key is 1312 bytes, signature 2420 bytes, with a FIPS 204
|
|
@@ -124,7 +124,7 @@ domain-separation context.
|
|
|
124
124
|
|
|
125
125
|
## Limitations
|
|
126
126
|
|
|
127
|
-
- **No Promises
|
|
127
|
+
- **No Promises**, every call is synchronous.
|
|
128
128
|
- **No RSA** and **no JWK** key format.
|
|
129
129
|
- **P-521** is not supported (P-256 and P-384 are).
|
|
130
130
|
- Signature *generation* for ML-DSA is client-side only; the server verifies.
|
package/docs/data.md
CHANGED
|
@@ -17,13 +17,13 @@ class Player {
|
|
|
17
17
|
|
|
18
18
|
From that the compiler synthesizes, on the class:
|
|
19
19
|
|
|
20
|
-
- `encode(): Uint8Array` / `static decode(buf): T
|
|
20
|
+
- `encode(): Uint8Array` / `static decode(buf): T`, the binary codec (with a
|
|
21
21
|
4-byte type id prefix).
|
|
22
|
-
- `encodeInto(w: DataWriter)` / `static decodeFrom(r: DataReader)
|
|
22
|
+
- `encodeInto(w: DataWriter)` / `static decodeFrom(r: DataReader)`, the codec
|
|
23
23
|
without the type-id frame, for nesting.
|
|
24
|
-
- `toJSON()` / `static fromJSON(v)
|
|
24
|
+
- `toJSON()` / `static fromJSON(v)`, the JSON codec (64-bit-and-larger integers
|
|
25
25
|
as decimal strings, so they survive `JSON.parse` exactly).
|
|
26
|
-
- `static dataId(): u32
|
|
26
|
+
- `static dataId(): u32`, a stable FNV-1a hash of the class name, written as the
|
|
27
27
|
type-id prefix by `encode()`.
|
|
28
28
|
|
|
29
29
|
Fields may be scalars (`u8`..`u256`, `i8`..`i256`, `f32`, `f64`, `bool`),
|
|
@@ -47,8 +47,8 @@ public blob(input: FileData): FileResult { /* input.decode, result.encode */ }
|
|
|
47
47
|
|
|
48
48
|
## The binary codec: `DataWriter` / `DataReader`
|
|
49
49
|
|
|
50
|
-
When you need to lay out bytes yourself
|
|
51
|
-
challenge messages
|
|
50
|
+
When you need to lay out bytes yourself, custom bodies, session payloads,
|
|
51
|
+
challenge messages, use the codec directly. It lives in the `data` module:
|
|
52
52
|
|
|
53
53
|
```ts
|
|
54
54
|
import { DataWriter, DataReader } from 'data';
|