toiljs 0.0.68 → 0.0.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/build/cli/.tsbuildinfo +1 -1
- package/build/client/.tsbuildinfo +1 -1
- package/build/client/rpc.js +10 -4
- package/build/client/stream/client.js +108 -5
- package/build/compiler/.tsbuildinfo +1 -1
- package/build/compiler/index.d.ts +2 -0
- package/build/compiler/index.js +282 -2
- package/build/compiler/toil-docs.generated.js +3 -2
- package/build/compiler/vite.js +8 -0
- package/build/devserver/.tsbuildinfo +1 -1
- package/build/devserver/daemon/host.d.ts +1 -7
- package/build/devserver/daemon/host.js +5 -59
- package/build/devserver/daemon/index.d.ts +1 -0
- package/build/devserver/daemon/index.js +17 -4
- package/build/devserver/db/database.js +1 -1
- package/build/devserver/db/routeKinds.d.ts +6 -0
- package/build/devserver/db/routeKinds.js +40 -0
- package/build/devserver/index.d.ts +0 -1
- package/build/devserver/index.js +0 -1
- package/build/devserver/runtime/module.d.ts +1 -0
- package/build/devserver/runtime/module.js +18 -2
- package/build/devserver/stream/index.js +4 -3
- package/build/devserver/wasm/surface.d.ts +2 -0
- package/build/devserver/wasm/surface.js +35 -4
- package/docs/derive.md +159 -0
- package/docs/index.md +1 -1
- package/docs/streams.md +49 -18
- package/examples/basic/server/services/Stats.ts +11 -3
- package/examples/basic/server/services/remotes.ts +8 -2
- package/package.json +3 -2
- package/server/runtime/exports/index.ts +8 -1
- package/server/runtime/index.ts +1 -0
- package/server/runtime/rpc/Rpc.ts +66 -0
- package/src/client/rpc.ts +21 -12
- package/src/client/stream/client.ts +138 -8
- package/src/compiler/index.ts +352 -2
- package/src/compiler/toil-docs.generated.ts +3 -2
- package/src/compiler/vite.ts +16 -0
- package/src/devserver/daemon/host.ts +10 -110
- package/src/devserver/daemon/index.ts +19 -6
- package/src/devserver/db/database.ts +1 -1
- package/src/devserver/db/routeKinds.ts +44 -0
- package/src/devserver/index.ts +0 -1
- package/src/devserver/runtime/host.ts +3 -7
- package/src/devserver/runtime/module.ts +30 -4
- package/src/devserver/stream/index.ts +8 -4
- package/src/devserver/wasm/surface.ts +33 -4
- package/test/daemon-build.test.ts +53 -0
- package/test/daemon-catalog.test.ts +78 -3
- package/test/daemon-emulation.test.ts +27 -29
- package/test/devserver-database.test.ts +93 -0
- package/test/fixtures/bignum-wire/spec.ts +3 -5
- package/test/fixtures/daemon-app.ts +25 -21
- package/test/fixtures/stream-typed.ts +41 -0
- package/test/rpc-dispatch.test.ts +132 -0
- package/test/rpc-kinds.test.ts +18 -0
- package/test/rpc.test.ts +20 -4
- package/test/stream-emulation.test.ts +39 -0
- package/build/devserver/mstore/store.d.ts +0 -18
- package/build/devserver/mstore/store.js +0 -82
- package/src/devserver/mstore/store.ts +0 -121
|
@@ -102,7 +102,7 @@ export function readBytes(ref: MemoryRef, ptr: number, len: number): Buffer {
|
|
|
102
102
|
* Bounds-checked write of a variable-length result into a guest out-buffer, with
|
|
103
103
|
* the edge's inline-drain return protocol: the byte length on success, or `-1`
|
|
104
104
|
* (STATUS_TOO_SMALL) when `outCap` is too small (the guest retries with a bigger
|
|
105
|
-
* buffer).
|
|
105
|
+
* buffer).
|
|
106
106
|
*/
|
|
107
107
|
export function writeBytesOut(
|
|
108
108
|
ref: MemoryRef,
|
|
@@ -217,12 +217,8 @@ export function buildEnvImports(
|
|
|
217
217
|
// the buffer is too small (the guest retries bigger), -2 if absent.
|
|
218
218
|
env_get: (keyPtr: number, keyLen: number, outPtr: number, outCap: number): number =>
|
|
219
219
|
envLookup(ref, keyPtr, keyLen, outPtr, outCap, false),
|
|
220
|
-
env_get_secure: (
|
|
221
|
-
keyPtr
|
|
222
|
-
keyLen: number,
|
|
223
|
-
outPtr: number,
|
|
224
|
-
outCap: number,
|
|
225
|
-
): number => envLookup(ref, keyPtr, keyLen, outPtr, outCap, true),
|
|
220
|
+
env_get_secure: (keyPtr: number, keyLen: number, outPtr: number, outCap: number): number =>
|
|
221
|
+
envLookup(ref, keyPtr, keyLen, outPtr, outCap, true),
|
|
226
222
|
|
|
227
223
|
thread_spawn: (_startArg: number): number => -1,
|
|
228
224
|
|
|
@@ -21,7 +21,14 @@ import {
|
|
|
21
21
|
persistDb,
|
|
22
22
|
setDbCatalog,
|
|
23
23
|
} from '../db/index.js';
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
parseRouteKinds,
|
|
26
|
+
parseRpcKinds,
|
|
27
|
+
routeKindForRequest,
|
|
28
|
+
rpcKindForId,
|
|
29
|
+
type RouteKindEntry,
|
|
30
|
+
type RpcKindEntry,
|
|
31
|
+
} from '../db/routeKinds.js';
|
|
25
32
|
import {
|
|
26
33
|
decodeResponseEnvelope,
|
|
27
34
|
encodeRequestEnvelope,
|
|
@@ -157,6 +164,7 @@ export class WasmServerModule {
|
|
|
157
164
|
private module: WebAssembly.Module | null = null;
|
|
158
165
|
private loadedMtimeMs = -1;
|
|
159
166
|
private routeKinds: readonly RouteKindEntry[] = [];
|
|
167
|
+
private rpcKinds: readonly RpcKindEntry[] = [];
|
|
160
168
|
private derives: readonly DeriveEntry[] = [];
|
|
161
169
|
// Set when a (re)compile loaded a module with @derive methods; the first
|
|
162
170
|
// dispatch afterward rebuilds every materialized view from its sources.
|
|
@@ -182,6 +190,7 @@ export class WasmServerModule {
|
|
|
182
190
|
} catch {
|
|
183
191
|
this.module = null;
|
|
184
192
|
this.routeKinds = [];
|
|
193
|
+
this.rpcKinds = [];
|
|
185
194
|
this.derives = [];
|
|
186
195
|
this.derivesDirty = false;
|
|
187
196
|
this.loadedMtimeMs = -1;
|
|
@@ -197,6 +206,7 @@ export class WasmServerModule {
|
|
|
197
206
|
// after a @data type evolves + rebuild, old on-disk rows now look out of date.
|
|
198
207
|
setDbCatalog(bytes);
|
|
199
208
|
this.routeKinds = parseRouteKinds(bytes);
|
|
209
|
+
this.rpcKinds = parseRpcKinds(bytes);
|
|
200
210
|
this.derives = parseDerives(bytes);
|
|
201
211
|
this.module = module;
|
|
202
212
|
this.loadedMtimeMs = mtimeMs;
|
|
@@ -224,9 +234,25 @@ export class WasmServerModule {
|
|
|
224
234
|
const ref: MemoryRef = { memory: null };
|
|
225
235
|
const state = freshDispatchState();
|
|
226
236
|
state.clientIp = req.clientIp ?? '';
|
|
227
|
-
// Match the edge DB gate: the HTTP method is the baseline authority,
|
|
228
|
-
//
|
|
229
|
-
|
|
237
|
+
// Match the edge DB gate: the HTTP method is the baseline authority, and `toildb.route_kinds`
|
|
238
|
+
// can only tighten a mutating route to query. The reserved /__toil_rpc endpoint is the inverse -
|
|
239
|
+
// a @remote defaults to read-only (Query); only an @action @remote (in rpc_kinds) may write.
|
|
240
|
+
const rpcPath = req.path.split('?')[0] ?? req.path;
|
|
241
|
+
const rpcMethod = req.method.toUpperCase();
|
|
242
|
+
const rpcMutating =
|
|
243
|
+
rpcMethod === 'POST' || rpcMethod === 'PUT' || rpcMethod === 'PATCH' || rpcMethod === 'DELETE';
|
|
244
|
+
if (rpcPath === '/__toil_rpc' && rpcMutating) {
|
|
245
|
+
const idHeader = req.headers.find(([n]) => n.toLowerCase() === 'toil-rpc')?.[1];
|
|
246
|
+
// Strict u32 parse, mirroring the host's `v.parse::<u32>()`: reject trailing garbage/whitespace
|
|
247
|
+
// and out-of-range ids so a malformed header falls through to read-only Query, exactly as prod.
|
|
248
|
+
const id = idHeader !== undefined && /^\d+$/.test(idHeader) ? Number(idHeader) : NaN;
|
|
249
|
+
state.db.functionKind =
|
|
250
|
+
Number.isInteger(id) && id <= 0xffffffff
|
|
251
|
+
? rpcKindForId(this.rpcKinds, id)
|
|
252
|
+
: DbFunctionKind.Query;
|
|
253
|
+
} else {
|
|
254
|
+
state.db.functionKind = dbFunctionKindForRequest(this.routeKinds, req.method, req.path);
|
|
255
|
+
}
|
|
230
256
|
const instance = new WebAssembly.Instance(this.module, buildHostImports(ref, state));
|
|
231
257
|
const exports = instance.exports as unknown as HandleExports;
|
|
232
258
|
ref.memory = exports.memory;
|
|
@@ -98,8 +98,8 @@ export class DevStreamBox {
|
|
|
98
98
|
* cold artifact, a missing `stream_dispatch`/`memory`, or a bad module throws. */
|
|
99
99
|
static load(wasm: Buffer): DevStreamBox {
|
|
100
100
|
const surface = parseSurface(wasm);
|
|
101
|
-
if (surface === 'invalid' || surface.targetMode !== 'hot') {
|
|
102
|
-
throw new Error('stream box requires a hot artifact with a valid toil.surface');
|
|
101
|
+
if (surface === 'invalid' || surface.targetMode !== 'hot' || !surface.flags.stream) {
|
|
102
|
+
throw new Error('stream box requires a hot stream artifact with a valid toil.surface');
|
|
103
103
|
}
|
|
104
104
|
const ref: { memory: WebAssembly.Memory | null } = { memory: null };
|
|
105
105
|
const state = freshStreamBoxState();
|
|
@@ -192,7 +192,10 @@ export class DevStreamBox {
|
|
|
192
192
|
}
|
|
193
193
|
|
|
194
194
|
private static resolveStreamInfo(e: StreamExports): StreamInfo | null {
|
|
195
|
-
if (
|
|
195
|
+
if (
|
|
196
|
+
typeof e.stream_info_offset !== 'function' ||
|
|
197
|
+
typeof e.stream_info_capacity !== 'function'
|
|
198
|
+
) {
|
|
196
199
|
return null;
|
|
197
200
|
}
|
|
198
201
|
return { offset: e.stream_info_offset() >>> 0, cap: e.stream_info_capacity() >>> 0 };
|
|
@@ -277,7 +280,8 @@ export class DevStreamBox {
|
|
|
277
280
|
dv.setUint16(f + 2, 0, true); // flags
|
|
278
281
|
dv.setUint32(f + 4, n, true); // length
|
|
279
282
|
dv.setUint32(f + 8, 0, true); // msg_seq
|
|
280
|
-
if (n > 0)
|
|
283
|
+
if (n > 0)
|
|
284
|
+
new Uint8Array(this.exports.memory.buffer, f + RING_FRAME_HEADER, n).set(inbound);
|
|
281
285
|
dv.setUint32(base + RC_WRITE, w + frameLen, true);
|
|
282
286
|
}
|
|
283
287
|
|
|
@@ -26,6 +26,20 @@ import { DataReader } from 'toiljs/io';
|
|
|
26
26
|
|
|
27
27
|
import { customSection } from './sections.js';
|
|
28
28
|
|
|
29
|
+
export const SURFACE_FORMAT_VERSION = 1;
|
|
30
|
+
export const SURFACE_ABI_VERSION = 1;
|
|
31
|
+
|
|
32
|
+
const TARGET_HOT = 0;
|
|
33
|
+
const TARGET_COLD = 1;
|
|
34
|
+
const FLAG_REST = 1 << 0;
|
|
35
|
+
const FLAG_STREAM = 1 << 1;
|
|
36
|
+
const FLAG_DAEMON = 1 << 2;
|
|
37
|
+
const FLAG_SCHEDULED = 1 << 3;
|
|
38
|
+
const FLAG_DATABASE = 1 << 4;
|
|
39
|
+
const FLAG_RENDER = 1 << 5;
|
|
40
|
+
const FLAG_KNOWN_MASK =
|
|
41
|
+
FLAG_REST | FLAG_STREAM | FLAG_DAEMON | FLAG_SCHEDULED | FLAG_DATABASE | FLAG_RENDER;
|
|
42
|
+
|
|
29
43
|
export interface SurfaceFlags {
|
|
30
44
|
readonly rest: boolean;
|
|
31
45
|
readonly stream: boolean;
|
|
@@ -57,16 +71,31 @@ export function parseSurface(wasm: Buffer): Surface | 'invalid' {
|
|
|
57
71
|
if (sec === null) return 'invalid';
|
|
58
72
|
|
|
59
73
|
const r = new DataReader(sec);
|
|
60
|
-
r.readU16(); // format_version
|
|
61
|
-
|
|
62
|
-
r.readU8();
|
|
74
|
+
const version = r.readU16(); // format_version
|
|
75
|
+
if (!r.ok || version !== SURFACE_FORMAT_VERSION) return 'invalid';
|
|
76
|
+
const targetModeByte = r.readU8();
|
|
77
|
+
if (!r.ok || (targetModeByte !== TARGET_HOT && targetModeByte !== TARGET_COLD)) {
|
|
78
|
+
return 'invalid';
|
|
79
|
+
}
|
|
80
|
+
const targetMode = targetModeByte === TARGET_COLD ? 'cold' : 'hot';
|
|
81
|
+
const reserved0 = r.readU8(); // reserved0
|
|
82
|
+
if (!r.ok || reserved0 !== 0) return 'invalid';
|
|
63
83
|
const f = r.readU32(); // surface_flags
|
|
84
|
+
if (!r.ok || (f & ~FLAG_KNOWN_MASK) !== 0) return 'invalid';
|
|
85
|
+
if ((f & FLAG_SCHEDULED) !== 0 && (f & FLAG_DAEMON) === 0) return 'invalid';
|
|
86
|
+
if (targetMode === 'hot' && (f & (FLAG_DAEMON | FLAG_SCHEDULED)) !== 0) {
|
|
87
|
+
return 'invalid';
|
|
88
|
+
}
|
|
89
|
+
if (targetMode === 'cold' && (f & (FLAG_REST | FLAG_STREAM | FLAG_RENDER)) !== 0) {
|
|
90
|
+
return 'invalid';
|
|
91
|
+
}
|
|
64
92
|
const abiVersion = r.readU16();
|
|
93
|
+
if (!r.ok || abiVersion !== SURFACE_ABI_VERSION) return 'invalid';
|
|
65
94
|
const buildId = r.readString();
|
|
66
95
|
const fingerprint = r.readU32();
|
|
67
96
|
const dataCoherenceHash = r.readU32(); // exactly THREE u32 after build_id
|
|
68
97
|
const pairCoherenceHash = r.readU32();
|
|
69
|
-
if (!r.ok) return 'invalid'; // PRESENT but corrupt => fail closed
|
|
98
|
+
if (!r.ok || r.remaining() !== 0) return 'invalid'; // PRESENT but corrupt => fail closed
|
|
70
99
|
return {
|
|
71
100
|
targetMode,
|
|
72
101
|
flags: {
|
|
@@ -33,6 +33,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
33
33
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
34
34
|
|
|
35
35
|
import {
|
|
36
|
+
assertNoStreamInRequestTier,
|
|
36
37
|
buildServer,
|
|
37
38
|
serverArtifacts,
|
|
38
39
|
splitSurfaceFiles,
|
|
@@ -183,6 +184,58 @@ describe('splitSurfaceFiles per-pass classification', () => {
|
|
|
183
184
|
expect(split.request).toContain('server/both.ts');
|
|
184
185
|
expect(split.stream).not.toContain('server/both.ts');
|
|
185
186
|
});
|
|
187
|
+
|
|
188
|
+
it('exposes the actual @stream modules in streamModules (not @rest or shared helpers)', () => {
|
|
189
|
+
const rels = lay({
|
|
190
|
+
'server/chat.ts': "@stream('chat')\nclass C {}\n",
|
|
191
|
+
'server/api.ts': '@rest\nclass A {}\n',
|
|
192
|
+
'server/util.ts': 'export function helper(): i32 { return 1; }\n',
|
|
193
|
+
});
|
|
194
|
+
const split = splitSurfaceFiles(tmp, rels);
|
|
195
|
+
expect(split.streamModules).toEqual(['server/chat.ts']);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('REJECTS a value-import of a @stream module; allows clean/type-only/commented (audit #17/#7)', () => {
|
|
199
|
+
const rels = lay({
|
|
200
|
+
'server/main.ts': "import { A } from './api';\n", // request-tier (shared) entry
|
|
201
|
+
'server/api.ts': '@rest\nclass A {}\n',
|
|
202
|
+
'server/chat.ts': "@stream('chat')\nclass C {}\n",
|
|
203
|
+
});
|
|
204
|
+
const check = () => assertNoStreamInRequestTier(tmp, splitSurfaceFiles(tmp, rels));
|
|
205
|
+
|
|
206
|
+
// main imports only the @rest surface -> no stream code leaks into release.wasm.
|
|
207
|
+
expect(check).not.toThrow();
|
|
208
|
+
|
|
209
|
+
// The footgun: a request-tier file value-imports the @stream module.
|
|
210
|
+
writeFileSync(join(tmp, 'server', 'main.ts'), "import { C } from './chat';\nvoid C;\n");
|
|
211
|
+
expect(check).toThrow(/@stream class would be compiled into the REQUEST tier/);
|
|
212
|
+
|
|
213
|
+
// A type-only import is erased and never compiles @stream code -> allowed.
|
|
214
|
+
writeFileSync(join(tmp, 'server', 'main.ts'), "import type { C } from './chat';\n");
|
|
215
|
+
expect(check).not.toThrow();
|
|
216
|
+
|
|
217
|
+
// A COMMENTED-OUT import is dead code and must not false-positive (audit #7).
|
|
218
|
+
writeFileSync(
|
|
219
|
+
join(tmp, 'server', 'main.ts'),
|
|
220
|
+
"// import { C } from './chat';\nimport { A } from './api';\n",
|
|
221
|
+
);
|
|
222
|
+
expect(check).not.toThrow();
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it('catches a @stream reached transitively through a plain helper NOT in the surface list (audit #6)', () => {
|
|
226
|
+
// util.ts is an UNDECORATED helper -> not in serverEntryFiles, so production hands splitSurfaceFiles
|
|
227
|
+
// only the entries+surfaces. The filesystem-resolving BFS must still traverse util.ts to reach the
|
|
228
|
+
// @stream module behind it.
|
|
229
|
+
lay({
|
|
230
|
+
'server/main.ts': "import './util';\n",
|
|
231
|
+
'server/util.ts': "import { C } from './chat';\nexport const x = 1;\n",
|
|
232
|
+
'server/chat.ts': "@stream('chat')\nclass C {}\n",
|
|
233
|
+
});
|
|
234
|
+
const surfacesOnly = ['server/main.ts', 'server/chat.ts']; // no plain util.ts, as in production
|
|
235
|
+
expect(() => assertNoStreamInRequestTier(tmp, splitSurfaceFiles(tmp, surfacesOnly))).toThrow(
|
|
236
|
+
/@stream class would be compiled into the REQUEST tier/,
|
|
237
|
+
);
|
|
238
|
+
});
|
|
186
239
|
});
|
|
187
240
|
|
|
188
241
|
// Needs the local toilscript dev build (with --targetMode) linked as a sibling
|
|
@@ -158,15 +158,17 @@ describe('parseDaemonCatalog (Part 5)', () => {
|
|
|
158
158
|
// --- toil.surface (Part 5) -------------------------------------------------
|
|
159
159
|
|
|
160
160
|
function buildSurfaceBytes(opts: {
|
|
161
|
-
mode:
|
|
161
|
+
mode: number;
|
|
162
162
|
flags: number;
|
|
163
|
+
version?: number;
|
|
164
|
+
reserved0?: number;
|
|
163
165
|
abi?: number;
|
|
164
166
|
buildId?: string;
|
|
165
167
|
}): Uint8Array {
|
|
166
168
|
const w = new DataWriter();
|
|
167
|
-
w.writeU16(1); // format_version
|
|
169
|
+
w.writeU16(opts.version ?? 1); // format_version
|
|
168
170
|
w.writeU8(opts.mode); // target_mode
|
|
169
|
-
w.writeU8(0); // reserved0
|
|
171
|
+
w.writeU8(opts.reserved0 ?? 0); // reserved0
|
|
170
172
|
w.writeU32(opts.flags); // surface_flags
|
|
171
173
|
w.writeU16(opts.abi ?? 1); // abi_version
|
|
172
174
|
w.writeString(opts.buildId ?? ''); // build_id
|
|
@@ -188,6 +190,7 @@ describe('parseSurface (Part 5)', () => {
|
|
|
188
190
|
expect(s.flags.daemon).toBe(true);
|
|
189
191
|
expect(s.flags.scheduled).toBe(true);
|
|
190
192
|
expect(s.flags.rest).toBe(false);
|
|
193
|
+
expect(s.abiVersion).toBe(1);
|
|
191
194
|
expect(s.fingerprint).toBe(0xdeadbeef);
|
|
192
195
|
expect(s.dataCoherenceHash).toBe(0x11111111);
|
|
193
196
|
expect(s.pairCoherenceHash).toBe(0x22222222);
|
|
@@ -211,6 +214,78 @@ describe('parseSurface (Part 5)', () => {
|
|
|
211
214
|
const truncated = full.subarray(0, full.length - 3); // chop a trailing hash
|
|
212
215
|
expect(parseSurface(wasmWithSection('toil.surface', truncated))).toBe('invalid');
|
|
213
216
|
});
|
|
217
|
+
|
|
218
|
+
it("fails closed: trailing bytes after toil.surface are 'invalid'", () => {
|
|
219
|
+
const full = Buffer.from(buildSurfaceBytes({ mode: 0, flags: 1 }));
|
|
220
|
+
const withGarbage = Buffer.concat([full, Buffer.from([0xff])]);
|
|
221
|
+
expect(parseSurface(wasmWithSection('toil.surface', withGarbage))).toBe('invalid');
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it("fails closed: unsupported format version is 'invalid'", () => {
|
|
225
|
+
expect(
|
|
226
|
+
parseSurface(
|
|
227
|
+
wasmWithSection(
|
|
228
|
+
'toil.surface',
|
|
229
|
+
buildSurfaceBytes({ version: 2, mode: 0, flags: 1 }),
|
|
230
|
+
),
|
|
231
|
+
),
|
|
232
|
+
).toBe('invalid');
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
it("fails closed: unsupported abi_version is 'invalid'", () => {
|
|
236
|
+
expect(
|
|
237
|
+
parseSurface(
|
|
238
|
+
wasmWithSection('toil.surface', buildSurfaceBytes({ mode: 0, flags: 1, abi: 2 })),
|
|
239
|
+
),
|
|
240
|
+
).toBe('invalid');
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
it('fails closed: target_mode 2 is not silently treated as hot', () => {
|
|
244
|
+
expect(
|
|
245
|
+
parseSurface(wasmWithSection('toil.surface', buildSurfaceBytes({ mode: 2, flags: 1 }))),
|
|
246
|
+
).toBe('invalid');
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
it("fails closed: nonzero reserved byte is 'invalid'", () => {
|
|
250
|
+
expect(
|
|
251
|
+
parseSurface(
|
|
252
|
+
wasmWithSection(
|
|
253
|
+
'toil.surface',
|
|
254
|
+
buildSurfaceBytes({ mode: 0, flags: 1, reserved0: 1 }),
|
|
255
|
+
),
|
|
256
|
+
),
|
|
257
|
+
).toBe('invalid');
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
it("fails closed: reserved surface flag bits are 'invalid'", () => {
|
|
261
|
+
expect(
|
|
262
|
+
parseSurface(
|
|
263
|
+
wasmWithSection('toil.surface', buildSurfaceBytes({ mode: 0, flags: 1 << 6 })),
|
|
264
|
+
),
|
|
265
|
+
).toBe('invalid');
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
it('fails closed: daemon flags are illegal on hot artifacts', () => {
|
|
269
|
+
expect(
|
|
270
|
+
parseSurface(
|
|
271
|
+
wasmWithSection('toil.surface', buildSurfaceBytes({ mode: 0, flags: 1 | 4 })),
|
|
272
|
+
),
|
|
273
|
+
).toBe('invalid');
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
it('fails closed: stream flags are illegal on cold artifacts', () => {
|
|
277
|
+
expect(
|
|
278
|
+
parseSurface(
|
|
279
|
+
wasmWithSection('toil.surface', buildSurfaceBytes({ mode: 1, flags: 4 | 2 })),
|
|
280
|
+
),
|
|
281
|
+
).toBe('invalid');
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
it('fails closed: scheduled requires daemon', () => {
|
|
285
|
+
expect(
|
|
286
|
+
parseSurface(wasmWithSection('toil.surface', buildSurfaceBytes({ mode: 1, flags: 8 }))),
|
|
287
|
+
).toBe('invalid');
|
|
288
|
+
});
|
|
214
289
|
});
|
|
215
290
|
|
|
216
291
|
describe('customSection bounds-checking', () => {
|
|
@@ -10,9 +10,9 @@
|
|
|
10
10
|
* - `daemon.is_leader()` / `current_epoch()` / `task_count()` stubs answer.
|
|
11
11
|
* - the epoch bumps on a cold-artifact reload (the fencing token).
|
|
12
12
|
*
|
|
13
|
-
* The fixture records its activity into
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* The fixture records its activity into resident daemon wasm memory and exports
|
|
14
|
+
* scalar accessors for the test. Interval ticks are driven with vitest fake
|
|
15
|
+
* timers.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import { spawnSync } from 'node:child_process';
|
|
@@ -25,7 +25,6 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest
|
|
|
25
25
|
|
|
26
26
|
import { DaemonHost } from '../src/devserver/daemon/index.js';
|
|
27
27
|
import type { ResolvedDaemonConfig } from '../src/devserver/daemon/host.js';
|
|
28
|
-
import { devMemoryStore } from '../src/devserver/mstore/store.js';
|
|
29
28
|
|
|
30
29
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
31
30
|
const FIXTURE = join(here, 'fixtures', 'daemon-app.ts');
|
|
@@ -50,7 +49,10 @@ function compileCold(src: string, outWasm: string): { ok: boolean; output: strin
|
|
|
50
49
|
[LOCAL_TOILSCRIPT_BIN, src, '-o', outWasm, '--runtime', 'stub', '--targetMode', 'cold'],
|
|
51
50
|
{ encoding: 'utf8' },
|
|
52
51
|
);
|
|
53
|
-
return {
|
|
52
|
+
return {
|
|
53
|
+
ok: r.status === 0 && existsSync(outWasm),
|
|
54
|
+
output: (r.stdout ?? '') + (r.stderr ?? ''),
|
|
55
|
+
};
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
let tmp: string;
|
|
@@ -72,13 +74,9 @@ afterAll(() => {
|
|
|
72
74
|
|
|
73
75
|
afterEach(() => {
|
|
74
76
|
vi.useRealTimers();
|
|
75
|
-
devMemoryStore.__reset();
|
|
76
77
|
});
|
|
77
78
|
|
|
78
|
-
const counter = (
|
|
79
|
-
const v = devMemoryStore.get(key);
|
|
80
|
-
return v === null ? 0 : Number(v.toString('utf8'));
|
|
81
|
-
};
|
|
79
|
+
const counter = (host: DaemonHost, name: string): number => host.callI32Export(`${name}Count`) ?? 0;
|
|
82
80
|
|
|
83
81
|
// Needs the local toilscript dev build (with --targetMode); skip where the
|
|
84
82
|
// sibling repo is absent (e.g. CI, which has only the published dep).
|
|
@@ -86,9 +84,10 @@ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT_BIN))('dev daemon emulation', () =>
|
|
|
86
84
|
it('compiles the @daemon fixture to a cold artifact', () => {
|
|
87
85
|
// Guard: every assertion below depends on the local toilscript link. A hard
|
|
88
86
|
// failure here (rather than a silent skip) surfaces the cross-repo break.
|
|
89
|
-
expect(
|
|
90
|
-
|
|
91
|
-
|
|
87
|
+
expect(
|
|
88
|
+
existsSync(LOCAL_TOILSCRIPT_BIN),
|
|
89
|
+
`local toilscript not found at ${LOCAL_TOILSCRIPT_BIN}`,
|
|
90
|
+
).toBe(true);
|
|
92
91
|
expect(toilscriptAvailable, 'cold compile of the @daemon fixture failed').toBe(true);
|
|
93
92
|
});
|
|
94
93
|
|
|
@@ -98,14 +97,14 @@ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT_BIN))('dev daemon emulation', () =>
|
|
|
98
97
|
try {
|
|
99
98
|
expect(host.active).toBe(true);
|
|
100
99
|
// onStart ran once -> "started" counter is exactly 1.
|
|
101
|
-
expect(counter('started')).toBe(1);
|
|
100
|
+
expect(counter(host, 'started')).toBe(1);
|
|
102
101
|
// The leader / epoch / task_count stubs all answered during onStart.
|
|
103
|
-
expect(counter('leader')).toBe(1);
|
|
104
|
-
expect(counter('
|
|
105
|
-
expect(counter('
|
|
102
|
+
expect(counter(host, 'leader')).toBe(1);
|
|
103
|
+
expect(counter(host, 'epochNonneg')).toBe(1);
|
|
104
|
+
expect(counter(host, 'taskcount2')).toBe(1);
|
|
106
105
|
// refresh() with no mtime change is a no-op -> daemon_start does NOT re-run.
|
|
107
106
|
expect(host.refresh()).toBe(false);
|
|
108
|
-
expect(counter('started')).toBe(1);
|
|
107
|
+
expect(counter(host, 'started')).toBe(1);
|
|
109
108
|
} finally {
|
|
110
109
|
host.close();
|
|
111
110
|
}
|
|
@@ -116,13 +115,13 @@ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT_BIN))('dev daemon emulation', () =>
|
|
|
116
115
|
const host = new DaemonHost(coldWasm, DAEMON_CFG, 'all', () => {});
|
|
117
116
|
host.refresh();
|
|
118
117
|
try {
|
|
119
|
-
expect(counter('
|
|
118
|
+
expect(counter(host, 'tickFast')).toBe(0); // not fired yet
|
|
120
119
|
vi.advanceTimersByTime(1000);
|
|
121
|
-
expect(counter('
|
|
120
|
+
expect(counter(host, 'tickFast')).toBe(1); // one interval elapsed -> one tick
|
|
122
121
|
vi.advanceTimersByTime(3000);
|
|
123
|
-
expect(counter('
|
|
122
|
+
expect(counter(host, 'tickFast')).toBe(4); // three more ticks
|
|
124
123
|
// The cron task ("0 */6 * * *") must NOT have fired in 4 simulated seconds.
|
|
125
|
-
expect(counter('
|
|
124
|
+
expect(counter(host, 'tickCron')).toBe(0);
|
|
126
125
|
} finally {
|
|
127
126
|
host.close();
|
|
128
127
|
}
|
|
@@ -135,13 +134,13 @@ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT_BIN))('dev daemon emulation', () =>
|
|
|
135
134
|
const host = new DaemonHost(coldWasm, DAEMON_CFG, 'all', () => {});
|
|
136
135
|
host.refresh();
|
|
137
136
|
try {
|
|
138
|
-
expect(counter('
|
|
137
|
+
expect(counter(host, 'tickCron')).toBe(0);
|
|
139
138
|
// Advance to 06:00:00 (30s) -> the cron one-shot fires exactly once.
|
|
140
139
|
vi.advanceTimersByTime(30_000);
|
|
141
|
-
expect(counter('
|
|
140
|
+
expect(counter(host, 'tickCron')).toBe(1);
|
|
142
141
|
// It dispatched task_index 1 (sixHourly), NOT the interval task body.
|
|
143
142
|
// (tick:fast also advanced on its own 1s timer; assert cron fired once.)
|
|
144
|
-
expect(counter('
|
|
143
|
+
expect(counter(host, 'tickCron')).toBe(1);
|
|
145
144
|
} finally {
|
|
146
145
|
host.close();
|
|
147
146
|
}
|
|
@@ -152,7 +151,7 @@ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT_BIN))('dev daemon emulation', () =>
|
|
|
152
151
|
host.refresh();
|
|
153
152
|
try {
|
|
154
153
|
expect(host.active).toBe(false);
|
|
155
|
-
expect(counter('started')).toBe(0);
|
|
154
|
+
expect(counter(host, 'started')).toBe(0);
|
|
156
155
|
} finally {
|
|
157
156
|
host.close();
|
|
158
157
|
}
|
|
@@ -170,9 +169,8 @@ describe.skipIf(!existsSync(LOCAL_TOILSCRIPT_BIN))('dev daemon emulation', () =>
|
|
|
170
169
|
const reloaded = host.refresh();
|
|
171
170
|
expect(reloaded).toBe(true);
|
|
172
171
|
expect(host.epoch()).toBe(e1 + 1n);
|
|
173
|
-
// A reload is a full restart ->
|
|
174
|
-
|
|
175
|
-
expect(counter('started')).toBe(2);
|
|
172
|
+
// A reload is a full restart -> the new resident daemon memory starts fresh.
|
|
173
|
+
expect(counter(host, 'started')).toBe(1);
|
|
176
174
|
} finally {
|
|
177
175
|
host.close();
|
|
178
176
|
}
|
|
@@ -11,7 +11,9 @@ import {
|
|
|
11
11
|
__setDbCatalogForTests,
|
|
12
12
|
buildDatabaseImports,
|
|
13
13
|
configureDbPersistence,
|
|
14
|
+
derivesForWrites,
|
|
14
15
|
freshDbState,
|
|
16
|
+
parseDerives,
|
|
15
17
|
persistDb,
|
|
16
18
|
setDbCatalog,
|
|
17
19
|
} from '../src/devserver/db/index.js';
|
|
@@ -92,6 +94,24 @@ function routeKindsSection(routes: readonly (readonly [number, number, string])[
|
|
|
92
94
|
return Buffer.concat(chunks);
|
|
93
95
|
}
|
|
94
96
|
|
|
97
|
+
function derivesSection(entries: readonly (readonly [number, string, string])[]): Buffer {
|
|
98
|
+
const chunks: Buffer[] = [Buffer.alloc(4)];
|
|
99
|
+
chunks[0].writeUInt16LE(1, 0); // format_version
|
|
100
|
+
chunks[0].writeUInt16LE(entries.length, 2); // n_derives
|
|
101
|
+
for (const [id, db, method] of entries) {
|
|
102
|
+
const head = Buffer.alloc(2);
|
|
103
|
+
head.writeUInt16LE(id, 0);
|
|
104
|
+
chunks.push(head);
|
|
105
|
+
for (const s of [db, method]) {
|
|
106
|
+
const b = Buffer.from(s, 'utf8');
|
|
107
|
+
const len = Buffer.alloc(4);
|
|
108
|
+
len.writeUInt32LE(b.length, 0);
|
|
109
|
+
chunks.push(len, b);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return Buffer.concat(chunks);
|
|
113
|
+
}
|
|
114
|
+
|
|
95
115
|
function catalogSectionV1(fillMaxWaitMs: number, fillAllowStale: number, replication = 5): Buffer {
|
|
96
116
|
const chunks: Buffer[] = [];
|
|
97
117
|
const u8 = (v: number) => chunks.push(Buffer.from([v & 0xff]));
|
|
@@ -761,6 +781,20 @@ describe('toildb dev emulator (migration + persistence)', () => {
|
|
|
761
781
|
expect(rsv(imports)).toBe(0x1234n); // the woven decoder dispatches on this
|
|
762
782
|
});
|
|
763
783
|
|
|
784
|
+
it('patch surfaces the current catalog schema_version (regression: was -1)', () => {
|
|
785
|
+
const { imports, buf } = setup();
|
|
786
|
+
__setDbCatalogForTests({ 'App/users': 0x1234 });
|
|
787
|
+
const h = resolve(imports, buf, 'App/users');
|
|
788
|
+
const [kPtr, kLen] = put(buf, 32, 'u1');
|
|
789
|
+
const [vPtr, vLen] = put(buf, 64, 'orig');
|
|
790
|
+
imports['data.create'](h, kPtr, kLen, vPtr, vLen, 0);
|
|
791
|
+
const [pPtr, pLen] = put(buf, 96, 'patched');
|
|
792
|
+
// patch stashes the patched record + its schema_version; the guest's woven
|
|
793
|
+
// decoder dispatches on that version, so it MUST be the current one, not -1.
|
|
794
|
+
imports['data.patch'](h, kPtr, kLen, pPtr, pLen, 0);
|
|
795
|
+
expect(rsv(imports)).toBe(0x1234n);
|
|
796
|
+
});
|
|
797
|
+
|
|
764
798
|
it('an evolved @data type leaves old rows stamped with the OLD version', () => {
|
|
765
799
|
const { imports, buf } = setup();
|
|
766
800
|
__setDbCatalogForTests({ 'App/users': 100 }); // version A
|
|
@@ -894,3 +928,62 @@ describe('toildb dev emulator (migration + persistence)', () => {
|
|
|
894
928
|
}
|
|
895
929
|
});
|
|
896
930
|
});
|
|
931
|
+
|
|
932
|
+
describe('toildb dev derives (toildb.derives parser + write routing)', () => {
|
|
933
|
+
const id = (d: { deriveId: number }) => d.deriveId;
|
|
934
|
+
|
|
935
|
+
it('parses a valid toildb.derives section in declaration order', () => {
|
|
936
|
+
const wasm = wasmWithSection(
|
|
937
|
+
'toildb.derives',
|
|
938
|
+
derivesSection([
|
|
939
|
+
[0, 'App', 'rebuild'],
|
|
940
|
+
[1, 'App', 'refresh'],
|
|
941
|
+
[2, 'Other', 'rollup'],
|
|
942
|
+
]),
|
|
943
|
+
);
|
|
944
|
+
expect(parseDerives(wasm).map((d) => `${d.deriveId}:${d.dbName}.${d.methodName}`)).toEqual([
|
|
945
|
+
'0:App.rebuild',
|
|
946
|
+
'1:App.refresh',
|
|
947
|
+
'2:Other.rollup',
|
|
948
|
+
]);
|
|
949
|
+
});
|
|
950
|
+
|
|
951
|
+
it('returns [] when there is no toildb.derives section', () => {
|
|
952
|
+
expect(parseDerives(wasmWithSection('toildb.other', Buffer.from([1, 2, 3])))).toEqual([]);
|
|
953
|
+
});
|
|
954
|
+
|
|
955
|
+
it('fails closed on a bad version, trailing bytes, or an empty db name', () => {
|
|
956
|
+
const badVersion = derivesSection([[0, 'App', 'r']]);
|
|
957
|
+
badVersion.writeUInt16LE(2, 0); // unknown format_version
|
|
958
|
+
expect(parseDerives(wasmWithSection('toildb.derives', badVersion))).toEqual([]);
|
|
959
|
+
|
|
960
|
+
const trailing = Buffer.concat([derivesSection([[0, 'App', 'r']]), Buffer.from([0])]);
|
|
961
|
+
expect(parseDerives(wasmWithSection('toildb.derives', trailing))).toEqual([]);
|
|
962
|
+
|
|
963
|
+
const emptyDb = derivesSection([[0, '', 'r']]);
|
|
964
|
+
expect(parseDerives(wasmWithSection('toildb.derives', emptyDb))).toEqual([]);
|
|
965
|
+
});
|
|
966
|
+
|
|
967
|
+
it('routes a source-collection write to its database derives, coalesced', () => {
|
|
968
|
+
const derives = parseDerives(
|
|
969
|
+
wasmWithSection(
|
|
970
|
+
'toildb.derives',
|
|
971
|
+
derivesSection([
|
|
972
|
+
[0, 'App', 'rebuild'],
|
|
973
|
+
[1, 'App', 'refresh'],
|
|
974
|
+
[2, 'Other', 'rollup'],
|
|
975
|
+
]),
|
|
976
|
+
),
|
|
977
|
+
);
|
|
978
|
+
// two App collections written in one dispatch -> App's two derives, once each
|
|
979
|
+
expect(derivesForWrites(derives, new Set(['App/users', 'App/posts'])).map(id)).toEqual([
|
|
980
|
+
0, 1,
|
|
981
|
+
]);
|
|
982
|
+
// a write to the other database -> only its derive
|
|
983
|
+
expect(derivesForWrites(derives, new Set(['Other/log'])).map(id)).toEqual([2]);
|
|
984
|
+
// a write to an unrelated database -> nothing runs
|
|
985
|
+
expect(derivesForWrites(derives, new Set(['Nope/x'])).map(id)).toEqual([]);
|
|
986
|
+
// no writes -> nothing runs
|
|
987
|
+
expect(derivesForWrites(derives, new Set<string>()).map(id)).toEqual([]);
|
|
988
|
+
});
|
|
989
|
+
});
|
|
@@ -20,8 +20,6 @@ class Account {
|
|
|
20
20
|
ids: u256[] = [];
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
//
|
|
24
|
-
@remote
|
|
25
|
-
|
|
26
|
-
return n;
|
|
27
|
-
}
|
|
23
|
+
// The two @data classes above ARE the surface: buildServerModule emits their codec + the bignum
|
|
24
|
+
// helpers for any @data. (No @remote here - a @remote injects a server-side Rpc registration that
|
|
25
|
+
// needs the full server runtime, which this `--runtime stub` fixture deliberately does not link.)
|