toiljs 0.0.67 → 0.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +63 -61
- package/build/backend/.tsbuildinfo +1 -1
- package/build/cli/.tsbuildinfo +1 -1
- package/build/cli/index.js +13 -1
- package/build/client/.tsbuildinfo +1 -1
- package/build/client/index.d.ts +2 -0
- package/build/client/index.js +1 -0
- package/build/client/rpc.js +21 -1
- package/build/client/stream/client.d.ts +11 -0
- package/build/client/stream/client.js +59 -0
- package/build/compiler/.tsbuildinfo +1 -1
- package/build/compiler/config.d.ts +2 -0
- package/build/compiler/config.js +9 -7
- package/build/compiler/index.d.ts +1 -0
- package/build/compiler/index.js +16 -2
- package/build/compiler/toil-docs.generated.js +5 -4
- package/build/devserver/.tsbuildinfo +1 -1
- package/build/devserver/daemon/runtime.d.ts +13 -0
- package/build/devserver/daemon/runtime.js +29 -0
- package/build/devserver/db/database.d.ts +1 -0
- package/build/devserver/db/database.js +10 -0
- package/build/devserver/db/derives.d.ts +7 -0
- package/build/devserver/db/derives.js +94 -0
- package/build/devserver/db/index.d.ts +1 -0
- package/build/devserver/db/index.js +1 -0
- package/build/devserver/db/types.d.ts +1 -0
- package/build/devserver/db/types.js +1 -0
- package/build/devserver/http/proxy.d.ts +5 -1
- package/build/devserver/http/proxy.js +39 -36
- package/build/devserver/http/runtime.d.ts +62 -0
- package/build/devserver/http/runtime.js +194 -0
- package/build/devserver/index.d.ts +2 -0
- package/build/devserver/index.js +1 -0
- package/build/devserver/production-ipc.d.ts +50 -0
- package/build/devserver/production-ipc.js +21 -0
- package/build/devserver/production-worker.d.ts +1 -0
- package/build/devserver/production-worker.js +73 -0
- package/build/devserver/production.d.ts +35 -0
- package/build/devserver/production.js +502 -0
- package/build/devserver/runtime/module.d.ts +5 -0
- package/build/devserver/runtime/module.js +47 -1
- package/build/devserver/server.d.ts +1 -0
- package/build/devserver/server.js +32 -145
- package/build/devserver/ssr.d.ts +2 -0
- package/build/devserver/ssr.js +19 -2
- package/build/devserver/stream/catalog.d.ts +20 -0
- package/build/devserver/stream/catalog.js +54 -0
- package/build/devserver/stream/host.d.ts +9 -0
- package/build/devserver/stream/host.js +15 -0
- package/build/devserver/stream/index.d.ts +37 -0
- package/build/devserver/stream/index.js +220 -0
- package/build/devserver/stream/manager.d.ts +34 -0
- package/build/devserver/stream/manager.js +103 -0
- package/build/devserver/stream/router.d.ts +25 -0
- package/build/devserver/stream/router.js +64 -0
- package/build/devserver/stream/wire.d.ts +5 -0
- package/build/devserver/stream/wire.js +33 -0
- package/build/devserver/stream/ws.d.ts +18 -0
- package/build/devserver/stream/ws.js +46 -0
- package/docs/cli.md +3 -1
- package/docs/derive.md +159 -0
- package/docs/getting-started.md +7 -7
- package/docs/index.md +1 -1
- package/docs/streams.md +46 -14
- package/examples/basic/server/routes/Guestbook.ts +38 -13
- package/package.json +2 -2
- package/src/cli/index.ts +14 -1
- package/src/client/index.ts +2 -0
- package/src/client/rpc.ts +25 -1
- package/src/client/stream/client.ts +109 -0
- package/src/compiler/config.ts +15 -7
- package/src/compiler/index.ts +24 -5
- package/src/compiler/toil-docs.generated.ts +5 -4
- package/src/devserver/daemon/runtime.ts +48 -0
- package/src/devserver/db/database.ts +14 -0
- package/src/devserver/db/derives.ts +121 -0
- package/src/devserver/db/index.ts +1 -0
- package/src/devserver/db/types.ts +6 -0
- package/src/devserver/http/proxy.ts +53 -39
- package/src/devserver/http/runtime.ts +287 -0
- package/src/devserver/index.ts +2 -0
- package/src/devserver/production-ipc.ts +63 -0
- package/src/devserver/production-worker.ts +83 -0
- package/src/devserver/production.ts +706 -0
- package/src/devserver/runtime/module.ts +95 -1
- package/src/devserver/server.ts +52 -201
- package/src/devserver/ssr.ts +23 -3
- package/src/devserver/stream/catalog.ts +106 -0
- package/src/devserver/stream/host.ts +42 -0
- package/src/devserver/stream/index.ts +308 -0
- package/src/devserver/stream/manager.ts +163 -0
- package/src/devserver/stream/router.ts +101 -0
- package/src/devserver/stream/wire.ts +58 -0
- package/src/devserver/stream/ws.ts +76 -0
- package/test/built-ssr.test.ts +98 -0
- package/test/devserver.test.ts +20 -4
- package/test/example-guestbook.test.ts +8 -5
- package/test/fixtures/stream-echo.ts +26 -0
- package/test/fixtures/stream-gate.ts +24 -0
- package/test/fixtures/stream-trap.ts +18 -0
- package/test/fixtures/stream-typed.ts +41 -0
- package/test/stream-emulation.test.ts +433 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { DevStreamBox } from './index.js';
|
|
3
|
+
export const STREAM_REJECTED = 0x0208;
|
|
4
|
+
export const STREAM_HOOK_TRAPPED = 0x0200;
|
|
5
|
+
export class StreamDevHost {
|
|
6
|
+
streamWasmPath;
|
|
7
|
+
bytes = null;
|
|
8
|
+
loadedMtimeMs = -1;
|
|
9
|
+
conns = new Map();
|
|
10
|
+
nextStreamId = 1n;
|
|
11
|
+
constructor(streamWasmPath) {
|
|
12
|
+
this.streamWasmPath = streamWasmPath;
|
|
13
|
+
}
|
|
14
|
+
get activeConnections() {
|
|
15
|
+
return this.conns.size;
|
|
16
|
+
}
|
|
17
|
+
has(connId) {
|
|
18
|
+
return this.conns.has(connId);
|
|
19
|
+
}
|
|
20
|
+
acceptUpgrade(connId, authority, path) {
|
|
21
|
+
if (this.conns.has(connId))
|
|
22
|
+
throw new Error(`stream connection '${connId}' is already open`);
|
|
23
|
+
this.refresh();
|
|
24
|
+
if (!this.bytes)
|
|
25
|
+
return { kind: 'rejected', code: STREAM_REJECTED };
|
|
26
|
+
let box;
|
|
27
|
+
try {
|
|
28
|
+
box = DevStreamBox.load(this.bytes);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return { kind: 'rejected', code: STREAM_REJECTED };
|
|
32
|
+
}
|
|
33
|
+
const streamId = this.allocStreamId();
|
|
34
|
+
let outcome;
|
|
35
|
+
try {
|
|
36
|
+
outcome = box.onConnect(streamId, authority, path);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return { kind: 'rejected', code: STREAM_HOOK_TRAPPED };
|
|
40
|
+
}
|
|
41
|
+
if (outcome.kind === 'reject')
|
|
42
|
+
return { kind: 'rejected', code: outcome.code };
|
|
43
|
+
this.conns.set(connId, { box, streamId });
|
|
44
|
+
return { kind: 'accepted', streamId };
|
|
45
|
+
}
|
|
46
|
+
dispatch(connId, inbound) {
|
|
47
|
+
const conn = this.conns.get(connId);
|
|
48
|
+
if (!conn)
|
|
49
|
+
return { kind: 'noConnection' };
|
|
50
|
+
try {
|
|
51
|
+
const out = conn.box.onMessage(conn.streamId, inbound);
|
|
52
|
+
if (out.kind === 'reply')
|
|
53
|
+
return { kind: 'reply', frames: out.frames };
|
|
54
|
+
return { kind: 'close', code: out.code };
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
this.conns.delete(connId);
|
|
58
|
+
return { kind: 'close', code: STREAM_HOOK_TRAPPED };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
close(connId) {
|
|
62
|
+
const conn = this.conns.get(connId);
|
|
63
|
+
if (!conn)
|
|
64
|
+
return;
|
|
65
|
+
try {
|
|
66
|
+
conn.box.onClose(conn.streamId);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
}
|
|
70
|
+
this.conns.delete(connId);
|
|
71
|
+
}
|
|
72
|
+
disconnect(connId) {
|
|
73
|
+
const conn = this.conns.get(connId);
|
|
74
|
+
if (!conn)
|
|
75
|
+
return;
|
|
76
|
+
try {
|
|
77
|
+
conn.box.onDisconnect(conn.streamId);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
}
|
|
81
|
+
this.conns.delete(connId);
|
|
82
|
+
}
|
|
83
|
+
refresh() {
|
|
84
|
+
let mtimeMs;
|
|
85
|
+
try {
|
|
86
|
+
mtimeMs = fs.statSync(this.streamWasmPath).mtimeMs;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
this.bytes = null;
|
|
90
|
+
this.loadedMtimeMs = -1;
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
if (mtimeMs === this.loadedMtimeMs && this.bytes)
|
|
94
|
+
return;
|
|
95
|
+
this.bytes = fs.readFileSync(this.streamWasmPath);
|
|
96
|
+
this.loadedMtimeMs = mtimeMs;
|
|
97
|
+
}
|
|
98
|
+
allocStreamId() {
|
|
99
|
+
const id = this.nextStreamId;
|
|
100
|
+
this.nextStreamId = id + 1n;
|
|
101
|
+
return id;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type StreamDef } from './catalog.js';
|
|
2
|
+
export interface StreamWs {
|
|
3
|
+
send(data: Buffer, isBinary: boolean): void;
|
|
4
|
+
close(code: number): void;
|
|
5
|
+
on(event: 'message', cb: (message: Buffer, isBinary: boolean) => void): void;
|
|
6
|
+
on(event: 'close', cb: () => void): void;
|
|
7
|
+
}
|
|
8
|
+
export interface StreamUpgradeContext {
|
|
9
|
+
readonly kind: 'stream';
|
|
10
|
+
readonly route: string;
|
|
11
|
+
readonly url: string;
|
|
12
|
+
readonly authority: string;
|
|
13
|
+
}
|
|
14
|
+
export declare class StreamRouter {
|
|
15
|
+
private readonly streamWasmPath;
|
|
16
|
+
private catalog;
|
|
17
|
+
private catalogMtimeMs;
|
|
18
|
+
private readonly host;
|
|
19
|
+
private connSeq;
|
|
20
|
+
constructor(streamWasmPath: string);
|
|
21
|
+
get activeConnections(): number;
|
|
22
|
+
matchRoute(path: string): StreamDef | null;
|
|
23
|
+
onUpgrade(ws: StreamWs, ctx: StreamUpgradeContext): void;
|
|
24
|
+
private refreshCatalog;
|
|
25
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { matchStreamRoute, parseStreamCatalog, } from './catalog.js';
|
|
3
|
+
import { StreamDevHost } from './manager.js';
|
|
4
|
+
import { StreamWsSession } from './ws.js';
|
|
5
|
+
export class StreamRouter {
|
|
6
|
+
streamWasmPath;
|
|
7
|
+
catalog = new Map();
|
|
8
|
+
catalogMtimeMs = -1;
|
|
9
|
+
host;
|
|
10
|
+
connSeq = 0;
|
|
11
|
+
constructor(streamWasmPath) {
|
|
12
|
+
this.streamWasmPath = streamWasmPath;
|
|
13
|
+
this.host = new StreamDevHost(streamWasmPath);
|
|
14
|
+
this.refreshCatalog();
|
|
15
|
+
}
|
|
16
|
+
get activeConnections() {
|
|
17
|
+
return this.host.activeConnections;
|
|
18
|
+
}
|
|
19
|
+
matchRoute(path) {
|
|
20
|
+
this.refreshCatalog();
|
|
21
|
+
return matchStreamRoute(this.catalog, path);
|
|
22
|
+
}
|
|
23
|
+
onUpgrade(ws, ctx) {
|
|
24
|
+
const connId = `s${String(++this.connSeq)}`;
|
|
25
|
+
const q = ctx.url.indexOf('?');
|
|
26
|
+
const path = q >= 0 ? ctx.url.slice(0, q) : ctx.url;
|
|
27
|
+
const session = new StreamWsSession(this.host, connId, ctx.authority, path, {
|
|
28
|
+
send: (frame) => {
|
|
29
|
+
ws.send(frame, true);
|
|
30
|
+
},
|
|
31
|
+
close: (code) => {
|
|
32
|
+
ws.close(code);
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
if (!session.onOpen())
|
|
36
|
+
return;
|
|
37
|
+
ws.on('message', (message) => {
|
|
38
|
+
session.onMessage(message);
|
|
39
|
+
});
|
|
40
|
+
ws.on('close', () => {
|
|
41
|
+
session.onClose();
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
refreshCatalog() {
|
|
45
|
+
let mtimeMs;
|
|
46
|
+
try {
|
|
47
|
+
mtimeMs = fs.statSync(this.streamWasmPath).mtimeMs;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
this.catalog = new Map();
|
|
51
|
+
this.catalogMtimeMs = -1;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (mtimeMs === this.catalogMtimeMs && this.catalog.size > 0)
|
|
55
|
+
return;
|
|
56
|
+
try {
|
|
57
|
+
this.catalog = parseStreamCatalog(fs.readFileSync(this.streamWasmPath));
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
this.catalog = new Map();
|
|
61
|
+
}
|
|
62
|
+
this.catalogMtimeMs = mtimeMs;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type Server } from '@dacely/hyper-express';
|
|
2
|
+
import { type ViteTarget } from '../http/proxy.js';
|
|
3
|
+
import { type StreamRouter } from './router.js';
|
|
4
|
+
export declare function streamEmulationEnabled(nodeMode: string): boolean;
|
|
5
|
+
export declare function wireStreams(app: Server, vite: ViteTarget, router: StreamRouter): void;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { pipeToVite } from '../http/proxy.js';
|
|
2
|
+
export function streamEmulationEnabled(nodeMode) {
|
|
3
|
+
return nodeMode === 'regional' || nodeMode === 'continental' || nodeMode === 'all';
|
|
4
|
+
}
|
|
5
|
+
export function wireStreams(app, vite, router) {
|
|
6
|
+
app.upgrade('/*', (request, response) => {
|
|
7
|
+
const def = router.matchRoute(request.path);
|
|
8
|
+
if (def !== null) {
|
|
9
|
+
response.upgrade({
|
|
10
|
+
kind: 'stream',
|
|
11
|
+
route: def.route,
|
|
12
|
+
url: request.url,
|
|
13
|
+
authority: request.headers['host'] ?? '',
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
response.upgrade({
|
|
18
|
+
kind: 'vite',
|
|
19
|
+
url: request.url,
|
|
20
|
+
protocol: request.headers['sec-websocket-protocol'] ?? '',
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
app.ws('/*', { message_type: 'Buffer', idle_timeout: 0, max_payload_length: 16 * 1024 * 1024 }, (ws) => {
|
|
25
|
+
const ctx = ws.context;
|
|
26
|
+
if (ctx.kind === 'stream') {
|
|
27
|
+
router.onUpgrade(ws, ws.context);
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
pipeToVite(ws, vite, ws.context);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { StreamDevHost } from './manager.js';
|
|
2
|
+
export interface StreamWsTransport {
|
|
3
|
+
send(frame: Buffer): void;
|
|
4
|
+
close(code: number): void;
|
|
5
|
+
}
|
|
6
|
+
export declare class StreamWsSession {
|
|
7
|
+
private readonly host;
|
|
8
|
+
private readonly connId;
|
|
9
|
+
private readonly authority;
|
|
10
|
+
private readonly path;
|
|
11
|
+
private readonly transport;
|
|
12
|
+
private open;
|
|
13
|
+
constructor(host: StreamDevHost, connId: string, authority: string, path: string, transport: StreamWsTransport);
|
|
14
|
+
get isOpen(): boolean;
|
|
15
|
+
onOpen(): boolean;
|
|
16
|
+
onMessage(inbound: Buffer): void;
|
|
17
|
+
onClose(): void;
|
|
18
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export class StreamWsSession {
|
|
2
|
+
host;
|
|
3
|
+
connId;
|
|
4
|
+
authority;
|
|
5
|
+
path;
|
|
6
|
+
transport;
|
|
7
|
+
open = false;
|
|
8
|
+
constructor(host, connId, authority, path, transport) {
|
|
9
|
+
this.host = host;
|
|
10
|
+
this.connId = connId;
|
|
11
|
+
this.authority = authority;
|
|
12
|
+
this.path = path;
|
|
13
|
+
this.transport = transport;
|
|
14
|
+
}
|
|
15
|
+
get isOpen() {
|
|
16
|
+
return this.open;
|
|
17
|
+
}
|
|
18
|
+
onOpen() {
|
|
19
|
+
const up = this.host.acceptUpgrade(this.connId, this.authority, this.path);
|
|
20
|
+
if (up.kind === 'rejected') {
|
|
21
|
+
this.transport.close(up.code);
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
this.open = true;
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
onMessage(inbound) {
|
|
28
|
+
if (!this.open)
|
|
29
|
+
return;
|
|
30
|
+
const r = this.host.dispatch(this.connId, inbound);
|
|
31
|
+
if (r.kind === 'reply') {
|
|
32
|
+
for (const frame of r.frames)
|
|
33
|
+
this.transport.send(frame);
|
|
34
|
+
}
|
|
35
|
+
else if (r.kind === 'close') {
|
|
36
|
+
this.open = false;
|
|
37
|
+
this.transport.close(r.code);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
onClose() {
|
|
41
|
+
if (!this.open && !this.host.has(this.connId))
|
|
42
|
+
return;
|
|
43
|
+
this.open = false;
|
|
44
|
+
this.host.close(this.connId);
|
|
45
|
+
}
|
|
46
|
+
}
|
package/docs/cli.md
CHANGED
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
- `toiljs build`, production build. With a `toilconfig.json` it builds the server (toilscript,
|
|
9
9
|
regenerating `shared/server.ts`) first, then the client → `build/client`. `--server` builds
|
|
10
10
|
only the server. Every `server/` file declaring a surface (`@data`/`@rest`/...) is compiled.
|
|
11
|
-
- `toiljs start`, self-host the built app
|
|
11
|
+
- `toiljs start`, self-host the built app with production hyper-express/uWS static workers,
|
|
12
|
+
SSR/wasm dispatch, daemon support, and a `/_toil` WebSocket channel. Use `--threads <n>`
|
|
13
|
+
(or `server.threads`) to set the worker count; `1` disables the pool.
|
|
12
14
|
- `toiljs configure`, toggle styling features on an existing project (see [styling.md](./styling.md)).
|
|
13
15
|
- `toiljs doctor`, diagnose project setup (`--json` for CI). `--fix` auto-wires the typed-RPC
|
|
14
16
|
setup (build scripts, tsconfig `shared` + alias, `.gitignore`, toilscript version, and the
|
package/docs/derive.md
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Derive (materialized views)
|
|
2
|
+
|
|
3
|
+
`@derive` precomputes a read-optimized **view** from your data so reads stay
|
|
4
|
+
fast and never scan. A request handler (`@get` runs as a *query*, `@post`/`@put`/
|
|
5
|
+
`@delete` as an *action*) is not allowed to scan, reading "the latest N events"
|
|
6
|
+
or "every member of a set" could fan out across unbounded rows, so those scans
|
|
7
|
+
are barred on the request path. A `@derive` does the scan **off** the request
|
|
8
|
+
path: it folds your event log / counters into a `View`, and your route serves
|
|
9
|
+
that view with a single keyed read.
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
@database
|
|
13
|
+
class GuestbookDb {
|
|
14
|
+
@collection static entries: Events<GuestKey, GuestEntry>;
|
|
15
|
+
@collection static totals: Counter<GuestKey>;
|
|
16
|
+
@collection static book: View<GuestKey, GuestbookView>;
|
|
17
|
+
|
|
18
|
+
// Recompute the view from the sources. Runs after a signature is written
|
|
19
|
+
// (and when a box first loads). A derive MAY scan + publish; a route may not.
|
|
20
|
+
@derive
|
|
21
|
+
recompute(): void {
|
|
22
|
+
const key = new GuestKey('main');
|
|
23
|
+
const view = new GuestbookView();
|
|
24
|
+
view.total = GuestbookDb.totals.get(key); // counter read
|
|
25
|
+
view.entries = GuestbookDb.entries.latest(key, 10); // scan, allowed here
|
|
26
|
+
GuestbookDb.book.publish(key, view); // publish the materialized view
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Why a derive
|
|
32
|
+
|
|
33
|
+
ToilDB gates every data op by the *function kind* it runs under:
|
|
34
|
+
|
|
35
|
+
- **query** (`@get`/`@head`) and **action** (`@post`/`@put`/`@patch`/`@delete`)
|
|
36
|
+
may do keyed reads and (actions only) writes, but **not scans**
|
|
37
|
+
(`events.latest`, `membership.list`).
|
|
38
|
+
- **derive** may do everything a read can, **plus** scans, plus
|
|
39
|
+
`view.publish`/`append`/`counter.add`.
|
|
40
|
+
|
|
41
|
+
So if a page needs "the 10 newest entries" or "the leaderboard", you cannot read
|
|
42
|
+
that directly in the `@get`. Instead a `@derive` builds it once into a `View`,
|
|
43
|
+
and the `@get` reads the view by key, which is not a scan.
|
|
44
|
+
|
|
45
|
+
## Declaring a derive
|
|
46
|
+
|
|
47
|
+
A derive is a method on your `@database` class, alongside the collections it
|
|
48
|
+
reads and the `View` it writes:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
@database
|
|
52
|
+
class MyDb {
|
|
53
|
+
@collection static events: Events<Key, Fact>; // a source
|
|
54
|
+
@collection static home: View<Key, HomePage>; // the materialized view
|
|
55
|
+
|
|
56
|
+
@derive
|
|
57
|
+
rebuild(): void {
|
|
58
|
+
// read sources, build the value, publish it
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Rules:
|
|
64
|
+
|
|
65
|
+
- A `@derive` method takes **no arguments and returns `void`**.
|
|
66
|
+
- A `@database` may declare **multiple** `@derive` methods; each is run
|
|
67
|
+
independently.
|
|
68
|
+
- The view value (`HomePage` above) and the key are ordinary `@data` types, so
|
|
69
|
+
they round-trip through the codec like any other stored value.
|
|
70
|
+
|
|
71
|
+
## `View<K, V>`
|
|
72
|
+
|
|
73
|
+
A `View` is a published, read-optimized projection. Its API:
|
|
74
|
+
|
|
75
|
+
```ts
|
|
76
|
+
view.get(key) // V | null - the published view, or null if none yet
|
|
77
|
+
view.require(key) // V - like get, but traps if nothing is published
|
|
78
|
+
view.publish(key, value) // void - overwrite the view (derive/job only)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`publish` is only allowed from a `@derive` (or a `@job`); the host assigns the
|
|
82
|
+
version so a later publish always supersedes an earlier one. `get`/`require` are
|
|
83
|
+
plain keyed reads, allowed from any handler, including a `@get` route.
|
|
84
|
+
|
|
85
|
+
## When derives run
|
|
86
|
+
|
|
87
|
+
You never call a derive yourself. The runtime runs it for you:
|
|
88
|
+
|
|
89
|
+
- **After a write to a source.** When a request writes one of a database's
|
|
90
|
+
source collections (an `events.append`/`append_once`, a `counter.add`, or a
|
|
91
|
+
record `create`/`patch`), that database's derives run right after the response
|
|
92
|
+
is produced, so the view reflects the new data on the next read. Many writes to
|
|
93
|
+
one database in a single request coalesce into one recompute.
|
|
94
|
+
- **On box load.** When a server box starts or hot-reloads (or the underlying
|
|
95
|
+
source data changed out of band), the views are rebuilt from their sources
|
|
96
|
+
before the first read is served. This is also where a value type's `@migrate`
|
|
97
|
+
runs against old stored events, as the derive re-reads and republishes them.
|
|
98
|
+
|
|
99
|
+
A derive's own writes (its `view.publish`) never re-trigger it.
|
|
100
|
+
|
|
101
|
+
The same code runs under `toiljs dev` (the in-process emulator) and on the
|
|
102
|
+
production edge, no flags or wiring to change.
|
|
103
|
+
|
|
104
|
+
## Reading a view from a route
|
|
105
|
+
|
|
106
|
+
The route just reads the view by key, which is a non-scan read and so is legal in
|
|
107
|
+
a `@get`:
|
|
108
|
+
|
|
109
|
+
```ts
|
|
110
|
+
@rest('guestbook')
|
|
111
|
+
class Guestbook {
|
|
112
|
+
@get('/')
|
|
113
|
+
list(): GuestbookView {
|
|
114
|
+
const key = new GuestKey('main');
|
|
115
|
+
const view = GuestbookDb.book.get(key);
|
|
116
|
+
return view == null ? new GuestbookView() : view; // empty until first publish
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
@post('/')
|
|
120
|
+
sign(input: NewMessage): GuestbookView {
|
|
121
|
+
const key = new GuestKey('main');
|
|
122
|
+
GuestbookDb.entries.append(key, new GuestEntry(input.author, input.message, 0));
|
|
123
|
+
GuestbookDb.totals.add(key, 1);
|
|
124
|
+
// The @derive republishes `book` right after this action returns, so the
|
|
125
|
+
// entries list is served by GET. The action just acks with the new total
|
|
126
|
+
// (a counter read is allowed here; a scan is not).
|
|
127
|
+
const view = new GuestbookView();
|
|
128
|
+
view.total = GuestbookDb.totals.get(key);
|
|
129
|
+
return view;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## How it fits together (the guestbook)
|
|
135
|
+
|
|
136
|
+
The `examples/basic` guestbook is the end-to-end demo:
|
|
137
|
+
|
|
138
|
+
1. `POST /guestbook` (an action) appends the signature to an `Events` stream and
|
|
139
|
+
bumps a `Counter`. It returns the running total, but it does **not** read the
|
|
140
|
+
entry list (that would be a scan).
|
|
141
|
+
2. The runtime then runs `@derive recompute()` under the derive kind: it scans
|
|
142
|
+
`entries.latest(...)`, reads the `totals` counter, and `publish`es a fresh
|
|
143
|
+
`GuestbookView`.
|
|
144
|
+
3. `GET /guestbook` (a query) reads `book.get(...)`, a single keyed read, and
|
|
145
|
+
returns the precomputed total + newest entries.
|
|
146
|
+
|
|
147
|
+
Sign twice and the total climbs across requests, because the data lives in
|
|
148
|
+
ToilDB (and its view), not in module memory.
|
|
149
|
+
|
|
150
|
+
## Notes
|
|
151
|
+
|
|
152
|
+
- A derive **recomputes** the view from whatever its method reads (here, the
|
|
153
|
+
latest 10 events). It is a fresh recompute on each trigger, so it suits views
|
|
154
|
+
built from a bounded read (latest N, a counter total, a small set). Folding an
|
|
155
|
+
unbounded full event log incrementally is a separate, more advanced pattern.
|
|
156
|
+
- Because publishes are last-writer-wins and a derive recomputes from the source
|
|
157
|
+
of truth, a view always converges to a correct snapshot of its sources.
|
|
158
|
+
- See also: [`data.md`](data.md) for `@data` value types, and the ToilDB host
|
|
159
|
+
ABI for the exact `derive_run` / `toildb.derives` contract.
|
package/docs/getting-started.md
CHANGED
|
@@ -95,13 +95,13 @@ store reached through a host binding.
|
|
|
95
95
|
|
|
96
96
|
The `toiljs` CLI drives both halves:
|
|
97
97
|
|
|
98
|
-
| Command
|
|
99
|
-
|
|
|
100
|
-
| `toiljs create [name]` | Scaffold a new app (templates, styling, options).
|
|
101
|
-
| `toiljs dev`
|
|
102
|
-
| `toiljs build`
|
|
103
|
-
| `toiljs start`
|
|
104
|
-
| `toiljs doctor`
|
|
98
|
+
| Command | What it does |
|
|
99
|
+
| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
100
|
+
| `toiljs create [name]` | Scaffold a new app (templates, styling, options). |
|
|
101
|
+
| `toiljs dev` | Dev server with hot reload: watches `server/`, rebuilds the wasm via toilscript, regenerates `shared/server.ts`, and runs Vite for the client. Flags: `--root <dir>`, `--port <n>`, `--host`. |
|
|
102
|
+
| `toiljs build` | Production build: server wasm first (so `shared/server.ts` is fresh), then the Vite client + static prerender. Flags: `--root <dir>`, `--server` (server only). |
|
|
103
|
+
| `toiljs start` | Self-host a built app with production uWS/static workers, no Vite. Flags: `--root`, `--port`, `--host`, `--threads`. |
|
|
104
|
+
| `toiljs doctor` | Diagnose setup/deps (`--json`, `--fix`). |
|
|
105
105
|
|
|
106
106
|
In dev, requests whose method matches a dispatchable verb go into the wasm
|
|
107
107
|
first; if the guest reports "no route matched" (the `x-toil-unhandled` marker)
|
package/docs/index.md
CHANGED
|
@@ -27,4 +27,4 @@ toilscript-to-WebAssembly server target.
|
|
|
27
27
|
|
|
28
28
|
See [routing.md](./routing.md), [client.md](./client.md), [styling.md](./styling.md),
|
|
29
29
|
[server.md](./server.md), [ssr.md](./ssr.md), [rpc.md](./rpc.md), [tiers.md](./tiers.md),
|
|
30
|
-
[streams.md](./streams.md), [daemon.md](./daemon.md), [cli.md](./cli.md).
|
|
30
|
+
[streams.md](./streams.md), [daemon.md](./daemon.md), [derive.md](./derive.md), [cli.md](./cli.md).
|
package/docs/streams.md
CHANGED
|
@@ -117,30 +117,62 @@ build/server/release-cold.wasm # L4 daemon (exports: daemon_start, sched
|
|
|
117
117
|
|
|
118
118
|
See [Tiers](./tiers.md) for how the three artifacts map to the deployment tiers.
|
|
119
119
|
|
|
120
|
-
##
|
|
120
|
+
## Reading and replying to messages
|
|
121
121
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
yet available. That bridge is the `StreamPacket` / `StreamOutbound` API and the
|
|
128
|
-
typed `Server.STREAM.echo.connect()` client. The intended shape, once it lands:
|
|
122
|
+
`@message` receives the inbound frame as a `StreamPacket` and returns a
|
|
123
|
+
`StreamOutbound`. `StreamPacket.bytes()` is the raw frame payload;
|
|
124
|
+
`StreamOutbound.reply(bytes)` stages one frame back to the client (return an empty
|
|
125
|
+
`StreamOutbound` to accept the frame without replying). The same resident box
|
|
126
|
+
handles every frame, so state on its fields persists across messages.
|
|
129
127
|
|
|
130
128
|
```ts
|
|
131
|
-
@message
|
|
129
|
+
@message
|
|
130
|
+
reply(packet: StreamPacket): StreamOutbound {
|
|
132
131
|
return StreamOutbound.reply(packet.bytes()); // echo the bytes back
|
|
133
132
|
}
|
|
134
133
|
```
|
|
135
134
|
|
|
135
|
+
## Typed messages
|
|
136
|
+
|
|
137
|
+
By default a `@message` payload is **raw bytes**. Opt into a decoded `@data` value
|
|
138
|
+
with `@stream({ message: T })`: the `@message` hook then receives the named `@data`
|
|
139
|
+
class, decoded from the frame for you. The reply stays raw (`StreamOutbound`).
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
@data
|
|
143
|
+
class ChatMsg { text: string = ''; }
|
|
144
|
+
|
|
145
|
+
@stream({ message: ChatMsg })
|
|
146
|
+
class Chat {
|
|
147
|
+
@message
|
|
148
|
+
onMessage(msg: ChatMsg): StreamOutbound { // decoded @data, not raw bytes
|
|
149
|
+
return StreamOutbound.reply(new TextEncoder().encode(msg.text));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## The client
|
|
155
|
+
|
|
156
|
+
A `@stream` class is reachable from the browser as `Server.Stream.<ClassName>`. The
|
|
157
|
+
typed client is generated into `shared/server.ts` (the same place `Server.REST`
|
|
158
|
+
lands), so no manual wiring is needed. `connect()` opens a WebSocket to the class's
|
|
159
|
+
route and resolves a channel:
|
|
160
|
+
|
|
136
161
|
```ts
|
|
137
|
-
const
|
|
138
|
-
|
|
162
|
+
const chat = await Server.Stream.Chat.connect();
|
|
163
|
+
chat.onMessage((bytes) => { /* a reply frame, always raw bytes */ });
|
|
164
|
+
chat.send(new ChatMsg('hello')); // a typed stream: send() encodes the @data for you
|
|
165
|
+
chat.onClose((code) => { /* a 0x02xx stream close code */ });
|
|
166
|
+
chat.close();
|
|
139
167
|
```
|
|
140
168
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
`
|
|
169
|
+
- The channel key is the **class name** (`Server.Stream.Chat`); it connects to the
|
|
170
|
+
class's mount route (`/Chat`).
|
|
171
|
+
- A **raw** `@stream` channel sends `Uint8Array`; a **typed** `@stream({ message: T })`
|
|
172
|
+
channel sends the `@data` class and encodes it on the wire for you.
|
|
173
|
+
- The inbound reply is **always raw bytes** - the server's `StreamOutbound` is raw.
|
|
174
|
+
- `connect()` resolves once the upgrade completes; a `@connect` reject (or any
|
|
175
|
+
later server close) surfaces through `onClose(code)`.
|
|
144
176
|
|
|
145
177
|
---
|
|
146
178
|
|
|
@@ -13,6 +13,12 @@ import { NewMessage } from '../models/NewMessage';
|
|
|
13
13
|
*
|
|
14
14
|
* await Server.REST.guestbook.sign({ body: new NewMessage('Ada', 'hi!') });
|
|
15
15
|
* const book = await Server.REST.guestbook.list(); // { total, entries: [...] }
|
|
16
|
+
*
|
|
17
|
+
* Reading the newest entries is a SCAN (`events.latest`), which is barred in a
|
|
18
|
+
* request handler (a `@get` runs as a Query, a `@post` as an Action) because a
|
|
19
|
+
* scan can fan out across unbounded rows. So a `@derive` does the scan off the
|
|
20
|
+
* request path and `publish`es a materialized `GuestbookView`; the GET then
|
|
21
|
+
* serves that view with a single non-scan `view.get`.
|
|
16
22
|
*/
|
|
17
23
|
|
|
18
24
|
// The guestbook is one global stream; a single fixed key addresses it.
|
|
@@ -28,33 +34,52 @@ class GuestKey {
|
|
|
28
34
|
class GuestbookDb {
|
|
29
35
|
@collection static entries: Events<GuestKey, GuestEntry>;
|
|
30
36
|
@collection static totals: Counter<GuestKey>;
|
|
31
|
-
|
|
37
|
+
// The materialized snapshot the GET serves: total + newest entries.
|
|
38
|
+
@collection static book: View<GuestKey, GuestbookView>;
|
|
32
39
|
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Recompute the materialized view from the source of truth (the event log +
|
|
42
|
+
* the counter). The runtime runs this under FunctionKind=Derive after a
|
|
43
|
+
* signature is appended (and rebuilds it when a box first loads), so the
|
|
44
|
+
* scan (`events.latest`) and the `view.publish` - both barred in a request
|
|
45
|
+
* handler - are allowed here. This is also where `GuestEntry`'s `@migrate`
|
|
46
|
+
* fires: `events.latest` decodes each stored event at ITS schema version, so
|
|
47
|
+
* an old pre-`at` entry is migrated as the view is rebuilt.
|
|
48
|
+
*/
|
|
49
|
+
@derive
|
|
50
|
+
recompute(): void {
|
|
51
|
+
const key = new GuestKey('main');
|
|
52
|
+
const view = new GuestbookView();
|
|
53
|
+
view.total = GuestbookDb.totals.get(key);
|
|
54
|
+
view.entries = GuestbookDb.entries.latest(key, 10);
|
|
55
|
+
GuestbookDb.book.publish(key, view);
|
|
56
|
+
}
|
|
40
57
|
}
|
|
41
58
|
|
|
42
59
|
@rest('guestbook')
|
|
43
60
|
class Guestbook {
|
|
44
|
-
/** `GET /guestbook` - the running total + the most recent signatures
|
|
61
|
+
/** `GET /guestbook` - the running total + the most recent signatures, served
|
|
62
|
+
* from the materialized view (a non-scan `view.get`). */
|
|
45
63
|
@get('/')
|
|
46
64
|
public list(): GuestbookView {
|
|
47
|
-
|
|
65
|
+
const key = new GuestKey('main');
|
|
66
|
+
const view = GuestbookDb.book.get(key);
|
|
67
|
+
if (view == null) return new GuestbookView();
|
|
68
|
+
return view;
|
|
48
69
|
}
|
|
49
70
|
|
|
50
|
-
/** `POST /guestbook` - append a signature (PERSISTED) and
|
|
51
|
-
*
|
|
71
|
+
/** `POST /guestbook` - append a signature (PERSISTED) and acknowledge with
|
|
72
|
+
* the new running total. The entries list is served by the GET above from
|
|
73
|
+
* the view the `@derive` republishes right after this action. Sign twice
|
|
74
|
+
* and the total keeps climbing across requests. */
|
|
52
75
|
@post('/')
|
|
53
76
|
public sign(input: NewMessage): GuestbookView {
|
|
54
77
|
const key = new GuestKey('main');
|
|
55
78
|
const at = <u64>(Date.now() / 1000);
|
|
56
79
|
GuestbookDb.entries.append(key, new GuestEntry(input.author, input.message, at));
|
|
57
80
|
GuestbookDb.totals.add(key, 1);
|
|
58
|
-
|
|
81
|
+
const view = new GuestbookView();
|
|
82
|
+
view.total = GuestbookDb.totals.get(key); // Counter get: non-scan, action-legal
|
|
83
|
+
return view;
|
|
59
84
|
}
|
|
60
85
|
}
|