toiljs 0.0.67 → 0.0.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/README.md +63 -61
  3. package/build/backend/.tsbuildinfo +1 -1
  4. package/build/cli/.tsbuildinfo +1 -1
  5. package/build/cli/index.js +13 -1
  6. package/build/client/.tsbuildinfo +1 -1
  7. package/build/client/index.d.ts +2 -0
  8. package/build/client/index.js +1 -0
  9. package/build/client/rpc.js +21 -1
  10. package/build/client/stream/client.d.ts +11 -0
  11. package/build/client/stream/client.js +59 -0
  12. package/build/compiler/.tsbuildinfo +1 -1
  13. package/build/compiler/config.d.ts +2 -0
  14. package/build/compiler/config.js +9 -7
  15. package/build/compiler/index.d.ts +1 -0
  16. package/build/compiler/index.js +16 -2
  17. package/build/compiler/toil-docs.generated.js +5 -4
  18. package/build/devserver/.tsbuildinfo +1 -1
  19. package/build/devserver/daemon/runtime.d.ts +13 -0
  20. package/build/devserver/daemon/runtime.js +29 -0
  21. package/build/devserver/db/database.d.ts +1 -0
  22. package/build/devserver/db/database.js +10 -0
  23. package/build/devserver/db/derives.d.ts +7 -0
  24. package/build/devserver/db/derives.js +94 -0
  25. package/build/devserver/db/index.d.ts +1 -0
  26. package/build/devserver/db/index.js +1 -0
  27. package/build/devserver/db/types.d.ts +1 -0
  28. package/build/devserver/db/types.js +1 -0
  29. package/build/devserver/http/proxy.d.ts +5 -1
  30. package/build/devserver/http/proxy.js +39 -36
  31. package/build/devserver/http/runtime.d.ts +62 -0
  32. package/build/devserver/http/runtime.js +194 -0
  33. package/build/devserver/index.d.ts +2 -0
  34. package/build/devserver/index.js +1 -0
  35. package/build/devserver/production-ipc.d.ts +50 -0
  36. package/build/devserver/production-ipc.js +21 -0
  37. package/build/devserver/production-worker.d.ts +1 -0
  38. package/build/devserver/production-worker.js +73 -0
  39. package/build/devserver/production.d.ts +35 -0
  40. package/build/devserver/production.js +502 -0
  41. package/build/devserver/runtime/module.d.ts +5 -0
  42. package/build/devserver/runtime/module.js +47 -1
  43. package/build/devserver/server.d.ts +1 -0
  44. package/build/devserver/server.js +32 -145
  45. package/build/devserver/ssr.d.ts +2 -0
  46. package/build/devserver/ssr.js +19 -2
  47. package/build/devserver/stream/catalog.d.ts +20 -0
  48. package/build/devserver/stream/catalog.js +54 -0
  49. package/build/devserver/stream/host.d.ts +9 -0
  50. package/build/devserver/stream/host.js +15 -0
  51. package/build/devserver/stream/index.d.ts +37 -0
  52. package/build/devserver/stream/index.js +220 -0
  53. package/build/devserver/stream/manager.d.ts +34 -0
  54. package/build/devserver/stream/manager.js +103 -0
  55. package/build/devserver/stream/router.d.ts +25 -0
  56. package/build/devserver/stream/router.js +64 -0
  57. package/build/devserver/stream/wire.d.ts +5 -0
  58. package/build/devserver/stream/wire.js +33 -0
  59. package/build/devserver/stream/ws.d.ts +18 -0
  60. package/build/devserver/stream/ws.js +46 -0
  61. package/docs/cli.md +3 -1
  62. package/docs/derive.md +159 -0
  63. package/docs/getting-started.md +7 -7
  64. package/docs/index.md +1 -1
  65. package/docs/streams.md +46 -14
  66. package/examples/basic/server/routes/Guestbook.ts +38 -13
  67. package/package.json +2 -2
  68. package/src/cli/index.ts +14 -1
  69. package/src/client/index.ts +2 -0
  70. package/src/client/rpc.ts +25 -1
  71. package/src/client/stream/client.ts +109 -0
  72. package/src/compiler/config.ts +15 -7
  73. package/src/compiler/index.ts +24 -5
  74. package/src/compiler/toil-docs.generated.ts +5 -4
  75. package/src/devserver/daemon/runtime.ts +48 -0
  76. package/src/devserver/db/database.ts +14 -0
  77. package/src/devserver/db/derives.ts +121 -0
  78. package/src/devserver/db/index.ts +1 -0
  79. package/src/devserver/db/types.ts +6 -0
  80. package/src/devserver/http/proxy.ts +53 -39
  81. package/src/devserver/http/runtime.ts +287 -0
  82. package/src/devserver/index.ts +2 -0
  83. package/src/devserver/production-ipc.ts +63 -0
  84. package/src/devserver/production-worker.ts +83 -0
  85. package/src/devserver/production.ts +706 -0
  86. package/src/devserver/runtime/module.ts +95 -1
  87. package/src/devserver/server.ts +52 -201
  88. package/src/devserver/ssr.ts +23 -3
  89. package/src/devserver/stream/catalog.ts +106 -0
  90. package/src/devserver/stream/host.ts +42 -0
  91. package/src/devserver/stream/index.ts +308 -0
  92. package/src/devserver/stream/manager.ts +163 -0
  93. package/src/devserver/stream/router.ts +101 -0
  94. package/src/devserver/stream/wire.ts +58 -0
  95. package/src/devserver/stream/ws.ts +76 -0
  96. package/test/built-ssr.test.ts +98 -0
  97. package/test/devserver.test.ts +20 -4
  98. package/test/example-guestbook.test.ts +8 -5
  99. package/test/fixtures/stream-echo.ts +26 -0
  100. package/test/fixtures/stream-gate.ts +24 -0
  101. package/test/fixtures/stream-trap.ts +18 -0
  102. package/test/fixtures/stream-typed.ts +41 -0
  103. package/test/stream-emulation.test.ts +433 -0
@@ -0,0 +1,103 @@
1
+ import fs from 'node:fs';
2
+ import { DevStreamBox } from './index.js';
3
+ export const STREAM_REJECTED = 0x0208;
4
+ export const STREAM_HOOK_TRAPPED = 0x0200;
5
+ export class StreamDevHost {
6
+ streamWasmPath;
7
+ bytes = null;
8
+ loadedMtimeMs = -1;
9
+ conns = new Map();
10
+ nextStreamId = 1n;
11
+ constructor(streamWasmPath) {
12
+ this.streamWasmPath = streamWasmPath;
13
+ }
14
+ get activeConnections() {
15
+ return this.conns.size;
16
+ }
17
+ has(connId) {
18
+ return this.conns.has(connId);
19
+ }
20
+ acceptUpgrade(connId, authority, path) {
21
+ if (this.conns.has(connId))
22
+ throw new Error(`stream connection '${connId}' is already open`);
23
+ this.refresh();
24
+ if (!this.bytes)
25
+ return { kind: 'rejected', code: STREAM_REJECTED };
26
+ let box;
27
+ try {
28
+ box = DevStreamBox.load(this.bytes);
29
+ }
30
+ catch {
31
+ return { kind: 'rejected', code: STREAM_REJECTED };
32
+ }
33
+ const streamId = this.allocStreamId();
34
+ let outcome;
35
+ try {
36
+ outcome = box.onConnect(streamId, authority, path);
37
+ }
38
+ catch {
39
+ return { kind: 'rejected', code: STREAM_HOOK_TRAPPED };
40
+ }
41
+ if (outcome.kind === 'reject')
42
+ return { kind: 'rejected', code: outcome.code };
43
+ this.conns.set(connId, { box, streamId });
44
+ return { kind: 'accepted', streamId };
45
+ }
46
+ dispatch(connId, inbound) {
47
+ const conn = this.conns.get(connId);
48
+ if (!conn)
49
+ return { kind: 'noConnection' };
50
+ try {
51
+ const out = conn.box.onMessage(conn.streamId, inbound);
52
+ if (out.kind === 'reply')
53
+ return { kind: 'reply', frames: out.frames };
54
+ return { kind: 'close', code: out.code };
55
+ }
56
+ catch {
57
+ this.conns.delete(connId);
58
+ return { kind: 'close', code: STREAM_HOOK_TRAPPED };
59
+ }
60
+ }
61
+ close(connId) {
62
+ const conn = this.conns.get(connId);
63
+ if (!conn)
64
+ return;
65
+ try {
66
+ conn.box.onClose(conn.streamId);
67
+ }
68
+ catch {
69
+ }
70
+ this.conns.delete(connId);
71
+ }
72
+ disconnect(connId) {
73
+ const conn = this.conns.get(connId);
74
+ if (!conn)
75
+ return;
76
+ try {
77
+ conn.box.onDisconnect(conn.streamId);
78
+ }
79
+ catch {
80
+ }
81
+ this.conns.delete(connId);
82
+ }
83
+ refresh() {
84
+ let mtimeMs;
85
+ try {
86
+ mtimeMs = fs.statSync(this.streamWasmPath).mtimeMs;
87
+ }
88
+ catch {
89
+ this.bytes = null;
90
+ this.loadedMtimeMs = -1;
91
+ return;
92
+ }
93
+ if (mtimeMs === this.loadedMtimeMs && this.bytes)
94
+ return;
95
+ this.bytes = fs.readFileSync(this.streamWasmPath);
96
+ this.loadedMtimeMs = mtimeMs;
97
+ }
98
+ allocStreamId() {
99
+ const id = this.nextStreamId;
100
+ this.nextStreamId = id + 1n;
101
+ return id;
102
+ }
103
+ }
@@ -0,0 +1,25 @@
1
+ import { type StreamDef } from './catalog.js';
2
+ export interface StreamWs {
3
+ send(data: Buffer, isBinary: boolean): void;
4
+ close(code: number): void;
5
+ on(event: 'message', cb: (message: Buffer, isBinary: boolean) => void): void;
6
+ on(event: 'close', cb: () => void): void;
7
+ }
8
+ export interface StreamUpgradeContext {
9
+ readonly kind: 'stream';
10
+ readonly route: string;
11
+ readonly url: string;
12
+ readonly authority: string;
13
+ }
14
+ export declare class StreamRouter {
15
+ private readonly streamWasmPath;
16
+ private catalog;
17
+ private catalogMtimeMs;
18
+ private readonly host;
19
+ private connSeq;
20
+ constructor(streamWasmPath: string);
21
+ get activeConnections(): number;
22
+ matchRoute(path: string): StreamDef | null;
23
+ onUpgrade(ws: StreamWs, ctx: StreamUpgradeContext): void;
24
+ private refreshCatalog;
25
+ }
@@ -0,0 +1,64 @@
1
+ import fs from 'node:fs';
2
+ import { matchStreamRoute, parseStreamCatalog, } from './catalog.js';
3
+ import { StreamDevHost } from './manager.js';
4
+ import { StreamWsSession } from './ws.js';
5
+ export class StreamRouter {
6
+ streamWasmPath;
7
+ catalog = new Map();
8
+ catalogMtimeMs = -1;
9
+ host;
10
+ connSeq = 0;
11
+ constructor(streamWasmPath) {
12
+ this.streamWasmPath = streamWasmPath;
13
+ this.host = new StreamDevHost(streamWasmPath);
14
+ this.refreshCatalog();
15
+ }
16
+ get activeConnections() {
17
+ return this.host.activeConnections;
18
+ }
19
+ matchRoute(path) {
20
+ this.refreshCatalog();
21
+ return matchStreamRoute(this.catalog, path);
22
+ }
23
+ onUpgrade(ws, ctx) {
24
+ const connId = `s${String(++this.connSeq)}`;
25
+ const q = ctx.url.indexOf('?');
26
+ const path = q >= 0 ? ctx.url.slice(0, q) : ctx.url;
27
+ const session = new StreamWsSession(this.host, connId, ctx.authority, path, {
28
+ send: (frame) => {
29
+ ws.send(frame, true);
30
+ },
31
+ close: (code) => {
32
+ ws.close(code);
33
+ },
34
+ });
35
+ if (!session.onOpen())
36
+ return;
37
+ ws.on('message', (message) => {
38
+ session.onMessage(message);
39
+ });
40
+ ws.on('close', () => {
41
+ session.onClose();
42
+ });
43
+ }
44
+ refreshCatalog() {
45
+ let mtimeMs;
46
+ try {
47
+ mtimeMs = fs.statSync(this.streamWasmPath).mtimeMs;
48
+ }
49
+ catch {
50
+ this.catalog = new Map();
51
+ this.catalogMtimeMs = -1;
52
+ return;
53
+ }
54
+ if (mtimeMs === this.catalogMtimeMs && this.catalog.size > 0)
55
+ return;
56
+ try {
57
+ this.catalog = parseStreamCatalog(fs.readFileSync(this.streamWasmPath));
58
+ }
59
+ catch {
60
+ this.catalog = new Map();
61
+ }
62
+ this.catalogMtimeMs = mtimeMs;
63
+ }
64
+ }
@@ -0,0 +1,5 @@
1
+ import { type Server } from '@dacely/hyper-express';
2
+ import { type ViteTarget } from '../http/proxy.js';
3
+ import { type StreamRouter } from './router.js';
4
+ export declare function streamEmulationEnabled(nodeMode: string): boolean;
5
+ export declare function wireStreams(app: Server, vite: ViteTarget, router: StreamRouter): void;
@@ -0,0 +1,33 @@
1
+ import { pipeToVite } from '../http/proxy.js';
2
+ export function streamEmulationEnabled(nodeMode) {
3
+ return nodeMode === 'regional' || nodeMode === 'continental' || nodeMode === 'all';
4
+ }
5
+ export function wireStreams(app, vite, router) {
6
+ app.upgrade('/*', (request, response) => {
7
+ const def = router.matchRoute(request.path);
8
+ if (def !== null) {
9
+ response.upgrade({
10
+ kind: 'stream',
11
+ route: def.route,
12
+ url: request.url,
13
+ authority: request.headers['host'] ?? '',
14
+ });
15
+ }
16
+ else {
17
+ response.upgrade({
18
+ kind: 'vite',
19
+ url: request.url,
20
+ protocol: request.headers['sec-websocket-protocol'] ?? '',
21
+ });
22
+ }
23
+ });
24
+ app.ws('/*', { message_type: 'Buffer', idle_timeout: 0, max_payload_length: 16 * 1024 * 1024 }, (ws) => {
25
+ const ctx = ws.context;
26
+ if (ctx.kind === 'stream') {
27
+ router.onUpgrade(ws, ws.context);
28
+ }
29
+ else {
30
+ pipeToVite(ws, vite, ws.context);
31
+ }
32
+ });
33
+ }
@@ -0,0 +1,18 @@
1
+ import type { StreamDevHost } from './manager.js';
2
+ export interface StreamWsTransport {
3
+ send(frame: Buffer): void;
4
+ close(code: number): void;
5
+ }
6
+ export declare class StreamWsSession {
7
+ private readonly host;
8
+ private readonly connId;
9
+ private readonly authority;
10
+ private readonly path;
11
+ private readonly transport;
12
+ private open;
13
+ constructor(host: StreamDevHost, connId: string, authority: string, path: string, transport: StreamWsTransport);
14
+ get isOpen(): boolean;
15
+ onOpen(): boolean;
16
+ onMessage(inbound: Buffer): void;
17
+ onClose(): void;
18
+ }
@@ -0,0 +1,46 @@
1
+ export class StreamWsSession {
2
+ host;
3
+ connId;
4
+ authority;
5
+ path;
6
+ transport;
7
+ open = false;
8
+ constructor(host, connId, authority, path, transport) {
9
+ this.host = host;
10
+ this.connId = connId;
11
+ this.authority = authority;
12
+ this.path = path;
13
+ this.transport = transport;
14
+ }
15
+ get isOpen() {
16
+ return this.open;
17
+ }
18
+ onOpen() {
19
+ const up = this.host.acceptUpgrade(this.connId, this.authority, this.path);
20
+ if (up.kind === 'rejected') {
21
+ this.transport.close(up.code);
22
+ return false;
23
+ }
24
+ this.open = true;
25
+ return true;
26
+ }
27
+ onMessage(inbound) {
28
+ if (!this.open)
29
+ return;
30
+ const r = this.host.dispatch(this.connId, inbound);
31
+ if (r.kind === 'reply') {
32
+ for (const frame of r.frames)
33
+ this.transport.send(frame);
34
+ }
35
+ else if (r.kind === 'close') {
36
+ this.open = false;
37
+ this.transport.close(r.code);
38
+ }
39
+ }
40
+ onClose() {
41
+ if (!this.open && !this.host.has(this.connId))
42
+ return;
43
+ this.open = false;
44
+ this.host.close(this.connId);
45
+ }
46
+ }
package/docs/cli.md CHANGED
@@ -8,7 +8,9 @@
8
8
  - `toiljs build`, production build. With a `toilconfig.json` it builds the server (toilscript,
9
9
  regenerating `shared/server.ts`) first, then the client → `build/client`. `--server` builds
10
10
  only the server. Every `server/` file declaring a surface (`@data`/`@rest`/...) is compiled.
11
- - `toiljs start`, self-host the built app (hyper-express) with a `/_toil` WebSocket channel.
11
+ - `toiljs start`, self-host the built app with production hyper-express/uWS static workers,
12
+ SSR/wasm dispatch, daemon support, and a `/_toil` WebSocket channel. Use `--threads <n>`
13
+ (or `server.threads`) to set the worker count; `1` disables the pool.
12
14
  - `toiljs configure`, toggle styling features on an existing project (see [styling.md](./styling.md)).
13
15
  - `toiljs doctor`, diagnose project setup (`--json` for CI). `--fix` auto-wires the typed-RPC
14
16
  setup (build scripts, tsconfig `shared` + alias, `.gitignore`, toilscript version, and the
package/docs/derive.md ADDED
@@ -0,0 +1,159 @@
1
+ # Derive (materialized views)
2
+
3
+ `@derive` precomputes a read-optimized **view** from your data so reads stay
4
+ fast and never scan. A request handler (`@get` runs as a *query*, `@post`/`@put`/
5
+ `@delete` as an *action*) is not allowed to scan, reading "the latest N events"
6
+ or "every member of a set" could fan out across unbounded rows, so those scans
7
+ are barred on the request path. A `@derive` does the scan **off** the request
8
+ path: it folds your event log / counters into a `View`, and your route serves
9
+ that view with a single keyed read.
10
+
11
+ ```ts
12
+ @database
13
+ class GuestbookDb {
14
+ @collection static entries: Events<GuestKey, GuestEntry>;
15
+ @collection static totals: Counter<GuestKey>;
16
+ @collection static book: View<GuestKey, GuestbookView>;
17
+
18
+ // Recompute the view from the sources. Runs after a signature is written
19
+ // (and when a box first loads). A derive MAY scan + publish; a route may not.
20
+ @derive
21
+ recompute(): void {
22
+ const key = new GuestKey('main');
23
+ const view = new GuestbookView();
24
+ view.total = GuestbookDb.totals.get(key); // counter read
25
+ view.entries = GuestbookDb.entries.latest(key, 10); // scan, allowed here
26
+ GuestbookDb.book.publish(key, view); // publish the materialized view
27
+ }
28
+ }
29
+ ```
30
+
31
+ ## Why a derive
32
+
33
+ ToilDB gates every data op by the *function kind* it runs under:
34
+
35
+ - **query** (`@get`/`@head`) and **action** (`@post`/`@put`/`@patch`/`@delete`)
36
+ may do keyed reads and (actions only) writes, but **not scans**
37
+ (`events.latest`, `membership.list`).
38
+ - **derive** may do everything a read can, **plus** scans, plus
39
+ `view.publish`/`append`/`counter.add`.
40
+
41
+ So if a page needs "the 10 newest entries" or "the leaderboard", you cannot read
42
+ that directly in the `@get`. Instead a `@derive` builds it once into a `View`,
43
+ and the `@get` reads the view by key, which is not a scan.
44
+
45
+ ## Declaring a derive
46
+
47
+ A derive is a method on your `@database` class, alongside the collections it
48
+ reads and the `View` it writes:
49
+
50
+ ```ts
51
+ @database
52
+ class MyDb {
53
+ @collection static events: Events<Key, Fact>; // a source
54
+ @collection static home: View<Key, HomePage>; // the materialized view
55
+
56
+ @derive
57
+ rebuild(): void {
58
+ // read sources, build the value, publish it
59
+ }
60
+ }
61
+ ```
62
+
63
+ Rules:
64
+
65
+ - A `@derive` method takes **no arguments and returns `void`**.
66
+ - A `@database` may declare **multiple** `@derive` methods; each is run
67
+ independently.
68
+ - The view value (`HomePage` above) and the key are ordinary `@data` types, so
69
+ they round-trip through the codec like any other stored value.
70
+
71
+ ## `View<K, V>`
72
+
73
+ A `View` is a published, read-optimized projection. Its API:
74
+
75
+ ```ts
76
+ view.get(key) // V | null - the published view, or null if none yet
77
+ view.require(key) // V - like get, but traps if nothing is published
78
+ view.publish(key, value) // void - overwrite the view (derive/job only)
79
+ ```
80
+
81
+ `publish` is only allowed from a `@derive` (or a `@job`); the host assigns the
82
+ version so a later publish always supersedes an earlier one. `get`/`require` are
83
+ plain keyed reads, allowed from any handler, including a `@get` route.
84
+
85
+ ## When derives run
86
+
87
+ You never call a derive yourself. The runtime runs it for you:
88
+
89
+ - **After a write to a source.** When a request writes one of a database's
90
+ source collections (an `events.append`/`append_once`, a `counter.add`, or a
91
+ record `create`/`patch`), that database's derives run right after the response
92
+ is produced, so the view reflects the new data on the next read. Many writes to
93
+ one database in a single request coalesce into one recompute.
94
+ - **On box load.** When a server box starts or hot-reloads (or the underlying
95
+ source data changed out of band), the views are rebuilt from their sources
96
+ before the first read is served. This is also where a value type's `@migrate`
97
+ runs against old stored events, as the derive re-reads and republishes them.
98
+
99
+ A derive's own writes (its `view.publish`) never re-trigger it.
100
+
101
+ The same code runs under `toiljs dev` (the in-process emulator) and on the
102
+ production edge, no flags or wiring to change.
103
+
104
+ ## Reading a view from a route
105
+
106
+ The route just reads the view by key, which is a non-scan read and so is legal in
107
+ a `@get`:
108
+
109
+ ```ts
110
+ @rest('guestbook')
111
+ class Guestbook {
112
+ @get('/')
113
+ list(): GuestbookView {
114
+ const key = new GuestKey('main');
115
+ const view = GuestbookDb.book.get(key);
116
+ return view == null ? new GuestbookView() : view; // empty until first publish
117
+ }
118
+
119
+ @post('/')
120
+ sign(input: NewMessage): GuestbookView {
121
+ const key = new GuestKey('main');
122
+ GuestbookDb.entries.append(key, new GuestEntry(input.author, input.message, 0));
123
+ GuestbookDb.totals.add(key, 1);
124
+ // The @derive republishes `book` right after this action returns, so the
125
+ // entries list is served by GET. The action just acks with the new total
126
+ // (a counter read is allowed here; a scan is not).
127
+ const view = new GuestbookView();
128
+ view.total = GuestbookDb.totals.get(key);
129
+ return view;
130
+ }
131
+ }
132
+ ```
133
+
134
+ ## How it fits together (the guestbook)
135
+
136
+ The `examples/basic` guestbook is the end-to-end demo:
137
+
138
+ 1. `POST /guestbook` (an action) appends the signature to an `Events` stream and
139
+ bumps a `Counter`. It returns the running total, but it does **not** read the
140
+ entry list (that would be a scan).
141
+ 2. The runtime then runs `@derive recompute()` under the derive kind: it scans
142
+ `entries.latest(...)`, reads the `totals` counter, and `publish`es a fresh
143
+ `GuestbookView`.
144
+ 3. `GET /guestbook` (a query) reads `book.get(...)`, a single keyed read, and
145
+ returns the precomputed total + newest entries.
146
+
147
+ Sign twice and the total climbs across requests, because the data lives in
148
+ ToilDB (and its view), not in module memory.
149
+
150
+ ## Notes
151
+
152
+ - A derive **recomputes** the view from whatever its method reads (here, the
153
+ latest 10 events). It is a fresh recompute on each trigger, so it suits views
154
+ built from a bounded read (latest N, a counter total, a small set). Folding an
155
+ unbounded full event log incrementally is a separate, more advanced pattern.
156
+ - Because publishes are last-writer-wins and a derive recomputes from the source
157
+ of truth, a view always converges to a correct snapshot of its sources.
158
+ - See also: [`data.md`](data.md) for `@data` value types, and the ToilDB host
159
+ ABI for the exact `derive_run` / `toildb.derives` contract.
@@ -95,13 +95,13 @@ store reached through a host binding.
95
95
 
96
96
  The `toiljs` CLI drives both halves:
97
97
 
98
- | Command | What it does |
99
- | --- | --- |
100
- | `toiljs create [name]` | Scaffold a new app (templates, styling, options). |
101
- | `toiljs dev` | Dev server with hot reload: watches `server/`, rebuilds the wasm via toilscript, regenerates `shared/server.ts`, and runs Vite for the client. Flags: `--root <dir>`, `--port <n>`, `--host`. |
102
- | `toiljs build` | Production build: server wasm first (so `shared/server.ts` is fresh), then the Vite client + static prerender. Flags: `--root <dir>`, `--server` (server only). |
103
- | `toiljs start` | Self-host a built app. Flags: `--root`, `--port`, `--host`. |
104
- | `toiljs doctor` | Diagnose setup/deps (`--json`, `--fix`). |
98
+ | Command | What it does |
99
+ | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
100
+ | `toiljs create [name]` | Scaffold a new app (templates, styling, options). |
101
+ | `toiljs dev` | Dev server with hot reload: watches `server/`, rebuilds the wasm via toilscript, regenerates `shared/server.ts`, and runs Vite for the client. Flags: `--root <dir>`, `--port <n>`, `--host`. |
102
+ | `toiljs build` | Production build: server wasm first (so `shared/server.ts` is fresh), then the Vite client + static prerender. Flags: `--root <dir>`, `--server` (server only). |
103
+ | `toiljs start` | Self-host a built app with production uWS/static workers, no Vite. Flags: `--root`, `--port`, `--host`, `--threads`. |
104
+ | `toiljs doctor` | Diagnose setup/deps (`--json`, `--fix`). |
105
105
 
106
106
  In dev, requests whose method matches a dispatchable verb go into the wasm
107
107
  first; if the guest reports "no route matched" (the `x-toil-unhandled` marker)
package/docs/index.md CHANGED
@@ -27,4 +27,4 @@ toilscript-to-WebAssembly server target.
27
27
 
28
28
  See [routing.md](./routing.md), [client.md](./client.md), [styling.md](./styling.md),
29
29
  [server.md](./server.md), [ssr.md](./ssr.md), [rpc.md](./rpc.md), [tiers.md](./tiers.md),
30
- [streams.md](./streams.md), [daemon.md](./daemon.md), [cli.md](./cli.md).
30
+ [streams.md](./streams.md), [daemon.md](./daemon.md), [derive.md](./derive.md), [cli.md](./cli.md).
package/docs/streams.md CHANGED
@@ -117,30 +117,62 @@ build/server/release-cold.wasm # L4 daemon (exports: daemon_start, sched
117
117
 
118
118
  See [Tiers](./tiers.md) for how the three artifacts map to the deployment tiers.
119
119
 
120
- ## What runs today
120
+ ## Reading and replying to messages
121
121
 
122
- The stream lifecycle hooks (`@connect` / `@message` / `@close` / `@disconnect`)
123
- run **today**, and this proves a resident box keeps state across events - that is
124
- exactly what the `Echo` example demonstrates by counting frames.
125
-
126
- Reading the inbound frame **bytes** and replying is the **next increment**, not
127
- yet available. That bridge is the `StreamPacket` / `StreamOutbound` API and the
128
- typed `Server.STREAM.echo.connect()` client. The intended shape, once it lands:
122
+ `@message` receives the inbound frame as a `StreamPacket` and returns a
123
+ `StreamOutbound`. `StreamPacket.bytes()` is the raw frame payload;
124
+ `StreamOutbound.reply(bytes)` stages one frame back to the client (return an empty
125
+ `StreamOutbound` to accept the frame without replying). The same resident box
126
+ handles every frame, so state on its fields persists across messages.
129
127
 
130
128
  ```ts
131
- @message reply(packet: StreamPacket): StreamOutbound {
129
+ @message
130
+ reply(packet: StreamPacket): StreamOutbound {
132
131
  return StreamOutbound.reply(packet.bytes()); // echo the bytes back
133
132
  }
134
133
  ```
135
134
 
135
+ ## Typed messages
136
+
137
+ By default a `@message` payload is **raw bytes**. Opt into a decoded `@data` value
138
+ with `@stream({ message: T })`: the `@message` hook then receives the named `@data`
139
+ class, decoded from the frame for you. The reply stays raw (`StreamOutbound`).
140
+
141
+ ```ts
142
+ @data
143
+ class ChatMsg { text: string = ''; }
144
+
145
+ @stream({ message: ChatMsg })
146
+ class Chat {
147
+ @message
148
+ onMessage(msg: ChatMsg): StreamOutbound { // decoded @data, not raw bytes
149
+ return StreamOutbound.reply(new TextEncoder().encode(msg.text));
150
+ }
151
+ }
152
+ ```
153
+
154
+ ## The client
155
+
156
+ A `@stream` class is reachable from the browser as `Server.Stream.<ClassName>`. The
157
+ typed client is generated into `shared/server.ts` (the same place `Server.REST`
158
+ lands), so no manual wiring is needed. `connect()` opens a WebSocket to the class's
159
+ route and resolves a channel:
160
+
136
161
  ```ts
137
- const stream = await Server.STREAM.echo.connect();
138
- stream.send(new TextEncoder().encode('hello'));
162
+ const chat = await Server.Stream.Chat.connect();
163
+ chat.onMessage((bytes) => { /* a reply frame, always raw bytes */ });
164
+ chat.send(new ChatMsg('hello')); // a typed stream: send() encodes the @data for you
165
+ chat.onClose((code) => { /* a 0x02xx stream close code */ });
166
+ chat.close();
139
167
  ```
140
168
 
141
- Until then, the hooks run on the connection lifecycle and you observe state
142
- through fields, as `Echo` does. See the comments in
143
- `examples/streams/server/streams/Echo.ts` for the authoritative note.
169
+ - The channel key is the **class name** (`Server.Stream.Chat`); it connects to the
170
+ class's mount route (`/Chat`).
171
+ - A **raw** `@stream` channel sends `Uint8Array`; a **typed** `@stream({ message: T })`
172
+ channel sends the `@data` class and encodes it on the wire for you.
173
+ - The inbound reply is **always raw bytes** - the server's `StreamOutbound` is raw.
174
+ - `connect()` resolves once the upgrade completes; a `@connect` reject (or any
175
+ later server close) surfaces through `onClose(code)`.
144
176
 
145
177
  ---
146
178
 
@@ -13,6 +13,12 @@ import { NewMessage } from '../models/NewMessage';
13
13
  *
14
14
  * await Server.REST.guestbook.sign({ body: new NewMessage('Ada', 'hi!') });
15
15
  * const book = await Server.REST.guestbook.list(); // { total, entries: [...] }
16
+ *
17
+ * Reading the newest entries is a SCAN (`events.latest`), which is barred in a
18
+ * request handler (a `@get` runs as a Query, a `@post` as an Action) because a
19
+ * scan can fan out across unbounded rows. So a `@derive` does the scan off the
20
+ * request path and `publish`es a materialized `GuestbookView`; the GET then
21
+ * serves that view with a single non-scan `view.get`.
16
22
  */
17
23
 
18
24
  // The guestbook is one global stream; a single fixed key addresses it.
@@ -28,33 +34,52 @@ class GuestKey {
28
34
  class GuestbookDb {
29
35
  @collection static entries: Events<GuestKey, GuestEntry>;
30
36
  @collection static totals: Counter<GuestKey>;
31
- }
37
+ // The materialized snapshot the GET serves: total + newest entries.
38
+ @collection static book: View<GuestKey, GuestbookView>;
32
39
 
33
- /** The current total + the 10 newest entries. */
34
- function snapshot(): GuestbookView {
35
- const key = new GuestKey('main');
36
- const view = new GuestbookView();
37
- view.total = GuestbookDb.totals.get(key);
38
- view.entries = GuestbookDb.entries.latest(key, 10);
39
- return view;
40
+ /**
41
+ * Recompute the materialized view from the source of truth (the event log +
42
+ * the counter). The runtime runs this under FunctionKind=Derive after a
43
+ * signature is appended (and rebuilds it when a box first loads), so the
44
+ * scan (`events.latest`) and the `view.publish` - both barred in a request
45
+ * handler - are allowed here. This is also where `GuestEntry`'s `@migrate`
46
+ * fires: `events.latest` decodes each stored event at ITS schema version, so
47
+ * an old pre-`at` entry is migrated as the view is rebuilt.
48
+ */
49
+ @derive
50
+ recompute(): void {
51
+ const key = new GuestKey('main');
52
+ const view = new GuestbookView();
53
+ view.total = GuestbookDb.totals.get(key);
54
+ view.entries = GuestbookDb.entries.latest(key, 10);
55
+ GuestbookDb.book.publish(key, view);
56
+ }
40
57
  }
41
58
 
42
59
  @rest('guestbook')
43
60
  class Guestbook {
44
- /** `GET /guestbook` - the running total + the most recent signatures. */
61
+ /** `GET /guestbook` - the running total + the most recent signatures, served
62
+ * from the materialized view (a non-scan `view.get`). */
45
63
  @get('/')
46
64
  public list(): GuestbookView {
47
- return snapshot();
65
+ const key = new GuestKey('main');
66
+ const view = GuestbookDb.book.get(key);
67
+ if (view == null) return new GuestbookView();
68
+ return view;
48
69
  }
49
70
 
50
- /** `POST /guestbook` - append a signature (PERSISTED) and return the
51
- * updated book. Sign twice and the total keeps climbing across requests. */
71
+ /** `POST /guestbook` - append a signature (PERSISTED) and acknowledge with
72
+ * the new running total. The entries list is served by the GET above from
73
+ * the view the `@derive` republishes right after this action. Sign twice
74
+ * and the total keeps climbing across requests. */
52
75
  @post('/')
53
76
  public sign(input: NewMessage): GuestbookView {
54
77
  const key = new GuestKey('main');
55
78
  const at = <u64>(Date.now() / 1000);
56
79
  GuestbookDb.entries.append(key, new GuestEntry(input.author, input.message, at));
57
80
  GuestbookDb.totals.add(key, 1);
58
- return snapshot();
81
+ const view = new GuestbookView();
82
+ view.total = GuestbookDb.totals.get(key); // Counter get: non-scan, action-legal
83
+ return view;
59
84
  }
60
85
  }