toiljs 0.0.33 → 0.0.36

package/src/compiler/docs.ts

@@ -21,6 +21,7 @@ the conventions you must follow:
 - \`.toil/docs/client.md\`, the \`Toil\` global, Link / NavLink, router hooks
 - \`.toil/docs/styling.md\`, CSS / Sass / Less / Stylus / Tailwind (via \`toiljs configure\`)
 - \`.toil/docs/server.md\`, the toilscript server target
+- \`.toil/docs/ssr.md\`, server-side rendering (\`ssr = true\`, hole markers, the server \`render\`)
 - \`.toil/docs/cli.md\`, toiljs CLI commands
 
 \`.toil/docs/\` is regenerated by toiljs; do not edit it by hand. This pointer file is yours to edit.
@@ -99,7 +100,7 @@ export const TOIL_DOCS: Record<string, string> = {
         '- Scripts: `npm run dev` (HMR), `npm run build` (→ `build/client` + `build/server`),',
         '  `npm start` (self-host the build).',
         '',
-        'See `routing.md`, `client.md`, `styling.md`, `server.md`, `cli.md`.',
+        'See `routing.md`, `client.md`, `styling.md`, `server.md`, `ssr.md`, `cli.md`.',
     ]),
     'routing.md': doc([
         '# Routing',
@@ -136,6 +137,11 @@ export const TOIL_DOCS: Record<string, string> = {
         '',
         '`Toil.useRouter().refresh()` re-runs loaders.',
         '',
+        '## Server rendering',
+        '',
+        'Add `export const ssr = true` to render a route on the server (real first-paint HTML + SEO,',
+        'then hydration) with near-static speed. See `ssr.md`.',
+        '',
         '## Navigation',
         '',
         '- `Toil.Link` / `Toil.NavLink` (adds an active class) / `Toil.useRouter()`',
@@ -263,6 +269,87 @@ export const TOIL_DOCS: Record<string, string> = {
         'For a REST-only project, `Server.handler = () => new RestHandler()` does the same with no',
         'boilerplate. On the client: `Server.REST.todos.getTodo({ params: { id } })` (see client.md).',
     ]),
+    'ssr.md': doc([
+        '# Server-side rendering (SSR)',
+        '',
+        'Opt a route into SSR with `export const ssr = true`. toiljs renders the page ONCE at build',
+        'time into a template with holes; at request time the server fills only the dynamic holes and',
+        'serves the page, then the browser hydrates it in place. The page is never re-rendered per',
+        'request, so an SSR route is served about as fast as a static file while still delivering real',
+        'first-paint HTML and SEO.',
+        '',
+        '## Marking the holes',
+        '',
+        'Wrap the dynamic bits of the page in hole markers from `toiljs/client`. They are transparent',
+        'in the browser (they just render their children); only the build and the server treat them',
+        'specially, so the same component is your normal client UI.',
+        '',
+        '- `<Hole id="name">{value}</Hole>`, a text hole (the value is HTML-escaped for you).',
+        '- `<RawHtml id="bio" html={s} />`, a raw-HTML block (you own sanitisation, like',
+        '  `dangerouslySetInnerHTML`).',
+        '- `<Repeat id="rows" each={items}>{(item) => <li>...</li>}</Repeat>`, a repeated region (a',
+        '  list); the row markup is captured once and stamped per item.',
+        '- `<Island>{...}</Island>`, a client-only escape hatch: empty in the server HTML, rendered',
+        '  after hydration (so it gets no first paint or SEO). Put router-hook-driven or otherwise',
+        '  non-server-safe bits here.',
+        '',
+        '    import { Hole, Repeat, RawHtml, useLoaderData } from "toiljs/client";',
+        '    export const ssr = true;',
+        '    export const loader = ({ params }: Toil.LoaderArgs) => loadProfile(params.name);',
+        '    export default function Profile() {',
+        '        const d = useLoaderData<typeof loader>();',
+        '        return (',
+        '            <main>',
+        '                <h1>@<Hole id="username">{d.username}</Hole></h1>',
+        '                <RawHtml id="bio" html={d.bioHtml} />',
+        '                <ul>',
+        '                    <Repeat id="posts" each={d.posts}>',
+        '                        {(p) => <li><Hole id="title">{p.title}</Hole></li>}',
+        '                    </Repeat>',
+        '                </ul>',
+        '            </main>',
+        '        );',
+        '    }',
+        '',
+        '## The server render',
+        '',
+        'For each SSR route the build emits a typed `Slot` enum and a coherence hash (a',
+        '`<route>.slots.ts` module). In the server you write a `render(req)` that fills each slot from',
+        'the request and data, using the `SlotValues` API, and register it with `Ssr`:',
+        '',
+        '```ts',
+        'import { Request, SlotValues, HtmlBuilder, Ssr } from "toiljs/server/runtime";',
+        'import { Slot, HASH } from "./profile.slots";',
+        '',
+        'function renderProfile(req: Request): SlotValues | null {',
+        '    if (!req.path.startsWith("/u/")) return null; // not this route',
+        '    const v = new SlotValues(HASH);',
+        '    v.setText(Slot.username, usernameFor(req));   // escaped',
+        '    v.setRaw(Slot.bio, bioHtml);                  // verbatim',
+        '    const rows = new HtmlBuilder();',
+        '    for (let i = 0; i < posts.length; i++) rows.raw("<li>").text(posts[i]).raw("</li>");',
+        '    v.setRepeat(Slot.posts, rows);                // stamped rows',
+        '    return v;',
+        '}',
+        'Ssr.register(renderProfile);',
+        '```',
+        '',
+        '`SlotValues`: `setText` (escaped), `setRaw` (verbatim), `setAttr` (attribute value),',
+        '`setRepeat` (a stamped `HtmlBuilder`), plus `setHeader` and `setStatus`. The hole values you',
+        'return are filled into the template and the page is served; the browser then hydrates against',
+        'the same data, so the markup matches with no client re-render.',
+        '',
+        '## Rules of thumb',
+        '',
+        '- An SSR route (and the layouts above it) must render under static markup. Use the hole',
+        '  markers and `useLoaderData`; move anything that needs router hooks or browser-only APIs into',
+        '  an `<Island>`. A route that cannot render this way is skipped at build (with a warning) and',
+        '  simply falls back to normal client rendering.',
+        '- Hole values are HTML-escaped exactly as React escapes them, so hydration is byte-for-byte',
+        '  clean. Keep a repeat row\'s structure the same across items (only the leaf hole values vary).',
+        '- Build output for an SSR route lands in `build/client/_ssr/` (the template + its manifest)',
+        '  alongside the generated `Slot` module; routes without `ssr = true` are unaffected.',
+    ]),
     'cli.md': doc([
         '# CLI',
         '',

package/src/compiler/generate.ts

@@ -90,7 +90,7 @@ function relFromToil(cfg: ResolvedToilConfig, abs: string): string {
     return rel;
 }
 
-function findLayout(cfg: ResolvedToilConfig): string | undefined {
+export function findLayout(cfg: ResolvedToilConfig): string | undefined {
     return ['layout.tsx', 'layout.jsx']
         .map((name) => path.join(cfg.clientAbsDir, name))
         .find((p) => fs.existsSync(p));
@@ -189,7 +189,7 @@ function specialIn(dir: string, base: string): string | undefined {
  * ancestor directory down to the file's own. With `includeClientRoot`, `client/<base>` is prepended
  * as the outermost (used by templates; the root `client/layout.tsx` is instead the top-level layout).
  */
-function findSpecialChain(
+export function findSpecialChain(
     cfg: ResolvedToilConfig,
     routeFile: string,
     base: string,

package/src/compiler/index.ts

@@ -14,6 +14,7 @@ import type { RunningBackend } from 'toiljs/backend';
 import { loadConfig } from './config.js';
 import { generate } from './generate.js';
 import { prerenderStaticParams } from './ssg.js';
+import { extractTemplates } from './template-build.js';
 import { createViteConfig } from './vite.js';
 
 /**
@@ -312,6 +313,10 @@ export async function build(opts: ToilCommandOptions = {}): Promise<void> {
     await viteBuild(await createViteConfig(cfg));
     // SSG: bake per-URL HTML + sitemap for dynamic routes that opt in via `generateStaticParams`.
     await prerenderStaticParams(cfg);
+    // Edge SSR: render `export const ssr = true` routes to template-with-holes
+    // (`_ssr/*.tmpl|slots` + the guest `Slot` module), copied into the edge host
+    // bundle. No-op when no route opts in.
+    await extractTemplates(cfg);
 }
 
 /**

package/src/compiler/ssr-codegen.ts

@@ -0,0 +1,85 @@
+/**
+ * Build-time codegen of the AssemblyScript surface a route's guest `render`
+ * uses. From the extracted holes (see `template.ts`) this emits:
+ *
+ *   - a `Slot` enum (hole name -> stable numeric id, the SAME ids the host
+ *     `.slots` manifest carries), so host and guest agree by construction; and
+ *   - the coherence `HASH` (32 bytes) compiled into the guest and echoed in
+ *     every values envelope; the host 500s on a mismatch (deploy skew).
+ *
+ * The render BODY is authored against `SlotValues` (the typed, hand-writable
+ * API). A route's `render(req)` derives its data from the request/edge and
+ * fills the slots:
+ *
+ * ```ts
+ * import { Request } from 'toiljs/server/runtime';
+ * import { SlotValues } from 'toiljs/server/runtime/ssr/slots';
+ * import { Slot, HASH } from './u_name.slots';
+ *
+ * export function renderUName(req: Request): SlotValues {
+ *   const v = new SlotValues(HASH);
+ *   v.setText(Slot.username, pathParam(req, 'name'));
+ *   v.setRaw(Slot.bio, bio);
+ *   const rows = new HtmlBuilder();
+ *   for (let i = 0; i < posts.length; i++) rows.raw('<li>').text(posts[i]).raw('</li>');
+ *   v.setRepeat(Slot.posts, rows);
+ *   return v;
+ * }
+ * ```
+ *
+ * Generating that body automatically from the JSX-subset bindings is a later
+ * phase; the typed enum + hash here are what make a hand-written or generated
+ * body coherent with the deployed template.
+ */
+
+import type { SlotRecord } from './template.js';
+import { assignSlotIds } from './template.js';
+
+/** Emit a `StaticArray<u8>` literal of the 32 hash bytes. */
+function hashLiteral(hash: Buffer): string {
+    const bytes = Array.from(hash.values())
+        .map((b) => `0x${b.toString(16).padStart(2, '0')}`)
+        .join(', ');
+    return `[${bytes}]`;
+}
+
+/** A slot name is a valid AS enum member iff it is a bare identifier; anything
+ * else (e.g. a hyphenated id) is rejected so the generated module can't break
+ * or smuggle code. */
+function isIdent(name: string): boolean {
+    return /^[A-Za-z_$][\w$]*$/.test(name);
+}
+
+/**
+ * Generate the `<route>.slots.ts` AssemblyScript module: the `Slot` enum and
+ * the `HASH` constant. `slots` is the top-level slot list from
+ * `extractFromHtml`; ids are assigned in document order to match the host
+ * `.slots` manifest written by `encodeSlots`.
+ */
+export function generateSlotsModule(routeName: string, slots: SlotRecord[], hash: Buffer): string {
+    if (hash.length !== 32) throw new Error('toil ssr: coherence hash must be 32 bytes');
+    const ids = assignSlotIds(slots);
+    const members: string[] = [];
+    for (const [name, id] of ids) {
+        if (!isIdent(name)) {
+            throw new Error(
+                `toil ssr: hole id "${name}" in route "${routeName}" is not a valid identifier ` +
+                    `(use [A-Za-z_][\\w]*) so it can be a Slot enum member`,
+            );
+        }
+        members.push(`    ${name} = ${id},`);
+    }
+    return `// AUTO-GENERATED by toil (edge SSR). Do not edit.
+// Route: ${routeName}. Slot ids match the deployed .slots manifest; HASH is the
+// coherence hash the host checks against the template (deploy-skew guard).
+
+/** Stable hole ids for this route's template. */
+export enum Slot {
+${members.join('\n')}
+}
+
+/** Coherence hash (32 bytes) baked into the guest and echoed in every values
+ * envelope; the host rejects a response whose hash != the deployed template. */
+export const HASH: StaticArray<u8> = ${hashLiteral(hash)};
+`;
+}

package/src/compiler/template-build.ts

@@ -0,0 +1,275 @@
+/**
+ * Build-time orchestration for edge SSR: render each opted-in route to a
+ * template-with-holes and emit the artifacts the edge + guest consume.
+ *
+ * The deterministic core (`extractRouteTemplate`, `injectIntoShell`,
+ * `writeTemplateArtifacts`) is unit-tested with controlled components; the
+ * `extractTemplates` driver loads real route + layout modules through a short-
+ * lived Vite SSR server (the same pattern as `ssg.ts`).
+ *
+ * A route opts in with `export const ssr = true`. Its page + layout chain are
+ * rendered under the loader-data provider with sample data, in sentinel mode;
+ * the result is spliced into the built shell's `#root`, stripped to a `.tmpl`,
+ * and written alongside the binary `.slots` manifest and the generated AS
+ * `Slot` enum + `HASH`. A route (or layout) that throws under static markup
+ * (e.g. it uses router hooks outside the supported subset) is skipped with a
+ * warning and falls back to pure client rendering.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {
+    createElement,
+    type ComponentType,
+    type Context,
+    type ReactNode,
+} from 'react';
+import { renderToStaticMarkup } from 'react-dom/server';
+import { createServer } from 'vite';
+
+import { type ResolvedToilConfig } from './config.js';
+import { findLayout, findSpecialChain } from './generate.js';
+import { scanRoutes } from './routes.js';
+import { generateSlotsModule } from './ssr-codegen.js';
+import {
+    assignSlotIds,
+    coherenceHash,
+    encodeSlots,
+    extractFromHtml,
+    type Extracted,
+} from './template.js';
+import { createViteConfig } from './vite.js';
+
+/** Marker element the client `mount` looks for to switch to `hydrateRoot`. */
+const SSR_MARKER = '<template id="__toil_ssr"></template>';
+const ROOT_DIV = '<div id="root"></div>';
+
+export interface RouteRenderInput {
+    /** File-safe route name (the `<name>.tmpl` stem). */
+    name: string;
+    Page: ComponentType;
+    /** Layout chain, OUTERMOST first (root layout, then nested). */
+    layouts: ComponentType<{ children?: ReactNode }>[];
+    /** Sample loader data provided via the loader context during the render. */
+    loaderData: unknown;
+    /** The client's `LoaderDataContext` (loaded via the SSR module graph so it
+     * is the same instance the page's `useLoaderData` reads). */
+    loaderContext: Context<unknown> | null;
+    /** The markers' build switch (same module instance the page imports). */
+    setSsrBuild: (on: boolean) => void;
+    /** The built HTML shell (with hashed script tags) to splice into. */
+    shell: string;
+}
+
+export interface TemplateArtifacts {
+    name: string;
+    tmpl: Buffer;
+    /** Binary `.slots` manifest for the Rust host. */
+    slotsBin: Buffer;
+    /** Generated AssemblyScript `Slot` enum + `HASH` module source. */
+    slotsModule: string;
+    hash: Buffer;
+    slotCount: number;
+}
+
+/** Build the route element tree: layouts (outermost first) wrapping the page,
+ * under the loader-data provider. The Suspense/RoutePage wrappers the client
+ * adds contribute no DOM, so this reproduces the client's markup. */
+export function assembleRouteElement(
+    Page: ComponentType,
+    layouts: ComponentType<{ children?: ReactNode }>[],
+    loaderData: unknown,
+    loaderContext: Context<unknown> | null,
+): ReactNode {
+    let node: ReactNode = createElement(Page);
+    if (loaderContext) {
+        node = createElement(loaderContext.Provider, { value: loaderData }, node);
+    }
+    for (let i = layouts.length - 1; i >= 0; i--) {
+        node = createElement(layouts[i], null, node);
+    }
+    return node;
+}
+
+/** Splice the rendered route HTML into the shell's `#root` and add the SSR
+ * marker so the client hydrates rather than client-renders. */
+export function injectIntoShell(shell: string, routeHtml: string): string {
+    if (!shell.includes(ROOT_DIV)) {
+        throw new Error('toil ssr: built shell has no empty <div id="root"></div> to splice into');
+    }
+    return shell.replace(ROOT_DIV, `<div id="root">${routeHtml}</div>${SSR_MARKER}`);
+}
+
+/** Render one route to its template artifacts (pure given its inputs). */
+export function extractRouteTemplate(input: RouteRenderInput): TemplateArtifacts {
+    const element = assembleRouteElement(
+        input.Page,
+        input.layouts,
+        input.loaderData,
+        input.loaderContext,
+    );
+    input.setSsrBuild(true);
+    let routeHtml: string;
+    try {
+        routeHtml = renderToStaticMarkup(element);
+    } finally {
+        input.setSsrBuild(false);
+    }
+    const full = injectIntoShell(input.shell, routeHtml);
+    const extracted: Extracted = extractFromHtml(full);
+    const ids = assignSlotIds(extracted.slots);
+    const hash = coherenceHash(extracted.tmpl, extracted.slots);
+    return {
+        name: input.name,
+        tmpl: extracted.tmpl,
+        slotsBin: encodeSlots(extracted.tmpl.length, hash, extracted.slots, ids),
+        slotsModule: generateSlotsModule(input.name, extracted.slots, hash),
+        hash,
+        slotCount: extracted.slots.length,
+    };
+}
+
+/** Write a route's `.tmpl` / `.slots` / `.slots.ts` into `ssrDir`. */
+export function writeTemplateArtifacts(ssrDir: string, art: TemplateArtifacts): void {
+    fs.mkdirSync(ssrDir, { recursive: true });
+    fs.writeFileSync(path.join(ssrDir, `${art.name}.tmpl`), art.tmpl);
+    fs.writeFileSync(path.join(ssrDir, `${art.name}.slots`), art.slotsBin);
+    fs.writeFileSync(path.join(ssrDir, `${art.name}.slots.ts`), art.slotsModule);
+}
+
+/** A file-safe, identifier-ish name for a route pattern (`/u/:name` -> `u_name`). */
+export function routeTemplateName(pattern: string): string {
+    const n = pattern.replace(/[^A-Za-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
+    return n.length > 0 ? n : 'index';
+}
+
+/** Synthesize a sample param set for a pattern's dynamic segments. */
+function sampleParams(pattern: string): Record<string, string> {
+    const params: Record<string, string> = {};
+    for (const m of pattern.matchAll(/[:*]+([A-Za-z0-9_]+)/g)) {
+        params[m[1]] = 'sample';
+    }
+    return params;
+}
+
+interface RouteModule {
+    default: ComponentType;
+    ssr?: boolean;
+    loader?: (args: { params: Record<string, string>; searchParams: URLSearchParams }) => unknown;
+}
+
+/**
+ * Render every `export const ssr = true` route to `<outDir>/_ssr/<name>.{tmpl,
+ * slots,slots.ts}` + a `templates.json` index, and copy the `.tmpl`/`.slots`
+ * into the edge host bundle at `hosts/<host>/_tmpl/`. Returns the patterns
+ * generated. Skips (with a warning) any route that throws under static markup.
+ */
+export async function extractTemplates(
+    cfg: ResolvedToilConfig,
+    hostName = 'edge',
+): Promise<string[]> {
+    const routes = scanRoutes(cfg.routesAbsDir).filter((r) => r.slot === undefined && !r.intercept);
+    if (routes.length === 0) return [];
+
+    const outDir = path.resolve(cfg.root, cfg.outDir);
+    const stashed = path.join(cfg.toilDir, 'shell.html');
+    const shellPath = fs.existsSync(stashed) ? stashed : path.join(outDir, 'index.html');
+    if (!fs.existsSync(shellPath)) return [];
+    const shell = fs.readFileSync(shellPath, 'utf8');
+
+    const warn = (msg: string): void => {
+        process.stderr.write(`  toil: SSR ${msg}\n`);
+    };
+    const server = await createServer({
+        ...(await createViteConfig(cfg)),
+        server: { middlewareMode: true, hmr: false },
+        appType: 'custom',
+        logLevel: 'silent',
+    });
+
+    const client = (await server.ssrLoadModule('toiljs/client')) as unknown as {
+        __setSsrBuild: (on: boolean) => void;
+        LoaderDataContext: Context<unknown>;
+    };
+
+    const ssrDir = path.join(outDir, '_ssr');
+    const hostsTmplDir = path.join(cfg.root, 'hosts', hostName, '_tmpl');
+    const generated: string[] = [];
+    const index: { route: string; name: string; hash: string }[] = [];
+
+    try {
+        for (const r of routes) {
+            let mod: RouteModule;
+            try {
+                mod = (await server.ssrLoadModule(r.file)) as unknown as RouteModule;
+            } catch (err) {
+                warn(`skipped ${r.pattern} (${err instanceof Error ? err.message : String(err)})`);
+                continue;
+            }
+            if (mod.ssr !== true) continue;
+
+            try {
+                const params = sampleParams(r.pattern);
+                const loaderData =
+                    typeof mod.loader === 'function'
+                        ? await mod.loader({ params, searchParams: new URLSearchParams() })
+                        : undefined;
+
+                const layoutFiles = [
+                    ...(findLayout(cfg) ? [findLayout(cfg)!] : []),
+                    ...findSpecialChain(cfg, r.file, 'layout', false),
+                ];
+                const layouts: ComponentType<{ children?: ReactNode }>[] = [];
+                for (const lf of layoutFiles) {
+                    const lm = (await server.ssrLoadModule(lf)) as unknown as {
+                        default: ComponentType<{ children?: ReactNode }>;
+                    };
+                    layouts.push(lm.default);
+                }
+
+                const name = routeTemplateName(r.pattern);
+                const art = extractRouteTemplate({
+                    name,
+                    Page: mod.default,
+                    layouts,
+                    loaderData,
+                    loaderContext: client.LoaderDataContext,
+                    setSsrBuild: client.__setSsrBuild,
+                    shell,
+                });
+                writeTemplateArtifacts(ssrDir, art);
+                fs.mkdirSync(hostsTmplDir, { recursive: true });
+                fs.copyFileSync(
+                    path.join(ssrDir, `${name}.tmpl`),
+                    path.join(hostsTmplDir, `${name}.tmpl`),
+                );
+                fs.copyFileSync(
+                    path.join(ssrDir, `${name}.slots`),
+                    path.join(hostsTmplDir, `${name}.slots`),
+                );
+                index.push({ route: r.pattern, name, hash: art.hash.toString('hex') });
+                generated.push(r.pattern);
+            } catch (err) {
+                warn(
+                    `skipped ${r.pattern} (render failed: ${
+                        err instanceof Error ? err.message : String(err)
+                    }) — falls back to client rendering`,
+                );
+            }
+        }
+    } finally {
+        await server.close();
+    }
+
+    if (generated.length > 0) {
+        fs.mkdirSync(ssrDir, { recursive: true });
+        fs.writeFileSync(path.join(ssrDir, 'templates.json'), JSON.stringify(index, null, 2));
+        process.stdout.write(
+            `  ✓ extracted ${String(generated.length)} SSR template${
+                generated.length === 1 ? '' : 's'
+            }\n`,
+        );
+    }
+    return generated;
+}