toiljs 0.0.14 → 0.0.16

package/src/compiler/index.ts

@@ -1,54 +1,64 @@
-import fs from 'node:fs';
-import path from 'node:path';
-
-import { build as viteBuild, createServer, type ViteDevServer } from 'vite';
-import { startBackend, type RunningBackend } from 'toiljs/backend';
-
-import { loadConfig } from './config.js';
-import { generate } from './generate.js';
-import { createViteConfig } from './vite.js';
-
-export interface ToilCommandOptions {
-    readonly root?: string;
-    readonly port?: number;
-}
-
-/** Starts the Vite dev server (HMR + transforms) for the client app. Returns the running server. */
-export async function dev(opts: ToilCommandOptions = {}): Promise<ViteDevServer> {
-    const cfg = await loadConfig(opts);
-    generate(cfg);
-    const server = await createServer(await createViteConfig(cfg));
-    await server.listen();
-    server.printUrls();
-    return server;
-}
-
-/** Produces an optimized production SPA bundle in the configured `outDir`. */
-export async function build(opts: ToilCommandOptions = {}): Promise<void> {
-    const cfg = await loadConfig(opts);
-    generate(cfg);
-    await viteBuild(await createViteConfig(cfg));
-}
-
-/**
- * Self-hosts the built client over the high-performance hyper-express backend (uWebSockets.js),
- * serving the configured `outDir` with an SPA fallback plus a WebSocket channel. Requires a prior
- * `build`. Returns the running backend.
- */
-export async function start(opts: ToilCommandOptions = {}): Promise<RunningBackend> {
-    const cfg = await loadConfig(opts);
-    const outDir = path.resolve(cfg.root, cfg.outDir);
-    if (!fs.existsSync(path.join(outDir, 'index.html'))) {
-        throw new Error(`No build found in ${outDir}. Run \`toiljs build\` first.`);
-    }
-    return startBackend({ root: outDir, port: cfg.port });
-}
-
-export { defineConfig, loadConfig } from './config.js';
-export { scanRoutes } from './routes.js';
-export type { ScannedRoute } from './routes.js';
-export { TOIL_ENV_DTS } from './generate.js';
-export { AI_HELPERS, AI_HELPER_IDS, aiHelperFiles, TOIL_DOCS } from './docs.js';
-export type { AiHelper } from './docs.js';
-export type { ToilConfig, ResolvedToilConfig } from './config.js';
-export type { RunningBackend, BackendOptions } from 'toiljs/backend';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { build as viteBuild, createServer, type ViteDevServer } from 'vite';
+import { startBackend, type RunningBackend } from 'toiljs/backend';
+
+import { loadConfig } from './config.js';
+import { generate } from './generate.js';
+import { prerenderStaticParams } from './ssg.js';
+import { createViteConfig } from './vite.js';
+
+export interface ToilCommandOptions {
+    readonly root?: string;
+    readonly port?: number;
+}
+
+/** Starts the Vite dev server (HMR + transforms) for the client app. Returns the running server. */
+export async function dev(opts: ToilCommandOptions = {}): Promise<ViteDevServer> {
+    const cfg = await loadConfig(opts);
+    generate(cfg);
+    const server = await createServer(await createViteConfig(cfg));
+    await server.listen();
+    server.printUrls();
+    return server;
+}
+
+/** Produces an optimized production SPA bundle in the configured `outDir`. */
+export async function build(opts: ToilCommandOptions = {}): Promise<void> {
+    const cfg = await loadConfig(opts);
+    generate(cfg);
+    await viteBuild(await createViteConfig(cfg));
+    // SSG: bake per-URL HTML + sitemap for dynamic routes that opt in via `generateStaticParams`.
+    await prerenderStaticParams(cfg);
+}
+
+/**
+ * Self-hosts the built client over the high-performance hyper-express backend (uWebSockets.js),
+ * serving the configured `outDir` with an SPA fallback plus a WebSocket channel. Requires a prior
+ * `build`. Returns the running backend.
+ */
+export async function start(opts: ToilCommandOptions = {}): Promise<RunningBackend> {
+    const cfg = await loadConfig(opts);
+    const outDir = path.resolve(cfg.root, cfg.outDir);
+    if (!fs.existsSync(path.join(outDir, 'index.html'))) {
+        throw new Error(`No build found in ${outDir}. Run \`toiljs build\` first.`);
+    }
+    return startBackend({ root: outDir, port: cfg.port });
+}
+
+export { defineConfig, loadConfig, AiProvider } from './config.js';
+export { scanRoutes } from './routes.js';
+export type { ScannedRoute } from './routes.js';
+export { TOIL_ENV_DTS } from './generate.js';
+export { AI_HELPERS, AI_HELPER_IDS, aiHelperFiles, TOIL_DOCS } from './docs.js';
+export type { AiHelper } from './docs.js';
+export type {
+    ToilConfig,
+    ResolvedToilConfig,
+    ClientConfig,
+    ServerConfig,
+    DevtoolsConfig,
+    DevtoolsAiConfig,
+} from './config.js';
+export type { RunningBackend, BackendOptions } from 'toiljs/backend';

package/src/compiler/pages.ts

@@ -1,70 +1,70 @@
-import { createRequire } from 'node:module';
-import path from 'node:path';
-
-import type * as TS from 'typescript';
-
-import { extractStaticExports } from './prerender.js';
-import type { ScannedRoute } from './routes.js';
-
-type Ts = typeof TS;
-
-/**
- * A page in the build-time search index: its URL pattern, whether it's dynamic, and the
- * statically-extracted `metadata` literal (the searchable subset; dynamic `generateMetadata` and
- * computed values can't be known at build, so they're absent). Serialized into the generated
- * `routes` module and registered client-side for {@link searchPages}.
- */
-export interface PageIndexEntry {
-    readonly path: string;
-    readonly dynamic: boolean;
-    readonly metadata: Record<string, unknown>;
-}
-
-/**
- * Loads the project's TypeScript synchronously (so {@link buildPageIndex} can run inside the sync
- * `generate()`), or `null` if it isn't installed, in which case pages are indexed by path only.
- */
-function loadTypeScriptSync(root: string): Ts | null {
-    try {
-        const require = createRequire(path.join(root, 'package.json'));
-        const mod = require('typescript') as { default?: Ts } & Ts;
-        return mod.default ?? mod;
-    } catch {
-        return null;
-    }
-}
-
-/** True when a route pattern has dynamic (`:param` / `*catch-all`) segments. */
-function isDynamic(pattern: string): boolean {
-    return /[:*]/.test(pattern);
-}
-
-/**
- * Builds the searchable page index from the scanned routes: every main-tree page (slots and
- * intercepting routes are excluded, they don't own a distinct URL) paired with its statically
- * extracted `metadata`. A route may also `export const searchHints` (a static `title`/`description`/
- * `keywords` object) to feed the index even when its real metadata is dynamic (`generateMetadata`);
- * hints are merged over the static `metadata`, winning ties. Reads each route file once.
- */
-export function buildPageIndex(root: string, routes: readonly ScannedRoute[]): PageIndexEntry[] {
-    const ts = loadTypeScriptSync(root);
-    const seen = new Set<string>();
-    const pages: PageIndexEntry[] = [];
-    for (const route of routes) {
-        if (route.slot !== undefined || route.intercept) continue;
-        if (seen.has(route.pattern)) continue;
-        seen.add(route.pattern);
-        const exports = ts ? extractStaticExports(ts, route.file, ['metadata', 'searchHints']) : {};
-        const metadata = { ...exports.metadata, ...exports.searchHints };
-        pages.push({ path: route.pattern, dynamic: isDynamic(route.pattern), metadata });
-    }
-    // Stable order (by path) so the generated module is deterministic across runs.
-    pages.sort((a, b) => a.path.localeCompare(b.path));
-    return pages;
-}
-
-/** Serializes the page index to the `export const pages` literal embedded in the routes module. */
-export function pagesModuleSource(pages: readonly PageIndexEntry[]): string {
-    const body = pages.map((p) => `  ${JSON.stringify(p)},`).join('\n');
-    return `export const pages: PageMeta[] = [\n${body}\n];\n`;
-}
+import { createRequire } from 'node:module';
+import path from 'node:path';
+
+import type * as TS from 'typescript';
+
+import { extractStaticExports } from './prerender.js';
+import type { ScannedRoute } from './routes.js';
+
+type Ts = typeof TS;
+
+/**
+ * A page in the build-time search index: its URL pattern, whether it's dynamic, and the
+ * statically-extracted `metadata` literal (the searchable subset; dynamic `generateMetadata` and
+ * computed values can't be known at build, so they're absent). Serialized into the generated
+ * `routes` module and registered client-side for {@link searchPages}.
+ */
+export interface PageIndexEntry {
+    readonly path: string;
+    readonly dynamic: boolean;
+    readonly metadata: Record<string, unknown>;
+}
+
+/**
+ * Loads the project's TypeScript synchronously (so {@link buildPageIndex} can run inside the sync
+ * `generate()`), or `null` if it isn't installed, in which case pages are indexed by path only.
+ */
+function loadTypeScriptSync(root: string): Ts | null {
+    try {
+        const require = createRequire(path.join(root, 'package.json'));
+        const mod = require('typescript') as { default?: Ts } & Ts;
+        return mod.default ?? mod;
+    } catch {
+        return null;
+    }
+}
+
+/** True when a route pattern has dynamic (`:param` / `*catch-all`) segments. */
+function isDynamic(pattern: string): boolean {
+    return /[:*]/.test(pattern);
+}
+
+/**
+ * Builds the searchable page index from the scanned routes: every main-tree page (slots and
+ * intercepting routes are excluded, they don't own a distinct URL) paired with its statically
+ * extracted `metadata`. A route may also `export const searchHints` (a static `title`/`description`/
+ * `keywords` object) to feed the index even when its real metadata is dynamic (`generateMetadata`);
+ * hints are merged over the static `metadata`, winning ties. Reads each route file once.
+ */
+export function buildPageIndex(root: string, routes: readonly ScannedRoute[]): PageIndexEntry[] {
+    const ts = loadTypeScriptSync(root);
+    const seen = new Set<string>();
+    const pages: PageIndexEntry[] = [];
+    for (const route of routes) {
+        if (route.slot !== undefined || route.intercept) continue;
+        if (seen.has(route.pattern)) continue;
+        seen.add(route.pattern);
+        const exports = ts ? extractStaticExports(ts, route.file, ['metadata', 'searchHints']) : {};
+        const metadata = { ...exports.metadata, ...exports.searchHints };
+        pages.push({ path: route.pattern, dynamic: isDynamic(route.pattern), metadata });
+    }
+    // Stable order (by path) so the generated module is deterministic across runs.
+    pages.sort((a, b) => a.path.localeCompare(b.path));
+    return pages;
+}
+
+/** Serializes the page index to the `export const pages` literal embedded in the routes module. */
+export function pagesModuleSource(pages: readonly PageIndexEntry[]): string {
+    const body = pages.map((p) => `  ${JSON.stringify(p)},`).join('\n');
+    return `export const pages: PageMeta[] = [\n${body}\n];\n`;
+}

package/src/compiler/plugin.ts

@@ -1,7 +1,120 @@
-import { type Plugin } from 'vite';
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 
-import { type ResolvedToilConfig } from './config.js';
+import { type Plugin, version as viteVersion } from 'vite';
+
+import { AiProvider, type DevtoolsAiConfig, type ResolvedToilConfig } from './config.js';
 import { generate } from './generate.js';
+import { scanRoutes } from './routes.js';
+
+/** Calls the configured AI provider (server-side, so the key never reaches the browser). */
+async function aiComplete(ai: DevtoolsAiConfig, prompt: string): Promise<string> {
+    const key = ai.apiKeyEnv ? process.env[ai.apiKeyEnv] : undefined;
+    if (ai.endpoint) {
+        const r = await fetch(ai.endpoint, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ prompt }),
+        });
+        const j = (await r.json()) as { text?: string };
+        return j.text ?? '';
+    }
+    if (ai.provider === AiProvider.OpenAI) {
+        if (!key) throw new Error(`missing API key (set env ${ai.apiKeyEnv ?? 'OPENAI_API_KEY'})`);
+        const r = await fetch('https://api.openai.com/v1/chat/completions', {
+            method: 'POST',
+            headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
+            body: JSON.stringify({
+                model: ai.model ?? 'gpt-4o',
+                messages: [{ role: 'user', content: prompt }],
+            }),
+        });
+        const j = (await r.json()) as { choices?: { message?: { content?: string } }[] };
+        return j.choices?.[0]?.message?.content ?? '';
+    }
+    // default: anthropic
+    if (!key) throw new Error(`missing API key (set env ${ai.apiKeyEnv ?? 'ANTHROPIC_API_KEY'})`);
+    const r = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+            'content-type': 'application/json',
+            'x-api-key': key,
+            'anthropic-version': '2023-06-01',
+        },
+        body: JSON.stringify({
+            model: ai.model ?? 'claude-sonnet-4-6',
+            max_tokens: 1024,
+            messages: [{ role: 'user', content: prompt }],
+        }),
+    });
+    const j = (await r.json()) as { content?: { text?: string }[] };
+    return (j.content ?? []).map((c) => c.text ?? '').join('');
+}
+
+/** Reads a package's version resolved from `<fromDir>`, or 'unknown'. */
+function depVersion(fromDir: string, name: string): string {
+    try {
+        const pkgPath = createRequire(path.join(fromDir, 'package.json')).resolve(`${name}/package.json`);
+        const raw = JSON.parse(fs.readFileSync(pkgPath, 'utf8')) as { version?: string };
+        return raw.version ?? 'unknown';
+    } catch {
+        return 'unknown';
+    }
+}
+
+/** toiljs's own version (package.json two levels up from build/compiler). */
+function frameworkVersion(): string {
+    try {
+        const p = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'package.json');
+        const raw = JSON.parse(fs.readFileSync(p, 'utf8')) as { version?: string };
+        return raw.version ?? '0.0.0';
+    } catch {
+        return '0.0.0';
+    }
+}
+
+/** Build/config snapshot served to the dev toolbar at `/__toil/devinfo`. */
+function devInfo(cfg: ResolvedToilConfig, port: number): Record<string, unknown> {
+    const routes: Record<string, string> = {};
+    for (const r of scanRoutes(cfg.routesAbsDir)) {
+        if (r.slot === undefined && !r.intercept) routes[r.pattern] = r.file;
+    }
+    return {
+        toiljs: frameworkVersion(),
+        // vite is a dependency of the framework, not the app, so resolving it from the app root
+        // fails; read the running vite's own exported version instead.
+        vite: viteVersion,
+        react: depVersion(cfg.root, 'react'),
+        port,
+        enabled: cfg.devtools,
+        flags: {
+            images: cfg.images,
+            fonts: cfg.fonts,
+            viewTransitions: cfg.viewTransitions,
+            transitions: cfg.transitions,
+            seo: cfg.seo != null,
+        },
+        routes,
+        ai: cfg.devtoolsAi !== null,
+    };
+}
+
+/** Opens `file` in the user's editor (best-effort): `$EDITOR file`, else `code -g file`. */
+function openInEditor(file: string): void {
+    try {
+        const editor = process.env.EDITOR ?? process.env.VISUAL;
+        const child = editor
+            ? spawn(editor, [file], { stdio: 'ignore', detached: true })
+            : spawn('code', ['-g', file], { stdio: 'ignore', detached: true });
+        child.on('error', () => undefined);
+        child.unref();
+    } catch {
+        /* ignore */
+    }
+}
 
 /**
  * Vite plugin that keeps the generated route table in sync during dev: when a route file is
@@ -35,6 +148,61 @@ export function toilPlugin(cfg: ResolvedToilConfig): Plugin {
             return null;
         },
         configureServer(server) {
+            // Dev toolbar endpoints (dev only). `/__toil/devinfo` -> build/config snapshot;
+            // `/__toil/open?file=` -> open the file in the editor.
+            server.middlewares.use('/__toil/devinfo', (_req, res) => {
+                const port = server.config.server.port ?? cfg.port;
+                res.setHeader('content-type', 'application/json');
+                res.end(JSON.stringify(devInfo(cfg, port)));
+            });
+            // `/__toil/ai` -> server-side AI proxy. The key is read from the env here and never
+            // reaches the browser; 404 when AI isn't configured (the toolbar then only hands off).
+            server.middlewares.use('/__toil/ai', (req, res) => {
+                if (req.method !== 'POST') {
+                    res.statusCode = 405;
+                    res.end();
+                    return;
+                }
+                const ai = cfg.devtoolsAi;
+                if (!ai) {
+                    res.statusCode = 404;
+                    res.end();
+                    return;
+                }
+                let body = '';
+                req.on('data', (chunk) => (body += String(chunk)));
+                req.on('end', () => {
+                    void (async () => {
+                        try {
+                            const { prompt } = JSON.parse(body || '{}') as { prompt?: string };
+                            const text = await aiComplete(ai, prompt ?? '');
+                            res.setHeader('content-type', 'application/json');
+                            res.end(JSON.stringify({ text }));
+                        } catch (e) {
+                            res.statusCode = 500;
+                            res.setHeader('content-type', 'application/json');
+                            res.end(
+                                JSON.stringify({ error: e instanceof Error ? e.message : String(e) }),
+                            );
+                        }
+                    })();
+                });
+            });
+            server.middlewares.use('/__toil/open', (req, res) => {
+                try {
+                    const url = new URL(req.url ?? '', 'http://localhost');
+                    const file = url.searchParams.get('file');
+                    const abs = file ? path.resolve(file) : '';
+                    // Only files inside the project root, never an arbitrary path.
+                    if (abs && abs.startsWith(cfg.root) && fs.existsSync(abs)) openInEditor(abs);
+                    res.statusCode = 204;
+                    res.end();
+                } catch {
+                    res.statusCode = 400;
+                    res.end();
+                }
+            });
+
             // Trailing slash so a sibling like `routes-extra/` doesn't match the `routes/` prefix.
             const routesPrefix = cfg.routesAbsDir.replace(/\\/g, '/').replace(/\/?$/, '/');
             const onChange = (file: string): void => {

package/src/compiler/prerender.ts

@@ -13,7 +13,7 @@ import { injectSeoHtml, routeSeo } from './seo.js';
 type Ts = typeof TS;
 
 /** Loads the project's TypeScript (used to read each route's static `metadata`), or `null` if absent. */
-async function loadTypeScript(root: string): Promise<Ts | null> {
+export async function loadTypeScript(root: string): Promise<Ts | null> {
     try {
         const resolved = createRequire(path.join(root, 'package.json')).resolve('typescript');
         const mod = (await import(pathToFileURL(resolved).href)) as { default?: Ts } & Ts;
@@ -132,6 +132,10 @@ export function prerenderPlugin(cfg: ResolvedToilConfig): Plugin {
             const shellPath = path.join(outDir, 'index.html');
             if (!fs.existsSync(shellPath)) return;
             const shell = fs.readFileSync(shellPath, 'utf8');
+            // Stash the clean built shell (asset tags, no per-route SEO yet) so the post-build SSG
+            // pass bakes dynamic routes from it rather than from this file once it's been overwritten
+            // with the `/` route's head (which would duplicate canonical/og tags).
+            fs.writeFileSync(path.join(cfg.toilDir, 'shell.html'), shell);
             const ts = await loadTypeScript(cfg.root);
 
             const routes = scanRoutes(cfg.routesAbsDir).filter(

package/src/compiler/seo.ts

@@ -340,17 +340,30 @@ export function robotsTxt(seo: SeoConfig): string {
     return blocks.join('\n\n') + '\n';
 }
 
-/** `sitemap.xml` from the site's static routes (requires `seo.url`); empty when no base URL. */
-export function sitemapXml(seo: SeoConfig, routes: readonly ScannedRoute[]): string {
+/**
+ * `sitemap.xml` from the site's static routes plus any `extra` concrete paths (e.g. SSG URLs from
+ * `generateStaticParams`); requires `seo.url`, empty when no base URL. `extra` is deduped against the
+ * static paths.
+ */
+export function sitemapXml(
+    seo: SeoConfig,
+    routes: readonly ScannedRoute[],
+    extra: readonly string[] = [],
+): string {
     if (seo.url === undefined || seo.sitemap === false) return '';
-    const urls = staticPaths(routes)
+    const paths = [...new Set([...staticPaths(routes), ...extra])];
+    const urls = paths
         .map((p) => `  <url><loc>${escapeHtml(joinUrl(seo.url ?? '', p))}</loc></url>`)
         .join('\n');
     return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`;
 }
 
 /** `llms.txt` (AI-crawler guidance) contents; empty when disabled. */
-export function llmsTxt(seo: SeoConfig, routes: readonly ScannedRoute[]): string {
+export function llmsTxt(
+    seo: SeoConfig,
+    routes: readonly ScannedRoute[],
+    pages?: readonly LlmsPage[],
+): string {
     if (seo.llms === false) return '';
     const cfg: LlmsConfig = seo.llms === true || seo.llms === undefined ? {} : seo.llms;
     const title = cfg.title ?? seo.title ?? seo.url ?? 'Site';
@@ -359,8 +372,11 @@ export function llmsTxt(seo: SeoConfig, routes: readonly ScannedRoute[]): string
     if (summary !== undefined) out.push(`\n> ${summary}`);
     if (cfg.instructions !== undefined) out.push(`\n${cfg.instructions}`);
 
-    const pages: readonly LlmsPage[] =
+    // Page list precedence: an explicit `seo.llms.pages`, else the build-supplied list (every route's
+    // resolved title/description, including SSG-enumerated dynamic pages), else just the static paths.
+    const resolvedPages: readonly LlmsPage[] =
         cfg.pages ??
+        pages ??
         (seo.url !== undefined
             ? staticPaths(routes).map(
                   (p): LlmsPage => ({
@@ -369,9 +385,9 @@ export function llmsTxt(seo: SeoConfig, routes: readonly ScannedRoute[]): string
                   }),
               )
             : []);
-    if (pages.length) {
+    if (resolvedPages.length) {
         out.push('\n## Pages\n');
-        for (const page of pages) {
+        for (const page of resolvedPages) {
             out.push(
                 `- [${page.title}](${page.url})${page.description !== undefined ? `: ${page.description}` : ''}`,
             );