toiljs 0.0.11 → 0.0.14

package/src/compiler/generate.ts

@@ -3,6 +3,7 @@ import path from 'node:path';
 
 import { type ResolvedToilConfig } from './config.js';
 import { writeDocs } from './docs.js';
+import { buildPageIndex, pagesModuleSource } from './pages.js';
 import { scanRoutes, type ScannedRoute } from './routes.js';
 import { llmsTxt, robotsTxt, sitemapXml } from './seo.js';
 
@@ -17,28 +18,28 @@ import { llmsTxt, robotsTxt, sitemapXml } from './seo.js';
 /** Side-effect style imports (e.g. `import './styles/main.css'`). */
 const STYLE_EXTENSIONS = ['css', 'scss', 'sass', 'less', 'styl', 'stylus', 'pcss', 'sss'];
 /** Asset imports whose default export is the resolved URL string (e.g. `import logo from './logo.svg'`). */
-const ASSET_EXTENSIONS = [
-    'svg',
-    'png',
-    'jpg',
-    'jpeg',
-    'gif',
-    'webp',
-    'avif',
-    'ico',
-    'bmp',
-    'apng',
-];
+const ASSET_EXTENSIONS = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'avif', 'ico', 'bmp', 'apng'];
 
 const STYLE_MODULES = STYLE_EXTENSIONS.map((ext) => `declare module '*.${ext}' {}`).join('\n');
 const ASSET_MODULES = ASSET_EXTENSIONS.map(
     (ext) => `declare module '*.${ext}' {\n    const src: string;\n    export default src;\n}`,
 ).join('\n');
 
+/**
+ * Types for vite-imagetools query imports (e.g. `import src from './hero.png?w=800&as=srcset'`).
+ * Declared inline rather than via `/// <reference types="vite-imagetools/client" />` because
+ * vite-imagetools v10 ships no `client` types entry, and the package lives in toiljs's own
+ * node_modules (unresolvable from a symlinked consumer). One `*` wildcard each, matching any
+ * specifier ending in the directive.
+ */
+const IMAGETOOLS_MODULES = [
+    `declare module '*as=srcset' {\n    const src: string;\n    export default src;\n}`,
+    `declare module '*as=url' {\n    const src: string;\n    export default src;\n}`,
+    `declare module '*as=metadata' {\n    const metadata: { src: string; width?: number; height?: number; format?: string }[];\n    export default metadata;\n}`,
+].join('\n');
+
 export const TOIL_ENV_DTS =
     `// AUTO-GENERATED by toil, do not edit.\n` +
-    // Types for image-optimization query imports (`import img from './x.png?w=400&format=webp'`).
-    `/// <reference types="vite-imagetools/client" />\n` +
     `declare const Toil: typeof import('toiljs/client');\n` +
     `declare namespace Toil {\n` +
     `    type LoaderArgs = import('toiljs/client').LoaderArgs;\n` +
@@ -49,6 +50,8 @@ export const TOIL_ENV_DTS =
     `    type RouteErrorProps = import('toiljs/client').RouteErrorProps;\n` +
     `    type Href = import('toiljs/client').Href;\n` +
     `    type RoutePath = import('toiljs/client').RoutePath;\n` +
+    `    type PageMeta = import('toiljs/client').PageMeta;\n` +
+    `    type SearchHints = import('toiljs/client').SearchHints;\n` +
     `}\n` +
     `declare const BinaryWriter: typeof import('toiljs/io').BinaryWriter;\n` +
     `declare const BinaryReader: typeof import('toiljs/io').BinaryReader;\n` +
@@ -59,12 +62,15 @@ export const TOIL_ENV_DTS =
     `\n` +
     `${ASSET_MODULES}\n` +
     `\n` +
+    `${IMAGETOOLS_MODULES}\n` +
+    `\n` +
     `declare module 'toiljs/routes' {\n` +
     `    export const routes: import('toiljs/client').RouteDef[];\n` +
     `    export const layout: import('toiljs/client').LayoutLoader;\n` +
     `    export const notFound: import('toiljs/client').NotFoundLoader;\n` +
     `    export const globalError: import('toiljs/client').ErrorComponentLoader;\n` +
     `    export const slots: Record<string, import('toiljs/client').RouteDef[]>;\n` +
+    `    export const pages: import('toiljs/client').PageMeta[];\n` +
     `}\n`;
 
 /**
@@ -73,7 +79,10 @@ export const TOIL_ENV_DTS =
  * `allowImportingTsExtensions` (TS5097) when the generated files are checked; Vite still resolves it.
  */
 function relFromToil(cfg: ResolvedToilConfig, abs: string): string {
-    let rel = path.relative(cfg.toilDir, abs).replace(/\\/g, '/').replace(/\.(tsx|jsx)$/, '');
+    let rel = path
+        .relative(cfg.toilDir, abs)
+        .replace(/\\/g, '/')
+        .replace(/\.(tsx|jsx)$/, '');
     if (!rel.startsWith('.')) rel = './' + rel;
     return rel;
 }
@@ -111,7 +120,9 @@ function routePathUnion(routes: ScannedRoute[]): string {
             members.add(`'${route.pattern}'`);
             continue;
         }
-        const parts = segments.map((s) => (s.startsWith(':') || s.startsWith('*') ? '${string}' : s));
+        const parts = segments.map((s) =>
+            s.startsWith(':') || s.startsWith('*') ? '${string}' : s,
+        );
         members.add('`/' + parts.join('/') + '`');
         const optionalIdx = segments.findIndex((s) => s.startsWith('**'));
         if (optionalIdx !== -1) {
@@ -128,14 +139,19 @@ function routePathUnion(routes: ScannedRoute[]): string {
  */
 function routesDts(cfg: ResolvedToilConfig, routes: ScannedRoute[]): string {
     // Type-only namespace import of every route module (erased at build) so editors don't flag a
-    // route's `loader` / `metadata` / `generateMetadata` / `revalidate` / `default` exports as unused
-    // — the compiler consumes them via dynamic `import()`, which editors don't count as a reference.
+    // route's `loader` / `metadata` / `generateMetadata` / `revalidate` / `default` exports as unused,
+    // the compiler consumes them via dynamic `import()`, which editors don't count as a reference.
     const refs = routes.map((route, i) => {
-        let rel = path.relative(cfg.root, route.file).replace(/\\/g, '/').replace(/\.(tsx|jsx)$/, '');
+        let rel = path
+            .relative(cfg.root, route.file)
+            .replace(/\\/g, '/')
+            .replace(/\.(tsx|jsx)$/, '');
         if (!rel.startsWith('.')) rel = `./${rel}`;
         return { name: `_toilRoute${String(i)}`, rel };
     });
-    const imports = refs.map((m) => `import type * as ${m.name} from ${JSON.stringify(m.rel)};\n`).join('');
+    const imports = refs
+        .map((m) => `import type * as ${m.name} from ${JSON.stringify(m.rel)};\n`)
+        .join('');
     const referenced = refs.length
         ? `export type _ToilRouteModules = [${refs.map((m) => `typeof ${m.name}`).join(', ')}];\n`
         : `export {};\n`;
@@ -255,24 +271,28 @@ export function generate(cfg: ResolvedToilConfig): ScannedRoute[] {
             return `  ${JSON.stringify(name)}: [\n${items}\n  ],`;
         })
         .join('\n');
+    const pages = buildPageIndex(cfg.root, routes);
     const routesSrc =
         `// @ts-nocheck\n` +
         `// AUTO-GENERATED by toil, do not edit.\n` +
-        `import type { RouteDef, LayoutLoader, NotFoundLoader } from 'toiljs/client';\n\n` +
+        `import type { RouteDef, LayoutLoader, NotFoundLoader, PageMeta } from 'toiljs/client';\n\n` +
         `export const routes: RouteDef[] = [\n${mainRoutes.map((r) => `  ${routeObj(r)},`).join('\n')}\n];\n\n` +
         `export const slots: Record<string, RouteDef[]> = {\n${slotsBody}\n};\n\n` +
         `export const layout: LayoutLoader = ${layoutFile ? `() => import(${JSON.stringify(relFromToil(cfg, layoutFile))})` : 'null'};\n` +
         `export const notFound: NotFoundLoader = ${notFoundFile ? `() => import(${JSON.stringify(relFromToil(cfg, notFoundFile))})` : 'null'};\n` +
-        `export const globalError = ${globalErrorFile ? `() => import(${JSON.stringify(relFromToil(cfg, globalErrorFile))})` : 'null'};\n`;
+        `export const globalError = ${globalErrorFile ? `() => import(${JSON.stringify(relFromToil(cfg, globalErrorFile))})` : 'null'};\n\n` +
+        pagesModuleSource(pages);
     fs.writeFileSync(path.join(cfg.toilDir, 'routes.ts'), routesSrc);
 
     const globalsSrc =
         `// @ts-nocheck\n` +
         `// AUTO-GENERATED by toil, do not edit.\n` +
         `import * as Toil from 'toiljs/client';\n` +
-        `import { BinaryWriter, BinaryReader, FastMap, FastSet } from 'toiljs/io';\n\n` +
+        `import { BinaryWriter, BinaryReader, FastMap, FastSet } from 'toiljs/io';\n` +
+        `import { pages } from './routes';\n\n` +
         `Object.assign(globalThis, { Toil, BinaryWriter, BinaryReader, FastMap, FastSet });\n` +
-        `Toil.setViewTransitions(${String(cfg.viewTransitions)});\n`;
+        `Toil.setViewTransitions(${String(cfg.viewTransitions)});\n` +
+        `Toil.registerPages(pages);\n`;
     fs.writeFileSync(path.join(cfg.toilDir, 'globals.ts'), globalsSrc);
 
     const entryFile = findEntry(cfg);
@@ -338,9 +358,7 @@ const ENTRY_SCRIPT = `<script type="module" src="./entry.tsx"></script>`;
  */
 function buildHtml(cfg: ResolvedToilConfig): string {
     const templatePath = path.join(cfg.publicDir, 'index.html');
-    let html = fs.existsSync(templatePath)
-        ? fs.readFileSync(templatePath, 'utf8')
-        : DEFAULT_HTML;
+    let html = fs.existsSync(templatePath) ? fs.readFileSync(templatePath, 'utf8') : DEFAULT_HTML;
     // Inject the entry only if the template doesn't already reference it as a module script
     // (matching the literal filename anywhere in the file would be too eager).
     if (!/src=["']\.\/entry\.tsx["']/.test(html)) {

package/src/compiler/image-report.ts

@@ -1,85 +1,88 @@
-import fs from 'node:fs';
-import path from 'node:path';
-
-import pc from 'picocolors';
-import type { Logger, Plugin } from 'vite';
-
-/** Raster/vector outputs the image pipeline may emit. */
-const IMAGE_RE = /\.(png|jpe?g|webp|avif|gif|tiff|svg)$/i;
-
-/** Formats a byte count like Vite's asset table (kB, base 1000). */
-function kb(bytes: number): string {
-    return `${(bytes / 1000).toFixed(2)} kB`;
-}
-
-interface Variant {
-    readonly out: string;
-    readonly outSize: number;
-}
-
-/**
- * Build-only plugin that reports which imported images the pipeline optimized, each source image,
- * its emitted variant(s), and the size saved. `public/` assets (copied as-is) never enter the
- * bundle, so they don't appear here. Logs nothing when no images were processed.
- *
- * `viteRoot` is Vite's root (the `.toil` dir) that emitted assets' `originalFileNames` are relative
- * to; `projectRoot` is used only to print friendly source paths.
- */
-export function imageReportPlugin(projectRoot: string, viteRoot: string): Plugin {
-    let logger: Logger | undefined;
-    return {
-        name: 'toil:image-report',
-        apply: 'build',
-        configResolved(config) {
-            logger = config.logger;
-        },
-        writeBundle(_options, bundle) {
-            // Group emitted image assets by their source file.
-            const bySource = new Map<string, { label: string; inSize: number | null; variants: Variant[] }>();
-            for (const file of Object.values(bundle)) {
-                if (file.type !== 'asset' || !IMAGE_RE.test(file.fileName)) continue;
-                const source = file.originalFileNames[0];
-                const key = source ?? file.fileName;
-                const outSize =
-                    typeof file.source === 'string'
-                        ? Buffer.byteLength(file.source)
-                        : file.source.byteLength;
-
-                let entry = bySource.get(key);
-                if (!entry) {
-                    let inSize: number | null = null;
-                    let label = '(generated)';
-                    if (source !== undefined) {
-                        const abs = path.resolve(viteRoot, source);
-                        label = path.relative(projectRoot, abs);
-                        try {
-                            inSize = fs.statSync(abs).size;
-                        } catch {
-                            inSize = null;
-                        }
-                    }
-                    entry = { label, inSize, variants: [] };
-                    bySource.set(key, entry);
-                }
-                entry.variants.push({ out: file.fileName, outSize });
-            }
-
-            if (bySource.size === 0 || !logger) return;
-
-            const count = bySource.size;
-            logger.info('');
-            logger.info(pc.green(`  ✓ optimized ${String(count)} image${count === 1 ? '' : 's'}`));
-            for (const { label, inSize, variants } of bySource.values()) {
-                logger.info(`  ${pc.dim(label)}`);
-                for (const v of variants) {
-                    const saved =
-                        inSize && inSize > 0
-                            ? pc.green(`  -${String(Math.round((1 - v.outSize / inSize) * 100))}%`)
-                            : '';
-                    const from = inSize ? `${kb(inSize)} → ` : '';
-                    logger.info(`    ${pc.dim('→')} ${v.out}  ${from}${kb(v.outSize)}${saved}`);
-                }
-            }
-        },
-    };
-}
+import fs from 'node:fs';
+import path from 'node:path';
+
+import pc from 'picocolors';
+import type { Logger, Plugin } from 'vite';
+
+/** Raster/vector outputs the image pipeline may emit. */
+const IMAGE_RE = /\.(png|jpe?g|webp|avif|gif|tiff|svg)$/i;
+
+/** Formats a byte count like Vite's asset table (kB, base 1000). */
+function kb(bytes: number): string {
+    return `${(bytes / 1000).toFixed(2)} kB`;
+}
+
+interface Variant {
+    readonly out: string;
+    readonly outSize: number;
+}
+
+/**
+ * Build-only plugin that reports which imported images the pipeline optimized, each source image,
+ * its emitted variant(s), and the size saved. `public/` assets (copied as-is) never enter the
+ * bundle, so they don't appear here. Logs nothing when no images were processed.
+ *
+ * `viteRoot` is Vite's root (the `.toil` dir) that emitted assets' `originalFileNames` are relative
+ * to; `projectRoot` is used only to print friendly source paths.
+ */
+export function imageReportPlugin(projectRoot: string, viteRoot: string): Plugin {
+    let logger: Logger | undefined;
+    return {
+        name: 'toil:image-report',
+        apply: 'build',
+        configResolved(config) {
+            logger = config.logger;
+        },
+        writeBundle(_options, bundle) {
+            // Group emitted image assets by their source file.
+            const bySource = new Map<
+                string,
+                { label: string; inSize: number | null; variants: Variant[] }
+            >();
+            for (const file of Object.values(bundle)) {
+                if (file.type !== 'asset' || !IMAGE_RE.test(file.fileName)) continue;
+                const source = file.originalFileNames[0];
+                const key = source ?? file.fileName;
+                const outSize =
+                    typeof file.source === 'string'
+                        ? Buffer.byteLength(file.source)
+                        : file.source.byteLength;
+
+                let entry = bySource.get(key);
+                if (!entry) {
+                    let inSize: number | null = null;
+                    let label = '(generated)';
+                    if (source !== undefined) {
+                        const abs = path.resolve(viteRoot, source);
+                        label = path.relative(projectRoot, abs);
+                        try {
+                            inSize = fs.statSync(abs).size;
+                        } catch {
+                            inSize = null;
+                        }
+                    }
+                    entry = { label, inSize, variants: [] };
+                    bySource.set(key, entry);
+                }
+                entry.variants.push({ out: file.fileName, outSize });
+            }
+
+            if (bySource.size === 0 || !logger) return;
+
+            const count = bySource.size;
+            logger.info('');
+            logger.info(pc.green(`  ✓ optimized ${String(count)} image${count === 1 ? '' : 's'}`));
+            for (const { label, inSize, variants } of bySource.values()) {
+                logger.info(`  ${pc.dim(label)}`);
+                for (const v of variants) {
+                    const saved =
+                        inSize && inSize > 0
+                            ? pc.green(`  -${String(Math.round((1 - v.outSize / inSize) * 100))}%`)
+                            : '';
+                    const from = inSize ? `${kb(inSize)} → ` : '';
+                    logger.info(`    ${pc.dim('→')} ${v.out}  ${from}${kb(v.outSize)}${saved}`);
+                }
+            }
+        },
+    };
+}

package/src/compiler/index.ts

@@ -45,6 +45,8 @@ export async function start(opts: ToilCommandOptions = {}): Promise<RunningBacke
 }
 
 export { defineConfig, loadConfig } from './config.js';
+export { scanRoutes } from './routes.js';
+export type { ScannedRoute } from './routes.js';
 export { TOIL_ENV_DTS } from './generate.js';
 export { AI_HELPERS, AI_HELPER_IDS, aiHelperFiles, TOIL_DOCS } from './docs.js';
 export type { AiHelper } from './docs.js';

package/src/compiler/pages.ts

@@ -0,0 +1,70 @@
+import { createRequire } from 'node:module';
+import path from 'node:path';
+
+import type * as TS from 'typescript';
+
+import { extractStaticExports } from './prerender.js';
+import type { ScannedRoute } from './routes.js';
+
+type Ts = typeof TS;
+
+/**
+ * A page in the build-time search index: its URL pattern, whether it's dynamic, and the
+ * statically-extracted `metadata` literal (the searchable subset; dynamic `generateMetadata` and
+ * computed values can't be known at build, so they're absent). Serialized into the generated
+ * `routes` module and registered client-side for {@link searchPages}.
+ */
+export interface PageIndexEntry {
+    readonly path: string;
+    readonly dynamic: boolean;
+    readonly metadata: Record<string, unknown>;
+}
+
+/**
+ * Loads the project's TypeScript synchronously (so {@link buildPageIndex} can run inside the sync
+ * `generate()`), or `null` if it isn't installed, in which case pages are indexed by path only.
+ */
+function loadTypeScriptSync(root: string): Ts | null {
+    try {
+        const require = createRequire(path.join(root, 'package.json'));
+        const mod = require('typescript') as { default?: Ts } & Ts;
+        return mod.default ?? mod;
+    } catch {
+        return null;
+    }
+}
+
+/** True when a route pattern has dynamic (`:param` / `*catch-all`) segments. */
+function isDynamic(pattern: string): boolean {
+    return /[:*]/.test(pattern);
+}
+
+/**
+ * Builds the searchable page index from the scanned routes: every main-tree page (slots and
+ * intercepting routes are excluded, they don't own a distinct URL) paired with its statically
+ * extracted `metadata`. A route may also `export const searchHints` (a static `title`/`description`/
+ * `keywords` object) to feed the index even when its real metadata is dynamic (`generateMetadata`);
+ * hints are merged over the static `metadata`, winning ties. Reads each route file once.
+ */
+export function buildPageIndex(root: string, routes: readonly ScannedRoute[]): PageIndexEntry[] {
+    const ts = loadTypeScriptSync(root);
+    const seen = new Set<string>();
+    const pages: PageIndexEntry[] = [];
+    for (const route of routes) {
+        if (route.slot !== undefined || route.intercept) continue;
+        if (seen.has(route.pattern)) continue;
+        seen.add(route.pattern);
+        const exports = ts ? extractStaticExports(ts, route.file, ['metadata', 'searchHints']) : {};
+        const metadata = { ...exports.metadata, ...exports.searchHints };
+        pages.push({ path: route.pattern, dynamic: isDynamic(route.pattern), metadata });
+    }
+    // Stable order (by path) so the generated module is deterministic across runs.
+    pages.sort((a, b) => a.path.localeCompare(b.path));
+    return pages;
+}
+
+/** Serializes the page index to the `export const pages` literal embedded in the routes module. */
+export function pagesModuleSource(pages: readonly PageIndexEntry[]): string {
+    const body = pages.map((p) => `  ${JSON.stringify(p)},`).join('\n');
+    return `export const pages: PageMeta[] = [\n${body}\n];\n`;
+}

package/src/compiler/plugin.ts

@@ -1,47 +1,51 @@
-import { type Plugin } from 'vite';
-
-import { type ResolvedToilConfig } from './config.js';
-import { generate } from './generate.js';
-
-/**
- * Vite plugin that keeps the generated route table in sync during dev: when a route file is
- * added or removed, it regenerates `.toil/routes.ts` and triggers a full reload. Editing a
- * route file's contents hot-reloads through `@vitejs/plugin-react` as usual.
- */
-export function toilPlugin(cfg: ResolvedToilConfig): Plugin {
-    return {
-        name: 'toil',
-        // Catch empty import specifiers in source and report the file, rolldown otherwise fails
-        // resolution with a cryptic "The specifiers must be a non-empty string. Received ''".
-        transform(code, id) {
-            const file = id.split('?')[0];
-            if (id.includes('\0') || file.includes('/node_modules/') || !/\.[mc]?[jt]sx?$/.test(file)) {
-                return null;
-            }
-            const empty =
-                /^[ \t]*import\s+(['"])\1\s*;?[ \t]*$/m.test(code) ||
-                /^[ \t]*import\b[^'"\n]*\bfrom\s+(['"])\1/m.test(code) ||
-                /^[ \t]*export\b[^'"\n]*\bfrom\s+(['"])\1/m.test(code) ||
-                /\bimport\s*\(\s*(['"])\1\s*\)/.test(code);
-            if (empty) {
-                throw new Error(
-                    `toil: empty import specifier (e.g. \`import '';\`) in ${file}, remove or complete the import.`,
-                );
-            }
-            return null;
-        },
-        configureServer(server) {
-            // Trailing slash so a sibling like `routes-extra/` doesn't match the `routes/` prefix.
-            const routesPrefix = cfg.routesAbsDir.replace(/\\/g, '/').replace(/\/?$/, '/');
-            const onChange = (file: string): void => {
-                if (file.replace(/\\/g, '/').startsWith(routesPrefix)) {
-                    generate(cfg);
-                    server.ws.send({ type: 'full-reload' });
-                }
-            };
-            server.watcher.add(cfg.routesAbsDir);
-            server.watcher.on('add', onChange);
-            server.watcher.on('unlink', onChange);
-        },
-    };
-}
+import { type Plugin } from 'vite';
+
+import { type ResolvedToilConfig } from './config.js';
+import { generate } from './generate.js';
+
+/**
+ * Vite plugin that keeps the generated route table in sync during dev: when a route file is
+ * added or removed, it regenerates `.toil/routes.ts` and triggers a full reload. Editing a
+ * route file's contents hot-reloads through `@vitejs/plugin-react` as usual.
+ */
+export function toilPlugin(cfg: ResolvedToilConfig): Plugin {
+    return {
+        name: 'toil',
+        // Catch empty import specifiers in source and report the file, rolldown otherwise fails
+        // resolution with a cryptic "The specifiers must be a non-empty string. Received ''".
+        transform(code, id) {
+            const file = id.split('?')[0];
+            if (
+                id.includes('\0') ||
+                file.includes('/node_modules/') ||
+                !/\.[mc]?[jt]sx?$/.test(file)
+            ) {
+                return null;
+            }
+            const empty =
+                /^[ \t]*import\s+(['"])\1\s*;?[ \t]*$/m.test(code) ||
+                /^[ \t]*import\b[^'"\n]*\bfrom\s+(['"])\1/m.test(code) ||
+                /^[ \t]*export\b[^'"\n]*\bfrom\s+(['"])\1/m.test(code) ||
+                /\bimport\s*\(\s*(['"])\1\s*\)/.test(code);
+            if (empty) {
+                throw new Error(
+                    `toil: empty import specifier (e.g. \`import '';\`) in ${file}, remove or complete the import.`,
+                );
+            }
+            return null;
+        },
+        configureServer(server) {
+            // Trailing slash so a sibling like `routes-extra/` doesn't match the `routes/` prefix.
+            const routesPrefix = cfg.routesAbsDir.replace(/\\/g, '/').replace(/\/?$/, '/');
+            const onChange = (file: string): void => {
+                if (file.replace(/\\/g, '/').startsWith(routesPrefix)) {
+                    generate(cfg);
+                    server.ws.send({ type: 'full-reload' });
+                }
+            };
+            server.watcher.add(cfg.routesAbsDir);
+            server.watcher.on('add', onChange);
+            server.watcher.on('unlink', onChange);
+        },
+    };
+}