toiljs 0.0.15 → 0.0.19

package/src/compiler/ssg.ts

@@ -1,126 +1,171 @@
-/**
- * Build-time SSG for dynamic routes. After the client bundle is written, this loads each dynamic
- * route that exports `generateStaticParams`, enumerates its concrete URLs, runs the route's
- * `generateMetadata` per URL, and bakes a `<url>/index.html` (so JS-less crawlers get per-page tags)
- * plus a `sitemap.xml` entry. Opt-in: a route without `generateStaticParams` is untouched, and the
- * whole pass is skipped when no such route exists or `seo` is unconfigured. Build-only.
- *
- * Runs from `build()` (not the prerender Vite plugin) so it can reuse `createViteConfig` without the
- * `vite.ts` <-> `prerender.ts` import cycle; it spins up a short-lived SSR server to load route source.
- */
-import fs from 'node:fs';
-import path from 'node:path';
-
-import { createServer } from 'vite';
-
-import { type ResolvedToilConfig } from './config.js';
-import { scanRoutes } from './routes.js';
-import { injectSeoHtml, routeSeo, sitemapXml } from './seo.js';
-import { createViteConfig } from './vite.js';
-
-type StaticParams = Record<string, string | string[]>;
-
-interface RouteModule {
-    generateStaticParams?: () => StaticParams[] | Promise<StaticParams[]>;
-    generateMetadata?: (args: {
-        params: StaticParams;
-        searchParams: URLSearchParams;
-        data: unknown;
-    }) => unknown;
-    loader?: (args: { params: StaticParams; searchParams: URLSearchParams }) => unknown;
-    metadata?: Record<string, unknown>;
-}
-
-/** Substitutes `:param` / `*catch-all` segments in a route pattern with concrete param values. */
-export function fillPattern(pattern: string, params: StaticParams): string {
-    return pattern.replace(/[:*]([A-Za-z0-9_]+)/g, (_m, name: string) => {
-        const value = params[name] as string | string[] | undefined;
-        if (Array.isArray(value)) return value.join('/');
-        return value ?? '';
-    });
-}
-
-/** Coerces an unknown module export to a typed Metadata-ish record, or null. */
-function asMetadata(value: unknown): Record<string, unknown> | null {
-    return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null;
-}
-
-/**
- * Pre-renders every dynamic route that opts in via `generateStaticParams`. Bakes per-URL HTML into
- * `outDir` and rewrites `sitemap.xml` with the generated URLs. Returns the list of generated URLs.
- */
-export async function prerenderStaticParams(cfg: ResolvedToilConfig): Promise<string[]> {
-    if (!cfg.seo) return [];
-    const outDir = path.resolve(cfg.root, cfg.outDir);
-    // Prefer the clean shell stashed by the prerender plugin (no per-route SEO baked in); fall back
-    // to the built index.html.
-    const stashed = path.join(cfg.toilDir, 'shell.html');
-    const shellPath = fs.existsSync(stashed) ? stashed : path.join(outDir, 'index.html');
-    if (!fs.existsSync(shellPath)) return [];
-
-    const allRoutes = scanRoutes(cfg.routesAbsDir);
-    const dynamic = allRoutes.filter(
-        (r) => r.slot === undefined && !r.intercept && /[:*]/.test(r.pattern),
-    );
-    if (dynamic.length === 0) return [];
-
-    const shell = fs.readFileSync(shellPath, 'utf8');
-    const server = await createServer({
-        ...(await createViteConfig(cfg)),
-        server: { middlewareMode: true, hmr: false },
-        appType: 'custom',
-        logLevel: 'silent',
-    });
-
-    const generated: string[] = [];
-    const warn = (msg: string): void => {
-        process.stderr.write(`  toil: SSG ${msg}\n`);
-    };
-    try {
-        for (const route of dynamic) {
-            let mod: RouteModule;
-            try {
-                mod = (await server.ssrLoadModule(route.file)) as RouteModule;
-            } catch (err) {
-                warn(`skipped ${route.pattern} (${err instanceof Error ? err.message : String(err)})`);
-                continue;
-            }
-            if (typeof mod.generateStaticParams !== 'function') continue;
-            const paramSets = await mod.generateStaticParams();
-            for (const params of paramSets) {
-                const url = fillPattern(route.pattern, params);
-                let metadata: Record<string, unknown> | null = null;
-                try {
-                    if (typeof mod.generateMetadata === 'function') {
-                        const searchParams = new URLSearchParams();
-                        const data =
-                            typeof mod.loader === 'function'
-                                ? await mod.loader({ params, searchParams })
-                                : undefined;
-                        metadata = asMetadata(await mod.generateMetadata({ params, searchParams, data }));
-                    } else if (mod.metadata) {
-                        metadata = asMetadata(mod.metadata);
-                    }
-                } catch (err) {
-                    warn(`metadata failed for ${url} (${err instanceof Error ? err.message : String(err)})`);
-                }
-                const html = injectSeoHtml(shell, routeSeo(cfg.seo, metadata, url));
-                const target = path.join(outDir, url.replace(/^\//, ''), 'index.html');
-                fs.mkdirSync(path.dirname(target), { recursive: true });
-                fs.writeFileSync(target, html);
-                generated.push(url);
-            }
-        }
-    } finally {
-        await server.close();
-    }
-
-    if (generated.length > 0) {
-        const sitemap = sitemapXml(cfg.seo, allRoutes, generated);
-        if (sitemap) fs.writeFileSync(path.join(outDir, 'sitemap.xml'), sitemap);
-        process.stdout.write(
-            `  ✓ prerendered ${String(generated.length)} dynamic route${generated.length === 1 ? '' : 's'}\n`,
-        );
-    }
-    return generated;
-}
+/**
+ * Build-time SSG for dynamic routes. After the client bundle is written, this loads each dynamic
+ * route that exports `generateStaticParams`, enumerates its concrete URLs, runs the route's
+ * `generateMetadata` per URL, and bakes a `<url>/index.html` (so JS-less crawlers get per-page tags)
+ * plus a `sitemap.xml` entry. Opt-in: a route without `generateStaticParams` is untouched, and the
+ * whole pass is skipped when no such route exists or `seo` is unconfigured. Build-only.
+ *
+ * Runs from `build()` (not the prerender Vite plugin) so it can reuse `createViteConfig` without the
+ * `vite.ts` <-> `prerender.ts` import cycle; it spins up a short-lived SSR server to load route source.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { createServer } from 'vite';
+
+import { type ResolvedToilConfig } from './config.js';
+import { extractStaticMetadata, loadTypeScript } from './prerender.js';
+import { scanRoutes } from './routes.js';
+import { injectSeoHtml, joinUrl, llmsTxt, routeSeo, sitemapXml, type LlmsPage } from './seo.js';
+import { createViteConfig } from './vite.js';
+
+/** Reads a string field off a metadata record, or undefined. */
+function metaString(meta: Record<string, unknown> | null, key: string): string | undefined {
+    const value = meta?.[key];
+    return typeof value === 'string' ? value : undefined;
+}
+
+type StaticParams = Record<string, string | string[]>;
+
+interface RouteModule {
+    generateStaticParams?: () => StaticParams[] | Promise<StaticParams[]>;
+    generateMetadata?: (args: {
+        params: StaticParams;
+        searchParams: URLSearchParams;
+        data: unknown;
+    }) => unknown;
+    loader?: (args: { params: StaticParams; searchParams: URLSearchParams }) => unknown;
+    metadata?: Record<string, unknown>;
+}
+
+/** Substitutes `:param` / `*catch-all` segments in a route pattern with concrete param values. */
+export function fillPattern(pattern: string, params: StaticParams): string {
+    return pattern.replace(/[:*]([A-Za-z0-9_]+)/g, (_m, name: string) => {
+        const value = params[name] as string | string[] | undefined;
+        if (Array.isArray(value)) return value.join('/');
+        return value ?? '';
+    });
+}
+
+/** Coerces an unknown module export to a typed Metadata-ish record, or null. */
+function asMetadata(value: unknown): Record<string, unknown> | null {
+    return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : null;
+}
+
+/**
+ * Pre-renders every dynamic route that opts in via `generateStaticParams`. Bakes per-URL HTML into
+ * `outDir` and rewrites `sitemap.xml` with the generated URLs. Returns the list of generated URLs.
+ */
+export async function prerenderStaticParams(cfg: ResolvedToilConfig): Promise<string[]> {
+    if (!cfg.seo) return [];
+    const outDir = path.resolve(cfg.root, cfg.outDir);
+    // Prefer the clean shell stashed by the prerender plugin (no per-route SEO baked in); fall back
+    // to the built index.html.
+    const stashed = path.join(cfg.toilDir, 'shell.html');
+    const shellPath = fs.existsSync(stashed) ? stashed : path.join(outDir, 'index.html');
+    if (!fs.existsSync(shellPath)) return [];
+
+    const allRoutes = scanRoutes(cfg.routesAbsDir);
+    const dynamic = allRoutes.filter(
+        (r) => r.slot === undefined && !r.intercept && /[:*]/.test(r.pattern),
+    );
+    if (dynamic.length === 0) return [];
+
+    const shell = fs.readFileSync(shellPath, 'utf8');
+    const server = await createServer({
+        ...(await createViteConfig(cfg)),
+        server: { middlewareMode: true, hmr: false },
+        appType: 'custom',
+        logLevel: 'silent',
+    });
+
+    const baseUrl = cfg.seo.url;
+    const generated: string[] = [];
+    const dynamicPages: LlmsPage[] = [];
+    const warn = (msg: string): void => {
+        process.stderr.write(`  toil: SSG ${msg}\n`);
+    };
+    try {
+        for (const route of dynamic) {
+            let mod: RouteModule;
+            try {
+                mod = (await server.ssrLoadModule(route.file)) as RouteModule;
+            } catch (err) {
+                warn(`skipped ${route.pattern} (${err instanceof Error ? err.message : String(err)})`);
+                continue;
+            }
+            if (typeof mod.generateStaticParams !== 'function') continue;
+            const paramSets = await mod.generateStaticParams();
+            for (const params of paramSets) {
+                const url = fillPattern(route.pattern, params);
+                // Containment guard: a param value with `..`/separators could make `url` resolve the
+                // output path outside `outDir`. Resolve the target up front and skip anything that
+                // escapes, so a (possibly externally-derived) param can't clobber files elsewhere.
+                const target = path.join(outDir, url.replace(/^\//, ''), 'index.html');
+                const outRoot = path.resolve(outDir);
+                const absTarget = path.resolve(target);
+                if (absTarget !== outRoot && !absTarget.startsWith(outRoot + path.sep)) {
+                    warn(`skipped ${route.pattern}: params escape outDir (${url})`);
+                    continue;
+                }
+                let metadata: Record<string, unknown> | null = null;
+                try {
+                    if (typeof mod.generateMetadata === 'function') {
+                        const searchParams = new URLSearchParams();
+                        const data =
+                            typeof mod.loader === 'function'
+                                ? await mod.loader({ params, searchParams })
+                                : undefined;
+                        metadata = asMetadata(await mod.generateMetadata({ params, searchParams, data }));
+                    } else if (mod.metadata) {
+                        metadata = asMetadata(mod.metadata);
+                    }
+                } catch (err) {
+                    warn(`metadata failed for ${url} (${err instanceof Error ? err.message : String(err)})`);
+                }
+                const html = injectSeoHtml(shell, routeSeo(cfg.seo, metadata, url));
+                fs.mkdirSync(path.dirname(target), { recursive: true });
+                fs.writeFileSync(target, html);
+                generated.push(url);
+                dynamicPages.push({
+                    title: metaString(metadata, 'title') ?? url,
+                    url: baseUrl !== undefined ? joinUrl(baseUrl, url) : url,
+                    description: metaString(metadata, 'description'),
+                });
+            }
+        }
+    } finally {
+        await server.close();
+    }
+
+    if (generated.length > 0) {
+        const sitemap = sitemapXml(cfg.seo, allRoutes, generated);
+        if (sitemap) fs.writeFileSync(path.join(outDir, 'sitemap.xml'), sitemap);
+        process.stdout.write(
+            `  ✓ prerendered ${String(generated.length)} dynamic route${generated.length === 1 ? '' : 's'}\n`,
+        );
+    }
+
+    // Rewrite llms.txt with the full page index: every static route's resolved title/description plus
+    // the enumerated dynamic pages, so AI crawlers get the whole site, not just the static paths.
+    if (baseUrl !== undefined) {
+        const ts = await loadTypeScript(cfg.root);
+        const staticPages: LlmsPage[] = allRoutes
+            .filter((r) => r.slot === undefined && !r.intercept && !/[:*]/.test(r.pattern))
+            .map((r): LlmsPage => {
+                const meta = ts ? extractStaticMetadata(ts, r.file) : null;
+                return {
+                    title: metaString(meta, 'title') ?? (r.pattern === '/' ? 'Home' : r.pattern),
+                    url: joinUrl(baseUrl, r.pattern),
+                    description: metaString(meta, 'description'),
+                };
+            });
+        const pages = [...staticPages, ...dynamicPages];
+        if (pages.length > 0) {
+            const llms = llmsTxt(cfg.seo, allRoutes, pages);
+            if (llms) fs.writeFileSync(path.join(outDir, 'llms.txt'), llms);
+        }
+    }
+
+    return generated;
+}

package/src/compiler/vite.ts

@@ -1,3 +1,4 @@
+import fs from 'node:fs';
 import { createRequire } from 'node:module';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
@@ -41,6 +42,36 @@ const IMAGE_EXT = /^(png|jpe?g|svg|gif|tiff|bmp|ico|webp|avif)$/i;
 /** Font extensions routed to `fonts/`. */
 const FONT_EXT = /^(woff|woff2|eot|ttf|otf)$/i;
 
+/**
+ * Resolves bare `shared/*` imports to the project's `shared/` folder (replacing a plain alias so it
+ * can run early), and fails `shared/server` with an actionable message when the module has not been
+ * generated yet (it comes from the server build, which must run first), instead of Vite's opaque
+ * `UNLOADABLE_DEPENDENCY`.
+ */
+function sharedResolverPlugin(cfg: ResolvedToilConfig): PluginOption {
+    const sharedDir = path.join(cfg.root, 'shared');
+    return {
+        name: 'toiljs:shared-resolver',
+        enforce: 'pre',
+        resolveId(source: string) {
+            if (source !== 'shared' && !source.startsWith('shared/')) return null;
+            const rel = source === 'shared' ? 'index' : source.slice('shared/'.length);
+            for (const ext of ['', '.ts', '.tsx', '.js', '.jsx', '.mjs']) {
+                const candidate = path.join(sharedDir, rel + ext);
+                if (fs.existsSync(candidate)) return candidate;
+            }
+            if (source === 'shared/server') {
+                throw new Error(
+                    'toiljs: "shared/server" is generated by the server build but is missing. ' +
+                        'Run the server build first (it emits shared/server.ts from your @data/@remote code): ' +
+                        '`npm run build:server`  (toilscript --target release --rpcModule shared/server.ts).',
+                );
+            }
+            return null;
+        },
+    };
+}
+
 /** Routes a built asset to a typed sub-folder (`images/`, `fonts/`, `css/`, else `assets/`). */
 function assetFileName(name: string): string {
     const ext = name.split('.').pop() ?? '';
@@ -116,12 +147,15 @@ export async function createViteConfig(cfg: ResolvedToilConfig): Promise<InlineC
             cfg.fonts ? fontPreloadPlugin(cfg) : undefined,
             nodePolyfills({ globals: { Buffer: true, global: true, process: true } }),
             react(),
+            sharedResolverPlugin(cfg),
             toilPlugin(cfg),
         ],
         resolve: {
             alias: {
                 'toiljs/client': cfg.runtimePath,
                 'toiljs/routes': path.join(cfg.toilDir, 'routes.ts'),
+                // `shared/*` is resolved by sharedResolverPlugin (above) so a missing generated
+                // shared/server.ts gives an actionable error instead of an opaque load failure.
                 ...polyfillShimAliases,
             },
             dedupe: ['react', 'react-dom'],

package/src/io/FastMap.ts

@@ -1,127 +1,151 @@
-export type PropertyExtendedKey = PropertyKey | bigint;
-
-/**
- * Like Record, but supports bigint keys (which JS auto-converts to strings).
- * Reflects actual JavaScript behavior where obj[123n] becomes obj["123"].
- */
-export type FastRecord<V> = {
-    [key: string]: V;
-};
-
-export type IndexKey = string | number;
-
-export class FastMap<K extends PropertyExtendedKey, V> implements Disposable {
-    protected _keys: K[] = [];
-    protected _values: FastRecord<V> = {};
-
-    constructor(iterable?: ReadonlyArray<readonly [K, V]> | null | FastMap<K, V>) {
-        if (iterable instanceof FastMap) {
-            this.setAll(iterable);
-        } else {
-            if (iterable) {
-                for (const [key, value] of iterable) {
-                    this.set(key, value);
-                }
-            }
-        }
-    }
-
-    public get size(): number {
-        return this._keys.length;
-    }
-
-    public setAll(map: FastMap<K, V>): void {
-        this._keys = [...map._keys];
-        this._values = { ...map._values };
-    }
-
-    public addAll(map: FastMap<K, V>): void {
-        for (const [key, value] of map.entries()) {
-            this.set(key, value);
-        }
-    }
-
-    public *keys(): IterableIterator<K> {
-        yield* this._keys;
-    }
-
-    public *values(): IterableIterator<V> {
-        for (const key of this._keys) {
-            yield this._values[key as IndexKey] as V;
-        }
-    }
-
-    public *entries(): IterableIterator<[K, V]> {
-        for (const key of this._keys) {
-            yield [key, this._values[key as IndexKey] as V];
-        }
-    }
-
-    public set(key: K, value: V): this {
-        if (!this.has(key)) {
-            this._keys.push(key);
-        }
-
-        this._values[key as IndexKey] = value;
-
-        return this;
-    }
-
-    public indexOf(key: K): number {
-        if (!this.has(key)) {
-            return -1;
-        }
-
-        for (let i = 0; i < this._keys.length; i++) {
-            if (this._keys[i] === key) {
-                return i;
-            }
-        }
-
-        throw new Error('Key not found, this should not happen.');
-    }
-
-    public get(key: K): V | undefined {
-        return this._values[key as IndexKey];
-    }
-
-    public has(key: K): boolean {
-        return Object.prototype.hasOwnProperty.call(this._values, key as IndexKey);
-    }
-
-    public delete(key: K): boolean {
-        if (!this.has(key)) {
-            return false;
-        }
-
-        const index = this.indexOf(key);
-        this._keys.splice(index, 1);
-
-        delete this._values[key as IndexKey];
-        return true;
-    }
-
-    public clear(): void {
-        this._keys = [];
-        this._values = {};
-    }
-
-    public [Symbol.dispose](): void {
-        this.clear();
-    }
-
-    public forEach(
-        callback: (value: V, key: K, map: FastMap<K, V>) => void,
-        thisArg?: unknown,
-    ): void {
-        for (const key of this._keys) {
-            callback.call(thisArg, this._values[key as IndexKey] as V, key, this);
-        }
-    }
-
-    *[Symbol.iterator](): IterableIterator<[K, V]> {
-        for (const key of this._keys) {
-            yield [key, this._values[key as IndexKey] as V];
-        }
-    }
-}
+/** A key usable with {@link FastMap}: any `PropertyKey` plus `bigint`. */
+export type PropertyExtendedKey = PropertyKey | bigint;
+
+/**
+ * Like Record, but supports bigint keys (which JS auto-converts to strings).
+ * Reflects actual JavaScript behavior where obj[123n] becomes obj["123"].
+ */
+export type FastRecord<V> = {
+    [key: string]: V;
+};
+
+/** The string/number form a key takes once used to index the backing object. */
+export type IndexKey = string | number;
+
+/**
+ * An insertion-ordered map backed by a key array plus a plain object, supporting
+ * `bigint` keys (coerced to their string form, like native property access). Exposed
+ * to the client as a global (no import). Implements `Disposable`, so a `using` binding
+ * clears it on scope exit.
+ */
+export class FastMap<K extends PropertyExtendedKey, V> implements Disposable {
+    protected _keys: K[] = [];
+    protected _values: FastRecord<V> = {};
+
+    /** @param iterable - initial entries, or another FastMap to copy. */
+    constructor(iterable?: ReadonlyArray<readonly [K, V]> | null | FastMap<K, V>) {
+        if (iterable instanceof FastMap) {
+            this.setAll(iterable);
+        } else {
+            if (iterable) {
+                for (const [key, value] of iterable) {
+                    this.set(key, value);
+                }
+            }
+        }
+    }
+
+    /** Number of entries. */
+    public get size(): number {
+        return this._keys.length;
+    }
+
+    /** Replaces all entries with a copy of `map`'s entries. */
+    public setAll(map: FastMap<K, V>): void {
+        this._keys = [...map._keys];
+        this._values = { ...map._values };
+    }
+
+    /** Merges `map`'s entries into this one (existing keys are overwritten). */
+    public addAll(map: FastMap<K, V>): void {
+        for (const [key, value] of map.entries()) {
+            this.set(key, value);
+        }
+    }
+
+    /** Iterates the keys in insertion order. */
+    public *keys(): IterableIterator<K> {
+        yield* this._keys;
+    }
+
+    /** Iterates the values in key-insertion order. */
+    public *values(): IterableIterator<V> {
+        for (const key of this._keys) {
+            yield this._values[key as IndexKey] as V;
+        }
+    }
+
+    /** Iterates `[key, value]` pairs in insertion order. */
+    public *entries(): IterableIterator<[K, V]> {
+        for (const key of this._keys) {
+            yield [key, this._values[key as IndexKey] as V];
+        }
+    }
+
+    /** Sets `key` to `value` (appending the key if new), and returns `this` for chaining. */
+    public set(key: K, value: V): this {
+        if (!this.has(key)) {
+            this._keys.push(key);
+        }
+
+        this._values[key as IndexKey] = value;
+
+        return this;
+    }
+
+    /** Returns the insertion index of `key`, or -1 if absent. */
+    public indexOf(key: K): number {
+        if (!this.has(key)) {
+            return -1;
+        }
+
+        for (let i = 0; i < this._keys.length; i++) {
+            if (this._keys[i] === key) {
+                return i;
+            }
+        }
+
+        throw new Error('Key not found, this should not happen.');
+    }
+
+    /** Returns the value for `key`, or `undefined` if absent. */
+    public get(key: K): V | undefined {
+        return this._values[key as IndexKey];
+    }
+
+    /** Whether `key` is present. */
+    public has(key: K): boolean {
+        return Object.prototype.hasOwnProperty.call(this._values, key as IndexKey);
+    }
+
+    /** Removes `key`; returns true if it was present. */
+    public delete(key: K): boolean {
+        if (!this.has(key)) {
+            return false;
+        }
+
+        const index = this.indexOf(key);
+        this._keys.splice(index, 1);
+
+        delete this._values[key as IndexKey];
+        return true;
+    }
+
+    /** Removes all entries. */
+    public clear(): void {
+        this._keys = [];
+        this._values = {};
+    }
+
+    /** `Disposable` hook: clears the map (so `using m = new FastMap()` frees it on scope exit). */
+    public [Symbol.dispose](): void {
+        this.clear();
+    }
+
+    /** Calls `callback(value, key, map)` for each entry in insertion order. */
+    public forEach(
+        callback: (value: V, key: K, map: FastMap<K, V>) => void,
+        thisArg?: unknown,
+    ): void {
+        for (const key of this._keys) {
+            callback.call(thisArg, this._values[key as IndexKey] as V, key, this);
+        }
+    }
+
+    /** Default iterator: `[key, value]` pairs in insertion order. */
+    *[Symbol.iterator](): IterableIterator<[K, V]> {
+        for (const key of this._keys) {
+            yield [key, this._values[key as IndexKey] as V];
+        }
+    }
+}