reroute-js 1.0.0-alpha.7 → 1.0.0-alpha.9

package/README.md

@@ -1169,6 +1169,7 @@ Your site now supports:
 - Accept header content negotiation (`Accept: text/plain`, `Accept: text/markdown`)
 - `/llms.txt` - Index of all content: title, description, and link per page
 - `/llms-full.txt` - Complete site content in one file (optional; large)
+- HTML discovery: `Link: rel=alternate` + `<link type="text/markdown">` pointing at the `.md` twin, `Link: rel=llms-txt`, `Vary: Accept`, and `Content-Signal` (also a `Content-Signal:` line in `robots.txt` when robots is enabled)
 
 #### How It Works
 
@@ -1201,6 +1202,7 @@ curl -H "Accept: text/html" https://example.com/blog/hello-world  # Default
 - **Flexible exclusions** - Strings, RegExp, or custom functions
 - **Works everywhere** - Collections, SSR routes, static pages
 - **Auto-discovery** - All content automatically indexed
+- **Crawler hints** - HTML `Link` / `<link rel="alternate">`, `Content-Signal`, `Vary: Accept`
 
 #### Discovering LLM Files in Your UI
 

package/cli/bin.d.ts

@@ -1,5 +1,5 @@
 /**
- * reroute-js v1.0.0-alpha.7
+ * reroute-js v1.0.0-alpha.9
  *
  * @license MIT
  * @copyright 2026 stewones <hi@stewan.io>

package/cli/bin.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env bun
 // @bun
 /**
- * reroute-js v1.0.0-alpha.7
+ * reroute-js v1.0.0-alpha.9
  *
  * @license MIT
  * @copyright 2026 stewones <hi@stewan.io>
@@ -68,7 +68,7 @@ async function getVersionString() {
 }
 async function getVersion() {
   if (true) {
-    return "1.0.0-alpha.7";
+    return "1.0.0-alpha.9";
   }
   const possiblePaths = [
     path.join(import.meta.dir, "../../../../package.json"),
@@ -86,7 +86,7 @@ async function getVersion() {
 }
 async function getCommit() {
   if (true) {
-    return "19dcaf7";
+    return "f5dd5ea";
   }
   return "dev";
 }
@@ -1488,6 +1488,21 @@ function setGlobalCollections(byCollectionForSSR) {
     globalThis.__REROUTE_COLLECTIONS__ = byCollectionForSSR;
   } catch {}
 }
+function resetSSRContentResults() {
+  try {
+    globalThis.__REROUTE_SSR_RESULTS__ = {};
+  } catch {}
+}
+function takeSSRContentResults() {
+  try {
+    const g = globalThis;
+    const results = g.__REROUTE_SSR_RESULTS__;
+    g.__REROUTE_SSR_RESULTS__ = {};
+    return results;
+  } catch {
+    return;
+  }
+}
 var init_collections = __esm(() => {
   init_imports();
 });
@@ -3608,23 +3623,34 @@ function parseAccessTracking(accessedCollections) {
   }
   return usage;
 }
-function processCollections(accessedCollections, byCollectionForSSR) {
+function processCapturedResults(capturedResults, byCollectionForSSR) {
   const subset = {};
   const partial = {};
   try {
     const g = globalThis;
-    const ssrResults = g.__REROUTE_SSR_RESULTS__;
-    if (ssrResults && typeof ssrResults === "object") {
-      for (const [collection, items] of Object.entries(ssrResults)) {
-        if (Array.isArray(items) && items.length > 0) {
-          subset[collection] = items;
-          const full = byCollectionForSSR?.[collection] || [];
-          partial[collection] = items.length < full.length;
-        }
+    const ssrResults = capturedResults ?? g.__REROUTE_SSR_RESULTS__;
+    if (!ssrResults || typeof ssrResults !== "object")
+      return;
+    for (const [collection, items] of Object.entries(ssrResults)) {
+      if (Array.isArray(items) && items.length > 0) {
+        subset[collection] = items;
+        const full = byCollectionForSSR?.[collection] || [];
+        partial[collection] = items.length < full.length;
       }
-      return { subset, partial };
     }
-  } catch {}
+  } catch {
+    return;
+  }
+  if (Object.keys(subset).length === 0)
+    return;
+  return { subset, partial };
+}
+function processCollections(accessedCollections, byCollectionForSSR, capturedResults) {
+  const captured = processCapturedResults(capturedResults, byCollectionForSSR);
+  if (captured)
+    return captured;
+  const subset = {};
+  const partial = {};
   const sortByDate = (order) => (a, b) => {
     const da = a?.meta?.date ? Date.parse(String(a.meta.date)) : 0;
     const db = b?.meta?.date ? Date.parse(String(b.meta.date)) : 0;
@@ -3913,6 +3939,7 @@ async function performSSRSetupInternal(options, streaming, timeoutState) {
   });
   assertNotTimedOut();
   resetSSRAccessTracking();
+  resetSSRContentResults();
   const bundlePreload = createBundlePreload(bundleUrl);
   const preloadResult = await withSpan("ssr.preload", async (span) => {
     span.setAttribute("pathname", pathname);
@@ -3988,12 +4015,12 @@ async function performSSRSetupInternal(options, streaming, timeoutState) {
     pending: computeResult.pending
   };
 }
-function generateCollectionScripts(byCollectionForSSR) {
+function generateCollectionScripts(byCollectionForSSR, capturedResults) {
   try {
     const g = globalThis;
     const acc = g.__REROUTE_SSR_ACCESSED__;
     if (acc) {
-      const { subset, partial } = processCollections(acc, byCollectionForSSR);
+      const { subset, partial } = processCollections(acc, byCollectionForSSR, capturedResults);
       if (Object.keys(subset).length > 0) {
         return createCollectionScript(subset, partial);
       }
@@ -4145,13 +4172,16 @@ async function renderSSRDocument(options) {
   }, false);
   const duplicatedData = duplicateQueryParamData(setup2.ssrData);
   setGlobalSSRData(duplicatedData);
+  let capturedResults;
   const appHtml = await withSpan("ssr.render.react", async (span) => {
     const componentWithPathname = cloneElement(rootComponent, {
       pathname,
       searchParams
     });
     span.setAttribute("reroute.pathname", pathname);
+    resetSSRContentResults();
     const html3 = renderToString(componentWithPathname);
+    capturedResults = takeSSRContentResults();
     span.setAttribute("reroute.html.final.size", html3.length);
     return html3;
   });
@@ -4160,7 +4190,7 @@ async function renderSSRDocument(options) {
     span.setAttribute("reroute.template.size", template2.length);
     return template2;
   });
-  const collectionScripts = generateCollectionScripts(setup2.byCollectionForSSR);
+  const collectionScripts = generateCollectionScripts(setup2.byCollectionForSSR, capturedResults);
   const config = await loadConfig(cwd);
   const browserTelemetryConfigRaw = config.telemetry?.browser;
   const { serializeBrowserTelemetryConfig: serializeBrowserTelemetryConfig2 } = await Promise.resolve().then(() => exports_serialize);
@@ -4178,7 +4208,7 @@ async function renderSSRDocument(options) {
   if (setup2.metadataResult.statusOverride !== undefined) {
     statusOverride = setup2.metadataResult.statusOverride;
   }
-  const extraHead = setup2.bundlePreload + setup2.preloadExtraHead;
+  const extraHead = appendLlmsAlternateHead(setup2.bundlePreload + setup2.preloadExtraHead, pathname, statusOverride || 200, config.llms);
   const combinedHead = dedent([
     dedent(head) || "",
     dedent(extraHead),
@@ -4208,6 +4238,7 @@ async function renderSSRDocument(options) {
 var init_render = __esm(() => {
   init_server2();
   init_config();
+  init_discovery3();
   init_lib();
   init_html();
 });
@@ -4534,6 +4565,112 @@ var init_extractor = __esm(() => {
   init_render();
 });
 
+// packages/core/src/llms/discovery.ts
+function yesNo(value) {
+  return value === false ? "no" : "yes";
+}
+function formatContentSignal(policy, whenUndefined = "omit") {
+  if (policy === false)
+    return;
+  if (policy === undefined && whenUndefined === "omit")
+    return;
+  const resolved = {
+    search: true,
+    aiInput: true,
+    aiTrain: true,
+    ...policy
+  };
+  return `ai-train=${yesNo(resolved.aiTrain)}, search=${yesNo(resolved.search)}, ai-input=${yesNo(resolved.aiInput)}`;
+}
+function markdownTwinPath(pathname) {
+  const path3 = pathname.split("?")[0];
+  if (path3 === "/" || path3 === "")
+    return "/index.md";
+  return `${path3.replace(/\/$/, "")}.md`;
+}
+function shouldAdvertiseMarkdownTwin(pathname, llmsConfig) {
+  if (!llmsConfig?.enabled)
+    return false;
+  if (llmsConfig.enableMarkdownExtension === false)
+    return false;
+  if (shouldExcludeRoute(pathname, llmsConfig.excludeRoutes))
+    return false;
+  return true;
+}
+function llmsAlternateLinkTag(pathname, llmsConfig) {
+  if (!shouldAdvertiseMarkdownTwin(pathname, llmsConfig))
+    return "";
+  return `<link rel="alternate" type="text/markdown" href="${markdownTwinPath(pathname)}" />`;
+}
+function mergeVary(existing, token) {
+  const parts = (existing ?? "").split(",").map((part) => part.trim()).filter(Boolean);
+  if (!parts.some((part) => part.toLowerCase() === token.toLowerCase())) {
+    parts.push(token);
+  }
+  return parts.join(", ");
+}
+function appendLinkValues(existing, values) {
+  const current = (existing ?? "").split(",").map((part) => part.trim()).filter(Boolean);
+  for (const value of values) {
+    if (!current.includes(value))
+      current.push(value);
+  }
+  return current.join(", ");
+}
+function buildLlmsDiscovery(pathname, llmsConfig) {
+  if (!llmsConfig?.enabled)
+    return null;
+  const linkValues = [];
+  if (shouldAdvertiseMarkdownTwin(pathname, llmsConfig)) {
+    linkValues.push(`<${markdownTwinPath(pathname)}>; rel="alternate"; type="text/markdown"`);
+  }
+  if (llmsConfig.enableIndex !== false) {
+    linkValues.push('</llms.txt>; rel="llms-txt"');
+  }
+  const contentSignal = formatContentSignal(llmsConfig.contentSignal, "default");
+  const vary = llmsConfig.enableContentNegotiation !== false ? "Accept" : undefined;
+  if (linkValues.length === 0 && !contentSignal && !vary)
+    return null;
+  return { linkValues, contentSignal, vary };
+}
+function writeHeader(headers2, canonical, value) {
+  const lower = canonical.toLowerCase();
+  if (headers2[lower] !== undefined)
+    headers2[lower] = value;
+  if (headers2[canonical] !== undefined || headers2[lower] === undefined) {
+    headers2[canonical] = value;
+  }
+}
+function readHeader(headers2, canonical) {
+  return headers2[canonical] ?? headers2[canonical.toLowerCase()];
+}
+function applyLlmsDiscoveryHeaders(headers2, pathname, llmsConfig, status = 200) {
+  if (status < 200 || status >= 300)
+    return;
+  const discovery2 = buildLlmsDiscovery(pathname, llmsConfig);
+  if (!discovery2)
+    return;
+  if (discovery2.linkValues.length > 0) {
+    writeHeader(headers2, "Link", appendLinkValues(readHeader(headers2, "Link"), discovery2.linkValues));
+  }
+  if (discovery2.contentSignal) {
+    writeHeader(headers2, "Content-Signal", discovery2.contentSignal);
+  }
+  if (discovery2.vary) {
+    writeHeader(headers2, "Vary", mergeVary(readHeader(headers2, "Vary"), discovery2.vary));
+  }
+}
+function appendLlmsAlternateHead(extraHead, pathname, status, llmsConfig) {
+  if (status !== 200)
+    return extraHead;
+  const tag = llmsAlternateLinkTag(pathname, llmsConfig);
+  return tag ? `${extraHead}
+${tag}` : extraHead;
+}
+var init_discovery3 = __esm(() => {
+  init_extractor();
+});
+
 // packages/core/src/llms/index-generator.ts
 var exports_index_generator = {};
 __export(exports_index_generator, {
@@ -5036,6 +5173,7 @@ var init_full_generator = __esm(() => {
 
 // packages/core/src/llms/index.ts
 var init_llms = __esm(() => {
+  init_discovery3();
   init_extractor();
   init_full_generator();
   init_index_generator();
@@ -5202,7 +5340,7 @@ async function discoverRobotsData(cwd, isWatchMode, robotsConfig, sitemapConfig,
     redirectSourcePaths
   };
 }
-var init_discovery3 = () => {};
+var init_discovery4 = () => {};
 
 // packages/core/src/robots/generator.ts
 function getEffectivePolicies(policies) {
@@ -5245,8 +5383,16 @@ function addDisallowDirectives(lines, policy, autoDisallows) {
     }
   }
 }
-function addPolicyBlock(lines, policy, autoDisallows) {
+function addContentSignalLine(lines, policy, robotsConfig) {
+  if (policy.userAgent !== "*")
+    return;
+  const value = formatContentSignal(robotsConfig.contentSignal);
+  if (value)
+    lines.push(`Content-Signal: ${value}`);
+}
+function addPolicyBlock(lines, policy, autoDisallows, robotsConfig) {
   lines.push(`User-agent: ${policy.userAgent}`);
+  addContentSignalLine(lines, policy, robotsConfig);
   addAllowDirectives(lines, policy);
   addDisallowDirectives(lines, policy, autoDisallows);
   if (policy.crawlDelay !== undefined) {
@@ -5272,7 +5418,7 @@ function generateRobots(discovery2, robotsConfig, sitemapEnabled) {
   const effectivePolicies = getEffectivePolicies(policies);
   const autoDisallows = getAutoDisallows(discovery2);
   for (const policy of effectivePolicies) {
-    addPolicyBlock(lines, policy, autoDisallows);
+    addPolicyBlock(lines, policy, autoDisallows, robotsConfig);
   }
   addGlobalCrawlDelay(lines, crawlDelay, effectivePolicies);
   if (customRules) {
@@ -5284,6 +5430,9 @@ function generateRobots(discovery2, robotsConfig, sitemapEnabled) {
 `).trim()}
 `;
 }
+var init_generator = __esm(() => {
+  init_discovery3();
+});
 
 // packages/core/src/robots/policies.ts
 function blockAICrawlers(crawlers) {
@@ -5354,7 +5503,8 @@ var init_policies = __esm(() => {
 
 // packages/core/src/robots/index.ts
 var init_robots = __esm(() => {
-  init_discovery3();
+  init_discovery4();
+  init_generator();
   init_policies();
 });
 
@@ -5687,7 +5837,7 @@ async function discoverSitemapData(cwd, collections2, baseUrl, isWatchMode, site
     ssrDataRoutes
   };
 }
-var init_discovery4 = () => {};
+var init_discovery5 = () => {};
 
 // packages/core/src/rss/discovery.ts
 import { pathToFileURL as pathToFileURL7 } from "node:url";
@@ -5955,7 +6105,7 @@ async function discoverRSSData(cwd, collections2, baseUrl, isWatchMode, rssConfi
   }
   const ssrDataFeeds = new Map;
   const bundledRoutes = globalThis.__REROUTE_ROUTES__;
-  const { discoverRoutes: discoverRoutes2 } = await Promise.resolve().then(() => (init_discovery4(), exports_discovery));
+  const { discoverRoutes: discoverRoutes2 } = await Promise.resolve().then(() => (init_discovery5(), exports_discovery));
   const { staticRoutes } = await discoverRoutes2(cwd, isWatchMode);
   if (bundledRoutes && Array.isArray(bundledRoutes)) {
     const routePromises = staticRoutes.map(async (route) => {
@@ -6012,7 +6162,7 @@ async function discoverRSSData(cwd, collections2, baseUrl, isWatchMode, rssConfi
     ssrDataFeeds
   };
 }
-var init_discovery5 = () => {};
+var init_discovery6 = () => {};
 
 // packages/core/src/rss/generator.ts
 function generateRSSXML(feed, options = {}) {
@@ -6125,7 +6275,7 @@ function escapeXml(unsafe) {
 
 // packages/core/src/rss/index.ts
 var init_rss = __esm(() => {
-  init_discovery5();
+  init_discovery6();
 });
 
 // packages/core/src/sitemap/generator.ts
@@ -6270,12 +6420,12 @@ function escapeXml2(unsafe) {
   return unsafe.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
 }
 var DEFAULT_MAX_URLS = 50000;
-var init_generator = () => {};
+var init_generator2 = () => {};
 
 // packages/core/src/sitemap/index.ts
 var init_sitemap = __esm(() => {
-  init_discovery4();
-  init_generator();
+  init_discovery5();
+  init_generator2();
 });
 
 // packages/core/src/ssr/stream.ts
@@ -6353,15 +6503,16 @@ async function renderSSRDocumentStream(options) {
     const newAttrs = hasLang ? attrs.replace(/lang\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/i, `lang="${pageLang}"`) : `${attrs} lang="${pageLang}"`;
     return `<html${newAttrs}>`;
   });
-  const extraHead = setup2.bundlePreload + setup2.preloadExtraHead + setup2.metadataResult.perPageHead;
-  if (/<title[\s\S]*?<\/title>/i.test(extraHead)) {
+  const extraHeadBase = setup2.bundlePreload + setup2.preloadExtraHead + setup2.metadataResult.perPageHead;
+  if (/<title[\s\S]*?<\/title>/i.test(extraHeadBase)) {
     headWithLang = headWithLang.replace(/<title[\s\S]*?<\/title>/i, "");
   }
-  if (/<meta\s+name\s*=\s*['"]description['"][^>]*>/i.test(extraHead)) {
+  if (/<meta\s+name\s*=\s*['"]description['"][^>]*>/i.test(extraHeadBase)) {
     headWithLang = headWithLang.replace(/<meta\s+name\s*=\s*['"]description['"][^>]*>/i, "");
   }
-  headWithLang = removeOverriddenMetaTags(headWithLang, extraHead);
+  headWithLang = removeOverriddenMetaTags(headWithLang, extraHeadBase);
   const config = await loadConfig(cwd);
+  const extraHead = appendLlmsAlternateHead(extraHeadBase, pathname, statusOverride || 200, config.llms);
   const browserTelemetryConfigRaw = config.telemetry?.browser;
   const { serializeBrowserTelemetryConfig: serializeBrowserTelemetryConfig2 } = await Promise.resolve().then(() => exports_serialize);
   const browserTelemetryConfig = serializeBrowserTelemetryConfig2(browserTelemetryConfigRaw);
@@ -6619,6 +6770,7 @@ function unescapeHtmlEntities(str) {
 var init_stream = __esm(() => {
   init_server2();
   init_config();
+  init_discovery3();
   init_lib();
   init_html();
   init_imports();
@@ -6661,12 +6813,15 @@ __export(exports_core, {
   acceptsGzip: () => acceptsGzip,
   allowAICrawlers: () => allowAICrawlers,
   allowSearchCrawlers: () => allowSearchCrawlers,
+  appendLlmsAlternateHead: () => appendLlmsAlternateHead,
   applyIndexTemplate: () => applyIndexTemplate,
+  applyLlmsDiscoveryHeaders: () => applyLlmsDiscoveryHeaders,
   basename: () => basename,
   blockAICrawlers: () => blockAICrawlers,
   blockAITrainingCrawlers: () => blockAITrainingCrawlers,
   blockSearchCrawlers: () => blockSearchCrawlers,
   buildHeadFromMeta: () => buildHeadFromMeta,
+  buildLlmsDiscovery: () => buildLlmsDiscovery,
   buildLocaleCookie: () => buildLocaleCookie,
   cleanupPathScopedSSRData: () => cleanupPathScopedSSRData,
   clearSSRModuleStores: () => clearSSRModuleStores,
@@ -6712,6 +6867,7 @@ __export(exports_core, {
   findOGImageForPath: () => findOGImageForPath,
   formatAsMarkdown: () => formatAsMarkdown,
   formatAsText: () => formatAsText,
+  formatContentSignal: () => formatContentSignal,
   formatMessage: () => formatMessage,
   generateAtomXML: () => generateAtomXML,
   generateCollectionScripts: () => generateCollectionScripts,
@@ -6743,6 +6899,7 @@ __export(exports_core, {
   isThenable: () => isThenable,
   join: () => join2,
   listContentFiles: () => listContentFiles,
+  llmsAlternateLinkTag: () => llmsAlternateLinkTag,
   loadCollections: () => loadCollections,
   loadConfig: () => loadConfig,
   loadIndexHtml: () => loadIndexHtml,
@@ -6752,9 +6909,11 @@ __export(exports_core, {
   localizeHref: () => localizeHref,
   looksLikeLocale: () => looksLikeLocale,
   lookupMessage: () => lookupMessage,
+  markdownTwinPath: () => markdownTwinPath,
   matchRoutePattern: () => matchRoutePattern,
   matchRoutes: () => matchRoutes,
   mergeLayoutData: () => mergeLayoutData,
+  mergeVary: () => mergeVary,
   negotiateLocale: () => negotiateLocale,
   normalizePathname: () => normalizePathname,
   parseContentPath: () => parseContentPath,
@@ -6767,10 +6926,12 @@ __export(exports_core, {
   renderSSRDocument: () => renderSSRDocument,
   renderSSRDocumentStream: () => renderSSRDocumentStream,
   resetSSRAccessTracking: () => resetSSRAccessTracking,
+  resetSSRContentResults: () => resetSSRContentResults,
   resolveLocale: () => resolveLocale,
   seedSSRModuleForPath: () => seedSSRModuleForPath,
   setGlobalCollections: () => setGlobalCollections,
   setGlobalSSRData: () => setGlobalSSRData,
+  shouldAdvertiseMarkdownTwin: () => shouldAdvertiseMarkdownTwin,
   shouldExcludeRoute: () => shouldExcludeRoute,
   splitTemplate: () => splitTemplate,
   ssrCache: () => ssrCache,
@@ -6778,6 +6939,7 @@ __export(exports_core, {
   stripEnd: () => stripEnd,
   stripHtmlTags: () => stripHtmlTags,
   stripStart: () => stripStart,
+  takeSSRContentResults: () => takeSSRContentResults,
   toBytes: () => toBytes,
   toOgLocale: () => toOgLocale,
   toPromise: () => toPromise
@@ -107156,7 +107318,7 @@ async function getVersionString2() {
 }
 async function getVersion2() {
   if (true) {
-    return "1.0.0-alpha.7";
+    return "1.0.0-alpha.9";
   }
   const possiblePaths = [
     path3.join(import.meta.dir, "../../../package.json"),
@@ -107173,10 +107335,10 @@ async function getVersion2() {
 }
 async function getCommit2() {
   if (true) {
-    return "19dcaf7";
+    return "f5dd5ea";
   }
   return "dev";
 }
 main();
 
-//# debugId=958B942751A1FF3B64756E2164756E21
+//# debugId=621D582CE8E376F164756E2164756E21