reroute-js 1.0.0-alpha.8 → 1.0.0-alpha.9

package/README.md

@@ -1169,6 +1169,7 @@ Your site now supports:
 - Accept header content negotiation (`Accept: text/plain`, `Accept: text/markdown`)
 - `/llms.txt` - Index of all content: title, description, and link per page
 - `/llms-full.txt` - Complete site content in one file (optional; large)
+- HTML discovery: `Link: rel=alternate` + `<link type="text/markdown">` pointing at the `.md` twin, `Link: rel=llms-txt`, `Vary: Accept`, and `Content-Signal` (also a `Content-Signal:` line in `robots.txt` when robots is enabled)
 
 #### How It Works
 
@@ -1201,6 +1202,7 @@ curl -H "Accept: text/html" https://example.com/blog/hello-world  # Default
 - **Flexible exclusions** - Strings, RegExp, or custom functions
 - **Works everywhere** - Collections, SSR routes, static pages
 - **Auto-discovery** - All content automatically indexed
+- **Crawler hints** - HTML `Link` / `<link rel="alternate">`, `Content-Signal`, `Vary: Accept`
 
 #### Discovering LLM Files in Your UI
 

package/cli/bin.d.ts

@@ -1,5 +1,5 @@
 /**
- * reroute-js v1.0.0-alpha.8
+ * reroute-js v1.0.0-alpha.9
  *
  * @license MIT
  * @copyright 2026 stewones <hi@stewan.io>

package/cli/bin.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env bun
 // @bun
 /**
- * reroute-js v1.0.0-alpha.8
+ * reroute-js v1.0.0-alpha.9
  *
  * @license MIT
  * @copyright 2026 stewones <hi@stewan.io>
@@ -68,7 +68,7 @@ async function getVersionString() {
 }
 async function getVersion() {
   if (true) {
-    return "1.0.0-alpha.8";
+    return "1.0.0-alpha.9";
   }
   const possiblePaths = [
     path.join(import.meta.dir, "../../../../package.json"),
@@ -86,7 +86,7 @@ async function getVersion() {
 }
 async function getCommit() {
   if (true) {
-    return "2c48a1e";
+    return "f5dd5ea";
   }
   return "dev";
 }
@@ -4208,7 +4208,7 @@ async function renderSSRDocument(options) {
   if (setup2.metadataResult.statusOverride !== undefined) {
     statusOverride = setup2.metadataResult.statusOverride;
   }
-  const extraHead = setup2.bundlePreload + setup2.preloadExtraHead;
+  const extraHead = appendLlmsAlternateHead(setup2.bundlePreload + setup2.preloadExtraHead, pathname, statusOverride || 200, config.llms);
   const combinedHead = dedent([
     dedent(head) || "",
     dedent(extraHead),
@@ -4238,6 +4238,7 @@ async function renderSSRDocument(options) {
 var init_render = __esm(() => {
   init_server2();
   init_config();
+  init_discovery3();
   init_lib();
   init_html();
 });
@@ -4564,6 +4565,112 @@ var init_extractor = __esm(() => {
   init_render();
 });
 
+// packages/core/src/llms/discovery.ts
+function yesNo(value) {
+  return value === false ? "no" : "yes";
+}
+function formatContentSignal(policy, whenUndefined = "omit") {
+  if (policy === false)
+    return;
+  if (policy === undefined && whenUndefined === "omit")
+    return;
+  const resolved = {
+    search: true,
+    aiInput: true,
+    aiTrain: true,
+    ...policy
+  };
+  return `ai-train=${yesNo(resolved.aiTrain)}, search=${yesNo(resolved.search)}, ai-input=${yesNo(resolved.aiInput)}`;
+}
+function markdownTwinPath(pathname) {
+  const path3 = pathname.split("?")[0];
+  if (path3 === "/" || path3 === "")
+    return "/index.md";
+  return `${path3.replace(/\/$/, "")}.md`;
+}
+function shouldAdvertiseMarkdownTwin(pathname, llmsConfig) {
+  if (!llmsConfig?.enabled)
+    return false;
+  if (llmsConfig.enableMarkdownExtension === false)
+    return false;
+  if (shouldExcludeRoute(pathname, llmsConfig.excludeRoutes))
+    return false;
+  return true;
+}
+function llmsAlternateLinkTag(pathname, llmsConfig) {
+  if (!shouldAdvertiseMarkdownTwin(pathname, llmsConfig))
+    return "";
+  return `<link rel="alternate" type="text/markdown" href="${markdownTwinPath(pathname)}" />`;
+}
+function mergeVary(existing, token) {
+  const parts = (existing ?? "").split(",").map((part) => part.trim()).filter(Boolean);
+  if (!parts.some((part) => part.toLowerCase() === token.toLowerCase())) {
+    parts.push(token);
+  }
+  return parts.join(", ");
+}
+function appendLinkValues(existing, values) {
+  const current = (existing ?? "").split(",").map((part) => part.trim()).filter(Boolean);
+  for (const value of values) {
+    if (!current.includes(value))
+      current.push(value);
+  }
+  return current.join(", ");
+}
+function buildLlmsDiscovery(pathname, llmsConfig) {
+  if (!llmsConfig?.enabled)
+    return null;
+  const linkValues = [];
+  if (shouldAdvertiseMarkdownTwin(pathname, llmsConfig)) {
+    linkValues.push(`<${markdownTwinPath(pathname)}>; rel="alternate"; type="text/markdown"`);
+  }
+  if (llmsConfig.enableIndex !== false) {
+    linkValues.push('</llms.txt>; rel="llms-txt"');
+  }
+  const contentSignal = formatContentSignal(llmsConfig.contentSignal, "default");
+  const vary = llmsConfig.enableContentNegotiation !== false ? "Accept" : undefined;
+  if (linkValues.length === 0 && !contentSignal && !vary)
+    return null;
+  return { linkValues, contentSignal, vary };
+}
+function writeHeader(headers2, canonical, value) {
+  const lower = canonical.toLowerCase();
+  if (headers2[lower] !== undefined)
+    headers2[lower] = value;
+  if (headers2[canonical] !== undefined || headers2[lower] === undefined) {
+    headers2[canonical] = value;
+  }
+}
+function readHeader(headers2, canonical) {
+  return headers2[canonical] ?? headers2[canonical.toLowerCase()];
+}
+function applyLlmsDiscoveryHeaders(headers2, pathname, llmsConfig, status = 200) {
+  if (status < 200 || status >= 300)
+    return;
+  const discovery2 = buildLlmsDiscovery(pathname, llmsConfig);
+  if (!discovery2)
+    return;
+  if (discovery2.linkValues.length > 0) {
+    writeHeader(headers2, "Link", appendLinkValues(readHeader(headers2, "Link"), discovery2.linkValues));
+  }
+  if (discovery2.contentSignal) {
+    writeHeader(headers2, "Content-Signal", discovery2.contentSignal);
+  }
+  if (discovery2.vary) {
+    writeHeader(headers2, "Vary", mergeVary(readHeader(headers2, "Vary"), discovery2.vary));
+  }
+}
+function appendLlmsAlternateHead(extraHead, pathname, status, llmsConfig) {
+  if (status !== 200)
+    return extraHead;
+  const tag = llmsAlternateLinkTag(pathname, llmsConfig);
+  return tag ? `${extraHead}
+${tag}` : extraHead;
+}
+var init_discovery3 = __esm(() => {
+  init_extractor();
+});
+
 // packages/core/src/llms/index-generator.ts
 var exports_index_generator = {};
 __export(exports_index_generator, {
@@ -5066,6 +5173,7 @@ var init_full_generator = __esm(() => {
 
 // packages/core/src/llms/index.ts
 var init_llms = __esm(() => {
+  init_discovery3();
   init_extractor();
   init_full_generator();
   init_index_generator();
@@ -5232,7 +5340,7 @@ async function discoverRobotsData(cwd, isWatchMode, robotsConfig, sitemapConfig,
     redirectSourcePaths
   };
 }
-var init_discovery3 = () => {};
+var init_discovery4 = () => {};
 
 // packages/core/src/robots/generator.ts
 function getEffectivePolicies(policies) {
@@ -5275,8 +5383,16 @@ function addDisallowDirectives(lines, policy, autoDisallows) {
     }
   }
 }
-function addPolicyBlock(lines, policy, autoDisallows) {
+function addContentSignalLine(lines, policy, robotsConfig) {
+  if (policy.userAgent !== "*")
+    return;
+  const value = formatContentSignal(robotsConfig.contentSignal);
+  if (value)
+    lines.push(`Content-Signal: ${value}`);
+}
+function addPolicyBlock(lines, policy, autoDisallows, robotsConfig) {
   lines.push(`User-agent: ${policy.userAgent}`);
+  addContentSignalLine(lines, policy, robotsConfig);
   addAllowDirectives(lines, policy);
   addDisallowDirectives(lines, policy, autoDisallows);
   if (policy.crawlDelay !== undefined) {
@@ -5302,7 +5418,7 @@ function generateRobots(discovery2, robotsConfig, sitemapEnabled) {
   const effectivePolicies = getEffectivePolicies(policies);
   const autoDisallows = getAutoDisallows(discovery2);
   for (const policy of effectivePolicies) {
-    addPolicyBlock(lines, policy, autoDisallows);
+    addPolicyBlock(lines, policy, autoDisallows, robotsConfig);
   }
   addGlobalCrawlDelay(lines, crawlDelay, effectivePolicies);
   if (customRules) {
@@ -5314,6 +5430,9 @@ function generateRobots(discovery2, robotsConfig, sitemapEnabled) {
 `).trim()}
 `;
 }
+var init_generator = __esm(() => {
+  init_discovery3();
+});
 
 // packages/core/src/robots/policies.ts
 function blockAICrawlers(crawlers) {
@@ -5384,7 +5503,8 @@ var init_policies = __esm(() => {
 
 // packages/core/src/robots/index.ts
 var init_robots = __esm(() => {
-  init_discovery3();
+  init_discovery4();
+  init_generator();
   init_policies();
 });
 
@@ -5717,7 +5837,7 @@ async function discoverSitemapData(cwd, collections2, baseUrl, isWatchMode, site
     ssrDataRoutes
   };
 }
-var init_discovery4 = () => {};
+var init_discovery5 = () => {};
 
 // packages/core/src/rss/discovery.ts
 import { pathToFileURL as pathToFileURL7 } from "node:url";
@@ -5985,7 +6105,7 @@ async function discoverRSSData(cwd, collections2, baseUrl, isWatchMode, rssConfi
   }
   const ssrDataFeeds = new Map;
   const bundledRoutes = globalThis.__REROUTE_ROUTES__;
-  const { discoverRoutes: discoverRoutes2 } = await Promise.resolve().then(() => (init_discovery4(), exports_discovery));
+  const { discoverRoutes: discoverRoutes2 } = await Promise.resolve().then(() => (init_discovery5(), exports_discovery));
   const { staticRoutes } = await discoverRoutes2(cwd, isWatchMode);
   if (bundledRoutes && Array.isArray(bundledRoutes)) {
     const routePromises = staticRoutes.map(async (route) => {
@@ -6042,7 +6162,7 @@ async function discoverRSSData(cwd, collections2, baseUrl, isWatchMode, rssConfi
     ssrDataFeeds
   };
 }
-var init_discovery5 = () => {};
+var init_discovery6 = () => {};
 
 // packages/core/src/rss/generator.ts
 function generateRSSXML(feed, options = {}) {
@@ -6155,7 +6275,7 @@ function escapeXml(unsafe) {
 
 // packages/core/src/rss/index.ts
 var init_rss = __esm(() => {
-  init_discovery5();
+  init_discovery6();
 });
 
 // packages/core/src/sitemap/generator.ts
@@ -6300,12 +6420,12 @@ function escapeXml2(unsafe) {
   return unsafe.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
 }
 var DEFAULT_MAX_URLS = 50000;
-var init_generator = () => {};
+var init_generator2 = () => {};
 
 // packages/core/src/sitemap/index.ts
 var init_sitemap = __esm(() => {
-  init_discovery4();
-  init_generator();
+  init_discovery5();
+  init_generator2();
 });
 
 // packages/core/src/ssr/stream.ts
@@ -6383,15 +6503,16 @@ async function renderSSRDocumentStream(options) {
     const newAttrs = hasLang ? attrs.replace(/lang\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/i, `lang="${pageLang}"`) : `${attrs} lang="${pageLang}"`;
     return `<html${newAttrs}>`;
   });
-  const extraHead = setup2.bundlePreload + setup2.preloadExtraHead + setup2.metadataResult.perPageHead;
-  if (/<title[\s\S]*?<\/title>/i.test(extraHead)) {
+  const extraHeadBase = setup2.bundlePreload + setup2.preloadExtraHead + setup2.metadataResult.perPageHead;
+  if (/<title[\s\S]*?<\/title>/i.test(extraHeadBase)) {
     headWithLang = headWithLang.replace(/<title[\s\S]*?<\/title>/i, "");
   }
-  if (/<meta\s+name\s*=\s*['"]description['"][^>]*>/i.test(extraHead)) {
+  if (/<meta\s+name\s*=\s*['"]description['"][^>]*>/i.test(extraHeadBase)) {
     headWithLang = headWithLang.replace(/<meta\s+name\s*=\s*['"]description['"][^>]*>/i, "");
   }
-  headWithLang = removeOverriddenMetaTags(headWithLang, extraHead);
+  headWithLang = removeOverriddenMetaTags(headWithLang, extraHeadBase);
   const config = await loadConfig(cwd);
+  const extraHead = appendLlmsAlternateHead(extraHeadBase, pathname, statusOverride || 200, config.llms);
   const browserTelemetryConfigRaw = config.telemetry?.browser;
   const { serializeBrowserTelemetryConfig: serializeBrowserTelemetryConfig2 } = await Promise.resolve().then(() => exports_serialize);
   const browserTelemetryConfig = serializeBrowserTelemetryConfig2(browserTelemetryConfigRaw);
@@ -6649,6 +6770,7 @@ function unescapeHtmlEntities(str) {
 var init_stream = __esm(() => {
   init_server2();
   init_config();
+  init_discovery3();
   init_lib();
   init_html();
   init_imports();
@@ -6691,12 +6813,15 @@ __export(exports_core, {
   acceptsGzip: () => acceptsGzip,
   allowAICrawlers: () => allowAICrawlers,
   allowSearchCrawlers: () => allowSearchCrawlers,
+  appendLlmsAlternateHead: () => appendLlmsAlternateHead,
   applyIndexTemplate: () => applyIndexTemplate,
+  applyLlmsDiscoveryHeaders: () => applyLlmsDiscoveryHeaders,
   basename: () => basename,
   blockAICrawlers: () => blockAICrawlers,
   blockAITrainingCrawlers: () => blockAITrainingCrawlers,
   blockSearchCrawlers: () => blockSearchCrawlers,
   buildHeadFromMeta: () => buildHeadFromMeta,
+  buildLlmsDiscovery: () => buildLlmsDiscovery,
   buildLocaleCookie: () => buildLocaleCookie,
   cleanupPathScopedSSRData: () => cleanupPathScopedSSRData,
   clearSSRModuleStores: () => clearSSRModuleStores,
@@ -6742,6 +6867,7 @@ __export(exports_core, {
   findOGImageForPath: () => findOGImageForPath,
   formatAsMarkdown: () => formatAsMarkdown,
   formatAsText: () => formatAsText,
+  formatContentSignal: () => formatContentSignal,
   formatMessage: () => formatMessage,
   generateAtomXML: () => generateAtomXML,
   generateCollectionScripts: () => generateCollectionScripts,
@@ -6773,6 +6899,7 @@ __export(exports_core, {
   isThenable: () => isThenable,
   join: () => join2,
   listContentFiles: () => listContentFiles,
+  llmsAlternateLinkTag: () => llmsAlternateLinkTag,
   loadCollections: () => loadCollections,
   loadConfig: () => loadConfig,
   loadIndexHtml: () => loadIndexHtml,
@@ -6782,9 +6909,11 @@ __export(exports_core, {
   localizeHref: () => localizeHref,
   looksLikeLocale: () => looksLikeLocale,
   lookupMessage: () => lookupMessage,
+  markdownTwinPath: () => markdownTwinPath,
   matchRoutePattern: () => matchRoutePattern,
   matchRoutes: () => matchRoutes,
   mergeLayoutData: () => mergeLayoutData,
+  mergeVary: () => mergeVary,
   negotiateLocale: () => negotiateLocale,
   normalizePathname: () => normalizePathname,
   parseContentPath: () => parseContentPath,
@@ -6802,6 +6931,7 @@ __export(exports_core, {
   seedSSRModuleForPath: () => seedSSRModuleForPath,
   setGlobalCollections: () => setGlobalCollections,
   setGlobalSSRData: () => setGlobalSSRData,
+  shouldAdvertiseMarkdownTwin: () => shouldAdvertiseMarkdownTwin,
   shouldExcludeRoute: () => shouldExcludeRoute,
   splitTemplate: () => splitTemplate,
   ssrCache: () => ssrCache,
@@ -107188,7 +107318,7 @@ async function getVersionString2() {
 }
 async function getVersion2() {
   if (true) {
-    return "1.0.0-alpha.8";
+    return "1.0.0-alpha.9";
   }
   const possiblePaths = [
     path3.join(import.meta.dir, "../../../package.json"),
@@ -107205,10 +107335,10 @@ async function getVersion2() {
 }
 async function getCommit2() {
   if (true) {
-    return "2c48a1e";
+    return "f5dd5ea";
   }
   return "dev";
 }
 main();
 
-//# debugId=2ADD943B3E15E9F264756E2164756E21
+//# debugId=621D582CE8E376F164756E2164756E21