reroute-js 1.0.0-alpha.7 → 1.0.0-alpha.9

package/core/index.js

@@ -1,5 +1,5 @@
 /**
- * reroute-js v1.0.0-alpha.7
+ * reroute-js v1.0.0-alpha.9
  *
  * @license MIT
  * @copyright 2026 stewones <hi@stewan.io>
@@ -1150,6 +1150,21 @@ function setGlobalCollections(byCollectionForSSR) {
     globalThis.__REROUTE_COLLECTIONS__ = byCollectionForSSR;
   } catch {}
 }
+function resetSSRContentResults() {
+  try {
+    globalThis.__REROUTE_SSR_RESULTS__ = {};
+  } catch {}
+}
+function takeSSRContentResults() {
+  try {
+    const g = globalThis;
+    const results = g.__REROUTE_SSR_RESULTS__;
+    g.__REROUTE_SSR_RESULTS__ = {};
+    return results;
+  } catch {
+    return;
+  }
+}
 var init_collections = __esm(() => {
   init_imports();
 });
@@ -3270,23 +3285,34 @@ function parseAccessTracking(accessedCollections) {
   }
   return usage;
 }
-function processCollections(accessedCollections, byCollectionForSSR) {
+function processCapturedResults(capturedResults, byCollectionForSSR) {
   const subset = {};
   const partial = {};
   try {
     const g = globalThis;
-    const ssrResults = g.__REROUTE_SSR_RESULTS__;
-    if (ssrResults && typeof ssrResults === "object") {
-      for (const [collection, items] of Object.entries(ssrResults)) {
-        if (Array.isArray(items) && items.length > 0) {
-          subset[collection] = items;
-          const full = byCollectionForSSR?.[collection] || [];
-          partial[collection] = items.length < full.length;
-        }
+    const ssrResults = capturedResults ?? g.__REROUTE_SSR_RESULTS__;
+    if (!ssrResults || typeof ssrResults !== "object")
+      return;
+    for (const [collection, items] of Object.entries(ssrResults)) {
+      if (Array.isArray(items) && items.length > 0) {
+        subset[collection] = items;
+        const full = byCollectionForSSR?.[collection] || [];
+        partial[collection] = items.length < full.length;
       }
-      return { subset, partial };
     }
-  } catch {}
+  } catch {
+    return;
+  }
+  if (Object.keys(subset).length === 0)
+    return;
+  return { subset, partial };
+}
+function processCollections(accessedCollections, byCollectionForSSR, capturedResults) {
+  const captured = processCapturedResults(capturedResults, byCollectionForSSR);
+  if (captured)
+    return captured;
+  const subset = {};
+  const partial = {};
   const sortByDate = (order) => (a, b) => {
     const da = a?.meta?.date ? Date.parse(String(a.meta.date)) : 0;
     const db = b?.meta?.date ? Date.parse(String(b.meta.date)) : 0;
@@ -3575,6 +3601,7 @@ async function performSSRSetupInternal(options, streaming, timeoutState) {
   });
   assertNotTimedOut();
   resetSSRAccessTracking();
+  resetSSRContentResults();
   const bundlePreload = createBundlePreload(bundleUrl);
   const preloadResult = await withSpan("ssr.preload", async (span) => {
     span.setAttribute("pathname", pathname);
@@ -3650,12 +3677,12 @@ async function performSSRSetupInternal(options, streaming, timeoutState) {
     pending: computeResult.pending
   };
 }
-function generateCollectionScripts(byCollectionForSSR) {
+function generateCollectionScripts(byCollectionForSSR, capturedResults) {
   try {
     const g = globalThis;
     const acc = g.__REROUTE_SSR_ACCESSED__;
     if (acc) {
-      const { subset, partial } = processCollections(acc, byCollectionForSSR);
+      const { subset, partial } = processCollections(acc, byCollectionForSSR, capturedResults);
       if (Object.keys(subset).length > 0) {
         return createCollectionScript(subset, partial);
       }
@@ -3807,13 +3834,16 @@ async function renderSSRDocument(options) {
   }, false);
   const duplicatedData = duplicateQueryParamData(setup2.ssrData);
   setGlobalSSRData(duplicatedData);
+  let capturedResults;
   const appHtml = await withSpan("ssr.render.react", async (span) => {
     const componentWithPathname = cloneElement(rootComponent, {
       pathname,
       searchParams
     });
     span.setAttribute("reroute.pathname", pathname);
+    resetSSRContentResults();
     const html3 = renderToString(componentWithPathname);
+    capturedResults = takeSSRContentResults();
     span.setAttribute("reroute.html.final.size", html3.length);
     return html3;
   });
@@ -3822,7 +3852,7 @@ async function renderSSRDocument(options) {
     span.setAttribute("reroute.template.size", template2.length);
     return template2;
   });
-  const collectionScripts = generateCollectionScripts(setup2.byCollectionForSSR);
+  const collectionScripts = generateCollectionScripts(setup2.byCollectionForSSR, capturedResults);
   const config = await loadConfig(cwd);
   const browserTelemetryConfigRaw = config.telemetry?.browser;
   const { serializeBrowserTelemetryConfig: serializeBrowserTelemetryConfig2 } = await Promise.resolve().then(() => exports_serialize);
@@ -3840,7 +3870,7 @@ async function renderSSRDocument(options) {
   if (setup2.metadataResult.statusOverride !== undefined) {
     statusOverride = setup2.metadataResult.statusOverride;
   }
-  const extraHead = setup2.bundlePreload + setup2.preloadExtraHead;
+  const extraHead = appendLlmsAlternateHead(setup2.bundlePreload + setup2.preloadExtraHead, pathname, statusOverride || 200, config.llms);
   const combinedHead = dedent([
     dedent(head) || "",
     dedent(extraHead),
@@ -3870,6 +3900,7 @@ async function renderSSRDocument(options) {
 var init_render = __esm(() => {
   init_server2();
   init_config();
+  init_discovery3();
   init_lib();
   init_html();
 });
@@ -4196,6 +4227,112 @@ var init_extractor = __esm(() => {
   init_render();
 });
 
+// packages/core/src/llms/discovery.ts
+function yesNo(value) {
+  return value === false ? "no" : "yes";
+}
+function formatContentSignal(policy, whenUndefined = "omit") {
+  if (policy === false)
+    return;
+  if (policy === undefined && whenUndefined === "omit")
+    return;
+  const resolved = {
+    search: true,
+    aiInput: true,
+    aiTrain: true,
+    ...policy
+  };
+  return `ai-train=${yesNo(resolved.aiTrain)}, search=${yesNo(resolved.search)}, ai-input=${yesNo(resolved.aiInput)}`;
+}
+function markdownTwinPath(pathname) {
+  const path2 = pathname.split("?")[0];
+  if (path2 === "/" || path2 === "")
+    return "/index.md";
+  return `${path2.replace(/\/$/, "")}.md`;
+}
+function shouldAdvertiseMarkdownTwin(pathname, llmsConfig) {
+  if (!llmsConfig?.enabled)
+    return false;
+  if (llmsConfig.enableMarkdownExtension === false)
+    return false;
+  if (shouldExcludeRoute(pathname, llmsConfig.excludeRoutes))
+    return false;
+  return true;
+}
+function llmsAlternateLinkTag(pathname, llmsConfig) {
+  if (!shouldAdvertiseMarkdownTwin(pathname, llmsConfig))
+    return "";
+  return `<link rel="alternate" type="text/markdown" href="${markdownTwinPath(pathname)}" />`;
+}
+function mergeVary(existing, token) {
+  const parts = (existing ?? "").split(",").map((part) => part.trim()).filter(Boolean);
+  if (!parts.some((part) => part.toLowerCase() === token.toLowerCase())) {
+    parts.push(token);
+  }
+  return parts.join(", ");
+}
+function appendLinkValues(existing, values) {
+  const current = (existing ?? "").split(",").map((part) => part.trim()).filter(Boolean);
+  for (const value of values) {
+    if (!current.includes(value))
+      current.push(value);
+  }
+  return current.join(", ");
+}
+function buildLlmsDiscovery(pathname, llmsConfig) {
+  if (!llmsConfig?.enabled)
+    return null;
+  const linkValues = [];
+  if (shouldAdvertiseMarkdownTwin(pathname, llmsConfig)) {
+    linkValues.push(`<${markdownTwinPath(pathname)}>; rel="alternate"; type="text/markdown"`);
+  }
+  if (llmsConfig.enableIndex !== false) {
+    linkValues.push('</llms.txt>; rel="llms-txt"');
+  }
+  const contentSignal = formatContentSignal(llmsConfig.contentSignal, "default");
+  const vary = llmsConfig.enableContentNegotiation !== false ? "Accept" : undefined;
+  if (linkValues.length === 0 && !contentSignal && !vary)
+    return null;
+  return { linkValues, contentSignal, vary };
+}
+function writeHeader(headers2, canonical, value) {
+  const lower = canonical.toLowerCase();
+  if (headers2[lower] !== undefined)
+    headers2[lower] = value;
+  if (headers2[canonical] !== undefined || headers2[lower] === undefined) {
+    headers2[canonical] = value;
+  }
+}
+function readHeader(headers2, canonical) {
+  return headers2[canonical] ?? headers2[canonical.toLowerCase()];
+}
+function applyLlmsDiscoveryHeaders(headers2, pathname, llmsConfig, status = 200) {
+  if (status < 200 || status >= 300)
+    return;
+  const discovery2 = buildLlmsDiscovery(pathname, llmsConfig);
+  if (!discovery2)
+    return;
+  if (discovery2.linkValues.length > 0) {
+    writeHeader(headers2, "Link", appendLinkValues(readHeader(headers2, "Link"), discovery2.linkValues));
+  }
+  if (discovery2.contentSignal) {
+    writeHeader(headers2, "Content-Signal", discovery2.contentSignal);
+  }
+  if (discovery2.vary) {
+    writeHeader(headers2, "Vary", mergeVary(readHeader(headers2, "Vary"), discovery2.vary));
+  }
+}
+function appendLlmsAlternateHead(extraHead, pathname, status, llmsConfig) {
+  if (status !== 200)
+    return extraHead;
+  const tag = llmsAlternateLinkTag(pathname, llmsConfig);
+  return tag ? `${extraHead}
+${tag}` : extraHead;
+}
+var init_discovery3 = __esm(() => {
+  init_extractor();
+});
+
 // packages/core/src/llms/index-generator.ts
 var exports_index_generator = {};
 __export(exports_index_generator, {
@@ -4698,6 +4835,7 @@ var init_full_generator = __esm(() => {
 
 // packages/core/src/llms/index.ts
 var init_llms = __esm(() => {
+  init_discovery3();
   init_extractor();
   init_full_generator();
   init_index_generator();
@@ -4864,7 +5002,7 @@ async function discoverRobotsData(cwd, isWatchMode, robotsConfig, sitemapConfig,
     redirectSourcePaths
   };
 }
-var init_discovery3 = () => {};
+var init_discovery4 = () => {};
 
 // packages/core/src/robots/generator.ts
 function getEffectivePolicies(policies) {
@@ -4907,8 +5045,16 @@ function addDisallowDirectives(lines, policy, autoDisallows) {
     }
   }
 }
-function addPolicyBlock(lines, policy, autoDisallows) {
+function addContentSignalLine(lines, policy, robotsConfig) {
+  if (policy.userAgent !== "*")
+    return;
+  const value = formatContentSignal(robotsConfig.contentSignal);
+  if (value)
+    lines.push(`Content-Signal: ${value}`);
+}
+function addPolicyBlock(lines, policy, autoDisallows, robotsConfig) {
   lines.push(`User-agent: ${policy.userAgent}`);
+  addContentSignalLine(lines, policy, robotsConfig);
   addAllowDirectives(lines, policy);
   addDisallowDirectives(lines, policy, autoDisallows);
   if (policy.crawlDelay !== undefined) {
@@ -4934,7 +5080,7 @@ function generateRobots(discovery2, robotsConfig, sitemapEnabled) {
   const effectivePolicies = getEffectivePolicies(policies);
   const autoDisallows = getAutoDisallows(discovery2);
   for (const policy of effectivePolicies) {
-    addPolicyBlock(lines, policy, autoDisallows);
+    addPolicyBlock(lines, policy, autoDisallows, robotsConfig);
   }
   addGlobalCrawlDelay(lines, crawlDelay, effectivePolicies);
   if (customRules) {
@@ -4946,6 +5092,9 @@ function generateRobots(discovery2, robotsConfig, sitemapEnabled) {
 `).trim()}
 `;
 }
+var init_generator = __esm(() => {
+  init_discovery3();
+});
 
 // packages/core/src/robots/policies.ts
 function blockAICrawlers(crawlers) {
@@ -5016,7 +5165,8 @@ var init_policies = __esm(() => {
 
 // packages/core/src/robots/index.ts
 var init_robots = __esm(() => {
-  init_discovery3();
+  init_discovery4();
+  init_generator();
   init_policies();
 });
 
@@ -5349,7 +5499,7 @@ async function discoverSitemapData(cwd, collections2, baseUrl, isWatchMode, site
     ssrDataRoutes
   };
 }
-var init_discovery4 = () => {};
+var init_discovery5 = () => {};
 
 // packages/core/src/rss/discovery.ts
 import { pathToFileURL as pathToFileURL7 } from "node:url";
@@ -5617,7 +5767,7 @@ async function discoverRSSData(cwd, collections2, baseUrl, isWatchMode, rssConfi
   }
   const ssrDataFeeds = new Map;
   const bundledRoutes = globalThis.__REROUTE_ROUTES__;
-  const { discoverRoutes: discoverRoutes2 } = await Promise.resolve().then(() => (init_discovery4(), exports_discovery));
+  const { discoverRoutes: discoverRoutes2 } = await Promise.resolve().then(() => (init_discovery5(), exports_discovery));
   const { staticRoutes } = await discoverRoutes2(cwd, isWatchMode);
   if (bundledRoutes && Array.isArray(bundledRoutes)) {
     const routePromises = staticRoutes.map(async (route) => {
@@ -5674,7 +5824,7 @@ async function discoverRSSData(cwd, collections2, baseUrl, isWatchMode, rssConfi
     ssrDataFeeds
   };
 }
-var init_discovery5 = () => {};
+var init_discovery6 = () => {};
 
 // packages/core/src/rss/generator.ts
 function generateRSSXML(feed, options = {}) {
@@ -5787,7 +5937,7 @@ function escapeXml(unsafe) {
 
 // packages/core/src/rss/index.ts
 var init_rss = __esm(() => {
-  init_discovery5();
+  init_discovery6();
 });
 
 // packages/core/src/sitemap/generator.ts
@@ -5932,12 +6082,12 @@ function escapeXml2(unsafe) {
   return unsafe.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
 }
 var DEFAULT_MAX_URLS = 50000;
-var init_generator = () => {};
+var init_generator2 = () => {};
 
 // packages/core/src/sitemap/index.ts
 var init_sitemap = __esm(() => {
-  init_discovery4();
-  init_generator();
+  init_discovery5();
+  init_generator2();
 });
 
 // packages/core/src/ssr/stream.ts
@@ -6015,15 +6165,16 @@ async function renderSSRDocumentStream(options) {
     const newAttrs = hasLang ? attrs.replace(/lang\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/i, `lang="${pageLang}"`) : `${attrs} lang="${pageLang}"`;
     return `<html${newAttrs}>`;
   });
-  const extraHead = setup2.bundlePreload + setup2.preloadExtraHead + setup2.metadataResult.perPageHead;
-  if (/<title[\s\S]*?<\/title>/i.test(extraHead)) {
+  const extraHeadBase = setup2.bundlePreload + setup2.preloadExtraHead + setup2.metadataResult.perPageHead;
+  if (/<title[\s\S]*?<\/title>/i.test(extraHeadBase)) {
     headWithLang = headWithLang.replace(/<title[\s\S]*?<\/title>/i, "");
   }
-  if (/<meta\s+name\s*=\s*['"]description['"][^>]*>/i.test(extraHead)) {
+  if (/<meta\s+name\s*=\s*['"]description['"][^>]*>/i.test(extraHeadBase)) {
     headWithLang = headWithLang.replace(/<meta\s+name\s*=\s*['"]description['"][^>]*>/i, "");
   }
-  headWithLang = removeOverriddenMetaTags(headWithLang, extraHead);
+  headWithLang = removeOverriddenMetaTags(headWithLang, extraHeadBase);
   const config = await loadConfig(cwd);
+  const extraHead = appendLlmsAlternateHead(extraHeadBase, pathname, statusOverride || 200, config.llms);
   const browserTelemetryConfigRaw = config.telemetry?.browser;
   const { serializeBrowserTelemetryConfig: serializeBrowserTelemetryConfig2 } = await Promise.resolve().then(() => exports_serialize);
   const browserTelemetryConfig = serializeBrowserTelemetryConfig2(browserTelemetryConfigRaw);
@@ -6281,6 +6432,7 @@ function unescapeHtmlEntities(str) {
 var init_stream = __esm(() => {
   init_server2();
   init_config();
+  init_discovery3();
   init_lib();
   init_html();
   init_imports();
@@ -6327,12 +6479,15 @@ export {
   acceptsGzip,
   allowAICrawlers,
   allowSearchCrawlers,
+  appendLlmsAlternateHead,
   applyIndexTemplate,
+  applyLlmsDiscoveryHeaders,
   basename,
   blockAICrawlers,
   blockAITrainingCrawlers,
   blockSearchCrawlers,
   buildHeadFromMeta,
+  buildLlmsDiscovery,
   buildLocaleCookie,
   cleanupPathScopedSSRData,
   clearSSRModuleStores,
@@ -6378,6 +6533,7 @@ export {
   findOGImageForPath,
   formatAsMarkdown,
   formatAsText,
+  formatContentSignal,
   formatMessage,
   generateAtomXML,
   generateCollectionScripts,
@@ -6409,6 +6565,7 @@ export {
   isThenable,
   join,
   listContentFiles,
+  llmsAlternateLinkTag,
   loadCollections,
   loadConfig,
   loadIndexHtml,
@@ -6418,9 +6575,11 @@ export {
   localizeHref,
   looksLikeLocale,
   lookupMessage,
+  markdownTwinPath,
   matchRoutePattern,
   matchRoutes,
   mergeLayoutData,
+  mergeVary,
   negotiateLocale,
   normalizePathname,
   parseContentPath,
@@ -6433,10 +6592,12 @@ export {
   renderSSRDocument,
   renderSSRDocumentStream,
   resetSSRAccessTracking,
+  resetSSRContentResults,
   resolveLocale,
   seedSSRModuleForPath,
   setGlobalCollections,
   setGlobalSSRData,
+  shouldAdvertiseMarkdownTwin,
   shouldExcludeRoute,
   splitTemplate,
   ssrCache,
@@ -6444,9 +6605,10 @@ export {
   stripEnd,
   stripHtmlTags,
   stripStart,
+  takeSSRContentResults,
   toBytes,
   toOgLocale,
   toPromise
 };
 
-//# debugId=59426A6F0EC6560464756E2164756E21
+//# debugId=B1DFB764A57413D364756E2164756E21