simple-dynamsoft-mcp 7.6.1 → 7.7.1

package/src/rag/lexical-provider.js

@@ -1,5 +1,5 @@
 import Fuse from "fuse.js";
-import { expandQueryTokens, extractSnippet } from "./search-utils.js";
+import { chunkMarkdown, expandQueryTokens, extractSnippet } from "./search-utils.js";
 
 const DEFAULT_FUSE_OPTIONS = {
   keys: [
@@ -12,27 +12,40 @@ const DEFAULT_FUSE_OPTIONS = {
   includeScore: true
 };
 
-const BODY_INDEX_CHARS = 4000;
+// Section size and overlap for body scoring; match the embedding chunking so
+// lexical and Gemini see the same units, and so a term cut at a chunk boundary
+// still appears whole in the neighbouring chunk.
+const SECTION_CHARS = 1200;
+const SECTION_OVERLAP_CHARS = 200;
 
-// Field-weighted BM25 haystack: title/tags outrank summary; doc/sample body text
-// is indexed at low weight so body-only content is findable; the URI's last slug
+// Field-weighted header: title/tags outrank summary; the URI's last slug
 // segment is kept (it is often the most descriptive token) while scheme/version
 // noise is dropped (issue #147).
-function buildLexicalHaystack(entry) {
+function buildHeaderText(entry) {
   const title = entry.title || "";
   const tags = Array.isArray(entry.tags) ? entry.tags.join(" ") : "";
   const summary = entry.summary || "";
-  const body = entry.embedText ? String(entry.embedText).slice(0, BODY_INDEX_CHARS) : "";
   const uriSlug = String(entry.uri || "").split("/").filter(Boolean).pop() || "";
-  return [
-    title, title, title,
-    tags, tags,
-    summary,
-    uriSlug,
-    body
-  ].join(" \n ");
+  return [title, title, title, tags, tags, summary, uriSlug].join(" \n ");
 }
 
+// The whole body, split on headings (each section keeps its heading path), so
+// content deep in long guides is searchable and a page is ranked by its best
+// matching section instead of by one length-diluted blob.
+function buildBodySections(entry) {
+  if (!entry.embedText) return [];
+  return chunkMarkdown(entry.embedText, SECTION_CHARS, SECTION_OVERLAP_CHARS, 0);
+}
+
+const MAX_SCOPED_FUSE_INDEXES = 64;
+
+// Fuse matches long natural-language questions only through fragments of the
+// pattern, giving a weak, noisy signal at a cost of seconds on large scopes.
+// On the golden set (scripts/eval-retrieval.mjs), skipping it above this length
+// kept recall, raised MRR, and cut warm p95 from ~2.2s to ~0.2s. BM25 carries
+// long queries.
+const MAX_FUSE_QUERY_CHARS = 64;
+
 const BM25_K1 = 1.2;
 const BM25_B = 0.75;
 
@@ -61,63 +74,86 @@ function compareLexicalResults(a, b) {
   return String(a.entry.uri || "").localeCompare(String(b.entry.uri || ""));
 }
 
-function buildBm25Index(entries) {
-  const documents = [];
+function buildBm25Stats(units) {
   const documentFrequency = new Map();
   let totalLength = 0;
-
-  entries.forEach((entry, index) => {
-    const haystack = buildLexicalHaystack(entry);
-
-    const tokens = tokenize(haystack);
-    const termFreq = new Map();
-    tokens.forEach((token) => {
-      termFreq.set(token, (termFreq.get(token) || 0) + 1);
-    });
-
-    const seen = new Set();
-    for (const token of tokens) {
-      if (seen.has(token)) continue;
-      seen.add(token);
+  for (const unit of units) {
+    unit.termFreq = new Map();
+    for (const token of unit.tokens) {
+      unit.termFreq.set(token, (unit.termFreq.get(token) || 0) + 1);
+    }
+    for (const token of unit.termFreq.keys()) {
       documentFrequency.set(token, (documentFrequency.get(token) || 0) + 1);
     }
+    unit.length = unit.tokens.length;
+    totalLength += unit.length;
+    delete unit.tokens;
+  }
+  return {
+    documentFrequency,
+    totalDocuments: units.length,
+    avgLength: units.length > 0 ? totalLength / units.length : 0
+  };
+}
 
-    totalLength += tokens.length;
-    documents.push({
-      index,
-      entry,
-      length: tokens.length,
-      termFreq,
-      snippetText: entry.embedText || entry.summary || entry.title || ""
-    });
-  });
-
-  const avgLength = documents.length > 0 ? totalLength / documents.length : 0;
+// Two BM25 indexes: one header unit per entry, one unit per body section.
+// A page scores header + best section.
+function buildBm25Index(entries) {
+  const documents = entries.map((entry, index) => ({
+    index,
+    entry,
+    tokens: tokenize(buildHeaderText(entry)),
+    sections: [],
+    fallbackSnippet: entry.summary || entry.title || ""
+  }));
+  const sections = [];
+  for (const doc of documents) {
+    for (const text of buildBodySections(doc.entry)) {
+      const section = { text, tokens: tokenize(text) };
+      doc.sections.push(section);
+      sections.push(section);
+    }
+  }
   return {
     documents,
-    documentFrequency,
-    totalDocuments: documents.length,
-    avgLength
+    headerStats: buildBm25Stats(documents),
+    sectionStats: buildBm25Stats(sections)
   };
 }
 
-function computeBm25Score(indexState, docState, terms) {
+function computeBm25Score(stats, unit, terms) {
   if (terms.length === 0) return 0;
-  if (!docState.length || !indexState.avgLength) return 0;
+  if (!unit.length || !stats.avgLength) return 0;
 
   let score = 0;
   for (const term of terms) {
-    const tf = docState.termFreq.get(term) || 0;
+    const tf = unit.termFreq.get(term) || 0;
     if (!tf) continue;
-    const df = indexState.documentFrequency.get(term) || 0;
-    const idf = Math.log(1 + (indexState.totalDocuments - df + 0.5) / (df + 0.5));
-    const denom = tf + BM25_K1 * (1 - BM25_B + BM25_B * (docState.length / indexState.avgLength));
+    const df = stats.documentFrequency.get(term) || 0;
+    const idf = Math.log(1 + (stats.totalDocuments - df + 0.5) / (df + 0.5));
+    const denom = tf + BM25_K1 * (1 - BM25_B + BM25_B * (unit.length / stats.avgLength));
     score += idf * ((tf * (BM25_K1 + 1)) / denom);
   }
 
   return score;
 }
 
+function scoreDocument(index, doc, terms) {
+  let best = null;
+  let bestScore = 0;
+  for (const section of doc.sections) {
+    const score = computeBm25Score(index.sectionStats, section, terms);
+    if (score > bestScore) {
+      bestScore = score;
+      best = section;
+    }
+  }
+  return {
+    score: computeBm25Score(index.headerStats, doc, terms) + bestScore,
+    snippetText: best ? best.text : doc.fallbackSnippet
+  };
+}
+
 function createLexicalProvider({
   entries,
   entryMatchesScope,
@@ -126,10 +162,24 @@ function createLexicalProvider({
   bm25Weight = 0.7,
   fuseWeight = 0.3
 }) {
-  const fuse = new Fuse(entries, fuseOptions);
   const bm25Index = buildBm25Index(entries);
   const entryByUri = new Map(entries.map((entry) => [entry.uri, entry]));
-  const snippetTextByUri = new Map(bm25Index.documents.map((doc) => [doc.entry.uri, doc.snippetText]));
+  // Fuse cost grows with the number of entries scanned, and its hits were
+  // filtered by scope only afterwards. One Fuse index per scope keeps scoped
+  // searches proportional to that scope; the scope set is small and bounded.
+  const fuseByScope = new Map();
+  const getScopedFuse = (filters) => {
+    const key = [filters?.product, filters?.edition, filters?.platform, filters?.type].map((v) => v || "").join("|");
+    let scoped = fuseByScope.get(key);
+    if (!scoped) {
+      scoped = new Fuse(entries.filter((entry) => entryMatchesScope(entry, filters || {})), fuseOptions);
+      if (fuseByScope.size >= MAX_SCOPED_FUSE_INDEXES) {
+        fuseByScope.delete(fuseByScope.keys().next().value);
+      }
+      fuseByScope.set(key, scoped);
+    }
+    return scoped;
+  };
 
   return {
     name: "lexical",
@@ -137,7 +187,7 @@ function createLexicalProvider({
       // Expand with domain synonyms/stems so vocabulary gaps (webcam/camera,
       // datamatrix/data matrix, scanning/scan) do not zero the BM25 component.
       const terms = expandQueryTokens([...new Set(tokenize(query))]);
-      const fuseHits = fuse.search(query);
+      const fuseHits = query.length <= MAX_FUSE_QUERY_CHARS ? getScopedFuse(filters).search(query) : [];
       const fuseScoreByUri = new Map();
 
       for (const hit of fuseHits) {
@@ -154,9 +204,11 @@ function createLexicalProvider({
       // out-of-scope fuzzy match deflate every in-scope fuse contribution (issue #147).
       let maxFuse = 0;
 
+      const snippetTextByUri = new Map();
       for (const doc of bm25Index.documents) {
         if (!entryMatchesScope(doc.entry, filters)) continue;
-        const score = computeBm25Score(bm25Index, doc, terms);
+        const { score, snippetText } = scoreDocument(bm25Index, doc, terms);
+        snippetTextByUri.set(doc.entry.uri, snippetText);
         if (score > 0) {
           bm25ScoreByUri.set(doc.entry.uri, score);
           if (score > maxBm25) maxBm25 = score;

package/src/rag/search-utils.js

@@ -78,9 +78,11 @@ function chunkMarkdown(text, chunkSize, chunkOverlap, maxChunks) {
   const sections = [];
   let current = { path: "", lines: [] };
 
+  // Heading-only sections (e.g. index pages whose headings are links) are
+  // kept: their heading path is the only text that makes them findable.
   const flush = () => {
     const body = current.lines.join(" ").replace(/\s+/g, " ").trim();
-    if (body) sections.push({ path: current.path, body });
+    if (body || current.path) sections.push({ path: current.path, body });
   };
 
   let inFence = false;
@@ -116,6 +118,11 @@ function chunkMarkdown(text, chunkSize, chunkOverlap, maxChunks) {
   const chunks = [];
   for (const section of sections) {
     const prefix = section.path ? `${section.path}\n` : "";
+    if (!section.body) {
+      chunks.push(section.path);
+      if (maxChunks && chunks.length >= maxChunks) return chunks;
+      continue;
+    }
     const pieces = chunkText(section.body, chunkSize, chunkOverlap, maxChunks);
     for (const piece of pieces) {
       chunks.push(`${prefix}${piece}`.trim());
@@ -248,6 +255,7 @@ function buildIndexSignature({ pkgVersion, signatureData, ragConfig }) {
     dbrWebDocCount: signatureData.dbrWebDocCount,
     dbrMobileDocCount: signatureData.dbrMobileDocCount,
     dbrServerDocCount: signatureData.dbrServerDocCount,
+    dbrSharedDocCount: signatureData.dbrSharedDocCount,
     dwtDocCount: signatureData.dwtDocCount,
     ddvDocCount: signatureData.ddvDocCount,
     versions: signatureData.versions,
@@ -257,7 +265,9 @@ function buildIndexSignature({ pkgVersion, signatureData, ragConfig }) {
     maxChunksPerDoc: ragConfig.maxChunksPerDoc,
     maxTextChars: ragConfig.maxTextChars,
     // Bump when the chunking strategy changes so cached vector indexes rebuild.
-    chunkStrategy: "heading-aware-v1"
+    chunkStrategy: "heading-aware-v2",
+    // Bump when doc:// URIs change shape; cached vectors are keyed by URI.
+    docUriScheme: "source-path-v1"
   });
 }
 

package/src/rag/vector-cache.js

@@ -20,6 +20,70 @@ function makeCheckpointFileName(provider, model, cacheKey) {
   return `rag-${provider}-${safeModel}-${cacheKey.slice(0, 12)}.checkpoint.json`;
 }
 
+// Cache layout: a JSON header (`<name>.json`: cacheKey, meta, items, vector
+// count/dimensions) plus a sibling `<name>.f32` file holding all vectors as
+// packed little-endian float32. Serializing vectors into JSON overflowed V8's
+// ~512MB max string length once the index grew; neither file here is built as
+// one large string.
+const VECTOR_FORMAT = "float32-le-v1";
+
+function vectorFileFor(headerFile) {
+  return headerFile.replace(/\.json$/i, ".f32");
+}
+
+function writeVectorFiles(headerFile, payload) {
+  const { vectors, ...header } = payload;
+  const list = Array.isArray(vectors) ? vectors : [];
+  const dimensions = list.length > 0 ? list[0].length : 0;
+  const packed = new Float32Array(list.length * dimensions);
+  list.forEach((vector, index) => {
+    if (vector.length !== dimensions) {
+      throw new Error(`Vector ${index} has ${vector.length} dimensions; expected ${dimensions}`);
+    }
+    packed.set(vector, index * dimensions);
+  });
+  // Vectors first: a header always describes a complete vector file, and a
+  // stale header next to a newer vector file fails the size check on load.
+  writeFileSync(vectorFileFor(headerFile), Buffer.from(packed.buffer, packed.byteOffset, packed.byteLength));
+  writeFileSync(headerFile, JSON.stringify({
+    ...header,
+    vectorFormat: VECTOR_FORMAT,
+    vectorCount: list.length,
+    dimensions
+  }));
+}
+
+// Returns the parsed header with `vectors` as Float32Array views over one
+// buffer, or { error } describing why the files cannot be used.
+function readVectorFiles(headerFile) {
+  const header = JSON.parse(readFileSync(headerFile, "utf8"));
+  if (!header || header.vectorFormat !== VECTOR_FORMAT || !Array.isArray(header.items)) {
+    return { error: "invalid_payload" };
+  }
+  const vectorFile = vectorFileFor(headerFile);
+  if (!existsSync(vectorFile)) {
+    return { error: "vectors_missing" };
+  }
+  const count = Number(header.vectorCount) || 0;
+  const dimensions = Number(header.dimensions) || 0;
+  const raw = readFileSync(vectorFile);
+  if (raw.byteLength !== count * dimensions * Float32Array.BYTES_PER_ELEMENT) {
+    return { error: "vectors_size_mismatch" };
+  }
+  // Copy into an aligned buffer; Buffer offsets are not guaranteed 4-byte aligned.
+  const all = new Float32Array(raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength));
+  const vectors = new Array(count);
+  for (let i = 0; i < count; i += 1) {
+    vectors[i] = all.subarray(i * dimensions, (i + 1) * dimensions);
+  }
+  return { payload: { ...header, vectors } };
+}
+
+function removeVectorFiles(headerFile) {
+  rmSync(headerFile, { force: true });
+  rmSync(vectorFileFor(headerFile), { force: true });
+}
+
 function loadVectorIndexCache(
   cacheFile,
   { cacheKey, signature, provider, model, requireSignature = false } = {}
@@ -28,12 +92,12 @@ function loadVectorIndexCache(
     return { hit: false, reason: "missing", payload: null };
   }
   try {
-    const parsed = JSON.parse(readFileSync(cacheFile, "utf8"));
-    if (!parsed || (cacheKey && parsed.cacheKey !== cacheKey)) {
-      return { hit: false, reason: "cache_key_mismatch", payload: null };
+    const { payload: parsed, error } = readVectorFiles(cacheFile);
+    if (error) {
+      return { hit: false, reason: error, payload: null };
     }
-    if (!Array.isArray(parsed.items) || !Array.isArray(parsed.vectors)) {
-      return { hit: false, reason: "invalid_payload", payload: null };
+    if (cacheKey && parsed.cacheKey !== cacheKey) {
+      return { hit: false, reason: "cache_key_mismatch", payload: null };
     }
     const meta = parsed.meta || {};
     if (provider && meta.provider && meta.provider !== provider) {
@@ -59,23 +123,7 @@ function loadVectorIndexCache(
 
 function saveVectorIndexCache(cacheDir, cacheFile, payload) {
   ensureDirectory(cacheDir);
-  let serialized;
-  try {
-    serialized = JSON.stringify(payload);
-  } catch (error) {
-    // JSON.stringify throws "Invalid string length" once the serialized cache
-    // exceeds V8's ~512MB max string length (reached with a large index of
-    // high-dimensional vectors). Fail with an actionable message instead.
-    if (error instanceof RangeError) {
-      const itemCount = Array.isArray(payload?.items) ? payload.items.length : "unknown";
-      throw new Error(
-        `Vector cache too large to serialize (${itemCount} items) — exceeds the JSON string limit. ` +
-        "Lower RAG_MAX_CHUNKS_PER_DOC or adopt a ceiling-proof cache format."
-      );
-    }
-    throw error;
-  }
-  writeFileSync(cacheFile, serialized);
+  writeVectorFiles(cacheFile, payload);
 }
 
 function loadVectorIndexCheckpoint(checkpointFile, expectedKey, expectedItems) {
@@ -83,12 +131,12 @@ function loadVectorIndexCheckpoint(checkpointFile, expectedKey, expectedItems) {
     return { hit: false, reason: "missing", payload: null };
   }
   try {
-    const parsed = JSON.parse(readFileSync(checkpointFile, "utf8"));
-    if (!parsed || parsed.cacheKey !== expectedKey) {
-      return { hit: false, reason: "cache_key_mismatch", payload: null };
+    const { payload: parsed, error } = readVectorFiles(checkpointFile);
+    if (error) {
+      return { hit: false, reason: error, payload: null };
     }
-    if (!Array.isArray(parsed.items) || !Array.isArray(parsed.vectors)) {
-      return { hit: false, reason: "invalid_payload", payload: null };
+    if (parsed.cacheKey !== expectedKey) {
+      return { hit: false, reason: "cache_key_mismatch", payload: null };
     }
     if (parsed.items.length !== expectedItems.length) {
       return { hit: false, reason: "items_length_mismatch", payload: null };
@@ -109,13 +157,11 @@ function loadVectorIndexCheckpoint(checkpointFile, expectedKey, expectedItems) {
 
 function saveVectorIndexCheckpoint(cacheDir, checkpointFile, payload) {
   ensureDirectory(cacheDir);
-  writeFileSync(checkpointFile, JSON.stringify(payload));
+  writeVectorFiles(checkpointFile, payload);
 }
 
 function clearVectorIndexCheckpoint(checkpointFile) {
-  if (existsSync(checkpointFile)) {
-    rmSync(checkpointFile, { force: true });
-  }
+  removeVectorFiles(checkpointFile);
 }
 
 function listFilesRecursive(rootDir) {

package/src/server/resource-index/builders.js

@@ -1,6 +1,7 @@
 import { existsSync, readdirSync, statSync } from "node:fs";
 import { extname, join } from "node:path";
 import { DDV_PREFERRED_ENTRY_FILES } from "./config.js";
+import { shortDocPath } from "./docs-loader.js";
 import { PRODUCT_SELECTION_GUIDANCE, PUBLIC_OFFERING_PRODUCTS } from "../public-offerings.js";
 
 const MRZ_MATCHER = /(?:\bmrz\b|machine[-\s]?readable[-\s]?zone|passport)/i;
@@ -201,6 +202,29 @@ function addPublicResourceToIndex(addResourceToIndex, entry) {
   }
 }
 
+function encodeDocSlug(slug) {
+  return String(slug).split("/").map((segment) => encodeURIComponent(segment)).join("/");
+}
+
+// One doc:// slug per article: its short path, or (on a collision within the
+// same URI prefix + platform, e.g. `user-guide.md` vs `user-guide/index.md`)
+// its full source path. Shared pages keep their `slugPrefix` namespace
+// (`dcv`, `dcv-core`) in both forms.
+function assignDocSlugs(docs, defaultPlatform) {
+  const withPrefix = (article, slug) => (article.slugPrefix ? `${article.slugPrefix}/${slug}` : slug);
+  const keyFor = (article, slug) => `${article.platform || defaultPlatform}/${slug}`;
+  const shortSlugs = docs.map((article) => withPrefix(article, shortDocPath(article.path || article.title)));
+  const counts = new Map();
+  docs.forEach((article, i) => {
+    const key = keyFor(article, shortSlugs[i]);
+    counts.set(key, (counts.get(key) || 0) + 1);
+  });
+  return docs.map((article, i) => {
+    if (counts.get(keyFor(article, shortSlugs[i])) === 1) return shortSlugs[i];
+    return withPrefix(article, String(article.path || article.title).replace(/\\/g, "/").replace(/\.md$/i, ""));
+  });
+}
+
 function addMarkdownDocResources({
   addResourceToIndex,
   docs,
@@ -214,12 +238,18 @@ function addMarkdownDocResources({
   defaultSummary,
   baseTags
 }) {
+  const slugs = assignDocSlugs(docs, defaultPlatform);
   for (let i = 0; i < docs.length; i++) {
     const article = docs[i];
     if (!article?.title) continue;
-    const slug = `${encodeURIComponent(article.title)}-${i}`;
+    const slug = encodeDocSlug(slugs[i]);
     const platform = article.platform || defaultPlatform;
     const tags = [...baseTags, platform];
+    if (Array.isArray(article.platforms)) {
+      for (const extra of article.platforms) {
+        if (!tags.includes(extra)) tags.push(extra);
+      }
+    }
     if (article.breadcrumb) tags.push(...article.breadcrumb.toLowerCase().split(/\s*>\s*/));
     // Fold frontmatter keywords into tags so domain-vocabulary search hits (#142).
     if (Array.isArray(article.keywords)) {
@@ -239,7 +269,9 @@ function addMarkdownDocResources({
       product,
       edition,
       platform,
+      ...(Array.isArray(article.platforms) ? { platforms: article.platforms } : {}),
       version,
+      ...(article.sourceVersion ? { sourceVersion: article.sourceVersion } : {}),
       majorVersion,
       url: article.url,
       title: product === "mrz" || product === "mds"
@@ -601,6 +633,7 @@ function buildResourceIndex({
   dbrWebDocs,
   dbrMobileDocs,
   dbrServerDocs,
+  dbrSharedDocs,
   dwtDocs,
   ddvDocs,
   discoverDcvMobileSamples,
@@ -967,6 +1000,27 @@ function buildResourceIndex({
     baseTags: ["doc", "dbr", "server"]
   });
 
+  const sharedEditions = [
+    ["web", dbrWebVersion, "Web"],
+    ["mobile", dbrMobileVersion, "Mobile"],
+    ["server", dbrServerVersion, "Server/Desktop"]
+  ];
+  for (const [edition, version, label] of sharedEditions) {
+    addMarkdownDocResources({
+      addResourceToIndex,
+      docs: dbrSharedDocs?.[edition] || [],
+      idPrefix: `dbr-${edition}-shared-doc`,
+      uriPrefix: `doc://dbr/${edition}`,
+      product: "dbr",
+      edition,
+      version,
+      majorVersion: LATEST_MAJOR.dbr,
+      defaultPlatform: edition,
+      defaultSummary: `Shared Capture Vision API used by Dynamsoft Barcode Reader ${label}`,
+      baseTags: ["doc", "dbr", edition, "shared", "capture-vision"]
+    });
+  }
+
   for (const platform of getDbrMobilePlatforms()) {
     const samples = discoverMobileSamples(platform);
     for (const level of ["high-level", "low-level"]) {
@@ -1155,6 +1209,7 @@ function buildResourceIndex({
 }
 
 export {
+  assignDocSlugs,
   buildIndexData,
   buildResourceIndex
 };

package/src/server/resource-index/docs-loader.js

@@ -98,9 +98,56 @@ function markdownPathToUrl(baseUrl, relativePath) {
   return `${baseUrl}${normalized}`;
 }
 
-function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs = [], excludeFiles = [] }) {
+// Path used in doc:// URIs. The platform is already a URI segment, so the
+// `programming/<lang>/` prefix is dropped, and a directory's index page is
+// addressed by the directory.
+function shortDocPath(relativePath) {
+  const withoutExt = String(relativePath || "").replace(/\\/g, "/").replace(/\.md$/i, "");
+  const withoutPlatform = withoutExt.replace(/^programming\/[^/]+\//, "");
+  const withoutIndex = withoutPlatform.replace(/(^|\/)index$/, "");
+  return withoutIndex || "index";
+}
+
+const SITE_ORIGIN = "https://www.dynamsoft.com";
+
+// Top-level `key: value` pairs from a Jekyll _config.yml. Only path/URL values
+// are kept: they are what docs interpolate as `{{ site.key }}` link prefixes.
+function loadSiteVariables(siteRoot) {
+  const vars = new Map();
+  const configPath = join(siteRoot || "", "_config.yml");
+  if (!siteRoot || !existsSync(configPath)) return vars;
+  for (const line of readFileSync(configPath, "utf8").split(/\r?\n/)) {
+    const match = line.match(/^([A-Za-z0-9_-]+):\s*["']?([^"'#\s]+)["']?\s*(?:#.*)?$/);
+    if (!match) continue;
+    const [, key, value] = match;
+    if (value.startsWith("/")) {
+      vars.set(key, `${SITE_ORIGIN}${value}`);
+    } else if (/^https?:\/\//i.test(value)) {
+      vars.set(key, value);
+    }
+  }
+  return vars;
+}
+
+// Replace `{{ site.key }}` with the absolute URL from _config.yml so links in
+// returned docs (and the shared-doc link map) point at the real public pages.
+// Unknown keys are left untouched.
+function resolveSiteLinks(text, siteVars) {
+  if (!text || !siteVars || siteVars.size === 0) return text;
+  return String(text).replace(/\{\{\s*site\.([A-Za-z0-9_-]+)\s*\}\}/g, (whole, key) => siteVars.get(key) ?? whole);
+}
+
+function loadMarkdownDocs({
+  rootDir,
+  urlBase,
+  siteRoot = rootDir,
+  includeDirNames = [],
+  excludeDirs = [],
+  excludeFiles = []
+}) {
   if (!existsSync(rootDir)) return { articles: [] };
 
+  const siteVars = loadSiteVariables(siteRoot);
   const files = collectMarkdownFiles(rootDir, { includeDirNames, excludeDirs, excludeFiles });
   const articles = [];
 
@@ -108,6 +155,9 @@ function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs
     const relativePath = relative(rootDir, filePath).replace(/\\/g, "/");
     const raw = readFileSync(filePath, "utf8");
     const parsed = parseFrontMatter(raw);
+    // Jekyll redirect stubs have no content of their own; indexing them yields
+    // empty hits whose URL only redirects elsewhere.
+    if (parsed.meta.redirect_to) continue;
     const title = parsed.meta.title || getHeadingTitle(parsed.body) || formatSegmentLabel(relativePath.replace(/\.md$/i, "").split("/").pop());
     if (!title) continue;
     const breadcrumb = parsed.meta.breadcrumbText || buildBreadcrumbFromPath(relativePath) || title;
@@ -121,7 +171,7 @@ function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs
     articles.push({
       title,
       url: markdownPathToUrl(urlBase, relativePath),
-      content: parsed.body.trim(),
+      content: resolveSiteLinks(parsed.body.trim(), siteVars),
       breadcrumb,
       description,
       keywords,
@@ -129,9 +179,12 @@ function loadMarkdownDocs({ rootDir, urlBase, includeDirNames = [], excludeDirs
     });
   }
 
-  return { articles };
+  return { articles, siteVars };
 }
 
 export {
-  loadMarkdownDocs
+  loadMarkdownDocs,
+  loadSiteVariables,
+  resolveSiteLinks,
+  shortDocPath
 };