unity-agentic-tools 0.2.0 → 0.3.1

package/dist/doc-indexer-cli.js

@@ -5,15 +5,29 @@ var __getProtoOf = Object.getPrototypeOf;
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+function __accessProp(key) {
+  return this[key];
+}
+var __toESMCache_node;
+var __toESMCache_esm;
 var __toESM = (mod, isNodeMode, target) => {
+  var canCache = mod != null && typeof mod === "object";
+  if (canCache) {
+    var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
+    var cached = cache.get(mod);
+    if (cached)
+      return cached;
+  }
   target = mod != null ? __create(__getProtoOf(mod)) : {};
   const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
   for (let key of __getOwnPropNames(mod))
     if (!__hasOwnProp.call(to, key))
       __defProp(to, key, {
-        get: () => mod[key],
+        get: __accessProp.bind(mod, key),
         enumerable: true
       });
+  if (canCache)
+    cache.set(mod, to);
   return to;
 };
 var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
@@ -2271,18 +2285,216 @@ function chunkProse(content, filePath) {
   }
   return chunks;
 }
-function stripHtml(html) {
+function removeBalancedBlock(html, openPattern, tagName) {
+  let result = html;
+  let match;
+  while ((match = openPattern.exec(result)) !== null) {
+    const startIdx = match.index;
+    let depth = 1;
+    let i = startIdx + match[0].length;
+    const openTag = new RegExp(`<${tagName}[\\s>]`, "gi");
+    const closeTag = `</${tagName}>`;
+    while (depth > 0 && i < result.length) {
+      const nextClose = result.indexOf(closeTag, i);
+      if (nextClose === -1)
+        break;
+      openTag.lastIndex = i;
+      let openMatch;
+      while ((openMatch = openTag.exec(result)) !== null && openMatch.index < nextClose) {
+        depth++;
+      }
+      depth--;
+      i = nextClose + closeTag.length;
+    }
+    result = result.substring(0, startIdx) + result.substring(i);
+    openPattern.lastIndex = 0;
+  }
+  return result;
+}
+function decodeEntities(text) {
+  return text.replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+}
+function htmlToMarkdown(html) {
+  let text = html;
+  text = text.replace(/<script[\s\S]*?<\/script>/gi, "");
+  text = text.replace(/<style[\s\S]*?<\/style>/gi, "");
+  text = text.replace(/<form[\s\S]*?<\/form>/gi, "");
+  const noisePatterns = [
+    [/<div[^>]*class="[^"]*header-wrapper[^"]*"[^>]*>/gi, "div"],
+    [/<div[^>]*id="sidebar"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*footer-wrapper[^"]*"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*suggest-wrap[^"]*"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*suggest-form[^"]*"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*suggest"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*scrollToFeedback[^"]*"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*nextprev[^"]*"[^>]*>/gi, "div"],
+    [/<div[^>]*class="[^"]*toolbar[^"]*"[^>]*>/gi, "div"],
+    [/<nav[^>]*>/gi, "nav"]
+  ];
+  for (const [pattern, tag] of noisePatterns) {
+    text = removeBalancedBlock(text, pattern, tag);
+  }
+  const contentWrapMatch = text.match(/<div[^>]*id="content-wrap"[^>]*>([\s\S]*)/i);
+  if (contentWrapMatch) {
+    const afterOpen = contentWrapMatch.index + contentWrapMatch[0].indexOf(">") + 1;
+    let depth = 1;
+    let i = afterOpen;
+    while (depth > 0 && i < text.length) {
+      const nextOpen = text.indexOf("<div", i);
+      const nextClose = text.indexOf("</div>", i);
+      if (nextClose === -1)
+        break;
+      if (nextOpen !== -1 && nextOpen < nextClose) {
+        depth++;
+        i = nextOpen + 4;
+      } else {
+        depth--;
+        if (depth === 0) {
+          text = text.substring(afterOpen, nextClose);
+        } else {
+          i = nextClose + 6;
+        }
+      }
+    }
+  } else {
+    const bodyMatch = text.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    if (bodyMatch) {
+      text = bodyMatch[1];
+    }
+  }
+  text = text.replace(/<a[^>]*class=['"][^'"]*switch-link[^'"]*['"][^>]*>[\s\S]*?<\/a>/gi, "");
+  text = text.replace(/<span[^>]*class="[^"]*tooltiptext[^"]*"[^>]*>[\s\S]*?<\/span>/gi, "");
+  text = text.replace(/<span[^>]*class="[^"]*search-words[^"]*"[^>]*>[\s\S]*?<\/span>/gi, "");
+  text = text.replace(/<div[^>]*class="[^"]*breadcrumbs[^"]*"[^>]*>([\s\S]*?)<\/div>/gi, (_match, inner) => {
+    const crumbs = inner.replace(/<a[^>]*>([\s\S]*?)<\/a>/gi, "$1").replace(/<[^>]+>/g, "").split(/\s*[>\/]\s*/).map((c) => c.trim()).filter(Boolean);
+    return crumbs.length > 0 ? `
+> ${crumbs.join(" > ")}
+
+` : "";
+  });
+  text = text.replace(/<pre[^>]*>\s*<code[^>]*>([\s\S]*?)<\/code>\s*<\/pre>/gi, (_m, code) => {
+    const decoded = decodeEntities(code.replace(/<[^>]+>/g, "").trim());
+    return `
+\`\`\`csharp
+${decoded}
+\`\`\`
+`;
+  });
+  text = text.replace(/<pre[^>]*class="[^"]*codeExample[^"]*"[^>]*>([\s\S]*?)<\/pre>/gi, (_m, code) => {
+    const decoded = decodeEntities(code.replace(/<[^>]+>/g, "").trim());
+    return `
+\`\`\`csharp
+${decoded}
+\`\`\`
+`;
+  });
+  text = text.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_m, code) => {
+    const decoded = decodeEntities(code.replace(/<[^>]+>/g, "").trim());
+    return `
+\`\`\`
+${decoded}
+\`\`\`
+`;
+  });
+  text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, (_m, t) => `
+# ${t.replace(/<[^>]+>/g, "").trim()}
+
+`);
+  text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, (_m, t) => `
+## ${t.replace(/<[^>]+>/g, "").trim()}
+
+`);
+  text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, (_m, t) => `
+### ${t.replace(/<[^>]+>/g, "").trim()}
+
+`);
+  text = text.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, (_m, t) => `
+#### ${t.replace(/<[^>]+>/g, "").trim()}
+
+`);
+  text = text.replace(/<table[^>]*>([\s\S]*?)<\/table>/gi, (_m, tableHtml) => {
+    const rows = [];
+    let hasHeader = false;
+    const theadMatch = tableHtml.match(/<thead[^>]*>([\s\S]*?)<\/thead>/i);
+    const headerRowMatch = theadMatch ? theadMatch[1].match(/<tr[^>]*>([\s\S]*?)<\/tr>/i) : tableHtml.match(/<tr[^>]*>([\s\S]*?)<\/tr>/i);
+    if (headerRowMatch) {
+      const thCells = Array.from(headerRowMatch[1].matchAll(/<th[^>]*>([\s\S]*?)<\/th>/gi));
+      if (thCells.length > 0) {
+        hasHeader = true;
+        const headerRow = thCells.map((c) => convertInlineHtml(c[1]).trim()).join(" | ");
+        rows.push(`| ${headerRow} |`);
+        rows.push(`|${thCells.map(() => "---").join("|")}|`);
+      }
+    }
+    const tbodyMatch = tableHtml.match(/<tbody[^>]*>([\s\S]*?)<\/tbody>/i);
+    const bodyHtml = tbodyMatch ? tbodyMatch[1] : tableHtml;
+    const trMatches = Array.from(bodyHtml.matchAll(/<tr[^>]*>([\s\S]*?)<\/tr>/gi));
+    for (const tr of trMatches) {
+      const tdCells = Array.from(tr[1].matchAll(/<td[^>]*>([\s\S]*?)<\/td>/gi));
+      if (tdCells.length === 0)
+        continue;
+      if (!hasHeader && rows.length === 0) {
+        const thCheck = Array.from(tr[1].matchAll(/<th[^>]*>([\s\S]*?)<\/th>/gi));
+        if (thCheck.length > 0)
+          continue;
+      }
+      const cellValues = tdCells.map((c) => convertInlineHtml(c[1]).trim());
+      rows.push(`| ${cellValues.join(" | ")} |`);
+      if (!hasHeader && rows.length === 1) {
+        rows.unshift(`|${tdCells.map(() => "---").join("|")}|`);
+        hasHeader = true;
+      }
+    }
+    return rows.length > 0 ? `
+${rows.join(`
+`)}
+` : "";
+  });
+  text = text.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, (_m, href, linkText) => {
+    const clean = linkText.replace(/<[^>]+>/g, "").trim();
+    if (!clean)
+      return "";
+    return `[${clean}](${href})`;
+  });
+  text = text.replace(/<(?:strong|b)>([\s\S]*?)<\/(?:strong|b)>/gi, "**$1**");
+  text = text.replace(/<(?:em|i)>([\s\S]*?)<\/(?:em|i)>/gi, "*$1*");
+  text = text.replace(/<code>([\s\S]*?)<\/code>/gi, "`$1`");
+  text = text.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_m, content) => {
+    const clean = content.replace(/<[^>]+>/g, "").trim();
+    return `
+- ${clean}`;
+  });
+  text = text.replace(/<\/?(?:ul|ol)[^>]*>/gi, `
+`);
+  text = text.replace(/<br\s*\/?>/gi, `
+`);
+  text = text.replace(/<\/p>/gi, `
+
+`);
+  text = text.replace(/<p[^>]*>/gi, "");
+  text = text.replace(/<hr[^>]*>/gi, `
+---
+`);
+  text = text.replace(/<[^>]+>/g, "");
+  text = decodeEntities(text);
+  text = text.replace(/[ \t]+$/gm, "");
+  text = text.replace(/\n{3,}/g, `
+
+`);
+  text = text.trim();
+  return text;
+}
+function convertInlineHtml(html) {
   let text = html;
-  text = text.replace(/<script[\s\S]*?<\/script>/gi, " ");
-  text = text.replace(/<style[\s\S]*?<\/style>/gi, " ");
-  text = text.replace(/<[^>]+>/g, " ");
-  text = text.replace(/&nbsp;/g, " ");
-  text = text.replace(/&amp;/g, "&");
-  text = text.replace(/&lt;/g, "<");
-  text = text.replace(/&gt;/g, ">");
-  text = text.replace(/&quot;/g, '"');
-  text = text.replace(/&#39;/g, "'");
-  text = text.replace(/\s+/g, " ").trim();
+  text = text.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, (_m, href, linkText) => {
+    const clean = linkText.replace(/<[^>]+>/g, "").trim();
+    return clean ? `[${clean}](${href})` : "";
+  });
+  text = text.replace(/<(?:strong|b)>([\s\S]*?)<\/(?:strong|b)>/gi, "**$1**");
+  text = text.replace(/<(?:em|i)>([\s\S]*?)<\/(?:em|i)>/gi, "*$1*");
+  text = text.replace(/<code>([\s\S]*?)<\/code>/gi, "`$1`");
+  text = text.replace(/<[^>]+>/g, "");
+  text = decodeEntities(text);
   return text;
 }
 function walkDirectory(dir) {
@@ -2347,7 +2559,7 @@ function indexMarkdownFile(filePath, storage) {
 function indexHtmlFile(filePath, storage) {
   const startTime = Date.now();
   const html = import_fs.readFileSync(filePath, "utf-8");
-  const text = stripHtml(html);
+  const text = htmlToMarkdown(html);
   const chunks = chunkProse(text, filePath);
   let embeddingsGenerated = 0;
   if (storage) {
@@ -2375,7 +2587,7 @@ async function indexDocsDirectory(dirPath, extensions = [".md", ".txt", ".html"]
       continue;
     const content = import_fs.readFileSync(fullPath, "utf-8");
     if (ext === ".html") {
-      const text = stripHtml(content);
+      const text = htmlToMarkdown(content);
       allChunks.push(...chunkProse(text, fullPath));
     } else {
       allChunks.push(...extractCodeBlocks(content));
@@ -2483,7 +2695,7 @@ async function indexSource(source, storage) {
     const ext = file.fullPath.substring(file.fullPath.lastIndexOf("."));
     const fileChunks = [];
     if (ext === ".html") {
-      const text = stripHtml(content);
+      const text = htmlToMarkdown(content);
       fileChunks.push(...chunkProse(text, file.fullPath));
     } else {
       fileChunks.push(...extractCodeBlocks(content, file.fullPath));
@@ -2617,28 +2829,37 @@ class DocStorage {
     await this.init();
     const lowerQuery = query.toLowerCase();
     const queryTerms = lowerQuery.split(/\s+/).filter((t) => t.length > 0);
+    if (queryTerms.length === 0)
+      return [];
+    const minTermRatio = 0.5;
     const results = [];
     for (const [id, chunk] of this.chunks) {
       const lowerContent = chunk.content.toLowerCase();
-      if (lowerContent.includes(lowerQuery)) {
-        let termHits = 0;
-        for (const term of queryTerms) {
-          let idx = 0;
-          while ((idx = lowerContent.indexOf(term, idx)) !== -1) {
-            termHits++;
-            idx += term.length;
-          }
-        }
-        const wordCount = lowerContent.split(/\s+/).length;
-        const score = termHits / Math.max(wordCount, 1);
-        if (score > 0) {
-          results.push({
-            id,
-            content: chunk.content,
-            score,
-            metadata: chunk.metadata
-          });
-        }
+      let matchedTerms = 0;
+      let termHits = 0;
+      for (const term of queryTerms) {
+        let found = false;
+        let idx = 0;
+        while ((idx = lowerContent.indexOf(term, idx)) !== -1) {
+          termHits++;
+          found = true;
+          idx += term.length;
+        }
+        if (found)
+          matchedTerms++;
+      }
+      const termRatio = matchedTerms / queryTerms.length;
+      if (termRatio < minTermRatio)
+        continue;
+      const wordCount = lowerContent.split(/\s+/).length;
+      const score = termHits / Math.max(wordCount, 1);
+      if (score > 0) {
+        results.push({
+          id,
+          content: chunk.content,
+          score,
+          metadata: chunk.metadata
+        });
       }
     }
     results.sort((a, b) => b.score - a.score);
@@ -2868,6 +3089,20 @@ function read_unity_version(projectRoot) {
 // src/cli.ts
 var program2 = new Command;
 program2.name("unity-doc-indexer").description("Fast Unity documentation indexer with local embeddings").version("1.0.0").option("--project-root <path>", "Unity project root (auto-detected if omitted)").option("--storage-path <path>", "Index storage file path (auto-resolved if omitted)");
+function extract_doc_title(filePath) {
+  if (!filePath)
+    return "Unknown";
+  const filename = filePath.split("/").pop()?.replace(/\.[^.]+$/, "") ?? "Unknown";
+  return filename.replace(/^class-/, "").replace(/-/g, ".");
+}
+function extract_relative_source(filePath) {
+  if (!filePath)
+    return;
+  const match = filePath.match(/((?:ScriptReference|Manual|Documentation)\/[^/]*\.html?)$/i);
+  if (match)
+    return match[1];
+  return filePath.split("/").pop();
+}
 function get_storage_path(opts) {
   if (opts.storagePath)
     return opts.storagePath;
@@ -2892,7 +3127,7 @@ async function auto_index(storage, projectRoot) {
   }
   return reindexed;
 }
-program2.command("search <query>").description("Search documentation (auto-discovers and indexes on first use)").option("-s, --summarize", "Summarize results (truncate content)").option("-c, --compress", "Compress results (minimal output)").option("-j, --json", "Output as JSON").action(async (query, options) => {
+program2.command("search <query>").description("Search documentation (auto-discovers and indexes on first use)").option("-j, --json", "Output as JSON").action(async (query, options) => {
   const globalOpts = program2.opts();
   const storagePath = get_storage_path(globalOpts);
   const projectRoot = globalOpts.projectRoot || find_project_root() || null;
@@ -2915,27 +3150,26 @@ program2.command("search <query>").description("Search documentation (auto-disco
     keyword_weight: 0.4
   });
   if (options.json) {
-    const output = options.summarize ? { ...results, results: results.results.map((r) => ({ ...r, content: r.content.slice(0, 200) })) } : options.compress ? { ...results, results: results.results.map(({ content, ...r }) => r) } : results;
-    console.log(JSON.stringify(output, null, 2));
+    console.log(JSON.stringify(results, null, 2));
     return;
   }
-  if (options.compress) {
-    for (const result of results.results) {
-      const title = result.metadata?.section || result.metadata?.unity_class || result.metadata?.file_path;
-      console.log(`${title} (${result.score.toFixed(4)})`);
+  const count = results.results.length;
+  console.log(`# Unity Docs: "${query}" (${count} result${count !== 1 ? "s" : ""})
+`);
+  for (const result of results.results) {
+    const meta = result.metadata;
+    const title = meta?.section || meta?.unity_class || extract_doc_title(meta?.file_path);
+    const source = extract_relative_source(meta?.file_path);
+    console.log(`## ${title}
+`);
+    console.log(result.content);
+    if (source) {
+      console.log(`
+*Source: ${source}*`);
     }
-    return;
-  }
-  console.log(`Found ${results.results.length} results in ${results.elapsed_ms}ms`);
-  console.log(`Semantic: ${results.semantic_count}, Keyword: ${results.keyword_count}`);
-  for (let i = 0;i < results.results.length; i++) {
-    const result = results.results[i];
-    const title = result.metadata?.section || result.metadata?.unity_class || result.metadata?.file_path;
-    const content = options.summarize ? result.content.slice(0, 200) + "..." : result.content;
     console.log(`
-[${i + 1}] ${title}`);
-    console.log(content);
-    console.log(`Score: ${result.score.toFixed(4)}`);
+---
+`);
   }
 });
 program2.command("index [path]").description("Index documentation (auto-discovers sources if no path given)").action(async (path) => {