soup-chop 1.0.4 → 1.0.5

package/dist/index.js

@@ -650,6 +650,18 @@ var DOCS_PATH_HINTS = ["/docs", "/documentation", "/guide", "/guides", "/api", "
 var DOC_LABEL_PATTERN = /\b(docs?|documentation|guide|guides|api|reference|learn|tutorial|get started)\b/i;
 var EXCLUDED_DOC_HOST_PATTERN = /(^|\.)(github\.com|npmjs\.com|www\.npmjs\.com|unpkg\.com)$/i;
 var EXCLUDED_FILE_EXTENSION_PATTERN = /\.(?:png|jpe?g|gif|svg|webp|ico|pdf|zip|gz|tgz|woff2?|ttf|eot|mp4|webm|mov|mp3|wav)$/i;
+function discoveryMethodRank(discoveryMethod) {
+  switch (discoveryMethod) {
+    case "explicit":
+      return 0;
+    case "readme_link":
+      return 1;
+    case "homepage":
+      return 2;
+    case "derived_pages":
+      return 3;
+  }
+}
 var turndown = new TurndownService({
   bulletListMarker: "-",
   codeBlockStyle: "fenced",
@@ -706,24 +718,40 @@ function extractMarkdownLinks(markdown) {
   }
   return links;
 }
-function detectDocsUrl(readmeContent) {
-  const best = extractMarkdownLinks(readmeContent).filter((candidate) => {
+function createDocsCandidate(url, discoveryMethod, score) {
+  return {
+    url,
+    discoveryMethod,
+    score
+  };
+}
+function dedupeDocsCandidates(candidates) {
+  const deduped = /* @__PURE__ */ new Map();
+  for (const candidate of candidates) {
+    const existing = deduped.get(candidate.url);
+    if (existing === void 0 || discoveryMethodRank(candidate.discoveryMethod) < discoveryMethodRank(existing.discoveryMethod) || discoveryMethodRank(candidate.discoveryMethod) === discoveryMethodRank(existing.discoveryMethod) && candidate.score > existing.score) {
+      deduped.set(candidate.url, candidate);
+    }
+  }
+  return [...deduped.values()].sort((left, right) => {
+    if (right.score !== left.score) {
+      return right.score - left.score;
+    }
+    if (left.discoveryMethod !== right.discoveryMethod) {
+      return discoveryMethodRank(left.discoveryMethod) - discoveryMethodRank(right.discoveryMethod);
+    }
+    return left.url.localeCompare(right.url);
+  });
+}
+function detectDocsUrls(readmeContent) {
+  return dedupeDocsCandidates(extractMarkdownLinks(readmeContent).filter((candidate) => {
     try {
       const url = new URL(candidate.url);
       return url.protocol === "http:" || url.protocol === "https:";
     } catch {
       return false;
     }
-  }).map((candidate) => ({ candidate, score: scoreDocsCandidate(candidate) })).filter((entry) => entry.score > 0).sort((left, right) => {
-    if (right.score !== left.score) {
-      return right.score - left.score;
-    }
-    return left.candidate.url.localeCompare(right.candidate.url);
-  })[0];
-  if (best === void 0) {
-    return null;
-  }
-  return normalizeUrl(new URL(best.candidate.url));
+  }).map((candidate) => ({ candidate, score: scoreDocsCandidate(candidate) })).filter((entry) => entry.score > 0).map((entry) => createDocsCandidate(normalizeUrl(new URL(entry.candidate.url)), "readme_link", entry.score)));
 }
 function readRepositoryUrl(repository) {
   if (typeof repository === "string" && repository.trim().length > 0) {
@@ -780,6 +808,55 @@ function detectPagesUrl(packageJson) {
   }
   return null;
 }
+function scoreHomepageCandidate(homepage) {
+  const host = homepage.hostname.toLowerCase();
+  const path = normalizePathname(homepage.pathname).toLowerCase();
+  let score = homepage.protocol === "https:" ? 1 : 0;
+  if (DOC_LABEL_PATTERN.test(path)) {
+    score += 6;
+  }
+  if (DOCS_PATH_HINTS.some((hint) => path === hint || path.startsWith(`${hint}/`))) {
+    score += 3;
+  }
+  if (EXCLUDED_DOC_HOST_PATTERN.test(host)) {
+    score -= 10;
+  }
+  return score;
+}
+function collectDocsCandidates(packageJson, readmeContent, explicitDocsUrl) {
+  const candidates = [];
+  if (explicitDocsUrl !== void 0) {
+    try {
+      candidates.push(createDocsCandidate(normalizeUrl(new URL(explicitDocsUrl)), "explicit", 100));
+    } catch {
+    }
+  }
+  candidates.push(...detectDocsUrls(readmeContent));
+  if (typeof packageJson.homepage === "string" && packageJson.homepage.trim().length > 0) {
+    try {
+      const homepage = new URL(packageJson.homepage.trim());
+      if (!/\.github\.io$/i.test(homepage.hostname) && !/\.gitlab\.io$/i.test(homepage.hostname)) {
+        const score = scoreHomepageCandidate(homepage);
+        if (score > 0) {
+          candidates.push(createDocsCandidate(normalizeUrl(homepage), "homepage", score));
+        }
+      }
+    } catch {
+    }
+  }
+  const pagesUrl = detectPagesUrl(packageJson);
+  if (pagesUrl !== null) {
+    candidates.push(createDocsCandidate(pagesUrl, "derived_pages", 1));
+  }
+  return dedupeDocsCandidates(candidates);
+}
+function isLikelySpaShell(html) {
+  const markdown = convertHtmlToMarkdown(html);
+  if (markdown.replace(/\s+/g, " ").trim().length >= 200) {
+    return false;
+  }
+  return /<div\b[^>]+id=["'](?:app|root|__next)["']/i.test(html);
+}
 function extractPrimaryHtml(html) {
   const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
   if (mainMatch?.[1]) {
@@ -877,9 +954,10 @@ async function fetchHtmlPage(fetchImpl, url) {
   if (!/text\/html|application\/xhtml\+xml/i.test(contentType)) {
     return null;
   }
+  const responseUrl = response.url.length > 0 ? response.url : url;
   return {
     html: await response.text(),
-    pageUrl: new URL(url)
+    pageUrl: new URL(responseUrl)
   };
 }
 function createWebsiteSource(html, pageUrl) {
@@ -954,6 +1032,46 @@ async function discoverStructuredRouteUrls(fetchImpl, initialUrl, initialHtml) {
   }
   return [];
 }
+async function probeDocsCandidate(candidate, options = {}) {
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const page = await fetchHtmlPage(fetchImpl, candidate.url);
+  if (page === null) {
+    return null;
+  }
+  const structuredRouteUrls = await discoverStructuredRouteUrls(fetchImpl, page.pageUrl, page.html);
+  if (isLikelySpaShell(page.html) && structuredRouteUrls.length === 0) {
+    return null;
+  }
+  const links = extractHtmlLinks(page.html, page.pageUrl);
+  const docsPrefix = deriveDocsPrefix(page.pageUrl, links);
+  const docsScopedLinks = links.filter((link) => matchesDocsScope(link, page.pageUrl, docsPrefix));
+  const markdown = convertHtmlToMarkdown(page.html).replace(/\s+/g, " ").trim();
+  let score = candidate.score;
+  if (structuredRouteUrls.length > 0) {
+    score += 6;
+  }
+  if (docsScopedLinks.length > 0) {
+    score += 4;
+  }
+  if (markdown.length >= 200) {
+    score += 2;
+  }
+  return {
+    ...candidate,
+    url: normalizeUrl(page.pageUrl),
+    score
+  };
+}
+async function probeDocsCandidates(candidates, options = {}) {
+  const probed = await Promise.all(candidates.map(async (candidate) => {
+    try {
+      return await probeDocsCandidate(candidate, options);
+    } catch {
+      return null;
+    }
+  }));
+  return dedupeDocsCandidates(probed.filter((candidate) => candidate !== null));
+}
 async function crawlDocsSite(entryUrl, options = {}) {
   const fetchImpl = options.fetchImpl ?? fetch;
   const maxPages = options.maxPages ?? DEFAULT_MAX_WEBSITE_PAGES;
@@ -1008,7 +1126,6 @@ async function crawlDocsSite(entryUrl, options = {}) {
       continue;
     }
     visited.add(current);
-    const currentUrl = new URL(current);
     const page = await fetchHtmlPage(fetchImpl, current);
     if (page === null) {
       if (sources.length === 0) {
@@ -1017,7 +1134,7 @@ async function crawlDocsSite(entryUrl, options = {}) {
       continue;
     }
     const html = page.html;
-    const links = extractHtmlLinks(html, currentUrl);
+    const links = extractHtmlLinks(html, page.pageUrl);
     docsPrefix ??= deriveDocsPrefix(initialUrl, links);
     for (const link of links) {
       const normalized = normalizeUrl(link);
@@ -1031,10 +1148,10 @@ async function crawlDocsSite(entryUrl, options = {}) {
         queue.push(normalized);
       }
     }
-    if (!matchesDocsScope(currentUrl, initialUrl, docsPrefix)) {
+    if (!matchesDocsScope(page.pageUrl, initialUrl, docsPrefix)) {
       continue;
     }
-    const source = createWebsiteSource(html, currentUrl);
+    const source = createWebsiteSource(html, page.pageUrl);
     if (source === null) {
       continue;
     }
@@ -1555,18 +1672,16 @@ function extractRepositoryUrl(repository) {
 
 // src/lib/discoverSources.ts
 var UNPKG_BASE2 = "https://unpkg.com";
-var SOURCE_MANIFEST_VERSION = 3;
+var SOURCE_MANIFEST_VERSION = 4;
 var TOP_LEVEL_DOC_ALLOWLIST = /* @__PURE__ */ new Set(["API.MD", "FAQ.MD", "MIGRATING.MD", "UPGRADING.MD", "CHANGELOG.MD", "CONTRIBUTING.MD"]);
 var LOCAL_TS_DIR_EXCLUDES = /* @__PURE__ */ new Set([".git", ".windsurf", "coverage", "dist", "dist-test", "node_modules", "sandbox"]);
+var MAX_WEBSITE_CANDIDATES = 2;
 function normalizeOrigin(origin) {
   return origin.replace(/^\//, "");
 }
 function canonicalTopLevelDocName(path) {
   return path.replace(/\.md$/i, ".md").toUpperCase();
 }
-function isSearchSource(source) {
-  return source !== null;
-}
 function cacheFileForOrigin(origin) {
   const normalized = normalizeOrigin(origin);
   if (normalized === "README.md") {
@@ -1577,39 +1692,84 @@ function cacheFileForOrigin(origin) {
 function createSourceId(origin) {
   return normalizeOrigin(origin).replace(/[^a-zA-Z0-9]+/g, "__").replace(/^__+|__+$/g, "").toLowerCase();
 }
-function createSource(sourceKind, origin, content) {
+function createSource(sourceKind, origin, content, discoveryMethod) {
   const normalized = normalizeOrigin(origin);
   return {
     sourceId: createSourceId(normalized),
     sourceKind,
     origin: normalized,
     title: posix2.basename(normalized),
-    content
+    content,
+    discoveryMethod
   };
 }
 function mapSourceKind(source, sourceKind, originPrefix) {
   const origin = originPrefix === void 0 ? source.origin : `${originPrefix}/${source.origin}`;
-  return createSourceWithTitle(sourceKind, origin, source.title, source.content);
+  return createSourceWithTitle(sourceKind, origin, source.title, source.content, source.discoveryMethod);
 }
-function createSourceWithTitle(sourceKind, origin, title, content) {
+function createSourceWithTitle(sourceKind, origin, title, content, discoveryMethod) {
   const normalized = normalizeOrigin(origin);
   return {
     sourceId: createSourceId(normalized),
     sourceKind,
     origin: normalized,
     title,
-    content
+    content,
+    discoveryMethod
   };
 }
 function countLines(content) {
   return content.length === 0 ? 0 : content.split("\n").length;
 }
 async function discoverWebsiteSourcesFromTarget(target, readme, canCache, options = {}) {
-  const docsUrl = await detectResolvedDocsUrl(target, readme, canCache, options.docsUrl);
-  if (docsUrl === null) {
+  const packageJson = await readPackageJsonFromTarget(target, canCache);
+  const candidates = await probeDocsCandidates(collectDocsCandidates(packageJson, readme, options.docsUrl));
+  if (candidates.length === 0) {
     return [];
   }
-  return crawlDocsSite(docsUrl);
+  return discoverWebsiteSourcesFromCandidates(candidates);
+}
+async function discoverWebsiteSourcesFromCandidates(candidates) {
+  const crawled = await Promise.all(candidates.slice(0, MAX_WEBSITE_CANDIDATES).map(async (candidate) => {
+    try {
+      const sources = await crawlDocsSite(candidate.url);
+      return sources.map((source) => ({ ...source, discoveryMethod: candidate.discoveryMethod }));
+    } catch {
+      return [];
+    }
+  }));
+  return dedupeWebsiteSources(crawled.flat());
+}
+function dedupeWebsiteSources(sources) {
+  const deduped = /* @__PURE__ */ new Map();
+  for (const source of sources) {
+    const key = source.content.replace(/\s+/g, " ").trim();
+    if (key.length === 0) {
+      continue;
+    }
+    const existing = deduped.get(key);
+    if (existing === void 0 || compareDiscoveryMethod(existing.discoveryMethod, source.discoveryMethod) > 0) {
+      deduped.set(key, source);
+    }
+  }
+  return [...deduped.values()].sort((left, right) => left.origin.localeCompare(right.origin));
+}
+function compareDiscoveryMethod(left, right) {
+  const rank = (value) => {
+    switch (value) {
+      case "explicit":
+        return 0;
+      case "readme_link":
+        return 1;
+      case "homepage":
+        return 2;
+      case "derived_pages":
+        return 3;
+      default:
+        return 4;
+    }
+  };
+  return rank(left) - rank(right);
 }
 async function discoverWikiSourcesFromTarget(target, canCache) {
   const packageJson = await readPackageJsonFromTarget(target, canCache);
@@ -1696,17 +1856,6 @@ async function readPackageJsonFromTarget(target, canCache) {
   const parsed = JSON.parse(raw);
   return isPackageJsonLike(parsed) ? parsed : {};
 }
-async function detectResolvedDocsUrl(target, readme, canCache, explicitDocsUrl) {
-  if (explicitDocsUrl !== void 0) {
-    return explicitDocsUrl;
-  }
-  const packageJson = await readPackageJsonFromTarget(target, canCache);
-  const pagesUrl = detectPagesUrl(packageJson);
-  if (pagesUrl !== null) {
-    return pagesUrl;
-  }
-  return detectDocsUrl(readme);
-}
 async function discoverJsDocSourcesFromTarget(target, packageJson, canCache) {
   const fetcher = await discoverSourceCodeFetcher(target, packageJson, { canCache });
   if (fetcher === null) {
@@ -1740,7 +1889,8 @@ function buildSourceCatalogEntries(sources) {
     sourceKind: source.sourceKind,
     origin: source.origin,
     title: source.title,
-    lineCount: countLines(source.content)
+    lineCount: countLines(source.content),
+    discoveryMethod: source.discoveryMethod
   }));
 }
 async function collectWorkspacePackages(workspaceRoot) {
@@ -1847,11 +1997,19 @@ async function readManifestSources(pkg, version) {
         sourceKind: entry.sourceKind,
         origin: entry.origin,
         title: entry.title,
-        content
+        content,
+        discoveryMethod: entry.discoveryMethod
       };
     })
   );
-  return sources.every(isSearchSource) ? sources : null;
+  const resolvedSources = [];
+  for (const source of sources) {
+    if (source === null) {
+      return null;
+    }
+    resolvedSources.push(source);
+  }
+  return resolvedSources;
 }
 async function writeManifestSources(pkg, version, sources) {
   await Promise.all(
@@ -1864,7 +2022,8 @@ async function writeManifestSources(pkg, version, sources) {
       sourceKind: source.sourceKind,
       origin: source.origin,
       title: source.title,
-      cacheFile: cacheFileForOrigin(source.origin)
+      cacheFile: cacheFileForOrigin(source.origin),
+      discoveryMethod: source.discoveryMethod
     }))
   };
   await writeCache(pkg, version, "sources-manifest.json", JSON.stringify(manifest, null, 2));
@@ -2465,7 +2624,8 @@ function extractMarkdownChunks(source) {
       path: source.origin,
       startLine: 1,
       endLine: totalLines,
-      content
+      content,
+      discoveryMethod: source.discoveryMethod
     }];
   }
   return entries.map((entry, index) => ({
@@ -2478,7 +2638,8 @@ function extractMarkdownChunks(source) {
     path: entry.path,
     startLine: entry.startLine,
     endLine: entry.endLine,
-    content: sliceLines(source.content, entry)
+    content: sliceLines(source.content, entry),
+    discoveryMethod: source.discoveryMethod
   }));
 }
 
@@ -2782,7 +2943,7 @@ function stemPorter(word) {
 // src/lib/searchDocs.ts
 var STOPWORDS = /* @__PURE__ */ new Set(["a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "how", "in", "is", "it", "of", "on", "or", "that", "the", "to", "with"]);
 var FIELD_WEIGHTS = { title: 5, path: 3, body: 1 };
-var SEARCH_INDEX_VERSION = 4;
+var SEARCH_INDEX_VERSION = 5;
 function searchIndexCacheFile() {
   return "search-index.json";
 }
@@ -2927,6 +3088,20 @@ function buildExampleMetadata(chunk) {
     topicId
   };
 }
+function discoveryMethodScore(discoveryMethod) {
+  switch (discoveryMethod) {
+    case "explicit":
+      return 0.3;
+    case "readme_link":
+      return 0.2;
+    case "homepage":
+      return 0.1;
+    case "derived_pages":
+      return 0;
+    default:
+      return 0;
+  }
+}
 function rankIndexEntries(index, query) {
   const queryTokens = [...new Set(tokenize(query))];
   if (queryTokens.length === 0) {
@@ -2960,6 +3135,7 @@ function rankIndexEntries(index, query) {
         score += 1.5;
       }
     }
+    score += discoveryMethodScore(entry.chunk.discoveryMethod);
     return {
       chunk: entry.chunk,
       indexPosition,
@@ -3866,7 +4042,7 @@ async function buildCompareVersionsResponse(pkg, v_old, v_new) {
 function createServer() {
   const server = new McpServer({
     name: "soup-chop",
-    version: "1.0.4"
+    version: "1.0.5"
   });
   server.registerResource(
     "capabilities",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "soup-chop",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "A JIT Documentation Proxy for the Model Context Protocol",
   "repository": {
     "type": "git",