smart-web-mcp 0.44.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/CHANGELOG.md +342 -0
  2. package/README.md +33 -568
  3. package/dist/assessment.d.ts +10 -3
  4. package/dist/assessment.js +21 -6
  5. package/dist/assessment.js.map +1 -1
  6. package/dist/body-export.d.ts +10 -0
  7. package/dist/body-export.js +28 -0
  8. package/dist/body-export.js.map +1 -0
  9. package/dist/browser-session.d.ts +32 -1
  10. package/dist/browser-session.js +373 -3
  11. package/dist/browser-session.js.map +1 -1
  12. package/dist/cli-fetch-options.d.ts +7 -0
  13. package/dist/cli-fetch-options.js +41 -0
  14. package/dist/cli-fetch-options.js.map +1 -0
  15. package/dist/cli.js +28 -11
  16. package/dist/cli.js.map +1 -1
  17. package/dist/comment-export.d.ts +10 -0
  18. package/dist/comment-export.js +39 -0
  19. package/dist/comment-export.js.map +1 -0
  20. package/dist/composition.d.ts +1 -22
  21. package/dist/composition.js +0 -16
  22. package/dist/composition.js.map +1 -1
  23. package/dist/extraction/budget.d.ts +3 -0
  24. package/dist/extraction/budget.js +88 -7
  25. package/dist/extraction/budget.js.map +1 -1
  26. package/dist/extraction/document.js +134 -9
  27. package/dist/extraction/document.js.map +1 -1
  28. package/dist/korea/coupang-filters.d.ts +5 -0
  29. package/dist/korea/coupang-filters.js +87 -0
  30. package/dist/korea/coupang-filters.js.map +1 -0
  31. package/dist/korea/coupang.d.ts +27 -6
  32. package/dist/korea/coupang.js +202 -82
  33. package/dist/korea/coupang.js.map +1 -1
  34. package/dist/korea/proxy-client.d.ts +1 -0
  35. package/dist/korea/proxy-client.js +6 -2
  36. package/dist/korea/proxy-client.js.map +1 -1
  37. package/dist/korea/smartvertical.d.ts +1 -0
  38. package/dist/korea/smartvertical.js +39 -25
  39. package/dist/korea/smartvertical.js.map +1 -1
  40. package/dist/lib.d.ts +4 -2
  41. package/dist/lib.js +2 -1
  42. package/dist/lib.js.map +1 -1
  43. package/dist/mcp-server.js +30 -15
  44. package/dist/mcp-server.js.map +1 -1
  45. package/dist/operator-browser-fixture.d.ts +7 -0
  46. package/dist/operator-browser-fixture.js +19 -0
  47. package/dist/operator-browser-fixture.js.map +1 -0
  48. package/dist/operator-canary-cli.d.ts +6 -0
  49. package/dist/operator-canary-cli.js +190 -0
  50. package/dist/operator-canary-cli.js.map +1 -0
  51. package/dist/operator-canary.d.ts +74 -0
  52. package/dist/operator-canary.js +135 -0
  53. package/dist/operator-canary.js.map +1 -0
  54. package/dist/operator-cdp.d.ts +15 -0
  55. package/dist/operator-cdp.js +195 -0
  56. package/dist/operator-cdp.js.map +1 -0
  57. package/dist/operator-frame.d.ts +3 -0
  58. package/dist/operator-frame.js +55 -0
  59. package/dist/operator-frame.js.map +1 -0
  60. package/dist/settings.d.ts +3 -0
  61. package/dist/settings.js +1 -0
  62. package/dist/settings.js.map +1 -1
  63. package/dist/shared.d.ts +33 -8
  64. package/dist/shared.js +15 -9
  65. package/dist/shared.js.map +1 -1
  66. package/dist/smartbrowser.d.ts +169 -0
  67. package/dist/smartbrowser.js +288 -0
  68. package/dist/smartbrowser.js.map +1 -0
  69. package/dist/smartcrawl.js +16 -21
  70. package/dist/smartcrawl.js.map +1 -1
  71. package/dist/smartfetch/academic-fallback.js +3 -5
  72. package/dist/smartfetch/academic-fallback.js.map +1 -1
  73. package/dist/smartfetch/archive-fallback.js +2 -5
  74. package/dist/smartfetch/archive-fallback.js.map +1 -1
  75. package/dist/smartfetch/assets.d.ts +1 -1
  76. package/dist/smartfetch/assets.js +28 -1
  77. package/dist/smartfetch/assets.js.map +1 -1
  78. package/dist/smartfetch/jina-reader.d.ts +1 -0
  79. package/dist/smartfetch/jina-reader.js +17 -3
  80. package/dist/smartfetch/jina-reader.js.map +1 -1
  81. package/dist/smartfetch/pipeline.js +15 -25
  82. package/dist/smartfetch/pipeline.js.map +1 -1
  83. package/dist/smartfetch/provider-policy.d.ts +3 -0
  84. package/dist/smartfetch/provider-policy.js +45 -2
  85. package/dist/smartfetch/provider-policy.js.map +1 -1
  86. package/dist/smartfetch/provider-types.d.ts +8 -0
  87. package/dist/smartfetch/providers/article.js +76 -10
  88. package/dist/smartfetch/providers/article.js.map +1 -1
  89. package/dist/smartfetch/providers/blind-post.d.ts +2 -0
  90. package/dist/smartfetch/providers/blind-post.js +167 -0
  91. package/dist/smartfetch/providers/blind-post.js.map +1 -0
  92. package/dist/smartfetch/providers/clien.d.ts +2 -0
  93. package/dist/smartfetch/providers/clien.js +157 -0
  94. package/dist/smartfetch/providers/clien.js.map +1 -0
  95. package/dist/smartfetch/providers/commerce.d.ts +15 -0
  96. package/dist/smartfetch/providers/commerce.js +123 -7
  97. package/dist/smartfetch/providers/commerce.js.map +1 -1
  98. package/dist/smartfetch/providers/dcinside.js +313 -20
  99. package/dist/smartfetch/providers/dcinside.js.map +1 -1
  100. package/dist/smartfetch/providers/hackernews-html.d.ts +21 -0
  101. package/dist/smartfetch/providers/hackernews-html.js +148 -0
  102. package/dist/smartfetch/providers/hackernews-html.js.map +1 -0
  103. package/dist/smartfetch/providers/hackernews.js +76 -14
  104. package/dist/smartfetch/providers/hackernews.js.map +1 -1
  105. package/dist/smartfetch/providers/index.js +7 -1
  106. package/dist/smartfetch/providers/index.js.map +1 -1
  107. package/dist/smartfetch/providers/linkedin.js +2 -1
  108. package/dist/smartfetch/providers/linkedin.js.map +1 -1
  109. package/dist/smartfetch/providers/naver-blog.d.ts +2 -0
  110. package/dist/smartfetch/providers/naver-blog.js +87 -45
  111. package/dist/smartfetch/providers/naver-blog.js.map +1 -1
  112. package/dist/smartfetch/providers/naver-cafe-native.d.ts +7 -0
  113. package/dist/smartfetch/providers/naver-cafe-native.js +188 -0
  114. package/dist/smartfetch/providers/naver-cafe-native.js.map +1 -0
  115. package/dist/smartfetch/providers/naver-cafe.js +165 -4
  116. package/dist/smartfetch/providers/naver-cafe.js.map +1 -1
  117. package/dist/smartfetch/providers/naver-map-search.js +1 -1
  118. package/dist/smartfetch/providers/naver-map-search.js.map +1 -1
  119. package/dist/smartfetch/providers/ppomppu.d.ts +2 -0
  120. package/dist/smartfetch/providers/ppomppu.js +289 -0
  121. package/dist/smartfetch/providers/ppomppu.js.map +1 -0
  122. package/dist/smartfetch/providers/reddit.js +24 -3
  123. package/dist/smartfetch/providers/reddit.js.map +1 -1
  124. package/dist/smartfetch/providers/threads.js +1 -1
  125. package/dist/smartfetch/providers/threads.js.map +1 -1
  126. package/dist/smartfetch/providers/wikipedia.js +138 -0
  127. package/dist/smartfetch/providers/wikipedia.js.map +1 -1
  128. package/dist/smartfetch.d.ts +7 -1
  129. package/dist/smartfetch.js +353 -76
  130. package/dist/smartfetch.js.map +1 -1
  131. package/dist/smartsearch-brave.d.ts +2 -0
  132. package/dist/smartsearch-brave.js +43 -0
  133. package/dist/smartsearch-brave.js.map +1 -0
  134. package/dist/smartsearch-browser.d.ts +5 -0
  135. package/dist/smartsearch-browser.js +116 -0
  136. package/dist/smartsearch-browser.js.map +1 -0
  137. package/dist/smartsearch-ddg.d.ts +2 -0
  138. package/dist/smartsearch-ddg.js +46 -0
  139. package/dist/smartsearch-ddg.js.map +1 -0
  140. package/dist/smartsearch-node.d.ts +6 -0
  141. package/dist/smartsearch-node.js +21 -0
  142. package/dist/smartsearch-node.js.map +1 -0
  143. package/dist/smartsearch-postgresql.d.ts +7 -0
  144. package/dist/smartsearch-postgresql.js +44 -0
  145. package/dist/smartsearch-postgresql.js.map +1 -0
  146. package/dist/smartsearch-python.d.ts +8 -0
  147. package/dist/smartsearch-python.js +87 -0
  148. package/dist/smartsearch-python.js.map +1 -0
  149. package/dist/smartsearch-quality.d.ts +6 -0
  150. package/dist/smartsearch-quality.js +47 -0
  151. package/dist/smartsearch-quality.js.map +1 -0
  152. package/dist/smartsearch-youtube.d.ts +7 -0
  153. package/dist/smartsearch-youtube.js +76 -0
  154. package/dist/smartsearch-youtube.js.map +1 -0
  155. package/dist/smartsearch.d.ts +2 -0
  156. package/dist/smartsearch.js +506 -84
  157. package/dist/smartsearch.js.map +1 -1
  158. package/package.json +8 -5
  159. package/python/requirements-undetected-chromedriver.txt +2 -2
@@ -1,9 +1,19 @@
1
- import { configEnabled, fetchText, resolveDuckDuckGoUrl, smartWebProfile, stripTags, uniqueResults, validateOutboundUrl } from "./shared.js";
1
+ import { configEnabled, fetchText, smartWebProfile, stripTags, uniqueResults, validateOutboundUrl } from "./shared.js";
2
2
  import { createAssessment, createHandoffSuggestion, detectInteractiveLikeText, firstMeaningfulBlockedReason } from "./assessment.js";
3
3
  import { configText } from "./settings.js";
4
4
  import { runSmartcrawl } from "./smartcrawl.js";
5
5
  import { budgetText, projectRetrievalAssessment } from "./extraction/budget.js";
6
6
  import { parseFeed } from "./smartfetch/generic-fallbacks.js";
7
+ import { runBrowserSession } from "./browser-session.js";
8
+ import { browserSearchMatchesQuery, parseNaverBrowserSearch, searchResultMatchesSite } from "./smartsearch-browser.js";
9
+ import { parseBraveSearch } from "./smartsearch-brave.js";
10
+ import { parseHTML } from "linkedom";
11
+ import { parseDuckDuckGoSearch } from "./smartsearch-ddg.js";
12
+ import { parsePythonDocsSearch, pythonDocsSearchUrl, unscopedPythonDocsTerms } from "./smartsearch-python.js";
13
+ import { postgresqlDocsBodyHasSymbol, postgresqlDocsCandidates, postgresqlDocsIntent } from "./smartsearch-postgresql.js";
14
+ import { nodeDocsIntent, nodeDocsOriginal } from "./smartsearch-node.js";
15
+ import { parseYoutubeSearch, youtubeSearchUrl } from "./smartsearch-youtube.js";
16
+ import { discoveryEvidenceScore, hasDiscoveryYield, isSearchTrackingRelay, koreanDiscoveryTerms, rankDiscoveryResults } from "./smartsearch-quality.js";
7
17
  const DEFAULT_TIMEOUT_MS = 12000;
8
18
  const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
9
19
  const SMARTSEARCH_BUDGET_PRESETS = {
@@ -94,18 +104,31 @@ export function projectSmartsearchOutput(response, preset = "compact") {
94
104
  metadata[path] = projected.metadata;
95
105
  return projected.value;
96
106
  };
97
- const results = budgetList(response.results, policy.maxResults);
98
- recordListMetadata(metadata, "results", results.metadata);
107
+ // A clipped URL is not an original the caller can open; omit it rather than
108
+ // returning a plausible-looking but invalid link, and fill from later rows.
109
+ const usableResults = response.results.filter((item) => item.url.length <= policy.maxUrlChars);
110
+ const omittedUrls = response.results.length - usableResults.length;
111
+ const results = budgetList(usableResults, policy.maxResults);
112
+ if (omittedUrls > 0 || results.metadata.truncated) {
113
+ metadata.results = {
114
+ truncated: true,
115
+ original_items: response.results.length,
116
+ returned_items: results.items.length,
117
+ omitted_items: response.results.length - results.items.length,
118
+ };
119
+ }
99
120
  const projectedResults = results.items.map((item, index) => ({
100
121
  ...item,
101
122
  title: projectText(item.title, policy.maxTitleChars, `results.${index}.title`),
102
- url: projectText(item.url, policy.maxUrlChars, `results.${index}.url`),
123
+ url: item.url,
103
124
  snippet: projectText(item.snippet || "", policy.maxSnippetChars, `results.${index}.snippet`),
104
125
  engine: projectText(item.engine, policy.maxEngineChars, `results.${index}.engine`),
105
126
  }));
106
127
  const notes = budgetList(response.notes, policy.maxNotes);
107
128
  recordListMetadata(metadata, "notes", notes.metadata);
108
129
  const projectedNotes = notes.items.map((note, index) => projectText(note, policy.maxNoteChars, `notes.${index}`));
130
+ if (omittedUrls > 0)
131
+ projectedNotes.push(`${omittedUrls} original URL(s) omitted because they exceed the ${policy.maxUrlChars}-character output limit.`);
109
132
  const fallbacks = budgetList(response.fallbacks_tried, policy.maxFallbacks);
110
133
  recordListMetadata(metadata, "fallbacks_tried", fallbacks.metadata);
111
134
  const projectedFallbacks = fallbacks.items.map((fallback, index) => projectText(fallback, policy.maxFallbackChars, `fallbacks_tried.${index}`));
@@ -151,6 +174,8 @@ export function projectSmartsearchOutput(response, preset = "compact") {
151
174
  maxUrlChars: policy.maxUrlChars,
152
175
  }, metadata)
153
176
  : undefined;
177
+ if (omittedUrls > 0 && assessment)
178
+ assessment.confidence = "low";
154
179
  const engine = projectText(response.engine, policy.maxEngineChars, "engine");
155
180
  const query = projectText(response.query, policy.maxQueryChars, "query");
156
181
  const truncated = Object.values(metadata).some((item) => item.truncated);
@@ -357,9 +382,13 @@ function buildSearchResponse(response, handoff) {
357
382
  return handoff ? { ...response, handoff, assessment } : { ...response, assessment };
358
383
  }
359
384
  function assessSearchResponse(response, handoff) {
360
- const blockedReason = firstMeaningfulBlockedReason(response.notes);
361
- const publicHtmlEngine = response.engine === "duckduckgo" || response.engine === "brave_html";
362
- const confidence = response.results.length === 0
385
+ // A failed earlier engine remains in notes, but does not block an exact
386
+ // watch-page original recovered from the site's own search.
387
+ const blockedReason = response.engine === "youtube_browser" && response.results.length > 0 ? "" : firstMeaningfulBlockedReason(response.notes);
388
+ const publicHtmlEngine = ["duckduckgo", "brave_html", "naver_browser", "youtube_browser", "mixed"].includes(response.engine);
389
+ const weakDiscovery = (publicHtmlEngine || response.engine === "searxng")
390
+ && !hasDiscoveryYield(response.results, koreanDiscoveryTerms(response.query), response.results.length || 1);
391
+ const confidence = weakDiscovery ? "low" : response.results.length === 0
363
392
  ? (handoff?.performed ? "medium" : "low")
364
393
  : (response.results.length >= 3 && !publicHtmlEngine ? "high" : "medium");
365
394
  const interactiveFlowLikely = Boolean(handoff && !handoff.performed && detectInteractiveLikeText(handoff.reason));
@@ -407,6 +436,15 @@ async function maybeRunCrawlHandoff(response, options, runtime) {
407
436
  }
408
437
  const crawlQuery = stripSiteOperators(options.query);
409
438
  const mode = inferCrawlMode(startUrl, options.crawlMode);
439
+ const siteTokens = searchQueryTokens(options.query).filter(token => /^site:[^"']+$/i.test(token));
440
+ if (siteTokens.length === 1 && extractSiteScope(siteTokens[0])?.host === "news.ycombinator.com" && parseHnDateQuery(options.query).bounded) {
441
+ const reason = "site_crawl not performed: HN date constraints require native publication timestamps that crawl items do not verify";
442
+ const declined = buildSearchResponse({ ...response, notes: [...response.notes, reason] }, {
443
+ tool: "smartcrawl", performed: false, reason, start_url: startUrl, query: options.query, mode,
444
+ items: [], pages_attempted: 0, pages_loaded: 0, errors: [reason],
445
+ });
446
+ return { ...declined, assessment: createAssessment({ ...declined.assessment, recommended_handoff: null }) };
447
+ }
410
448
  const crawl = await (runtime.runSmartcrawl || runSmartcrawl)({
411
449
  url: startUrl,
412
450
  ...(crawlQuery ? { query: crawlQuery } : {}),
@@ -508,9 +546,46 @@ async function searchStackExchange(query, numResults, timeoutMs) {
508
546
  return failedProviderSearch(`StackExchange parse error: ${message}`);
509
547
  }
510
548
  }
511
- async function searchHackerNews(query, numResults, timeoutMs) {
512
- const q = stripSiteOperators(query) || query;
513
- const result = await fetchText(`https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(q)}&tags=story&hitsPerPage=${Math.min(numResults, 20)}`, timeoutMs, {
549
+ function searchQueryTokens(query) {
550
+ return query.match(/"(?:\\.|[^"\\])*"|'[^']*'|[()]|[^\s()]+/g) || [];
551
+ }
552
+ function parseHnDateQuery(query) {
553
+ const tokens = searchQueryTokens(query).filter(token => !/^site:[^"']+$/i.test(token));
554
+ const parsed = { text: "", bounded: false };
555
+ const terms = [];
556
+ for (const token of tokens) {
557
+ const operator = /^(after|before):(.*)$/i.exec(token);
558
+ if (!operator) {
559
+ terms.push(token);
560
+ continue;
561
+ }
562
+ parsed.bounded = true;
563
+ const date = operator[2];
564
+ const timestamp = Date.parse(`${date}T00:00:00Z`);
565
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(timestamp) || new Date(timestamp).toISOString().slice(0, 10) !== date) {
566
+ parsed.error = `Invalid HN date constraint: ${token}; use a real UTC calendar date YYYY-MM-DD`;
567
+ continue;
568
+ }
569
+ const seconds = timestamp / 1000;
570
+ if (operator[1].toLowerCase() === "after")
571
+ parsed.after = Math.max(parsed.after ?? -Infinity, seconds);
572
+ else
573
+ parsed.before = Math.min(parsed.before ?? Infinity, seconds);
574
+ }
575
+ parsed.text = terms.join(" ");
576
+ if (parsed.bounded && terms.some(term => /^(?:AND|OR|NOT)$/.test(term) || /[()]/.test(term.replace(/^(["']).*\1$/, ""))))
577
+ parsed.error = "HN date constraints with Boolean/grouped expressions are unsupported; refusing to change their scope";
578
+ if (parsed.after !== undefined && parsed.before !== undefined && parsed.after >= parsed.before)
579
+ parsed.error = "HN date constraints have an empty or reversed UTC interval";
580
+ return parsed;
581
+ }
582
+ async function searchHackerNews(query, numResults, timeoutMs, dates) {
583
+ const q = dates.bounded ? dates.text : stripSiteOperators(query) || query;
584
+ const params = new URLSearchParams({ query: q, tags: "story", hitsPerPage: String(Math.min(numResults, 20)) });
585
+ const filters = [dates.after === undefined ? "" : `created_at_i>=${dates.after}`, dates.before === undefined ? "" : `created_at_i<${dates.before}`].filter(Boolean);
586
+ if (filters.length)
587
+ params.set("numericFilters", filters.join(","));
588
+ const result = await fetchText(`https://hn.algolia.com/api/v1/search?${params}`, timeoutMs, {
514
589
  headers: {
515
590
  accept: "application/json,text/plain,*/*",
516
591
  "user-agent": DEFAULT_USER_AGENT,
@@ -520,8 +595,15 @@ async function searchHackerNews(query, numResults, timeoutMs) {
520
595
  return failedProviderSearch(result.error || `Hacker News error ${result.status}`);
521
596
  try {
522
597
  const data = result.text ? JSON.parse(result.text) : {};
523
- const raw = Array.isArray(data?.hits) ? data.hits : [];
524
- const results = uniqueResults(raw.map((item) => {
598
+ if (!Array.isArray(data?.hits))
599
+ return failedProviderSearch("Hacker News returned an invalid hits payload");
600
+ const raw = data.hits;
601
+ const valid = dates.bounded ? raw.filter((item) => item && typeof item.objectID === "string" && /^\d+$/.test(item.objectID)
602
+ && typeof item.title === "string" && item.title.trim() && Number.isSafeInteger(item.created_at_i) && Math.abs(item.created_at_i) <= 8640000000000
603
+ && (dates.after === undefined || item.created_at_i >= dates.after) && (dates.before === undefined || item.created_at_i < dates.before)) : raw;
604
+ if (dates.bounded && raw.length && !valid.length)
605
+ return failedProviderSearch("Hacker News returned no verifiable stories satisfying the requested date constraints");
606
+ const results = uniqueResults(valid.map((item) => {
525
607
  const objectId = String(item?.objectID || "").trim();
526
608
  let linkedHost = "";
527
609
  try {
@@ -534,15 +616,16 @@ async function searchHackerNews(query, numResults, timeoutMs) {
534
616
  title: String(item?.title || item?.story_title || `HN #${objectId}`).trim(),
535
617
  url: objectId ? `https://news.ycombinator.com/item?id=${objectId}` : "",
536
618
  snippet: [
619
+ dates.bounded ? new Date(item.created_at_i * 1000).toISOString() : "",
537
620
  item?.points ? `▲ ${item.points}` : "",
538
621
  item?.author ? `by ${item.author}` : "",
539
- item?.created_at ? String(item.created_at).slice(0, 10) : "",
622
+ !dates.bounded && item?.created_at ? String(item.created_at).slice(0, 10) : "",
540
623
  linkedHost ? `→ ${linkedHost}` : "",
541
624
  ].filter(Boolean).join(" · "),
542
625
  engine: "hackernews",
543
626
  };
544
627
  }).filter((item) => item.url)).slice(0, numResults);
545
- return completedProviderSearch(results, results.length > 0 ? "" : "Hacker News search returned no results");
628
+ return { ...completedProviderSearch(results, results.length > 0 ? "" : "Hacker News search returned no results"), hnDateBoundsVerified: dates.bounded };
546
629
  }
547
630
  catch (error) {
548
631
  const message = error instanceof Error ? error.message : String(error);
@@ -825,6 +908,9 @@ async function searchGitHubRepoDocs(query, numResults, timeoutMs) {
825
908
  }
826
909
  async function searchMdn(query, numResults, timeoutMs) {
827
910
  const q = stripSiteOperators(query) || query;
911
+ const quoted = searchQueryTokens(query)
912
+ .filter(token => token.startsWith('"') && token.endsWith('"') && token.length > 2 && !token.includes("\\"))
913
+ .map(token => token.slice(1, -1).toLocaleLowerCase());
828
914
  const result = await fetchText(`https://developer.mozilla.org/api/v1/search?q=${encodeURIComponent(q)}&locale=en-US`, timeoutMs, {
829
915
  headers: {
830
916
  accept: "application/json,text/plain,*/*",
@@ -835,8 +921,10 @@ async function searchMdn(query, numResults, timeoutMs) {
835
921
  return failedProviderSearch(result.error || `MDN error ${result.status}`);
836
922
  try {
837
923
  const data = result.text ? JSON.parse(result.text) : {};
838
- const raw = Array.isArray(data?.documents) ? data.documents : [];
839
- const results = uniqueResults(raw.map((item) => {
924
+ if (!Array.isArray(data?.documents))
925
+ return failedProviderSearch("MDN returned an invalid documents payload");
926
+ const raw = data.documents;
927
+ const candidates = uniqueResults(raw.map((item) => {
840
928
  const mdnUrl = String(item?.mdn_url || item?.url || "").trim();
841
929
  return {
842
930
  title: String(item?.title || item?.slug || mdnUrl).trim(),
@@ -844,8 +932,14 @@ async function searchMdn(query, numResults, timeoutMs) {
844
932
  snippet: String(item?.summary || item?.excerpt || "").trim(),
845
933
  engine: "mdn",
846
934
  };
847
- }).filter((item) => item.url && item.url !== "https://developer.mozilla.org/")).slice(0, numResults);
848
- return completedProviderSearch(results, results.length > 0 ? "" : "MDN search returned no results");
935
+ }).filter((item) => item.url && item.url !== "https://developer.mozilla.org/"));
936
+ // Native index neighbors do not satisfy an explicit quote by proximity.
937
+ const results = quoted.length ? candidates.filter(item => quoted.every(phrase => item.title.toLocaleLowerCase().includes(phrase) || item.snippet.toLocaleLowerCase().includes(phrase))) : candidates;
938
+ if (quoted.length && candidates.length && !results.length)
939
+ return failedProviderSearch("MDN index has no title/summary evidence for the quoted phrase; continuing to full-query search");
940
+ return completedProviderSearch(results.slice(0, numResults), quoted.length && results.length
941
+ ? "MDN quoted phrase matched index title/summary only; original body remains unverified"
942
+ : results.length ? "" : "MDN search returned no results");
849
943
  }
850
944
  catch (error) {
851
945
  const message = error instanceof Error ? error.message : String(error);
@@ -864,7 +958,9 @@ async function searchCrates(query, numResults, timeoutMs) {
864
958
  return failedProviderSearch(result.error || `crates.io error ${result.status}`);
865
959
  try {
866
960
  const data = result.text ? JSON.parse(result.text) : {};
867
- const raw = Array.isArray(data?.crates) ? data.crates : [];
961
+ if (!Array.isArray(data?.crates))
962
+ return failedProviderSearch("crates.io returned an invalid crates payload");
963
+ const raw = data.crates;
868
964
  const results = uniqueResults(raw.map((item) => {
869
965
  const name = String(item?.name || item?.id || "").trim();
870
966
  const latest = String(item?.max_version || item?.newest_version || "").trim();
@@ -886,7 +982,7 @@ async function searchCrates(query, numResults, timeoutMs) {
886
982
  return failedProviderSearch(`crates.io parse error: ${message}`);
887
983
  }
888
984
  }
889
- async function searchNpmPackage(packageName, timeoutMs) {
985
+ async function searchNpmPackage(packageName, timeoutMs, quoted = []) {
890
986
  const result = await fetchText(`https://registry.npmjs.org/${encodeURIComponent(packageName)}`, timeoutMs, {
891
987
  headers: {
892
988
  accept: "application/json,text/plain,*/*",
@@ -898,20 +994,29 @@ async function searchNpmPackage(packageName, timeoutMs) {
898
994
  try {
899
995
  const data = result.text ? JSON.parse(result.text) : {};
900
996
  const record = data;
901
- const name = String(record.name || packageName).trim();
997
+ // The requested URL is not evidence that a successful response describes
998
+ // that package. An empty or mismatched registry payload must not invent it.
999
+ if (typeof record.name !== "string" || record.name !== packageName) {
1000
+ return failedProviderSearch("npm package metadata did not identify the requested package");
1001
+ }
1002
+ const name = record.name;
1003
+ const description = typeof record.description === "string" ? record.description : "";
1004
+ if (quoted.length && !quoted.every(phrase => name.toLocaleLowerCase().includes(phrase) || description.toLocaleLowerCase().includes(phrase))) {
1005
+ return failedProviderSearch("npm package metadata has no evidence for the quoted phrase; continuing to full-query search");
1006
+ }
902
1007
  const latest = String(record["dist-tags"]?.latest || record.version || "").trim();
903
1008
  const maintainer = Array.isArray(record.maintainers) ? String(record.maintainers[0]?.name || "").trim() : "";
904
1009
  const results = uniqueResults([{
905
1010
  title: name,
906
1011
  url: `https://www.npmjs.com/package/${name}`,
907
1012
  snippet: [
908
- String(record.description || "").trim(),
1013
+ description.trim(),
909
1014
  latest ? `v${latest}` : "",
910
1015
  maintainer ? `by ${maintainer}` : "",
911
1016
  ].filter(Boolean).join(" · "),
912
1017
  engine: "npm",
913
1018
  }]);
914
- return completedProviderSearch(results, "");
1019
+ return completedProviderSearch(results, quoted.length ? "npm quoted phrase matched package metadata only; original body remains unverified" : "");
915
1020
  }
916
1021
  catch (error) {
917
1022
  const message = error instanceof Error ? error.message : String(error);
@@ -930,7 +1035,9 @@ async function searchNpm(query, numResults, timeoutMs) {
930
1035
  return failedProviderSearch(result.error || `npm error ${result.status}`);
931
1036
  try {
932
1037
  const data = result.text ? JSON.parse(result.text) : {};
933
- const raw = Array.isArray(data?.objects) ? data.objects : [];
1038
+ if (!Array.isArray(data?.objects))
1039
+ return failedProviderSearch("npm search returned an invalid objects payload");
1040
+ const raw = data.objects;
934
1041
  const results = uniqueResults(raw.map((item) => {
935
1042
  const pkg = item?.package || {};
936
1043
  const name = String(pkg?.name || "").trim();
@@ -972,7 +1079,9 @@ async function searchWikipedia(query, numResults, timeoutMs) {
972
1079
  return failedProviderSearch(result.error || `Wikipedia error ${result.status}`);
973
1080
  try {
974
1081
  const data = result.text ? JSON.parse(result.text) : {};
975
- const raw = Array.isArray(data?.query?.search) ? data.query.search : [];
1082
+ if (!Array.isArray(data?.query?.search))
1083
+ return failedProviderSearch("Wikipedia returned an invalid search payload");
1084
+ const raw = data.query.search;
976
1085
  const results = uniqueResults(raw.map((item) => {
977
1086
  const title = String(item?.title || "").trim();
978
1087
  const pageId = item?.pageid;
@@ -984,7 +1093,20 @@ async function searchWikipedia(query, numResults, timeoutMs) {
984
1093
  engine: "wikipedia",
985
1094
  };
986
1095
  }).filter((item) => item.url)).slice(0, numResults);
987
- return completedProviderSearch(results, results.length > 0 ? "" : "Wikipedia search returned no results");
1096
+ // Native API ordering can fill a Korean two-term query with articles whose
1097
+ // excerpt only contains one incidental word (e.g. 기계식 계산기 for a keyboard
1098
+ // query). Do not hide all leads if none has enough visible evidence.
1099
+ const terms = wikiLang === "ko" ? koreanDiscoveryTerms(query) : [];
1100
+ const topical = terms.length >= 2 ? results.filter(row => discoveryEvidenceScore(row, terms) >= 2) : results;
1101
+ const phrase = terms.join("");
1102
+ const folded = (value) => value.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "");
1103
+ const phraseScore = (row) => !phrase ? 0
1104
+ : folded(row.title).includes(phrase) ? 2 : folded(row.snippet).includes(phrase) ? 1 : 0;
1105
+ const selected = topical.length ? topical.sort((a, b) => phraseScore(b) - phraseScore(a)) : results;
1106
+ const note = selected.length < results.length
1107
+ ? `Excluded ${results.length - selected.length} Wikipedia result(s) without both query terms in title/snippet`
1108
+ : selected.length ? "" : "Wikipedia search returned no results";
1109
+ return completedProviderSearch(selected, note);
988
1110
  }
989
1111
  catch (error) {
990
1112
  const message = error instanceof Error ? error.message : String(error);
@@ -1002,19 +1124,7 @@ async function searchDuckDuckGo(query, numResults, timeoutMs) {
1002
1124
  if (/anomaly\.js|sending automated queries|captcha/i.test(result.text)) {
1003
1125
  return failedProviderSearch("DuckDuckGo blocked the automated HTML fallback");
1004
1126
  }
1005
- const blocks = result.text.split(/<div class="result[^"]*"/i).slice(1);
1006
- const results = uniqueResults(blocks.map((block) => {
1007
- const anchor = block.match(/<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
1008
- const snippet = block.match(/<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/i)
1009
- || block.match(/<div[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/div>/i);
1010
- const rawUrl = anchor?.[1] || "";
1011
- return {
1012
- title: stripTags(anchor?.[2] || ""),
1013
- url: resolveDuckDuckGoUrl(rawUrl),
1014
- snippet: stripTags(snippet?.[1] || ""),
1015
- engine: "duckduckgo",
1016
- };
1017
- }).filter((item) => item.url)).slice(0, numResults);
1127
+ const results = parseDuckDuckGoSearch(result.text);
1018
1128
  return completedProviderSearch(results, results.length > 0 ? "" : "DuckDuckGo returned no results");
1019
1129
  }
1020
1130
  async function searchBraveHtml(query, numResults, timeoutMs) {
@@ -1026,19 +1136,14 @@ async function searchBraveHtml(query, numResults, timeoutMs) {
1026
1136
  });
1027
1137
  if (!result.ok)
1028
1138
  return failedProviderSearch(result.error || `Brave HTML error ${result.status}`);
1029
- const results = [];
1030
- const regex = /<a href="(https?:\/\/[^"#]+)"[^>]*>[\s\S]*?<div class="title[^>]*"[^>]*>([\s\S]*?)<\/div>[\s\S]*?<\/a>[\s\S]*?<div class="generic-snippet[\s\S]*?<div class="content[^>]*"[^>]*>([\s\S]*?)<\/div>/gi;
1031
- for (const match of result.text.matchAll(regex)) {
1032
- results.push({
1033
- title: stripTags(match[2] || ""),
1034
- url: String(match[1] || "").trim(),
1035
- snippet: stripTags(match[3] || ""),
1036
- engine: "brave_html",
1037
- });
1038
- if (results.length >= numResults)
1039
- break;
1139
+ const unique = parseBraveSearch(result.text);
1140
+ if (unique.length === 0 && result.text.trim()) {
1141
+ const { document } = parseHTML(result.text);
1142
+ const visible = `${document.title} ${document.body?.textContent || ""}`.slice(0, 10000);
1143
+ if (/verify you are human|captcha|unusual traffic|automated quer(?:y|ies)|비정상적인 접근|자동입력 방지/i.test(visible)) {
1144
+ return failedProviderSearch("Brave HTML stopped at a challenge");
1145
+ }
1040
1146
  }
1041
- const unique = uniqueResults(results);
1042
1147
  return completedProviderSearch(unique, unique.length > 0 ? "" : "Brave HTML returned no results");
1043
1148
  }
1044
1149
  async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
@@ -1052,14 +1157,17 @@ async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
1052
1157
  if (!result.ok)
1053
1158
  return failedProviderSearch(result.error || `SearXNG error ${result.status}`);
1054
1159
  try {
1055
- const data = result.text ? JSON.parse(result.text) : {};
1056
- const raw = Array.isArray(data?.results) ? data.results : [];
1160
+ const data = JSON.parse(result.text);
1161
+ if (!data || typeof data !== "object" || !Array.isArray(data.results)) {
1162
+ return failedProviderSearch("SearXNG invalid result envelope: expected a results array");
1163
+ }
1164
+ const raw = data.results;
1057
1165
  const results = uniqueResults(raw.map((item) => ({
1058
1166
  title: String(item?.title || item?.url || "").trim(),
1059
1167
  url: String(item?.url || "").trim(),
1060
1168
  snippet: String(item?.content || item?.description || item?.text || "").trim(),
1061
1169
  engine: "searxng",
1062
- }))).slice(0, numResults);
1170
+ }))).slice(0, Math.max(100, numResults));
1063
1171
  return completedProviderSearch(results, results.length > 0 ? "" : "SearXNG returned no results");
1064
1172
  }
1065
1173
  catch (error) {
@@ -1068,15 +1176,56 @@ async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
1068
1176
  }
1069
1177
  }
1070
1178
  export async function runSmartsearch(options, runtime = {}) {
1179
+ const startedAt = Date.now();
1071
1180
  const query = options.query.trim();
1072
1181
  const numResults = Math.max(1, Math.min(20, Math.floor(Number(options.numResults || 8))));
1073
1182
  const searchType = options.searchType || "auto";
1074
1183
  const contextMaxCharacters = Math.max(1000, Math.floor(Number(options.contextMaxCharacters || 10000)));
1075
1184
  const timeoutMs = Math.max(1000, Math.floor(Number(options.timeoutMs || DEFAULT_TIMEOUT_MS)));
1185
+ const cdpUrl = configText("fetch.browserCdpUrl");
1186
+ const browserEnabled = Boolean(cdpUrl && /[가-힣a-z]/iu.test(stripSiteOperators(query)) && !localOnlyMode() && !privateProfileMode() && configEnabled("search.enableBrowserFallback", true));
1187
+ const attemptTimeout = () => Math.max(1, Math.floor((timeoutMs - (Date.now() - startedAt)) / (browserEnabled ? 4 : 1)));
1076
1188
  const searxngBaseUrl = (options.searxngBaseUrl || configText("search.searxngBaseUrl") || "").trim();
1077
1189
  const fallbacks = [];
1078
1190
  const notes = [];
1079
1191
  const attempts = [];
1192
+ // A result outside an explicit site/path is not a useful provider success.
1193
+ // Multiple positive site operators are left to the general engine; native
1194
+ // first-site routing cannot preserve their Boolean semantics.
1195
+ const tokens = searchQueryTokens(query);
1196
+ const siteMatches = tokens.filter((token) => /^site:[^"']+$/i.test(token));
1197
+ const scope = siteMatches.length === 1 ? extractSiteScope(siteMatches[0]) : null;
1198
+ const unscopedDocsTerms = !siteMatches.length ? unscopedPythonDocsTerms(query) : undefined;
1199
+ const pythonSearchUrl = browserEnabled ? pythonDocsSearchUrl(unscopedDocsTerms ? { host: "docs.python.org", path: "3" } : scope, unscopedDocsTerms || tokens.filter(token => token !== siteMatches[0]).join(" ")) : undefined;
1200
+ const nativeYoutubeUrl = browserEnabled ? youtubeSearchUrl(scope, tokens.filter(token => token !== siteMatches[0]).join(" ")) : undefined;
1201
+ const hnDates = scope?.host === "news.ycombinator.com" ? parseHnDateQuery(query) : undefined;
1202
+ const nativeCandidateLimit = scope ? 20 : numResults;
1203
+ const constrain = (outcome, limit = numResults) => {
1204
+ if (hnDates?.bounded && outcome.status === "succeeded" && !outcome.hnDateBoundsVerified)
1205
+ return failedProviderSearch("HN date constraints require verified native publication timestamps; undated search snippets were not accepted");
1206
+ if (!outcome.ok)
1207
+ return outcome;
1208
+ const originals = outcome.results.filter((item) => !isSearchTrackingRelay(item));
1209
+ if (originals.length !== outcome.results.length) {
1210
+ notes.push(`Excluded ${outcome.results.length - originals.length} outbound tracking relay(s); discovery requires original pages`);
1211
+ outcome = completedProviderSearch(originals, outcome.note);
1212
+ if (!originals.length)
1213
+ return failedProviderSearch("Search returned tracking relays without original results");
1214
+ }
1215
+ if (!scope)
1216
+ return { ...outcome, results: outcome.results.slice(0, limit) };
1217
+ const scoped = outcome.results.filter((item) => searchResultMatchesSite(item.url, scope.host, scope.path));
1218
+ if (scoped.length !== outcome.results.length)
1219
+ notes.push(`Filtered ${outcome.results.length - scoped.length} result(s) outside site:${scope.host}${scope.path ? `/${scope.path}` : ""}`);
1220
+ const wikiArticleIntent = scope.host === "github.com" && /(?:^|\/)wiki(?:\/|$)/i.test(scope.path)
1221
+ && !/\b(?:history|revisions?|edit)\b/i.test(stripSiteOperators(query));
1222
+ const results = wikiArticleIntent
1223
+ ? scoped.filter((item) => !/\/wiki\/(?:[^?#]+\/)?_(?:history|edit)(?:[/?#]|$)/i.test(new URL(item.url).pathname))
1224
+ : scoped;
1225
+ if (results.length !== scoped.length)
1226
+ notes.push(`Excluded ${scoped.length - results.length} GitHub wiki history/edit page(s); original articles were requested`);
1227
+ return results.length ? completedProviderSearch(results.slice(0, limit), outcome.note) : failedProviderSearch("No results satisfied the explicit site constraint");
1228
+ };
1080
1229
  const recordAttempt = (provider, outcome) => {
1081
1230
  attempts.push({
1082
1231
  provider,
@@ -1085,15 +1234,122 @@ export async function runSmartsearch(options, runtime = {}) {
1085
1234
  ...(outcome.status === "failed" && outcome.note ? { error: outcome.note } : {}),
1086
1235
  });
1087
1236
  };
1237
+ if (hnDates?.error)
1238
+ return buildSearchResponse({ engine: "error", query, results: [], fallbacks_tried: [], notes: [hnDates.error], execution: { status: "failed", attempts: [{ provider: "query_constraints", status: "failed", result_count: 0, error: hnDates.error }] } });
1239
+ const discoveryTerms = koreanDiscoveryTerms(query);
1240
+ let discoveryResults = [];
1241
+ const collectDiscovery = (outcome) => {
1242
+ if (!outcome.ok)
1243
+ return false;
1244
+ discoveryResults = rankDiscoveryResults([...discoveryResults, ...outcome.results], discoveryTerms);
1245
+ if (hasDiscoveryYield(discoveryResults, discoveryTerms, numResults))
1246
+ return true;
1247
+ notes.push("Low discovery yield in title/snippet evidence; continuing within the existing search budget. Unmatched originals are retained, not declared irrelevant.");
1248
+ return false;
1249
+ };
1250
+ const finishDiscovery = () => {
1251
+ const results = discoveryResults.slice(0, numResults);
1252
+ const engines = new Set(results.map((row) => row.engine));
1253
+ if (!hasDiscoveryYield(results, discoveryTerms, numResults))
1254
+ notes.push("Limited topical evidence in discovery metadata; original-page relevance remains unverified");
1255
+ return maybeRunCrawlHandoff({ engine: engines.size > 1 ? "mixed" : results[0]?.engine || "none", query, results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1256
+ };
1257
+ const budgetExhausted = () => {
1258
+ if (Date.now() - startedAt < timeoutMs)
1259
+ return false;
1260
+ notes.push("Search budget exhausted; no further discovery engine was started");
1261
+ return true;
1262
+ };
1088
1263
  try {
1089
- const siteOperator = extractSiteOperator(query);
1264
+ // A full site:path can itself be the requested original. Check its authored
1265
+ // body before relying on search indexes that only expose titles/summaries.
1266
+ // Only quote-only queries are safe here: extra terms/operators need a real
1267
+ // search engine rather than being silently discarded.
1268
+ const remainder = tokens.filter(token => !/^site:[^"']+$/i.test(token));
1269
+ const quoted = remainder.filter(token => /^"[^"\\]+"$/.test(token)).map(token => token.slice(1, -1));
1270
+ if (scope?.path.includes("/") && !siteNativeFallbackBlockedReason("Scoped original") && !parseNpmPackageScope(query)
1271
+ && quoted.length > 0 && quoted.length === remainder.length && !budgetExhausted()) {
1272
+ const url = siteScopeToStartUrl(scope);
1273
+ const validation = await validateOutboundUrl(url);
1274
+ if (validation.ok && !budgetExhausted()) {
1275
+ const original = await fetchText(url, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
1276
+ if (original.ok && !budgetExhausted()) {
1277
+ const document = parseHTML(original.text).document;
1278
+ const article = document.querySelector("main article") || document.querySelector("main") || document.querySelector("article");
1279
+ if (article) {
1280
+ for (const node of article.querySelectorAll("script, style, nav, aside, header, footer, [hidden], [inert], [aria-hidden='true'], [style*='display:none'], [style*='display: none']"))
1281
+ node.remove();
1282
+ const body = (article.textContent || "").replace(/\s+/g, " ").trim();
1283
+ const heading = (document.title || article.querySelector("h1")?.textContent || "").replace(/\s+/g, " ").trim();
1284
+ const lower = body.toLocaleLowerCase();
1285
+ if (body.length >= 40 && heading.length >= 3 && quoted.every(phrase => lower.includes(phrase.toLocaleLowerCase()))
1286
+ && !detectInteractiveLikeText(`${heading} ${body.slice(0, 300)}`)) {
1287
+ const position = lower.indexOf(quoted[0].toLocaleLowerCase());
1288
+ const snippet = body.slice(Math.max(0, position - 90), position + quoted[0].length + 110);
1289
+ const outcome = constrain(completedProviderSearch([{ title: heading, url, snippet, engine: "scoped_original" }], ""));
1290
+ if (outcome.ok) {
1291
+ fallbacks.push("scoped_original");
1292
+ recordAttempt("scoped_original", outcome);
1293
+ return maybeRunCrawlHandoff({ engine: "scoped_original", query, results: outcome.results, fallbacks_tried: fallbacks, notes: ["Quoted phrase verified in the exact scoped original body"], execution: smartsearchExecution(attempts) }, options, runtime);
1294
+ }
1295
+ }
1296
+ }
1297
+ }
1298
+ }
1299
+ }
1300
+ const nodeIntent = !siteMatches.length ? nodeDocsIntent(query) : undefined;
1301
+ if (nodeIntent && !budgetExhausted()) {
1302
+ fallbacks.push("node_docs");
1303
+ const original = await fetchText(nodeIntent.url, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
1304
+ const result = original.ok && !budgetExhausted() ? nodeDocsOriginal(original.text, nodeIntent.url, nodeIntent.symbol) : undefined;
1305
+ const outcome = result ? completedProviderSearch([result], "") : failedProviderSearch(original.ok
1306
+ ? "Node.js 24 official filesystem original did not verify the requested symbol"
1307
+ : `Node.js 24 official original unavailable: ${original.error || original.status}`);
1308
+ recordAttempt("node_docs", outcome);
1309
+ if (result) {
1310
+ notes.push("Official Node.js 24 documentation section verified; complete page extraction remains a separate read");
1311
+ collectDiscovery(outcome);
1312
+ return finishDiscovery();
1313
+ }
1314
+ notes.push(outcome.note);
1315
+ }
1316
+ const postgresIntent = !siteMatches.length ? postgresqlDocsIntent(query) : undefined;
1317
+ if (postgresIntent && !budgetExhausted()) {
1318
+ fallbacks.push("postgresql_docs");
1319
+ const searchUrl = new URL("https://www.postgresql.org/search/");
1320
+ searchUrl.searchParams.set("q", `${postgresIntent.version} ${postgresIntent.symbol}`);
1321
+ const index = await fetchText(searchUrl.href, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
1322
+ let verified;
1323
+ if (index.ok && !budgetExhausted()) {
1324
+ for (const candidate of postgresqlDocsCandidates(index.text, postgresIntent.version)) {
1325
+ if (budgetExhausted())
1326
+ break;
1327
+ const original = await fetchText(candidate.url, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
1328
+ if (original.ok && postgresqlDocsBodyHasSymbol(original.text, postgresIntent.symbol) && !budgetExhausted()) {
1329
+ verified = candidate;
1330
+ break;
1331
+ }
1332
+ }
1333
+ }
1334
+ const outcome = verified ? completedProviderSearch([verified], "") : failedProviderSearch(index.ok
1335
+ ? "PostgreSQL search did not verify the requested symbol in a version-matched original"
1336
+ : `PostgreSQL official search unavailable: ${index.error || index.status}`);
1337
+ recordAttempt("postgresql_docs", outcome);
1338
+ if (verified) {
1339
+ notes.push("Official versioned documentation original contains the requested symbol; complete page extraction remains a separate read");
1340
+ collectDiscovery(outcome);
1341
+ return finishDiscovery();
1342
+ }
1343
+ notes.push(outcome.note);
1344
+ }
1345
+ const siteOperator = siteMatches.length !== 1 || tokens.some(token => token === "(" || token === ")") ? "" : extractSiteOperator(siteMatches[0]);
1090
1346
  if (siteOperator === "velog.io") {
1091
1347
  const blockedReason = siteNativeFallbackBlockedReason("Velog");
1092
1348
  if (blockedReason) {
1093
1349
  notes.push(blockedReason);
1094
1350
  }
1095
1351
  else {
1096
- const velog = await searchVelog(query, numResults, timeoutMs);
1352
+ const velog = constrain(await searchVelog(query, nativeCandidateLimit, attemptTimeout()));
1097
1353
  fallbacks.push("velog");
1098
1354
  recordAttempt("velog", velog);
1099
1355
  if (velog.ok)
@@ -1108,7 +1364,7 @@ export async function runSmartsearch(options, runtime = {}) {
1108
1364
  notes.push(blockedReason);
1109
1365
  }
1110
1366
  else {
1111
- const stackexchange = await searchStackExchange(query, numResults, timeoutMs);
1367
+ const stackexchange = constrain(await searchStackExchange(query, nativeCandidateLimit, attemptTimeout()));
1112
1368
  fallbacks.push("stackexchange");
1113
1369
  recordAttempt("stackexchange", stackexchange);
1114
1370
  if (stackexchange.ok)
@@ -1123,11 +1379,11 @@ export async function runSmartsearch(options, runtime = {}) {
1123
1379
  notes.push(blockedReason);
1124
1380
  }
1125
1381
  else {
1126
- const hackernews = await searchHackerNews(query, numResults, timeoutMs);
1382
+ const hackernews = constrain(await searchHackerNews(query, nativeCandidateLimit, attemptTimeout(), hnDates));
1127
1383
  fallbacks.push("hackernews");
1128
1384
  recordAttempt("hackernews", hackernews);
1129
- if (hackernews.ok)
1130
- return maybeRunCrawlHandoff({ engine: "hackernews", query, results: hackernews.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1385
+ if (hackernews.ok || (hnDates?.bounded && hackernews.status === "succeeded"))
1386
+ return maybeRunCrawlHandoff({ engine: "hackernews", query, results: hackernews.results, fallbacks_tried: fallbacks, notes: hackernews.note ? [...notes, hackernews.note] : notes, execution: smartsearchExecution(attempts) }, options, runtime);
1131
1387
  if (hackernews.note)
1132
1388
  notes.push(hackernews.note);
1133
1389
  }
@@ -1138,7 +1394,9 @@ export async function runSmartsearch(options, runtime = {}) {
1138
1394
  notes.push(blockedReason);
1139
1395
  }
1140
1396
  else {
1141
- const reddit = await searchReddit(query, numResults, timeoutMs);
1397
+ // Reddit already preserves subreddit scope while filling this count;
1398
+ // oversampling here would trigger unnecessary secondary-provider reads.
1399
+ const reddit = constrain(await searchReddit(query, numResults, attemptTimeout()));
1142
1400
  fallbacks.push("reddit");
1143
1401
  recordAttempt("reddit", reddit);
1144
1402
  if (reddit.ok)
@@ -1149,11 +1407,15 @@ export async function runSmartsearch(options, runtime = {}) {
1149
1407
  }
1150
1408
  if (siteOperator === "github.com") {
1151
1409
  const blockedReason = siteNativeFallbackBlockedReason("GitHub");
1410
+ const repoScope = parseGitHubRepoScope(query);
1152
1411
  if (blockedReason) {
1153
1412
  notes.push(blockedReason);
1154
1413
  }
1155
- else if (parseGitHubRepoScope(query)) {
1156
- const githubRepoDocs = await searchGitHubRepoDocs(query, numResults, timeoutMs);
1414
+ else if (repoScope?.scopePath && !repoScope.scopePath.startsWith("blob/")) {
1415
+ notes.push("GitHub repository tree lookup cannot search this URL section; preserving the full query for general search");
1416
+ }
1417
+ else if (repoScope) {
1418
+ const githubRepoDocs = constrain(await searchGitHubRepoDocs(query, nativeCandidateLimit, attemptTimeout()));
1157
1419
  fallbacks.push("github_repo_docs");
1158
1420
  recordAttempt("github_repo_docs", githubRepoDocs);
1159
1421
  if (githubRepoDocs.ok)
@@ -1165,7 +1427,7 @@ export async function runSmartsearch(options, runtime = {}) {
1165
1427
  notes.push("GitHub native repo fallback skipped for non-repository intent");
1166
1428
  }
1167
1429
  else {
1168
- const github = await searchGitHub(query, numResults, timeoutMs);
1430
+ const github = constrain(await searchGitHub(query, nativeCandidateLimit, attemptTimeout()));
1169
1431
  fallbacks.push("github");
1170
1432
  recordAttempt("github", github);
1171
1433
  if (github.ok)
@@ -1176,16 +1438,27 @@ export async function runSmartsearch(options, runtime = {}) {
1176
1438
  }
1177
1439
  if (siteOperator === "npmjs.com") {
1178
1440
  const blockedReason = siteNativeFallbackBlockedReason("npm");
1441
+ const exactPackage = parseNpmPackageScope(query);
1442
+ const remainder = searchQueryTokens(query).filter(token => !/^site:[^"']+$/i.test(token));
1443
+ const quoted = remainder.filter(token => token.startsWith('"') && token.endsWith('"') && token.length > 2 && !token.includes("\\"))
1444
+ .map(token => token.slice(1, -1).toLocaleLowerCase());
1445
+ const nativeExact = Boolean(exactPackage && (remainder.length === 0 || (quoted.length === remainder.length && quoted.length > 0)));
1179
1446
  if (blockedReason) {
1180
1447
  notes.push(blockedReason);
1181
1448
  }
1449
+ else if (exactPackage && !nativeExact) {
1450
+ // Metadata cannot evaluate free words, exclusions, dates or Boolean
1451
+ // expressions. Only standalone quotes can be checked in its own fields.
1452
+ notes.push("npm package metadata cannot evaluate the remaining query constraints; preserving the full query for general search");
1453
+ }
1182
1454
  else {
1183
- const exactPackage = parseNpmPackageScope(query);
1184
- const npm = exactPackage
1185
- ? await searchNpmPackage(exactPackage, timeoutMs)
1186
- : await searchNpm(query, numResults, timeoutMs);
1455
+ const npm = constrain(exactPackage
1456
+ ? await searchNpmPackage(exactPackage, attemptTimeout(), quoted)
1457
+ : await searchNpm(query, nativeCandidateLimit, attemptTimeout()));
1187
1458
  fallbacks.push("npm");
1188
1459
  recordAttempt("npm", npm);
1460
+ if (npm.ok && npm.note)
1461
+ notes.push(npm.note);
1189
1462
  if (npm.ok)
1190
1463
  return maybeRunCrawlHandoff({ engine: "npm", query, results: npm.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1191
1464
  if (npm.note)
@@ -1198,9 +1471,11 @@ export async function runSmartsearch(options, runtime = {}) {
1198
1471
  notes.push(blockedReason);
1199
1472
  }
1200
1473
  else {
1201
- const mdn = await searchMdn(query, numResults, timeoutMs);
1474
+ const mdn = constrain(await searchMdn(query, nativeCandidateLimit, attemptTimeout()));
1202
1475
  fallbacks.push("mdn");
1203
1476
  recordAttempt("mdn", mdn);
1477
+ if (mdn.ok && mdn.note)
1478
+ notes.push(mdn.note);
1204
1479
  if (mdn.ok)
1205
1480
  return maybeRunCrawlHandoff({ engine: "mdn", query, results: mdn.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1206
1481
  if (mdn.note)
@@ -1213,7 +1488,7 @@ export async function runSmartsearch(options, runtime = {}) {
1213
1488
  notes.push(blockedReason);
1214
1489
  }
1215
1490
  else {
1216
- const crates = await searchCrates(query, numResults, timeoutMs);
1491
+ const crates = constrain(await searchCrates(query, nativeCandidateLimit, attemptTimeout()));
1217
1492
  fallbacks.push("crates");
1218
1493
  recordAttempt("crates", crates);
1219
1494
  if (crates.ok)
@@ -1228,7 +1503,7 @@ export async function runSmartsearch(options, runtime = {}) {
1228
1503
  notes.push(blockedReason);
1229
1504
  }
1230
1505
  else {
1231
- const wikipedia = await searchWikipedia(query, numResults, timeoutMs);
1506
+ const wikipedia = constrain(await searchWikipedia(query, nativeCandidateLimit, attemptTimeout()));
1232
1507
  fallbacks.push("wikipedia");
1233
1508
  recordAttempt("wikipedia", wikipedia);
1234
1509
  if (wikipedia.ok)
@@ -1237,17 +1512,19 @@ export async function runSmartsearch(options, runtime = {}) {
1237
1512
  notes.push(wikipedia.note);
1238
1513
  }
1239
1514
  }
1515
+ if (budgetExhausted())
1516
+ return finishDiscovery();
1240
1517
  if (searxngEnabled() && searxngBaseUrl) {
1241
1518
  const privacySensitive = localOnlyMode() || privateProfileMode();
1242
1519
  if (privacySensitive && !await privacySafeSearxngBase(searxngBaseUrl)) {
1243
1520
  notes.push("SearXNG disabled by privacy mode because the configured base URL is not local or private");
1244
1521
  }
1245
1522
  else {
1246
- const searxng = await searchSearxng(searxngBaseUrl, query, numResults, timeoutMs);
1523
+ const searxng = constrain(await searchSearxng(searxngBaseUrl, query, numResults, attemptTimeout()), 100);
1247
1524
  fallbacks.push("searxng");
1248
1525
  recordAttempt("searxng", searxng);
1249
- if (searxng.ok)
1250
- return maybeRunCrawlHandoff({ engine: "searxng", query, results: searxng.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1526
+ if (collectDiscovery(searxng))
1527
+ return finishDiscovery();
1251
1528
  if (searxng.note)
1252
1529
  notes.push(searxng.note);
1253
1530
  }
@@ -1255,31 +1532,179 @@ export async function runSmartsearch(options, runtime = {}) {
1255
1532
  else if (!searxngEnabled()) {
1256
1533
  notes.push("SearXNG disabled by configuration");
1257
1534
  }
1535
+ if (budgetExhausted())
1536
+ return finishDiscovery();
1258
1537
  if (duckduckgoEnabled()) {
1259
- const ddg = await searchDuckDuckGo(query, numResults, timeoutMs);
1538
+ const ddg = constrain(await searchDuckDuckGo(query, numResults, attemptTimeout()), 100);
1260
1539
  fallbacks.push("duckduckgo");
1261
1540
  recordAttempt("duckduckgo", ddg);
1262
- if (ddg.ok)
1263
- return maybeRunCrawlHandoff({ engine: "duckduckgo", query, results: ddg.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1541
+ if (collectDiscovery(ddg))
1542
+ return finishDiscovery();
1264
1543
  if (ddg.note)
1265
1544
  notes.push(ddg.note);
1266
1545
  }
1267
1546
  else {
1268
1547
  notes.push("DuckDuckGo disabled by configuration");
1269
1548
  }
1549
+ if (budgetExhausted())
1550
+ return finishDiscovery();
1270
1551
  if (braveHtmlEnabled()) {
1271
- const braveHtml = await searchBraveHtml(query, numResults, timeoutMs);
1552
+ const braveHtml = constrain(await searchBraveHtml(query, numResults, attemptTimeout()), 100);
1272
1553
  fallbacks.push("brave_html");
1273
1554
  recordAttempt("brave_html", braveHtml);
1274
- if (braveHtml.ok)
1275
- return maybeRunCrawlHandoff({ engine: "brave_html", query, results: braveHtml.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1555
+ if (collectDiscovery(braveHtml))
1556
+ return finishDiscovery();
1276
1557
  if (braveHtml.note)
1277
1558
  notes.push(braveHtml.note);
1278
1559
  }
1279
1560
  else {
1280
1561
  notes.push("Brave HTML disabled by configuration");
1281
1562
  }
1282
- return maybeRunCrawlHandoff({ engine: "none", query, results: [], fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
1563
+ const remaining = timeoutMs - (Date.now() - startedAt);
1564
+ let canContinueNative = false;
1565
+ if (browserEnabled) {
1566
+ fallbacks.push("naver_browser");
1567
+ let outcome;
1568
+ if (remaining <= 0)
1569
+ outcome = failedProviderSearch("Search budget exhausted before operator-browser continuation");
1570
+ else {
1571
+ try {
1572
+ // Native search sometimes treats a site path as text. A second,
1573
+ // host-only discovery query may recover the same originals; the
1574
+ // original path remains a hard result constraint below.
1575
+ const searchQueries = scope?.path && siteMatches.length === 1 && /[가-힣]/u.test(query)
1576
+ && ["clien.net", "dcinside.com", "ppomppu.co.kr"].includes(scope.host)
1577
+ ? [query, query.replace(siteMatches[0], `site:${scope.host}`)] : [query];
1578
+ outcome = failedProviderSearch("Browser search exposed no verified result headlines satisfying supported query constraints");
1579
+ for (const [index, searchQuery] of searchQueries.entries()) {
1580
+ const budget = timeoutMs - (Date.now() - startedAt);
1581
+ if (budget <= 0)
1582
+ break;
1583
+ const url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}`;
1584
+ const page = await (runtime.runBrowserSession || runBrowserSession)({ url, cdpUrl, timeoutMs: Math.max(1, Math.floor(budget / ((pythonSearchUrl || nativeYoutubeUrl ? 1 : 0) + searchQueries.length - index))), allowPrivateHosts: false });
1585
+ if (!page.ok) {
1586
+ outcome = failedProviderSearch(`Approved browser: ${page.error.code}: ${page.error.message}`);
1587
+ break;
1588
+ }
1589
+ const final = new URL(page.final_url);
1590
+ const sameQuery = final.origin === "https://search.naver.com" && !final.username && !final.password
1591
+ && final.pathname === "/search.naver" && final.searchParams.getAll("query").length === 1 && final.searchParams.get("query") === searchQuery;
1592
+ if (!sameQuery || /captcha|verify you are human|unusual traffic|자동입력 방지|비정상적인 접근/i.test(page.text.slice(0, 1000))) {
1593
+ outcome = failedProviderSearch("Browser search stopped at a challenge or changed query; user interaction may be required");
1594
+ break;
1595
+ }
1596
+ canContinueNative = true;
1597
+ const safeResults = [];
1598
+ for (const item of rankDiscoveryResults(parseNaverBrowserSearch(page), discoveryTerms)) {
1599
+ if (Date.now() - startedAt >= timeoutMs)
1600
+ break;
1601
+ if (!browserSearchMatchesQuery(item, query))
1602
+ continue;
1603
+ if (isSearchTrackingRelay(item))
1604
+ continue;
1605
+ if (scope && !searchResultMatchesSite(item.url, scope.host, scope.path))
1606
+ continue;
1607
+ let timer;
1608
+ const validation = await Promise.race([
1609
+ validateOutboundUrl(item.url, { allowPrivateHosts: false }),
1610
+ new Promise((resolve) => { timer = setTimeout(() => resolve({ ok: false }), Math.max(1, timeoutMs - (Date.now() - startedAt))); }),
1611
+ ]).finally(() => { if (timer)
1612
+ clearTimeout(timer); });
1613
+ if (Date.now() - startedAt >= timeoutMs)
1614
+ break;
1615
+ if (validation.ok)
1616
+ safeResults.push(item);
1617
+ if (safeResults.length >= numResults)
1618
+ break;
1619
+ }
1620
+ outcome = safeResults.length ? constrain(completedProviderSearch(safeResults, "")) : failedProviderSearch("Browser search exposed no verified result headlines satisfying supported query constraints");
1621
+ if (outcome.ok) {
1622
+ if (index > 0)
1623
+ notes.push("Host-only discovery preserved the requested site path on every returned original");
1624
+ notes.push("Recovered through the approved local browser. Search snippets are discovery evidence, not verification of article bodies or all comments.");
1625
+ break;
1626
+ }
1627
+ }
1628
+ }
1629
+ catch (error) {
1630
+ outcome = failedProviderSearch(`Approved browser search failed: ${error instanceof Error ? error.message : String(error)}`);
1631
+ }
1632
+ }
1633
+ recordAttempt("naver_browser", outcome);
1634
+ if (outcome.ok) {
1635
+ collectDiscovery(outcome);
1636
+ if (!unscopedDocsTerms)
1637
+ return finishDiscovery();
1638
+ }
1639
+ if (outcome.note)
1640
+ notes.push(outcome.note);
1641
+ }
1642
+ if (nativeYoutubeUrl && canContinueNative && !budgetExhausted()) {
1643
+ fallbacks.push("youtube_browser");
1644
+ let outcome;
1645
+ try {
1646
+ const page = await (runtime.runBrowserSession || runBrowserSession)({
1647
+ url: nativeYoutubeUrl, cdpUrl, timeoutMs: Math.max(1, timeoutMs - (Date.now() - startedAt)), allowPrivateHosts: false,
1648
+ waitForSelector: "ytd-video-renderer a#video-title",
1649
+ });
1650
+ if (Date.now() - startedAt >= timeoutMs)
1651
+ outcome = failedProviderSearch("YouTube native search exceeded the shared search budget");
1652
+ else if (!page.ok)
1653
+ outcome = failedProviderSearch(`YouTube native browser: ${page.error.code}: ${page.error.message}`);
1654
+ else {
1655
+ const rows = parseYoutubeSearch(page, nativeYoutubeUrl, query);
1656
+ outcome = Date.now() - startedAt >= timeoutMs ? failedProviderSearch("YouTube native search exceeded the shared search budget")
1657
+ : rows.length ? constrain(completedProviderSearch(rows, "")) : failedProviderSearch("YouTube native search exposed no exact watch-page headlines satisfying the quoted query");
1658
+ if (outcome.ok)
1659
+ notes.push("Recovered watch-page discovery through the approved local browser. Video title evidence does not verify the full video or transcript.");
1660
+ }
1661
+ }
1662
+ catch (error) {
1663
+ outcome = failedProviderSearch(`YouTube native search failed: ${error instanceof Error ? error.message : String(error)}`);
1664
+ }
1665
+ recordAttempt("youtube_browser", outcome);
1666
+ if (outcome.ok) {
1667
+ collectDiscovery(outcome);
1668
+ return finishDiscovery();
1669
+ }
1670
+ if (outcome.note)
1671
+ notes.push(outcome.note);
1672
+ }
1673
+ if (pythonSearchUrl && canContinueNative && !budgetExhausted()) {
1674
+ fallbacks.push("python_docs_browser");
1675
+ let outcome;
1676
+ try {
1677
+ const page = await (runtime.runBrowserSession || runBrowserSession)({
1678
+ url: pythonSearchUrl, cdpUrl, timeoutMs: Math.max(1, timeoutMs - (Date.now() - startedAt)), allowPrivateHosts: false,
1679
+ waitForSelector: "#search-results .search-summary:not(:empty)",
1680
+ });
1681
+ if (Date.now() - startedAt >= timeoutMs)
1682
+ outcome = failedProviderSearch("Python native search exceeded the shared search budget");
1683
+ else if (!page.ok)
1684
+ outcome = failedProviderSearch(`Python native browser: ${page.error.code}: ${page.error.message}`);
1685
+ else {
1686
+ const rows = parsePythonDocsSearch(page, pythonSearchUrl, unscopedDocsTerms || query);
1687
+ outcome = Date.now() - startedAt >= timeoutMs ? failedProviderSearch("Python native search exceeded the shared search budget")
1688
+ : rows.length ? constrain(completedProviderSearch(rows, "")) : failedProviderSearch("Python native search exposed no completed results satisfying the original query");
1689
+ if (outcome.ok)
1690
+ notes.push("Recovered through official Python documentation search in the approved browser. Native index/headline evidence is discovery, not complete original-body verification.");
1691
+ }
1692
+ }
1693
+ catch (error) {
1694
+ outcome = failedProviderSearch(`Python native search failed: ${error instanceof Error ? error.message : String(error)}`);
1695
+ }
1696
+ recordAttempt("python_docs_browser", outcome);
1697
+ if (outcome.ok) {
1698
+ // Explicit documentation intent: a verified official index supersedes
1699
+ // adjacent general-web candidates; retain them if the native lane fails.
1700
+ if (unscopedDocsTerms)
1701
+ discoveryResults = [];
1702
+ collectDiscovery(outcome);
1703
+ }
1704
+ if (outcome.note)
1705
+ notes.push(outcome.note);
1706
+ }
1707
+ return finishDiscovery();
1283
1708
  }
1284
1709
  catch (error) {
1285
1710
  const message = error instanceof Error ? error.message : String(error);
@@ -1330,9 +1755,6 @@ export function renderSmartsearch(response) {
1330
1755
  lines.push(item.snippet);
1331
1756
  parts.push(lines.join("\n"));
1332
1757
  });
1333
- if (assessment.recommended_handoff && assessment.recommended_handoff.tool !== "smartcrawl") {
1334
- parts.push(`Recommended handoff:\n- tool: ${assessment.recommended_handoff.tool}\n- reason: ${assessment.recommended_handoff.reason}\n- url: ${assessment.recommended_handoff.url}`);
1335
- }
1336
1758
  if (response.notes.length > 0)
1337
1759
  parts.push(`Notes:\n${response.notes.map((item) => `- ${item}`).join("\n")}`);
1338
1760
  return parts.join("\n\n").trim();