smart-web-mcp 0.44.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +342 -0
- package/README.md +33 -568
- package/dist/assessment.d.ts +10 -3
- package/dist/assessment.js +21 -6
- package/dist/assessment.js.map +1 -1
- package/dist/body-export.d.ts +10 -0
- package/dist/body-export.js +28 -0
- package/dist/body-export.js.map +1 -0
- package/dist/browser-session.d.ts +32 -1
- package/dist/browser-session.js +373 -3
- package/dist/browser-session.js.map +1 -1
- package/dist/cli-fetch-options.d.ts +7 -0
- package/dist/cli-fetch-options.js +41 -0
- package/dist/cli-fetch-options.js.map +1 -0
- package/dist/cli.js +28 -11
- package/dist/cli.js.map +1 -1
- package/dist/comment-export.d.ts +10 -0
- package/dist/comment-export.js +39 -0
- package/dist/comment-export.js.map +1 -0
- package/dist/composition.d.ts +1 -22
- package/dist/composition.js +0 -16
- package/dist/composition.js.map +1 -1
- package/dist/extraction/budget.d.ts +3 -0
- package/dist/extraction/budget.js +88 -7
- package/dist/extraction/budget.js.map +1 -1
- package/dist/extraction/document.js +134 -9
- package/dist/extraction/document.js.map +1 -1
- package/dist/korea/coupang-filters.d.ts +5 -0
- package/dist/korea/coupang-filters.js +87 -0
- package/dist/korea/coupang-filters.js.map +1 -0
- package/dist/korea/coupang.d.ts +27 -6
- package/dist/korea/coupang.js +202 -82
- package/dist/korea/coupang.js.map +1 -1
- package/dist/korea/proxy-client.d.ts +1 -0
- package/dist/korea/proxy-client.js +6 -2
- package/dist/korea/proxy-client.js.map +1 -1
- package/dist/korea/smartvertical.d.ts +1 -0
- package/dist/korea/smartvertical.js +39 -25
- package/dist/korea/smartvertical.js.map +1 -1
- package/dist/lib.d.ts +4 -2
- package/dist/lib.js +2 -1
- package/dist/lib.js.map +1 -1
- package/dist/mcp-server.js +30 -15
- package/dist/mcp-server.js.map +1 -1
- package/dist/operator-browser-fixture.d.ts +7 -0
- package/dist/operator-browser-fixture.js +19 -0
- package/dist/operator-browser-fixture.js.map +1 -0
- package/dist/operator-canary-cli.d.ts +6 -0
- package/dist/operator-canary-cli.js +190 -0
- package/dist/operator-canary-cli.js.map +1 -0
- package/dist/operator-canary.d.ts +74 -0
- package/dist/operator-canary.js +135 -0
- package/dist/operator-canary.js.map +1 -0
- package/dist/operator-cdp.d.ts +15 -0
- package/dist/operator-cdp.js +195 -0
- package/dist/operator-cdp.js.map +1 -0
- package/dist/operator-frame.d.ts +3 -0
- package/dist/operator-frame.js +55 -0
- package/dist/operator-frame.js.map +1 -0
- package/dist/settings.d.ts +3 -0
- package/dist/settings.js +1 -0
- package/dist/settings.js.map +1 -1
- package/dist/shared.d.ts +33 -8
- package/dist/shared.js +15 -9
- package/dist/shared.js.map +1 -1
- package/dist/smartbrowser.d.ts +169 -0
- package/dist/smartbrowser.js +288 -0
- package/dist/smartbrowser.js.map +1 -0
- package/dist/smartcrawl.js +16 -21
- package/dist/smartcrawl.js.map +1 -1
- package/dist/smartfetch/academic-fallback.js +3 -5
- package/dist/smartfetch/academic-fallback.js.map +1 -1
- package/dist/smartfetch/archive-fallback.js +2 -5
- package/dist/smartfetch/archive-fallback.js.map +1 -1
- package/dist/smartfetch/assets.d.ts +1 -1
- package/dist/smartfetch/assets.js +28 -1
- package/dist/smartfetch/assets.js.map +1 -1
- package/dist/smartfetch/jina-reader.d.ts +1 -0
- package/dist/smartfetch/jina-reader.js +17 -3
- package/dist/smartfetch/jina-reader.js.map +1 -1
- package/dist/smartfetch/pipeline.js +15 -25
- package/dist/smartfetch/pipeline.js.map +1 -1
- package/dist/smartfetch/provider-policy.d.ts +3 -0
- package/dist/smartfetch/provider-policy.js +45 -2
- package/dist/smartfetch/provider-policy.js.map +1 -1
- package/dist/smartfetch/provider-types.d.ts +8 -0
- package/dist/smartfetch/providers/article.js +76 -10
- package/dist/smartfetch/providers/article.js.map +1 -1
- package/dist/smartfetch/providers/blind-post.d.ts +2 -0
- package/dist/smartfetch/providers/blind-post.js +167 -0
- package/dist/smartfetch/providers/blind-post.js.map +1 -0
- package/dist/smartfetch/providers/clien.d.ts +2 -0
- package/dist/smartfetch/providers/clien.js +157 -0
- package/dist/smartfetch/providers/clien.js.map +1 -0
- package/dist/smartfetch/providers/commerce.d.ts +15 -0
- package/dist/smartfetch/providers/commerce.js +123 -7
- package/dist/smartfetch/providers/commerce.js.map +1 -1
- package/dist/smartfetch/providers/dcinside.js +313 -20
- package/dist/smartfetch/providers/dcinside.js.map +1 -1
- package/dist/smartfetch/providers/hackernews-html.d.ts +21 -0
- package/dist/smartfetch/providers/hackernews-html.js +148 -0
- package/dist/smartfetch/providers/hackernews-html.js.map +1 -0
- package/dist/smartfetch/providers/hackernews.js +76 -14
- package/dist/smartfetch/providers/hackernews.js.map +1 -1
- package/dist/smartfetch/providers/index.js +7 -1
- package/dist/smartfetch/providers/index.js.map +1 -1
- package/dist/smartfetch/providers/linkedin.js +2 -1
- package/dist/smartfetch/providers/linkedin.js.map +1 -1
- package/dist/smartfetch/providers/naver-blog.d.ts +2 -0
- package/dist/smartfetch/providers/naver-blog.js +87 -45
- package/dist/smartfetch/providers/naver-blog.js.map +1 -1
- package/dist/smartfetch/providers/naver-cafe-native.d.ts +7 -0
- package/dist/smartfetch/providers/naver-cafe-native.js +188 -0
- package/dist/smartfetch/providers/naver-cafe-native.js.map +1 -0
- package/dist/smartfetch/providers/naver-cafe.js +165 -4
- package/dist/smartfetch/providers/naver-cafe.js.map +1 -1
- package/dist/smartfetch/providers/naver-map-search.js +1 -1
- package/dist/smartfetch/providers/naver-map-search.js.map +1 -1
- package/dist/smartfetch/providers/ppomppu.d.ts +2 -0
- package/dist/smartfetch/providers/ppomppu.js +289 -0
- package/dist/smartfetch/providers/ppomppu.js.map +1 -0
- package/dist/smartfetch/providers/reddit.js +24 -3
- package/dist/smartfetch/providers/reddit.js.map +1 -1
- package/dist/smartfetch/providers/threads.js +1 -1
- package/dist/smartfetch/providers/threads.js.map +1 -1
- package/dist/smartfetch/providers/wikipedia.js +138 -0
- package/dist/smartfetch/providers/wikipedia.js.map +1 -1
- package/dist/smartfetch.d.ts +7 -1
- package/dist/smartfetch.js +353 -76
- package/dist/smartfetch.js.map +1 -1
- package/dist/smartsearch-brave.d.ts +2 -0
- package/dist/smartsearch-brave.js +43 -0
- package/dist/smartsearch-brave.js.map +1 -0
- package/dist/smartsearch-browser.d.ts +5 -0
- package/dist/smartsearch-browser.js +116 -0
- package/dist/smartsearch-browser.js.map +1 -0
- package/dist/smartsearch-ddg.d.ts +2 -0
- package/dist/smartsearch-ddg.js +46 -0
- package/dist/smartsearch-ddg.js.map +1 -0
- package/dist/smartsearch-node.d.ts +6 -0
- package/dist/smartsearch-node.js +21 -0
- package/dist/smartsearch-node.js.map +1 -0
- package/dist/smartsearch-postgresql.d.ts +7 -0
- package/dist/smartsearch-postgresql.js +44 -0
- package/dist/smartsearch-postgresql.js.map +1 -0
- package/dist/smartsearch-python.d.ts +8 -0
- package/dist/smartsearch-python.js +87 -0
- package/dist/smartsearch-python.js.map +1 -0
- package/dist/smartsearch-quality.d.ts +6 -0
- package/dist/smartsearch-quality.js +47 -0
- package/dist/smartsearch-quality.js.map +1 -0
- package/dist/smartsearch-youtube.d.ts +7 -0
- package/dist/smartsearch-youtube.js +76 -0
- package/dist/smartsearch-youtube.js.map +1 -0
- package/dist/smartsearch.d.ts +2 -0
- package/dist/smartsearch.js +506 -84
- package/dist/smartsearch.js.map +1 -1
- package/package.json +8 -5
- package/python/requirements-undetected-chromedriver.txt +2 -2
package/dist/smartsearch.js
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
|
-
import { configEnabled, fetchText,
|
|
1
|
+
import { configEnabled, fetchText, smartWebProfile, stripTags, uniqueResults, validateOutboundUrl } from "./shared.js";
|
|
2
2
|
import { createAssessment, createHandoffSuggestion, detectInteractiveLikeText, firstMeaningfulBlockedReason } from "./assessment.js";
|
|
3
3
|
import { configText } from "./settings.js";
|
|
4
4
|
import { runSmartcrawl } from "./smartcrawl.js";
|
|
5
5
|
import { budgetText, projectRetrievalAssessment } from "./extraction/budget.js";
|
|
6
6
|
import { parseFeed } from "./smartfetch/generic-fallbacks.js";
|
|
7
|
+
import { runBrowserSession } from "./browser-session.js";
|
|
8
|
+
import { browserSearchMatchesQuery, parseNaverBrowserSearch, searchResultMatchesSite } from "./smartsearch-browser.js";
|
|
9
|
+
import { parseBraveSearch } from "./smartsearch-brave.js";
|
|
10
|
+
import { parseHTML } from "linkedom";
|
|
11
|
+
import { parseDuckDuckGoSearch } from "./smartsearch-ddg.js";
|
|
12
|
+
import { parsePythonDocsSearch, pythonDocsSearchUrl, unscopedPythonDocsTerms } from "./smartsearch-python.js";
|
|
13
|
+
import { postgresqlDocsBodyHasSymbol, postgresqlDocsCandidates, postgresqlDocsIntent } from "./smartsearch-postgresql.js";
|
|
14
|
+
import { nodeDocsIntent, nodeDocsOriginal } from "./smartsearch-node.js";
|
|
15
|
+
import { parseYoutubeSearch, youtubeSearchUrl } from "./smartsearch-youtube.js";
|
|
16
|
+
import { discoveryEvidenceScore, hasDiscoveryYield, isSearchTrackingRelay, koreanDiscoveryTerms, rankDiscoveryResults } from "./smartsearch-quality.js";
|
|
7
17
|
const DEFAULT_TIMEOUT_MS = 12000;
|
|
8
18
|
const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
|
|
9
19
|
const SMARTSEARCH_BUDGET_PRESETS = {
|
|
@@ -94,18 +104,31 @@ export function projectSmartsearchOutput(response, preset = "compact") {
|
|
|
94
104
|
metadata[path] = projected.metadata;
|
|
95
105
|
return projected.value;
|
|
96
106
|
};
|
|
97
|
-
|
|
98
|
-
|
|
107
|
+
// A clipped URL is not an original the caller can open; omit it rather than
|
|
108
|
+
// returning a plausible-looking but invalid link, and fill from later rows.
|
|
109
|
+
const usableResults = response.results.filter((item) => item.url.length <= policy.maxUrlChars);
|
|
110
|
+
const omittedUrls = response.results.length - usableResults.length;
|
|
111
|
+
const results = budgetList(usableResults, policy.maxResults);
|
|
112
|
+
if (omittedUrls > 0 || results.metadata.truncated) {
|
|
113
|
+
metadata.results = {
|
|
114
|
+
truncated: true,
|
|
115
|
+
original_items: response.results.length,
|
|
116
|
+
returned_items: results.items.length,
|
|
117
|
+
omitted_items: response.results.length - results.items.length,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
99
120
|
const projectedResults = results.items.map((item, index) => ({
|
|
100
121
|
...item,
|
|
101
122
|
title: projectText(item.title, policy.maxTitleChars, `results.${index}.title`),
|
|
102
|
-
url:
|
|
123
|
+
url: item.url,
|
|
103
124
|
snippet: projectText(item.snippet || "", policy.maxSnippetChars, `results.${index}.snippet`),
|
|
104
125
|
engine: projectText(item.engine, policy.maxEngineChars, `results.${index}.engine`),
|
|
105
126
|
}));
|
|
106
127
|
const notes = budgetList(response.notes, policy.maxNotes);
|
|
107
128
|
recordListMetadata(metadata, "notes", notes.metadata);
|
|
108
129
|
const projectedNotes = notes.items.map((note, index) => projectText(note, policy.maxNoteChars, `notes.${index}`));
|
|
130
|
+
if (omittedUrls > 0)
|
|
131
|
+
projectedNotes.push(`${omittedUrls} original URL(s) omitted because they exceed the ${policy.maxUrlChars}-character output limit.`);
|
|
109
132
|
const fallbacks = budgetList(response.fallbacks_tried, policy.maxFallbacks);
|
|
110
133
|
recordListMetadata(metadata, "fallbacks_tried", fallbacks.metadata);
|
|
111
134
|
const projectedFallbacks = fallbacks.items.map((fallback, index) => projectText(fallback, policy.maxFallbackChars, `fallbacks_tried.${index}`));
|
|
@@ -151,6 +174,8 @@ export function projectSmartsearchOutput(response, preset = "compact") {
|
|
|
151
174
|
maxUrlChars: policy.maxUrlChars,
|
|
152
175
|
}, metadata)
|
|
153
176
|
: undefined;
|
|
177
|
+
if (omittedUrls > 0 && assessment)
|
|
178
|
+
assessment.confidence = "low";
|
|
154
179
|
const engine = projectText(response.engine, policy.maxEngineChars, "engine");
|
|
155
180
|
const query = projectText(response.query, policy.maxQueryChars, "query");
|
|
156
181
|
const truncated = Object.values(metadata).some((item) => item.truncated);
|
|
@@ -357,9 +382,13 @@ function buildSearchResponse(response, handoff) {
|
|
|
357
382
|
return handoff ? { ...response, handoff, assessment } : { ...response, assessment };
|
|
358
383
|
}
|
|
359
384
|
function assessSearchResponse(response, handoff) {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const
|
|
385
|
+
// A failed earlier engine remains in notes, but does not block an exact
|
|
386
|
+
// watch-page original recovered from the site's own search.
|
|
387
|
+
const blockedReason = response.engine === "youtube_browser" && response.results.length > 0 ? "" : firstMeaningfulBlockedReason(response.notes);
|
|
388
|
+
const publicHtmlEngine = ["duckduckgo", "brave_html", "naver_browser", "youtube_browser", "mixed"].includes(response.engine);
|
|
389
|
+
const weakDiscovery = (publicHtmlEngine || response.engine === "searxng")
|
|
390
|
+
&& !hasDiscoveryYield(response.results, koreanDiscoveryTerms(response.query), response.results.length || 1);
|
|
391
|
+
const confidence = weakDiscovery ? "low" : response.results.length === 0
|
|
363
392
|
? (handoff?.performed ? "medium" : "low")
|
|
364
393
|
: (response.results.length >= 3 && !publicHtmlEngine ? "high" : "medium");
|
|
365
394
|
const interactiveFlowLikely = Boolean(handoff && !handoff.performed && detectInteractiveLikeText(handoff.reason));
|
|
@@ -407,6 +436,15 @@ async function maybeRunCrawlHandoff(response, options, runtime) {
|
|
|
407
436
|
}
|
|
408
437
|
const crawlQuery = stripSiteOperators(options.query);
|
|
409
438
|
const mode = inferCrawlMode(startUrl, options.crawlMode);
|
|
439
|
+
const siteTokens = searchQueryTokens(options.query).filter(token => /^site:[^"']+$/i.test(token));
|
|
440
|
+
if (siteTokens.length === 1 && extractSiteScope(siteTokens[0])?.host === "news.ycombinator.com" && parseHnDateQuery(options.query).bounded) {
|
|
441
|
+
const reason = "site_crawl not performed: HN date constraints require native publication timestamps that crawl items do not verify";
|
|
442
|
+
const declined = buildSearchResponse({ ...response, notes: [...response.notes, reason] }, {
|
|
443
|
+
tool: "smartcrawl", performed: false, reason, start_url: startUrl, query: options.query, mode,
|
|
444
|
+
items: [], pages_attempted: 0, pages_loaded: 0, errors: [reason],
|
|
445
|
+
});
|
|
446
|
+
return { ...declined, assessment: createAssessment({ ...declined.assessment, recommended_handoff: null }) };
|
|
447
|
+
}
|
|
410
448
|
const crawl = await (runtime.runSmartcrawl || runSmartcrawl)({
|
|
411
449
|
url: startUrl,
|
|
412
450
|
...(crawlQuery ? { query: crawlQuery } : {}),
|
|
@@ -508,9 +546,46 @@ async function searchStackExchange(query, numResults, timeoutMs) {
|
|
|
508
546
|
return failedProviderSearch(`StackExchange parse error: ${message}`);
|
|
509
547
|
}
|
|
510
548
|
}
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
549
|
+
function searchQueryTokens(query) {
|
|
550
|
+
return query.match(/"(?:\\.|[^"\\])*"|'[^']*'|[()]|[^\s()]+/g) || [];
|
|
551
|
+
}
|
|
552
|
+
function parseHnDateQuery(query) {
|
|
553
|
+
const tokens = searchQueryTokens(query).filter(token => !/^site:[^"']+$/i.test(token));
|
|
554
|
+
const parsed = { text: "", bounded: false };
|
|
555
|
+
const terms = [];
|
|
556
|
+
for (const token of tokens) {
|
|
557
|
+
const operator = /^(after|before):(.*)$/i.exec(token);
|
|
558
|
+
if (!operator) {
|
|
559
|
+
terms.push(token);
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
parsed.bounded = true;
|
|
563
|
+
const date = operator[2];
|
|
564
|
+
const timestamp = Date.parse(`${date}T00:00:00Z`);
|
|
565
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(timestamp) || new Date(timestamp).toISOString().slice(0, 10) !== date) {
|
|
566
|
+
parsed.error = `Invalid HN date constraint: ${token}; use a real UTC calendar date YYYY-MM-DD`;
|
|
567
|
+
continue;
|
|
568
|
+
}
|
|
569
|
+
const seconds = timestamp / 1000;
|
|
570
|
+
if (operator[1].toLowerCase() === "after")
|
|
571
|
+
parsed.after = Math.max(parsed.after ?? -Infinity, seconds);
|
|
572
|
+
else
|
|
573
|
+
parsed.before = Math.min(parsed.before ?? Infinity, seconds);
|
|
574
|
+
}
|
|
575
|
+
parsed.text = terms.join(" ");
|
|
576
|
+
if (parsed.bounded && terms.some(term => /^(?:AND|OR|NOT)$/.test(term) || /[()]/.test(term.replace(/^(["']).*\1$/, ""))))
|
|
577
|
+
parsed.error = "HN date constraints with Boolean/grouped expressions are unsupported; refusing to change their scope";
|
|
578
|
+
if (parsed.after !== undefined && parsed.before !== undefined && parsed.after >= parsed.before)
|
|
579
|
+
parsed.error = "HN date constraints have an empty or reversed UTC interval";
|
|
580
|
+
return parsed;
|
|
581
|
+
}
|
|
582
|
+
async function searchHackerNews(query, numResults, timeoutMs, dates) {
|
|
583
|
+
const q = dates.bounded ? dates.text : stripSiteOperators(query) || query;
|
|
584
|
+
const params = new URLSearchParams({ query: q, tags: "story", hitsPerPage: String(Math.min(numResults, 20)) });
|
|
585
|
+
const filters = [dates.after === undefined ? "" : `created_at_i>=${dates.after}`, dates.before === undefined ? "" : `created_at_i<${dates.before}`].filter(Boolean);
|
|
586
|
+
if (filters.length)
|
|
587
|
+
params.set("numericFilters", filters.join(","));
|
|
588
|
+
const result = await fetchText(`https://hn.algolia.com/api/v1/search?${params}`, timeoutMs, {
|
|
514
589
|
headers: {
|
|
515
590
|
accept: "application/json,text/plain,*/*",
|
|
516
591
|
"user-agent": DEFAULT_USER_AGENT,
|
|
@@ -520,8 +595,15 @@ async function searchHackerNews(query, numResults, timeoutMs) {
|
|
|
520
595
|
return failedProviderSearch(result.error || `Hacker News error ${result.status}`);
|
|
521
596
|
try {
|
|
522
597
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
523
|
-
|
|
524
|
-
|
|
598
|
+
if (!Array.isArray(data?.hits))
|
|
599
|
+
return failedProviderSearch("Hacker News returned an invalid hits payload");
|
|
600
|
+
const raw = data.hits;
|
|
601
|
+
const valid = dates.bounded ? raw.filter((item) => item && typeof item.objectID === "string" && /^\d+$/.test(item.objectID)
|
|
602
|
+
&& typeof item.title === "string" && item.title.trim() && Number.isSafeInteger(item.created_at_i) && Math.abs(item.created_at_i) <= 8640000000000
|
|
603
|
+
&& (dates.after === undefined || item.created_at_i >= dates.after) && (dates.before === undefined || item.created_at_i < dates.before)) : raw;
|
|
604
|
+
if (dates.bounded && raw.length && !valid.length)
|
|
605
|
+
return failedProviderSearch("Hacker News returned no verifiable stories satisfying the requested date constraints");
|
|
606
|
+
const results = uniqueResults(valid.map((item) => {
|
|
525
607
|
const objectId = String(item?.objectID || "").trim();
|
|
526
608
|
let linkedHost = "";
|
|
527
609
|
try {
|
|
@@ -534,15 +616,16 @@ async function searchHackerNews(query, numResults, timeoutMs) {
|
|
|
534
616
|
title: String(item?.title || item?.story_title || `HN #${objectId}`).trim(),
|
|
535
617
|
url: objectId ? `https://news.ycombinator.com/item?id=${objectId}` : "",
|
|
536
618
|
snippet: [
|
|
619
|
+
dates.bounded ? new Date(item.created_at_i * 1000).toISOString() : "",
|
|
537
620
|
item?.points ? `▲ ${item.points}` : "",
|
|
538
621
|
item?.author ? `by ${item.author}` : "",
|
|
539
|
-
item?.created_at ? String(item.created_at).slice(0, 10) : "",
|
|
622
|
+
!dates.bounded && item?.created_at ? String(item.created_at).slice(0, 10) : "",
|
|
540
623
|
linkedHost ? `→ ${linkedHost}` : "",
|
|
541
624
|
].filter(Boolean).join(" · "),
|
|
542
625
|
engine: "hackernews",
|
|
543
626
|
};
|
|
544
627
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
545
|
-
return completedProviderSearch(results, results.length > 0 ? "" : "Hacker News search returned no results");
|
|
628
|
+
return { ...completedProviderSearch(results, results.length > 0 ? "" : "Hacker News search returned no results"), hnDateBoundsVerified: dates.bounded };
|
|
546
629
|
}
|
|
547
630
|
catch (error) {
|
|
548
631
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -825,6 +908,9 @@ async function searchGitHubRepoDocs(query, numResults, timeoutMs) {
|
|
|
825
908
|
}
|
|
826
909
|
async function searchMdn(query, numResults, timeoutMs) {
|
|
827
910
|
const q = stripSiteOperators(query) || query;
|
|
911
|
+
const quoted = searchQueryTokens(query)
|
|
912
|
+
.filter(token => token.startsWith('"') && token.endsWith('"') && token.length > 2 && !token.includes("\\"))
|
|
913
|
+
.map(token => token.slice(1, -1).toLocaleLowerCase());
|
|
828
914
|
const result = await fetchText(`https://developer.mozilla.org/api/v1/search?q=${encodeURIComponent(q)}&locale=en-US`, timeoutMs, {
|
|
829
915
|
headers: {
|
|
830
916
|
accept: "application/json,text/plain,*/*",
|
|
@@ -835,8 +921,10 @@ async function searchMdn(query, numResults, timeoutMs) {
|
|
|
835
921
|
return failedProviderSearch(result.error || `MDN error ${result.status}`);
|
|
836
922
|
try {
|
|
837
923
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
838
|
-
|
|
839
|
-
|
|
924
|
+
if (!Array.isArray(data?.documents))
|
|
925
|
+
return failedProviderSearch("MDN returned an invalid documents payload");
|
|
926
|
+
const raw = data.documents;
|
|
927
|
+
const candidates = uniqueResults(raw.map((item) => {
|
|
840
928
|
const mdnUrl = String(item?.mdn_url || item?.url || "").trim();
|
|
841
929
|
return {
|
|
842
930
|
title: String(item?.title || item?.slug || mdnUrl).trim(),
|
|
@@ -844,8 +932,14 @@ async function searchMdn(query, numResults, timeoutMs) {
|
|
|
844
932
|
snippet: String(item?.summary || item?.excerpt || "").trim(),
|
|
845
933
|
engine: "mdn",
|
|
846
934
|
};
|
|
847
|
-
}).filter((item) => item.url && item.url !== "https://developer.mozilla.org/"))
|
|
848
|
-
|
|
935
|
+
}).filter((item) => item.url && item.url !== "https://developer.mozilla.org/"));
|
|
936
|
+
// Native index neighbors do not satisfy an explicit quote by proximity.
|
|
937
|
+
const results = quoted.length ? candidates.filter(item => quoted.every(phrase => item.title.toLocaleLowerCase().includes(phrase) || item.snippet.toLocaleLowerCase().includes(phrase))) : candidates;
|
|
938
|
+
if (quoted.length && candidates.length && !results.length)
|
|
939
|
+
return failedProviderSearch("MDN index has no title/summary evidence for the quoted phrase; continuing to full-query search");
|
|
940
|
+
return completedProviderSearch(results.slice(0, numResults), quoted.length && results.length
|
|
941
|
+
? "MDN quoted phrase matched index title/summary only; original body remains unverified"
|
|
942
|
+
: results.length ? "" : "MDN search returned no results");
|
|
849
943
|
}
|
|
850
944
|
catch (error) {
|
|
851
945
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -864,7 +958,9 @@ async function searchCrates(query, numResults, timeoutMs) {
|
|
|
864
958
|
return failedProviderSearch(result.error || `crates.io error ${result.status}`);
|
|
865
959
|
try {
|
|
866
960
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
867
|
-
|
|
961
|
+
if (!Array.isArray(data?.crates))
|
|
962
|
+
return failedProviderSearch("crates.io returned an invalid crates payload");
|
|
963
|
+
const raw = data.crates;
|
|
868
964
|
const results = uniqueResults(raw.map((item) => {
|
|
869
965
|
const name = String(item?.name || item?.id || "").trim();
|
|
870
966
|
const latest = String(item?.max_version || item?.newest_version || "").trim();
|
|
@@ -886,7 +982,7 @@ async function searchCrates(query, numResults, timeoutMs) {
|
|
|
886
982
|
return failedProviderSearch(`crates.io parse error: ${message}`);
|
|
887
983
|
}
|
|
888
984
|
}
|
|
889
|
-
async function searchNpmPackage(packageName, timeoutMs) {
|
|
985
|
+
async function searchNpmPackage(packageName, timeoutMs, quoted = []) {
|
|
890
986
|
const result = await fetchText(`https://registry.npmjs.org/${encodeURIComponent(packageName)}`, timeoutMs, {
|
|
891
987
|
headers: {
|
|
892
988
|
accept: "application/json,text/plain,*/*",
|
|
@@ -898,20 +994,29 @@ async function searchNpmPackage(packageName, timeoutMs) {
|
|
|
898
994
|
try {
|
|
899
995
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
900
996
|
const record = data;
|
|
901
|
-
|
|
997
|
+
// The requested URL is not evidence that a successful response describes
|
|
998
|
+
// that package. An empty or mismatched registry payload must not invent it.
|
|
999
|
+
if (typeof record.name !== "string" || record.name !== packageName) {
|
|
1000
|
+
return failedProviderSearch("npm package metadata did not identify the requested package");
|
|
1001
|
+
}
|
|
1002
|
+
const name = record.name;
|
|
1003
|
+
const description = typeof record.description === "string" ? record.description : "";
|
|
1004
|
+
if (quoted.length && !quoted.every(phrase => name.toLocaleLowerCase().includes(phrase) || description.toLocaleLowerCase().includes(phrase))) {
|
|
1005
|
+
return failedProviderSearch("npm package metadata has no evidence for the quoted phrase; continuing to full-query search");
|
|
1006
|
+
}
|
|
902
1007
|
const latest = String(record["dist-tags"]?.latest || record.version || "").trim();
|
|
903
1008
|
const maintainer = Array.isArray(record.maintainers) ? String(record.maintainers[0]?.name || "").trim() : "";
|
|
904
1009
|
const results = uniqueResults([{
|
|
905
1010
|
title: name,
|
|
906
1011
|
url: `https://www.npmjs.com/package/${name}`,
|
|
907
1012
|
snippet: [
|
|
908
|
-
|
|
1013
|
+
description.trim(),
|
|
909
1014
|
latest ? `v${latest}` : "",
|
|
910
1015
|
maintainer ? `by ${maintainer}` : "",
|
|
911
1016
|
].filter(Boolean).join(" · "),
|
|
912
1017
|
engine: "npm",
|
|
913
1018
|
}]);
|
|
914
|
-
return completedProviderSearch(results, "");
|
|
1019
|
+
return completedProviderSearch(results, quoted.length ? "npm quoted phrase matched package metadata only; original body remains unverified" : "");
|
|
915
1020
|
}
|
|
916
1021
|
catch (error) {
|
|
917
1022
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -930,7 +1035,9 @@ async function searchNpm(query, numResults, timeoutMs) {
|
|
|
930
1035
|
return failedProviderSearch(result.error || `npm error ${result.status}`);
|
|
931
1036
|
try {
|
|
932
1037
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
933
|
-
|
|
1038
|
+
if (!Array.isArray(data?.objects))
|
|
1039
|
+
return failedProviderSearch("npm search returned an invalid objects payload");
|
|
1040
|
+
const raw = data.objects;
|
|
934
1041
|
const results = uniqueResults(raw.map((item) => {
|
|
935
1042
|
const pkg = item?.package || {};
|
|
936
1043
|
const name = String(pkg?.name || "").trim();
|
|
@@ -972,7 +1079,9 @@ async function searchWikipedia(query, numResults, timeoutMs) {
|
|
|
972
1079
|
return failedProviderSearch(result.error || `Wikipedia error ${result.status}`);
|
|
973
1080
|
try {
|
|
974
1081
|
const data = result.text ? JSON.parse(result.text) : {};
|
|
975
|
-
|
|
1082
|
+
if (!Array.isArray(data?.query?.search))
|
|
1083
|
+
return failedProviderSearch("Wikipedia returned an invalid search payload");
|
|
1084
|
+
const raw = data.query.search;
|
|
976
1085
|
const results = uniqueResults(raw.map((item) => {
|
|
977
1086
|
const title = String(item?.title || "").trim();
|
|
978
1087
|
const pageId = item?.pageid;
|
|
@@ -984,7 +1093,20 @@ async function searchWikipedia(query, numResults, timeoutMs) {
|
|
|
984
1093
|
engine: "wikipedia",
|
|
985
1094
|
};
|
|
986
1095
|
}).filter((item) => item.url)).slice(0, numResults);
|
|
987
|
-
|
|
1096
|
+
// Native API ordering can fill a Korean two-term query with articles whose
|
|
1097
|
+
// excerpt only contains one incidental word (e.g. 기계식 계산기 for a keyboard
|
|
1098
|
+
// query). Do not hide all leads if none has enough visible evidence.
|
|
1099
|
+
const terms = wikiLang === "ko" ? koreanDiscoveryTerms(query) : [];
|
|
1100
|
+
const topical = terms.length >= 2 ? results.filter(row => discoveryEvidenceScore(row, terms) >= 2) : results;
|
|
1101
|
+
const phrase = terms.join("");
|
|
1102
|
+
const folded = (value) => value.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "");
|
|
1103
|
+
const phraseScore = (row) => !phrase ? 0
|
|
1104
|
+
: folded(row.title).includes(phrase) ? 2 : folded(row.snippet).includes(phrase) ? 1 : 0;
|
|
1105
|
+
const selected = topical.length ? topical.sort((a, b) => phraseScore(b) - phraseScore(a)) : results;
|
|
1106
|
+
const note = selected.length < results.length
|
|
1107
|
+
? `Excluded ${results.length - selected.length} Wikipedia result(s) without both query terms in title/snippet`
|
|
1108
|
+
: selected.length ? "" : "Wikipedia search returned no results";
|
|
1109
|
+
return completedProviderSearch(selected, note);
|
|
988
1110
|
}
|
|
989
1111
|
catch (error) {
|
|
990
1112
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1002,19 +1124,7 @@ async function searchDuckDuckGo(query, numResults, timeoutMs) {
|
|
|
1002
1124
|
if (/anomaly\.js|sending automated queries|captcha/i.test(result.text)) {
|
|
1003
1125
|
return failedProviderSearch("DuckDuckGo blocked the automated HTML fallback");
|
|
1004
1126
|
}
|
|
1005
|
-
const
|
|
1006
|
-
const results = uniqueResults(blocks.map((block) => {
|
|
1007
|
-
const anchor = block.match(/<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
1008
|
-
const snippet = block.match(/<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/i)
|
|
1009
|
-
|| block.match(/<div[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/div>/i);
|
|
1010
|
-
const rawUrl = anchor?.[1] || "";
|
|
1011
|
-
return {
|
|
1012
|
-
title: stripTags(anchor?.[2] || ""),
|
|
1013
|
-
url: resolveDuckDuckGoUrl(rawUrl),
|
|
1014
|
-
snippet: stripTags(snippet?.[1] || ""),
|
|
1015
|
-
engine: "duckduckgo",
|
|
1016
|
-
};
|
|
1017
|
-
}).filter((item) => item.url)).slice(0, numResults);
|
|
1127
|
+
const results = parseDuckDuckGoSearch(result.text);
|
|
1018
1128
|
return completedProviderSearch(results, results.length > 0 ? "" : "DuckDuckGo returned no results");
|
|
1019
1129
|
}
|
|
1020
1130
|
async function searchBraveHtml(query, numResults, timeoutMs) {
|
|
@@ -1026,19 +1136,14 @@ async function searchBraveHtml(query, numResults, timeoutMs) {
|
|
|
1026
1136
|
});
|
|
1027
1137
|
if (!result.ok)
|
|
1028
1138
|
return failedProviderSearch(result.error || `Brave HTML error ${result.status}`);
|
|
1029
|
-
const
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
engine: "brave_html",
|
|
1037
|
-
});
|
|
1038
|
-
if (results.length >= numResults)
|
|
1039
|
-
break;
|
|
1139
|
+
const unique = parseBraveSearch(result.text);
|
|
1140
|
+
if (unique.length === 0 && result.text.trim()) {
|
|
1141
|
+
const { document } = parseHTML(result.text);
|
|
1142
|
+
const visible = `${document.title} ${document.body?.textContent || ""}`.slice(0, 10000);
|
|
1143
|
+
if (/verify you are human|captcha|unusual traffic|automated quer(?:y|ies)|비정상적인 접근|자동입력 방지/i.test(visible)) {
|
|
1144
|
+
return failedProviderSearch("Brave HTML stopped at a challenge");
|
|
1145
|
+
}
|
|
1040
1146
|
}
|
|
1041
|
-
const unique = uniqueResults(results);
|
|
1042
1147
|
return completedProviderSearch(unique, unique.length > 0 ? "" : "Brave HTML returned no results");
|
|
1043
1148
|
}
|
|
1044
1149
|
async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
|
|
@@ -1052,14 +1157,17 @@ async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
|
|
|
1052
1157
|
if (!result.ok)
|
|
1053
1158
|
return failedProviderSearch(result.error || `SearXNG error ${result.status}`);
|
|
1054
1159
|
try {
|
|
1055
|
-
const data =
|
|
1056
|
-
|
|
1160
|
+
const data = JSON.parse(result.text);
|
|
1161
|
+
if (!data || typeof data !== "object" || !Array.isArray(data.results)) {
|
|
1162
|
+
return failedProviderSearch("SearXNG invalid result envelope: expected a results array");
|
|
1163
|
+
}
|
|
1164
|
+
const raw = data.results;
|
|
1057
1165
|
const results = uniqueResults(raw.map((item) => ({
|
|
1058
1166
|
title: String(item?.title || item?.url || "").trim(),
|
|
1059
1167
|
url: String(item?.url || "").trim(),
|
|
1060
1168
|
snippet: String(item?.content || item?.description || item?.text || "").trim(),
|
|
1061
1169
|
engine: "searxng",
|
|
1062
|
-
}))).slice(0, numResults);
|
|
1170
|
+
}))).slice(0, Math.max(100, numResults));
|
|
1063
1171
|
return completedProviderSearch(results, results.length > 0 ? "" : "SearXNG returned no results");
|
|
1064
1172
|
}
|
|
1065
1173
|
catch (error) {
|
|
@@ -1068,15 +1176,56 @@ async function searchSearxng(baseUrl, query, numResults, timeoutMs) {
|
|
|
1068
1176
|
}
|
|
1069
1177
|
}
|
|
1070
1178
|
export async function runSmartsearch(options, runtime = {}) {
|
|
1179
|
+
const startedAt = Date.now();
|
|
1071
1180
|
const query = options.query.trim();
|
|
1072
1181
|
const numResults = Math.max(1, Math.min(20, Math.floor(Number(options.numResults || 8))));
|
|
1073
1182
|
const searchType = options.searchType || "auto";
|
|
1074
1183
|
const contextMaxCharacters = Math.max(1000, Math.floor(Number(options.contextMaxCharacters || 10000)));
|
|
1075
1184
|
const timeoutMs = Math.max(1000, Math.floor(Number(options.timeoutMs || DEFAULT_TIMEOUT_MS)));
|
|
1185
|
+
const cdpUrl = configText("fetch.browserCdpUrl");
|
|
1186
|
+
const browserEnabled = Boolean(cdpUrl && /[가-힣a-z]/iu.test(stripSiteOperators(query)) && !localOnlyMode() && !privateProfileMode() && configEnabled("search.enableBrowserFallback", true));
|
|
1187
|
+
const attemptTimeout = () => Math.max(1, Math.floor((timeoutMs - (Date.now() - startedAt)) / (browserEnabled ? 4 : 1)));
|
|
1076
1188
|
const searxngBaseUrl = (options.searxngBaseUrl || configText("search.searxngBaseUrl") || "").trim();
|
|
1077
1189
|
const fallbacks = [];
|
|
1078
1190
|
const notes = [];
|
|
1079
1191
|
const attempts = [];
|
|
1192
|
+
// A result outside an explicit site/path is not a useful provider success.
|
|
1193
|
+
// Multiple positive site operators are left to the general engine; native
|
|
1194
|
+
// first-site routing cannot preserve their Boolean semantics.
|
|
1195
|
+
const tokens = searchQueryTokens(query);
|
|
1196
|
+
const siteMatches = tokens.filter((token) => /^site:[^"']+$/i.test(token));
|
|
1197
|
+
const scope = siteMatches.length === 1 ? extractSiteScope(siteMatches[0]) : null;
|
|
1198
|
+
const unscopedDocsTerms = !siteMatches.length ? unscopedPythonDocsTerms(query) : undefined;
|
|
1199
|
+
const pythonSearchUrl = browserEnabled ? pythonDocsSearchUrl(unscopedDocsTerms ? { host: "docs.python.org", path: "3" } : scope, unscopedDocsTerms || tokens.filter(token => token !== siteMatches[0]).join(" ")) : undefined;
|
|
1200
|
+
const nativeYoutubeUrl = browserEnabled ? youtubeSearchUrl(scope, tokens.filter(token => token !== siteMatches[0]).join(" ")) : undefined;
|
|
1201
|
+
const hnDates = scope?.host === "news.ycombinator.com" ? parseHnDateQuery(query) : undefined;
|
|
1202
|
+
const nativeCandidateLimit = scope ? 20 : numResults;
|
|
1203
|
+
const constrain = (outcome, limit = numResults) => {
|
|
1204
|
+
if (hnDates?.bounded && outcome.status === "succeeded" && !outcome.hnDateBoundsVerified)
|
|
1205
|
+
return failedProviderSearch("HN date constraints require verified native publication timestamps; undated search snippets were not accepted");
|
|
1206
|
+
if (!outcome.ok)
|
|
1207
|
+
return outcome;
|
|
1208
|
+
const originals = outcome.results.filter((item) => !isSearchTrackingRelay(item));
|
|
1209
|
+
if (originals.length !== outcome.results.length) {
|
|
1210
|
+
notes.push(`Excluded ${outcome.results.length - originals.length} outbound tracking relay(s); discovery requires original pages`);
|
|
1211
|
+
outcome = completedProviderSearch(originals, outcome.note);
|
|
1212
|
+
if (!originals.length)
|
|
1213
|
+
return failedProviderSearch("Search returned tracking relays without original results");
|
|
1214
|
+
}
|
|
1215
|
+
if (!scope)
|
|
1216
|
+
return { ...outcome, results: outcome.results.slice(0, limit) };
|
|
1217
|
+
const scoped = outcome.results.filter((item) => searchResultMatchesSite(item.url, scope.host, scope.path));
|
|
1218
|
+
if (scoped.length !== outcome.results.length)
|
|
1219
|
+
notes.push(`Filtered ${outcome.results.length - scoped.length} result(s) outside site:${scope.host}${scope.path ? `/${scope.path}` : ""}`);
|
|
1220
|
+
const wikiArticleIntent = scope.host === "github.com" && /(?:^|\/)wiki(?:\/|$)/i.test(scope.path)
|
|
1221
|
+
&& !/\b(?:history|revisions?|edit)\b/i.test(stripSiteOperators(query));
|
|
1222
|
+
const results = wikiArticleIntent
|
|
1223
|
+
? scoped.filter((item) => !/\/wiki\/(?:[^?#]+\/)?_(?:history|edit)(?:[/?#]|$)/i.test(new URL(item.url).pathname))
|
|
1224
|
+
: scoped;
|
|
1225
|
+
if (results.length !== scoped.length)
|
|
1226
|
+
notes.push(`Excluded ${scoped.length - results.length} GitHub wiki history/edit page(s); original articles were requested`);
|
|
1227
|
+
return results.length ? completedProviderSearch(results.slice(0, limit), outcome.note) : failedProviderSearch("No results satisfied the explicit site constraint");
|
|
1228
|
+
};
|
|
1080
1229
|
const recordAttempt = (provider, outcome) => {
|
|
1081
1230
|
attempts.push({
|
|
1082
1231
|
provider,
|
|
@@ -1085,15 +1234,122 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1085
1234
|
...(outcome.status === "failed" && outcome.note ? { error: outcome.note } : {}),
|
|
1086
1235
|
});
|
|
1087
1236
|
};
|
|
1237
|
+
if (hnDates?.error)
|
|
1238
|
+
return buildSearchResponse({ engine: "error", query, results: [], fallbacks_tried: [], notes: [hnDates.error], execution: { status: "failed", attempts: [{ provider: "query_constraints", status: "failed", result_count: 0, error: hnDates.error }] } });
|
|
1239
|
+
const discoveryTerms = koreanDiscoveryTerms(query);
|
|
1240
|
+
let discoveryResults = [];
|
|
1241
|
+
const collectDiscovery = (outcome) => {
|
|
1242
|
+
if (!outcome.ok)
|
|
1243
|
+
return false;
|
|
1244
|
+
discoveryResults = rankDiscoveryResults([...discoveryResults, ...outcome.results], discoveryTerms);
|
|
1245
|
+
if (hasDiscoveryYield(discoveryResults, discoveryTerms, numResults))
|
|
1246
|
+
return true;
|
|
1247
|
+
notes.push("Low discovery yield in title/snippet evidence; continuing within the existing search budget. Unmatched originals are retained, not declared irrelevant.");
|
|
1248
|
+
return false;
|
|
1249
|
+
};
|
|
1250
|
+
const finishDiscovery = () => {
|
|
1251
|
+
const results = discoveryResults.slice(0, numResults);
|
|
1252
|
+
const engines = new Set(results.map((row) => row.engine));
|
|
1253
|
+
if (!hasDiscoveryYield(results, discoveryTerms, numResults))
|
|
1254
|
+
notes.push("Limited topical evidence in discovery metadata; original-page relevance remains unverified");
|
|
1255
|
+
return maybeRunCrawlHandoff({ engine: engines.size > 1 ? "mixed" : results[0]?.engine || "none", query, results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1256
|
+
};
|
|
1257
|
+
const budgetExhausted = () => {
|
|
1258
|
+
if (Date.now() - startedAt < timeoutMs)
|
|
1259
|
+
return false;
|
|
1260
|
+
notes.push("Search budget exhausted; no further discovery engine was started");
|
|
1261
|
+
return true;
|
|
1262
|
+
};
|
|
1088
1263
|
try {
|
|
1089
|
-
|
|
1264
|
+
// A full site:path can itself be the requested original. Check its authored
|
|
1265
|
+
// body before relying on search indexes that only expose titles/summaries.
|
|
1266
|
+
// Only quote-only queries are safe here: extra terms/operators need a real
|
|
1267
|
+
// search engine rather than being silently discarded.
|
|
1268
|
+
const remainder = tokens.filter(token => !/^site:[^"']+$/i.test(token));
|
|
1269
|
+
const quoted = remainder.filter(token => /^"[^"\\]+"$/.test(token)).map(token => token.slice(1, -1));
|
|
1270
|
+
if (scope?.path.includes("/") && !siteNativeFallbackBlockedReason("Scoped original") && !parseNpmPackageScope(query)
|
|
1271
|
+
&& quoted.length > 0 && quoted.length === remainder.length && !budgetExhausted()) {
|
|
1272
|
+
const url = siteScopeToStartUrl(scope);
|
|
1273
|
+
const validation = await validateOutboundUrl(url);
|
|
1274
|
+
if (validation.ok && !budgetExhausted()) {
|
|
1275
|
+
const original = await fetchText(url, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
|
|
1276
|
+
if (original.ok && !budgetExhausted()) {
|
|
1277
|
+
const document = parseHTML(original.text).document;
|
|
1278
|
+
const article = document.querySelector("main article") || document.querySelector("main") || document.querySelector("article");
|
|
1279
|
+
if (article) {
|
|
1280
|
+
for (const node of article.querySelectorAll("script, style, nav, aside, header, footer, [hidden], [inert], [aria-hidden='true'], [style*='display:none'], [style*='display: none']"))
|
|
1281
|
+
node.remove();
|
|
1282
|
+
const body = (article.textContent || "").replace(/\s+/g, " ").trim();
|
|
1283
|
+
const heading = (document.title || article.querySelector("h1")?.textContent || "").replace(/\s+/g, " ").trim();
|
|
1284
|
+
const lower = body.toLocaleLowerCase();
|
|
1285
|
+
if (body.length >= 40 && heading.length >= 3 && quoted.every(phrase => lower.includes(phrase.toLocaleLowerCase()))
|
|
1286
|
+
&& !detectInteractiveLikeText(`${heading} ${body.slice(0, 300)}`)) {
|
|
1287
|
+
const position = lower.indexOf(quoted[0].toLocaleLowerCase());
|
|
1288
|
+
const snippet = body.slice(Math.max(0, position - 90), position + quoted[0].length + 110);
|
|
1289
|
+
const outcome = constrain(completedProviderSearch([{ title: heading, url, snippet, engine: "scoped_original" }], ""));
|
|
1290
|
+
if (outcome.ok) {
|
|
1291
|
+
fallbacks.push("scoped_original");
|
|
1292
|
+
recordAttempt("scoped_original", outcome);
|
|
1293
|
+
return maybeRunCrawlHandoff({ engine: "scoped_original", query, results: outcome.results, fallbacks_tried: fallbacks, notes: ["Quoted phrase verified in the exact scoped original body"], execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
const nodeIntent = !siteMatches.length ? nodeDocsIntent(query) : undefined;
|
|
1301
|
+
if (nodeIntent && !budgetExhausted()) {
|
|
1302
|
+
fallbacks.push("node_docs");
|
|
1303
|
+
const original = await fetchText(nodeIntent.url, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
|
|
1304
|
+
const result = original.ok && !budgetExhausted() ? nodeDocsOriginal(original.text, nodeIntent.url, nodeIntent.symbol) : undefined;
|
|
1305
|
+
const outcome = result ? completedProviderSearch([result], "") : failedProviderSearch(original.ok
|
|
1306
|
+
? "Node.js 24 official filesystem original did not verify the requested symbol"
|
|
1307
|
+
: `Node.js 24 official original unavailable: ${original.error || original.status}`);
|
|
1308
|
+
recordAttempt("node_docs", outcome);
|
|
1309
|
+
if (result) {
|
|
1310
|
+
notes.push("Official Node.js 24 documentation section verified; complete page extraction remains a separate read");
|
|
1311
|
+
collectDiscovery(outcome);
|
|
1312
|
+
return finishDiscovery();
|
|
1313
|
+
}
|
|
1314
|
+
notes.push(outcome.note);
|
|
1315
|
+
}
|
|
1316
|
+
const postgresIntent = !siteMatches.length ? postgresqlDocsIntent(query) : undefined;
|
|
1317
|
+
if (postgresIntent && !budgetExhausted()) {
|
|
1318
|
+
fallbacks.push("postgresql_docs");
|
|
1319
|
+
const searchUrl = new URL("https://www.postgresql.org/search/");
|
|
1320
|
+
searchUrl.searchParams.set("q", `${postgresIntent.version} ${postgresIntent.symbol}`);
|
|
1321
|
+
const index = await fetchText(searchUrl.href, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
|
|
1322
|
+
let verified;
|
|
1323
|
+
if (index.ok && !budgetExhausted()) {
|
|
1324
|
+
for (const candidate of postgresqlDocsCandidates(index.text, postgresIntent.version)) {
|
|
1325
|
+
if (budgetExhausted())
|
|
1326
|
+
break;
|
|
1327
|
+
const original = await fetchText(candidate.url, attemptTimeout(), { redirect: "error", headers: { "user-agent": DEFAULT_USER_AGENT } });
|
|
1328
|
+
if (original.ok && postgresqlDocsBodyHasSymbol(original.text, postgresIntent.symbol) && !budgetExhausted()) {
|
|
1329
|
+
verified = candidate;
|
|
1330
|
+
break;
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
const outcome = verified ? completedProviderSearch([verified], "") : failedProviderSearch(index.ok
|
|
1335
|
+
? "PostgreSQL search did not verify the requested symbol in a version-matched original"
|
|
1336
|
+
: `PostgreSQL official search unavailable: ${index.error || index.status}`);
|
|
1337
|
+
recordAttempt("postgresql_docs", outcome);
|
|
1338
|
+
if (verified) {
|
|
1339
|
+
notes.push("Official versioned documentation original contains the requested symbol; complete page extraction remains a separate read");
|
|
1340
|
+
collectDiscovery(outcome);
|
|
1341
|
+
return finishDiscovery();
|
|
1342
|
+
}
|
|
1343
|
+
notes.push(outcome.note);
|
|
1344
|
+
}
|
|
1345
|
+
const siteOperator = siteMatches.length !== 1 || tokens.some(token => token === "(" || token === ")") ? "" : extractSiteOperator(siteMatches[0]);
|
|
1090
1346
|
if (siteOperator === "velog.io") {
|
|
1091
1347
|
const blockedReason = siteNativeFallbackBlockedReason("Velog");
|
|
1092
1348
|
if (blockedReason) {
|
|
1093
1349
|
notes.push(blockedReason);
|
|
1094
1350
|
}
|
|
1095
1351
|
else {
|
|
1096
|
-
const velog = await searchVelog(query,
|
|
1352
|
+
const velog = constrain(await searchVelog(query, nativeCandidateLimit, attemptTimeout()));
|
|
1097
1353
|
fallbacks.push("velog");
|
|
1098
1354
|
recordAttempt("velog", velog);
|
|
1099
1355
|
if (velog.ok)
|
|
@@ -1108,7 +1364,7 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1108
1364
|
notes.push(blockedReason);
|
|
1109
1365
|
}
|
|
1110
1366
|
else {
|
|
1111
|
-
const stackexchange = await searchStackExchange(query,
|
|
1367
|
+
const stackexchange = constrain(await searchStackExchange(query, nativeCandidateLimit, attemptTimeout()));
|
|
1112
1368
|
fallbacks.push("stackexchange");
|
|
1113
1369
|
recordAttempt("stackexchange", stackexchange);
|
|
1114
1370
|
if (stackexchange.ok)
|
|
@@ -1123,11 +1379,11 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1123
1379
|
notes.push(blockedReason);
|
|
1124
1380
|
}
|
|
1125
1381
|
else {
|
|
1126
|
-
const hackernews = await searchHackerNews(query,
|
|
1382
|
+
const hackernews = constrain(await searchHackerNews(query, nativeCandidateLimit, attemptTimeout(), hnDates));
|
|
1127
1383
|
fallbacks.push("hackernews");
|
|
1128
1384
|
recordAttempt("hackernews", hackernews);
|
|
1129
|
-
if (hackernews.ok)
|
|
1130
|
-
return maybeRunCrawlHandoff({ engine: "hackernews", query, results: hackernews.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1385
|
+
if (hackernews.ok || (hnDates?.bounded && hackernews.status === "succeeded"))
|
|
1386
|
+
return maybeRunCrawlHandoff({ engine: "hackernews", query, results: hackernews.results, fallbacks_tried: fallbacks, notes: hackernews.note ? [...notes, hackernews.note] : notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1131
1387
|
if (hackernews.note)
|
|
1132
1388
|
notes.push(hackernews.note);
|
|
1133
1389
|
}
|
|
@@ -1138,7 +1394,9 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1138
1394
|
notes.push(blockedReason);
|
|
1139
1395
|
}
|
|
1140
1396
|
else {
|
|
1141
|
-
|
|
1397
|
+
// Reddit already preserves subreddit scope while filling this count;
|
|
1398
|
+
// oversampling here would trigger unnecessary secondary-provider reads.
|
|
1399
|
+
const reddit = constrain(await searchReddit(query, numResults, attemptTimeout()));
|
|
1142
1400
|
fallbacks.push("reddit");
|
|
1143
1401
|
recordAttempt("reddit", reddit);
|
|
1144
1402
|
if (reddit.ok)
|
|
@@ -1149,11 +1407,15 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1149
1407
|
}
|
|
1150
1408
|
if (siteOperator === "github.com") {
|
|
1151
1409
|
const blockedReason = siteNativeFallbackBlockedReason("GitHub");
|
|
1410
|
+
const repoScope = parseGitHubRepoScope(query);
|
|
1152
1411
|
if (blockedReason) {
|
|
1153
1412
|
notes.push(blockedReason);
|
|
1154
1413
|
}
|
|
1155
|
-
else if (
|
|
1156
|
-
|
|
1414
|
+
else if (repoScope?.scopePath && !repoScope.scopePath.startsWith("blob/")) {
|
|
1415
|
+
notes.push("GitHub repository tree lookup cannot search this URL section; preserving the full query for general search");
|
|
1416
|
+
}
|
|
1417
|
+
else if (repoScope) {
|
|
1418
|
+
const githubRepoDocs = constrain(await searchGitHubRepoDocs(query, nativeCandidateLimit, attemptTimeout()));
|
|
1157
1419
|
fallbacks.push("github_repo_docs");
|
|
1158
1420
|
recordAttempt("github_repo_docs", githubRepoDocs);
|
|
1159
1421
|
if (githubRepoDocs.ok)
|
|
@@ -1165,7 +1427,7 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1165
1427
|
notes.push("GitHub native repo fallback skipped for non-repository intent");
|
|
1166
1428
|
}
|
|
1167
1429
|
else {
|
|
1168
|
-
const github = await searchGitHub(query,
|
|
1430
|
+
const github = constrain(await searchGitHub(query, nativeCandidateLimit, attemptTimeout()));
|
|
1169
1431
|
fallbacks.push("github");
|
|
1170
1432
|
recordAttempt("github", github);
|
|
1171
1433
|
if (github.ok)
|
|
@@ -1176,16 +1438,27 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1176
1438
|
}
|
|
1177
1439
|
if (siteOperator === "npmjs.com") {
|
|
1178
1440
|
const blockedReason = siteNativeFallbackBlockedReason("npm");
|
|
1441
|
+
const exactPackage = parseNpmPackageScope(query);
|
|
1442
|
+
const remainder = searchQueryTokens(query).filter(token => !/^site:[^"']+$/i.test(token));
|
|
1443
|
+
const quoted = remainder.filter(token => token.startsWith('"') && token.endsWith('"') && token.length > 2 && !token.includes("\\"))
|
|
1444
|
+
.map(token => token.slice(1, -1).toLocaleLowerCase());
|
|
1445
|
+
const nativeExact = Boolean(exactPackage && (remainder.length === 0 || (quoted.length === remainder.length && quoted.length > 0)));
|
|
1179
1446
|
if (blockedReason) {
|
|
1180
1447
|
notes.push(blockedReason);
|
|
1181
1448
|
}
|
|
1449
|
+
else if (exactPackage && !nativeExact) {
|
|
1450
|
+
// Metadata cannot evaluate free words, exclusions, dates or Boolean
|
|
1451
|
+
// expressions. Only standalone quotes can be checked in its own fields.
|
|
1452
|
+
notes.push("npm package metadata cannot evaluate the remaining query constraints; preserving the full query for general search");
|
|
1453
|
+
}
|
|
1182
1454
|
else {
|
|
1183
|
-
const
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
: await searchNpm(query, numResults, timeoutMs);
|
|
1455
|
+
const npm = constrain(exactPackage
|
|
1456
|
+
? await searchNpmPackage(exactPackage, attemptTimeout(), quoted)
|
|
1457
|
+
: await searchNpm(query, nativeCandidateLimit, attemptTimeout()));
|
|
1187
1458
|
fallbacks.push("npm");
|
|
1188
1459
|
recordAttempt("npm", npm);
|
|
1460
|
+
if (npm.ok && npm.note)
|
|
1461
|
+
notes.push(npm.note);
|
|
1189
1462
|
if (npm.ok)
|
|
1190
1463
|
return maybeRunCrawlHandoff({ engine: "npm", query, results: npm.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1191
1464
|
if (npm.note)
|
|
@@ -1198,9 +1471,11 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1198
1471
|
notes.push(blockedReason);
|
|
1199
1472
|
}
|
|
1200
1473
|
else {
|
|
1201
|
-
const mdn = await searchMdn(query,
|
|
1474
|
+
const mdn = constrain(await searchMdn(query, nativeCandidateLimit, attemptTimeout()));
|
|
1202
1475
|
fallbacks.push("mdn");
|
|
1203
1476
|
recordAttempt("mdn", mdn);
|
|
1477
|
+
if (mdn.ok && mdn.note)
|
|
1478
|
+
notes.push(mdn.note);
|
|
1204
1479
|
if (mdn.ok)
|
|
1205
1480
|
return maybeRunCrawlHandoff({ engine: "mdn", query, results: mdn.results, fallbacks_tried: fallbacks, notes, execution: smartsearchExecution(attempts) }, options, runtime);
|
|
1206
1481
|
if (mdn.note)
|
|
@@ -1213,7 +1488,7 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1213
1488
|
notes.push(blockedReason);
|
|
1214
1489
|
}
|
|
1215
1490
|
else {
|
|
1216
|
-
const crates = await searchCrates(query,
|
|
1491
|
+
const crates = constrain(await searchCrates(query, nativeCandidateLimit, attemptTimeout()));
|
|
1217
1492
|
fallbacks.push("crates");
|
|
1218
1493
|
recordAttempt("crates", crates);
|
|
1219
1494
|
if (crates.ok)
|
|
@@ -1228,7 +1503,7 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1228
1503
|
notes.push(blockedReason);
|
|
1229
1504
|
}
|
|
1230
1505
|
else {
|
|
1231
|
-
const wikipedia = await searchWikipedia(query,
|
|
1506
|
+
const wikipedia = constrain(await searchWikipedia(query, nativeCandidateLimit, attemptTimeout()));
|
|
1232
1507
|
fallbacks.push("wikipedia");
|
|
1233
1508
|
recordAttempt("wikipedia", wikipedia);
|
|
1234
1509
|
if (wikipedia.ok)
|
|
@@ -1237,17 +1512,19 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1237
1512
|
notes.push(wikipedia.note);
|
|
1238
1513
|
}
|
|
1239
1514
|
}
|
|
1515
|
+
if (budgetExhausted())
|
|
1516
|
+
return finishDiscovery();
|
|
1240
1517
|
if (searxngEnabled() && searxngBaseUrl) {
|
|
1241
1518
|
const privacySensitive = localOnlyMode() || privateProfileMode();
|
|
1242
1519
|
if (privacySensitive && !await privacySafeSearxngBase(searxngBaseUrl)) {
|
|
1243
1520
|
notes.push("SearXNG disabled by privacy mode because the configured base URL is not local or private");
|
|
1244
1521
|
}
|
|
1245
1522
|
else {
|
|
1246
|
-
const searxng = await searchSearxng(searxngBaseUrl, query, numResults,
|
|
1523
|
+
const searxng = constrain(await searchSearxng(searxngBaseUrl, query, numResults, attemptTimeout()), 100);
|
|
1247
1524
|
fallbacks.push("searxng");
|
|
1248
1525
|
recordAttempt("searxng", searxng);
|
|
1249
|
-
if (searxng
|
|
1250
|
-
return
|
|
1526
|
+
if (collectDiscovery(searxng))
|
|
1527
|
+
return finishDiscovery();
|
|
1251
1528
|
if (searxng.note)
|
|
1252
1529
|
notes.push(searxng.note);
|
|
1253
1530
|
}
|
|
@@ -1255,31 +1532,179 @@ export async function runSmartsearch(options, runtime = {}) {
|
|
|
1255
1532
|
else if (!searxngEnabled()) {
|
|
1256
1533
|
notes.push("SearXNG disabled by configuration");
|
|
1257
1534
|
}
|
|
1535
|
+
if (budgetExhausted())
|
|
1536
|
+
return finishDiscovery();
|
|
1258
1537
|
if (duckduckgoEnabled()) {
|
|
1259
|
-
const ddg = await searchDuckDuckGo(query, numResults,
|
|
1538
|
+
const ddg = constrain(await searchDuckDuckGo(query, numResults, attemptTimeout()), 100);
|
|
1260
1539
|
fallbacks.push("duckduckgo");
|
|
1261
1540
|
recordAttempt("duckduckgo", ddg);
|
|
1262
|
-
if (ddg
|
|
1263
|
-
return
|
|
1541
|
+
if (collectDiscovery(ddg))
|
|
1542
|
+
return finishDiscovery();
|
|
1264
1543
|
if (ddg.note)
|
|
1265
1544
|
notes.push(ddg.note);
|
|
1266
1545
|
}
|
|
1267
1546
|
else {
|
|
1268
1547
|
notes.push("DuckDuckGo disabled by configuration");
|
|
1269
1548
|
}
|
|
1549
|
+
if (budgetExhausted())
|
|
1550
|
+
return finishDiscovery();
|
|
1270
1551
|
if (braveHtmlEnabled()) {
|
|
1271
|
-
const braveHtml = await searchBraveHtml(query, numResults,
|
|
1552
|
+
const braveHtml = constrain(await searchBraveHtml(query, numResults, attemptTimeout()), 100);
|
|
1272
1553
|
fallbacks.push("brave_html");
|
|
1273
1554
|
recordAttempt("brave_html", braveHtml);
|
|
1274
|
-
if (braveHtml
|
|
1275
|
-
return
|
|
1555
|
+
if (collectDiscovery(braveHtml))
|
|
1556
|
+
return finishDiscovery();
|
|
1276
1557
|
if (braveHtml.note)
|
|
1277
1558
|
notes.push(braveHtml.note);
|
|
1278
1559
|
}
|
|
1279
1560
|
else {
|
|
1280
1561
|
notes.push("Brave HTML disabled by configuration");
|
|
1281
1562
|
}
|
|
1282
|
-
|
|
1563
|
+
const remaining = timeoutMs - (Date.now() - startedAt);
|
|
1564
|
+
let canContinueNative = false;
|
|
1565
|
+
if (browserEnabled) {
|
|
1566
|
+
fallbacks.push("naver_browser");
|
|
1567
|
+
let outcome;
|
|
1568
|
+
if (remaining <= 0)
|
|
1569
|
+
outcome = failedProviderSearch("Search budget exhausted before operator-browser continuation");
|
|
1570
|
+
else {
|
|
1571
|
+
try {
|
|
1572
|
+
// Native search sometimes treats a site path as text. A second,
|
|
1573
|
+
// host-only discovery query may recover the same originals; the
|
|
1574
|
+
// original path remains a hard result constraint below.
|
|
1575
|
+
const searchQueries = scope?.path && siteMatches.length === 1 && /[가-힣]/u.test(query)
|
|
1576
|
+
&& ["clien.net", "dcinside.com", "ppomppu.co.kr"].includes(scope.host)
|
|
1577
|
+
? [query, query.replace(siteMatches[0], `site:${scope.host}`)] : [query];
|
|
1578
|
+
outcome = failedProviderSearch("Browser search exposed no verified result headlines satisfying supported query constraints");
|
|
1579
|
+
for (const [index, searchQuery] of searchQueries.entries()) {
|
|
1580
|
+
const budget = timeoutMs - (Date.now() - startedAt);
|
|
1581
|
+
if (budget <= 0)
|
|
1582
|
+
break;
|
|
1583
|
+
const url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}`;
|
|
1584
|
+
const page = await (runtime.runBrowserSession || runBrowserSession)({ url, cdpUrl, timeoutMs: Math.max(1, Math.floor(budget / ((pythonSearchUrl || nativeYoutubeUrl ? 1 : 0) + searchQueries.length - index))), allowPrivateHosts: false });
|
|
1585
|
+
if (!page.ok) {
|
|
1586
|
+
outcome = failedProviderSearch(`Approved browser: ${page.error.code}: ${page.error.message}`);
|
|
1587
|
+
break;
|
|
1588
|
+
}
|
|
1589
|
+
const final = new URL(page.final_url);
|
|
1590
|
+
const sameQuery = final.origin === "https://search.naver.com" && !final.username && !final.password
|
|
1591
|
+
&& final.pathname === "/search.naver" && final.searchParams.getAll("query").length === 1 && final.searchParams.get("query") === searchQuery;
|
|
1592
|
+
if (!sameQuery || /captcha|verify you are human|unusual traffic|자동입력 방지|비정상적인 접근/i.test(page.text.slice(0, 1000))) {
|
|
1593
|
+
outcome = failedProviderSearch("Browser search stopped at a challenge or changed query; user interaction may be required");
|
|
1594
|
+
break;
|
|
1595
|
+
}
|
|
1596
|
+
canContinueNative = true;
|
|
1597
|
+
const safeResults = [];
|
|
1598
|
+
for (const item of rankDiscoveryResults(parseNaverBrowserSearch(page), discoveryTerms)) {
|
|
1599
|
+
if (Date.now() - startedAt >= timeoutMs)
|
|
1600
|
+
break;
|
|
1601
|
+
if (!browserSearchMatchesQuery(item, query))
|
|
1602
|
+
continue;
|
|
1603
|
+
if (isSearchTrackingRelay(item))
|
|
1604
|
+
continue;
|
|
1605
|
+
if (scope && !searchResultMatchesSite(item.url, scope.host, scope.path))
|
|
1606
|
+
continue;
|
|
1607
|
+
let timer;
|
|
1608
|
+
const validation = await Promise.race([
|
|
1609
|
+
validateOutboundUrl(item.url, { allowPrivateHosts: false }),
|
|
1610
|
+
new Promise((resolve) => { timer = setTimeout(() => resolve({ ok: false }), Math.max(1, timeoutMs - (Date.now() - startedAt))); }),
|
|
1611
|
+
]).finally(() => { if (timer)
|
|
1612
|
+
clearTimeout(timer); });
|
|
1613
|
+
if (Date.now() - startedAt >= timeoutMs)
|
|
1614
|
+
break;
|
|
1615
|
+
if (validation.ok)
|
|
1616
|
+
safeResults.push(item);
|
|
1617
|
+
if (safeResults.length >= numResults)
|
|
1618
|
+
break;
|
|
1619
|
+
}
|
|
1620
|
+
outcome = safeResults.length ? constrain(completedProviderSearch(safeResults, "")) : failedProviderSearch("Browser search exposed no verified result headlines satisfying supported query constraints");
|
|
1621
|
+
if (outcome.ok) {
|
|
1622
|
+
if (index > 0)
|
|
1623
|
+
notes.push("Host-only discovery preserved the requested site path on every returned original");
|
|
1624
|
+
notes.push("Recovered through the approved local browser. Search snippets are discovery evidence, not verification of article bodies or all comments.");
|
|
1625
|
+
break;
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
catch (error) {
|
|
1630
|
+
outcome = failedProviderSearch(`Approved browser search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
recordAttempt("naver_browser", outcome);
|
|
1634
|
+
if (outcome.ok) {
|
|
1635
|
+
collectDiscovery(outcome);
|
|
1636
|
+
if (!unscopedDocsTerms)
|
|
1637
|
+
return finishDiscovery();
|
|
1638
|
+
}
|
|
1639
|
+
if (outcome.note)
|
|
1640
|
+
notes.push(outcome.note);
|
|
1641
|
+
}
|
|
1642
|
+
if (nativeYoutubeUrl && canContinueNative && !budgetExhausted()) {
|
|
1643
|
+
fallbacks.push("youtube_browser");
|
|
1644
|
+
let outcome;
|
|
1645
|
+
try {
|
|
1646
|
+
const page = await (runtime.runBrowserSession || runBrowserSession)({
|
|
1647
|
+
url: nativeYoutubeUrl, cdpUrl, timeoutMs: Math.max(1, timeoutMs - (Date.now() - startedAt)), allowPrivateHosts: false,
|
|
1648
|
+
waitForSelector: "ytd-video-renderer a#video-title",
|
|
1649
|
+
});
|
|
1650
|
+
if (Date.now() - startedAt >= timeoutMs)
|
|
1651
|
+
outcome = failedProviderSearch("YouTube native search exceeded the shared search budget");
|
|
1652
|
+
else if (!page.ok)
|
|
1653
|
+
outcome = failedProviderSearch(`YouTube native browser: ${page.error.code}: ${page.error.message}`);
|
|
1654
|
+
else {
|
|
1655
|
+
const rows = parseYoutubeSearch(page, nativeYoutubeUrl, query);
|
|
1656
|
+
outcome = Date.now() - startedAt >= timeoutMs ? failedProviderSearch("YouTube native search exceeded the shared search budget")
|
|
1657
|
+
: rows.length ? constrain(completedProviderSearch(rows, "")) : failedProviderSearch("YouTube native search exposed no exact watch-page headlines satisfying the quoted query");
|
|
1658
|
+
if (outcome.ok)
|
|
1659
|
+
notes.push("Recovered watch-page discovery through the approved local browser. Video title evidence does not verify the full video or transcript.");
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
catch (error) {
|
|
1663
|
+
outcome = failedProviderSearch(`YouTube native search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1664
|
+
}
|
|
1665
|
+
recordAttempt("youtube_browser", outcome);
|
|
1666
|
+
if (outcome.ok) {
|
|
1667
|
+
collectDiscovery(outcome);
|
|
1668
|
+
return finishDiscovery();
|
|
1669
|
+
}
|
|
1670
|
+
if (outcome.note)
|
|
1671
|
+
notes.push(outcome.note);
|
|
1672
|
+
}
|
|
1673
|
+
if (pythonSearchUrl && canContinueNative && !budgetExhausted()) {
|
|
1674
|
+
fallbacks.push("python_docs_browser");
|
|
1675
|
+
let outcome;
|
|
1676
|
+
try {
|
|
1677
|
+
const page = await (runtime.runBrowserSession || runBrowserSession)({
|
|
1678
|
+
url: pythonSearchUrl, cdpUrl, timeoutMs: Math.max(1, timeoutMs - (Date.now() - startedAt)), allowPrivateHosts: false,
|
|
1679
|
+
waitForSelector: "#search-results .search-summary:not(:empty)",
|
|
1680
|
+
});
|
|
1681
|
+
if (Date.now() - startedAt >= timeoutMs)
|
|
1682
|
+
outcome = failedProviderSearch("Python native search exceeded the shared search budget");
|
|
1683
|
+
else if (!page.ok)
|
|
1684
|
+
outcome = failedProviderSearch(`Python native browser: ${page.error.code}: ${page.error.message}`);
|
|
1685
|
+
else {
|
|
1686
|
+
const rows = parsePythonDocsSearch(page, pythonSearchUrl, unscopedDocsTerms || query);
|
|
1687
|
+
outcome = Date.now() - startedAt >= timeoutMs ? failedProviderSearch("Python native search exceeded the shared search budget")
|
|
1688
|
+
: rows.length ? constrain(completedProviderSearch(rows, "")) : failedProviderSearch("Python native search exposed no completed results satisfying the original query");
|
|
1689
|
+
if (outcome.ok)
|
|
1690
|
+
notes.push("Recovered through official Python documentation search in the approved browser. Native index/headline evidence is discovery, not complete original-body verification.");
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
catch (error) {
|
|
1694
|
+
outcome = failedProviderSearch(`Python native search failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1695
|
+
}
|
|
1696
|
+
recordAttempt("python_docs_browser", outcome);
|
|
1697
|
+
if (outcome.ok) {
|
|
1698
|
+
// Explicit documentation intent: a verified official index supersedes
|
|
1699
|
+
// adjacent general-web candidates; retain them if the native lane fails.
|
|
1700
|
+
if (unscopedDocsTerms)
|
|
1701
|
+
discoveryResults = [];
|
|
1702
|
+
collectDiscovery(outcome);
|
|
1703
|
+
}
|
|
1704
|
+
if (outcome.note)
|
|
1705
|
+
notes.push(outcome.note);
|
|
1706
|
+
}
|
|
1707
|
+
return finishDiscovery();
|
|
1283
1708
|
}
|
|
1284
1709
|
catch (error) {
|
|
1285
1710
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1330,9 +1755,6 @@ export function renderSmartsearch(response) {
|
|
|
1330
1755
|
lines.push(item.snippet);
|
|
1331
1756
|
parts.push(lines.join("\n"));
|
|
1332
1757
|
});
|
|
1333
|
-
if (assessment.recommended_handoff && assessment.recommended_handoff.tool !== "smartcrawl") {
|
|
1334
|
-
parts.push(`Recommended handoff:\n- tool: ${assessment.recommended_handoff.tool}\n- reason: ${assessment.recommended_handoff.reason}\n- url: ${assessment.recommended_handoff.url}`);
|
|
1335
|
-
}
|
|
1336
1758
|
if (response.notes.length > 0)
|
|
1337
1759
|
parts.push(`Notes:\n${response.notes.map((item) => `- ${item}`).join("\n")}`);
|
|
1338
1760
|
return parts.join("\n\n").trim();
|