pi-browser-harness 0.15.1 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,14 @@
2
2
 
3
3
  All notable changes to pi-browser-harness will be documented in this file.
4
4
 
5
+ ## 0.16.0 — 2026-11-07
6
+
7
+ ### Added
8
+
9
+ - **`browser_web_search` falls back across search engines.** With no `engine` given it tries Google, then Bing, DuckDuckGo and Brave, until one returns results, so a Google CAPTCHA or markup change no longer breaks search. `details.engine` says which engine answered and `details.fallbacks` lists the ones that failed first. When every engine fails the error is `all_engines_failed` with each engine's reason; when none found anything it stays `no_results`.
10
+ - **`engine` parameter.** Name one engine (`google`, `bing`, `duckduckgo`, `brave`) to search it alone, with no fallback.
11
+ - **Pluggable engines.** Each engine is a self-contained module mounted into a registry (`createEngineRegistry().mount(engine)`) that is injected into `createWebSearchTool`; mount order is the fallback order. Adding an engine is one module and one `mount`.
12
+
5
13
  ## 0.15.1 — 2026-11-07
6
14
 
7
15
  - Re-release of 0.15.0, whose npm publish failed (the provenance log returned 502, then rejected the retry as a duplicate). No code changes.
package/README.md CHANGED
@@ -119,7 +119,7 @@ Three things follow from that:
119
119
 
120
120
  | Tool | Purpose |
121
121
  |---|---|
122
- | `browser_web_search` | Query → ranked `{title, url, snippet, rank}`, by scraping a real Google SERP in your Chrome. No API key. Links only. |
122
+ | `browser_web_search` | Query → ranked `{title, url, snippet, rank}`, by scraping a real search results page in your Chrome. No API key. Links only. Walks the fallback ladder Google → Bing → DuckDuckGo → Brave until one answers, or searches only the `engine` you name. |
123
123
  | `browser_read_page` | Reader mode. A `url` (opened and closed in its own tab) or an owned `targetId` → clean article text, boilerplate stripped. |
124
124
 
125
125
  Both run in isolated tabs and never disturb your current page.
@@ -245,8 +245,14 @@ browser_network_requests({ // did a request 4xx/5xx?
245
245
  ```
246
246
  browser_web_search({ query: "chrome devtools protocol overview", limit: 5 })
247
247
  browser_read_page({ url: "<the best result's url>" })
248
+
249
+ // Search one engine only, with no fallback:
250
+ browser_web_search({ query: "chrome devtools protocol overview", engine: "bing" })
248
251
  ```
249
252
 
253
+ `details.engine` names the engine that answered, and `details.fallbacks` lists any engines that failed
254
+ first (CAPTCHA, no results, markup change, load failure).
255
+
250
256
  Both tools open their own isolated tab, so neither disturbs the tab you are working in.
251
257
 
252
258
  ### Keyboard modifiers
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-browser-harness",
3
- "version": "0.15.1",
3
+ "version": "0.16.0",
4
4
  "description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP",
5
5
  "keywords": [
6
6
  "pi-package",
@@ -1,4 +1,5 @@
1
- // Page-side code for browser_web_search: pulls the organic result anchors off a Google results page. See fill.page.ts for the rules.
1
+ // Page-side code for browser_web_search: pulls the organic result anchors off a search engine's results page, driven
2
+ // by the engine's selectors. See fill.page.ts for the rules.
2
3
 
3
4
  export type SerpAnchor = {
4
5
  readonly href: string;
@@ -12,29 +13,41 @@ export type SerpExtraction = {
12
13
  readonly pageText: string;
13
14
  };
14
15
 
16
+ export type SerpSelectors = {
17
+ readonly root: string;
18
+ readonly heading: string;
19
+ readonly containers: ReadonlyArray<string>;
20
+ readonly snippet?: string;
21
+ readonly cite: string;
22
+ };
23
+
15
24
  // Returns the extraction as JSON text (not an object) so the isolated-tab evaluator can keep its string guard.
16
- export function extractSerp(pageTextLimit: number): string {
17
- const main = document.querySelector<HTMLElement>("#search, #rso, #main") ?? document.body;
18
- const headings = Array.from(main.querySelectorAll<HTMLElement>("a h3"));
25
+ export function extractSerp(selectors: SerpSelectors, pageTextLimit: number): string {
26
+ const main = document.querySelector<HTMLElement>(selectors.root) ?? document.body;
27
+ const headings = Array.from(main.querySelectorAll<HTMLElement>(selectors.heading));
19
28
  const seen = new Set<string>();
20
29
  const anchors: SerpAnchor[] = [];
21
- for (const h3 of headings) {
22
- const a = h3.closest<HTMLAnchorElement>("a[href]");
30
+ for (const heading of headings) {
31
+ // The heading is the link (DuckDuckGo), sits inside it (Google, Brave) or wraps it (Bing).
32
+ const a = heading.closest<HTMLAnchorElement>("a[href]") ?? heading.querySelector<HTMLAnchorElement>("a[href]");
23
33
  if (!a) continue;
24
34
  const href = a.href;
25
35
  if (!href || seen.has(href)) continue;
26
36
  seen.add(href);
27
- const container: HTMLElement | null =
28
- a.closest<HTMLElement>("div.g") ??
29
- a.closest<HTMLElement>("div[data-hveid]") ??
30
- a.parentElement?.parentElement ??
31
- a.parentElement;
32
- const containerText = container ? container.innerText : "";
33
- const heading = h3.innerText || h3.textContent;
34
- const snippet = containerText.replace(heading, "").replace(/\s+/g, " ").trim().slice(0, 500);
35
- const citeEl = container ? container.querySelector<HTMLElement>("cite") : null;
37
+ let container: HTMLElement | null = null;
38
+ for (const selector of selectors.containers) {
39
+ container = a.closest<HTMLElement>(selector);
40
+ if (container) break;
41
+ }
42
+ container = container ?? a.parentElement?.parentElement ?? a.parentElement;
43
+ const title = heading.innerText || heading.textContent || "";
44
+ const snippetEl =
45
+ selectors.snippet !== undefined && container ? container.querySelector<HTMLElement>(selectors.snippet) : null;
46
+ const snippetSource = snippetEl ? snippetEl.innerText : container ? container.innerText.replace(title, "") : "";
47
+ const snippet = snippetSource.replace(/\s+/g, " ").trim().slice(0, 500);
48
+ const citeEl = container ? container.querySelector<HTMLElement>(selectors.cite) : null;
36
49
  const cite = citeEl ? (citeEl.innerText || citeEl.textContent || "").trim() : "";
37
- anchors.push({ href, heading, snippet, cite });
50
+ anchors.push({ href, heading: title, snippet, cite });
38
51
  }
39
52
  const pageText = document.body.innerText.slice(0, pageTextLimit);
40
53
  const extraction: SerpExtraction = { anchors, pageText };
@@ -0,0 +1,47 @@
1
+ import { externalResult, isWebUrl, type SearchEngine, type SerpAnchor, urlFromCite } from "../serp";
2
+
3
+ const BING_PROPERTY = /(?:^|\.)bing\.(?:com|net)$/;
4
+ const isExternalResult = externalResult((host) => BING_PROPERTY.test(host));
5
+
6
+ // Bing wraps organic links as bing.com/ck/a?...&u=a1<base64url(target)>. The "a1" prefix tags the encoding.
7
+ export const decodeBingClick = (href: string): string | undefined => {
8
+ try {
9
+ const u = new URL(href);
10
+ if (!BING_PROPERTY.test(u.hostname.toLowerCase()) || u.pathname !== "/ck/a") return undefined;
11
+ const encoded = u.searchParams.get("u");
12
+ if (!encoded?.startsWith("a1")) return undefined;
13
+ const decoded = Buffer.from(encoded.slice(2), "base64url").toString("utf8");
14
+ return isWebUrl(decoded) ? decoded : undefined;
15
+ } catch {
16
+ return undefined;
17
+ }
18
+ };
19
+
20
+ const resultUrl = (anchor: SerpAnchor): string | undefined => {
21
+ const target = decodeBingClick(anchor.href) ?? anchor.href;
22
+ if (isExternalResult(target)) return target;
23
+ if (anchor.cite !== undefined && anchor.cite.trim() !== "") return urlFromCite(anchor.cite, isExternalResult);
24
+ return undefined;
25
+ };
26
+
27
+ export const bing: SearchEngine = {
28
+ id: "bing",
29
+ name: "Bing",
30
+ buildUrl: (query, limit) =>
31
+ `https://www.bing.com/search?q=${encodeURIComponent(query)}&count=${limit}&setlang=en`,
32
+ selectors: {
33
+ root: "#b_results, #b_content",
34
+ heading: "li.b_algo h2",
35
+ containers: ["li.b_algo"],
36
+ snippet: ".b_caption p, p[class*='b_lineclamp'], .b_snippet",
37
+ cite: "cite",
38
+ },
39
+ captchaMarkers: [
40
+ "one last step",
41
+ "solve the challenge",
42
+ "verify you are a human",
43
+ "unusual traffic",
44
+ "are you a robot",
45
+ ],
46
+ resultUrl,
47
+ };
@@ -0,0 +1,23 @@
1
+ import { externalResult, type SearchEngine, type SerpAnchor } from "../serp";
2
+
3
+ const BRAVE_PROPERTY = /(?:^|\.)brave\.com$/;
4
+ const isExternalResult = externalResult((host) => BRAVE_PROPERTY.test(host));
5
+
6
+ // Brave links straight to the target; there is no redirect wrapper to undo.
7
+ const resultUrl = (anchor: SerpAnchor): string | undefined =>
8
+ isExternalResult(anchor.href) ? anchor.href : undefined;
9
+
10
+ export const brave: SearchEngine = {
11
+ id: "brave",
12
+ name: "Brave Search",
13
+ buildUrl: (query) => `https://search.brave.com/search?q=${encodeURIComponent(query)}&source=web`,
14
+ selectors: {
15
+ root: "#results, main",
16
+ heading: "[data-type='web'] a .title",
17
+ containers: ["[data-type='web']"],
18
+ snippet: ".generic-snippet .content, .snippet-description",
19
+ cite: "cite",
20
+ },
21
+ captchaMarkers: ["captcha", "verify you are human", "unusual activity", "not a robot"],
22
+ resultUrl,
23
+ };
@@ -0,0 +1,42 @@
1
+ import { externalResult, type SearchEngine, type SerpAnchor, urlFromCite } from "../serp";
2
+
3
+ const DDG_PROPERTY = /(?:^|\.)duckduckgo\.com$/;
4
+ const isExternalResult = externalResult((host) => DDG_PROPERTY.test(host));
5
+
6
+ // Results normally link straight to the target; a duckduckgo.com/l/?uddg=<encoded target> wrapper is undone, and ads
7
+ // (routed through /y.js) are dropped by the own-property filter.
8
+ export const unwrapDuckDuckGo = (href: string): string => {
9
+ try {
10
+ const u = new URL(href);
11
+ if (!DDG_PROPERTY.test(u.hostname.toLowerCase()) || u.pathname !== "/l/") return href;
12
+ return u.searchParams.get("uddg") ?? href;
13
+ } catch {
14
+ return href;
15
+ }
16
+ };
17
+
18
+ const resultUrl = (anchor: SerpAnchor): string | undefined => {
19
+ const target = unwrapDuckDuckGo(anchor.href);
20
+ if (isExternalResult(target)) return target;
21
+ if (anchor.cite !== undefined && anchor.cite.trim() !== "") return urlFromCite(anchor.cite, isExternalResult);
22
+ return undefined;
23
+ };
24
+
25
+ // The main results page. It renders results with JavaScript after the load event, hence readySelector. (The no-JS
26
+ // html.duckduckgo.com endpoint serves a bot challenge even to a real Chrome, so it is not used.)
27
+ export const duckduckgo: SearchEngine = {
28
+ id: "duckduckgo",
29
+ name: "DuckDuckGo",
30
+ buildUrl: (query) => `https://duckduckgo.com/?q=${encodeURIComponent(query)}&kl=us-en&ia=web`,
31
+ selectors: {
32
+ // The page has two mainline containers and the first is empty, so the root is the results list itself.
33
+ root: ".react-results--main, #react-layout",
34
+ heading: "article[data-testid='result'] a[data-testid='result-title-a']",
35
+ containers: ["article[data-testid='result']"],
36
+ snippet: "[data-result='snippet']",
37
+ cite: "a[data-testid='result-extras-url-link']",
38
+ },
39
+ captchaMarkers: ["bots use duckduckgo too", "complete the following challenge"],
40
+ readySelector: "article[data-testid='result']",
41
+ resultUrl,
42
+ };
@@ -0,0 +1,95 @@
1
+ import {
2
+ classifySerpWith,
3
+ externalResult,
4
+ parseSerp,
5
+ type ResolvedTargets,
6
+ type SearchEngine,
7
+ type SearchResult,
8
+ type SerpAnchor,
9
+ type SerpVerdict,
10
+ urlFromCite,
11
+ } from "../serp";
12
+
13
+ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
14
+ "unusual traffic",
15
+ "not a robot",
16
+ "recaptcha",
17
+ "detected unusual",
18
+ "systems have detected",
19
+ "before you continue to google",
20
+ "our systems have detected unusual traffic",
21
+ ];
22
+
23
+ // Any Google property, on any TLD (google.com, google.co.uk, consent.google.de, googleusercontent.com), is never a
24
+ // result: a sorry or consent page is where a rate-limited /goto redirect lands.
25
+ const GOOGLE_PROPERTY = /(?:^|\.)google(?:usercontent)?\.(?:com?\.[a-z]{2}|[a-z]{2,3})$/;
26
+
27
+ const isExternalResult = externalResult((host) => GOOGLE_PROPERTY.test(host));
28
+
29
+ const unwrapRedirect = (href: string): string => {
30
+ const wrapper = /^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i.exec(href);
31
+ if (!wrapper) return href;
32
+ const query = href.slice(href.indexOf("?") + 1);
33
+ const target = new URLSearchParams(query).get("q");
34
+ return target ?? href;
35
+ };
36
+
37
+ // Rebuilds a url from Google's visible breadcrumb cite, the only place the target survives once Google wraps hrefs
38
+ // in /goto.
39
+ export const googleUrlFromCite = (cite: string): string | undefined => urlFromCite(cite, isExternalResult);
40
+
41
+ // Only a real Google /goto url counts: the daemon fetches these, so a look-alike host, userinfo, a port or http
42
+ // must never qualify. Hosts are google.com or www.google.<tld>, nothing else.
43
+ const GOOGLE_GOTO_HOST = /^(?:www\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?$/;
44
+ export const isGotoRedirect = (href: string): boolean => {
45
+ try {
46
+ const u = new URL(href);
47
+ return (
48
+ u.protocol === "https:" &&
49
+ u.username === "" &&
50
+ u.password === "" &&
51
+ u.port === "" &&
52
+ GOOGLE_GOTO_HOST.test(u.hostname.toLowerCase()) &&
53
+ u.pathname === "/goto"
54
+ );
55
+ } catch {
56
+ return false;
57
+ }
58
+ };
59
+
60
+ // Direct external hrefs win. A /goto wrapper is opaque: the server-resolved redirect target is exact, and the cite is
61
+ // a display breadcrumb that only approximates the path, so it is the fallback when no target was resolved.
62
+ const resultUrl = (anchor: SerpAnchor, resolved: ResolvedTargets): string | undefined => {
63
+ if (!isGotoRedirect(anchor.href)) {
64
+ const unwrapped = unwrapRedirect(anchor.href);
65
+ return isExternalResult(unwrapped) ? unwrapped : undefined;
66
+ }
67
+ const target = resolved.get(anchor.href);
68
+ if (target !== undefined && isExternalResult(target)) return target;
69
+ if (anchor.cite !== undefined && anchor.cite.trim() !== "") return googleUrlFromCite(anchor.cite);
70
+ return undefined;
71
+ };
72
+
73
+ export const google: SearchEngine = {
74
+ id: "google",
75
+ name: "Google",
76
+ buildUrl: (query, limit) => `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`,
77
+ selectors: {
78
+ root: "#search, #rso, #main",
79
+ heading: "a h3",
80
+ containers: ["div.g", "div[data-hveid]"],
81
+ cite: "cite",
82
+ },
83
+ captchaMarkers: CAPTCHA_MARKERS,
84
+ resultUrl,
85
+ needsRedirectResolve: isGotoRedirect,
86
+ };
87
+
88
+ export const parseGoogleSerp = (
89
+ anchors: ReadonlyArray<SerpAnchor>,
90
+ limit: number,
91
+ resolved: ResolvedTargets = new Map(),
92
+ ): ReadonlyArray<SearchResult> => parseSerp(google, anchors, limit, resolved);
93
+
94
+ export const classifyGoogleSerp = (pageText: string, resultCount: number, anchorCount: number): SerpVerdict =>
95
+ classifySerpWith(CAPTCHA_MARKERS, pageText, resultCount, anchorCount);
@@ -0,0 +1,13 @@
1
+ // Composition root for the built-in engines. The order of mounts is the default fallback ladder. To add an engine,
2
+ // write engines/<id>.ts exporting a SearchEngine and mount it here — or mount it onto defaultEngines() and inject the
3
+ // result into createWebSearchTool from outside.
4
+ import { bing } from "./bing";
5
+ import { brave } from "./brave";
6
+ import { duckduckgo } from "./duckduckgo";
7
+ import { google } from "./google";
8
+ import { createEngineRegistry, type EngineRegistry } from "./registry";
9
+
10
+ export const defaultEngines = (): EngineRegistry =>
11
+ createEngineRegistry().mount(google).mount(bing).mount(duckduckgo).mount(brave);
12
+
13
+ export { createEngineRegistry, type EngineRegistry } from "./registry";
@@ -0,0 +1,29 @@
1
+ import type { SearchEngine } from "../serp";
2
+
3
+ // An immutable set of mounted engines. Mount order is the fallback ladder: the first engine mounted is tried first.
4
+ export type EngineRegistry = {
5
+ // A new registry with `engine` appended to the ladder. Throws on a duplicate id: that is a wiring bug, caught at
6
+ // composition time rather than surfacing as a confusing search.
7
+ readonly mount: (engine: SearchEngine) => EngineRegistry;
8
+ readonly get: (id: string) => SearchEngine | undefined;
9
+ readonly ids: () => ReadonlyArray<string>;
10
+ readonly ladder: () => ReadonlyArray<SearchEngine>;
11
+ };
12
+
13
+ const ENGINE_ID = /^[a-z][a-z0-9-]*$/;
14
+
15
+ const fromEngines = (engines: ReadonlyArray<SearchEngine>): EngineRegistry => {
16
+ const byId = new Map(engines.map((e) => [e.id, e] as const));
17
+ return {
18
+ mount: (engine) => {
19
+ if (!ENGINE_ID.test(engine.id)) throw new Error(`search engine id "${engine.id}" must match ${String(ENGINE_ID)}`);
20
+ if (byId.has(engine.id)) throw new Error(`search engine "${engine.id}" is already mounted`);
21
+ return fromEngines([...engines, engine]);
22
+ },
23
+ get: (id) => byId.get(id),
24
+ ids: () => engines.map((e) => e.id),
25
+ ladder: () => engines,
26
+ };
27
+ };
28
+
29
+ export const createEngineRegistry = (): EngineRegistry => fromEngines([]);
@@ -2,7 +2,7 @@
2
2
  // following it. The blob cannot be decoded client-side and the page's <cite> is only a display breadcrumb, so this
3
3
  // is the one source of the exact url. Google answers the redirect to a cookie-less GET; HEAD returns 200 instead.
4
4
 
5
- import { isGotoRedirect } from "./google-serp";
5
+ import { isGotoRedirect } from "./engines/google";
6
6
 
7
7
  const REDIRECT_STATUSES: ReadonlySet<number> = new Set([301, 302, 303, 307, 308]);
8
8
 
@@ -0,0 +1,157 @@
1
+ // Engine-neutral SERP vocabulary and parsing. Each engine (engines/*.ts) supplies how to find result anchors on its
2
+ // page and how to turn an anchor into its target url; ranking, dedupe and verdicts are shared here.
3
+
4
+ export type SerpAnchor = {
5
+ readonly href: string;
6
+ readonly heading: string;
7
+ readonly snippet: string;
8
+ // Visible breadcrumb url ("https://host › path › seg"); the fallback when the href is an opaque redirect.
9
+ readonly cite?: string;
10
+ };
11
+
12
+ export type SerpExtraction = {
13
+ readonly anchors: ReadonlyArray<SerpAnchor>;
14
+ readonly pageText: string;
15
+ };
16
+
17
+ export type SearchResult = {
18
+ readonly title: string;
19
+ readonly url: string;
20
+ readonly snippet: string;
21
+ readonly rank: number;
22
+ };
23
+
24
+ export type SerpVerdict = "ok" | "captcha" | "no_results" | "extraction_mismatch";
25
+
26
+ // Redirect hrefs resolved server-side (Google /goto), keyed by href.
27
+ export type ResolvedTargets = ReadonlyMap<string, string>;
28
+
29
+ // Selectors handed to the page-side extractor. Every result is found from its heading: the title element is either
30
+ // the result link itself or sits inside / around it.
31
+ export type SerpSelectors = {
32
+ // Results column; the extractor falls back to <body> when none match.
33
+ readonly root: string;
34
+ // The title element of each organic result.
35
+ readonly heading: string;
36
+ // Result-block selectors, tried in order with closest(); the first match is the result's container.
37
+ readonly containers: ReadonlyArray<string>;
38
+ // Snippet element inside the container; without it the snippet is the container text minus the heading.
39
+ readonly snippet?: string;
40
+ // Breadcrumb element inside the container.
41
+ readonly cite: string;
42
+ };
43
+
44
+ // A pluggable search engine. Engines are plain values mounted into an EngineRegistry (engines/registry.ts); nothing
45
+ // else in the search code knows which engines exist.
46
+ export type SearchEngine = {
47
+ // Stable lower-case identifier the agent passes as `engine` ("google", "bing").
48
+ readonly id: string;
49
+ // Human name, used in error messages ("Bing served a CAPTCHA").
50
+ readonly name: string;
51
+ readonly buildUrl: (query: string, limit: number) => string;
52
+ readonly selectors: SerpSelectors;
53
+ // Lower-case phrases that mark a bot wall / challenge page.
54
+ readonly captchaMarkers: ReadonlyArray<string>;
55
+ // The anchor's external target, or undefined when it is not an organic web result.
56
+ readonly resultUrl: (anchor: SerpAnchor, resolved: ResolvedTargets) => string | undefined;
57
+ // For engines that render results with JavaScript after the load event: the tool waits (briefly) for this selector
58
+ // to match before extracting. A page that never matches is still extracted and classified (captcha / no_results).
59
+ readonly readySelector?: string;
60
+ // Hrefs that can only be decoded by asking the engine's server for the redirect (Google /goto).
61
+ readonly needsRedirectResolve?: (href: string) => boolean;
62
+ };
63
+
64
+ const CITE_SEPARATOR = /\s*›\s*/;
65
+ const TRUNCATION = /(?:\.\.\.|…)/;
66
+
67
+ export const isWebUrl = (url: string): boolean => {
68
+ if (!/^https?:\/\//i.test(url)) return false;
69
+ try {
70
+ new URL(url);
71
+ return true;
72
+ } catch {
73
+ return false;
74
+ }
75
+ };
76
+
77
+ export const hostOf = (url: string): string | undefined => {
78
+ try {
79
+ return new URL(url).hostname.toLowerCase();
80
+ } catch {
81
+ return undefined;
82
+ }
83
+ };
84
+
85
+ // Rebuilds a url from a visible breadcrumb cite. Segments after a truncation marker are unknowable, so the path stops
86
+ // at the last complete segment rather than guessing. Returns only urls `isResult` accepts.
87
+ export const urlFromCite = (cite: string, isResult: (url: string) => boolean): string | undefined => {
88
+ const segments = cite
89
+ .trim()
90
+ .split(CITE_SEPARATOR)
91
+ .map((segment) => segment.trim())
92
+ .filter((segment) => segment.length > 0);
93
+ const origin = segments[0];
94
+ if (origin === undefined || TRUNCATION.test(origin)) return undefined;
95
+ // A cite naming any other scheme (ftp:, mailto:) is not a web result; prefixing it would forge a bogus https url.
96
+ if (/^[a-z][a-z0-9+.-]*:/i.test(origin) && !/^https?:\/\//i.test(origin)) return undefined;
97
+ const withScheme = /^https?:\/\//i.test(origin) ? origin : `https://${origin}`;
98
+ const path: string[] = [];
99
+ for (const segment of segments.slice(1)) {
100
+ if (TRUNCATION.test(segment)) break;
101
+ path.push(segment);
102
+ }
103
+ const url = [withScheme.replace(/\/$/, ""), ...path].join("/");
104
+ return isResult(url) ? url : undefined;
105
+ };
106
+
107
+ // A web url whose host is not one of the engine's own properties.
108
+ export const externalResult =
109
+ (isOwnHost: (host: string) => boolean) =>
110
+ (url: string): boolean => {
111
+ if (!isWebUrl(url)) return false;
112
+ const host = hostOf(url);
113
+ return host !== undefined && !isOwnHost(host);
114
+ };
115
+
116
+ const dedupeKey = (url: string): string => {
117
+ try {
118
+ const u = new URL(url);
119
+ return `${u.origin}${u.pathname.replace(/\/$/, "")}`.toLowerCase();
120
+ } catch {
121
+ return url.toLowerCase();
122
+ }
123
+ };
124
+
125
+ export const parseSerp = (
126
+ engine: SearchEngine,
127
+ anchors: ReadonlyArray<SerpAnchor>,
128
+ limit: number,
129
+ resolved: ResolvedTargets = new Map(),
130
+ ): ReadonlyArray<SearchResult> => {
131
+ const seen = new Set<string>();
132
+ const results: SearchResult[] = [];
133
+ for (const anchor of anchors) {
134
+ if (results.length >= limit) break;
135
+ const url = engine.resultUrl(anchor, resolved);
136
+ if (url === undefined) continue;
137
+ const key = dedupeKey(url);
138
+ if (seen.has(key)) continue;
139
+ seen.add(key);
140
+ results.push({ title: anchor.heading.trim(), url, snippet: anchor.snippet.trim(), rank: results.length + 1 });
141
+ }
142
+ return results;
143
+ };
144
+
145
+ export const classifySerpWith = (
146
+ captchaMarkers: ReadonlyArray<string>,
147
+ pageText: string,
148
+ resultCount: number,
149
+ anchorCount: number,
150
+ ): SerpVerdict => {
151
+ if (resultCount > 0) return "ok";
152
+ const lower = pageText.toLowerCase();
153
+ if (captchaMarkers.some((marker) => lower.includes(marker))) return "captcha";
154
+ // Anchors were found but every one was filtered out: the markup changed, not the result set.
155
+ if (anchorCount > 0) return "extraction_mismatch";
156
+ return "no_results";
157
+ };
@@ -4,46 +4,58 @@ import { parseJson } from "../../schemas/parse";
4
4
  import { type Result, err, ok } from "../../util/result";
5
5
  import { applyTruncation } from "../../util/truncate";
6
6
  import { type DetailsOf, defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
7
+ import type { BrowserClient } from "../../client";
7
8
  import {
8
9
  closeIsolatedTab,
9
10
  evalInIsolatedTab,
11
+ type IsolatedTab,
10
12
  isJsonText,
11
13
  navigateIsolatedTab,
12
14
  openIsolatedTab,
13
15
  waitForIsolatedLoad,
14
16
  } from "../isolated-tab";
15
17
  import { pageCallExpression } from "../page/inject";
18
+ import { pollUntil } from "../poll";
19
+ import { safeJs } from "../../util/js-template";
16
20
  import { extractSerp } from "../page/serp.page";
17
21
  import type * as extracted from "../page/serp.page";
18
22
  import { ExpandableTextSchema, renderExpandableText } from "../render";
19
23
  import { deadlineSignal, runLimited } from "../concurrency";
24
+ import { defaultEngines, type EngineRegistry } from "./engines";
20
25
  import {
21
- classifySerp,
22
- isGotoRedirect,
23
- parseGoogleSerp,
26
+ classifySerpWith,
27
+ parseSerp,
24
28
  type ResolvedTargets,
29
+ type SearchEngine,
25
30
  type SearchResult,
26
31
  type SerpExtraction,
27
- } from "./google-serp";
32
+ type SerpSelectors,
33
+ } from "./serp";
28
34
  import { fetchRedirectTarget, type RedirectResolver } from "./redirect-resolver";
29
35
 
30
36
  const LOAD_TIMEOUT_MS = 15_000;
31
37
  const RESOLVE_BUDGET_MS = 5_000;
32
38
  const RESOLVE_CONCURRENCY = 5;
39
+ const RENDER_WAIT_MS = 4_000;
33
40
 
34
41
  export type WebSearchDeps = {
42
+ // The mounted engines; mount order is the fallback ladder.
43
+ readonly engines: EngineRegistry;
35
44
  readonly resolveRedirect: RedirectResolver;
36
45
  readonly resolveBudgetMs?: number;
37
46
  };
38
47
 
39
- // Resolves every /goto anchor in parallel under one time budget; a slot that fails or runs out of time is simply
40
- // absent from the map, and the parser falls back to that anchor's cite.
41
- const resolveGotoTargets = async (
48
+ // Resolves every server-side redirect anchor (Google /goto) in parallel under one time budget; a slot that fails or
49
+ // runs out of time is simply absent from the map, and the engine falls back to that anchor's cite.
50
+ const resolveRedirectTargets = async (
51
+ engine: SearchEngine,
42
52
  anchors: ReadonlyArray<SerpExtraction["anchors"][number]>,
43
53
  deps: WebSearchDeps,
44
54
  signal: AbortSignal | undefined,
45
55
  ): Promise<ResolvedTargets> => {
46
- const hrefs = anchors.map((a) => a.href).filter(isGotoRedirect);
56
+ const needsResolve = engine.needsRedirectResolve;
57
+ if (needsResolve === undefined) return new Map();
58
+ const hrefs = anchors.map((a) => a.href).filter(needsResolve);
47
59
  if (hrefs.length === 0) return new Map();
48
60
  const deadline = deadlineSignal(deps.resolveBudgetMs ?? RESOLVE_BUDGET_MS, signal);
49
61
  const targets = await runLimited(hrefs, { limit: RESOLVE_CONCURRENCY, signal: deadline.signal }, (href, s) =>
@@ -58,25 +70,17 @@ const resolveGotoTargets = async (
58
70
  return resolved;
59
71
  };
60
72
 
61
- const WebSearchArgs = Type.Object({
62
- query: Type.String({ description: "The search query." }),
63
- limit: Type.Optional(
64
- Type.Integer({ default: 10, minimum: 1, maximum: 30, description: "Max results to return. Default: 10." }),
65
- ),
66
- });
67
-
68
73
  const PAGE_TEXT_LIMIT = 4000;
69
74
 
70
75
  // The page function declares the extraction shape it produces; this pins it to the shape the SERP parser consumes,
71
76
  // so the two cannot drift apart silently (the runtime validator below still guards the actual wire value).
72
77
  type Pinned<T extends SerpExtraction> = T;
73
78
  export type ExtractionFromPage = Pinned<extracted.SerpExtraction>;
79
+ type PinnedSelectors<T extends extracted.SerpSelectors> = T;
80
+ export type SelectorsForPage = PinnedSelectors<SerpSelectors>;
74
81
 
75
- // Built once: stringifying the page function per call was the only cost of the thunk.
76
- const SERP_EXTRACTION_EXPR = pageCallExpression(extractSerp, PAGE_TEXT_LIMIT);
77
-
78
- const buildSerpUrl = (query: string, limit: number): string =>
79
- `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
82
+ const extractionExpression = (engine: SearchEngine): string =>
83
+ pageCallExpression(extractSerp, engine.selectors, PAGE_TEXT_LIMIT);
80
84
 
81
85
  const SerpExtractionSchema = Type.Object({
82
86
  anchors: Type.Array(
@@ -102,105 +106,231 @@ const SearchResultSchema = Type.Object({
102
106
  rank: Type.Number(),
103
107
  });
104
108
 
109
+ const AttemptSchema = Type.Object({
110
+ engine: Type.String(),
111
+ kind: Type.String(),
112
+ reason: Type.Optional(Type.String()),
113
+ message: Type.String(),
114
+ });
115
+
105
116
  const WebSearchDetailsSchema = Type.Object({
106
117
  results: Type.Array(SearchResultSchema),
107
- engine: Type.Literal("google"),
118
+ engine: Type.String(),
108
119
  query: Type.String(),
120
+ // Engines tried and failed before `engine` answered; absent when the first engine answered.
121
+ fallbacks: Type.Optional(Type.Array(AttemptSchema)),
109
122
  render: ExpandableTextSchema,
110
123
  fullOutputPath: Type.Optional(Type.String()),
111
124
  });
112
125
 
113
126
  export type WebSearchDetails = DetailsOf<typeof WebSearchDetailsSchema>;
127
+ type Attempt = DetailsOf<typeof AttemptSchema>;
114
128
 
115
129
  const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
116
130
  const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
117
131
  return `${results.length} result(s) for "${query}":\n\n${lines.join("\n\n")}`;
118
132
  };
119
133
 
120
- export const createWebSearchTool = (deps: WebSearchDeps) => defineBrowserTool({
121
- name: "browser_web_search",
122
- label: "Browser Web Search",
123
- description:
124
- "Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.",
125
- promptSnippet: "Search the web for a query",
126
- promptGuidelines: [
127
- "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
128
- "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
129
- "If results were on the page but none could be parsed it fails with details.reason:'extraction_mismatch' (plus anchorCount) — Google's markup changed; report it rather than retrying.",
130
- "Runs in its own isolated tab and never disturbs the user's current tab.",
131
- ],
132
- parameters: WebSearchArgs,
133
- details: WebSearchDetailsSchema,
134
- concurrency: "serialized",
135
- async handler(args, { client, signal }): Promise<Result<ToolOk<WebSearchDetails>, ToolErr>> {
136
- const limit = args.limit ?? 10;
137
- const serpUrl = buildSerpUrl(args.query, limit);
138
-
139
- const opened = await openIsolatedTab(client);
140
- if (!opened.success) return err(opened.error);
141
- const tab = opened.data;
142
-
143
- try {
144
- const navigated = await navigateIsolatedTab(client, tab, serpUrl);
145
- if (!navigated.success) return err(navigated.error);
146
-
147
- const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
148
- if (!loaded.success) return err(loaded.error);
149
-
150
- const extracted = await evalInIsolatedTab(client, tab, SERP_EXTRACTION_EXPR, isJsonText);
151
- if (!extracted.success) return err(extracted.error);
152
-
153
- const parsedJson = parseSerpExtraction(extracted.data);
154
- if (!parsedJson.success) {
155
- return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
156
- }
157
- const extraction: SerpExtraction = parsedJson.data;
158
-
159
- const resolved = await resolveGotoTargets(extraction.anchors, deps, signal);
160
- const results = parseGoogleSerp(extraction.anchors, limit, resolved);
161
- const anchorCount = extraction.anchors.length;
162
- const verdict = classifySerp(extraction.pageText, results.length, anchorCount);
163
- if (verdict === "captcha") {
164
- return err({
165
- kind: "invalid_state",
166
- message: "Google served a CAPTCHA / bot wall",
167
- details: { reason: verdict, serpUrl },
168
- });
169
- }
170
- if (verdict === "extraction_mismatch") {
171
- return err({
172
- kind: "invalid_state",
173
- message: `found ${anchorCount} result anchor(s) but none parsed to an external url (Google markup may have changed)`,
174
- details: { reason: verdict, anchorCount, serpUrl },
175
- });
176
- }
177
- if (verdict === "no_results") {
178
- return err({ kind: "invalid_state", message: "no results found", details: { reason: verdict, serpUrl } });
179
- }
180
-
181
- const body = formatResults(args.query, results);
182
- const truncated = await applyTruncation(body, "search");
183
- return ok({
184
- text: truncated.text,
185
- details: {
186
- results,
187
- engine: "google",
188
- query: args.query,
189
- render: {
190
- summary: `${results.length} result(s) · google · "${args.query}"`,
191
- body,
192
- ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
193
- },
194
- ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
195
- },
196
- });
197
- } finally {
198
- await closeIsolatedTab(client, tab);
199
- }
200
- },
201
- renderResult(result, expanded) {
202
- return renderExpandableText(result.details.render, expanded);
134
+ // One engine, one SERP, in the already-open isolated tab.
135
+ const searchEngine = async (
136
+ engine: SearchEngine,
137
+ query: string,
138
+ limit: number,
139
+ ctx: {
140
+ readonly client: BrowserClient;
141
+ readonly tab: IsolatedTab;
142
+ readonly signal: AbortSignal | undefined;
143
+ readonly expression: string;
203
144
  },
145
+ deps: WebSearchDeps,
146
+ ): Promise<Result<ReadonlyArray<SearchResult>, ToolErr>> => {
147
+ const { client, tab, signal, expression } = ctx;
148
+ const serpUrl = engine.buildUrl(query, limit);
149
+
150
+ const navigated = await navigateIsolatedTab(client, tab, serpUrl);
151
+ if (!navigated.success) return err(navigated.error);
152
+
153
+ const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
154
+ if (!loaded.success) return err(loaded.error);
155
+
156
+ if (engine.readySelector !== undefined) {
157
+ // A miss is not a failure: a bot wall or an empty SERP never renders results, and extraction classifies it.
158
+ await pollUntil(client, safeJs`document.querySelector(${engine.readySelector}) !== null`, {
159
+ timeoutMs: RENDER_WAIT_MS,
160
+ signal,
161
+ sessionId: tab.sessionId,
162
+ what: `${engine.name} results`,
163
+ });
164
+ if (signal?.aborted === true) return err({ kind: "timeout", message: `Aborted while waiting for ${engine.name} results` });
165
+ }
166
+
167
+ const evaluated = await evalInIsolatedTab(client, tab, expression, isJsonText);
168
+ if (!evaluated.success) return err(evaluated.error);
169
+
170
+ const parsedJson = parseSerpExtraction(evaluated.data);
171
+ if (!parsedJson.success) {
172
+ return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
173
+ }
174
+ const extraction: SerpExtraction = parsedJson.data;
175
+
176
+ const resolved = await resolveRedirectTargets(engine, extraction.anchors, deps, signal);
177
+ const results = parseSerp(engine, extraction.anchors, limit, resolved);
178
+ const anchorCount = extraction.anchors.length;
179
+ const verdict = classifySerpWith(engine.captchaMarkers, extraction.pageText, results.length, anchorCount);
180
+ if (verdict === "captcha") {
181
+ return err({
182
+ kind: "invalid_state",
183
+ message: `${engine.name} served a CAPTCHA / bot wall`,
184
+ details: { reason: verdict, serpUrl },
185
+ });
186
+ }
187
+ if (verdict === "extraction_mismatch") {
188
+ return err({
189
+ kind: "invalid_state",
190
+ message: `found ${anchorCount} result anchor(s) but none parsed to an external url (${engine.name} markup may have changed)`,
191
+ details: { reason: verdict, anchorCount, serpUrl },
192
+ });
193
+ }
194
+ if (verdict === "no_results") {
195
+ return err({ kind: "invalid_state", message: "no results found", details: { reason: verdict, serpUrl } });
196
+ }
197
+ return ok(results);
198
+ };
199
+
200
+ const withEngine = (engine: string, error: ToolErr): ToolErr => ({
201
+ ...error,
202
+ details: { ...error.details, engine },
204
203
  });
205
204
 
206
- export const webSearchTool = createWebSearchTool({ resolveRedirect: fetchRedirectTarget });
205
+ const toAttempt = (engine: string, error: ToolErr): Attempt => {
206
+ const reason = error.details?.["reason"];
207
+ return {
208
+ engine,
209
+ kind: error.kind,
210
+ ...(typeof reason === "string" ? { reason } : {}),
211
+ message: error.message,
212
+ };
213
+ };
214
+
215
+ // A dead browser or a cancelled call fails every engine the same way, so the ladder stops instead of trying the next.
216
+ const stopsLadder = (error: ToolErr, signal: AbortSignal | undefined): boolean =>
217
+ signal?.aborted === true || error.kind === "not_connected";
218
+
219
+ const exhausted = (attempts: ReadonlyArray<Attempt>): ToolErr => {
220
+ const allEmpty = attempts.every((a) => a.reason === "no_results");
221
+ const summary = attempts.map((a) => `${a.engine}: ${a.message}`).join("; ");
222
+ return {
223
+ kind: "invalid_state",
224
+ message: allEmpty ? `no results found on any engine (${summary})` : `every search engine failed (${summary})`,
225
+ details: { reason: allEmpty ? "no_results" : "all_engines_failed", attempts },
226
+ };
227
+ };
228
+
229
+ // A named engine is tried alone; no name means the whole mounted ladder.
230
+ const pickLadder = (
231
+ engines: EngineRegistry,
232
+ name: string | undefined,
233
+ ): Result<ReadonlyArray<SearchEngine>, ToolErr> => {
234
+ if (name === undefined) return ok(engines.ladder());
235
+ const named = engines.get(name);
236
+ if (named !== undefined) return ok([named]);
237
+ return err({
238
+ kind: "invalid_args",
239
+ message: `unknown search engine "${name}"; mounted engines: ${engines.ids().join(", ")}`,
240
+ details: { engine: name, engines: engines.ids() },
241
+ });
242
+ };
243
+
244
+ const webSearchArgs = (engines: EngineRegistry) =>
245
+ Type.Object({
246
+ query: Type.String({ description: "The search query." }),
247
+ limit: Type.Optional(
248
+ Type.Integer({ default: 10, minimum: 1, maximum: 30, description: "Max results to return. Default: 10." }),
249
+ ),
250
+ engine: Type.Optional(
251
+ Type.String({
252
+ enum: [...engines.ids()],
253
+ description: `Search only this engine (${engines.ids().join(", ")}), with no fallback. Omit to walk the fallback ladder ${engines.ids().join(" → ")} until one returns results.`,
254
+ }),
255
+ ),
256
+ });
257
+
258
+ export const createWebSearchTool = (deps: WebSearchDeps) => {
259
+ const { engines } = deps;
260
+ const ids = engines.ids();
261
+ if (ids.length === 0) throw new Error("browser_web_search needs at least one mounted search engine");
262
+ const ladderText = ids.join(" → ");
263
+ // Built once per engine: stringifying the page function per call was the only cost of the thunk.
264
+ const expressions = new Map(engines.ladder().map((e) => [e.id, extractionExpression(e)] as const));
265
+
266
+ return defineBrowserTool({
267
+ name: "browser_web_search",
268
+ label: "Browser Web Search",
269
+ description:
270
+ `Search the web and return ranked results (title, url, snippet, rank). Scrapes a search engine results page in the real Chrome — no API key. Engines: ${ids.join(", ")}. With no engine given it walks the fallback ladder ${ladderText} until one returns results. Returns links only; use browser_read_page to read a result's content.`,
271
+ promptSnippet: "Search the web for a query",
272
+ promptGuidelines: [
273
+ `Omit engine to get the fallback ladder (${ladderText}); details.engine says which engine answered and details.fallbacks lists the engines that failed first.`,
274
+ "Pass engine only when one engine is specifically needed; a named engine is tried alone, with no fallback.",
275
+ "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl and engine) — surface it to the user; do not retry in a tight loop.",
276
+ "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
277
+ "If results were on the page but none could be parsed it fails with details.reason:'extraction_mismatch' (plus anchorCount) — the engine's markup changed; report it rather than retrying.",
278
+ "When every engine on the ladder fails it is details.reason:'all_engines_failed' with details.attempts naming each engine's failure.",
279
+ "Runs in its own isolated tab and never disturbs the user's current tab.",
280
+ ],
281
+ parameters: webSearchArgs(engines),
282
+ details: WebSearchDetailsSchema,
283
+ concurrency: "serialized",
284
+ async handler(args, { client, signal }): Promise<Result<ToolOk<WebSearchDetails>, ToolErr>> {
285
+ const limit = args.limit ?? 10;
286
+ const ladder = pickLadder(engines, args.engine);
287
+ if (!ladder.success) return err(ladder.error);
288
+
289
+ const opened = await openIsolatedTab(client);
290
+ if (!opened.success) return err(opened.error);
291
+ const tab = opened.data;
292
+
293
+ try {
294
+ const attempts: Attempt[] = [];
295
+ for (const engine of ladder.data) {
296
+ const id = engine.id;
297
+ const expression = expressions.get(id) ?? extractionExpression(engine);
298
+ const searched = await searchEngine(engine, args.query, limit, { client, tab, signal, expression }, deps);
299
+ if (!searched.success) {
300
+ // A single named engine reports its own failure exactly, as if there were no ladder.
301
+ if (ladder.data.length === 1 || stopsLadder(searched.error, signal)) return err(withEngine(id, searched.error));
302
+ attempts.push(toAttempt(id, searched.error));
303
+ continue;
304
+ }
305
+ const results = searched.data;
306
+ const body = formatResults(args.query, results);
307
+ const truncated = await applyTruncation(body, "search");
308
+ const via = attempts.length > 0 ? ` (after ${attempts.map((a) => a.engine).join(", ")} failed)` : "";
309
+ return ok({
310
+ text: truncated.text,
311
+ details: {
312
+ results,
313
+ engine: id,
314
+ query: args.query,
315
+ ...(attempts.length > 0 ? { fallbacks: attempts } : {}),
316
+ render: {
317
+ summary: `${results.length} result(s) · ${id}${via} · "${args.query}"`,
318
+ body,
319
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
320
+ },
321
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
322
+ },
323
+ });
324
+ }
325
+ return err(exhausted(attempts));
326
+ } finally {
327
+ await closeIsolatedTab(client, tab);
328
+ }
329
+ },
330
+ renderResult(result, expanded) {
331
+ return renderExpandableText(result.details.render, expanded);
332
+ },
333
+ });
334
+ };
335
+
336
+ export const webSearchTool = createWebSearchTool({ engines: defaultEngines(), resolveRedirect: fetchRedirectTarget });
@@ -1,155 +0,0 @@
1
- export type SerpAnchor = {
2
- readonly href: string;
3
- readonly heading: string;
4
- readonly snippet: string;
5
- // Visible breadcrumb url ("https://host › path › seg"); the only place the target survives once Google wraps hrefs in /goto.
6
- readonly cite?: string;
7
- };
8
-
9
- export type SerpExtraction = {
10
- readonly anchors: ReadonlyArray<SerpAnchor>;
11
- readonly pageText: string;
12
- };
13
-
14
- export type SearchResult = {
15
- readonly title: string;
16
- readonly url: string;
17
- readonly snippet: string;
18
- readonly rank: number;
19
- };
20
-
21
- export type SerpVerdict = "ok" | "captcha" | "no_results" | "extraction_mismatch";
22
-
23
- const CAPTCHA_MARKERS: ReadonlyArray<string> = [
24
- "unusual traffic",
25
- "not a robot",
26
- "recaptcha",
27
- "detected unusual",
28
- "systems have detected",
29
- "before you continue to google",
30
- "our systems have detected unusual traffic",
31
- ];
32
-
33
- const unwrapRedirect = (href: string): string => {
34
- const wrapper = /^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i.exec(href);
35
- if (!wrapper) return href;
36
- const query = href.slice(href.indexOf("?") + 1);
37
- const target = new URLSearchParams(query).get("q");
38
- return target ?? href;
39
- };
40
-
41
- const CITE_SEPARATOR = /\s*›\s*/;
42
- const TRUNCATION = /(?:\.\.\.|…)/;
43
-
44
- // Rebuilds a url from Google's visible breadcrumb cite. Segments after a truncation marker are unknowable, so the
45
- // path stops at the last complete segment rather than guessing.
46
- export const urlFromCite = (cite: string): string | undefined => {
47
- const segments = cite
48
- .trim()
49
- .split(CITE_SEPARATOR)
50
- .map((segment) => segment.trim())
51
- .filter((segment) => segment.length > 0);
52
- const origin = segments[0];
53
- if (origin === undefined || TRUNCATION.test(origin)) return undefined;
54
- // A cite naming any other scheme (ftp:, mailto:) is not a web result; prefixing it would forge a bogus https url.
55
- if (/^[a-z][a-z0-9+.-]*:/i.test(origin) && !/^https?:\/\//i.test(origin)) return undefined;
56
- const withScheme = /^https?:\/\//i.test(origin) ? origin : `https://${origin}`;
57
- const path: string[] = [];
58
- for (const segment of segments.slice(1)) {
59
- if (TRUNCATION.test(segment)) break;
60
- path.push(segment);
61
- }
62
- const url = [withScheme.replace(/\/$/, ""), ...path].join("/");
63
- return isExternalResult(url) ? url : undefined;
64
- };
65
-
66
- // Only a real Google /goto url counts: the daemon fetches these, so a look-alike host, userinfo, a port or http
67
- // must never qualify. Hosts are google.com or www.google.<tld>, nothing else.
68
- const GOOGLE_GOTO_HOST = /^(?:www\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?$/;
69
- export const isGotoRedirect = (href: string): boolean => {
70
- try {
71
- const u = new URL(href);
72
- return (
73
- u.protocol === "https:" &&
74
- u.username === "" &&
75
- u.password === "" &&
76
- u.port === "" &&
77
- GOOGLE_GOTO_HOST.test(u.hostname.toLowerCase()) &&
78
- u.pathname === "/goto"
79
- );
80
- } catch {
81
- return false;
82
- }
83
- };
84
-
85
- export type ResolvedTargets = ReadonlyMap<string, string>;
86
-
87
- // Direct external hrefs win. A /goto wrapper is opaque: the server-resolved redirect target is exact, and the cite is
88
- // a display breadcrumb that only approximates the path, so it is the fallback when no target was resolved.
89
- const resolveResultUrl = (anchor: SerpAnchor, resolved: ResolvedTargets): string | undefined => {
90
- if (!isGotoRedirect(anchor.href)) {
91
- const unwrapped = unwrapRedirect(anchor.href);
92
- return isExternalResult(unwrapped) ? unwrapped : undefined;
93
- }
94
- const target = resolved.get(anchor.href);
95
- if (target !== undefined && isExternalResult(target)) return target;
96
- if (anchor.cite !== undefined && anchor.cite.trim() !== "") return urlFromCite(anchor.cite);
97
- return undefined;
98
- };
99
-
100
- // Any Google property, on any TLD (google.com, google.co.uk, consent.google.de, googleusercontent.com), is never a
101
- // result: a sorry or consent page is where a rate-limited /goto redirect lands.
102
- const GOOGLE_PROPERTY = /(?:^|\.)google(?:usercontent)?\.(?:com?\.[a-z]{2}|[a-z]{2,3})$/;
103
-
104
- const isExternalResult = (url: string): boolean => {
105
- if (!/^https?:\/\//i.test(url)) return false;
106
- try {
107
- const host = new URL(url).hostname.toLowerCase();
108
- if (GOOGLE_PROPERTY.test(host)) return false;
109
- return true;
110
- } catch {
111
- return false;
112
- }
113
- };
114
-
115
- const dedupeKey = (url: string): string => {
116
- try {
117
- const u = new URL(url);
118
- return `${u.origin}${u.pathname.replace(/\/$/, "")}`.toLowerCase();
119
- } catch {
120
- return url.toLowerCase();
121
- }
122
- };
123
-
124
- export const parseGoogleSerp = (
125
- anchors: ReadonlyArray<SerpAnchor>,
126
- limit: number,
127
- resolved: ResolvedTargets = new Map(),
128
- ): ReadonlyArray<SearchResult> => {
129
- const seen = new Set<string>();
130
- const results: SearchResult[] = [];
131
- for (const anchor of anchors) {
132
- if (results.length >= limit) break;
133
- const url = resolveResultUrl(anchor, resolved);
134
- if (url === undefined) continue;
135
- const key = dedupeKey(url);
136
- if (seen.has(key)) continue;
137
- seen.add(key);
138
- results.push({
139
- title: anchor.heading.trim(),
140
- url,
141
- snippet: anchor.snippet.trim(),
142
- rank: results.length + 1,
143
- });
144
- }
145
- return results;
146
- };
147
-
148
- export const classifySerp = (pageText: string, resultCount: number, anchorCount: number): SerpVerdict => {
149
- if (resultCount > 0) return "ok";
150
- const lower = pageText.toLowerCase();
151
- if (CAPTCHA_MARKERS.some((marker) => lower.includes(marker))) return "captcha";
152
- // Anchors were found but every one was filtered out: the markup changed, not the result set.
153
- if (anchorCount > 0) return "extraction_mismatch";
154
- return "no_results";
155
- };