pi-browser-harness 0.14.1 → 0.16.0

package/src/domains/search/engines/google.ts

@@ -0,0 +1,95 @@
+import {
+  classifySerpWith,
+  externalResult,
+  parseSerp,
+  type ResolvedTargets,
+  type SearchEngine,
+  type SearchResult,
+  type SerpAnchor,
+  type SerpVerdict,
+  urlFromCite,
+} from "../serp";
+
+const CAPTCHA_MARKERS: ReadonlyArray<string> = [
+  "unusual traffic",
+  "not a robot",
+  "recaptcha",
+  "detected unusual",
+  "systems have detected",
+  "before you continue to google",
+  "our systems have detected unusual traffic",
+];
+
+// Any Google property, on any TLD (google.com, google.co.uk, consent.google.de, googleusercontent.com), is never a
+// result: a sorry or consent page is where a rate-limited /goto redirect lands.
+const GOOGLE_PROPERTY = /(?:^|\.)google(?:usercontent)?\.(?:com?\.[a-z]{2}|[a-z]{2,3})$/;
+
+const isExternalResult = externalResult((host) => GOOGLE_PROPERTY.test(host));
+
+const unwrapRedirect = (href: string): string => {
+  const wrapper = /^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i.exec(href);
+  if (!wrapper) return href;
+  const query = href.slice(href.indexOf("?") + 1);
+  const target = new URLSearchParams(query).get("q");
+  return target ?? href;
+};
+
+// Rebuilds a url from Google's visible breadcrumb cite, the only place the target survives once Google wraps hrefs
+// in /goto.
+export const googleUrlFromCite = (cite: string): string | undefined => urlFromCite(cite, isExternalResult);
+
+// Only a real Google /goto url counts: the daemon fetches these, so a look-alike host, userinfo, a port or http
+// must never qualify. Hosts are google.com or www.google.<tld>, nothing else.
+const GOOGLE_GOTO_HOST = /^(?:www\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?$/;
+export const isGotoRedirect = (href: string): boolean => {
+  try {
+    const u = new URL(href);
+    return (
+      u.protocol === "https:" &&
+      u.username === "" &&
+      u.password === "" &&
+      u.port === "" &&
+      GOOGLE_GOTO_HOST.test(u.hostname.toLowerCase()) &&
+      u.pathname === "/goto"
+    );
+  } catch {
+    return false;
+  }
+};
+
+// Direct external hrefs win. A /goto wrapper is opaque: the server-resolved redirect target is exact, and the cite is
+// a display breadcrumb that only approximates the path, so it is the fallback when no target was resolved.
+const resultUrl = (anchor: SerpAnchor, resolved: ResolvedTargets): string | undefined => {
+  if (!isGotoRedirect(anchor.href)) {
+    const unwrapped = unwrapRedirect(anchor.href);
+    return isExternalResult(unwrapped) ? unwrapped : undefined;
+  }
+  const target = resolved.get(anchor.href);
+  if (target !== undefined && isExternalResult(target)) return target;
+  if (anchor.cite !== undefined && anchor.cite.trim() !== "") return googleUrlFromCite(anchor.cite);
+  return undefined;
+};
+
+export const google: SearchEngine = {
+  id: "google",
+  name: "Google",
+  buildUrl: (query, limit) => `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`,
+  selectors: {
+    root: "#search, #rso, #main",
+    heading: "a h3",
+    containers: ["div.g", "div[data-hveid]"],
+    cite: "cite",
+  },
+  captchaMarkers: CAPTCHA_MARKERS,
+  resultUrl,
+  needsRedirectResolve: isGotoRedirect,
+};
+
+export const parseGoogleSerp = (
+  anchors: ReadonlyArray<SerpAnchor>,
+  limit: number,
+  resolved: ResolvedTargets = new Map(),
+): ReadonlyArray<SearchResult> => parseSerp(google, anchors, limit, resolved);
+
+export const classifyGoogleSerp = (pageText: string, resultCount: number, anchorCount: number): SerpVerdict =>
+  classifySerpWith(CAPTCHA_MARKERS, pageText, resultCount, anchorCount);

package/src/domains/search/engines/index.ts

@@ -0,0 +1,13 @@
+// Composition root for the built-in engines. The order of mounts is the default fallback ladder. To add an engine,
+// write engines/<id>.ts exporting a SearchEngine and mount it here — or mount it onto defaultEngines() and inject the
+// result into createWebSearchTool from outside.
+import { bing } from "./bing";
+import { brave } from "./brave";
+import { duckduckgo } from "./duckduckgo";
+import { google } from "./google";
+import { createEngineRegistry, type EngineRegistry } from "./registry";
+
+export const defaultEngines = (): EngineRegistry =>
+  createEngineRegistry().mount(google).mount(bing).mount(duckduckgo).mount(brave);
+
+export { createEngineRegistry, type EngineRegistry } from "./registry";

package/src/domains/search/engines/registry.ts

@@ -0,0 +1,29 @@
+import type { SearchEngine } from "../serp";
+
+// An immutable set of mounted engines. Mount order is the fallback ladder: the first engine mounted is tried first.
+export type EngineRegistry = {
+  // A new registry with `engine` appended to the ladder. Throws on a duplicate id: that is a wiring bug, caught at
+  // composition time rather than surfacing as a confusing search.
+  readonly mount: (engine: SearchEngine) => EngineRegistry;
+  readonly get: (id: string) => SearchEngine | undefined;
+  readonly ids: () => ReadonlyArray<string>;
+  readonly ladder: () => ReadonlyArray<SearchEngine>;
+};
+
+const ENGINE_ID = /^[a-z][a-z0-9-]*$/;
+
+const fromEngines = (engines: ReadonlyArray<SearchEngine>): EngineRegistry => {
+  const byId = new Map(engines.map((e) => [e.id, e] as const));
+  return {
+    mount: (engine) => {
+      if (!ENGINE_ID.test(engine.id)) throw new Error(`search engine id "${engine.id}" must match ${String(ENGINE_ID)}`);
+      if (byId.has(engine.id)) throw new Error(`search engine "${engine.id}" is already mounted`);
+      return fromEngines([...engines, engine]);
+    },
+    get: (id) => byId.get(id),
+    ids: () => engines.map((e) => e.id),
+    ladder: () => engines,
+  };
+};
+
+export const createEngineRegistry = (): EngineRegistry => fromEngines([]);

package/src/domains/search/redirect-resolver.ts

@@ -2,7 +2,7 @@
 // following it. The blob cannot be decoded client-side and the page's <cite> is only a display breadcrumb, so this
 // is the one source of the exact url. Google answers the redirect to a cookie-less GET; HEAD returns 200 instead.
 
-import { isGotoRedirect } from "./google-serp";
+import { isGotoRedirect } from "./engines/google";
 
 const REDIRECT_STATUSES: ReadonlySet<number> = new Set([301, 302, 303, 307, 308]);
 

package/src/domains/search/serp.ts

@@ -0,0 +1,157 @@
+// Engine-neutral SERP vocabulary and parsing. Each engine (engines/*.ts) supplies how to find result anchors on its
+// page and how to turn an anchor into its target url; ranking, dedupe and verdicts are shared here.
+
+export type SerpAnchor = {
+  readonly href: string;
+  readonly heading: string;
+  readonly snippet: string;
+  // Visible breadcrumb url ("https://host › path › seg"); the fallback when the href is an opaque redirect.
+  readonly cite?: string;
+};
+
+export type SerpExtraction = {
+  readonly anchors: ReadonlyArray<SerpAnchor>;
+  readonly pageText: string;
+};
+
+export type SearchResult = {
+  readonly title: string;
+  readonly url: string;
+  readonly snippet: string;
+  readonly rank: number;
+};
+
+export type SerpVerdict = "ok" | "captcha" | "no_results" | "extraction_mismatch";
+
+// Redirect hrefs resolved server-side (Google /goto), keyed by href.
+export type ResolvedTargets = ReadonlyMap<string, string>;
+
+// Selectors handed to the page-side extractor. Every result is found from its heading: the title element is either
+// the result link itself or sits inside / around it.
+export type SerpSelectors = {
+  // Results column; the extractor falls back to <body> when none match.
+  readonly root: string;
+  // The title element of each organic result.
+  readonly heading: string;
+  // Result-block selectors, tried in order with closest(); the first match is the result's container.
+  readonly containers: ReadonlyArray<string>;
+  // Snippet element inside the container; without it the snippet is the container text minus the heading.
+  readonly snippet?: string;
+  // Breadcrumb element inside the container.
+  readonly cite: string;
+};
+
+// A pluggable search engine. Engines are plain values mounted into an EngineRegistry (engines/registry.ts); nothing
+// else in the search code knows which engines exist.
+export type SearchEngine = {
+  // Stable lower-case identifier the agent passes as `engine` ("google", "bing").
+  readonly id: string;
+  // Human name, used in error messages ("Bing served a CAPTCHA").
+  readonly name: string;
+  readonly buildUrl: (query: string, limit: number) => string;
+  readonly selectors: SerpSelectors;
+  // Lower-case phrases that mark a bot wall / challenge page.
+  readonly captchaMarkers: ReadonlyArray<string>;
+  // The anchor's external target, or undefined when it is not an organic web result.
+  readonly resultUrl: (anchor: SerpAnchor, resolved: ResolvedTargets) => string | undefined;
+  // For engines that render results with JavaScript after the load event: the tool waits (briefly) for this selector
+  // to match before extracting. A page that never matches is still extracted and classified (captcha / no_results).
+  readonly readySelector?: string;
+  // Hrefs that can only be decoded by asking the engine's server for the redirect (Google /goto).
+  readonly needsRedirectResolve?: (href: string) => boolean;
+};
+
+const CITE_SEPARATOR = /\s*›\s*/;
+const TRUNCATION = /(?:\.\.\.|…)/;
+
+export const isWebUrl = (url: string): boolean => {
+  if (!/^https?:\/\//i.test(url)) return false;
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+export const hostOf = (url: string): string | undefined => {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return undefined;
+  }
+};
+
+// Rebuilds a url from a visible breadcrumb cite. Segments after a truncation marker are unknowable, so the path stops
+// at the last complete segment rather than guessing. Returns only urls `isResult` accepts.
+export const urlFromCite = (cite: string, isResult: (url: string) => boolean): string | undefined => {
+  const segments = cite
+    .trim()
+    .split(CITE_SEPARATOR)
+    .map((segment) => segment.trim())
+    .filter((segment) => segment.length > 0);
+  const origin = segments[0];
+  if (origin === undefined || TRUNCATION.test(origin)) return undefined;
+  // A cite naming any other scheme (ftp:, mailto:) is not a web result; prefixing it would forge a bogus https url.
+  if (/^[a-z][a-z0-9+.-]*:/i.test(origin) && !/^https?:\/\//i.test(origin)) return undefined;
+  const withScheme = /^https?:\/\//i.test(origin) ? origin : `https://${origin}`;
+  const path: string[] = [];
+  for (const segment of segments.slice(1)) {
+    if (TRUNCATION.test(segment)) break;
+    path.push(segment);
+  }
+  const url = [withScheme.replace(/\/$/, ""), ...path].join("/");
+  return isResult(url) ? url : undefined;
+};
+
+// A web url whose host is not one of the engine's own properties.
+export const externalResult =
+  (isOwnHost: (host: string) => boolean) =>
+  (url: string): boolean => {
+    if (!isWebUrl(url)) return false;
+    const host = hostOf(url);
+    return host !== undefined && !isOwnHost(host);
+  };
+
+const dedupeKey = (url: string): string => {
+  try {
+    const u = new URL(url);
+    return `${u.origin}${u.pathname.replace(/\/$/, "")}`.toLowerCase();
+  } catch {
+    return url.toLowerCase();
+  }
+};
+
+export const parseSerp = (
+  engine: SearchEngine,
+  anchors: ReadonlyArray<SerpAnchor>,
+  limit: number,
+  resolved: ResolvedTargets = new Map(),
+): ReadonlyArray<SearchResult> => {
+  const seen = new Set<string>();
+  const results: SearchResult[] = [];
+  for (const anchor of anchors) {
+    if (results.length >= limit) break;
+    const url = engine.resultUrl(anchor, resolved);
+    if (url === undefined) continue;
+    const key = dedupeKey(url);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    results.push({ title: anchor.heading.trim(), url, snippet: anchor.snippet.trim(), rank: results.length + 1 });
+  }
+  return results;
+};
+
+export const classifySerpWith = (
+  captchaMarkers: ReadonlyArray<string>,
+  pageText: string,
+  resultCount: number,
+  anchorCount: number,
+): SerpVerdict => {
+  if (resultCount > 0) return "ok";
+  const lower = pageText.toLowerCase();
+  if (captchaMarkers.some((marker) => lower.includes(marker))) return "captcha";
+  // Anchors were found but every one was filtered out: the markup changed, not the result set.
+  if (anchorCount > 0) return "extraction_mismatch";
+  return "no_results";
+};