pi-chrome 0.10.1 → 0.11.0

package/README.md

@@ -69,7 +69,7 @@ pi-chrome v<version>
 
 By default, `chrome_*` clicks and keystrokes are **synthetic** DOM events (`event.isTrusted === false`). They drive React/Vue/Angular state correctly but **do not** satisfy Chrome's user-activation gates: clipboard write, fullscreen, file picker, and autoplay all need a real user gesture.
 
-pi-chrome can optionally route input through `chrome.debugger` (CDP `Input.dispatchMouseEvent` / `Input.dispatchKeyEvent`) so each event arrives as `isTrusted=true`, satisfies user-activation, and bypasses site bot-detection that filters synthetic events. The tradeoff: Chrome pins a yellow *"Pi Existing Chrome Profile Bridge started debugging this browser"* banner to the top of any debugged tab.
+pi-chrome can optionally route input through `chrome.debugger` (CDP `Input.dispatchMouseEvent` / `Input.dispatchKeyEvent`) so each event arrives as `isTrusted=true`, satisfies user-activation, and bypasses site bot-detection that filters synthetic events. The tradeoff: Chrome pins a yellow *"Pi Chrome Connector started debugging this browser"* banner to the top of any debugged tab.
 
 Usage:
 

package/extensions/chrome-profile-bridge/browser-extension/manifest.json

@@ -1,14 +1,14 @@
 {
   "manifest_version": 3,
-  "name": "Pi Existing Chrome Profile Bridge",
-  "version": "0.10.1",
-  "description": "Lets Pi control tabs in this existing Chrome profile via a local bridge at 127.0.0.1.",
+  "name": "Pi Chrome Connector",
+  "version": "0.11.0",
+  "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
   "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"],
   "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"],
   "background": {
     "service_worker": "service_worker.js"
   },
   "action": {
-    "default_title": "Pi Chrome Bridge"
+    "default_title": "Pi Chrome Connector"
   }
 }

package/extensions/chrome-profile-bridge/browser-extension/service_worker.js

@@ -1,12 +1,12 @@
 const BRIDGE_URL = "http://127.0.0.1:17318";
-const CLIENT_NAME = `Pi Chrome Bridge ${chrome.runtime.id}`;
+const CLIENT_NAME = `Pi Chrome Connector ${chrome.runtime.id}`;
 const POLL_ERROR_BACKOFF_MS = 2000;
 let polling = false;
 
 // =================== Trusted-input (CDP) layer ===================
 // Tracks which tabs we have attached chrome.debugger to, plus session-level mode.
 const attachedTabs = new Map(); // tabId -> { detachAt: number, pointer: {x,y} }
-let TRUSTED_MODE = "off"; // "off" | "on" | "auto"
+let TRUSTED_MODE = "auto"; // "off" | "on" | "auto" (default: smart retry only)
 const TRUSTED_IDLE_DETACH_MS = 15_000;
 const CDP_VERSION = "1.3";
 
@@ -35,6 +35,31 @@ function trustedStatus() {
   };
 }
 
+// Auto-upgrade: if synthetic result carries suggestTrusted=true, the bridge mode is "auto"
+// (default) or "on", and the caller didn't explicitly opt out, retry once with trusted CDP
+// path. Surfaces both results so callers can see what happened.
+async function maybeUpgradeToTrusted(kind, params, syntheticResult, trustedFn) {
+  if (!syntheticResult || !syntheticResult.suggestTrusted) return syntheticResult;
+  if (params && params.trusted === false) return syntheticResult;
+  if (TRUSTED_MODE === "off") return syntheticResult;
+  if (!chrome.debugger) return syntheticResult;
+  try {
+    const trustedResult = await trustedFn();
+    return {
+      ...trustedResult,
+      autoRetried: true,
+      autoRetryReason: syntheticResult.suggestReason || `${kind} produced no mutation`,
+      syntheticAttempt: { pageMutated: syntheticResult.pageMutated, suggestReason: syntheticResult.suggestReason },
+    };
+  } catch (error) {
+    return {
+      ...syntheticResult,
+      autoRetryAttempted: true,
+      autoRetryError: error?.message || String(error),
+    };
+  }
+}
+
 async function attachDebugger(tabId) {
   if (!chrome.debugger) throw new Error("chrome.debugger API unavailable; reload the extension to grant the new permission");
   if (attachedTabs.has(tabId)) {
@@ -485,9 +510,11 @@ async function dispatch(action, params) {
       ]);
     case "page.evaluate":
       return evaluateInTab(params);
-    case "page.click":
+    case "page.click": {
       if (await wantsTrusted(params)) return trustedClick(params);
-      return executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]);
+      const synth = await executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]);
+      return await maybeUpgradeToTrusted("click", params, synth, () => trustedClick(params));
+    }
     case "page.hover":
       if (await wantsTrusted(params)) return trustedHover(params);
       return executeActionInTab(params, hoverPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]);
@@ -496,9 +523,11 @@ async function dispatch(action, params) {
       return executeActionInTab(params, dragPage, [params.fromUid ?? null, params.fromSelector ?? null, params.fromX ?? null, params.fromY ?? null, params.toUid ?? null, params.toSelector ?? null, params.toX ?? null, params.toY ?? null, params.steps ?? 12]);
     case "page.upload":
       return executeActionInTab(params, uploadFiles, [params.selector ?? null, params.uid ?? null, params.files || []]);
-    case "page.type":
+    case "page.type": {
       if (await wantsTrusted(params)) return trustedType(params);
-      return executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]);
+      const synth = await executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]);
+      return await maybeUpgradeToTrusted("type", params, synth, () => trustedType(params));
+    }
     case "page.fill":
       if (await wantsTrusted(params)) return trustedFill(params);
       return executeActionInTab(params, fillPage, [params.selector ?? null, params.uid ?? null, params.text || "", params.submit === true]);
@@ -1262,23 +1291,46 @@ async function clickPage(selector, uid, x, y) {
   // Heuristic: if the clicked thing looks like a media play affordance and the page has paused
   // audio/video, the synthetic click may not unlock autoplay. Surface a warning.
   let autoplayHint;
-  const label = (point.element.getAttribute("aria-label") || point.element.textContent || "").toLowerCase();
-  if (/^(play|start|begin|next|continue|unmute)/.test(label.trim())) {
+  const labelRaw = (point.element.getAttribute("aria-label") || point.element.textContent || "").trim();
+  const label = labelRaw.toLowerCase();
+  if (/^(play|start|begin|next|continue|unmute)/.test(label)) {
     const idleMedia = Array.from(document.querySelectorAll("audio,video")).some((m) => m.paused);
     if (idleMedia) autoplayHint = "This element looks like a media affordance and the page has paused media. Synthetic clicks do not satisfy user-activation gates; audio/video may not start.";
   }
+  const pageMutated = pageHash() !== before;
+  // Smart-auto retry hint: only set when synthetic produced no observable change AND the
+  // element looks gated, OR the page just emitted a user-activation rejection. The dispatcher
+  // uses this to decide whether to retry with trusted mode.
+  let suggestTrusted = false;
+  let suggestReason;
+  if (!pageMutated) {
+    if (autoplayHint) { suggestTrusted = true; suggestReason = "play/media affordance + idle media"; }
+    else if (/copy(\s|$)|paste|share|download|fullscreen|sign in with|continue with|allow|enable/i.test(label)) {
+      suggestTrusted = true; suggestReason = `label '${labelRaw.slice(0, 40)}' looks gated`;
+    } else {
+      // Inspect recent console errors for activation-gate rejections.
+      const recent = (state.console || []).slice(-8);
+      const hit = recent.find((e) => /NotAllowedError|Document is not focused|requires transient activation|gesture is required/.test(
+        (e.args || []).map((a) => typeof a === "string" ? a : (a && a.message) || JSON.stringify(a)).join(" ")
+      ));
+      if (hit) { suggestTrusted = true; suggestReason = "recent console error indicates user-activation gate"; }
+    }
+  }
   return {
     x: point.x,
     y: point.y,
     selector,
     uid,
     tag: point.element.tagName,
+    label: labelRaw.slice(0, 80) || undefined,
     isTrusted: false,
     defaultPrevented,
     elementVisible: visible,
     occludedBy: occluded || undefined,
-    pageMutated: pageHash() !== before,
+    pageMutated,
     autoplayHint,
+    suggestTrusted: suggestTrusted || undefined,
+    suggestReason,
   };
 }
 
@@ -1526,15 +1578,27 @@ async function typeIntoPage(selector, uid, text, pressEnter) {
   const before = pageHash();
   let element = elementBySelectorOrUid(selector, uid) || document.activeElement;
   if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element");
+  const initialValue = "value" in element ? element.value : (element.isContentEditable ? element.textContent : null);
   element.focus();
   if (!(element.isContentEditable || "value" in element)) throw new Error("Focused element is not text-editable");
   for (const ch of Array.from(text)) await typeCharacter(element, ch);
   if (pressEnter) pressKeyInPage("Enter");
+  const finalValue = "value" in element ? element.value : element.textContent;
+  const valueMatches = "value" in element ? element.value.includes(text) : (element.textContent || "").includes(text);
+  const pageMutated = pageHash() !== before;
+  // Smart-auto retry hint when typing didn't land at all (e.g., editor blocks synthetic input).
+  let suggestTrusted = false, suggestReason;
+  if (text.length > 0 && initialValue === finalValue) {
+    suggestTrusted = true;
+    suggestReason = "value did not change — editor likely rejects synthetic input";
+  }
   return {
     selector, uid, length: text.length, pressEnter,
     isTrusted: false,
-    valueMatches: "value" in element ? element.value.includes(text) : undefined,
-    pageMutated: pageHash() !== before,
+    valueMatches,
+    pageMutated,
+    suggestTrusted: suggestTrusted || undefined,
+    suggestReason,
   };
 }
 

package/extensions/chrome-profile-bridge/index.ts

@@ -471,7 +471,7 @@ Usage rules:
 					lines.push(
 						`✗ EXTENSION VERSION MISMATCH: companion extension is v${version.extensionVersion}, but pi-chrome is v${PI_CHROME_VERSION}.`,
 						`  All chrome_* tools will run with the OLD extension code until this is fixed.`,
-						`  Fix: open chrome://extensions and click reload on "Pi Existing Chrome Profile Bridge".`,
+						`  Fix: open chrome://extensions and click reload on "Pi Chrome Connector".`,
 						`  (Future version drifts will self-heal: the extension now polls pi-chrome's expected version and reloads itself.)`,
 					);
 				} else {
@@ -518,7 +518,14 @@ Usage rules:
 						permissionGranted?: boolean;
 					};
 					if (status.permissionGranted) {
-						lines.push(`✓ Trusted-input mode available via chrome.debugger (current: ${status.mode ?? "off"}${status.attachedTabs && status.attachedTabs.length ? `; attached to tab ${status.attachedTabs.join(",")}` : ""}). Pass trusted=true on chrome_click/type/etc, or run /chrome-trusted on, to satisfy isTrusted + user-activation gates.`);
+						const attached = status.attachedTabs && status.attachedTabs.length ? `; attached to tab ${status.attachedTabs.join(",")}` : "";
+						const note =
+							status.mode === "auto"
+								? " — smart-retry enabled: synthetic input runs first; if a click/type produced no page change AND the target looks gated, the call is automatically re-run with trusted CDP (yellow debugger banner appears only for that retry)."
+								: status.mode === "on"
+									? " — every chrome_* call goes through CDP; the yellow debugger banner is visible while attached."
+									: " — synthetic events only; pass trusted=true on chrome_click/type/etc, or switch to auto/on with /chrome-trusted, when isTrusted or user-activation gates matter.";
+						lines.push(`✓ Trusted-input mode available via chrome.debugger (current: ${status.mode ?? "off"}${attached}).${note}`);
 					} else {
 						lines.push(`⚠ chrome.debugger API unavailable. The extension is missing the "debugger" permission — reload the extension in chrome://extensions and accept the new permission prompt.`);
 					}
@@ -546,27 +553,83 @@ Usage rules:
 			return matches.length > 0 ? matches : null;
 		},
 		handler: async (args, ctx) => {
-			const arg = (args || "").trim().toLowerCase();
-			if (arg === "status" || arg === "") {
-				try {
-					const status = (await bridge.send("trusted.status", {}, 5_000)) as { mode: string; attachedTabs: number[]; permissionGranted: boolean };
-					const attached = status.attachedTabs?.length ? ` (attached to tab ${status.attachedTabs.join(",")})` : "";
-					const perm = status.permissionGranted ? "" : " — chrome.debugger API unavailable; reload the extension and accept the new permission.";
-					ctx.ui.notify(`Trusted-input mode: ${status.mode}${attached}${perm}`, "info");
-				} catch (error) {
-					ctx.ui.notify(`Failed to read trusted mode: ${(error as Error).message}`, "warning");
+			const rawArg = (args || "").trim().toLowerCase();
+
+			// Resolve current status once for both branches (interactive picker + direct args).
+			let status: { mode: string; attachedTabs: number[]; permissionGranted: boolean } | undefined;
+			try {
+				status = (await bridge.send("trusted.status", {}, 5_000)) as typeof status;
+			} catch (error) {
+				ctx.ui.notify(`Failed to read trusted mode: ${(error as Error).message}`, "warning");
+				return;
+			}
+			if (!status) return;
+
+			if (!status.permissionGranted) {
+				ctx.ui.notify(
+					"chrome.debugger API unavailable — the extension is missing the 'debugger' permission. Open chrome://extensions, reload 'Pi Chrome Connector', and accept the new permission prompt.",
+					"warning",
+				);
+				return;
+			}
+
+			const attached = status.attachedTabs?.length ? ` — currently attached to tab ${status.attachedTabs.join(",")}` : "";
+			const current = status.mode;
+
+			let target = rawArg;
+			if (target === "status") {
+				ctx.ui.notify(`Trusted-input mode: ${current}${attached}`, "info");
+				return;
+			}
+			if (!target) {
+				// Interactive picker. Show current mode + tradeoffs in each label.
+				const options = [
+					`auto${current === "auto" ? " (current)" : ""} — default; synthetic first, retry with CDP only when a call looks gated`,
+					`off${current === "off" ? " (current)" : ""} — synthetic DOM events only; never auto-retry`,
+					`on${current === "on" ? " (current)" : ""} — every chrome_* call goes through CDP (yellow debugger banner permanently visible)`,
+					`status — print current mode and any attached tabs\u2026`,
+				];
+				const picked = await ctx.ui.select(
+					`Trusted-input mode (current: ${current}${attached})`,
+					options,
+				);
+				if (!picked) return; // cancelled
+				if (picked.startsWith("on")) target = "on";
+				else if (picked.startsWith("off")) target = "off";
+				else if (picked.startsWith("auto")) target = "auto";
+				else if (picked.startsWith("status")) {
+					ctx.ui.notify(`Trusted-input mode: ${current}${attached}`, "info");
+					return;
 				}
+			}
+
+			if (!["on", "off", "auto"].includes(target)) {
+				ctx.ui.notify(`Unknown argument '${rawArg}'. Use: on | off | auto | status, or run /chrome-trusted with no args for a picker.`, "warning");
 				return;
 			}
-			if (!["on", "off", "auto"].includes(arg)) {
-				ctx.ui.notify(`Unknown argument '${arg}'. Use: on | off | auto | status`, "warning");
+
+			if (target === current) {
+				ctx.ui.notify(`Trusted-input mode already ${current}.`, "info");
 				return;
 			}
+
+			// Extra confirmation only on first-time "on" (warn about banner).
+			if (target === "on" && current === "off") {
+				const ok = await ctx.ui.confirm(
+					"Turn on trusted-input mode?",
+					"All chrome_* tools will dispatch through chrome.debugger (CDP). Events will arrive as isTrusted=true and satisfy user-activation gates (clipboard, fullscreen, autoplay, file picker).\n\nChrome will pin a yellow 'Pi Chrome Connector started debugging this browser' banner to the top of any debugged tab while attached. Clicking 'Cancel' on that banner detaches the debugger.",
+				);
+				if (!ok) {
+					ctx.ui.notify("Trusted-input mode unchanged.", "info");
+					return;
+				}
+			}
+
 			try {
-				const result = (await bridge.send("trusted.mode", { mode: arg }, 5_000)) as { mode: string };
+				const result = (await bridge.send("trusted.mode", { mode: target }, 5_000)) as { mode: string };
 				if (result.mode === "on") {
 					ctx.ui.notify(
-						"Trusted-input mode ON. All chrome_* tools now dispatch through chrome.debugger (CDP). Chrome will show a yellow 'started debugging this browser' banner. Events arrive as isTrusted=true and satisfy user-activation gates.",
+						"Trusted-input mode ON. chrome_* tools now dispatch through chrome.debugger. The yellow debugger banner will appear when Chrome is next driven.",
 						"info",
 					);
 				} else if (result.mode === "off") {
@@ -779,7 +842,7 @@ Usage rules:
 		name: "chrome_click",
 		label: "Chrome Click",
 		description:
-			"Click a snapshot uid, CSS selector, or viewport coordinate in an existing Chrome tab through the companion extension. Defaults to synthetic DOM events (isTrusted=false). Pass trusted=true (or run /chrome-trusted on) to route through chrome.debugger so events arrive as browser-trusted and satisfy user-activation gates — Chrome shows a yellow 'started debugging' banner while attached. Pass includeSnapshot=true to return a fresh snapshot after the click.",
+			"Click a snapshot uid, CSS selector, or viewport coordinate. Default 'auto' mode runs synthetic DOM events first and silently retries with trusted CDP only when the click looks gated (no page change + affordance label matches play/copy/share/sign-in/etc, or a recent NotAllowedError). The yellow 'started debugging' banner appears only when the retry actually happens. Pass trusted=true to force CDP for this call (banner appears immediately). Pass trusted=false to skip retry. Pass includeSnapshot=true to return a fresh snapshot after the click.",
 		promptSnippet: "Click page elements in Chrome by snapshot uid, selector, or viewport coordinate.",
 		parameters: Type.Object({
 			uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot. Prefer uid over selector after taking a snapshot." })),
@@ -812,7 +875,7 @@ Usage rules:
 		name: "chrome_type",
 		label: "Chrome Type",
 		description:
-			"Focus an optional snapshot uid or CSS selector, then type text into an existing Chrome tab. Defaults to synthetic per-character keydown/beforeinput/input/keyup sequence. Pass trusted=true (or run /chrome-trusted on) to route through chrome.debugger so each keystroke is browser-trusted (isTrusted=true). Pass includeSnapshot=true to return a fresh snapshot after typing.",
+			"Focus an optional snapshot uid or CSS selector, then type text. Default 'auto' mode runs synthetic per-character keydown/beforeinput/input/keyup first; if the input value doesn't change at all (editor rejected synthetic input) the call is silently retried through chrome.debugger so each keystroke is browser-trusted (isTrusted=true). Pass trusted=true to force CDP for this call. Pass trusted=false to skip retry. Pass includeSnapshot=true to return a fresh snapshot after typing.",
 		promptSnippet: "Type text into Chrome, optionally focusing a snapshot uid or selector first.",
 		parameters: Type.Object({
 			text: Type.String(),

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pi-chrome",
-	"version": "0.10.1",
+	"version": "0.11.0",
 	"description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).",
 	"keywords": [
 		"pi-package",