pi-agent-browser-native 0.2.34 → 0.2.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +25 -15
- package/docs/ARCHITECTURE.md +19 -13
- package/docs/COMMAND_REFERENCE.md +274 -44
- package/docs/ELECTRON.md +3 -3
- package/docs/RELEASE.md +11 -11
- package/docs/REQUIREMENTS.md +5 -5
- package/docs/SUPPORT_MATRIX.md +43 -24
- package/docs/TOOL_CONTRACT.md +50 -30
- package/extensions/agent-browser/index.ts +518 -2402
- package/extensions/agent-browser/lib/argv-descriptor.ts +90 -0
- package/extensions/agent-browser/lib/argv-grammar.ts +128 -0
- package/extensions/agent-browser/lib/command-policy.ts +71 -0
- package/extensions/agent-browser/lib/command-taxonomy.ts +336 -0
- package/extensions/agent-browser/lib/electron/cleanup.ts +1 -0
- package/extensions/agent-browser/lib/executable-path.ts +19 -0
- package/extensions/agent-browser/lib/input-modes/params.ts +6 -6
- package/extensions/agent-browser/lib/orchestration/batch-stdin.ts +65 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/browser-action-model.ts +154 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts +149 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +56 -30
- package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts +13 -3
- package/extensions/agent-browser/lib/orchestration/browser-run/index.ts +33 -27
- package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +48 -22
- package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +39 -10
- package/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts +93 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts +98 -124
- package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +40 -1
- package/extensions/agent-browser/lib/orchestration/electron-host/index.ts +860 -0
- package/extensions/agent-browser/lib/playbook.ts +10 -10
- package/extensions/agent-browser/lib/prompt-policy.ts +122 -0
- package/extensions/agent-browser/lib/results/action-recommendations.ts +3 -23
- package/extensions/agent-browser/lib/results/presentation/navigation.ts +2 -34
- package/extensions/agent-browser/lib/runtime.ts +93 -227
- package/extensions/agent-browser/lib/session-page-state.ts +31 -14
- package/extensions/agent-browser/lib/temp.ts +148 -23
- package/package.json +4 -4
- package/scripts/agent-browser-capability-baseline.mjs +198 -1
|
@@ -45,7 +45,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
45
45
|
role: Type.Optional(Type.String({ description: "Role locator value for locator=role. May be used instead of value; when both are set they must match." })),
|
|
46
46
|
name: Type.Optional(Type.String({ description: "Accessible name filter for locator=role; compiles to --name <name>." })),
|
|
47
47
|
session: Type.Optional(Type.String({ description: "Optional upstream session name; prepends --session <name> before the compiled command." })),
|
|
48
|
-
}),
|
|
48
|
+
}, { additionalProperties: false }),
|
|
49
49
|
),
|
|
50
50
|
qa: Type.Optional(
|
|
51
51
|
Type.Union([
|
|
@@ -79,7 +79,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
79
79
|
componentName: Type.Optional(Type.String({ description: "Component name to correlate with react tree output and bounded local workspace search." })),
|
|
80
80
|
includeDomHints: Type.Optional(Type.Boolean({ description: "Whether selector lookups should inspect DOM HTML attributes for source-like metadata. Defaults to true." })),
|
|
81
81
|
maxWorkspaceFiles: Type.Optional(Type.Number({ description: "Maximum local source files to scan when componentName is provided. Defaults to 2000 and cannot exceed 5000.", minimum: 1, maximum: SOURCE_LOOKUP_MAX_WORKSPACE_FILES })),
|
|
82
|
-
}, { description: "EXPERIMENTAL: local UI-to-source candidates only (confidence/evidence, not guaranteed mappings). Compiles to batch; mutually exclusive with other input modes." }),
|
|
82
|
+
}, { additionalProperties: false, description: "EXPERIMENTAL: local UI-to-source candidates only (confidence/evidence, not guaranteed mappings). Compiles to batch; mutually exclusive with other input modes." }),
|
|
83
83
|
),
|
|
84
84
|
networkSourceLookup: Type.Optional(
|
|
85
85
|
Type.Object({
|
|
@@ -88,7 +88,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
88
88
|
session: Type.Optional(Type.String({ description: "Optional upstream session name; prepends --session <name> before the generated batch." })),
|
|
89
89
|
url: Type.Optional(Type.String({ description: "Optional failed request URL or URL fragment to correlate with local source." })),
|
|
90
90
|
maxWorkspaceFiles: Type.Optional(Type.Number({ description: "Maximum local source files to scan for URL literals. Defaults to 2000 and cannot exceed 5000.", minimum: 1, maximum: SOURCE_LOOKUP_MAX_WORKSPACE_FILES })),
|
|
91
|
-
}, { description: "EXPERIMENTAL: failed-request-to-source candidates only (initiator metadata and bounded workspace URL literals; not definitive blame). Compiles to batch; mutually exclusive with other input modes." }),
|
|
91
|
+
}, { additionalProperties: false, description: "EXPERIMENTAL: failed-request-to-source candidates only (initiator metadata and bounded workspace URL literals; not definitive blame). Compiles to batch; mutually exclusive with other input modes." }),
|
|
92
92
|
),
|
|
93
93
|
electron: Type.Optional(
|
|
94
94
|
Type.Union([
|
|
@@ -172,10 +172,10 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
172
172
|
values: Type.Optional(Type.Array(Type.String({ description: "Option value for select steps." }), { description: "One or more option values for select steps.", minItems: 1 })),
|
|
173
173
|
path: Type.Optional(Type.String({ description: "Artifact/download path for waitForDownload or screenshot steps." })),
|
|
174
174
|
milliseconds: Type.Optional(Type.Number({ description: "Milliseconds for wait steps." })),
|
|
175
|
-
}),
|
|
175
|
+
}, { additionalProperties: false }),
|
|
176
176
|
{ minItems: 1 },
|
|
177
177
|
),
|
|
178
|
-
}),
|
|
178
|
+
}, { additionalProperties: false }),
|
|
179
179
|
),
|
|
180
180
|
stdin: Type.Optional(Type.String({ description: "Optional raw stdin content; only supported for batch, eval --stdin, auth save --password-stdin, and is generated internally by job, qa, sourceLookup, or networkSourceLookup mode. Do not use with electron mode." })),
|
|
181
181
|
sessionMode: Type.Optional(
|
|
@@ -185,4 +185,4 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
185
185
|
default: DEFAULT_SESSION_MODE,
|
|
186
186
|
}),
|
|
187
187
|
),
|
|
188
|
-
});
|
|
188
|
+
}, { additionalProperties: false });
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export type BatchCommandStep = [string, ...string[]];
|
|
2
|
+
|
|
3
|
+
function validateUserBatchStep(step: unknown, index: number): { error: string; ok: false } | { ok: true; step: BatchCommandStep } {
|
|
4
|
+
if (!Array.isArray(step)) {
|
|
5
|
+
return {
|
|
6
|
+
error: `agent_browser batch stdin step ${index} must be a non-empty array of string command tokens.`,
|
|
7
|
+
ok: false,
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
if (step.length === 0) {
|
|
11
|
+
return {
|
|
12
|
+
error: `agent_browser batch stdin step ${index} must not be empty.`,
|
|
13
|
+
ok: false,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
const invalidTokenIndex = step.findIndex((token) => typeof token !== "string");
|
|
17
|
+
if (invalidTokenIndex !== -1) {
|
|
18
|
+
return {
|
|
19
|
+
error: `agent_browser batch stdin step ${index} token ${invalidTokenIndex} must be a string.`,
|
|
20
|
+
ok: false,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
return { ok: true, step: step as BatchCommandStep };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function parseBatchStdinJsonArray(stdin: string | undefined): { error?: string; steps?: unknown[] } {
|
|
27
|
+
if (stdin === undefined) {
|
|
28
|
+
return { steps: [] };
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
const parsed = JSON.parse(stdin) as unknown;
|
|
32
|
+
if (!Array.isArray(parsed)) {
|
|
33
|
+
return { error: "agent_browser batch stdin must be a JSON array of command steps." };
|
|
34
|
+
}
|
|
35
|
+
return { steps: parsed };
|
|
36
|
+
} catch (error) {
|
|
37
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
38
|
+
return { error: `agent_browser batch stdin could not be parsed as JSON: ${message}` };
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function parseUserBatchStdin(stdin: string | undefined): { error?: string; steps?: BatchCommandStep[] } {
|
|
43
|
+
const parsed = parseBatchStdinJsonArray(stdin);
|
|
44
|
+
if (parsed.error || parsed.steps === undefined) {
|
|
45
|
+
return parsed.error ? { error: parsed.error } : { steps: [] };
|
|
46
|
+
}
|
|
47
|
+
const steps: BatchCommandStep[] = [];
|
|
48
|
+
for (const [index, rawStep] of parsed.steps.entries()) {
|
|
49
|
+
const validated = validateUserBatchStep(rawStep, index);
|
|
50
|
+
if (!validated.ok) {
|
|
51
|
+
return { error: validated.error };
|
|
52
|
+
}
|
|
53
|
+
steps.push(validated.step);
|
|
54
|
+
}
|
|
55
|
+
return { steps };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function parseValidBatchStepEntries(stdin: string | undefined): Array<{ index: number; step: BatchCommandStep }> {
|
|
59
|
+
const parsed = parseBatchStdinJsonArray(stdin);
|
|
60
|
+
if (parsed.error || parsed.steps === undefined) return [];
|
|
61
|
+
return parsed.steps.flatMap((step, index) => {
|
|
62
|
+
const validated = validateUserBatchStep(step, index);
|
|
63
|
+
return validated.ok ? [{ index, step: validated.step }] : [];
|
|
64
|
+
});
|
|
65
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Purpose: Normalize planned browser argv into a small action model for prompt-derived guards.
|
|
3
|
+
* Responsibilities: Map command tokens and batch stdin steps to click-like and keyboard-submit actions with target labels.
|
|
4
|
+
* Scope: Best-effort finalizing-action detection only; does not model eval, generic fill/type, or non-Enter keyboard flows.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { SessionRefSnapshot } from "../../session-page-state.js";
|
|
8
|
+
import { parseValidBatchStepEntries } from "../batch-stdin.js";
|
|
9
|
+
|
|
10
|
+
const FINAL_ACTION_PATTERN = /\b(?:finish|place\s+(?:the\s+)?order|submit\s+(?:the\s+)?order|complete\s+(?:the\s+)?order|confirm\s+(?:the\s+)?order|purchase|buy\s+now|pay\s+now|finali[sz]e|submit\s+payment|checkout\s+complete)\b/i;
|
|
11
|
+
|
|
12
|
+
const CLICK_LIKE_COMMANDS = new Set(["click", "dblclick", "tap"]);
|
|
13
|
+
const FIND_CLICK_ACTIONS = new Set(["click", "dblclick", "tap"]);
|
|
14
|
+
const KEYBOARD_SUBMIT_KEYS = new Set(["enter", "return"]);
|
|
15
|
+
|
|
16
|
+
export type BrowserFinalizingActionKind = "click-like" | "keyboard-submit";
|
|
17
|
+
|
|
18
|
+
export interface BrowserFinalizingAction {
|
|
19
|
+
command: string[];
|
|
20
|
+
kind: BrowserFinalizingActionKind;
|
|
21
|
+
stepIndex?: number;
|
|
22
|
+
targetLabel?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const STOP_BOUNDARY_GUARD_SCOPE = {
|
|
26
|
+
covered: [
|
|
27
|
+
"standalone click, dblclick, and tap",
|
|
28
|
+
"find … click|dblclick|tap",
|
|
29
|
+
"batch steps with the click-like shapes above",
|
|
30
|
+
"press <key> and key <key> when key is Enter or Return",
|
|
31
|
+
],
|
|
32
|
+
excluded: [
|
|
33
|
+
"eval --stdin and other scripted activation",
|
|
34
|
+
"fill, type, select, drag, and upload without an explicit click-like command",
|
|
35
|
+
"keyboard type/inserttext and keyboard shortcuts other than Enter/Return",
|
|
36
|
+
"semanticAction and job/qa compiled plans unless their batch stdin contains a covered step",
|
|
37
|
+
],
|
|
38
|
+
} as const;
|
|
39
|
+
|
|
40
|
+
function normalizeTargetText(value: string): string {
|
|
41
|
+
return value
|
|
42
|
+
.replace(/[_-]+/g, " ")
|
|
43
|
+
.replace(/[\[\]{}()#.'\"=:/]+/g, " ")
|
|
44
|
+
.replace(/\s+/g, " ")
|
|
45
|
+
.trim();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function matchesFinalActionLabel(value: string | undefined): boolean {
|
|
49
|
+
return value !== undefined && FINAL_ACTION_PATTERN.test(normalizeTargetText(value));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function parseRefId(value: string | undefined): string | undefined {
|
|
53
|
+
if (!value) return undefined;
|
|
54
|
+
const trimmed = value.trim();
|
|
55
|
+
const candidate = trimmed.startsWith("@") ? trimmed.slice(1) : trimmed.startsWith("ref=") ? trimmed.slice(4) : trimmed;
|
|
56
|
+
return /^e\d+$/.test(candidate) ? candidate : undefined;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function getRefTargetLabel(refSnapshot: SessionRefSnapshot | undefined, refId: string | undefined): string | undefined {
|
|
60
|
+
if (!refId) return undefined;
|
|
61
|
+
const ref = refSnapshot?.refs?.[refId];
|
|
62
|
+
return ref ? [ref.role, ref.name].filter(Boolean).join(" ") : undefined;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function getFlagValue(tokens: string[], flag: string): string | undefined {
|
|
66
|
+
for (const [index, token] of tokens.entries()) {
|
|
67
|
+
if (token === flag) return tokens[index + 1];
|
|
68
|
+
if (token.startsWith(`${flag}=`)) return token.slice(flag.length + 1);
|
|
69
|
+
}
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function getClickLikeTargetLabel(command: string[], refSnapshot: SessionRefSnapshot | undefined): string | undefined {
|
|
74
|
+
const target = command[1];
|
|
75
|
+
return getRefTargetLabel(refSnapshot, parseRefId(target)) ?? target;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function getFindClickTargetLabel(command: string[]): string | undefined {
|
|
79
|
+
if (command[0] !== "find") return undefined;
|
|
80
|
+
const actionIndex = command.findIndex((token, index) => index >= 3 && FIND_CLICK_ACTIONS.has(token));
|
|
81
|
+
if (actionIndex === -1) return undefined;
|
|
82
|
+
return getFlagValue(command, "--name") ?? command[2];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function getKeyboardSubmitKey(command: string[]): string | undefined {
|
|
86
|
+
const commandName = command[0];
|
|
87
|
+
if (commandName === "press" || commandName === "key") return command[1];
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function collectActionsFromCommand(command: string[], refSnapshot: SessionRefSnapshot | undefined, stepIndex?: number): BrowserFinalizingAction[] {
|
|
92
|
+
const actions: BrowserFinalizingAction[] = [];
|
|
93
|
+
if (CLICK_LIKE_COMMANDS.has(command[0] ?? "")) {
|
|
94
|
+
actions.push({
|
|
95
|
+
command,
|
|
96
|
+
kind: "click-like",
|
|
97
|
+
stepIndex,
|
|
98
|
+
targetLabel: getClickLikeTargetLabel(command, refSnapshot),
|
|
99
|
+
});
|
|
100
|
+
return actions;
|
|
101
|
+
}
|
|
102
|
+
if (command[0] === "find") {
|
|
103
|
+
const actionIndex = command.findIndex((token, index) => index >= 3 && FIND_CLICK_ACTIONS.has(token));
|
|
104
|
+
if (actionIndex !== -1) {
|
|
105
|
+
actions.push({
|
|
106
|
+
command,
|
|
107
|
+
kind: "click-like",
|
|
108
|
+
stepIndex,
|
|
109
|
+
targetLabel: getFindClickTargetLabel(command),
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return actions;
|
|
113
|
+
}
|
|
114
|
+
const submitKey = getKeyboardSubmitKey(command)?.trim().toLowerCase();
|
|
115
|
+
if (submitKey && KEYBOARD_SUBMIT_KEYS.has(submitKey)) {
|
|
116
|
+
actions.push({
|
|
117
|
+
command,
|
|
118
|
+
kind: "keyboard-submit",
|
|
119
|
+
stepIndex,
|
|
120
|
+
targetLabel: submitKey,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return actions;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function collectBrowserFinalizingActions(options: {
|
|
127
|
+
commandTokens: string[];
|
|
128
|
+
refSnapshot?: SessionRefSnapshot;
|
|
129
|
+
stdin?: string;
|
|
130
|
+
}): BrowserFinalizingAction[] {
|
|
131
|
+
const actions = collectActionsFromCommand(options.commandTokens, options.refSnapshot);
|
|
132
|
+
if (options.commandTokens[0] !== "batch") return actions;
|
|
133
|
+
for (const { index, step } of parseValidBatchStepEntries(options.stdin)) {
|
|
134
|
+
actions.push(...collectActionsFromCommand(step, options.refSnapshot, index));
|
|
135
|
+
}
|
|
136
|
+
return actions;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function shouldBlockFinalizingAction(action: BrowserFinalizingAction): boolean {
|
|
140
|
+
if (action.kind === "keyboard-submit") return true;
|
|
141
|
+
return matchesFinalActionLabel(action.targetLabel);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function findBlockedFinalizingAction(options: {
|
|
145
|
+
commandTokens: string[];
|
|
146
|
+
refSnapshot?: SessionRefSnapshot;
|
|
147
|
+
stdin?: string;
|
|
148
|
+
}): BrowserFinalizingAction | undefined {
|
|
149
|
+
for (const action of collectBrowserFinalizingActions(options)) {
|
|
150
|
+
if (!shouldBlockFinalizingAction(action)) continue;
|
|
151
|
+
return action;
|
|
152
|
+
}
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { isRecord } from "../../parsing.js";
|
|
2
|
+
import { redactSensitiveText } from "../../runtime.js";
|
|
3
|
+
import { withOptionalSessionArgs, type AgentBrowserNextAction } from "../../results/next-actions.js";
|
|
4
|
+
import { runSessionCommandData } from "./session-state.js";
|
|
5
|
+
import type { ClickDispatchDiagnostic, ClickDispatchProbe, ClickDispatchProbeTarget } from "./types.js";
|
|
6
|
+
|
|
7
|
+
const CLICK_DISPATCH_MARKER_PREFIX = "__piAgentBrowserClickDispatchProbe_";
|
|
8
|
+
const CLICK_DISPATCH_CLEANUP_TIMEOUT_MS = 2_000;
|
|
9
|
+
|
|
10
|
+
function parseClickRefId(selector: string): string | undefined {
|
|
11
|
+
const trimmed = selector.trim();
|
|
12
|
+
const candidate = trimmed.startsWith("@") ? trimmed.slice(1) : trimmed.startsWith("ref=") ? trimmed.slice(4) : trimmed;
|
|
13
|
+
return /^e\d+$/.test(candidate) ? candidate : undefined;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function getClickDispatchSelectorTarget(commandTokens: string[]): ClickDispatchProbeTarget | undefined {
|
|
17
|
+
if (commandTokens[0] !== "click" || commandTokens.includes("--new-tab")) return undefined;
|
|
18
|
+
const selector = commandTokens[1];
|
|
19
|
+
if (!selector || selector.startsWith("-")) return undefined;
|
|
20
|
+
if (parseClickRefId(selector)) return undefined;
|
|
21
|
+
if (selector.startsWith("xpath=")) return { kind: "xpath", selector: selector.slice("xpath=".length) };
|
|
22
|
+
return { kind: "selector", selector };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function getEvalResultRecord(data: unknown): Record<string, unknown> | undefined {
|
|
26
|
+
return isRecord(data) && isRecord(data.result) ? data.result : undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function buildClickDispatchProbeInstallScript(probe: ClickDispatchProbe): string {
|
|
30
|
+
const target = probe.target;
|
|
31
|
+
const resolveTarget = target.kind === "selector"
|
|
32
|
+
? `(() => { try { return document.querySelector(${JSON.stringify(target.selector)}); } catch { return null; } })()`
|
|
33
|
+
: `(() => { try { return document.evaluate(${JSON.stringify(target.selector)}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; } catch { return null; } })()`;
|
|
34
|
+
return `(() => {
|
|
35
|
+
const marker = ${JSON.stringify(probe.marker)};
|
|
36
|
+
const element = ${resolveTarget};
|
|
37
|
+
if (!element) return { status: "target-not-found", marker };
|
|
38
|
+
const state = { events: [], target: { tagName: element.tagName.toLowerCase() } };
|
|
39
|
+
const eventTypes = ["pointerdown", "mousedown", "pointerup", "mouseup", "click"];
|
|
40
|
+
const listeners = eventTypes.map((type) => {
|
|
41
|
+
const listener = (event) => {
|
|
42
|
+
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
|
|
43
|
+
const eventTarget = event.target;
|
|
44
|
+
const targetMatched = path.includes(element) || eventTarget === element || (eventTarget instanceof Node && element.contains(eventTarget));
|
|
45
|
+
state.events.push({ type: event.type, isTrusted: event.isTrusted === true, targetMatched });
|
|
46
|
+
};
|
|
47
|
+
document.addEventListener(type, listener, true);
|
|
48
|
+
return [type, listener];
|
|
49
|
+
});
|
|
50
|
+
state.cleanup = () => listeners.forEach(([type, listener]) => document.removeEventListener(type, listener, true));
|
|
51
|
+
window[marker] = state;
|
|
52
|
+
return { status: "installed", marker, target: state.target };
|
|
53
|
+
})()`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function buildClickDispatchProbeCheckScript(probe: ClickDispatchProbe): string {
|
|
57
|
+
return `(() => {
|
|
58
|
+
const marker = ${JSON.stringify(probe.marker)};
|
|
59
|
+
const state = window[marker];
|
|
60
|
+
const finish = (payload) => {
|
|
61
|
+
if (state && typeof state.cleanup === "function") state.cleanup();
|
|
62
|
+
try { delete window[marker]; } catch {}
|
|
63
|
+
return payload;
|
|
64
|
+
};
|
|
65
|
+
if (!state || !Array.isArray(state.events)) return finish({ status: "probe-missing", nativeEventCount: 0 });
|
|
66
|
+
const nativeEventCount = state.events.filter((event) => event && event.isTrusted === true && event.targetMatched === true).length;
|
|
67
|
+
if (nativeEventCount > 0) return finish({ status: "native-event-observed", nativeEventCount, target: state.target });
|
|
68
|
+
return finish({ status: "no-native-event-observed", nativeEventCount, target: state.target });
|
|
69
|
+
})()`;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function buildClickDispatchProbeCleanupScript(probe: ClickDispatchProbe): string {
|
|
73
|
+
return `(() => {
|
|
74
|
+
const marker = ${JSON.stringify(probe.marker)};
|
|
75
|
+
const state = window[marker];
|
|
76
|
+
if (state && typeof state.cleanup === "function") state.cleanup();
|
|
77
|
+
try { delete window[marker]; } catch {}
|
|
78
|
+
return { status: "cleaned-up" };
|
|
79
|
+
})()`;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function redactClickDispatchTarget(target: ClickDispatchProbeTarget): ClickDispatchProbeTarget {
|
|
83
|
+
return target.kind === "selector" || target.kind === "xpath"
|
|
84
|
+
? { ...target, selector: redactSensitiveText(target.selector) }
|
|
85
|
+
: target;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function formatClickDispatchDiagnosticText(diagnostic: ClickDispatchDiagnostic): string {
|
|
89
|
+
return `Click dispatch diagnostic: ${diagnostic.summary}`;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function buildClickDispatchNextActions(options: { commandTokens: string[]; sessionName?: string }): AgentBrowserNextAction[] {
|
|
93
|
+
const retryArgs = options.commandTokens[0] === "click" ? options.commandTokens : ["click", ...options.commandTokens];
|
|
94
|
+
return [
|
|
95
|
+
{
|
|
96
|
+
id: "inspect-click-dispatch-miss",
|
|
97
|
+
params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) },
|
|
98
|
+
reason: "Refresh interactive refs and verify the intended click target before retrying upstream click.",
|
|
99
|
+
safety: "Read-only snapshot; the wrapper does not replay clicks in-page when upstream reports success without DOM events.",
|
|
100
|
+
tool: "agent_browser",
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
id: "retry-click-after-dispatch-miss",
|
|
104
|
+
params: { args: withOptionalSessionArgs(options.sessionName, retryArgs) },
|
|
105
|
+
reason: "Retry the same upstream click after confirming the target is visible; do not assume the prior success mutated the page.",
|
|
106
|
+
safety: "Only retry when the target is still intended; use page-change evidence or a fresh snapshot before continuing the workflow.",
|
|
107
|
+
tool: "agent_browser",
|
|
108
|
+
},
|
|
109
|
+
];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function prepareClickDispatchProbe(options: { commandTokens: string[]; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchProbe | undefined> {
|
|
113
|
+
if (!options.sessionName || options.commandTokens[0] !== "click" || options.commandTokens.includes("--new-tab")) return undefined;
|
|
114
|
+
const target = getClickDispatchSelectorTarget(options.commandTokens);
|
|
115
|
+
if (!target) return undefined;
|
|
116
|
+
const probe: ClickDispatchProbe = { marker: `${CLICK_DISPATCH_MARKER_PREFIX}${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`, target };
|
|
117
|
+
const installData = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeInstallScript(probe) });
|
|
118
|
+
const installResult = getEvalResultRecord(installData);
|
|
119
|
+
return installResult?.status === "installed" ? probe : undefined;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export async function collectClickDispatchDiagnostic(options: { cwd: string; probe?: ClickDispatchProbe; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchDiagnostic | undefined> {
|
|
123
|
+
if (!options.probe || !options.sessionName) return undefined;
|
|
124
|
+
const data = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeCheckScript(options.probe) });
|
|
125
|
+
const result = getEvalResultRecord(data);
|
|
126
|
+
if (!result) return undefined;
|
|
127
|
+
const status = typeof result.status === "string" ? result.status : undefined;
|
|
128
|
+
if (status !== "no-native-event-observed") return undefined;
|
|
129
|
+
const nativeEventCount = typeof result.nativeEventCount === "number" ? result.nativeEventCount : 0;
|
|
130
|
+
const summary = "Upstream click reported success but no trusted DOM event reached the selected element. Gather evidence with snapshot or page-change checks, then retry upstream click or report the workflow issue; the wrapper does not replay clicks in-page.";
|
|
131
|
+
return {
|
|
132
|
+
nativeEventCount,
|
|
133
|
+
reason: "native-click-produced-no-target-dom-event",
|
|
134
|
+
status,
|
|
135
|
+
summary,
|
|
136
|
+
target: redactClickDispatchTarget(options.probe.target),
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export async function cleanupClickDispatchProbe(options: { cwd: string; probe?: ClickDispatchProbe; sessionName?: string }): Promise<void> {
|
|
141
|
+
if (!options.probe || !options.sessionName) return;
|
|
142
|
+
await runSessionCommandData({
|
|
143
|
+
args: ["eval", "--stdin"],
|
|
144
|
+
cwd: options.cwd,
|
|
145
|
+
sessionName: options.sessionName,
|
|
146
|
+
stdin: buildClickDispatchProbeCleanupScript(options.probe),
|
|
147
|
+
timeoutMs: CLICK_DISPATCH_CLEANUP_TIMEOUT_MS,
|
|
148
|
+
}).catch(() => undefined);
|
|
149
|
+
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { delimiter, isAbsolute, join, resolve } from "node:path";
|
|
1
|
+
import { stat } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, resolve } from "node:path";
|
|
4
3
|
|
|
4
|
+
import { isCloseCommand, isOpenNavigationCommand } from "../../command-taxonomy.js";
|
|
5
5
|
import type { ElectronLaunchRecord } from "../../electron/launch.js";
|
|
6
|
+
import { executableExistsOnPath } from "../../executable-path.js";
|
|
6
7
|
import type { AgentBrowserSourceLookupAnalysis, CompiledAgentBrowserJob, CompiledAgentBrowserSemanticAction } from "../../input-modes.js";
|
|
7
8
|
import { isHttpOrHttpsUrl } from "../../input-modes/job.js";
|
|
8
9
|
import type { AgentBrowserNextAction } from "../../results.js";
|
|
@@ -20,12 +21,14 @@ import {
|
|
|
20
21
|
getGuardedRefUsage,
|
|
21
22
|
runSessionCommandData,
|
|
22
23
|
} from "./session-state.js";
|
|
24
|
+
import { parseValidBatchStepEntries } from "../batch-stdin.js";
|
|
23
25
|
import { getScreenshotPathTokenIndex } from "./prepare.js";
|
|
24
26
|
import type {
|
|
25
27
|
ArtifactCleanupGuidance,
|
|
26
28
|
ComboboxFocusDiagnostic,
|
|
27
29
|
ElectronBroadGetTextScopeDiagnostic,
|
|
28
30
|
ElectronHandoffSummary,
|
|
31
|
+
ElectronManagedSessionTarget,
|
|
29
32
|
FillVerificationDiagnostic,
|
|
30
33
|
NavigationSummary,
|
|
31
34
|
OverlayBlockerCandidate,
|
|
@@ -238,23 +241,6 @@ function getRecordStartLikeCommand(command: string | undefined, commandTokens: s
|
|
|
238
241
|
return undefined;
|
|
239
242
|
}
|
|
240
243
|
|
|
241
|
-
async function executableExistsOnPath(command: string): Promise<boolean> {
|
|
242
|
-
const pathValue = process.env.PATH ?? "";
|
|
243
|
-
const extensions = process.platform === "win32" ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean) : [""];
|
|
244
|
-
for (const directory of pathValue.split(delimiter).filter(Boolean)) {
|
|
245
|
-
for (const extension of extensions) {
|
|
246
|
-
try {
|
|
247
|
-
const candidate = join(directory, `${command}${extension}`);
|
|
248
|
-
await access(candidate, fsConstants.X_OK);
|
|
249
|
-
if ((await stat(candidate)).isFile()) return true;
|
|
250
|
-
} catch {
|
|
251
|
-
// Try the next candidate.
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
return false;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
244
|
export async function collectRecordingDependencyWarning(options: { command: string | undefined; commandTokens: string[]; succeeded: boolean }): Promise<RecordingDependencyWarning | undefined> {
|
|
259
245
|
if (!options.succeeded) return undefined;
|
|
260
246
|
const recordCommand = getRecordStartLikeCommand(options.command, options.commandTokens);
|
|
@@ -314,8 +300,21 @@ export async function collectOverlayBlockerDiagnostic(options: { command?: strin
|
|
|
314
300
|
return { candidates, snapshot, summary: `Click completed but the page stayed on ${currentUrl}; a fresh snapshot contains likely overlay close/dismiss controls.` };
|
|
315
301
|
}
|
|
316
302
|
|
|
303
|
+
const SELECTOR_TEXT_VISIBILITY_CANDIDATE_LIMIT = 8;
|
|
304
|
+
|
|
317
305
|
function buildVisibleTextProbeScript(selector: string): string {
|
|
318
|
-
return `(() => {\n const selector = ${JSON.stringify(selector)};\n const isVisible = (element) => {\n const style = window.getComputedStyle(element);\n if (!style || style.display === 'none' || style.visibility === 'hidden' || style.visibility === 'collapse' || Number(style.opacity) === 0) return false;\n return Array.from(element.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0);\n };\n let matches = [];\n try {\n matches = Array.from(document.querySelectorAll(selector));\n } catch (error) {\n return JSON.stringify({ selector, error: error instanceof Error ? error.message : String(error) });\n }\n const visible = matches.filter(isVisible);\n const trim = (value) => typeof value === 'string' ? value.trim().replace(/\\s+/g, ' ').slice(0, 200) : undefined;\n return JSON.stringify({ selector, matchCount: matches.length, visibleCount: visible.length, firstMatchVisible: matches[0] ? isVisible(matches[0]) : undefined, firstTextPreview: trim(matches[0]?.textContent), firstVisibleTextPreview: trim(visible[0]?.textContent) });\n})()`;
|
|
306
|
+
return `(() => {\n const selector = ${JSON.stringify(selector)};\n const isVisible = (element) => {\n const style = window.getComputedStyle(element);\n if (!style || style.display === 'none' || style.visibility === 'hidden' || style.visibility === 'collapse' || Number(style.opacity) === 0) return false;\n return Array.from(element.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0);\n };\n let matches = [];\n try {\n matches = Array.from(document.querySelectorAll(selector));\n } catch (error) {\n return JSON.stringify({ selector, error: error instanceof Error ? error.message : String(error) });\n }\n const visible = matches.filter(isVisible);\n const trim = (value) => typeof value === 'string' ? value.trim().replace(/\\s+/g, ' ').slice(0, 200) : undefined;\n const describeCandidate = (element) => {\n const index = matches.indexOf(element);\n const role = element.getAttribute('role');\n const candidate = { index, tagName: element.tagName.toLowerCase(), textPreview: trim(element.textContent) };\n if (role) candidate.role = role;\n return candidate;\n };\n const visibleCandidates = visible.slice(0, ${SELECTOR_TEXT_VISIBILITY_CANDIDATE_LIMIT}).map(describeCandidate);\n return JSON.stringify({ selector, matchCount: matches.length, visibleCount: visible.length, firstMatchVisible: matches[0] ? isVisible(matches[0]) : undefined, firstTextPreview: trim(matches[0]?.textContent), firstVisibleTextPreview: trim(visible[0]?.textContent), visibleCandidates });\n})()`;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function parseSelectorTextVisibilityCandidates(value: unknown): SelectorTextVisibilityDiagnostic["visibleCandidates"] {
|
|
310
|
+
if (!Array.isArray(value)) return undefined;
|
|
311
|
+
const candidates = value.flatMap((entry): NonNullable<SelectorTextVisibilityDiagnostic["visibleCandidates"]> => {
|
|
312
|
+
if (!isRecord(entry) || typeof entry.index !== "number" || typeof entry.tagName !== "string") return [];
|
|
313
|
+
const role = typeof entry.role === "string" && entry.role.length > 0 ? entry.role : undefined;
|
|
314
|
+
const textPreview = typeof entry.textPreview === "string" && entry.textPreview.length > 0 ? redactSensitiveText(entry.textPreview) : undefined;
|
|
315
|
+
return [{ index: entry.index, tagName: entry.tagName, ...(role ? { role } : {}), ...(textPreview ? { textPreview } : {}) }];
|
|
316
|
+
});
|
|
317
|
+
return candidates.length > 0 ? candidates : undefined;
|
|
319
318
|
}
|
|
320
319
|
|
|
321
320
|
function parseSelectorTextVisibilityProbe(data: unknown, selector: string): Omit<SelectorTextVisibilityDiagnostic, "summary"> | undefined {
|
|
@@ -327,7 +326,14 @@ function parseSelectorTextVisibilityProbe(data: unknown, selector: string): Omit
|
|
|
327
326
|
const matchCount = typeof parsed.matchCount === "number" ? parsed.matchCount : undefined;
|
|
328
327
|
const visibleCount = typeof parsed.visibleCount === "number" ? parsed.visibleCount : undefined;
|
|
329
328
|
if (matchCount === undefined || visibleCount === undefined) return undefined;
|
|
330
|
-
return {
|
|
329
|
+
return {
|
|
330
|
+
firstMatchVisible: typeof parsed.firstMatchVisible === "boolean" ? parsed.firstMatchVisible : undefined,
|
|
331
|
+
firstVisibleTextPreview: typeof parsed.firstVisibleTextPreview === "string" && parsed.firstVisibleTextPreview.length > 0 ? redactSensitiveText(parsed.firstVisibleTextPreview) : undefined,
|
|
332
|
+
matchCount,
|
|
333
|
+
selector,
|
|
334
|
+
visibleCandidates: parseSelectorTextVisibilityCandidates(parsed.visibleCandidates),
|
|
335
|
+
visibleCount,
|
|
336
|
+
};
|
|
331
337
|
}
|
|
332
338
|
|
|
333
339
|
function selectorMayExposeSensitiveLiteral(selector: string): boolean {
|
|
@@ -380,6 +386,14 @@ export function formatSelectorTextVisibilityText(diagnostics: SelectorTextVisibi
|
|
|
380
386
|
const actionId = index === 0 ? "inspect-visible-text-candidates" : `inspect-visible-text-candidates-${index + 1}`;
|
|
381
387
|
const lines = [`Selector text visibility warning: ${diagnostic.summary}`];
|
|
382
388
|
if (diagnostic.firstVisibleTextPreview) lines.push(`First visible text preview: ${JSON.stringify(diagnostic.firstVisibleTextPreview)}`);
|
|
389
|
+
if (diagnostic.visibleCandidates && diagnostic.visibleCandidates.length > 0) {
|
|
390
|
+
lines.push(`Visible candidates (${diagnostic.visibleCandidates.length} shown, querySelectorAll index):`);
|
|
391
|
+
for (const candidate of diagnostic.visibleCandidates) {
|
|
392
|
+
const rolePart = candidate.role ? ` role=${candidate.role}` : "";
|
|
393
|
+
const previewPart = candidate.textPreview ? `: ${JSON.stringify(candidate.textPreview)}` : "";
|
|
394
|
+
lines.push(`- [${candidate.index}] ${candidate.tagName}${rolePart}${previewPart}`);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
383
397
|
lines.push(`Next action: use details.nextActions ${actionId} before trusting this selector text.`);
|
|
384
398
|
return lines;
|
|
385
399
|
}).join("\n");
|
|
@@ -465,8 +479,24 @@ export function formatEvalStdinHintText(hint: ReturnType<typeof getEvalStdinHint
|
|
|
465
479
|
return hint ? `Eval stdin hint: ${hint.reason} ${hint.suggestion}` : undefined;
|
|
466
480
|
}
|
|
467
481
|
|
|
482
|
+
export function getEvalResultWarning(options: { command?: string; data: unknown; navigationSummary?: { url?: string }; pageUrl?: string; stdin?: string }) {
|
|
483
|
+
if (options.command !== "eval" || !options.stdin?.trim() || !isRecord(options.data) || options.data.result !== null) return undefined;
|
|
484
|
+
const pageUrl = options.pageUrl?.trim() ?? options.navigationSummary?.url?.trim() ?? extractNavigationSummaryFromData(options.data)?.url;
|
|
485
|
+
if (!pageUrl || !/^file:/i.test(pageUrl)) return undefined;
|
|
486
|
+
const trimmed = options.stdin.trim();
|
|
487
|
+
if (/^(?:null|undefined)$/i.test(trimmed)) return undefined;
|
|
488
|
+
return {
|
|
489
|
+
reason: "eval --stdin returned null on a file:// page; upstream may not expose full DOM semantics for local fixtures.",
|
|
490
|
+
suggestion: "Treat this as inconclusive verification. Use snapshot -i, get text on current @refs, screenshot evidence, or a reachable http(s) fixture before concluding DOM state.",
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
export function formatEvalResultWarningText(warning: ReturnType<typeof getEvalResultWarning>): string | undefined {
|
|
495
|
+
return warning ? `Eval result warning: ${warning.reason} ${warning.suggestion}` : undefined;
|
|
496
|
+
}
|
|
497
|
+
|
|
468
498
|
export async function getArtifactCleanupGuidance(options: { command?: string; cwd: string; manifest?: SessionArtifactManifest; succeeded: boolean }): Promise<ArtifactCleanupGuidance | undefined> {
|
|
469
|
-
if (!options.succeeded || options.command
|
|
499
|
+
if (!options.succeeded || !isCloseCommand(options.command) || !options.manifest || options.manifest.entries.length === 0) return undefined;
|
|
470
500
|
const explicitEntries = options.manifest.entries.filter((entry) => entry.storageScope === "explicit-path");
|
|
471
501
|
const explicitArtifactPaths: string[] = [];
|
|
472
502
|
const seenPaths = new Set<string>();
|
|
@@ -505,7 +535,7 @@ async function collectElectronManagedSessionUrl(options: { cwd: string; sessionN
|
|
|
505
535
|
return urlResult.error ? { error: urlResult.error } : { url };
|
|
506
536
|
}
|
|
507
537
|
|
|
508
|
-
async function collectElectronManagedSessionTarget(options: { cwd: string; sessionName?: string; signal?: AbortSignal; timeoutMs?: number }): Promise<
|
|
538
|
+
export async function collectElectronManagedSessionTarget(options: { cwd: string; sessionName?: string; signal?: AbortSignal; timeoutMs?: number }): Promise<ElectronManagedSessionTarget | undefined> {
|
|
509
539
|
if (!options.sessionName) return undefined;
|
|
510
540
|
const [titleResult, urlResult] = await Promise.all([
|
|
511
541
|
collectManagedSessionCommandData({ args: ["get", "title"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, timeoutMs: options.timeoutMs }),
|
|
@@ -647,11 +677,7 @@ export async function collectElectronHandoff(options: { cwd: string; handoff: "c
|
|
|
647
677
|
function getTimeoutProgressSteps(compiledJob: CompiledAgentBrowserJob | undefined, command: string | undefined, stdin: string | undefined): Array<{ args: string[]; index: number }> {
|
|
648
678
|
if (compiledJob) return compiledJob.steps.map((step, index) => ({ args: step.args, index: index + 1 }));
|
|
649
679
|
if (command !== "batch" || !stdin) return [];
|
|
650
|
-
|
|
651
|
-
const parsed = JSON.parse(stdin) as unknown;
|
|
652
|
-
if (!Array.isArray(parsed)) return [];
|
|
653
|
-
return parsed.flatMap((step, index) => Array.isArray(step) && step.every((token) => typeof token === "string") ? [{ args: step as string[], index: index + 1 }] : []);
|
|
654
|
-
} catch { return []; }
|
|
680
|
+
return parseValidBatchStepEntries(stdin).map(({ index, step }) => ({ args: step, index: index + 1 }));
|
|
655
681
|
}
|
|
656
682
|
|
|
657
683
|
function getLastPositionalToken(args: string[], startIndex = 1): string | undefined {
|
|
@@ -709,7 +735,7 @@ async function collectTimeoutArtifactEvidence(cwd: string, steps: Array<{ args:
|
|
|
709
735
|
function getPlannedCurrentPageUrl(steps: Array<{ args: string[]; index: number }>): string | undefined {
|
|
710
736
|
for (let index = steps.length - 1; index >= 0; index -= 1) {
|
|
711
737
|
const args = steps[index]?.args ?? [];
|
|
712
|
-
if (args[0]
|
|
738
|
+
if (isOpenNavigationCommand(args[0]) || args[0] === "pushstate") return getLastPositionalToken(args);
|
|
713
739
|
}
|
|
714
740
|
return undefined;
|
|
715
741
|
}
|