pi-agent-browser-native 0.2.33 → 0.2.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +47 -17
- package/docs/ARCHITECTURE.md +25 -13
- package/docs/COMMAND_REFERENCE.md +285 -47
- package/docs/ELECTRON.md +3 -3
- package/docs/RELEASE.md +22 -14
- package/docs/REQUIREMENTS.md +5 -5
- package/docs/SUPPORT_MATRIX.md +26 -22
- package/docs/TOOL_CONTRACT.md +97 -32
- package/extensions/agent-browser/index.ts +519 -2402
- package/extensions/agent-browser/lib/argv-descriptor.ts +90 -0
- package/extensions/agent-browser/lib/argv-grammar.ts +128 -0
- package/extensions/agent-browser/lib/command-policy.ts +71 -0
- package/extensions/agent-browser/lib/command-taxonomy.ts +336 -0
- package/extensions/agent-browser/lib/electron/cleanup.ts +1 -0
- package/extensions/agent-browser/lib/executable-path.ts +19 -0
- package/extensions/agent-browser/lib/input-modes/job.ts +62 -0
- package/extensions/agent-browser/lib/input-modes/params.ts +8 -8
- package/extensions/agent-browser/lib/input-modes.ts +3 -0
- package/extensions/agent-browser/lib/orchestration/batch-stdin.ts +65 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/browser-action-model.ts +154 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/click-dispatch.ts +149 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/diagnostics.ts +77 -29
- package/extensions/agent-browser/lib/orchestration/browser-run/final-result.ts +6 -2
- package/extensions/agent-browser/lib/orchestration/browser-run/index.ts +33 -27
- package/extensions/agent-browser/lib/orchestration/browser-run/prepare.ts +74 -23
- package/extensions/agent-browser/lib/orchestration/browser-run/process-output.ts +67 -17
- package/extensions/agent-browser/lib/orchestration/browser-run/prompt-guards.ts +93 -0
- package/extensions/agent-browser/lib/orchestration/browser-run/session-state.ts +19 -123
- package/extensions/agent-browser/lib/orchestration/browser-run/types.ts +32 -1
- package/extensions/agent-browser/lib/orchestration/electron-host/index.ts +860 -0
- package/extensions/agent-browser/lib/playbook.ts +24 -23
- package/extensions/agent-browser/lib/prompt-policy.ts +122 -0
- package/extensions/agent-browser/lib/results/action-recommendations.ts +3 -23
- package/extensions/agent-browser/lib/results/categories.ts +1 -1
- package/extensions/agent-browser/lib/results/presentation/navigation.ts +2 -34
- package/extensions/agent-browser/lib/results/presentation/registry.ts +34 -6
- package/extensions/agent-browser/lib/results/presentation/semantic-action.ts +133 -0
- package/extensions/agent-browser/lib/results/presentation.ts +11 -6
- package/extensions/agent-browser/lib/runtime.ts +93 -227
- package/extensions/agent-browser/lib/session-page-state.ts +31 -14
- package/extensions/agent-browser/lib/temp.ts +148 -23
- package/package.json +4 -4
- package/scripts/agent-browser-capability-baseline.mjs +198 -1
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Scope: Job and QA modes only.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import type { ArtifactVerificationSummary } from "../results/contracts.js";
|
|
7
8
|
import { isRecord } from "../parsing.js";
|
|
8
9
|
import { summarizeNetworkFailures } from "../results/network.js";
|
|
9
10
|
import { getBatchResultItems, getCommandNameFromBatchItem, getSelectValues } from "./shared.js";
|
|
@@ -93,6 +94,67 @@ export function compileAgentBrowserJob(input: unknown): { compiled?: CompiledAge
|
|
|
93
94
|
return { compiled: { args: ["batch"], stdin: JSON.stringify(steps.map((step) => step.args)), steps } };
|
|
94
95
|
}
|
|
95
96
|
|
|
97
|
+
export function isHttpOrHttpsUrl(url: string): boolean {
|
|
98
|
+
try {
|
|
99
|
+
const protocol = new URL(url).protocol;
|
|
100
|
+
return protocol === "http:" || protocol === "https:";
|
|
101
|
+
} catch {
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function describeQaChecksRun(checks: CompiledAgentBrowserQaPreset["checks"]): string {
|
|
107
|
+
const parts = [`load:${checks.loadState}`];
|
|
108
|
+
if (checks.expectedText.length > 0) parts.push(`text×${checks.expectedText.length}`);
|
|
109
|
+
if (checks.expectedSelector) parts.push("selector");
|
|
110
|
+
if (checks.checkNetwork) parts.push("network");
|
|
111
|
+
if (checks.checkConsole) parts.push("console");
|
|
112
|
+
if (checks.checkErrors) parts.push("errors");
|
|
113
|
+
if (checks.screenshotPath) parts.push("screenshot");
|
|
114
|
+
return parts.join(", ");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function extractQaPageContext(options: {
|
|
118
|
+
attachedTarget?: { title?: string; url?: string };
|
|
119
|
+
batchData?: unknown;
|
|
120
|
+
compiled?: CompiledAgentBrowserQaPreset;
|
|
121
|
+
}): { title?: string; url?: string } {
|
|
122
|
+
if (options.attachedTarget?.title || options.attachedTarget?.url) {
|
|
123
|
+
return { title: options.attachedTarget.title, url: options.attachedTarget.url };
|
|
124
|
+
}
|
|
125
|
+
for (const item of getBatchResultItems(options.batchData)) {
|
|
126
|
+
if (getCommandNameFromBatchItem(item) !== "open" || !isRecord(item.result)) continue;
|
|
127
|
+
const url = typeof item.result.url === "string" ? item.result.url : undefined;
|
|
128
|
+
const title = typeof item.result.title === "string" ? item.result.title : undefined;
|
|
129
|
+
if (url || title) return { title, url };
|
|
130
|
+
}
|
|
131
|
+
if (options.compiled?.checks.url) {
|
|
132
|
+
return { url: options.compiled.checks.url };
|
|
133
|
+
}
|
|
134
|
+
return {};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function buildQaCompactPassText(options: {
|
|
138
|
+
artifactVerification?: ArtifactVerificationSummary;
|
|
139
|
+
batchStepCount: number;
|
|
140
|
+
checks: CompiledAgentBrowserQaPreset["checks"];
|
|
141
|
+
page?: { title?: string; url?: string };
|
|
142
|
+
qaPreset: AgentBrowserQaPresetAnalysis;
|
|
143
|
+
}): string {
|
|
144
|
+
const lines = [options.qaPreset.summary];
|
|
145
|
+
const pageParts = [options.page?.title, options.page?.url].filter((part): part is string => typeof part === "string" && part.length > 0);
|
|
146
|
+
if (pageParts.length > 0) lines.push(`Page: ${pageParts.join(" — ")}`);
|
|
147
|
+
lines.push(`Checks run: ${describeQaChecksRun(options.checks)} (${options.batchStepCount} batch step${options.batchStepCount === 1 ? "" : "s"})`);
|
|
148
|
+
if (options.checks.screenshotPath) {
|
|
149
|
+
const verification = options.artifactVerification;
|
|
150
|
+
lines.push(verification
|
|
151
|
+
? `Screenshot: ${options.checks.screenshotPath} (${verification.verifiedCount}/${verification.artifacts.length} verified on disk)`
|
|
152
|
+
: `Screenshot: ${options.checks.screenshotPath}`);
|
|
153
|
+
}
|
|
154
|
+
lines.push("Full diagnostic matrix: see details.qaPreset and details.batchSteps.");
|
|
155
|
+
return lines.join("\n");
|
|
156
|
+
}
|
|
157
|
+
|
|
96
158
|
export function analyzeQaPresetResults(data: unknown): AgentBrowserQaPresetAnalysis | undefined {
|
|
97
159
|
const items = getBatchResultItems(data);
|
|
98
160
|
if (items.length === 0) return undefined;
|
|
@@ -25,8 +25,8 @@ import {
|
|
|
25
25
|
export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
26
26
|
|
|
27
27
|
args: Type.Optional(
|
|
28
|
-
Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name." }), {
|
|
29
|
-
description: "Exact agent-browser CLI arguments, excluding the binary name and any shell operators. Required unless semanticAction, job, qa, sourceLookup, networkSourceLookup, or electron is provided.",
|
|
28
|
+
Type.Array(Type.String({ description: "Exact agent-browser CLI arguments, excluding the binary name. Do not pass --json; the wrapper injects it. First-call recipe: open → snapshot -i → click/fill @eN → snapshot -i." }), {
|
|
29
|
+
description: "Exact agent-browser CLI arguments, excluding the binary name and any shell operators. Required unless semanticAction, job, qa, sourceLookup, networkSourceLookup, or electron is provided. Do not include --json (wrapper injects it). Typical first calls: open, snapshot -i, click/fill current @refs, then snapshot -i again after navigation or DOM changes.",
|
|
30
30
|
minItems: 1,
|
|
31
31
|
}),
|
|
32
32
|
),
|
|
@@ -45,7 +45,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
45
45
|
role: Type.Optional(Type.String({ description: "Role locator value for locator=role. May be used instead of value; when both are set they must match." })),
|
|
46
46
|
name: Type.Optional(Type.String({ description: "Accessible name filter for locator=role; compiles to --name <name>." })),
|
|
47
47
|
session: Type.Optional(Type.String({ description: "Optional upstream session name; prepends --session <name> before the compiled command." })),
|
|
48
|
-
}),
|
|
48
|
+
}, { additionalProperties: false }),
|
|
49
49
|
),
|
|
50
50
|
qa: Type.Optional(
|
|
51
51
|
Type.Union([
|
|
@@ -79,7 +79,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
79
79
|
componentName: Type.Optional(Type.String({ description: "Component name to correlate with react tree output and bounded local workspace search." })),
|
|
80
80
|
includeDomHints: Type.Optional(Type.Boolean({ description: "Whether selector lookups should inspect DOM HTML attributes for source-like metadata. Defaults to true." })),
|
|
81
81
|
maxWorkspaceFiles: Type.Optional(Type.Number({ description: "Maximum local source files to scan when componentName is provided. Defaults to 2000 and cannot exceed 5000.", minimum: 1, maximum: SOURCE_LOOKUP_MAX_WORKSPACE_FILES })),
|
|
82
|
-
}),
|
|
82
|
+
}, { additionalProperties: false, description: "EXPERIMENTAL: local UI-to-source candidates only (confidence/evidence, not guaranteed mappings). Compiles to batch; mutually exclusive with other input modes." }),
|
|
83
83
|
),
|
|
84
84
|
networkSourceLookup: Type.Optional(
|
|
85
85
|
Type.Object({
|
|
@@ -88,7 +88,7 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
88
88
|
session: Type.Optional(Type.String({ description: "Optional upstream session name; prepends --session <name> before the generated batch." })),
|
|
89
89
|
url: Type.Optional(Type.String({ description: "Optional failed request URL or URL fragment to correlate with local source." })),
|
|
90
90
|
maxWorkspaceFiles: Type.Optional(Type.Number({ description: "Maximum local source files to scan for URL literals. Defaults to 2000 and cannot exceed 5000.", minimum: 1, maximum: SOURCE_LOOKUP_MAX_WORKSPACE_FILES })),
|
|
91
|
-
}),
|
|
91
|
+
}, { additionalProperties: false, description: "EXPERIMENTAL: failed-request-to-source candidates only (initiator metadata and bounded workspace URL literals; not definitive blame). Compiles to batch; mutually exclusive with other input modes." }),
|
|
92
92
|
),
|
|
93
93
|
electron: Type.Optional(
|
|
94
94
|
Type.Union([
|
|
@@ -172,10 +172,10 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
172
172
|
values: Type.Optional(Type.Array(Type.String({ description: "Option value for select steps." }), { description: "One or more option values for select steps.", minItems: 1 })),
|
|
173
173
|
path: Type.Optional(Type.String({ description: "Artifact/download path for waitForDownload or screenshot steps." })),
|
|
174
174
|
milliseconds: Type.Optional(Type.Number({ description: "Milliseconds for wait steps." })),
|
|
175
|
-
}),
|
|
175
|
+
}, { additionalProperties: false }),
|
|
176
176
|
{ minItems: 1 },
|
|
177
177
|
),
|
|
178
|
-
}),
|
|
178
|
+
}, { additionalProperties: false }),
|
|
179
179
|
),
|
|
180
180
|
stdin: Type.Optional(Type.String({ description: "Optional raw stdin content; only supported for batch, eval --stdin, auth save --password-stdin, and is generated internally by job, qa, sourceLookup, or networkSourceLookup mode. Do not use with electron mode." })),
|
|
181
181
|
sessionMode: Type.Optional(
|
|
@@ -185,4 +185,4 @@ export const AGENT_BROWSER_PARAMS = Type.Object({
|
|
|
185
185
|
default: DEFAULT_SESSION_MODE,
|
|
186
186
|
}),
|
|
187
187
|
),
|
|
188
|
-
});
|
|
188
|
+
}, { additionalProperties: false });
|
|
@@ -7,8 +7,11 @@
|
|
|
7
7
|
export { AGENT_BROWSER_PARAMS } from "./input-modes/params.js";
|
|
8
8
|
export {
|
|
9
9
|
analyzeQaPresetResults,
|
|
10
|
+
buildQaCompactPassText,
|
|
10
11
|
compileAgentBrowserJob,
|
|
11
12
|
compileAgentBrowserQaPreset,
|
|
13
|
+
extractQaPageContext,
|
|
14
|
+
isHttpOrHttpsUrl,
|
|
12
15
|
} from "./input-modes/job.js";
|
|
13
16
|
export {
|
|
14
17
|
analyzeNetworkSourceLookupResults,
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export type BatchCommandStep = [string, ...string[]];
|
|
2
|
+
|
|
3
|
+
function validateUserBatchStep(step: unknown, index: number): { error: string; ok: false } | { ok: true; step: BatchCommandStep } {
|
|
4
|
+
if (!Array.isArray(step)) {
|
|
5
|
+
return {
|
|
6
|
+
error: `agent_browser batch stdin step ${index} must be a non-empty array of string command tokens.`,
|
|
7
|
+
ok: false,
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
if (step.length === 0) {
|
|
11
|
+
return {
|
|
12
|
+
error: `agent_browser batch stdin step ${index} must not be empty.`,
|
|
13
|
+
ok: false,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
const invalidTokenIndex = step.findIndex((token) => typeof token !== "string");
|
|
17
|
+
if (invalidTokenIndex !== -1) {
|
|
18
|
+
return {
|
|
19
|
+
error: `agent_browser batch stdin step ${index} token ${invalidTokenIndex} must be a string.`,
|
|
20
|
+
ok: false,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
return { ok: true, step: step as BatchCommandStep };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function parseBatchStdinJsonArray(stdin: string | undefined): { error?: string; steps?: unknown[] } {
|
|
27
|
+
if (stdin === undefined) {
|
|
28
|
+
return { steps: [] };
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
const parsed = JSON.parse(stdin) as unknown;
|
|
32
|
+
if (!Array.isArray(parsed)) {
|
|
33
|
+
return { error: "agent_browser batch stdin must be a JSON array of command steps." };
|
|
34
|
+
}
|
|
35
|
+
return { steps: parsed };
|
|
36
|
+
} catch (error) {
|
|
37
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
38
|
+
return { error: `agent_browser batch stdin could not be parsed as JSON: ${message}` };
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function parseUserBatchStdin(stdin: string | undefined): { error?: string; steps?: BatchCommandStep[] } {
|
|
43
|
+
const parsed = parseBatchStdinJsonArray(stdin);
|
|
44
|
+
if (parsed.error || parsed.steps === undefined) {
|
|
45
|
+
return parsed.error ? { error: parsed.error } : { steps: [] };
|
|
46
|
+
}
|
|
47
|
+
const steps: BatchCommandStep[] = [];
|
|
48
|
+
for (const [index, rawStep] of parsed.steps.entries()) {
|
|
49
|
+
const validated = validateUserBatchStep(rawStep, index);
|
|
50
|
+
if (!validated.ok) {
|
|
51
|
+
return { error: validated.error };
|
|
52
|
+
}
|
|
53
|
+
steps.push(validated.step);
|
|
54
|
+
}
|
|
55
|
+
return { steps };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function parseValidBatchStepEntries(stdin: string | undefined): Array<{ index: number; step: BatchCommandStep }> {
|
|
59
|
+
const parsed = parseBatchStdinJsonArray(stdin);
|
|
60
|
+
if (parsed.error || parsed.steps === undefined) return [];
|
|
61
|
+
return parsed.steps.flatMap((step, index) => {
|
|
62
|
+
const validated = validateUserBatchStep(step, index);
|
|
63
|
+
return validated.ok ? [{ index, step: validated.step }] : [];
|
|
64
|
+
});
|
|
65
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Purpose: Normalize planned browser argv into a small action model for prompt-derived guards.
|
|
3
|
+
* Responsibilities: Map command tokens and batch stdin steps to click-like and keyboard-submit actions with target labels.
|
|
4
|
+
* Scope: Best-effort finalizing-action detection only; does not model eval, generic fill/type, or non-Enter keyboard flows.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { SessionRefSnapshot } from "../../session-page-state.js";
|
|
8
|
+
import { parseValidBatchStepEntries } from "../batch-stdin.js";
|
|
9
|
+
|
|
10
|
+
const FINAL_ACTION_PATTERN = /\b(?:finish|place\s+(?:the\s+)?order|submit\s+(?:the\s+)?order|complete\s+(?:the\s+)?order|confirm\s+(?:the\s+)?order|purchase|buy\s+now|pay\s+now|finali[sz]e|submit\s+payment|checkout\s+complete)\b/i;
|
|
11
|
+
|
|
12
|
+
const CLICK_LIKE_COMMANDS = new Set(["click", "dblclick", "tap"]);
|
|
13
|
+
const FIND_CLICK_ACTIONS = new Set(["click", "dblclick", "tap"]);
|
|
14
|
+
const KEYBOARD_SUBMIT_KEYS = new Set(["enter", "return"]);
|
|
15
|
+
|
|
16
|
+
export type BrowserFinalizingActionKind = "click-like" | "keyboard-submit";
|
|
17
|
+
|
|
18
|
+
export interface BrowserFinalizingAction {
|
|
19
|
+
command: string[];
|
|
20
|
+
kind: BrowserFinalizingActionKind;
|
|
21
|
+
stepIndex?: number;
|
|
22
|
+
targetLabel?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const STOP_BOUNDARY_GUARD_SCOPE = {
|
|
26
|
+
covered: [
|
|
27
|
+
"standalone click, dblclick, and tap",
|
|
28
|
+
"find … click|dblclick|tap",
|
|
29
|
+
"batch steps with the click-like shapes above",
|
|
30
|
+
"press <key> and key <key> when key is Enter or Return",
|
|
31
|
+
],
|
|
32
|
+
excluded: [
|
|
33
|
+
"eval --stdin and other scripted activation",
|
|
34
|
+
"fill, type, select, drag, and upload without an explicit click-like command",
|
|
35
|
+
"keyboard type/inserttext and keyboard shortcuts other than Enter/Return",
|
|
36
|
+
"semanticAction and job/qa compiled plans unless their batch stdin contains a covered step",
|
|
37
|
+
],
|
|
38
|
+
} as const;
|
|
39
|
+
|
|
40
|
+
function normalizeTargetText(value: string): string {
|
|
41
|
+
return value
|
|
42
|
+
.replace(/[_-]+/g, " ")
|
|
43
|
+
.replace(/[\[\]{}()#.'\"=:/]+/g, " ")
|
|
44
|
+
.replace(/\s+/g, " ")
|
|
45
|
+
.trim();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function matchesFinalActionLabel(value: string | undefined): boolean {
|
|
49
|
+
return value !== undefined && FINAL_ACTION_PATTERN.test(normalizeTargetText(value));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function parseRefId(value: string | undefined): string | undefined {
|
|
53
|
+
if (!value) return undefined;
|
|
54
|
+
const trimmed = value.trim();
|
|
55
|
+
const candidate = trimmed.startsWith("@") ? trimmed.slice(1) : trimmed.startsWith("ref=") ? trimmed.slice(4) : trimmed;
|
|
56
|
+
return /^e\d+$/.test(candidate) ? candidate : undefined;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function getRefTargetLabel(refSnapshot: SessionRefSnapshot | undefined, refId: string | undefined): string | undefined {
|
|
60
|
+
if (!refId) return undefined;
|
|
61
|
+
const ref = refSnapshot?.refs?.[refId];
|
|
62
|
+
return ref ? [ref.role, ref.name].filter(Boolean).join(" ") : undefined;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function getFlagValue(tokens: string[], flag: string): string | undefined {
|
|
66
|
+
for (const [index, token] of tokens.entries()) {
|
|
67
|
+
if (token === flag) return tokens[index + 1];
|
|
68
|
+
if (token.startsWith(`${flag}=`)) return token.slice(flag.length + 1);
|
|
69
|
+
}
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function getClickLikeTargetLabel(command: string[], refSnapshot: SessionRefSnapshot | undefined): string | undefined {
|
|
74
|
+
const target = command[1];
|
|
75
|
+
return getRefTargetLabel(refSnapshot, parseRefId(target)) ?? target;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function getFindClickTargetLabel(command: string[]): string | undefined {
|
|
79
|
+
if (command[0] !== "find") return undefined;
|
|
80
|
+
const actionIndex = command.findIndex((token, index) => index >= 3 && FIND_CLICK_ACTIONS.has(token));
|
|
81
|
+
if (actionIndex === -1) return undefined;
|
|
82
|
+
return getFlagValue(command, "--name") ?? command[2];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function getKeyboardSubmitKey(command: string[]): string | undefined {
|
|
86
|
+
const commandName = command[0];
|
|
87
|
+
if (commandName === "press" || commandName === "key") return command[1];
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function collectActionsFromCommand(command: string[], refSnapshot: SessionRefSnapshot | undefined, stepIndex?: number): BrowserFinalizingAction[] {
|
|
92
|
+
const actions: BrowserFinalizingAction[] = [];
|
|
93
|
+
if (CLICK_LIKE_COMMANDS.has(command[0] ?? "")) {
|
|
94
|
+
actions.push({
|
|
95
|
+
command,
|
|
96
|
+
kind: "click-like",
|
|
97
|
+
stepIndex,
|
|
98
|
+
targetLabel: getClickLikeTargetLabel(command, refSnapshot),
|
|
99
|
+
});
|
|
100
|
+
return actions;
|
|
101
|
+
}
|
|
102
|
+
if (command[0] === "find") {
|
|
103
|
+
const actionIndex = command.findIndex((token, index) => index >= 3 && FIND_CLICK_ACTIONS.has(token));
|
|
104
|
+
if (actionIndex !== -1) {
|
|
105
|
+
actions.push({
|
|
106
|
+
command,
|
|
107
|
+
kind: "click-like",
|
|
108
|
+
stepIndex,
|
|
109
|
+
targetLabel: getFindClickTargetLabel(command),
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return actions;
|
|
113
|
+
}
|
|
114
|
+
const submitKey = getKeyboardSubmitKey(command)?.trim().toLowerCase();
|
|
115
|
+
if (submitKey && KEYBOARD_SUBMIT_KEYS.has(submitKey)) {
|
|
116
|
+
actions.push({
|
|
117
|
+
command,
|
|
118
|
+
kind: "keyboard-submit",
|
|
119
|
+
stepIndex,
|
|
120
|
+
targetLabel: submitKey,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return actions;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function collectBrowserFinalizingActions(options: {
|
|
127
|
+
commandTokens: string[];
|
|
128
|
+
refSnapshot?: SessionRefSnapshot;
|
|
129
|
+
stdin?: string;
|
|
130
|
+
}): BrowserFinalizingAction[] {
|
|
131
|
+
const actions = collectActionsFromCommand(options.commandTokens, options.refSnapshot);
|
|
132
|
+
if (options.commandTokens[0] !== "batch") return actions;
|
|
133
|
+
for (const { index, step } of parseValidBatchStepEntries(options.stdin)) {
|
|
134
|
+
actions.push(...collectActionsFromCommand(step, options.refSnapshot, index));
|
|
135
|
+
}
|
|
136
|
+
return actions;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function shouldBlockFinalizingAction(action: BrowserFinalizingAction): boolean {
|
|
140
|
+
if (action.kind === "keyboard-submit") return true;
|
|
141
|
+
return matchesFinalActionLabel(action.targetLabel);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function findBlockedFinalizingAction(options: {
|
|
145
|
+
commandTokens: string[];
|
|
146
|
+
refSnapshot?: SessionRefSnapshot;
|
|
147
|
+
stdin?: string;
|
|
148
|
+
}): BrowserFinalizingAction | undefined {
|
|
149
|
+
for (const action of collectBrowserFinalizingActions(options)) {
|
|
150
|
+
if (!shouldBlockFinalizingAction(action)) continue;
|
|
151
|
+
return action;
|
|
152
|
+
}
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { isRecord } from "../../parsing.js";
|
|
2
|
+
import { redactSensitiveText } from "../../runtime.js";
|
|
3
|
+
import { withOptionalSessionArgs, type AgentBrowserNextAction } from "../../results/next-actions.js";
|
|
4
|
+
import { runSessionCommandData } from "./session-state.js";
|
|
5
|
+
import type { ClickDispatchDiagnostic, ClickDispatchProbe, ClickDispatchProbeTarget } from "./types.js";
|
|
6
|
+
|
|
7
|
+
const CLICK_DISPATCH_MARKER_PREFIX = "__piAgentBrowserClickDispatchProbe_";
|
|
8
|
+
const CLICK_DISPATCH_CLEANUP_TIMEOUT_MS = 2_000;
|
|
9
|
+
|
|
10
|
+
function parseClickRefId(selector: string): string | undefined {
|
|
11
|
+
const trimmed = selector.trim();
|
|
12
|
+
const candidate = trimmed.startsWith("@") ? trimmed.slice(1) : trimmed.startsWith("ref=") ? trimmed.slice(4) : trimmed;
|
|
13
|
+
return /^e\d+$/.test(candidate) ? candidate : undefined;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function getClickDispatchSelectorTarget(commandTokens: string[]): ClickDispatchProbeTarget | undefined {
|
|
17
|
+
if (commandTokens[0] !== "click" || commandTokens.includes("--new-tab")) return undefined;
|
|
18
|
+
const selector = commandTokens[1];
|
|
19
|
+
if (!selector || selector.startsWith("-")) return undefined;
|
|
20
|
+
if (parseClickRefId(selector)) return undefined;
|
|
21
|
+
if (selector.startsWith("xpath=")) return { kind: "xpath", selector: selector.slice("xpath=".length) };
|
|
22
|
+
return { kind: "selector", selector };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function getEvalResultRecord(data: unknown): Record<string, unknown> | undefined {
|
|
26
|
+
return isRecord(data) && isRecord(data.result) ? data.result : undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function buildClickDispatchProbeInstallScript(probe: ClickDispatchProbe): string {
|
|
30
|
+
const target = probe.target;
|
|
31
|
+
const resolveTarget = target.kind === "selector"
|
|
32
|
+
? `(() => { try { return document.querySelector(${JSON.stringify(target.selector)}); } catch { return null; } })()`
|
|
33
|
+
: `(() => { try { return document.evaluate(${JSON.stringify(target.selector)}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; } catch { return null; } })()`;
|
|
34
|
+
return `(() => {
|
|
35
|
+
const marker = ${JSON.stringify(probe.marker)};
|
|
36
|
+
const element = ${resolveTarget};
|
|
37
|
+
if (!element) return { status: "target-not-found", marker };
|
|
38
|
+
const state = { events: [], target: { tagName: element.tagName.toLowerCase() } };
|
|
39
|
+
const eventTypes = ["pointerdown", "mousedown", "pointerup", "mouseup", "click"];
|
|
40
|
+
const listeners = eventTypes.map((type) => {
|
|
41
|
+
const listener = (event) => {
|
|
42
|
+
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
|
|
43
|
+
const eventTarget = event.target;
|
|
44
|
+
const targetMatched = path.includes(element) || eventTarget === element || (eventTarget instanceof Node && element.contains(eventTarget));
|
|
45
|
+
state.events.push({ type: event.type, isTrusted: event.isTrusted === true, targetMatched });
|
|
46
|
+
};
|
|
47
|
+
document.addEventListener(type, listener, true);
|
|
48
|
+
return [type, listener];
|
|
49
|
+
});
|
|
50
|
+
state.cleanup = () => listeners.forEach(([type, listener]) => document.removeEventListener(type, listener, true));
|
|
51
|
+
window[marker] = state;
|
|
52
|
+
return { status: "installed", marker, target: state.target };
|
|
53
|
+
})()`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function buildClickDispatchProbeCheckScript(probe: ClickDispatchProbe): string {
|
|
57
|
+
return `(() => {
|
|
58
|
+
const marker = ${JSON.stringify(probe.marker)};
|
|
59
|
+
const state = window[marker];
|
|
60
|
+
const finish = (payload) => {
|
|
61
|
+
if (state && typeof state.cleanup === "function") state.cleanup();
|
|
62
|
+
try { delete window[marker]; } catch {}
|
|
63
|
+
return payload;
|
|
64
|
+
};
|
|
65
|
+
if (!state || !Array.isArray(state.events)) return finish({ status: "probe-missing", nativeEventCount: 0 });
|
|
66
|
+
const nativeEventCount = state.events.filter((event) => event && event.isTrusted === true && event.targetMatched === true).length;
|
|
67
|
+
if (nativeEventCount > 0) return finish({ status: "native-event-observed", nativeEventCount, target: state.target });
|
|
68
|
+
return finish({ status: "no-native-event-observed", nativeEventCount, target: state.target });
|
|
69
|
+
})()`;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function buildClickDispatchProbeCleanupScript(probe: ClickDispatchProbe): string {
|
|
73
|
+
return `(() => {
|
|
74
|
+
const marker = ${JSON.stringify(probe.marker)};
|
|
75
|
+
const state = window[marker];
|
|
76
|
+
if (state && typeof state.cleanup === "function") state.cleanup();
|
|
77
|
+
try { delete window[marker]; } catch {}
|
|
78
|
+
return { status: "cleaned-up" };
|
|
79
|
+
})()`;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function redactClickDispatchTarget(target: ClickDispatchProbeTarget): ClickDispatchProbeTarget {
|
|
83
|
+
return target.kind === "selector" || target.kind === "xpath"
|
|
84
|
+
? { ...target, selector: redactSensitiveText(target.selector) }
|
|
85
|
+
: target;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function formatClickDispatchDiagnosticText(diagnostic: ClickDispatchDiagnostic): string {
|
|
89
|
+
return `Click dispatch diagnostic: ${diagnostic.summary}`;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function buildClickDispatchNextActions(options: { commandTokens: string[]; sessionName?: string }): AgentBrowserNextAction[] {
|
|
93
|
+
const retryArgs = options.commandTokens[0] === "click" ? options.commandTokens : ["click", ...options.commandTokens];
|
|
94
|
+
return [
|
|
95
|
+
{
|
|
96
|
+
id: "inspect-click-dispatch-miss",
|
|
97
|
+
params: { args: withOptionalSessionArgs(options.sessionName, ["snapshot", "-i"]) },
|
|
98
|
+
reason: "Refresh interactive refs and verify the intended click target before retrying upstream click.",
|
|
99
|
+
safety: "Read-only snapshot; the wrapper does not replay clicks in-page when upstream reports success without DOM events.",
|
|
100
|
+
tool: "agent_browser",
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
id: "retry-click-after-dispatch-miss",
|
|
104
|
+
params: { args: withOptionalSessionArgs(options.sessionName, retryArgs) },
|
|
105
|
+
reason: "Retry the same upstream click after confirming the target is visible; do not assume the prior success mutated the page.",
|
|
106
|
+
safety: "Only retry when the target is still intended; use page-change evidence or a fresh snapshot before continuing the workflow.",
|
|
107
|
+
tool: "agent_browser",
|
|
108
|
+
},
|
|
109
|
+
];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function prepareClickDispatchProbe(options: { commandTokens: string[]; cwd: string; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchProbe | undefined> {
|
|
113
|
+
if (!options.sessionName || options.commandTokens[0] !== "click" || options.commandTokens.includes("--new-tab")) return undefined;
|
|
114
|
+
const target = getClickDispatchSelectorTarget(options.commandTokens);
|
|
115
|
+
if (!target) return undefined;
|
|
116
|
+
const probe: ClickDispatchProbe = { marker: `${CLICK_DISPATCH_MARKER_PREFIX}${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`, target };
|
|
117
|
+
const installData = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeInstallScript(probe) });
|
|
118
|
+
const installResult = getEvalResultRecord(installData);
|
|
119
|
+
return installResult?.status === "installed" ? probe : undefined;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export async function collectClickDispatchDiagnostic(options: { cwd: string; probe?: ClickDispatchProbe; sessionName?: string; signal?: AbortSignal }): Promise<ClickDispatchDiagnostic | undefined> {
|
|
123
|
+
if (!options.probe || !options.sessionName) return undefined;
|
|
124
|
+
const data = await runSessionCommandData({ args: ["eval", "--stdin"], cwd: options.cwd, sessionName: options.sessionName, signal: options.signal, stdin: buildClickDispatchProbeCheckScript(options.probe) });
|
|
125
|
+
const result = getEvalResultRecord(data);
|
|
126
|
+
if (!result) return undefined;
|
|
127
|
+
const status = typeof result.status === "string" ? result.status : undefined;
|
|
128
|
+
if (status !== "no-native-event-observed") return undefined;
|
|
129
|
+
const nativeEventCount = typeof result.nativeEventCount === "number" ? result.nativeEventCount : 0;
|
|
130
|
+
const summary = "Upstream click reported success but no trusted DOM event reached the selected element. Gather evidence with snapshot or page-change checks, then retry upstream click or report the workflow issue; the wrapper does not replay clicks in-page.";
|
|
131
|
+
return {
|
|
132
|
+
nativeEventCount,
|
|
133
|
+
reason: "native-click-produced-no-target-dom-event",
|
|
134
|
+
status,
|
|
135
|
+
summary,
|
|
136
|
+
target: redactClickDispatchTarget(options.probe.target),
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export async function cleanupClickDispatchProbe(options: { cwd: string; probe?: ClickDispatchProbe; sessionName?: string }): Promise<void> {
|
|
141
|
+
if (!options.probe || !options.sessionName) return;
|
|
142
|
+
await runSessionCommandData({
|
|
143
|
+
args: ["eval", "--stdin"],
|
|
144
|
+
cwd: options.cwd,
|
|
145
|
+
sessionName: options.sessionName,
|
|
146
|
+
stdin: buildClickDispatchProbeCleanupScript(options.probe),
|
|
147
|
+
timeoutMs: CLICK_DISPATCH_CLEANUP_TIMEOUT_MS,
|
|
148
|
+
}).catch(() => undefined);
|
|
149
|
+
}
|