pi-feats 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +24 -1
  2. package/THIRD_PARTY_NOTICES.md +12 -0
  3. package/assets/pi-feats-preview.png +0 -0
  4. package/extensions/api-server/application-log-store.ts +31 -0
  5. package/extensions/api-server/application-runtime.ts +80 -18
  6. package/extensions/api-server/application-store.ts +6 -2
  7. package/extensions/api-server/profile-store.ts +47 -20
  8. package/extensions/api-server/server.ts +182 -47
  9. package/extensions/browser-harness/cdp/commands.ts +260 -0
  10. package/extensions/browser-harness/cdp/console-buffer.ts +188 -0
  11. package/extensions/browser-harness/cdp/discovery.ts +198 -0
  12. package/extensions/browser-harness/cdp/errors.ts +23 -0
  13. package/extensions/browser-harness/cdp/events.ts +170 -0
  14. package/extensions/browser-harness/cdp/network-buffer.ts +146 -0
  15. package/extensions/browser-harness/cdp/ownership.ts +73 -0
  16. package/extensions/browser-harness/cdp/record-store.ts +61 -0
  17. package/extensions/browser-harness/cdp/session.ts +403 -0
  18. package/extensions/browser-harness/cdp/steel-transport.ts +151 -0
  19. package/extensions/browser-harness/cdp/target-factory.ts +153 -0
  20. package/extensions/browser-harness/cdp/types.ts +82 -0
  21. package/extensions/browser-harness/client.ts +366 -0
  22. package/extensions/browser-harness/daemon/bridge.ts +531 -0
  23. package/extensions/browser-harness/daemon/index.ts +86 -0
  24. package/extensions/browser-harness/daemon/protocol.ts +110 -0
  25. package/extensions/browser-harness/daemon/queue.ts +85 -0
  26. package/extensions/browser-harness/daemon/server.ts +170 -0
  27. package/extensions/browser-harness/daemon/spawn.ts +124 -0
  28. package/extensions/browser-harness/daemon/transport.ts +178 -0
  29. package/extensions/browser-harness/domains/ax-tree.ts +179 -0
  30. package/extensions/browser-harness/domains/box.ts +51 -0
  31. package/extensions/browser-harness/domains/cdp-call.ts +55 -0
  32. package/extensions/browser-harness/domains/click.ts +97 -0
  33. package/extensions/browser-harness/domains/console.ts +219 -0
  34. package/extensions/browser-harness/domains/dialog.ts +34 -0
  35. package/extensions/browser-harness/domains/drag.ts +67 -0
  36. package/extensions/browser-harness/domains/element-call.ts +117 -0
  37. package/extensions/browser-harness/domains/files.ts +219 -0
  38. package/extensions/browser-harness/domains/fill-engine.ts +80 -0
  39. package/extensions/browser-harness/domains/form-batch.ts +65 -0
  40. package/extensions/browser-harness/domains/form.ts +283 -0
  41. package/extensions/browser-harness/domains/headers.ts +89 -0
  42. package/extensions/browser-harness/domains/history.ts +80 -0
  43. package/extensions/browser-harness/domains/isolated-tab.ts +91 -0
  44. package/extensions/browser-harness/domains/js.ts +241 -0
  45. package/extensions/browser-harness/domains/keyboard.ts +197 -0
  46. package/extensions/browser-harness/domains/navigate.ts +161 -0
  47. package/extensions/browser-harness/domains/network.ts +301 -0
  48. package/extensions/browser-harness/domains/page.ts +154 -0
  49. package/extensions/browser-harness/domains/readpage/read-page.ts +174 -0
  50. package/extensions/browser-harness/domains/readpage/readability.ts +65 -0
  51. package/extensions/browser-harness/domains/ref-resolve.ts +118 -0
  52. package/extensions/browser-harness/domains/render.ts +43 -0
  53. package/extensions/browser-harness/domains/screenshot-capture.ts +80 -0
  54. package/extensions/browser-harness/domains/screenshot.ts +76 -0
  55. package/extensions/browser-harness/domains/scroll.ts +59 -0
  56. package/extensions/browser-harness/domains/search/google-serp.ts +85 -0
  57. package/extensions/browser-harness/domains/search/web-search.ts +144 -0
  58. package/extensions/browser-harness/domains/setup.ts +35 -0
  59. package/extensions/browser-harness/domains/snapshot.ts +256 -0
  60. package/extensions/browser-harness/domains/tabs.ts +202 -0
  61. package/extensions/browser-harness/domains/target.ts +35 -0
  62. package/extensions/browser-harness/domains/viewport.ts +42 -0
  63. package/extensions/browser-harness/index.ts +88 -0
  64. package/extensions/browser-harness/profile/bind.ts +90 -0
  65. package/extensions/browser-harness/profile/browser-process.ts +320 -0
  66. package/extensions/browser-harness/profile/command.ts +189 -0
  67. package/extensions/browser-harness/profile/launch.ts +80 -0
  68. package/extensions/browser-harness/profile/list.ts +131 -0
  69. package/extensions/browser-harness/profile/paths.ts +103 -0
  70. package/extensions/browser-harness/profile/store.ts +84 -0
  71. package/extensions/browser-harness/prompt.ts +16 -0
  72. package/extensions/browser-harness/registry.ts +80 -0
  73. package/extensions/browser-harness/schemas/common.ts +11 -0
  74. package/extensions/browser-harness/schemas/parse.ts +23 -0
  75. package/extensions/browser-harness/setup.ts +47 -0
  76. package/extensions/browser-harness/state.ts +51 -0
  77. package/extensions/browser-harness/steel.ts +215 -0
  78. package/extensions/browser-harness/util/debug.ts +8 -0
  79. package/extensions/browser-harness/util/guards.ts +30 -0
  80. package/extensions/browser-harness/util/js-template.ts +12 -0
  81. package/extensions/browser-harness/util/mutex.ts +20 -0
  82. package/extensions/browser-harness/util/paths.ts +31 -0
  83. package/extensions/browser-harness/util/result.ts +9 -0
  84. package/extensions/browser-harness/util/sharp-shim.ts +42 -0
  85. package/extensions/browser-harness/util/tool.ts +185 -0
  86. package/extensions/browser-harness/util/truncate.ts +41 -0
  87. package/extensions/cli-resources.ts +35 -16
  88. package/extensions/composio/index.ts +163 -0
  89. package/extensions/context-memory/index.ts +58 -0
  90. package/extensions/lib/application-context.ts +22 -0
  91. package/extensions/lib/application-handler-templates.ts +17 -1
  92. package/extensions/lib/context-memory.ts +165 -0
  93. package/extensions/lib/profile-sandbox.ts +3 -0
  94. package/extensions/lib/workflow-harness.ts +238 -0
  95. package/extensions/pi-console-webui/app/[section]/page.tsx +6 -1
  96. package/extensions/pi-console-webui/app/applications/[slug]/page.tsx +15 -0
  97. package/extensions/pi-console-webui/app/layout.tsx +6 -1
  98. package/extensions/pi-console-webui/app/login/page.tsx +5 -9
  99. package/extensions/pi-console-webui/app/not-found.tsx +5 -0
  100. package/extensions/pi-console-webui/app/page.tsx +6 -0
  101. package/extensions/pi-console-webui/app/terminal/page.tsx +3 -0
  102. package/extensions/pi-console-webui/components/application-handler-editor.tsx +4 -4
  103. package/extensions/pi-console-webui/components/application-logs.tsx +12 -8
  104. package/extensions/pi-console-webui/components/application-settings.tsx +5 -1
  105. package/extensions/pi-console-webui/components/chat-workspace.tsx +7 -4
  106. package/extensions/pi-console-webui/components/console-page.tsx +3 -1
  107. package/extensions/pi-console-webui/components/console.tsx +53 -7
  108. package/extensions/pi-console-webui/components/login-form.tsx +14 -0
  109. package/extensions/pi-console-webui/components/pulse-resources.tsx +10 -6
  110. package/extensions/pi-console-webui/components/skill-resources.tsx +3 -3
  111. package/extensions/pi-console-webui/lib/page-metadata.ts +33 -0
  112. package/extensions/profiles.ts +18 -7
  113. package/extensions/pulse/store.ts +5 -2
  114. package/extensions/sequential-workflow.ts +340 -256
  115. package/extensions/skill-sources/store.ts +24 -3
  116. package/licenses/pi-browser-harness-MIT.txt +21 -0
  117. package/package.json +25 -4
  118. package/schemas/sequential-workflow-template.schema.json +62 -0
  119. package/skills/browser-harness/SKILL.md +33 -0
  120. package/skills/composio-connections/SKILL.md +75 -0
  121. package/skills/sequential-workflow/SKILL.md +70 -0
  122. package/skills/sequential-workflow-template/SKILL.md +44 -0
@@ -0,0 +1,65 @@
1
+ export type ContentBlock = {
2
+ readonly kind: "paragraph" | "heading" | "listitem" | "blockquote" | "other";
3
+ readonly text: string;
4
+ readonly linkTextLength: number;
5
+ readonly inBoilerplate: boolean;
6
+ };
7
+
8
+ export type PageCapture = {
9
+ readonly url: string;
10
+ readonly title: string;
11
+ readonly blocks: ReadonlyArray<ContentBlock>;
12
+ readonly bodyText: string;
13
+ };
14
+
15
+ export type ReadablePage = {
16
+ readonly title: string;
17
+ readonly url: string;
18
+ readonly text: string;
19
+ readonly wordCount: number;
20
+ };
21
+
22
+ const MIN_PARAGRAPH_CHARS = 25;
23
+ const MAX_LINK_DENSITY = 0.5;
24
+ const MIN_ARTICLE_WORDS = 40;
25
+
26
+ const linkDensity = (block: ContentBlock): number => {
27
+ if (block.text.length === 0) return 1;
28
+ return block.linkTextLength / block.text.length;
29
+ };
30
+
31
+ const isArticleBlock = (block: ContentBlock): boolean => {
32
+ if (block.inBoilerplate) return false;
33
+ if (linkDensity(block) > MAX_LINK_DENSITY) return false;
34
+ if (block.kind === "heading" || block.kind === "blockquote" || block.kind === "listitem") {
35
+ return block.text.trim().length > 0;
36
+ }
37
+ return block.text.trim().length >= MIN_PARAGRAPH_CHARS;
38
+ };
39
+
40
+ const render = (block: ContentBlock): string => {
41
+ const text = block.text.trim();
42
+ if (block.kind === "heading") return `\n## ${text}\n`;
43
+ if (block.kind === "listitem") return `- ${text}`;
44
+ if (block.kind === "blockquote") return `> ${text}`;
45
+ return text;
46
+ };
47
+
48
+ const wordCountOf = (text: string): number => {
49
+ const trimmed = text.trim();
50
+ return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length;
51
+ };
52
+
53
+ export const extractReadable = (capture: PageCapture): ReadablePage => {
54
+ const articleText = capture.blocks.filter(isArticleBlock).map(render).join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
55
+
56
+ const useFallback = wordCountOf(articleText) < MIN_ARTICLE_WORDS;
57
+ const text = useFallback ? capture.bodyText.trim() : articleText;
58
+
59
+ return {
60
+ title: capture.title.trim(),
61
+ url: capture.url,
62
+ text,
63
+ wordCount: wordCountOf(text),
64
+ };
65
+ };
@@ -0,0 +1,118 @@
1
+ import type { BrowserClient } from "../client";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import type { ToolErr } from "../util/tool";
4
+ import { type Box, boxOf } from "./box";
5
+ import { cdpCall } from "./cdp-call";
6
+ import { buildTree, collectInteractiveTargets, type SlimNode } from "./ax-tree";
7
+
8
+ const staleErr = (ref: string): ToolErr => ({
9
+ kind: "invalid_state",
10
+ message: `Ref ${ref} is unknown or stale — re-run browser_snapshot to get fresh refs.`,
11
+ details: { ref },
12
+ });
13
+
14
+ export const resolveRefToObjectId = async (
15
+ client: BrowserClient,
16
+ ref: string,
17
+ ): Promise<Result<string, ToolErr>> => {
18
+ const backendId = client.session().resolveRef(ref);
19
+ if (backendId === undefined) return err(staleErr(ref));
20
+ const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId });
21
+ if (!resolved.success) {
22
+ return err(staleErr(ref));
23
+ }
24
+ const objectId = resolved.data.object.objectId;
25
+ if (!objectId) return err(staleErr(ref));
26
+ return ok(objectId);
27
+ };
28
+
29
+ export const resolveRefToBackendId = (client: BrowserClient, ref: string): Result<number, ToolErr> => {
30
+ const backendId = client.session().resolveRef(ref);
31
+ if (backendId === undefined) return err(staleErr(ref));
32
+ return ok(backendId);
33
+ };
34
+
35
+ export const resolveRefToBox = async (
36
+ client: BrowserClient,
37
+ ref: string,
38
+ ): Promise<Result<Box, ToolErr>> => {
39
+ const backendId = client.session().resolveRef(ref);
40
+ if (backendId === undefined) return err(staleErr(ref));
41
+ const box = await boxOf(client, backendId);
42
+ if (!box.success) return err(staleErr(ref));
43
+ return ok(box.data);
44
+ };
45
+
46
+ const sigOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}|${n.value ?? ""}|${n.state ?? ""}`;
47
+ const keyOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}`;
48
+
49
+ export const interactiveDiff = async (client: BrowserClient): Promise<string> => {
50
+ const session = client.session();
51
+ const prev = session.refSignatures();
52
+
53
+ const axRes = await cdpCall(client, "Accessibility.getFullAXTree", {});
54
+ if (!axRes.success) return "";
55
+ const rawNodes = axRes.data.nodes;
56
+ const slim = buildTree(rawNodes, { interestingOnly: true, maxNodes: 1000 });
57
+ const targets = collectInteractiveTargets(slim);
58
+
59
+ const priorRefs = session.refMappings();
60
+ const refByBackendId = new Map<number, string>();
61
+ for (const [ref, backendId] of priorRefs) refByBackendId.set(backendId, ref);
62
+ let nextIndex = 1;
63
+ for (const ref of priorRefs.keys()) {
64
+ const digits = /^e(\d+)$/.exec(ref)?.[1];
65
+ if (digits !== undefined) nextIndex = Math.max(nextIndex, Number(digits) + 1);
66
+ }
67
+ const refMap = new Map<string, number>();
68
+ const refSig = new Map<string, string>();
69
+ const taken = new Set<string>();
70
+ for (const { node, backendId } of targets) {
71
+ const prior = refByBackendId.get(backendId);
72
+ const ref = prior !== undefined && !taken.has(prior) ? prior : `e${nextIndex++}`;
73
+ taken.add(ref);
74
+ node.ref = ref;
75
+ refMap.set(ref, backendId);
76
+ refSig.set(ref, sigOf(node));
77
+ }
78
+ session.setRefMap(refMap, refSig);
79
+
80
+ const prevByKey = new Map<string, string>();
81
+ for (const sig of prev.values()) {
82
+ const parts = sig.split("|");
83
+ prevByKey.set(`${parts[0]}|${parts[1]}`, sig);
84
+ }
85
+ const curByKey = new Map<string, { ref: string; sig: string }>();
86
+ for (const { node } of targets) {
87
+ if (node.ref) curByKey.set(keyOf(node), { ref: node.ref, sig: sigOf(node) });
88
+ }
89
+
90
+ const appeared: string[] = [];
91
+ const changed: string[] = [];
92
+ for (const [key, { ref, sig }] of curByKey) {
93
+ const before = prevByKey.get(key);
94
+ if (before === undefined) {
95
+ appeared.push(` *[${ref}] ${key.replace("|", ' "')}"`);
96
+ } else if (before !== sig) {
97
+ const va = before.split("|")[2] ?? "";
98
+ const vb = sig.split("|")[2] ?? "";
99
+ const sa = before.split("|")[3] ?? "";
100
+ const sb = sig.split("|")[3] ?? "";
101
+ const detail = va !== vb ? `value ${JSON.stringify(va)} → ${JSON.stringify(vb)}` : `state ${JSON.stringify(sa)} → ${JSON.stringify(sb)}`;
102
+ changed.push(` [${ref}] ${key.replace("|", ' "')}": ${detail}`);
103
+ }
104
+ }
105
+ const removed: string[] = [];
106
+ for (const key of prevByKey.keys()) {
107
+ if (!curByKey.has(key)) removed.push(` ${key.replace("|", ' "')}"`);
108
+ }
109
+
110
+ if (appeared.length === 0 && changed.length === 0 && removed.length === 0) return "";
111
+ const lines: string[] = ["", "Page changes (re-snapshot for full tree):"];
112
+ if (changed.length) lines.push("Changed:", ...changed.slice(0, 12));
113
+ if (appeared.length) lines.push("New (*):", ...appeared.slice(0, 12));
114
+ if (removed.length) lines.push("Removed:", ...removed.slice(0, 8));
115
+ const overflow = Math.max(0, changed.length - 12) + Math.max(0, appeared.length - 12) + Math.max(0, removed.length - 8);
116
+ if (overflow > 0) lines.push(` …and ${overflow} more (browser_snapshot for full state)`);
117
+ return lines.join("\n");
118
+ };
@@ -0,0 +1,43 @@
1
+ import { Markdown, Text } from "@earendil-works/pi-tui";
2
+ import { getMarkdownTheme, keyHint } from "@earendil-works/pi-coding-agent";
3
+ import type { Component } from "@earendil-works/pi-tui";
4
+ import type { Theme } from "@earendil-works/pi-coding-agent";
5
+ import { isRecord } from "../util/guards";
6
+
7
+ const COMPACT_PREVIEW_LINES = 6;
8
+
9
+ type ExpandableText = {
10
+ readonly summary: string;
11
+ readonly body: string;
12
+ readonly fullOutputPath?: string;
13
+ };
14
+
15
+ const isExpandableText = (value: unknown): value is ExpandableText =>
16
+ isRecord(value) && typeof value["summary"] === "string" && typeof value["body"] === "string";
17
+
18
+ const extractRender = (details: unknown): unknown =>
19
+ isRecord(details) ? details["render"] : undefined;
20
+
21
+ export const renderExpandableText = (
22
+ label: string,
23
+ result: { readonly details?: unknown },
24
+ expanded: boolean,
25
+ theme: Theme,
26
+ ): Component => {
27
+ const render = extractRender(result.details);
28
+ if (!isExpandableText(render)) return new Text(theme.fg("error", `${label}: no details`), 0, 0);
29
+
30
+ if (!expanded) {
31
+ const lines = render.body.split("\n");
32
+ const preview = lines.slice(0, COMPACT_PREVIEW_LINES).join("\n");
33
+ const more = lines.length > COMPACT_PREVIEW_LINES ? `\n… ${lines.length - COMPACT_PREVIEW_LINES} more lines` : "";
34
+ const md = `**${render.summary}**\n\n${preview}${more}\n\n${keyHint("app.tools.expand", "to expand")}`;
35
+ return new Markdown(md, 0, 0, getMarkdownTheme());
36
+ }
37
+
38
+ const tail = render.fullOutputPath
39
+ ? `\n\nFull output at \`${render.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
40
+ : `\n\n${keyHint("app.tools.expand", "to collapse")}`;
41
+ const md = `**${render.summary}**\n\n${render.body}${tail}`;
42
+ return new Markdown(md, 0, 0, getMarkdownTheme());
43
+ };
@@ -0,0 +1,80 @@
1
+ import { rename, writeFile } from "node:fs/promises";
2
+ import type { BrowserClient } from "../client";
3
+ import { type Result, ok } from "../util/result";
4
+ import type { ToolErr } from "../util/tool";
5
+ import { screenshotPath } from "../util/paths";
6
+ import { loadSharp } from "../util/sharp-shim";
7
+ import { safeJs } from "../util/js-template";
8
+ import { cdpCall, evalJs } from "./cdp-call";
9
+
10
+ export type CaptureArgs = {
11
+ readonly fullPage?: boolean;
12
+ readonly format?: "png" | "jpeg";
13
+ readonly quality?: number;
14
+ };
15
+
16
+ export const captureBase = async (
17
+ client: BrowserClient,
18
+ args: CaptureArgs,
19
+ ): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
20
+ const format = args.format ?? "png";
21
+ const quality = args.quality ?? 80;
22
+ const path = screenshotPath(client.namespace, format);
23
+ const r = await cdpCall(client, "Page.captureScreenshot", {
24
+ format,
25
+ captureBeyondViewport: args.fullPage ?? false,
26
+ ...(format === "jpeg" ? { quality } : {}),
27
+ });
28
+ if (!r.success) return r;
29
+ await writeFile(path, Buffer.from(r.data.data, "base64"));
30
+ return ok({ path, format });
31
+ };
32
+
33
+ export const resizeIfNeeded = async (
34
+ path: string,
35
+ maxDim: number,
36
+ ): Promise<{ readonly note: string }> => {
37
+ const load = await loadSharp();
38
+ if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
39
+ if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
40
+ try {
41
+ const meta = await load.sharp(path).metadata();
42
+ const w = meta.width ?? 0;
43
+ const h = meta.height ?? 0;
44
+ if (Math.max(w, h) <= maxDim) return { note: "" };
45
+ const tmp = `${path}.resized`;
46
+ await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
47
+ await rename(tmp, path);
48
+ return { note: ` (resized to fit ${maxDim}px)` };
49
+ } catch (e) {
50
+ return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
51
+ }
52
+ };
53
+
54
+ export const captureWithCrosshair = async (
55
+ client: BrowserClient,
56
+ args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
57
+ ): Promise<Result<{ readonly path: string }, ToolErr>> => {
58
+ const cap = await captureBase(client, args);
59
+ if (!cap.success) return cap;
60
+ const load = await loadSharp();
61
+ if (load.kind !== "ok") return ok({ path: cap.data.path });
62
+ try {
63
+ const dprR = await evalJs(client, safeJs`window.devicePixelRatio`);
64
+ const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
65
+ const meta = await load.sharp(cap.data.path).metadata();
66
+ const w = meta.width ?? 0;
67
+ const h = meta.height ?? 0;
68
+ const px = Math.round(args.x * dpr);
69
+ const py = Math.round(args.y * dpr);
70
+ const r = Math.round(15 * dpr);
71
+ const stroke = Math.max(2, Math.round(3 * dpr));
72
+ const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
73
+ const tmp = `${cap.data.path}.debug`;
74
+ await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
75
+ await rename(tmp, cap.data.path);
76
+ return ok({ path: cap.data.path });
77
+ } catch {
78
+ return ok({ path: cap.data.path });
79
+ }
80
+ };
@@ -0,0 +1,76 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { Type } from "typebox";
3
+ import { Image, type ImageTheme, Text, truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
4
+ import { type Result, ok } from "../util/result";
5
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
6
+ import { asRecord, asString } from "../util/guards";
7
+ import { captureBase, resizeIfNeeded } from "./screenshot-capture";
8
+
9
+ const ScreenshotArgs = Type.Object({
10
+ fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })),
11
+ format: Type.Optional(Type.Union([Type.Literal("png"), Type.Literal("jpeg")], { default: "png" })),
12
+ quality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100, default: 80, description: "JPEG quality 1-100" })),
13
+ maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
14
+ });
15
+
16
+ export const screenshotTool = defineBrowserTool({
17
+ name: "browser_screenshot",
18
+ label: "Browser Screenshot",
19
+ description:
20
+ "Capture the page as a JPEG/PNG image. NOT a default exploration tool. Use ONLY when (a) you need to verify visual properties (layout, color, spacing, rendered text legibility), or (b) browser_snapshot and browser_execute_js cannot answer your question. For understanding what's on the page use browser_snapshot. For specific element values use browser_execute_js. For click coordinates use the @(x,y) hints in browser_snapshot's outline.",
21
+ promptSnippet: "Capture a screenshot — visual verification only",
22
+ promptGuidelines: [
23
+ "DO NOT use as a default exploration tool. browser_snapshot is the default for understanding page structure — far cheaper and more reliable.",
24
+ "DO NOT use to find click coordinates. browser_snapshot already includes @(x,y) for every interactive element. If a target isn't there, use browser_execute_js with element.getBoundingClientRect().",
25
+ "DO use to verify visual rendering after an action: did the modal animate in correctly, did the chart render, are colors right, did the layout reflow as expected.",
26
+ "DO use as a debugging fallback when browser_execute_js or browser_snapshot return surprising results and you suspect the rendered page differs from the DOM.",
27
+ "format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.",
28
+ ],
29
+ parameters: ScreenshotArgs,
30
+ concurrency: "parallel",
31
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
32
+ const cap = await captureBase(client, args);
33
+ if (!cap.success) return cap;
34
+ let note = "";
35
+ if (args.maxDim !== undefined) {
36
+ const resized = await resizeIfNeeded(cap.data.path, args.maxDim);
37
+ note = resized.note;
38
+ }
39
+ return ok({
40
+ text: `Screenshot saved: ${cap.data.path}${note}`,
41
+ details: { path: cap.data.path, format: cap.data.format, attached: false },
42
+ });
43
+ },
44
+
45
+ renderResult(result, _expanded, theme) {
46
+ const raw = asRecord(result.details);
47
+ const filePath = raw === undefined ? undefined : asString(raw["path"]);
48
+ if (!filePath) {
49
+ return new Text(theme.fg("error", "Screenshot path missing"), 0, 0);
50
+ }
51
+ const format = raw === undefined ? undefined : asString(raw["format"]);
52
+ try {
53
+ const buf = readFileSync(filePath);
54
+ const b64 = buf.toString("base64");
55
+ const mimeType = format === "jpeg" ? "image/jpeg" : "image/png";
56
+ const imageTheme: ImageTheme = {
57
+ fallbackColor: (str: string) => theme.fg("dim", str),
58
+ };
59
+ const image = new Image(b64, mimeType, imageTheme, {
60
+ maxWidthCells: 80,
61
+ maxHeightCells: 24,
62
+ filename: filePath,
63
+ });
64
+ // Image's text fallback does not respect the rendered width, so an unwrapped long path can blow past the terminal width and crash the host TUI.
65
+ return {
66
+ invalidate: () => image.invalidate(),
67
+ render: (width: number) =>
68
+ image.render(width).map((line) =>
69
+ visibleWidth(line) > width ? truncateToWidth(line, width) : line,
70
+ ),
71
+ };
72
+ } catch {
73
+ return new Text(theme.fg("warning", `Screenshot saved: ${filePath}`), 0, 0);
74
+ }
75
+ },
76
+ });
@@ -0,0 +1,59 @@
1
+ import { Type } from "typebox";
2
+ import { type Result, err, ok } from "../util/result";
3
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
4
+ import { cdpCall } from "./cdp-call";
5
+
6
+ const ScrollArgs = Type.Object({
7
+ x: Type.Optional(Type.Number({ description: "X coordinate where to scroll. Default: viewport center" })),
8
+ y: Type.Optional(Type.Number({ description: "Y coordinate where to scroll. Default: viewport center" })),
9
+ deltaX: Type.Optional(Type.Number({ default: 0, description: "Horizontal scroll delta (CSS pixels). Positive = right." })),
10
+ deltaY: Type.Optional(Type.Number({
11
+ default: 300,
12
+ description: "Vertical scroll delta (CSS pixels). Positive = down, negative = up (W3C wheel-event convention). Default: 300 (scroll down).",
13
+ })),
14
+ });
15
+
16
+ export const scrollTool = defineBrowserTool({
17
+ name: "browser_scroll",
18
+ label: "Browser Scroll",
19
+ description:
20
+ "Scroll the page at given coordinates. deltaY follows W3C wheel-event convention: positive = scroll down, negative = scroll up. Default scrolls down (deltaY = 300).",
21
+ promptSnippet: "Scroll the page (deltaY positive=down, negative=up)",
22
+ promptGuidelines: [
23
+ "Default behavior scrolls down 300px (deltaY=300). Pass a negative deltaY to scroll up.",
24
+ "Pass x/y to target a specific scrollable region (e.g., a div with overflow); otherwise scrolls the page at viewport center.",
25
+ ],
26
+ parameters: ScrollArgs,
27
+ concurrency: "serialized",
28
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
29
+ const info = await client.pageInfo();
30
+ if (!info.success) return err({ kind: "cdp_error", message: info.error.message });
31
+ if ("dialog" in info.data) {
32
+ return err({
33
+ kind: "invalid_state",
34
+ message: `Dialog open: ${info.data.dialog.type} — ${info.data.dialog.message}`,
35
+ });
36
+ }
37
+ const cx = args.x ?? Math.round(info.data.width / 2);
38
+ const cy = args.y ?? Math.round(info.data.height / 2);
39
+ const dx = args.deltaX ?? 0;
40
+ const dy = args.deltaY ?? 300;
41
+ // The page target must be focused or Input events are dropped silently: mouseWheel hangs and synthesizeScrollGesture returns ok without scrolling.
42
+ const front = await cdpCall(client, "Page.bringToFront", {});
43
+ if (!front.success) return front;
44
+ // Establish compositor mouse position first, or CDP sometimes drops the wheel event with a timeout.
45
+ const moved = await cdpCall(client, "Input.dispatchMouseEvent", {
46
+ type: "mouseMoved", x: cx, y: cy, button: "none", buttons: 0,
47
+ });
48
+ if (!moved.success) return moved;
49
+ const wheel = await cdpCall(client, "Input.dispatchMouseEvent", {
50
+ type: "mouseWheel", x: cx, y: cy, deltaX: dx, deltaY: dy,
51
+ button: "none", buttons: 0, pointerType: "mouse",
52
+ });
53
+ if (!wheel.success) return wheel;
54
+ return ok({
55
+ text: `Scrolled at (${cx}, ${cy}) by (${dx}, ${dy})`,
56
+ details: { x: cx, y: cy, deltaX: dx, deltaY: dy },
57
+ });
58
+ },
59
+ });
@@ -0,0 +1,85 @@
1
+ export type SerpAnchor = {
2
+ readonly href: string;
3
+ readonly heading: string;
4
+ readonly snippet: string;
5
+ };
6
+
7
+ export type SerpExtraction = {
8
+ readonly anchors: ReadonlyArray<SerpAnchor>;
9
+ readonly pageText: string;
10
+ };
11
+
12
+ export type SearchResult = {
13
+ readonly title: string;
14
+ readonly url: string;
15
+ readonly snippet: string;
16
+ readonly rank: number;
17
+ };
18
+
19
+ export type SerpVerdict = "ok" | "captcha" | "no_results";
20
+
21
+ const CAPTCHA_MARKERS: ReadonlyArray<string> = [
22
+ "unusual traffic",
23
+ "not a robot",
24
+ "recaptcha",
25
+ "detected unusual",
26
+ "systems have detected",
27
+ "before you continue to google",
28
+ "our systems have detected unusual traffic",
29
+ ];
30
+
31
+ const unwrapRedirect = (href: string): string => {
32
+ const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i);
33
+ if (!wrapper) return href;
34
+ const query = href.slice(href.indexOf("?") + 1);
35
+ const target = new URLSearchParams(query).get("q");
36
+ return target ?? href;
37
+ };
38
+
39
+ const isExternalResult = (url: string): boolean => {
40
+ if (!/^https?:\/\//i.test(url)) return false;
41
+ try {
42
+ const host = new URL(url).hostname.toLowerCase();
43
+ if (host === "google.com" || host.endsWith(".google.com")) return false;
44
+ if (host === "googleusercontent.com" || host.endsWith(".googleusercontent.com")) return false;
45
+ return true;
46
+ } catch {
47
+ return false;
48
+ }
49
+ };
50
+
51
+ const dedupeKey = (url: string): string => {
52
+ try {
53
+ const u = new URL(url);
54
+ return `${u.origin}${u.pathname.replace(/\/$/, "")}`.toLowerCase();
55
+ } catch {
56
+ return url.toLowerCase();
57
+ }
58
+ };
59
+
60
+ export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => {
61
+ const seen = new Set<string>();
62
+ const results: SearchResult[] = [];
63
+ for (const anchor of anchors) {
64
+ if (results.length >= limit) break;
65
+ const url = unwrapRedirect(anchor.href);
66
+ if (!isExternalResult(url)) continue;
67
+ const key = dedupeKey(url);
68
+ if (seen.has(key)) continue;
69
+ seen.add(key);
70
+ results.push({
71
+ title: anchor.heading.trim(),
72
+ url,
73
+ snippet: anchor.snippet.trim(),
74
+ rank: results.length + 1,
75
+ });
76
+ }
77
+ return results;
78
+ };
79
+
80
+ export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => {
81
+ if (resultCount > 0) return "ok";
82
+ const lower = pageText.toLowerCase();
83
+ if (CAPTCHA_MARKERS.some((marker) => lower.includes(marker))) return "captcha";
84
+ return "no_results";
85
+ };
@@ -0,0 +1,144 @@
1
+ import { Type } from "typebox";
2
+ import { Compile } from "typebox/compile";
3
+ import { parseJson } from "../../schemas/parse";
4
+ import { type Result, err, ok } from "../../util/result";
5
+ import { applyTruncation } from "../../util/truncate";
6
+ import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
7
+ import {
8
+ closeIsolatedTab,
9
+ evalInIsolatedTab,
10
+ isJsonText,
11
+ navigateIsolatedTab,
12
+ openIsolatedTab,
13
+ waitForIsolatedLoad,
14
+ } from "../isolated-tab";
15
+ import { renderExpandableText } from "../render";
16
+ import { classifySerp, parseGoogleSerp, type SearchResult, type SerpExtraction } from "./google-serp";
17
+
18
+ const LOAD_TIMEOUT_MS = 15_000;
19
+
20
+ const WebSearchArgs = Type.Object({
21
+ query: Type.String({ description: "The search query." }),
22
+ limit: Type.Optional(
23
+ Type.Integer({ default: 10, minimum: 1, maximum: 30, description: "Max results to return. Default: 10." }),
24
+ ),
25
+ });
26
+
27
+ const PAGE_TEXT_LIMIT = 4000;
28
+
29
+ const buildSerpExtractionExpr = (): string => `
30
+ (() => {
31
+ const main = document.querySelector('#search, #rso, #main') || document.body;
32
+ const headings = Array.from(main.querySelectorAll('a h3'));
33
+ const seen = new Set();
34
+ const anchors = [];
35
+ for (const h3 of headings) {
36
+ const a = h3.closest('a[href]');
37
+ if (!a) continue;
38
+ const href = a.href;
39
+ if (!href || seen.has(href)) continue;
40
+ seen.add(href);
41
+ const container =
42
+ a.closest('div.g') ||
43
+ a.closest('div[data-hveid]') ||
44
+ a.parentElement?.parentElement ||
45
+ a.parentElement;
46
+ const containerText = container ? container.innerText || '' : '';
47
+ const heading = h3.innerText || h3.textContent || '';
48
+ const snippet = containerText.replace(heading, '').replace(/\\s+/g, ' ').trim().slice(0, 500);
49
+ anchors.push({ href, heading, snippet });
50
+ }
51
+ const pageText = (document.body.innerText || '').slice(0, ${PAGE_TEXT_LIMIT});
52
+ return JSON.stringify({ anchors, pageText });
53
+ })()
54
+ `;
55
+
56
+ const buildSerpUrl = (query: string, limit: number): string =>
57
+ `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${limit}&hl=en`;
58
+
59
+ const SerpExtractionSchema = Type.Object({
60
+ anchors: Type.Array(
61
+ Type.Object({ href: Type.String(), heading: Type.String(), snippet: Type.String() }),
62
+ ),
63
+ pageText: Type.String(),
64
+ });
65
+
66
+ const serpExtractionValidator = Compile(SerpExtractionSchema);
67
+
68
+ const formatResults = (query: string, results: ReadonlyArray<SearchResult>): string => {
69
+ const lines = results.map((r) => `${r.rank}. ${r.title}\n ${r.url}\n ${r.snippet}`);
70
+ return `${results.length} result(s) for "${query}":\n\n${lines.join("\n\n")}`;
71
+ };
72
+
73
+ export const webSearchTool = defineBrowserTool({
74
+ name: "browser_web_search",
75
+ label: "Browser Web Search",
76
+ description:
77
+ "Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.",
78
+ promptSnippet: "Search the web for a query",
79
+ promptGuidelines: [
80
+ "Returns ranked {title, url, snippet, rank} — links only, no page content. Follow up with browser_read_page.",
81
+ "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.",
82
+ "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.",
83
+ "Runs in its own isolated tab and never disturbs the user's current tab.",
84
+ ],
85
+ parameters: WebSearchArgs,
86
+ concurrency: "serialized",
87
+ async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
88
+ const limit = args.limit ?? 10;
89
+ const serpUrl = buildSerpUrl(args.query, limit);
90
+
91
+ const opened = await openIsolatedTab(client);
92
+ if (!opened.success) return err(opened.error);
93
+ const tab = opened.data;
94
+
95
+ try {
96
+ const navigated = await navigateIsolatedTab(client, tab, serpUrl);
97
+ if (!navigated.success) return err(navigated.error);
98
+
99
+ const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
100
+ if (!loaded.success) return err(loaded.error);
101
+
102
+ const extracted = await evalInIsolatedTab(client, tab, buildSerpExtractionExpr(), isJsonText);
103
+ if (!extracted.success) return err(extracted.error);
104
+
105
+ const parsedJson = parseJson(extracted.data, serpExtractionValidator);
106
+ if (!parsedJson.success) {
107
+ return err({ kind: "internal", message: `SERP extraction returned an unexpected shape: ${parsedJson.error}` });
108
+ }
109
+ const extraction: SerpExtraction = parsedJson.data;
110
+
111
+ const results = parseGoogleSerp(extraction.anchors, limit);
112
+ const verdict = classifySerp(extraction.pageText, results.length);
113
+ if (verdict !== "ok") {
114
+ return err({
115
+ kind: "invalid_state",
116
+ message: verdict === "captcha" ? "Google served a CAPTCHA / bot wall" : "no results found",
117
+ details: { reason: verdict, serpUrl },
118
+ });
119
+ }
120
+
121
+ const body = formatResults(args.query, results);
122
+ const truncated = await applyTruncation(body, "search");
123
+ return ok({
124
+ text: truncated.text,
125
+ details: {
126
+ results,
127
+ engine: "google",
128
+ query: args.query,
129
+ render: {
130
+ summary: `${results.length} result(s) · google · "${args.query}"`,
131
+ body,
132
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
133
+ },
134
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
135
+ },
136
+ });
137
+ } finally {
138
+ await closeIsolatedTab(client, tab);
139
+ }
140
+ },
141
+ renderResult(result, expanded, theme) {
142
+ return renderExpandableText("web_search", result, expanded, theme);
143
+ },
144
+ });