patchright-core 1.57.0 → 1.58.2

package/lib/server/agent/pageAgent.js

@@ -0,0 +1,204 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pageAgent_exports = {};
+__export(pageAgent_exports, {
+  pageAgentExpect: () => pageAgentExpect,
+  pageAgentExtract: () => pageAgentExtract,
+  pageAgentPerform: () => pageAgentPerform
+});
+module.exports = __toCommonJS(pageAgent_exports);
+var import_fs = __toESM(require("fs"));
+var import_path = __toESM(require("path"));
+var import_tool = require("./tool");
+var import_utilsBundle = require("../../utilsBundle");
+var import_mcpBundle = require("../../mcpBundle");
+var import_actionRunner = require("./actionRunner");
+var import_performTools = __toESM(require("./performTools"));
+var import_expectTools = __toESM(require("./expectTools"));
+var actions = __toESM(require("./actions"));
+async function pageAgentPerform(progress, context, userTask, callParams) {
+  const cacheKey = (callParams.cacheKey ?? userTask).trim();
+  if (await cachedPerform(progress, context, cacheKey))
+    return;
+  const task = `
+### Instructions
+- Perform the following task on the page.
+- Your reply should be a tool call that performs action the page".
+
+### Task
+${userTask}
+`;
+  progress.disableTimeout();
+  await runLoop(progress, context, import_performTools.default, task, void 0, callParams);
+  await updateCache(context, cacheKey);
+}
+async function pageAgentExpect(progress, context, expectation, callParams) {
+  const cacheKey = (callParams.cacheKey ?? expectation).trim();
+  if (await cachedPerform(progress, context, cacheKey))
+    return;
+  const task = `
+### Instructions
+- Call one of the "browser_expect_*" tools to verify / assert the condition.
+- You can call exactly one tool and it can't be report_results, must be one of the assertion tools.
+
+### Expectation
+${expectation}
+`;
+  progress.disableTimeout();
+  await runLoop(progress, context, import_expectTools.default, task, void 0, callParams);
+  await updateCache(context, cacheKey);
+}
+async function pageAgentExtract(progress, context, query, schema, callParams) {
+  const task = `
+### Instructions
+Extract the following information from the page. Do not perform any actions, just extract the information.
+
+### Query
+${query}`;
+  const { result } = await runLoop(progress, context, [], task, schema, callParams);
+  return result;
+}
+async function runLoop(progress, context, toolDefinitions, userTask, resultSchema, params) {
+  if (!context.agentParams.api || !context.agentParams.model)
+    throw new Error(`This action requires the API and API key to be set on the page agent. Did you mean to --run-agents=missing?`);
+  if (!context.agentParams.apiKey)
+    throw new Error(`This action requires API key to be set on the page agent.`);
+  if (context.agentParams.apiEndpoint && !URL.canParse(context.agentParams.apiEndpoint))
+    throw new Error(`Agent API endpoint "${context.agentParams.apiEndpoint}" is not a valid URL.`);
+  const snapshot = await context.takeSnapshot(progress);
+  const { tools, callTool, reportedResult, refusedToPerformReason } = (0, import_tool.toolsForLoop)(progress, context, toolDefinitions, { resultSchema, refuseToPerform: "allow" });
+  const secrets = Object.fromEntries((context.agentParams.secrets || [])?.map((s) => [s.name, s.value]));
+  const apiCacheTextBefore = context.agentParams.apiCacheFile ? await import_fs.default.promises.readFile(context.agentParams.apiCacheFile, "utf-8").catch(() => "{}") : "{}";
+  const apiCacheBefore = JSON.parse(apiCacheTextBefore || "{}");
+  const loop = new import_mcpBundle.Loop({
+    api: context.agentParams.api,
+    apiEndpoint: context.agentParams.apiEndpoint,
+    apiKey: context.agentParams.apiKey,
+    apiTimeout: context.agentParams.apiTimeout ?? 0,
+    model: context.agentParams.model,
+    maxTokens: params.maxTokens ?? context.maxTokensRemaining(),
+    maxToolCalls: params.maxActions ?? context.agentParams.maxActions ?? 10,
+    maxToolCallRetries: params.maxActionRetries ?? context.agentParams.maxActionRetries ?? 3,
+    summarize: true,
+    debug: import_utilsBundle.debug,
+    callTool,
+    tools,
+    secrets,
+    cache: apiCacheBefore,
+    ...context.events
+  });
+  const task = [];
+  if (context.agentParams.systemPrompt) {
+    task.push("### System");
+    task.push(context.agentParams.systemPrompt);
+    task.push("");
+  }
+  task.push("### Task");
+  task.push(userTask);
+  if (context.history().length) {
+    task.push("### Context history");
+    task.push(context.history().map((h) => `- ${h.type}: ${h.description}`).join("\n"));
+    task.push("");
+  }
+  task.push("### Page snapshot");
+  task.push(snapshot);
+  task.push("");
+  const { error, usage } = await loop.run(task.join("\n"), { signal: progress.signal });
+  context.consumeTokens(usage.input + usage.output);
+  if (context.agentParams.apiCacheFile) {
+    const apiCacheAfter = { ...apiCacheBefore, ...loop.cache() };
+    const sortedCache = Object.fromEntries(Object.entries(apiCacheAfter).sort(([a], [b]) => a.localeCompare(b)));
+    const apiCacheTextAfter = JSON.stringify(sortedCache, void 0, 2);
+    if (apiCacheTextAfter !== apiCacheTextBefore) {
+      await import_fs.default.promises.mkdir(import_path.default.dirname(context.agentParams.apiCacheFile), { recursive: true });
+      await import_fs.default.promises.writeFile(context.agentParams.apiCacheFile, apiCacheTextAfter);
+    }
+  }
+  if (refusedToPerformReason())
+    throw new Error(`Agent refused to perform action: ${refusedToPerformReason()}`);
+  if (error)
+    throw new Error(`Agentic loop failed: ${error}`);
+  return { result: reportedResult ? reportedResult() : void 0 };
+}
+async function cachedPerform(progress, context, cacheKey) {
+  if (!context.agentParams?.cacheFile)
+    return;
+  const cache = await cachedActions(context.agentParams?.cacheFile);
+  const entry = cache.actions[cacheKey];
+  if (!entry)
+    return;
+  for (const action of entry.actions)
+    await (0, import_actionRunner.runAction)(progress, "run", context.page, action, context.agentParams.secrets ?? []);
+  return entry.actions;
+}
+async function updateCache(context, cacheKey) {
+  const cacheFile = context.agentParams?.cacheFile;
+  const cacheOutFile = context.agentParams?.cacheOutFile;
+  const cacheFileKey = cacheFile ?? cacheOutFile;
+  const cache = cacheFileKey ? await cachedActions(cacheFileKey) : { actions: {}, newActions: {} };
+  const newEntry = { actions: context.actions() };
+  cache.actions[cacheKey] = newEntry;
+  cache.newActions[cacheKey] = newEntry;
+  if (cacheOutFile) {
+    const entries = Object.entries(cache.newActions);
+    entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));
+    await import_fs.default.promises.writeFile(cacheOutFile, JSON.stringify(Object.fromEntries(entries), void 0, 2));
+  } else if (cacheFile) {
+    const entries = Object.entries(cache.actions);
+    entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));
+    await import_fs.default.promises.writeFile(cacheFile, JSON.stringify(Object.fromEntries(entries), void 0, 2));
+  }
+}
+const allCaches = /* @__PURE__ */ new Map();
+async function cachedActions(cacheFile) {
+  let cache = allCaches.get(cacheFile);
+  if (!cache) {
+    const content = await import_fs.default.promises.readFile(cacheFile, "utf-8").catch(() => "");
+    let json;
+    try {
+      json = JSON.parse(content.trim() || "{}");
+    } catch (error) {
+      throw new Error(`Failed to parse cache file ${cacheFile}:
+${error.message}`);
+    }
+    const parsed = actions.cachedActionsSchema.safeParse(json);
+    if (parsed.error)
+      throw new Error(`Failed to parse cache file ${cacheFile}:
+${import_mcpBundle.z.prettifyError(parsed.error)}`);
+    cache = { actions: parsed.data, newActions: {} };
+    allCaches.set(cacheFile, cache);
+  }
+  return cache;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  pageAgentExpect,
+  pageAgentExtract,
+  pageAgentPerform
+});

package/lib/server/agent/performTools.js

@@ -0,0 +1,262 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var performTools_exports = {};
+__export(performTools_exports, {
+  default: () => performTools_default
+});
+module.exports = __toCommonJS(performTools_exports);
+var import_mcpBundle = require("../../mcpBundle");
+var import_tool = require("./tool");
+const navigateSchema = import_mcpBundle.z.object({
+  url: import_mcpBundle.z.string().describe("URL to navigate to")
+});
+const navigate = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_navigate",
+    title: "Navigate to URL",
+    description: "Navigate to a URL",
+    inputSchema: navigateSchema
+  },
+  handle: async (progress, context, params) => {
+    return await context.runActionNoWait(progress, {
+      method: "navigate",
+      url: params.url
+    });
+  }
+});
+const snapshot = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_snapshot",
+    title: "Page snapshot",
+    description: "Capture accessibility snapshot of the current page, this is better than screenshot",
+    inputSchema: import_mcpBundle.z.object({})
+  },
+  handle: async (progress, context, params) => {
+    return await context.snapshotResult(progress);
+  }
+});
+const elementSchema = import_mcpBundle.z.object({
+  element: import_mcpBundle.z.string().describe("Human-readable element description used to obtain permission to interact with the element"),
+  ref: import_mcpBundle.z.string().describe("Exact target element reference from the page snapshot")
+});
+const clickSchema = elementSchema.extend({
+  doubleClick: import_mcpBundle.z.boolean().optional().describe("Whether to perform a double click instead of a single click"),
+  button: import_mcpBundle.z.enum(["left", "right", "middle"]).optional().describe("Button to click, defaults to left"),
+  modifiers: import_mcpBundle.z.array(import_mcpBundle.z.enum(["Alt", "Control", "ControlOrMeta", "Meta", "Shift"])).optional().describe("Modifier keys to press")
+});
+const click = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_click",
+    title: "Click",
+    description: "Perform click on a web page",
+    inputSchema: clickSchema
+  },
+  handle: async (progress, context, params) => {
+    const [selector] = await context.refSelectors(progress, [params]);
+    return await context.runActionAndWait(progress, {
+      method: "click",
+      selector,
+      button: params.button,
+      modifiers: params.modifiers,
+      clickCount: params.doubleClick ? 2 : void 0
+    });
+  }
+});
+const drag = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_drag",
+    title: "Drag mouse",
+    description: "Perform drag and drop between two elements",
+    inputSchema: import_mcpBundle.z.object({
+      startElement: import_mcpBundle.z.string().describe("Human-readable source element description used to obtain the permission to interact with the element"),
+      startRef: import_mcpBundle.z.string().describe("Exact source element reference from the page snapshot"),
+      endElement: import_mcpBundle.z.string().describe("Human-readable target element description used to obtain the permission to interact with the element"),
+      endRef: import_mcpBundle.z.string().describe("Exact target element reference from the page snapshot")
+    })
+  },
+  handle: async (progress, context, params) => {
+    const [sourceSelector, targetSelector] = await context.refSelectors(progress, [
+      { ref: params.startRef, element: params.startElement },
+      { ref: params.endRef, element: params.endElement }
+    ]);
+    return await context.runActionAndWait(progress, {
+      method: "drag",
+      sourceSelector,
+      targetSelector
+    });
+  }
+});
+const hoverSchema = elementSchema.extend({
+  modifiers: import_mcpBundle.z.array(import_mcpBundle.z.enum(["Alt", "Control", "ControlOrMeta", "Meta", "Shift"])).optional().describe("Modifier keys to press")
+});
+const hover = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_hover",
+    title: "Hover mouse",
+    description: "Hover over element on page",
+    inputSchema: hoverSchema
+  },
+  handle: async (progress, context, params) => {
+    const [selector] = await context.refSelectors(progress, [params]);
+    return await context.runActionAndWait(progress, {
+      method: "hover",
+      selector,
+      modifiers: params.modifiers
+    });
+  }
+});
+const selectOptionSchema = elementSchema.extend({
+  values: import_mcpBundle.z.array(import_mcpBundle.z.string()).describe("Array of values to select in the dropdown. This can be a single value or multiple values.")
+});
+const selectOption = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_select_option",
+    title: "Select option",
+    description: "Select an option in a dropdown",
+    inputSchema: selectOptionSchema
+  },
+  handle: async (progress, context, params) => {
+    const [selector] = await context.refSelectors(progress, [params]);
+    return await context.runActionAndWait(progress, {
+      method: "selectOption",
+      selector,
+      labels: params.values
+    });
+  }
+});
+const pressKey = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_press_key",
+    title: "Press a key",
+    description: "Press a key on the keyboard",
+    inputSchema: import_mcpBundle.z.object({
+      key: import_mcpBundle.z.string().describe("Name of the key to press or a character to generate, such as `ArrowLeft` or `a`"),
+      modifiers: import_mcpBundle.z.array(import_mcpBundle.z.enum(["Alt", "Control", "ControlOrMeta", "Meta", "Shift"])).optional().describe("Modifier keys to press")
+    })
+  },
+  handle: async (progress, context, params) => {
+    return await context.runActionAndWait(progress, {
+      method: "pressKey",
+      key: params.modifiers ? [...params.modifiers, params.key].join("+") : params.key
+    });
+  }
+});
+const typeSchema = elementSchema.extend({
+  text: import_mcpBundle.z.string().describe("Text to type into the element"),
+  submit: import_mcpBundle.z.boolean().optional().describe("Whether to submit entered text (press Enter after)"),
+  slowly: import_mcpBundle.z.boolean().optional().describe("Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.")
+});
+const type = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_type",
+    title: "Type text",
+    description: "Type text into editable element",
+    inputSchema: typeSchema
+  },
+  handle: async (progress, context, params) => {
+    const [selector] = await context.refSelectors(progress, [params]);
+    if (params.slowly) {
+      return await context.runActionAndWait(progress, {
+        method: "pressSequentially",
+        selector,
+        text: params.text,
+        submit: params.submit
+      });
+    } else {
+      return await context.runActionAndWait(progress, {
+        method: "fill",
+        selector,
+        text: params.text,
+        submit: params.submit
+      });
+    }
+  }
+});
+const fillForm = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_fill_form",
+    title: "Fill form",
+    description: "Fill multiple form fields. Always use this tool when you can fill more than one field at a time.",
+    inputSchema: import_mcpBundle.z.object({
+      fields: import_mcpBundle.z.array(import_mcpBundle.z.object({
+        name: import_mcpBundle.z.string().describe("Human-readable field name"),
+        type: import_mcpBundle.z.enum(["textbox", "checkbox", "radio", "combobox", "slider"]).describe("Type of the field"),
+        ref: import_mcpBundle.z.string().describe("Exact target field reference from the page snapshot"),
+        value: import_mcpBundle.z.string().describe("Value to fill in the field. If the field is a checkbox, the value should be `true` or `false`. If the field is a combobox, the value should be the text of the option.")
+      })).describe("Fields to fill in")
+    })
+  },
+  handle: async (progress, context, params) => {
+    const actions = [];
+    for (const field of params.fields) {
+      const [selector] = await context.refSelectors(progress, [{ ref: field.ref, element: field.name }]);
+      if (field.type === "textbox" || field.type === "slider") {
+        actions.push({
+          method: "fill",
+          selector,
+          text: field.value
+        });
+      } else if (field.type === "checkbox" || field.type === "radio") {
+        actions.push({
+          method: "setChecked",
+          selector,
+          checked: field.value === "true"
+        });
+      } else if (field.type === "combobox") {
+        actions.push({
+          method: "selectOption",
+          selector,
+          labels: [field.value]
+        });
+      }
+    }
+    return await context.runActionsAndWait(progress, actions);
+  }
+});
+const setCheckedSchema = elementSchema.extend({
+  checked: import_mcpBundle.z.boolean().describe("Whether to check the checkbox")
+});
+const setChecked = (0, import_tool.defineTool)({
+  schema: {
+    name: "browser_set_checked",
+    title: "Set checked",
+    description: "Set the checked state of a checkbox",
+    inputSchema: setCheckedSchema
+  },
+  handle: async (progress, context, params) => {
+    const [selector] = await context.refSelectors(progress, [params]);
+    return await context.runActionAndWait(progress, {
+      method: "setChecked",
+      selector,
+      checked: params.checked
+    });
+  }
+});
+var performTools_default = [
+  navigate,
+  snapshot,
+  click,
+  drag,
+  hover,
+  selectOption,
+  pressKey,
+  type,
+  fillForm,
+  setChecked
+];

package/lib/server/agent/tool.js

@@ -0,0 +1,109 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var tool_exports = {};
+__export(tool_exports, {
+  defineTool: () => defineTool,
+  toolsForLoop: () => toolsForLoop
+});
+module.exports = __toCommonJS(tool_exports);
+var import_mcpBundle = require("../../mcpBundle");
+var import_stringUtils = require("../../utils/isomorphic/stringUtils");
+function defineTool(tool) {
+  return tool;
+}
+function toolsForLoop(progress, context, toolDefinitions, options = {}) {
+  const tools = toolDefinitions.map((tool) => {
+    const result = {
+      name: tool.schema.name,
+      description: tool.schema.description,
+      inputSchema: import_mcpBundle.z.toJSONSchema(tool.schema.inputSchema)
+    };
+    return result;
+  });
+  if (options.resultSchema) {
+    tools.push({
+      name: "report_result",
+      description: "Report the result of the task.",
+      inputSchema: options.resultSchema
+    });
+  }
+  if (options.refuseToPerform === "allow") {
+    tools.push({
+      name: "refuse_to_perform",
+      description: "Refuse to perform action.",
+      inputSchema: {
+        type: "object",
+        properties: {
+          reason: {
+            type: "string",
+            description: `Call this when you believe that you can't perform the action because something is wrong with the page. The reason will be reported to the user.`
+          }
+        },
+        required: ["reason"]
+      }
+    });
+  }
+  let reportedResult;
+  let refusedToPerformReason;
+  const callTool = async (params) => {
+    if (params.name === "report_result") {
+      reportedResult = params.arguments;
+      return {
+        content: [{ type: "text", text: "Done" }],
+        isError: false
+      };
+    }
+    if (params.name === "refuse_to_perform") {
+      refusedToPerformReason = params.arguments.reason;
+      return {
+        content: [{ type: "text", text: "Done" }],
+        isError: false
+      };
+    }
+    const tool = toolDefinitions.find((t) => t.schema.name === params.name);
+    if (!tool) {
+      return {
+        content: [{
+          type: "text",
+          text: `Tool ${params.name} not found. Available tools: ${toolDefinitions.map((t) => t.schema.name)}`
+        }],
+        isError: true
+      };
+    }
+    try {
+      return await tool.handle(progress, context, params.arguments);
+    } catch (error) {
+      return {
+        content: [{ type: "text", text: (0, import_stringUtils.stripAnsiEscapes)(error.message) }],
+        isError: true
+      };
+    }
+  };
+  return {
+    tools,
+    callTool,
+    reportedResult: options.resultSchema ? () => reportedResult : void 0,
+    refusedToPerformReason: () => refusedToPerformReason
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  defineTool,
+  toolsForLoop
+});

package/lib/server/artifact.js

@@ -103,7 +103,7 @@ class Artifact extends import_instrumentation.SdkObject {
     if (!this._unaccessibleErrorMessage)
       await import_fs.default.promises.unlink(this._localPath).catch((e) => {
       });
-    await this.reportFinished(new import_errors.TargetClosedError());
+    await this.reportFinished(new import_errors.TargetClosedError(this.closeReason()));
   }
   async reportFinished(error) {
     if (this._finished)