webmarker-js 0.0.6 → 0.1.0

package/README.md

@@ -11,60 +11,146 @@ Mark web pages for use with vision-language models.
 
 ## Overview
 
-🚧 Under Construction
+**WebMarker** adds visual markings with labels to elements on a web page. This can be used for [Set-of-Mark (SoM)](https://github.com/microsoft/SoM) prompting, which improves visual grounding abilities of vision-language models such as GPT-4o, Claude 3.5, and Google Gemini 1.5.
 
-**WebMarker** adds visual markings with labels to elements on a web page. This can be used for [Set-of-Mark (SoM)](https://github.com/microsoft/SoM) prompting, which improves visual grounding abilities of vision-language models such as GPT-4V, Claude 3, and Google Gemini 1.5.
+![Screenshot of marked Google homepage](https://github.com/user-attachments/assets/722e1034-06d4-4ccd-a7d6-f03749435681)
 
-## Usage
 
-The `mark()` function will add markings for all interactive elements on a web page, and return an object containing the marked elements. The returned objects's keys are the mark labels, and the values are an object with the element (`element`), mark element (`markElement`), and mask element (`maskElement`).
+## How it works
+
+**1. Call the `mark()` function**
+
+This marks the interactive elements on the page, and returns an object containing the marked elements, where each key is a mark label string, and each value is an object with the following properties:
+
+- `element`: The interactive element that was marked.
+- `markElement`: The label element that was added to the page.
+- `maskElement`: The bounding box element that was added to the page.
+
+You can use this information to build your prompt for the vision-language model.
+
+**2. Send a screenshot of the marked page to a vision-language model, along with your prompt**
+
+Example prompt:
 
 ```javascript
-import { mark, unmark } from "webmarker-js";
+let markedElements = await mark();
+
+let prompt = `The following is a screenshot of a web page.
 
-// Mark interactive elements on the document
-let elements = await mark();
+Interactive elements have been marked with red bounding boxes and labels.
 
-// Reference an element by label
-console.log(elements["0"].element);
+When referring to elements, use the labels to identify them.
 
-// Remove markings
-unmark();
+Return an action and element to perform the action on.
+
+Available actions: click, hover
+
+Available elements:
+${Object.keys(markedElements)
+  .map((label) => `- ${label}`)
+  .join("\n")}
+
+Example response: click 0
+`;
+```
+
+**3. Programmatically interact with the marked elements.**
+
+In a web browser (i.e. via Playwright), interact with elements as needed.
+
+For prompting or agent ideas, see the [WebVoyager](https://arxiv.org/abs/2401.13919) paper.
+
+## Playwright example
+
+```javascript
+// Inject the WebMarker library into the page
+await page.addScriptTag({
+  url: "https://cdn.jsdelivr.net/npm/webmarker-js/dist/main.js",
+});
+
+// Mark the page and get the marked elements
+let markedElements = await page.evaluate(async () => await WebMarker.mark());
+
+// (Optional) Check if page is marked
+let isMarked = await page.evaluate(async () => await WebMarker.isMarked());
+
+// (Optional) Unmark the page
+await page.evaluate(async () => await WebMarker.unmark());
 ```
 
 ## Options
 
-- **selector**: A custom CSS selector to specify which elements to mark.
-  - Type: `string`
-  - Default: `"button, input, a, select, textarea"`
-- **markStyle**: A CSS style to apply to the label element. You can also specify a function that returns a CSS style object.
-  - Type: `Readonly<Partial<CSSStyleDeclaration>> or (element: Element) => Readonly<Partial<CSSStyleDeclaration>>`
-  - Default: `{backgroundColor: "red", color: "white", padding: "2px 4px", fontSize: "12px", fontWeight: "bold"}`
-- **markPlacement**: The placement of the mark relative to the element.
-  - Type: `'top' | 'top-start' | 'top-end' | 'right' | 'right-start' | 'right-end' | 'bottom' | 'bottom-start' | 'bottom-end' | 'left' | 'left-start' | 'left-end'`
-  - Default: `'top-start'`
-- **maskStyle**: A CSS style to apply to the bounding box element. You can also specify a function that returns a CSS style object. Bounding boxes are only shown if showMasks is true.
-  - Type: `Readonly<Partial<CSSStyleDeclaration>> or (element: Element) => Readonly<Partial<CSSStyleDeclaration>>`
-  - Default: `{outline: "2px dashed red", backgroundColor: "transparent"}`
-- **showMasks**: Whether or not to show bounding boxes around the elements.
-  - Type: `boolean`
-  - Default: `true`
-- **labelGenerator**: Provide a function for generating labels. By default, labels are generated as numbers starting from 0.
-  - Type: `(element: Element, index: number) => string`
-  - Default: `(_, index) => index.toString()`
-- **containerElement**: Provide a container element to query the elements to be marked. By default, the container element is document.body.
-  - Type: `Element`
-  - Default: `document.body`
-- **viewPortOnly**: Only mark elements that are visible in the current viewport.
-  - Type: `boolean`
-  - Default: `false`
+### selector
+
+A custom CSS selector to specify which elements to mark.
+
+- Type: `string`
+- Default: `"button, input, a, select, textarea"`
+
+### markAttribute
+
+A custom attribute to add to the marked elements. This attribute contains the label of the mark.
+
+- Type: `string`
+- Default: `"data-mark-id"`
+
+### markStyle
+
+A CSS style to apply to the label element. You can also specify a function that returns a CSS style object.
+
+- Type: `Readonly<Partial<CSSStyleDeclaration>> or (element: Element) => Readonly<Partial<CSSStyleDeclaration>>`
+- Default: `{backgroundColor: "red", color: "white", padding: "2px 4px", fontSize: "12px", fontWeight: "bold"}`
+
+### markPlacement
+
+The placement of the mark relative to the element.
+
+- Type: `'top' | 'top-start' | 'top-end' | 'right' | 'right-start' | 'right-end' | 'bottom' | 'bottom-start' | 'bottom-end' | 'left' | 'left-start' | 'left-end'`
+- Default: `'top-start'`
+
+### maskStyle
+
+A CSS style to apply to the bounding box element. You can also specify a function that returns a CSS style object. Bounding boxes are only shown if showMasks is true.
+
+- Type: `Readonly<Partial<CSSStyleDeclaration>> or (element: Element) => Readonly<Partial<CSSStyleDeclaration>>`
+- Default: `{outline: "2px dashed red", backgroundColor: "transparent"}`
+
+### showMasks
+
+Whether or not to show bounding boxes around the elements.
+
+- Type: `boolean`
+- Default: `true`
+
+### labelGenerator
+
+Provide a function for generating labels. By default, labels are generated as integers starting from 0.
+
+- Type: `(element: Element, index: number) => string`
+- Default: `(_, index) => index.toString()`
+
+### containerElement
+
+Provide a container element to query the elements to be marked. By default, the container element is document.body.
+
+- Type: `Element`
+- Default: `document.body`
+
+### viewPortOnly
+
+Only mark elements that are visible in the current viewport.
+
+- Type: `boolean`
+- Default: `false`
 
 ### Advanced example
 
 ```typescript
-let elements = mark({
+const markedElements = await mark({
   // Only mark buttons and inputs
   selector: "button, input",
+  // Use test id attribute for marker labels
+  markAttribute: "data-test-id",
   // Use a blue mark with white text
   markStyle: { color: "white", backgroundColor: "blue", padding: 5 },
   // Use a blue dashed outline mask with a transparent and slighly blue background
@@ -82,11 +168,4 @@ let elements = mark({
   // Only mark elements that are visible in the current viewport
   viewPortOnly: true,
 });
-
-// Cleanup
-unmark();
 ```
-
-## Use with Playwright
-
-Coming soon

package/dist/index.d.ts

@@ -4,6 +4,12 @@ interface MarkOptions {
      * A CSS selector to specify the elements to be marked.
      */
     selector?: string;
+    /**
+     * Name for the attribute added to the marked elements. This attribute is used to store the label.
+     *
+     * @default 'data-mark-id'
+     */
+    markAttribute?: string;
     /**
      * A CSS style to apply to the label element.
      * You can also specify a function that returns a CSS style object.

package/dist/main.js

@@ -932,6 +932,7 @@ var WebMarker = (() => {
     return __async(this, arguments, function* (options = {}) {
       const {
         selector = "button, input, a, select, textarea",
+        markAttribute = "data-mark-id",
         markStyle = {
           backgroundColor: "red",
           color: "white",
@@ -961,7 +962,7 @@ var WebMarker = (() => {
           const markElement = createMark(element, markStyle, label, markPlacement);
           const maskElement = showMasks ? createMask(element, maskStyle, label) : void 0;
           markedElements[label] = { element, markElement, maskElement };
-          element.setAttribute("data-webmarkeredby", `webmarker-${label}`);
+          element.setAttribute(markAttribute, label);
         }))
       );
       document.documentElement.dataset.webmarkered = "true";
@@ -988,8 +989,8 @@ var WebMarker = (() => {
   }
   function createMask(element, style, label) {
     const maskElement = document.createElement("div");
-    maskElement.className = "webmarkermask";
-    maskElement.id = `webmarkermask-${label}`;
+    maskElement.className = "webmarker-mask";
+    maskElement.id = `webmarker-mask-${label}`;
     document.body.appendChild(maskElement);
     positionMask(maskElement, element);
     applyStyle(
@@ -1040,7 +1041,7 @@ var WebMarker = (() => {
     Object.assign(element.style, defaultStyle, customStyle);
   }
   function unmark() {
-    document.querySelectorAll(".webmarker, .webmarkermask").forEach((el) => el.remove());
+    document.querySelectorAll(".webmarker, .webmarker-mask").forEach((el) => el.remove());
     document.documentElement.removeAttribute("data-webmarkered");
     cleanupFns.forEach((fn) => fn());
     cleanupFns = [];

package/dist/module.js

@@ -908,6 +908,7 @@ function mark() {
   return __async(this, arguments, function* (options = {}) {
     const {
       selector = "button, input, a, select, textarea",
+      markAttribute = "data-mark-id",
       markStyle = {
         backgroundColor: "red",
         color: "white",
@@ -937,7 +938,7 @@ function mark() {
         const markElement = createMark(element, markStyle, label, markPlacement);
         const maskElement = showMasks ? createMask(element, maskStyle, label) : void 0;
         markedElements[label] = { element, markElement, maskElement };
-        element.setAttribute("data-webmarkeredby", `webmarker-${label}`);
+        element.setAttribute(markAttribute, label);
       }))
     );
     document.documentElement.dataset.webmarkered = "true";
@@ -964,8 +965,8 @@ function createMark(element, style, label, markPlacement = "top-start") {
 }
 function createMask(element, style, label) {
   const maskElement = document.createElement("div");
-  maskElement.className = "webmarkermask";
-  maskElement.id = `webmarkermask-${label}`;
+  maskElement.className = "webmarker-mask";
+  maskElement.id = `webmarker-mask-${label}`;
   document.body.appendChild(maskElement);
   positionMask(maskElement, element);
   applyStyle(
@@ -1016,7 +1017,7 @@ function applyStyle(element, defaultStyle, customStyle) {
   Object.assign(element.style, defaultStyle, customStyle);
 }
 function unmark() {
-  document.querySelectorAll(".webmarker, .webmarkermask").forEach((el) => el.remove());
+  document.querySelectorAll(".webmarker, .webmarker-mask").forEach((el) => el.remove());
   document.documentElement.removeAttribute("data-webmarkered");
   cleanupFns.forEach((fn) => fn());
   cleanupFns = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webmarker-js",
-  "version": "0.0.6",
+  "version": "0.1.0",
   "description": "A library for marking web pages for Set-of-Mark (SoM) prompting with vision-language models.",
   "source": "src/index.ts",
   "main": "dist/main.js",

package/src/index.ts

@@ -19,6 +19,12 @@ interface MarkOptions {
    * A CSS selector to specify the elements to be marked.
    */
   selector?: string;
+  /**
+   * Name for the attribute added to the marked elements. This attribute is used to store the label.
+   *
+   * @default 'data-mark-id'
+   */
+  markAttribute?: string;
   /**
    * A CSS style to apply to the label element.
    * You can also specify a function that returns a CSS style object.
@@ -77,6 +83,7 @@ async function mark(
 ): Promise<Record<string, MarkedElement>> {
   const {
     selector = "button, input, a, select, textarea",
+    markAttribute = "data-mark-id",
     markStyle = {
       backgroundColor: "red",
       color: "white",
@@ -113,7 +120,7 @@ async function mark(
         : undefined;
 
       markedElements[label] = { element, markElement, maskElement };
-      element.setAttribute("data-webmarkeredby", `webmarker-${label}`);
+      element.setAttribute(markAttribute, label);
     })
   );
 
@@ -155,8 +162,8 @@ function createMask(
   label: string
 ): HTMLElement {
   const maskElement = document.createElement("div");
-  maskElement.className = "webmarkermask";
-  maskElement.id = `webmarkermask-${label}`;
+  maskElement.className = "webmarker-mask";
+  maskElement.id = `webmarker-mask-${label}`;
   document.body.appendChild(maskElement);
   positionMask(maskElement, element);
   applyStyle(
@@ -215,7 +222,7 @@ function applyStyle(
 
 function unmark(): void {
   document
-    .querySelectorAll(".webmarker, .webmarkermask")
+    .querySelectorAll(".webmarker, .webmarker-mask")
     .forEach((el) => el.remove());
   document.documentElement.removeAttribute("data-webmarkered");
   cleanupFns.forEach((fn) => fn());

package/test-results/.last-run.json

@@ -1,4 +1,6 @@
 {
-  "status": "passed",
-  "failedTests": []
+  "status": "failed",
+  "failedTests": [
+    "bcbd25456d68a5d49fac-61d7a7d69e576e88625b"
+  ]
 }