wuying-agentbay-sdk 0.12.0 → 0.13.0

package/docs/api/mobile-use/mobile-simulate.md

@@ -0,0 +1,135 @@
+# Class: MobileSimulateService
+
+Provides methods to manage persistent mobile dev info and sync to the mobile device.
+
+## Table of contents
+
+
+### Methods
+
+- [hasMobileInfo](#hasmobileinfo)
+- [setSimulateContextId](#setsimulatecontextid)
+- [setSimulateEnable](#setsimulateenable)
+- [setSimulateMode](#setsimulatemode)
+- [uploadMobileInfo](#uploadmobileinfo)
+
+## Methods
+
+### hasMobileInfo
+
+▸ **hasMobileInfo**(`contextSync`): `Promise`\<`boolean`\>
+
+Check if the mobile dev info file exists in one context sync. (Only for user provided context sync)
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `contextSync` | [`ContextSync`](../common-features/basics/context-sync.md) | The context sync to check. |
+
+#### Returns
+
+`Promise`\<`boolean`\>
+
+True if the mobile dev info file exists, False otherwise.
+
+**`Throws`**
+
+Error if context_sync is not provided or context_sync.context_id or context_sync.path is not provided.
+
+**`Remarks`**
+
+This method can only be used when mobile simulate context sync is managed by user side. For internal mobile simulate
+context sync, this method will not work.
+
+___
+
+### setSimulateContextId
+
+▸ **setSimulateContextId**(`contextId`): `void`
+
+Set a previously saved simulate context id. Please make sure the context id is provided by MobileSimulateService
+but not user side created context.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `contextId` | `string` | The context ID of the previously saved mobile simulate context. |
+
+#### Returns
+
+`void`
+
+___
+
+### setSimulateEnable
+
+▸ **setSimulateEnable**(`enable`): `void`
+
+Set the simulate enable flag.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `enable` | `boolean` | The simulate feature enable flag. |
+
+#### Returns
+
+`void`
+
+___
+
+### setSimulateMode
+
+▸ **setSimulateMode**(`mode`): `void`
+
+Set the simulate mode.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `mode` | ``MobileSimulateMode`` | The simulate mode. - PropertiesOnly: Simulate only device properties. - SensorsOnly: Simulate only device sensors. - PackagesOnly: Simulate only installed packages. - ServicesOnly: Simulate only system services. - All: Simulate all aspects of the device. |
+
+#### Returns
+
+`void`
+
+___
+
+### uploadMobileInfo
+
+▸ **uploadMobileInfo**(`mobileDevInfoContent`, `contextSync?`): `Promise`\<``MobileSimulateUploadResult``\>
+
+Upload the mobile simulate dev info.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `mobileDevInfoContent` | `string` | The mobile simulate dev info content to upload. |
+| `contextSync?` | [`ContextSync`](../common-features/basics/context-sync.md) | Optional - If not provided, a new context sync will be created for the mobile simulate service and this context id will be returned by the MobileSimulateUploadResult. User can use this context id to do persistent mobile simulate across sessions. - If provided, the mobile simulate dev info will be uploaded to the context sync in a specific path. |
+
+#### Returns
+
+`Promise`\<``MobileSimulateUploadResult``\>
+
+The result of the upload operation.
+
+**`Throws`**
+
+Error if mobile_dev_info_content is not provided or not a valid JSON string.
+
+**`Throws`**
+
+Error if context_sync is provided but context_sync.context_id is not provided.
+
+**`Remarks`**
+
+If context_sync is not provided, a new context sync will be created for the mobile simulate.
+If context_sync is provided, the mobile simulate dev info will be uploaded to the context sync.
+If the mobile simulate dev info already exists in the context sync, the context sync will be updated.
+If the mobile simulate dev info does not exist in the context sync, the context sync will be created.
+If the upload operation fails, the error message will be returned.

package/docs/examples/browser-use/browser/basic-usage.ts

@@ -3,14 +3,19 @@
  * This example demonstrates the core browser operations without external dependencies.
  */
 
-import { AgentBay, CreateSessionParams, ActOptions, ObserveOptions, ExtractOptions } from 'wuying-agentbay-sdk';
-
-// Simple schema for demonstration
-class PageInfo {
-  title: string = "";
-  url: string = "";
-}
-
+import {
+  AgentBay,
+  CreateSessionParams,
+  ActOptions,
+  ObserveOptions,
+  ExtractOptions,
+} from "wuying-agentbay-sdk";
+import { z } from "zod";
+
+const PageInfoSchema = z.object({
+  title: z.string().describe("page title"),
+  url: z.string().describe("page url"),
+});
 async function main() {
   // Get API key from environment variable
   const apiKey = process.env.AGENTBAY_API_KEY;
@@ -57,7 +62,7 @@ async function main() {
     // Example: Mock page object (in real usage, you'd get this from Playwright)
     const mockPage = {
       url: () => "https://example.com",
-      title: () => "Example Domain"
+      title: () => "Example Domain",
     };
 
     // Example 1: Perform an action
@@ -65,7 +70,7 @@ async function main() {
     try {
       const actOptions: ActOptions = {
         action: "Click the 'More information...' link",
-        timeoutMS: 5000
+        timeout: 5,
       };
 
       const actResult = await session.browser.agent.act(actOptions, mockPage);
@@ -81,10 +86,11 @@ async function main() {
     console.log("\n--- Example 2: Observing page elements ---");
     try {
       const observeOptions: ObserveOptions = {
-        instruction: "Find all links and buttons on the page"
+        instruction: "Find all links and buttons on the page",
       };
 
-      const [observeSuccess, observations] = await session.browser.agent.observe(observeOptions, mockPage);
+      const [observeSuccess, observations] =
+        await session.browser.agent.observe(observeOptions, mockPage);
       console.log("Observe success:", observeSuccess);
       console.log("Number of observations:", observations.length);
 
@@ -92,7 +98,7 @@ async function main() {
         console.log(`Observation ${index + 1}:`, {
           selector: obs.selector,
           description: obs.description,
-          method: obs.method
+          method: obs.method,
         });
       });
     } catch (error) {
@@ -102,22 +108,24 @@ async function main() {
     // Example 3: Extract structured data from the page
     console.log("\n--- Example 3: Extracting structured data ---");
     try {
-      const extractOptions: ExtractOptions<PageInfo> = {
+      const extractOptions: ExtractOptions<typeof PageInfoSchema> = {
         instruction: "Extract the page title and URL",
-        schema: PageInfo,
-        use_text_extract: false
+        schema: PageInfoSchema,
+        use_text_extract: false,
       };
 
-      const [extractSuccess, extractedData] = await session.browser.agent.extract(extractOptions, mockPage);
+      const [extractSuccess, extracted] = await session.browser.agent.extract(
+        extractOptions,
+        mockPage
+      );
       console.log("Extract success:", extractSuccess);
-      console.log("Extracted data count:", extractedData.length);
 
-      extractedData.forEach((data: PageInfo, index: number) => {
-        console.log(`Extracted item ${index + 1}:`, {
-          title: data.title,
-          url: data.url
+      if (extractSuccess && extracted) {
+        console.log("Extracted data:", {
+          title: extracted.title,
+          url: extracted.url,
         });
-      });
+      }
     } catch (error) {
       console.log("Extraction failed:", error);
     }
@@ -125,7 +133,6 @@ async function main() {
     // Clean up
     console.log("\n--- Cleanup ---");
     console.log("Browser session completed successfully");
-
   } catch (error) {
     console.error("Error in main function:", error);
   }

package/docs/examples/browser-use/browser/browser-type-example.ts

@@ -66,7 +66,7 @@ async function testBrowserType(
     console.log('   ✓ Browser initialized successfully');
 
     // Get endpoint URL
-    const endpointUrl = session.browser.getEndpointUrl();
+    const endpointUrl = await session.browser.getEndpointUrl();
     console.log(`\n3. CDP endpoint: ${String(endpointUrl).substring(0, 50)}...`);
 
     // Connect Playwright and verify browser
@@ -179,7 +179,7 @@ async function quickExample(): Promise<void> {
       console.log('✓ Chrome browser initialized successfully');
 
       // Get endpoint and use with Playwright
-      const endpointUrl = session.browser.getEndpointUrl();
+      const endpointUrl = await session.browser.getEndpointUrl();
       const browser = await chromium.connectOverCDP(endpointUrl);
       const page = await browser.contexts()[0].newPage();
 
@@ -262,5 +262,4 @@ main()
   .catch(error => {
     console.error('Error:', error.message);
     process.exit(1);
-  });
-
+  });

package/docs/examples/browser-use/browser/captcha_tongcheng.ts

@@ -8,8 +8,30 @@
  */
 
 import { AgentBay, CreateSessionParams } from 'wuying-agentbay-sdk';
-import { BrowserOption } from 'wuying-agentbay-sdk';
-import { chromium } from 'playwright';
+import { BrowserOption } from 'wuying-agentbay-sdk/dist/browser';
+import { chromium, Page, ConsoleMessage } from 'playwright';
+
+// Polling detection function, continuously checks until condition is met or timeout
+async function waitForCondition(
+    page: Page,
+    conditionCode: string | (() => any),
+    timeout: number = 30000,
+    interval: number = 200
+): Promise<boolean> {
+    const startTime = Date.now();
+    while (Date.now() - startTime < timeout) {
+        try {
+            const result = await page.evaluate(conditionCode);
+            if (result) {
+                return true;
+            }
+        } catch (error) {
+            // Ignore errors, continue polling
+        }
+        await new Promise(resolve => setTimeout(resolve, interval));
+    }
+    return false;
+}
 
 async function main(): Promise<void> {
     // Get API key from environment variable
@@ -81,42 +103,52 @@ async function main(): Promise<void> {
         console.log('Clicking next step button...');
         await page.click('#next_step1');
 
-        // Listen for captcha processing messages
-        let captchaSolvingStarted = false;
-        let captchaSolvingFinished = false;
-
         // Listen for console messages
-        const handleConsole = (msg: any) => {
+        page.on('console', async (msg: ConsoleMessage) => {
             console.log(`🔍 Received console message: ${msg.text()}`);
+            
             if (msg.text() === 'wuying-captcha-solving-started') {
-                captchaSolvingStarted = true;
-                console.log('🎯 Setting captchaSolvingStarted = true');
-                page.evaluate('window.captchaSolvingStarted = true; window.captchaSolvingFinished = false;');
+                console.log('🎯 Captcha processing started');
+                await page.evaluate('window.captchaSolvingStarted = true; window.captchaSolvingFinished = false;');
             } else if (msg.text() === 'wuying-captcha-solving-finished') {
-                captchaSolvingFinished = true;
-                console.log('✅ Setting captchaSolvingFinished = true');
-                page.evaluate('window.captchaSolvingFinished = true;');
+                console.log('✅ Captcha processing finished');
+                await page.evaluate('window.captchaSolvingFinished = true;');
             }
-        };
-
-        page.on('console', handleConsole);
+        });
 
         // Wait 1 second first, then check if captcha processing has started
-        try {
-            await page.waitForTimeout(1000);
-            await page.waitForFunction('() => window.captchaSolvingStarted === true', { timeout: 1000 });
+        await page.waitForTimeout(1000);
+        
+        // Use function form like JavaScript version
+        // The function will be serialized and executed in browser context where window exists
+        const started = await waitForCondition(
+            page,
+            () => {
+                // @ts-expect-error - window exists in browser context where this function will be executed
+                return window.captchaSolvingStarted === true;
+            },
+            3000,
+            200
+        );
+        
+        if (started) {
             console.log('🎯 Detected captcha processing started, waiting for completion...');
-            
-            // If start is detected, wait for completion (max 30 seconds)
-            try {
-                await page.waitForFunction('() => window.captchaSolvingFinished === true', { timeout: 30000 });
+            const finished = await waitForCondition(
+                page,
+                () => {
+                    // @ts-expect-error - window exists in browser context where this function will be executed
+                    return window.captchaSolvingFinished === true;
+                },
+                30000,
+                200
+            );
+            if (finished) {
                 console.log('✅ Captcha processing completed');
-            } catch (error) {
-                console.log('⚠️ Captcha processing timeout, may still be in progress');
+            } else {
+                console.log('⚠️ Captcha processing timeout, may still be in progress, continuing execution');
             }
-                
-        } catch (error) {
-            console.log('⏭️ No captcha processing detected, continuing execution');
+        } else {
+            console.log('⏭️ No captcha processing detected, may not need to handle captcha');
         }
         
         await page.waitForTimeout(2000);

package/docs/examples/browser-use/browser/run-2048.ts

@@ -6,35 +6,29 @@
  * - Utilize PageUseAgent to run 2048 game
  */
 
-import { AgentBay, CreateSessionParams } from '../../../../typescript/src/agent-bay';
-import { BrowserOption, ExtractOptions, ActOptions } from '../../../../typescript/src/browser';
-// import { chromium } from 'playwright';
-
-class GameState {
-  score?: number;
-  highestTile?: number;
-  grid: number[][] = [];
-}
+import { AgentBay, CreateSessionParams } from "wuying-agentbay-sdk";
+import { BrowserOption, ExtractOptions } from "wuying-agentbay-sdk";
+import { z } from "zod";
 
-class MoveAnalysis {
-  move?: number;
-  confidence: number = 0;
-  reasoning: string = "";
-}
+const GameStateSchema = z.object({
+  score: z.number().optional(),
+  highestTile: z.number().optional(),
+  grid: z.array(z.array(z.number())),
+});
 
 function transposeGrid(grid: number[][]): number[][] {
   if (!grid || grid.length === 0) {
     return [];
   }
-  return grid[0].map((_, colIndex) => grid.map(row => row[colIndex]));
+  return grid[0].map((_, colIndex) => grid.map((row) => row[colIndex]));
 }
 
 function formatGridForLlmInstruction(gridData: number[][]): string {
   const formattedRows: string[] = [];
   for (let i = 0; i < gridData.length; i++) {
-    formattedRows.push(`row${i + 1}: [${gridData[i].join(', ')}]`);
+    formattedRows.push(`row${i + 1}: [${gridData[i].join(", ")}]`);
   }
-  return formattedRows.join('\n');
+  return formattedRows.join("\n");
 }
 
 async function main() {
@@ -66,17 +60,17 @@ async function main() {
       console.log("endpoint_url =", endpointUrl);
 
       // Note: Install playwright with: npm install playwright
-      const { chromium } = require('playwright');
+      const { chromium } = require("playwright");
       const browser = await chromium.connectOverCDP(endpointUrl);
       let page = null;
 
       try {
-        const context = browser.contexts()[0]
+        const context = browser.contexts()[0];
         page = await context.newPage();
         console.log("🌐 Navigating to 2048...");
         await page.goto("https://ovolve.github.io/2048-AI/", {
           waitUntil: "domcontentloaded",
-          timeout: 180000
+          timeout: 180000,
         });
         console.log("🌐 Navigated to 2048 done");
         await page.waitForSelector(".grid-container", { timeout: 10000 });
@@ -87,11 +81,11 @@ async function main() {
 
         while (true) {
           console.log("🔄 Game loop iteration...");
-          await new Promise(resolve => setTimeout(resolve, 300));
+          await new Promise((resolve) => setTimeout(resolve, 300));
 
           // Get current game state
           console.log("📊 Extracting game state...");
-          const gameStateOptions: ExtractOptions<GameState> = {
+          const gameStateOptions: ExtractOptions<typeof GameStateSchema> = {
             instruction: `
               Extract the current game state:
               1. Score from the score counter
@@ -105,20 +99,25 @@ async function main() {
                  For instance, if the only tiles present are the two above, the grid should be:[[0, 0, 0, 2], [0, 0, 0, 0], [0, 0, 0, 0], [2, 0, 0, 0]]
               3. Highest tile value present
             `,
-            schema: GameState,
-            use_text_extract: false
-
+            schema: GameStateSchema,
+            use_text_extract: false,
           };
 
-          const [success, gameStates] = await session.browser.agent.extract(gameStateOptions, page);
-          if (success && gameStates.length > 0) {
-            const gameState = gameStates[0];
+          const [success, gameState] = await session.browser.agent.extract(
+            gameStateOptions,
+            page
+          );
+          if (success && gameState) {
             const transposedGrid = transposeGrid(gameState.grid);
             console.log(`transposed grid: ${JSON.stringify(transposedGrid)}`);
             console.log(`gameState: ${JSON.stringify(gameState)}`);
             const gridInstruction = formatGridForLlmInstruction(transposedGrid);
 
-            if (lastTransposedGrid !== null && JSON.stringify(transposedGrid) === JSON.stringify(lastTransposedGrid)) {
+            if (
+              lastTransposedGrid !== null &&
+              JSON.stringify(transposedGrid) ===
+                JSON.stringify(lastTransposedGrid)
+            ) {
               transposedGridNotChangedTimes += 1;
             } else {
               transposedGridNotChangedTimes = 0;
@@ -144,22 +143,33 @@ async function main() {
 
             if (transposedGridNotChangedTimes >= 1) {
               instructionStr += `
-                9. Do not generate move value in ${JSON.stringify(lastMoveHistory)}
-                10. If last move value ${lastMoveHistory[lastMoveHistory.length - 1]} moves up or down, then generate move value with left or right direction, otherwise generate move value with up or down direction
+                9. Do not generate move value in ${JSON.stringify(
+                  lastMoveHistory
+                )}
+                10. If last move value ${
+                  lastMoveHistory[lastMoveHistory.length - 1]
+                } moves up or down, then generate move value with left or right direction, otherwise generate move value with up or down direction
               `;
             }
 
-            const nextMoveOptions: ExtractOptions<MoveAnalysis> = {
+            const nextMoveOptions: ExtractOptions<any> = {
               instruction: instructionStr,
-              schema: MoveAnalysis,
-              use_text_extract: false
+              schema: z.object({
+                move: z.number().optional(),
+                confidence: z.number().optional(),
+                reasoning: z.string().optional(),
+              }),
+              use_text_extract: false,
             };
 
-            const [moveSuccess, nextMove] = await session.browser.agent.extract(nextMoveOptions, page);
+            const [moveSuccess, nextMove] = await session.browser.agent.extract(
+              nextMoveOptions,
+              page
+            );
             let selectedMove = 4; // Default to no move
 
-            if (moveSuccess && nextMove.length > 0) {
-              selectedMove = nextMove[0].move ?? 4;
+            if (moveSuccess && nextMove && typeof nextMove === "object") {
+              selectedMove = (nextMove as any).move ?? 4;
             } else {
               console.log("❌ Failed to extract next move, retry observing");
               continue;