wuying-agentbay-sdk 0.10.2 → 0.12.0

package/docs/examples/browser-use/browser/browser-viewport.ts

@@ -0,0 +1,129 @@
+/**
+ * Example demonstrating Browser Viewport configuration with AgentBay SDK.
+ *
+ * This example shows how to initialize the browser with custom viewport and user-agent settings:
+ * - Create AIBrowser session with custom viewport and user-agent.
+ * - Use playwright to connect to AIBrowser instance through CDP protocol
+ * - Verify custom viewport and screen size
+ * - Verify custom user agent
+ */
+
+// @ts-nocheck
+import { AgentBay, CreateSessionParams } from 'wuying-agentbay-sdk';
+import { BrowserOption, BrowserViewport } from 'wuying-agentbay-sdk';
+import { chromium } from 'playwright';
+
+interface ScreenInfo {
+    outerWidth: number;
+    outerHeight: number;
+    innerWidth: number;
+    innerHeight: number;
+    width: number;
+    height: number;
+    availWidth: number;
+    availHeight: number;
+    colorDepth: number;
+    pixelDepth: number;
+}
+
+interface WindowInfo {
+    screen: ScreenInfo | null;
+}
+
+interface UserAgentResponse {
+    'user-agent': string;
+}
+
+async function main(): Promise<void> {
+    // Get API key from environment variable
+    const apiKey = process.env.AGENTBAY_API_KEY;
+    if (!apiKey) {
+        console.log('Error: AGENTBAY_API_KEY environment variable not set');
+        return;
+    }
+
+    try {
+        // Initialize AgentBay client
+        console.log('Initializing AgentBay client...');
+        const agentBay = new AgentBay({ apiKey });
+
+        // Create a session
+        console.log('Creating a new session...');
+        const params: CreateSessionParams = {
+            imageId: 'browser_latest',
+        };
+        const sessionResult = await agentBay.create(params);
+
+        if (!sessionResult.success) {
+            console.log('Failed to create session');
+            return;
+        }
+
+        const session = sessionResult.session;
+        console.log(`Session created with ID: ${session.sessionId}`);
+
+        // Create browser option with viewport and user-agent
+        const browserOption: BrowserOption = {
+            userAgent: 'Mozilla/5.0 (Mocked Windows Desktop)',
+            viewport: { width: 1920, height: 1080 },
+            // screen: { width: 1920, height: 1080 },
+        };
+
+        const initialized = await session.browser.initializeAsync(browserOption);
+        if (initialized) {
+            const endpointUrl = await session.browser.getEndpointUrl();
+            console.log('endpoint_url =', endpointUrl);
+
+            const browser = await chromium.connectOverCDP(endpointUrl);
+            const context = browser.contexts()[0];
+            const page = await context.newPage();
+
+            try {
+                // Check custom viewport and screen size
+                console.log('\n--- Check window Properties ---');
+                const windowInfo = await page.evaluate((): WindowInfo => {
+                    const screenInfo = window.screen ? {
+                        outerWidth: window.outerWidth,
+                        outerHeight: window.outerHeight,
+                        innerWidth: window.innerWidth,
+                        innerHeight: window.innerHeight,
+                        width: window.screen.width,
+                        height: window.screen.height,
+                        availWidth: window.screen.availWidth,
+                        availHeight: window.screen.availHeight,
+                        colorDepth: window.screen.colorDepth,
+                        pixelDepth: window.screen.pixelDepth
+                    } : null;
+                    
+                    return {
+                        screen: screenInfo
+                    };
+                });
+                console.log('Screen Info:', windowInfo.screen);
+                
+                // Check custom user agent
+                console.log('\n--- Check User Agent ---');
+                await page.goto('https://httpbin.org/user-agent');
+
+                const response = await page.evaluate((): UserAgentResponse => {
+                    return JSON.parse(document.body.textContent || '{}');
+                });
+                const userAgent = response['user-agent'] || '';
+                console.log('User Agent:', userAgent);
+
+                await page.waitForTimeout(3000);
+            } finally {
+                await browser.close();
+            }
+        }
+
+        // Clean up session
+        await agentBay.delete(session);
+    } catch (error) {
+        console.error('Error:', error);
+    }
+}
+
+if (require.main === module) {
+    main().catch(console.error);
+}

package/docs/examples/browser-use/browser/call_for_user_jd.ts

@@ -0,0 +1,184 @@
+/**
+ * Example demonstrating wuying-call-for-user message handling with AgentBay SDK.
+ *
+ * This example shows how to handle the 'wuying-call-for-user' message that can be received
+ * during browser automation sessions. The wuying-call-for-user message is triggered when:
+ *
+ * 1. The browser encounters a situation that requires human intervention
+ * 2. Authentication challenges that cannot be automatically resolved
+ * 3. Complex verification processes that need user input
+ * 4. Security measures that require manual verification
+ *
+ * When you receive a 'wuying-call-for-user' message, the recommended handling flow is:
+ *
+ * 1. Parse the console message to identify the message type
+ * 2. When 'wuying-call-for-user' is detected, open the session resource URL in a browser
+ * 3. Allow the user to interact with the browser to complete the required action
+ * 4. Wait for the user to complete the interaction (typically 20-30 seconds)
+ * 5. Continue with the automation flow
+ *
+ * This example demonstrates:
+ * - Creating an AgentBay session with browser capabilities
+ * - Connecting to the browser via CDP protocol using Playwright
+ * - Setting up console message listeners to detect wuying-call-for-user messages
+ * - Opening the session resource URL for user interaction
+ * - Implementing a wait mechanism for user completion
+ * - Taking screenshots for debugging purposes
+ */
+
+import { AgentBay, CreateSessionParams } from 'wuying-agentbay-sdk';
+import { BrowserOption } from 'wuying-agentbay-sdk';
+import { chromium } from 'playwright';
+import { exec } from 'child_process';
+import * as os from 'os';
+
+async function main(): Promise<void> {
+    /**
+     * Main function demonstrating wuying-call-for-user message handling.
+     * This function sets up a browser session and navigates to JD.com to trigger
+     * scenarios that might require user intervention.
+     */
+    // Get API key from environment variable
+    const apiKey = process.env.AGENTBAY_API_KEY;
+    if (!apiKey) {
+        console.log('Error: AGENTBAY_API_KEY environment variable not set');
+        return;
+    }
+
+    try {
+        // Initialize AgentBay client
+        console.log('Initializing AgentBay client...');
+        const agentBay = new AgentBay({ apiKey });
+
+        // Create a session
+        console.log('Creating a new session...');
+        const params: CreateSessionParams = {
+            imageId: 'browser_latest', // Specify the image ID
+        };
+        const sessionResult = await agentBay.create(params);
+
+        if (!sessionResult.success) {
+            console.log('Failed to create session');
+            return;
+        }
+
+        const session = sessionResult.session;
+        console.log(`Session created with ID: ${session.sessionId}`);
+
+        const browserOption: BrowserOption = {};
+        const initialized = await session.browser.initializeAsync(browserOption);
+        if (!initialized) {
+            console.log('Failed to initialize browser');
+            return;
+        }
+
+        console.log('Browser initialized successfully');
+        const endpointUrl = await session.browser.getEndpointUrl();
+        console.log('endpoint_url =', endpointUrl);
+        
+        const result = await session.info();
+        const info = result.data;
+        console.log(`session resource url is ${info.resourceUrl}`);
+
+        // Connect to browser using Playwright
+        const browser = await chromium.connectOverCDP(endpointUrl);
+        const context = browser.contexts()[0]
+        const page = await context.newPage();
+        console.log('🌐 Navigating to jd site...');
+        const url = 'https://www.jd.com/';
+        await page.goto(url);
+
+        // Listen for console messages
+        const handleConsole = async (msg: any) => {
+            console.log(`🔍 Received console message: ${msg.text()}`);
+            
+            // Parse JSON message
+            let messageType: string;
+            try {
+                const messageData = JSON.parse(msg.text());
+                messageType = messageData.type || '';
+                console.log(`📋 Parsed message type: ${messageType}`);
+            } catch (error) {
+                // If not JSON, treat as plain text
+                messageType = msg.text();
+                console.log(`📋 Plain text message: ${messageType}`);
+            }
+
+            if (messageType === 'wuying-call-for-user') {
+                console.log('📞 Received wuying-call-for-user message');
+                console.log(`session resource url is ${info.resourceUrl}`);
+                // You can skip this message or use chrome to open url for user handle
+                // Following sample code shows how to use chrome open url
+                console.log('🌐 Opening browser with session resource URL...');
+                
+                // Open URL in default browser (cross-platform)
+                const platformName = os.platform();
+                
+                let command: string;
+                if (platformName === 'win32') {
+                    // Windows: use double quotes to handle special characters
+                    command = `start "" "${info.resourceUrl}"`;
+                } else if (platformName === 'darwin') {
+                    // macOS: use single quotes to handle special characters
+                    command = `open '${info.resourceUrl}'`;
+                } else {
+                    // Linux: use single quotes to handle special characters
+                    command = `xdg-open '${info.resourceUrl}'`;
+                }
+                
+                exec(command, (error) => {
+                    if (error) {
+                        console.log('Failed to open browser:', error);
+                    } else {
+                        console.log('Browser opened successfully');
+                    }
+                });
+                
+                // wait user to interact with the browser
+                console.log('⏳ Starting 20 second wait for user interaction...');
+                // Use setTimeout to block for 20 seconds for user interaction, also you can check user input
+                await new Promise<void>((resolve) => {
+                    setTimeout(() => {
+                        console.log('⏳ User interaction wait completed');
+                        resolve();
+                    }, 20000);
+                });
+            }
+        };
+
+        page.on('console', handleConsole);
+
+        await page.waitForTimeout(5000);
+        console.log('click login');
+        await page.click('.link-login');
+        await page.waitForTimeout(25000);
+        
+        console.log('Test completed');
+        
+        // Keep browser open for a while to observe results
+        await page.waitForTimeout(5000);
+
+        // Take screenshot and print base64, can be pasted directly into Chrome address bar
+        try {
+            const screenshotBytes = await page.screenshot({ fullPage: false });
+            const b64 = screenshotBytes.toString('base64');
+            console.log('page_screenshot_base64 = data:image/png;base64,', b64);
+        } catch (error) {
+            console.log('screenshot failed:', error);
+        }
+        
+        await browser.close();
+
+        // Clean up session
+        await agentBay.delete(session);
+        console.log('Session cleaned up successfully');
+
+    } catch (error) {
+        console.error('Error in main function:', error);
+    }
+}
+
+// Run the main function if this file is executed directly
+if (typeof require !== 'undefined' && require.main === module) {
+    main().catch(console.error);
+}

package/docs/examples/browser-use/browser/captcha_tongcheng.ts

@@ -0,0 +1,151 @@
+/**
+ * Example demonstrating AIBrowser capabilities with AgentBay SDK.
+ * This example shows how to use AIBrowser to solve captcha automatically, including:
+ * - Create AIBrowser session
+ * - Use playwright to connect to AIBrowser instance through CDP protocol
+ * - Set solve_captchas to be True and goto tongcheng website
+ * - We will encounter a captcha and we will solve it automatically.
+ */
+
+import { AgentBay, CreateSessionParams } from 'wuying-agentbay-sdk';
+import { BrowserOption } from 'wuying-agentbay-sdk';
+import { chromium } from 'playwright';
+
+async function main(): Promise<void> {
+    // Get API key from environment variable
+    const apiKey = process.env.AGENTBAY_API_KEY;
+    if (!apiKey) {
+        console.log('Error: AGENTBAY_API_KEY environment variable not set');
+        return;
+    }
+
+    try {
+        // Initialize AgentBay client
+        console.log('Initializing AgentBay client...');
+        const agentBay = new AgentBay({ apiKey });
+
+        // Create a session
+        console.log('Creating a new session...');
+        const params: CreateSessionParams = {
+            imageId: 'browser_latest', // Specify the image ID
+        };
+        const sessionResult = await agentBay.create(params);
+
+        if (!sessionResult.success) {
+            console.log('Failed to create session');
+            return;
+        }
+
+        const session = sessionResult.session;
+        console.log(`Session created with ID: ${session.sessionId}`);
+
+        const browserOption: BrowserOption = {
+            useStealth: true,
+            solveCaptchas: true,
+        };
+
+        const initialized = await session.browser.initializeAsync(browserOption);
+        if (!initialized) {
+            console.log('Failed to initialize browser');
+            return;
+        }
+
+        console.log('Browser initialized successfully');
+        const endpointUrl = await session.browser.getEndpointUrl();
+        console.log('endpoint_url =', endpointUrl);
+
+        // Connect to browser using Playwright
+        const browser = await chromium.connectOverCDP(endpointUrl);
+        const context = browser.contexts()[0]
+        const page = await context.newPage();
+        console.log('🌐 Navigating to tongcheng site...');
+        const url = 'https://passport.ly.com/Passport/GetPassword';
+        await page.goto(url, { waitUntil: 'domcontentloaded' });
+
+        // Use selector to locate input field
+        const inputElement = await page.waitForSelector('#name_in', { timeout: 10000 });
+        console.log('Found login name input field: #name_in');
+        
+        // Clear input field and enter phone number
+        const phoneNumber = '15011556760';
+        console.log(`Entering phone number: ${phoneNumber}`);
+        
+        await inputElement.click();
+        await inputElement.fill(''); // Clear input field
+        await inputElement.type(phoneNumber);
+        console.log('Waiting for captcha');
+
+        // Wait a moment to ensure input is complete
+        await page.waitForTimeout(1000);
+        
+        console.log('Clicking next step button...');
+        await page.click('#next_step1');
+
+        // Listen for captcha processing messages
+        let captchaSolvingStarted = false;
+        let captchaSolvingFinished = false;
+
+        // Listen for console messages
+        const handleConsole = (msg: any) => {
+            console.log(`🔍 Received console message: ${msg.text()}`);
+            if (msg.text() === 'wuying-captcha-solving-started') {
+                captchaSolvingStarted = true;
+                console.log('🎯 Setting captchaSolvingStarted = true');
+                page.evaluate('window.captchaSolvingStarted = true; window.captchaSolvingFinished = false;');
+            } else if (msg.text() === 'wuying-captcha-solving-finished') {
+                captchaSolvingFinished = true;
+                console.log('✅ Setting captchaSolvingFinished = true');
+                page.evaluate('window.captchaSolvingFinished = true;');
+            }
+        };
+
+        page.on('console', handleConsole);
+
+        // Wait 1 second first, then check if captcha processing has started
+        try {
+            await page.waitForTimeout(1000);
+            await page.waitForFunction('() => window.captchaSolvingStarted === true', { timeout: 1000 });
+            console.log('🎯 Detected captcha processing started, waiting for completion...');
+            
+            // If start is detected, wait for completion (max 30 seconds)
+            try {
+                await page.waitForFunction('() => window.captchaSolvingFinished === true', { timeout: 30000 });
+                console.log('✅ Captcha processing completed');
+            } catch (error) {
+                console.log('⚠️ Captcha processing timeout, may still be in progress');
+            }
+                
+        } catch (error) {
+            console.log('⏭️ No captcha processing detected, continuing execution');
+        }
+        
+        await page.waitForTimeout(2000);
+        await page.type('#step2_yzm', '1234');
+        console.log('Test completed');
+        
+        // Keep browser open for a while to observe results
+        await page.waitForTimeout(5000);
+
+        // Take screenshot and print base64, can be pasted directly into Chrome address bar
+        try {
+            const screenshotBytes = await page.screenshot({ fullPage: false });
+            const b64 = screenshotBytes.toString('base64');
+            console.log('page_screenshot_base64 = data:image/png;base64,', b64);
+        } catch (error) {
+            console.log('screenshot failed:', error);
+        }
+        
+        await browser.close();
+
+        // Clean up session
+        await agentBay.delete(session);
+        console.log('Session cleaned up successfully');
+
+    } catch (error) {
+        console.error('Error in main function:', error);
+    }
+}
+
+if (require.main === module) {
+    main().catch(console.error);
+}

package/docs/examples/browser-use/browser/run-2048.ts

@@ -0,0 +1,209 @@
+/**
+ * Example demonstrating AIBrowser capabilities with AgentBay SDK.
+ * This example shows how to use PageUseAgent to run 2048 game, including:
+ * - Create AIBrowser session
+ * - Use playwright to connect to AIBrowser instance through CDP protocol
+ * - Utilize PageUseAgent to run 2048 game
+ */
+
+import { AgentBay, CreateSessionParams } from '../../../../typescript/src/agent-bay';
+import { BrowserOption, ExtractOptions, ActOptions } from '../../../../typescript/src/browser';
+// import { chromium } from 'playwright';
+
+class GameState {
+  score?: number;
+  highestTile?: number;
+  grid: number[][] = [];
+}
+
+class MoveAnalysis {
+  move?: number;
+  confidence: number = 0;
+  reasoning: string = "";
+}
+
+function transposeGrid(grid: number[][]): number[][] {
+  if (!grid || grid.length === 0) {
+    return [];
+  }
+  return grid[0].map((_, colIndex) => grid.map(row => row[colIndex]));
+}
+
+function formatGridForLlmInstruction(gridData: number[][]): string {
+  const formattedRows: string[] = [];
+  for (let i = 0; i < gridData.length; i++) {
+    formattedRows.push(`row${i + 1}: [${gridData[i].join(', ')}]`);
+  }
+  return formattedRows.join('\n');
+}
+
+async function main() {
+  // Get API key from environment variable
+  const apiKey = process.env.AGENTBAY_API_KEY;
+  if (!apiKey) {
+    console.log("Error: AGENTBAY_API_KEY environment variable not set");
+    return;
+  }
+
+  // Initialize AgentBay client
+  console.log("Initializing AgentBay client...");
+  const agentBay = new AgentBay({ apiKey });
+
+  // Create a session
+  console.log("Creating a new session...");
+  const params: CreateSessionParams = {
+    imageId: "browser_latest",
+  };
+  const sessionResult = await agentBay.create(params);
+
+  if (sessionResult.success) {
+    const session = sessionResult.session;
+    console.log(`Session created with ID: ${session.sessionId}`);
+
+    if (await session.browser.initializeAsync({} as BrowserOption)) {
+      console.log("Browser initialized successfully");
+      const endpointUrl = await session.browser.getEndpointUrl();
+      console.log("endpoint_url =", endpointUrl);
+
+      // Note: Install playwright with: npm install playwright
+      const { chromium } = require('playwright');
+      const browser = await chromium.connectOverCDP(endpointUrl);
+      let page = null;
+
+      try {
+        const context = browser.contexts()[0]
+        page = await context.newPage();
+        console.log("🌐 Navigating to 2048...");
+        await page.goto("https://ovolve.github.io/2048-AI/", {
+          waitUntil: "domcontentloaded",
+          timeout: 180000
+        });
+        console.log("🌐 Navigated to 2048 done");
+        await page.waitForSelector(".grid-container", { timeout: 10000 });
+
+        let lastTransposedGrid: number[][] | null = null;
+        let transposedGridNotChangedTimes = 0;
+        let lastMoveHistory: number[] = [];
+
+        while (true) {
+          console.log("🔄 Game loop iteration...");
+          await new Promise(resolve => setTimeout(resolve, 300));
+
+          // Get current game state
+          console.log("📊 Extracting game state...");
+          const gameStateOptions: ExtractOptions<GameState> = {
+            instruction: `
+              Extract the current game state:
+              1. Score from the score counter
+              2. All tile values and their positions in the 4x4 grid must be extracted.
+                 Each tile's value and position can be obtained from the tile-position-x-y class, where x (1 to 4) is the column and y (1 to 4) is the row.
+                 For example, tile-position-4-1 means the tile is in column 4, row 1.
+                 The value of the tile is given by the number in the tile's class.
+                 For example, <div class='tile tile-2 tile-position-1-4 tile-new'>2</div> means a tile with value 2 at column 1, row 4;
+                 and <div class='tile tile-2 tile-position-4-1 tile-new'>2</div> means a tile with value 2 at column 4, row 1.
+                 Empty spaces should be represented as 0 in the grid.
+                 For instance, if the only tiles present are the two above, the grid should be:[[0, 0, 0, 2], [0, 0, 0, 0], [0, 0, 0, 0], [2, 0, 0, 0]]
+              3. Highest tile value present
+            `,
+            schema: GameState,
+            use_text_extract: false
+
+          };
+
+          const [success, gameStates] = await session.browser.agent.extract(gameStateOptions, page);
+          if (success && gameStates.length > 0) {
+            const gameState = gameStates[0];
+            const transposedGrid = transposeGrid(gameState.grid);
+            console.log(`transposed grid: ${JSON.stringify(transposedGrid)}`);
+            console.log(`gameState: ${JSON.stringify(gameState)}`);
+            const gridInstruction = formatGridForLlmInstruction(transposedGrid);
+
+            if (lastTransposedGrid !== null && JSON.stringify(transposedGrid) === JSON.stringify(lastTransposedGrid)) {
+              transposedGridNotChangedTimes += 1;
+            } else {
+              transposedGridNotChangedTimes = 0;
+              lastMoveHistory = [];
+            }
+            lastTransposedGrid = transposedGrid;
+
+            let instructionStr = `
+              Based on the current game state:
+              - Score: ${gameState.score}
+              - Highest tile: ${gameState.highestTile}
+              - Grid: This is a 4x4 matrix ordered by row (top to bottom) and column (left to right). The rows are stacked vertically, and tiles can move vertically between rows or horizontally between columns:${gridInstruction}
+              What is the best move (up/down/left/right)? Consider:
+              1. Keeping high value tiles in corners (bottom left, bottom right, top left, top right)
+              2. Maintaining a clear path to merge tiles
+              3. Avoiding moves that could block merges
+              4. Avoiding moves that merge no blocks as possible
+              5. Only adjacent tiles of the same value can merge
+              6. Making a move will move all tiles in that direction until they hit a tile of a different value or the edge of the board
+              7. Tiles cannot move past the edge of the board
+              8. Each move must move at least one tile
+            `;
+
+            if (transposedGridNotChangedTimes >= 1) {
+              instructionStr += `
+                9. Do not generate move value in ${JSON.stringify(lastMoveHistory)}
+                10. If last move value ${lastMoveHistory[lastMoveHistory.length - 1]} moves up or down, then generate move value with left or right direction, otherwise generate move value with up or down direction
+              `;
+            }
+
+            const nextMoveOptions: ExtractOptions<MoveAnalysis> = {
+              instruction: instructionStr,
+              schema: MoveAnalysis,
+              use_text_extract: false
+            };
+
+            const [moveSuccess, nextMove] = await session.browser.agent.extract(nextMoveOptions, page);
+            let selectedMove = 4; // Default to no move
+
+            if (moveSuccess && nextMove.length > 0) {
+              selectedMove = nextMove[0].move ?? 4;
+            } else {
+              console.log("❌ Failed to extract next move, retry observing");
+              continue;
+            }
+
+            lastMoveHistory.push(selectedMove);
+
+            const moveKeyMap: { [key: number]: string } = {
+              0: "ArrowUp",
+              1: "ArrowDown",
+              2: "ArrowLeft",
+              3: "ArrowRight",
+              4: "Escape",
+            };
+
+            const moveKey = moveKeyMap[selectedMove];
+            await page.keyboard.press(moveKey);
+          } else {
+            console.log("❌ Failed to extract game state, retry observing");
+          }
+        }
+      } catch (error) {
+        console.log(`❌ Error in game loop: ${error}`);
+        try {
+          if (page !== null) {
+            const isGameOver = await page.evaluate(
+              "() => document.querySelector('.game-over') !== null"
+            );
+            if (isGameOver) {
+              console.log("🏁 Game Over!");
+              return;
+            }
+          }
+        } catch (innerError) {
+          console.log(`Could not check game over status: ${innerError}`);
+        }
+        throw error;
+      }
+    } else {
+      console.log("Failed to initialize browser");
+    }
+  }
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}