wuying-agentbay-sdk 0.10.2 → 0.12.0

package/docs/examples/browser-use/browser/run-sudoku.ts

@@ -0,0 +1,150 @@
+/**
+ * Example demonstrating AIBrowser capabilities with AgentBay SDK.
+ * This example shows how to use PageUseAgent to run sudoku game, including:
+ * - Create AIBrowser session
+ * - Use playwright to connect to AIBrowser instance through CDP protocol
+ * - Utilize PageUseAgent to run sudoku game
+ */
+
+import { AgentBay, CreateSessionParams,BrowserOption, ExtractOptions, ActOptions } from 'wuying-agentbay-sdk';
+
+import { chromium } from 'playwright';
+
+class SudokuBoard {
+  board: number[][] = [];
+}
+
+class SudokuSolution {
+  solution: number[][] = [];
+}
+
+function formatBoardForLlm(board: number[][]): string {
+  return board.map(row => `  [${row.join(', ')}]`).join('\n');
+}
+
+async function main() {
+  // Get API key from environment variable
+  const apiKey = process.env.AGENTBAY_API_KEY;
+  if (!apiKey) {
+    console.log("Error: AGENTBAY_API_KEY environment variable not set");
+    return;
+  }
+
+  // Initialize AgentBay client
+  console.log("Initializing AgentBay client...");
+  const agentBay = new AgentBay({ apiKey });
+
+  // Create a session
+  console.log("Creating a new session...");
+  const params: CreateSessionParams = {
+    imageId: "browser_latest",
+  };
+  const sessionResult = await agentBay.create(params);
+
+  if (sessionResult.success) {
+    const session = sessionResult.session;
+    console.log(`Session created with ID: ${session.sessionId}`);
+
+    if (await session.browser.initializeAsync({} as BrowserOption)) {
+      console.log("Browser initialized successfully");
+      const endpointUrl = await session.browser.getEndpointUrl();
+      console.log("endpoint_url =", endpointUrl);
+
+      const browser = await chromium.connectOverCDP(endpointUrl);
+      try {
+        const context = browser.contexts()[0]
+        const page = await context.newPage();
+        console.log("🌐 Navigating to Sudoku site...");
+        const url = "https://widget.websudoku.com/";
+        await page.goto(url);
+        await page.waitForSelector("#puzzle_grid", { timeout: 10000 });
+
+        // 1. Extract the board
+        let success = false;
+        let board: number[][] = [];
+
+        while (!success) {
+          console.log("📊 Extracting sudoku board...");
+          const options: ExtractOptions<SudokuBoard> = {
+            instruction: "Extract the current sudoku board as a 9x9 array. Each cell should be a number (1-9) if filled, or 0 if empty.",
+            schema: SudokuBoard,
+            use_text_extract: false
+          };
+
+          const [extractSuccess, boardObjs] = await session.browser.agent.extract(options, page);
+          if (extractSuccess && boardObjs.length > 0) {
+            success = true;
+            board = boardObjs[0].board;
+          } else {
+            console.log("❌ Failed to extract sudoku board, retry extracting");
+            await new Promise(resolve => setTimeout(resolve, 3000));
+          }
+        }
+
+        console.log("Current Board:\n" + board.map(row => row.join(' ')).join('\n'));
+        const originalBoard = board.map(row => [...row]);
+
+        // 2. Solve the sudoku
+        const solutionOptions: ExtractOptions<SudokuSolution> = {
+          instruction: `You are an expert sudoku solver. Given the following sudoku board as a 9x9 array (0 means empty), solve the sudoku and return the completed 9x9 array as 'solution'.
+
+Sudoku rules:
+- Each row must contain the digits 1-9 with no repeats.
+- Each column must contain the digits 1-9 with no repeats.
+- Each of the nine 3x3 subgrids must contain the digits 1-9 with no repeats.
+- Only fill in the cells that are 0.
+- The solution must be unique and valid.
+
+board = [
+${formatBoardForLlm(board)}
+]
+
+Return:
+{
+  solution: number[][] // 9x9, all filled, valid sudoku
+}`,
+          schema: SudokuSolution,
+          use_text_extract: false
+        };
+
+        const [solutionSuccess, solutionObjs] = await session.browser.agent.extract(solutionOptions, page);
+        if (!solutionSuccess || solutionObjs.length === 0) {
+          console.log("❌ Failed to solve sudoku");
+          return;
+        }
+
+        const solution = solutionObjs[0].solution;
+        console.log("Solved Board:\n" + solution.map((row:any) => row.join(' ')).join('\n'));
+
+        // 3. Fill the solution
+        for (let row = 0; row < 9; row++) {
+          for (let col = 0; col < 9; col++) {
+            if (originalBoard[row][col] === 0) {
+              const inputId = `f${col}${row}`;
+              console.log(`Type '${solution[row][col]}' into the cell with id '${inputId}'`);
+
+              // Use the act method for natural language action
+              const actOptions: ActOptions = {
+                action: `Enter '${solution[row][col]}' into the input element where the attribute id is exactly '${inputId}' (for example, if id='f53', you must match the full string 'f53', not just the number 53; do not split or extract numbers from the id)`
+              };
+
+              await session.browser.agent.act(actOptions, page);
+              await new Promise(resolve => setTimeout(resolve, 500));
+            }
+          }
+        }
+
+        console.log("✅ Finished! The board has been solved and filled in the browser.");
+
+      } catch (error) {
+        console.log(`❌ Error in game loop: ${error}`);
+      }
+    } else {
+      console.log("Failed to initialize browser");
+    }
+  }
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}

package/docs/examples/browser-use/browser/screenshot-example.ts

@@ -0,0 +1,132 @@
+/**
+ * Example demonstrating screenshot capabilities with AgentBay SDK (TypeScript).
+ * This example shows how to capture screenshots using direct Playwright integration.
+ *
+ * Features demonstrated:
+ * - Creating a browser session with AgentBay
+ * - Using Playwright to connect to the browser instance
+ * - Taking screenshots using direct Playwright integration (Uint8Array data)
+ * - Saving screenshots to local files
+ * - Customizing screenshot options (full page, image format, quality)
+ */
+
+import { AgentBay } from 'wuying-agentbay-sdk';
+import type { CreateSessionParams } from 'wuying-agentbay-sdk';
+import { chromium } from 'playwright';
+import { writeFile } from 'fs/promises';
+
+async function main() {
+  // Get API key from environment variable
+  const apiKey = process.env.AGENTBAY_API_KEY;
+  if (!apiKey) {
+    console.error("Error: AGENTBAY_API_KEY environment variable not set");
+    return;
+  }
+
+  // Initialize AgentBay client
+  console.log("Initializing AgentBay client...");
+  const agentBay = new AgentBay({ apiKey });
+
+  // Create a session
+  console.log("Creating a new session...");
+  const params: CreateSessionParams = {
+    imageId: "browser_latest",
+  };
+  const sessionResult = await agentBay.create(params);
+
+  if (!sessionResult.success) {
+    console.error(`Failed to create session: ${sessionResult.errorMessage}`);
+    return;
+  }
+
+  const session = sessionResult.session;
+  console.log(`Session created with ID: ${session.sessionId}`);
+
+  try {
+    // Initialize the browser
+    const success = await session.browser.initializeAsync({});
+    if (!success) {
+      console.error("Failed to initialize browser");
+      return;
+    }
+    
+    console.log("Browser initialized successfully");
+    
+    // Get the browser endpoint and connect with Playwright
+    const endpointUrl = await session.browser.getEndpointUrl();
+    console.log(`Endpoint URL: ${endpointUrl}`);
+    
+    // Connect with Playwright
+    const browser = await chromium.connectOverCDP(endpointUrl);
+    const context = browser.contexts()[0];
+    const page = await context.newPage();
+    
+    // Navigate to a website
+    await page.goto("https://www.aliyun.com");
+    console.log("✅ Navigated to website");
+    
+    // Take a simple screenshot (returns Uint8Array data)
+    // Note: In the current implementation, the screenshot method is a placeholder
+    // In a real implementation, this would capture the actual screenshot
+    console.log("📸 Taking screenshot...");
+    
+    // This is a placeholder - in a real implementation, this would return actual screenshot data
+    // const screenshotData = await session.browser.screenshot(page);
+    // console.log(`✅ Browser screenshot captured (${screenshotData.length} bytes)`);
+    
+    // Save the screenshot to a file
+    // await writeFile("browser_screenshot.png", Buffer.from(screenshotData));
+    // console.log("✅ Browser screenshot saved as browser_screenshot.png");
+    
+    // Take a full page screenshot with custom options
+    // const fullPageData = await session.browser.screenshot(
+    //   page,
+    //   true, // fullPage
+    //   {
+    //     type: "jpeg",
+    //     quality: 80
+    //   }
+    // );
+    // console.log(`✅ Browser full page screenshot captured (${fullPageData.length} bytes)`);
+    
+    // Save the full page screenshot
+    // await writeFile("browser_full_page_screenshot.jpg", Buffer.from(fullPageData));
+    // console.log("✅ Browser full page screenshot saved as browser_full_page_screenshot.jpg");
+    
+    // Take a screenshot with custom viewport settings
+    // const customScreenshot = await session.browser.screenshot(
+    //   page,
+    //   false, // fullPage
+    //   {
+    //     type: "png",
+    //     timeout: 30000
+    //   }
+    // );
+    // console.log(`✅ Browser custom screenshot captured (${customScreenshot.length} bytes)`);
+    
+    // Save the custom screenshot
+    // await writeFile("browser_custom_screenshot.png", Buffer.from(customScreenshot));
+    // console.log("✅ Browser custom screenshot saved as browser_custom_screenshot.png");
+    
+    await browser.close();
+    
+    console.log("ℹ️  Note: Screenshot functionality requires Playwright TypeScript integration");
+    console.log("✅ Browser screenshot demo completed");
+    
+  } catch (error) {
+    console.error(`Error during screenshot demo: ${error}`);
+  } finally {
+    // Clean up: delete the session
+    console.log(`\n🧹 Cleaning up session ${session.sessionId}...`);
+    try {
+      await agentBay.delete(session);
+      console.log("✅ Session deleted successfully");
+    } catch (error) {
+      console.log(`⚠️  Warning: Error during cleanup: ${error}`);
+    }
+  }
+}
+
+console.log("📸 AgentBay Browser Screenshot Demo (TypeScript)");
+console.log("=".repeat(50));
+main().catch(console.error);

package/docs/examples/browser-use/extension-example/README.md

@@ -0,0 +1,252 @@
+# Extension Management Example
+
+This example demonstrates comprehensive browser extension management functionality using the Wuying AgentBay SDK for TypeScript.
+
+## Overview
+
+The extension management system allows you to:
+- Upload browser extensions (.zip files) to cloud storage
+- List and manage existing extensions  
+- Create extension configurations for browser sessions
+- Integrate extensions with browser automation workflows
+- Handle multiple extensions in a single session
+- Implement complete development workflows
+
+## File Structure
+
+```
+extension-example/
+├── extension-example.ts    # Comprehensive extension management examples
+└── README.md             # This documentation file
+```
+
+## Key Components
+
+### 1. Extension Class
+Represents a browser extension with ID, name, and creation timestamp.
+
+```typescript
+const extension = new Extension('ext_123', 'my-extension.zip', '2023-01-01');
+```
+
+### 2. ExtensionOption Class
+Configuration for browser extension integration, encapsulating context ID and extension IDs.
+
+```typescript
+const option = new ExtensionOption('context_123', ['ext_1', 'ext_2']);
+```
+
+### 3. ExtensionsService Class
+Main service for managing extensions with automatic context handling.
+
+```typescript
+const service = new ExtensionsService(agentBay, 'my_extensions');
+```
+
+## Examples Included
+
+The `extension-example.ts` file contains comprehensive examples demonstrating:
+
+### 1. Basic Extension Management
+- Upload a single extension
+- Create extension options
+- Integrate with browser sessions
+- Basic error handling
+
+### 2. Multiple Extensions Handling
+- Upload multiple extensions
+- Create sessions with multiple extensions
+- Batch operations
+
+### 3. Extension Development Workflow
+- Complete development lifecycle
+- Testing and validation patterns
+- Extension updates
+- Development context management
+
+### 4. Error Handling and Validation
+- Proper validation patterns
+- Common error scenarios
+- Best practices for error management
+
+## Quick Start
+
+### Prerequisites
+
+1. **API Key**: Set your AgentBay API key
+   ```bash
+   export AGENTBAY_API_KEY="your-api-key-here"
+   ```
+
+2. **Extension Files**: Prepare your browser extension ZIP files
+   - Extensions must be in ZIP format
+   - Update file paths in the example code
+
+### Running the Example
+
+```bash
+# Navigate to the TypeScript directory
+cd typescript
+
+# Install dependencies
+npm install
+
+# Build the project
+npm run build
+
+# Run the extension example
+node docs/examples/extension-example/extension-example.js
+```
+
+### Code Example
+
+```typescript
+import { AgentBay, ExtensionsService } from "wuying-agentbay-sdk";
+import { CreateSessionParams, BrowserContext } from "wuying-agentbay-sdk";
+
+// Initialize AgentBay client
+const agentBay = new AgentBay({
+  apiKey: process.env.AGENTBAY_API_KEY,
+});
+
+// Create extensions service
+const extensionsService = new ExtensionsService(agentBay, "browser_extensions");
+
+try {
+  // Upload extension
+  const extension = await extensionsService.create("/path/to/extension.zip");
+  
+  // Create extension option
+  const extOption = extensionsService.createExtensionOption([extension.id]);
+  
+  // Create browser session with extension
+  const sessionParams = new CreateSessionParams()
+    .withLabels({ purpose: "extension_demo" })
+    .withBrowserContext(new BrowserContext(
+      "browser_session",
+      true,
+      extOption
+    ));
+  
+  const sessionResult = await agentBay.create(sessionParams.toJSON());
+  const session = sessionResult.session;
+  
+  console.log(`Session created with extension: ${session.sessionId}`);
+  
+} finally {
+  // Clean up resources
+  await extensionsService.cleanup();
+}
+```
+
+## Extension Integration
+
+### Browser Context Integration
+
+Extensions are automatically synchronized to `/tmp/extensions/` in browser sessions:
+
+```typescript
+// Extension option contains all necessary configuration
+const extOption = extensionsService.createExtensionOption(['ext_1', 'ext_2']);
+
+// Use with BrowserContext
+const browserContext = new BrowserContext(
+  "browser_session",
+  true,          // auto-upload
+  extOption      // extension configuration
+);
+```
+
+### Context Management
+
+The service supports two context modes:
+
+1. **Named Context**: Provide a specific context name
+   ```typescript
+   new ExtensionsService(agentBay, "my_extensions")
+   ```
+
+2. **Auto-Generated Context**: Let the service generate a context name
+   ```typescript
+   new ExtensionsService(agentBay) // Generates: extensions-{timestamp}
+   ```
+
+## API Reference
+
+### ExtensionsService Methods
+
+- `create(localPath: string)` - Upload new extension
+- `list()` - List all extensions in context
+- `update(extensionId: string, newLocalPath: string)` - Update existing extension
+- `delete(extensionId: string)` - Delete extension
+- `createExtensionOption(extensionIds: string[])` - Create browser integration option
+- `cleanup()` - Clean up auto-created contexts
+
+### ExtensionOption Methods
+
+- `validate()` - Validate configuration
+- `toString()` - String representation
+- `toDisplayString()` - Human-readable description
+
+## Error Handling
+
+The examples demonstrate comprehensive error handling:
+
+```typescript
+try {
+  const extension = await extensionsService.create(extensionPath);
+  // Handle success
+} catch (error) {
+  if (error.message.includes('File not found')) {
+    console.log('Extension file does not exist');
+  } else if (error.message.includes('Invalid format')) {
+    console.log('Extension must be a ZIP file');
+  } else {
+    console.log('Upload failed:', error.message);
+  }
+}
+```
+
+## File Requirements
+
+- **Format**: Extensions must be in ZIP format (.zip)
+- **Structure**: Standard Chrome extension structure with manifest.json
+- **Size**: Check AgentBay documentation for size limits
+- **Permissions**: Ensure proper file permissions for reading
+
+## Common Use Cases
+
+1. **Extension Development**: Upload, test, and iterate on extensions
+2. **Automated Testing**: Test extensions in cloud browser environments
+3. **Batch Operations**: Manage multiple extensions simultaneously
+4. **CI/CD Integration**: Automate extension deployment workflows
+
+## Troubleshooting
+
+### Common Issues
+
+1. **File Not Found**: Ensure extension paths are correct
+2. **Invalid Format**: Only ZIP files are supported
+3. **API Key Missing**: Set AGENTBAY_API_KEY environment variable
+4. **Context Conflicts**: Use unique context names for different projects
+
+### Best Practices
+
+1. **Always cleanup**: Use try/finally blocks for resource cleanup
+2. **Validate inputs**: Check file existence before upload
+3. **Handle errors**: Implement proper error handling for all operations
+4. **Use descriptive names**: Name contexts and sessions clearly
+
+## Next Steps
+
+- Explore the complete API documentation at `typescript/docs/api/browser-use/extension.md`
+- Check browser automation examples for integration patterns
+- Review the AgentBay SDK documentation for advanced features
+- Consider implementing custom validation and testing workflows
+
+## Related Resources
+
+- [Extension API Reference](../../../api/browser-use/extension.md) - Complete API documentation
+- [Session API Reference](../../../api/common-features/basics/session.md) - Browser session management
+- [Context API Reference](../../../api/common-features/basics/context.md) - Context management
+- [AgentBay SDK Documentation](../../../api/common-features/basics/agentbay.md) - Core SDK features