camel-ai 0.2.74a4__py3-none-any.whl → 0.2.75__py3-none-any.whl

camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts

@@ -1,4 +1,4 @@
-import { Page, Browser, BrowserContext, chromium } from 'playwright';
+import { Page, Browser, BrowserContext, chromium, ConsoleMessage } from 'playwright';
 import { BrowserToolkitConfig, SnapshotResult, SnapshotElement, ActionResult, TabInfo, BrowserAction, DetailedTiming } from './types';
 import { ConfigLoader, StealthConfig } from './config-loader';
 
@@ -6,18 +6,43 @@ export class HybridBrowserSession {
   private browser: Browser | null = null;
   private context: BrowserContext | null = null;
   private pages: Map<string, Page> = new Map();
+  private consoleLogs: Map<string, ConsoleMessage[]> = new Map();
   private currentTabId: string | null = null;
   private tabCounter = 0;
   private configLoader: ConfigLoader;
   private scrollPosition: { x: number; y: number } = {x: 0, y: 0};
   private hasNavigatedBefore = false; //  Track if we've navigated before
+  private logLimit: number; 
 
   constructor(config: BrowserToolkitConfig = {}) {
     // Use ConfigLoader's fromPythonConfig to handle conversion properly
     this.configLoader = ConfigLoader.fromPythonConfig(config);
+    // Load browser configuration for console log limit, default to 1000
+    this.logLimit = this.configLoader.getBrowserConfig().consoleLogLimit || 1000;
   }
 
-  async ensureBrowser(): Promise<void> {
+  private registerNewPage(tabId: string, page: Page): void {
+    // Register page and logs with tabId
+    this.pages.set(tabId, page);
+    this.consoleLogs.set(tabId, []);
+    // Set up console log listener for the page
+    page.on('console', (msg: ConsoleMessage) => {
+      const logs = this.consoleLogs.get(tabId);
+      if (logs) {
+        logs.push(msg);
+        if (logs.length >  this.logLimit) {
+          logs.shift();
+        }
+      }
+    });
+
+    // Clean logs on page close
+    page.on('close', () => {
+      this.consoleLogs.delete(tabId);
+    });
+  }
+
+  async ensureBrowser(): Promise<void> {  
     if (this.browser) {
       return;
     }
@@ -47,21 +72,19 @@ export class HybridBrowserSession {
         this.context = await this.browser.newContext(contextOptions);
       }
       
-      // Handle existing pages
       const pages = this.context.pages();
       if (pages.length > 0) {
-        // Map existing pages - for CDP, only use pages with about:blank URL
+        // Map existing pages - for CDP, find ONE available blank page
         let availablePageFound = false;
         for (const page of pages) {
           const pageUrl = page.url();
-          // In CDP mode, only consider pages with about:blank as available
-          if (pageUrl === 'about:blank') {
+          if (this.isBlankPageUrl(pageUrl)) {
             const tabId = this.generateTabId();
-            this.pages.set(tabId, page);
-            if (!this.currentTabId) {
-              this.currentTabId = tabId;
-              availablePageFound = true;
-            }
+            this.registerNewPage(tabId, page);
+            this.currentTabId = tabId;
+            availablePageFound = true;
+            console.log(`[CDP] Registered blank page as initial tab: ${tabId}, URL: ${pageUrl}`);
+            break;  // Only register ONE page initially
           }
         }
         
@@ -97,7 +120,7 @@ export class HybridBrowserSession {
         const pages = this.context.pages();
         if (pages.length > 0) {
           const initialTabId = this.generateTabId();
-          this.pages.set(initialTabId, pages[0]);
+          this.registerNewPage(initialTabId, pages[0]);
           this.currentTabId = initialTabId;
         }
       } else {
@@ -115,7 +138,7 @@ export class HybridBrowserSession {
         
         const initialPage = await this.context.newPage();
         const initialTabId = this.generateTabId();
-        this.pages.set(initialTabId, initialPage);
+        this.registerNewPage(initialTabId, initialPage);
         this.currentTabId = initialTabId;
       }
     }
@@ -132,13 +155,57 @@ export class HybridBrowserSession {
     return `${browserConfig.tabIdPrefix}${String(++this.tabCounter).padStart(browserConfig.tabCounterPadding, '0')}`;
   }
 
+  private isBlankPageUrl(url: string): boolean {
+    // Unified blank page detection logic used across the codebase
+    const browserConfig = this.configLoader.getBrowserConfig();
+    return (
+      // Standard about:blank variations (prefix match for query params)
+      url === 'about:blank' || 
+      url.startsWith('about:blank?') ||
+      // Configured blank page URLs (exact match for compatibility)
+      browserConfig.blankPageUrls.includes(url) ||
+      // Empty URL
+      url === '' ||
+      // Data URLs (often used for blank pages)
+      url.startsWith(browserConfig.dataUrlPrefix || 'data:')
+    );
+  }
+
   async getCurrentPage(): Promise<Page> {
     if (!this.currentTabId || !this.pages.has(this.currentTabId)) {
+      // In CDP mode, try to create a new page if none exists
+      const browserConfig = this.configLoader.getBrowserConfig();
+      if (browserConfig.connectOverCdp && this.context) {
+        console.log('[CDP] No active page found, attempting to create new page...');
+        try {
+          const newPage = await this.context.newPage();
+          const newTabId = this.generateTabId();
+          this.registerNewPage(newTabId, newPage);
+          this.currentTabId = newTabId;
+          
+          // Set page timeouts
+          newPage.setDefaultNavigationTimeout(browserConfig.navigationTimeout);
+          newPage.setDefaultTimeout(browserConfig.navigationTimeout);
+          
+          console.log(`[CDP] Created new page with tab ID: ${newTabId}`);
+          return newPage;
+        } catch (error) {
+          console.error('[CDP] Failed to create new page:', error);
+          throw new Error('No active page available and failed to create new page in CDP mode');
+        }
+      }
       throw new Error('No active page available');
     }
     return this.pages.get(this.currentTabId)!;
   }
 
+  async getCurrentLogs(): Promise<ConsoleMessage[]> {
+    if (!this.currentTabId || !this.consoleLogs.has(this.currentTabId)) {
+      return [];
+    }
+    return this.consoleLogs.get(this.currentTabId) || [];
+  }
+
   /**
    * Get current scroll position from the page
    */
@@ -343,7 +410,7 @@ export class HybridBrowserSession {
           
           // Generate tab ID for the new page
           const newTabId = this.generateTabId();
-          this.pages.set(newTabId, newPage);
+          this.registerNewPage(newTabId, newPage);
           
           // Set up page properties
           const browserConfig = this.configLoader.getBrowserConfig();
@@ -384,25 +451,67 @@ export class HybridBrowserSession {
 
   /**
    *  Simplified type implementation using Playwright's aria-ref selector
+   *  Supports both single and multiple input operations
    */
-  private async performType(page: Page, ref: string, text: string): Promise<{ success: boolean; error?: string }> {
+  private async performType(page: Page, ref: string | undefined, text: string | undefined, inputs?: Array<{ ref: string; text: string }>): Promise<{ success: boolean; error?: string; details?: Record<string, any> }> {
     try {
       // Ensure we have the latest snapshot
       await (page as any)._snapshotForAI();
       
-      // Use Playwright's aria-ref selector
-      const selector = `aria-ref=${ref}`;
-      const element = await page.locator(selector).first();
-      
-      const exists = await element.count() > 0;
-      if (!exists) {
-        return { success: false, error: `Element with ref ${ref} not found` };
+      // Handle multiple inputs if provided
+      if (inputs && inputs.length > 0) {
+        const results: Record<string, { success: boolean; error?: string }> = {};
+        
+        for (const input of inputs) {
+          const selector = `aria-ref=${input.ref}`;
+          const element = await page.locator(selector).first();
+          
+          const exists = await element.count() > 0;
+          if (!exists) {
+            results[input.ref] = { success: false, error: `Element with ref ${input.ref} not found` };
+            continue;
+          }
+          
+          try {
+            // Type text using Playwright's built-in fill method
+            await element.fill(input.text);
+            results[input.ref] = { success: true };
+          } catch (error) {
+            results[input.ref] = { success: false, error: `Type failed: ${error}` };
+          }
+        }
+        
+        // Check if all inputs were successful
+        const allSuccess = Object.values(results).every(r => r.success);
+        const errors = Object.entries(results)
+          .filter(([_, r]) => !r.success)
+          .map(([ref, r]) => `${ref}: ${r.error}`)
+          .join('; ');
+        
+        return {
+          success: allSuccess,
+          error: allSuccess ? undefined : `Some inputs failed: ${errors}`,
+          details: results
+        };
       }
       
-      // Type text using Playwright's built-in fill method
-      await element.fill(text);
+      // Handle single input (backward compatibility)
+      if (ref && text !== undefined) {
+        const selector = `aria-ref=${ref}`;
+        const element = await page.locator(selector).first();
+        
+        const exists = await element.count() > 0;
+        if (!exists) {
+          return { success: false, error: `Element with ref ${ref} not found` };
+        }
+        
+        // Type text using Playwright's built-in fill method
+        await element.fill(text);
+        
+        return { success: true };
+      }
       
-      return { success: true };
+      return { success: false, error: 'No valid input provided' };
     } catch (error) {
       return { success: false, error: `Type failed: ${error}` };
     }
@@ -434,7 +543,97 @@ export class HybridBrowserSession {
     }
   }
 
+  /**
+   *  Simplified mouse control implementation
+   */
+  private async performMouseControl(page: Page, control: string, x: number, y: number): Promise<{ success: boolean; error?: string }> {
+    try {
+      const viewport = page.viewportSize();
+      if (!viewport) {
+        return { success: false, error: 'Viewport size not available from page.' };
+      }
+      if (x < 0 || y < 0 || x > viewport.width || y > viewport.height) {
+        return { success: false, error: `Invalid coordinates, outside viewport bounds: (${x}, ${y})` };
+      }
+      switch (control) {
+        case 'click': {
+          await page.mouse.click(x, y);
+          break;
+        }
+        case 'right_click': {
+          await page.mouse.click(x, y, { button: 'right' });
+          break;
+        }
+        case 'dblclick': {
+          await page.mouse.dblclick(x, y);
+          break;
+        }
+        default:
+          return { success: false, error: `Invalid control action: ${control}` };
+      }
+      
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: `Mouse action failed: ${error}` };
+    }
+  }
+
+  /**
+   *  Enhanced mouse drag and drop implementation using ref IDs
+   */
+  private async performMouseDrag(page: Page, fromRef: string, toRef: string): Promise<{ success: boolean; error?: string }> {
+    try {
+      // Ensure we have the latest snapshot
+      await (page as any)._snapshotForAI();
+      
+      // Get elements using Playwright's aria-ref selector
+      const fromSelector = `aria-ref=${fromRef}`;
+      const toSelector = `aria-ref=${toRef}`;
+      
+      const fromElement = await page.locator(fromSelector).first();
+      const toElement = await page.locator(toSelector).first();
+      
+      // Check if elements exist
+      const fromExists = await fromElement.count() > 0;
+      const toExists = await toElement.count() > 0;
+      
+      if (!fromExists) {
+        return { success: false, error: `Source element with ref ${fromRef} not found` };
+      }
+      
+      if (!toExists) {
+        return { success: false, error: `Target element with ref ${toRef} not found` };
+      }
+      
+      // Get the center coordinates of both elements
+      const fromBox = await fromElement.boundingBox();
+      const toBox = await toElement.boundingBox();
+      
+      if (!fromBox) {
+        return { success: false, error: `Could not get bounding box for source element with ref ${fromRef}` };
+      }
+      
+      if (!toBox) {
+        return { success: false, error: `Could not get bounding box for target element with ref ${toRef}` };
+      }
+      
+      const fromX = fromBox.x + fromBox.width / 2;
+      const fromY = fromBox.y + fromBox.height / 2;
+      const toX = toBox.x + toBox.width / 2;
+      const toY = toBox.y + toBox.height / 2;
+
+      // Perform the drag operation
+      await page.mouse.move(fromX, fromY);
+      await page.mouse.down();
+      // Destination coordinates
+      await page.mouse.move(toX, toY);
+      await page.mouse.up();
 
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: `Mouse drag action failed: ${error}` };
+    }
+  }
 
   async executeAction(action: BrowserAction): Promise<ActionResult> {
     const startTime = Date.now();
@@ -450,6 +649,8 @@ export class HybridBrowserSession {
       //  No need to pre-fetch snapshot - each action method handles this
       
       let newTabId: string | undefined;
+      let customMessage: string | undefined;
+      let actionDetails: Record<string, any> | undefined;
       
       switch (action.type) {
         case 'click': {
@@ -474,12 +675,20 @@ export class HybridBrowserSession {
           elementSearchTime = Date.now() - elementSearchStart;
           const typeStart = Date.now();
 
-          const typeResult = await this.performType(page, action.ref, action.text);
+          const typeResult = await this.performType(page, action.ref, action.text, action.inputs);
           
           if (!typeResult.success) {
             throw new Error(`Type failed: ${typeResult.error}`);
           }
           
+          // Set custom message and details if multiple inputs were used
+          if (typeResult.details) {
+            const successCount = Object.values(typeResult.details).filter((r: any) => r.success).length;
+            const totalCount = Object.keys(typeResult.details).length;
+            customMessage = `Typed text into ${successCount}/${totalCount} elements`;
+            actionDetails = typeResult.details;
+          }
+          
           actionExecutionTime = Date.now() - typeStart;
           break;
         }
@@ -519,6 +728,40 @@ export class HybridBrowserSession {
           actionExecutionTime = Date.now() - enterStart;
           break;
         }
+
+        case 'mouse_control': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const mouseControlStart = Date.now();
+          const mouseControlResult = await this.performMouseControl(page, action.control, action.x, action.y);
+
+          if (!mouseControlResult.success) {
+            throw new Error(`Action failed: ${mouseControlResult.error}`);
+          }
+          actionExecutionTime = Date.now() - mouseControlStart;
+          break;
+        }
+
+        case 'mouse_drag': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const mouseDragStart = Date.now();
+          const mouseDragResult = await this.performMouseDrag(page, action.from_ref, action.to_ref);
+
+          if (!mouseDragResult.success) {
+            throw new Error(`Action failed: ${mouseDragResult.error}`);
+          }
+          actionExecutionTime = Date.now() - mouseDragStart;
+          break;
+        }
+
+        case 'press_key': {
+          elementSearchTime = Date.now() - elementSearchStart;
+          const keyPressStart = Date.now();
+          // concatenate keys with '+' for key combinations
+          const keys = action.keys.join('+');
+          await page.keyboard.press(keys);
+          actionExecutionTime = Date.now() - keyPressStart;
+          break;
+        }
           
         default:
           throw new Error(`Unknown action type: ${(action as any).type}`);
@@ -533,7 +776,7 @@ export class HybridBrowserSession {
       
       return {
         success: true,
-        message: `Action ${action.type} executed successfully`,
+        message: customMessage || `Action ${action.type} executed successfully`,
         timing: {
           total_time_ms: totalTime,
           element_search_time_ms: elementSearchTime,
@@ -543,6 +786,7 @@ export class HybridBrowserSession {
           network_idle_time_ms: stabilityResult.networkIdleTime,
         },
         ...(newTabId && { newTabId }), //  Include new tab ID if present
+        ...(actionDetails && { details: actionDetails }), // Include action details if present
       };
     } catch (error) {
       const totalTime = Date.now() - startTime;
@@ -584,16 +828,23 @@ export class HybridBrowserSession {
     
     try {
       // Get current page to check if it's blank
-      const currentPage = await this.getCurrentPage();
-      const currentUrl = currentPage.url();
+      let currentPage: Page;
+      let currentUrl: string;
+      
+      try {
+        currentPage = await this.getCurrentPage();
+        currentUrl = currentPage.url();
+      } catch (error: any) {
+        // If no active page is available, getCurrentPage() will create one in CDP mode
+        console.log('[visitPage] Failed to get current page:', error);
+        throw new Error(`No active page available: ${error?.message || error}`);
+      }
       
       //  Check if current page is blank or if this is the first navigation
       const browserConfig = this.configLoader.getBrowserConfig();
-      const isBlankPage = (
-        browserConfig.blankPageUrls.includes(currentUrl) ||
-        currentUrl === browserConfig.defaultStartUrl ||
-        currentUrl.startsWith(browserConfig.dataUrlPrefix) // data URLs are often used for blank pages
-      );
+      
+      // Use unified blank page detection
+      const isBlankPage = this.isBlankPageUrl(currentUrl) || currentUrl === browserConfig.defaultStartUrl;
       
       const shouldUseCurrentTab = isBlankPage || !this.hasNavigatedBefore;
       
@@ -648,10 +899,10 @@ export class HybridBrowserSession {
             const pageUrl = page.url();
             // Check if this page is not already tracked and is blank
             const isTracked = Array.from(this.pages.values()).includes(page);
-            if (!isTracked && pageUrl === 'about:blank') {
+            if (!isTracked && this.isBlankPageUrl(pageUrl)) {
               newPage = page;
               newTabId = this.generateTabId();
-              this.pages.set(newTabId, newPage);
+              this.registerNewPage(newTabId, newPage);
               break;
             }
           }
@@ -663,7 +914,7 @@ export class HybridBrowserSession {
           // Non-CDP mode: create new page as usual
           newPage = await this.context.newPage();
           newTabId = this.generateTabId();
-          this.pages.set(newTabId, newPage);
+          this.registerNewPage(newTabId, newPage);
         }
         
         // Set up page properties

camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts

@@ -30,6 +30,7 @@ export interface BrowserConfig {
   // Tab management
   tabIdPrefix: string;
   tabCounterPadding: number;
+  consoleLogLimit: number;
   
   // Scroll and positioning
   scrollPositionScale: number;
@@ -113,9 +114,10 @@ function getDefaultBrowserConfig(): BrowserConfig {
     clickTimeout: 3000,
     tabIdPrefix: 'tab-',
     tabCounterPadding: 3,
+    consoleLogLimit: 1000,
     scrollPositionScale: 0.1,
     navigationDelay: 100,
-    blankPageUrls: ['about:blank', ''],
+    blankPageUrls: [],
     dataUrlPrefix: 'data:',
     domContentLoadedState: 'domcontentloaded',
     networkIdleState: 'networkidle',