real-browser-mcp-server 1.1.7 → 1.1.9

package/dist/src/ai/action-parser.js

@@ -0,0 +1,229 @@
+"use strict";
+// @ts-nocheck
+/**
+ * AI Action Parser - Parse natural language commands into actions
+ *
+ * Converts commands like:
+ * - "click the login button" -> { type: 'click', target: 'login button' }
+ * - "type hello in the search box" -> { type: 'type', target: 'search box', text: 'hello' }
+ * - "scroll down" -> { type: 'scroll', direction: 'down' }
+ */
+class ActionParser {
+    constructor() {
+        // Action patterns
+        this.actionPatterns = [
+            // Click patterns
+            {
+                pattern: /^(click|tap|press|hit|select)\s+(?:on\s+)?(?:the\s+)?(.+)/i,
+                type: 'click',
+                extract: (match) => ({ target: match[2].trim() })
+            },
+            // Type patterns
+            {
+                pattern: /^(type|enter|write|input)\s+["']?([^"']+)["']?\s+(?:in|into|in the|into the)\s+(.+)/i,
+                type: 'type',
+                extract: (match) => ({ text: match[2].trim(), target: match[3].trim() })
+            },
+            {
+                pattern: /^(type|enter|write|input)\s+(.+)\s+(?:in|into)\s+["']?([^"']+)["']?/i,
+                type: 'type',
+                extract: (match) => ({ target: match[2].trim(), text: match[3].trim() })
+            },
+            {
+                pattern: /^(fill|fill in|complete)\s+(?:the\s+)?(.+)\s+(?:with|as)\s+["']?([^"']+)["']?/i,
+                type: 'type',
+                extract: (match) => ({ target: match[2].trim(), text: match[3].trim() })
+            },
+            // Navigate patterns
+            {
+                pattern: /^(go to|navigate to|open|visit)\s+(.+)/i,
+                type: 'navigate',
+                extract: (match) => {
+                    let url = match[2].trim();
+                    if (!url.startsWith('http')) {
+                        url = 'https://' + url;
+                    }
+                    return { url };
+                }
+            },
+            // Scroll patterns
+            {
+                pattern: /^scroll\s+(up|down|left|right)(?:\s+(\d+)\s*(?:px|pixels)?)?/i,
+                type: 'scroll',
+                extract: (match) => ({
+                    direction: match[1].toLowerCase(),
+                    amount: parseInt(match[2]) || 300
+                })
+            },
+            {
+                pattern: /^scroll\s+to\s+(?:the\s+)?(top|bottom|footer|header)/i,
+                type: 'scroll',
+                extract: (match) => {
+                    const target = match[1].toLowerCase();
+                    return {
+                        direction: target === 'top' || target === 'header' ? 'up' : 'down',
+                        amount: 10000,
+                        scrollTo: target
+                    };
+                }
+            },
+            // Wait patterns
+            {
+                pattern: /^wait\s+(?:for\s+)?(\d+)\s*(?:ms|milliseconds?|s|seconds?)?/i,
+                type: 'wait',
+                extract: (match) => {
+                    let duration = parseInt(match[1]);
+                    const unit = match[0].toLowerCase();
+                    if (unit.includes('s') && !unit.includes('ms')) {
+                        duration *= 1000;
+                    }
+                    return { duration };
+                }
+            },
+            {
+                pattern: /^wait\s+(?:for\s+)?(?:the\s+)?(.+?)(?:\s+to\s+(?:appear|load|show))?$/i,
+                type: 'waitFor',
+                extract: (match) => ({ target: match[1].trim() })
+            },
+            // Find/Search patterns
+            {
+                pattern: /^(find|search|look for|locate)\s+(?:the\s+)?(.+)/i,
+                type: 'find',
+                extract: (match) => ({ query: match[2].trim() })
+            },
+            // Hover patterns
+            {
+                pattern: /^(hover|mouse over|move to)\s+(?:the\s+)?(.+)/i,
+                type: 'hover',
+                extract: (match) => ({ target: match[2].trim() })
+            },
+            // Clear patterns
+            {
+                pattern: /^(clear|empty|delete)\s+(?:the\s+)?(.+)/i,
+                type: 'clear',
+                extract: (match) => ({ target: match[2].trim() })
+            },
+            // Submit patterns
+            {
+                pattern: /^submit\s+(?:the\s+)?(?:form)?(.*)$/i,
+                type: 'submit',
+                extract: (match) => ({ target: match[1].trim() || 'form' })
+            },
+            // Go back/forward patterns
+            {
+                pattern: /^go\s+(back|forward)/i,
+                type: 'navigation',
+                extract: (match) => ({ direction: match[1].toLowerCase() })
+            },
+            // Refresh patterns
+            {
+                pattern: /^(refresh|reload)\s*(?:the\s+)?(?:page)?/i,
+                type: 'refresh',
+                extract: () => ({})
+            }
+        ];
+        // Context variable patterns (for substitution)
+        this.variablePattern = /\{(\w+)\}/g;
+    }
+    /**
+     * Parse a natural language command
+     */
+    async parse(command, context = {}) {
+        // Substitute context variables
+        let processedCommand = command.replace(this.variablePattern, (match, varName) => {
+            return context[varName] !== undefined ? context[varName] : match;
+        });
+        // Trim and normalize
+        processedCommand = processedCommand.trim();
+        // Try each pattern
+        for (const pattern of this.actionPatterns) {
+            const match = processedCommand.match(pattern.pattern);
+            if (match) {
+                const extracted = pattern.extract(match);
+                return {
+                    type: pattern.type,
+                    ...extracted,
+                    originalCommand: command,
+                    confidence: 0.9
+                };
+            }
+        }
+        // Fallback: Try to guess action from keywords
+        const fallback = this.guessFallback(processedCommand);
+        if (fallback) {
+            return {
+                ...fallback,
+                originalCommand: command,
+                confidence: 0.5
+            };
+        }
+        // Could not parse
+        return {
+            type: 'unknown',
+            originalCommand: command,
+            confidence: 0,
+            error: 'Could not understand command'
+        };
+    }
+    /**
+     * Try to guess action from keywords
+     */
+    guessFallback(command) {
+        const lower = command.toLowerCase();
+        // Check for action keywords
+        if (lower.includes('button') || lower.includes('link') || lower.includes('click')) {
+            return { type: 'click', target: command };
+        }
+        if (lower.includes('type') || lower.includes('enter') || lower.includes('input')) {
+            return { type: 'find', query: command, suggestedAction: 'type' };
+        }
+        if (lower.includes('search') || lower.includes('find') || lower.includes('look')) {
+            return { type: 'find', query: command };
+        }
+        if (lower.includes('scroll')) {
+            return { type: 'scroll', direction: 'down', amount: 300 };
+        }
+        // Default to find
+        return { type: 'find', query: command };
+    }
+    /**
+     * Parse multiple commands (separated by 'then', 'and', or newlines)
+     */
+    async parseMultiple(commands, context = {}) {
+        // Split by separators
+        const parts = commands.split(/\s+(?:then|and)\s+|\n|;/i).filter(p => p.trim());
+        const results = [];
+        for (const part of parts) {
+            const parsed = await this.parse(part.trim(), context);
+            results.push(parsed);
+        }
+        return results;
+    }
+    /**
+     * Get suggestions for incomplete commands
+     */
+    getSuggestions(partialCommand) {
+        const lower = partialCommand.toLowerCase();
+        const suggestions = [];
+        if (lower.startsWith('click')) {
+            suggestions.push('click the login button', 'click the submit button', 'click the link', 'click the menu');
+        }
+        else if (lower.startsWith('type')) {
+            suggestions.push('type "text" in the search box', 'type "username" in the login field', 'type "hello" in the input');
+        }
+        else if (lower.startsWith('go')) {
+            suggestions.push('go to google.com', 'go back', 'go forward');
+        }
+        else if (lower.startsWith('scroll')) {
+            suggestions.push('scroll down', 'scroll up', 'scroll to the bottom', 'scroll to the top');
+        }
+        else if (lower.startsWith('wait')) {
+            suggestions.push('wait 2 seconds', 'wait for the button to appear', 'wait 500ms');
+        }
+        else if (lower.startsWith('find')) {
+            suggestions.push('find the login button', 'find the search input', 'find all links');
+        }
+        return suggestions;
+    }
+}
+module.exports = ActionParser;

package/dist/src/ai/core.js

@@ -0,0 +1,367 @@
+"use strict";
+// @ts-nocheck
+/**
+ * AI Core Module - Foundation for all AI-powered features
+ *
+ * This module provides AI capabilities that are automatically available
+ * to ALL tools in the project. Any new tool added will automatically
+ * benefit from these AI features.
+ *
+ * Features:
+ * - Smart element finding with multiple strategies
+ * - Auto-healing selectors when they break
+ * - Page understanding and structure analysis
+ * - Natural language command parsing
+ * - Confidence scoring for element matches
+ * - Fallback strategies when primary method fails
+ */
+const ElementFinder = require('./element-finder');
+const SelectorHealer = require('./selector-healer');
+const PageAnalyzer = require('./page-analyzer');
+const ActionParser = require('./action-parser');
+const fs = require('fs');
+const path = require('path');
+/**
+ * AI Core class - Central AI intelligence for the browser automation
+ */
+class AICore {
+    constructor() {
+        this.elementFinder = new ElementFinder();
+        this.selectorHealer = new SelectorHealer();
+        this.pageAnalyzer = new PageAnalyzer();
+        this.actionParser = new ActionParser();
+        // Cache for performance
+        this.pageCache = new Map();
+        this.selectorCache = new Map();
+        // Configuration
+        this.config = {
+            defaultConfidence: 0.7,
+            maxCacheAge: 30000, // 30 seconds
+            enableAutoHeal: true,
+            enableSmartFind: true,
+            logLevel: 'info', // 'debug' | 'info' | 'warn' | 'error'
+            cacheFile: path.join(process.cwd(), '.cache', 'ai_cache.json')
+        };
+        this.loadCache();
+    }
+    /**
+     * Load cache from disk
+     */
+    loadCache() {
+        try {
+            if (fs.existsSync(this.config.cacheFile)) {
+                const data = JSON.parse(fs.readFileSync(this.config.cacheFile, 'utf8'));
+                if (data.pageCache) {
+                    this.pageCache = new Map(Object.entries(data.pageCache));
+                }
+                if (data.selectorCache) {
+                    this.selectorCache = new Map(Object.entries(data.selectorCache));
+                }
+                this.log('info', 'Loaded AI cache from disk');
+            }
+        }
+        catch (e) {
+            this.log('warn', `Failed to load cache: ${e.message}`);
+        }
+    }
+    /**
+     * Save cache to disk
+     */
+    saveCache() {
+        try {
+            const dir = path.dirname(this.config.cacheFile);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            const data = {
+                pageCache: Object.fromEntries(this.pageCache),
+                selectorCache: Object.fromEntries(this.selectorCache)
+            };
+            fs.writeFileSync(this.config.cacheFile, JSON.stringify(data, null, 2));
+        }
+        catch (e) {
+            this.log('warn', `Failed to save cache: ${e.message}`);
+        }
+    }
+    /**
+     * Configure AI Core settings
+     */
+    configure(options = {}) {
+        this.config = { ...this.config, ...options };
+        return this;
+    }
+    /**
+     * AI-Enhanced element finding
+     * Tries multiple strategies to find an element
+     */
+    async smartFind(page, query, options = {}) {
+        const { strategy = 'auto', context = null, confidence = this.config.defaultConfidence, returnMultiple = false } = options;
+        this.log('debug', `SmartFind: "${query}" with strategy: ${strategy}`);
+        const results = await this.elementFinder.find(page, query, {
+            strategy,
+            context,
+            confidence,
+            returnMultiple
+        });
+        this.log('info', `SmartFind found ${results.length} elements with confidence >= ${confidence}`);
+        return results;
+    }
+    /**
+     * AI-Enhanced click with auto-healing
+     * If selector fails, tries to find the element using AI
+     */
+    async smartClick(page, selector, options = {}) {
+        const { humanLike = true, autoHeal = this.config.enableAutoHeal } = options;
+        try {
+            // Try original selector first
+            const element = await page.$(selector);
+            if (element) {
+                if (humanLike) {
+                    try {
+                        const { createCursor } = require('ghost-cursor-patchright');
+                        const cursor = createCursor(page);
+                        await cursor.click(selector);
+                    }
+                    catch {
+                        await element.click();
+                    }
+                }
+                else {
+                    await element.click();
+                }
+                return { success: true, selector, healed: false };
+            }
+        }
+        catch (e) {
+            this.log('warn', `Original selector failed: ${selector}`);
+        }
+        // Auto-heal if enabled
+        if (autoHeal) {
+            this.log('info', `Attempting to heal selector: ${selector}`);
+            const healed = await this.healAndExecute(page, selector, 'click', options);
+            if (healed.success) {
+                return { ...healed, healed: true };
+            }
+        }
+        return { success: false, error: `Element not found: ${selector}` };
+    }
+    /**
+     * AI-Enhanced type with auto-healing
+     */
+    async smartType(page, selector, text, options = {}) {
+        const { delay = 50, clear = false, autoHeal = this.config.enableAutoHeal } = options;
+        try {
+            const element = await page.$(selector);
+            if (element) {
+                if (clear) {
+                    await element.click({ clickCount: 3 });
+                    await page.keyboard.press('Backspace');
+                }
+                await element.type(text, { delay });
+                return { success: true, selector, healed: false };
+            }
+        }
+        catch (e) {
+            this.log('warn', `Original selector failed: ${selector}`);
+        }
+        if (autoHeal) {
+            const healed = await this.healAndExecute(page, selector, 'type', { text, ...options });
+            if (healed.success) {
+                return { ...healed, healed: true };
+            }
+        }
+        return { success: false, error: `Element not found: ${selector}` };
+    }
+    /**
+     * Heal a broken selector and execute action
+     */
+    async healAndExecute(page, brokenSelector, action, options = {}) {
+        const alternatives = await this.selectorHealer.heal(page, brokenSelector, {
+            maxAlternatives: 5
+        });
+        for (const alt of alternatives) {
+            try {
+                const element = await page.$(alt.selector);
+                if (element) {
+                    this.log('info', `Healed selector: ${brokenSelector} -> ${alt.selector} (confidence: ${alt.confidence})`);
+                    // Cache the healed selector
+                    this.selectorCache.set(brokenSelector, {
+                        healed: alt.selector,
+                        timestamp: Date.now()
+                    });
+                    this.saveCache();
+                    // Execute action
+                    if (action === 'click') {
+                        await element.click();
+                    }
+                    else if (action === 'type') {
+                        if (options.clear) {
+                            await element.click({ clickCount: 3 });
+                            await page.keyboard.press('Backspace');
+                        }
+                        await element.type(options.text, { delay: options.delay || 50 });
+                    }
+                    return { success: true, selector: alt.selector, originalSelector: brokenSelector };
+                }
+            }
+            catch (e) {
+                continue;
+            }
+        }
+        return { success: false, error: 'Could not heal selector' };
+    }
+    /**
+     * Understand page structure
+     */
+    async understandPage(page, options = {}) {
+        const cacheKey = page.url();
+        const cached = this.pageCache.get(cacheKey);
+        if (cached && (Date.now() - cached.timestamp) < this.config.maxCacheAge) {
+            this.log('debug', 'Using cached page analysis');
+            return cached.analysis;
+        }
+        const analysis = await this.pageAnalyzer.analyze(page, options);
+        this.pageCache.set(cacheKey, {
+            analysis,
+            timestamp: Date.now()
+        });
+        this.saveCache();
+        return analysis;
+    }
+    /**
+     * Parse natural language command and execute
+     */
+    async executeCommand(page, command, options = {}) {
+        const { context = {}, dryRun = false, humanLike = true } = options;
+        this.log('info', `Parsing command: "${command}"`);
+        const parsed = await this.actionParser.parse(command, context);
+        if (dryRun) {
+            return { success: true, dryRun: true, parsed };
+        }
+        // Execute parsed action
+        return await this.executeAction(page, parsed, { humanLike });
+    }
+    /**
+     * Execute a parsed action
+     */
+    async executeAction(page, action, options = {}) {
+        const { humanLike = true } = options;
+        switch (action.type) {
+            case 'click':
+                return await this.smartClick(page, action.target, { humanLike });
+            case 'type':
+                return await this.smartType(page, action.target, action.text, { humanLike });
+            case 'navigate':
+                // Playwright/Patchright waitUntil: load|domcontentloaded|networkidle|commit
+                await page.goto(action.url, { waitUntil: 'networkidle' });
+                return { success: true, url: action.url };
+            case 'scroll':
+                await page.evaluate((direction, amount) => {
+                    window.scrollBy({ top: direction === 'up' ? -amount : amount, behavior: 'smooth' });
+                }, action.direction || 'down', action.amount || 300);
+                return { success: true, direction: action.direction };
+            case 'wait':
+                await new Promise(r => setTimeout(r, action.duration || 1000));
+                return { success: true, waited: action.duration };
+            case 'find':
+                const results = await this.smartFind(page, action.query);
+                return { success: true, found: results.length, elements: results };
+            default:
+                return { success: false, error: `Unknown action type: ${action.type}` };
+        }
+    }
+    /**
+     * Wrap any handler with AI capabilities
+     * This allows existing handlers to benefit from AI features
+     */
+    wrapHandler(handler, handlerName) {
+        const aiCore = this;
+        return async function aiEnhancedHandler(params = {}) {
+            const startTime = Date.now();
+            // Check if AI features are requested
+            const useAI = params._useAI !== false;
+            const autoHeal = params._autoHeal !== false && aiCore.config.enableAutoHeal;
+            try {
+                // Execute original handler
+                const result = await handler(params);
+                // If success, return result
+                if (result.success) {
+                    return {
+                        ...result,
+                        _ai: { used: false, duration: Date.now() - startTime }
+                    };
+                }
+                // If failed and autoHeal is enabled, try AI recovery
+                if (autoHeal && result.error?.includes('not found')) {
+                    aiCore.log('info', `AI attempting recovery for ${handlerName}`);
+                    // Extract selector from params
+                    const selector = params.selector || params.target;
+                    if (selector) {
+                        const healed = await aiCore.selectorHealer.heal(params._page, selector, { maxAlternatives: 3 });
+                        if (healed.length > 0) {
+                            // Retry with healed selector
+                            const retryParams = { ...params, selector: healed[0].selector };
+                            const retryResult = await handler(retryParams);
+                            return {
+                                ...retryResult,
+                                _ai: {
+                                    used: true,
+                                    healed: true,
+                                    originalSelector: selector,
+                                    healedSelector: healed[0].selector,
+                                    duration: Date.now() - startTime
+                                }
+                            };
+                        }
+                    }
+                }
+                return {
+                    ...result,
+                    _ai: { used: useAI, duration: Date.now() - startTime }
+                };
+            }
+            catch (error) {
+                aiCore.log('error', `Handler ${handlerName} failed: ${error.message}`);
+                return {
+                    success: false,
+                    error: error.message,
+                    _ai: { used: useAI, duration: Date.now() - startTime }
+                };
+            }
+        };
+    }
+    /**
+     * Clear caches
+     */
+    clearCache() {
+        this.pageCache.clear();
+        this.selectorCache.clear();
+        this.saveCache();
+        this.log('info', 'AI caches cleared');
+    }
+    /**
+     * Logging utility
+     */
+    log(level, message) {
+        const levels = ['debug', 'info', 'warn', 'error'];
+        const configLevel = levels.indexOf(this.config.logLevel);
+        const msgLevel = levels.indexOf(level);
+        if (msgLevel >= configLevel) {
+            const emoji = { debug: '🔍', info: '🤖', warn: '⚠️', error: '❌' }[level];
+            console.error(`${emoji} [AI] ${message}`);
+        }
+    }
+}
+// Singleton instance
+const aiCore = new AICore();
+module.exports = {
+    AICore,
+    aiCore,
+    smartFind: (page, query, options) => aiCore.smartFind(page, query, options),
+    smartClick: (page, selector, options) => aiCore.smartClick(page, selector, options),
+    smartType: (page, selector, text, options) => aiCore.smartType(page, selector, text, options),
+    understandPage: (page, options) => aiCore.understandPage(page, options),
+    executeCommand: (page, command, options) => aiCore.executeCommand(page, command, options),
+    wrapHandler: (handler, name) => aiCore.wrapHandler(handler, name),
+    configure: (options) => aiCore.configure(options)
+};