vision-navigator 1.0.0

package/README.md

@@ -0,0 +1,113 @@
+# Vision Navigator
+
+A raw CDP (Chrome DevTools Protocol) browser automation tool powered by local LLMs (Ollama) or OpenRouter.
+It natively handles navigation, simplified DOM extraction, and browser interaction via a custom CDP driver, avoiding heavy dependencies like Puppeteer or Playwright.
+
+## Features
+- **Custom CDP Driver**: Communicates directly with Chromium over WebSockets.
+- **Local AI Powered**: Uses `qwen2.5:3b` via Ollama by default, capable of running entirely locally.
+- **Web UI Dashboard**: Includes a sleek dashboard to submit YAML workflows and view step-by-step results, screenshots, and logs.
+- **AI Diagnostics**: Automatically monitors console errors, network issues, and performance metrics, and uses the AI to provide usability/performance suggestions.
+- **CLI Interface**: Can be run as an NPM package/CLI to execute workflows headlessly and get a pass/fail report.
+
+## 🚀 How to Run (3 Ways)
+
+Vision Navigator is flexible and can be run in three different modes depending on your needs.
+
+### Environment Configuration (.env)
+
+You can configure the model, LLM provider, and storage settings using environment variables. Create a `.env` file in the root of the `vision_navigator_ts` directory (or wherever you are running the CLI from):
+
+```env
+# Default is qwen2.5:3b via local Ollama
+OLLAMA_URL=http://localhost:11434
+MODEL=qwen2.5:3b
+
+# If using OpenRouter instead of local Ollama
+OPENROUTER_API_KEY=your_api_key_here
+OPENROUTER_MODEL=google/gemini-2.0-flash-exp:free
+
+# Storage configurations (optional)
+POCKETBASE_URL=http://127.0.0.1:8091
+MINIO_ENDPOINT=127.0.0.1
+MINIO_PORT=9002
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_BUCKET=store-runs
+```
+
+### 1. As a Standalone NPM CLI
+
+If you just want to run tests locally via your terminal, you can install and use it as an NPM package.
+
+```bash
+# Install dependencies and build
+npm install
+npm run build
+
+# Link the package globally
+npm link
+
+# Run a single workflow
+vision-navigator run ./workflows/riskely-test.yaml
+
+# Run a directory of tests
+vision-navigator test ./workflows
+
+# Start the built-in web server (runs on port 8000)
+vision-navigator serve
+```
+
+### 2. Standalone Docker Container
+
+If you want an isolated environment without installing Node.js or Chromium on your host machine, you can run the tool as a standalone Docker container.
+
+```bash
+# Build the Docker image
+docker build -t vision-navigator .
+
+# Run a single workflow (mount your local workflows directory)
+docker run --rm -v $(pwd)/workflows:/usr/src/app/workflows vision-navigator npm run start -- run workflows/riskely-test.yaml
+
+# Start the Web UI only
+docker run -p 8000:8000 vision-navigator
+```
+*Note: If you use a local Ollama instance on your host, make sure to pass the correct `OLLAMA_URL` environment variable (e.g. `-e OLLAMA_URL=http://host.docker.internal:11434`).*
+
+### 3. Full Stack with Docker Compose (Recommended for Self-Hosting)
+
+The most comprehensive way to run Vision Navigator. This spins up the full environment including the main server, a dedicated **Ollama** container (pre-configured with `qwen2.5:3b`), **PocketBase** (for test run history), and **Minio** (for storing screenshot artifacts).
+
+1. Ensure you have Docker and Docker Compose installed.
+2. Run the following command in the root directory:
+
+```bash
+docker-compose up -d --build
+```
+
+This will automatically:
+- Start **Vision Navigator** Web UI at `http://localhost:8000`
+- Start **PocketBase** at `http://localhost:8091`
+- Start **Minio** at `http://localhost:9002`
+- Start **Ollama** and automatically pull the `qwen2.5:3b` model.
+
+**Running CLI tests within Docker Compose:**
+```bash
+# Run a single workflow
+docker exec -it vision-navigator npm run start -- run workflows/riskely-test.yaml
+
+# Run all workflows in a directory
+docker exec -it vision-navigator npm run start -- test workflows
+```
+
+## 📝 Writing Workflows
+Workflows are written in simple YAML format.
+
+```yaml
+steps:
+  - instruction: "Navigate to http://quotes.toscrape.com/"
+  - instruction: "Click the 'Login' link near the top right"
+  - instruction: "Type 'admin' into the username field"
+  - instruction: "Type 'password123' into the password field"
+  - instruction: "Click the Login button"
+```

package/dist/cdp-driver.js

@@ -0,0 +1,484 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CDPDriver = void 0;
+const child_process_1 = require("child_process");
+const axios_1 = __importDefault(require("axios"));
+const ws_1 = __importDefault(require("ws"));
+const injected_scripts_1 = require("./injected-scripts");
+const os_1 = __importDefault(require("os"));
+class CDPDriver {
+    constructor(port = 9222) {
+        this.chromeProcess = null;
+        this.ws = null;
+        this.logs = [];
+        this.networkIssues = [];
+        this.performanceMetrics = null;
+        this.messageId = 1;
+        this.pendingRequests = new Map();
+        this.eventListeners = new Map();
+        this.port = port;
+    }
+    getChromePath() {
+        if (process.env.CHROME_BIN)
+            return process.env.CHROME_BIN;
+        const platform = os_1.default.platform();
+        if (platform === 'darwin')
+            return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+        if (platform === 'win32')
+            return 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe';
+        return '/usr/bin/google-chrome'; // Linux
+    }
+    async start() {
+        console.log("Starting custom CDP driver without Puppeteer...");
+        const chromePath = this.getChromePath();
+        this.chromeProcess = (0, child_process_1.spawn)(chromePath, [
+            `--remote-debugging-port=${this.port}`,
+            '--headless=new', // Modern headless fixes blank screenshot issues
+            '--no-sandbox',
+            '--disable-dev-shm-usage',
+            '--window-size=1280,800',
+            '--disable-web-security',
+            '--ignore-certificate-errors'
+        ]);
+        // Wait for Chrome to start CDP server
+        let wsUrl = '';
+        for (let i = 0; i < 20; i++) {
+            try {
+                await new Promise(r => setTimeout(r, 500));
+                const res = await axios_1.default.get(`http://127.0.0.1:${this.port}/json`);
+                const page = res.data.find((t) => t.type === 'page');
+                if (page && page.webSocketDebuggerUrl) {
+                    wsUrl = page.webSocketDebuggerUrl;
+                    break;
+                }
+            }
+            catch (e) {
+                // Ignore and retry
+            }
+        }
+        if (!wsUrl) {
+            this.stop();
+            throw new Error("Failed to connect to Chrome CDP");
+        }
+        console.log(`Connected to native CDP WebSocket: ${wsUrl}`);
+        await this.connectWs(wsUrl);
+        // Enable domains
+        await this.send('Page.enable');
+        await this.send('Runtime.enable');
+        await this.send('Log.enable');
+        await this.send('Network.enable');
+        await this.send('Performance.enable');
+        // Listen to console events
+        this.on('Runtime.consoleAPICalled', (params) => {
+            const type = params.type.toUpperCase();
+            if (['WARNING', 'ERROR'].includes(type) || type === 'WARN') {
+                const args = params.args.map((a) => a.value || a.description).join(' ');
+                this.logs.push(`[CONSOLE ${type}] ${args}`);
+            }
+        });
+        this.on('Runtime.exceptionThrown', (params) => {
+            this.logs.push(`[CONSOLE EXCEPTION] ${params.exceptionDetails.text}`);
+        });
+        this.on('Network.requestFailed', (params) => {
+            const { request, errorText } = params;
+            this.networkIssues.push(`[NETWORK FAILED] ${request?.url || params.requestId} - ${errorText}`);
+        });
+        this.on('Network.responseReceived', (params) => {
+            const { response } = params;
+            if (response && response.status >= 400) {
+                this.networkIssues.push(`[NETWORK ${response.status}] ${response.url}`);
+            }
+        });
+    }
+    connectWs(url) {
+        return new Promise((resolve, reject) => {
+            this.ws = new ws_1.default(url);
+            this.ws.on('open', resolve);
+            this.ws.on('error', reject);
+            this.ws.on('message', (data) => {
+                const msg = JSON.parse(data.toString());
+                if (msg.id) {
+                    const cb = this.pendingRequests.get(msg.id);
+                    if (cb) {
+                        if (msg.error)
+                            cb.reject(new Error(msg.error.message));
+                        else
+                            cb.resolve(msg.result);
+                        this.pendingRequests.delete(msg.id);
+                    }
+                }
+                else if (msg.method) {
+                    const listeners = this.eventListeners.get(msg.method) || [];
+                    listeners.forEach(fn => fn(msg.params));
+                }
+            });
+        });
+    }
+    send(method, params = {}) {
+        return new Promise((resolve, reject) => {
+            if (!this.ws)
+                return reject(new Error("No WS connection"));
+            const id = this.messageId++;
+            this.pendingRequests.set(id, { resolve, reject });
+            this.ws.send(JSON.stringify({ id, method, params }));
+        });
+    }
+    on(method, callback) {
+        const listeners = this.eventListeners.get(method) || [];
+        listeners.push(callback);
+        this.eventListeners.set(method, listeners);
+    }
+    once(method) {
+        return new Promise(resolve => {
+            const cb = (params) => {
+                resolve(params);
+                const listeners = this.eventListeners.get(method) || [];
+                this.eventListeners.set(method, listeners.filter(fn => fn !== cb));
+            };
+            this.on(method, cb);
+        });
+    }
+    async stop() {
+        if (this.ws)
+            this.ws.close();
+        if (this.chromeProcess) {
+            this.chromeProcess.kill();
+        }
+    }
+    getLogs() {
+        const logs = [...this.logs];
+        this.logs = [];
+        return logs;
+    }
+    getNetworkIssues() {
+        const issues = [...this.networkIssues];
+        this.networkIssues = [];
+        return issues;
+    }
+    async getPerformanceMetrics() {
+        try {
+            const metrics = await this.send('Performance.getMetrics');
+            return metrics.metrics.reduce((acc, m) => {
+                acc[m.name] = m.value;
+                return acc;
+            }, {});
+        }
+        catch (e) {
+            return {};
+        }
+    }
+    async executeScript(script, args = []) {
+        let expression = script;
+        if (args.length > 0) {
+            expression = `(${script})(...${JSON.stringify(args)})`;
+        }
+        const res = await this.send('Runtime.evaluate', {
+            expression: expression,
+            returnByValue: true,
+            awaitPromise: true
+        });
+        if (res.exceptionDetails) {
+            throw new Error(`Script error: ${res.exceptionDetails.exception?.description || res.exceptionDetails.text}`);
+        }
+        return res.result.value;
+    }
+    async navigate(url) {
+        console.log(`Navigating to ${url}...`);
+        const navPromise = this.once('Page.loadEventFired');
+        await this.send('Page.navigate', { url });
+        await navPromise;
+        // Add a small delay after navigation to ensure initial render is complete
+        await new Promise(r => setTimeout(r, 1000));
+        const res = await this.executeScript(`document.title`);
+        console.log(`Page loaded. Title: ${res}`);
+    }
+    async getScreenshot() {
+        await new Promise(r => setTimeout(r, 500));
+        const res = await this.send('Page.captureScreenshot', { format: 'png' });
+        return res.data;
+    }
+    async click(x, y) {
+        await this.send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 });
+        await this.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 });
+    }
+    async typeText(text) {
+        for (const char of text) {
+            await this.send('Input.dispatchKeyEvent', { type: 'char', text: char });
+        }
+    }
+    async scroll(direction) {
+        const yDelta = direction === "down" ? 500 : -500;
+        await this.executeScript(`window.scrollBy(0, ${yDelta})`);
+        await new Promise(r => setTimeout(r, 500));
+    }
+    async getSimplifiedDOM() {
+        const result = await this.executeScript(injected_scripts_1.PAGE_AGENT_SCRIPT);
+        if (result && result.tree) {
+            return this.formatDOMTree(result.tree);
+        }
+        return "<body></body>";
+    }
+    formatDOMTree(node, depth = 0) {
+        if (!node)
+            return '';
+        if (node.type === 'TEXT_NODE') {
+            const text = node.text.replace(/\s+/g, ' ').trim();
+            return text ? (text.length > 50 ? text.substring(0, 50) + '...' : text) + ' ' : '';
+        }
+        const tagName = node.tagName;
+        let attributes = '';
+        if (node.isInteractive && node.highlightIndex !== null) {
+            attributes += ` id="${node.highlightIndex}"`;
+        }
+        const importantAttrs = ['placeholder', 'aria-label', 'title', 'name', 'value', 'alt'];
+        if (node.attributes) {
+            for (const [key, val] of Object.entries(node.attributes)) {
+                if (importantAttrs.includes(key) && val) {
+                    const valStr = String(val);
+                    const truncated = valStr.length > 30 ? valStr.substring(0, 30) + '...' : valStr;
+                    attributes += ` ${key}="${truncated}"`;
+                }
+            }
+        }
+        let childrenHTML = '';
+        if (node.children && node.children.length > 0) {
+            for (const child of node.children) {
+                childrenHTML += this.formatDOMTree(child, depth + 1);
+            }
+        }
+        const hasContent = childrenHTML.trim().length > 0;
+        if (!node.isInteractive && !hasContent) {
+            return '';
+        }
+        if (node.isInteractive) {
+            if (hasContent) {
+                return `<${tagName}${attributes}>${childrenHTML}</${tagName}>`;
+            }
+            else {
+                return `<${tagName}${attributes}/>`;
+            }
+        }
+        else {
+            return childrenHTML + ' ';
+        }
+    }
+    async clickById(id) {
+        console.log(`Clicking element with ID: ${id}`);
+        await this.executeScript(`
+            (function() {
+                const el = document.querySelector('[data-vnav-id="${id}"]');
+                if (el) {
+                    // Visual feedback
+                    const rect = el.getBoundingClientRect();
+                    const overlay = document.createElement('div');
+                    Object.assign(overlay.style, {
+                        position: 'fixed', left: rect.left + 'px', top: rect.top + 'px',
+                        width: rect.width + 'px', height: rect.height + 'px',
+                        backgroundColor: 'rgba(255, 0, 0, 0.3)', border: '2px solid red',
+                        zIndex: '2147483647', pointerEvents: 'none'
+                    });
+                    document.body.appendChild(overlay);
+                    setTimeout(() => overlay.remove(), 500);
+
+                    el.scrollIntoView({ behavior: 'smooth', block: 'center' });
+                    el.click();
+                }
+            })();
+        `);
+        await new Promise(r => setTimeout(r, 600));
+    }
+    async typeById(id, text) {
+        console.log(`Typing "${text}" into element with ID: ${id}`);
+        await this.executeScript(`
+            (function() {
+                const el = document.querySelector('[data-vnav-id="${id}"]');
+                if (el) {
+                    // Visual feedback
+                    const rect = el.getBoundingClientRect();
+                    const overlay = document.createElement('div');
+                    Object.assign(overlay.style, {
+                        position: 'fixed', left: rect.left + 'px', top: rect.top + 'px',
+                        width: rect.width + 'px', height: rect.height + 'px',
+                        backgroundColor: 'rgba(0, 0, 255, 0.3)', border: '2px solid blue',
+                        zIndex: '2147483647', pointerEvents: 'none'
+                    });
+                    document.body.appendChild(overlay);
+                    setTimeout(() => overlay.remove(), 500);
+
+                    el.scrollIntoView({ behavior: 'smooth', block: 'center' });
+                    el.focus();
+                    
+                    if (el.isContentEditable) {
+                        if (!document.execCommand('insertText', false, ${JSON.stringify(text)})) {
+                            el.innerText = ${JSON.stringify(text)};
+                        }
+                    } else {
+                        // React-friendly value setter
+                        const proto = el instanceof HTMLTextAreaElement ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
+                        const nativeInputValueSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
+                        if (nativeInputValueSetter) {
+                            nativeInputValueSetter.call(el, ${JSON.stringify(text)});
+                        } else {
+                            el.value = ${JSON.stringify(text)};
+                        }
+                    }
+                    
+                    el.dispatchEvent(new Event('input', { bubbles: true }));
+                    el.dispatchEvent(new Event('change', { bubbles: true }));
+                }
+            })();
+        `);
+        await new Promise(r => setTimeout(r, 600));
+    }
+    async dismissCookieBanners(opts) {
+        try {
+            const aggressive = Boolean(opts?.aggressive);
+            const res = await this.executeScript(`
+                (() => {
+                    const aggressive = ${JSON.stringify(aggressive)};
+                    const COOKIE_WORDS = [
+                        'cookie', 'cookies', 'kaka', 'kakor', 'samtycke', 'samtyck', 'consent', 'gdpr', 'integritet', 'privacy'
+                    ];
+                    const ACCEPT = [
+                        'accept', 'accept all', 'agree', 'i agree', 'allow', 'allow all', 'ok', 'okay',
+                        'acceptera', 'acceptera alla', 'godkänn', 'godkänn alla', 'tillåt', 'tillåt alla', 'jag godkänner', 'jag accepterar',
+                        'samtyck', 'samtycker'
+                    ];
+                    const DENY = [
+                        'reject', 'decline', 'deny', 'no thanks', 'manage', 'settings', 'preferences', 'customize', 'only necessary',
+                        'avvisa', 'neka', 'inställ', 'inställningar', 'anpassa', 'nödvänd', 'bara nödvändiga', 'hantera', 'preferenser'
+                    ];
+
+                    const normalize = (s) => (s || '')
+                        .toString()
+                        .toLowerCase()
+                        .normalize('NFD')
+                        .replace(/[\\u0300-\\u036f]/g, '')
+                        .replace(/\\s+/g, ' ')
+                        .trim();
+
+                    const isVisible = (el) => {
+                        try {
+                            const style = window.getComputedStyle(el);
+                            if (!style || style.visibility === 'hidden' || style.display === 'none') return false;
+                            const rect = el.getBoundingClientRect();
+                            if (!rect || rect.width < 2 || rect.height < 2) return false;
+                            if (rect.bottom < 0 || rect.right < 0) return false;
+                            return true;
+                        } catch {
+                            return false;
+                        }
+                    };
+
+                    const hasCookieContext = (el) => {
+                        try {
+                            const bodyText = normalize(document.body?.innerText || document.body?.textContent || '');
+                            const globalCookie = COOKIE_WORDS.some(w => bodyText.includes(normalize(w)));
+                            if (globalCookie) return true;
+                            let node = el;
+                            for (let i = 0; i < 6 && node; i++) {
+                                const t = normalize(node.innerText || node.textContent || '');
+                                if (COOKIE_WORDS.some(w => t.includes(normalize(w)))) return true;
+                                node = node.parentElement;
+                            }
+                            return false;
+                        } catch {
+                            return false;
+                        }
+                    };
+
+                    const scoreEl = (el) => {
+                        const text = normalize(el.innerText || el.textContent || '');
+                        const label = normalize(el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('value') || '');
+                        const combined = (text + ' ' + label).trim();
+                        if (!combined) return { score: 0, combined };
+                        for (const bad of DENY) {
+                            if (combined.includes(normalize(bad))) return { score: 0, combined };
+                        }
+                        let score = 0;
+                        for (const ok of ACCEPT) {
+                            const tok = normalize(ok);
+                            if (!tok) continue;
+                            if (combined === tok) score += 50;
+                            else if (combined.includes(tok)) score += 20;
+                        }
+                        if (!aggressive && !hasCookieContext(el)) score = 0;
+                        const tag = (el.tagName || '').toLowerCase();
+                        if (tag === 'button') score += 5;
+                        const role = normalize(el.getAttribute('role'));
+                        if (role === 'button') score += 2;
+                        return { score, combined };
+                    };
+
+                    const knownSelectors = [
+                        '#onetrust-accept-btn-handler',
+                        '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',
+                        '#CybotCookiebotDialogBodyButtonAccept',
+                        'button[id*="cookie"][id*="accept"]',
+                        'button[id*="consent"][id*="accept"]',
+                        'button[data-testid*="cookie"][data-testid*="accept"]'
+                    ];
+
+                    const clickFirst = (el) => {
+                        try {
+                            el.scrollIntoView({ block: 'center', inline: 'center' });
+                            el.click();
+                            return true;
+                        } catch {
+                            return false;
+                        }
+                    };
+
+                    for (const sel of knownSelectors) {
+                        const el = document.querySelector(sel);
+                        if (el && isVisible(el)) {
+                            const meta = scoreEl(el);
+                            if (clickFirst(el)) {
+                                return { clicked: true, label: meta.combined || sel, candidates: 1 };
+                            }
+                        }
+                    }
+
+                    const collect = (root, out) => {
+                        try {
+                            const nodes = root.querySelectorAll('button, a, input[type="button"], input[type="submit"], [role="button"]');
+                            nodes.forEach(n => out.push(n));
+                            const all = root.querySelectorAll('*');
+                            all.forEach(n => {
+                                if (n && n.shadowRoot) collect(n.shadowRoot, out);
+                            });
+                        } catch {}
+                    };
+
+                    const candidates = [];
+                    collect(document, candidates);
+
+                    const scored = candidates
+                        .filter(isVisible)
+                        .map(el => ({ el, ...scoreEl(el) }))
+                        .filter(x => x.score > 0)
+                        .sort((a, b) => b.score - a.score);
+
+                    if (scored.length === 0) return { clicked: false, candidates: 0 };
+
+                    const best = scored[0];
+                    const ok = clickFirst(best.el);
+                    return { clicked: !!ok, label: best.combined, candidates: scored.length };
+                })()
+            `);
+            return { clicked: Boolean(res?.clicked), label: res?.label, candidates: res?.candidates };
+        }
+        catch {
+            return { clicked: false };
+        }
+    }
+    // Keep legacy method signature if strictly needed, or reimplement
+    async getInteractiveElements() {
+        // Not strictly used by Navigator currently, but good to have
+        return [];
+    }
+}
+exports.CDPDriver = CDPDriver;