vision-navigator 1.0.0

package/dist/cli.js

@@ -0,0 +1,207 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("dotenv/config");
+const server_1 = require("./server");
+const navigator_1 = require("./navigator");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const rawArgs = process.argv.slice(2);
+const args = [];
+for (let i = 0; i < rawArgs.length; i++) {
+    const a = rawArgs[i];
+    if (a === '--groq-key') {
+        const v = rawArgs[i + 1];
+        if (v) {
+            process.env.GROQ_API_KEY = v;
+            i++;
+        }
+        continue;
+    }
+    if (a === '--groq-model') {
+        const v = rawArgs[i + 1];
+        if (v) {
+            process.env.GROQ_MODEL = v;
+            i++;
+        }
+        continue;
+    }
+    if (a === '--fast') {
+        process.env.FAST_MODE = '1';
+        continue;
+    }
+    if (a === '--verify') {
+        process.env.VERIFY_WITH_AI = '1';
+        continue;
+    }
+    if (a === '--no-analysis') {
+        process.env.SKIP_ANALYSIS = '1';
+        continue;
+    }
+    if (a === '--analysis-async') {
+        process.env.ANALYSIS_ASYNC = '1';
+        continue;
+    }
+    args.push(a);
+}
+const command = args[0];
+if (command === 'run') {
+    const filePath = args[1];
+    if (!filePath) {
+        console.error("Error: Please provide a path to a YAML workflow file.");
+        console.log("Usage: vision-navigator run <workflow.yaml>");
+        process.exit(1);
+    }
+    const fullPath = path_1.default.resolve(process.cwd(), filePath);
+    if (!fs_1.default.existsSync(fullPath)) {
+        console.error(`Error: File not found: ${fullPath}`);
+        process.exit(1);
+    }
+    console.log(`Loading workflow from ${filePath}...`);
+    const fileContent = fs_1.default.readFileSync(fullPath, 'utf-8');
+    let workflow;
+    try {
+        workflow = js_yaml_1.default.load(fileContent);
+    }
+    catch (e) {
+        console.error(`Error parsing YAML: ${e}`);
+        process.exit(1);
+    }
+    const nav = new navigator_1.Navigator();
+    nav.runWorkflow(workflow).then(run => {
+        console.log("\n========================================");
+        console.log("      WORKFLOW EXECUTION RESULTS        ");
+        console.log("========================================\n");
+        let allPassed = true;
+        run.results.forEach((r, i) => {
+            const statusIcon = (r.status === 'executed' || r.status === 'completed') ? '✅' : '❌';
+            console.log(`${statusIcon} Step ${i + 1}: ${r.step}`);
+            console.log(`   Status: ${r.status}`);
+            if (r.action) {
+                console.log(`   Action: ${r.action.replace(/\n/g, ' ')}`);
+            }
+            if (r.ai_suggestions) {
+                console.log(`   💡 AI Feedback: ${r.ai_suggestions.split('\n').map((l) => l.trim()).filter((l) => l).join(' ')}`);
+            }
+            if (r.has_console_errors || (r.network_issues && r.network_issues.length > 0)) {
+                console.log(`   ⚠️  Issues detected (check web UI or logs for details)`);
+            }
+            console.log("");
+            if (r.status !== 'executed' && r.status !== 'completed') {
+                allPassed = false;
+            }
+        });
+        console.log("========================================");
+        if (allPassed) {
+            console.log("🎉 All steps executed successfully!");
+            process.exit(0);
+        }
+        else {
+            console.log("⚠️  Some steps failed or encountered issues.");
+            process.exit(1);
+        }
+    }).catch(e => {
+        console.error(`Workflow execution failed: ${e}`);
+        process.exit(1);
+    });
+}
+else if (command === 'test') {
+    const dirPath = args[1] || '.';
+    const fullDirPath = path_1.default.resolve(process.cwd(), dirPath);
+    if (!fs_1.default.existsSync(fullDirPath) || !fs_1.default.statSync(fullDirPath).isDirectory()) {
+        console.error(`Error: Directory not found: ${fullDirPath}`);
+        process.exit(1);
+    }
+    const files = fs_1.default.readdirSync(fullDirPath)
+        .filter(f => f.endsWith('.yaml') || f.endsWith('.yml'))
+        .map(f => path_1.default.join(fullDirPath, f));
+    if (files.length === 0) {
+        console.log(`No YAML files found in ${fullDirPath}`);
+        process.exit(0);
+    }
+    console.log(`Found ${files.length} test files. Starting execution...\n`);
+    (async () => {
+        const nav = new navigator_1.Navigator();
+        let allTestsPassed = true;
+        const testResults = [];
+        for (const file of files) {
+            console.log(`\n--- Running Test: ${path_1.default.basename(file)} ---`);
+            const fileContent = fs_1.default.readFileSync(file, 'utf-8');
+            let workflow;
+            try {
+                workflow = js_yaml_1.default.load(fileContent);
+            }
+            catch (e) {
+                console.error(`Error parsing YAML: ${e}`);
+                testResults.push({ file: path_1.default.basename(file), passed: false, steps: 0 });
+                allTestsPassed = false;
+                continue;
+            }
+            try {
+                const run = await nav.runWorkflow(workflow);
+                let passed = true;
+                run.results.forEach((r) => {
+                    if (r.status !== 'executed' && r.status !== 'completed') {
+                        passed = false;
+                    }
+                });
+                if (!passed)
+                    allTestsPassed = false;
+                console.log(`\nResult for ${path_1.default.basename(file)}: ${passed ? '✅ PASSED' : '❌ FAILED'}`);
+                testResults.push({ file: path_1.default.basename(file), passed, steps: run.results.length });
+            }
+            catch (e) {
+                console.error(`Failed executing ${path_1.default.basename(file)}: ${e}`);
+                testResults.push({ file: path_1.default.basename(file), passed: false, steps: 0 });
+                allTestsPassed = false;
+            }
+        }
+        console.log("\n========================================");
+        console.log("            TEST SUITE RESULTS          ");
+        console.log("========================================");
+        testResults.forEach(tr => {
+            console.log(`${tr.passed ? '✅' : '❌'} ${tr.file} (${tr.steps} steps)`);
+        });
+        console.log("========================================");
+        if (allTestsPassed) {
+            console.log("🎉 ALL TESTS PASSED!");
+            process.exit(0);
+        }
+        else {
+            console.log("⚠️  SOME TESTS FAILED.");
+            process.exit(1);
+        }
+    })();
+}
+else if (command === 'serve' || !command) {
+    const port = process.env.PORT ? parseInt(process.env.PORT, 10) : 8000;
+    (0, server_1.startServer)(port);
+}
+else {
+    console.log("Usage: vision-navigator [serve | run <file.yaml>]");
+    console.log("");
+    console.log("Commands:");
+    console.log("  serve              Start the web UI server (default)");
+    console.log("  run <file.yaml>    Execute a workflow file in the terminal");
+    console.log("  test [dir]         Run all YAML workflows in a directory (default: current dir)");
+    console.log("");
+    console.log("Environment:");
+    console.log("  GROQ_API_KEY        Use Groq for actions/verification/analysis");
+    console.log("  GROQ_MODEL          Override Groq model (default: llama-3.1-8b-instant)");
+    console.log("  FAST_MODE=1         Faster runs (shorter prompts, less waiting)");
+    console.log("  VERIFY_WITH_AI=1    Verify critical steps (login/submit/continue etc)");
+    console.log("  SKIP_ANALYSIS=1     Skip long step analysis (keeps verifier if enabled)");
+    console.log("  ANALYSIS_ASYNC=1    Run analysis in background (faster step loop)");
+    console.log("");
+    console.log("Flags:");
+    console.log("  --groq-key <key>    Set GROQ_API_KEY for this run");
+    console.log("  --groq-model <id>   Set GROQ_MODEL for this run");
+    console.log("  --fast              Set FAST_MODE=1");
+    console.log("  --verify            Set VERIFY_WITH_AI=1");
+    console.log("  --no-analysis       Set SKIP_ANALYSIS=1");
+    console.log("  --analysis-async    Set ANALYSIS_ASYNC=1");
+}

package/dist/injected-scripts.js

@@ -0,0 +1,328 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PAGE_AGENT_SCRIPT = void 0;
+exports.PAGE_AGENT_SCRIPT = `
+(function() {
+  const DOM_HASH_MAP = {};
+  const ID = { current: 0 };
+  const HIGHLIGHT_CONTAINER_ID = 'playwright-highlight-container';
+  let xpathCache = new WeakMap();
+  
+  // Cache for DOM queries
+  const DOM_CACHE = {
+    boundingRects: new WeakMap(),
+    computedStyles: new WeakMap(),
+    clearCache: () => {
+      DOM_CACHE.boundingRects = new WeakMap();
+      DOM_CACHE.computedStyles = new WeakMap();
+    }
+  };
+
+  function getCachedBoundingRect(element) {
+    if (!element) return null;
+    if (DOM_CACHE.boundingRects.has(element)) {
+      return DOM_CACHE.boundingRects.get(element);
+    }
+    const rect = element.getBoundingClientRect();
+    if (rect) {
+      DOM_CACHE.boundingRects.set(element, rect);
+    }
+    return rect;
+  }
+
+  function getCachedComputedStyle(element) {
+    if (!element) return null;
+    if (DOM_CACHE.computedStyles.has(element)) {
+      return DOM_CACHE.computedStyles.get(element);
+    }
+    const style = window.getComputedStyle(element);
+    if (style) {
+      DOM_CACHE.computedStyles.set(element, style);
+    }
+    return style;
+  }
+
+  function getElementPosition(currentElement) {
+    if (!currentElement.parentElement) return 0;
+    const tagName = currentElement.nodeName.toLowerCase();
+    const siblings = Array.from(currentElement.parentElement.children).filter(
+      (sib) => sib.nodeName.toLowerCase() === tagName
+    );
+    if (siblings.length === 1) return 0;
+    return siblings.indexOf(currentElement) + 1;
+  }
+
+  function getXPathTree(element, stopAtBoundary = true) {
+    if (xpathCache.has(element)) return xpathCache.get(element);
+    const segments = [];
+    let currentElement = element;
+    while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
+      if (stopAtBoundary && (currentElement.parentNode instanceof ShadowRoot || currentElement.parentNode instanceof HTMLIFrameElement)) {
+        break;
+      }
+      const position = getElementPosition(currentElement);
+      const tagName = currentElement.nodeName.toLowerCase();
+      const xpathIndex = position > 0 ? \`[\${position}]\` : '';
+      segments.unshift(\`\${tagName}\${xpathIndex}\`);
+      currentElement = currentElement.parentNode;
+    }
+    const result = segments.join('/');
+    xpathCache.set(element, result);
+    return result;
+  }
+
+  function isScrollableElement(element) {
+    if (!element || element.nodeType !== Node.ELEMENT_NODE) return null;
+    const style = getCachedComputedStyle(element);
+    if (!style) return null;
+    const display = style.display;
+    if (display === 'inline' || display === 'inline-block') return null;
+    
+    const overflowX = style.overflowX;
+    const overflowY = style.overflowY;
+    const scrollableX = overflowX === 'auto' || overflowX === 'scroll';
+    const scrollableY = overflowY === 'auto' || overflowY === 'scroll';
+    
+    if (!scrollableX && !scrollableY) return null;
+    
+    const scrollWidth = element.scrollWidth - element.clientWidth;
+    const scrollHeight = element.scrollHeight - element.clientHeight;
+    const threshold = 4;
+    
+    if (scrollWidth < threshold && scrollHeight < threshold) return null;
+    if (!scrollableY && scrollWidth < threshold) return null;
+    if (!scrollableX && scrollHeight < threshold) return null;
+    
+    return {
+      top: element.scrollTop,
+      right: element.scrollWidth - element.clientWidth - element.scrollLeft,
+      bottom: element.scrollHeight - element.clientHeight - element.scrollTop,
+      left: element.scrollLeft
+    };
+  }
+
+  function isElementVisible(element) {
+    const style = getCachedComputedStyle(element);
+    return (
+      element.offsetWidth > 0 &&
+      element.offsetHeight > 0 &&
+      style?.visibility !== 'hidden' &&
+      style?.display !== 'none'
+    );
+  }
+
+  function isInteractiveElement(element) {
+    if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
+    
+    const tagName = element.tagName.toLowerCase();
+    const style = getCachedComputedStyle(element);
+    
+    // Ignore elements that cannot receive pointer events
+    if (style?.pointerEvents === 'none') return false;
+    
+    // Explicit interactive tags
+    const interactiveTags = new Set(['a', 'button', 'input', 'select', 'textarea', 'details', 'summary', 'label', 'option']);
+    if (interactiveTags.has(tagName)) {
+        // Check disabled
+        if (element.disabled || element.readOnly || element.inert) return false;
+        return true;
+    }
+    
+    // Roles
+    const role = element.getAttribute('role');
+    const interactiveRoles = new Set(['button', 'link', 'menuitem', 'tab', 'checkbox', 'radio', 'switch', 'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'slider', 'spinbutton', 'scrollbar']);
+    if (role && interactiveRoles.has(role)) return true;
+    
+    // Content editable
+    if (element.getAttribute('contenteditable') === 'true' || element.getAttribute('contenteditable') === '') return true;
+    if (element.isContentEditable) {
+        // Only mark as interactive if parent is not also contentEditable (to avoid highlighting every single child span)
+        if (element.parentElement && !element.parentElement.isContentEditable) {
+            return true;
+        }
+    }
+
+    // Scrollable
+    if (isScrollableElement(element)) return true;
+
+    // Event listeners (requires getEventListeners or heuristic)
+    try {
+        if (typeof getEventListeners === 'function') {
+            const listeners = getEventListeners(element);
+            if (listeners && (listeners.click || listeners.mousedown || listeners.mouseup)) return true;
+        }
+    } catch(e) {}
+    
+    if (element.onclick || element.onmousedown || element.onmouseup) return true;
+
+    // Interactive cursors
+    const interactiveCursors = new Set(['pointer', 'cell', 'crosshair', 'move', 'grab', 'grabbing', 'zoom-in', 'zoom-out']);
+    if (style?.cursor && interactiveCursors.has(style.cursor)) {
+        // Check if parent has the same cursor (to avoid highlighting children just due to inheritance)
+        let parent = element.parentElement;
+        if (parent) {
+            const parentStyle = getCachedComputedStyle(parent);
+            if (parentStyle && parentStyle.cursor === style.cursor) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    return false;
+  }
+
+  function highlightElement(element, index, parentIframe) {
+    let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
+    if (!container) {
+      container = document.createElement('div');
+      container.id = HIGHLIGHT_CONTAINER_ID;
+      container.style.position = 'fixed';
+      container.style.top = '0';
+      container.style.left = '0';
+      container.style.width = '100%';
+      container.style.height = '100%';
+      container.style.zIndex = '2147483647';
+      container.style.pointerEvents = 'none';
+      document.body.appendChild(container);
+    }
+
+    const rect = element.getBoundingClientRect();
+    if (!rect || rect.width === 0 || rect.height === 0) return;
+
+    const colors = [
+        '#FF0000', '#00FF00', '#0000FF', '#FFA500', '#800080', '#008080', '#FF69B4', '#4B0082', '#FF4500', '#2E8B57', '#DC143C', '#4682B4'
+    ];
+    const baseColor = colors[index % colors.length];
+    const backgroundColor = baseColor + '1A'; // Low opacity
+
+    const fragment = document.createDocumentFragment();
+    let iframeOffset = { x: 0, y: 0 };
+    if (parentIframe) {
+        const iframeRect = parentIframe.getBoundingClientRect();
+        iframeOffset.x = iframeRect.left;
+        iframeOffset.y = iframeRect.top;
+    }
+
+    const overlay = document.createElement('div');
+    overlay.style.position = 'fixed';
+    overlay.style.border = \`2px solid \${baseColor}\`;
+    overlay.style.backgroundColor = backgroundColor;
+    overlay.style.pointerEvents = 'none';
+    overlay.style.boxSizing = 'border-box';
+    overlay.style.top = \`\${rect.top + iframeOffset.y}px\`;
+    overlay.style.left = \`\${rect.left + iframeOffset.x}px\`;
+    overlay.style.width = \`\${rect.width}px\`;
+    overlay.style.height = \`\${rect.height}px\`;
+    fragment.appendChild(overlay);
+
+    // Label
+    const label = document.createElement('div');
+    label.className = 'vnav-highlight-label';
+    label.style.position = 'fixed';
+    label.style.background = baseColor;
+    label.style.color = 'white';
+    label.style.padding = '1px 4px';
+    label.style.borderRadius = '4px';
+    label.style.fontSize = '12px';
+    label.style.fontWeight = 'bold';
+    label.textContent = index.toString();
+    
+    let labelTop = rect.top + iframeOffset.y;
+    let labelLeft = rect.left + iframeOffset.x; // Position at top-left
+    
+    // Adjust slightly so it doesn't overlap completely with the border
+    labelTop -= 10;
+    labelLeft -= 10;
+    
+    if (labelLeft < 0) labelLeft = 0;
+    if (labelTop < 0) labelTop = 0;
+    
+    label.style.top = \`\${labelTop}px\`;
+    label.style.left = \`\${labelLeft}px\`;
+    
+    fragment.appendChild(label);
+    container.appendChild(fragment);
+  }
+
+  // Helper to remove existing highlights
+  function removeHighlights() {
+    const container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
+    if (container) container.remove();
+  }
+
+  function buildDomTree(node, highlightIndexRef) {
+    if (!node) return null;
+    if (node.id === HIGHLIGHT_CONTAINER_ID) return null;
+    
+    // Skip script/style etc
+    const denyList = new Set(['script', 'style', 'noscript', 'meta', 'link', 'svg']);
+    if (denyList.has(node.tagName?.toLowerCase())) return null;
+
+    if (node.nodeType === Node.TEXT_NODE) {
+        const text = node.textContent.trim();
+        if (!text) return null;
+        if (!isElementVisible(node.parentElement)) return null;
+        return {
+            type: 'TEXT_NODE',
+            text: text,
+            isVisible: true
+        };
+    }
+
+    if (node.nodeType !== Node.ELEMENT_NODE) return null;
+    if (!isElementVisible(node)) return null;
+
+    const nodeData = {
+        tagName: node.tagName.toLowerCase(),
+        attributes: {},
+        children: [],
+        isInteractive: false,
+        highlightIndex: null
+    };
+
+    // Attributes
+    if (node.hasAttributes()) {
+        for (const attr of node.attributes) {
+            nodeData.attributes[attr.name] = attr.value;
+        }
+    }
+
+    // Interactivity
+    if (isInteractiveElement(node)) {
+        nodeData.isInteractive = true;
+        nodeData.highlightIndex = highlightIndexRef.current++;
+        highlightElement(node, nodeData.highlightIndex, null);
+        
+        // Add vnav-id to element for easy lookup later if needed
+        node.setAttribute('data-vnav-id', nodeData.highlightIndex);
+    }
+
+    // Children
+    if (node.shadowRoot) {
+        for (const child of node.shadowRoot.childNodes) {
+            const childData = buildDomTree(child, highlightIndexRef);
+            if (childData) nodeData.children.push(childData);
+        }
+    }
+    for (const child of node.childNodes) {
+        const childData = buildDomTree(child, highlightIndexRef);
+        if (childData) nodeData.children.push(childData);
+    }
+
+    return nodeData;
+  }
+
+  // Main execution
+  removeHighlights();
+  DOM_CACHE.clearCache();
+  const highlightIndexRef = { current: 1 };
+  
+  const tree = buildDomTree(document.body, highlightIndexRef);
+  
+  return {
+    tree,
+    elementCount: highlightIndexRef.current - 1
+  };
+})()
+`;