sonance-brand-mcp 1.3.94 → 1.3.95

package/dist/assets/dev-tools/components/ChatInterface.tsx

@@ -113,6 +113,8 @@ export function ChatInterface({
   const [isAnnotating, setIsAnnotating] = useState(false);
   const [annotatedScreenshot, setAnnotatedScreenshot] = useState<string | null>(null);
   const [manualFocusBounds, setManualFocusBounds] = useState<Rectangle | null>(null);
+  // Discovered elements from annotation tool (for targeting when no element was clicked)
+  const [annotationDiscoveredElements, setAnnotationDiscoveredElements] = useState<VisionFocusedElement[]>([]);
   
   // Auto-dismiss toast after 5 seconds
   useEffect(() => {
@@ -177,10 +179,20 @@ export function ChatInterface({
   }, []);
 
   // Handle annotation confirmation - screenshot is already captured and annotated
-  const handleAnnotationConfirm = useCallback((annotated: string, bounds: Rectangle) => {
-    console.log("[Vision Mode] Annotation confirmed:", { bounds });
+  // Now also receives discovered elements from within the drawn rectangle
+  const handleAnnotationConfirm = useCallback((annotated: string, bounds: Rectangle, discoveredElements: VisionFocusedElement[]) => {
+    console.log("[Vision Mode] Annotation confirmed:", { 
+      bounds, 
+      discoveredElementsCount: discoveredElements.length,
+      discoveredElements: discoveredElements.map(e => ({
+        name: e.name,
+        text: e.textContent?.substring(0, 30),
+        id: e.elementId,
+      })),
+    });
     setAnnotatedScreenshot(annotated);
     setManualFocusBounds(bounds);
+    setAnnotationDiscoveredElements(discoveredElements);
     setIsAnnotating(false);
     // Focus the input so user can type their prompt
     setTimeout(() => inputRef.current?.focus(), 100);
@@ -191,10 +203,11 @@ export function ChatInterface({
     setIsAnnotating(false);
   }, []);
 
-  // Clear the current annotation
+  // Clear the current annotation and discovered elements
   const clearAnnotation = useCallback(() => {
     setAnnotatedScreenshot(null);
     setManualFocusBounds(null);
+    setAnnotationDiscoveredElements([]);
   }, []);
 
   // Handle vision mode edit request
@@ -202,9 +215,18 @@ export function ChatInterface({
     // Use Apply-First mode if callback is provided (new Cursor-style workflow)
     const useApplyFirst = !!onApplyFirstComplete;
 
+    // Determine which focused elements to use:
+    // - If user clicked an element, use visionFocusedElements (passed from parent)
+    // - If user used annotation tool without clicking, use annotationDiscoveredElements
+    const effectiveFocusedElements = visionFocusedElements.length > 0 
+      ? visionFocusedElements 
+      : annotationDiscoveredElements;
+
     console.log("[Vision Mode] Starting edit request:", {
       prompt,
-      focusedElements: visionFocusedElements.length,
+      focusedElementsFromClick: visionFocusedElements.length,
+      focusedElementsFromAnnotation: annotationDiscoveredElements.length,
+      effectiveFocusedElements: effectiveFocusedElements.length,
       mode: useApplyFirst ? "apply-first" : "preview-first"
     });
 
@@ -226,9 +248,16 @@ export function ChatInterface({
       // PRIORITY 1: Use manually annotated screenshot if available
       // This is when user drew a focus area using the annotation tool
       if (annotatedScreenshot) {
-        console.log("[Vision Mode] Using manually annotated screenshot");
+        console.log("[Vision Mode] Using manually annotated screenshot with discovered elements:", {
+          discoveredCount: annotationDiscoveredElements.length,
+          elements: annotationDiscoveredElements.slice(0, 3).map(e => ({
+            name: e.name,
+            text: e.textContent?.substring(0, 20),
+            id: e.elementId,
+          })),
+        });
         screenshot = annotatedScreenshot;
-        // Clear the annotation after use
+        // Clear the annotation after use (but keep discovered elements for the API call)
         setAnnotatedScreenshot(null);
         setManualFocusBounds(null);
       } else {
@@ -240,8 +269,8 @@ export function ChatInterface({
         // Annotate screenshot with section highlight if parent section exists
         // This helps the LLM visually identify the target area for modifications
         screenshot = rawScreenshot;
-        if (rawScreenshot && visionFocusedElements.length > 0) {
-          const parentSection = visionFocusedElements[0].parentSection;
+        if (rawScreenshot && effectiveFocusedElements.length > 0) {
+          const parentSection = effectiveFocusedElements[0].parentSection;
           if (parentSection?.coordinates) {
             screenshot = await drawSectionHighlight(rawScreenshot, parentSection.coordinates);
             console.log("[Vision Mode] Added section highlight to screenshot:", {
@@ -254,7 +283,9 @@ export function ChatInterface({
 
       // Choose API endpoint based on mode
       const endpoint = useApplyFirst ? "/api/sonance-vision-apply" : "/api/sonance-vision-edit";
-      console.log("[Vision Mode] Sending to API:", endpoint);
+      console.log("[Vision Mode] Sending to API:", endpoint, {
+        effectiveFocusedElements: effectiveFocusedElements.length,
+      });
 
       const response = await fetch(endpoint, {
         method: "POST",
@@ -265,9 +296,13 @@ export function ChatInterface({
           screenshot,
           pageRoute: window.location.pathname,
           userPrompt: prompt,
-          focusedElements: visionFocusedElements,
+          // Use effective focused elements (from click OR from annotation discovery)
+          focusedElements: effectiveFocusedElements,
         }),
       });
+      
+      // Clear annotation discovered elements after API call
+      setAnnotationDiscoveredElements([]);
 
       const data = await response.json();
       console.log("[Vision Mode] API response:", {

package/dist/assets/dev-tools/components/ScreenshotAnnotator.tsx

@@ -4,6 +4,7 @@ import React, { useEffect, useState, useCallback } from "react";
 import { createPortal } from "react-dom";
 import { Check, X, RotateCcw, Crop } from "lucide-react";
 import html2canvas from "html2canvas-pro";
+import { VisionFocusedElement } from "../types";
 
 export interface Rectangle {
   x: number;
@@ -12,9 +13,157 @@ export interface Rectangle {
   height: number;
 }
 
+/** Discovered element info for scoring and ranking */
+interface DiscoveredElement {
+  element: Element;
+  score: number;
+  textContent: string;
+  className: string;
+  elementId: string;
+  tagName: string;
+  rect: DOMRect;
+}
+
+/**
+ * Discover DOM elements within the given rectangle bounds.
+ * Uses a grid sampling approach to find all elements in the area,
+ * then scores and ranks them for targeting accuracy.
+ */
+function discoverElementsInBounds(rect: Rectangle): VisionFocusedElement[] {
+  const discoveredMap = new Map<Element, DiscoveredElement>();
+  
+  // Tags to skip - generic containers and non-content elements
+  const skipTags = new Set(['html', 'body', 'head', 'script', 'style', 'meta', 'link', 'noscript']);
+  
+  // Semantic elements get bonus points
+  const semanticElements = new Set(['section', 'article', 'form', 'header', 'footer', 'main', 'nav', 'aside', 'dialog']);
+  
+  // Sample points in a grid pattern within the rectangle
+  const gridSize = 5; // 5x5 grid = 25 sample points
+  const stepX = rect.width / (gridSize + 1);
+  const stepY = rect.height / (gridSize + 1);
+  
+  for (let i = 1; i <= gridSize; i++) {
+    for (let j = 1; j <= gridSize; j++) {
+      const x = rect.x + stepX * i;
+      const y = rect.y + stepY * j;
+      
+      // Get all elements at this point (from top to bottom)
+      const elementsAtPoint = document.elementsFromPoint(x, y);
+      
+      for (const el of elementsAtPoint) {
+        // Skip if already processed
+        if (discoveredMap.has(el)) continue;
+        
+        // Skip DevTools elements
+        if (el.hasAttribute('data-sonance-devtools') || 
+            el.hasAttribute('data-annotator-overlay') ||
+            el.hasAttribute('data-annotator-toolbar') ||
+            el.hasAttribute('data-vision-mode-border')) {
+          continue;
+        }
+        
+        const tagName = el.tagName.toLowerCase();
+        
+        // Skip generic/non-content elements
+        if (skipTags.has(tagName)) continue;
+        
+        // Get element info
+        const elRect = el.getBoundingClientRect();
+        const id = el.id || '';
+        const className = el.className && typeof el.className === 'string' ? el.className : '';
+        
+        // Extract meaningful text content (not from children with their own text)
+        let textContent = '';
+        for (const node of el.childNodes) {
+          if (node.nodeType === Node.TEXT_NODE) {
+            const text = node.textContent?.trim();
+            if (text) textContent += text + ' ';
+          }
+        }
+        textContent = textContent.trim().substring(0, 100);
+        
+        // If no direct text, try to get visible text from element
+        if (!textContent && el instanceof HTMLElement) {
+          // For inputs, use placeholder or value
+          if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
+            textContent = el.placeholder || el.value || '';
+          } else if (el instanceof HTMLButtonElement || tagName === 'a') {
+            textContent = el.textContent?.trim().substring(0, 100) || '';
+          }
+        }
+        
+        // Calculate score for ranking
+        let score = 0;
+        
+        // ID is most valuable for targeting
+        if (id) score += 100;
+        
+        // Text content helps identify the element
+        if (textContent) score += 50;
+        
+        // Semantic elements are better targets
+        if (semanticElements.has(tagName)) score += 30;
+        
+        // Interactive elements are often targets
+        if (['button', 'a', 'input', 'select', 'textarea'].includes(tagName)) score += 25;
+        
+        // Component-like classNames (PascalCase patterns, not Tailwind utilities)
+        if (className) {
+          const classes = className.split(/\s+/);
+          const hasComponentClass = classes.some(c => 
+            /^[A-Z][a-zA-Z0-9]+/.test(c) || // PascalCase
+            /^[a-z]+-[a-z]+-/.test(c) // kebab-case with multiple segments (likely BEM)
+          );
+          if (hasComponentClass) score += 20;
+        }
+        
+        // Heading elements are important
+        if (['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tagName)) score += 20;
+        
+        // Penalize very large elements (probably containers)
+        if (elRect.width > window.innerWidth * 0.8 && elRect.height > window.innerHeight * 0.8) {
+          score -= 30;
+        }
+        
+        discoveredMap.set(el, {
+          element: el,
+          score,
+          textContent,
+          className,
+          elementId: id,
+          tagName,
+          rect: elRect,
+        });
+      }
+    }
+  }
+  
+  // Convert to array, sort by score descending, take top 10
+  const sorted = Array.from(discoveredMap.values())
+    .sort((a, b) => b.score - a.score)
+    .slice(0, 10);
+  
+  // Convert to VisionFocusedElement format
+  return sorted.map((item): VisionFocusedElement => ({
+    name: item.elementId || item.tagName,
+    type: 'component', // Generic type since we're discovering
+    coordinates: {
+      x: item.rect.left + window.scrollX,
+      y: item.rect.top + window.scrollY,
+      width: item.rect.width,
+      height: item.rect.height,
+    },
+    textContent: item.textContent || undefined,
+    className: item.className || undefined,
+    elementId: item.elementId || undefined,
+    description: `${item.tagName}${item.elementId ? '#' + item.elementId : ''}${item.textContent ? ': "' + item.textContent.substring(0, 30) + '"' : ''}`,
+  }));
+}
+
 interface ScreenshotAnnotatorProps {
-  /** Called when user confirms their selection with captured screenshot */
-  onConfirm: (annotatedScreenshot: string, bounds: Rectangle) => void;
+  /** Called when user confirms their selection with captured screenshot and discovered elements */
+  onConfirm: (annotatedScreenshot: string, bounds: Rectangle, discoveredElements: VisionFocusedElement[]) => void;
   /** Called when user cancels */
   onCancel: () => void;
 }
@@ -88,13 +237,26 @@ export function ScreenshotAnnotator({
     setStartPos(null);
   }, []);
 
-  // Confirm: capture screenshot and annotate it
+  // Confirm: discover elements in bounds, capture screenshot and annotate it
   const handleConfirm = useCallback(async () => {
     if (!currentRect || isCapturing) return;
     
     setIsCapturing(true);
 
     try {
+      // FIRST: Discover DOM elements within the drawn rectangle
+      // This must happen BEFORE the overlay is removed to get accurate results
+      console.log("[ScreenshotAnnotator] Discovering elements in bounds:", currentRect);
+      const discoveredElements = discoverElementsInBounds(currentRect);
+      console.log("[ScreenshotAnnotator] Discovered elements:", {
+        count: discoveredElements.length,
+        elements: discoveredElements.map(e => ({
+          name: e.name,
+          text: e.textContent?.substring(0, 30),
+          id: e.elementId,
+        })),
+      });
+
       // Capture the full page screenshot (excluding DevTools elements)
       const canvas = await html2canvas(document.body, {
         ignoreElements: (element) => {
@@ -140,7 +302,9 @@ export function ScreenshotAnnotator({
       ctx.fillText(labelText, labelX + labelPadding, labelY + 16);
 
       const annotatedScreenshot = canvas.toDataURL("image/png", 0.9);
-      onConfirm(annotatedScreenshot, currentRect);
+      
+      // Pass screenshot, bounds, AND discovered elements to callback
+      onConfirm(annotatedScreenshot, currentRect, discoveredElements);
     } catch (error) {
       console.error("Failed to capture screenshot:", error);
       setIsCapturing(false);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sonance-brand-mcp",
-  "version": "1.3.94",
+  "version": "1.3.95",
   "description": "MCP Server for Sonance Brand Guidelines and Component Library - gives Claude instant access to brand colors, typography, and UI components.",
   "main": "dist/index.js",
   "type": "module",