cct-cli 0.7.9.0__py3-none-any.whl

calc_terminal/vision/capture.py

@@ -0,0 +1,171 @@
+"""
+CAT v0.8.a — Screen Capture Engine.
+
+Manages getDisplayMedia() / getMedia() capture via the frontend.
+The server side handles config, state, and fallback logic.
+Actual pixel capture happens in the browser; the server receives
+base64-encoded frames via WebSocket or HTTP POST.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Dict, List, Optional
+
+
+class CaptureMode(str, Enum):
+    DISPLAY = "display"        # getDisplayMedia — full screen / window / tab
+    MEDIA = "media"            # getMedia with displaySurface hints
+    AUDIO_VIDEO = "audio_video"
+    AUDIO_ONLY = "audio_only"
+    TAB = "tab"                # Browser tab capture
+    WINDOW = "window"          # Specific window capture
+    NONE = "none"              # No capture available
+
+
+@dataclass
+class CaptureConfig:
+    mode: CaptureMode = CaptureMode.DISPLAY
+    fps: float = 1.0
+    quality: int = 85
+    max_frames: int = 500
+    width: Optional[int] = None
+    height: Optional[int] = None
+    auto_analyze: bool = True
+    change_threshold: float = 0.05
+    blur_detection: bool = True
+    max_session_minutes: int = 60
+    privacy_mode: bool = True
+    suppress_notifications: bool = True
+    suppress_system_audio: bool = False
+    cursor_capture: bool = True
+    annotations_enabled: bool = True
+    saved_frames_max: int = 20
+    context_window_frames: int = 5
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "mode": self.mode.value,
+            "fps": self.fps,
+            "quality": self.quality,
+            "max_frames": self.max_frames,
+            "width": self.width,
+            "height": self.height,
+            "auto_analyze": self.auto_analyze,
+            "change_threshold": self.change_threshold,
+            "blur_detection": self.blur_detection,
+            "max_session_minutes": self.max_session_minutes,
+            "privacy_mode": self.privacy_mode,
+            "suppress_notifications": self.suppress_notifications,
+            "cursor_capture": self.cursor_capture,
+            "annotations_enabled": self.annotations_enabled,
+        }
+
+
+class CaptureEngine:
+    """Server-side capture controller.
+
+    The actual screen capture runs in the browser via getDisplayMedia().
+    This engine:
+      1. Generates the frontend capture config/instructions
+      2. Processes incoming frames from the browser
+      3. Detects capture capability and falls back gracefully
+    """
+
+    def __init__(self, config: CaptureConfig):
+        self.config = config
+        self.paused = False
+        self._initialized = False
+        self._frame_callback: Optional[Callable] = None
+        self._stats = {
+            "frames_received": 0,
+            "frames_dropped": 0,
+            "bytes_received": 0,
+            "errors": 0,
+            "started_at": None,
+        }
+
+    async def initialize(self):
+        self._initialized = True
+        self._stats["started_at"] = time.time()
+
+    async def cleanup(self):
+        self._initialized = False
+        self._frame_callback = None
+
+    def get_capture_config(self) -> Dict[str, Any]:
+        """Return config dict for the frontend to initialize getDisplayMedia."""
+        display_surface = "monitor"
+        if self.config.mode == CaptureMode.TAB:
+            display_surface = "browser"
+        elif self.config.mode == CaptureMode.WINDOW:
+            display_surface = "window"
+
+        return {
+            "video": {
+                "displaySurface": display_surface,
+                "logicalSurface": True,
+                "cursor": "always" if self.config.cursor_capture else "never",
+                "noiseSuppression": not self.config.suppress_system_audio,
+                "suppressLocalAudioPlayback": self.config.suppress_system_audio,
+                "selfBrowserSurface": "include",
+                "systemAudio": "exclude" if self.config.suppress_system_audio else "include",
+                "preferCurrentTab": self.config.mode == CaptureMode.TAB,
+            },
+            "audio": self.config.mode in (
+                CaptureMode.AUDIO_VIDEO, CaptureMode.AUDIO_ONLY),
+            "videoConstraints": {
+                "width": {"ideal": self.config.width or 1920},
+                "height": {"ideal": self.config.height or 1080},
+                "frameRate": {"ideal": self.config.fps, "max": self.config.fps * 2},
+            },
+        }
+
+    def get_capability_report(self) -> Dict[str, Any]:
+        """Report what capture modes the frontend should attempt."""
+        return {
+            "display_api": True,
+            "media_api": True,
+            "audio_capture": self.config.mode in (
+                CaptureMode.AUDIO_VIDEO, CaptureMode.AUDIO_ONLY),
+            "tab_capture": True,
+            "window_capture": True,
+            "frame_rate": self.config.fps,
+            "max_resolution": f"{self.config.width or 1920}x{self.config.height or 1080}",
+            "browser_requirements": "getDisplayMedia() requires HTTPS or localhost",
+        }
+
+    def process_incoming_frame(self, frame_b64: str,
+                               timestamp: Optional[float] = None,
+                               metadata: Optional[Dict] = None) -> Dict[str, Any]:
+        """Process a base64 frame received from the browser."""
+        if self.paused:
+            self._stats["frames_dropped"] += 1
+            return {"processed": False, "reason": "paused"}
+
+        self._stats["frames_received"] += 1
+        self._stats["bytes_received"] += len(frame_b64)
+
+        return {
+            "processed": True,
+            "frame_index": self._stats["frames_received"],
+            "size_bytes": len(frame_b64),
+            "timestamp": timestamp or time.time(),
+            "metadata": metadata or {},
+        }
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        elapsed = 0.0
+        if self._stats["started_at"]:
+            elapsed = time.time() - self._stats["started_at"]
+        return {
+            **self._stats,
+            "elapsed_seconds": round(elapsed, 2),
+            "avg_fps": round(
+                self._stats["frames_received"] / max(elapsed, 0.01), 2
+            ),
+        }

calc_terminal/vision/context.py

@@ -0,0 +1,231 @@
+"""
+CAT v0.8.a — Vision Context Engine.
+
+Fuses screen frame + code context + project structure + chat history
+into a single VisualContext object for AI analysis.
+"""
+
+from __future__ import annotations
+
+import base64
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class VisualContext:
+    """Complete visual context for AI analysis."""
+    frame_b64: str
+    frame_meta: Dict[str, Any]
+    annotations: List[Dict[str, Any]]
+    code_context: Optional[Dict[str, Any]] = None
+    project_context: Optional[Dict[str, Any]] = None
+    chat_history: List[Dict[str, str]] = field(default_factory=list)
+    pointer_telemetry: Optional[Dict[str, Any]] = None
+    before_frame_b64: Optional[str] = None
+    before_meta: Optional[Dict[str, Any]] = None
+    analysis_prompt: str = ""
+    created_at: float = 0.0
+
+    def __post_init__(self):
+        if self.created_at == 0.0:
+            self.created_at = time.time()
+
+    def to_dict(self, include_frames: bool = False) -> Dict[str, Any]:
+        result = {
+            "frame_meta": self.frame_meta,
+            "annotations": self.annotations,
+            "code_context": self.code_context,
+            "project_context": self.project_context,
+            "chat_history": self.chat_history[-10:],
+            "pointer_telemetry": self.pointer_telemetry,
+            "analysis_prompt": self.analysis_prompt,
+            "created_at": self.created_at,
+            "has_frame": bool(self.frame_b64),
+            "has_before_frame": bool(self.before_frame_b64),
+        }
+        if include_frames:
+            result["frame_b64"] = self.frame_b64
+            result["before_frame_b64"] = self.before_frame_b64
+        return result
+
+    def to_ai_messages(self) -> List[Dict[str, Any]]:
+        """Build the message payload for the AI vision model."""
+        messages = []
+        system_parts = [
+            "You are CAT Vision Agent. Analyze the screen state and provide "
+            "actionable insights. Be specific about UI elements, errors, and "
+            "suggested fixes."
+        ]
+        if self.code_context:
+            system_parts.append(
+                f"\nActive file: {self.code_context.get('file_path', 'unknown')}"
+            )
+            if self.code_context.get("cursor_line"):
+                system_parts.append(
+                    f"Cursor at line: {self.code_context['cursor_line']}"
+                )
+        if self.project_context:
+            system_parts.append(
+                f"Project type: {self.project_context.get('type', 'unknown')}"
+            )
+        messages.append({"role": "system", "content": "\n".join(system_parts)})
+
+        user_parts = []
+        if self.analysis_prompt:
+            user_parts.append(self.analysis_prompt)
+        else:
+            user_parts.append(
+                "Analyze this screen capture. Describe what you see, identify "
+                "any issues, and suggest next steps."
+            )
+
+        if self.pointer_telemetry:
+            pt = self.pointer_telemetry
+            if pt.get("active_element"):
+                user_parts.append(
+                    f"\nFocused element: {pt['active_element']}"
+                )
+            if pt.get("cursor_moving"):
+                user_parts.append("\nUser is actively moving the cursor.")
+
+        if self.annotations:
+            ann_desc = []
+            for ann in self.annotations:
+                tool = ann.get("tool", "")
+                if tool == "text":
+                    ann_desc.append(f"Text annotation: {ann.get('text', '')}")
+                elif tool == "arrow":
+                    ann_desc.append("Arrow annotation")
+                elif tool == "highlight":
+                    ann_desc.append("Highlighted region")
+                else:
+                    ann_desc.append(f"{tool} annotation")
+            user_parts.append(
+                "\nUser annotations: " + "; ".join(ann_desc)
+            )
+
+        user_content = [
+            {"type": "text", "text": "\n".join(user_parts)}
+        ]
+
+        if self.frame_b64:
+            mime = "image/png"
+            user_content.append({
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:{mime};base64,{self.frame_b64}",
+                    "detail": "high",
+                }
+            })
+
+        if self.before_frame_b64:
+            user_content.append({
+                "type": "text",
+                "text": "\n--- BEFORE state (for comparison) ---"
+            })
+            user_content.append({
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:image/png;base64,{self.before_frame_b64}",
+                    "detail": "high",
+                }
+            })
+
+        messages.append({"role": "user", "content": user_content})
+        return messages
+
+
+class VisionContextEngine:
+    """Builds VisualContext from multiple sources."""
+
+    def __init__(self):
+        self._code_context: Optional[Dict[str, Any]] = None
+        self._project_context: Optional[Dict[str, Any]] = None
+        self._chat_history: List[Dict[str, str]] = []
+        self._max_chat = 20
+
+    def set_code_context(self, file_path: str = "", content: str = "",
+                         cursor_line: int = 0, language: str = "",
+                         diagnostics: Optional[List[str]] = None):
+        self._code_context = {
+            "file_path": file_path,
+            "content": content[:5000] if content else "",
+            "cursor_line": cursor_line,
+            "language": language,
+            "diagnostics": diagnostics or [],
+        }
+
+    def set_project_context(self, project_type: str = "",
+                            structure: Optional[Dict] = None,
+                            dependencies: Optional[List[str]] = None):
+        self._project_context = {
+            "type": project_type,
+            "structure": structure or {},
+            "dependencies": dependencies or [],
+        }
+
+    def add_chat_turn(self, role: str, content: str):
+        self._chat_history.append({"role": role, "content": content[:2000]})
+        if len(self._chat_history) > self._max_chat:
+            self._chat_history = self._chat_history[-self._max_chat:]
+
+    def build_context(self, frame_b64: str,
+                      frame_meta: Optional[Dict] = None,
+                      annotations: Optional[List[Dict]] = None,
+                      pointer_telemetry: Optional[Dict] = None,
+                      analysis_prompt: str = "",
+                      before_frame_b64: Optional[str] = None,
+                      before_meta: Optional[Dict] = None) -> VisualContext:
+        return VisualContext(
+            frame_b64=frame_b64,
+            frame_meta=frame_meta or {},
+            annotations=annotations or [],
+            code_context=self._code_context,
+            project_context=self._project_context,
+            chat_history=list(self._chat_history),
+            pointer_telemetry=pointer_telemetry,
+            analysis_prompt=analysis_prompt,
+            before_frame_b64=before_frame_b64,
+            before_meta=before_meta,
+        )
+
+    def build_visual_debug_prompt(self, error_message: str = "",
+                                  user_intent: str = "") -> str:
+        parts = [
+            "VISUAL DEBUG SESSION — Analyze the current screen state.",
+            "The user is experiencing an issue or wants to understand what's on screen.",
+        ]
+        if error_message:
+            parts.append(f"\nError reported: {error_message}")
+        if user_intent:
+            parts.append(f"\nUser intent: {user_intent}")
+        parts.append(
+            "\nProvide:\n"
+            "1. What you see on screen (UI elements, text, state)\n"
+            "2. Any errors, warnings, or anomalies\n"
+            "3. Root cause hypothesis\n"
+            "4. Specific fix suggestion with code if applicable\n"
+            "5. Before/after comparison if previous frame is available"
+        )
+        return "\n".join(parts)
+
+    def build_code_review_prompt(self, file_path: str = "",
+                                 code: str = "") -> str:
+        parts = [
+            "CODE-VISUAL REVIEW — The screen shows the current code state.",
+            "Review the visible code and provide feedback.",
+        ]
+        if file_path:
+            parts.append(f"\nFile: {file_path}")
+        if code:
+            parts.append(f"\nCode preview:\n```\n{code[:3000]}\n```")
+        parts.append(
+            "\nAnalyze:\n"
+            "1. Code quality and correctness\n"
+            "2. Visual rendering issues (if web app)\n"
+            "3. UI/UX observations\n"
+            "4. Suggested improvements"
+        )
+        return "\n".join(parts)

calc_terminal/vision/correlation.py

@@ -0,0 +1,169 @@
+"""
+CAT v0.8.a — Screen → Code Correlation (spec 20).
+
+Maps a visual annotation/cursor region to likely project files by:
+  - DOM/preview context (if live preview is running)
+  - filename / component heuristics
+  - workspace search for component names near the annotation label
+
+Never hallucinates an exact source location from pixels alone.
+Returns a ranked list with confidence + evidence.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+
+def _norm_annotation_label(ann: Dict[str, Any]) -> str:
+    return (ann.get("text") or ann.get("label") or ann.get("description") or "").strip()
+
+
+def _guess_component_from_label(label: str) -> List[str]:
+    """Extract likely component/file hints from a short label like 'Submit button glitches'."""
+    if not label:
+        return []
+    words = re.findall(r"[A-Za-z][A-Za-z0-9_-]*", label)
+    # keep capitalized or component-like words, also "button", "form", "submit"
+    hints = []
+    for w in words:
+        lw = w.lower()
+        if len(w) >= 3 and lw not in {"this", "that", "when", "with", "glitches", "broken", "jumps"}:
+            hints.append(w)
+    return hints[:8]
+
+
+def _search_workspace(hints: List[str], root: str, max_hits: int = 12) -> List[Dict[str, Any]]:
+    """Cheap filename + grep search. Returns [{path, reason, score}]."""
+    if not root or not os.path.isdir(root):
+        return []
+    root_p = Path(root)
+    results: List[Dict[str, Any]] = []
+    # filename scan
+    all_files = []
+    for p in root_p.rglob("*"):
+        if p.is_file() and p.suffix.lower() in {".tsx",".ts",".jsx",".js",".py",".html",".css",".vue",".svelte"}:
+            if any(part.startswith(".") or part == "node_modules" or part == "__pycache__" for part in p.parts):
+                continue
+            all_files.append(p)
+            if len(all_files) > 8000:
+                break
+    for hint in hints:
+        hl = hint.lower()
+        for f in all_files:
+            name_l = f.name.lower()
+            if hl in name_l or hl.replace("-","") in name_l.replace("-",""):
+                results.append({"path": str(f.relative_to(root_p)), "reason": f"filename contains '{hint}'", "score": 0.7})
+                if len(results) >= max_hits:
+                    return results
+    # content grep for component names (cheap, capped)
+    for hint in hints[:3]:
+        hl = hint.lower()
+        if len(hl) < 3:
+            continue
+        for f in all_files[:1200]:
+            try:
+                text = f.read_text(encoding="utf-8", errors="ignore")[:20000]
+            except Exception:
+                continue
+            if hl in text.lower():
+                # boost if it's a component definition
+                score = 0.6
+                if re.search(rf"(class|function|const)\s+{re.escape(hint)}\b", text):
+                    score = 0.85
+                elif re.search(rf"<{re.escape(hint)}\b", text):
+                    score = 0.75
+                results.append({"path": str(f.relative_to(root_p)), "reason": f"mentions '{hint}'", "score": score})
+                if len(results) >= max_hits:
+                    return results
+    # dedupe, keep highest score per path
+    best: Dict[str, Dict] = {}
+    for r in results:
+        prev = best.get(r["path"])
+        if prev is None or r["score"] > prev["score"]:
+            best[r["path"]] = r
+    ranked = sorted(best.values(), key=lambda r: r["score"], reverse=True)
+    return ranked[:max_hits]
+
+
+def correlate(
+    annotations: List[Dict[str, Any]],
+    cursor: Optional[Dict[str, Any]] = None,
+    preview_context: Optional[Dict[str, Any]] = None,
+    workspace_root: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Returns {
+      "has_mapping": bool,
+      "candidates": [{path, reason, score}],
+      "preview_hint": str | None,
+      "notes": str  # honest about uncertainty
+    }
+    """
+    # preview / DOM hint is strongest — when live preview controller knows the entry URL + component
+    preview_hint = None
+    candidates: List[Dict[str, Any]] = []
+    notes_parts: List[str] = []
+
+    if preview_context:
+        # preview_context may contain entry file / component map produced by browser/preview.py
+        entry = preview_context.get("entry") or preview_context.get("url") or ""
+        if entry:
+            preview_hint = entry
+            candidates.append({"path": entry, "reason": "live preview entry", "score": 0.65})
+        dom = preview_context.get("dom_match") or preview_context.get("selector") or ""
+        if dom:
+            notes_parts.append(f"DOM context: {dom}")
+
+    # annotation labels → hints → workspace search
+    all_hints: List[str] = []
+    for ann in (annotations or []):
+        label = _norm_annotation_label(ann)
+        hints = _guess_component_from_label(label)
+        tool = ann.get("tool", "")
+        if tool:
+            hints.append(tool)
+        all_hints.extend(hints)
+
+    # also from cursor active_element
+    if cursor and cursor.get("active_element"):
+        all_hints.append(str(cursor["active_element"])[:40])
+
+    # dedupe hints
+    seen = set()
+    uniq_hints: List[str] = []
+    for h in all_hints:
+        hl = h.lower()
+        if hl not in seen:
+            seen.add(hl)
+            uniq_hints.append(h)
+
+    if uniq_hints and workspace_root:
+        hits = _search_workspace(uniq_hints, workspace_root, max_hits=10)
+        candidates.extend(hits)
+
+    # sort + dedupe candidates
+    best: Dict[str, Dict] = {}
+    for c in candidates:
+        prev = best.get(c["path"])
+        if prev is None or c["score"] > prev["score"]:
+            best[c["path"]] = c
+    ranked = sorted(best.values(), key=lambda c: c["score"], reverse=True)
+
+    has_mapping = bool(ranked and ranked[0]["score"] >= 0.6)
+    if not ranked:
+        notes_parts.append("No code mapping could be inferred from the current visual context.")
+        notes_parts.append("Try adding a short annotation label (e.g. 'Submit button') or opening the preview so CAT can see the component tree.")
+    elif not has_mapping:
+        notes_parts.append("Weak mapping only — candidates are guesses, not exact source locations.")
+
+    return {
+        "has_mapping": has_mapping,
+        "candidates": ranked[:8],
+        "preview_hint": preview_hint,
+        "hints_used": uniq_hints[:10],
+        "notes": " ".join(notes_parts),
+    }