cct-cli 0.7.9.0__py3-none-any.whl

calc_terminal/vision/__init__.py

@@ -0,0 +1,89 @@
+"""
+CAT v0.8.a Vision Agent — screen capture, frame pipeline, annotation,
+visual context, and before/after verification.
+
+Usage (server):
+    from calc_terminal.vision.session import VisionSessionManager
+    from calc_terminal.vision.context import VisionContextEngine
+    from calc_terminal.vision.verify import BeforeAfterVerifier
+
+Usage (frontend JS calls /api/vision/* endpoints on the server).
+"""
+
+__version__ = "0.8.a"
+
+from .session import VisionSessionManager, VisionSession, SessionState
+from .capture import CaptureEngine, CaptureMode, CaptureConfig
+from .frame_pipeline import FramePipeline, FrameMeta, ChangeLevel
+from .annotations import AnnotationEngine, Annotation, AnnotationTool
+from .context import VisionContextEngine, VisualContext
+from .verify import BeforeAfterVerifier, VerificationResult
+from .safety import PrivacyController, SafetyConfig
+from .provider import VisionProviderRouter
+from .cursor import CursorTracker, PointerEvent, DwellRegion
+from .priority import FramePriorityEngine, PriorityLevel
+from . import events as vision_events
+
+__all__ = [
+    "VisionSessionManager", "VisionSession", "SessionState",
+    "CaptureEngine", "CaptureMode", "CaptureConfig",
+    "FramePipeline", "FrameMeta", "ChangeLevel",
+    "AnnotationEngine", "Annotation", "AnnotationTool",
+    "VisionContextEngine", "VisualContext",
+    "BeforeAfterVerifier", "VerificationResult",
+    "PrivacyController", "SafetyConfig",
+    "VisionProviderRouter",
+    "CursorTracker", "PointerEvent", "DwellRegion",
+    "FramePriorityEngine", "PriorityLevel",
+    "vision_events",
+]
+
+# ---------------------------------------------------------------------------
+# Legacy image-pipeline compatibility (calc_terminal/vision.py)
+# The file calc_terminal/vision.py (image validation/normalization +
+# vision-capable model routing) and the package calc_terminal/vision/
+# (screen-capture agent) share the import name.  Python prefers the
+# package directory, so `import calc_terminal.vision` would hide the file.
+# Re-expose the file's public symbols here so `from calc_terminal import
+# vision` and `vision.pil_available()` keep working without a rename.
+try:
+    import importlib.util as _ilu
+    import pathlib as _pl
+    _vision_file = _pl.Path(__file__).with_name("vision.py")
+    # When this package is loaded as `calc_terminal.vision`, the file is
+    # `calc_terminal/vision.py`'s sibling (one directory up).
+    if not _vision_file.exists():
+        _vision_file = _pl.Path(__file__).parent.parent / "vision.py"
+        # fallback: the classic location is calc_terminal/vision.py next
+        # to the package directory.
+        if not _vision_file.exists():
+            _vision_file = _pl.Path(__file__).parent / "vision.py"
+    if _vision_file.exists():
+        _spec = _ilu.spec_from_file_location(
+            "calc_terminal._vision_image", str(_vision_file))
+        _mod = _ilu.module_from_spec(_spec)
+        _spec.loader.exec_module(_mod)  # type: ignore[union-attr]
+        # Re-export the image pipeline API
+        for _name in (
+            "MAX_DIMENSION", "MAX_BYTES", "JPEG_QUALITY",
+            "PROVIDER_NATIVE", "ALL_IMAGE_EXTS",
+            "pil_available", "ocr_available", "ImageError",
+            "validate", "classify_image", "inspect",
+            "normalize", "encode_for_model",
+            "analysis_prompt", "BASE_ANALYSIS_PROMPT",
+            "run_local_ocr", "find_vision_config", "analyze",
+        ):
+            if hasattr(_mod, _name):
+                globals()[_name] = getattr(_mod, _name)
+        # Backward-compat: `vision.vision` self-reference (some probes do
+        # `from calc_terminal import vision; vision.vision`?)
+        __all__.extend([n for n in (
+            "pil_available", "ocr_available", "ImageError",
+            "validate", "classify_image", "inspect",
+            "normalize", "encode_for_model",
+            "analysis_prompt", "BASE_ANALYSIS_PROMPT",
+            "run_local_ocr", "find_vision_config", "analyze",
+            "MAX_DIMENSION", "MAX_BYTES",
+        ) if n not in __all__])
+except Exception:
+    pass

calc_terminal/vision/analysis.py

@@ -0,0 +1,194 @@
+"""
+CAT v0.8.a — AI Analysis Modes (spec 19).
+
+Three internal levels:
+  QUICK  — low latency, current frame + current annotation + recent interaction
+  DEEP   — selected frame(s) + annotations + cursor + project + chat
+  DEBUG  — full pipeline: vision → identify UI region → locate code → propose fix → apply → verify
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class AnalysisLevel(str, Enum):
+    QUICK = "quick"
+    DEEP = "deep"
+    DEBUG = "debug"
+
+
+@dataclass
+class AnalysisRequest:
+    level: AnalysisLevel
+    session_id: str
+    prompt: str = ""
+    include_code: bool = False
+    include_before_after: bool = False
+    error_message: str = ""
+    user_intent: str = ""
+
+
+@dataclass
+class AnalysisResult:
+    level: AnalysisLevel
+    analysis: str
+    frame_index: Optional[int] = None
+    confidence: float = 0.0
+    findings: List[str] = field(default_factory=list)
+    suggested_fix: Optional[str] = None
+    related_files: List[str] = field(default_factory=list)
+    duration_ms: float = 0.0
+    provider: str = ""
+    model: str = ""
+
+
+def build_system_prompt(level: AnalysisLevel) -> str:
+    if level == AnalysisLevel.QUICK:
+        return (
+            "You are CAT Vision — QUICK mode. Low latency.\n"
+            "Analyze the current frame + current annotation + recent cursor context.\n"
+            "Be concise, point to the immediate issue, suggest one next step."
+        )
+    if level == AnalysisLevel.DEEP:
+        return (
+            "You are CAT Vision — DEEP analysis. Thorough reasoning.\n"
+            "Analyze selected frame(s), annotations, cursor context, project/code context, and chat history.\n"
+            "Explain what you see, root cause, and a specific fix with file/function references when possible."
+        )
+    # DEBUG
+    return (
+        "You are CAT Vision — DEBUG mode. Full engineering workflow.\n"
+        "Pipeline: Vision → Identify UI region → Determine likely behavior → Inspect project\n"
+        "→ Locate related code → Reproduce where possible → Propose fix → Wait for permission → Verify.\n"
+        "Correlate the visual region with project code. If exact mapping is unavailable, say so — never hallucinate file locations.\n"
+        "Propose a concrete code change and flag the permission gate."
+    )
+
+
+def build_user_prompt(req: AnalysisRequest, ctx: Dict[str, Any]) -> str:
+    parts: List[str] = []
+    if req.level == AnalysisLevel.QUICK:
+        parts.append(req.prompt or "Quick analyze: what is visible and what should the user check?")
+        if ctx.get("annotation_summary"):
+            parts.append(f"Annotation: {ctx['annotation_summary']}")
+        if ctx.get("cursor_summary"):
+            parts.append(f"Cursor: {ctx['cursor_summary']}")
+    elif req.level == AnalysisLevel.DEEP:
+        parts.append(req.prompt or "Deep analyze this screen. What is wrong and how to fix it?")
+        if req.error_message:
+            parts.append(f"Error reported: {req.error_message}")
+        if req.user_intent:
+            parts.append(f"User intent: {req.user_intent}")
+        if ctx.get("annotation_summary"):
+            parts.append(f"Annotations: {ctx['annotation_summary']}")
+        if ctx.get("cursor_summary"):
+            parts.append(f"Cursor context: {ctx['cursor_summary']}")
+        if ctx.get("project_hint"):
+            parts.append(f"Project: {ctx['project_hint']}")
+    else:  # DEBUG
+        parts.append("VISUAL DEBUG SESSION — full pipeline.")
+        if req.prompt:
+            parts.append(f"User: {req.prompt}")
+        if req.error_message:
+            parts.append(f"Error: {req.error_message}")
+        if req.user_intent:
+            parts.append(f"Intent: {req.user_intent}")
+        parts.append(
+            "Provide:\n"
+            "1. What you see (UI elements, text, state)\n"
+            "2. Errors/warnings/anomalies\n"
+            "3. Root cause hypothesis with code reference if locatable\n"
+            "4. Specific fix suggestion with code\n"
+            "5. Observation: say when exact source mapping is unavailable\n"
+            "6. Do NOT apply fixes yourself — propose and wait for permission"
+        )
+    return "\n\n".join(parts)
+
+
+def run_analysis(req: AnalysisRequest, visual_ctx, config: Optional[Dict] = None) -> AnalysisResult:
+    """Run one analysis level against a VisualContext (vision/context.py)."""
+    t0 = time.time()
+    system_prompt = build_system_prompt(req.level)
+    # build minimal ctx hints from visual_ctx
+    ctx_hints: Dict[str, Any] = {}
+    try:
+        if visual_ctx.annotations:
+            ctx_hints["annotation_summary"] = "; ".join(
+                f"{a.get('tool','?')}" for a in visual_ctx.annotations[:5]
+            )
+        if visual_ctx.pointer_telemetry:
+            pt = visual_ctx.pointer_telemetry
+            ctx_hints["cursor_summary"] = f"active={pt.get('active_element','')} moving={pt.get('cursor_moving',False)}"
+        if visual_ctx.project_context:
+            ctx_hints["project_hint"] = visual_ctx.project_context.get("type", "")
+    except Exception:
+        pass
+    user_prompt = build_user_prompt(req, ctx_hints)
+    # enrich visual_ctx with our prompts before calling provider
+    visual_ctx.analysis_prompt = user_prompt
+    # route
+    try:
+        from .provider import VisionProviderRouter
+        from .. import aicore
+        from ..attachments import Attachment
+        import base64
+        router = VisionProviderRouter(config)
+        cfg, caps = router.find_vision_capable(config) or (None, None)
+        if cfg is None:
+            return AnalysisResult(level=req.level, analysis="No vision-capable model configured. Run /model to choose one.", duration_ms=(time.time()-t0)*1000)
+        # Build attachment from frame_b64
+        att = Attachment.__new__(Attachment)
+        att.id = f"att-vision-{req.level.value}-{int(time.time()*1000)%1000000}"
+        att.name = "vision_frame"
+        att.path = ""
+        att.extension = ".png"
+        att.mime_type = "image/png"
+        att.size = len(visual_ctx.frame_b64 or "")
+        att.kind = "image"
+        att.content = "[vision frame]"
+        att.metadata = {"inline_b64": visual_ctx.frame_b64 or ""}
+        att.extraction_status = "ready"
+        att.error = None
+        att.created_at = time.time()
+        att.source = "vision_analysis"
+        # Attachments: frame + optional before frame
+        attachments = [att]
+        if req.include_before_after and visual_ctx.before_frame_b64:
+            att2 = Attachment.__new__(Attachment)
+            att2.id = "att-vision-before"
+            att2.name = "before_frame"
+            att2.path = ""
+            att2.extension = ".png"
+            att2.mime_type = "image/png"
+            att2.size = len(visual_ctx.before_frame_b64)
+            att2.kind = "image"
+            att2.content = "[before frame]"
+            att2.metadata = {"inline_b64": visual_ctx.before_frame_b64}
+            att2.extraction_status = "ready"
+            att2.error = None
+            att2.created_at = time.time()
+            att2.source = "vision_verify"
+            attachments.append(att2)
+        analysis_text = aicore.query_ai(
+            user_prompt,
+            system_prompt=system_prompt,
+            history=None,
+            config=cfg,
+            attachments=attachments,
+            size_class="large" if req.level != AnalysisLevel.QUICK else "normal",
+        )
+        return AnalysisResult(
+            level=req.level,
+            analysis=analysis_text or "",
+            frame_index=None,
+            confidence=0.7,
+            duration_ms=(time.time()-t0)*1000,
+            provider=cfg.get("provider",""),
+            model=cfg.get("model",""),
+        )
+    except Exception as e:
+        return AnalysisResult(level=req.level, analysis=f"Analysis failed: {e}", duration_ms=(time.time()-t0)*1000)

calc_terminal/vision/annotations.py

@@ -0,0 +1,297 @@
+"""
+CAT v0.8.a — Annotation Engine.
+
+Handles pen, arrow, rect, circle, text, and highlight annotations.
+Manages undo/redo stack, serialization, and frame annotation binding.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class AnnotationTool(str, Enum):
+    PEN = "pen"
+    ARROW = "arrow"
+    RECT = "rect"
+    CIRCLE = "circle"
+    TEXT = "text"
+    HIGHLIGHT = "highlight"
+    CROSSHAIR = "crosshair"
+    ERASER = "eraser"
+
+
+@dataclass
+class Color:
+    r: int = 255
+    g: int = 0
+    b: int = 0
+    a: float = 1.0
+
+    def to_css(self) -> str:
+        return f"rgba({self.r},{self.g},{self.b},{self.a})"
+
+    def to_hex(self) -> str:
+        return f"#{self.r:02x}{self.g:02x}{self.b:02x}"
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"r": self.r, "g": self.g, "b": self.b, "a": self.a}
+
+    @classmethod
+    def from_hex(cls, hex_str: str) -> "Color":
+        h = hex_str.lstrip("#")
+        if len(h) == 3:
+            h = "".join(c * 2 for c in h)
+        if len(h) == 6:
+            return cls(int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16))
+        elif len(h) == 8:
+            return cls(
+                int(h[0:2], 16), int(h[2:4], 16),
+                int(h[4:6], 16), int(h[6:8], 16) / 255.0
+            )
+        return cls()
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> "Color":
+        return cls(d.get("r", 255), d.get("g", 0), d.get("b", 0), d.get("a", 1.0))
+
+
+@dataclass
+class Annotation:
+    id: str
+    tool: AnnotationTool
+    points: List[Tuple[float, float]]
+    color: Color
+    line_width: float
+    text: str = ""
+    text_position: Optional[Tuple[float, float]] = None
+    opacity: float = 1.0
+    created_at: float = 0.0
+    frame_index: int = -1
+    bounding_box: Optional[Dict[str, float]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "tool": self.tool.value,
+            "points": [[p[0], p[1]] for p in self.points],
+            "color": self.color.to_dict(),
+            "line_width": self.line_width,
+            "text": self.text,
+            "text_position": list(self.text_position) if self.text_position else None,
+            "opacity": self.opacity,
+            "created_at": self.created_at,
+            "frame_index": self.frame_index,
+            "bounding_box": self.bounding_box,
+        }
+
+    @classmethod
+    def from_dict(cls, d: Dict) -> "Annotation":
+        return cls(
+            id=d.get("id", str(uuid.uuid4())),
+            tool=AnnotationTool(d.get("tool", "pen")),
+            points=[(p[0], p[1]) for p in d.get("points", [])],
+            color=Color.from_dict(d.get("color", {})),
+            line_width=d.get("line_width", 3),
+            text=d.get("text", ""),
+            text_position=tuple(d["text_position"]) if d.get("text_position") else None,
+            opacity=d.get("opacity", 1.0),
+            created_at=d.get("created_at", 0.0),
+            frame_index=d.get("frame_index", -1),
+            bounding_box=d.get("bounding_box"),
+        )
+
+
+class AnnotationEngine:
+    """Manages annotation creation, undo/redo, and serialization."""
+
+    def __init__(self):
+        self.annotations: List[Annotation] = []
+        self._undo_stack: List[Annotation] = []
+        self._redo_stack: List[Annotation] = []
+        self._max_undo = 50
+        self._default_color = Color(255, 0, 0)
+        self._default_width = 3.0
+
+    def set_default_color(self, color: Color):
+        self._default_color = color
+
+    def set_default_width(self, width: float):
+        self._default_width = width
+
+    def add_annotation(self, tool: AnnotationTool,
+                       points: List[Tuple[float, float]],
+                       color: Optional[Color] = None,
+                       line_width: Optional[float] = None,
+                       text: str = "",
+                       frame_index: int = -1,
+                       annotation_id: Optional[str] = None) -> Annotation:
+        ann = Annotation(
+            id=annotation_id or str(uuid.uuid4()),
+            tool=tool,
+            points=points,
+            color=color or self._default_color,
+            line_width=line_width or self._default_width,
+            text=text,
+            text_position=points[0] if points else None,
+            created_at=time.time(),
+            frame_index=frame_index,
+            bounding_box=self._compute_bbox(points),
+        )
+        self.annotations.append(ann)
+        self._undo_stack.append(ann)
+        if len(self._undo_stack) > self._max_undo:
+            self._undo_stack.pop(0)
+        self._redo_stack.clear()
+        return ann
+
+    def undo(self) -> Optional[Annotation]:
+        if not self._undo_stack:
+            return None
+        ann = self._undo_stack.pop()
+        self.annotations = [a for a in self.annotations if a.id != ann.id]
+        self._redo_stack.append(ann)
+        return ann
+
+    def redo(self) -> Optional[Annotation]:
+        if not self._redo_stack:
+            return None
+        ann = self._redo_stack.pop()
+        self.annotations.append(ann)
+        self._undo_stack.append(ann)
+        return ann
+
+    def clear(self):
+        self.annotations.clear()
+        self._undo_stack.clear()
+        self._redo_stack.clear()
+
+    def clear_frame(self, frame_index: int):
+        self.annotations = [
+            a for a in self.annotations if a.frame_index != frame_index
+        ]
+
+    def update_annotation(self, annotation_id: str, **kwargs) -> Optional[Annotation]:
+        for ann in self.annotations:
+            if ann.id == annotation_id:
+                for key, val in kwargs.items():
+                    if key == "color" and isinstance(val, dict):
+                        ann.color = Color.from_dict(val)
+                    elif key == "points" and isinstance(val, list):
+                        ann.points = [(p[0], p[1]) for p in val]
+                        ann.bounding_box = self._compute_bbox(ann.points)
+                    elif hasattr(ann, key):
+                        setattr(ann, key, val)
+                return ann
+        return None
+
+    def delete_annotation(self, annotation_id: str) -> bool:
+        before = len(self.annotations)
+        self.annotations = [a for a in self.annotations if a.id != annotation_id]
+        return len(self.annotations) < before
+
+    def get_annotations_for_frame(self, frame_index: int) -> List[Dict]:
+        return [a.to_dict() for a in self.annotations if a.frame_index == frame_index]
+
+    def get_pointer_annotation(self, x: float, y: float,
+                               radius: float = 15.0) -> Optional[Annotation]:
+        """Find annotation near a pointer position."""
+        for ann in reversed(self.annotations):
+            if ann.tool == AnnotationTool.CROSSHAIR:
+                continue
+            if ann.bounding_box:
+                bb = ann.bounding_box
+                if (bb["x"] - radius <= x <= bb["x"] + bb["width"] + radius and
+                        bb["y"] - radius <= y <= bb["y"] + bb["height"] + radius):
+                    return ann
+        return None
+
+    def to_list(self) -> List[Dict]:
+        return [a.to_dict() for a in self.annotations]
+
+    def from_list(self, data: List[Dict]):
+        self.clear()
+        for d in data:
+            self.annotations.append(Annotation.from_dict(d))
+        self._undo_stack = list(self.annotations)
+        self._redo_stack.clear()
+
+    def _compute_bbox(self, points: List[Tuple[float, float]]) -> Optional[Dict[str, float]]:
+        if not points:
+            return None
+        xs = [p[0] for p in points]
+        ys = [p[1] for p in points]
+        return {
+            "x": min(xs), "y": min(ys),
+            "width": max(xs) - min(xs),
+            "height": max(ys) - min(ys),
+        }
+
+    def export_svg(self, width: int = 1920, height: int = 1080) -> str:
+        lines = [
+            f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}">',
+        ]
+        for ann in self.annotations:
+            color_hex = ann.color.to_hex()
+            opacity = ann.opacity
+            if ann.tool == AnnotationTool.PEN and len(ann.points) >= 2:
+                path_d = "M " + " L ".join(
+                    f"{p[0]:.1f},{p[1]:.1f}" for p in ann.points
+                )
+                lines.append(
+                    f'  <path d="{path_d}" stroke="{color_hex}" '
+                    f'stroke-width="{ann.line_width}" fill="none" '
+                    f'opacity="{opacity}" stroke-linecap="round"/>'
+                )
+            elif ann.tool == AnnotationTool.ARROW and len(ann.points) >= 2:
+                x1, y1 = ann.points[0]
+                x2, y2 = ann.points[-1]
+                lines.append(
+                    f'  <line x1="{x1:.1f}" y1="{y1:.1f}" x2="{x2:.1f}" y2="{y2:.1f}" '
+                    f'stroke="{color_hex}" stroke-width="{ann.line_width}" '
+                    f'opacity="{opacity}" marker-end="url(#arrowhead)"/>'
+                )
+            elif ann.tool == AnnotationTool.RECT and len(ann.points) >= 2:
+                x1, y1 = ann.points[0]
+                x2, y2 = ann.points[-1]
+                lines.append(
+                    f'  <rect x="{min(x1,x2):.1f}" y="{min(y1,y2):.1f}" '
+                    f'width="{abs(x2-x1):.1f}" height="{abs(y2-y1):.1f}" '
+                    f'stroke="{color_hex}" stroke-width="{ann.line_width}" '
+                    f'fill="none" opacity="{opacity}"/>'
+                )
+            elif ann.tool == AnnotationTool.CIRCLE and len(ann.points) >= 2:
+                x1, y1 = ann.points[0]
+                x2, y2 = ann.points[-1]
+                cx = (x1 + x2) / 2
+                cy = (y1 + y2) / 2
+                rx = abs(x2 - x1) / 2
+                ry = abs(y2 - y1) / 2
+                lines.append(
+                    f'  <ellipse cx="{cx:.1f}" cy="{cy:.1f}" rx="{rx:.1f}" ry="{ry:.1f}" '
+                    f'stroke="{color_hex}" stroke-width="{ann.line_width}" '
+                    f'fill="none" opacity="{opacity}"/>'
+                )
+            elif ann.tool == AnnotationTool.TEXT and ann.text_position:
+                tx, ty = ann.text_position
+                escaped = ann.text.replace("&", "&amp;").replace("<", "&lt;")
+                lines.append(
+                    f'  <text x="{tx:.1f}" y="{ty:.1f}" fill="{color_hex}" '
+                    f'font-size="{ann.line_width * 5:.0f}" opacity="{opacity}">'
+                    f'{escaped}</text>'
+                )
+            elif ann.tool == AnnotationTool.HIGHLIGHT and len(ann.points) >= 2:
+                x1, y1 = ann.points[0]
+                x2, y2 = ann.points[-1]
+                highlight_color = Color(ann.color.r, ann.color.g, ann.color.b, 0.3)
+                lines.append(
+                    f'  <rect x="{min(x1,x2):.1f}" y="{min(y1,y2):.1f}" '
+                    f'width="{abs(x2-x1):.1f}" height="{abs(y2-y1):.1f}" '
+                    f'fill="{highlight_color.to_hex()}" opacity="{opacity}"/>'
+                )
+        lines.append("</svg>")
+        return "\n".join(lines)