cct-cli 0.7.9.0__py3-none-any.whl

calc_terminal/vision/cursor.py

@@ -0,0 +1,258 @@
+"""
+CAT v0.8.a — Cursor / Pointer Tracking & Context Inference.
+
+Records Pointer Events (mouse/pen/touch) with throttling, dwell detection,
+and probabilistic attention inference — never "mind reading".
+
+Spec 10/11: pointer metadata + Cursor Context Inference.
+"""
+
+from __future__ import annotations
+
+import math
+import time
+from dataclasses import dataclass, field, asdict
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class PointerType(str, Enum):
+    MOUSE = "mouse"
+    PEN = "pen"
+    TOUCH = "touch"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class PointerEvent:
+    timestamp: float
+    x: float  # normalized 0.0-1.0
+    y: float  # normalized 0.0-1.0
+    pointer_id: int = 0
+    pointer_type: str = "mouse"
+    button: int = 0
+    pressure: float = 0.5
+    movement_x: float = 0.0
+    movement_y: float = 0.0
+    is_click: bool = False
+    is_double_click: bool = False
+    is_drag: bool = False
+    is_scroll: bool = False
+    scroll_delta: float = 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "x": round(self.x, 4),
+            "y": round(self.y, 4),
+            "pointerId": self.pointer_id,
+            "pointerType": self.pointer_type,
+            "button": self.button,
+            "pressure": round(self.pressure, 3),
+            "movement": [round(self.movement_x, 4), round(self.movement_y, 4)],
+            "click": self.is_click,
+            "doubleClick": self.is_double_click,
+            "drag": self.is_drag,
+            "scroll": self.is_scroll,
+        }
+
+
+@dataclass
+class DwellRegion:
+    x: float
+    y: float
+    radius: float
+    dwell_ms: float
+    event_count: int
+    started_at: float
+    ended_at: float
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "x": round(self.x, 4),
+            "y": round(self.y, 4),
+            "radius": round(self.radius, 4),
+            "dwell_ms": round(self.dwell_ms, 1),
+            "event_count": self.event_count,
+            "started_at": self.started_at,
+            "ended_at": self.ended_at,
+        }
+
+
+class CursorTracker:
+    """
+    Throttled pointer tracking + dwell detection + attention inference.
+
+    - Throttle: coalesce high-rate pointermove (16ms min interval)
+    - Dwell: cursor stays within radius for >= dwell_threshold_ms
+    - Inference: probabilistic observations, never mind-reading
+    """
+
+    def __init__(self,
+                 throttle_ms: float = 16.0,
+                 dwell_threshold_ms: float = 800.0,
+                 dwell_radius: float = 0.03,
+                 max_events: int = 2000):
+        self.throttle_ms = throttle_ms
+        self.dwell_threshold_ms = dwell_threshold_ms
+        self.dwell_radius = dwell_radius
+        self.max_events = max_events
+
+        self._events: List[PointerEvent] = []
+        self._last_emit_time: float = 0.0
+        self._last_pos: Tuple[float, float] = (0.0, 0.0)
+        self._dwell_start: Optional[float] = None
+        self._dwell_center: Tuple[float, float] = (0.0, 0.0)
+        self._dwells: List[DwellRegion] = []
+        self._click_count: int = 0
+        self._scroll_total: float = 0.0
+
+    # -- ingestion --
+
+    def add_event(self, evt: PointerEvent) -> bool:
+        """Add event with throttling. Returns True if stored."""
+        now = evt.timestamp or time.time()
+        # throttle pointermove without click/drag
+        is_move_only = not (evt.is_click or evt.is_drag or evt.is_scroll or evt.is_double_click)
+        if is_move_only and self._events:
+            elapsed_ms = (now - self._last_emit_time) * 1000
+            if elapsed_ms < self.throttle_ms:
+                # coalesce: update last move position instead of appending
+                if self._events and not self._events[-1].is_click:
+                    self._events[-1].x = evt.x
+                    self._events[-1].y = evt.y
+                    self._events[-1].timestamp = now
+                    self._events[-1].movement_x = evt.movement_x
+                    self._events[-1].movement_y = evt.movement_y
+                    self._update_dwell(evt.x, evt.y, now)
+                    return False
+        self._events.append(evt)
+        self._last_emit_time = now
+        self._last_pos = (evt.x, evt.y)
+        if len(self._events) > self.max_events:
+            self._events = self._events[-self.max_events:]
+        if evt.is_click:
+            self._click_count += 1
+        if evt.is_scroll:
+            self._scroll_total += abs(evt.scroll_delta)
+        self._update_dwell(evt.x, evt.y, now)
+        return True
+
+    def add_raw(self, x: float, y: float,
+                pointer_type: str = "mouse",
+                pointer_id: int = 0,
+                button: int = 0,
+                pressure: float = 0.5,
+                movement_x: float = 0.0,
+                movement_y: float = 0.0,
+                is_click: bool = False,
+                is_double_click: bool = False,
+                is_drag: bool = False,
+                is_scroll: bool = False,
+                scroll_delta: float = 0.0,
+                timestamp: Optional[float] = None) -> bool:
+        return self.add_event(PointerEvent(
+            timestamp=timestamp or time.time(),
+            x=max(0.0, min(1.0, x)),
+            y=max(0.0, min(1.0, y)),
+            pointer_id=pointer_id,
+            pointer_type=pointer_type,
+            button=button,
+            pressure=pressure,
+            movement_x=movement_x,
+            movement_y=movement_y,
+            is_click=is_click,
+            is_double_click=is_double_click,
+            is_drag=is_drag,
+            is_scroll=is_scroll,
+            scroll_delta=scroll_delta,
+        ))
+
+    # -- dwell --
+
+    def _update_dwell(self, x: float, y: float, now: float):
+        if self._dwell_start is None:
+            self._dwell_start = now
+            self._dwell_center = (x, y)
+            return
+        dist = math.hypot(x - self._dwell_center[0], y - self._dwell_center[1])
+        if dist > self.dwell_radius:
+            # moved away: finalize previous dwell if significant
+            dwell_ms = (now - self._dwell_start) * 1000
+            if dwell_ms >= self.dwell_threshold_ms:
+                self._dwells.append(DwellRegion(
+                    x=self._dwell_center[0],
+                    y=self._dwell_center[1],
+                    radius=self.dwell_radius,
+                    dwell_ms=dwell_ms,
+                    event_count=1,
+                    started_at=self._dwell_start,
+                    ended_at=now,
+                ))
+                if len(self._dwells) > 100:
+                    self._dwells = self._dwells[-50:]
+            self._dwell_start = now
+            self._dwell_center = (x, y)
+        # else still dwelling — no action until move away or query
+
+    def get_current_dwell_ms(self) -> float:
+        if self._dwell_start is None:
+            return 0.0
+        return (time.time() - self._dwell_start) * 1000
+
+    # -- queries --
+
+    @property
+    def events(self) -> List[PointerEvent]:
+        return list(self._events)
+
+    @property
+    def dwells(self) -> List[DwellRegion]:
+        return list(self._dwells)
+
+    def recent_events(self, n: int = 50) -> List[Dict]:
+        return [e.to_dict() for e in self._events[-n:]]
+
+    def get_cursor_context(self) -> Dict[str, Any]:
+        """Probabilistic cursor context for AI — never mind-reading."""
+        if not self._events:
+            return {"has_data": False}
+        latest = self._events[-1]
+        dwell_ms = self.get_current_dwell_ms()
+        recent = self._events[-20:]
+        # attention inference
+        observations: List[str] = []
+        inferences: List[str] = []
+        if dwell_ms >= self.dwell_threshold_ms:
+            observations.append(f"Cursor remained near ({latest.x:.2f}, {latest.y:.2f}) for {dwell_ms:.0f}ms.")
+            inferences.append("User is likely directing attention to this region.")
+        if self._click_count >= 2:
+            clicks_near = [e for e in recent if e.is_click and math.hypot(e.x - latest.x, e.y - latest.y) < 0.05]
+            if len(clicks_near) >= 2:
+                observations.append(f"User clicked near ({latest.x:.2f}, {latest.y:.2f}) {len(clicks_near)} times.")
+                inferences.append("Repeated clicks suggest the target may be unresponsive or needs attention.")
+        if any(e.is_drag for e in recent):
+            observations.append("User performed a drag gesture recently.")
+            inferences.append("User may be repositioning or selecting content.")
+        if abs(self._scroll_total) > 0.5:
+            observations.append(f"User scrolled (total delta {self._scroll_total:.1f}).")
+        return {
+            "has_data": True,
+            "latest": latest.to_dict(),
+            "dwell_ms": round(dwell_ms, 1),
+            "is_dwelling": dwell_ms >= self.dwell_threshold_ms,
+            "click_count": self._click_count,
+            "scroll_total": round(self._scroll_total, 2),
+            "recent_dwell_regions": [d.to_dict() for d in self._dwells[-5:]],
+            "observations": observations,
+            "inferences": inferences,
+            "cursor_label": f"Cursor: X {int(latest.x*1920)}  Y {int(latest.y*1080)}  (normalized {latest.x:.3f}, {latest.y:.3f})",
+        }
+
+    def clear(self):
+        self._events.clear()
+        self._dwells.clear()
+        self._click_count = 0
+        self._scroll_total = 0
+        self._dwell_start = None
+        self._last_emit_time = 0.0

calc_terminal/vision/events.py

@@ -0,0 +1,66 @@
+"""
+CAT v0.8.a — Vision Event System (spec 41).
+
+Shared event names + helper to emit on both the backend bus (eventbus)
+and the UI event stream (event_stream) where appropriate.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, Optional
+
+# Vision event names (spec 41)
+VISION_SESSION_STARTED = "VISION_SESSION_STARTED"
+VISION_SESSION_STOPPED = "VISION_SESSION_STOPPED"
+VISION_CAPTURE_STARTED = "VISION_CAPTURE_STARTED"
+VISION_CAPTURE_ENDED = "VISION_CAPTURE_ENDED"
+VISION_FRAME_SELECTED = "VISION_FRAME_SELECTED"
+VISION_FRAME_ANALYZED = "VISION_FRAME_ANALYZED"
+VISION_CURSOR_EVENT = "VISION_CURSOR_EVENT"
+VISION_ANNOTATION_CREATED = "VISION_ANNOTATION_CREATED"
+VISION_ANNOTATION_UPDATED = "VISION_ANNOTATION_UPDATED"
+VISION_ANNOTATION_DELETED = "VISION_ANNOTATION_DELETED"
+VISION_USER_MESSAGE = "VISION_USER_MESSAGE"
+VISION_AI_ANALYSIS = "VISION_AI_ANALYSIS"
+VISION_FIX_PROPOSED = "VISION_FIX_PROPOSED"
+VISION_FIX_APPLIED = "VISION_FIX_APPLIED"
+VISION_PREVIEW_REFRESHED = "VISION_PREVIEW_REFRESHED"
+VISION_VERIFICATION_COMPLETE = "VISION_VERIFICATION_COMPLETE"
+
+ALL_VISION_EVENTS = [
+    VISION_SESSION_STARTED,
+    VISION_SESSION_STOPPED,
+    VISION_CAPTURE_STARTED,
+    VISION_CAPTURE_ENDED,
+    VISION_FRAME_SELECTED,
+    VISION_FRAME_ANALYZED,
+    VISION_CURSOR_EVENT,
+    VISION_ANNOTATION_CREATED,
+    VISION_ANNOTATION_UPDATED,
+    VISION_ANNOTATION_DELETED,
+    VISION_USER_MESSAGE,
+    VISION_AI_ANALYSIS,
+    VISION_FIX_PROPOSED,
+    VISION_FIX_APPLIED,
+    VISION_PREVIEW_REFRESHED,
+    VISION_VERIFICATION_COMPLETE,
+]
+
+
+def emit_vision_event(event: str, **data: Any):
+    """Emit a vision event on the backend bus + event_stream (best effort)."""
+    data.setdefault("ts", time.time())
+    data.setdefault("source", "vision")
+    # backend bus (sync pub/sub) — ui/app.py can subscribe
+    try:
+        from .. import eventbus
+        eventbus.bus.publish(event, **data)
+    except Exception:
+        pass
+    # event_stream (bounded buffer with subscribers)
+    try:
+        from .. import event_stream
+        event_stream.stream.emit(event, **data)
+    except Exception:
+        pass

calc_terminal/vision/frame_pipeline.py

@@ -0,0 +1,259 @@
+"""
+CAT v0.8.a — Frame Pipeline.
+
+Adaptive sampling, change detection, quality scoring, and frame selection.
+The pipeline decides which frames are worth sending to the AI model,
+discards duplicates/blur, and maintains a sliding window of context frames.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class ChangeLevel(str, Enum):
+    NONE = "none"           # No meaningful change
+    MINOR = "minor"         # Small UI update
+    MODERATE = "moderate"   # Noticeable change, worth keeping
+    MAJOR = "major"         # Significant change, must keep
+    CRITICAL = "critical"   # Error/screen change, always keep
+
+
+@dataclass
+class FrameMeta:
+    timestamp: float = 0.0
+    width: int = 0
+    height: int = 0
+    pointer_x: float = 0.0
+    pointer_y: float = 0.0
+    pointer_down: bool = False
+    scroll_y: int = 0
+    active_element: str = ""
+    url: str = ""
+    title: str = ""
+    annotations: List[Dict] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "width": self.width,
+            "height": self.height,
+            "pointer_x": self.pointer_x,
+            "pointer_y": self.pointer_y,
+            "pointer_down": self.pointer_down,
+            "scroll_y": self.scroll_y,
+            "active_element": self.active_element,
+            "url": self.url,
+            "title": self.title,
+            "annotations": self.annotations,
+        }
+
+
+@dataclass
+class FrameEntry:
+    frame_b64: str
+    meta: FrameMeta
+    index: int
+    change_level: ChangeLevel
+    quality_score: float
+    hash: str
+    kept: bool = True
+
+
+class FramePipeline:
+    """Manages frame ingestion, change detection, and selection."""
+
+    def __init__(self, config: Any = None):
+        self._config = config
+        self._frames: List[FrameEntry] = []
+        self._last_hash: str = ""
+        self._last_change_time: float = 0.0
+        self._frame_count: int = 0
+        self._change_threshold = getattr(config, "change_threshold", 0.05)
+        self._max_frames = getattr(config, "max_frames", 500)
+        self._saved_max = getattr(config, "saved_frames_max", 20)
+        self._context_window = getattr(config, "context_window_frames", 5)
+        self._min_frame_interval = 1.0 / max(getattr(config, "fps", 1.0), 0.1)
+
+    @property
+    def frame_count(self) -> int:
+        return self._frame_count
+
+    def ingest(self, frame_b64: str,
+               meta: Optional[FrameMeta] = None) -> Optional[Dict[str, Any]]:
+        """Ingest a frame, detect change, decide whether to keep it.
+        Returns frame info dict if the frame was selected, None otherwise.
+        """
+        if meta is None:
+            meta = FrameMeta(timestamp=time.time())
+
+        self._frame_count += 1
+        frame_hash = self._compute_hash(frame_b64)
+        change_level = self._detect_change(frame_b64, frame_hash)
+        quality = self._score_quality(frame_b64, meta)
+
+        if change_level == ChangeLevel.NONE and quality < 0.3:
+            return None
+
+        entry = FrameEntry(
+            frame_b64=frame_b64,
+            meta=meta,
+            index=self._frame_count,
+            change_level=change_level,
+            quality_score=quality,
+            hash=frame_hash,
+            kept=True,
+        )
+
+        if len(self._frames) >= self._max_frames:
+            self._prune()
+
+        self._frames.append(entry)
+        self._last_hash = frame_hash
+        if change_level.value in ("moderate", "major", "critical"):
+            self._last_change_time = time.time()
+
+        return {
+            "frame_index": entry.index,
+            "change_level": change_level.value,
+            "quality_score": round(quality, 3),
+            "kept": True,
+            "meta": meta.to_dict(),
+            "total_frames": self._frame_count,
+            "stored_frames": len(self._frames),
+        }
+
+    def get_context_frames(self, n: Optional[int] = None) -> List[Dict]:
+        """Return the last N frames for AI context (without full base64)."""
+        n = n or self._context_window
+        recent = [f for f in self._frames if f.kept][-n:]
+        return [
+            {
+                "frame_index": f.index,
+                "change_level": f.change_level.value,
+                "quality_score": round(f.quality_score, 3),
+                "meta": f.meta.to_dict(),
+                "has_annotations": bool(f.meta.annotations),
+            }
+            for f in recent
+        ]
+
+    def get_latest_frame(self) -> Optional[FrameEntry]:
+        kept = [f for f in self._frames if f.kept]
+        return kept[-1] if kept else None
+
+    def get_frame_b64(self, frame_index: int) -> Optional[str]:
+        for f in self._frames:
+            if f.index == frame_index:
+                return f.frame_b64
+        return None
+
+    def get_best_frame(self) -> Optional[FrameEntry]:
+        """Select the best frame for AI analysis: highest change + quality."""
+        kept = [f for f in self._frames if f.kept]
+        if not kept:
+            return None
+        ranked = sorted(
+            kept,
+            key=lambda f: (
+                list(ChangeLevel).index(f.change_level) * 10
+                + f.quality_score
+            ),
+            reverse=True,
+        )
+        return ranked[0]
+
+    def get_summary(self) -> Dict[str, Any]:
+        kept = [f for f in self._frames if f.kept]
+        return {
+            "total_frames": self._frame_count,
+            "stored_frames": len(self._frames),
+            "kept_frames": len(kept),
+            "change_levels": {
+                level.value: sum(
+                    1 for f in kept if f.change_level == level
+                )
+                for level in ChangeLevel
+            },
+            "avg_quality": round(
+                sum(f.quality_score for f in kept) / max(len(kept), 1), 3
+            ),
+            "context_window": self._context_window,
+        }
+
+    def clear(self):
+        self._frames.clear()
+        self._last_hash = ""
+        self._frame_count = 0
+
+    def _compute_hash(self, frame_b64: str) -> str:
+        sample = frame_b64[:4096] if len(frame_b64) > 4096 else frame_b64
+        return hashlib.md5(sample.encode("ascii", errors="ignore")).hexdigest()
+
+    def _detect_change(self, frame_b64: str, frame_hash: str) -> ChangeLevel:
+        if not self._last_hash:
+            return ChangeLevel.MAJOR
+
+        if frame_hash == self._last_hash:
+            return ChangeLevel.NONE
+
+        similarity = self._compare_hashes(self._last_hash, frame_hash)
+        if similarity > 0.97:
+            return ChangeLevel.NONE
+        elif similarity > 0.90:
+            return ChangeLevel.MINOR
+        elif similarity > 0.75:
+            return ChangeLevel.MODERATE
+        elif similarity > 0.50:
+            return ChangeLevel.MAJOR
+        else:
+            return ChangeLevel.CRITICAL
+
+    def _compare_hashes(self, h1: str, h2: str) -> float:
+        """Compare two MD5 hashes by hex char similarity."""
+        if not h1 or not h2 or len(h1) != len(h2):
+            return 0.0
+        matches = sum(a == b for a, b in zip(h1, h2))
+        return matches / len(h1)
+
+    def _score_quality(self, frame_b64: str, meta: FrameMeta) -> float:
+        """Score frame quality 0.0-1.0. Higher = clearer, more useful."""
+        score = 0.5
+        data_len = len(frame_b64)
+        if data_len < 1000:
+            return 0.1
+        if 5000 < data_len < 500000:
+            score += 0.15
+        elif data_len > 500000:
+            score += 0.1
+        if meta.width > 800 and meta.height > 600:
+            score += 0.15
+        elif meta.width > 400 and meta.height > 300:
+            score += 0.05
+        if meta.pointer_down:
+            score += 0.1
+        if meta.active_element:
+            score += 0.1
+        if meta.annotations:
+            score += 0.1
+        return min(score, 1.0)
+
+    def _prune(self):
+        """Remove oldest low-priority frames when at capacity."""
+        if len(self._frames) <= self._max_frames:
+            return
+        removable = [
+            f for f in self._frames
+            if f.change_level in (ChangeLevel.NONE, ChangeLevel.MINOR)
+        ]
+        removable.sort(key=lambda f: f.index)
+        to_remove = len(self._frames) - self._max_frames + 50
+        for f in removable[:to_remove]:
+            self._frames.remove(f)
+        if len(self._frames) > self._max_frames:
+            self._frames = self._frames[-self._max_frames:]