activity-frames 0.1.0__py3-none-any.whl

activity_frames/db.py

@@ -0,0 +1,86 @@
+"""Read-only access to the local capture database.
+
+The capture engine owns this database; activity-frames never writes to
+it. Connections are opened with SQLite's read-only URI flag so a bug
+here cannot corrupt capture data.
+"""
+from __future__ import annotations
+
+import os
+import sqlite3
+from pathlib import Path
+from typing import Any, Iterable
+
+DEFAULT_DB_CANDIDATES = (
+    "~/.screenpipe/db.sqlite",       # default engine data dir
+    "~/.screenpipe/screenpipe.db",
+)
+
+
+class RecorderDBNotFound(FileNotFoundError):
+    """Raised when no capture database can be located."""
+
+
+def find_default_db() -> str:
+    """Locate the capture DB, honoring $AFRAMES_DB (or $SCREENPIPE_DB)."""
+    for var in ("AFRAMES_DB", "SCREENPIPE_DB"):
+        env = os.environ.get(var)
+        if env:
+            p = Path(env).expanduser()
+            if p.exists():
+                return str(p)
+            raise RecorderDBNotFound(f"${var} points to a missing file: {env}")
+    for cand in DEFAULT_DB_CANDIDATES:
+        p = Path(cand).expanduser()
+        if p.exists():
+            return str(p)
+    raise RecorderDBNotFound(
+        "No capture database found. Start recording with: aframes record "
+        "(or point $AFRAMES_DB at an existing capture database)."
+    )
+
+
+class Database:
+    """Minimal read-only SQLite wrapper (port of Nocta's SQLiteDB.swift)."""
+
+    def __init__(self, path: str | None = None):
+        self.path = path or find_default_db()
+        p = Path(self.path).expanduser()
+        if not p.exists():
+            raise RecorderDBNotFound(f"Database not found: {self.path}")
+        # as_uri() percent-encodes special characters and handles Windows
+        # drive letters, unlike naive f"file:{path}" interpolation.
+        uri = p.resolve().as_uri() + "?mode=ro"
+        self._conn = sqlite3.connect(uri, uri=True, timeout=3.0)
+        self._conn.execute("PRAGMA query_only = ON")
+
+    def rows(self, sql: str, params: Iterable[Any] = ()) -> list[tuple]:
+        cur = self._conn.execute(sql, tuple(params))
+        try:
+            return cur.fetchall()
+        finally:
+            cur.close()
+
+    def scalar(self, sql: str, params: Iterable[Any] = (), default: Any = 0) -> Any:
+        rows = self.rows(sql, params)
+        if rows and rows[0] and rows[0][0] is not None:
+            return rows[0][0]
+        return default
+
+    def table_exists(self, name: str) -> bool:
+        return (
+            self.scalar(
+                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?",
+                (name,),
+            )
+            > 0
+        )
+
+    def close(self) -> None:
+        self._conn.close()
+
+    def __enter__(self) -> "Database":
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()

activity_frames/emit.py

@@ -0,0 +1,118 @@
+"""Output formats: JSON, YAML, markdown, and the agent context block.
+
+The context block is the flagship output: a compact, token-efficient
+plaintext representation of an ActivityDocument designed to be pasted
+into an agent's system prompt or returned from an MCP tool.
+"""
+from __future__ import annotations
+
+import json
+
+from .frames import ActivityDocument
+
+
+def to_json(doc: ActivityDocument, *, include_input_text: bool = False, indent: int = 2) -> str:
+    return json.dumps(doc.to_dict(include_input_text), indent=indent, ensure_ascii=False)
+
+
+def to_yaml(doc: ActivityDocument, *, include_input_text: bool = False) -> str:
+    try:
+        import yaml
+    except ImportError as e:
+        raise ImportError(
+            "PyYAML is required for YAML output: pip install activity-frames[yaml]"
+        ) from e
+    return yaml.safe_dump(
+        doc.to_dict(include_input_text),
+        sort_keys=False,
+        allow_unicode=True,
+        width=100,
+    )
+
+
+def to_markdown(doc: ActivityDocument, *, include_input_text: bool = False) -> str:
+    d = doc.to_dict(include_input_text)
+    cov = d["coverage"]
+    lines = [
+        f"# Activity {d['window'].get('day', d['window']['start_utc'][:10])}",
+        "",
+        f"**Coverage:** {cov['first_activity']} to {cov['last_activity']} local, "
+        f"{cov['active_minutes']} active min across {cov['distinct_apps']} apps "
+        f"({cov['coverage_pct']}% of span)",
+        "",
+        "| # | Time | App / Site | Active | What was on screen |",
+        "|---|------|-----------|--------|--------------------|",
+    ]
+    for f in d["frames"]:
+        where = f["app"] + (f" ({f['site']})" if f.get("site") else "")
+        what = "; ".join(
+            (f"{p['kind']}" + (f": {p['entity']}" if p.get("entity") else ""))
+            for p in f.get("pages", [])[:3]
+        ) or "; ".join(f.get("windows", [])[:2])
+        lines.append(
+            f"| {f['id']} | {f['start'][:5]}-{f['end'][:5]} | {where} "
+            f"| {f['duration_min']}m | {what} |"
+        )
+    if cov.get("gaps"):
+        lines.append("")
+        gaps = ", ".join(f"{g['start']}-{g['end']} ({g['minutes']}m)" for g in cov["gaps"])
+        lines.append(f"**Away:** {gaps}")
+    return "\n".join(lines)
+
+
+def context_block(doc: ActivityDocument, *, max_frames: int = 40) -> str:
+    """Compact plaintext block for an agent's prompt.
+
+    Chronological, one line per frame, entities inline. Roughly 15-25
+    tokens per frame; a full working day fits in well under 1.5k tokens.
+    """
+    d = doc.to_dict(False)
+    cov = d["coverage"]
+    frames = d["frames"]
+
+    # If over budget, keep the longest frames but preserve chronology.
+    if len(frames) > max_frames:
+        keep = sorted(frames, key=lambda f: -f["duration_min"])[:max_frames]
+        keep_ids = {f["id"] for f in keep}
+        dropped = len(frames) - len(keep)
+        frames = [f for f in frames if f["id"] in keep_ids]
+    else:
+        dropped = 0
+
+    day = d["window"].get("day", d["window"]["start_utc"][:10])
+    lines = [
+        f"USER ACTIVITY ({day}, local time; measured from screen capture, "
+        "no interpretation):",
+        f"coverage: {cov['first_activity']}-{cov['last_activity']}, "
+        f"{cov['active_minutes']} active min, {cov['distinct_apps']} apps",
+    ]
+    for g in cov.get("gaps", []):
+        lines.append(f"away: {g['start']}-{g['end']} ({g['minutes']}m)")
+    for f in frames:
+        where = f["app"] + (f"/{f['site']}" if f.get("site") else "")
+        bits = []
+        for p in f.get("pages", [])[:4]:
+            b = p["kind"]
+            if p.get("entity"):
+                b += f":{p['entity']}"
+            if p.get("count"):
+                b += f" x{p['count']}"
+            bits.append(b)
+        if not bits and f.get("windows"):
+            bits = [f["windows"][0][:60]]
+        inp = f.get("input", {})
+        if inp.get("keys", 0) > 50:
+            bits.append(f"typed ~{inp['keys']} chars")
+        lines.append(
+            f"- {f['start'][:5]}-{f['end'][:5]} {where} ({f['duration_min']}m): "
+            + ("; ".join(bits) if bits else "on screen")
+        )
+    if dropped:
+        lines.append(f"(+{dropped} frames over the size budget omitted)")
+    omitted = d.get("omitted", {}).get("below_min_minutes", 0)
+    if omitted:
+        lines.append(
+            f"(+{omitted} brief frames under "
+            f"{d['omitted']['min_minutes']} min omitted)"
+        )
+    return "\n".join(lines)

activity_frames/enrich.py

@@ -0,0 +1,342 @@
+"""Deterministic event enrichment (port of Nocta's EnrichmentEngine.swift).
+
+Raw `ui_events` rows have three reliability problems this module fixes
+with plain code:
+
+1. Stale app attribution: the recorder sometimes tags an event with the
+   previously focused app. Fix: attribute each event to the nearest
+   frame in time (frames carry authoritative app/window/url).
+2. Keyboard-layout mismatch: some capture setups record physical key
+   positions interpreted as QWERTY while the user types another layout
+   (e.g. AZERTY). Fix: an optional, explicit translation map. Off by
+   default; never guessed.
+3. Anonymous clicks: many click events carry no element name. Fix:
+   resolve the click's coordinates against the frame's accessibility /
+   OCR element tree (exact containment, then tolerance, then a coarse
+   screen-zone). Every resolution is tagged with how it was obtained.
+
+Everything is confidence-tagged. Nothing is invented: an unresolvable
+click stays unresolved.
+"""
+from __future__ import annotations
+
+from bisect import bisect_left
+from dataclasses import dataclass
+
+from ._time import parse_epoch
+from .db import Database
+
+# ---- Keyboard layout translation --------------------------------------
+
+# Maps are "recorded char -> actually typed char". The azerty map covers
+# the mismatched positions between AZERTY hardware and QWERTY decoding;
+# characters not in the map pass through unchanged.
+LAYOUTS: dict[str, dict[str, str]] = {
+    "azerty": {
+        "q": "a", "a": "q", "w": "z", "z": "w",
+        "Q": "A", "A": "Q", "W": "Z", "Z": "W",
+        ";": "m", "m": ",", ",": ";",
+    },
+}
+
+
+def decode_text(text: str, layout: str | dict[str, str] | None) -> str:
+    """Translate recorded text through a layout map. None = identity."""
+    if not text or layout is None:
+        return text
+    mapping = LAYOUTS[layout] if isinstance(layout, str) else layout
+    return "".join(mapping.get(ch, ch) for ch in text)
+
+
+# ---- Enriched events ---------------------------------------------------
+
+@dataclass
+class EnrichedEvent:
+    epoch: float
+    event_type: str           # click | key | text | app_switch | clipboard
+    app: str                  # authoritative (nearest frame), falls back to event's
+    window: str | None
+    url: str | None
+    frame_id: int | None
+    frame_dt_ms: int | None   # distance to the attributing frame
+    label: str | None         # element name (native or resolved)
+    resolution: str           # native | exact | tolerance | zone | none
+    text: str | None          # decoded text content (None unless requested)
+    confidence: str           # high | medium | low
+
+
+# Primary-monitor logical resolution used to normalize click coordinates.
+# The capture engine stores element bounds normalized to 0..1; clicks are
+# absolute points. Detected from the main display when possible (macOS
+# CoreGraphics via ctypes, no dependency); override with $AFRAMES_SCREEN
+# ("1728x1117") or the enrich_events() arguments. The fallback constant
+# matches a 16-inch MacBook Pro.
+def _detect_screen() -> tuple[float, float]:
+    import os
+
+    env = os.environ.get("AFRAMES_SCREEN", "")
+    if "x" in env:
+        try:
+            w, h = env.lower().split("x", 1)
+            return float(w), float(h)
+        except ValueError:
+            pass
+    try:  # macOS main display, logical points
+        import ctypes
+
+        cg = ctypes.CDLL(
+            "/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics"
+        )
+        cg.CGMainDisplayID.restype = ctypes.c_uint32
+        cg.CGDisplayPixelsWide.restype = ctypes.c_size_t
+        cg.CGDisplayPixelsHigh.restype = ctypes.c_size_t
+        did = cg.CGMainDisplayID()
+        w, h = cg.CGDisplayPixelsWide(did), cg.CGDisplayPixelsHigh(did)
+        if w > 0 and h > 0:
+            return float(w), float(h)
+    except Exception:
+        pass
+    return 1728.0, 1117.0
+
+
+DEFAULT_SCREEN_W, DEFAULT_SCREEN_H = _detect_screen()
+_TOL_PX = 40.0  # tolerance ring in logical pixels
+
+
+def nearest_index(sorted_epochs: list[float], target: float) -> int | None:
+    """Index of the value nearest to target in a sorted list."""
+    if not sorted_epochs:
+        return None
+    lo = bisect_left(sorted_epochs, target)
+    if lo >= len(sorted_epochs):
+        return len(sorted_epochs) - 1
+    if lo > 0 and abs(sorted_epochs[lo - 1] - target) <= abs(sorted_epochs[lo] - target):
+        return lo - 1
+    return lo
+
+
+# A click on a frame with no recorded element tree can often be resolved
+# against a neighboring frame captured moments earlier/later that DOES have
+# one (capture is event-driven; the tree is not extracted on every frame).
+ELEMENT_RESCUE_WINDOW_S = 5.0
+
+
+def enrich_events(
+    db: Database,
+    start_utc: str,
+    end_utc: str,
+    *,
+    layout: str | dict[str, str] | None = None,
+    include_text: bool = False,
+    resolve_clicks: bool = True,
+    screen_w: float = DEFAULT_SCREEN_W,
+    screen_h: float = DEFAULT_SCREEN_H,
+    element_rescue_window: float = ELEMENT_RESCUE_WINDOW_S,
+) -> list[EnrichedEvent]:
+    """Enrich all ui_events in [start_utc, end_utc). In-memory, read-only."""
+    if not db.table_exists("ui_events"):
+        return []
+
+    frame_rows = db.rows(
+        """
+        SELECT id, timestamp, app_name, window_name, browser_url FROM frames
+        WHERE timestamp >= ? AND timestamp < ?
+          AND app_name IS NOT NULL AND app_name != ''
+        ORDER BY timestamp ASC
+        """,
+        (start_utc, end_utc),
+    )
+    frames = [
+        (int(r[0]), e, r[2] or "", r[3], r[4])
+        for r in frame_rows
+        if (e := parse_epoch(r[1])) > 0  # malformed rows would unsort the bisect
+    ]
+    frame_epochs = [f[1] for f in frames]
+
+    # Sorted (epoch, frame_id) of frames that actually have an element tree,
+    # used to rescue click resolution when the nearest frame has none.
+    elem_frames: list[tuple[float, int]] = []
+    if resolve_clicks and frames and db.table_exists("elements"):
+        lo_id = min(f[0] for f in frames)
+        hi_id = max(f[0] for f in frames)
+        with_elements = {
+            int(r[0])
+            for r in db.rows(
+                "SELECT DISTINCT frame_id FROM elements"
+                " WHERE frame_id >= ? AND frame_id <= ?",
+                (lo_id, hi_id),
+            )
+        }
+        elem_frames = sorted(
+            (f[1], f[0]) for f in frames if f[0] in with_elements
+        )
+    elem_epochs = [e for e, _ in elem_frames]
+
+    event_rows = db.rows(
+        """
+        SELECT timestamp, event_type, x, y, element_name, element_role,
+               text_content, app_name
+        FROM ui_events
+        WHERE timestamp >= ? AND timestamp < ?
+        ORDER BY timestamp ASC
+        """,
+        (start_utc, end_utc),
+    )
+
+    elem_cache: dict[int, list[tuple[str, float, float, float, float]]] = {}
+    out: list[EnrichedEvent] = []
+
+    for ts, etype, x, y, elem_name, elem_role, text_content, event_app in event_rows:
+        epoch = parse_epoch(ts or "")
+        etype = etype or ""
+
+        # 1. Authoritative context from the nearest frame.
+        app = event_app or ""
+        window = url = None
+        frame_id = frame_dt_ms = None
+        idx = nearest_index(frame_epochs, epoch)
+        if idx is not None:
+            fid, fepoch, fapp, fwindow, furl = frames[idx]
+            frame_id = fid
+            frame_dt_ms = int(abs(epoch - fepoch) * 1000)
+            if fapp:
+                app = fapp
+            window, url = fwindow, furl
+
+        # 3. Click-label resolution. Resolve against the nearest frame that
+        # HAS an element tree (within the rescue window); the truly nearest
+        # frame often has none because tree extraction is event-driven.
+        native = (elem_name or "").strip()
+        label: str | None = native or None
+        resolution = "native" if native else "none"
+        if (
+            not native
+            and resolve_clicks
+            and etype == "click"
+            and x is not None
+            and y is not None
+            and frame_id is not None
+        ):
+            res_frame_id = frame_id
+            res_dt_ms = frame_dt_ms
+            ei = nearest_index(elem_epochs, epoch)
+            if ei is not None and abs(elem_epochs[ei] - epoch) <= element_rescue_window:
+                res_frame_id = elem_frames[ei][1]
+                res_dt_ms = int(abs(epoch - elem_epochs[ei]) * 1000)
+            label, resolution = _resolve_click(
+                db, res_frame_id, float(x), float(y), elem_cache, screen_w, screen_h
+            )
+            if resolution != "none":
+                # Confidence should reflect distance to the frame we actually
+                # resolved against, not the attribution frame.
+                frame_dt_for_conf = res_dt_ms
+            else:
+                frame_dt_for_conf = frame_dt_ms
+        else:
+            frame_dt_for_conf = frame_dt_ms
+
+        # 2. Layout decode (opt-in).
+        text = None
+        if text_content and include_text:
+            text = decode_text(text_content, layout)
+
+        out.append(
+            EnrichedEvent(
+                epoch=epoch,
+                event_type=etype,
+                app=app,
+                window=window,
+                url=url,
+                frame_id=frame_id,
+                frame_dt_ms=frame_dt_ms,
+                label=label,
+                resolution=resolution,
+                text=text,
+                confidence=_confidence(etype, resolution, frame_dt_for_conf),
+            )
+        )
+    return out
+
+
+def _resolve_click(
+    db: Database,
+    frame_id: int,
+    x: float,
+    y: float,
+    cache: dict,
+    screen_w: float,
+    screen_h: float,
+) -> tuple[str | None, str]:
+    # Only the primary monitor is calibrated; off-screen coords stay unresolved.
+    if not (0 <= x <= screen_w and 0 <= y <= screen_h):
+        return None, "none"
+    if not db.table_exists("elements"):
+        return None, "none"
+    xn, yn = x / screen_w, y / screen_h
+
+    elems = cache.get(frame_id)
+    if elems is None:
+        rows = db.rows(
+            """
+            SELECT text, left_bound, top_bound, width_bound, height_bound
+            FROM elements WHERE frame_id = ? AND text IS NOT NULL AND text != ''
+            """,
+            (frame_id,),
+        )
+        elems = [
+            (r[0], float(r[1]), float(r[2]), float(r[3]), float(r[4]))
+            for r in rows
+            if r[1] is not None and r[2] is not None and r[3] is not None and r[4] is not None
+        ]
+        cache[frame_id] = elems
+    if not elems:
+        return None, "none"
+
+    # Exact containment; smallest area wins (most specific element).
+    best = None
+    for text, l, t, w, h in elems:
+        if l <= xn <= l + w and t <= yn <= t + h:
+            area = w * h
+            if best is None or area < best[1]:
+                best = (text, area)
+    if best:
+        return _clean(best[0]), "exact"
+
+    # Tolerance: expanded boxes (ring in logical pixels, normalized here).
+    tol_n = _TOL_PX / screen_w
+    tol = None
+    for text, l, t, w, h in elems:
+        if l - tol_n <= xn <= l + w + tol_n and t - tol_n <= yn <= t + h + tol_n:
+            area = w * h
+            if tol is None or area < tol[1]:
+                tol = (text, area)
+    if tol:
+        return _clean(tol[0]), "tolerance"
+
+    # Coarse screen zone (low confidence, still honest).
+    if yn < 0.04:
+        return "menu bar", "zone"
+    if yn < 0.08:
+        return "browser chrome", "zone"
+    if xn < 0.15:
+        return "left sidebar", "zone"
+    return "main content", "zone"
+
+
+def _clean(s: str) -> str:
+    t = s.strip()
+    return t[:80] + "..." if len(t) > 80 else t
+
+
+def _confidence(etype: str, resolution: str, frame_dt_ms: int | None) -> str:
+    if etype in ("text", "app_switch", "clipboard"):
+        return "high"
+    if etype == "click":
+        if resolution == "native":
+            return "high"
+        if resolution == "exact":
+            return "high" if (frame_dt_ms or 9999) < 2000 else "medium"
+        if resolution == "tolerance":
+            return "medium"
+        return "low"
+    return "low"