threadkeeper 0.4.0__py3-none-any.whl

threadkeeper/tools/extract.py

@@ -0,0 +1,411 @@
+"""Auto-extraction MCP tools.
+
+Extracted from server.py. Heuristic candidates for note/concept/distill/
+verbatim from recent dialog_messages. Each candidate lands status='pending';
+session reviews in batch via review_candidates() then accept/reject.
+"""
+
+import sqlite3
+import time
+import re as _re_extract
+
+from .._mcp import mcp
+from ..db import get_db
+from ..config import SEMANTIC_AVAILABLE
+from ..helpers import fmt_age, q, gen_concept_id, gen_distill_id
+from .. import identity
+from ..identity import _ensure_session, _detect_self_cid, _emit
+from ..embeddings import _embed
+
+
+# Locale-aware heuristic matchers — patterns live in i18n.py so this
+# module stays English-only. Locale-independent patterns (header,
+# bullet list) stay inline.
+from ..i18n import (
+    WANT_RE as _WANT_RE,
+    INSIGHT_MARKERS_RE as _INSIGHT_MARKERS_RE,
+    EXAMPLE_RE as _EXAMPLE_RE,
+    FRAME_RE as _FRAME_RE,
+)
+_HEADER_RE = _re_extract.compile(r"^##+\s", _re_extract.MULTILINE)
+_BULLET_RE = _re_extract.compile(
+    r"^\s*(?:[-*•]|\d+[.)])\s", _re_extract.MULTILINE
+)
+
+
+# Message-level noise filter (complements session-level
+# _INTERNAL_PROMPT_PREFIXES). These prefixes appear in INDIVIDUAL
+# messages of otherwise-valid sessions — context-compaction summaries
+# from Anthropic CLI, subagent task prompts, SKILL.md injections, and
+# `[Request interrupted by user for tool use]` service markers. None
+# of them are user-intent signals; all of them were polluting extract
+# candidates in the first calibration pass (2026-05-16 audit, 13
+# candidates → ~25% precision; ~half of misses were these patterns).
+_NOISE_CONTENT_PREFIXES: tuple[str, ...] = (
+    "This session is being continued from a previous conversation",
+    "Base directory for this skill:",
+    "In the repo at /",
+    "[Request interrupted by user",
+    # Generic subagent/spawn role prompts. Subagents and `claude -p`
+    # children commonly open with one of these. They're never user-
+    # intent signals — they're task framing injected by the parent.
+    "You are the ",
+    "You are a ",
+    "You are an ",
+    "Research task",
+    "Design task",
+    "Context:",
+)
+
+
+# Verbatim-specific minimum length. The global 30-char floor is too
+# permissive for "I want X"-style user_want matches — short fragments
+# like "[Request interrupted by user for tool use]" (39 chars) and
+# CLI metadata strings sneak through. 50 chars is the empirical
+# threshold below which user_want matches are almost always noise.
+_VERBATIM_MIN_LEN = 50
+
+
+def _is_log_content(text: str) -> bool:
+    """Heuristic: high density of pass/fail/checkmark glyphs → this is
+    a test runner / CI log output, not natural-language signal.
+
+    Returns True if marker count ≥ 3 in the first 2KB of content. Logs
+    from Detox / Maestro / mocha / pytest etc. trigger user_want false
+    matches on lines like "✓ runFixture:registerUser → $ahmed (2.0s)"
+    and entire blocks dump into a single dialog_message turn.
+    """
+    if len(text) < 100:
+        return False
+    sample = text[:2000]
+    markers = ("✓", "✗", "[OK]", "[FAIL]", "PASS", "FAIL", "skipped",
+               "runFixture:")
+    return sum(sample.count(m) for m in markers) >= 3
+
+
+def _candidate_exists(conn, source_uuid, content):
+    if source_uuid:
+        if conn.execute(
+            "SELECT 1 FROM extract_candidates WHERE source_uuid=? "
+            "AND status IN ('pending','accepted') LIMIT 1",
+            (source_uuid,),
+        ).fetchone():
+            return True
+    return bool(
+        conn.execute(
+            "SELECT 1 FROM extract_candidates WHERE content=? "
+            "AND status IN ('pending','accepted') LIMIT 1",
+            (content[:500],),
+        ).fetchone()
+    )
+
+
+def _enqueue(conn, kind, source_uuid, source_cid, content, rationale):
+    if _candidate_exists(conn, source_uuid, content):
+        return None
+    cur = conn.execute(
+        "INSERT INTO extract_candidates (kind, source_uuid, source_cid, "
+        "content, rationale, status, created_at) VALUES (?,?,?,?,?,?,?)",
+        (kind, source_uuid, source_cid, content, rationale,
+         "pending", int(time.time())),
+    )
+    return cur.lastrowid
+
+
+@mcp.tool()
+def extract_recent(window_min: int = 60, max_messages: int = 500) -> str:
+    """Scan recent dialog_messages and enqueue heuristic candidates.
+
+    H1 user_want         → verbatim (normative phrasing)
+    H2 long_insight      → distill (assistant ≥500ch + ## headers + conclusion marker)
+    H3 example_regularity→ concept (bullets≥3 OR example-marker≥2 + abstract frame)
+    H4 paraphrase_repeat → note (≥3 msgs cosine ≥0.80 within same session)"""
+    conn = get_db()
+    _ensure_session(conn)
+    now = int(time.time())
+    cutoff = now - max(1, int(window_min)) * 60
+    # Exclude our own internal-prompted child sessions (shadow_review
+    # observer + close_thread auto-reviewer + curator) — otherwise their
+    # promp text becomes extract candidates, the same self-pollution
+    # we fixed in shadow_review._collect_window.
+    from ..shadow_review import _INTERNAL_PROMPT_PREFIXES
+    # Session-level filter — drop entire sessions started by our own
+    # spawn prompts.
+    sess_prefix_clauses = " OR ".join(
+        ["substr(content, 1, ?) = ?"] * len(_INTERNAL_PROMPT_PREFIXES)
+    )
+    sess_prefix_params: list = []
+    for p in _INTERNAL_PROMPT_PREFIXES:
+        sess_prefix_params.extend([len(p), p])
+    # Message-level filter — drop individual noise messages (compaction
+    # summaries, SKILL injections, subagent prompts) inside otherwise
+    # valid sessions. SQL LIKE with '%' suffix on user-controlled prefix
+    # is safe: each pattern is a literal in source code, not user input.
+    msg_noise_clauses = " AND ".join(
+        [f"content NOT LIKE ?"] * len(_NOISE_CONTENT_PREFIXES)
+    )
+    msg_noise_params = [p + "%" for p in _NOISE_CONTENT_PREFIXES]
+    rows = conn.execute(
+        "SELECT uuid, role, content, session_id, created_at, embedding "
+        "FROM dialog_messages WHERE created_at >= ? "
+        "AND role IN ('user','assistant') "
+        "AND content NOT LIKE '[tool_result]%' AND content NOT LIKE '[Image%' "
+        f"AND {msg_noise_clauses} "
+        "AND length(content) >= 30 "
+        "AND session_id NOT IN ("
+        "  SELECT DISTINCT session_id FROM dialog_messages "
+        f"  WHERE role = 'user' AND ({sess_prefix_clauses})"
+        ") "
+        "ORDER BY created_at ASC LIMIT ?",
+        (cutoff, *msg_noise_params, *sess_prefix_params,
+         max(10, int(max_messages))),
+    ).fetchall()
+    if not rows:
+        return f"no_dialog window={window_min}m"
+    counts = {"verbatim": 0, "distill": 0, "concept": 0, "note": 0}
+    skipped = 0
+    for r in rows:
+        uuid, cid, content, role = (
+            r["uuid"], r["session_id"], r["content"], r["role"]
+        )
+        # Test-runner / CI log dumps trigger user_want false matches on
+        # log labels like "Earnings Withdrawal → ✓ runFixture:…"
+        if _is_log_content(content):
+            skipped += 1
+            continue
+        if (role == "user" and _WANT_RE.search(content)
+                and len(content) >= _VERBATIM_MIN_LEN):
+            res = _enqueue(conn, "verbatim", uuid, cid, content[:2000],
+                           "H1 user_want pattern")
+            if res:
+                counts["verbatim"] += 1
+            else:
+                skipped += 1
+        if role == "assistant":
+            if (len(content) >= 500 and _HEADER_RE.search(content)
+                    and _INSIGHT_MARKERS_RE.search(content)):
+                res = _enqueue(conn, "distill", uuid, cid, content[:4000],
+                               "H2 long_insight (headers + conclusion marker)")
+                if res:
+                    counts["distill"] += 1
+                else:
+                    skipped += 1
+            bullets = len(_BULLET_RE.findall(content))
+            examples = len(_EXAMPLE_RE.findall(content))
+            if (bullets >= 3 or examples >= 2) and _FRAME_RE.search(content):
+                res = _enqueue(
+                    conn, "concept", uuid, cid, content[:3000],
+                    f"H3 example_regularity (bullets={bullets}, examples={examples})",
+                )
+                if res:
+                    counts["concept"] += 1
+                else:
+                    skipped += 1
+    if SEMANTIC_AVAILABLE:
+        try:
+            import numpy as _np  # type: ignore
+        except ImportError:
+            _np = None
+        if _np is not None:
+            with_emb = [r for r in rows if r["embedding"]]
+            by_sess: dict = {}
+            for r in with_emb:
+                by_sess.setdefault(r["session_id"] or "", []).append(r)
+            for sid, msgs in by_sess.items():
+                if len(msgs) < 3:
+                    continue
+                embs = _np.stack([
+                    _np.frombuffer(m["embedding"], dtype="float32") for m in msgs
+                ])
+                sim = embs @ embs.T
+                clustered = [False] * len(msgs)
+                for i in range(len(msgs)):
+                    if clustered[i]:
+                        continue
+                    members = [i]
+                    for j in range(i + 1, len(msgs)):
+                        if not clustered[j] and sim[i, j] >= 0.80:
+                            members.append(j)
+                    if len(members) >= 3:
+                        for k in members:
+                            clustered[k] = True
+                        sub = sim[_np.ix_(members, members)]
+                        rep_idx = members[int(_np.argmax(sub.mean(axis=1)))]
+                        rep = msgs[rep_idx]
+                        member_uuids = sorted(msgs[k]["uuid"] for k in members)
+                        cluster_key = "cluster:" + ",".join(
+                            u[:8] for u in member_uuids[:6]
+                        )
+                        if conn.execute(
+                            "SELECT 1 FROM extract_candidates WHERE source_uuid=? "
+                            "AND status IN ('pending','accepted')",
+                            (cluster_key,),
+                        ).fetchone():
+                            skipped += 1
+                            continue
+                        conn.execute(
+                            "INSERT INTO extract_candidates (kind, source_uuid, "
+                            "source_cid, content, rationale, status, created_at) "
+                            "VALUES (?,?,?,?,?,?,?)",
+                            ("note", cluster_key, sid, rep["content"][:2000],
+                             f"H4 paraphrase_repeat n={len(members)} "
+                             f"sess={sid[:8]} centroid={rep['uuid'][:8]}",
+                             "pending", now),
+                        )
+                        counts["note"] += 1
+    _emit(conn, "extract_recent",
+          summary=" ".join(f"{k}={v}" for k, v in counts.items()))
+    conn.commit()
+    return (
+        f"ok window={window_min}m scanned={len(rows)} "
+        f"verbatim={counts['verbatim']} distill={counts['distill']} "
+        f"concept={counts['concept']} note={counts['note']} "
+        f"skipped_existing={skipped}"
+    )
+
+
+@mcp.tool()
+def review_candidates(status: str = "pending", k: int = 20) -> str:
+    """status ∈ {pending, accepted, rejected, all}. Newest first."""
+    valid = ("pending", "accepted", "rejected", "all")
+    if status not in valid:
+        return f"ERR bad_status={status}"
+    conn = get_db()
+    sql = (
+        "SELECT id, kind, source_uuid, source_cid, content, rationale, "
+        "status, created_at FROM extract_candidates "
+    )
+    if status == "all":
+        rows = conn.execute(
+            sql + "ORDER BY created_at DESC LIMIT ?", (max(1, int(k)),)
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            sql + "WHERE status=? ORDER BY created_at DESC LIMIT ?",
+            (status, max(1, int(k))),
+        ).fetchall()
+    if not rows:
+        return f"no_candidates status={status}"
+    now = int(time.time())
+    out = [f"candidates n={len(rows)} status={status}"]
+    for r in rows:
+        snip = r["content"][:240].replace("\n", " ")
+        if len(r["content"]) > 240:
+            snip += "…"
+        out.append(
+            f"  #{r['id']} {r['kind']} cid={(r['source_cid'] or '-')[:8]} "
+            f"age={fmt_age(now - r['created_at'])}_ago"
+        )
+        out.append(f"    why={r['rationale'] or '?'}")
+        out.append(f"    {q(snip)}")
+    return "\n".join(out)
+
+
+_VALID_TARGET_KINDS = ("note", "concept", "distill", "verbatim")
+
+
+@mcp.tool()
+def accept_candidate(id: int, target_kind: str = "",
+                     thread_id: str = "") -> str:
+    """Materialize candidate into its target table.
+    target_kind overrides candidate's kind. thread_id optional."""
+    conn = get_db()
+    _ensure_session(conn)
+    r = conn.execute(
+        "SELECT * FROM extract_candidates WHERE id=?", (int(id),)
+    ).fetchone()
+    if not r:
+        return f"ERR candidate_not_found={id}"
+    if r["status"] != "pending":
+        return f"ERR not_pending status={r['status']}"
+    kind = (target_kind or r["kind"]).strip()
+    if kind not in _VALID_TARGET_KINDS:
+        return f"ERR bad_target_kind={kind}"
+    tid = thread_id.strip() or None
+    if tid and not conn.execute(
+        "SELECT 1 FROM threads WHERE id=?", (tid,)
+    ).fetchone():
+        return f"ERR thread_not_found={tid}"
+    now = int(time.time())
+    content = r["content"]
+    placed = ""
+    if kind == "verbatim":
+        cur = conn.execute(
+            "INSERT INTO verbatim (speaker, content, thread_id, created_at, "
+            "session_id) VALUES (?,?,?,?,?)",
+            ("user", content, tid, now, identity._session_id),
+        )
+        placed = f"verbatim id={cur.lastrowid}"
+    elif kind == "note":
+        emb = _embed(content)
+        cur = conn.execute(
+            "INSERT INTO notes (thread_id, content, kind, created_at, "
+            "session_id, embedding) VALUES (?,?,?,?,?,?)",
+            (tid, content, "insight", now, identity._session_id, emb),
+        )
+        placed = f"note id={cur.lastrowid} thread={tid or '-'}"
+    elif kind == "concept":
+        pid = gen_concept_id(conn)
+        cid = _detect_self_cid()
+        conn.execute(
+            "INSERT INTO concepts (id, description, triangulation_notes, "
+            "confidence, source_thread, registered_by_cid, registered_at, "
+            "last_evidence_at) VALUES (?,?,?,?,?,?,?,?)",
+            (pid, content, r["rationale"], "low", tid, cid, now, now),
+        )
+        placed = f"concept id={pid}"
+    elif kind == "distill":
+        pid = gen_distill_id(conn)
+        cid = _detect_self_cid()
+        conn.execute(
+            "INSERT INTO distill (id, content, kind, confidence, "
+            "source_thread, source_cid, created_at) "
+            "VALUES (?,?,?,?,?,?,?)",
+            (pid, content, "insight", "medium", tid, cid, now),
+        )
+        if cid:
+            conn.execute(
+                "INSERT INTO distill_votes (distill_id, voter_cid, weight, "
+                "voted_at) VALUES (?,?,?,?)",
+                (pid, cid, 1.0, now),
+            )
+            conn.execute(
+                "UPDATE distill SET vote_sum=1.0, vote_count=1 WHERE id=?",
+                (pid,),
+            )
+        placed = f"distill id={pid}"
+    conn.execute(
+        "UPDATE extract_candidates SET status='accepted', decided_at=? "
+        "WHERE id=?",
+        (now, int(id)),
+    )
+    _emit(conn, f"accept_candidate:{kind}", target=str(id), summary=placed)
+    conn.commit()
+    return f"ok accepted #{id} → {placed}"
+
+
+@mcp.tool()
+def reject_candidate(id: int, reason: str = "") -> str:
+    """Mark rejected. Reason appended to rationale for heuristic tuning."""
+    conn = get_db()
+    _ensure_session(conn)
+    r = conn.execute(
+        "SELECT id, rationale, status FROM extract_candidates WHERE id=?",
+        (int(id),),
+    ).fetchone()
+    if not r:
+        return f"ERR candidate_not_found={id}"
+    if r["status"] != "pending":
+        return f"ERR not_pending status={r['status']}"
+    now = int(time.time())
+    new_r = r["rationale"] or ""
+    if reason:
+        new_r = (new_r + f" | rejected: {reason}").lstrip(" |")[:500]
+    conn.execute(
+        "UPDATE extract_candidates SET status='rejected', decided_at=?, "
+        "rationale=? WHERE id=?",
+        (now, new_r, int(id)),
+    )
+    _emit(conn, "reject_candidate", target=str(id), summary=reason)
+    conn.commit()
+    return f"ok rejected #{id}"

threadkeeper/tools/graph.py

@@ -0,0 +1,183 @@
+"""Knowledge-graph MCP tools.
+
+Extracted from server.py. Provides typed edges between entities
+(threads, notes, concepts, distillates, tasks, signals, probes)
+with link/unlink primitives and a BFS traversal (`neighbors`).
+"""
+
+import sqlite3
+import time
+from typing import Optional
+
+from .._mcp import mcp
+from ..db import get_db
+from ..helpers import fmt_age
+from ..identity import _ensure_session, _detect_self_cid, _emit
+
+
+EDGE_KINDS = ("thread", "note", "concept", "distill", "task", "signal", "probe")
+EDGE_RELATIONS_HINT = (
+    "refines", "contradicts", "exemplifies", "depends_on",
+    "mentions", "elaborates", "supersedes",
+)
+
+
+def _entity_table(kind: str) -> Optional[str]:
+    """Map kind → table name for existence checks."""
+    return {
+        "thread": "threads", "note": "notes", "concept": "concepts",
+        "distill": "distill", "task": "tasks", "signal": "signals",
+        "probe": "probes",
+    }.get(kind)
+
+
+def _entity_exists(conn: sqlite3.Connection, kind: str, eid: str) -> bool:
+    table = _entity_table(kind)
+    if not table:
+        return False
+    try:
+        return conn.execute(
+            f"SELECT 1 FROM {table} WHERE id=?", (eid,)
+        ).fetchone() is not None
+    except sqlite3.OperationalError:
+        return False
+
+
+def _snippet_for(conn: sqlite3.Connection, kind: str, eid: str) -> str:
+    """Pull a short content preview for an entity, regardless of kind."""
+    table = _entity_table(kind)
+    if not table:
+        return "?"
+    field_map = {
+        "thread": "question", "note": "content", "concept": "description",
+        "distill": "content", "task": "prompt", "signal": "content",
+        "probe": "prompt",
+    }
+    field = field_map.get(kind, "")
+    if not field:
+        return "?"
+    try:
+        row = conn.execute(
+            f"SELECT {field} v FROM {table} WHERE id=?", (eid,)
+        ).fetchone()
+    except sqlite3.OperationalError:
+        return "?"
+    if not row or not row["v"]:
+        return "(empty)"
+    text = row["v"][:90].replace("\n", " ")
+    if len(row["v"]) > 90:
+        text += "…"
+    return text
+
+
+@mcp.tool()
+def link(from_kind: str, from_id: str, to_kind: str, to_id: str,
+         relation: str, weight: float = 1.0) -> str:
+    """Create a typed edge between two entities.
+
+    Kinds: thread, note, concept, distill, task, signal, probe.
+    Relations (suggested, free-form ok): refines, contradicts, exemplifies,
+    depends_on, mentions, elaborates, supersedes.
+
+    Existing edge with same (from, to, relation) is replaced (re-linking
+    means updating weight/timestamp, not duplicating)."""
+    if from_kind not in EDGE_KINDS:
+        return f"ERR bad_from_kind={from_kind} (use {','.join(EDGE_KINDS)})"
+    if to_kind not in EDGE_KINDS:
+        return f"ERR bad_to_kind={to_kind}"
+    if not relation.strip():
+        return "ERR empty_relation"
+    if not (-10 <= weight <= 10):
+        return f"ERR weight_out_of_range={weight}"
+    conn = get_db()
+    _ensure_session(conn)
+    if not _entity_exists(conn, from_kind, from_id.strip()):
+        return f"ERR from_not_found={from_kind}:{from_id}"
+    if not _entity_exists(conn, to_kind, to_id.strip()):
+        return f"ERR to_not_found={to_kind}:{to_id}"
+    existing = conn.execute(
+        "SELECT id FROM edges WHERE from_kind=? AND from_id=? AND "
+        "to_kind=? AND to_id=? AND relation=?",
+        (from_kind, from_id.strip(), to_kind, to_id.strip(), relation.strip()),
+    ).fetchone()
+    now_t = int(time.time())
+    cid = _detect_self_cid()
+    if existing:
+        conn.execute(
+            "UPDATE edges SET weight=?, created_by_cid=?, created_at=? "
+            "WHERE id=?",
+            (weight, cid, now_t, existing["id"]),
+        )
+        eid = existing["id"]
+    else:
+        cur = conn.execute(
+            "INSERT INTO edges (from_kind, from_id, to_kind, to_id, "
+            "relation, weight, created_by_cid, created_at) "
+            "VALUES (?,?,?,?,?,?,?,?)",
+            (from_kind, from_id.strip(), to_kind, to_id.strip(),
+             relation.strip(), weight, cid, now_t),
+        )
+        eid = cur.lastrowid
+    _emit(conn, "link", target=f"{from_kind}:{from_id}",
+          summary=f"-{relation}-> {to_kind}:{to_id}")
+    conn.commit()
+    return f"ok edge={eid} {from_kind}:{from_id} -{relation}-> {to_kind}:{to_id}"
+
+
+@mcp.tool()
+def unlink(edge_id: int) -> str:
+    """Remove an edge by id."""
+    conn = get_db()
+    _ensure_session(conn)
+    cur = conn.execute("DELETE FROM edges WHERE id=?", (int(edge_id),))
+    if cur.rowcount == 0:
+        return f"ERR edge_not_found={edge_id}"
+    _emit(conn, "unlink", target=str(edge_id))
+    conn.commit()
+    return "ok"
+
+
+@mcp.tool()
+def neighbors(kind: str, id: str, depth: int = 1, max_n: int = 12) -> str:
+    """BFS the graph from a starting node up to `depth` hops away.
+    Returns each visited node with its kind, id, and a short content snippet
+    pulled from its native table. Both directions of edges traversed."""
+    if kind not in EDGE_KINDS:
+        return f"ERR bad_kind={kind}"
+    conn = get_db()
+    if not _entity_exists(conn, kind, id.strip()):
+        return f"ERR not_found={kind}:{id}"
+    depth = max(1, min(int(depth), 4))
+    max_n = max(1, min(int(max_n), 50))
+    visited: set[tuple[str, str]] = {(kind, id.strip())}
+    frontier = [(kind, id.strip(), 0)]
+    nodes_out: list[tuple[str, str, int, str]] = []
+    while frontier and len(nodes_out) < max_n:
+        k, eid, d = frontier.pop(0)
+        if d >= depth:
+            continue
+        rows = conn.execute(
+            "SELECT to_kind AS nk, to_id AS nid, relation, weight FROM edges "
+            "WHERE from_kind=? AND from_id=? "
+            "UNION "
+            "SELECT from_kind AS nk, from_id AS nid, relation, weight FROM edges "
+            "WHERE to_kind=? AND to_id=?",
+            (k, eid, k, eid),
+        ).fetchall()
+        for r in rows:
+            nk, nid = r["nk"], r["nid"]
+            key = (nk, nid)
+            if key in visited:
+                continue
+            visited.add(key)
+            snippet = _snippet_for(conn, nk, nid)
+            nodes_out.append((nk, nid, d + 1, snippet))
+            frontier.append((nk, nid, d + 1))
+            if len(nodes_out) >= max_n:
+                break
+    if not nodes_out:
+        return f"no_neighbors {kind}:{id} depth={depth}"
+    lines = [f"neighbors {kind}:{id} depth={depth} n={len(nodes_out)}"]
+    for k, eid, d, sn in nodes_out:
+        lines.append(f"  [+{d}] {k}:{eid} — {sn}")
+    return "\n".join(lines)