threadkeeper 0.4.0__py3-none-any.whl

threadkeeper/tools/invariants.py

@@ -0,0 +1,177 @@
+"""Invariance detection MCP tool.
+
+Extracted from server.py. Finds recurring assistant-side response patterns
+that survive prompt variance — clusters of responses with high mutual
+similarity whose preceding user prompts are diverse. High-scoring clusters
+are candidates for "things I always say" regardless of what was asked.
+
+Requires semantic embeddings (sentence-transformers) — without them the
+tool returns ERR.
+"""
+
+import sqlite3
+import time
+from typing import Optional
+
+from .._mcp import mcp
+from ..db import get_db
+from ..config import SEMANTIC_AVAILABLE
+from ..helpers import fmt_age, q
+from ..identity import _ensure_session
+
+
+@mcp.tool()
+def find_invariants(window_days: int = 14,
+                    min_cluster_size: int = 3,
+                    response_cohesion: float = 0.85,
+                    top_n: int = 10,
+                    max_messages: int = 10000) -> str:
+    """Find recurring assistant-side patterns that survive prompt variance.
+
+    Algorithm:
+      1. Pull recent assistant messages from dialog_messages (with embeddings).
+      2. Greedy cluster by response embedding cosine ≥ response_cohesion.
+      3. For each cluster (size ≥ min_cluster_size), find each response's
+         immediately-preceding user prompt in the same conversation.
+      4. Score = avg_response_similarity × (1 - avg_prompt_similarity).
+         High = my response stays the same shape while prompts vary widely.
+
+    Returns top_n clusters with sample response, scores, and counts.
+    Requires semantic embeddings (sentence-transformers) — without them
+    returns ERR.
+    """
+    if not SEMANTIC_AVAILABLE:
+        return "ERR semantic_off (need sentence-transformers + embeddings)"
+    try:
+        import numpy as _np  # type: ignore
+    except ImportError:
+        return "ERR numpy_unavailable"
+
+    conn = get_db()
+    cutoff = int(time.time()) - max(1, int(window_days)) * 86400
+    # Aggressive filter: subagent jsonls (project='subagents') are mostly
+    # boilerplate role-intros and pollute clusters. Skip those + common
+    # subagent-shape kickoff phrases. We want main-conversation responses.
+    rows = conn.execute(
+        "SELECT uuid, session_id, content, created_at, embedding "
+        "FROM dialog_messages "
+        "WHERE role='assistant' AND embedding IS NOT NULL "
+        "AND created_at >= ? "
+        "AND project != 'subagents' "
+        "AND content NOT LIKE '[thinking]%' "
+        "AND content NOT LIKE 'I''m Claude Code%' "
+        "AND content NOT LIKE 'Hello! I''m Claude Code%' "
+        "AND content NOT LIKE 'I''ll help you%' "
+        "AND content NOT LIKE 'I understand you want me to%' "
+        "AND content NOT LIKE '<summary>%' "
+        "AND length(content) >= 120 "
+        "ORDER BY created_at DESC LIMIT ?",
+        (cutoff, max(100, int(max_messages))),
+    ).fetchall()
+    if len(rows) < min_cluster_size:
+        return f"insufficient_data n={len(rows)} need>={min_cluster_size}"
+
+    embs = _np.stack([
+        _np.frombuffer(r["embedding"], dtype="float32") for r in rows
+    ])  # (N, D)
+    N = embs.shape[0]
+    sim = embs @ embs.T  # (N, N), embeddings already normalized
+
+    # Greedy single-link clustering from each unassigned seed.
+    assigned = [False] * N
+    clusters: list[list[int]] = []
+    threshold = float(response_cohesion)
+    for i in range(N):
+        if assigned[i]:
+            continue
+        cluster = [i]
+        assigned[i] = True
+        # vectorized scan of remaining
+        for j in range(i + 1, N):
+            if assigned[j]:
+                continue
+            if sim[i, j] >= threshold:
+                cluster.append(j)
+                assigned[j] = True
+        if len(cluster) >= min_cluster_size:
+            clusters.append(cluster)
+
+    if not clusters:
+        return (
+            f"no_clusters (n={N}, threshold={threshold}, "
+            f"min={min_cluster_size}) — try lower threshold"
+        )
+
+    invariants = []
+    for cl in clusters:
+        cl_arr = _np.array(cl)
+        sub_sim = sim[_np.ix_(cl_arr, cl_arr)]
+        n = len(cl)
+        # mean of off-diagonal
+        if n > 1:
+            cohesion = (sub_sim.sum() - n) / (n * (n - 1))
+        else:
+            cohesion = 1.0
+
+        # gather preceding user prompts (one per cluster member, same session)
+        prompt_embs = []
+        for idx in cl:
+            r = rows[idx]
+            ts = r["created_at"]
+            sid = r["session_id"]
+            if not sid:
+                continue
+            ur = conn.execute(
+                "SELECT embedding FROM dialog_messages "
+                "WHERE session_id=? AND role='user' AND created_at < ? "
+                "AND embedding IS NOT NULL "
+                "AND content NOT LIKE '[tool_result]%' "
+                "AND content NOT LIKE '[Image%' "
+                "ORDER BY created_at DESC LIMIT 1",
+                (sid, ts),
+            ).fetchone()
+            if ur and ur["embedding"]:
+                prompt_embs.append(
+                    _np.frombuffer(ur["embedding"], dtype="float32")
+                )
+        if len(prompt_embs) < min_cluster_size:
+            continue
+        pe = _np.stack(prompt_embs)
+        psim = pe @ pe.T
+        pn = len(prompt_embs)
+        if pn > 1:
+            avg_psim = (psim.sum() - pn) / (pn * (pn - 1))
+        else:
+            avg_psim = 1.0
+        diversity = 1.0 - float(avg_psim)
+        score = float(cohesion) * diversity
+
+        # representative: longest message in cluster
+        rep_idx = max(cl, key=lambda i: len(rows[i]["content"]))
+        rep = rows[rep_idx]["content"][:240].replace("\n", " ")
+        if len(rows[rep_idx]["content"]) > 240:
+            rep += "…"
+        invariants.append({
+            "size": n,
+            "cohesion": float(cohesion),
+            "diversity": diversity,
+            "score": score,
+            "sample": rep,
+        })
+
+    invariants.sort(key=lambda x: x["score"], reverse=True)
+    invariants = invariants[: max(1, int(top_n))]
+    if not invariants:
+        return f"no_invariants (clusters had insufficient prompt variety)"
+
+    out = [
+        f"invariants n={len(invariants)} window={window_days}d "
+        f"threshold={threshold} pool={N}"
+    ]
+    for inv in invariants:
+        out.append(
+            f"  size={inv['size']} cohesion={inv['cohesion']:.2f} "
+            f"diversity={inv['diversity']:.2f} score={inv['score']:.2f}"
+        )
+        out.append(f"    sample: {inv['sample']}")
+    return "\n".join(out)

threadkeeper/tools/lessons.py

@@ -0,0 +1,110 @@
+"""MCP tools that expose the CLI-agnostic lessons store.
+
+  lesson_append(title, body, summary, source)
+    Materialize a class-level lesson into ~/.threadkeeper/lessons.md.
+    Idempotent on slug — re-calling with the same title overwrites the
+    existing section.
+
+  lesson_list(k=20)
+    Compact listing for inspection / diagnostics.
+
+  lesson_get(slug)
+    Return the full body of a single lesson by slug.
+
+The learning loop (review_thread + shadow_review) writes here instead
+of (or in addition to) ~/.claude/skills/*/SKILL.md so non-Claude CLIs
+share the procedural-knowledge surface. Each CLI's per-user
+instructions file references this path via the managed thread-keeper
+block written by `_setup.py`.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from .._mcp import mcp
+from ..identity import _ensure_session
+from ..db import get_db
+from ..lessons import (
+    append_lesson,
+    iter_lessons,
+    count_lessons,
+    get_path,
+)
+
+
+@mcp.tool()
+def lesson_append(
+    title: str,
+    body: str,
+    summary: str = "",
+    source: str = "",
+) -> str:
+    """Materialize a class-level lesson into ~/.threadkeeper/lessons.md.
+
+    `title` is sluggified to a stable key — repeated calls with the same
+    title overwrite the existing section (idempotent).
+
+    `body` is markdown; goes verbatim into the section body.
+
+    `summary` is an optional one-liner rendered as a blockquote right
+    after the header. Use when the body is long and a TL;DR helps the
+    next agent decide whether to read further.
+
+    `source` is a provenance tag — typically a thread id (\"Tabc123\")
+    when written by review_thread, or \"shadow\" when written by the
+    shadow_review observer. Empty is fine.
+    """
+    conn = get_db()
+    _ensure_session(conn)
+    if not title.strip():
+        return "ERR empty_title"
+    if not body.strip():
+        return "ERR empty_body"
+    slug = append_lesson(
+        title=title, body=body, summary=summary, source=source,
+    )
+    return f"ok slug={slug} path={get_path()}"
+
+
+@mcp.tool()
+def lesson_list(k: int = 20) -> str:
+    """Compact listing of materialized lessons, newest first.
+
+    Format per line: `<age>  <slug>  source=<src>  <first 60 chars of body>`
+    """
+    conn = get_db()
+    _ensure_session(conn)
+    items = list(iter_lessons())
+    if not items:
+        return "no_lessons"
+    items.sort(key=lambda x: x["ts"], reverse=True)
+    now = int(datetime.now().timestamp())
+    out: list[str] = [f"lessons total={len(items)} path={get_path()}"]
+    for it in items[:max(1, k)]:
+        age_s = max(0, now - it["ts"])
+        age = (
+            f"{age_s}s"
+            if age_s < 60
+            else f"{age_s // 60}m"
+            if age_s < 3600
+            else f"{age_s // 3600}h"
+            if age_s < 86400
+            else f"{age_s // 86400}d"
+        )
+        snippet = " ".join(it["body"].split())[:60]
+        src = it["source"] or "?"
+        out.append(f"  {age:>5s}  {it['slug']:30s}  src={src:8s}  {snippet}")
+    return "\n".join(out)
+
+
+@mcp.tool()
+def lesson_get(slug: str) -> str:
+    """Return the full body of one lesson by slug. Useful when
+    `lesson_list` surfaced something you want to read in full."""
+    conn = get_db()
+    _ensure_session(conn)
+    for it in iter_lessons():
+        if it["slug"] == slug:
+            return it["body"]
+    return f"ERR not_found slug={slug}"

threadkeeper/tools/missed_spawns.py

@@ -0,0 +1,142 @@
+"""Missed-spawn detection MCP tool.
+
+Scans recent assistant messages for response shapes that signal
+decomposable work (multiple top-level numbered items or multiple
+markdown section headers) and checks whether the conversation
+actually called spawn() around the same time. Responses with
+decomposable shape but no nearby spawn() are flagged as
+`missed_spawn` candidates — places where the agent answered
+linearly when it could have parallelized.
+
+This is a behavioral mirror: it doesn't change anything, it tells
+you how often spawn() reflex actually fires.
+"""
+
+import re
+import sqlite3
+import time
+from datetime import datetime, timezone
+
+from .._mcp import mcp
+from ..db import get_db
+from ..helpers import fmt_age, q
+from ..identity import _ensure_session
+
+
+# Top-level numbered enumeration in markdown. Allows up to 3 leading
+# spaces, optional ** wrap. Each match = one numbered item.
+_NUMBERED_RE = re.compile(r"(?m)^[ \t]{0,3}(?:\*\*)?\d+[\.\)][ \t]+")
+
+# H2 / H3 markdown headers. We don't count H1 (rare in chat replies).
+_HEADER_RE = re.compile(r"(?m)^#{2,3}\s+\S")
+
+# Time window (seconds) around an assistant message in which a tasks row
+# counts as "the spawn for this response". 10 min is generous.
+_SPAWN_PROXIMITY_S = 600
+
+
+@mcp.tool()
+def find_missed_spawns(window_days: int = 14,
+                       min_response_len: int = 400,
+                       min_numbered: int = 2,
+                       min_headers: int = 3,
+                       top_n: int = 10,
+                       max_messages: int = 5000) -> str:
+    """Find assistant responses that decomposed into independent blocks
+    but were answered linearly (no spawn() call nearby).
+
+    Algorithm:
+      1. Pull recent assistant messages (last `window_days` days,
+         length ≥ `min_response_len`, excluding subagent jsonls).
+      2. For each, count top-level numbered items and H2/H3 headers.
+      3. Mark as `decomposable` if numbered ≥ `min_numbered` OR
+         headers ≥ `min_headers`.
+      4. For each decomposable response, check whether any tasks row
+         with parent_cid = response's session_id has started_at within
+         ±10 min of the response. If none → missed_spawn.
+      5. Return top `top_n` by score (numbered + headers).
+
+    Use this to calibrate the spawn_hint: a high missed-spawn count
+    means the hint isn't strong enough, or thresholds need tuning.
+    """
+    conn = get_db()
+    _ensure_session(conn)
+    now = int(time.time())
+    cutoff = now - max(1, int(window_days)) * 86400
+
+    rows = conn.execute(
+        "SELECT uuid, session_id, content, created_at "
+        "FROM dialog_messages "
+        "WHERE role='assistant' AND created_at >= ? "
+        "AND project != 'subagents' "
+        "AND length(content) >= ? "
+        "AND content NOT LIKE '[thinking]%' "
+        "AND content NOT LIKE '[tool_result]%' "
+        "AND content NOT LIKE '<summary>%' "
+        "ORDER BY created_at DESC LIMIT ?",
+        (cutoff, max(100, int(min_response_len)), max(100, int(max_messages))),
+    ).fetchall()
+
+    if not rows:
+        return f"insufficient_data scanned=0 window_days={window_days}"
+
+    candidates = []
+    for r in rows:
+        content = r["content"] or ""
+        n_num = len(_NUMBERED_RE.findall(content))
+        n_hdr = len(_HEADER_RE.findall(content))
+        if n_num < min_numbered and n_hdr < min_headers:
+            continue
+        candidates.append({
+            "uuid": r["uuid"],
+            "session_id": r["session_id"],
+            "content": content,
+            "created_at": r["created_at"],
+            "numbered": n_num,
+            "headers": n_hdr,
+        })
+
+    if not candidates:
+        return (f"scanned={len(rows)} decomposable=0 — no responses "
+                "matched decomp shape thresholds")
+
+    # For each candidate, look for a tasks row close in time.
+    missed = []
+    for c in candidates:
+        spawned = conn.execute(
+            "SELECT COUNT(*) cnt FROM tasks "
+            "WHERE parent_cid = ? "
+            "AND started_at BETWEEN ? AND ?",
+            (c["session_id"],
+             c["created_at"] - _SPAWN_PROXIMITY_S,
+             c["created_at"] + _SPAWN_PROXIMITY_S),
+        ).fetchone()["cnt"]
+        if spawned == 0:
+            missed.append(c)
+
+    if not missed:
+        return (f"scanned={len(rows)} decomposable={len(candidates)} "
+                "missed=0 — every decomposable response had a nearby spawn()")
+
+    # Rank by score = numbered + headers (rough decomposition intensity).
+    missed.sort(key=lambda x: -(x["numbered"] + x["headers"]))
+    top = missed[: max(1, int(top_n))]
+
+    out = [
+        f"missed_spawn scanned={len(rows)} decomposable={len(candidates)} "
+        f"missed={len(missed)} window={window_days}d"
+    ]
+    for c in top:
+        iso = datetime.fromtimestamp(c["created_at"], tz=timezone.utc).strftime(
+            "%Y-%m-%dT%H:%MZ"
+        )
+        sid_short = (c["session_id"] or "?")[:8]
+        sample = c["content"][:120].replace("\n", " ")
+        if len(c["content"]) > 120:
+            sample += "…"
+        out.append(
+            f"  {iso} sid={sid_short} nbr={c['numbered']} "
+            f"hdr={c['headers']} age={fmt_age(now - c['created_at'])}_ago "
+            f"{q(sample)}"
+        )
+    return "\n".join(out)