cherry-docs 0.2.0__py3-none-any.whl

scripts/eval_projects.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""cherry eval — evaluate CherryDocs memory quality across all projects.
+
+Runs in two passes:
+  1. Heuristic  — memory count, confidence distribution, kind diversity
+  2. LLM judge  — Ollama grades each project's answer on a 1-5 usefulness scale
+
+Reads exclusively from ~/.cherrydocs/ — never touches downstream project repos.
+
+Usage:
+  python scripts/eval_projects.py
+  python scripts/eval_projects.py --project footcorn
+  python scripts/eval_projects.py --no-llm      # heuristic only
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+import httpx
+
+from app.services.promoted_memory_answer import answer_from_promoted_memory
+from app.services.promoted_memory_store import DEFAULT_PROMOTED_ROOT, LocalPromotedMemoryStore
+
+_HOME_CHERRY = Path.home() / ".cherrydocs"
+_DEFAULT_BUFFER = os.environ.get("CHERRY_CAPTURE_BUFFER_DIR", str(_HOME_CHERRY / "capture"))
+_OLLAMA_URL = os.getenv("OLLAMA_CHAT_URL", "http://127.0.0.1:11434/api/chat")
+_OLLAMA_MODEL = os.getenv("CHERRY_OLLAMA_MODEL", "qwen2.5:7b-instruct")
+
+# Questions used for every project — generic enough to work without knowing the domain
+_EVAL_QUESTIONS = [
+    "What is this project and what is its current state?",
+    "What are the key technical decisions made so far?",
+    "What should I know before making changes to this codebase?",
+]
+
+_JUDGE_PROMPT = """\
+You are evaluating whether an AI memory system gave a useful answer to a developer.
+
+Question: {question}
+
+Answer received:
+{answer}
+
+Rate the answer on a scale from 1 to 5:
+1 = useless (empty, generic, no project-specific info)
+2 = weak (vague, mostly generic)
+3 = ok (some specific info but incomplete)
+4 = good (specific and actionable)
+5 = excellent (highly specific, immediately actionable, would save real time)
+
+Respond with JSON only: {{"score": <1-5>, "reason": "<one sentence>"}}"""
+
+
+
+def _list_projects(promoted_root: str) -> list[str]:
+    root = Path(promoted_root)
+    if not root.exists():
+        return []
+    return sorted(p.stem for p in root.glob("*.json"))
+
+
+def _heuristic(project_id: str, promoted_root: str) -> dict:
+    store = LocalPromotedMemoryStore(promoted_root)
+    records = store.load_records(project_id)
+    active = [r for r in records if r.status == "active"]
+    inactive = [r for r in records if r.status != "active"]
+
+    confidences = [r.confidence for r in active]
+    avg_conf = sum(confidences) / len(confidences) if confidences else 0.0
+    kinds = {r.kind for r in active}
+
+    score = "❌"
+    if len(active) >= 3 and avg_conf >= 0.7 and len(kinds) >= 2:
+        score = "✅"
+    elif len(active) >= 1:
+        score = "⚠️ "
+
+    return {
+        "score": score,
+        "active": len(active),
+        "inactive": len(inactive),
+        "avg_conf": avg_conf,
+        "kinds": sorted(kinds),
+    }
+
+
+def _ask(project_id: str, question: str, promoted_root: str, buffer_dir: str) -> str:
+    try:
+        answer = answer_from_promoted_memory(
+            project_id=project_id,
+            question=question,
+            buffer_dir=buffer_dir,
+            promoted_root=promoted_root,
+        )
+        return answer.answer or ""
+    except Exception as e:
+        return f"[error: {e}]"
+
+
+def _judge(question: str, answer_text: str, timeout: float = 60.0) -> dict:
+    if not answer_text or answer_text.startswith("[error"):
+        return {"score": 1, "reason": "no answer returned"}
+    prompt = _JUDGE_PROMPT.format(question=question, answer=answer_text[:800])
+    payload = {
+        "model": _OLLAMA_MODEL,
+        "messages": [{"role": "user", "content": prompt}],
+        "stream": False,
+        "format": "json",
+        "options": {"temperature": 0.1},
+    }
+    try:
+        with httpx.Client(timeout=timeout) as client:
+            r = client.post(_OLLAMA_URL, json=payload)
+            r.raise_for_status()
+        content = (r.json().get("message") or {}).get("content") or ""
+        return json.loads(content)
+    except Exception as e:
+        return {"score": 0, "reason": f"ollama error: {e}"}
+
+
+def _bar(score: int, max_score: int = 5) -> str:
+    filled = round(score / max_score * 10)
+    return "█" * filled + "░" * (10 - filled)
+
+
+def run_eval(
+    projects: list[str],
+    promoted_root: str,
+    buffer_dir: str,
+    use_llm: bool = True,
+    verbose: bool = False,
+) -> list[dict]:
+    results = []
+    for pid in projects:
+        print(f"\n{'─'*60}")
+        print(f"  Project: {pid}")
+        print(f"{'─'*60}")
+
+        # --- Heuristic pass ---
+        h = _heuristic(pid, promoted_root)
+        print(f"  Memories : {h['active']} active  ({h['inactive']} inactive)")
+        print(f"  Avg conf : {h['avg_conf']:.2f}")
+        print(f"  Kinds    : {', '.join(h['kinds']) or 'none'}")
+        print(f"  Heuristic: {h['score']}")
+
+        # --- LLM judge pass ---
+        llm_scores = []
+        if use_llm:
+            print()
+            for q in _EVAL_QUESTIONS:
+                print(f"  Q: {q}")
+                ans = _ask(pid, q, promoted_root, buffer_dir)
+                verdict = _judge(q, ans)
+                s = verdict.get("score", 0)
+                llm_scores.append(s)
+                bar = _bar(s)
+                print(f"     [{bar}] {s}/5 — {verdict.get('reason', '')}")
+                if verbose and ans:
+                    print(f"     Answer: {ans[:200]}")
+                print()
+
+        avg_llm = sum(llm_scores) / len(llm_scores) if llm_scores else None
+        results.append({
+            "project_id": pid,
+            "heuristic": h,
+            "llm_avg": avg_llm,
+            "llm_scores": llm_scores,
+        })
+
+    return results
+
+
+def _summary(results: list[dict], use_llm: bool) -> None:
+    print(f"\n{'═'*60}")
+    print("  SUMMARY")
+    print(f"{'═'*60}")
+    header = f"  {'Project':<30} {'Mem':>4} {'Conf':>5} {'H':>3}"
+    if use_llm:
+        header += f"  {'LLM':>5}"
+    print(header)
+    print(f"  {'─'*56}")
+    for r in results:
+        h = r["heuristic"]
+        row = f"  {r['project_id']:<30} {h['active']:>4} {h['avg_conf']:>5.2f} {h['score']:>3}"
+        if use_llm and r["llm_avg"] is not None:
+            row += f"  {r['llm_avg']:>5.1f}"
+        print(row)
+
+    if use_llm:
+        all_scores = [s for r in results for s in r["llm_scores"]]
+        if all_scores:
+            overall = sum(all_scores) / len(all_scores)
+            bar = _bar(round(overall))
+            print(f"\n  Overall LLM score: {bar} {overall:.1f}/5")
+            if overall >= 4:
+                print("  → Memory quality is GOOD. CherryDocs is delivering value.")
+            elif overall >= 2.5:
+                print("  → Memory quality is OK. Run /bootstrap-project to improve.")
+            else:
+                print("  → Memory quality is WEAK. Sessions may not have been distilled yet.")
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(description="Evaluate CherryDocs memory quality across projects")
+    p.add_argument("--project", help="Evaluate a single project (default: all)")
+    p.add_argument("--promoted-root", default=DEFAULT_PROMOTED_ROOT)
+    p.add_argument("--buffer-dir", default=_DEFAULT_BUFFER)
+    p.add_argument("--no-llm", action="store_true", help="Heuristic only (no Ollama)")
+    p.add_argument("--verbose", action="store_true", help="Print answer text")
+    return p
+
+
+def main() -> int:
+    args = _parser().parse_args()
+
+    promoted_root = args.promoted_root
+    buffer_dir = args.buffer_dir
+    use_llm = not args.no_llm
+
+    projects = [args.project] if args.project else _list_projects(promoted_root)
+    if not projects:
+        print(f"No projects found in {promoted_root}")
+        print("Run /bootstrap-project in a Claude Code session first.")
+        return 1
+
+    print(f"\n🍒 CherryDocs Eval — {len(projects)} project(s)")
+    print(f"   Promoted root: {promoted_root}")
+    print(f"   Buffer dir:    {buffer_dir}")
+    print(f"   LLM judge:     {'on (' + _OLLAMA_MODEL + ')' if use_llm else 'off'}")
+
+    results = run_eval(projects, promoted_root, buffer_dir, use_llm=use_llm, verbose=args.verbose)
+    _summary(results, use_llm)
+    print()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/install.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""cherry install — wire CherryDocs into Claude Code globally.
+
+Works two ways:
+  A) Package install (recommended):
+       pip install cherry-docs
+       cherry install
+
+  B) Source install (development):
+       python /path/to/cherry-docs/scripts/install.py
+
+What it does:
+  1. Adds cherry-docs-mcp to Claude Code user scope
+     (every project gets log_activity / onboard / answer / save_checkpoint)
+  2. Adds cherry-hook capture hooks to ~/.claude/settings.json
+     (every session is captured, auto-distilled on stop)
+  3. Creates ~/.cherrydocs/ as the central store
+  4. Migrates any project-local .cherrydocs/promoted/ data to the central store
+
+After install, open any project in Claude Code — CherryDocs is active.
+Run /bootstrap-project on existing projects to seed their memory.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+CHERRY_ROOT = Path(__file__).resolve().parent.parent
+GLOBAL_SETTINGS = Path.home() / ".claude" / "settings.json"
+CENTRAL_DIR = Path.home() / ".cherrydocs"
+CENTRAL_CAPTURE = CENTRAL_DIR / "capture"
+CENTRAL_PROMOTED = CENTRAL_DIR / "promoted"
+
+
+def _run(cmd: list[str], check: bool = True) -> subprocess.CompletedProcess:
+    return subprocess.run(cmd, capture_output=True, text=True, check=check)
+
+
+def _mcp_command() -> list[str]:
+    """Return the command to launch the MCP server.
+
+    Prefers the installed `cherry-docs-mcp` entry point.
+    Falls back to `python mcp_server.py` from source root.
+    """
+    ep = shutil.which("cherry-docs-mcp")
+    if ep:
+        return [ep]
+    mcp_server = CHERRY_ROOT / "mcp_server.py"
+    return [sys.executable, str(mcp_server)]
+
+
+def _hook_command(subcommand: str) -> str:
+    """Return the shell command string for a hook subcommand.
+
+    Sets both CHERRY_CAPTURE_BUFFER_DIR and CHERRY_PROMOTED_ROOT so all
+    hook subprocesses (including auto-distill) use the central store.
+
+    Prefers the installed `cherry-hook` entry point.
+    Falls back to `python state_manager.py` from source root.
+    """
+    env = (
+        f"CHERRY_CAPTURE_BUFFER_DIR={CENTRAL_CAPTURE}"
+        f" CHERRY_PROMOTED_ROOT={CENTRAL_PROMOTED}"
+    )
+    ep = shutil.which("cherry-hook")
+    if ep:
+        return f"{env} {ep} {subcommand}"
+    sm = CHERRY_ROOT / "scripts" / "claude_hooks" / "state_manager.py"
+    return f"{env} {sys.executable} {sm} {subcommand}"
+
+
+def _install_mcp() -> bool:
+    """Add cherry-docs MCP server at user scope via claude CLI."""
+    print("→ Adding cherry-docs MCP server (user scope)…")
+    result = _run(["claude", "mcp", "get", "cherry-docs"], check=False)
+    if "user" in result.stdout.lower():
+        print("  ✓ Already installed at user scope.")
+        return True
+
+    # Remove project/local-scoped entries (they would shadow user scope)
+    _run(["claude", "mcp", "remove", "cherry-docs", "-s", "project"], check=False)
+    _run(["claude", "mcp", "remove", "cherry-docs", "-s", "local"], check=False)
+
+    cmd = _mcp_command()
+    result = _run(
+        ["claude", "mcp", "add", "cherry-docs", *cmd, "--scope", "user"],
+        check=False,
+    )
+    if result.returncode != 0:
+        print(f"  ✗ claude mcp add failed: {result.stderr.strip()}")
+        print(f"    Add manually: claude mcp add cherry-docs {' '.join(cmd)} --scope user")
+        return False
+
+    print(f"  ✓ MCP server registered ({' '.join(cmd)}).")
+    return True
+
+
+def _build_hooks() -> dict:
+    return {
+        "UserPromptSubmit": [{"hooks": [
+            {"type": "command", "command": _hook_command("session-start")},
+        ]}],
+        "PostToolUse": [{
+            "matcher": "Edit|Write|NotebookEdit|Bash|mcp__cherry-docs__log_activity|mcp__cherry-docs__save_checkpoint",
+            "hooks": [{"type": "command", "command": _hook_command("post-tool-use")}],
+        }],
+        "Stop": [{"hooks": [
+            {"type": "command", "command": _hook_command("stop")},
+        ]}],
+    }
+
+
+def _install_hooks() -> None:
+    """Merge CherryDocs hooks into ~/.claude/settings.json."""
+    print("→ Writing capture hooks to ~/.claude/settings.json…")
+    settings: dict = {}
+    if GLOBAL_SETTINGS.exists():
+        try:
+            settings = json.loads(GLOBAL_SETTINGS.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            pass
+
+    existing = settings.get("hooks", {})
+    new_hooks = _build_hooks()
+
+    for event, matchers in new_hooks.items():
+        current = existing.get(event, [])
+        # Drop stale CherryDocs hook entries (identified by state_manager or cherry-hook)
+        current = [
+            m for m in current
+            if not any(
+                "state_manager.py" in h.get("command", "") or "cherry-hook" in h.get("command", "")
+                for h in m.get("hooks", [])
+            )
+        ]
+        existing[event] = matchers + current
+
+    settings["hooks"] = existing
+    GLOBAL_SETTINGS.parent.mkdir(parents=True, exist_ok=True)
+    GLOBAL_SETTINGS.write_text(json.dumps(settings, indent=2), encoding="utf-8")
+    print("  ✓ Hooks installed.")
+
+
+def _create_store() -> None:
+    """Create central ~/.cherrydocs/ directory structure."""
+    print(f"→ Creating central store at {CENTRAL_DIR}…")
+    CENTRAL_CAPTURE.mkdir(parents=True, exist_ok=True)
+    CENTRAL_PROMOTED.mkdir(parents=True, exist_ok=True)
+    print("  ✓ Store ready.")
+
+
+def _migrate_local_promoted() -> None:
+    """Copy any project-local .cherrydocs/promoted/*.json to the central store.
+
+    Before this fix, cherry-docs-mcp wrote promoted memories relative to its
+    working directory (.cherrydocs/promoted/). This migrates that data to the
+    central ~/.cherrydocs/promoted/ so it isn't lost.
+    """
+    local_promoted = CHERRY_ROOT / ".cherrydocs" / "promoted"
+    if not local_promoted.exists():
+        return
+    jsons = list(local_promoted.glob("*.json"))
+    if not jsons:
+        return
+    print(f"→ Migrating {len(jsons)} project file(s) from local store to central store…")
+    for src in jsons:
+        dst = CENTRAL_PROMOTED / src.name
+        if dst.exists():
+            print(f"  ↷ Skipping {src.name} (already in central store)")
+        else:
+            shutil.copy2(src, dst)
+            print(f"  ✓ Migrated {src.name}")
+    print("  ✓ Migration complete.")
+
+
+def main() -> int:
+    print(f"\nCherryDocs install — v{_version()}\n")
+
+    _create_store()
+    _migrate_local_promoted()
+    _install_mcp()
+    _install_hooks()
+
+    print("""
+✅ CherryDocs is now active for ALL Claude Code projects.
+
+Next steps:
+  • Open any project in Claude Code — hooks + MCP are live immediately
+  • Run:  /bootstrap-project
+    (reads codebase, seeds 10-20 memories via log_activity)
+  • Use 'cherry status' to verify at any time
+
+Project IDs are detected from git remote (e.g. freebeiro-myapp)
+or fall back to the directory name.
+""")
+    return 0
+
+
+def _version() -> str:
+    try:
+        from importlib.metadata import version
+        return version("cherry-docs")
+    except Exception:
+        return "dev"
+
+
+if __name__ == "__main__":
+    sys.exit(main())