foldcrumbs 0.3.0__py3-none-any.whl

foldcrumbs/__init__.py

@@ -0,0 +1,9 @@
+"""foldcrumbs — persistent cross-session memory for coding agents.
+
+File-based memory store + lifecycle hooks for Claude Code. No external
+service, no vector DB: retrieval is done by the agent's own grep over the
+memory folder, and a local OpenAI-compatible LLM is used only for async
+distillation. Pure stdlib, so hook scripts never fail on a missing import.
+"""
+
+__version__ = "0.3.0"

foldcrumbs/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

foldcrumbs/audit.py

@@ -0,0 +1,117 @@
+"""Store integrity audit + pruning.
+
+Link integrity: the index links to real files on disk (``store.rebuild_index``
+is path-based), so a dead link — or an active memory the index doesn't link —
+just means the index is stale; ``heal_index`` rebuilds it. Pollution: a memory
+whose title/content is a structural tooling artifact (markdown table, code
+fence, status glyphs, the local-command caveat — distill's strict detector) is
+never durable knowledge and can be pruned. The strict detector deliberately
+excludes prose that merely mentions MEMORY.md so legitimate foldcrumbs design notes
+are never deleted. Superseded/deleted records keep their
+files but drop out of the index/recall; ``prune`` clears those too.
+"""
+
+from __future__ import annotations
+
+import re
+
+from . import config, store
+from .distill import _is_hard_artifact
+
+_LINK_RE = re.compile(r"\]\(([^)]+\.md)\)")
+# compute_confidence below this is low-trust (stale/contradicted); prune only on
+# explicit request, never automatically.
+STALE_CONF = 0.3
+
+
+def _name(m) -> str:
+    return m.source_path or m.filename()
+
+
+def _index_links(cwd=None) -> set[str]:
+    p = config.index_path(cwd)
+    if not p.exists():
+        return set()
+    try:
+        return set(_LINK_RE.findall(p.read_text(encoding="utf-8")))
+    except OSError:
+        return set()
+
+
+def audit(cwd=None) -> dict:
+    """Read-only report: dead index links, orphaned active memories (on disk but
+    unlinked), artifact pollution, and low-trust/stale memories."""
+    linked = _index_links(cwd)
+    mems = list(store.iter_memories(cwd))
+    active = [m for m in mems if m.status == "active"]
+    on_disk = {_name(m) for m in mems}
+    active_names = {_name(m) for m in active}
+    return {
+        "dead_links": sorted(t for t in linked if t not in on_disk),
+        "orphans": sorted(n for n in active_names if n not in linked),
+        "pollution": sorted(_name(m) for m in active
+                            if _is_hard_artifact(m.title) or _is_hard_artifact(m.content)),
+        "stale": sorted(_name(m) for m in active
+                        if m.compute_confidence() < STALE_CONF),
+        "active": len(active),
+        "total": len(mems),
+    }
+
+
+def heal_index(cwd=None) -> bool:
+    """Rebuild the index if it is stale (dead links or unlinked active memories).
+
+    Cheap and idempotent; returns True iff it rebuilt. Callers that share a store
+    across machines should gate this on ``config.distill_enabled()`` so only a
+    writing machine repairs (avoids sync churn)."""
+    a = audit(cwd)
+    if a["dead_links"] or a["orphans"]:
+        store.rebuild_index(cwd)
+        return True
+    return False
+
+
+def _delete(name: str, cwd=None) -> bool:
+    try:
+        (config.memory_dir(cwd) / name).unlink()
+        return True
+    except OSError:
+        return False
+
+
+def prune_artifacts(cwd=None) -> list[str]:
+    """Delete active memories whose text is a clear tooling artifact, then rebuild
+    the index. Conservative — only unambiguous artifacts. Returns deleted names."""
+    removed = [
+        _name(m)
+        for m in list(store.iter_memories(cwd))
+        if m.status == "active" and (_is_hard_artifact(m.title) or _is_hard_artifact(m.content))
+    ]
+    removed = [n for n in removed if _delete(n, cwd)]
+    if removed:
+        store.rebuild_index(cwd)
+    return removed
+
+
+def prune(cwd=None, apply: bool = False, include_stale: bool = False) -> dict:
+    """Find (and with ``apply``, delete) prune candidates.
+
+    Candidates: superseded/deleted records (files left behind), active artifact
+    pollution, and — only with ``include_stale`` — low-trust active memories.
+    Dry-run by default; rebuilds the index when it deletes anything."""
+    candidates: dict[str, str] = {}
+    for m in store.iter_memories(cwd):
+        name = _name(m)
+        if m.status in ("deleted", "superseded"):
+            candidates[name] = "superseded/deleted"
+        elif m.status == "active" and (_is_hard_artifact(m.title) or _is_hard_artifact(m.content)):
+            candidates[name] = "artifact"
+        elif (include_stale and m.status == "active"
+              and m.compute_confidence() < STALE_CONF):
+            candidates[name] = "stale"
+    removed: list[str] = []
+    if apply and candidates:
+        removed = [n for n in candidates if _delete(n, cwd)]
+        if removed:
+            store.rebuild_index(cwd)
+    return {"candidates": candidates, "removed": removed, "applied": apply}

foldcrumbs/cli.py

@@ -0,0 +1,358 @@
+"""foldcrumbs CLI (stdlib argparse).
+
+Commands:
+  remember   store a memory
+  recall     search the store (substring + fuzzy) and render a context block
+  index      rebuild MEMORY.md
+  distill    distill a transcript/text file into memories (uses the LLM)
+  status     show config + store stats
+  install    merge hooks into Claude Code settings.json
+  migrate    move a legacy engram install to foldcrumbs
+  uninstall  remove foldcrumbs hooks
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import config, distill, install, llm, store
+from .profile import format_context_block
+from .schema import VALID_TYPES, MemoryRecord
+
+
+def _cmd_remember(args: argparse.Namespace) -> int:
+    rec = MemoryRecord(
+        title=args.title or args.text[:80],
+        content=args.text,
+        type=args.type,
+        confidence=args.confidence,
+        provenance="explicit_statement",
+        source="cli",
+        tags=args.tag or [],
+    )
+    action, path = store.upsert(rec)
+    store.rebuild_index()
+    print(f"{action}: {path}")
+    return 0
+
+
+def _cmd_recall(args: argparse.Namespace) -> int:
+    top = store.search(args.query, limit=args.limit)
+    block = format_context_block(top, heading=args.query)
+    print(block or "(no matching memories)")
+    return 0
+
+
+def _cmd_answer(args: argparse.Namespace) -> int:
+    mems = store.search(args.question, limit=args.limit)
+    if not mems:
+        print("(no relevant memories found)")
+        return 0
+    context = "\n".join(f"- [{m.type}] {m.content}" for m in mems)
+    out = llm.chat(
+        messages=[
+            {"role": "system", "content": "Answer the question using ONLY the "
+             "provided project memories. If they don't cover it, say so."},
+            {"role": "user", "content": f"Memories:\n{context}\n\nQuestion: {args.question}"},
+        ],
+        temperature=0.1,
+    )
+    print(out or f"(LLM unavailable — relevant memories)\n{context}")
+    return 0
+
+
+def _cmd_checkpoint(args: argparse.Namespace) -> int:
+    text = Path(args.file).read_text(encoding="utf-8") if args.file else sys.stdin.read()
+    handoff = distill.make_handoff(text)
+    if not handoff:
+        print("(nothing to checkpoint)")
+        return 0
+    path = store.write_handoff(handoff)
+    print(f"handoff written: {path}")
+    return 0
+
+
+def _cmd_handoff(_: argparse.Namespace) -> int:
+    print(store.read_handoff() or "(no handoff yet)")
+    return 0
+
+
+def _cmd_index(_: argparse.Namespace) -> int:
+    print(f"rebuilt: {store.rebuild_index()}")
+    return 0
+
+
+def _cmd_distill(args: argparse.Namespace) -> int:
+    text = Path(args.file).read_text(encoding="utf-8") if args.file else sys.stdin.read()
+    if not llm.available():
+        print("warning: LLM endpoint unreachable — using heuristic fallback",
+              file=sys.stderr)
+    res = distill.distill_and_store(text, source="cli-distill")
+    print(f"distilled: {res}")
+    return 0
+
+
+def _cmd_doctor(_: argparse.Namespace) -> int:
+    from foldcrumbs import audit
+    a = audit.audit()
+    print(f"memories   : {a['active']} active / {a['total']} total")
+    print(f"dead links : {len(a['dead_links'])}" + (f"  {a['dead_links']}" if a['dead_links'] else ""))
+    print(f"orphans    : {len(a['orphans'])}" + (f"  {a['orphans']}" if a['orphans'] else ""))
+    print(f"pollution  : {len(a['pollution'])}" + (f"  {a['pollution']}" if a['pollution'] else ""))
+    print(f"low-trust  : {len(a['stale'])}" + (f"  {a['stale']}" if a['stale'] else ""))
+    if a["dead_links"] or a["orphans"]:
+        print("hint: run `foldcrumbs index` to rebuild, or `foldcrumbs doctor` after a distill.")
+    if a["pollution"]:
+        print("hint: run `foldcrumbs prune` (dry-run) then `foldcrumbs prune --apply`.")
+    return 0
+
+
+def _cmd_prune(args: argparse.Namespace) -> int:
+    from foldcrumbs import audit
+    res = audit.prune(apply=args.apply, include_stale=args.include_stale)
+    if not res["candidates"]:
+        print("nothing to prune.")
+        return 0
+    for name, reason in sorted(res["candidates"].items()):
+        mark = "removed" if name in res["removed"] else ("would remove" if not args.apply else "kept")
+        print(f"  [{reason}] {name} — {mark}")
+    if not args.apply:
+        print(f"\n{len(res['candidates'])} candidate(s). Re-run with --apply to delete.")
+    else:
+        print(f"\nremoved {len(res['removed'])} file(s); index rebuilt.")
+    return 0
+
+
+def _cmd_status(_: argparse.Namespace) -> int:
+    mems = store.load_all()
+    active = [m for m in mems if m.status == "active"]
+    print(f"memory dir : {config.memory_dir()}")
+    print(f"index      : {config.index_path()}")
+    print(f"memories   : {len(active)} active / {len(mems)} total")
+    backend = config.llm_backend()
+    if backend == "claude-cli":
+        print(f"LLM backend: claude-cli ({config.claude_bin()})")
+    elif backend == "codex":
+        print(f"LLM backend: codex ({config.codex_bin()})")
+    elif backend in config._NO_LLM_BACKENDS:
+        print("LLM backend: none — keyword heuristic only")
+    else:
+        print(f"LLM backend: openai — {config.LLM_ENDPOINT} (model {config.LLM_MODEL})")
+    print(f"LLM reachable: {llm.available()}")
+    print(f"distill here : {'on' if config.distill_enabled() else 'off (read-only consumer)'}")
+    print(f"context budget: {config.CONTEXT_BUDGET} @ {int(config.CONTEXT_PCT*100)}%")
+    return 0
+
+
+def _cmd_migrate(args: argparse.Namespace) -> int:
+    """Migrate a legacy engram install to foldcrumbs (non-destructive).
+
+    1. State dir: copy ~/.engram -> ~/.foldcrumbs (backend choice, CLI bins,
+       checkpoint flags) when the new dir doesn't exist yet. The source is never
+       deleted, so a machine can be rolled back.
+    2. Memory (opt-in): with --from <old-project-dir>, copy that project's memory
+       store into the *current* project's memory dir (deterministic slug). Never
+       overwrites a non-empty target unless --force; never deletes the source.
+
+    Recall still needs no LLM; this only moves files. Back-compat in config.py
+    means foldcrumbs already reads a legacy ~/.engram, so this is about making the
+    move explicit, not about restoring function.
+    """
+    import shutil
+
+    old_state = Path.home() / ".engram"
+    new_state = Path.home() / ".foldcrumbs"
+    if new_state.exists():
+        print(f"state : {new_state} already exists — skipped")
+    elif old_state.exists():
+        shutil.copytree(old_state, new_state)
+        print(f"state : copied {old_state} -> {new_state}")
+    else:
+        print("state : no ~/.engram to migrate")
+
+    if args.from_dir:
+        src = config.memory_dir(args.from_dir)
+        dst = config.memory_dir()
+        if not src.exists():
+            print(f"memory: source {src} not found — nothing to copy")
+        elif src.resolve() == dst.resolve():
+            print(f"memory: source and target are the same ({dst}) — skipped")
+        elif dst.exists() and any(dst.iterdir()) and not args.force:
+            print(f"memory: target {dst} not empty — pass --force to merge")
+            return 1
+        else:
+            dst.mkdir(parents=True, exist_ok=True)
+            for item in src.iterdir():
+                target = dst / item.name
+                if item.is_dir():
+                    shutil.copytree(item, target, dirs_exist_ok=True)
+                else:
+                    shutil.copy2(item, target)
+            print(f"memory: copied {src} -> {dst}")
+    else:
+        print(f"memory: (skipped) pass --from <old-project-dir> to copy its store "
+              f"into {config.memory_dir()}")
+    print("done. reinstall hooks with `foldcrumbs install` if not already.")
+    return 0
+
+
+def _cmd_install(args: argparse.Namespace) -> int:
+    agent = args.agent
+    if agent == "opencode":
+        paths = install.opencode_paths(global_scope=not args.local)
+        mcp = install.install_opencode_mcp(paths["config"])
+        plugin = install.write_opencode_plugin(paths["plugins"])
+        agents = install.append_agents_md(paths["agents"])
+        print(f"opencode.json mcp: {mcp or '(already present)'} ({paths['config']})")
+        print(f"plugin: {plugin}")
+        print(f"AGENTS.md: {agents or '(block already present)'}")
+        return 0
+
+    path = Path(args.settings) if args.settings else install.default_settings_path(
+        agent=agent, global_scope=not args.local
+    )
+    changes = install.install_hooks(path, agent=agent)
+    print(f"settings: {path}")
+    print("added:", changes or "(nothing — already installed)")
+    if agent == "codex":
+        print("codex MCP (config.toml):", install.install_codex_mcp_toml())
+    _configure_backend_at_install(args)
+    return 0
+
+
+def _configure_backend_at_install(args: argparse.Namespace) -> None:
+    """Pick the LLM distillation backend during install.
+
+    Explicit ``--backend`` wins. Otherwise prompt interactively when on a TTY;
+    when non-interactive (piped/CI) leave the existing choice untouched and say
+    how to set it later.
+    """
+    if getattr(args, "no_backend_prompt", False):
+        return
+    choice = getattr(args, "backend", None)
+    if not choice:
+        if not (sys.stdin.isatty() and sys.stdout.isatty()):
+            print("LLM backend: left as-is "
+                  f"({config.llm_backend()}); set later with `foldcrumbs backend <name>`")
+            return
+        choice = install.prompt_backend()
+    if not choice:
+        return
+    written = install.configure_backend(choice)
+    print(f"LLM backend: {choice} -> wrote {', '.join(written)} in {config.STATE_DIR}")
+
+
+def _cmd_backend(args: argparse.Namespace) -> int:
+    """Set (or, with no argument, show) the machine-local LLM backend."""
+    if not args.choice:
+        backend = config.llm_backend()
+        print(f"LLM backend: {backend}")
+        print(f"reachable  : {llm.available()}")
+        print("choices    :", ", ".join(k for k, _ in install.BACKEND_CHOICES))
+        return 0
+    written = install.configure_backend(
+        args.choice, bin_path=args.bin, endpoint=args.endpoint, model=args.model)
+    print(f"LLM backend: {args.choice} -> wrote {', '.join(written)} in {config.STATE_DIR}")
+    print(f"reachable  : {llm.available()}")
+    return 0
+
+
+def _cmd_uninstall(args: argparse.Namespace) -> int:
+    path = Path(args.settings) if args.settings else install.default_settings_path(
+        agent=args.agent, global_scope=not args.local
+    )
+    removed = install.uninstall_hooks(path)
+    print(f"removed from {path}: {removed or '(nothing)'}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="foldcrumbs", description=__doc__)
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    r = sub.add_parser("remember", help="store a memory")
+    r.add_argument("text")
+    r.add_argument("--type", default="fact", choices=sorted(VALID_TYPES))
+    r.add_argument("--title", default="")
+    r.add_argument("--confidence", type=float, default=0.85)
+    r.add_argument("--tag", action="append")
+    r.set_defaults(func=_cmd_remember)
+
+    rc = sub.add_parser("recall", help="search the store")
+    rc.add_argument("query")
+    rc.add_argument("--limit", type=int, default=10)
+    rc.set_defaults(func=_cmd_recall)
+
+    an = sub.add_parser("answer", help="answer a question grounded in memory (LLM)")
+    an.add_argument("question")
+    an.add_argument("--limit", type=int, default=8)
+    an.set_defaults(func=_cmd_answer)
+
+    sub.add_parser("index", help="rebuild MEMORY.md").set_defaults(func=_cmd_index)
+
+    cp = sub.add_parser("checkpoint", help="write a working-state handoff (LLM)")
+    cp.add_argument("file", nargs="?", help="transcript/text file (default: stdin)")
+    cp.set_defaults(func=_cmd_checkpoint)
+
+    sub.add_parser("handoff", help="print the current handoff").set_defaults(
+        func=_cmd_handoff)
+
+    d = sub.add_parser("distill", help="distill a transcript/text into memories")
+    d.add_argument("file", nargs="?", help="path to text file (default: stdin)")
+    d.set_defaults(func=_cmd_distill)
+
+    sub.add_parser("status", help="show config + stats").set_defaults(func=_cmd_status)
+
+    mg = sub.add_parser("migrate", help="migrate a legacy engram install to foldcrumbs")
+    mg.add_argument("--from", dest="from_dir", metavar="OLD_PROJECT_DIR",
+                    help="also copy this project's memory store into the current one")
+    mg.add_argument("--force", action="store_true",
+                    help="merge into a non-empty target memory dir")
+    mg.set_defaults(func=_cmd_migrate)
+
+    sub.add_parser("doctor", help="audit store: dead links, orphans, pollution"
+                   ).set_defaults(func=_cmd_doctor)
+
+    pr = sub.add_parser("prune", help="delete pollution / superseded memories (dry-run by default)")
+    pr.add_argument("--apply", action="store_true", help="actually delete (default: dry-run)")
+    pr.add_argument("--include-stale", action="store_true",
+                    help="also prune low-trust memories")
+    pr.set_defaults(func=_cmd_prune)
+
+    ins = sub.add_parser("install", help="wire foldcrumbs into a coding agent")
+    ins.add_argument("--agent", choices=["claude", "codex", "opencode"], default="claude")
+    ins.add_argument("--local", action="store_true", help="project scope instead of global")
+    ins.add_argument("--settings", help="explicit settings.json path")
+    ins.add_argument("--backend", choices=list(config.BACKENDS),
+                     help="LLM distill backend (skip the interactive prompt)")
+    ins.add_argument("--no-backend-prompt", action="store_true",
+                     dest="no_backend_prompt",
+                     help="don't ask about / change the LLM backend")
+    ins.set_defaults(func=_cmd_install)
+
+    bk = sub.add_parser("backend", help="show or set the LLM distill backend")
+    bk.add_argument("choice", nargs="?", choices=list(config.BACKENDS),
+                    help="backend to select (omit to show current)")
+    bk.add_argument("--bin", help="explicit CLI path for claude-cli/codex")
+    bk.add_argument("--endpoint", help="HTTP endpoint for the openai backend")
+    bk.add_argument("--model", help="model id for the openai backend")
+    bk.set_defaults(func=_cmd_backend)
+
+    uns = sub.add_parser("uninstall", help="remove foldcrumbs hooks")
+    uns.add_argument("--agent", choices=["claude", "codex", "opencode"], default="claude")
+    uns.add_argument("--local", action="store_true")
+    uns.add_argument("--settings")
+    uns.set_defaults(func=_cmd_uninstall)
+
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())