memcrumbs 0.2.0__py3-none-any.whl

memcrumbs/__init__.py

@@ -0,0 +1,6 @@
+"""memcrumbs - breadcrumbs of project memory for coding agents: a wiki plus a token-efficient index.
+
+Layers: core (this package: wiki, index, outline, checks) -> packs (memcrumbs.packs.*: reusable domain add-ons)
+-> project (<projects_dir>/<name>/: memcrumbs.toml, wiki/, plugins/ tailored to one project).
+"""
+__version__ = "0.2.0"

memcrumbs/__main__.py

@@ -0,0 +1,3 @@
+from memcrumbs.cli import main
+
+main()

memcrumbs/bench.py

@@ -0,0 +1,138 @@
+"""`memcrumbs bench QUESTIONS.toml`: recall@1 / recall@3 / MRR, tokens per answer and latency, against any
+project (public or private) with no code changes.
+
+    [[q]]
+    id = "rb-01"
+    q = "how do I roll back a bad deploy?"
+    expect = ["wiki/runbooks/rollback.md"]      # path suffixes; any one in the top k counts
+    category = "runbook"                        # optional; results are also broken down by category
+
+A hit matches an expected path when its file path ends with it (`wiki/x.md`, `repo/app.py`); `path#Section`
+also requires the hit's section to contain that text. Tokens are the `find` output's chars/4 (estimate)."""
+import io
+import json
+import time
+import tomllib
+from contextlib import redirect_stdout
+from pathlib import Path
+
+from memcrumbs import cmd_read, writeset
+from memcrumbs.output import Out, est_tokens
+from memcrumbs.util import posix
+
+
+def load(path):
+    with open(path, "rb") as f:
+        qs = tomllib.load(f).get("q", [])
+    for i, q in enumerate(qs):
+        if not q.get("q") or not q.get("expect"):
+            raise SystemExit("bench: question %d needs q and expect" % (i + 1))
+        q.setdefault("id", "q%d" % (i + 1))
+    return qs
+
+
+def _match(hit, expect):
+    path = "/" + posix(hit["path"]).lstrip("/")
+    for e in expect:
+        target, _, section = e.partition("#")
+        if path.lower().endswith("/" + target.strip("/").lower()) and \
+                (not section or section.lower() in (hit.get("section") or "").lower()):
+            return True
+    return False
+
+
+def run(project, store, questions, k=8):
+    rows = []
+    for q in questions:
+        t0 = time.perf_counter()
+        hits = store.search(q["q"], kinds=q.get("kinds"), limit=k)
+        ms = (time.perf_counter() - t0) * 1000
+        buf = io.StringIO()
+        with redirect_stdout(buf):                                 # the exact text an agent would read
+            out = Out("find", None, project.redact)
+            cmd_read.find(out, project, store, q["q"], q.get("kinds"), k, track=False)
+            out.close()
+        rank = next((i + 1 for i, h in enumerate(hits) if _match(h, q["expect"])), None)
+        followed = next((i + 1 for i, h in enumerate(hits) if _match(h, q["expect"]) or
+                         (h.get("see") and _match(h["see"], q["expect"]))), None)
+        rows.append(dict(id=q["id"], q=q["q"], category=q.get("category", ""), paraphrase=bool(q.get("paraphrase")),
+                         rank=rank, followed=followed, ms=round(ms, 1), tokens=est_tokens(len(buf.getvalue())),
+                         top3=[project.rel(h["path"]) for h in hits[:3]]))
+    return rows
+
+
+def _agg(rows):
+    n = len(rows) or 1
+    ms, toks = sorted(r["ms"] for r in rows), sorted(r["tokens"] for r in rows)
+    pct = (lambda xs, p: xs[min(len(xs) - 1, int(p * len(xs)))] if xs else 0)
+    return dict(n=len(rows), recall_at_1=round(sum(1 for r in rows if r["rank"] == 1) / n, 3),
+                recall_at_3=round(sum(1 for r in rows if r["rank"] and r["rank"] <= 3) / n, 3),
+                recall_at_3_with_pointers=round(sum(1 for r in rows if r.get("followed") and r["followed"] <= 3) / n, 3),
+                mrr=round(sum(1 / r["rank"] for r in rows if r["rank"]) / n, 3),
+                tokens_mean=round(sum(toks) / n), tokens_p95=pct(toks, 0.95),
+                ms_mean=round(sum(ms) / n, 1), ms_p95=pct(ms, 0.95))
+
+
+def default_path(project):
+    """The project's own bench, grown from real use with `memcrumbs bench --add` (kept with the wiki)."""
+    return project.wiki / "bench.toml"
+
+
+def add(out, project, question, expects, category=""):
+    """Record a question the wiki should answer, with the file(s) that do. Run it when a lookup missed."""
+    from memcrumbs.curate import exact_page
+    if not expects:
+        raise SystemExit("memcrumbs: --add needs --expect <page or file>")
+    targets = []
+    for e in expects:
+        try:
+            targets.append(project.rel(exact_page(project, e)))
+        except SystemExit:
+            p = Path(e).expanduser().resolve()
+            if not p.exists():
+                raise SystemExit("memcrumbs: no page or file %r" % e)
+            try:
+                targets.append(posix(p.relative_to(project.dir)))
+            except ValueError:
+                targets.append(posix(p))
+    path = default_path(project)
+    head = "" if path.exists() else ("# Questions this wiki must answer (memcrumbs bench). Added with "
+                                      "`memcrumbs bench --add`.\n")
+    n = len(load(path)) + 1 if path.exists() else 1
+    entry = '%s\n[[q]]\nid = "real-%03d"\nq = %s\nexpect = [%s]\n%s' % (
+        head, n, json.dumps(project.redact(question), ensure_ascii=False), ", ".join(json.dumps(t) for t in targets),
+        ("category = %s\n" % json.dumps(category)) if category else "")
+    writeset.append_text(project, path, entry)
+    out.set(path=project.rel(path), id="real-%03d" % n, expect=targets)
+    out("added real-%03d to %s (%d questions) — memcrumbs bench to measure" % (n, project.rel(path), n))
+
+
+def report(out, project, store, path, min_recall3=None, k=8):
+    path = path or default_path(project)
+    if not Path(path).exists():
+        raise SystemExit("memcrumbs: no questions at %s — record real ones with: memcrumbs bench --add \"<question>\" "
+                         "--expect <page>" % posix(path))
+    rows = run(project, store, load(path), k)
+    total = _agg(rows)
+    cats = {c: _agg([r for r in rows if r["category"] == c]) for c in sorted({r["category"] for r in rows if r["category"]})}
+    para = _agg([r for r in rows if r["paraphrase"]])
+    misses = [r for r in rows if not r["rank"] or r["rank"] > 3]
+    out.list_key = "questions"
+    out.set(project=project.name, questions_file=posix(path), total=total, by_category=cats, paraphrase=para,
+            questions=rows)
+    out.hint("memcrumbs bench %s --json --full" % posix(path))
+    out("bench %s · %d questions · recall@1 %.3f · recall@3 %.3f · MRR %.3f" % (
+        project.name, total["n"], total["recall_at_1"], total["recall_at_3"], total["mrr"]))
+    out("recall@3 counting '→ see' pointers under a hit: %.3f" % total["recall_at_3_with_pointers"])
+    out("tokens/answer mean %d p95 %d (est.) · search latency mean %.1f ms p95 %.1f ms" % (
+        total["tokens_mean"], total["tokens_p95"], total["ms_mean"], total["ms_p95"]))
+    if para["n"]:
+        out("paraphrased questions: %d · recall@3 %.3f" % (para["n"], para["recall_at_3"]))
+    for c, a in cats.items():
+        out("  %-12s n=%-3d r@1 %.2f  r@3 %.2f" % (c, a["n"], a["recall_at_1"], a["recall_at_3"]))
+    for r in misses:
+        out("miss %s (rank %s): %s → %s" % (r["id"], r["rank"] or "-", r["q"], ", ".join(r["top3"]) or "no hits"))
+    if min_recall3 is not None and total["recall_at_3"] < min_recall3:
+        out("FAIL: recall@3 %.3f < %.3f" % (total["recall_at_3"], min_recall3))
+        return 1
+    return 0

memcrumbs/cites.py

@@ -0,0 +1,136 @@
+"""Reverse lookup: what the wiki records about a file. At index time every markdown doc's `sources:` and
+the file-like tokens in its text become citation rows; `about <path>` lists the pages, decision rows, incidents
+and gotchas that cite a file, verdicts first, and code hits and `outline` say "cited by: …" in one line."""
+import re
+from pathlib import Path
+
+EXT = ("py|pyw|ps1|psm1|psd1|js|mjs|cjs|ts|tsx|jsx|go|rs|cs|java|kt|rb|php|c|h|cpp|hpp|sql|sh|bash|bat|cmd|toml|json|"
+       "jsonc|yaml|yml|ini|cfg|conf|env|xml|csv|md|txt|mq5|mqh|mq4|service|timer|tf|kiln")
+PATH = re.compile(r"(?<![\w/\\.:@~-])((?:[A-Za-z]:[/\\]|~?/)?(?:\.{1,2}[/\\])*+(?:[\w.~-]+[/\\])*+[\w-][\w.-]*\.(?:%s))"
+                  r"(?![\w/\\-])" % EXT, re.I)
+CITING = ("decision", "incident", "runbook", "wiki", "journal", "doc", "research", "memory", "legacy", "bug")
+ORDER = {"decision": 0, "incident": 1, "runbook": 3, "wiki": 4, "journal": 6}
+HEADER_PAGES = {"gotchas": "gotcha", "rules": "rule"}
+NAMES = {"wiki": "page", "memory": "memory note", "legacy": "legacy note", "research": "research note",
+         "journal": "journal entry", "bug": "bug log entry", "doc": "doc"}       # every group named
+
+
+def norm(token):
+    """A cited path as stored: forward slashes, lower case, no leading ./ or ../ parts."""
+    t = token.replace("\\", "/").lower()
+    while t.startswith(("./", "../")):
+        t = t.split("/", 1)[1]
+    return t
+
+
+def tokens(text):
+    return {norm(m) for m in PATH.findall(text or "")}
+
+
+def rows(path, docs):
+    """Citation rows (target basename, token, page path, loc, kind, status, title, section) for one file's docs."""
+    out, seen = [], set()
+    for d in docs:
+        if d.kind not in CITING:
+            continue
+        for t in tokens(d.body) | tokens(getattr(d, "cites", "")) | tokens(d.section):
+            key = (t, d.loc)
+            if key not in seen:
+                seen.add(key)
+                out.append((t.rsplit("/", 1)[-1], t, path, d.loc, d.kind, (d.status or "").lower(), d.title, d.section))
+    return out
+
+
+def _matches(store, project, path):
+    """Rows citing `path`: a token with folders must be a suffix of the path; a bare name matches by name."""
+    p = Path(path)
+    rows = store.db.execute("SELECT token, path, loc, kind, status, title, section FROM cites WHERE target = ?",
+                            (p.name.lower(),)).fetchall()
+    if not rows:                                   # the usual case on a code hit: no filesystem calls at all
+        return []
+    if len(p.parts) == 1 and not p.exists():       # a bare file name: every citation of that name
+        return [dict(token=r[0], path=r[1], loc=r[2], kind=r[3], status=r[4], title=r[5], section=r[6], exact=True)
+                for r in rows]
+    forms = {p}
+    if p.exists() or p.is_absolute():               # as given and resolved: on macOS /var/… is /private/var/…
+        forms = {p.absolute(), p.resolve()}
+    rels = {f.as_posix().lower() for f in forms}
+    for root in list(getattr(project, "repos", [])) + [getattr(project, "dir", None)]:
+        if root is not None:
+            for f in forms:
+                for r in {Path(root), Path(root).resolve()}:
+                    try:
+                        rels.add(f.relative_to(r).as_posix().lower())
+                    except ValueError:
+                        pass
+    found = []
+    for r in rows:
+        token = r[0]
+        exact = "/" in token and any(x == token or x.endswith("/" + token) for x in rels)
+        if "/" in token and not exact:
+            continue
+        found.append(dict(token=token, path=r[1], loc=r[2], kind=r[3], status=r[4], title=r[5], section=r[6],
+                          exact=exact))
+    return found
+
+
+def _kind(h):
+    stem = Path(h["path"]).stem.lower()
+    return HEADER_PAGES.get(stem, h["kind"])
+
+
+def _bound_facts(project, path):
+    """Facts whose value is read from this file ([facts.bind.KEY] from = …): the file is their source."""
+    binds = (getattr(project, "cfg", {}).get("facts", {}) or {}).get("bind") or {}
+    if not binds:
+        return []
+    from memcrumbs import facts, freshness
+    try:
+        me = Path(path).resolve()
+    except OSError:
+        return []
+    out = []
+    for key, spec in binds.items():
+        if any(p.resolve() == me for p in freshness.source_paths(project, [spec.get("from", "")])):
+            out.append(dict(token=Path(path).name.lower(), path=str(facts.page_path(project)), loc=1, kind="fact",
+                            status="", title="fact %s (bound to this file)" % key, section="", exact=True, what="fact"))
+    return out
+
+
+def about(store, project, path):
+    """One entry per citing page (its best location), verdicts and incidents first."""
+    best = {}
+    for h in _matches(store, project, path):
+        cur = best.get(h["path"])
+        if cur is None or (bool(h["status"]), h["exact"], -h["loc"]) > (bool(cur["status"]), cur["exact"], -cur["loc"]):
+            best[h["path"]] = h
+    hits = list(best.values()) + _bound_facts(project, path)
+    for h in hits:
+        h["what"] = h.get("what") or _kind(h)
+    hits.sort(key=lambda h: (0 if h["status"] in ("rejected", "accepted", "parked", "superseded") else 1,
+                             {"gotcha": 2, "rule": 2}.get(h["what"], ORDER.get(h["kind"], 5)), not h["exact"], h["path"]))
+    return hits
+
+
+def summary(store, project, path):
+    """'1 decision (REJECTED), 1 incident, 1 gotcha' or '' — one line for code hits and outline. Only what changes
+    how a file is handled; plain pages that mention it are left to `about`."""
+    return describe([h for h in about(store, project, path) if h["what"] in ("decision", "incident", "gotcha", "rule")])
+
+
+def describe(hits):
+    if not hits:
+        return ""
+    counts = {}
+    for h in hits:
+        counts.setdefault(h["what"], []).append(h["status"])
+    parts, named = [], {}
+    for what in sorted(counts, key=lambda w: {"decision": 0, "incident": 1, "gotcha": 2, "rule": 3, "fact": 4}.get(w, 5)):
+        name = NAMES.get(what, what.replace("_", " "))
+        named.setdefault(name, []).extend(counts[what])
+    for name, statuses in named.items():
+        st = sorted({s.upper() for s in statuses if s})
+        n = len(statuses)
+        plural = (name[:-1] + "ies" if name.endswith("y") else name + "s") if n > 1 else name
+        parts.append("%d %s%s" % (n, plural, " (%s)" % "/".join(st) if st else ""))
+    return ", ".join(parts)

memcrumbs/claude_settings.py

@@ -0,0 +1,148 @@
+"""`.claude/settings.json` edits for `wire`: the SessionStart hook, and allow rules for the read-only lookups.
+
+Each edit is one textual insertion right after an opening brace or bracket (jsonedit.py), so `--remove` takes
+out exactly the inserted text and every other byte stays as it was. A hook written by memcrumbs 0.1 (whole file
+re-serialized by json.dumps) is removed at the JSON level when the file is still in that exact format."""
+import json
+
+from memcrumbs import jsonedit
+from memcrumbs.jsonedit import JsonEditError
+from memcrumbs.util import newline_of
+
+READ_ONLY = ("find", "page", "outline", "sym", "status", "about", "fact")   # never log/decision/incident/verify/plan
+SHELLS = ("Bash", "PowerShell")                              # Claude Code on Windows runs commands via PowerShell
+
+
+def hook_command(project):
+    return "memcrumbs -p %s status --hook" % project.name     # -p before the subcommand: valid for any argparse
+
+
+def hook_entry(project):
+    return {"matcher": "", "hooks": [{"type": "command", "command": hook_command(project), "timeout": 30}]}
+
+
+def allow_rules(project):
+    """Prefix rules for the read-only subcommands, with and without `-p NAME`, for both shell tools."""
+    return ["%s(memcrumbs %s%s:*)" % (sh, pre, cmd) for sh in SHELLS for pre in ("", "-p %s " % project.name)
+            for cmd in READ_ONLY]
+
+
+def ours(cmd):
+    return isinstance(cmd, str) and cmd.startswith("memcrumbs ") and " status" in cmd
+
+
+def has_hook(data):
+    hooks = data.get("hooks") if isinstance(data.get("hooks"), dict) else {}
+    return any(ours(h.get("command")) for e in hooks.get("SessionStart") or [] if isinstance(e, dict)
+               for h in e.get("hooks") or [] if isinstance(h, dict))
+
+
+def missing_allow(data, project):
+    perms = data.get("permissions") if isinstance(data.get("permissions"), dict) else {}
+    have = set(perms.get("allow") or [])
+    return [r for r in allow_rules(project) if r not in have]
+
+
+def _snippets(keys, entries):
+    k0, k1 = keys
+    for frag in ('"%s": {"%s": [%s]}' % (k0, k1, entries), '"%s": [%s]' % (k1, entries), entries):
+        for indent in ("  ", "    ", "      "):
+            for nl in ("\r\n", "\n"):                         # CRLF first: an LF snippet also matches inside it
+                for tail in (",", nl):
+                    yield nl + indent + frag + tail
+
+
+def insert(text, keys, entries):
+    """Insert `entries` (JSON array items as text) into data[k0][k1], creating the object and list as needed."""
+    k0, k1 = keys
+    top = jsonedit._ws(text, 0)
+    m = jsonedit.members(text, top)
+    if k0 not in m:
+        at, frag, indent = top, '"%s": {"%s": [%s]}' % (k0, k1, entries), "  "
+    elif text[m[k0][0]] == "{":
+        inner = jsonedit.members(text, m[k0][0])
+        if k1 not in inner:
+            at, frag, indent = m[k0][0], '"%s": [%s]' % (k1, entries), "    "
+        elif text[inner[k1][0]] == "[":
+            at, frag, indent = inner[k1][0], entries, "      "
+        else:
+            raise JsonEditError("%s.%s is not a list" % keys)
+    else:
+        raise JsonEditError("%s is not an object" % k0)
+    return jsonedit.insert_after_open(text, at, frag, indent, newline_of(text))[0]
+
+
+def take_out(text, keys, entries):
+    """Remove exactly what `insert` added; None if that text is not there (edited by hand)."""
+    for snip in _snippets(keys, entries):
+        if snip in text:
+            return text.replace(snip, "", 1)
+    return None
+
+
+def remove_items(text, keys, items):
+    """Fallback when an editor reformatted the list: drop each string in `items` from data[k0][k1] by its element
+    span, keeping every other byte. None if the list is missing or holds none of them."""
+    k0, k1 = keys
+    m = jsonedit.members(text, jsonedit._ws(text, 0))
+    if k0 not in m or text[m[k0][0]] != "{":
+        return None
+    inner = jsonedit.members(text, m[k0][0])
+    if k1 not in inner or text[inner[k1][0]] != "[":
+        return None
+    start, end = inner[k1]
+    spans, i = [], jsonedit._ws(text, start + 1)
+    while text[i] != "]":
+        j = jsonedit.value_end(text, i)
+        spans.append((i, j))
+        i = jsonedit._ws(text, j)
+        if text[i] == ",":
+            i = jsonedit._ws(text, i + 1)
+    want = set(items)
+    drop = [k for k, (a, b) in enumerate(spans) if text[a] == '"' and json.loads(text[a:b]) in want]
+    if not drop:
+        return None
+    runs = []
+    for k in drop:
+        if runs and runs[-1][1] == k - 1:
+            runs[-1][1] = k
+        else:
+            runs.append([k, k])
+    for lo, hi in reversed(runs):                     # an element goes with the separator after it (before, if last)
+        if hi < len(spans) - 1:
+            text = text[:spans[lo][0]] + text[spans[hi + 1][0]:]
+        elif lo > 0:
+            text = text[:spans[lo - 1][1]] + text[spans[hi][1]:]
+        else:
+            text = text[:start + 1] + text[end - 1:]
+    return text
+
+
+def hook_text(project):
+    return json.dumps(hook_entry(project))
+
+
+def allow_text(rules):
+    return ", ".join(json.dumps(r) for r in rules)
+
+
+def canonical_remove_hook(text, data):
+    """settings.json written by json.dumps(indent=2) (memcrumbs 0.1 did this): drop our hook at the JSON level and
+    re-serialize in the same style, which reproduces every other byte. None if the file is hand-formatted."""
+    nl = newline_of(text)
+    body = text.replace("\r\n", "\n")
+    for ascii_ in (True, False):
+        canon = json.dumps(data, indent=2, ensure_ascii=ascii_)
+        if body.rstrip("\n") != canon or len(body) - len(canon) > 1:
+            continue
+        hooks = data["hooks"]
+        kept = [e for e in hooks["SessionStart"] if not (isinstance(e, dict) and any(
+            ours(h.get("command")) for h in e.get("hooks") or [] if isinstance(h, dict)))]
+        if kept:
+            hooks["SessionStart"] = kept
+        else:
+            del hooks["SessionStart"]
+            if not hooks:
+                del data["hooks"]
+        return (json.dumps(data, indent=2, ensure_ascii=ascii_) + body[len(canon):]).replace("\n", nl)
+    return None