memware 0.1.1__py3-none-any.whl

memware/__init__.py

@@ -0,0 +1,25 @@
+"""memware — memory for AI agents that only remembers the latest truth.
+
+Two stores, one SQLite file:
+
+* **turns** — immutable evidence: what was said in past sessions, indexed with
+  FTS5 for cheap, model-free recall.
+* **beliefs** — a bi-temporal ledger of facts. A new value for the same
+  ``(subject, relation)`` key supersedes the old one; recall only ever returns
+  the currently valid belief. History is kept for audit, never surfaced.
+"""
+
+from memware.ledger import Outcome, Policy, approve, assert_belief, current, history, reject
+from memware.store import Store
+
+__all__ = [
+    "Outcome",
+    "Policy",
+    "Store",
+    "approve",
+    "assert_belief",
+    "current",
+    "history",
+    "reject",
+]
+__version__ = "0.1.1"

memware/cli.py

@@ -0,0 +1,311 @@
+"""``memware`` command-line interface. Every command is also usable from a hook:
+pass ``--from-hook`` to read the harness's JSON payload on stdin."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from memware import __version__
+from memware.index import (
+    read_turns,
+    search_beliefs,
+    search_beliefs_multi,
+    search_turns_multi,
+)
+from memware.ingest import capture_disabled, prune_sources, sync_file, sync_tree
+from memware.ledger import Policy, approve, assert_belief, current, history, reject
+from memware.review import HttpReviewBackend, JsonlReviewBackend, open_reviews, sync_reviews
+from memware.store import DEFAULT_DB, Store
+
+
+def _hook_payload() -> dict[str, object]:
+    try:
+        payload = json.load(sys.stdin) if not sys.stdin.isatty() else {}
+    except (json.JSONDecodeError, ValueError):
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _out(obj: object, as_json: bool) -> None:
+    if as_json:
+        print(json.dumps(obj, indent=2, default=str))
+    elif isinstance(obj, list):
+        for row in obj:
+            print(row if isinstance(row, str) else json.dumps(row, default=str))
+    else:
+        print(obj if isinstance(obj, str) else json.dumps(obj, default=str))
+
+
+def cmd_init(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        _out({"db": str(s.path), **s.stats()}, a.json)
+    return 0
+
+
+def cmd_sync(a: argparse.Namespace) -> int:
+    if a.from_hook and capture_disabled():
+        return 0  # MEMWARE_NO_CAPTURE=1: this run must not enter the store
+    paths = list(a.paths)
+    if a.from_hook:
+        tp = _hook_payload().get("transcript_path")
+        if tp:
+            paths.append(str(tp))
+    if not paths:
+        print("nothing to sync", file=sys.stderr)
+        return 0
+    with Store(a.db) as s:
+        report: dict[str, int] = {}
+        for p in paths:
+            path = Path(p).expanduser()
+            if path.is_dir():
+                report.update(
+                    sync_tree(
+                        s,
+                        path,
+                        harness=a.harness,
+                        skip_if_contains=a.skip_if_contains,
+                        exclude=a.exclude,
+                    )
+                )
+            elif path.exists():
+                report[str(path)] = sync_file(
+                    s, path, harness=a.harness, skip_if_contains=a.skip_if_contains
+                )
+        _out({"added": sum(report.values()), "files": len(report)}, a.json or a.from_hook)
+    return 0
+
+
+def cmd_recall(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        hits = []
+        if a.what in ("all", "beliefs"):
+            hits += search_beliefs_multi(s, a.queries, k=a.k, record_use=not a.no_touch)
+        if a.what in ("all", "turns"):
+            hits += search_turns_multi(
+                s, a.queries, k=a.k, record_use=not a.no_touch, snippet_tokens=a.snippet_tokens
+            )
+        rows = [
+            {
+                "kind": h.kind,
+                "id": h.id,
+                "score": round(h.score, 4),
+                "session": h.session,
+                "ts": h.ts,
+                "role": h.role,
+                "subject": h.subject,
+                "relation": h.relation,
+                "source": h.source,
+                "offset": h.offset,
+                "snippet": h.snippet,
+                "text": h.text if a.full else h.text[:300],
+            }
+            for h in hits
+        ]
+        _out(rows, a.json)
+    return 0
+
+
+def cmd_context(a: argparse.Namespace) -> int:
+    """Prompt-time helper: print currently valid beliefs relevant to the prompt."""
+    prompt = a.prompt or str(_hook_payload().get("prompt", ""))
+    if not prompt.strip():
+        return 0
+    with Store(a.db) as s:
+        hits = search_beliefs(s, prompt, k=a.k, require_subject=True)
+    if not hits:
+        return 0
+    lines = []
+    for h in hits:
+        value = h.text.removeprefix(f"{h.subject} {h.relation} ")
+        since = f" (since {h.ts[:10]})" if h.ts else ""
+        lines.append(f"- {h.subject} {h.relation}: {value}{since}")
+    block = "Known facts (currently valid, from your memory ledger):\n" + "\n".join(lines)
+    if a.from_hook:
+        print(
+            json.dumps(
+                {
+                    "hookSpecificOutput": {
+                        "hookEventName": "UserPromptSubmit",
+                        "additionalContext": block,
+                    }
+                }
+            )
+        )
+    else:
+        print(block)
+    return 0
+
+
+def cmd_assert(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        r = assert_belief(
+            s,
+            a.subject,
+            a.relation,
+            a.value,
+            valid_from=a.valid_from,
+            source=a.source,
+            reliability=a.reliability,
+            policy=Policy(a.policy),
+        )
+        _out(
+            {
+                "outcome": r.outcome.value,
+                "belief_id": r.belief_id,
+                "incumbent_id": r.incumbent_id,
+                "review_id": r.review_id,
+            },
+            a.json,
+        )
+    return 0
+
+
+def cmd_beliefs(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        rows = history(s, a.subject, a.relation) if a.relation else current(s, a.subject)
+        _out(rows, a.json)
+    return 0
+
+
+def cmd_read(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        _out(read_turns(s, a.session, around=a.around, window=a.window), a.json)
+    return 0
+
+
+def cmd_review(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        if a.action == "list":
+            _out([r.__dict__ for r in open_reviews(s)], a.json)
+        elif a.action == "approve":
+            _out(approve(s, a.id).__dict__, a.json)
+        elif a.action == "reject":
+            _out(reject(s, a.id).__dict__, a.json)
+        elif a.action == "sync":
+            backend: HttpReviewBackend | JsonlReviewBackend = (
+                HttpReviewBackend(a.url, a.token)
+                if a.url
+                else JsonlReviewBackend(a.outbox, a.inbox)
+            )
+            _out(sync_reviews(s, backend), a.json)
+    return 0
+
+
+def cmd_prune(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        rep = prune_sources(s, glob=a.glob, containing=a.containing)
+        _out({"sources_pruned": len(rep), "turns_removed": sum(rep.values())}, a.json)
+    return 0
+
+
+def cmd_stats(a: argparse.Namespace) -> int:
+    with Store(a.db) as s:
+        _out({"db": str(s.path), **s.stats()}, a.json)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="memware", description=__doc__)
+    p.add_argument("--db", default=str(DEFAULT_DB), help="SQLite file (env MEMWARE_DB)")
+    p.add_argument("--json", action="store_true", help="machine-readable output")
+    p.add_argument("--version", action="version", version=__version__)
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    def add(name: str, help: str) -> argparse.ArgumentParser:
+        sp = sub.add_parser(name, help=help)
+        sp.add_argument("--json", action="store_true", default=argparse.SUPPRESS)
+        return sp
+
+    s = add("init", "create the database")
+    s.set_defaults(fn=cmd_init)
+
+    s = add("sync", "index new turns from transcripts")
+    s.add_argument("paths", nargs="*")
+    s.add_argument("--harness", default="claude-code")
+    s.add_argument("--from-hook", action="store_true")
+    s.add_argument(
+        "--skip-if-contains",
+        metavar="TEXT",
+        help="skip (and un-index) files whose head contains TEXT, e.g. an eval marker",
+    )
+    s.add_argument(
+        "--exclude",
+        action="append",
+        default=[],
+        metavar="GLOB",
+        help="path glob to skip when syncing a directory (repeatable)",
+    )
+    s.set_defaults(fn=cmd_sync)
+
+    s = add("recall", "search turns and beliefs; pass several phrasings to fuse them")
+    s.add_argument(
+        "queries",
+        nargs="+",
+        metavar="QUERY",
+        help="one or more phrasings: synonyms, related terms, the literal value you expect",
+    )
+    s.add_argument("-k", type=int, default=8)
+    s.add_argument("--what", choices=["all", "turns", "beliefs"], default="all")
+    s.add_argument("--full", action="store_true")
+    s.add_argument("--no-touch", action="store_true")
+    s.add_argument("--snippet-tokens", type=int, default=96, help="FTS5 snippet window (tokens)")
+    s.set_defaults(fn=cmd_recall)
+
+    s = add("context", "print valid beliefs relevant to a prompt (hook-friendly)")
+    s.add_argument("prompt", nargs="?")
+    s.add_argument("-k", type=int, default=6)
+    s.add_argument("--from-hook", action="store_true")
+    s.set_defaults(fn=cmd_context)
+
+    s = add("assert", "record a belief; supersedes the previous value")
+    s.add_argument("subject")
+    s.add_argument("relation")
+    s.add_argument("value")
+    s.add_argument("--valid-from")
+    s.add_argument("--source")
+    s.add_argument("--reliability", type=float, default=0.5)
+    s.add_argument(
+        "--policy", choices=[x.value for x in Policy], default=Policy.GATE_CONFLICTS.value
+    )
+    s.set_defaults(fn=cmd_assert)
+
+    s = add("beliefs", "current beliefs, or the history of one key")
+    s.add_argument("subject", nargs="?")
+    s.add_argument("relation", nargs="?")
+    s.set_defaults(fn=cmd_beliefs)
+
+    s = add("read", "read a session's turns")
+    s.add_argument("session")
+    s.add_argument("--around", type=int)
+    s.add_argument("--window", type=int, default=5)
+    s.set_defaults(fn=cmd_read)
+
+    s = add("review", "list/approve/reject/sync contested supersessions")
+    s.add_argument("action", choices=["list", "approve", "reject", "sync"])
+    s.add_argument("id", nargs="?", type=int)
+    s.add_argument("--outbox", default="~/.memware/review-outbox.jsonl")
+    s.add_argument("--inbox", default="~/.memware/review-inbox.jsonl")
+    s.add_argument("--url")
+    s.add_argument("--token")
+    s.set_defaults(fn=cmd_review)
+
+    s = add("prune", "un-index sources by path glob and/or content marker")
+    s.add_argument("--glob", metavar="GLOB")
+    s.add_argument("--containing", metavar="TEXT")
+    s.set_defaults(fn=cmd_prune)
+
+    s = add("stats", "counts")
+    s.set_defaults(fn=cmd_stats)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    a = build_parser().parse_args(argv)
+    return int(a.fn(a))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

memware/eval.py

@@ -0,0 +1,208 @@
+"""Retrieval-level evaluation: does recall surface the right evidence, and never the stale value?
+
+A question set is JSONL, one object per line::
+
+    {"id": "q1", "question": "which port does the api listen on",
+     "expect_any": ["8443"], "not_expect": ["8080"], "type": "stale"}
+
+``type`` is ``fact`` (answer exists), ``stale`` (answer changed over time; the
+old value must not appear) or ``negative`` (nothing relevant should be found).
+Scores are containment against the retrieved context, so no model is needed
+and results are reproducible. Two contexts are scored for every question:
+``beliefs`` (currently valid beliefs only — what a prompt-time hook injects)
+and ``beliefs+turns`` (beliefs plus transcript evidence). Transcripts are
+evidence and legitimately contain old values, so a stale value appearing in
+``beliefs+turns`` is expected; appearing in ``beliefs`` is a defect. End-to-end
+runs with a model are described in docs/eval.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from memware.index import search_beliefs, search_turns
+from memware.ingest import sync_tree
+from memware.store import DEFAULT_DB, Store
+
+MARKER = "[memware-eval]"
+"""Put this in every prompt an evaluation sends. Runs that carry it are excluded when
+``--corpus`` rebuilds a clean store, and ``MEMWARE_NO_CAPTURE=1`` in the run's environment
+keeps hooks from indexing them in the first place."""
+
+
+def build_clean_store(
+    db: str,
+    corpus: Path,
+    *,
+    harness: str = "claude-code",
+    beliefs_from: str | None = None,
+    exclude: list[str] | None = None,
+    also_skip: list[str] | None = None,
+) -> dict[str, int]:
+    """Index ``corpus`` into a fresh store at ``db``, skipping files that contain MARKER
+    (or any of ``also_skip`` — e.g. the prompt text of an older evaluation harness),
+    and copy the belief ledger from ``beliefs_from`` so retrieval is judged against the
+    same beliefs the live system has."""
+    for suffix in ("", "-wal", "-shm"):
+        Path(db + suffix).unlink(missing_ok=True)
+    with Store(db) as s:
+        rep = sync_tree(
+            s,
+            corpus,
+            harness=harness,
+            skip_if_contains=[MARKER, *(also_skip or [])],
+            exclude=exclude,
+        )
+        copied = 0
+        if beliefs_from:
+            s.conn.execute("ATTACH ? AS live", (str(Path(beliefs_from).expanduser()),))
+            s.conn.execute("INSERT INTO belief SELECT * FROM live.belief")
+            copied = int(s.conn.execute("SELECT count(*) FROM belief").fetchone()[0])
+            s.conn.execute("DETACH live")
+        return {"files": len(rep), "turns": sum(rep.values()), "beliefs": copied}
+
+
+def retrieve(store: Store, question: str, k: int) -> tuple[str, str]:
+    """Return (beliefs_context, beliefs_plus_turns_context).
+
+    The beliefs context mirrors what the prompt-time hook injects, so it uses the
+    same subject gate (``require_subject=True``); the turns context mirrors an
+    explicit, broad recall.
+    """
+    beliefs = search_beliefs(store, question, k=k, record_use=False, require_subject=True)
+    turns = search_turns(store, question, k=k, record_use=False)
+    b = "\n".join(h.text for h in beliefs)
+    return b, "\n".join([b, *(h.text for h in turns)]).strip()
+
+
+def _first(ctx: str, needles: list[Any]) -> int:
+    pos = [ctx.find(str(n).lower()) for n in needles]
+    pos = [p for p in pos if p >= 0]
+    return min(pos) if pos else -1
+
+
+def _score(q: dict[str, Any], ctx: str) -> dict[str, Any]:
+    """``stale`` means the old value leads: it appears and the expected value does not
+    appear before it. Mentioning the old value as history after the current one is fine."""
+    ctx = ctx.lower()
+    found = any(str(e).lower() in ctx for e in q.get("expect_any", []))
+    old_at = _first(ctx, q.get("not_expect", []))
+    new_at = _first(ctx, q.get("expect_any", []))
+    stale = old_at >= 0 and (new_at < 0 or old_at < new_at)
+    qtype = str(q.get("type", "fact"))
+    ok = (not found) if qtype == "negative" else (found and not stale)
+    return {"found": found, "stale": stale, "ok": ok, "context_chars": len(ctx)}
+
+
+def run(db: str, questions: Path, *, k: int = 8) -> dict[str, Any]:
+    rows = [
+        json.loads(line)
+        for line in questions.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    results: list[dict[str, Any]] = []
+    latencies: list[float] = []
+    with Store(db) as s:
+        for q in rows:
+            t0 = time.perf_counter()
+            b_ctx, bt_ctx = retrieve(s, q["question"], k)
+            latencies.append((time.perf_counter() - t0) * 1000)
+            results.append(
+                {
+                    "id": q["id"],
+                    "type": q.get("type", "fact"),
+                    "beliefs_injected": bool(b_ctx),
+                    "beliefs": _score(q, b_ctx),
+                    "beliefs+turns": _score(q, bt_ctx),
+                }
+            )
+
+    def summarize(ctx_key: str) -> dict[str, Any]:
+        by_type: dict[str, list[bool]] = {}
+        for r in results:
+            by_type.setdefault(str(r["type"]), []).append(bool(r[ctx_key]["ok"]))
+        n = max(1, len(results))
+        chars = [int(r[ctx_key]["context_chars"]) for r in results]
+        return {
+            "accuracy": sum(bool(r[ctx_key]["ok"]) for r in results) / n,
+            "stale_rate": sum(bool(r[ctx_key]["stale"]) for r in results) / n,
+            "by_type": {t: sum(v) / len(v) for t, v in by_type.items()},
+            "context_chars_median": statistics.median(chars) if chars else 0,
+        }
+
+    n_all = max(1, len(results))
+    return {
+        "n": len(results),
+        "beliefs_injection_rate": sum(bool(r["beliefs_injected"]) for r in results) / n_all,
+        "beliefs_injection_rate_negatives": (
+            sum(bool(r["beliefs_injected"]) for r in results if r["type"] == "negative")
+            / max(1, sum(1 for r in results if r["type"] == "negative"))
+        ),
+        "beliefs": summarize("beliefs"),
+        "beliefs+turns": summarize("beliefs+turns"),
+        "latency_ms_median": statistics.median(latencies) if latencies else 0.0,
+        "results": results,
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(prog="memware-eval", description=__doc__)
+    p.add_argument("questions", type=Path)
+    p.add_argument("--db", default=str(DEFAULT_DB))
+    p.add_argument("-k", type=int, default=8)
+    p.add_argument("--json", action="store_true")
+    p.add_argument(
+        "--corpus",
+        type=Path,
+        help="rebuild --db as a clean scratch store from this transcript root first "
+        f"(files containing {MARKER!r} are skipped)",
+    )
+    p.add_argument(
+        "--beliefs-from", help="copy the belief ledger from this store into the scratch store"
+    )
+    p.add_argument("--harness", default="claude-code")
+    p.add_argument("--exclude", action="append", default=[], metavar="GLOB")
+    p.add_argument(
+        "--also-skip",
+        action="append",
+        default=[],
+        metavar="TEXT",
+        help="additional content markers that identify evaluation transcripts (repeatable)",
+    )
+    a = p.parse_args(argv)
+    if a.corpus:
+        if str(Path(a.db).expanduser()) == str(DEFAULT_DB):
+            raise SystemExit(
+                "--corpus rebuilds the store: pass a scratch --db, not the default one"
+            )
+        built = build_clean_store(
+            a.db, a.corpus, harness=a.harness, beliefs_from=a.beliefs_from, exclude=a.exclude
+        )
+        print(f"clean store: {built}", file=sys.stderr)
+    rep = run(a.db, a.questions, k=a.k)
+    if a.json:
+        print(json.dumps(rep, indent=2))
+    else:
+        for key in ("beliefs", "beliefs+turns"):
+            sm = rep[key]
+            print(
+                f"[{key}] n={rep['n']} accuracy={sm['accuracy']:.3f} "
+                f"stale_rate={sm['stale_rate']:.3f} by_type={sm['by_type']} "
+                f"context_chars={sm['context_chars_median']:.0f}"
+            )
+        print(
+            f"beliefs injected on {rep['beliefs_injection_rate']:.0%} of questions "
+            f"({rep['beliefs_injection_rate_negatives']:.0%} of negatives); "
+            f"median_latency={rep['latency_ms_median']:.1f}ms"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())