memd 0.3.0__py3-none-any.whl

memd/__init__.py

@@ -0,0 +1,113 @@
+"""memd — agent-driven project memory curator.
+
+Maintains ./.memory/{state,decisions,mistakes,todo}.md (+ archive/, inbox/)
+across projects, CLIs (claude-code, antigravity, others), and agent swarms.
+
+Triggers:
+  - claude-code hooks: SessionStart (context brief), SessionEnd / PreCompact
+    (background distill of the session transcript)
+  - systemd user timer: `memd sweep` catches missed sessions, inbox notes
+    from other CLIs/agents, pruning, and auto-detects new projects.
+
+Distillation brain: headless `claude -p` (haiku by default, sonnet for large
+end-of-session distills) with a curator charter prompt. memd itself enforces
+the invariants the model must not be trusted with: frontmatter, append-only
+mistakes.md, shrink guard, size budgets, archive overflow, git audit commits.
+"""
+
+__version__ = "0.3.0"
+
+from memd.errors import ConfigError, CuratorError, DigestError, MemdError
+from memd.config import (
+    AG_INDEX_PATH,
+    CLAUDE_PROJECTS_DIR,
+    CLAUDE_SETTINGS,
+    CONFIG_DIR,
+    CONFIG_PATH,
+    CURSORS_PATH,
+    DEFAULT_CONFIG,
+    HOME,
+    LOCK_DIR,
+    LOG_PATH,
+    MEMORY_FILES,
+    META_PATH,
+    STATE_DIR,
+    STATE_LOCK_PATH,
+    XDG_CONFIG_HOME,
+    XDG_STATE_HOME,
+    atomic_write,
+    git_toplevel,
+    load_config,
+    load_json,
+    log,
+    save_config,
+    save_json,
+    today,
+    update_json,
+)
+from memd.memory import (
+    BRIEF_NOTE,
+    FM_RE,
+    MODEL_STUB,
+    SKELETONS,
+    append_archive,
+    archive_path,
+    brief_sections,
+    enforce_budget_mistakes,
+    find_project,
+    git_commit_memory,
+    global_slice,
+    make_brief,
+    read_memory,
+    register,
+    render_frontmatter,
+    scaffold,
+    section_date,
+    split_frontmatter,
+    split_sections,
+    write_memory_file,
+)
+from memd.digest import (
+    AG_KNOWN_STEP_TYPES,
+    AG_STEP_ASSISTANT,
+    AG_STEP_ERROR,
+    AG_STEP_TOOL,
+    AG_STEP_USER,
+    REDACT_PATTERNS,
+    REDACT_PEM_RE,
+    REDACT_RE,
+    _pb_strings,
+    ag_conversations_dir,
+    ag_max_idx,
+    ag_workspace,
+    baseline_cursor,
+    cap_digest,
+    digest_ag_db,
+    digest_jsonl,
+    digest_source,
+    project_ag_dbs,
+    redact,
+)
+from memd.inbox import collect_inbox, write_inbox_note
+from memd.curator import CHARTER, build_prompt, call_curator, validate
+from memd.sync import (
+    encode_claude_dir,
+    project_lock,
+    source_pending,
+    sync_project,
+    transcript_files,
+)
+from memd.sweep import (
+    cmd_sweep,
+    detect_new_projects,
+    ensure_global,
+    write_global_index,
+)
+from memd.hooks import (
+    HOOK_DEFS,
+    cmd_hook,
+    cmd_install_hooks,
+    detach,
+    self_invocation,
+)
+from memd.cli import cmd_note, cmd_status, main

memd/cli.py

@@ -0,0 +1,196 @@
+"""CLI argument parsing and command dispatch."""
+
+import argparse
+import os
+import shutil
+import sys
+
+from memd import __doc__, __version__
+from memd.config import (CURSORS_PATH, META_PATH, git_toplevel, load_config,
+                         load_json, save_config, update_json)
+from memd.digest import baseline_cursor
+from memd.hooks import cmd_hook, cmd_install_hooks
+from memd.inbox import collect_inbox, write_inbox_note
+from memd.memory import (find_project, make_brief, register, scaffold,
+                         init_memory_repo, main_worktree_root, migrate_parent_untrack,
+                         memory_checkout_clean)
+from memd.sweep import cmd_sweep, ensure_global
+from memd.sync import source_pending, sync_project, transcript_files
+
+# --------------------------------------------------------------------------
+# status / CLI plumbing
+# --------------------------------------------------------------------------
+
+
+def cmd_status(cfg):
+    ensure_global(cfg)
+    meta = load_json(META_PATH, {})
+    cursors = load_json(CURSORS_PATH, {})
+    if not cfg["projects"]:
+        print("no projects registered (run: memd init [path])")
+        return
+    for path, entry in sorted(cfg["projects"].items()):
+        m = meta.get(path, {})
+        srcs = transcript_files(cfg, path)
+        pending = [s for s in srcs if source_pending(s, cursors)]
+        ag = sum(1 for s in srcs if s.endswith(".db"))
+        inbox = len(collect_inbox(path)[1])
+        print(f"{entry['name']}  ({path})")
+        print(f"  last distill : {m.get('last_sync', 'never')}"
+              + (f"  [{m.get('trigger')}/{m.get('model')}]" if m else ""))
+        if m.get("summary"):
+            print(f"  summary      : {m['summary']}")
+        print(f"  sources      : {len(srcs)} ({ag} antigravity), "
+              f"{len(pending)} with pending content, {inbox} inbox note(s)")
+
+
+def cmd_note(cfg, args):
+    text = args.message if args.message is not None else sys.stdin.read()
+    text = text.strip()
+    if not text:
+        print("empty note; nothing written", file=sys.stderr)
+        return 2
+    if args.is_global:
+        root = ensure_global(cfg)
+        if not root:
+            print("no global_root configured", file=sys.stderr)
+            return 2
+    else:
+        path = os.path.realpath(args.project)
+        root = find_project(cfg, path) or git_toplevel(path) or path
+    print(write_inbox_note(os.path.realpath(root), text, source=args.source))
+    return 0
+
+
+def main():
+    ap = argparse.ArgumentParser(prog="memd", description=__doc__.splitlines()[0])
+    ap.add_argument("--version", action="version", version=f"memd {__version__}")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p = sub.add_parser("init", help="scaffold .memory/ and register a project")
+    p.add_argument("path", nargs="?", default=".")
+    p.add_argument("--name")
+    p.add_argument("--global", dest="is_global", action="store_true",
+                   help="seed the global memory root (~/.memory) instead of a project")
+
+    p = sub.add_parser("sync", help="distill new session content into memory")
+    p.add_argument("--project", default=".")
+    p.add_argument("--transcript")
+    p.add_argument("--trigger", default="manual")
+    p.add_argument("--dry-run", action="store_true")
+
+    p = sub.add_parser("sweep",
+                       help="timer entry: catch up all projects, prune, detect")
+    p.add_argument("--jobs", type=int,
+                   help="parallel project syncs (default: config sweep_jobs, 4)")
+    sub.add_parser("status", help="show registry, backlog, last distills")
+    sub.add_parser("install-hooks", help="wire memd into ~/.claude/settings.json")
+
+    p = sub.add_parser("brief", help="print session-start memory brief")
+    p.add_argument("path", nargs="?", default=".")
+    p.add_argument("--max-chars", type=int,
+                   help="hard cap on total brief output for this run")
+    p.add_argument("--topic",
+                   help="only content sections mentioning this keyword "
+                        "(case-insensitive)")
+
+    p = sub.add_parser("hook", help="claude-code hook entry (reads JSON on stdin)")
+    p.add_argument("event", choices=["session-start", "session-end", "pre-compact"])
+
+    p = sub.add_parser("note", help="append a collision-safe note to a curator inbox")
+    p.add_argument("--project", default=".")
+    p.add_argument("--global", dest="is_global", action="store_true",
+                   help="write to the global memory inbox instead of a project")
+    p.add_argument("-m", "--message", help="note text (default: read from stdin)")
+    p.add_argument("--source", help="optional writer label recorded in the note")
+
+    p = sub.add_parser("exclude", help="never auto-manage a path")
+    p.add_argument("path")
+
+    args = ap.parse_args()
+    cfg = load_config()
+
+    if args.cmd == "init" and getattr(args, "is_global", False):
+        gr = ensure_global(cfg)
+        print(f"global memory root ready at {os.path.join(gr, '.memory')}" if gr
+              else "no global_root configured")
+    elif args.cmd == "init" and os.path.realpath(args.path) == os.path.realpath(cfg.get("global_root") or "\0"):
+        # Don't let a bare `memd init ~` register the global root as a normal
+        # project (model stub in $HOME, lost "global" flag). Route to --global.
+        print("that path is the global memory root; use: memd init --global",
+              file=sys.stderr)
+        sys.exit(2)  # config error
+    elif args.cmd == "init":
+        path = os.path.realpath(args.path)
+        mem = os.path.join(path, ".memory")
+        mem_pre = os.path.exists(mem)  # real .memory here before we scaffold?
+        # A pre-existing .memory is safe to relink only if it is a clean, tracked
+        # checkout (content already in git); check before scaffold dirties it.
+        mem_pre_clean = mem_pre and memory_checkout_clean(path)
+        created = scaffold(path, args.name)
+
+        if cfg.get("memory_own_repo") and cfg.get("git_commit"):
+            migrate_parent_untrack(path)
+            main_root = main_worktree_root(path)
+            if main_root != path:
+                # Linked worktree: share the main worktree's single store via a
+                # symlink so worktrees can't diverge — but never destroy unsaved
+                # memory (memd must not lose memory).
+                main_mem = os.path.join(main_root, ".memory")
+                created.extend(scaffold(main_root, name=args.name))
+                init_memory_repo(main_mem, "memd: seed project memory root")
+                if os.path.islink(mem):
+                    pass  # already linked (idempotent)
+                elif (not mem_pre) or mem_pre_clean:
+                    # Only the templates we just scaffolded, or a clean checkout
+                    # whose content already lives in git and the shared store.
+                    shutil.rmtree(mem)
+                    os.symlink(main_mem, mem)
+                else:
+                    print(f"warning: {mem} holds unsaved memory; not replacing it "
+                          f"with a symlink to {main_mem} (resolve manually)", file=sys.stderr)
+            else:
+                init_memory_repo(mem, "memd: seed project memory root")
+
+        register(cfg, path, args.name)
+        # Baseline: start memory from now. Pre-existing transcripts are
+        # presumed covered by existing memory (or not worth back-filling).
+        skipped = 0
+        def _baseline(cursors):
+            nonlocal skipped
+            for src in transcript_files(cfg, path):
+                if src not in cursors:
+                    cursors[src] = baseline_cursor(src)
+                    skipped += 1
+        update_json(CURSORS_PATH, _baseline)
+        print(f"registered {path}"
+              + (f", created {len(created)} file(s)" if created else "")
+              + (f", baselined {skipped} existing transcript(s)" if skipped else ""))
+    elif args.cmd == "sync":
+        ok = sync_project(cfg, args.project, trigger=args.trigger,
+                          transcript=args.transcript, dry_run=args.dry_run)
+        # exit contract: 0 = ok/no-op, 3 = curator/distill failure
+        sys.exit(3 if ok is False else 0)
+    elif args.cmd == "sweep":
+        # 1 = known error (failed distills)
+        sys.exit(1 if cmd_sweep(cfg, jobs=args.jobs) else 0)
+    elif args.cmd == "status":
+        cmd_status(cfg)
+    elif args.cmd == "install-hooks":
+        cmd_install_hooks()
+    elif args.cmd == "brief":
+        brief = make_brief(cfg, args.path, max_chars=args.max_chars,
+                           topic=args.topic)
+        print(brief or "(no .memory directory here)")
+    elif args.cmd == "note":
+        sys.exit(cmd_note(cfg, args))
+    elif args.cmd == "hook":
+        sys.exit(cmd_hook(cfg, args.event))
+    elif args.cmd == "exclude":
+        path = os.path.realpath(args.path)
+        if path not in cfg["exclude"]:
+            cfg["exclude"].append(path)
+        cfg["projects"].pop(path, None)
+        save_config(cfg)
+        print(f"excluded {path}")
+

memd/config.py

@@ -0,0 +1,164 @@
+"""Config loading, defaults, path constants, and shared low-level utilities
+(logging, atomic JSON, locked registry updates)."""
+
+import datetime as dt
+import fcntl
+import json
+import os
+import subprocess
+
+HOME = os.path.expanduser("~")
+XDG_CONFIG_HOME = os.environ.get("XDG_CONFIG_HOME") or os.path.join(HOME, ".config")
+XDG_STATE_HOME = os.environ.get("XDG_STATE_HOME") or os.path.join(HOME, ".local", "state")
+CONFIG_DIR = os.path.join(XDG_CONFIG_HOME, "memd")
+CONFIG_PATH = os.path.join(CONFIG_DIR, "config.json")
+STATE_DIR = os.path.join(XDG_STATE_HOME, "memd")
+CURSORS_PATH = os.path.join(STATE_DIR, "cursors.json")
+AG_INDEX_PATH = os.path.join(STATE_DIR, "ag_index.json")
+META_PATH = os.path.join(STATE_DIR, "meta.json")
+LOCK_DIR = os.path.join(STATE_DIR, "locks")
+STATE_LOCK_PATH = os.path.join(STATE_DIR, "state.lock")
+LOG_PATH = os.path.join(STATE_DIR, "memd.log")
+CLAUDE_PROJECTS_DIR = os.path.join(HOME, ".claude", "projects")
+CLAUDE_SETTINGS = os.path.join(HOME, ".claude", "settings.json")
+
+MEMORY_FILES = ("state.md", "decisions.md", "mistakes.md", "todo.md")
+
+DEFAULT_CONFIG = {
+    "claude_bin": "claude",
+    "curator_cmd": [],              # override distill backend: argv list, prompt on
+                                    # stdin, "{model}" substituted; output must
+                                    # contain one JSON object (fences/prose ok).
+                                    # Empty -> claude_bin headless invocation.
+    "antigravity_dir": os.path.join(HOME, ".gemini", "antigravity-cli"),
+    "model_small": "haiku",
+    "model_large": "sonnet",
+    "escalate_chars": 15000,        # session-end digests above this go to model_large
+    "digest_cap_chars": 60000,      # max transcript digest fed to the model
+    "quiet_seconds": 600,           # sweep skips transcripts modified more recently
+    "sweep_jobs": 4,                # parallel project syncs per sweep (threads;
+                                    # per-project flock still serializes any
+                                    # single project). CLI --jobs overrides.
+    "auto_scaffold": True,          # scaffold .memory/ in detected git-root projects
+    "git_commit": True,             # commit .memory/ changes after each distill
+    "memory_own_repo": True,        # use a standalone git repo for .memory/ to decouple history
+    "budgets": {                    # active-file size budgets (chars of body)
+        "state.md": 10000,
+        "decisions.md": 12000,
+        "todo.md": 10000,
+        "mistakes.md": 22000,
+    },
+    "REDACT_EXTRA_PATTERNS": [],    # extra raw regex strings redacted from
+                                    # digests/inbox notes on top of the
+                                    # built-in REDACT_PATTERNS
+    "exclude": [],                  # absolute paths never auto-managed
+    "projects": {},                 # path -> {"name": str, "extra_sources": [globs]}
+    "global_root": HOME,            # project path whose .memory holds system/user/
+                                    # cross-project truth (store: ~/.memory). The
+                                    # fallback when a session belongs to no project.
+    "global_brief_chars": 800,      # cap for the global state.md excerpt layered
+                                    # into a project session's brief. 0 = pointer
+                                    # only (no excerpt); <0 = omit the global slice.
+    "brief_chars": 2500,            # budget for the brief's content sections
+                                    # (todos / decisions / state / mistakes) —
+                                    # the fixed BRIEF_NOTE + metadata overhead
+                                    # (~300 chars) is not counted against it.
+    "brief_decisions_days": 30,     # recency window for decisions.md sections
+                                    # surfaced in the brief.
+}
+
+# --------------------------------------------------------------------------
+# small utilities
+# --------------------------------------------------------------------------
+
+
+LOG_BACKUPS = 3  # rotated copies kept: memd.log.1 (newest) .. memd.log.3
+
+
+def log(msg):
+    os.makedirs(STATE_DIR, exist_ok=True)
+    try:
+        if os.path.exists(LOG_PATH) and os.path.getsize(LOG_PATH) > 1_000_000:
+            for i in range(LOG_BACKUPS - 1, 0, -1):
+                if os.path.exists(f"{LOG_PATH}.{i}"):
+                    os.replace(f"{LOG_PATH}.{i}", f"{LOG_PATH}.{i + 1}")
+            os.replace(LOG_PATH, LOG_PATH + ".1")
+    except OSError:
+        pass
+    stamp = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    with open(LOG_PATH, "a") as f:
+        f.write(f"{stamp} {msg}\n")
+
+
+def load_json(path, default):
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return default
+
+
+def save_json(path, data):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    tmp = path + ".tmp"
+    with open(tmp, "w") as f:
+        json.dump(data, f, indent=2, sort_keys=True)
+        f.write("\n")
+    os.replace(tmp, path)
+
+
+def atomic_write(path, text):
+    tmp = path + ".tmp"
+    with open(tmp, "w") as f:
+        f.write(text)
+    os.replace(tmp, path)
+
+
+def update_json(path, mutate):
+    """Read-modify-write a shared registry JSON file (cursors, meta) under a
+    process-wide lock, so concurrent project syncs can't lose each other's
+    updates. The on-disk value is re-read inside the lock and only the caller's
+    delta is merged in — never a stale in-memory snapshot. The lock is held only
+    for the merge, never across a curator call."""
+    os.makedirs(STATE_DIR, exist_ok=True)
+    fd = open(STATE_LOCK_PATH, "w")
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        data = load_json(path, {})
+        mutate(data)
+        save_json(path, data)
+        return data
+    finally:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+        fd.close()
+
+
+def load_config():
+    cfg = dict(DEFAULT_CONFIG)
+    on_disk = load_json(CONFIG_PATH, {})
+    for k, v in on_disk.items():
+        if k == "budgets":
+            cfg["budgets"] = {**DEFAULT_CONFIG["budgets"], **v}
+        else:
+            cfg[k] = v
+    return cfg
+
+
+def save_config(cfg):
+    save_json(CONFIG_PATH, cfg)
+
+
+def git_toplevel(path):
+    try:
+        out = subprocess.run(
+            ["git", "-C", path, "rev-parse", "--show-toplevel"],
+            capture_output=True, text=True, timeout=10,
+        )
+        if out.returncode == 0:
+            return out.stdout.strip()
+    except (OSError, subprocess.TimeoutExpired):
+        pass
+    return None
+
+def today():
+    return dt.date.today().isoformat()