mnemofish 0.2.0__py3-none-any.whl

mnemo/__init__.py

@@ -0,0 +1,10 @@
+"""mnemo — persistent, portable, cross-AI memory over a markdown vault."""
+
+__version__ = "0.2.0"
+
+from .config import Config
+from .index import Index
+from .note import Note
+from .search import Search
+
+__all__ = ["Config", "Index", "Note", "Search", "__version__"]

mnemo/cli.py

@@ -0,0 +1,236 @@
+"""mnemo CLI.
+
+F1: reindex / search / get
+F2: recall (push, for the SessionStart hook) / write (with dedup)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from .config import Config
+from .index import Index
+from .recall import build_recall
+from .search import Search
+from .vault import detect_project
+from .writer import write_note
+
+
+def _csv(value: str) -> list[str]:
+    return [v.strip() for v in (value or "").split(",") if v.strip()]
+
+
+def _dest_from_url(url: str) -> str:
+    name = url.rstrip("/").split("/")[-1]
+    if name.endswith(".git"):
+        name = name[:-4]
+    return name or "vault"
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="mnemo", description="Persistent memory over a markdown vault"
+    )
+    p.add_argument("--vault", help="vault path (default: $MNEMO_VAULT or cwd)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    sp = sub.add_parser("reindex", help="refresh the derived index from the vault")
+    sp.add_argument("--full", action="store_true", help="reparse every note")
+
+    ss = sub.add_parser("search", help="search notes (returns summaries, not bodies)")
+    ss.add_argument("query")
+    ss.add_argument("--type")
+    ss.add_argument("--project")
+    ss.add_argument("-k", type=int, default=5)
+    ss.add_argument("--json", action="store_true")
+
+    sg = sub.add_parser("get", help="print a full note by id")
+    sg.add_argument("id")
+
+    sr = sub.add_parser("recall", help="build the session-start recall block")
+    sr.add_argument("--project", help="override project (else detected)")
+    sr.add_argument("--project-dir", help="dir to detect project from (default: cwd)")
+    sr.add_argument("--hook", action="store_true", help="emit SessionStart JSON")
+    sr.add_argument("--reindex", action="store_true", help="refresh index first")
+
+    sw = sub.add_parser("write", help="add or update a note (deduped)")
+    sw.add_argument("--type", required=True)
+    sw.add_argument("--title", required=True)
+    sw.add_argument("--summary", default="")
+    sw.add_argument("--body", help="body text; '-' or omitted reads stdin")
+    sw.add_argument("--project")
+    sw.add_argument("--tags", default="", help="comma-separated")
+    sw.add_argument("--links", default="", help="comma-separated ids")
+    sw.add_argument("--id")
+
+    sd = sub.add_parser("daily", help="append an entry to today's daily note")
+    sd.add_argument("text", nargs="?", help="entry text; omitted/'-' reads stdin")
+
+    sub.add_parser("serve", help="run the MCP server over stdio (cross-AI, pull)")
+
+    si = sub.add_parser("init", help="scaffold the vault as a git repo")
+    si.add_argument("--remote", help="git remote URL (use a PRIVATE repo)")
+
+    sy = sub.add_parser("sync", help="commit + pull --rebase + push the vault")
+    sy.add_argument("--message", "-m")
+
+    sc = sub.add_parser("clone", help="clone a vault on a new machine, then reindex")
+    sc.add_argument("url")
+    sc.add_argument("dest", nargs="?", help="destination dir (default: repo name)")
+
+    se = sub.add_parser("export", help="zip the vault markdown (for Drive/transfer)")
+    se.add_argument("out", help="output .zip path")
+
+    sm = sub.add_parser("import", help="unzip an archive into the vault, then reindex")
+    sm.add_argument("archive", help="input .zip path")
+
+    spj = sub.add_parser(
+        "project",
+        help="show the detected project, or write a .mnemo-project marker",
+    )
+    spj.add_argument("name", nargs="?", help="set this project name in the current dir")
+    spj.add_argument("--dir", default=".", help="directory (default: cwd)")
+    return p
+
+
+def _cmd_recall(args, cfg, idx) -> None:
+    if args.reindex:
+        idx.reindex(cfg.vault)
+    project = args.project or detect_project(args.project_dir or os.getcwd())
+    block = build_recall(idx, project)
+    if args.hook:
+        if block:
+            print(
+                json.dumps(
+                    {
+                        "hookSpecificOutput": {
+                            "hookEventName": "SessionStart",
+                            "additionalContext": block,
+                        }
+                    },
+                    ensure_ascii=False,
+                )
+            )
+    else:
+        print(block)
+
+
+def _cmd_write(args, cfg, idx) -> None:
+    body = args.body
+    if body in (None, "-"):
+        body = "" if sys.stdin.isatty() else sys.stdin.read()
+    result = write_note(
+        cfg,
+        idx,
+        type=args.type,
+        title=args.title,
+        summary=args.summary,
+        body=body,
+        project=args.project,
+        tags=_csv(args.tags),
+        links=_csv(args.links),
+        id=args.id,
+    )
+    print(json.dumps(result, ensure_ascii=False))
+
+
+def _force_utf8() -> None:
+    # Hooks and non-ASCII (Turkish) notes must not crash on Windows cp1252.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")
+        except (AttributeError, ValueError):
+            pass
+
+
+def main(argv: list[str] | None = None) -> int:
+    _force_utf8()
+    args = _build_parser().parse_args(argv)
+
+    if args.cmd == "serve":
+        from .server import run as serve
+        serve(args.vault)
+        return 0
+
+    if args.cmd == "project":
+        from .vault import MARKER, detect_project
+        target = Path(args.dir)
+        if args.name:
+            (target / MARKER).write_text(args.name + "\n", encoding="utf-8")
+            print(json.dumps({"marker": str(target / MARKER), "project": args.name}))
+        else:
+            print(json.dumps({"project": detect_project(target)}))
+        return 0
+
+    if args.cmd in ("init", "sync", "clone", "export", "import"):
+        from . import portability as port
+        cfg = Config(args.vault)
+        if args.cmd == "init":
+            if args.remote:
+                print(
+                    "WARNING: use a PRIVATE repo — your notes will be pushed there.",
+                    file=sys.stderr,
+                )
+            print(json.dumps(port.init_vault(cfg.vault, args.remote)))
+        elif args.cmd == "sync":
+            print(json.dumps(port.sync_vault(cfg.vault, args.message)))
+        elif args.cmd == "clone":
+            dest = args.dest or _dest_from_url(args.url)
+            print(json.dumps(port.clone_vault(args.url, dest)))
+        elif args.cmd == "export":
+            print(json.dumps(port.export_vault(cfg.vault, args.out)))
+        elif args.cmd == "import":
+            print(json.dumps(port.import_vault(args.archive, cfg.vault)))
+        return 0
+
+    cfg = Config(args.vault)
+    # Semantic (embedding) index for search/reindex/write; FTS-only fast path
+    # for recall/daily/get so the SessionStart hook never loads the model.
+    emb = None
+    if args.cmd in ("reindex", "search", "write"):
+        from .embed import Embedder
+        if Embedder.is_available():
+            emb = Embedder()
+    idx = Index(cfg.index_path, embedder=emb)
+    try:
+        if args.cmd == "reindex":
+            print(json.dumps(idx.reindex(cfg.vault, full=args.full)))
+        elif args.cmd == "search":
+            res = Search(idx).search(
+                args.query, type=args.type, project=args.project, k=args.k
+            )
+            if args.json:
+                print(json.dumps(res, ensure_ascii=False, indent=2))
+            else:
+                if not res:
+                    print("(no matches)")
+                for r in res:
+                    proj = r["project"] or "-"
+                    print(f"[{r['score']}] {r['title']}  ({r['type']}/{proj})")
+                    if r["summary"]:
+                        print(f"    {r['summary']}")
+                    print(f"    {r['path']}")
+        elif args.cmd == "get":
+            note = Search(idx).get(args.id)
+            print(json.dumps(note, ensure_ascii=False, indent=2) if note else "not found")
+        elif args.cmd == "recall":
+            _cmd_recall(args, cfg, idx)
+        elif args.cmd == "write":
+            _cmd_write(args, cfg, idx)
+        elif args.cmd == "daily":
+            from .daily import append_daily
+            text = args.text
+            if text in (None, "-"):
+                text = "" if sys.stdin.isatty() else sys.stdin.read().strip()
+            print(json.dumps(append_daily(cfg, idx, text), ensure_ascii=False))
+    finally:
+        idx.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mnemo/config.py

@@ -0,0 +1,32 @@
+"""Configuration — vault location + derived index path."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+INDEX_REL = Path(".mnemo") / "index.sqlite"
+
+
+class Config:
+    """Resolves where the vault lives and where the derived index goes.
+
+    Vault resolution order: explicit arg → $MNEMO_VAULT → current directory.
+    The index always lives at ``<vault>/.mnemo/index.sqlite`` (gitignored).
+    """
+
+    def __init__(self, vault: str | Path | None = None):
+        self.vault = self._resolve_vault(vault)
+        self.index_path = self.vault / INDEX_REL
+
+    @staticmethod
+    def _resolve_vault(vault: str | Path | None) -> Path:
+        if vault:
+            return Path(vault).expanduser().resolve()
+        env = os.environ.get("MNEMO_VAULT")
+        if env:
+            return Path(env).expanduser().resolve()
+        return Path.cwd().resolve()
+
+    def ensure_dirs(self) -> None:
+        self.index_path.parent.mkdir(parents=True, exist_ok=True)

mnemo/daily.py

@@ -0,0 +1,29 @@
+"""Daily journal — append timestamped entries to today's daily note."""
+
+from __future__ import annotations
+
+import datetime as _dt
+
+import frontmatter
+
+
+def append_daily(cfg, index, text: str) -> dict:
+    today = _dt.date.today().isoformat()
+    path = cfg.vault / "daily" / f"{today}.md"
+
+    if path.exists():
+        post = frontmatter.load(str(path))
+        meta = post.metadata
+        body = post.content
+    else:
+        meta = {"id": today, "type": "daily", "title": today, "created": today}
+        body = ""
+
+    stamp = _dt.datetime.now().strftime("%H:%M")
+    body = f"{body.rstrip()}\n- {stamp} {text}".strip()
+    meta["updated"] = today
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(frontmatter.dumps(frontmatter.Post(body, **meta)), encoding="utf-8")
+    index.reindex(cfg.vault)
+    return {"path": str(path.relative_to(cfg.vault)), "entry": text}

mnemo/embed.py

@@ -0,0 +1,50 @@
+"""Optional semantic embedding layer (pluggable, lazy).
+
+Backend: fastembed (ONNX) — fast cold-start, light install, friendly to
+`uv tool install`. Default model is multilingual (notes may be Turkish).
+Everything degrades to FTS5 when the ``embed`` extra is absent.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+# Multilingual MiniLM: fast + stable, handles Turkish. 384-dim.
+DEFAULT_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+DEFAULT_DIM = 384
+
+
+class Embedder:
+    def __init__(self, model_name: str = DEFAULT_MODEL, dim: int = DEFAULT_DIM):
+        self.model_name = model_name
+        self.dim = dim
+        self._model = None
+
+    @staticmethod
+    def is_available() -> bool:
+        try:
+            import fastembed  # noqa: F401
+            import sqlite_vec  # noqa: F401
+        except Exception:
+            return False
+        return True
+
+    def _ensure(self):
+        if self._model is None:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                from fastembed import TextEmbedding
+
+                self._model = TextEmbedding(self.model_name)
+        return self._model
+
+    def encode_one(self, text: str) -> list[float]:
+        model = self._ensure()
+        vec = next(iter(model.embed([text])))
+        return [float(x) for x in vec]
+
+    def encode(self, texts: Sequence[str]) -> list[list[float]]:
+        model = self._ensure()
+        return [[float(x) for x in v] for v in model.embed(list(texts))]

mnemo/index.py

@@ -0,0 +1,205 @@
+"""SQLite index: `notes` metadata + FTS5 full-text + optional vector store.
+
+The index is derived from the vault and fully rebuildable. Reindexing is
+incremental (a file is re-parsed only when its mtime changed). When an
+``Embedder`` is supplied and sqlite-vec is available, a cosine vector table is
+maintained alongside FTS for hybrid search; otherwise everything works on FTS5.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import sqlite3
+from pathlib import Path
+
+from .note import Note
+from .vault import iter_note_files
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS notes (
+    id       TEXT PRIMARY KEY,
+    path     TEXT UNIQUE,
+    mtime    REAL,
+    hash     TEXT,
+    type     TEXT,
+    project  TEXT,
+    title    TEXT,
+    summary  TEXT,
+    tags     TEXT,
+    created  TEXT,
+    updated  TEXT,
+    body     TEXT
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
+    id UNINDEXED,
+    title,
+    summary,
+    body,
+    tags,
+    tokenize = 'unicode61 remove_diacritics 2'
+);
+
+CREATE INDEX IF NOT EXISTS idx_notes_type ON notes(type);
+CREATE INDEX IF NOT EXISTS idx_notes_project ON notes(project);
+"""
+
+_WORD_RE = re.compile(r"\w+", re.UNICODE)
+
+
+def _hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def fts_query(text: str) -> str:
+    """Build a safe FTS5 query: prefix-matched terms joined by OR."""
+    terms = _WORD_RE.findall(text)
+    if not terms:
+        return '""'
+    return " OR ".join(f'"{t}"*' for t in terms)
+
+
+class Index:
+    def __init__(self, db_path: str | Path, embedder=None):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.con = sqlite3.connect(str(self.db_path))
+        self.con.row_factory = sqlite3.Row
+        self.embedder = embedder
+        self.vectors = False
+        self._vec = None
+        if embedder is not None:
+            self._enable_vectors()
+        self.con.executescript(SCHEMA)
+        if self.vectors:
+            self.con.execute(
+                "CREATE VIRTUAL TABLE IF NOT EXISTS vec_notes USING vec0("
+                f"note_id TEXT, embedding float[{embedder.dim}] distance_metric=cosine)"
+            )
+
+    def _enable_vectors(self) -> None:
+        try:
+            import sqlite_vec
+
+            self.con.enable_load_extension(True)
+            sqlite_vec.load(self.con)
+            self.con.enable_load_extension(False)
+            self._vec = sqlite_vec
+            self.vectors = True
+        except Exception:
+            self.vectors = False
+
+    def close(self) -> None:
+        self.con.close()
+
+    def __enter__(self) -> "Index":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    # ------------------------------------------------------------------ write
+    def _vec_delete(self, note_id: str) -> None:
+        if self.vectors:
+            self.con.execute("DELETE FROM vec_notes WHERE note_id = ?", (note_id,))
+
+    def _vec_insert(self, note: Note) -> None:
+        if not self.vectors:
+            return
+        vec = self.embedder.encode_one(note.search_text())
+        self.con.execute(
+            "INSERT INTO vec_notes(note_id, embedding) VALUES (?, ?)",
+            (note.id, self._vec.serialize_float32(vec)),
+        )
+
+    def _upsert(self, note: Note, mtime: float, h: str) -> None:
+        rel = str(note.path)
+        old = self.con.execute("SELECT id FROM notes WHERE path = ?", (rel,)).fetchone()
+        if old:
+            self.con.execute("DELETE FROM notes_fts WHERE id = ?", (old["id"],))
+            self._vec_delete(old["id"])
+        self.con.execute("DELETE FROM notes_fts WHERE id = ?", (note.id,))
+        self._vec_delete(note.id)
+        self.con.execute("DELETE FROM notes WHERE id = ? OR path = ?", (note.id, rel))
+        self.con.execute(
+            """INSERT INTO notes
+               (id, path, mtime, hash, type, project, title, summary, tags,
+                created, updated, body)
+               VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
+            (
+                note.id, rel, mtime, h, note.type, note.project, note.title,
+                note.summary, json.dumps(note.tags, ensure_ascii=False),
+                note.created, note.updated, note.body,
+            ),
+        )
+        self.con.execute(
+            "INSERT INTO notes_fts (id, title, summary, body, tags) VALUES (?,?,?,?,?)",
+            (note.id, note.title, note.summary, note.body, " ".join(note.tags)),
+        )
+        self._vec_insert(note)
+
+    def reindex(self, vault: str | Path, full: bool = False) -> dict[str, int]:
+        """Incrementally sync the index with the vault. Returns stats."""
+        vault = Path(vault)
+        existing = {
+            row["path"]: (row["mtime"], row["id"])
+            for row in self.con.execute("SELECT path, mtime, id FROM notes")
+        }
+        seen: set[str] = set()
+        added = updated = skipped = 0
+
+        for f in iter_note_files(vault):
+            rel = str(f.relative_to(vault))
+            seen.add(rel)
+            mtime = f.stat().st_mtime
+            if not full and rel in existing and abs(existing[rel][0] - mtime) < 1e-6:
+                skipped += 1
+                continue
+            note = Note.from_file(f)
+            note.path = Path(rel)
+            self._upsert(note, mtime, _hash(note.search_text()))
+            updated += 1 if rel in existing else 0
+            added += 0 if rel in existing else 1
+
+        removed = 0
+        for path, (_, nid) in existing.items():
+            if path not in seen:
+                self.con.execute("DELETE FROM notes WHERE path = ?", (path,))
+                self.con.execute("DELETE FROM notes_fts WHERE id = ?", (nid,))
+                self._vec_delete(nid)
+                removed += 1
+
+        self.con.commit()
+        return {"added": added, "updated": updated, "skipped": skipped, "removed": removed}
+
+    # ------------------------------------------------------------------- read
+    def count(self) -> int:
+        return self.con.execute("SELECT COUNT(*) FROM notes").fetchone()[0]
+
+    def fts_ids(self, query: str, limit: int) -> list[str]:
+        rows = self.con.execute(
+            """SELECT id, bm25(notes_fts) AS rank
+               FROM notes_fts WHERE notes_fts MATCH ?
+               ORDER BY rank LIMIT ?""",
+            (fts_query(query), limit),
+        ).fetchall()
+        return [r["id"] for r in rows]
+
+    def vec_search(self, query: str, limit: int) -> list[tuple[str, float]] | None:
+        """KNN over the vector store. Returns (id, cosine_distance) or None
+        when semantic search is unavailable."""
+        if not self.vectors:
+            return None
+        q = self.embedder.encode_one(query)
+        rows = self.con.execute(
+            """SELECT note_id, distance FROM vec_notes
+               WHERE embedding MATCH ? AND k = ? ORDER BY distance""",
+            (self._vec.serialize_float32(q), limit),
+        ).fetchall()
+        return [(r["note_id"], r["distance"]) for r in rows]
+
+    def vec_ids(self, query: str, limit: int) -> list[str] | None:
+        hits = self.vec_search(query, limit)
+        return None if hits is None else [h[0] for h in hits]

mnemo/note.py

@@ -0,0 +1,75 @@
+"""Note model — markdown + YAML frontmatter parsing/serialization."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import frontmatter
+
+NOTE_TYPES = {"decision", "lesson", "daily", "project", "reference", "note"}
+
+_SLUG_RE = re.compile(r"[^a-z0-9]+")
+
+
+def slugify(text: str) -> str:
+    s = _SLUG_RE.sub("-", text.lower()).strip("-")
+    return s or "note"
+
+
+def _as_list(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value.strip()] if value.strip() else []
+    return [str(v).strip() for v in value if str(v).strip()]
+
+
+@dataclass
+class Note:
+    """One atomic memory: a decision, lesson, daily entry, reference, etc."""
+
+    id: str
+    type: str
+    title: str
+    body: str = ""
+    summary: str = ""
+    project: str | None = None
+    tags: list[str] = field(default_factory=list)
+    created: str | None = None
+    updated: str | None = None
+    links: list[str] = field(default_factory=list)
+    path: Path | None = None
+
+    @classmethod
+    def from_post(cls, post: "frontmatter.Post", path: Path | None = None) -> "Note":
+        meta = post.metadata or {}
+        stem = path.stem if path else ""
+        title = str(meta.get("title") or stem).strip()
+        ntype = str(meta.get("type") or "note").strip()
+        nid = str(meta.get("id") or stem or slugify(title))
+        project = meta.get("project")
+        return cls(
+            id=nid,
+            type=ntype,
+            title=title,
+            body=(post.content or "").strip(),
+            summary=str(meta.get("summary") or "").strip(),
+            project=str(project).strip() if project else None,
+            tags=_as_list(meta.get("tags")),
+            created=str(meta["created"]) if meta.get("created") else None,
+            updated=str(meta["updated"]) if meta.get("updated") else None,
+            links=_as_list(meta.get("links")),
+            path=Path(path) if path else None,
+        )
+
+    @classmethod
+    def from_file(cls, path: str | Path) -> "Note":
+        path = Path(path)
+        post = frontmatter.load(str(path))
+        return cls.from_post(post, path)
+
+    def search_text(self) -> str:
+        """Concatenated text used for indexing / hashing."""
+        return "\n".join(p for p in (self.title, self.summary, self.body) if p)