keepm 0.2.0__py3-none-any.whl

keepm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

keepm/cli.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import sys
+from collections.abc import Sequence
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    """Dispatch the management CLI without contaminating MCP stdio mode."""
+    arguments = list(sys.argv[1:] if argv is None else argv)
+    if arguments[:1] == ["setup"]:
+        from keepm.setup import main as setup_main
+
+        setup_main(arguments[1:])
+        return
+
+    from keepm.server import main as server_main
+
+    server_main(arguments)

keepm/config.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+import os
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, cast
+
+
+MemoryLanguage = Literal["zh", "en"]
+
+
+def default_config_path() -> Path:
+    root = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config"))
+    return root / "keepm" / "config.toml"
+
+
+def default_data_root() -> Path:
+    root = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
+    return root / "keepm"
+
+
+@dataclass(frozen=True, slots=True)
+class KeepMConfig:
+    vault_path: Path
+    memory_dir: str = "Memory"
+    stale_after_seconds: int = 30
+    language: MemoryLanguage = "en"
+
+    def __post_init__(self) -> None:
+        memory_path = Path(self.memory_dir)
+        if (
+            not self.memory_dir.strip()
+            or memory_path.is_absolute()
+            or ".." in memory_path.parts
+        ):
+            raise ValueError("memory_dir must be relative and remain inside the Vault")
+        if self.stale_after_seconds < 0:
+            raise ValueError("stale_after_seconds must be non-negative")
+        if self.language not in ("zh", "en"):
+            raise ValueError("language must be 'zh' or 'en'")
+        object.__setattr__(self, "vault_path", self.vault_path.expanduser().resolve())
+
+    @property
+    def memory_root(self) -> Path:
+        return self.vault_path / self.memory_dir
+
+    @property
+    def global_root(self) -> Path:
+        return self.memory_root / "global"
+
+    @property
+    def projects_root(self) -> Path:
+        return self.memory_root / "projects"
+
+    @property
+    def trash_root(self) -> Path:
+        return self.memory_root / ".trash"
+
+
+class ConfigStore:
+    def __init__(self, path: Path | None = None) -> None:
+        self.path = path or default_config_path()
+
+    def load(self) -> KeepMConfig:
+        if not self.path.is_file():
+            raise FileNotFoundError(f"KeepM is not configured: {self.path}")
+        data = tomllib.loads(self.path.read_text(encoding="utf-8"))
+        return KeepMConfig(
+            vault_path=Path(data["vault_path"]),
+            memory_dir=str(data.get("memory_dir", "Memory")),
+            stale_after_seconds=int(data.get("stale_after_seconds", 30)),
+            language=cast(MemoryLanguage, str(data.get("language", "en"))),
+        )
+
+    def load_optional(self) -> KeepMConfig | None:
+        try:
+            return self.load()
+        except FileNotFoundError:
+            return None
+
+    def save(self, config: KeepMConfig) -> KeepMConfig:
+        config.vault_path.mkdir(parents=True, exist_ok=True)
+        for path in (config.global_root, config.projects_root, config.trash_root):
+            path.mkdir(parents=True, exist_ok=True)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        content = (
+            f"vault_path = {json.dumps(str(config.vault_path))}\n"
+            f"memory_dir = {json.dumps(config.memory_dir)}\n"
+            f"stale_after_seconds = {config.stale_after_seconds}\n"
+            f"language = {json.dumps(config.language)}\n"
+        )
+        temporary = self.path.with_suffix(".tmp")
+        temporary.write_text(content, encoding="utf-8", newline="\n")
+        os.replace(temporary, self.path)
+        return config

keepm/database.py

@@ -0,0 +1,572 @@
+from __future__ import annotations
+
+import datetime as dt
+import json
+import sqlite3
+import threading
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator
+
+from keepm.markdown import slug_filename
+from keepm.models import MemoryDocument, SearchHit
+
+SCHEMA_VERSION = "1"
+
+
+@dataclass(frozen=True, slots=True)
+class FileState:
+    id: str
+    path: Path
+    name: str
+    scope: str
+    mtime_ns: int
+    size: int
+    checksum: str
+
+
+@dataclass(frozen=True, slots=True)
+class IndexedMemory:
+    id: str
+    path: Path
+    name: str
+    description: str
+    type: str
+    scope: str
+    tags: tuple[str, ...]
+    pinned: bool
+    created: str
+    updated: str
+    mtime_ns: int
+    size: int
+    checksum: str
+
+
+class MemoryDatabase:
+    def __init__(self, path: Path) -> None:
+        self.path = path.expanduser().resolve()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.RLock()
+        self.connection = sqlite3.connect(
+            self.path,
+            timeout=5,
+            isolation_level=None,
+            check_same_thread=False,
+        )
+        self.connection.row_factory = sqlite3.Row
+        self.connection.execute("PRAGMA journal_mode=WAL")
+        self.connection.execute("PRAGMA busy_timeout=5000")
+        self.connection.execute("PRAGMA foreign_keys=ON")
+        self.tokenizer = self._create_schema()
+
+    def _create_schema(self) -> str:
+        self.connection.executescript(
+            """
+            CREATE TABLE IF NOT EXISTS memories (
+              id TEXT PRIMARY KEY,
+              path TEXT NOT NULL UNIQUE,
+              name TEXT NOT NULL,
+              description TEXT NOT NULL,
+              type TEXT NOT NULL,
+              scope TEXT NOT NULL,
+              tags_json TEXT NOT NULL,
+              pinned INTEGER NOT NULL DEFAULT 0,
+              created TEXT NOT NULL,
+              updated TEXT NOT NULL,
+              mtime_ns INTEGER NOT NULL,
+              size INTEGER NOT NULL,
+              checksum TEXT NOT NULL,
+              indexed_at TEXT NOT NULL,
+              UNIQUE(scope, name)
+            );
+
+            CREATE TABLE IF NOT EXISTS links (
+              source_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+              target_text TEXT NOT NULL,
+              target_id TEXT,
+              relation_type TEXT NOT NULL,
+              resolved INTEGER NOT NULL DEFAULT 0,
+              PRIMARY KEY(source_id, target_text, relation_type)
+            );
+
+            CREATE TABLE IF NOT EXISTS index_errors (
+              path TEXT PRIMARY KEY,
+              error TEXT NOT NULL,
+              mtime_ns INTEGER NOT NULL,
+              size INTEGER NOT NULL
+            );
+
+            CREATE TABLE IF NOT EXISTS index_state (
+              key TEXT PRIMARY KEY,
+              value TEXT NOT NULL
+            );
+            """
+        )
+
+        row = self.connection.execute(
+            "SELECT value FROM index_state WHERE key = 'tokenizer'"
+        ).fetchone()
+        fts_exists = self.connection.execute(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_fts'"
+        ).fetchone()
+        if fts_exists:
+            tokenizer = str(row[0]) if row else "unicode61"
+        else:
+            try:
+                self.connection.execute(
+                    """CREATE VIRTUAL TABLE memory_fts USING fts5(
+                    id UNINDEXED, name, description, tags, body, tokenize='trigram'
+                    )"""
+                )
+                tokenizer = "trigram"
+            except sqlite3.OperationalError:
+                self.connection.execute(
+                    """CREATE VIRTUAL TABLE memory_fts USING fts5(
+                    id UNINDEXED, name, description, tags, body, tokenize='unicode61'
+                    )"""
+                )
+                tokenizer = "unicode61"
+
+        self.connection.execute(
+            """INSERT INTO index_state(key, value) VALUES('tokenizer', ?)
+            ON CONFLICT(key) DO UPDATE SET value = excluded.value""",
+            (tokenizer,),
+        )
+        self.connection.execute(
+            """INSERT INTO index_state(key, value) VALUES('schema_version', ?)
+            ON CONFLICT(key) DO NOTHING""",
+            (SCHEMA_VERSION,),
+        )
+        return tokenizer
+
+    @contextmanager
+    def _write_transaction(self) -> Iterator[None]:
+        with self._lock:
+            self.connection.execute("BEGIN IMMEDIATE")
+            try:
+                yield
+            except Exception:
+                self.connection.rollback()
+                raise
+            else:
+                self.connection.commit()
+
+    @staticmethod
+    def _now() -> str:
+        return dt.datetime.now(dt.timezone.utc).isoformat()
+
+    @staticmethod
+    def _record(row: sqlite3.Row) -> IndexedMemory:
+        return IndexedMemory(
+            id=str(row["id"]),
+            path=Path(row["path"]),
+            name=str(row["name"]),
+            description=str(row["description"]),
+            type=str(row["type"]),
+            scope=str(row["scope"]),
+            tags=tuple(json.loads(row["tags_json"])),
+            pinned=bool(row["pinned"]),
+            created=str(row["created"]),
+            updated=str(row["updated"]),
+            mtime_ns=int(row["mtime_ns"]),
+            size=int(row["size"]),
+            checksum=str(row["checksum"]),
+        )
+
+    @staticmethod
+    def _search_hit(row: sqlite3.Row) -> SearchHit:
+        return SearchHit(
+            id=str(row["id"]),
+            name=str(row["name"]),
+            description=str(row["description"]),
+            scope=str(row["scope"]),
+            path=Path(row["path"]),
+            snippet=str(row["snippet"] or row["description"]),
+            rank=float(row["rank"] or 0.0),
+            pinned=bool(row["pinned"]),
+        )
+
+    def upsert(
+        self,
+        document: MemoryDocument,
+        *,
+        mtime_ns: int,
+        size: int,
+        checksum: str,
+    ) -> None:
+        path = document.path.expanduser().resolve()
+        with self._write_transaction():
+            replaced = self.connection.execute(
+                "SELECT id FROM memories WHERE path = ? AND id <> ?",
+                (str(path), document.id),
+            ).fetchall()
+            for row in replaced:
+                self.connection.execute("DELETE FROM memory_fts WHERE id = ?", (row["id"],))
+                self.connection.execute("DELETE FROM memories WHERE id = ?", (row["id"],))
+
+            self.connection.execute(
+                """
+                INSERT INTO memories(
+                  id, path, name, description, type, scope, tags_json, pinned,
+                  created, updated, mtime_ns, size, checksum, indexed_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(id) DO UPDATE SET
+                  path = excluded.path,
+                  name = excluded.name,
+                  description = excluded.description,
+                  type = excluded.type,
+                  scope = excluded.scope,
+                  tags_json = excluded.tags_json,
+                  pinned = excluded.pinned,
+                  created = excluded.created,
+                  updated = excluded.updated,
+                  mtime_ns = excluded.mtime_ns,
+                  size = excluded.size,
+                  checksum = excluded.checksum,
+                  indexed_at = excluded.indexed_at
+                """,
+                (
+                    document.id,
+                    str(path),
+                    document.name,
+                    document.description,
+                    document.type,
+                    document.scope,
+                    json.dumps(document.tags, ensure_ascii=False),
+                    int(document.pinned),
+                    document.created,
+                    document.updated,
+                    mtime_ns,
+                    size,
+                    checksum,
+                    self._now(),
+                ),
+            )
+            self.connection.execute("DELETE FROM memory_fts WHERE id = ?", (document.id,))
+            self.connection.execute(
+                "INSERT INTO memory_fts(id, name, description, tags, body) VALUES (?, ?, ?, ?, ?)",
+                (
+                    document.id,
+                    document.name,
+                    document.description,
+                    " ".join(document.tags),
+                    document.body,
+                ),
+            )
+            self.connection.execute("DELETE FROM links WHERE source_id = ?", (document.id,))
+            self.connection.executemany(
+                """INSERT INTO links(
+                source_id, target_text, target_id, relation_type, resolved
+                ) VALUES (?, ?, NULL, ?, 0)""",
+                (
+                    (document.id, link.target, link.relation_type)
+                    for link in document.links
+                ),
+            )
+            self.connection.execute("DELETE FROM index_errors WHERE path = ?", (str(path),))
+            self._resolve_links()
+
+    @staticmethod
+    def _target_names(target_text: str) -> tuple[str, ...]:
+        basename = Path(target_text).name.removesuffix(".md")
+        candidates = [target_text, basename]
+        try:
+            candidates.append(slug_filename(basename).removesuffix(".md"))
+        except ValueError:
+            pass
+        return tuple(dict.fromkeys(candidate.casefold() for candidate in candidates))
+
+    def _resolve_links(self) -> None:
+        self.connection.execute("UPDATE links SET target_id = NULL, resolved = 0")
+        rows = self.connection.execute(
+            """SELECT l.source_id, l.target_text, l.relation_type, m.scope
+            FROM links l JOIN memories m ON m.id = l.source_id"""
+        ).fetchall()
+        memories = self.connection.execute("SELECT id, name, scope FROM memories").fetchall()
+        for link in rows:
+            names = self._target_names(str(link["target_text"]))
+            source_scope = str(link["scope"])
+            eligible = [
+                row
+                for row in memories
+                if str(row["name"]).casefold() in names
+                and (row["scope"] == source_scope or row["scope"] == "global")
+            ]
+            eligible.sort(key=lambda row: 0 if row["scope"] == source_scope else 1)
+            if not eligible:
+                continue
+            self.connection.execute(
+                """UPDATE links SET target_id = ?, resolved = 1
+                WHERE source_id = ? AND target_text = ? AND relation_type = ?""",
+                (
+                    eligible[0]["id"],
+                    link["source_id"],
+                    link["target_text"],
+                    link["relation_type"],
+                ),
+            )
+
+    def delete_path(self, path: Path) -> bool:
+        resolved = path.expanduser().resolve()
+        with self._write_transaction():
+            row = self.connection.execute(
+                "SELECT id FROM memories WHERE path = ?", (str(resolved),)
+            ).fetchone()
+            if row:
+                self.connection.execute("DELETE FROM memory_fts WHERE id = ?", (row["id"],))
+                self.connection.execute("DELETE FROM memories WHERE id = ?", (row["id"],))
+            self.connection.execute(
+                "DELETE FROM index_errors WHERE path = ?", (str(resolved),)
+            )
+            self._resolve_links()
+            return row is not None
+
+    def mark_error(self, path: Path, error: str, *, mtime_ns: int, size: int) -> None:
+        resolved = path.expanduser().resolve()
+        with self._write_transaction():
+            row = self.connection.execute(
+                "SELECT id FROM memories WHERE path = ?", (str(resolved),)
+            ).fetchone()
+            if row:
+                self.connection.execute("DELETE FROM memory_fts WHERE id = ?", (row["id"],))
+                self.connection.execute("DELETE FROM memories WHERE id = ?", (row["id"],))
+            self.connection.execute(
+                """INSERT INTO index_errors(path, error, mtime_ns, size)
+                VALUES (?, ?, ?, ?)
+                ON CONFLICT(path) DO UPDATE SET
+                  error = excluded.error,
+                  mtime_ns = excluded.mtime_ns,
+                  size = excluded.size""",
+                (str(resolved), error, mtime_ns, size),
+            )
+            self._resolve_links()
+
+    def get_by_id(self, memory_id: str) -> IndexedMemory | None:
+        with self._lock:
+            row = self.connection.execute(
+                "SELECT * FROM memories WHERE id = ?", (memory_id,)
+            ).fetchone()
+        return self._record(row) if row else None
+
+    def get_by_path(self, path: Path) -> IndexedMemory | None:
+        resolved = path.expanduser().resolve()
+        with self._lock:
+            row = self.connection.execute(
+                "SELECT * FROM memories WHERE path = ?", (str(resolved),)
+            ).fetchone()
+        return self._record(row) if row else None
+
+    def get_by_name(
+        self, name: str, *, scopes: tuple[str, ...]
+    ) -> IndexedMemory | None:
+        names = self._target_names(name)
+        with self._lock:
+            for scope in scopes:
+                rows = self.connection.execute(
+                    "SELECT * FROM memories WHERE scope = ?", (scope,)
+                ).fetchall()
+                for row in rows:
+                    if str(row["name"]).casefold() in names:
+                        return self._record(row)
+        return None
+
+    def all_records(self) -> tuple[IndexedMemory, ...]:
+        with self._lock:
+            rows = self.connection.execute(
+                "SELECT * FROM memories ORDER BY scope, name"
+            ).fetchall()
+        return tuple(self._record(row) for row in rows)
+
+    def file_states(self) -> dict[Path, FileState]:
+        with self._lock:
+            rows = self.connection.execute(
+                "SELECT id, path, name, scope, mtime_ns, size, checksum FROM memories"
+            ).fetchall()
+        return {
+            Path(row["path"]): FileState(
+                id=str(row["id"]),
+                path=Path(row["path"]),
+                name=str(row["name"]),
+                scope=str(row["scope"]),
+                mtime_ns=int(row["mtime_ns"]),
+                size=int(row["size"]),
+                checksum=str(row["checksum"]),
+            )
+            for row in rows
+        }
+
+    def update_file_metadata(self, path: Path, *, mtime_ns: int, size: int) -> None:
+        resolved = path.expanduser().resolve()
+        with self._lock:
+            self.connection.execute(
+                """UPDATE memories SET mtime_ns = ?, size = ?, indexed_at = ?
+                WHERE path = ?""",
+                (mtime_ns, size, self._now(), str(resolved)),
+            )
+
+    def index_error_states(self) -> dict[Path, tuple[int, int]]:
+        with self._lock:
+            rows = self.connection.execute(
+                "SELECT path, mtime_ns, size FROM index_errors"
+            ).fetchall()
+        return {
+            Path(row["path"]): (int(row["mtime_ns"]), int(row["size"]))
+            for row in rows
+        }
+
+    def index_errors(self) -> tuple[tuple[Path, str, int, int], ...]:
+        with self._lock:
+            rows = self.connection.execute(
+                "SELECT path, error, mtime_ns, size FROM index_errors ORDER BY path"
+            ).fetchall()
+        return tuple(
+            (Path(row["path"]), str(row["error"]), int(row["mtime_ns"]), int(row["size"]))
+            for row in rows
+        )
+
+    def search(
+        self,
+        query: str,
+        *,
+        scopes: tuple[str, ...],
+        limit: int,
+    ) -> tuple[SearchHit, ...]:
+        cleaned = query.strip()
+        unique_scopes = tuple(dict.fromkeys(scopes))
+        if not cleaned or not unique_scopes or limit < 1:
+            return ()
+        bounded_limit = min(limit, 100)
+        placeholders = ",".join("?" for _ in unique_scopes)
+        with self._lock:
+            if self.tokenizer == "trigram" and len(cleaned) >= 3:
+                literal_query = '"' + cleaned.replace('"', '""') + '"'
+                sql = f"""
+                    SELECT m.*,
+                           snippet(memory_fts, 4, '[', ']', ' ... ', 24) AS snippet,
+                           bm25(memory_fts) AS rank
+                    FROM memory_fts JOIN memories m ON m.id = memory_fts.id
+                    WHERE memory_fts MATCH ? AND m.scope IN ({placeholders})
+                    ORDER BY m.pinned DESC, rank ASC LIMIT ?
+                """
+                rows = self.connection.execute(
+                    sql, (literal_query, *unique_scopes, bounded_limit)
+                ).fetchall()
+            else:
+                escaped = (
+                    cleaned.replace("\\", "\\\\")
+                    .replace("%", "\\%")
+                    .replace("_", "\\_")
+                )
+                pattern = "%" + escaped + "%"
+                sql = f"""
+                    SELECT m.*, substr(f.body, 1, 240) AS snippet, 0.0 AS rank
+                    FROM memory_fts f JOIN memories m ON m.id = f.id
+                    WHERE m.scope IN ({placeholders})
+                      AND (f.name LIKE ? ESCAPE '\\'
+                           OR f.description LIKE ? ESCAPE '\\'
+                           OR f.tags LIKE ? ESCAPE '\\'
+                           OR f.body LIKE ? ESCAPE '\\')
+                    ORDER BY m.pinned DESC, m.updated DESC LIMIT ?
+                """
+                rows = self.connection.execute(
+                    sql,
+                    (
+                        *unique_scopes,
+                        pattern,
+                        pattern,
+                        pattern,
+                        pattern,
+                        bounded_limit,
+                    ),
+                ).fetchall()
+        return tuple(self._search_hit(row) for row in rows)
+
+    def list_pinned(
+        self, *, scopes: tuple[str, ...], limit: int
+    ) -> tuple[SearchHit, ...]:
+        unique_scopes = tuple(dict.fromkeys(scopes))
+        if not unique_scopes or limit < 1:
+            return ()
+        placeholders = ",".join("?" for _ in unique_scopes)
+        with self._lock:
+            rows = self.connection.execute(
+                f"""SELECT m.*, m.description AS snippet, 0.0 AS rank
+                FROM memories m
+                WHERE m.pinned = 1 AND m.scope IN ({placeholders})
+                ORDER BY m.updated DESC LIMIT ?""",
+                (*unique_scopes, min(limit, 100)),
+            ).fetchall()
+        return tuple(self._search_hit(row) for row in rows)
+
+    def forward_links(
+        self, source_id: str
+    ) -> tuple[tuple[str, str | None, str, bool], ...]:
+        with self._lock:
+            rows = self.connection.execute(
+                """SELECT target_text, target_id, relation_type, resolved
+                FROM links WHERE source_id = ?
+                ORDER BY target_text, relation_type""",
+                (source_id,),
+            ).fetchall()
+        return tuple(
+            (
+                str(row["target_text"]),
+                str(row["target_id"]) if row["target_id"] else None,
+                str(row["relation_type"]),
+                bool(row["resolved"]),
+            )
+            for row in rows
+        )
+
+    def backlinks(self, target_id: str) -> tuple[tuple[str, str], ...]:
+        with self._lock:
+            rows = self.connection.execute(
+                """SELECT source_id, relation_type FROM links
+                WHERE target_id = ? ORDER BY source_id, relation_type""",
+                (target_id,),
+            ).fetchall()
+        return tuple((str(row["source_id"]), str(row["relation_type"])) for row in rows)
+
+    def unresolved_links(self) -> tuple[tuple[str, str, str], ...]:
+        with self._lock:
+            rows = self.connection.execute(
+                """SELECT source_id, target_text, relation_type FROM links
+                WHERE resolved = 0 ORDER BY source_id, target_text, relation_type"""
+            ).fetchall()
+        return tuple(
+            (str(row["source_id"]), str(row["target_text"]), str(row["relation_type"]))
+            for row in rows
+        )
+
+    def set_state(self, key: str, value: str) -> None:
+        with self._lock:
+            self.connection.execute(
+                """INSERT INTO index_state(key, value) VALUES (?, ?)
+                ON CONFLICT(key) DO UPDATE SET value = excluded.value""",
+                (key, value),
+            )
+
+    def get_state(self, key: str) -> str | None:
+        with self._lock:
+            row = self.connection.execute(
+                "SELECT value FROM index_state WHERE key = ?", (key,)
+            ).fetchone()
+        return str(row["value"]) if row else None
+
+    def quick_check(self) -> bool:
+        with self._lock:
+            row = self.connection.execute("PRAGMA quick_check").fetchone()
+        return bool(row and row[0] == "ok")
+
+    def clear(self) -> None:
+        with self._write_transaction():
+            self.connection.execute("DELETE FROM memory_fts")
+            self.connection.execute("DELETE FROM links")
+            self.connection.execute("DELETE FROM memories")
+            self.connection.execute("DELETE FROM index_errors")
+            self.connection.execute("DELETE FROM index_state WHERE key = 'last_reconcile'")
+
+    def close(self) -> None:
+        with self._lock:
+            self.connection.close()