git-memex 0.2.0__py3-none-any.whl

memex/config.py

@@ -0,0 +1,119 @@
+"""Configuration and path helpers."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+DEFAULT_DIMENSION = 384
+# MiniLM paraphrases of the same fact often land ~0.70–0.85. Pair with the
+# digit-set guard so "3 days" does not merge with "7 days" (~0.93 cosine).
+DEFAULT_DEDUP_SCORE = 0.70
+# Soft floor for "short fact vs longer elaboration" subsumption (below near-dup).
+DEFAULT_ELABORATION_SCORE = 0.50
+MEMORY_DIRNAME = ".memex"
+MODEL_FILENAME = "MODEL.json"
+NOTES_DIRNAME = "notes"
+DB_FILENAME = "memory.db"
+
+
+@dataclass(frozen=True)
+class ModelPin:
+    """Pinned embedding model metadata written to MODEL.json."""
+
+    model: str
+    dimension: int
+    version: int = 1
+
+    def to_dict(self) -> dict:
+        return {
+            "model": self.model,
+            "dimension": self.dimension,
+            "version": self.version,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> ModelPin:
+        return cls(
+            model=str(data["model"]),
+            dimension=int(data["dimension"]),
+            version=int(data.get("version", 1)),
+        )
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Runtime settings from environment."""
+
+    root_override: Path | None
+    model_override: str | None
+    min_score: float
+    dedup_score: float
+    elaboration_score: float
+
+    @classmethod
+    def from_env(cls) -> Settings:
+        root = os.environ.get("MEMEX_ROOT")
+        model = os.environ.get("MEMEX_MODEL")
+        raw_min = os.environ.get("MEMEX_MIN_SCORE", "0.0")
+        try:
+            min_score = float(raw_min)
+        except ValueError:
+            min_score = 0.0
+        raw_dedup = os.environ.get("MEMEX_DEDUP_SCORE", str(DEFAULT_DEDUP_SCORE))
+        try:
+            dedup_score = float(raw_dedup)
+        except ValueError:
+            dedup_score = DEFAULT_DEDUP_SCORE
+        raw_elab = os.environ.get(
+            "MEMEX_ELABORATION_SCORE", str(DEFAULT_ELABORATION_SCORE)
+        )
+        try:
+            elaboration_score = float(raw_elab)
+        except ValueError:
+            elaboration_score = DEFAULT_ELABORATION_SCORE
+        return cls(
+            root_override=Path(root).expanduser().resolve() if root else None,
+            model_override=model.strip() if model else None,
+            min_score=min_score,
+            dedup_score=dedup_score,
+            elaboration_score=elaboration_score,
+        )
+
+
+def memory_dir(git_root: Path) -> Path:
+    return git_root / MEMORY_DIRNAME
+
+
+def notes_dir(git_root: Path) -> Path:
+    return memory_dir(git_root) / NOTES_DIRNAME
+
+
+def db_path(git_root: Path) -> Path:
+    return memory_dir(git_root) / DB_FILENAME
+
+
+def model_path(git_root: Path) -> Path:
+    return memory_dir(git_root) / MODEL_FILENAME
+
+
+def load_model_pin(git_root: Path) -> ModelPin | None:
+    path = model_path(git_root)
+    if not path.is_file():
+        return None
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return ModelPin.from_dict(data)
+
+
+def save_model_pin(git_root: Path, pin: ModelPin) -> None:
+    path = model_path(git_root)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(pin.to_dict(), indent=2) + "\n", encoding="utf-8")
+
+
+def default_model_name() -> str:
+    settings = Settings.from_env()
+    return settings.model_override or DEFAULT_MODEL

memex/embedder.py

@@ -0,0 +1,118 @@
+"""Lazy in-process sentence-transformers embedder."""
+
+from __future__ import annotations
+
+import threading
+from typing import Any
+
+from memex.config import DEFAULT_DIMENSION, DEFAULT_MODEL, default_model_name
+
+
+class EmbedderError(RuntimeError):
+    """Embedding model failed to load or produce vectors."""
+
+
+class Embedder:
+    """Process-wide lazy SentenceTransformer wrapper."""
+
+    _lock = threading.Lock()
+    _instance: Embedder | None = None
+
+    def __init__(self, model_name: str | None = None) -> None:
+        self._model_name = model_name or default_model_name()
+        self._model: Any | None = None
+        self._dimension: int | None = None
+        self._load_lock = threading.Lock()
+
+    @classmethod
+    def get(cls, model_name: str | None = None) -> Embedder:
+        name = model_name or default_model_name()
+        with cls._lock:
+            if cls._instance is None or cls._instance.model_name != name:
+                cls._instance = Embedder(name)
+            return cls._instance
+
+    @classmethod
+    def reset_for_tests(cls) -> None:
+        with cls._lock:
+            cls._instance = None
+
+    @property
+    def model_name(self) -> str:
+        return self._model_name
+
+    @property
+    def dimension(self) -> int:
+        self._ensure_loaded()
+        assert self._dimension is not None
+        return self._dimension
+
+    def _ensure_loaded(self) -> None:
+        if self._model is not None:
+            return
+        with self._load_lock:
+            if self._model is not None:
+                return
+            try:
+                from sentence_transformers import SentenceTransformer
+            except ImportError as exc:
+                raise EmbedderError(
+                    "sentence-transformers is not installed"
+                ) from exc
+            try:
+                model = SentenceTransformer(self._model_name)
+                # Probe dimension without requiring a full encode of user text.
+                probe = model.encode(["dimension probe"], normalize_embeddings=True)
+                dim = int(probe.shape[1])
+            except Exception as exc:
+                raise EmbedderError(
+                    f"failed to load embedding model {self._model_name!r}: {exc}"
+                ) from exc
+            self._model = model
+            self._dimension = dim
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        cleaned = [t.strip() for t in texts]
+        if any(not t for t in cleaned):
+            raise ValueError("cannot embed empty text")
+        self._ensure_loaded()
+        assert self._model is not None
+        vectors = self._model.encode(cleaned, normalize_embeddings=True)
+        return [list(map(float, row)) for row in vectors]
+
+    def embed_one(self, text: str) -> list[float]:
+        return self.embed([text])[0]
+
+
+class StubEmbedder:
+    """Deterministic embedder for unit tests (no torch)."""
+
+    def __init__(self, dimension: int = DEFAULT_DIMENSION, model_name: str = DEFAULT_MODEL) -> None:
+        self._dimension = dimension
+        self._model_name = model_name
+
+    @property
+    def model_name(self) -> str:
+        return self._model_name
+
+    @property
+    def dimension(self) -> int:
+        return self._dimension
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        out: list[list[float]] = []
+        for text in texts:
+            if not text.strip():
+                raise ValueError("cannot embed empty text")
+            # Simple bag-of-chars hash into a unit-ish vector.
+            vec = [0.0] * self._dimension
+            for i, ch in enumerate(text.encode("utf-8")):
+                vec[i % self._dimension] += (ch % 31) / 31.0
+            norm = sum(v * v for v in vec) ** 0.5 or 1.0
+            out.append([v / norm for v in vec])
+        return out
+
+    def embed_one(self, text: str) -> list[float]:
+        return self.embed([text])[0]

memex/gitroot.py

@@ -0,0 +1,43 @@
+"""Locate the git repository root."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from memex.config import Settings
+
+
+class NotAGitRepositoryError(RuntimeError):
+    """Raised when no .git directory is found walking parents."""
+
+
+def find_git_root(start: Path | None = None) -> Path:
+    """Walk parents from *start* (default: cwd) until a ``.git`` entry is found.
+
+    Honors ``MEMEX_ROOT`` when set: that path must itself be inside a
+    git work tree (or be the root).
+    """
+    settings = Settings.from_env()
+    if settings.root_override is not None:
+        return _require_git_root(settings.root_override)
+
+    cur = (start or Path.cwd()).expanduser().resolve()
+    return _require_git_root(cur)
+
+
+def _require_git_root(start: Path) -> Path:
+    cur = start.expanduser().resolve()
+    if not cur.exists():
+        raise NotAGitRepositoryError(f"path does not exist: {cur}")
+    if cur.is_file():
+        cur = cur.parent
+
+    for candidate in [cur, *cur.parents]:
+        git_entry = candidate / ".git"
+        if git_entry.exists():
+            return candidate
+
+    raise NotAGitRepositoryError(
+        f"not a git repository (or any parent): {start}. "
+        "Set MEMEX_ROOT to the repo root, or run inside a git checkout."
+    )

memex/install.py

@@ -0,0 +1,381 @@
+"""Install Memex into a git repository for agents (layout, rules, MCP)."""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import sys
+from importlib import resources
+from pathlib import Path
+from typing import Any, Iterable, Sequence
+
+from memex.gitroot import find_git_root
+from memex.store import ensure_layout
+
+BEGIN_MARK = "<!-- BEGIN MEMEX AGENTS -->"
+END_MARK = "<!-- END MEMEX AGENTS -->"
+_BLOCK_RE = re.compile(
+    re.escape(BEGIN_MARK) + r".*?" + re.escape(END_MARK),
+    re.DOTALL,
+)
+
+BEGIN_CODEX_MCP = "# BEGIN MEMEX MCP"
+END_CODEX_MCP = "# END MEMEX MCP"
+_CODEX_MCP_RE = re.compile(
+    re.escape(BEGIN_CODEX_MCP) + r".*?" + re.escape(END_CODEX_MCP),
+    re.DOTALL,
+)
+
+GITATTRIBUTES_LINE = ".memex/memory.db binary"
+
+PLATFORMS = ("cursor", "claude", "codex")
+TARGETS = (*PLATFORMS, "all")
+
+
+def load_template(name: str) -> str:
+    """Return a packaged template file's text."""
+    root = resources.files("memex") / "templates"
+    return (root / name).read_text(encoding="utf-8")
+
+
+def resolve_memex_command() -> str:
+    """Absolute path to the ``memex`` binary when possible (IDEs need this)."""
+    argv0 = Path(sys.argv[0]).expanduser()
+    try:
+        resolved = argv0.resolve()
+    except OSError:
+        resolved = argv0
+    if resolved.is_file() and resolved.name in {"memex", "memex.exe"}:
+        return str(resolved)
+
+    found = shutil.which("memex")
+    if found:
+        return str(Path(found).resolve())
+    return "memex"
+
+
+def normalize_targets(targets: Sequence[str]) -> list[str]:
+    """Expand ``all`` and de-duplicate while preserving order."""
+    out: list[str] = []
+    for raw in targets:
+        name = raw.strip().lower()
+        if name == "all":
+            for platform in PLATFORMS:
+                if platform not in out:
+                    out.append(platform)
+            continue
+        if name not in PLATFORMS:
+            raise ValueError(
+                f"unknown install target {raw!r}; choose from: "
+                + ", ".join(TARGETS)
+            )
+        if name not in out:
+            out.append(name)
+    if not out:
+        raise ValueError("specify at least one target: " + ", ".join(TARGETS))
+    return out
+
+
+def upsert_marked_block(path: Path, block: str) -> str:
+    """Insert or replace a BEGIN/END Memex markdown block. Returns action."""
+    block = block.strip() + "\n"
+    if not path.exists():
+        path.write_text(block, encoding="utf-8")
+        return "created"
+
+    text = path.read_text(encoding="utf-8")
+    if BEGIN_MARK in text and END_MARK in text:
+        if not _BLOCK_RE.search(text):
+            raise ValueError(
+                f"{path}: found {BEGIN_MARK!r} / {END_MARK!r} but "
+                "could not parse the block (markers out of order?)"
+            )
+        new_text = _BLOCK_RE.sub(block.rstrip("\n"), text, count=1)
+        if not new_text.endswith("\n"):
+            new_text += "\n"
+        if new_text == text:
+            return "unchanged"
+        path.write_text(new_text, encoding="utf-8")
+        return "updated"
+
+    stripped = text.rstrip()
+    path.write_text(stripped + "\n\n" + block, encoding="utf-8")
+    return "appended"
+
+
+def upsert_agents_block(agents_path: Path, block: str) -> str:
+    """Insert or replace the Memex block in ``AGENTS.md``."""
+    return upsert_marked_block(agents_path, block)
+
+
+def write_text_file(path: Path, content: str) -> str:
+    """Create or update a text file. Returns created|updated|unchanged."""
+    body = content if content.endswith("\n") else content + "\n"
+    existed = path.is_file()
+    if existed and path.read_text(encoding="utf-8") == body:
+        return "unchanged"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(body, encoding="utf-8")
+    return "updated" if existed else "created"
+
+
+def upsert_mcp_json(
+    mcp_path: Path,
+    *,
+    command: str,
+    memex_root_env: str,
+) -> str:
+    """Merge the memex server into an MCP JSON config. Returns action."""
+    mcp_path.parent.mkdir(parents=True, exist_ok=True)
+    entry = {
+        "command": command,
+        "args": ["serve"],
+        "env": {"MEMEX_ROOT": memex_root_env},
+    }
+
+    if mcp_path.is_file():
+        try:
+            data = json.loads(mcp_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"invalid JSON in {mcp_path}: {exc}") from exc
+        if not isinstance(data, dict):
+            raise ValueError(f"{mcp_path} must contain a JSON object")
+        servers = data.setdefault("mcpServers", {})
+        if not isinstance(servers, dict):
+            raise ValueError(f"{mcp_path}: mcpServers must be an object")
+        if servers.get("memex") == entry:
+            return "unchanged"
+        action = "updated" if "memex" in servers else "merged"
+        servers["memex"] = entry
+    else:
+        data = {"mcpServers": {"memex": entry}}
+        action = "created"
+
+    mcp_path.write_text(
+        json.dumps(data, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+    return action
+
+
+def _toml_string(value: str) -> str:
+    return json.dumps(value, ensure_ascii=False)
+
+
+def format_codex_mcp_block(command: str, memex_root: str) -> str:
+    """TOML fragment for Codex project MCP config."""
+    return (
+        f"{BEGIN_CODEX_MCP}\n"
+        f"[mcp_servers.memex]\n"
+        f"command = {_toml_string(command)}\n"
+        f'args = ["serve"]\n'
+        f"\n"
+        f"[mcp_servers.memex.env]\n"
+        f"MEMEX_ROOT = {_toml_string(memex_root)}\n"
+        f"{END_CODEX_MCP}\n"
+    )
+
+
+def upsert_codex_mcp(config_path: Path, *, command: str, memex_root: str) -> str:
+    """Insert or replace the Memex MCP block in ``.codex/config.toml``."""
+    block = format_codex_mcp_block(command, memex_root).rstrip("\n")
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if not config_path.exists():
+        config_path.write_text(block + "\n", encoding="utf-8")
+        return "created"
+
+    text = config_path.read_text(encoding="utf-8")
+    if BEGIN_CODEX_MCP in text and END_CODEX_MCP in text:
+        if not _CODEX_MCP_RE.search(text):
+            raise ValueError(
+                f"{config_path}: found Memex MCP markers but could not parse block"
+            )
+        new_text = _CODEX_MCP_RE.sub(block, text, count=1)
+        if not new_text.endswith("\n"):
+            new_text += "\n"
+        if new_text == text:
+            return "unchanged"
+        config_path.write_text(new_text, encoding="utf-8")
+        return "updated"
+
+    stripped = text.rstrip()
+    config_path.write_text(stripped + "\n\n" + block + "\n", encoding="utf-8")
+    return "appended"
+
+
+def ensure_gitattributes(git_root: Path) -> str:
+    """Ensure ``.gitattributes`` marks ``memory.db`` as binary."""
+    path = git_root / ".gitattributes"
+    if path.is_file():
+        lines = path.read_text(encoding="utf-8").splitlines()
+        if GITATTRIBUTES_LINE in lines:
+            return "unchanged"
+        text = path.read_text(encoding="utf-8")
+        suffix = "" if text.endswith("\n") or not text else "\n"
+        path.write_text(text + suffix + GITATTRIBUTES_LINE + "\n", encoding="utf-8")
+        return "appended"
+    path.write_text(GITATTRIBUTES_LINE + "\n", encoding="utf-8")
+    return "created"
+
+
+def _install_shared(
+    root: Path,
+    *,
+    agents: bool,
+    gitattributes: bool,
+) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    if agents:
+        out["agents"] = {
+            "path": "AGENTS.md",
+            "action": upsert_marked_block(
+                root / "AGENTS.md", load_template("AGENTS.md")
+            ),
+        }
+    else:
+        out["agents"] = {"skipped": True}
+
+    if gitattributes:
+        out["gitattributes"] = {
+            "path": ".gitattributes",
+            "action": ensure_gitattributes(root),
+        }
+    else:
+        out["gitattributes"] = {"skipped": True}
+    return out
+
+
+def _install_cursor(
+    root: Path,
+    *,
+    command: str,
+    rules: bool,
+    mcp: bool,
+) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    if rules:
+        out["cursor_rule"] = {
+            "path": ".cursor/rules/memex.mdc",
+            "action": write_text_file(
+                root / ".cursor" / "rules" / "memex.mdc",
+                load_template("memex.mdc"),
+            ),
+        }
+    else:
+        out["cursor_rule"] = {"skipped": True}
+
+    if mcp:
+        out["cursor_mcp"] = {
+            "path": ".cursor/mcp.json",
+            "action": upsert_mcp_json(
+                root / ".cursor" / "mcp.json",
+                command=command,
+                memex_root_env="${workspaceFolder}",
+            ),
+            "command": command,
+        }
+    else:
+        out["cursor_mcp"] = {"skipped": True}
+    return out
+
+
+def _install_claude(
+    root: Path,
+    *,
+    command: str,
+    instructions: bool,
+    mcp: bool,
+) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    if instructions:
+        out["claude_md"] = {
+            "path": "CLAUDE.md",
+            "action": upsert_marked_block(
+                root / "CLAUDE.md", load_template("AGENTS.md")
+            ),
+        }
+    else:
+        out["claude_md"] = {"skipped": True}
+
+    if mcp:
+        out["claude_mcp"] = {
+            "path": ".mcp.json",
+            "action": upsert_mcp_json(
+                root / ".mcp.json",
+                command=command,
+                memex_root_env="${CLAUDE_PROJECT_DIR:-.}",
+            ),
+            "command": command,
+        }
+    else:
+        out["claude_mcp"] = {"skipped": True}
+    return out
+
+
+def _install_codex(
+    root: Path,
+    *,
+    command: str,
+    mcp: bool,
+) -> dict[str, Any]:
+    if not mcp:
+        return {"codex_mcp": {"skipped": True}}
+    return {
+        "codex_mcp": {
+            "path": ".codex/config.toml",
+            "action": upsert_codex_mcp(
+                root / ".codex" / "config.toml",
+                command=command,
+                memex_root=str(root),
+            ),
+            "command": command,
+        }
+    }
+
+
+def install_memex(
+    git_root: Path | None = None,
+    *,
+    targets: Sequence[str] | Iterable[str] = ("cursor",),
+    agents: bool = True,
+    instructions: bool = True,
+    rules: bool = True,
+    mcp: bool = True,
+    gitattributes: bool = True,
+    memex_command: str | None = None,
+) -> dict[str, Any]:
+    """Set up Memex layout + agent wiring for one or more platforms."""
+    root = (git_root or find_git_root()).resolve()
+    platforms = normalize_targets(list(targets))
+    ensure_layout(root)
+    command = memex_command or resolve_memex_command()
+
+    actions: dict[str, Any] = {
+        "ok": True,
+        "git_root": str(root),
+        "targets": platforms,
+        "layout": "ensured",
+        "memex_command": command,
+    }
+    actions.update(_install_shared(root, agents=agents, gitattributes=gitattributes))
+
+    for platform in platforms:
+        if platform == "cursor":
+            actions.update(
+                _install_cursor(root, command=command, rules=rules, mcp=mcp)
+            )
+        elif platform == "claude":
+            actions.update(
+                _install_claude(
+                    root,
+                    command=command,
+                    instructions=instructions,
+                    mcp=mcp,
+                )
+            )
+        elif platform == "codex":
+            actions.update(_install_codex(root, command=command, mcp=mcp))
+
+    return actions

memex/models.py

@@ -0,0 +1,34 @@
+"""Dataclasses for memory records and recall hits."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class MemoryRecord:
+    id: str
+    content: str
+    tags: list[str] = field(default_factory=list)
+    created_at: str = ""
+    path: str = ""
+    content_hash: str = ""
+    source: str = "agent"
+
+
+@dataclass(frozen=True)
+class RecallHit:
+    id: str
+    content: str
+    score: float
+    path: str
+    tags: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "content": self.content,
+            "score": self.score,
+            "path": self.path,
+            "tags": list(self.tags),
+        }