markdown-memory 0.1.0__py3-none-any.whl

markdown_memory/__init__.py

@@ -0,0 +1,47 @@
+"""markdown-memory: AST-aware Markdown indexing and hybrid retrieval over MCP."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from markdown_memory.exceptions import (
+    ASTParseError,
+    DatabaseError,
+    DocumentNotFoundError,
+    EmbeddingError,
+    IndexingError,
+    MarkdownMemoryError,
+    SectionNotFoundError,
+)
+from markdown_memory.models import (
+    Document,
+    DocumentSummary,
+    IndexReport,
+    OutlineNode,
+    SearchResult,
+    Section,
+    SectionDraft,
+)
+
+try:
+    # pyproject.toml is the one place the version is written; this reads it back from the
+    # installed metadata rather than repeating it.
+    __version__ = version("markdown-memory")
+except PackageNotFoundError:  # a bare source tree nobody installed
+    __version__ = "0+unknown"
+
+__all__ = [
+    "ASTParseError",
+    "DatabaseError",
+    "Document",
+    "DocumentNotFoundError",
+    "DocumentSummary",
+    "EmbeddingError",
+    "IndexReport",
+    "IndexingError",
+    "MarkdownMemoryError",
+    "OutlineNode",
+    "SearchResult",
+    "Section",
+    "SectionDraft",
+    "SectionNotFoundError",
+    "__version__",
+]

markdown_memory/autoindex.py

@@ -0,0 +1,170 @@
+"""Keeping the server's own documentation root indexed without being asked.
+
+Nothing watches the filesystem. The server already looks at it on every search - the
+freshness sweep counts indexed files that moved on - so that look is what decides when to
+re-index. The first search after a start finds the runner has never run, and that
+catch-up run picks up whatever changed while no server was running. A run is the ordinary
+incremental `index_directory`, in one background thread at a time.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Callable
+
+from markdown_memory.exceptions import IndexBusyError, IndexCancelled
+from markdown_memory.models import IndexReport, IndexStatus
+
+logger = logging.getLogger(__name__)
+
+#: How soon after a run finishes an edit found by a search may start the next one. Short
+#: enough that an agent's edit is searchable within a turn or two; long enough that a
+#: burst of saves is one run rather than one each.
+CHANGE_GAP_SECONDS = 10.0
+#: How long a run speaks for when nothing edited was seen. Only a walk finds a file
+#: nobody indexed yet, or an edit that put its modification time back.
+WALK_GAP_SECONDS = 300.0
+
+
+class AutoIndexer:
+    """One background index run at a time, started by what the server sees on use.
+
+    Every field below is read and written under one lock, so a request can never slip
+    between a run finishing and the runner forgetting it, and nothing starts once `stop`
+    has been called.
+    """
+
+    def __init__(
+        self,
+        run: Callable[[Callable[[], bool]], IndexReport],
+        measure: Callable[[], IndexStatus],
+        *,
+        clock: Callable[[], float] = time.monotonic,
+        change_gap: float = CHANGE_GAP_SECONDS,
+        walk_gap: float = WALK_GAP_SECONDS,
+    ) -> None:
+        self._run = run
+        self._measure = measure
+        self._clock = clock
+        self._change_gap = change_gap
+        self._walk_gap = walk_gap
+        self._lock = threading.Lock()
+        self._thread: threading.Thread | None = None
+        self._stopping = False
+        self._last_finished: float | None = None
+        #: Nothing starts before this: set when another process held the lock, so that
+        #: contention costs one attempt per change gap, not one per search.
+        self._retry_after = float("-inf")
+        #: Changed files the last finished run could not clear - files it failed on. A
+        #: search that still sees exactly those is not a reason to walk again; one more
+        #: is. Reset to zero by a run that failed nothing, so an edit made while it ran
+        #: is picked up by the next search.
+        self._baseline = 0
+        #: The weights mismatch as the last run left it: a new one is repaired at once,
+        #: one this runner could not repair does not start a run per search.
+        self._seen_mismatch: str | None = None
+
+    @property
+    def active(self) -> bool:
+        with self._lock:
+            return self._thread is not None
+
+    def request(self) -> bool:
+        """Start a run now unless one is running or the runner is stopping."""
+        with self._lock:
+            return self._start()
+
+    def consider(self, status: IndexStatus) -> bool:
+        """Start a run if what a search just measured says one is due."""
+        with self._lock:
+            if self._stopping or self._thread is not None or self._clock() < self._retry_after:
+                return False
+            since = (
+                float("inf") if self._last_finished is None else self._clock() - self._last_finished
+            )
+            due = (
+                (status.changed_files > self._baseline and since >= self._change_gap)
+                or (
+                    status.weights_mismatch is not None
+                    and status.weights_mismatch != self._seen_mismatch
+                )
+                or since >= self._walk_gap
+            )
+            return self._start() if due else False
+
+    def stop(self) -> None:
+        """Stop the running run between two documents, and wait for it to let go."""
+        with self._lock:
+            self._stopping = True
+            thread = self._thread
+        if thread is not None:
+            thread.join()
+
+    def _start(self) -> bool:
+        if self._stopping or self._thread is not None:
+            return False
+        # Not a daemon: the interpreter would kill it mid-write at exit. `stop` is what
+        # ends it, and the service calls that before it closes the database.
+        self._thread = threading.Thread(target=self._work, name="mdmem-autoindex")
+        self._thread.start()
+        return True
+
+    def _is_stopping(self) -> bool:
+        with self._lock:
+            return self._stopping
+
+    def _work(self) -> None:
+        baseline: int | None = None
+        busy = False
+        before = self._mismatch()
+        try:
+            report = self._run(self._is_stopping)
+            baseline = self._changed() if report.errors else 0
+        except IndexCancelled:
+            logger.info("Automatic index run stopped")
+        except IndexBusyError as exc:
+            # Someone else holds the lock - maybe on another root of a shared database -
+            # so nothing is learnt about this tree; the next search asks again.
+            logger.info("Automatic index run skipped: %s", exc)
+            busy = True
+        except Exception:
+            # The whole run failed - a model that will not load, say. Retrying it for the
+            # same edit every few seconds would load the model every few seconds: what
+            # it left behind becomes the baseline, and only a further change is news.
+            logger.exception("Automatic index run failed")
+            baseline = self._changed()
+        after = self._mismatch()
+        with self._lock:
+            self._last_finished = self._clock()
+            if busy:
+                # Nothing ran, so nothing is known: ask again once the change gap has
+                # passed rather than a whole walk interval later - a root nobody has
+                # indexed has no changed files to prompt the next attempt.
+                self._last_finished -= self._walk_gap - self._change_gap
+                self._retry_after = self._last_finished + self._walk_gap
+                self._thread = None
+                return
+            if baseline is not None:
+                self._baseline = baseline
+            # Seen only if this run met it and could not clear it. One a search recorded
+            # while the run was busy is news the run never acted on.
+            if after is None or after == before:
+                self._seen_mismatch = after
+            self._thread = None
+
+    def _changed(self) -> int | None:
+        try:
+            return self._measure().changed_files
+        except Exception:
+            logger.exception("Cannot read the index status around an automatic run")
+            return None
+
+    def _mismatch(self) -> str | None:
+        try:
+            return self._measure().weights_mismatch
+        except Exception:
+            logger.exception("Cannot read the index status around an automatic run")
+            with self._lock:
+                return self._seen_mismatch

markdown_memory/config.py

@@ -0,0 +1,235 @@
+"""Where the server reads its settings from, and in what order.
+
+One precedence for every entry point: an explicit argument beats the environment, which
+beats the project default. ``resolve_config`` is the only place that order lives.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from markdown_memory.embedders import DEFAULT_EMBEDDER
+from markdown_memory.exceptions import ConfigurationError
+from markdown_memory.indexer import DEFAULT_INDEX_WORKERS
+
+ENV_DB_PATH = "MARKDOWN_MEMORY_DB"
+
+
+ENV_DOCS_DIR = "MARKDOWN_MEMORY_DOCS_DIR"
+
+
+ENV_MODEL_CACHE = "MARKDOWN_MEMORY_MODEL_CACHE"
+
+
+ENV_EMBEDDER = "MARKDOWN_MEMORY_EMBEDDER"
+
+
+ENV_LOG_LEVEL = "MARKDOWN_MEMORY_LOG_LEVEL"
+
+
+ENV_EXCLUDE = "MARKDOWN_MEMORY_EXCLUDE"
+
+
+ENV_INDEX_WORKERS = "MARKDOWN_MEMORY_INDEX_WORKERS"
+
+
+ENV_AUTO_INDEX = "MARKDOWN_MEMORY_AUTO_INDEX"
+
+
+# Claude Code exports this to every stdio MCP server it spawns, set to the project root.
+# `.mcp.json` cannot interpolate it - measured on Claude Code 2.1.278, `${CLAUDE_PROJECT_DIR}`
+# and `${workspaceFolder}` are both reported as "Missing environment variables" and passed
+# through literally - so a project-scoped config uses relative paths and the server resolves
+# them here instead.
+ENV_PROJECT_DIR = "CLAUDE_PROJECT_DIR"
+
+
+def _xdg_dir(variable: str, fallback: str) -> Path:
+    configured = os.environ.get(variable, "").strip()
+    return Path(configured) if configured else Path.home() / fallback
+
+
+def _project_database(docs_dir: Path) -> Path:
+    """Where one documentation root's index lives when nothing configured it.
+
+    Keyed on the documentation root, never on the working directory. The working directory
+    belongs to whoever launched the server, so two servers started from one directory for
+    two different projects would share a database - which is the cross-project leak this
+    default exists to close, arrived at from the other side. Search is scoped to the docs
+    root, but a document stays resolvable across a whole database by path or unique suffix,
+    so sharing the file is enough to leak one project's documentation into another's answers.
+
+    Kept out of the project too. A database inside the repository is committed by accident,
+    deleted by `git clean -xdf`, rebuilt per worktree, unwritable when the checkout is
+    read-only, and - on a network share - sits where SQLite's WAL cannot take the locks it
+    needs. The name carries the root's own basename so a person can tell the indexes apart,
+    and a digest of its resolved path so two projects called `docs` cannot collide.
+    """
+    try:
+        resolved = docs_dir.expanduser().resolve()
+    except (OSError, RuntimeError):  # symlink loop, or a path the OS will not resolve
+        resolved = docs_dir.expanduser().absolute()
+    digest = hashlib.sha256(os.fsencode(str(resolved))).hexdigest()[:12]
+    label = re.sub(r"[^A-Za-z0-9_.-]", "-", resolved.name) or "root"
+    return (
+        _xdg_dir("XDG_DATA_HOME", ".local/share")
+        / "markdown-memory"
+        / "projects"
+        / f"{label}-{digest}"
+        / "index.db"
+    )
+
+
+@dataclass(slots=True, frozen=True)
+class ServerConfig:
+    """Runtime configuration, resolved from the environment (CLI flags override)."""
+
+    db_path: Path
+    docs_dir: Path
+    embedder: str = DEFAULT_EMBEDDER
+    model_cache_dir: Path | None = None
+    # Glob patterns, relative to the docs root, that indexing must not descend into: a
+    # repository's own fixtures, vendored documentation or test corpus are not its docs.
+    exclude: tuple[str, ...] = ()
+    #: Files embedded at the same time while indexing.
+    index_workers: int = DEFAULT_INDEX_WORKERS
+    #: Whether the stdio server keeps its own docs root indexed in the background.
+    auto_index: bool = True
+
+    @classmethod
+    def from_env(cls) -> ServerConfig:
+        root = _project_root()
+        db_path = _configured_path(ENV_DB_PATH, root)
+        docs_dir = _configured_path(ENV_DOCS_DIR, root)
+        model_cache = _configured_path(ENV_MODEL_CACHE, root)
+        return cls(
+            # One index per documentation root, rather than one for the whole machine.
+            # Isolation should not depend on the user having set an environment variable.
+            # The model cache below stays shared on purpose: 218 MB of read-only weights,
+            # identical everywhere, and copying it per project would be pure waste.
+            db_path=(db_path if db_path else _project_database(docs_dir if docs_dir else root)),
+            docs_dir=docs_dir if docs_dir else root,
+            embedder=os.environ.get(ENV_EMBEDDER, "").strip() or DEFAULT_EMBEDDER,
+            model_cache_dir=(
+                model_cache
+                if model_cache
+                else _xdg_dir("XDG_CACHE_HOME", ".cache") / "markdown-memory" / "models"
+            ),
+            exclude=parse_exclusions(os.environ.get(ENV_EXCLUDE, "")),
+            index_workers=_positive_int(ENV_INDEX_WORKERS, DEFAULT_INDEX_WORKERS),
+            auto_index=os.environ.get(ENV_AUTO_INDEX, "").strip().lower()
+            not in {"0", "false", "off", "no"},
+        )
+
+
+def resolve_config(
+    *,
+    db: Path | None = None,
+    docs_dir: Path | None = None,
+    embedder: str | None = None,
+    exclude: Sequence[str] = (),
+    auto_index: bool | None = None,
+) -> ServerConfig:
+    """Environment configuration with explicit overrides laid over it.
+
+    Naming a different documentation root re-keys the database, because the default is
+    keyed on that root: taking `ServerConfig.from_env().db_path` as the fallback reads a
+    path derived from the *environment's* root, and two callers pointed at different roots
+    from one directory would land in the launcher's single database - exactly the
+    cross-project leak keying was added to close. An explicitly configured database still
+    wins, from the argument or the environment, in that order.
+
+    Every entry point that takes overrides resolves them here - the server's own flags
+    and the scripts alike - so that precedence is written once and cannot drift between
+    them. The paths that accept none (the in-process service, `eval_retrieval.py`) go
+    straight to `ServerConfig.from_env`, which is the same answer with nothing laid over
+    it.
+    """
+    base = ServerConfig.from_env()
+    root = docs_dir.expanduser() if docs_dir else base.docs_dir
+    configured_db = _configured_path(ENV_DB_PATH, _project_root())
+    return ServerConfig(
+        db_path=(
+            db.expanduser() if db else configured_db if configured_db else _project_database(root)
+        ),
+        docs_dir=root,
+        embedder=embedder or base.embedder,
+        model_cache_dir=base.model_cache_dir,
+        exclude=tuple(exclude) or base.exclude,
+        index_workers=base.index_workers,
+        auto_index=base.auto_index if auto_index is None else auto_index,
+    )
+
+
+def _config_from_cli(arguments: argparse.Namespace) -> ServerConfig:
+    """The command line laid over the environment."""
+    return resolve_config(
+        db=arguments.db,
+        docs_dir=arguments.docs_dir,
+        embedder=arguments.embedder,
+        exclude=arguments.exclude,
+        auto_index=False if getattr(arguments, "no_auto_index", False) else None,
+    )
+
+
+def _project_root() -> Path:
+    """The directory a relative configured path is relative to.
+
+    Claude Code sets the working directory of a project-scoped server to the project root
+    as well, so the fallback agrees with the export in that case; it differs only for a
+    server started by hand from somewhere else.
+    """
+    exported = os.environ.get(ENV_PROJECT_DIR, "").strip()
+    return Path(exported).expanduser() if exported else Path.cwd()
+
+
+def _positive_int(variable: str, default: int) -> int:
+    """A count read from the environment; anything that is not one keeps the default."""
+    value = os.environ.get(variable, "").strip()
+    return int(value) if value.isdigit() and int(value) > 0 else default
+
+
+def _configured_path(variable: str, root: Path) -> Path | None:
+    """One configured path, resolved against ``root`` when it is relative.
+
+    An unexpanded `${...}` is rejected rather than used as a directory name: Claude Code
+    loads a config whose variables it could not expand and passes the literal text through,
+    which would otherwise index a directory named `${workspaceFolder}` and report success
+    over zero files.
+    """
+    value = os.environ.get(variable, "").strip()
+    if not value:
+        return None
+    # A directory really named `docs/${version}` is allowed: if the literal path exists,
+    # it is a path, not a variable nobody expanded.
+    if "${" in value and not Path(value).expanduser().exists():
+        raise ConfigurationError(
+            f"{variable} is set to {value!r}, which still contains an unexpanded variable. "
+            "Claude Code expands only environment variables in .mcp.json - not "
+            "${workspaceFolder} or ${CLAUDE_PROJECT_DIR} - so write the path relative to the "
+            "project root instead (for example '.markdown-memory/index.db')."
+        )
+    path = Path(value).expanduser()
+    return path if path.is_absolute() else (root / path)
+
+
+def parse_exclusions(value: str) -> tuple[str, ...]:
+    """Split a configured exclusion list on commas; blanks and stray ``./`` dropped.
+
+    Comma only: a colon separator would split a pattern that contains one, and silently
+    excluding the wrong thing is worse than not accepting the separator.
+    """
+    patterns = []
+    for part in value.split(","):
+        # One leading "./" only: `lstrip("./")` would eat the dot of `.hidden` and
+        # exclude a `hidden` directory instead of the one that was named.
+        cleaned = part.strip().removeprefix("./").rstrip("/")
+        if cleaned:
+            patterns.append(cleaned)
+    return tuple(patterns)