sessionmemory 0.2.0__py3-none-any.whl

sessionmemory/lib/bootstrap.py

@@ -0,0 +1,138 @@
+"""Create the parts of a vault that cannot be created on demand.
+
+Note directories are not among them. Git does not track an empty directory, so a
+pre-created `learnings/` would not survive the vault's first commit, and every note
+write already creates its own parent. What has to exist up front is what nothing else
+will ever write: the marker that identifies the directory as a vault, the gitignore that
+keeps each field's index out of the backup, and a README for the human who opens
+the vault in Obsidian.
+
+Nothing here overwrites an existing file. Initialization has to be safe to re-run
+against a vault that has been in use for a year, because the reason to re-run it is that
+a later version of the CLI adds a file to this list.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import tomli_w
+
+from sessionmemory.lib import atomic
+from sessionmemory.lib.config import VAULT_MARKER, is_initialized, today
+from sessionmemory.lib.embed import MODEL_CODE
+from sessionmemory.lib.paths import SYSTEM_DIR
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+GITIGNORE = """\
+# Each field's vector index is derived from the pages beside it and is
+# rebuilt by `sessionmemory reindex`. Committing it would churn history on every write.
+*.sqlite3
+*.sqlite3-journal
+
+# Obsidian's per-machine window state, which conflicts on every sync.
+.obsidian/workspace*.json
+
+.DS_Store
+"""
+
+README = f"""\
+# Session Memory Vault
+
+One folder per project under `projects/`. Inside each:
+
+- `learnings/` is a field: flat markdown pages, embedded and searchable.
+- `logs/` is a second field, one page per session, searched on request.
+- `specs/`, `plans/`, and `backlog.md` are plain files, never indexed.
+- `backlog.md` is the list of open items for the project, one line each, sized by
+  effort and grouped by commit type.
+
+The `{MODEL_CODE}.sqlite3` file inside a field is its vector index. It is
+derived from the pages beside it, gitignored, and rebuilt by `sessionmemory reindex`.
+
+`_system/registry.toml` maps each project's git remote and root to its folder here, and
+`_system/vault.toml` is the marker that tells the CLI this directory is a vault.
+
+Pages are created with `sessionmemory new learning` and searched with
+`sessionmemory search`. Everything else is an ordinary file you read and edit directly.
+The format follows the memoryfield spec: https://github.com/calpaterson/memoryfield-spec
+"""
+
+
+@dataclass(frozen=True)
+class InitResult:
+    """What initialization changed, and what it found already in place."""
+
+    vault: Path
+    created: tuple[str, ...]
+    existed: tuple[str, ...]
+
+
+class NotAVaultError(RuntimeError):
+    """Raised when a non-empty directory holding no marker is asked to become a vault."""
+
+
+def is_empty(vault: Path) -> bool:
+    """Report whether `vault` holds nothing but dot entries.
+
+    Shared with `commands/_common.require_vault`, which needs the same check to decide
+    between naming `sessionmemory init` and `sessionmemory init --force` in its fix-it message.
+
+    Args:
+        vault (Path): The directory to check.
+
+    Returns:
+        bool: True when the directory has no non-dot entries.
+    """
+    return all(entry.name.startswith(".") for entry in vault.iterdir())
+
+
+def _marker_contents() -> str:
+    """Build the marker file's contents.
+
+    Returns:
+        str: The TOML text for `_system/vault.toml`.
+    """
+    return tomli_w.dumps({"created": today()})
+
+
+def initialize(vault: Path, *, force: bool = False) -> InitResult:
+    """Create every file a vault needs and report what was and was not already there.
+
+    Args:
+        vault (Path): The directory to initialize. It is created if absent.
+        force (bool): Accept a non-empty directory that holds no vault marker.
+
+    Returns:
+        InitResult: The vault-relative names created and the ones already present.
+
+    Raises:
+        NotAVaultError: If the directory holds files but no marker and `force` is unset.
+    """
+    vault.mkdir(parents=True, exist_ok=True)
+
+    marker = f"{SYSTEM_DIR}/{VAULT_MARKER}"
+    if not force and not is_empty(vault) and not is_initialized(vault):
+        msg = f"{vault} is not empty and has no vault marker; pass --force to initialize it anyway"
+        raise NotAVaultError(msg)
+
+    contents: dict[str, str] = {
+        marker: _marker_contents(),
+        ".gitignore": GITIGNORE,
+        "README.md": README,
+    }
+
+    created: list[str] = []
+    existed: list[str] = []
+    for relative_path, text in contents.items():
+        destination = vault / relative_path
+        if destination.exists():
+            existed.append(relative_path)
+            continue
+        atomic.write_text(destination, text)
+        created.append(relative_path)
+
+    return InitResult(vault=vault, created=tuple(sorted(created)), existed=tuple(sorted(existed)))

sessionmemory/lib/config.py

@@ -0,0 +1,102 @@
+"""Locate the vault."""
+
+from __future__ import annotations
+
+import datetime
+import os
+import tomllib
+from pathlib import Path
+
+from sessionmemory.lib.paths import SYSTEM_DIR
+
+VAULT_ENV_VAR = "SESSIONMEMORY_VAULT"
+
+VAULT_MARKER = "vault.toml"
+
+
+class VaultNotConfiguredError(RuntimeError):
+    """Raised when the vault location is unset or does not exist."""
+
+
+def vault_root() -> Path:
+    """Return the vault directory named by the environment.
+
+    The path is resolved, so every directory the CLI prints is absolute, such as the
+    `project_dir` in `sessionmemory project --json`. A relative value would otherwise be
+    printed as given, and usable only from the directory the variable was written for.
+
+    Returns:
+        Path: The resolved vault directory.
+
+    Raises:
+        VaultNotConfiguredError: If the variable is unset or names a missing directory.
+    """
+    raw = os.environ.get(VAULT_ENV_VAR)
+    if not raw:
+        msg = f"{VAULT_ENV_VAR} is not set. Point it at your vault repository."
+        raise VaultNotConfiguredError(msg)
+
+    root = Path(raw).expanduser().resolve()
+    if not root.is_dir():
+        msg = f"{VAULT_ENV_VAR} is {root}, which does not exist or is not a directory."
+        raise VaultNotConfiguredError(msg)
+
+    return root
+
+
+def today() -> str:
+    """Return today's date as an ISO string.
+
+    Returns:
+        str: Today's date.
+    """
+    return datetime.datetime.now(tz=datetime.UTC).date().isoformat()
+
+
+def now() -> str:
+    """Return the current UTC time as the quoted-string ISO form the memoryfield spec wants.
+
+    Second precision and a `Z` suffix: PyYAML quotes a value of this shape on dump, which
+    is what keeps a YAML 1.1 parser from coercing it to a datetime on the way back in.
+    """
+    stamp = datetime.datetime.now(tz=datetime.UTC).replace(microsecond=0)
+    return stamp.isoformat().replace("+00:00", "Z")
+
+
+def marker_path(vault: Path) -> Path:
+    """Return the path of the file that proves a directory is a vault.
+
+    Args:
+        vault (Path): The vault root.
+
+    Returns:
+        Path: The marker file, whether or not it exists.
+    """
+    return vault / SYSTEM_DIR / VAULT_MARKER
+
+
+def is_initialized(vault: Path) -> bool:
+    """Report whether `vault` has been initialized by `sessionmemory init`.
+
+    A directory that merely exists is not a vault. `SESSIONMEMORY_VAULT` pointing at a
+    home directory or a mistyped path would otherwise be accepted, and the first note
+    written would scatter a `learnings/` tree into it.
+
+    A marker that cannot be parsed is treated as absent rather than as proof, so a
+    truncated file produces "run sessionmemory init" instead of an error from deep inside a
+    later command.
+
+    Args:
+        vault (Path): The vault root.
+
+    Returns:
+        bool: True when a readable marker file is present.
+    """
+    path = marker_path(vault)
+    if not path.is_file():
+        return False
+    try:
+        tomllib.loads(path.read_text(encoding="utf-8"))
+    except (tomllib.TOMLDecodeError, OSError, UnicodeDecodeError):
+        return False
+    return True

sessionmemory/lib/doctor.py

@@ -0,0 +1,186 @@
+"""The six things that can be wrong with a vault, stated without severity.
+
+Every check is a suggestion. The article's argument holds: an irrelevant or imperfect
+page is never surfaced by semantic search, so nothing here fails a build. There is no
+repair, because repair is what grew to three hundred lines last time.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from sessionmemory.lib import field, fieldindex, paths, registry
+from sessionmemory.lib.frontmatter import (
+    FrontmatterError,
+    MissingFrontmatterError,
+    parse,
+    unquoted_datetime_keys,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from sessionmemory.lib.embed import Embedder
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One thing a check found."""
+
+    check: str
+    path: str
+    message: str
+
+
+def _fields(vault: Path) -> list[Path]:
+    return [
+        paths.project_dir(vault, slug) / name
+        for slug in paths.iter_project_slugs(vault)
+        for name in paths.FIELD_DIRS
+        if (paths.project_dir(vault, slug) / name).is_dir()
+    ]
+
+
+def nonconformant_names(vault: Path, _embedder: Embedder) -> list[Finding]:
+    """Report a markdown file in a field whose name breaks the spec's filename rule."""
+    return [
+        Finding("filename", str(path), "not lowercase ascii letters, digits, and hyphens")
+        for directory in _fields(vault)
+        for path in sorted(directory.glob("*.md"))
+        if path.is_file() and not field.is_debris(path.name) and not field.is_page_name(path.name)
+    ]
+
+
+def oversized_pages(vault: Path, _embedder: Embedder) -> list[Finding]:
+    """Report a page over the 8KB limit; only its first 8KB is embedded."""
+    return [
+        Finding(
+            "size",
+            str(path),
+            f"{path.stat().st_size} bytes; split it, the limit is {field.PAGE_LIMIT}",
+        )
+        for directory in _fields(vault)
+        for path in field.iter_pages(directory)
+        if path.stat().st_size > field.PAGE_LIMIT
+    ]
+
+
+def _blank(value: object) -> bool:
+    """Report whether a frontmatter value is missing or an empty/whitespace string."""
+    return not (isinstance(value, str) and value.strip())
+
+
+def malformed_frontmatter(vault: Path, _embedder: Embedder) -> list[Finding]:
+    """Report a page whose frontmatter cannot be parsed, or a learning missing title or summary.
+
+    A logs page with no frontmatter block is fine; a learnings page needs a title and a
+    summary to be worth anything a search result shows, so it has nothing to be missing
+    when it has no block at all.
+    """
+    findings = []
+    for directory in _fields(vault):
+        for path in field.iter_pages(directory):
+            raw = path.read_bytes()
+            try:
+                text = raw.decode("utf-8")
+            except UnicodeDecodeError:
+                findings.append(Finding("frontmatter", str(path), "not valid UTF-8"))
+                continue
+            try:
+                meta, _body = parse(text)
+            except MissingFrontmatterError:
+                if path.parent.name == paths.LEARNINGS_DIR:
+                    findings.append(Finding("frontmatter", str(path), "missing title or summary"))
+                continue
+            except FrontmatterError as error:
+                findings.append(Finding("frontmatter", str(path), str(error)))
+                continue
+            if path.parent.name == paths.LEARNINGS_DIR and (
+                _blank(meta.get("title")) or _blank(meta.get("summary"))
+            ):
+                findings.append(Finding("frontmatter", str(path), "missing title or summary"))
+    return findings
+
+
+def unquoted_datetimes(vault: Path, _embedder: Embedder) -> list[Finding]:
+    """Report a page whose frontmatter carries a bare date or datetime.
+
+    The spec requires quoting them: a YAML 1.1 parser types a bare value and a YAML 1.2
+    parser leaves it a string, so what the page says would depend on which reads it. A
+    block that cannot be parsed at all is left to the frontmatter check.
+    """
+    findings = []
+    for directory in _fields(vault):
+        for path in field.iter_pages(directory):
+            try:
+                keys = unquoted_datetime_keys(path.read_text(encoding="utf-8"))
+            except (UnicodeDecodeError, FrontmatterError):
+                continue
+            if keys:
+                message = f"{', '.join(keys)}: unquoted; quote the value"
+                findings.append(Finding("datetime", str(path), message))
+    return findings
+
+
+def dead_projects(vault: Path, _embedder: Embedder) -> list[Finding]:
+    """Report a registered project whose repository root no longer exists."""
+    try:
+        projects = registry.load(vault)
+    except registry.RegistryError as error:
+        return [Finding("registry", "", str(error))]
+    return [
+        Finding("project", slug, f"root {project.root} does not exist")
+        for slug, project in sorted(projects.items())
+        if project.root and not Path(project.root).is_dir()
+    ]
+
+
+def _is_stale(directory: Path, embedder: Embedder) -> str | None:
+    index = fieldindex.index_path(directory, embedder)
+    if not index.is_file():
+        return None
+    try:
+        conn = fieldindex.connect(index)
+    except sqlite3.DatabaseError:
+        return "index is not a readable database; run: sessionmemory reindex"
+    try:
+        stored = {
+            row["filename"]: row["sha256_hash"]
+            for row in conn.execute("SELECT filename, sha256_hash FROM pages")
+        }
+    finally:
+        conn.close()
+    current = {
+        path.name: hashlib.sha256(path.read_bytes()).digest()
+        for path in field.iter_pages(directory)
+    }
+    return None if stored == current else "index is behind its pages; run: sessionmemory reindex"
+
+
+def stale_indexes(vault: Path, embedder: Embedder) -> list[Finding]:
+    """Report a field whose index file is unreadable or behind its pages."""
+    findings = []
+    for directory in _fields(vault):
+        message = _is_stale(directory, embedder)
+        if message:
+            findings.append(Finding("index", str(directory), message))
+    return findings
+
+
+CHECKS: tuple[Callable[[Path, Embedder], list[Finding]], ...] = (
+    nonconformant_names,
+    oversized_pages,
+    malformed_frontmatter,
+    unquoted_datetimes,
+    dead_projects,
+    stale_indexes,
+)
+
+
+def run(vault: Path, embedder: Embedder) -> list[Finding]:
+    """Run every check and return what they found."""
+    return [finding for check in CHECKS for finding in check(vault, embedder)]

sessionmemory/lib/embed.py

@@ -0,0 +1,123 @@
+"""Turn note text into vectors, in-process and without a daemon.
+
+`fastembed` runs `nomic-embed-text-v1.5` on ONNX Runtime inside this process. The
+alternative considered was Ollama, and it was rejected for one reason: it is a service,
+and a service is something every note write would depend on being up. A model file on
+disk cannot be down.
+
+The model cache is pinned rather than left at fastembed's default, which lives under the
+system temp directory. macOS purges that periodically, and a silent re-download of 520MB
+in the middle of `sessionmemory new` is indistinguishable from a hang.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import random
+from pathlib import Path
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from fastembed import TextEmbedding
+
+MODEL_CODE = "nomic-embed-text-v1.5"
+MODEL_NAME = "nomic-ai/nomic-embed-text-v1.5"
+DIM = 768
+
+# nomic is trained with task prefixes and fastembed 0.8 does not add them, so they are
+# added here, which the memoryfield spec permits for a model that mandates them.
+DOCUMENT_PREFIX = "search_document: "
+QUERY_PREFIX = "search_query: "
+
+CACHE_ENV_VAR = "SESSIONMEMORY_MODEL_CACHE"
+DEFAULT_CACHE = Path.home() / ".cache" / "sessionmemory" / "models"
+
+
+class Embedder(Protocol):
+    """Anything that turns pages and queries into vectors of `dim` floats.
+
+    `name` is the model code the index file is named for, so two embedders with
+    different names never share an index.
+    """
+
+    name: str
+    dim: int
+
+    def encode_documents(self, texts: Sequence[str]) -> list[list[float]]:
+        """Return one vector per page text, in the order given."""
+        ...  # pragma: no cover
+
+    def encode_query(self, text: str) -> list[float]:
+        """Return the vector for one search query."""
+        ...  # pragma: no cover
+
+
+class FastEmbedder:
+    """Embeds with the real model, loaded on first use rather than at construction."""
+
+    def __init__(self, cache_dir: Path = DEFAULT_CACHE) -> None:
+        self.name = MODEL_CODE
+        self.dim = DIM
+        self.cache_dir = cache_dir
+        self._model: TextEmbedding | None = None
+
+    def _load(self) -> TextEmbedding:
+        if self._model is None:
+            from fastembed import TextEmbedding
+
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            self._model = TextEmbedding(model_name=MODEL_NAME, cache_dir=str(self.cache_dir))
+        return self._model
+
+    def _encode(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        return [vector.tolist() for vector in self._load().embed(texts)]
+
+    def encode_documents(self, texts: Sequence[str]) -> list[list[float]]:
+        """Return one vector per page text, in the order given."""
+        return self._encode([f"{DOCUMENT_PREFIX}{text}" for text in texts])
+
+    def encode_query(self, text: str) -> list[float]:
+        """Return the vector for one search query."""
+        return self._encode([f"{QUERY_PREFIX}{text}"])[0]
+
+
+class StubEmbedder:
+    """Embeds deterministically from a hash, with no model and no network."""
+
+    def __init__(self) -> None:
+        self.name = "stub"
+        self.dim = DIM
+
+    def _one(self, text: str) -> list[float]:
+        seed = int.from_bytes(hashlib.sha256(text.encode("utf-8")).digest()[:8], "big")
+        rng = random.Random(seed)
+        raw = [rng.gauss(0.0, 1.0) for _ in range(self.dim)]
+        norm = math.sqrt(sum(value * value for value in raw)) or 1.0
+        return [value / norm for value in raw]
+
+    def encode_documents(self, texts: Sequence[str]) -> list[list[float]]:
+        """Return one vector per page text, in the order given."""
+        return [self._one(text) for text in texts]
+
+    def encode_query(self, text: str) -> list[float]:
+        """Return the vector for one search query."""
+        return self._one(text)
+
+
+def default_embedder() -> FastEmbedder:
+    """Build the embedder commands use, reading the model cache location from the environment.
+
+    Returns:
+        FastEmbedder: An embedder caching to `SESSIONMEMORY_MODEL_CACHE`, or
+            `DEFAULT_CACHE` when that variable is unset or empty.
+    """
+    raw = os.environ.get(CACHE_ENV_VAR)
+    if not raw:
+        return FastEmbedder(cache_dir=DEFAULT_CACHE)
+    return FastEmbedder(cache_dir=Path(raw).expanduser())

sessionmemory/lib/export.py

@@ -0,0 +1,25 @@
+"""Write one field as a `.memoryfield.zip`, the spec's archival transport."""
+
+from __future__ import annotations
+
+import zipfile
+from typing import TYPE_CHECKING
+
+from sessionmemory.lib import field, fieldindex
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from sessionmemory.lib.embed import Embedder
+
+
+def export_field(directory: Path, embedder: Embedder, output: Path) -> Path:
+    """Refresh the index, then archive every page and the index flat at the zip root."""
+    fieldindex.refresh(directory, embedder)
+    with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+        for path in field.iter_pages(directory):
+            archive.write(path, arcname=path.name)
+        index = fieldindex.index_path(directory, embedder)
+        if index.is_file():
+            archive.write(index, arcname=index.name)
+    return output