sessionmemory 0.2.0__py3-none-any.whl

sessionmemory/lib/field.py

@@ -0,0 +1,181 @@
+"""A page in a field: a markdown file with optional YAML frontmatter in a flat directory.
+
+This is the one place a page's shape is known. The filename rule, the debris rule, and
+the 8KB soft limit come from the memoryfield spec, and a page without frontmatter is a
+valid page there, so `read_page` never refuses one.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from dataclasses import dataclass
+from itertools import islice
+from typing import TYPE_CHECKING, Any
+
+from sessionmemory.lib import atomic
+from sessionmemory.lib.frontmatter import FrontmatterError, parse, serialize
+from sessionmemory.lib.ids import id_candidates, slugify
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from pathlib import Path
+
+PAGE_LIMIT = 8192
+
+_PAGE_NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.md$")
+_DEBRIS_NAMES = frozenset({".DS_Store", "desktop.ini", "Thumbs.db"})
+_MAX_CLAIM_ATTEMPTS = 1000
+
+
+class PageError(ValueError):
+    """Raised when a page cannot be created as asked."""
+
+
+@dataclass(frozen=True)
+class Page:
+    """One page as read from disk."""
+
+    path: Path
+    meta: dict[str, Any]
+    body: str
+
+    def _text(self, key: str) -> str:
+        value = self.meta.get(key)
+        return value if isinstance(value, str) else ""
+
+    @property
+    def title(self) -> str:
+        """The page's title, or empty."""
+        return self._text("title")
+
+    @property
+    def uuid(self) -> str:
+        """The page's uuid, or empty."""
+        return self._text("uuid")
+
+    @property
+    def summary(self) -> str:
+        """The page's one-line summary, or empty."""
+        return self._text("summary")
+
+    @property
+    def created(self) -> str:
+        """The page's creation datetime, or empty."""
+        return self._text("created")
+
+    @property
+    def updated(self) -> str:
+        """The page's last-updated datetime, or empty."""
+        return self._text("updated")
+
+    @property
+    def size(self) -> int:
+        """The file's size in bytes."""
+        return self.path.stat().st_size
+
+
+def is_page_name(name: str) -> bool:
+    """Report whether a filename conforms to the spec's page filename rule."""
+    return _PAGE_NAME.match(name) is not None
+
+
+def is_debris(name: str) -> bool:
+    """Report whether a filename is sync, editor, or OS debris the spec says to ignore."""
+    return name in _DEBRIS_NAMES or ".sync-conflict-" in name or name.endswith("~")
+
+
+def iter_pages(directory: Path) -> list[Path]:
+    """List the pages in one field, sorted by name.
+
+    Only conformant names at the top level count; the spec forbids indexing pages in
+    sub-directories, and a non-conformant name is reported by `doctor` rather than read.
+    """
+    if not directory.is_dir():
+        return []
+    return sorted(
+        path
+        for path in directory.iterdir()
+        if path.is_file() and not is_debris(path.name) and is_page_name(path.name)
+    )
+
+
+def read_page(path: Path) -> Page:
+    """Read a page, tolerating a missing frontmatter block or invalid UTF-8."""
+    text = path.read_bytes().decode("utf-8", errors="replace")
+    try:
+        meta, body = parse(text)
+    except FrontmatterError:
+        return Page(path=path, meta={}, body=text)
+    return Page(path=path, meta=meta, body=body)
+
+
+def _already_says(path: Path, meta: Mapping[str, Any], body: str) -> bool:
+    try:
+        current_meta, current_body = parse(path.read_text(encoding="utf-8"))
+    except (OSError, FrontmatterError, UnicodeDecodeError):
+        return False
+    return current_meta == dict(meta) and current_body.rstrip() == body.rstrip()
+
+
+def write_page(path: Path, meta: Mapping[str, Any], body: str) -> None:
+    """Write a page atomically, leaving a file that already says this untouched.
+
+    A formatter restyles frontmatter quoting and spacing; comparing parsed content rather
+    than bytes keeps this CLI and a formatter from trading edits forever.
+    """
+    if _already_says(path, meta, body):
+        return
+    atomic.write_text(path, serialize(meta, body))
+
+
+def claim_filename(directory: Path, title: str, *, stem: str | None = None) -> Path:
+    """Create an empty file for a new page, taking the first free name.
+
+    Exclusive creation is the gate, so two writers racing for one title get two files.
+
+    Raises:
+        PageError: If the title yields no slug or no name is free.
+    """
+    if stem is None:
+        try:
+            stem = slugify(title)
+        except ValueError as error:
+            raise PageError(str(error)) from error
+    taken = {path.stem for path in iter_pages(directory)}
+    for candidate in islice(id_candidates(stem, taken), _MAX_CLAIM_ATTEMPTS):
+        path = directory / f"{candidate}.md"
+        if atomic.claim(path):
+            return path
+    msg = f"no free filename for {stem!r} in {directory}"
+    raise PageError(msg)
+
+
+def _create(directory: Path, meta: dict[str, Any], body: str, *, stem: str | None) -> Path:
+    path = claim_filename(directory, str(meta["title"]), stem=stem)
+    try:
+        atomic.write_text(path, serialize(meta, body))
+    except BaseException:
+        path.unlink(missing_ok=True)
+        raise
+    return path
+
+
+def new_page(directory: Path, *, title: str, summary: str, body: str, now: str) -> Path:
+    """Create a memory page with the five fields the spec defines."""
+    meta = {
+        "title": title,
+        "uuid": str(uuid.uuid4()),
+        "summary": summary,
+        "created": now,
+        "updated": now,
+    }
+    return _create(directory, meta, body, stem=None)
+
+
+def new_document(
+    directory: Path, *, title: str, body: str, now: str, stem: str | None = None
+) -> Path:
+    """Create a spec, plan, or log: a titled, dated file that is not a memory page."""
+    meta = {"title": title, "created": now, "updated": now}
+    return _create(directory, meta, body, stem=stem)

sessionmemory/lib/fieldindex.py

@@ -0,0 +1,214 @@
+"""One field's vector index: the spec's `pages` table in one SQLite file.
+
+The index is derived from the pages beside it and can be deleted at any time. Freshness
+is the sha256 of each file, so a page edited in any editor is re-embedded on the next
+read, and every read refreshes first so nobody has to remember `reindex`.
+
+There is no application lock. A page write is an atomic rename, SQLite serializes its
+own writers, and a corrupt file is discarded and rebuilt rather than reported.
+"""
+
+from __future__ import annotations
+
+import datetime
+import hashlib
+import json
+import sqlite3
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import sqlite_vec
+
+from sessionmemory.lib import field
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from sessionmemory.lib.embed import Embedder
+
+# Measured on a real vault with nomic-embed-text-v1.5: a page that answers the query sits
+# under 0.25, a related neighbor under 0.40, and the nearest page to an unrelated query
+# sits at 0.45 or beyond. It matches the reference implementation's default for the model.
+DEFAULT_MAX_DISTANCE = 0.45
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS pages (
+    filename      TEXT PRIMARY KEY,
+    frontmatter   JSON NOT NULL,
+    last_modified DATETIME NOT NULL,
+    sha256_hash   BLOB NOT NULL,
+    embedding     BLOB NOT NULL
+);
+"""
+
+
+@dataclass(frozen=True)
+class Hit:
+    """One search result, nearest first when ordered by `distance`."""
+
+    path: Path
+    title: str
+    summary: str
+    distance: float
+
+
+@dataclass(frozen=True)
+class Refresh:
+    """What one refresh changed."""
+
+    added: int
+    updated: int
+    removed: int
+    unchanged: int
+
+
+def index_path(field_dir: Path, embedder: Embedder) -> Path:
+    """Return the index file for one field, named for the model that fills it."""
+    return field_dir / f"{embedder.name}.sqlite3"
+
+
+def connect(path: Path) -> sqlite3.Connection:
+    """Open an index file, creating the table if the file is new."""
+    conn = sqlite3.connect(path)
+    try:
+        conn.row_factory = sqlite3.Row
+        conn.enable_load_extension(True)  # noqa: FBT003
+        sqlite_vec.load(conn)
+        conn.enable_load_extension(False)  # noqa: FBT003
+        conn.execute("PRAGMA busy_timeout = 5000")
+        conn.executescript(_SCHEMA)
+    except BaseException:
+        conn.close()
+        raise
+    return conn
+
+
+def _open(path: Path) -> sqlite3.Connection:
+    """Open the index, discarding and recreating a file SQLite cannot read."""
+    try:
+        return connect(path)
+    except sqlite3.DatabaseError:
+        path.unlink(missing_ok=True)
+        path.with_name(f"{path.name}-journal").unlink(missing_ok=True)
+        return connect(path)
+
+
+def embedding_input(text: str) -> str:
+    """Return the first `PAGE_LIMIT` bytes of `text`, never splitting a character."""
+    encoded = text.encode("utf-8")
+    if len(encoded) <= field.PAGE_LIMIT:
+        return text
+    return encoded[: field.PAGE_LIMIT].decode("utf-8", errors="ignore")
+
+
+def _mtime_iso(path: Path) -> str:
+    stamp = datetime.datetime.fromtimestamp(path.stat().st_mtime, tz=datetime.UTC)
+    return stamp.replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def _refresh(conn: sqlite3.Connection, field_dir: Path, embedder: Embedder) -> Refresh:
+    stored = {
+        row["filename"]: row["sha256_hash"]
+        for row in conn.execute("SELECT filename, sha256_hash FROM pages")
+    }
+    pending: list[tuple[str, str, str, bytes, str]] = []
+    unchanged = 0
+    seen: set[str] = set()
+    for path in field.iter_pages(field_dir):
+        seen.add(path.name)
+        raw = path.read_bytes()
+        digest = hashlib.sha256(raw).digest()
+        # Hash the raw bytes so freshness tracks the file as written; a page saved
+        # with invalid UTF-8 is still indexed rather than crashing the refresh.
+        text = raw.decode("utf-8", errors="replace")
+        if stored.get(path.name) == digest:
+            unchanged += 1
+            continue
+        page = field.read_page(path)
+        pending.append((path.name, json.dumps(page.meta), _mtime_iso(path), digest, text))
+
+    gone = sorted(set(stored) - seen)
+    conn.executemany("DELETE FROM pages WHERE filename = ?", [(name,) for name in gone])
+
+    # The spec's embedding input is the whole file, frontmatter included.
+    vectors = embedder.encode_documents([embedding_input(text) for *_rest, text in pending])
+    conn.executemany(
+        "INSERT OR REPLACE INTO pages (filename, frontmatter, last_modified, sha256_hash, embedding)"
+        " VALUES (?, ?, ?, ?, ?)",
+        [
+            (name, meta, modified, digest, sqlite_vec.serialize_float32(vector))
+            for (name, meta, modified, digest, _text), vector in zip(pending, vectors, strict=True)
+        ],
+    )
+    conn.commit()
+    added = sum(1 for name, *_rest in pending if name not in stored)
+    return Refresh(
+        added=added, updated=len(pending) - added, removed=len(gone), unchanged=unchanged
+    )
+
+
+def refresh(field_dir: Path, embedder: Embedder) -> Refresh:
+    """Bring one field's index up to date with the pages beside it."""
+    if not field_dir.is_dir():
+        return Refresh(added=0, updated=0, removed=0, unchanged=0)
+    conn = _open(index_path(field_dir, embedder))
+    try:
+        return _refresh(conn, field_dir, embedder)
+    finally:
+        conn.close()
+
+
+def search(
+    field_dir: Path,
+    embedder: Embedder,
+    query: str,
+    *,
+    limit: int,
+    max_distance: float = DEFAULT_MAX_DISTANCE,
+) -> list[Hit]:
+    """Return the pages within `max_distance` of `query`, nearest first, refreshing the index first.
+
+    A cutoff rather than a bare top-k, so a query nothing answers returns nothing instead
+    of the nearest pages dressed up as hits.
+    """
+    if not field_dir.is_dir():
+        return []
+    conn = _open(index_path(field_dir, embedder))
+    try:
+        _refresh(conn, field_dir, embedder)
+        rows = conn.execute(
+            "SELECT * FROM ("
+            " SELECT filename, frontmatter, vec_distance_cosine(embedding, ?) AS distance"
+            " FROM pages)"
+            " WHERE distance <= ? ORDER BY distance LIMIT ?",
+            (sqlite_vec.serialize_float32(embedder.encode_query(query)), max_distance, limit),
+        ).fetchall()
+    finally:
+        conn.close()
+    hits = []
+    for row in rows:
+        meta = json.loads(row["frontmatter"])
+        title = meta.get("title")
+        summary = meta.get("summary")
+        hits.append(
+            Hit(
+                path=field_dir / row["filename"],
+                title=title if isinstance(title, str) else "",
+                summary=summary if isinstance(summary, str) else "",
+                distance=float(row["distance"]),
+            )
+        )
+    return hits
+
+
+def forget(field_dir: Path, embedder: Embedder, filename: str) -> None:
+    """Drop one page's row, so a deleted page stops matching before the next refresh."""
+    path = index_path(field_dir, embedder)
+    if not path.is_file():
+        return
+    conn = _open(path)
+    try:
+        conn.execute("DELETE FROM pages WHERE filename = ?", (filename,))
+        conn.commit()
+    finally:
+        conn.close()

sessionmemory/lib/frontmatter.py

@@ -0,0 +1,163 @@
+"""Read and write the YAML frontmatter block that heads every note.
+
+Dates are normalized to ISO strings on the way in and quoted on the way out. PyYAML
+would otherwise load `created: 2026-08-01` as a `datetime.date` and dump it back
+unquoted, so a value's type would depend on whether the note had been round tripped.
+"""
+
+from __future__ import annotations
+
+import datetime
+from typing import TYPE_CHECKING, Any
+
+import yaml
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+DELIMITER = "---"
+BOM = "\ufeff"
+
+
+class FrontmatterError(ValueError):
+    """Raised when a note's frontmatter is absent or malformed."""
+
+
+class MissingFrontmatterError(FrontmatterError):
+    """Raised when a note has no frontmatter block at all."""
+
+
+def _stringify_dates(value: Any) -> Any:  # noqa: ANN401
+    """Convert any date or datetime in a nested structure to an ISO string.
+
+    Args:
+        value (Any): A value that may contain dates at any depth.
+
+    Returns:
+        Any: The same structure with dates replaced by ISO strings.
+    """
+    if isinstance(value, datetime.date):
+        return value.isoformat()
+    if isinstance(value, dict):
+        return {k: _stringify_dates(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_stringify_dates(v) for v in value]
+    return value
+
+
+def _split(text: str) -> tuple[str, str]:
+    """Return the raw YAML block and the body, tolerating a BOM and CRLF endings.
+
+    Raises:
+        FrontmatterError: If the block is missing or unterminated.
+    """
+    text = text.removeprefix(BOM).replace("\r\n", "\n")
+
+    if not text.startswith(f"{DELIMITER}\n"):
+        msg = "note has no frontmatter block"
+        raise MissingFrontmatterError(msg)
+
+    rest = text[len(DELIMITER) + 1 :]
+    if rest.startswith(f"{DELIMITER}\n"):
+        # An empty block's closing delimiter sits immediately after the opening
+        # one, so the "\n---\n" search below never gets the leading newline it
+        # needs: the opening strip already consumed it.
+        raw_meta = ""
+        body = rest[len(DELIMITER) + 1 :]
+    elif rest == DELIMITER:
+        # Same empty-block case, but the closing delimiter is also the end of
+        # the file, so there is no body to slice off.
+        raw_meta = ""
+        body = ""
+    else:
+        end = rest.find(f"\n{DELIMITER}\n")
+        if end == -1:
+            if rest.endswith(f"\n{DELIMITER}"):
+                # Closing delimiter is the last thing in the file, so there is no
+                # trailing "\n" to anchor the usual end + len(DELIMITER) + 2 offset.
+                raw_meta = rest[: -len(DELIMITER) - 1]
+                body = ""
+            else:
+                msg = "frontmatter block was never closed"
+                raise FrontmatterError(msg)
+        else:
+            raw_meta = rest[:end]
+            body = rest[end + len(DELIMITER) + 2 :]
+    return raw_meta, body
+
+
+def _load(raw_meta: str) -> dict[str, Any]:
+    """Load a raw YAML block as the mapping it must be, with values as YAML typed them.
+
+    Raises:
+        FrontmatterError: If the block is not valid YAML or not a mapping.
+    """
+    try:
+        loaded = yaml.safe_load(raw_meta)
+    except yaml.YAMLError as error:
+        msg = f"frontmatter is not valid YAML: {error}"
+        raise FrontmatterError(msg) from error
+    loaded = {} if loaded is None else loaded
+    if not isinstance(loaded, dict):
+        msg = f"frontmatter must be a mapping, got {type(loaded).__name__}"
+        raise FrontmatterError(msg)
+    return loaded
+
+
+def parse(text: str) -> tuple[dict[str, Any], str]:
+    """Split a note into its frontmatter mapping and its markdown body.
+
+    A leading byte order mark and Windows line endings are tolerated. Notes are
+    hand-edited in whatever editor is at hand, and a note is still a note when it comes
+    back with a BOM or CRLF endings, so refusing it would refuse a legible file.
+
+    Args:
+        text (str): The full contents of a note file.
+
+    Returns:
+        tuple[dict[str, Any], str]: The metadata mapping and the body.
+
+    Raises:
+        FrontmatterError: If the block is missing, unterminated, or not a mapping.
+    """
+    raw_meta, body = _split(text)
+    return _stringify_dates(_load(raw_meta)), body
+
+
+def unquoted_datetime_keys(text: str) -> list[str]:
+    """Return the top-level keys whose bare values YAML typed as a date or datetime.
+
+    The memoryfield spec requires quoting datetimes, since a YAML 1.1 parser types a bare
+    one and a YAML 1.2 parser does not, so what the page says would depend on the reader.
+
+    Args:
+        text (str): The full contents of a note file.
+
+    Returns:
+        list[str]: The offending keys in file order, empty when every value is quoted.
+
+    Raises:
+        FrontmatterError: If the block is missing, unterminated, or not a mapping.
+    """
+    raw_meta, _body = _split(text)
+    return [key for key, value in _load(raw_meta).items() if isinstance(value, datetime.date)]
+
+
+def serialize(meta: Mapping[str, Any], body: str) -> str:
+    """Render a metadata mapping and a body back into note file contents.
+
+    Args:
+        meta (Mapping[str, Any]): The metadata to write.
+        body (str): The markdown body.
+
+    Returns:
+        str: The complete file contents, ending in a single newline.
+    """
+    dumped = yaml.safe_dump(
+        dict(meta),
+        sort_keys=False,
+        default_flow_style=False,
+        allow_unicode=True,
+        width=88,
+    )
+    return f"{DELIMITER}\n{dumped}{DELIMITER}\n{body.rstrip()}\n"