hotmemory 0.1.0__py3-none-any.whl

hotmemory/__init__.py

@@ -0,0 +1,26 @@
+"""Agent memory as tables on Hotdata."""
+
+from hotmemory.contract import Fact, Memory
+from hotmemory.filter import Filter, TimeRange
+from hotmemory.memory import MemoryStore, MemoryWriter
+from hotmemory.record import SCHEMA_VERSION, JSONValue, Kind, Record, normalize
+from hotmemory.store import Clock, Embedder, Hit, Store, Writer
+
+__all__ = [
+    "SCHEMA_VERSION",
+    "Clock",
+    "Embedder",
+    "Fact",
+    "Filter",
+    "Hit",
+    "JSONValue",
+    "Kind",
+    "Memory",
+    "MemoryStore",
+    "MemoryWriter",
+    "Record",
+    "Store",
+    "TimeRange",
+    "Writer",
+    "normalize",
+]

hotmemory/_chunks.py

@@ -0,0 +1,184 @@
+"""The chunking of a Markdown document for `Memory.load`, and of a text for `Memory.capture`."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+_HEADING = re.compile(r"^ {0,3}(#{1,6})(?:[ \t]+|$)")
+_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
+_TABLE_DELIMITER = re.compile(r"^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")
+PATH_SEPARATOR = " > "
+
+
+@dataclass(frozen=True)
+class Chunk:
+    """One chunk of a document: the heading path of its first line, and its text."""
+
+    path: tuple[str, ...]
+    text: str
+
+
+@dataclass(frozen=True)
+class _Line:
+    text: str
+    level: int
+    fenced: bool
+
+
+def chunk_document(text: str, chunk_chars: int) -> list[str]:
+    """Return the chunks of the Markdown `text`, each with its heading prefix.
+
+    A chunk is `<prefix>\\n\\n<body>`. The prefix is the first heading of the document,
+    then the heading path of the chunk's first line, joined by ` > `, with a heading that
+    repeats the one before it left out. A document with no heading gives chunks with no
+    prefix. The body holds at most `chunk_chars` characters, unless one line is longer.
+    A document of blank lines only gives no chunk.
+    """
+    lines = _scan(text)
+    title = next((line.text.strip() for line in lines if line.level), "")
+    chunks = []
+    for chunk in pack(lines, chunk_chars):
+        parts: list[str] = []
+        for heading in (title, *chunk.path):
+            if heading and (not parts or parts[-1] != heading):
+                parts.append(heading)
+        prefix = PATH_SEPARATOR.join(parts)
+        chunks.append(f"{prefix}\n\n{chunk.text}" if prefix else chunk.text)
+    return chunks
+
+
+def chunk_text(text: str, chunk_chars: int) -> list[str]:
+    """Return the chunks of `text`, cut at blank lines outside fences, with no prefix.
+
+    A heading line does not start a section. A block that does not fit is cut at line
+    ends, and a line that does not fit is cut at `chunk_chars`. The blocks are packed in
+    order into bodies of at most `chunk_chars`, joined by a blank line. A text of blank
+    lines only gives no chunk.
+    """
+    lines = [_Line(line.text, 0, line.fenced) for line in _scan(text)]
+    return [chunk.text for chunk in pack(lines, chunk_chars)]
+
+
+def pack(lines: list[_Line], chunk_chars: int) -> list[Chunk]:
+    """Return the sections of `lines`, cut and packed into bodies of at most `chunk_chars`.
+
+    The sections start at each heading outside a fence. A section that does not fit is
+    cut into blocks at blank lines outside fences, and its heading stays with the block
+    after it when both fit. A block that does not fit is cut at
+    line ends, and a line that does not fit is cut at `chunk_chars`. A cut pipe table
+    repeats its header row and delimiter row at the top of each later piece. Then the
+    pieces are packed in order, joined by a blank line.
+    """
+    pieces: list[Chunk] = []
+    for path, section in _sections(lines):
+        body = _join(section)
+        if not body:
+            continue
+        if len(body) <= chunk_chars:
+            pieces.append(Chunk(path, body))
+            continue
+        blocks = _blocks(section)
+        if len(blocks) > 1 and section[0].level and len(blocks[0]) == 1:
+            joined = [*blocks[0], "", *blocks[1]]
+            if len("\n".join(joined)) <= chunk_chars:
+                blocks[:2] = [joined]
+        for block in blocks:
+            pieces.extend(Chunk(path, piece) for piece in _cut_block(block, chunk_chars))
+    packed: list[Chunk] = []
+    for piece in pieces:
+        if packed and len(packed[-1].text) + 2 + len(piece.text) <= chunk_chars:
+            packed[-1] = Chunk(packed[-1].path, f"{packed[-1].text}\n\n{piece.text}")
+        else:
+            packed.append(piece)
+    return packed
+
+
+def _scan(text: str) -> list[_Line]:
+    """Return each line of `text` with its heading level, 0 for none, and its fence state."""
+    lines = []
+    fence = ""
+    for raw in text.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
+        opener = _FENCE.match(raw)
+        if fence:
+            marker = opener.group(1) if opener is not None else ""
+            if marker[:1] == fence[0] and len(marker) >= len(fence) and not raw.strip(" `~"):
+                fence = ""
+            lines.append(_Line(raw, 0, True))
+            continue
+        if opener is not None:
+            fence = opener.group(1)
+            lines.append(_Line(raw, 0, True))
+            continue
+        heading = _HEADING.match(raw)
+        lines.append(_Line(raw, len(heading.group(1)) if heading else 0, False))
+    return lines
+
+
+def _sections(lines: list[_Line]) -> list[tuple[tuple[str, ...], list[_Line]]]:
+    """Return the sections of `lines`, each with the heading path of its heading."""
+    sections: list[tuple[tuple[str, ...], list[_Line]]] = [((), [])]
+    stack: list[tuple[int, str]] = []
+    for line in lines:
+        if line.level:
+            while stack and stack[-1][0] >= line.level:
+                stack.pop()
+            stack.append((line.level, line.text.strip()))
+            sections.append((tuple(text for _, text in stack), []))
+        sections[-1][1].append(line)
+    return sections
+
+
+def _blocks(lines: list[_Line]) -> list[list[str]]:
+    """Return the runs of lines between blank lines outside fences."""
+    blocks: list[list[str]] = [[]]
+    for line in lines:
+        if not line.fenced and not line.text.strip():
+            if blocks[-1]:
+                blocks.append([])
+            continue
+        blocks[-1].append(line.text)
+    return [block for block in blocks if block]
+
+
+def _cut_block(block: list[str], chunk_chars: int) -> list[str]:
+    """Return `block` cut at line ends into pieces of at most `chunk_chars` characters.
+
+    A line longer than `chunk_chars` is cut at that size. A pipe table repeats its first
+    two rows at the top of each piece after the first, when they leave room for a row.
+    """
+    header: list[str] = []
+    if len(block) > 2 and block[0].lstrip().startswith("|") and _TABLE_DELIMITER.match(block[1]):
+        header = block[:2]
+        if len("\n".join(header)) * 2 >= chunk_chars:
+            header = []
+    pieces: list[str] = []
+    current: list[str] = []
+    for line in block:
+        for part in _cut_line(line, chunk_chars):
+            if current and len("\n".join([*current, part])) > chunk_chars:
+                pieces.append("\n".join(current))
+                current = list(header) if header and part.lstrip().startswith("|") else []
+                if current and len("\n".join([*current, part])) > chunk_chars:
+                    current = []
+            current.append(part)
+    if current:
+        pieces.append("\n".join(current))
+    return pieces
+
+
+def _cut_line(line: str, chunk_chars: int) -> list[str]:
+    """Return `line` cut into parts of at most `chunk_chars` characters."""
+    if len(line) <= chunk_chars:
+        return [line]
+    return [line[start : start + chunk_chars] for start in range(0, len(line), chunk_chars)]
+
+
+def _join(lines: list[_Line]) -> str:
+    """Return the text of `lines` without the blank lines at either end."""
+    texts = [line.text for line in lines]
+    while texts and not texts[0].strip():
+        texts.pop(0)
+    while texts and not texts[-1].strip():
+        texts.pop()
+    return "\n".join(texts)

hotmemory/_rules.py

@@ -0,0 +1,196 @@
+"""Plain functions that every driver calls, so the drivers agree on the contract."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass, replace
+from datetime import datetime
+
+from hotmemory.filter import Filter, kinds_of
+from hotmemory.record import JSONValue, Kind, Record, check_label, normalize
+
+
+@dataclass(frozen=True)
+class Draft:
+    """One put before a driver writes it: the record, and whether it closes the current one."""
+
+    record: Record
+    close_previous: bool = False
+
+
+def check_prefix(prefix: Sequence[str]) -> tuple[str, ...]:
+    """Return `prefix` as a tuple of labels. An empty prefix is allowed and matches all."""
+    if isinstance(prefix, str):
+        raise TypeError("prefix must be a sequence of labels, not a string")
+    labels = tuple(prefix)
+    for label in labels:
+        check_label(label)
+    return labels
+
+
+def under_prefix(namespace: tuple[str, ...], prefix: tuple[str, ...]) -> bool:
+    """Return True if the first labels of `namespace` are the labels of `prefix`."""
+    return namespace[: len(prefix)] == prefix
+
+
+def next_revision(current: Record | None) -> int:
+    """Return the revision number for the next put after `current`."""
+    return 1 if current is None else current.revision + 1
+
+
+def check_episode_line(current: Record | None, kind: Kind) -> None:
+    """Raise ValueError if a put of `kind` moves the key of `current` across the episode line.
+
+    A key holds episodes only, or holds no episode at all.
+    """
+    if current is not None and (current.kind == "episode") != (kind == "episode"):
+        raise ValueError(
+            f"key {current.key!r} holds kind {current.kind!r}; a put of kind {kind!r} "
+            "cannot move a key between 'episode' and another kind"
+        )
+
+
+def is_duplicate(current: Record, content: str, now: datetime) -> bool:
+    """Return True if a put of `content` writes nothing because `current` holds it.
+
+    That is the case when `current` is listed at `now` and its normalized content equals
+    the normalized `content`. A current revision past its `forget_after` is never a
+    duplicate, so a put brings the fact back as a new revision.
+    """
+    return is_listed(current, now) and normalize(current.content) == normalize(content)
+
+
+def check_close(current: Record, record: Record) -> None:
+    """Raise ValueError if a put with `close_previous` cannot close `current` for `record`.
+
+    The new `valid_from` must be set, and must not be earlier than the `valid_from` of
+    `current`.
+    """
+    if record.valid_from is None:
+        raise ValueError("close_previous needs valid_from or observed_at on the new revision")
+    if current.valid_from is not None and current.valid_from > record.valid_from:
+        raise ValueError(
+            f"key {current.key!r} is valid from {current.valid_from.isoformat()}, which is "
+            f"later than the new valid_from {record.valid_from.isoformat()}"
+        )
+
+
+def superseded(current: Record, record: Record, now: datetime, close_previous: bool) -> Record:
+    """Return `current` with `superseded_by` set to the id of `record`.
+
+    With `close_previous`, its `valid_until` also becomes the `valid_from` of `record`, and
+    its `expired_at` becomes `now`.
+    """
+    if not close_previous:
+        return replace(current, superseded_by=record.id)
+    return replace(current, superseded_by=record.id, valid_until=record.valid_from, expired_at=now)
+
+
+def with_new_sources(current: Record, record: Record) -> Record | None:
+    """Return `record` with the sources of `current` and then its own, or None if none is new.
+
+    A put whose content duplicates `current` calls this. The merged sources keep their
+    order and hold no repeats.
+    """
+    sources = tuple(dict.fromkeys((*current.sources, *record.sources)))
+    if sources == current.sources:
+        return None
+    return replace(record, sources=sources)
+
+
+def is_listed(record: Record, now: datetime) -> bool:
+    """Return True if `list` and `search` can return `record` at `now`.
+
+    A superseded revision is not listed. A record is not listed once its `forget_after`
+    is at or before `now`.
+    """
+    return record.superseded_by is None and not is_forgotten(record, now)
+
+
+def is_forgotten(record: Record, now: datetime) -> bool:
+    """Return True if `record` is past its `forget_after` at `now`."""
+    return record.forget_after is not None and record.forget_after <= now
+
+
+def matches(record: Record, filter: Filter | None) -> bool:
+    """Return True if `record` satisfies every field that `filter` sets."""
+    if filter is None:
+        return True
+    kinds = kinds_of(filter)
+    if kinds is not None and record.kind not in kinds:
+        return False
+    if filter.subject is not None and record.subject != filter.subject:
+        return False
+    if filter.actor is not None and record.actor != filter.actor:
+        return False
+    if filter.tags is not None and not set(filter.tags) <= set(record.tags):
+        return False
+    ranges = (
+        (filter.valid_from, record.valid_from),
+        (filter.valid_until, record.valid_until),
+        (filter.created_at, record.created_at),
+        (filter.expired_at, record.expired_at),
+    )
+    return all(window is None or window.contains(value) for window, value in ranges)
+
+
+def newest_first(record: Record) -> tuple[float, str]:
+    """Sort key for the order of `list`: newest `created_at` first, then by id."""
+    return (-record.created_at.timestamp(), record.id)
+
+
+def build_record(
+    namespace: Sequence[str],
+    key: str,
+    revision: int,
+    created_at: datetime,
+    *,
+    kind: Kind,
+    content: str,
+    subject: str,
+    cues: Sequence[str],
+    payload: dict[str, JSONValue] | None,
+    tags: Sequence[str],
+    sources: Sequence[str],
+    actor: str,
+    observed_at: datetime | None,
+    valid_from: datetime | None,
+    valid_until: datetime | None,
+    forget_after: datetime | None,
+    forget_reason: str,
+) -> Record:
+    """Return the record that a put writes. `valid_from` defaults to `observed_at`."""
+    return Record(
+        namespace=tuple(namespace),
+        key=key,
+        revision=revision,
+        kind=kind,
+        subject=subject,
+        content=content,
+        cues=tuple(cues),
+        payload={} if payload is None else payload,
+        tags=tuple(tags),
+        sources=tuple(sources),
+        actor=actor,
+        created_at=created_at,
+        observed_at=observed_at,
+        valid_from=observed_at if valid_from is None else valid_from,
+        valid_until=valid_until,
+        forget_after=forget_after,
+        forget_reason=forget_reason,
+    )
+
+
+def cosine_distance(a: Sequence[float], b: Sequence[float]) -> float:
+    """Return 1 minus the cosine similarity of `a` and `b`.
+
+    Raises ValueError if the lengths differ or a vector has zero length.
+    """
+    if len(a) != len(b):
+        raise ValueError(f"vectors differ in length: {len(a)} and {len(b)}")
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if norm_a == 0 or norm_b == 0:
+        raise ValueError("cannot compute a cosine distance for a zero vector")
+    return 1.0 - sum(x * y for x, y in zip(a, b, strict=True)) / (norm_a * norm_b)

hotmemory/_writer.py

@@ -0,0 +1,115 @@
+"""The buffer behind every driver's `Writer`."""
+
+from __future__ import annotations
+
+import builtins
+from collections.abc import Callable, Sequence
+from datetime import datetime, timedelta
+from types import TracebackType
+from typing import Self
+
+from hotmemory._rules import Draft, build_record
+from hotmemory.record import JSONValue, Kind
+from hotmemory.store import Clock
+
+
+def check_count(name: str, value: int) -> None:
+    """Raise TypeError if `value` is not an integer, or ValueError if it is negative."""
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise TypeError(f"{name} must be an integer")
+    if value < 0:
+        raise ValueError(f"{name} must not be negative")
+
+
+class BufferedWriter:
+    """A `Writer` that hands its buffer to `write` in one call on each flush.
+
+    `write` takes the buffered drafts in order and returns the id that each one wrote.
+    """
+
+    def __init__(
+        self,
+        write: Callable[[Sequence[Draft]], builtins.list[str]],
+        clock: Clock,
+        max_rows: int,
+        interval: timedelta,
+    ) -> None:
+        check_count("max_rows", max_rows)
+        if max_rows < 1:
+            raise ValueError("max_rows must be 1 or more")
+        self._write = write
+        self._clock = clock
+        self._max_rows = max_rows
+        self._interval = interval
+        self._buffer: builtins.list[Draft] = []
+        self._flushed: builtins.list[str] = []
+        self._last_flush = clock()
+
+    @property
+    def flushed(self) -> builtins.list[str]:
+        return builtins.list(self._flushed)
+
+    def put(
+        self,
+        namespace: Sequence[str],
+        key: str,
+        *,
+        kind: Kind,
+        content: str,
+        subject: str = "",
+        cues: Sequence[str] = (),
+        payload: dict[str, JSONValue] | None = None,
+        tags: Sequence[str] = (),
+        sources: Sequence[str] = (),
+        actor: str = "",
+        observed_at: datetime | None = None,
+        valid_from: datetime | None = None,
+        valid_until: datetime | None = None,
+        forget_after: datetime | None = None,
+        forget_reason: str = "",
+        close_previous: bool = False,
+    ) -> None:
+        now = self._clock()
+        record = build_record(
+            namespace,
+            key,
+            1,
+            now,
+            kind=kind,
+            content=content,
+            subject=subject,
+            cues=cues,
+            payload=payload,
+            tags=tags,
+            sources=sources,
+            actor=actor,
+            observed_at=observed_at,
+            valid_from=valid_from,
+            valid_until=valid_until,
+            forget_after=forget_after,
+            forget_reason=forget_reason,
+        )
+        self._buffer.append(Draft(record, close_previous))
+        if len(self._buffer) >= self._max_rows or now - self._last_flush >= self._interval:
+            self.flush()
+
+    def flush(self) -> builtins.list[str]:
+        ids = self._write(self._buffer) if self._buffer else []
+        self._buffer.clear()
+        self._flushed.extend(ids)
+        self._last_flush = self._clock()
+        return ids
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        if exc_type is None:
+            self.flush()
+        else:
+            self._buffer.clear()