monkeybot 2.1.1__py3-none-any.whl

monkeybot/core/memory/integrity.py

@@ -0,0 +1,180 @@
+"""Structural integrity checks for on-disk markdown memory trees.
+
+Used by ``scripts/verify_memory.py``, pytest evals (T1), and future tooling.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+_INDEX_LINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
+_TYPED_FOLDERS = frozenset({"episodic", "semantic", "procedural", "working"})
+DEFAULT_MIN_SUMMARY_CHARS = 20
+
+
+def _load_index_entries(memory_root: Path) -> list[tuple[str, str]]:
+    """Return [(raw_link, full_line), ...] for every [[...]] entry in INDEX.md."""
+    index = memory_root / "INDEX.md"
+    if not index.exists():
+        return []
+    entries: list[tuple[str, str]] = []
+    for line in index.read_text(encoding="utf-8").splitlines():
+        m = _INDEX_LINK_RE.search(line)
+        if m:
+            entries.append((m.group(1), line.strip()))
+    return entries
+
+
+def _typed_folders_present(memory_root: Path) -> list[Path]:
+    return [memory_root / f for f in sorted(_TYPED_FOLDERS) if (memory_root / f).is_dir()]
+
+
+def _collect_typed_files(memory_root: Path) -> set[str]:
+    """Relative paths (e.g. 'episodic/foo.md') for all files in typed folders."""
+    found: set[str] = set()
+    for folder in _typed_folders_present(memory_root):
+        for f in folder.rglob("*.md"):
+            if f.is_file():
+                found.add(str(f.relative_to(memory_root)))
+    return found
+
+
+@dataclass(frozen=True)
+class IntegrityResult:
+    """Counts from :meth:`MemoryIntegrityChecker.run`."""
+
+    orphan_count: int
+    unindexed_count: int
+    short_summary_count: int
+    raw_only_count: int
+    total_index_entries: int
+    total_typed_files: int
+
+    @property
+    def total_issues(self) -> int:
+        return (
+            self.orphan_count
+            + self.unindexed_count
+            + self.short_summary_count
+            + self.raw_only_count
+        )
+
+
+class MemoryIntegrityChecker:
+    """Verify memory directory structure (INDEX vs typed folders vs raw)."""
+
+    def __init__(self, memory_root: Path, min_summary_chars: int = DEFAULT_MIN_SUMMARY_CHARS) -> None:
+        self._memory_root = memory_root
+        self._min_summary_chars = min_summary_chars
+
+    def run(self) -> IntegrityResult:
+        """Run all checks. Missing ``memory_root`` yields zero issues (no crash)."""
+        root = self._memory_root
+        if not root.exists():
+            return IntegrityResult(0, 0, 0, 0, 0, 0)
+
+        index_entries = _load_index_entries(root)
+        indexed_paths: set[str] = set()
+
+        orphan_count = 0
+        for link, _line in index_entries:
+            target = root / link
+            indexed_paths.add(link)
+            if not target.exists():
+                orphan_count += 1
+
+        all_typed = _collect_typed_files(root)
+        unindexed_count = 0
+        for rel in sorted(all_typed):
+            if rel not in indexed_paths:
+                unindexed_count += 1
+
+        short_summary_count = 0
+        for link, _line in index_entries:
+            target = root / link
+            if not target.exists():
+                continue
+            text = target.read_text(encoding="utf-8").strip()
+            if len(text) < self._min_summary_chars:
+                short_summary_count += 1
+
+        raw_only_count = 0
+        processed_dir = root / "raw" / "processed"
+        if processed_dir.exists():
+            processed_names = {f.name for f in processed_dir.iterdir() if f.is_file()}
+            summarised_names = {Path(p).name for p in all_typed}
+            orphaned_raw = processed_names - summarised_names
+            raw_only_count = len(orphaned_raw)
+
+        return IntegrityResult(
+            orphan_count=orphan_count,
+            unindexed_count=unindexed_count,
+            short_summary_count=short_summary_count,
+            raw_only_count=raw_only_count,
+            total_index_entries=len(index_entries),
+            total_typed_files=len(all_typed),
+        )
+
+
+def verify_memory_cli(memory_root: Path) -> int:
+    """CLI entry: print human-readable report; return exit code (0 = ok).
+
+    If ``memory_root`` does not exist, prints an error and returns ``1``.
+    """
+    if not memory_root.exists():
+        print(f"ERROR: memory directory not found: {memory_root}")
+        return 1
+
+    index_entries = _load_index_entries(memory_root)
+    indexed_paths: set[str] = set()
+    issues = 0
+
+    for link, line in index_entries:
+        target = memory_root / link
+        indexed_paths.add(link)
+        if not target.exists():
+            print(f"[ORPHAN]   INDEX.md references missing file: {link}")
+            print(f"           entry: {line}")
+            issues += 1
+
+    all_typed = _collect_typed_files(memory_root)
+    for rel in sorted(all_typed):
+        if rel not in indexed_paths:
+            print(f"[UNINDEXED] file exists but has no INDEX.md entry: {rel}")
+            issues += 1
+
+    min_chars = DEFAULT_MIN_SUMMARY_CHARS
+    for link, _line in index_entries:
+        target = memory_root / link
+        if not target.exists():
+            continue
+        text = target.read_text(encoding="utf-8").strip()
+        if len(text) < min_chars:
+            print(f"[SHORT]    summary file may be truncated ({len(text)} chars): {link}")
+            print(f"           content: {text!r}")
+            issues += 1
+
+    processed_dir = memory_root / "raw" / "processed"
+    if processed_dir.exists():
+        processed_names = {f.name for f in processed_dir.iterdir() if f.is_file()}
+        summarised_names = {Path(p).name for p in all_typed}
+        orphaned_raw = processed_names - summarised_names
+        for name in sorted(orphaned_raw):
+            print(
+                f"[RAW_ONLY] processed raw exists but no summary in typed folder: {name}"
+            )
+            issues += 1
+
+    total_indexed = len(index_entries)
+    total_files = len(all_typed)
+    print(
+        f"\nMemory root : {memory_root}"
+        f"\nINDEX entries : {total_indexed}"
+        f"\nTyped files   : {total_files}"
+        f"\nIssues found  : {issues}"
+    )
+    if issues == 0:
+        print("All checks passed.")
+    return issues

monkeybot/core/memory/organizer.py

@@ -0,0 +1,270 @@
+"""Memory organizer — async post-processor for agent memory.
+
+Reads raw observation files from ``raw/``, summarizes each,
+classifies into a typed folder, and updates INDEX.md for deterministic retrieval.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Sequence
+from contextlib import aclosing
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Protocol, cast
+
+from monkeybot.core.memory.index_format import (
+    INDEX_ARCHIVE_FILENAME,
+    INDEX_FILENAME,
+    DEFAULT_INDEX_HEADER,
+    append_index_entries,
+    apply_index_entry_cap,
+    format_index_document,
+    index_cap_from_env,
+    merge_archive_content,
+    split_index_document,
+)
+from monkeybot.core.types.content_blocks import Text
+from monkeybot.core.types.interfaces import MonkeybotError
+from monkeybot.core.llm.provider import Done, Message, Provider, TextDelta
+from monkeybot.core.workspace.protocol import WorkspaceStorage
+
+logger = logging.getLogger(__name__)
+
+BUILT_IN_FOLDERS: list[str] = ["episodic", "semantic", "procedural", "working"]
+
+_SUMMARIZE_PROMPT = (
+    "Summarize the following agent observation in 3-5 sentences. "
+    "Be concrete and factual. Do not add opinions.\n\n"
+    "<observation>\n{content}\n</observation>\n\nSummary:"
+)
+
+_CLASSIFY_PROMPT = (
+    "Classify the following summary into exactly one memory folder. "
+    "Reply with ONLY the folder name — nothing else.\n\n"
+    "Folders:\n{folder_list}\n\nSummary:\n{summary}\n\nFolder:"
+)
+
+_INDEX_ENTRY_PROMPT = (
+    "Generate an INDEX.md entry. Reply in EXACTLY this format:\n"
+    "tags: <comma_separated>\nsummary: <one_sentence>\n\n"
+    "File: {filename}\nContent: {summary}\n\nEntry:"
+)
+
+
+class _CustomMemoryFolderLike(Protocol):
+    name: str
+    description: str
+
+
+class MemoryOrganizerError(MonkeybotError):
+    """Raised when INDEX.md cannot be written after an organizer run."""
+
+
+@dataclass
+class MemoryOrganizerResult:
+    files_processed: int
+    files_written: int
+    index_updated: bool
+    errors: list[str] = field(default_factory=list)
+
+
+@dataclass
+class IndexEntry:
+    folder: str    # destination folder name
+    filename: str  # filename inside folder (not full path)
+    tags: str      # "comma,separated"
+    summary: str   # one sentence
+
+
+async def _complete_text(
+    provider: Provider,
+    *,
+    model: str,
+    prompt: str,
+) -> str:
+    """Single-turn text completion (no tools) from streaming deltas."""
+    parts: list[str] = []
+    async with aclosing(
+        cast(Any, provider.stream([Message(role="user", content=[Text(text=prompt)])], [], model=model))
+    ) as stream:
+        async for ev in stream:
+            if isinstance(ev, TextDelta):
+                parts.append(ev.text)
+            elif isinstance(ev, Done):
+                break
+    return "".join(parts).strip()
+
+
+class MemoryOrganizer:
+    """Async post-processor that organises raw agent observations into typed memory."""
+
+    def __init__(
+        self,
+        provider: Provider,
+        model: str,
+        storage: WorkspaceStorage,
+        custom_folders: Sequence[_CustomMemoryFolderLike] | None = None,
+    ) -> None:
+        self._provider = provider
+        self._model = model
+        self._storage = storage
+        self._all_folders: list[str] = BUILT_IN_FOLDERS + [
+            cf.name for cf in (custom_folders or [])
+        ]
+        self._custom_folder_descriptions: dict[str, str] = {
+            cf.name: cf.description for cf in (custom_folders or [])
+        }
+
+    async def run(self) -> MemoryOrganizerResult:
+        """Process all unprocessed raw .md files directly under ``raw/``.
+
+        Returns immediately with zeros if ``raw/`` is missing or empty.
+        Per-file errors are captured in MemoryOrganizerResult.errors, not propagated.
+        """
+        all_under_raw = await self._storage.list_files("raw/")
+        raw_files = [
+            p
+            for p in all_under_raw
+            if p.startswith("raw/")
+            and p.endswith(".md")
+            and not p.startswith("raw/processed/")
+            and p.count("/") == 1
+        ]
+        if not raw_files:
+            return MemoryOrganizerResult(0, 0, False)
+
+        entries: list[IndexEntry] = []
+        files_written = 0
+        errors: list[str] = []
+
+        for raw_rel in sorted(raw_files):
+            raw_name = Path(raw_rel).name
+            try:
+                content = await self._storage.read_text(raw_rel)
+                summary = await self._summarize(content)
+                folder = await self._classify(summary)
+                filename = self._generate_filename(raw_name, folder)
+
+                dest_path = f"{folder}/{filename}"
+                await self._storage.write_text(dest_path, summary)
+
+                entries.append(
+                    IndexEntry(folder=folder, filename=filename, tags="", summary=summary)
+                )
+
+                processed_rel = f"raw/processed/{raw_name}"
+                await self._storage.move(raw_rel, processed_rel)
+
+                files_written += 1
+
+            except Exception as e:
+                logger.warning(
+                    "Memory organizer failed to process %s: %s", raw_name, str(e)
+                )
+                errors.append(raw_name)
+
+        if entries:
+            await self._update_index(entries)
+
+        return MemoryOrganizerResult(
+            files_processed=len(raw_files),
+            files_written=files_written,
+            index_updated=bool(entries),
+            errors=errors,
+        )
+
+    def _generate_filename(self, raw_name: str, folder: str) -> str:
+        del folder
+        stem = Path(raw_name).stem
+        return f"{stem}.md"
+
+    async def _summarize(self, raw_content: str) -> str:
+        return await _complete_text(
+            self._provider,
+            model=self._model,
+            prompt=_SUMMARIZE_PROMPT.format(content=raw_content),
+        )
+
+    async def _classify(self, summary: str) -> str:
+        folder_list = "\n".join(
+            f"- {f}: {self._custom_folder_descriptions.get(f, f)}"
+            for f in self._all_folders
+        )
+        text = await _complete_text(
+            self._provider,
+            model=self._model,
+            prompt=_CLASSIFY_PROMPT.format(folder_list=folder_list, summary=summary),
+        )
+        folder = text.strip().lower()
+        if folder not in self._all_folders:
+            logger.warning(
+                "Classify returned unknown folder '%s', using 'episodic'", folder
+            )
+            return "episodic"
+        return folder
+
+    async def _update_index(self, entries: list[IndexEntry]) -> None:
+        existing_content = ""
+        if await self._storage.exists(INDEX_FILENAME):
+            try:
+                existing_content = await self._storage.read_text(INDEX_FILENAME)
+            except Exception:
+                existing_content = ""
+
+        if not existing_content.strip():
+            existing_content = f"{DEFAULT_INDEX_HEADER}\n"
+        new_lines: list[str] = []
+        for entry in entries:
+            try:
+                response = await _complete_text(
+                    self._provider,
+                    model=self._model,
+                    prompt=_INDEX_ENTRY_PROMPT.format(
+                        filename=entry.filename, summary=entry.summary
+                    ),
+                )
+
+                tags = ""
+                summary_line = entry.summary
+                for line in response.splitlines():
+                    if line.lower().startswith("tags:"):
+                        tags = line[5:].strip()
+                    elif line.lower().startswith("summary:"):
+                        summary_line = line[8:].strip()
+
+                entry.tags = tags
+                entry.summary = summary_line
+            except Exception as exc:
+                logger.debug("index entry LLM call failed for %s: %r", entry.filename, exc)
+
+            formatted = (
+                f"- [[{entry.folder}/{entry.filename}]]"
+                f" | tags: {entry.tags} | {entry.summary}"
+            )
+            new_lines.append(formatted)
+
+        if not new_lines:
+            return
+
+        merged = append_index_entries(existing_content, new_lines)
+        _header, entry_lines = split_index_document(merged)
+        cap = index_cap_from_env()
+        kept, archived = apply_index_entry_cap(entry_lines, cap)
+        if archived:
+            archive_raw = ""
+            if await self._storage.exists(INDEX_ARCHIVE_FILENAME):
+                try:
+                    archive_raw = await self._storage.read_text(INDEX_ARCHIVE_FILENAME)
+                except Exception:
+                    archive_raw = ""
+            archive_out = merge_archive_content(archive_raw, archived)
+            await self._storage.write_text(INDEX_ARCHIVE_FILENAME, archive_out)
+
+        final_content = format_index_document(_header, kept)
+
+        try:
+            await self._storage.write_text(INDEX_FILENAME, final_content)
+        except Exception as e:
+            raise MemoryOrganizerError(f"Failed to write INDEX.md: {e}") from e

monkeybot/core/memory/storage_ops.py

@@ -0,0 +1,139 @@
+"""Async memory operations on :class:`~monkeybot.core.workspace.protocol.WorkspaceStorage`."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from monkeybot.core.memory.index_format import INDEX_FILENAME, parse_index_entry_lines
+from monkeybot.core.workspace.protocol import WorkspaceStorage
+
+
+class MemoryPromotionError(RuntimeError):
+    """Raised when promotion preconditions fail (path, run_id guard, missing file, etc.)."""
+
+
+def _parse_index_lines(raw: str) -> list[str]:
+    return parse_index_entry_lines(raw)
+
+
+async def async_load_index(storage: WorkspaceStorage) -> list[str]:
+    """Load ``INDEX.md`` lines, or ``[]`` if absent."""
+    if not await storage.exists(INDEX_FILENAME):
+        return []
+    raw = await storage.read_text(INDEX_FILENAME)
+    return _parse_index_lines(raw)
+
+
+def _matches_query(line_lower: str, tokens: list[str]) -> bool:
+    return all(tok in line_lower for tok in tokens)
+
+
+def _memory_rel_skipped(rel_posix: str, skip_relative_prefixes: tuple[str, ...]) -> bool:
+    if not skip_relative_prefixes:
+        return False
+    for raw_p in skip_relative_prefixes:
+        p = raw_p.replace("\\", "/").strip("/")
+        if not p:
+            continue
+        if rel_posix == p or rel_posix.startswith(p + "/"):
+            return True
+    return False
+
+
+async def async_search_memory(query: str, storage: WorkspaceStorage, top_k: int = 5) -> list[str]:
+    if top_k <= 0:
+        return []
+    q = query.strip()
+    if not q:
+        return []
+    tokens = [t.lower() for t in q.split() if t]
+    if not tokens:
+        return []
+
+    candidates = await async_load_index(storage)
+    matches: list[str] = []
+    for line in candidates:
+        if _matches_query(line.lower(), tokens):
+            matches.append(line)
+            if len(matches) >= top_k:
+                break
+    return matches
+
+
+async def async_search_memory_files(
+    storage: WorkspaceStorage,
+    query: str,
+    *,
+    max_hits: int = 40,
+    skip_relative_prefixes: tuple[str, ...] = (),
+) -> dict[str, Any]:
+    q = query.lower().strip()
+    if not q:
+        return {"ok": True, "query": query, "hits": [], "note": "empty query"}
+
+    all_paths = await storage.list_files("")
+    if not all_paths:
+        return {"ok": True, "query": query, "hits": [], "note": "empty memory tree"}
+
+    suffixes = {".md", ".txt", ".markdown"}
+    hits: list[dict[str, Any]] = []
+    for rel in sorted(all_paths):
+        if len(hits) >= max_hits:
+            break
+        low = rel.lower()
+        if not any(low.endswith(s) for s in suffixes):
+            continue
+        rel_posix = rel.replace("\\", "/")
+        if _memory_rel_skipped(rel_posix, skip_relative_prefixes):
+            continue
+        try:
+            text = await storage.read_text(rel)
+        except OSError:
+            continue
+        except FileNotFoundError:
+            continue
+        lower = text.lower()
+        pos = lower.find(q)
+        if pos < 0:
+            continue
+        start = max(0, pos - 60)
+        end = min(len(text), pos + len(q) + 80)
+        snippet = text[start:end].replace("\n", " ")
+        hits.append({"path": rel, "snippet": snippet, "match_offset": pos})
+    return {"ok": True, "query": query, "hits": hits, "truncated": len(hits) >= max_hits}
+
+
+async def async_promote_to_memory(run_id: str, file: Path, storage: WorkspaceStorage) -> None:
+    """Copy a local scratch file into ``semantic/`` on ``storage``, then delete the source.
+
+    ``file`` must be a path on the process filesystem (e.g. agent workspace scratch), not a
+    storage-relative key. The destination is always ``semantic/{filename}`` on ``storage``.
+    A future storage-key source variant would be needed if the artifact already lives only
+    in GCS/S3.
+    """
+    src = file.resolve()
+    if not src.is_file():
+        raise MemoryPromotionError("Source path must be an existing regular file.")
+
+    if run_id not in src.parts:
+        raise MemoryPromotionError(
+            "Source path must include the run identifier as a path segment.",
+        )
+
+    dest_key = f"semantic/{src.name}"
+    text = src.read_text(encoding="utf-8")
+    await storage.write_text(dest_key, text)
+    try:
+        src.unlink()
+    except OSError as exc:
+        raise MemoryPromotionError(f"Failed to remove source after promote: {exc}") from exc
+
+
+__all__ = [
+    "INDEX_FILENAME",
+    "MemoryPromotionError",
+    "async_load_index",
+    "async_search_memory_files",
+    "async_promote_to_memory",
+]

monkeybot/core/memory/subsystem.py

@@ -0,0 +1,91 @@
+"""Single injection point for memory (storage + hook + organizer)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from monkeybot.core.hooks import HookManager
+from monkeybot.core.llm.provider import Provider
+from monkeybot.core.memory.hook import MemoryHook
+from monkeybot.core.memory.organizer import MemoryOrganizer
+from monkeybot.core.memory.storage_ops import (
+    async_load_index,
+    async_promote_to_memory,
+    async_search_memory_files,
+)
+from monkeybot.core.workspace.protocol import WorkspaceStorage
+
+
+class MemorySubsystem:
+    """Owns :class:`WorkspaceStorage`, :class:`MemoryHook`, and :class:`MemoryOrganizer`.
+
+    External callers inject this object into :func:`~monkeybot.core.context.build_context`
+    and :class:`~monkeybot.core.tools.core_tool_executor.CoreToolExecutor` instead of
+    passing paths or storage separately.
+    """
+
+    def __init__(
+        self,
+        storage: WorkspaceStorage,
+        provider: Provider,
+        model: str,
+        *,
+        memory_uri: str,
+        max_retrieval_hits: int = 3,
+    ) -> None:
+        self._storage = storage
+        self._memory_uri = memory_uri.strip()
+        self._organizer = MemoryOrganizer(
+            provider=provider,
+            model=model,
+            storage=storage,
+        )
+        self._hook = MemoryHook(
+            storage=storage,
+            organizer_runner=self._organizer.run,
+            max_retrieval_hits=max_retrieval_hits,
+        )
+
+    @property
+    def uri(self) -> str:
+        """URI string used to reconstruct storage in subprocess workers."""
+        return self._memory_uri
+
+    @property
+    def storage(self) -> WorkspaceStorage:
+        return self._storage
+
+    def register_hooks(self, manager: HookManager) -> None:
+        self._hook.register(manager)
+
+    async def load_index(self) -> list[str]:
+        return await async_load_index(self._storage)
+
+    async def search_files(
+        self,
+        query: str,
+        *,
+        max_hits: int = 40,
+        skip_raw: bool = True,
+    ) -> dict[str, Any]:
+        skip: tuple[str, ...] = ("raw",) if skip_raw else ()
+        return await async_search_memory_files(
+            self._storage,
+            query,
+            max_hits=max_hits,
+            skip_relative_prefixes=skip,
+        )
+
+    async def promote(self, run_id: str, file: Path) -> None:
+        await async_promote_to_memory(run_id, file, self._storage)
+
+    async def gc_processed(self, *, max_age_sec: float = 7 * 24 * 60 * 60) -> dict[str, int]:
+        return await self._storage.gc_prefix("raw/processed/", max_age_sec)
+
+    async def flush(self) -> None:
+        """Await pending memory organizer task. See :meth:`MemoryHook.flush`."""
+        await self._hook.flush()
+
+
+__all__ = ["MemorySubsystem"]

monkeybot/core/messages/__init__.py

@@ -0,0 +1,16 @@
+"""Message normalization helpers for provider replay.
+
+Pipeline (Pi-style)::
+
+    history Messages → transform_context() → convert_to_provider() → Provider.stream
+"""
+
+from monkeybot.core.messages.convert_provider import convert_to_provider
+from monkeybot.core.messages.tool_integrity import repair_tool_turn_integrity
+from monkeybot.core.messages.transform_context import transform_context
+
+__all__ = [
+    "convert_to_provider",
+    "repair_tool_turn_integrity",
+    "transform_context",
+]