deeptutor-memory 1.7.0__py3-none-any.whl

deeptutor/services/memory/store.py

@@ -0,0 +1,453 @@
+"""High-level facade for the three-layer memory subsystem.
+
+All callers — API routers, LLM tools, surface event hooks — go through
+:class:`MemoryStore`. The store is stateless; per-user isolation is
+inherited from :func:`paths.memory_root` which resolves :class:`PathService`
+lazily via context variables.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import logging
+from pathlib import Path
+import shutil
+from typing import Literal
+
+from deeptutor.services.file_io import atomic_write_text
+from deeptutor.services.memory import consolidator, paths, trace
+from deeptutor.services.memory.consolidator import ConsolidateResult, OnEvent
+from deeptutor.services.memory.document import Document, parse, serialize
+from deeptutor.services.memory.ops import AddOp, ApplyReport, EditOp, OpResult
+from deeptutor.services.memory.ops import apply as ops_apply
+from deeptutor.services.memory.paths import L3Slot, Surface
+from deeptutor.services.memory.trace import TraceEvent
+
+logger = logging.getLogger(__name__)
+
+Layer = Literal["L2", "L3"]
+
+_V1_FILES = ("PROFILE.md", "SOUL.md", "SUMMARY.md")
+
+
+def _normalize_pref_text(text: str) -> str:
+    """Whitespace/case-insensitive key for duplicate-preference detection."""
+    return " ".join(str(text or "").split()).casefold()
+
+
+def _find_duplicate_preference(doc: Document, section: str, text: str):
+    """Return an existing entry in ``section`` whose text matches ``text``.
+
+    Read-only — does not create the section if it is absent.
+    """
+    target = _normalize_pref_text(text)
+    if not target:
+        return None
+    for entry in doc.all_entries():
+        if entry.section == section and _normalize_pref_text(entry.text) == target:
+            return entry
+    return None
+
+
+_NO_MEMORY = (
+    "(No memory available — interact with DeepTutor and update from the Memory page to build one.)"
+)
+
+
+@dataclass
+class DocOverview:
+    layer: Layer
+    key: str  # surface name (L2) or slot name (L3)
+    exists: bool
+    updated_at: str | None
+    entry_count: int
+    backlog: int  # L1 events since last update (L2 only; 0 for L3)
+
+
+class MemoryStore:
+    """Stateless facade. Safe to call as a process-wide singleton."""
+
+    def __init__(self) -> None:
+        self._write_locks: dict[str, asyncio.Lock] = {}
+
+    # ── L1 ────────────────────────────────────────────────────────────────
+
+    async def emit(self, event: TraceEvent) -> None:
+        await trace.append(event)
+
+    # ── L2 / L3 read ──────────────────────────────────────────────────────
+
+    def read_doc(self, layer: Layer, key: str) -> Document:
+        path = self._path(layer, key)
+        if not path.exists():
+            return Document(title=_default_title(layer, key))
+        return parse(path.read_text(encoding="utf-8"))
+
+    def read_raw(self, layer: Layer, key: str) -> str:
+        path = self._path(layer, key)
+        if not path.exists():
+            return ""
+        return path.read_text(encoding="utf-8")
+
+    def read_l3_concat(self) -> str:
+        """Concatenate all four L3 docs for the ``read_memory`` tool."""
+        parts: list[str] = []
+        for slot in paths.L3_SLOTS:
+            body = self.read_raw("L3", slot).strip()
+            if body:
+                parts.append(body)
+        if not parts:
+            return _NO_MEMORY
+        return "\n\n---\n\n".join(parts) + "\n"
+
+    # ── L2 / L3 write (manual paths) ──────────────────────────────────────
+
+    async def overwrite_doc(self, layer: Layer, key: str, md: str) -> None:
+        """Direct user-driven save from the workbench editor."""
+        path = self._path(layer, key)
+        async with self._lock_for(path):
+            await asyncio.to_thread(atomic_write_text, path, md)
+
+    async def delete_entry(self, layer: Layer, key: str, entry_id: str) -> bool:
+        path = self._path(layer, key)
+        if not path.exists():
+            return False
+        async with self._lock_for(path):
+            doc = parse(path.read_text(encoding="utf-8"))
+            if not doc.remove(entry_id):
+                return False
+            await asyncio.to_thread(atomic_write_text, path, serialize(doc))
+            return True
+
+    # ── L2 / L3 write (consolidator paths) ────────────────────────────────
+
+    async def update_l2(
+        self,
+        surface: Surface,
+        *,
+        language: str = "en",
+        user_label: str = "anonymous",
+        on_event: OnEvent | None = None,
+        apply_ops: bool = True,
+    ) -> ConsolidateResult:
+        path = paths.l2_file(surface)
+        async with self._lock_for(path):
+            return await consolidator.consolidate_l2(
+                surface,
+                language=language,
+                user_label=user_label,
+                on_event=on_event,
+                apply_ops=apply_ops,
+            )
+
+    async def update_l3(
+        self,
+        slot: L3Slot,
+        *,
+        language: str = "en",
+        user_label: str = "anonymous",
+        on_event: OnEvent | None = None,
+        apply_ops: bool = True,
+    ) -> ConsolidateResult:
+        if slot == "preferences":
+            raise ValueError("preferences.md is not auto-consolidated")
+        path = paths.l3_file(slot)
+        async with self._lock_for(path):
+            return await consolidator.consolidate_l3(
+                slot,
+                language=language,
+                user_label=user_label,
+                on_event=on_event,
+                apply_ops=apply_ops,
+            )
+
+    async def apply_ops_payload(
+        self, layer: Layer, key: str, ops_payload: list[dict]
+    ) -> ApplyReport:
+        """Apply a list of ops-as-JSON to a layer doc atomically.
+
+        Used by the workbench's preview → apply two-step flow. The
+        payload typically comes from a previous ``apply_ops=False``
+        consolidate call surfaced to the user for review.
+        """
+        from deeptutor.services.memory.consolidator import _parse_ops_response
+
+        path = self._path(layer, key)
+        json_like = {"ops": ops_payload}
+        import json as _json
+
+        ops = _parse_ops_response(_json.dumps(json_like, ensure_ascii=False))
+        async with self._lock_for(path):
+            default_title = _default_title(layer, key)
+            doc = (
+                parse(path.read_text(encoding="utf-8"))
+                if path.exists()
+                else Document(title=default_title)
+            )
+            report = ops_apply(doc, ops)
+            if report.accepted and ops:
+                path.parent.mkdir(parents=True, exist_ok=True)
+                await asyncio.to_thread(atomic_write_text, path, serialize(doc))
+            return report
+
+    async def write_preference(
+        self,
+        *,
+        op: Literal["add", "edit"],
+        text: str,
+        target_id: str | None = None,
+        reason: str | None = None,
+        trace_id: str,
+    ) -> ApplyReport:
+        """Write the chat-mode preference signal. The ``write_memory`` tool
+        is the only caller; ``trace_id`` is the current chat turn's L1 id
+        injected by runtime."""
+        path = paths.l3_file("preferences")
+        async with self._lock_for(path):
+            doc = (
+                parse(path.read_text(encoding="utf-8"))
+                if path.exists()
+                else Document(title=_default_title("L3", "preferences"))
+            )
+            section = "Preferences"
+            if op == "add":
+                # Idempotent add: preferences.md is never auto-consolidated
+                # (see update_l3), so an identical bullet added again would
+                # persist forever as a duplicate. Guided-learning turns are
+                # highly tool-driven and long-running, so the model tends to
+                # re-issue the same write_memory across turns (issue #647).
+                # Short-circuit to a no-op that reports the existing entry as
+                # already saved instead of appending a duplicate.
+                duplicate = _find_duplicate_preference(doc, section, text)
+                if duplicate is not None:
+                    return ApplyReport(
+                        accepted=True,
+                        results=[
+                            OpResult(
+                                op=AddOp(section=section, text=text, refs=[trace_id]),
+                                status="applied",
+                                entry_id=duplicate.id,
+                                detail="duplicate",
+                            )
+                        ],
+                    )
+                report = ops_apply(
+                    doc,
+                    [AddOp(section=section, text=text, refs=[trace_id])],
+                )
+            else:
+                if not target_id:
+                    return ApplyReport(accepted=False, reason="edit requires target_id")
+                report = ops_apply(
+                    doc,
+                    [
+                        EditOp(
+                            target_id=target_id,
+                            new_text=text,
+                            new_refs=[trace_id],
+                        )
+                    ],
+                )
+            if report.accepted:
+                await asyncio.to_thread(atomic_write_text, path, serialize(doc))
+            if reason:
+                # Surface the reason in logs for workbench observability.
+                logger.info("write_memory %s id=%s reason=%s", op, target_id or "new", reason)
+            return report
+
+    # ── Workbench overview ────────────────────────────────────────────────
+
+    def overview(self) -> list[DocOverview]:
+        rows: list[DocOverview] = []
+        for surface in paths.SURFACES:
+            rows.append(self._overview_for("L2", surface))
+        for slot in paths.L3_SLOTS:
+            rows.append(self._overview_for("L3", slot))
+        return rows
+
+    def _overview_for(self, layer: Layer, key: str) -> DocOverview:
+        path = self._path(layer, key)
+        if not path.exists():
+            backlog = trace.count_since(key) if layer == "L2" else 0  # type: ignore[arg-type]
+            return DocOverview(
+                layer=layer,
+                key=key,
+                exists=False,
+                updated_at=None,
+                entry_count=0,
+                backlog=backlog,
+            )
+
+        stat = path.stat()
+        updated_at = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
+        try:
+            doc = parse(path.read_text(encoding="utf-8"))
+            entry_count = len(doc.all_entries())
+        except Exception:
+            entry_count = 0
+
+        backlog = 0
+        if layer == "L2":
+            try:
+                cutoff = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
+                backlog = trace.count_since(key, since=cutoff)  # type: ignore[arg-type]
+            except Exception:
+                backlog = 0
+
+        return DocOverview(
+            layer=layer,
+            key=key,
+            exists=True,
+            updated_at=updated_at,
+            entry_count=entry_count,
+            backlog=backlog,
+        )
+
+    # ── Internals ─────────────────────────────────────────────────────────
+
+    def _path(self, layer: Layer, key: str) -> Path:
+        if layer == "L2":
+            if key not in paths.SURFACES:
+                raise ValueError(f"unknown surface {key!r}")
+            return paths.l2_file(key)  # type: ignore[arg-type]
+        if key not in paths.L3_SLOTS:
+            raise ValueError(f"unknown L3 slot {key!r}")
+        return paths.l3_file(key)  # type: ignore[arg-type]
+
+    def _lock_for(self, path: Path) -> asyncio.Lock:
+        key = str(path)
+        lock = self._write_locks.get(key)
+        if lock is None:
+            lock = asyncio.Lock()
+            self._write_locks[key] = lock
+        return lock
+
+
+# ── v1 → v2 startup migration ─────────────────────────────────────────────
+
+
+def migrate_v1_if_needed() -> Path | None:
+    """If any v1 memory files are present under the memory root, move the
+    whole memory directory's loose files into ``memory/backup/<ts>/``.
+
+    Idempotent: if there's nothing v1-shaped at the root, this is a no-op.
+
+    Returns the backup directory path on migration, or ``None`` otherwise.
+    """
+    root = paths.memory_root()
+    if not root.exists():
+        return None
+    v1_present = [name for name in _V1_FILES if (root / name).exists()]
+    if not v1_present:
+        return None
+
+    ts = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    backup_dir = paths.backup_root() / ts
+    backup_dir.mkdir(parents=True, exist_ok=True)
+    for item in list(root.iterdir()):
+        if item.name in {"trace", "L2", "L3", "backup"}:
+            continue
+        try:
+            shutil.move(str(item), str(backup_dir / item.name))
+        except OSError:
+            logger.warning("v1 memory migration: failed to move %s", item, exc_info=True)
+    logger.info("v1 memory migrated to %s", backup_dir)
+    return backup_dir
+
+
+def migrate_partner_surface_if_needed() -> bool:
+    """Rename the legacy ``tutorbot`` memory surface to ``partner``.
+
+    The partner surface key used to be ``tutorbot`` — so footnote refs read
+    ``tutorbot:<id>`` and the consolidator even wrote "tutorbot" into L2
+    prose. It is now ``partner``. This moves any on-disk artifacts (L2 doc +
+    meta, snapshot dir, trace dir) to the new name, rewrites the ``tutorbot``
+    token to ``partner`` inside the L2 doc/meta (both the ``tutorbot:`` ref
+    prefix and the bare prose word), and renames the per-surface key inside
+    every L3 meta.
+
+    Idempotent: skips any target that already exists; a no-op when nothing
+    tutorbot-shaped lives under the memory root.
+    """
+    import json
+    import re
+
+    root = paths.memory_root()
+    if not root.exists():
+        return False
+
+    moved = False
+
+    l2 = paths.l2_dir()
+    old_md, new_md = l2 / "tutorbot.md", l2 / "partner.md"
+    if old_md.exists() and not new_md.exists():
+        text = old_md.read_text(encoding="utf-8")
+        text = text.replace("tutorbot:", "partner:")  # footnote/inline refs
+        text = re.sub(r"\btutorbot\b", "partner", text)  # bare prose word
+        text = re.sub(r"\bTutorbot\b", "Partner", text)
+        new_md.write_text(text, encoding="utf-8")
+        old_md.unlink()
+        moved = True
+    old_meta, new_meta = l2 / "tutorbot.meta.json", l2 / "partner.meta.json"
+    if old_meta.exists() and not new_meta.exists():
+        text = old_meta.read_text(encoding="utf-8").replace("tutorbot:", "partner:")
+        new_meta.write_text(text, encoding="utf-8")
+        old_meta.unlink()
+        moved = True
+
+    # snapshot/<surface>/ and trace/<surface>/ — plain directory moves
+    # (entity ids carry no surface prefix, so no content rewrite needed).
+    for sub in ("snapshot", "trace"):
+        old_dir, new_dir = root / sub / "tutorbot", root / sub / "partner"
+        if old_dir.is_dir() and not new_dir.exists():
+            shutil.move(str(old_dir), str(new_dir))
+            moved = True
+
+    # L3 metas track seen L2 entry ids per surface — rename that key.
+    l3 = paths.l3_dir()
+    if l3.is_dir():
+        for meta_path in l3.glob("*.meta.json"):
+            try:
+                data = json.loads(meta_path.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError):
+                continue
+            seen = data.get("seen_l2_entry_ids")
+            if isinstance(seen, dict) and "tutorbot" in seen and "partner" not in seen:
+                seen["partner"] = seen.pop("tutorbot")
+                meta_path.write_text(
+                    json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8"
+                )
+                moved = True
+
+    if moved:
+        logger.info("migrated legacy 'tutorbot' memory surface to 'partner'")
+    return moved
+
+
+# ── Singleton accessor ────────────────────────────────────────────────────
+
+
+_singleton: MemoryStore | None = None
+
+
+def get_memory_store() -> MemoryStore:
+    global _singleton
+    if _singleton is None:
+        _singleton = MemoryStore()
+    return _singleton
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────
+
+
+def _default_title(layer: Layer, key: str) -> str:
+    if layer == "L2":
+        return f"{key} memory"
+    return {
+        "recent": "Recent summary",
+        "profile": "User profile",
+        "scope": "Knowledge scope",
+        "preferences": "Preferences",
+    }.get(key, key)

deeptutor/services/memory/trace.py

@@ -0,0 +1,153 @@
+"""L1 raw event trace: append-only JSONL files, one per surface per UTC day.
+
+Trace capture must never break the producing surface — every append is
+wrapped and failures are logged-and-swallowed. Writes are serialized
+per-surface with an asyncio lock so multiple turns in the same process
+don't interleave JSON lines.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+import json
+import logging
+from pathlib import Path
+from typing import Any, Iterator
+
+from deeptutor.services.memory.ids import new_trace_id
+from deeptutor.services.memory.paths import SURFACES, Surface, trace_dir, trace_file
+
+logger = logging.getLogger(__name__)
+
+_locks: dict[str, asyncio.Lock] = {}
+
+
+def _lock_for(surface: Surface) -> asyncio.Lock:
+    lock = _locks.get(surface)
+    if lock is None:
+        lock = asyncio.Lock()
+        _locks[surface] = lock
+    return lock
+
+
+@dataclass
+class TraceEvent:
+    id: str
+    ts: str
+    surface: Surface
+    kind: str
+    payload: dict[str, Any]
+    session_id: str | None = None
+    turn_id: str | None = None
+
+    @classmethod
+    def new(
+        cls,
+        surface: Surface,
+        kind: str,
+        payload: dict[str, Any],
+        *,
+        session_id: str | None = None,
+        turn_id: str | None = None,
+    ) -> "TraceEvent":
+        return cls(
+            id=new_trace_id(surface),
+            ts=datetime.now(tz=timezone.utc).isoformat(),
+            surface=surface,
+            kind=kind,
+            payload=payload,
+            session_id=session_id,
+            turn_id=turn_id,
+        )
+
+
+async def append(event: TraceEvent) -> None:
+    """Append one event to today's surface trace file. Never raises."""
+    try:
+        path = trace_file(event.surface, datetime.now(tz=timezone.utc).date())
+        line = json.dumps(asdict(event), ensure_ascii=False, separators=(",", ":"))
+        async with _lock_for(event.surface):
+            await asyncio.to_thread(_append_line, path, line)
+    except Exception:
+        logger.warning(
+            "memory trace append failed surface=%s kind=%s",
+            event.surface,
+            event.kind,
+            exc_info=True,
+        )
+
+
+def _append_line(path: Path, line: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(line)
+        fh.write("\n")
+
+
+def iter_since(surface: Surface, since: datetime | None = None) -> Iterator[TraceEvent]:
+    """Yield events for ``surface`` in chronological order, optionally
+    filtering to events with ``ts >= since`` (UTC)."""
+    files = sorted(trace_dir(surface).glob("*.jsonl"))
+    cutoff_iso = since.isoformat() if since else ""
+    cutoff_date_iso = since.date().isoformat() if since else ""
+    for path in files:
+        if cutoff_date_iso and path.stem < cutoff_date_iso:
+            continue
+        try:
+            with path.open("r", encoding="utf-8") as fh:
+                for raw in fh:
+                    raw = raw.strip()
+                    if not raw:
+                        continue
+                    try:
+                        obj = json.loads(raw)
+                    except json.JSONDecodeError:
+                        continue
+                    if cutoff_iso and obj.get("ts", "") < cutoff_iso:
+                        continue
+                    yield TraceEvent(**obj)
+        except OSError:
+            continue
+
+
+def iter_by_ids(ids: list[str]) -> Iterator[TraceEvent]:
+    """Resolve trace ids back to their events. Cross-surface walk."""
+    wanted_by_surface: dict[str, set[str]] = {}
+    for tid in ids:
+        if ":" not in tid:
+            continue
+        surface, _ = tid.split(":", 1)
+        if surface in SURFACES:
+            wanted_by_surface.setdefault(surface, set()).add(tid)
+
+    for surface, wanted in wanted_by_surface.items():
+        for event in iter_since(surface):  # type: ignore[arg-type]
+            if event.id in wanted:
+                yield event
+
+
+def count_since(surface: Surface, since: datetime | None = None) -> int:
+    return sum(1 for _ in iter_since(surface, since))
+
+
+def latest_ts(surface: Surface) -> str | None:
+    """Most recent event timestamp for ``surface``, or None."""
+    files = sorted(trace_dir(surface).glob("*.jsonl"), reverse=True)
+    for path in files:
+        try:
+            last = ""
+            with path.open("r", encoding="utf-8") as fh:
+                for raw in fh:
+                    raw = raw.strip()
+                    if raw:
+                        last = raw
+            if last:
+                obj = json.loads(last)
+                ts = obj.get("ts")
+                if isinstance(ts, str):
+                    return ts
+        except (OSError, json.JSONDecodeError):
+            continue
+    return None