deeptutor-memory 1.7.0__py3-none-any.whl

deeptutor/services/memory/consolidator/line_doc.py

@@ -0,0 +1,486 @@
+"""Line-numbered view of a memory document + line-level edit ops.
+
+The audit / dedup modes ask the LLM to operate on a memory document
+the same way an IDE assistant operates on source code: it sees
+numbered lines and emits structured edits referencing those numbers.
+
+To keep the document's invariants intact, the LLM only ever sees a
+**sanitized** view — section headers (``## name``) and entry bullets
+(``- text [^m_xxx]``). The footnote block is hidden and rebuilt by
+:func:`apply_edits` from the surviving entries' refs.
+
+Editing model
+-------------
+Three op types: ``ReplaceLineOp``, ``DeleteLinesOp``, ``InsertAfterOp``.
+Apply in **descending line order** so earlier lines never shift under
+later edits. Each op carries a free-form ``reason`` for observability;
+audit/dedup prompts require it. Refs are mandatory on replace / insert
+of an entry (validated by the runtime).
+
+Public API
+----------
+* :func:`render_view` — turn a :class:`Document` into a list of numbered
+  :class:`Line` rows + lookup tables.
+* :func:`apply_edits` — apply a batch of edits to a document; pure
+  function (returns a new document) so callers can preview without
+  mutating shared state.
+* :func:`parse_edits_payload` — tolerant JSON → typed edit list parser.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+import logging
+import re
+from typing import Iterable, Literal, Union
+
+from deeptutor.services.memory.document import Document, Entry
+from deeptutor.services.memory.ids import is_entry_id, new_entry_id
+
+logger = logging.getLogger(__name__)
+
+LineKind = Literal["title", "blank", "section", "bullet"]
+
+
+@dataclass(frozen=True)
+class Line:
+    number: int  # 1-based, matches what the LLM sees
+    kind: LineKind
+    text: str  # rendered text (no leading "n: ")
+    entry_id: str | None = None  # for bullet lines, the m_xxx id
+    section: str | None = None  # for bullet lines, owning section name
+
+
+@dataclass(frozen=True)
+class LineView:
+    """Snapshot of the sanitized document seen by audit / dedup LLMs."""
+
+    lines: list[Line]
+    entry_by_id: dict[str, Entry]
+    entries_in_order: list[Entry]
+
+    def render(self, *, with_numbers: bool = True) -> str:
+        if with_numbers:
+            width = max(2, len(str(len(self.lines))))
+            return "\n".join(f"{line.number:>{width}}: {line.text}" for line in self.lines)
+        return "\n".join(line.text for line in self.lines)
+
+    def line(self, number: int) -> Line | None:
+        return self.lines[number - 1] if 1 <= number <= len(self.lines) else None
+
+
+# ── Edit ops ────────────────────────────────────────────────────────────
+
+
+@dataclass(frozen=True)
+class ReplaceLineOp:
+    line: int
+    new_text: str
+    refs: list[str]
+    reason: str = ""
+    op: Literal["replace"] = "replace"
+
+
+@dataclass(frozen=True)
+class DeleteLinesOp:
+    line_start: int
+    line_end: int  # inclusive
+    reason: str = ""
+    op: Literal["delete"] = "delete"
+
+
+@dataclass(frozen=True)
+class InsertAfterOp:
+    after_line: int
+    text: str
+    refs: list[str]
+    # ``section`` is optional: when None, the engine uses the section
+    # containing ``after_line``. If after_line is 0 (top-of-doc) or
+    # points at a title/blank, section MUST be provided.
+    section: str | None = None
+    reason: str = ""
+    op: Literal["insert"] = "insert"
+
+
+Edit = Union[ReplaceLineOp, DeleteLinesOp, InsertAfterOp]
+
+
+@dataclass
+class EditResult:
+    op: Edit
+    status: Literal["applied", "rejected"]
+    detail: str = ""
+
+
+@dataclass
+class EditReport:
+    applied: list[EditResult] = field(default_factory=list)
+    rejected: list[EditResult] = field(default_factory=list)
+
+    @property
+    def all_results(self) -> list[EditResult]:
+        return self.applied + self.rejected
+
+
+# ── Render: Document → LineView ─────────────────────────────────────────
+
+
+def render_view(doc: Document) -> LineView:
+    """Produce the numbered view the LLM operates on."""
+    lines: list[Line] = []
+    entries_in_order: list[Entry] = []
+    entry_by_id: dict[str, Entry] = {}
+
+    if doc.title:
+        lines.append(Line(number=len(lines) + 1, kind="title", text=f"# {doc.title}"))
+        lines.append(Line(number=len(lines) + 1, kind="blank", text=""))
+
+    for section_name, entries in doc.sections:
+        if not entries:
+            continue
+        lines.append(Line(number=len(lines) + 1, kind="section", text=f"## {section_name}"))
+        for entry in entries:
+            lines.append(
+                Line(
+                    number=len(lines) + 1,
+                    kind="bullet",
+                    text=f"- {entry.text} [^{entry.id}]",
+                    entry_id=entry.id,
+                    section=section_name,
+                )
+            )
+            entries_in_order.append(entry)
+            entry_by_id[entry.id] = entry
+        lines.append(Line(number=len(lines) + 1, kind="blank", text=""))
+
+    # Strip the trailing blank so the rendered view doesn't end with an
+    # empty line — keeps line counts predictable.
+    while lines and lines[-1].kind == "blank":
+        lines.pop()
+
+    return LineView(
+        lines=lines,
+        entry_by_id=entry_by_id,
+        entries_in_order=entries_in_order,
+    )
+
+
+# ── Apply edits ─────────────────────────────────────────────────────────
+
+
+def apply_edits(
+    doc: Document, edits: Iterable[Edit], *, view: LineView | None = None
+) -> tuple[Document, EditReport]:
+    """Apply a batch of edits, in reverse line order, to a fresh copy.
+
+    Returns ``(new_doc, report)``. ``new_doc`` is always returned; if
+    any edit was rejected, those are captured in ``report.rejected`` and
+    the rest are still applied. The caller decides what to do with a
+    partial-success batch (audit/dedup just write the partial result).
+
+    Reverse order avoids line-number drift: removing line 5 does not
+    affect the meaning of "line 3" since 3 < 5 and we process 5 first.
+    """
+    # Chunked callers retain the view from which all model prompts were built.
+    # Its entry IDs keep later edits anchored after earlier lines are deleted.
+    view = view if view is not None else render_view(doc)
+    edit_list = _sort_reverse(list(edits))
+    report = EditReport()
+
+    # Work on a deep-ish copy: the entry list per section is fresh, but
+    # Entry instances themselves are reused (and possibly mutated in
+    # place by replace).
+    new_doc = Document(
+        title=doc.title,
+        sections=[(name, list(entries)) for name, entries in doc.sections],
+    )
+
+    for edit in edit_list:
+        try:
+            detail = _apply_one(edit, new_doc, view)
+            report.applied.append(EditResult(op=edit, status="applied", detail=detail))
+        except _Reject as exc:
+            logger.warning("line-edit rejected: %s — %s", _short(edit), exc)
+            report.rejected.append(EditResult(op=edit, status="rejected", detail=str(exc)))
+
+    _drop_empty_sections(new_doc)
+    return new_doc, report
+
+
+class _Reject(Exception):
+    """Internal sentinel — signals one edit is unsafe; siblings still apply."""
+
+
+def _apply_one(edit: Edit, doc: Document, view: LineView) -> str:
+    if isinstance(edit, ReplaceLineOp):
+        return _apply_replace(edit, doc, view)
+    if isinstance(edit, DeleteLinesOp):
+        return _apply_delete(edit, doc, view)
+    if isinstance(edit, InsertAfterOp):
+        return _apply_insert(edit, doc, view)
+    raise _Reject(f"unknown edit type {type(edit).__name__}")
+
+
+def _apply_replace(edit: ReplaceLineOp, doc: Document, view: LineView) -> str:
+    line = view.line(edit.line)
+    if line is None:
+        raise _Reject(f"line {edit.line} out of range")
+    if line.kind != "bullet" or not line.entry_id:
+        raise _Reject(f"line {edit.line} is not an editable entry")
+    if not edit.new_text.strip():
+        raise _Reject("new_text empty")
+    if not edit.refs:
+        raise _Reject("replace requires non-empty refs")
+
+    entry = _entry_in_doc(doc, line.entry_id)
+    if entry is None:
+        raise _Reject(f"entry {line.entry_id} not found in current doc")
+    entry.text = edit.new_text.strip()
+    entry.refs = list(edit.refs)
+    return f"replace {entry.id}"
+
+
+def _apply_delete(edit: DeleteLinesOp, doc: Document, view: LineView) -> str:
+    if edit.line_end < edit.line_start:
+        raise _Reject(f"line_end {edit.line_end} < line_start {edit.line_start}")
+    ids_to_drop: set[str] = set()
+    for n in range(edit.line_start, edit.line_end + 1):
+        line = view.line(n)
+        if line is None or line.kind != "bullet" or not line.entry_id:
+            continue  # section/blank lines are removed only as a side-effect of empties
+        ids_to_drop.add(line.entry_id)
+    if not ids_to_drop:
+        raise _Reject("range covers no entries")
+    for _name, entries in doc.sections:
+        entries[:] = [e for e in entries if e.id not in ids_to_drop]
+    return f"deleted {len(ids_to_drop)} entries"
+
+
+def _apply_insert(edit: InsertAfterOp, doc: Document, view: LineView) -> str:
+    if not edit.text.strip():
+        raise _Reject("insert text empty")
+    if not edit.refs:
+        raise _Reject("insert requires non-empty refs")
+
+    section = edit.section
+    if section is None:
+        if edit.after_line < 1 or edit.after_line > len(view.lines):
+            raise _Reject(
+                "after_line out of range; for top-of-doc insert provide `section` explicitly"
+            )
+        anchor = view.line(edit.after_line)
+        section = anchor.section if anchor and anchor.section else None
+        if section is None and anchor and anchor.kind == "section":
+            section = anchor.text.lstrip("# ").strip()
+        if section is None:
+            raise _Reject("no section context for insert; supply `section`")
+
+    entry = Entry(
+        id=new_entry_id(),
+        section=section,
+        text=edit.text.strip(),
+        refs=list(edit.refs),
+    )
+    target = _section_entries(doc, section)
+    # When inserting after a bullet inside an existing section, honor
+    # the local position; otherwise append at end.
+    anchor = view.line(edit.after_line) if 1 <= edit.after_line <= len(view.lines) else None
+    if anchor and anchor.kind == "bullet" and anchor.section == section and anchor.entry_id:
+        for idx, existing in enumerate(target):
+            if existing.id == anchor.entry_id:
+                target.insert(idx + 1, entry)
+                break
+        else:
+            target.append(entry)
+    else:
+        target.append(entry)
+    return f"inserted {entry.id} into {section!r}"
+
+
+# ── Parse edits payload ─────────────────────────────────────────────────
+
+
+_REF_WRAPPER_CHARS = "`[](){}<>^ \t\n\r"
+_ENTRY_ID_REF_RE = re.compile(r"^m_[0-9A-HJKMNP-TV-Z]{26}$")
+
+
+def _clean_refs(raw_refs: object, *, layer: str | None) -> list[str]:
+    """Strip wrappers + drop garbage refs from one ``refs`` array.
+
+    Two cleanups, in order:
+
+    1. **Wrapper strip**. The audit / dedup line-numbered view shows each
+       bullet as ``- text [^m_xxx]``. LLMs sometimes copy the marker
+       (``^m_xxx``) wholesale into the new refs array — caret and all.
+       Strip ``` ` [ ] ( ) { } < > ^ ``` plus whitespace from both sides.
+
+    2. **Layer-shape filter**. After stripping, an L2 doc whose refs
+       still look like ``m_<ULID>`` (an entry id from the line view) is
+       almost certainly hallucinated — real L2 refs are ``surface:id``.
+       Drop them. L3 ref shape *is* ``m_<ULID>`` so the filter is a
+       no-op there.
+    """
+    if not isinstance(raw_refs, list):
+        return []
+    out: list[str] = []
+    for r in raw_refs:
+        if not r:
+            continue
+        s = str(r).strip(_REF_WRAPPER_CHARS).strip()
+        if not s:
+            continue
+        if layer == "L2" and _ENTRY_ID_REF_RE.match(s):
+            continue
+        out.append(s)
+    return out
+
+
+def parse_edits_payload(raw: str, *, layer: str | None = None) -> list[Edit]:
+    """Tolerant JSON parse → list[Edit].
+
+    Accepts ``{"edits": [...]}`` or a top-level ``[...]``. Each entry's
+    ``op`` field discriminates the type. Unknown ops are dropped.
+
+    ``layer`` (``"L2"`` / ``"L3"``) controls ref-shape filtering — see
+    :func:`_clean_refs`. Omit it (or pass ``None``) when the caller
+    can't or shouldn't filter by layer; refs are still stripped of
+    wrapper characters.
+    """
+    snippet = _extract_json(raw)
+    if snippet is None:
+        return []
+    try:
+        data = json.loads(snippet)
+    except json.JSONDecodeError:
+        logger.warning("line-edit parse: malformed JSON")
+        return []
+    items = data.get("edits") if isinstance(data, dict) else data
+    if not isinstance(items, list):
+        return []
+
+    edits: list[Edit] = []
+    for raw_op in items:
+        if not isinstance(raw_op, dict):
+            continue
+        kind = raw_op.get("op")
+        try:
+            if kind == "replace":
+                edits.append(
+                    ReplaceLineOp(
+                        line=int(raw_op.get("line", 0)),
+                        new_text=str(raw_op.get("new_text", "")).strip(),
+                        refs=_clean_refs(raw_op.get("refs", []), layer=layer),
+                        reason=str(raw_op.get("reason", "")).strip(),
+                    )
+                )
+            elif kind == "delete":
+                edits.append(
+                    DeleteLinesOp(
+                        line_start=int(raw_op.get("line_start", raw_op.get("line", 0))),
+                        line_end=int(raw_op.get("line_end", raw_op.get("line", 0))),
+                        reason=str(raw_op.get("reason", "")).strip(),
+                    )
+                )
+            elif kind == "insert":
+                section = raw_op.get("section")
+                edits.append(
+                    InsertAfterOp(
+                        after_line=int(raw_op.get("after_line", 0)),
+                        text=str(raw_op.get("text", "")).strip(),
+                        refs=_clean_refs(raw_op.get("refs", []), layer=layer),
+                        section=str(section).strip() if section else None,
+                        reason=str(raw_op.get("reason", "")).strip(),
+                    )
+                )
+        except (TypeError, ValueError):
+            continue
+    return edits
+
+
+# ── Helpers ─────────────────────────────────────────────────────────────
+
+
+def _sort_reverse(edits: list[Edit]) -> list[Edit]:
+    def key(e: Edit) -> tuple[int, int]:
+        if isinstance(e, ReplaceLineOp):
+            return (e.line, 0)
+        if isinstance(e, DeleteLinesOp):
+            return (e.line_end, 1)  # delete sorts before insert at same line
+        if isinstance(e, InsertAfterOp):
+            return (e.after_line, 2)
+        return (0, 9)
+
+    return sorted(edits, key=key, reverse=True)
+
+
+def _entry_in_doc(doc: Document, entry_id: str) -> Entry | None:
+    if not is_entry_id(entry_id):
+        return None
+    for _section, entries in doc.sections:
+        for entry in entries:
+            if entry.id == entry_id:
+                return entry
+    return None
+
+
+def _section_entries(doc: Document, name: str) -> list[Entry]:
+    for section, entries in doc.sections:
+        if section == name:
+            return entries
+    new_entries: list[Entry] = []
+    doc.sections.append((name, new_entries))
+    return new_entries
+
+
+def _drop_empty_sections(doc: Document) -> None:
+    doc.sections[:] = [(name, entries) for name, entries in doc.sections if entries]
+
+
+def _short(edit: Edit) -> str:
+    if isinstance(edit, ReplaceLineOp):
+        return f"replace L{edit.line}"
+    if isinstance(edit, DeleteLinesOp):
+        return f"delete L{edit.line_start}-{edit.line_end}"
+    if isinstance(edit, InsertAfterOp):
+        return f"insert@L{edit.after_line}"
+    return repr(edit)
+
+
+_FENCE_RE = re.compile(r"^```[a-zA-Z]*\s*|\s*```$")
+
+
+def _extract_json(raw: str) -> str | None:
+    text = _FENCE_RE.sub("", raw.strip())
+    # First {...} or [...] via raw_decode so trailing prose braces cannot
+    # extend the slice past the real JSON value.
+    obj_start = text.find("{")
+    arr_start = text.find("[")
+    if obj_start == -1 and arr_start == -1:
+        return None
+    if obj_start == -1:
+        start = arr_start
+    elif arr_start == -1:
+        start = obj_start
+    else:
+        start = min(obj_start, arr_start)
+    try:
+        _parsed, end = json.JSONDecoder().raw_decode(text[start:])
+    except json.JSONDecodeError:
+        return None
+    return text[start : start + end]
+
+
+__all__ = [
+    "DeleteLinesOp",
+    "Edit",
+    "EditReport",
+    "EditResult",
+    "InsertAfterOp",
+    "Line",
+    "LineView",
+    "ReplaceLineOp",
+    "apply_edits",
+    "parse_edits_payload",
+    "render_view",
+]

deeptutor/services/memory/consolidator/meta.py

@@ -0,0 +1,194 @@
+"""Per-doc consolidator metadata (``*.meta.json`` files).
+
+For each L2/L3 markdown doc we keep a sidecar JSON capturing the set of
+upstream ids "seen" at the last update. ``run_update`` uses set diff
+against the live state to compute "what's new since last update" — a
+purely id-based diff is robust against mtime / time-zone / replays.
+
+Files
+-----
+* ``memory/L2/<surface>.meta.json``::
+
+      {
+        "version": 1,
+        "last_update_at": "<iso-utc>",
+        "seen_entity_refs": ["chat:01HZK4...", ...]
+      }
+
+* ``memory/L3/<slot>.meta.json``::
+
+      {
+        "version": 1,
+        "last_update_at": "<iso-utc>",
+        "seen_l2_entry_ids": {
+          "chat": ["m_xxx", ...],
+          "notebook": ["m_yyy"],
+          ...
+        }
+      }
+
+Atomic writes via temp + rename. Missing files behave as "first run".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+import json
+import logging
+import os
+from pathlib import Path
+import tempfile
+
+from deeptutor.services.memory import paths
+from deeptutor.services.memory.paths import L3Slot, Surface
+
+logger = logging.getLogger(__name__)
+
+_META_VERSION = 1
+
+
+# ── L2 meta ─────────────────────────────────────────────────────────────
+
+
+@dataclass
+class L2Meta:
+    last_update_at: str | None = None
+    seen_entity_refs: set[str] = field(default_factory=set)
+
+
+def l2_meta_path(surface: Surface) -> Path:
+    return paths.l2_dir() / f"{surface}.meta.json"
+
+
+def load_l2_meta(surface: Surface) -> L2Meta:
+    return _load_meta_l2(l2_meta_path(surface))
+
+
+def save_l2_meta(surface: Surface, *, seen_entity_refs: set[str]) -> L2Meta:
+    path = l2_meta_path(surface)
+    meta = L2Meta(
+        last_update_at=_now_iso(),
+        seen_entity_refs=set(seen_entity_refs),
+    )
+    _atomic_write_json(
+        path,
+        {
+            "version": _META_VERSION,
+            "last_update_at": meta.last_update_at,
+            "seen_entity_refs": sorted(meta.seen_entity_refs),
+        },
+    )
+    return meta
+
+
+# ── L3 meta ─────────────────────────────────────────────────────────────
+
+
+@dataclass
+class L3Meta:
+    last_update_at: str | None = None
+    seen_l2_entry_ids: dict[str, set[str]] = field(default_factory=dict)
+
+
+def l3_meta_path(slot: L3Slot) -> Path:
+    return paths.l3_dir() / f"{slot}.meta.json"
+
+
+def load_l3_meta(slot: L3Slot) -> L3Meta:
+    return _load_meta_l3(l3_meta_path(slot))
+
+
+def save_l3_meta(
+    slot: L3Slot,
+    *,
+    seen_l2_entry_ids: dict[str, set[str]],
+) -> L3Meta:
+    path = l3_meta_path(slot)
+    meta = L3Meta(
+        last_update_at=_now_iso(),
+        seen_l2_entry_ids={surface: set(ids) for surface, ids in seen_l2_entry_ids.items()},
+    )
+    _atomic_write_json(
+        path,
+        {
+            "version": _META_VERSION,
+            "last_update_at": meta.last_update_at,
+            "seen_l2_entry_ids": {
+                surface: sorted(ids) for surface, ids in meta.seen_l2_entry_ids.items()
+            },
+        },
+    )
+    return meta
+
+
+# ── Internals ───────────────────────────────────────────────────────────
+
+
+def _load_meta_l2(path: Path) -> L2Meta:
+    data = _read_json(path)
+    if not data:
+        return L2Meta()
+    refs = data.get("seen_entity_refs") or []
+    return L2Meta(
+        last_update_at=data.get("last_update_at"),
+        seen_entity_refs=set(refs) if isinstance(refs, list) else set(),
+    )
+
+
+def _load_meta_l3(path: Path) -> L3Meta:
+    data = _read_json(path)
+    if not data:
+        return L3Meta()
+    raw = data.get("seen_l2_entry_ids") or {}
+    if not isinstance(raw, dict):
+        raw = {}
+    return L3Meta(
+        last_update_at=data.get("last_update_at"),
+        seen_l2_entry_ids={
+            surface: set(ids) if isinstance(ids, list) else set() for surface, ids in raw.items()
+        },
+    )
+
+
+def _read_json(path: Path) -> dict | None:
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        logger.warning("memory meta: failed to read %s: %s", path, exc)
+        return None
+
+
+def _atomic_write_json(path: Path, payload: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_str = tempfile.mkstemp(prefix=path.name + ".", dir=str(path.parent))
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            json.dump(payload, fh, ensure_ascii=False, indent=2, sort_keys=False)
+            fh.flush()
+            os.fsync(fh.fileno())
+        os.replace(tmp_str, path)
+    finally:
+        if os.path.exists(tmp_str):
+            try:
+                os.remove(tmp_str)
+            except OSError:
+                pass
+
+
+def _now_iso() -> str:
+    return datetime.now(tz=timezone.utc).isoformat()
+
+
+__all__ = [
+    "L2Meta",
+    "L3Meta",
+    "l2_meta_path",
+    "l3_meta_path",
+    "load_l2_meta",
+    "load_l3_meta",
+    "save_l2_meta",
+    "save_l3_meta",
+]

deeptutor/services/memory/consolidator/modes/__init__.py

@@ -0,0 +1,31 @@
+"""Four user-visible consolidation modes.
+
+* :func:`run_update`  — chunk-based incremental fact extraction.
+* :func:`run_audit`   — chunk-based line-level edits against raw evidence.
+* :func:`run_dedup`   — iterative line-level dedup over the full doc.
+* :func:`run_merge`   — no-LLM footnote consolidation (collapse duplicate refs).
+
+Plus thin shims (:func:`consolidate_l2`, :func:`consolidate_l3`) kept
+for :mod:`deeptutor.services.memory.store` so the public API surface
+stays stable while the implementation switches under the hood.
+"""
+
+from __future__ import annotations
+
+from deeptutor.services.memory.consolidator.modes._shims import (
+    consolidate_l2,
+    consolidate_l3,
+)
+from deeptutor.services.memory.consolidator.modes.audit import run_audit
+from deeptutor.services.memory.consolidator.modes.dedup import run_dedup
+from deeptutor.services.memory.consolidator.modes.merge import run_merge
+from deeptutor.services.memory.consolidator.modes.update import run_update
+
+__all__ = [
+    "consolidate_l2",
+    "consolidate_l3",
+    "run_audit",
+    "run_dedup",
+    "run_merge",
+    "run_update",
+]