deeptutor-memory 1.7.0__py3-none-any.whl

deeptutor/services/memory/__init__.py

@@ -0,0 +1,42 @@
+"""Three-layer memory subsystem.
+
+- ``trace``   : L1 raw event capture (append-only JSONL per surface per day)
+- ``document``: L2/L3 markdown + footnote-citation parser/serializer (pure)
+- ``ops``     : add/edit/delete batch applier (pure)
+- ``paths``   : per-user path resolution
+- ``ids``     : ULID-style trace and entry id generators
+- ``store``   : public facade — :class:`MemoryStore`
+- ``consolidator``: LLM-driven L1→L2 and L2→L3 ops
+
+The v1 two-file `MemoryService` is gone. All callers go through
+:func:`get_memory_store` and the :class:`MemoryStore` facade.
+"""
+
+from .ids import is_entry_id, is_trace_id, new_entry_id, new_trace_id
+from .paths import L3_SLOTS, SURFACES, L3Slot, Surface, memory_path_service_override
+from .store import (
+    DocOverview,
+    MemoryStore,
+    get_memory_store,
+    migrate_partner_surface_if_needed,
+    migrate_v1_if_needed,
+)
+from .trace import TraceEvent
+
+__all__ = [
+    "DocOverview",
+    "L3_SLOTS",
+    "L3Slot",
+    "MemoryStore",
+    "SURFACES",
+    "Surface",
+    "TraceEvent",
+    "get_memory_store",
+    "is_entry_id",
+    "is_trace_id",
+    "memory_path_service_override",
+    "migrate_partner_surface_if_needed",
+    "migrate_v1_if_needed",
+    "new_entry_id",
+    "new_trace_id",
+]

deeptutor/services/memory/consolidator/__init__.py

@@ -0,0 +1,62 @@
+"""Memory consolidator — chunk-based update / audit / dedup.
+
+Public surface (call these from the API router / store / tests):
+
+* :class:`ConsolidateResult`, :data:`OnEvent` — legacy types preserved
+  for :mod:`deeptutor.services.memory.store`.
+* :func:`consolidate_l2`, :func:`consolidate_l3` — legacy shims that
+  delegate to :func:`run_update`.
+* :func:`run_update`, :func:`run_audit`, :func:`run_dedup` — the three
+  modes the workbench drives directly.
+* :func:`_parse_ops_response`, :func:`_filter_banned`,
+  :func:`_has_banned` — kept for :meth:`store.MemoryStore.apply_ops_payload`
+  and the existing test suite.
+
+Submodule layout:
+
+    chunker.py         pure char-based chunker with boundary expansion
+    line_doc.py        line-numbered view + replace/delete/insert edits
+    meta.py            *.meta.json read/write for "seen ids" diffs
+    references.py      ref pool validation + raw-trace annotation
+    guards.py          banned-phrase filter (legacy + L3 enforcement)
+    parse.py           legacy ops-array parser (apply_ops_payload)
+    modes/
+      _runtime.py      shared prompt loading + LLM + atomic write
+      _shims.py        legacy consolidate_l2/l3 → run_update
+      update.py        chunk-based incremental fact extraction
+      audit.py         chunk-based line edits vs raw evidence
+      dedup.py         iterative line-level dedup over full doc
+    prompts/
+      {en,zh}/{update_l2,update_l3,audit_l2,audit_l3,dedup,_meta}.yaml
+"""
+
+from __future__ import annotations
+
+from deeptutor.services.memory.consolidator.guards import _filter_banned, _has_banned
+from deeptutor.services.memory.consolidator.modes import (
+    consolidate_l2,
+    consolidate_l3,
+    run_audit,
+    run_dedup,
+    run_merge,
+    run_update,
+)
+from deeptutor.services.memory.consolidator.modes._shims import (
+    ConsolidateResult,
+    OnEvent,
+)
+from deeptutor.services.memory.consolidator.parse import _parse_ops_response
+
+__all__ = [
+    "ConsolidateResult",
+    "OnEvent",
+    "_filter_banned",
+    "_has_banned",
+    "_parse_ops_response",
+    "consolidate_l2",
+    "consolidate_l3",
+    "run_audit",
+    "run_dedup",
+    "run_merge",
+    "run_update",
+]

deeptutor/services/memory/consolidator/chunker.py

@@ -0,0 +1,131 @@
+"""Character-based chunking with boundary expansion.
+
+The L2 / L3 update flow concatenates inputs into one string, then
+:func:`chunk_with_boundary` cuts it into ≤ budget pieces. Each piece's
+right edge is extended forward to the next paragraph or sentence
+boundary — content is **never truncated mid-statement**. Adjacent
+chunks overlap by a percentage of the target size so a fact straddling
+a cut still gets a fair read.
+
+Pure functions: no I/O, no LLM. Easy to unit-test.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import math
+import re
+from typing import Literal
+
+Boundary = Literal["paragraph", "sentence"]
+
+# Paragraph boundary: one or more blank lines.
+_PARA_BOUNDARY = re.compile(r"\n\s*\n+")
+# Sentence boundary: terminal punctuation followed by space/newline.
+# Covers ASCII (.!?) and CJK (。！？).
+_SENT_BOUNDARY = re.compile(r"[.!?。！？](?:[\")»」』]+)?(?=\s|$)")
+
+
+@dataclass(frozen=True)
+class ChunkSpan:
+    """One chunk's coordinates inside the source text.
+
+    ``start`` is inclusive, ``end`` exclusive. ``index`` is the 0-based
+    position in the returned list (useful for events).
+    """
+
+    index: int
+    start: int
+    end: int
+    text: str
+
+
+def chunk_with_boundary(
+    text: str,
+    *,
+    budget: int,
+    overlap_ratio: float,
+    min_chunk_chars: int,
+    max_chunk_chars: int,
+    boundary: Boundary = "paragraph",
+) -> list[ChunkSpan]:
+    """Cut ``text`` into ≤ ``budget`` chunks aligned to natural boundaries.
+
+    * Target size = ``clamp(ceil(len(text) / budget), min, max)``.
+    * Right edge of each chunk is extended forward to the next
+      ``boundary`` so no sentence/paragraph is split.
+    * Adjacent chunks overlap by ``round(target * overlap_ratio)`` chars.
+    * If the input is short enough to fit in a single chunk, returns
+      one ``ChunkSpan`` covering everything.
+    """
+    if not text.strip():
+        return []
+    if budget < 1:
+        budget = 1
+
+    n = len(text)
+    target = math.ceil(n / budget)
+    target = max(min_chunk_chars, min(max_chunk_chars, target))
+    overlap = max(0, min(target - 1, round(target * overlap_ratio)))
+
+    # Short-circuit: input fits in one chunk.
+    if n <= target:
+        return [ChunkSpan(index=0, start=0, end=n, text=text)]
+
+    # Hard cap on how far the right edge can be pulled to find a
+    # boundary. Beyond this we accept a non-boundary cut so chunks
+    # never grow past ``max_chunk_chars`` even in degenerate input
+    # (e.g. a single long line with no paragraph/sentence breaks).
+    spans: list[ChunkSpan] = []
+    cursor = 0
+    while cursor < n:
+        target_end = min(n, cursor + target)
+        hard_cap = min(n, cursor + max_chunk_chars)
+        if target_end >= n:
+            end = n
+        else:
+            end = _expand_to_boundary(text, target_end, boundary, limit=hard_cap)
+        # Guarantee forward motion: the boundary expansion may pull us
+        # to len(text), or — degenerate input — to ``target_end`` itself.
+        if end <= cursor:
+            end = min(n, cursor + max(1, target))
+        spans.append(
+            ChunkSpan(
+                index=len(spans),
+                start=cursor,
+                end=end,
+                text=text[cursor:end],
+            )
+        )
+        if end >= n:
+            break
+        next_cursor = end - overlap
+        # No infinite loop: must advance by at least one char even with
+        # huge overlap.
+        if next_cursor <= cursor:
+            next_cursor = cursor + 1
+        cursor = next_cursor
+
+    return spans
+
+
+# ── Internals ───────────────────────────────────────────────────────────
+
+
+def _expand_to_boundary(text: str, target_end: int, boundary: Boundary, *, limit: int) -> int:
+    """Push ``target_end`` forward to the next natural boundary.
+
+    The search is bounded by ``limit`` (exclusive). If no boundary is
+    found within that window the function returns ``limit`` — a non-
+    boundary cut, but a bounded one. Without this the chunker can
+    inflate a single chunk to the end of the input on pathological
+    text with no paragraph/sentence markers.
+    """
+    pattern = _PARA_BOUNDARY if boundary == "paragraph" else _SENT_BOUNDARY
+    match = pattern.search(text, target_end, limit)
+    if match is None:
+        return limit
+    return match.end()
+
+
+__all__ = ["Boundary", "ChunkSpan", "chunk_with_boundary"]

deeptutor/services/memory/consolidator/guards.py

@@ -0,0 +1,104 @@
+"""Op-emit guards: banned-phrase filter + budgets.
+
+Run at op-emit time (during the loop) so the model gets an observation
+back when an op is rejected and can rewrite. Today's pre-redesign code
+filtered banned phrases after the LLM call returned all ops at once,
+which meant rejection was a silent drop.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import logging
+import re
+from typing import Iterable
+
+from deeptutor.services.memory.ops import Op
+
+logger = logging.getLogger(__name__)
+
+# L3 objectivity guard: phrases the LLM is prompt-banned from emitting.
+# Runtime enforces by dropping any L3 op whose text contains one of these
+# (outside of quoted user verbatim 「」 / "…"). Logged as a warning so we
+# can tune the list against real prompt regressions.
+BANNED_PHRASES: tuple[str, ...] = (
+    # English absolutes
+    "deeply",
+    "truly",
+    "mastered",
+    "expert in",
+    "passionate",
+    "loves",
+    "hates",
+    "always",
+    "never",
+    "fully understands",
+    # Chinese absolutes
+    "深刻",
+    "彻底",
+    "完美掌握",
+    "完美理解",
+    "完全理解",
+    "完全掌握",
+    "专家",
+    "热爱",
+    "总是",
+    "从来不",
+)
+
+
+# Per-loop budgets. Beyond these the dispatcher emits a hint observation
+# instead of executing the action; the prompt nudges the model to finish.
+@dataclass(frozen=True)
+class ToolBudgets:
+    read_entity: int = 30
+    search: int = 20
+    list_pending: int = 50  # cheap nav, generous
+    list_sections: int = 50
+    recent_changes: int = 3
+    add_entry: int = 12
+    edit_entry: int = 12
+    delete_entry: int = 12
+    note: int = 8
+
+
+_QUOTED_RE = re.compile(r"「[^」]*」|\"[^\"]*\"")
+
+
+def _has_banned(text: str) -> bool:
+    """Return ``True`` iff a banned phrase appears outside every quote.
+
+    Quoted regions (CJK 「…」 or ASCII "…") are stripped first, because
+    the prompt allows verbatim user quotations to contain the otherwise-
+    banned absolutes.
+    """
+    stripped = _QUOTED_RE.sub("", text).lower()
+    for phrase in BANNED_PHRASES:
+        if phrase in stripped:
+            return True
+    return False
+
+
+def _op_text(op: Op) -> str:
+    text = getattr(op, "text", "") or getattr(op, "new_text", "")
+    return str(text)
+
+
+def _filter_banned(ops: Iterable[Op]) -> list[Op]:
+    """Drop ops whose text contains banned absolutist phrasing.
+
+    Used post-loop as a safety net even though the per-op emit path
+    already rejects them. Kept callable by name for legacy tests and
+    the apply_ops_payload preview/apply round-trip.
+    """
+    kept: list[Op] = []
+    for op in ops:
+        text = _op_text(op)
+        if text and _has_banned(text):
+            logger.warning(
+                "memory consolidate: dropped op with banned phrase: %s",
+                text[:80],
+            )
+            continue
+        kept.append(op)
+    return kept