deeptutor-memory 1.7.0__py3-none-any.whl

deeptutor/services/memory/recall.py

@@ -0,0 +1,237 @@
+"""Reading memory back — what the learner has been doing lately.
+
+L1 keeps two kinds of record, and answering "what happened lately" needs both:
+
+* the **snapshot** knows what *exists* — every chat session, notebook record,
+  quiz attempt, document, book and knowledge base, each with a label and a
+  timestamp. :func:`recent` reads that.
+* the **trace** knows what was *done* — the events surfaces emit as they run.
+  :func:`recent_queries` reads the one kind that records the learner asking
+  something in their own words.
+
+Both run on stamps: no message body, no document text, no answer is read. That
+is what makes them safe to call from an interactive path rather than only from
+consolidation, which is the whole reason this module exists separately from
+:mod:`deeptutor.services.memory.store`.
+
+Every hit carries ``days_ago`` rather than leaving an ISO timestamp for the
+caller to subtract. A model asked to do date arithmetic on
+``2026-08-16T11:03:00+00:00`` will sometimes get it wrong, and "how long ago"
+is the entire basis on which anything decides whether an item is still worth
+raising.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+import logging
+from typing import Any, Iterable
+
+from deeptutor.services.memory.paths import SURFACES, Surface
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_LIMIT = 20
+_MAX_LIMIT = 100
+# A week: long enough to survive a couple of quiet days, short enough that what
+# comes back is still recognisably "lately".
+_DEFAULT_DAYS = 7
+# A label is a title, a question or a filename. Anything longer is a body that
+# leaked into one, and carrying it would defeat the point of stamps-only reads.
+_MAX_LABEL_CHARS = 200
+
+
+@dataclass(frozen=True, slots=True)
+class RecallHit:
+    """One thing the learner touched, shaped for a caller to act on."""
+
+    surface: str
+    label: str
+    ts: str
+    days_ago: int | None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "surface": self.surface,
+            "label": self.label,
+            "ts": self.ts,
+            "days_ago": self.days_ago,
+        }
+
+
+# ── Time ─────────────────────────────────────────────────────────────────
+
+
+def _now() -> datetime:
+    return datetime.now(tz=timezone.utc)
+
+
+def _parse_ts(ts: str) -> datetime | None:
+    if not ts:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(ts).replace("Z", "+00:00"))
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
+def days_ago(ts: str, *, now: datetime | None = None) -> int | None:
+    """Whole days between ``ts`` and now, or ``None`` for an unusable stamp."""
+    parsed = _parse_ts(ts)
+    if parsed is None:
+        return None
+    delta = (now or _now()) - parsed
+    # A future stamp (clock skew on an imported record) reads as today rather
+    # than as a negative age.
+    return max(0, delta.days)
+
+
+def _cutoff(days: int | None, now: datetime) -> datetime | None:
+    if days is None or days < 0:
+        return None
+    return now - timedelta(days=days)
+
+
+# ── Shaping ──────────────────────────────────────────────────────────────
+
+
+def _clean_label(raw: str) -> str:
+    """One line, bounded. Empty when there is nothing worth showing."""
+    text = " ".join(str(raw or "").split())
+    return text[:_MAX_LABEL_CHARS]
+
+
+def _resolve_surfaces(surfaces: Iterable[str] | None) -> tuple[Surface, ...]:
+    if not surfaces:
+        return SURFACES
+    wanted = {str(surface).strip().lower() for surface in surfaces}
+    resolved = tuple(surface for surface in SURFACES if surface in wanted)
+    return resolved or SURFACES
+
+
+def _clamp_limit(limit: int | None) -> int:
+    if not limit or limit < 1:
+        return _DEFAULT_LIMIT
+    return min(int(limit), _MAX_LIMIT)
+
+
+def _newest_first(hits: list[RecallHit]) -> list[RecallHit]:
+    """Sort newest first, dropping repeats of the same label on the same surface.
+
+    A knowledge base queried five times, or a session reopened all afternoon,
+    is one thing the learner is working on — not five. Keeping the duplicates
+    would let a single topic crowd out everything else downstream.
+    """
+    hits.sort(key=lambda hit: hit.ts, reverse=True)
+    seen: set[tuple[str, str]] = set()
+    deduped: list[RecallHit] = []
+    for hit in hits:
+        key = (hit.surface, hit.label.casefold())
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(hit)
+    return deduped
+
+
+# ── What exists (snapshot) ───────────────────────────────────────────────
+
+
+def recent(
+    *,
+    days: int | None = _DEFAULT_DAYS,
+    limit: int | None = _DEFAULT_LIMIT,
+    surfaces: Iterable[str] | None = None,
+) -> list[RecallHit]:
+    """The latest workspace activity across surfaces, newest first.
+
+    Stamps only — no content is read, so this stays cheap enough to call on a
+    page load.
+
+    Items without a usable timestamp are dropped rather than kept: placing an
+    item of unknown age inside "the last seven days" asserts something the data
+    does not support. This notably excludes knowledge bases, whose snapshot
+    timestamp is their *earliest* index time — a KB in daily use would look
+    stale here. What the learner has actually been asking a KB lives in the
+    trace instead; see :func:`recent_queries`.
+    """
+    from deeptutor.services.memory.snapshot import adapters
+
+    now = _now()
+    cutoff = _cutoff(days, now)
+    hits: list[RecallHit] = []
+    for surface in _resolve_surfaces(surfaces):
+        try:
+            stamps = adapters.read_stamps(surface)
+        except Exception:
+            logger.warning("recall.recent: stamps failed surface=%s", surface, exc_info=True)
+            continue
+        for stamp in stamps:
+            parsed = _parse_ts(stamp.ts)
+            if parsed is None or (cutoff is not None and parsed < cutoff):
+                continue
+            label = _clean_label(stamp.label)
+            if not label:
+                continue
+            hits.append(
+                RecallHit(
+                    surface=surface,
+                    label=label,
+                    ts=stamp.ts,
+                    days_ago=days_ago(stamp.ts, now=now),
+                )
+            )
+    return _newest_first(hits)[: _clamp_limit(limit)]
+
+
+# ── What was asked (trace) ───────────────────────────────────────────────
+
+# The trace kinds that carry a learner-authored question. Knowledge-base
+# queries are the only ones today: the other emitters record system-side facts
+# (a stated preference, a partner turn), not something the learner typed as a
+# question.
+_QUERY_KINDS: dict[Surface, str] = {"kb": "query"}
+
+
+def recent_queries(
+    *,
+    days: int | None = _DEFAULT_DAYS,
+    limit: int | None = _DEFAULT_LIMIT,
+) -> list[RecallHit]:
+    """Searches the learner ran lately, in their own words, newest first.
+
+    Complements :func:`recent`: the snapshot can say a knowledge base exists,
+    but only the trace records that someone asked it about eigenvalues on
+    Tuesday.
+    """
+    from deeptutor.services.memory import trace
+
+    now = _now()
+    cutoff = _cutoff(days, now)
+    hits: list[RecallHit] = []
+    for surface, kind in _QUERY_KINDS.items():
+        try:
+            events = trace.iter_since(surface, cutoff)
+        except Exception:
+            logger.warning("recall.recent_queries: failed surface=%s", surface, exc_info=True)
+            continue
+        for event in events:
+            if event.kind != kind:
+                continue
+            label = _clean_label(str((event.payload or {}).get("query") or ""))
+            if not label:
+                continue
+            hits.append(
+                RecallHit(
+                    surface=surface,
+                    label=label,
+                    ts=event.ts,
+                    days_ago=days_ago(event.ts, now=now),
+                )
+            )
+    return _newest_first(hits)[: _clamp_limit(limit)]
+
+
+__all__ = ["RecallHit", "days_ago", "recent", "recent_queries"]

deeptutor/services/memory/settings.py

@@ -0,0 +1,198 @@
+"""User-tunable parameters for the memory consolidator.
+
+Single source of truth is ``data/user/settings/main.yaml`` under the
+``memory:`` subtree. Defaults live here. The frontend ``/settings/memory``
+page reads/writes the same subtree via the API.
+
+Decoupled from the algorithm code: every mode picks values up via
+:func:`load_memory_settings`, never via module-level constants.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field, fields, is_dataclass
+from typing import Any, Literal
+
+from deeptutor.utils.config_manager import ConfigManager
+
+_SETTINGS_KEY = "memory"
+
+
+@dataclass(frozen=True)
+class UpdateSettings:
+    l2_budget: int = 20
+    l3_budget: int = 10
+
+
+@dataclass(frozen=True)
+class AuditSettings:
+    l2_budget: int = 20
+    l3_budget: int = 10
+
+
+@dataclass(frozen=True)
+class DedupSettings:
+    iterations: int = 3
+    auto_after_update: bool = True
+
+
+@dataclass(frozen=True)
+class MergeSettings:
+    """No-LLM footnote consolidation (collapse duplicate refs into one footnote each)."""
+
+    auto_after_update: bool = True
+    auto_after_audit: bool = True
+    auto_after_dedup: bool = True
+
+
+@dataclass(frozen=True)
+class ChunkingSettings:
+    overlap_ratio: float = 0.10
+    boundary: Literal["paragraph", "sentence"] = "paragraph"
+    min_chunk_chars: int = 1000
+    max_chunk_chars: int = 64000
+
+
+@dataclass(frozen=True)
+class ReferenceSettings:
+    enforce_required: bool = True
+    drop_invalid_refs: bool = True
+
+
+@dataclass(frozen=True)
+class MemorySettings:
+    update: UpdateSettings = field(default_factory=UpdateSettings)
+    audit: AuditSettings = field(default_factory=AuditSettings)
+    dedup: DedupSettings = field(default_factory=DedupSettings)
+    merge: MergeSettings = field(default_factory=MergeSettings)
+    chunking: ChunkingSettings = field(default_factory=ChunkingSettings)
+    reference: ReferenceSettings = field(default_factory=ReferenceSettings)
+
+
+def load_memory_settings() -> MemorySettings:
+    """Return the current ``memory:`` subtree merged on top of defaults.
+
+    Missing keys fall back to defaults. Out-of-range numeric values are
+    clamped to safe ranges so a malformed YAML never crashes a run.
+    """
+    raw = ConfigManager().load_config().get(_SETTINGS_KEY) or {}
+    return _from_dict(MemorySettings, raw)
+
+
+def save_memory_settings(payload: dict[str, Any]) -> MemorySettings:
+    """Merge ``payload`` into the on-disk ``memory:`` subtree.
+
+    Unknown keys are dropped; values are coerced to the schema's types
+    so the YAML never picks up junk. Returns the post-merge settings.
+    """
+    merged = _from_dict(MemorySettings, payload)
+    coerced = asdict(merged)
+    ConfigManager().save_config({_SETTINGS_KEY: coerced})
+    return merged
+
+
+def memory_settings_dict() -> dict[str, Any]:
+    """Settings as a plain dict — JSON-safe for the API response."""
+    return asdict(load_memory_settings())
+
+
+# ── Coercion + clamping ─────────────────────────────────────────────────
+
+
+_MIN_BUDGET = 1
+_MAX_BUDGET = 200
+_MIN_DEDUP_ITER = 1
+_MAX_DEDUP_ITER = 20
+_MIN_OVERLAP = 0.0
+_MAX_OVERLAP = 0.5
+_MIN_CHUNK_CHARS = 200
+_MAX_CHUNK_CHARS = 64000
+_BOUNDARIES = ("paragraph", "sentence")
+
+
+def _from_dict(cls: type, raw: Any) -> Any:
+    """Build a frozen dataclass from a partial dict.
+
+    Strategy: walk fields, if a field is itself a dataclass and the input
+    has a matching dict, recurse. Otherwise coerce + clamp. Defaults fill
+    any missing field.
+    """
+    if not is_dataclass(cls):
+        raise TypeError(f"{cls!r} is not a dataclass")
+
+    instance_defaults = cls()  # type: ignore[call-arg]
+    if not isinstance(raw, dict):
+        return instance_defaults
+
+    kwargs: dict[str, Any] = {}
+    for f in fields(cls):
+        provided = raw.get(f.name)
+        default = getattr(instance_defaults, f.name)
+        if isinstance(f.type, type) and is_dataclass(f.type):
+            kwargs[f.name] = _from_dict(f.type, provided) if provided is not None else default
+            continue
+        # nested dataclass detection through the actual default type
+        if is_dataclass(default):
+            kwargs[f.name] = (
+                _from_dict(type(default), provided) if provided is not None else default
+            )
+            continue
+        kwargs[f.name] = _coerce_scalar(f.name, provided, default)
+    return cls(**kwargs)
+
+
+def _coerce_scalar(name: str, raw: Any, default: Any) -> Any:
+    if raw is None:
+        return default
+    if isinstance(default, bool):
+        return bool(raw)
+    if isinstance(default, int):
+        try:
+            int_value = int(raw)
+        except (TypeError, ValueError):
+            return default
+        return _clamp_int(name, int_value, default)
+    if isinstance(default, float):
+        try:
+            float_value = float(raw)
+        except (TypeError, ValueError):
+            return default
+        return _clamp_float(name, float_value, default)
+    if isinstance(default, str):
+        str_value = str(raw)
+        if name == "boundary" and str_value not in _BOUNDARIES:
+            return default
+        return str_value
+    return raw
+
+
+def _clamp_int(name: str, value: int, default: int) -> int:
+    if name.endswith("budget"):
+        return max(_MIN_BUDGET, min(_MAX_BUDGET, value))
+    if name == "iterations":
+        return max(_MIN_DEDUP_ITER, min(_MAX_DEDUP_ITER, value))
+    if name == "min_chunk_chars":
+        return max(_MIN_CHUNK_CHARS, min(_MAX_CHUNK_CHARS, value))
+    if name == "max_chunk_chars":
+        return max(_MIN_CHUNK_CHARS, min(_MAX_CHUNK_CHARS, value))
+    return max(0, value)
+
+
+def _clamp_float(name: str, value: float, default: float) -> float:
+    if name == "overlap_ratio":
+        return max(_MIN_OVERLAP, min(_MAX_OVERLAP, value))
+    return value
+
+
+__all__ = [
+    "AuditSettings",
+    "ChunkingSettings",
+    "DedupSettings",
+    "MemorySettings",
+    "MergeSettings",
+    "ReferenceSettings",
+    "UpdateSettings",
+    "load_memory_settings",
+    "memory_settings_dict",
+    "save_memory_settings",
+]

deeptutor/services/memory/snapshot/__init__.py

@@ -0,0 +1,127 @@
+"""Workspace snapshot subsystem for L1 memory.
+
+Public API:
+
+- :func:`read_snapshot` — current entities for a surface (no I/O on
+  ``state.json`` / ``changes.jsonl``).
+- :func:`read_stamps` — the same set without content: identity, label,
+  fingerprint, timestamp. What the diff and "what happened lately" both
+  actually need.
+- :func:`refresh_snapshot` — re-read workspace, diff against last
+  persisted state, append changes, persist new state. Returns the
+  computed change list. Idempotent: a no-change refresh writes nothing
+  to the changes log.
+- :func:`read_changes` — paginated history of past refreshes for one
+  surface (git-log-style display source).
+- :func:`current_state` — the persisted ``state.json`` for a surface
+  (consolidator uses ``last_refresh`` to gate L2 updates).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Iterable, Sequence
+
+from deeptutor.services.memory.paths import Surface
+from deeptutor.services.memory.snapshot import adapters, store
+from deeptutor.services.memory.snapshot.diff import diff_snapshots
+from deeptutor.services.memory.snapshot.entity import (
+    ChangeEntry,
+    Entity,
+    EntityStamp,
+    Stamped,
+)
+
+
+def read_snapshot(surface: Surface) -> list[Entity]:
+    return adapters.read_entities(surface)
+
+
+def read_stamps(surface: Surface) -> list[EntityStamp]:
+    """Diff-shaped view of a surface, without its content. See :mod:`.adapters`."""
+    return adapters.read_stamps(surface)
+
+
+def pending_changes(
+    surface: Surface, entities: Sequence[Stamped] | None = None
+) -> list[ChangeEntry]:
+    """Compute the diff between current workspace and the last persisted state.
+
+    Pure / read-only: never writes to ``state.json`` or ``changes.jsonl``.
+    Used by the L1 view to show "what would a refresh capture right now".
+
+    Accepts anything carrying ``id`` / ``label`` / ``fingerprint`` — a caller
+    that already has full :class:`Entity` objects (the L1 view, which needs
+    their content anyway) passes those; a caller that only needs the diff gets
+    the cheap :class:`EntityStamp` path by passing nothing.
+    """
+    stamps: Sequence[Stamped] = entities if entities is not None else adapters.read_stamps(surface)
+    curr_fp = {e.id: e.fingerprint for e in stamps}
+    curr_labels = {e.id: e.label for e in stamps}
+
+    prev = store.load_state(surface)
+    prev_fp = prev.get("fingerprints") or {}
+    prev_labels = prev.get("labels") or {}
+
+    return diff_snapshots(
+        prev_fp,
+        curr_fp,
+        label_map=curr_labels,
+        prev_label_map=prev_labels,
+    )
+
+
+def refresh_snapshot(surface: Surface) -> list[ChangeEntry]:
+    """Reconcile persisted state with the workspace and log what moved.
+
+    Runs on stamps, not full entities: a refresh only ever writes fingerprints
+    and labels, so reading message bodies here would be pure waste.
+    """
+    stamps = adapters.read_stamps(surface)
+    changes = pending_changes(surface, stamps)
+    curr_fp = {e.id: e.fingerprint for e in stamps}
+    curr_labels = {e.id: e.label for e in stamps}
+    store.append_changes(surface, changes)
+    store.save_state(
+        surface,
+        fingerprints=curr_fp,
+        labels=curr_labels,
+        last_refresh=datetime.now(tz=timezone.utc).isoformat(),
+    )
+    return changes
+
+
+def read_changes(surface: Surface, *, limit: int = 200, offset: int = 0) -> list[ChangeEntry]:
+    bound = max(1, min(limit, 1000))
+    all_changes: list[ChangeEntry] = list(store.iter_changes(surface))
+    # Most recent first — the file is append-order, reverse it.
+    all_changes.reverse()
+    return all_changes[offset : offset + bound]
+
+
+def current_state(surface: Surface) -> dict:
+    return store.load_state(surface)
+
+
+def clear_changes(surface: Surface) -> None:
+    store.clear_changes(surface)
+
+
+__all__ = [
+    "Entity",
+    "EntityStamp",
+    "Stamped",
+    "ChangeEntry",
+    "read_snapshot",
+    "read_stamps",
+    "pending_changes",
+    "refresh_snapshot",
+    "read_changes",
+    "current_state",
+    "clear_changes",
+    "adapters",
+]
+
+
+# Iterable kept for static-analyzer happiness when consumers import *.
+_: Iterable = []