typeclaw 0.36.8 → 0.37.0

package/src/bundled-plugins/memory/README.md

@@ -13,39 +13,64 @@ Auto-loaded by every TypeClaw agent. No `plugins[]` entry to add and no opt-out.
     "bufferBytes": 500000,
     "injectionBudgetBytes": 16384,
     "minIdleDeltaLines": 3,
-    "dreaming": { "schedule": "*/30 * * * *" }
+    "dreaming": { "schedule": "*/30 * * * *" },
+    "vector": { "enabled": false }
   }
 }
 ```
 
-| Field                         | Default          | Effect                                                                                                                                                                                                                                         |
-| ----------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `memory.idleMs`               | `60000`          | Debounce window before `memory-logger` spawns after a prompt completes. Minimum `1000`.                                                                                                                                                        |
-| `memory.bufferBytes`          | `500000`         | Size-based ceiling: spawns `memory-logger` when the transcript grows by this many bytes since the last run. `0` disables. Minimum `10000` when non-zero.                                                                                       |
-| `memory.injectionBudgetBytes` | `16384`          | Total shard-body budget for direct-mode memory injection. Above this, `loadMemory` switches to index-mode (headings + metadata only) and the agent must call `memory_search` to fetch specific topics or recent stream events. Minimum `4096`. |
-| `memory.minIdleDeltaLines`    | `3`              | Minimum JSONL line growth since the last `memory-logger` run required to fire an idle spawn. Below this, the idle timer ticks but no spawn fires. `0` disables (legacy always-fire-on-idle behavior). Independent of `bufferBytes`.            |
-| `memory.dreaming.schedule`    | `"*/30 * * * *"` | Five-field cron expression for the dreaming subagent.                                                                                                                                                                                          |
+| Field                         | Default          | Effect                                                                                                                                                                                                                                                                                                                                                                                |
+| ----------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `memory.idleMs`               | `60000`          | Debounce window before `memory-logger` spawns after a prompt completes. Minimum `1000`.                                                                                                                                                                                                                                                                                               |
+| `memory.bufferBytes`          | `500000`         | Size-based ceiling: spawns `memory-logger` when the transcript grows by this many bytes since the last run. `0` disables. Minimum `10000` when non-zero.                                                                                                                                                                                                                              |
+| `memory.injectionBudgetBytes` | `16384`          | Total shard-body budget for direct-mode memory injection. Above this, `loadMemory` switches to index-mode (headings + metadata only) and the agent must call `memory_search` to fetch specific topics or recent stream events. Minimum `4096`.                                                                                                                                        |
+| `memory.minIdleDeltaLines`    | `3`              | Minimum JSONL line growth since the last `memory-logger` run required to fire an idle spawn. Below this, the idle timer ticks but no spawn fires. `0` disables (legacy always-fire-on-idle behavior). Independent of `bufferBytes`.                                                                                                                                                   |
+| `memory.dreaming.schedule`    | `"*/30 * * * *"` | Five-field cron expression for the dreaming subagent.                                                                                                                                                                                                                                                                                                                                 |
+| `memory.vector.enabled`       | `false`          | Master switch for per-turn vector memory. When `true`, the `# Memory` system-prompt section is suppressed and memory is injected per turn into the user prompt via `hybridSearch` (over budget) or direct shards (under budget); the host downloads the embedding model and mounts it into the container. When `false`, memory lives in the system prompt and no model is downloaded. |
 
-All fields are **restart-required** — the plugin reads them once at boot.
+All fields are **restart-required** — the plugin reads them once at boot. The `memory` block is plugin-owned config that passes through core's schema (`.catchall`), so it is **outside the core `FIELD_EFFECTS` reload fence**: changing any `memory.*` field and running `typeclaw reload` silently no-ops with no "restart-required" warning. Restart the container to apply.
 
 ## What it contributes
 
-| Kind     | Name                       | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| -------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Subagent | `memory-logger`            | Reads a parent transcript past a watermark and appends fragments to `memory/streams/<today>.jsonl`. Coalesced per `agentDir`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| Subagent | `dreaming`                 | Reads shards under `memory/topics/` plus undreamed daily-stream events and rebalances the topic shards. Coalesced per `agentDir`. Citation-superset invariant enforced on every run.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
-| Subagent | `memory-retrieval`         | On `session.turn.start` when injection plan is `index` mode, reads the user's actual prompt for this turn + shard listing, writes a focused summary to `memory/.retrieval-cache/<sessionId>.md`. Coalesced per `parentSessionId`. Declares `profile: 'fast'` (retrieval is "≤3 keyword searches + 1 write", no reasoning required) and `timeoutMs: 30_000` so a wedged provider call releases the coalescing key instead of poisoning the cache for every subsequent turn.                                                                                                                                                                                                                                                            |
-| Tool     | `memory_search`            | Main-agent tool. Substring/regex search across BOTH topic shards (slugs, frontmatter, bodies) and undreamed daily-stream events (fragment topic/body, legacy prose). Plain queries are phrase-first: the whole query is tried as one substring, and if that finds nothing the query is split on whitespace and the distinct words are OR-matched, ranked by how many words each hit contains (regex queries never fall back). Results are discriminated by `source: "topic" \| "stream"`; exact-phrase (and regex) results list topics first, then streams newest-first, while word-fallback results are ranked by matched-word count with that order as the tiebreak (so a higher-scoring stream can precede a lower-scoring topic). |
-| Tool     | `delete_topic_shard`       | Subagent-only (dreaming). Deletes a topic shard at `memory/topics/<slug>.md`. Path-guarded.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| Cron     | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
-| Hook     | `session.idle`             | Per-session debouncer with size-based ceiling. Spawns `memory-logger` on idle or buffer-trip.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| Hook     | `session.end`              | Spawns `memory-logger` immediately; also unlinks the retrieval-cache file for this session.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| Hook     | `session.turn.start`       | When `buildInjectionPlan` returns `mode: 'index'` and origin is not a subagent, spawns `memory-retrieval` (detached) with the turn's `userPrompt` so the cache reflects the user's current question, not the assembling system prompt. Fire-and-forget; failures route through the plugin logger.                                                                                                                                                                                                                                                                                                                                                                                                                                     |
+| Kind     | Name                       | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| -------- | -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Subagent | `memory-logger`            | Reads a parent transcript past a watermark and appends fragments to `memory/streams/<today>.jsonl`. Coalesced per `agentDir`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| Subagent | `dreaming`                 | Reads shards under `memory/topics/` plus undreamed daily-stream events and rebalances the topic shards. Coalesced per `agentDir`. Citation-superset invariant enforced on every run.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| Subagent | `memory-retrieval`         | On `session.turn.start` when injection plan is `index` mode, reads the user's actual prompt for this turn + shard listing, writes a focused summary to `memory/.retrieval-cache/<sessionId>.md`. Coalesced per `parentSessionId`. Declares `profile: 'fast'` (retrieval is "≤3 keyword searches + 1 write", no reasoning required) and `timeoutMs: 30_000` so a wedged provider call releases the coalescing key instead of poisoning the cache for every subsequent turn.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
+| Tool     | `memory_search`            | Main-agent tool. Pass `query` to search, or `topic` to look up one shard by exact slug (not both). Search is substring/regex across BOTH topic shards (slugs, frontmatter, bodies) and undreamed daily-stream events (fragment topic/body, legacy prose). Plain queries are phrase-first: the whole query is tried as one substring, and if that finds nothing the query is split on whitespace and the distinct words are OR-matched, ranked by how many words each hit contains (regex queries never fall back). Results are discriminated by `source: "topic" \| "stream"`; exact-phrase (and regex) results list topics first, then streams newest-first, while word-fallback results are ranked by matched-word count with that order as the tiebreak (so a higher-scoring stream can precede a lower-scoring topic). The `topic` mode reads `memory/topics/<slug>.md` directly via `loadShard` and returns that one shard with its full body — no fuzzy search — so a channel agent can open a topic whose slug the per-turn injection already showed it, instead of re-searching for a body the retrieval layer already located. When no topic shard matches the slug it falls back to a reference of the same slug (`memory/references/<slug>.md`), because the per-turn injection renders topic and reference entries with the same `slug:` line and the same `memory_search({ topic: "<slug>" })` recovery hint — so a reference slug surfaced in channel memory must resolve here, not dead-end. Bad slugs (path traversal) come back as a structured error; a slug matching neither a topic nor a reference is an empty match set. |
+| Tool     | `delete_topic_shard`       | Subagent-only (dreaming). Deletes a topic shard at `memory/topics/<slug>.md`. Path-guarded.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| Cron     | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| Hook     | `session.idle`             | Per-session debouncer with size-based ceiling. Spawns `memory-logger` on idle or buffer-trip.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| Hook     | `session.end`              | Spawns `memory-logger` immediately; also unlinks the retrieval-cache file for this session.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| Hook     | `session.turn.start`       | **Vector on:** renders the per-turn user-prompt memory block into `event.retrievalContext.results` (direct-mode shards under budget, de-duplicated across turns per session; top-K `hybridSearch` over budget; channel origins always index/headings-only); runs for every origin that supplies a `retrievalContext`, including subagents. **Vector off:** when `buildInjectionPlan` returns `mode: 'index'` and origin is not a subagent, spawns `memory-retrieval` (detached) with the turn's `userPrompt` so the cache reflects the user's current question. Fire-and-forget; failures route through the plugin logger.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
 
 ## Memory injection (two-tier, topic shards only)
 
 Default budget is 16 KB. Direct mode when shard bytes sum ≤ budget: all shard bodies are injected verbatim. Index mode when sum > budget: only heading + `cites=N, days=N, lastReinforced=YYYY-MM-DD` per shard, plus a directive for the agent to call `memory_search` to fetch specific topics or recent stream events.
 
+**Where the section is injected depends on `memory.vector.enabled`:**
+
+- **Vector off (default): system prompt.** `loadMemory` renders the `# Memory` section once at session creation; it lives in the cacheable system-prompt prefix (ordered last, since it's the most volatile block). The two-tier direct/index split above applies, and the index-mode `memory-retrieval` subagent (below) covers the over-budget case lag-by-one.
+- **Vector on: user prompt, per turn.** The system-prompt `# Memory` section is suppressed entirely (`createSession`'s `suppressSystemMemory`, derived once at boot in `src/run/index.ts`). The `session.turn.start` hook instead renders memory into `event.retrievalContext.results`, which the turn-drivers append to the user message. Under budget it injects shard bodies in **direct mode**, **de-duplicated across turns** (see below); over budget it runs `hybridSearch` and injects the **top-K** most relevant shards/fragments for the current message. The direct path renders via `renderDedupedMemorySection` and the index path via `renderRetrievedMemorySection` (`load-memory.ts`). This keeps the system-prompt cache prefix stable across a session and lets retrieval track the live topic instead of a stale session-start snapshot.
+
+  **Cross-turn dedup (direct mode, non-channel).** Re-sending every shard body verbatim on every turn made a long conversation pay the same memory block dozens of times. The plugin keeps a per-session `slug → body-hash` record (`turn-dedup.ts`, `partitionDirectShards`) and, after a shard's full body has been injected once this session, replaces it on later turns with a one-line `slug:` reference plus a `memory_search({ topic: "<slug>" })` recovery hint. A shard whose body **changed** (a dreaming pass rewrote it) re-injects in full so the agent never reasons over a stale body. The "nothing the agent always had vanishes on an off-topic turn" guarantee is preserved by **availability, not literal presence**: every topic is still named (heading + slug) every turn, and the full body is one `memory_search` away. The per-session record is cleared on `session.end` (before the subagent-origin early-return, so subagent sessions don't leak it), so a resurrected session re-injects bodies in full rather than dangling a reference to text the rebuilt context no longer holds.
+
+  For **channel** origins, the per-turn render injects topic **headings only** — the excerpt body is stripped and the agent fetches it on demand via `memory_search`. A channel **direct-mode** turn is force-indexed via `forceIndexForChannel` in `renderVectorTurnMemory` to a headings/slugs section over **every** shard (not run through `hybridSearch`), so a full body can never reach a channel turn AND no heading is silently dropped by relevance-filtering or a stale vector index. Only channel turns that are genuinely **over budget** fall to `hybridSearch` top-K, where headings-only injection is the established channel policy regardless. This mirrors `forceIndexForChannel`'s direct-path policy that channel sessions never carry bodies: a topic heading is a self-contained belief sentence (the dreaming subagent writes exactly one), so the heading alone lets the model decide whether the topic is worth opening, while the ~10-shard excerpt block — which dominated the per-turn injection and was usually abandoned unread — no longer rides in the (uncacheable) user turn on every message. Non-channel origins (TUI/direct) keep the full excerpt on first injection, where the extra retrieval round-trip isn't worth it.
+
+  Each stripped topic is injected with its `slug:` line so the agent can re-fetch the exact shard with `memory_search({ topic: "<slug>" })` — a direct slug lookup, not a second fuzzy search over the heading text the retrieval layer already matched. Undreamed `source: "stream"` items have no topic shard yet (no slug to look up), so they instead carry a `memory_search({ query: ... })` recovery hint; the channel directive names both paths so every stripped entry — topic or stream — has an explicit way back to its full text.
+
+The invariant `suppressSystemMemory === memory.vector.enabled` is load-bearing: a session must never carry memory in both the system prompt and the user turn.
+
+### Vector retrieval is parent-child, not a flat pool
+
+When `memory.vector.enabled` is true, `hybridSearch` does **not** rank topic vectors and stream-fragment vectors together in one flat cosine pool. It uses a parent-child collapse so the result set is always topic-level current truth:
+
+- **Match surface.** The query matches against topic-belief vectors AND fragment vectors (both lanes, fused by RRF). Fragments are the fine-grained retrieval hooks; topics are the returned unit. The keyword lane is **phrase-first with a token-OR fallback** (mirroring `memory_search`): since `hybridSearch`'s query is the whole user prompt — which never appears verbatim in a shard — a phrase-only lane would return nothing every turn and silently degrade RRF to vector-only. The fallback (`searchAllRanked`) also ranks by matched-token-count and truncates after ranking, giving RRF a real second ranker instead of alphabetical topic order.
+- **Collapse to parent.** A matched fragment resolves to the topic that cites it (via the shard's `fragments:` list — the citation IS the parent foreign key) and contributes its score to that topic. It never appears as a standalone result. An **undreamed** fragment (no topic cites it yet) resolves to itself, preserving the ~30-min freshness window before dreaming consolidates it.
+- **MAX-child ranking.** A collapsed topic takes the **maximum** of its members' RRF scores, never the sum. Sum would over-rank often-revised topics purely for accumulating more citations to match (the PARADE result: max beats sum when relevance is concentrated).
+- **Dreamed-and-cited fragment vectors are pruned every dreaming pass.** Once a fragment is folded into a topic (dreamed AND cited), its standalone `stream:*` row is redundant — any match on it collapses to the parent topic, whose `topic:*` row is already a candidate. But the redundant row still consumes one of `store.query`'s finite `topK * 2` pre-fusion slots by raw cosine, so a topic cited on many days would otherwise hoard proportionally more slots and displace distinct topics — silently defeating the MAX-child anti-popularity design over a long container uptime. `deleteRedundantDreamedCitedStreamVectors` (`dreaming.ts`) removes these per pass; it is the same deletion startup `pruneStaleRows` performs (a dreamed-and-cited fragment leaves the undreamed passage set), advanced from per-restart to per-pass. Undreamed fragment rows are kept — they resolve to themselves and ARE the freshness window.
+- **Superseded fragments are excluded from the match surface.** When dreaming overturns a belief on contradiction, the old fragment id moves from `fragments:` to a `superseded:` list (see below). Superseded ids stay cited — so the citation-superset GC invariant keeps the fragment alive and history is auditable — but `passages.ts` does not embed them, so a stale "uses bun" fragment can never resurface as a hook for the current "uses pnpm" belief. `parseCitations` stays section-blind so GC and frontmatter recompute still see both sections; `splitCitationsBySection` is the status-aware view the retrieval layer reads (`parent-link.ts`).
+
 **Undreamed daily-stream events are NOT injected into the system prompt.** They are reachable only via `memory_search`, which discriminates results by `source: "topic" | "stream"`. The agent now decides per-query whether recent observations are relevant, instead of carrying every undreamed fragment in the cached prompt prefix. Three reasons this is the right shape:
 
 1. PR #314 made `memory_search` cover the stream surface, so the duplicate copy in the system prompt no longer earns its bytes.
@@ -70,6 +95,18 @@ The subagent uses these signals to:
 
 There is no `## Historical observations` bucket. Demoted topics live as their own shards; injection-time filtering (the index/direct split) handles the prompt-budget pressure.
 
+## Muscle memory (three forms)
+
+While reading streams, the dreaming subagent watches for **repeated multi-step procedures** the user has guided the main agent through, and codifies them. There are three forms, picked smallest-that-fits (top to bottom, stop at the first match):
+
+1. **Form C — plugin suggestion.** The procedure needs a runtime hook (`session.prompt` / `tool.before` / etc.), a custom tool, a cron job, or a subagent — things only a plugin can express. Recorded as a topic shard with a `proposal: plugin packages/<name>` line.
+2. **Form B — CLI suggestion.** The procedure boils down to "run this small script with these args." Recorded as a topic shard with a `proposal: cli packages/<name>` line.
+3. **Form A — skill.** The default, and where most procedures land. The procedure can be done with the tools the agent already has. Written directly to `memory/skills/<name>/SKILL.md` and auto-loaded as a first-class skill by the next session.
+
+The split exists because the forms have different write boundaries. The dreaming subagent can write skills directly (Form A) — `memory/skills/` is inside its sandbox — but it can only **suggest** CLIs and plugins (Forms B and C), because those live under `packages/`, outside its write sandbox. A suggestion is a passive recommendation: the topic shard sits in long-term memory, and the main agent scaffolds the package only when a current user request asks for the matching procedure. CLI/plugin proposal shards are the one exception to the one-sentence belief format (see [Memory saturation](#memory-saturation)) — they keep a rationale paragraph plus the `proposal:` line that names the form and package, and carry the same fragment citations as any other shard.
+
+The bar for codifying is identical across all three forms: the procedure is multi-step, has recurred (≥2 distinct fragments, ideally across different days), has a clearly statable trigger, and generalizes beyond the specific instances. No speculative skills or suggestions — anything the main agent never reaches for is dead weight in the prompt budget it reads on every prompt. A given CLI or plugin is suggested **once**; future dreaming runs leave the existing proposal shard alone unless new fragments show the procedure has shifted shape (e.g. a CLI that now needs a runtime hook, upgrading the proposal from `cli` to `plugin`).
+
 ## Citation-superset safety net
 
 `checkCitationSupersetAcrossShards` checks that the union of fragment ids cited in NEW shards is a superset of the union cited in OLD shards. Violation triggers:
@@ -83,13 +120,31 @@ A `[dreaming] citation-superset violation: …` warning logs the dropped ids and
 
 ## Files on disk
 
-- **`memory/topics/<slug>.md`** — per-topic shards with YAML frontmatter (`heading`, `cites`, `days`, `lastReinforced`, `tags?`) + body markdown. Runtime owns the frontmatter (recomputed after every dreaming run from the body's citations); dreaming subagent writes body only.
+- **`memory/topics/<slug>.md`** — per-topic shards with YAML frontmatter (`heading`, `cites`, `days`, `lastReinforced`, `tags?`) + body markdown. Runtime owns the frontmatter (recomputed after every dreaming run from the body's citations); dreaming subagent writes body only. The body is a **compact belief record**: one belief sentence stating the current truth (subject + predicate + essential scope qualifier; the strength wording — "mentioned" / "consistently" / "always" — is calibrated from `days`), followed by the `fragments:` list (active evidence behind the belief) and an optional `superseded:` list (evidence overturned by a later contradiction — kept cited for GC/history but excluded from vector retrieval). One sentence (not a paragraph) keeps bodies small so more topics stay under the direct-injection budget; the citation lists are the only load-bearing part of the body, so the prose shape is free to be terse. Existing verbose shards are rewritten into this form by the next dreaming run (no migration).
 - **`memory/streams/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`. Force-committed alongside the shards.
 - **`memory/MEMORY.md.pre-shard.bak`** — legacy pre-shard backup left by older TypeClaw versions. Safe to delete after verifying.
-- **`memory/skills/<name>/SKILL.md`** — muscle memory. Skills the dreaming subagent distills from repeated procedures. Auto-loaded as first-class skills.
+- **`memory/skills/<name>/SKILL.md`** — muscle memory (Form A). Skills the dreaming subagent distills from repeated procedures. Auto-loaded as first-class skills. CLI/plugin suggestions (Forms B and C) are recorded as topic shards instead — see [Muscle memory](#muscle-memory-three-forms).
+- **`memory/references/<slug>.md`** — verbatim reference artifacts (gated behind `memory.references.enabled`). Written by `store_reference`. When vector is enabled, a freshly stored reference is embedded into the vector index immediately via the `store_reference` on-write hook (`vector/reference-index-on-write.ts`), mirroring the stream-fragment on-write hook — so it is vector-retrievable within the same container uptime instead of only after the next startup index build. Chunks are derived by the same `referencePassagesForOne` the startup build uses, and re-storing a shorter body prunes the stale higher-index chunk rows.
 - **`memory/.dreaming-state.json`** — per-day dreamed-id sets.
 - **`memory/.retrieval-cache/<sessionId>.md`** — ephemeral retrieval summaries. Written by `memory-retrieval`, read by `loadMemory` on the next prompt of the same session, unlinked on `session.end`.
 
+## Observability
+
+The plugin emits structured `[plugin:memory]` log lines (no separate metrics infra). The load-bearing per-run signals:
+
+- **`[memory-logger] <session> done fragments_written=N elapsed_ms=…`** — how many fragments a logger run captured (delta of fragment events in today's stream).
+- **`[dreaming] done topics_created=N topics_removed=N superseded_new=N fragments_dropped=N over_budget=N elapsed_ms=…`** — consolidation activity per run: new/removed topic shards (by snapshot path diff), net citations moved into `superseded:`, fragments GC'd, and `over_budget` (the count of topic shards whose embeddable text exceeds the model token budget — see [Embedding token budget](#embedding-token-budget)). `topics_created` + reinforcement is the "is memory getting sharper" signal; `superseded_new` is contradiction-edit volume; a persistently non-zero `over_budget` means the subagent isn't compacting the flagged shards. Derived by `computeDreamingMetrics` from the pre/post shard snapshots.
+- **`[vector-retrieval] mode=index topic_results=N stream_results=N reference_results=N elapsed_ms=… [suppressed=1]`** (or `mode=direct topics=N`) — per-turn retrieval breakdown. `stream_results` counts undreamed-fragment hits that self-resolved (no citing topic yet) — the freshness-window usage signal that informs whether the undreamed surface earns its keep. `reference_results` counts verbatim-reference hits (only non-zero when `references.enabled`), broken out from `stream_results` so the two surfaces are distinguishable. `elapsed_ms` times the full `hybridSearch` (embed + cosine scan + keyword lane + RRF fusion) — the on-critical-path retrieval cost. `suppressed=1` appears when the query matched nothing after the relevance gate, so an empty memory block is distinguishable from a missing one.
+
+These are intentionally verbose for now so behavior is observable in logs; trim once the useful subset is known.
+
+## Embedding token budget
+
+The embedding model (`Xenova/multilingual-e5-base`) caps inputs at **512 tokens**; the transformers.js tokenizer truncates anything longer by default, silently dropping the tail. Canonical compact shards (heading + one belief sentence first) sit well under the cap, but legacy verbose shards, legacy prose migration events, long fragments, and very long user queries can exceed it. Handled in two layers (`vector/truncation.ts`):
+
+1. **Deterministic bound at embed time (primary, always on).** `embed()` calls `boundEmbeddableText` on every input before the tokenizer sees it, trimming to a character budget derived from the same conservative per-script token estimate (CJK counted 1:1, Latin ~3.5 chars/token; the tokenizer is never loaded, preserving the embedder's lazy-load). The cut is now explicit and owned, not a hidden tokenizer side effect, and the leading heading/belief sentence — the load-bearing retrieval signal — always survives because it comes first. This guarantees no silent loss even for inputs nothing ever rewrites (raw legacy prose, long queries).
+2. **Compaction routed to dreaming (remediation, vector-on only).** Bounding keeps retrieval correct but the bounded tail is still not embedded. The real fix is to make the shard fit. On each run the dreaming handler computes which topic shards are over budget (measuring `topicPassage(...).text` — the same citation-stripped string the embedder bounds — so the flag matches what is actually truncated) and renders an "Over the embedding budget" table into the subagent's prompt, alongside the strength-signal table. Dreaming rule 8 instructs it to compact those shards into the one-belief-sentence form **while preserving every `fragments:`/`superseded:` citation** — shrinking prose, never dropping a citation (a dropped citation would trip the citation-superset revert, leaving the shard both reverted and still over budget). The signal is gated on the vector index (`memory/.vectors/index.db`) existing: with vector off, nothing embeds these shards, so the budget doesn't apply and the table is suppressed. The `over_budget=N` count rides the `[dreaming] done` line.
+
 ## How `session.idle` works
 
 Core fires `session.idle` immediately after every `session.prompt()` completion. The plugin owns the debounce: a `Map<sessionId, Timeout>` reset on every event. When the timer fires, the plugin spawns `memory-logger` for that session — unless the min-delta gate suppresses the spawn (see below).
@@ -114,7 +169,9 @@ Each `memory-logger` spawn captures the line count of `memory/streams/<today>.js
 
 ## Tests
 
-Test files in this directory (kebab-case, `.test.ts` neighbors): `paths`, `slug`, `frontmatter`, `topics`, `shard-snapshot`, `delete-tool`, `citations`, `citation-superset`, `load-shards`, `load-memory`, `injection-plan`, `search-tool`, `memory-retrieval`, `memory-logger`, `dreaming`, `index`, `integration`. Plus guard policies in `../guard/policies/`: `memory-topics-delete`, `memory-topics-write`, `memory-retrieval-cache-write`.
+Tests live next to the code they cover as `.test.ts` neighbors; `vector/*.test.ts` covers the vector subsystem (store, embedder, hybrid search, relevance gate, startup/on-write indexing, doctor/inspect). The core surfaces are `paths`, `slug`, `frontmatter`, `topics`, `shard-snapshot`, `delete-tool`, `citations`, `citation-superset`, `load-shards`, `load-memory`, `injection-plan`, `turn-dedup`, `search-tool`, `memory-retrieval`, `memory-logger`, `dreaming`, `index`, `index-vector`, `index-vector-retrieval`, `integration` (not exhaustive — `git ls-files 'src/bundled-plugins/memory/**/*.test.ts'` is the source of truth). Plus guard policies in `../guard/policies/`: `memory-topics-delete`, `memory-topics-write`, `memory-retrieval-cache-write`.
+
+Both vector hook tests build the plugin through `createMemoryPluginForTests({ hybridSearch?, queryEmbedFn? })` — a per-instance dependency seam (the factory defaults to the real implementations) so a fake is scoped to one boot and never leaks across the suite. `index-vector` injects a fake `hybridSearch` to cover the budget-gating, channel force-index, and cross-turn dedup branches of the `session.turn.start` hook. `index-vector-retrieval` injects only a synthetic `queryEmbedFn` (so the ~279 MB model never loads) and runs the REAL `hybridSearch` → `VectorStore.query` → RRF → `renderRetrievedMemorySection` pipeline through the same hook against a real seeded vector store; the query token appears nowhere in the corpus, so a rendered topic proves the vector lane matched rather than the substring `memory_search` lane. Relevance-gate suppression is covered separately in `vector/hybrid` and `vector/relevance-gate`.
 
 ## Notes from before the plugin existed
 

package/src/bundled-plugins/memory/append-tool.ts

@@ -11,63 +11,84 @@ import { streamFilePath } from './paths'
 import { detectSecrets } from './secret-detector'
 import { newEventId, timestampFromId } from './stream-events'
 import type { FragmentEvent, WatermarkEvent } from './stream-events'
-import { appendEvents, readEvents } from './stream-io'
+import { appendEvents, readEvents, type FragmentsAppendedContext } from './stream-io'
 
-export const appendTool = defineTool({
-  description:
-    "Append a memory fragment to today's JSONL daily stream and advance the watermark. The runtime serializes your call into a JSON line and chooses the filename — do not emit raw JSON and do not pass a path. `topic`/`body` are the fragment's substance; `source` is the parent session id; `entry` is the transcript-entry-id this fragment anchors to; `latestEntryId` is the latest transcript-entry-id you evaluated in this run (advances the watermark, may equal `entry` or be later). Refuses content with recognized credential patterns and refuses byte-equivalent topic+body within the same daily stream.",
-  parameters: z.object({
-    topic: z.string().min(1),
-    body: z.string().min(1),
-    source: z.string().min(1),
-    entry: z.string().min(1),
-    latestEntryId: z.string().min(1),
-  }),
-  async execute({ topic, body, source, entry, latestEntryId }, ctx) {
-    const streamPath = dailyStreamPath(ctx.agentDir)
-    assertNoSecrets(`${topic}\n${body}`)
-
-    const hash = fragmentContentHash({ topic, body })
-    const events = await readEvents(streamPath)
-    const duplicate = events
-      .filter((event) => event.type === 'fragment')
-      .find((event) => fragmentContentHash(event) === hash)
-    if (duplicate !== undefined) {
-      throw new Error(
-        `Refusing to append: fragment "${duplicate.topic}" already exists in ${streamPath} with byte-equivalent content. ` +
-          `The dreaming subagent will see the existing fragment; do not write it again. If the new occurrence ` +
-          `is genuinely informative, write a fragment that says so explicitly rather than restating the original.`,
-      )
-    }
+export type FragmentsAppendedHook = (fragments: FragmentEvent[], context: FragmentsAppendedContext) => Promise<void>
 
-    const fragmentId = newEventId()
-    const watermarkId = newEventId()
-    const fragment: FragmentEvent = {
-      type: 'fragment',
-      id: fragmentId,
-      ts: timestampFromId(fragmentId),
-      source,
-      entry,
-      topic,
-      body,
-    }
-    const watermark: WatermarkEvent = {
-      type: 'watermark',
-      id: watermarkId,
-      ts: timestampFromId(watermarkId),
-      source,
-      entry: latestEntryId,
-    }
+export function createAppendTool(onFragmentsAppended?: FragmentsAppendedHook) {
+  return defineTool({
+    description:
+      "Append a memory fragment to today's JSONL daily stream and advance the watermark. The runtime serializes your call into a JSON line and chooses the filename — do not emit raw JSON and do not pass a path. `topic`/`body` are the fragment's substance; `source` is the parent session id; `entry` is the transcript-entry-id this fragment anchors to; `latestEntryId` is the latest transcript-entry-id you evaluated in this run (advances the watermark, may equal `entry` or be later). Refuses content with recognized credential patterns and refuses byte-equivalent topic+body within the same daily stream.",
+    parameters: z.object({
+      topic: z.string().min(1),
+      body: z.string().min(1),
+      source: z.string().min(1),
+      entry: z.string().min(1),
+      latestEntryId: z.string().min(1),
+      references: z.array(z.string()).optional(),
+    }),
+    async execute({ topic, body, source, entry, latestEntryId, references }, ctx) {
+      const streamPath = dailyStreamPath(ctx.agentDir)
+      assertNoSecrets(`${topic}\n${body}`)
 
-    await mkdir(dirname(streamPath), { recursive: true })
-    await appendEvents(streamPath, [fragment, watermark])
+      const hash = fragmentContentHash({ topic, body })
+      const events = await readEvents(streamPath)
+      const duplicate = events
+        .filter((event) => event.type === 'fragment')
+        .find((event) => fragmentContentHash(event) === hash)
+      if (duplicate !== undefined) {
+        throw new Error(
+          `Refusing to append: fragment "${duplicate.topic}" already exists in ${streamPath} with byte-equivalent content. ` +
+            `The dreaming subagent will see the existing fragment; do not write it again. If the new occurrence ` +
+            `is genuinely informative, write a fragment that says so explicitly rather than restating the original.`,
+        )
+      }
 
-    return {
-      content: [{ type: 'text' as const, text: `Appended memory fragment and watermark to ${streamPath}` }],
-      details: { path: streamPath, fragmentId: fragment.id, watermarkId: watermark.id },
-    }
-  },
-})
+      const fragmentId = newEventId()
+      const watermarkId = newEventId()
+      const fragment: FragmentEvent = {
+        type: 'fragment',
+        id: fragmentId,
+        ts: timestampFromId(fragmentId),
+        source,
+        entry,
+        topic,
+        body,
+      }
+      if (references !== undefined && references.length > 0) {
+        fragment.references = references
+      }
+      const watermark: WatermarkEvent = {
+        type: 'watermark',
+        id: watermarkId,
+        ts: timestampFromId(watermarkId),
+        source,
+        entry: latestEntryId,
+      }
+
+      await mkdir(dirname(streamPath), { recursive: true })
+      await appendEvents(
+        streamPath,
+        [fragment, watermark],
+        onFragmentsAppended,
+        onFragmentsAppended
+          ? (err) => {
+              ctx.logger?.warn(
+                `[memory] post-append vector hook failed: ${err instanceof Error ? err.message : String(err)}`,
+              )
+            }
+          : undefined,
+      )
+
+      return {
+        content: [{ type: 'text' as const, text: `Appended memory fragment and watermark to ${streamPath}` }],
+        details: { path: streamPath, fragmentId: fragment.id, watermarkId: watermark.id },
+      }
+    },
+  })
+}
+
+export const appendTool = createAppendTool()
 
 export const advanceWatermarkTool = defineTool({
   description:

package/src/bundled-plugins/memory/citation-superset.ts

@@ -20,6 +20,10 @@
 // to honor that — especially across hundreds of runs over months — so the
 // mechanical check is the safety floor.
 //
+// Reference citations (`references/<slug>`) use a different format and are
+// intentionally excluded from this check — they are not fragment ids and are
+// not subject to the citation-superset GC invariant.
+//
 // Detection only. The handler decides what to do with the verdict (revert
 // memory/topics/ to its pre-run bytes, skip daily-stream compaction, still
 // advance the dreamed-id set so we do not loop on the same fragments).

package/src/bundled-plugins/memory/citations.ts

@@ -54,3 +54,57 @@ export function parseCitations(text: string): Map<string, Set<string>> {
 export function isCitationLine(line: string): boolean {
   return CITATION_LINE.test(line)
 }
+
+// Drops `fragments:`/`superseded:` headings and citation lines, leaving only the
+// belief prose. The embedding input must exclude them: mean-pooling a body of one
+// belief sentence + dozens of `streams/<date>#<uuidv7>` lines dilutes the belief
+// and pulls every topic vector toward the shared citation-list structure. The
+// citations stay in the on-disk body (the load-bearing parent-child links); only
+// the text handed to the embedder is stripped.
+export function stripCitationLines(body: string): string {
+  const kept = body.split('\n').filter((line) => !isCitationLine(line) && !SECTION_HEADING.test(line))
+  return collapseBlankRuns(kept).join('\n').trim()
+}
+
+function collapseBlankRuns(lines: string[]): string[] {
+  const out: string[] = []
+  let prevBlank = false
+  for (const line of lines) {
+    const blank = line.trim() === ''
+    if (blank && prevBlank) continue
+    out.push(line)
+    prevBlank = blank
+  }
+  return out
+}
+
+// Superseded citations stay cited (so the citation-superset GC invariant never
+// drops them) but must be excluded from retrieval, so a superseded "uses bun"
+// fragment can't surface as a hook for the current "uses pnpm" belief.
+// `parseCitations` stays section-blind for GC; this is the status-aware view.
+// Citations before any heading count as active (legacy shards had no section).
+const SECTION_HEADING = /^[\s-]*(fragments|superseded)\s*:\s*$/i
+
+export type SectionedCitations = { active: Set<string>; superseded: Set<string> }
+
+export function splitCitationsBySection(body: string): SectionedCitations {
+  const active = new Set<string>()
+  const superseded = new Set<string>()
+  let current: 'active' | 'superseded' = 'active'
+
+  for (const line of body.split('\n')) {
+    const heading = SECTION_HEADING.exec(line)
+    if (heading !== null) {
+      current = heading[1]!.toLowerCase() === 'superseded' ? 'superseded' : 'active'
+      continue
+    }
+    const citation = CITATION_LINE.exec(line)
+    if (citation === null) continue
+    ;(current === 'superseded' ? superseded : active).add(citation[3]!)
+  }
+
+  // Re-affirmed fact (appears in both sections across edits): active wins.
+  for (const id of active) superseded.delete(id)
+
+  return { active, superseded }
+}

package/src/bundled-plugins/memory/dreaming-metrics.ts

@@ -0,0 +1,30 @@
+import { splitCitationsBySection } from './citations'
+
+export type DreamingMetrics = {
+  topicsCreated: number
+  topicsRemoved: number
+  supersededDelta: number
+  referencesDemoted: number
+  referencesEvicted: number
+}
+
+// Snapshots are keyed by absolute shard path → file bytes (captureShardSnapshot).
+// supersededDelta is the net change in citations under `superseded:` across all
+// shards, i.e. how many fragments were overturned this run.
+export function computeDreamingMetrics(before: Map<string, Buffer>, after: Map<string, Buffer>): DreamingMetrics {
+  let topicsCreated = 0
+  for (const path of after.keys()) if (!before.has(path)) topicsCreated += 1
+
+  let topicsRemoved = 0
+  for (const path of before.keys()) if (!after.has(path)) topicsRemoved += 1
+
+  const supersededDelta = countSuperseded(after) - countSuperseded(before)
+
+  return { topicsCreated, topicsRemoved, supersededDelta, referencesDemoted: 0, referencesEvicted: 0 }
+}
+
+function countSuperseded(snapshot: Map<string, Buffer>): number {
+  let total = 0
+  for (const bytes of snapshot.values()) total += splitCitationsBySection(bytes.toString('utf8')).superseded.size
+  return total
+}