typeclaw 0.7.0 → 0.9.0

package/src/bundled-plugins/guard/policies/memory-retrieval-cache-write.ts

@@ -0,0 +1,37 @@
+import path from 'node:path'
+
+import type { SessionOrigin } from '@/agent/session-origin'
+import type { SecuritySeverity } from '@/bundled-plugins/security/permissions'
+
+export const GUARD_MEMORY_RETRIEVAL_CACHE_WRITE = 'memoryRetrievalCacheWrite'
+export const GUARD_MEMORY_RETRIEVAL_CACHE_WRITE_SEVERITY: SecuritySeverity = 'low'
+
+const SESSION_ID_REGEX = /^[A-Za-z0-9._-]{1,128}$/
+
+export async function isMemoryRetrievalCacheWriteAllowed(options: {
+  tool: string
+  args: Record<string, unknown>
+  agentDir: string
+  origin?: SessionOrigin
+}): Promise<boolean> {
+  const { tool, args, agentDir, origin } = options
+  if (tool !== 'write') return false
+  if (origin?.kind !== 'subagent' || origin.subagent !== 'memory-retrieval') return false
+
+  const rawPath = args.path
+  if (typeof rawPath !== 'string') return false
+
+  const targetPath = path.resolve(agentDir, rawPath)
+  const expectedDir = path.resolve(agentDir, 'memory', '.retrieval-cache')
+  const relative = path.relative(expectedDir, targetPath)
+  if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) return false
+
+  const parts = relative.split(path.sep).filter(Boolean)
+  if (parts.length !== 1) return false
+
+  const fileName = parts[0]!
+  if (!fileName.endsWith('.md')) return false
+
+  const sessionId = fileName.slice(0, -3)
+  return SESSION_ID_REGEX.test(sessionId)
+}

package/src/bundled-plugins/guard/policies/memory-topics-delete.ts

@@ -0,0 +1,67 @@
+import path from 'node:path'
+
+import type { SessionOrigin } from '@/agent/session-origin'
+import { SLUG_REGEX } from '@/bundled-plugins/memory/slug'
+import type { SecuritySeverity } from '@/bundled-plugins/security/permissions'
+
+import type { GuardBlock } from '../policy'
+
+export const GUARD_MEMORY_TOPICS_DELETE = 'memoryTopicsDelete'
+
+export const GUARD_MEMORY_TOPICS_DELETE_SEVERITY: SecuritySeverity = 'medium'
+
+export function checkMemoryTopicsDeleteGuard(options: {
+  tool: string
+  args: Record<string, unknown>
+  agentDir: string
+  origin?: SessionOrigin
+}): GuardBlock | undefined {
+  const { tool, args, agentDir, origin } = options
+
+  if (tool !== 'delete_topic_shard') return undefined
+
+  const rawPath = args.path
+  if (typeof rawPath !== 'string') {
+    return block(tool, 'path argument must be a string')
+  }
+
+  if (origin?.kind !== 'subagent' || origin.subagent !== 'dreaming') {
+    return block(tool, 'only the dreaming subagent may delete topic shards')
+  }
+
+  if (rawPath.includes('\\')) {
+    return block(tool, 'path must use POSIX separators under memory/topics/')
+  }
+
+  const targetPath = path.resolve(agentDir, rawPath)
+  const topicsDir = path.resolve(agentDir, 'memory', 'topics')
+  const relative = path.relative(topicsDir, targetPath)
+
+  if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) {
+    return block(tool, `path must be a direct child of memory/topics/: ${targetPath}`)
+  }
+
+  const parts = relative.split(path.sep).filter(Boolean)
+  if (parts.length !== 1) {
+    return block(tool, `path must be a single .md file inside memory/topics/: ${targetPath}`)
+  }
+
+  const fileName = parts[0]!
+  if (!fileName.endsWith('.md')) {
+    return block(tool, `path must be a single .md file inside memory/topics/: ${targetPath}`)
+  }
+
+  const slug = fileName.slice(0, -3)
+  if (!SLUG_REGEX.test(slug)) {
+    return block(tool, `slug must match ${SLUG_REGEX}: ${slug}`)
+  }
+
+  return undefined
+}
+
+function block(tool: string, reason: string): GuardBlock {
+  return {
+    block: true,
+    reason: `Guard \`${GUARD_MEMORY_TOPICS_DELETE}\` blocked ${tool}: ${reason}.`,
+  }
+}

package/src/bundled-plugins/guard/policies/memory-topics-write.ts

@@ -0,0 +1,33 @@
+import path from 'node:path'
+
+import type { SessionOrigin } from '@/agent/session-origin'
+import { SLUG_REGEX } from '@/bundled-plugins/memory/slug'
+
+export async function isMemoryTopicsWriteAllowed(options: {
+  tool: string
+  args: Record<string, unknown>
+  agentDir: string
+  origin?: SessionOrigin
+}): Promise<boolean> {
+  if (options.tool !== 'write') return false
+
+  const { origin } = options
+  if (!origin || origin.kind !== 'subagent' || origin.subagent !== 'dreaming') return false
+
+  const rawPath = options.args.path
+  if (typeof rawPath !== 'string') return false
+
+  const target = path.resolve(options.agentDir, rawPath)
+  const expectedDir = path.resolve(options.agentDir, 'memory', 'topics')
+  const rel = path.relative(expectedDir, target)
+  if (rel === '' || rel.startsWith('..') || path.isAbsolute(rel)) return false
+
+  const parts = rel.split(path.sep).filter(Boolean)
+  const fileName = parts[0]
+  if (parts.length !== 1 || !fileName || !fileName.endsWith('.md')) return false
+
+  const slug = fileName.slice(0, -3)
+  if (!SLUG_REGEX.test(slug)) return false
+
+  return true
+}

package/src/bundled-plugins/guard/policies/non-workspace-write.ts

@@ -1,7 +1,11 @@
 import { realpath } from 'node:fs/promises'
 import path from 'node:path'
 
+import type { SessionOrigin } from '@/agent/session-origin'
+
 import { ACKNOWLEDGE_GUARDS, type GuardBlock, isGuardAcknowledged } from '../policy'
+import { isMemoryRetrievalCacheWriteAllowed } from './memory-retrieval-cache-write'
+import { isMemoryTopicsWriteAllowed } from './memory-topics-write'
 import { isSkillAuthoringAllowed } from './skill-authoring'
 
 export const GUARD_NON_WORKSPACE_WRITE = 'nonWorkspaceWrite'
@@ -9,7 +13,6 @@ export const GUARD_NON_WORKSPACE_WRITE = 'nonWorkspaceWrite'
 const AGENT_ROOT_WRITE_ALLOWLIST = new Set([
   'AGENTS.md',
   'IDENTITY.md',
-  'MEMORY.md',
   'SOUL.md',
   'USER.md',
   'cron.json',
@@ -28,8 +31,9 @@ export async function checkNonWorkspaceWriteGuard(options: {
   tool: string
   args: Record<string, unknown>
   agentDir: string
+  origin?: SessionOrigin
 }): Promise<GuardBlock | undefined> {
-  const { tool, args, agentDir } = options
+  const { tool, args, agentDir, origin } = options
   if (tool !== 'write' && tool !== 'edit') return undefined
 
   const rawPath = args.path
@@ -42,6 +46,8 @@ export async function checkNonWorkspaceWriteGuard(options: {
     resolveRealIntendedPath(workspacePath),
   ])
   if (await isSkillAuthoringAllowed({ tool, args, agentDir })) return undefined
+  if (await isMemoryRetrievalCacheWriteAllowed({ tool, args, agentDir, origin })) return undefined
+  if (await isMemoryTopicsWriteAllowed({ tool, args, agentDir, origin })) return undefined
   if (await isAllowedAgentRootWrite(agentDir, targetPath, realTargetPath)) return undefined
   if (isInside(realWorkspacePath, realTargetPath)) return undefined
   if (isGuardAcknowledged(args, GUARD_NON_WORKSPACE_WRITE)) return undefined

package/src/bundled-plugins/guard/policy.ts

@@ -16,4 +16,11 @@ export {
   checkSkillAuthoringGuard,
   isSkillAuthoringAllowed,
 } from './policies/skill-authoring'
+export { GUARD_MEMORY_TOPICS_DELETE, checkMemoryTopicsDeleteGuard } from './policies/memory-topics-delete'
+export {
+  GUARD_MEMORY_RETRIEVAL_CACHE_WRITE,
+  GUARD_MEMORY_RETRIEVAL_CACHE_WRITE_SEVERITY,
+  isMemoryRetrievalCacheWriteAllowed,
+} from './policies/memory-retrieval-cache-write'
+export { isMemoryTopicsWriteAllowed } from './policies/memory-topics-write'
 export { GUARD_UNCOMMITTED_CHANGES, checkUncommittedChangesAdvice } from './policies/uncommitted-changes'

package/src/bundled-plugins/memory/README.md

@@ -1,8 +1,8 @@
 # typeclaw-plugin-memory
 
-The bundled memory plugin. Owns `MEMORY.md` (long-term memory) and `memory/yyyy-MM-dd.jsonl` (daily streams) plus the two subagents that write them: `memory-logger` and `dreaming`.
+The bundled memory plugin. Owns `memory/topics/` (sharded long-term memory) and `memory/streams/yyyy-MM-dd.jsonl` (daily fragment streams) plus three subagents that read and write them: `memory-logger`, `dreaming`, `memory-retrieval`.
 
-This plugin is **auto-loaded** by every TypeClaw agent. There is no `plugins[]` entry to add and no opt-out. To configure it, add a `memory` block to `typeclaw.json`.
+Auto-loaded by every TypeClaw agent. No `plugins[]` entry to add and no opt-out. Configure via the `memory` block in `typeclaw.json`.
 
 ## Config
 
@@ -11,101 +11,115 @@ This plugin is **auto-loaded** by every TypeClaw agent. There is no `plugins[]`
   "memory": {
     "idleMs": 60000,
     "bufferBytes": 500000,
+    "injectionBudgetBytes": 16384,
     "dreaming": { "schedule": "*/30 * * * *" }
   }
 }
 ```
 
-| Field                      | Default            | Effect                                                                                                                                                                                                                                                                                                                                                             |
-| -------------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `memory.idleMs`            | `60000`            | Debounce window before `memory-logger` spawns after a prompt completes. Minimum `1000`. Default bumped from `10000` to `60000` to reduce spawn churn during conversational sessions where the agent goes idle for short periods between rapid back-and-forth turns.                                                                                                |
-| `memory.bufferBytes`       | `500000`           | Size-based ceiling: spawns `memory-logger` when the transcript grows by this many bytes since the last run, even during continuous activity. `0` disables. Minimum `10000` when non-zero. Default bumped from `100000` to `500000` so a single conversational session stays within one memory-logger run unless it grows past ~half a megabyte of transcript.      |
-| `memory.dreaming`          | `{}` (cron job on) | Dreaming cron job is always registered. Override `schedule` to change when it fires.                                                                                                                                                                                                                                                                               |
-| `memory.dreaming.schedule` | `"*/30 * * * *"`   | Five-field cron expression. Defaults to every 30 minutes; fires short-circuit with zero LLM cost when nothing sits past the watermark, so frequent no-op fires are cheap and let sporadic agents still consolidate while alive (`src/cron/scheduler.ts` has no catchup for missed fires). Second-level schedules are rejected to avoid noisy no-op dreaming loops. |
+| Field                         | Default          | Effect                                                                                                                                                                                                                                         |
+| ----------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `memory.idleMs`               | `60000`          | Debounce window before `memory-logger` spawns after a prompt completes. Minimum `1000`.                                                                                                                                                        |
+| `memory.bufferBytes`          | `500000`         | Size-based ceiling: spawns `memory-logger` when the transcript grows by this many bytes since the last run. `0` disables. Minimum `10000` when non-zero.                                                                                       |
+| `memory.injectionBudgetBytes` | `16384`          | Total shard-body budget for direct-mode memory injection. Above this, `loadMemory` switches to index-mode (headings + metadata only) and the agent must call `memory_search` to fetch specific topics or recent stream events. Minimum `4096`. |
+| `memory.dreaming.schedule`    | `"*/30 * * * *"` | Five-field cron expression for the dreaming subagent.                                                                                                                                                                                          |
 
 All fields are **restart-required** — the plugin reads them once at boot.
 
 ## What it contributes
 
-| Kind     | Name                       | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| -------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Subagent | `memory-logger`            | Reads a parent transcript past a watermark and appends fragments to `memory/<today>.jsonl`. Coalesced per `agentDir`; the plugin chains spawn calls onto a per-agent Promise so two concurrent channel sessions never race on the same daily stream file.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| Subagent | `dreaming`                 | Reads `MEMORY.md` plus undreamed daily-stream events, **rebalances** the existing topics using per-topic strength signals (citation count, distinct days, recency) injected into its user prompt, rewrites `MEMORY.md` with `memory/yyyy-MM-dd#<fragment-id>` citations, optionally writes muscle-memory skills under `memory/skills/<name>/SKILL.md`, advances the per-day dreamed-id set, **compacts daily streams** by dropping superseded watermarks and dreamed-but-uncited fragments, then commits the result with a summary message (`dream: <summary> <emoji>`, e.g. `dream: 3 fragments + new skill 'pr-review' 🔮`). Coalesced per `agentDir`. The runtime enforces a **citation-superset invariant** on every rewrite: a new MEMORY.md that drops any previously-cited fragment id is reverted to its pre-run bytes (dreamed-ids still advance so the run is not retried in a loop). |
-| Cron job | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| Hook     | `session.idle`             | Per-session debouncer with size-based ceiling. Resets a `setTimeout(idleMs)` on every event; on fire, calls `ctx.spawnSubagent('memory-logger', ...)`. Also `fs.stat`s the transcript on every event and spawns immediately when growth since the last run reaches `bufferBytes`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| Hook     | `session.end`              | Cancels the debounce timer and immediately spawns `memory-logger` (so the final transcript is captured even when the user disconnects right away).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| Kind     | Name                       | Notes                                                                                                                                                                                                                                                                                             |
+| -------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Subagent | `memory-logger`            | Reads a parent transcript past a watermark and appends fragments to `memory/streams/<today>.jsonl`. Coalesced per `agentDir`.                                                                                                                                                                     |
+| Subagent | `dreaming`                 | Reads shards under `memory/topics/` plus undreamed daily-stream events and rebalances the topic shards. Coalesced per `agentDir`. Citation-superset invariant enforced on every run.                                                                                                              |
+| Subagent | `memory-retrieval`         | On `session.turn.start` when injection plan is `index` mode, reads the user's actual prompt for this turn + shard listing, writes a focused summary to `memory/.retrieval-cache/<sessionId>.md`. Coalesced per `parentSessionId`.                                                                 |
+| Tool     | `memory_search`            | Main-agent tool. Substring/regex search across BOTH topic shards (slugs, frontmatter, bodies) and undreamed daily-stream events (fragment topic/body, legacy prose). Results are discriminated by `source: "topic" \| "stream"`; topics come first, then streams newest-first.                    |
+| Tool     | `delete_topic_shard`       | Subagent-only (dreaming). Deletes a topic shard at `memory/topics/<slug>.md`. Path-guarded.                                                                                                                                                                                                       |
+| Cron     | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.                                                                                                                                                                                                               |
+| Hook     | `session.idle`             | Per-session debouncer with size-based ceiling. Spawns `memory-logger` on idle or buffer-trip.                                                                                                                                                                                                     |
+| Hook     | `session.end`              | Spawns `memory-logger` immediately; also unlinks the retrieval-cache file for this session.                                                                                                                                                                                                       |
+| Hook     | `session.turn.start`       | When `buildInjectionPlan` returns `mode: 'index'` and origin is not a subagent, spawns `memory-retrieval` (detached) with the turn's `userPrompt` so the cache reflects the user's current question, not the assembling system prompt. Fire-and-forget; failures route through the plugin logger. |
 
-## Memory injection
+## Memory injection (two-tier, topic shards only)
 
-The rendered `# Memory` section (MEMORY.md + undreamed daily-stream tails) is injected into every session's system prompt by core (`src/agent/index.ts` `createResourceLoader` → `loadMemory`), **not** by a plugin hook. It is appended as the last block of the system prompt, after `gitNudge`, so the most-volatile content (daily streams that grow after every memory-logger fire) sits at the bottom of the cache-suffix region. This way a memory change only invalidates the memory section itself rather than everything downstream of it.
+Default budget is 16 KB. Direct mode when shard bytes sum ≤ budget: all shard bodies are injected verbatim. Index mode when sum > budget: only heading + `cites=N, days=N, lastReinforced=YYYY-MM-DD` per shard, plus a directive for the agent to call `memory_search` to fetch specific topics or recent stream events.
 
-## Memory saturation (LTP/LTD analogue)
+**Undreamed daily-stream events are NOT injected into the system prompt.** They are reachable only via `memory_search`, which discriminates results by `source: "topic" | "stream"`. The agent now decides per-query whether recent observations are relevant, instead of carrying every undreamed fragment in the cached prompt prefix. Three reasons this is the right shape:
 
-MEMORY.md is read into every session's system prompt, so its size is the prompt budget for everything else. Without a saturation policy it grows monotonically — every consolidated topic survives forever and citations accumulate across days. The dreaming subagent therefore treats MEMORY.md like human long-term memory: **repetition strengthens, lack of repetition saturates**.
+1. PR #314 made `memory_search` cover the stream surface, so the duplicate copy in the system prompt no longer earns its bytes.
+2. Streams grow unboundedly with usage (~360 KB at 30 days in the typical case, more under heavy use). The previous per-file 12 KB cap silently dropped each day's tail with no signal to the agent; on-demand search returns the relevant slice instead of "the first 12 KB by date".
+3. Streams sat inside the cached system-prompt prefix and appended new fragments on every memory-logger run, breaking cache reuse across prompts. Without injection, the prefix is stable until topic shards change.
 
-### How
+**Channel-origin always uses index mode regardless of total shard size** — defends against memory bleed into channel responses (the agent treats injected memory as instructions when channel users see it).
 
-On every run the runtime computes per-topic strength signals from MEMORY.md's existing citations — `cites` (total), `days` (distinct calendar days those citations span), `last reinforced` (most recent citation date), `age (d)` (whole days since `last reinforced`). The numbers are derived by `src/bundled-plugins/memory/strength.ts` and rendered as a table at the top of the dreaming subagent's user prompt. There is no sidecar file, no schema version, no migration — strength is recomputed on every run from MEMORY.md alone.
+When index mode is active, the `memory-retrieval` subagent fires on `session.turn.start` — the hook that brackets every actual `session.prompt(text)` call with the user's literal text — reads that user prompt, decides what's relevant across BOTH topic shards and undreamed stream events, pulls them via `memory_search`/`read`, and writes a focused ≤8 KB summary to `memory/.retrieval-cache/<sessionId>.md`. The NEXT `loadMemory` call for the same session reads and appends that cache file (lag-by-one-prompt). The cache file is unlinked on `session.end`.
 
-The subagent uses these numbers to:
+The hook trigger matters: `SessionPromptEvent.prompt` (`session.prompt`) carries the assembling system prompt (`basePrompt + IDENTITY + SOUL`) at session-creation time, NOT the user's message. Reading that field as if it were the user's prompt — which this plugin did before PR #340 — caused the retrieval subagent to keyword-mine TypeClaw's own framing prose (`TypeClaw`, `subagent`, `AGENTS.md`, `systemPromptLeak`, etc.) on every session. `session.turn.start` is the correct hook for "what is the user asking right now."
 
-1. **Promote strong topics.** `days = 1` → tentative ("the user mentioned"). `days >= 3` → confident ("the user consistently"). `days >= 7` → declarative ("the user always"). Promotion is gated on distinct days, not raw citation count — five citations on one day is one debugging session, five citations across five days is a recurring pattern.
-2. **Merge near-duplicates.** Topics that overlap in subject matter get folded into one, with the merged topic's `fragments:` list as the **union** of the source topics' fragment ids.
-3. **Demote decayed topics.** A topic with `cites = 1, days = 1, age >= 30` (or `cites <= 3, days <= 2, age >= 60`) routes into a `## Historical observations` bucket as a one-line bullet. The fact is preserved in the summary, the citation is preserved (so daily-stream GC keeps the underlying fragment), but the bytes shrink from a full topic+paragraph+citation-list to one line. Strong topics (`days >= 3`) are never demoted.
+## Memory saturation
 
-**There is no hard-deletion path** in this iteration. The historical bucket grows monotonically; the subagent is explicitly told not to attempt quarter-summary collapses because the safety net (below) would revert them. If the bucket becomes inconveniently long in practice, a future runtime change will provide a structured drop mechanism — until then every demoted citation stays alive forever via its one-line bullet.
+The dreaming subagent treats topic shards like human long-term memory: **repetition strengthens, lack of repetition saturates**. On every run the runtime computes per-shard strength signals from each shard's frontmatter (`cites`, `days`, `lastReinforced`) and renders them as a table at the top of the dreaming subagent's user prompt.
 
-### The citation-superset safety net
+The subagent uses these signals to:
 
-After every dreaming run that rewrote MEMORY.md, `src/bundled-plugins/memory/citation-superset.ts` checks that the union of fragment ids cited in the NEW file is a superset of the union cited in the OLD file. If any previously-cited id is missing from the rewrite, the runtime:
+1. **Promote strong topics.** `days = 1` → tentative ("the user mentioned"). `days >= 3` → confident ("the user consistently"). `days >= 7` → declarative ("the user always"). Promotion is gated on distinct days, not raw citation count.
+2. **Merge near-duplicates.** Topics that overlap get folded into one. The merged topic's citation set is the union.
+3. **Demote decayed topics.** A weak/decayed topic stays as its own shard but the body should be trimmed to a single line. When index-mode injection is in effect, demoted shards' bodies don't enter the system prompt at all — the index plus `memory_search` retrieval cover them on demand.
 
-1. Restores MEMORY.md to its pre-run bytes via `writeFile(memoryFilePath, memoryTextBefore)`. The pre-run bytes are captured **before** `runSession` so the revert always has a clean source.
-2. Skips daily-stream fragment GC for this run (no fragments are dropped).
-3. Advances the dreamed-id set anyway — the **conscious anti-loop tradeoff**: this means the run's NEW undreamed fragments are orphaned (they survive in the daily JSONL forever, force-committed, but will not be re-shown to a future dreaming run and therefore never make it into MEMORY.md). The alternative (don't advance) would infinite-loop if the LLM keeps making the same mistake on the same inputs. The orphaned fragments are recoverable from git history (`git log memory/`) by a human operator.
-4. Logs a `[dreaming] citation-superset violation: …` warning naming the dropped ids and explicitly stating the orphaning tradeoff.
+There is no `## Historical observations` bucket. Demoted topics live as their own shards; injection-time filtering (the index/direct split) handles the prompt-budget pressure.
 
-**Revert-write failure path.** If the `writeFile` in step 1 itself throws (disk full, EACCES, MEMORY.md replaced by a directory by a buggy subagent, etc.), MEMORY.md is in an unknown state. The runtime then:
+## Citation-superset safety net
 
-- Skips the dreamed-id advance (so the next run gets a second chance at the same input).
-- Skips compaction (so no fragments are GC'd against an ambiguous citation set).
-- Skips the commit (so a known-bad on-disk state is not snapshotted).
-- Logs a `[dreaming] citation-superset violation AND revert failed: …` ERROR with the recovery command (`git checkout -- MEMORY.md && typeclaw restart`).
+`checkCitationSupersetAcrossShards` checks that the union of fragment ids cited in NEW shards is a superset of the union cited in OLD shards. Violation triggers:
 
-The check exists because the daily-stream GC in `compactDailyStreams` drops any fragment that is `dreamedIds ∧ ¬citedIds`. Citations in MEMORY.md are the only thing that keeps a fragment alive past its first dreaming run — an omitted id means the underlying fragment would be permanently deleted on the next compaction.
+1. `restoreShardSnapshot` restores every pre-run shard to byte-identical bytes AND deletes any new shards created during the run.
+2. Daily-stream fragment GC is skipped.
+3. Dreamed-ids ADVANCE anyway — the **conscious anti-loop tradeoff**: orphaned fragments survive in the daily JSONL (force-committed) but won't be re-shown to a future dreaming run. The alternative (don't advance) would infinite-loop if the LLM keeps making the same mistake.
+4. The commit is skipped.
+
+A `[dreaming] citation-superset violation: …` warning logs the dropped ids and explicitly names the orphaning tradeoff.
 
 ## Files on disk
 
-- **`MEMORY.md`** — long-term memory. Created by the dreaming subagent on first run if absent. Force-committed by the runtime; `skip-worktree` flag is set so the human's `git status` stays clean.
-- **`memory/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`, lossy-preserving one-shot migration from older `.md` streams. Appended to by `memory-logger`. Created on demand. Gitignored at the agent's level but force-committed alongside `MEMORY.md` after each dreaming run.
-- **`memory/skills/<name>/SKILL.md`** — _muscle memory_. Skills the dreaming subagent distills from repeated procedures it sees in daily streams. Auto-discovered as first-class skills by `createResourceLoader`, and force-committed under the same `memory/` snapshot path as the daily streams. Written or refined with the standard `write` / `edit` tools; the bundled guard plugin enforces the exact `memory/skills/<name>/SKILL.md` path shape, single-segment kebab/snake-case names, matching frontmatter, and symlink/path-traversal safety. There is no runtime skill-delete tool; outright deletion of muscle-memory skills remains a user decision.
-- **`memory/.dreaming-state.json`** — per-day **dreamed-id sets**: which stream-event ids the dreaming subagent has already reasoned over. Plain JSON, schema version `2`. The next dreaming run reads only fragments whose id is NOT in the set. On malformed input or a version mismatch (including legacy `version: 1` line-count files from before the id-based switch), the plugin fails open with empty state — one extra dreaming run re-reads each day, then the file is stable.
+- **`memory/topics/<slug>.md`** — per-topic shards with YAML frontmatter (`heading`, `cites`, `days`, `lastReinforced`, `tags?`) + body markdown. Runtime owns the frontmatter (recomputed after every dreaming run from the body's citations); dreaming subagent writes body only.
+- **`memory/streams/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`. Force-committed alongside the shards.
+- **`memory/MEMORY.md.pre-shard.bak`** — one-shot pre-migration backup created by the boot migration. Safe to delete after verifying.
+- **`memory/skills/<name>/SKILL.md`** — muscle memory. Skills the dreaming subagent distills from repeated procedures. Auto-loaded as first-class skills.
+- **`memory/.dreaming-state.json`** — per-day dreamed-id sets.
+- **`memory/.retrieval-cache/<sessionId>.md`** — ephemeral retrieval summaries. Written by `memory-retrieval`, read by `loadMemory` on the next prompt of the same session, unlinked on `session.end`.
 
-`typeclaw init` does **not** scaffold these files. They appear when needed.
+## One-shot boot migration
 
-## How `session.idle` works
+When the plugin boots against an agent folder with a root `MEMORY.md` and no `memory/topics/`, it runs `runShardingMigration`. Steps:
 
-Core fires `session.idle` immediately after every `session.prompt()` completion (success or error). The plugin owns the debounce: it keeps a `Map<sessionId, Timeout>` and resets the timer on every event. When the timer fires, the plugin spawns `memory-logger` for that session.
+1. Detect prerequisites.
+2. Reset `memory/.migrating/` if a previous run crashed mid-flight.
+3. Run the legacy `.md → .jsonl` daily-stream migration (existing behavior).
+4. Parse `MEMORY.md` via `parseTopicsWithBodies`.
+5. **Stage topic shards** in `memory/.migrating/topics/` (originals untouched).
+6. **Stage streams** by COPY (not rename) to `memory/.migrating/streams/`.
+7. Stage pre-shard backup by COPY.
+8. **Verify staging** via `checkCitationSupersetAcrossShards`. On failure, abort and KEEP `memory/.migrating/` for human inspection. Originals untouched.
+9. **Atomic finalization**: rename three dirs (`topics`, `streams`, `.pre-shard.bak`), then unlink originals.
 
-If the user starts a new prompt before the timer fires, the next `session.idle` event resets the timer. If the user disconnects, `session.end` cancels the timer and fires `memory-logger` immediately so the final transcript is captured.
+Crash-recovery branches at boot: stale `memory/.migrating/` with no `topics/` → cleanup + retry; leftover `memory/.migrating/` alongside complete `topics/` → cleanup only; orphan root `MEMORY.md` or `memory/<date>.jsonl` alongside the new layout → delete orphans.
 
-In channel sessions, the agent rarely goes idle long enough to trip the timer because new participant messages keep arriving. The size-based ceiling handles this: on every `session.idle` the plugin `fs.stat`s the transcript and compares against the size at the last memory-logger run. Once growth reaches `memory.bufferBytes`, the timer is cancelled and `memory-logger` spawns immediately. The watermark on the output side absorbs any over-firing — if a buffer-trip arrives on a transcript chunk that's all tool noise, `memory-logger` reads it, decides nothing is worth logging, advances the watermark, and exits.
+The migration is idempotent and crash-safe.
 
-## Migration notes (from before the plugin existed)
+## How `session.idle` works
+
+Core fires `session.idle` immediately after every `session.prompt()` completion. The plugin owns the debounce: a `Map<sessionId, Timeout>` reset on every event. When the timer fires, the plugin spawns `memory-logger` for that session.
 
-- `memory.idleMs` and `memory.dreaming.schedule` already existed in core's `typeclaw.json` schema. They moved into this plugin's `configSchema` verbatim. Existing agents continue to work with no config change.
-- `memory.dreaming.schedule` was previously live-reloadable. It is now **restart-required** because plugin config is read once at boot. To change the schedule, edit `typeclaw.json` and run `typeclaw restart`.
-- The cron job ID changed from `__internal_dreaming` to `__plugin_memory_dreaming`. Anything that referenced the old ID (custom dashboards, scripts) needs updating.
+If the user starts a new prompt before the timer fires, the next `session.idle` resets it. If the user disconnects, `session.end` cancels the timer and fires `memory-logger` immediately.
+
+In busy channel sessions the agent rarely goes idle long enough to trip the timer. The size-based ceiling handles this: on every `session.idle` the plugin `fs.stat`s the transcript and compares against the size at the last memory-logger run. Once growth reaches `memory.bufferBytes`, the timer is cancelled and `memory-logger` spawns immediately.
 
 ## Tests
 
-- `index.test.ts` — composition tests (config schema, hook wiring, debounce semantics, MEMORY.md auto-create).
-- `memory-logger.test.ts` — system prompt invariants, watermark handling.
-- `dreaming.test.ts` — orchestration, watermark advancement, git snapshot (including muscle-memory skill files), system prompt + tool-surface invariants, citation-superset safety net (revert on dropped id, dreamed-ids still advance, no-revert on legitimate merge, no-revert on first-ever run), saturation-prompt invariants (rebalance-every-run, promotion ladder, historical bucket, demotion thresholds, bucket overflow synthesis).
-- `dreaming-state.test.ts` — fail-open semantics on malformed state.
-- `watermark.test.ts` — marker parsing.
-- `append-tool.test.ts` — append-only semantics.
-- `src/bundled-plugins/guard/policies/skill-authoring.test.ts` — runtime skill authoring guard: path sandboxing, name validation, YAML frontmatter, and write/edit final-content validation.
-- `load-memory.test.ts` — memory section rendering, undreamed-tail filtering, watermark stripping.
-- `topics.test.ts` — citation-attributing parser (per-topic citation grouping for strength signals).
-- `strength.test.ts` — per-topic strength computation (distinct days, recency, age clamping) and markdown table rendering.
-- `citation-superset.test.ts` — the safety-net check (superset semantics, missing-id reporting, summary truncation).
+Test files in this directory (kebab-case, `.test.ts` neighbors): `paths`, `slug`, `frontmatter`, `topics`, `shard-snapshot`, `delete-tool`, `citations`, `citation-superset`, `migration`, `load-shards`, `load-memory`, `injection-plan`, `search-tool`, `memory-retrieval`, `memory-logger`, `dreaming`, `index`, `integration`. Plus guard policies in `../guard/policies/`: `memory-topics-delete`, `memory-topics-write`, `memory-retrieval-cache-write`.
+
+## Migration notes (from before the plugin existed)
+
+- `memory.idleMs` and `memory.dreaming.schedule` already existed in core's `typeclaw.json` schema and moved into this plugin's `configSchema` verbatim.
+- `memory.dreaming.schedule` is now **restart-required** because plugin config is read once at boot.
+- The cron job ID is `__plugin_memory_dreaming` (previously `__internal_dreaming`).

package/src/bundled-plugins/memory/append-tool.ts

@@ -1,5 +1,5 @@
 import { mkdir } from 'node:fs/promises'
-import { dirname, join } from 'node:path'
+import { dirname } from 'node:path'
 
 import { z } from 'zod'
 
@@ -7,6 +7,7 @@ import { defineTool } from '@/plugin'
 import { formatLocalDate } from '@/shared'
 
 import { fragmentContentHash } from './fragment-parser'
+import { streamFilePath } from './paths'
 import { detectSecrets } from './secret-detector'
 import { newEventId, timestampFromId } from './stream-events'
 import type { FragmentEvent, WatermarkEvent } from './stream-events'
@@ -97,7 +98,7 @@ export const advanceWatermarkTool = defineTool({
 })
 
 function dailyStreamPath(agentDir: string): string {
-  return join(agentDir, 'memory', `${formatLocalDate()}.jsonl`)
+  return streamFilePath(agentDir, formatLocalDate())
 }
 
 function assertNoSecrets(content: string): void {

package/src/bundled-plugins/memory/citation-superset.ts

@@ -1,13 +1,13 @@
-// Citation-superset safety net for the dreaming subagent's MEMORY.md
-// rewrite. After every dreaming run that touched MEMORY.md, we check that
-// the union of fragment ids cited in the NEW file is a superset of the
-// union cited in the OLD file. If any previously-cited id is missing from
+// Citation-superset safety net for the dreaming subagent's topic-shard
+// rewrite. After every dreaming run that touched memory/topics/, we check that
+// the union of fragment ids cited in the NEW shard set is a superset of the
+// union cited in the OLD shard set. If any previously-cited id is missing from
 // the rewrite, the rewrite is rejected.
 //
 // Why this exists: the daily-stream GC in compactDailyStreams drops any
-// fragment that is `dreamedIds ∧ ¬citedIds`. Citations in MEMORY.md are
+// fragment that is `dreamedIds ∧ ¬citedIds`. Citations in topic shards are
 // the only thing that keeps a fragment alive past its first dreaming run.
-// If the subagent rewrites MEMORY.md and accidentally omits a citation —
+// If the subagent rewrites a topic shard and accidentally omits a citation —
 // either by garbling a merged topic's fragments: list or by dropping a
 // topic entirely — the next compaction call permanently deletes the
 // underlying fragment from the daily JSONL. There is no recovery beyond
@@ -21,22 +21,34 @@
 // mechanical check is the safety floor.
 //
 // Detection only. The handler decides what to do with the verdict (revert
-// MEMORY.md to its pre-run bytes, skip daily-stream compaction, still
+// memory/topics/ to its pre-run bytes, skip daily-stream compaction, still
 // advance the dreamed-id set so we do not loop on the same fragments).
 
 import { parseCitations } from './citations'
 
 export type CitationSupersetVerdict = { ok: true } | { ok: false; missing: Array<{ date: string; fragmentId: string }> }
 
-// Compare the OLD MEMORY.md to the NEW MEMORY.md and report any
+// Compare the OLD shard content to the NEW shard content and report any
 // fragment id that the OLD cited and the NEW does not. Empty old text
 // (first-ever dreaming run, prior file missing) is treated as the empty
 // citation set — any new file passes by construction.
 export function checkCitationSuperset(oldText: string, newText: string): CitationSupersetVerdict {
-  const oldCitations = parseCitations(oldText)
+  return checkCitationIndexSuperset(buildCitationIndex(oldText), buildCitationIndex(newText))
+}
+
+export function checkCitationSupersetAcrossShards(
+  oldShards: Map<string, string>,
+  newShards: Map<string, string>,
+): CitationSupersetVerdict {
+  return checkCitationIndexSuperset(buildCitationIndexFromShards(oldShards), buildCitationIndexFromShards(newShards))
+}
+
+function checkCitationIndexSuperset(
+  oldCitations: Map<string, Set<string>>,
+  newCitations: Map<string, Set<string>>,
+): CitationSupersetVerdict {
   if (oldCitations.size === 0) return { ok: true }
 
-  const newCitations = parseCitations(newText)
   const missing: Array<{ date: string; fragmentId: string }> = []
 
   const dates = [...oldCitations.keys()].sort()
@@ -52,9 +64,35 @@ export function checkCitationSuperset(oldText: string, newText: string): Citatio
   return missing.length === 0 ? { ok: true } : { ok: false, missing }
 }
 
+function buildCitationIndex(text: string): Map<string, Set<string>> {
+  return parseCitations(text)
+}
+
+function buildCitationIndexFromShards(shards: Map<string, string>): Map<string, Set<string>> {
+  const index = new Map<string, Set<string>>()
+
+  for (const text of shards.values()) {
+    mergeCitationIndex(index, buildCitationIndex(text))
+  }
+
+  return index
+}
+
+function mergeCitationIndex(target: Map<string, Set<string>>, source: Map<string, Set<string>>): void {
+  for (const [date, ids] of source) {
+    let targetIds = target.get(date)
+    if (targetIds === undefined) {
+      targetIds = new Set<string>()
+      target.set(date, targetIds)
+    }
+
+    for (const id of ids) targetIds.add(id)
+  }
+}
+
 // Pretty-print the verdict's missing ids for log output. Keeps the line
 // short by reporting count + first N ids; the full list is reconstructable
-// from MEMORY.md's git history if forensics are ever needed.
+// from memory/topics/ git history if forensics are ever needed.
 export function summarizeMissingCitations(missing: ReadonlyArray<{ date: string; fragmentId: string }>): string {
   const total = missing.length
   const sample = missing.slice(0, 3).map((m) => `${m.date}#${m.fragmentId}`)

package/src/bundled-plugins/memory/citations.ts

@@ -1,4 +1,4 @@
-// Citation format: `memory/yyyy-MM-dd#<fragment-id>`. The id is the full
+// Citation format: `streams/yyyy-MM-dd#<fragment-id>`. The id is the full
 // UUIDv7 of the fragment event in the daily JSONL stream. The date prefix is
 // redundant with the id's timestamp (UUIDv7 encodes minting time in the first
 // 48 bits) but kept for human grep-ability — readers should be able to see
@@ -6,30 +6,41 @@
 //
 // The format does NOT accept line ranges. The prior `:43-45` shape is gone
 // (see the "drop backward compat" decision in the PR description). Parsing
-// silently ignores any line in MEMORY.md that doesn't match this exact shape,
+// silently ignores any line in a topic shard that doesn't match this exact shape,
 // so legacy citations from before the cutover are dropped — they no longer
 // pin fragments alive against compaction.
 
-const CITATION_LINE = /^[\s-]*memory\/(\d{4}-\d{2}-\d{2})#([\w-]+)\s*$/im
+export const CITATION_FORMAT_CANONICAL = 'streams' as const
+export const acceptedPrefixes = ['streams', 'memory'] as const
 
-const CITATION_LINE_GLOBAL = /memory\/(\d{4}-\d{2}-\d{2})#([\w-]+)/g
+// Single alternation keeps line and global parsing on the same transitional
+// prefix set while dropping the prefix from the public Citation shape.
+const CITATION_LINE = /^[\s-]*(streams|memory)\/(\d{4}-\d{2}-\d{2})#([\w-]+)\s*$/im
+
+const CITATION_LINE_GLOBAL = /(streams|memory)\/(\d{4}-\d{2}-\d{2})#([\w-]+)/g
+
+const LEGACY_CITATION_GLOBAL = /memory\/(\d{4}-\d{2}-\d{2})#([\w-]+)/g
 
 export type Citation = { date: string; fragmentId: string }
 
 export function formatCitation(date: string, fragmentId: string): string {
-  return `memory/${date}#${fragmentId}`
+  return `${CITATION_FORMAT_CANONICAL}/${date}#${fragmentId}`
+}
+
+export function normalizeCitation(citation: string): string {
+  return citation.replace(LEGACY_CITATION_GLOBAL, `${CITATION_FORMAT_CANONICAL}/$1#$2`)
 }
 
 // Parse every citation in `text` and return them grouped by date. The
 // returned Map is empty when no citations appear. Used by:
 //   - dreaming.ts compaction to decide which fragments are still referenced
-//     by MEMORY.md and must survive GC.
+//     by topic shards and must survive GC.
 //   - tests pinning the format.
 export function parseCitations(text: string): Map<string, Set<string>> {
   const out = new Map<string, Set<string>>()
   for (const match of text.matchAll(CITATION_LINE_GLOBAL)) {
-    const date = match[1]!
-    const fragmentId = match[2]!
+    const date = match[2]!
+    const fragmentId = match[3]!
     let set = out.get(date)
     if (set === undefined) {
       set = new Set<string>()