typeclaw 0.39.7 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/dump-system-prompt.ts +4 -50
- package/src/agent/auth-reloadable.ts +24 -0
- package/src/agent/auth.ts +12 -0
- package/src/agent/index.ts +18 -58
- package/src/agent/multimodal/look-at.ts +87 -0
- package/src/bundled-plugins/guard/policies/non-workspace-write.ts +0 -2
- package/src/bundled-plugins/guard/policy.ts +0 -5
- package/src/bundled-plugins/memory/README.md +26 -37
- package/src/bundled-plugins/memory/index.ts +32 -147
- package/src/bundled-plugins/memory/load-memory.ts +7 -121
- package/src/bundled-plugins/memory/memory-logger.ts +8 -2
- package/src/bundled-plugins/memory/search-command.ts +2 -11
- package/src/bundled-plugins/memory/vector/doctor.ts +1 -1
- package/src/bundled-plugins/memory/vector/embedder.ts +4 -5
- package/src/bundled-plugins/memory/vector/hybrid.ts +47 -5
- package/src/bundled-plugins/memory/vector/index-on-write.ts +25 -20
- package/src/bundled-plugins/memory/vector/reference-index-on-write.ts +28 -23
- package/src/bundled-plugins/memory/vector/relevance-gate.ts +11 -7
- package/src/bundled-plugins/memory/vector/script.ts +87 -0
- package/src/bundled-plugins/memory/vector/store.ts +2 -0
- package/src/bundled-plugins/planner/planner.ts +4 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +9 -1
- package/src/channels/adapters/describe-error.ts +40 -0
- package/src/channels/adapters/discord.ts +9 -12
- package/src/channels/adapters/kakaotalk-classify.ts +31 -6
- package/src/channels/adapters/kakaotalk.ts +15 -18
- package/src/channels/adapters/line.ts +11 -14
- package/src/channels/adapters/slack-bot.ts +15 -18
- package/src/channels/adapters/slack.ts +45 -21
- package/src/channels/adapters/telegram-bot.ts +15 -18
- package/src/channels/adapters/webex-bot.ts +13 -14
- package/src/channels/adapters/webex.ts +14 -15
- package/src/cli/compose.ts +20 -0
- package/src/cli/docker-preflight.ts +42 -0
- package/src/cli/hostd.ts +12 -3
- package/src/cli/init.ts +24 -22
- package/src/cli/logs.ts +7 -0
- package/src/cli/provider.ts +8 -1
- package/src/cli/reload.ts +6 -0
- package/src/cli/restart.ts +7 -0
- package/src/cli/shell.ts +7 -0
- package/src/cli/start.ts +7 -0
- package/src/cli/status.ts +9 -0
- package/src/cli/stop.ts +7 -0
- package/src/cli/tui.ts +6 -0
- package/src/config/config.ts +31 -8
- package/src/container/docker-app.ts +253 -0
- package/src/container/index.ts +11 -0
- package/src/container/start.ts +85 -41
- package/src/hostd/current-host-daemon.ts +21 -0
- package/src/hostd/daemon.ts +75 -15
- package/src/hostd/kakao-renewal-manager.ts +35 -10
- package/src/hostd/supervisor.ts +18 -3
- package/src/hostd/tailscale.ts +41 -5
- package/src/hostd/webex-renewal-manager.ts +13 -4
- package/src/plugin/hooks.ts +3 -4
- package/src/plugin/index.ts +1 -0
- package/src/plugin/manager.ts +14 -0
- package/src/plugin/registry.ts +12 -0
- package/src/plugin/types.ts +5 -1
- package/src/run/channel-session-factory.ts +3 -5
- package/src/run/index.ts +63 -36
- package/src/server/command-runner.ts +2 -11
- package/src/skills/typeclaw-memory/SKILL.md +2 -2
- package/src/skills/typeclaw-plugins/SKILL.md +1 -1
- package/src/test-helpers/rm-temp-dir.ts +24 -0
- package/typeclaw.schema.json +1 -23
- package/src/bundled-plugins/guard/policies/memory-retrieval-cache-write.ts +0 -43
- package/src/bundled-plugins/memory/memory-retrieval.ts +0 -111
- package/src/bundled-plugins/memory/vector/cache-write.ts +0 -19
- package/src/bundled-plugins/memory/vector/config.ts +0 -28
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# typeclaw-plugin-memory
|
|
2
2
|
|
|
3
|
-
The bundled memory plugin. Owns `memory/topics/` (sharded long-term memory) and `memory/streams/yyyy-MM-dd.jsonl` (daily fragment streams) plus
|
|
3
|
+
The bundled memory plugin. Owns `memory/topics/` (sharded long-term memory) and `memory/streams/yyyy-MM-dd.jsonl` (daily fragment streams) plus two subagents that read and write them: `memory-logger` and `dreaming`.
|
|
4
4
|
|
|
5
5
|
Auto-loaded by every TypeClaw agent. No `plugins[]` entry to add and no opt-out. Configure via the `memory` block in `typeclaw.json`.
|
|
6
6
|
|
|
@@ -13,20 +13,18 @@ Auto-loaded by every TypeClaw agent. No `plugins[]` entry to add and no opt-out.
|
|
|
13
13
|
"bufferBytes": 500000,
|
|
14
14
|
"injectionBudgetBytes": 16384,
|
|
15
15
|
"minIdleDeltaLines": 3,
|
|
16
|
-
"dreaming": { "schedule": "*/30 * * * *" }
|
|
17
|
-
"vector": { "enabled": false }
|
|
16
|
+
"dreaming": { "schedule": "*/30 * * * *" }
|
|
18
17
|
}
|
|
19
18
|
}
|
|
20
19
|
```
|
|
21
20
|
|
|
22
|
-
| Field | Default | Effect
|
|
23
|
-
| ----------------------------- | ---------------- |
|
|
24
|
-
| `memory.idleMs` | `60000` | Debounce window before `memory-logger` spawns after a prompt completes. Minimum `1000`.
|
|
25
|
-
| `memory.bufferBytes` | `500000` | Size-based ceiling: spawns `memory-logger` when the transcript grows by this many bytes since the last run. `0` disables. Minimum `10000` when non-zero.
|
|
26
|
-
| `memory.injectionBudgetBytes` | `16384` | Total shard-body budget for direct-mode memory injection. Above this,
|
|
27
|
-
| `memory.minIdleDeltaLines` | `3` | Minimum JSONL line growth since the last `memory-logger` run required to fire an idle spawn. Below this, the idle timer ticks but no spawn fires. `0` disables (legacy always-fire-on-idle behavior). Independent of `bufferBytes`.
|
|
28
|
-
| `memory.dreaming.schedule` | `"*/30 * * * *"` | Five-field cron expression for the dreaming subagent.
|
|
29
|
-
| `memory.vector.enabled` | `false` | Master switch for per-turn vector memory. When `true`, the `# Memory` system-prompt section is suppressed and memory is injected per turn into the user prompt via `hybridSearch` (over budget) or direct shards (under budget); the host downloads the embedding model and mounts it into the container. When `false`, memory lives in the system prompt and no model is downloaded. |
|
|
21
|
+
| Field | Default | Effect |
|
|
22
|
+
| ----------------------------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
23
|
+
| `memory.idleMs` | `60000` | Debounce window before `memory-logger` spawns after a prompt completes. Minimum `1000`. |
|
|
24
|
+
| `memory.bufferBytes` | `500000` | Size-based ceiling: spawns `memory-logger` when the transcript grows by this many bytes since the last run. `0` disables. Minimum `10000` when non-zero. |
|
|
25
|
+
| `memory.injectionBudgetBytes` | `16384` | Total shard-body budget for direct-mode memory injection. Above this, per-turn retrieval switches to index mode and uses top-K hybrid search; channel turns always receive headings only. Minimum `4096`. |
|
|
26
|
+
| `memory.minIdleDeltaLines` | `3` | Minimum JSONL line growth since the last `memory-logger` run required to fire an idle spawn. Below this, the idle timer ticks but no spawn fires. `0` disables (legacy always-fire-on-idle behavior). Independent of `bufferBytes`. |
|
|
27
|
+
| `memory.dreaming.schedule` | `"*/30 * * * *"` | Five-field cron expression for the dreaming subagent. |
|
|
30
28
|
|
|
31
29
|
All fields are **restart-required** — the plugin reads them once at boot. The `memory` block is plugin-owned config that passes through core's schema (`.catchall`), so it is **outside the core `FIELD_EFFECTS` reload fence**: changing any `memory.*` field and running `typeclaw reload` silently no-ops with no "restart-required" warning. Restart the container to apply.
|
|
32
30
|
|
|
@@ -36,30 +34,24 @@ All fields are **restart-required** — the plugin reads them once at boot. The
|
|
|
36
34
|
| -------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
37
35
|
| Subagent | `memory-logger` | Reads a parent transcript past a watermark and appends fragments to `memory/streams/<today>.jsonl`. Coalesced per `agentDir`. |
|
|
38
36
|
| Subagent | `dreaming` | Reads shards under `memory/topics/` plus undreamed daily-stream events and rebalances the topic shards. Coalesced per `agentDir`. Citation-superset invariant enforced on every run. |
|
|
39
|
-
| Subagent | `memory-retrieval` | On `session.turn.start` when injection plan is `index` mode, reads the user's actual prompt for this turn + shard listing, writes a focused summary to `memory/.retrieval-cache/<sessionId>.md`. Coalesced per `parentSessionId`. Declares `profile: 'fast'` (retrieval is "≤3 keyword searches + 1 write", no reasoning required) and `timeoutMs: 30_000` so a wedged provider call releases the coalescing key instead of poisoning the cache for every subsequent turn. |
|
|
40
37
|
| Tool | `memory_search` | Main-agent tool. Pass `query` to search, or `topic` to look up one shard by exact slug (not both). Search is substring/regex across BOTH topic shards (slugs, frontmatter, bodies) and undreamed daily-stream events (fragment topic/body, legacy prose). Plain queries are phrase-first: the whole query is tried as one substring, and if that finds nothing the query is split on whitespace and the distinct words are OR-matched, ranked by how many words each hit contains (regex queries never fall back). Results are discriminated by `source: "topic" \| "stream"`; exact-phrase (and regex) results list topics first, then streams newest-first, while word-fallback results are ranked by matched-word count with that order as the tiebreak (so a higher-scoring stream can precede a lower-scoring topic). The `topic` mode reads `memory/topics/<slug>.md` directly via `loadShard` and returns that one shard with its full body — no fuzzy search — so a channel agent can open a topic whose slug the per-turn injection already showed it, instead of re-searching for a body the retrieval layer already located. When no topic shard matches the slug it falls back to a reference of the same slug (`memory/references/<slug>.md`), because the per-turn injection renders topic and reference entries with the same `slug:` line and the same `memory_search({ topic: "<slug>" })` recovery hint — so a reference slug surfaced in channel memory must resolve here, not dead-end. Bad slugs (path traversal) come back as a structured error; a slug matching neither a topic nor a reference is an empty match set. The optional `where` param scopes a query to one room (`"#incidents"` or its raw chat id): it keeps only stream fragments whose captured `where` matches the room by raw `chat` id or `chatName` (case-insensitive, leading `#` optional) and — because room is fragment-only provenance — returns ONLY stream matches, excluding topic shards and references. |
|
|
41
38
|
| Tool | `delete_topic_shard` | Subagent-only (dreaming). Deletes a topic shard at `memory/topics/<slug>.md`. Path-guarded. |
|
|
42
39
|
| Cron | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`. |
|
|
43
40
|
| Hook | `session.idle` | Per-session debouncer with size-based ceiling. Spawns `memory-logger` on idle or buffer-trip. |
|
|
44
|
-
| Hook | `session.end` | Spawns `memory-logger` immediately
|
|
45
|
-
| Hook | `session.turn.start` |
|
|
41
|
+
| Hook | `session.end` | Spawns `memory-logger` immediately and clears per-session retrieval de-dup state. |
|
|
42
|
+
| Hook | `session.turn.start` | Renders the per-turn user-prompt memory block into `event.retrievalContext.results` (direct-mode shards under budget, de-duplicated across turns per session; top-K `hybridSearch` over budget; channel origins always index/headings-only); runs for every origin that supplies a `retrievalContext`, including user-delegated subagents (researcher, reviewer) — but NOT TypeClaw-owned infrastructure turns (`system` origins and `system`-spawned subagents memory-logger/backup, plus the dreaming cron job matched by its `__plugin_memory_dreaming` job id), whose `userPrompt` is a fixed instruction block, not a user message (see [Internal-infrastructure turns skip retrieval](#internal-infrastructure-turns-skip-retrieval)). |
|
|
46
43
|
|
|
47
44
|
## Memory injection (two-tier, topic shards only)
|
|
48
45
|
|
|
49
|
-
|
|
46
|
+
Memory is injected per turn into the user prompt; the system prompt never contains long-term memory; vector memory is always on. The `session.turn.start` hook renders memory into `event.retrievalContext.results`, which the turn-drivers append to the user message. Under budget it injects shard bodies in **direct mode**, **de-duplicated across turns** (see below); over budget it runs `hybridSearch` and injects the **top-K** most relevant shards/fragments/references for the current message. The direct path renders via `renderDedupedRetrievedMemorySection` and the index path via `renderRetrievedMemorySection` (`load-memory.ts`). This keeps the system-prompt cache prefix stable across a session and lets retrieval track the live topic instead of a stale session-start snapshot.
|
|
50
47
|
|
|
51
|
-
**
|
|
48
|
+
**Cross-turn dedup (direct mode, non-channel).** Re-sending every shard body verbatim on every turn made a long conversation pay the same memory block dozens of times. The plugin keeps a per-session `slug → body-hash` record (`turn-dedup.ts`, `partitionDirectShards`) and, after a shard's full body has been injected once this session, replaces it on later turns with a one-line `slug:` reference plus a `memory_search({ topic: "<slug>" })` recovery hint. A shard whose body **changed** (a dreaming pass rewrote it) re-injects in full so the agent never reasons over a stale body. The "nothing the agent always had vanishes on an off-topic turn" guarantee is preserved by **availability, not literal presence**: every topic is still named (heading + slug) every turn, and the full body is one `memory_search` away. The per-session record is cleared on `session.end` (before the subagent-origin early-return, so subagent sessions don't leak it), so a resurrected session re-injects bodies in full rather than dangling a reference to text the rebuilt context no longer holds.
|
|
52
49
|
|
|
53
|
-
- **
|
|
54
|
-
- **Vector on: user prompt, per turn.** The system-prompt `# Memory` section is suppressed entirely (`createSession`'s `suppressSystemMemory`, derived once at boot in `src/run/index.ts`). The `session.turn.start` hook instead renders memory into `event.retrievalContext.results`, which the turn-drivers append to the user message. Under budget it injects shard bodies in **direct mode**, **de-duplicated across turns** (see below); over budget it runs `hybridSearch` and injects the **top-K** most relevant shards/fragments for the current message. The direct path renders via `renderDedupedMemorySection` and the index path via `renderRetrievedMemorySection` (`load-memory.ts`). This keeps the system-prompt cache prefix stable across a session and lets retrieval track the live topic instead of a stale session-start snapshot.
|
|
50
|
+
For **channel** origins, the per-turn render injects topic **headings only** — the excerpt body is stripped and the agent fetches it on demand via `memory_search`. A channel **direct-mode** turn is force-indexed to a headings/slugs section over **every** shard (not run through `hybridSearch`), so a full body can never reach a channel turn AND no heading is silently dropped by relevance-filtering or a stale vector index. Only channel turns that are genuinely **over budget** fall to `hybridSearch` top-K, where headings-only injection is the established channel policy regardless. This mirrors the direct-path policy that channel sessions never carry bodies: a topic heading is a self-contained belief sentence (the dreaming subagent writes exactly one), so the heading alone lets the model decide whether the topic is worth opening, while the ~10-shard excerpt block — which dominated the per-turn injection and was usually abandoned unread — no longer rides in the (uncacheable) user turn on every message. Non-channel origins (TUI/direct) keep the full excerpt on first injection, where the extra retrieval round-trip isn't worth it.
|
|
55
51
|
|
|
56
|
-
|
|
52
|
+
Each stripped topic is injected with its `slug:` line so the agent can re-fetch the exact shard with `memory_search({ topic: "<slug>" })` — a direct slug lookup, not a second fuzzy search over the heading text the retrieval layer already matched. Undreamed `source: "stream"` items have no topic shard yet (no slug to look up), so they instead carry a `memory_search({ query: ... })` recovery hint; the channel directive names both paths so every stripped entry — topic or stream — has an explicit way back to its full text.
|
|
57
53
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
Each stripped topic is injected with its `slug:` line so the agent can re-fetch the exact shard with `memory_search({ topic: "<slug>" })` — a direct slug lookup, not a second fuzzy search over the heading text the retrieval layer already matched. Undreamed `source: "stream"` items have no topic shard yet (no slug to look up), so they instead carry a `memory_search({ query: ... })` recovery hint; the channel directive names both paths so every stripped entry — topic or stream — has an explicit way back to its full text.
|
|
61
|
-
|
|
62
|
-
The invariant `suppressSystemMemory === memory.vector.enabled` is load-bearing: a session must never carry memory in both the system prompt and the user turn.
|
|
54
|
+
The system prompt never contains long-term memory, so a session cannot double-carry memory in both the system prompt and the user turn.
|
|
63
55
|
|
|
64
56
|
### Internal-infrastructure turns skip retrieval
|
|
65
57
|
|
|
@@ -70,11 +62,11 @@ The discriminator (`isInternalInfraTurn` in `index.ts`) matches two provenance s
|
|
|
70
62
|
- **`system` provenance** — `memory-logger` and `backup` are spawned directly with a `system` parent origin, so either a `system` origin itself or a subagent whose `spawnedByOrigin.kind === 'system'` is skipped.
|
|
71
63
|
- **The dreaming cron job** — `dreaming` is spawned by the memory plugin's own cron job (`__plugin_memory_dreaming`), so its parent origin is `cron`, NOT `system` (the cron consumer stamps a `cron` parentOrigin so the subagent resolves to the right role). It is matched by that specific job id — `spawnedByOrigin.kind === 'cron' && jobId === __plugin_memory_dreaming` — derived via `buildPluginCronGlobalId` so it can't drift from the registration. A **user-scheduled** cron subagent has a different job id and is real delegated work, so it keeps memory.
|
|
72
64
|
|
|
73
|
-
This is the same "don't mine framing prose" class of bug
|
|
65
|
+
This is the same "don't mine framing prose" class of bug applied to the subagent turn-start path.
|
|
74
66
|
|
|
75
67
|
### Vector retrieval is parent-child, not a flat pool
|
|
76
68
|
|
|
77
|
-
|
|
69
|
+
`hybridSearch` does **not** rank topic vectors and stream-fragment vectors together in one flat cosine pool. It uses a parent-child collapse so the result set is always topic-level current truth:
|
|
78
70
|
|
|
79
71
|
- **Match surface.** The query matches against topic-belief vectors AND fragment vectors (both lanes, fused by RRF). Fragments are the fine-grained retrieval hooks; topics are the returned unit. The keyword lane is **phrase-first with a token-OR fallback** (mirroring `memory_search`): since `hybridSearch`'s query is the whole user prompt — which never appears verbatim in a shard — a phrase-only lane would return nothing every turn and silently degrade RRF to vector-only. The fallback (`searchAllRanked`) also ranks by matched-token-count and truncates after ranking, giving RRF a real second ranker instead of alphabetical topic order.
|
|
80
72
|
- **Collapse to parent.** A matched fragment resolves to the topic that cites it (via the shard's `fragments:` list — the citation IS the parent foreign key) and contributes its score to that topic. It never appears as a standalone result. An **undreamed** fragment (no topic cites it yet) resolves to itself, preserving the ~30-min freshness window before dreaming consolidates it.
|
|
@@ -82,17 +74,15 @@ When `memory.vector.enabled` is true, `hybridSearch` does **not** rank topic vec
|
|
|
82
74
|
- **Dreamed-and-cited fragment vectors are pruned every dreaming pass.** Once a fragment is folded into a topic (dreamed AND cited), its standalone `stream:*` row is redundant — any match on it collapses to the parent topic, whose `topic:*` row is already a candidate. But the redundant row still consumes one of `store.query`'s finite `topK * 2` pre-fusion slots by raw cosine, so a topic cited on many days would otherwise hoard proportionally more slots and displace distinct topics — silently defeating the MAX-child anti-popularity design over a long container uptime. `deleteRedundantDreamedCitedStreamVectors` (`dreaming.ts`) removes these per pass; it is the same deletion startup `pruneStaleRows` performs (a dreamed-and-cited fragment leaves the undreamed passage set), advanced from per-restart to per-pass. Undreamed fragment rows are kept — they resolve to themselves and ARE the freshness window.
|
|
83
75
|
- **Superseded fragments are excluded from the match surface.** When dreaming overturns a belief on contradiction, the old fragment id moves from `fragments:` to a `superseded:` list (see below). Superseded ids stay cited — so the citation-superset GC invariant keeps the fragment alive and history is auditable — but `passages.ts` does not embed them, so a stale "uses bun" fragment can never resurface as a hook for the current "uses pnpm" belief. `parseCitations` stays section-blind so GC and frontmatter recompute still see both sections; `splitCitationsBySection` is the status-aware view the retrieval layer reads (`parent-link.ts`).
|
|
84
76
|
|
|
85
|
-
**Undreamed daily-stream events are NOT injected into the system prompt.** They are reachable
|
|
77
|
+
**Undreamed daily-stream events are NOT injected into the system prompt.** They are reachable via `memory_search` and per-turn vector retrieval, both of which discriminate results by `source: "topic" | "stream"`. The agent now decides per-query whether recent observations are relevant, instead of carrying every undreamed fragment in the cached prompt prefix. Three reasons this is the right shape:
|
|
86
78
|
|
|
87
79
|
1. PR #314 made `memory_search` cover the stream surface, so the duplicate copy in the system prompt no longer earns its bytes.
|
|
88
80
|
2. Streams grow unboundedly with usage (~360 KB at 30 days in the typical case, more under heavy use). The previous per-file 12 KB cap silently dropped each day's tail with no signal to the agent; on-demand search returns the relevant slice instead of "the first 12 KB by date".
|
|
89
|
-
3. Streams
|
|
81
|
+
3. Streams in the cached system-prompt prefix would append new fragments on every memory-logger run, breaking cache reuse across prompts. Without system-prompt injection, the prefix stays stable.
|
|
90
82
|
|
|
91
83
|
**Channel-origin always uses index mode regardless of total shard size** — defends against memory bleed into channel responses (the agent treats injected memory as instructions when channel users see it).
|
|
92
84
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
The hook trigger matters: `SessionPromptEvent.prompt` (`session.prompt`) carries the assembling system prompt (`basePrompt + IDENTITY + SOUL`) at session-creation time, NOT the user's message. Reading that field as if it were the user's prompt — which this plugin did before PR #340 — caused the retrieval subagent to keyword-mine TypeClaw's own framing prose (`TypeClaw`, `subagent`, `AGENTS.md`, `systemPromptLeak`, etc.) on every session. `session.turn.start` is the correct hook for "what is the user asking right now."
|
|
85
|
+
The hook trigger matters: `SessionPromptEvent.prompt` (`session.prompt`) carries the assembling system prompt (`basePrompt + IDENTITY + SOUL`) at session-creation time, NOT the user's message. `session.turn.start` is the correct hook for "what is the user asking right now," so per-turn retrieval embeds the actual user text instead of TypeClaw's framing prose.
|
|
96
86
|
|
|
97
87
|
## Memory saturation
|
|
98
88
|
|
|
@@ -102,7 +92,7 @@ The subagent uses these signals to:
|
|
|
102
92
|
|
|
103
93
|
1. **Promote strong topics.** `days = 1` → tentative ("the user mentioned"). `days >= 3` → confident ("the user consistently"). `days >= 7` → declarative ("the user always"). Promotion is gated on distinct days, not raw citation count.
|
|
104
94
|
2. **Merge near-duplicates.** Topics that overlap get folded into one. The merged topic's citation set is the union.
|
|
105
|
-
3. **Demote decayed topics.** A weak/decayed topic stays as its own shard but the body should be trimmed to a single line. When index-mode injection is in effect, demoted shards' bodies don't enter the
|
|
95
|
+
3. **Demote decayed topics.** A weak/decayed topic stays as its own shard but the body should be trimmed to a single line. When index-mode injection is in effect, demoted shards' bodies don't enter the user turn at all — the index plus `memory_search` retrieval cover them on demand.
|
|
106
96
|
|
|
107
97
|
There is no `## Historical observations` bucket. Demoted topics live as their own shards; injection-time filtering (the index/direct split) handles the prompt-budget pressure.
|
|
108
98
|
|
|
@@ -135,9 +125,8 @@ A `[dreaming] citation-superset violation: …` warning logs the dropped ids and
|
|
|
135
125
|
- **`memory/streams/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`. Force-committed alongside the shards. A `fragment` event carries optional **situational provenance** for "who said what, where, and when": `who` (display name/handle of the attributed speaker, LLM-supplied per-fragment by `memory-logger` only when one transcript speaker clearly owns the evidence), `where` (`{ adapter, workspace, workspaceName?, chat, chatName?, thread }` — runtime-stamped from the session origin in the append tool, channel origins only), and `when` (reuses the existing `ts`). Both are optional, so legacy fragments and non-channel (TUI) origins parse with them absent. `who`/`where` are deliberately NOT in `fragmentContentHash` (dedup stays content-identity) and NOT in `fragmentEmbeddableText` (provenance is retrieval metadata, not embedded — it must not burn the 512-token budget or let same-room/same-person dominate recall). **Secret-scanning covers provenance too**: `who` (LLM-supplied) clears the same `detectSecrets` guard as `topic`/`body` and a credential-shaped value is refused with the retryable error; the origin-derived `where` names (`workspaceName`/`chatName`) are external channel data the agent can't rewrite, so a credential-shaped name is silently dropped (raw id kept) rather than wedging the logger. Raw ids are platform-issued identifiers, not free text, so they are not scanned. `where` is runtime-stamped rather than LLM-supplied because the channel is stable for a whole logger run; `who` is per-fragment because a single run spans many speakers and `origin.lastInboundAuthorId` is only the spawn-time one — auto-stamping it would misattribute fragments about earlier speakers. Surfaced to the agent (not embedded) via `memory_search` (`StreamMatch.who/when/where`), `hybridSearch` (`HybridSearchResult.who/when/where` on self-resolved undreamed fragments), and the per-turn retrieval render (`renderProvenanceLine` → `_Jisoo in #incidents on 2026-06-24_`).
|
|
136
126
|
- **`memory/MEMORY.md.pre-shard.bak`** — legacy pre-shard backup left by older TypeClaw versions. Safe to delete after verifying.
|
|
137
127
|
- **`memory/skills/<name>/SKILL.md`** — muscle memory (Form A). Skills the dreaming subagent distills from repeated procedures. Auto-loaded as first-class skills. CLI/plugin suggestions (Forms B and C) are recorded as topic shards instead — see [Muscle memory](#muscle-memory-three-forms).
|
|
138
|
-
- **`memory/references/<slug>.md`** — verbatim reference artifacts. Written by `store_reference`, a `memory-logger` custom tool that is always registered (there is no `references.enabled` config gate — references are unconditionally available; they simply stay empty until the `memory-logger` chooses to store one). A fragment may cite a stored reference via `append`'s `references: ['<slug>']`, but the append tool validates each slug against the references that actually exist on disk and silently drops the rest (`resolveValidReferences`), so a hallucinated slug never lands a dangling citation in the stream.
|
|
128
|
+
- **`memory/references/<slug>.md`** — verbatim reference artifacts. Written by `store_reference`, a `memory-logger` custom tool that is always registered (there is no `references.enabled` config gate — references are unconditionally available; they simply stay empty until the `memory-logger` chooses to store one). A fragment may cite a stored reference via `append`'s `references: ['<slug>']`, but the append tool validates each slug against the references that actually exist on disk and silently drops the rest (`resolveValidReferences`), so a hallucinated slug never lands a dangling citation in the stream. A freshly stored reference is embedded into the vector index immediately via the `store_reference` on-write hook (`vector/reference-index-on-write.ts`), mirroring the stream-fragment on-write hook — so it is vector-retrievable within the same container uptime instead of only after the next startup index build. Chunks are derived by the same `referencePassagesForOne` the startup build uses, and re-storing a shorter body prunes the stale higher-index chunk rows.
|
|
139
129
|
- **`memory/.dreaming-state.json`** — per-day dreamed-id sets.
|
|
140
|
-
- **`memory/.retrieval-cache/<sessionId>.md`** — ephemeral retrieval summaries. Written by `memory-retrieval`, read by `loadMemory` on the next prompt of the same session, unlinked on `session.end`.
|
|
141
130
|
|
|
142
131
|
## Observability
|
|
143
132
|
|
|
@@ -145,7 +134,7 @@ The plugin emits structured `[plugin:memory]` log lines (no separate metrics inf
|
|
|
145
134
|
|
|
146
135
|
- **`[memory-logger] <session> done fragments_written=N elapsed_ms=…`** — how many fragments a logger run captured (delta of fragment events in today's stream).
|
|
147
136
|
- **`[dreaming] done topics_created=N topics_removed=N superseded_new=N fragments_dropped=N over_budget=N elapsed_ms=…`** — consolidation activity per run: new/removed topic shards (by snapshot path diff), net citations moved into `superseded:`, fragments GC'd, and `over_budget` (the count of topic shards whose embeddable text exceeds the model token budget — see [Embedding token budget](#embedding-token-budget)). `topics_created` + reinforcement is the "is memory getting sharper" signal; `superseded_new` is contradiction-edit volume; a persistently non-zero `over_budget` means the subagent isn't compacting the flagged shards. Derived by `computeDreamingMetrics` from the pre/post shard snapshots.
|
|
148
|
-
- **`[vector-retrieval] mode=index topic_results=N stream_results=N reference_results=N elapsed_ms=… [suppressed=1]`** (or `mode=direct topics=N`) — per-turn retrieval breakdown. `stream_results` counts undreamed-fragment hits that self-resolved (no citing topic yet) — the freshness-window usage signal that informs whether the undreamed surface earns its keep. `reference_results` counts verbatim-reference hits (non-zero once the `memory-logger` has stored at least one reference
|
|
137
|
+
- **`[vector-retrieval] mode=index topic_results=N stream_results=N reference_results=N elapsed_ms=… [suppressed=1]`** (or `mode=direct topics=N`) — per-turn retrieval breakdown. `stream_results` counts undreamed-fragment hits that self-resolved (no citing topic yet) — the freshness-window usage signal that informs whether the undreamed surface earns its keep. `reference_results` counts verbatim-reference hits (non-zero once the `memory-logger` has stored at least one reference), broken out from `stream_results` so the two surfaces are distinguishable. `elapsed_ms` times the full `hybridSearch` (embed + cosine scan + keyword lane + RRF fusion) — the on-critical-path retrieval cost. `suppressed=1` appears when the query matched nothing after the relevance gate, so an empty memory block is distinguishable from a missing one.
|
|
149
138
|
|
|
150
139
|
These are intentionally verbose for now so behavior is observable in logs; trim once the useful subset is known.
|
|
151
140
|
|
|
@@ -154,7 +143,7 @@ These are intentionally verbose for now so behavior is observable in logs; trim
|
|
|
154
143
|
The embedding model (`Xenova/multilingual-e5-base`) caps inputs at **512 tokens**; the transformers.js tokenizer truncates anything longer by default, silently dropping the tail. Canonical compact shards (heading + one belief sentence first) sit well under the cap, but legacy verbose shards, legacy prose migration events, long fragments, and very long user queries can exceed it. Handled in two layers (`vector/truncation.ts`):
|
|
155
144
|
|
|
156
145
|
1. **Deterministic bound at embed time (primary, always on).** `embed()` calls `boundEmbeddableText` on every input before the tokenizer sees it, trimming to a character budget derived from the same conservative per-script token estimate (CJK counted 1:1, Latin ~3.5 chars/token; the tokenizer is never loaded, preserving the embedder's lazy-load). The cut is now explicit and owned, not a hidden tokenizer side effect, and the leading heading/belief sentence — the load-bearing retrieval signal — always survives because it comes first. This guarantees no silent loss even for inputs nothing ever rewrites (raw legacy prose, long queries).
|
|
157
|
-
2. **Compaction routed to dreaming (remediation
|
|
146
|
+
2. **Compaction routed to dreaming (remediation).** Bounding keeps retrieval correct but the bounded tail is still not embedded. The real fix is to make the shard fit. On each run the dreaming handler computes which topic shards are over budget (measuring `topicPassage(...).text` — the same citation-stripped string the embedder bounds — so the flag matches what is actually truncated) and renders an "Over the embedding budget" table into the subagent's prompt, alongside the strength-signal table. Dreaming rule 8 instructs it to compact those shards into the one-belief-sentence form **while preserving every `fragments:`/`superseded:` citation** — shrinking prose, never dropping a citation (a dropped citation would trip the citation-superset revert, leaving the shard both reverted and still over budget). The signal is gated on the vector index (`memory/.vectors/index.db`) existing; before the first index exists, the table is suppressed. The `over_budget=N` count rides the `[dreaming] done` line.
|
|
158
147
|
|
|
159
148
|
## How `session.idle` works
|
|
160
149
|
|
|
@@ -180,7 +169,7 @@ Each `memory-logger` spawn captures the line count of `memory/streams/<today>.js
|
|
|
180
169
|
|
|
181
170
|
## Tests
|
|
182
171
|
|
|
183
|
-
Tests live next to the code they cover as `.test.ts` neighbors; `vector/*.test.ts` covers the vector subsystem (store, embedder, hybrid search, relevance gate, startup/on-write indexing, doctor/inspect). The core surfaces are `paths`, `slug`, `frontmatter`, `topics`, `shard-snapshot`, `delete-tool`, `citations`, `citation-superset`, `load-shards`, `load-memory`, `injection-plan`, `turn-dedup`, `search-tool`, `memory-
|
|
172
|
+
Tests live next to the code they cover as `.test.ts` neighbors; `vector/*.test.ts` covers the vector subsystem (store, embedder, hybrid search, relevance gate, startup/on-write indexing, doctor/inspect). The core surfaces are `paths`, `slug`, `frontmatter`, `topics`, `shard-snapshot`, `delete-tool`, `citations`, `citation-superset`, `load-shards`, `load-memory`, `injection-plan`, `turn-dedup`, `search-tool`, `memory-logger`, `dreaming`, `index`, `index-vector`, `index-vector-retrieval`, `integration` (not exhaustive — `git ls-files 'src/bundled-plugins/memory/**/*.test.ts'` is the source of truth). Plus guard policies in `../guard/policies/`: `memory-topics-delete`, `memory-topics-write`.
|
|
184
173
|
|
|
185
174
|
Both vector hook tests build the plugin through `createMemoryPluginForTests({ hybridSearch?, queryEmbedFn? })` — a per-instance dependency seam (the factory defaults to the real implementations) so a fake is scoped to one boot and never leaks across the suite. `index-vector` injects a fake `hybridSearch` to cover the budget-gating, channel force-index, and cross-turn dedup branches of the `session.turn.start` hook. `index-vector-retrieval` injects only a synthetic `queryEmbedFn` (so the ~279 MB model never loads) and runs the REAL `hybridSearch` → `VectorStore.query` → RRF → `renderRetrievedMemorySection` pipeline through the same hook against a real seeded vector store; the query token appears nowhere in the corpus, so a rendered topic proves the vector lane matched rather than the substring `memory_search` lane. Relevance-gate suppression is covered separately in `vector/hybrid` and `vector/relevance-gate`.
|
|
186
175
|
|
|
@@ -10,28 +10,25 @@ import { buildPluginCronGlobalId, definePlugin, type SpawnSubagentOptions } from
|
|
|
10
10
|
import { formatLocalDate } from '@/shared'
|
|
11
11
|
|
|
12
12
|
import { createDreamingSubagent, type DreamingPayload } from './dreaming'
|
|
13
|
-
import {
|
|
13
|
+
import { DEFAULT_INJECTION_BUDGET_BYTES, MIN_INJECTION_BUDGET_BYTES } from './injection-plan'
|
|
14
14
|
import {
|
|
15
15
|
loadMemoryInjectionPlan,
|
|
16
16
|
renderDedupedRetrievedMemorySection,
|
|
17
17
|
renderRetrievedMemorySection,
|
|
18
18
|
renderTopicIndexMemorySection,
|
|
19
19
|
} from './load-memory'
|
|
20
|
-
import { loadAllShards } from './load-shards'
|
|
21
20
|
import { createMemoryLoggerSubagent, type MemoryLoggerPayload } from './memory-logger'
|
|
22
|
-
import { createMemoryRetrievalSubagent, type MemoryRetrievalPayload } from './memory-retrieval'
|
|
23
21
|
import { preShardBackupPath, streamFilePath, streamsDir, topicsDir } from './paths'
|
|
24
22
|
import { bumpReferenceAccess } from './references/load-references'
|
|
25
23
|
import { memoryCommands } from './search-command'
|
|
26
24
|
import { createMemorySearchTool } from './search-tool'
|
|
27
25
|
import { type InjectedMemoryState, partitionRetrievedMemoryItems } from './turn-dedup'
|
|
28
|
-
import { vectorConfigSchema } from './vector/config'
|
|
29
26
|
import { runVectorIndexDoctor } from './vector/doctor'
|
|
30
27
|
import { embed } from './vector/embedder'
|
|
31
28
|
import { hybridSearch, type EmbedFn } from './vector/hybrid'
|
|
32
29
|
import { makeAppendHook } from './vector/index-on-write'
|
|
33
30
|
import { makeReferenceStoredHook } from './vector/reference-index-on-write'
|
|
34
|
-
import { VectorStore } from './vector/store'
|
|
31
|
+
import { type OpenVectorStore, VectorStore } from './vector/store'
|
|
35
32
|
|
|
36
33
|
const DEFAULT_IDLE_MS = 60_000
|
|
37
34
|
const DEFAULT_BUFFER_BYTES = 500_000
|
|
@@ -66,17 +63,6 @@ const DEFAULT_DREAMING_SCHEDULE = '*/30 * * * *'
|
|
|
66
63
|
const DREAMING_CRON_KEY = 'dreaming'
|
|
67
64
|
const DREAMING_CRON_JOB_ID = buildPluginCronGlobalId('memory', DREAMING_CRON_KEY)
|
|
68
65
|
|
|
69
|
-
// memory-retrieval's ceiling, enforced by the orchestration layer (see
|
|
70
|
-
// `awaitWithSubagentTimeout` in @/agent/subagents). 30s is sized for the
|
|
71
|
-
// declared workload — up to 3 `memory_search` calls + 1 `write` against a
|
|
72
|
-
// `fast`-profile model. The 5+ minute outliers observed in the wild
|
|
73
|
-
// (reasoning-model cold-start on the default profile) require either a
|
|
74
|
-
// genuinely wedged provider, a misconfigured profile that routes retrieval
|
|
75
|
-
// to a reasoning model anyway, or both. In all three cases, releasing the
|
|
76
|
-
// coalescing key after 30s lets the next channel turn spawn a fresh
|
|
77
|
-
// retrieval instead of staying skip-coalesced behind the stuck one.
|
|
78
|
-
const RETRIEVAL_SPAWN_TIMEOUT_MS = 30_000
|
|
79
|
-
|
|
80
66
|
// Hard ceiling on a single memory-logger spawn. The chain serializes spawns
|
|
81
67
|
// per agent, so a non-settling spawn would otherwise wedge every subsequent
|
|
82
68
|
// fire — including the session.end hook path that gates cron consumer's
|
|
@@ -137,13 +123,7 @@ const memoryConfigSchema = z
|
|
|
137
123
|
// the timeout in milliseconds instead of the production 50s. Kept
|
|
138
124
|
// undocumented for users.
|
|
139
125
|
spawnTimeoutMs: z.number().int().min(1).default(SPAWN_TIMEOUT_MS),
|
|
140
|
-
// Test seam: per-spawn ceiling for memory-retrieval. Same rationale as
|
|
141
|
-
// `spawnTimeoutMs` — operators have no reason to tune this; it exists
|
|
142
|
-
// so the wedge-recovery test for memory-retrieval can fire the timeout
|
|
143
|
-
// in milliseconds instead of the production 30s.
|
|
144
|
-
retrievalSpawnTimeoutMs: z.number().int().min(1).default(RETRIEVAL_SPAWN_TIMEOUT_MS),
|
|
145
126
|
dreaming: dreamingConfigSchema.optional(),
|
|
146
|
-
vector: vectorConfigSchema,
|
|
147
127
|
})
|
|
148
128
|
.default({
|
|
149
129
|
idleMs: DEFAULT_IDLE_MS,
|
|
@@ -151,8 +131,6 @@ const memoryConfigSchema = z
|
|
|
151
131
|
injectionBudgetBytes: DEFAULT_INJECTION_BUDGET_BYTES,
|
|
152
132
|
minIdleDeltaLines: DEFAULT_MIN_IDLE_DELTA_LINES,
|
|
153
133
|
spawnTimeoutMs: SPAWN_TIMEOUT_MS,
|
|
154
|
-
retrievalSpawnTimeoutMs: RETRIEVAL_SPAWN_TIMEOUT_MS,
|
|
155
|
-
vector: { enabled: false },
|
|
156
134
|
})
|
|
157
135
|
|
|
158
136
|
const VECTOR_TURN_TOP_K = 10
|
|
@@ -166,13 +144,13 @@ const VECTOR_TURN_TOP_K = 10
|
|
|
166
144
|
export type MemoryPluginDeps = {
|
|
167
145
|
hybridSearch: typeof hybridSearch
|
|
168
146
|
queryEmbedFn: EmbedFn
|
|
169
|
-
openAppendVectorStore: (agentDir: string) =>
|
|
147
|
+
openAppendVectorStore: (agentDir: string) => OpenVectorStore
|
|
170
148
|
}
|
|
171
149
|
|
|
172
150
|
const defaultDeps: MemoryPluginDeps = {
|
|
173
151
|
hybridSearch,
|
|
174
152
|
queryEmbedFn: embed,
|
|
175
|
-
openAppendVectorStore: (agentDir) => VectorStore.open(join(agentDir, 'memory', '.vectors', 'index.db')),
|
|
153
|
+
openAppendVectorStore: (agentDir) => () => VectorStore.open(join(agentDir, 'memory', '.vectors', 'index.db')),
|
|
176
154
|
}
|
|
177
155
|
|
|
178
156
|
// TypeClaw-owned infrastructure subagents (memory-logger, dreaming, backup) run
|
|
@@ -183,8 +161,7 @@ const defaultDeps: MemoryPluginDeps = {
|
|
|
183
161
|
// ~560-token query that trips the 512-token bound warning on every spawn. None
|
|
184
162
|
// of these subagents consume injected long-term memory the way a conversational
|
|
185
163
|
// turn does, so per-turn retrieval is pure waste here. This is the same class of
|
|
186
|
-
// "don't mine framing prose" bug
|
|
187
|
-
// the subagent turn-start path.
|
|
164
|
+
// "don't mine framing prose" bug applied to the subagent turn-start path.
|
|
188
165
|
//
|
|
189
166
|
// Two provenance shapes carry infra work, because the two infra subagents are
|
|
190
167
|
// spawned differently:
|
|
@@ -283,7 +260,6 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
283
260
|
const bufferBytes = ctx.config.bufferBytes
|
|
284
261
|
const minIdleDeltaLines = ctx.config.minIdleDeltaLines
|
|
285
262
|
const spawnTimeoutMs = ctx.config.spawnTimeoutMs
|
|
286
|
-
const retrievalSpawnTimeoutMs = ctx.config.retrievalSpawnTimeoutMs
|
|
287
263
|
const dreamingSchedule = ctx.config.dreaming?.schedule ?? DEFAULT_DREAMING_SCHEDULE
|
|
288
264
|
|
|
289
265
|
const idleTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
|
@@ -416,38 +392,6 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
416
392
|
return { skip: false }
|
|
417
393
|
}
|
|
418
394
|
|
|
419
|
-
const runMemoryRetrieval = async (event: {
|
|
420
|
-
sessionId: string
|
|
421
|
-
agentDir: string
|
|
422
|
-
userPrompt: string
|
|
423
|
-
origin?: SessionOrigin
|
|
424
|
-
}): Promise<void> => {
|
|
425
|
-
const shards = await loadAllShards(event.agentDir)
|
|
426
|
-
const plan = buildInjectionPlan(shards, { budgetBytes: ctx.config.injectionBudgetBytes })
|
|
427
|
-
if (plan.mode === 'direct') return
|
|
428
|
-
|
|
429
|
-
const cacheFilePath = join(event.agentDir, 'memory', '.retrieval-cache', `${event.sessionId}.md`)
|
|
430
|
-
const payload: MemoryRetrievalPayload = {
|
|
431
|
-
parentSessionId: event.sessionId,
|
|
432
|
-
agentDir: event.agentDir,
|
|
433
|
-
recentPrompt: event.userPrompt,
|
|
434
|
-
cacheFilePath,
|
|
435
|
-
...(event.origin !== undefined ? { origin: event.origin } : {}),
|
|
436
|
-
}
|
|
437
|
-
// System authority, not the triggering turn's role — see the
|
|
438
|
-
// memory-logger spawn above. memory-retrieval writes
|
|
439
|
-
// memory/.retrieval-cache/, which a guest-demoted role cannot.
|
|
440
|
-
const retrievalSpawnOptions: SpawnSubagentOptions = {
|
|
441
|
-
parentSessionId: event.sessionId,
|
|
442
|
-
spawnedByOrigin: {
|
|
443
|
-
kind: 'system',
|
|
444
|
-
component: 'memory-retrieval',
|
|
445
|
-
...(event.origin !== undefined ? { triggeredBy: event.origin } : {}),
|
|
446
|
-
},
|
|
447
|
-
}
|
|
448
|
-
await ctx.spawnSubagent('memory-retrieval', payload, retrievalSpawnOptions)
|
|
449
|
-
}
|
|
450
|
-
|
|
451
395
|
// Subagents are constructed at boot here (rather than imported as constants)
|
|
452
396
|
// so their lifecycle logs route through the plugin logger and pick up the
|
|
453
397
|
// `[plugin:memory]` prefix. Without this, they would write directly to
|
|
@@ -458,23 +402,18 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
458
402
|
error: (m: string) => ctx.logger.error(m),
|
|
459
403
|
}
|
|
460
404
|
|
|
461
|
-
//
|
|
462
|
-
|
|
405
|
+
// Opener (not an open handle) for append-time indexing. Each on-write hook
|
|
406
|
+
// opens its own short-lived store and closes it when the batch settles, so
|
|
407
|
+
// no bun:sqlite handle outlives a hook call to pin the agent dir (Bun
|
|
408
|
+
// #25964). Boot-only paths that never append never open the DB at all.
|
|
409
|
+
const openAppendVectorStore = deps.openAppendVectorStore(ctx.agentDir)
|
|
463
410
|
|
|
464
411
|
return {
|
|
465
412
|
subagents: {
|
|
466
413
|
'memory-logger': createMemoryLoggerSubagent({
|
|
467
414
|
logger: subagentLogger,
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
onFragmentsAppended: makeAppendHook(appendVectorStore),
|
|
471
|
-
onReferenceStored: makeReferenceStoredHook(appendVectorStore),
|
|
472
|
-
}
|
|
473
|
-
: {}),
|
|
474
|
-
}),
|
|
475
|
-
'memory-retrieval': createMemoryRetrievalSubagent({
|
|
476
|
-
logger: subagentLogger,
|
|
477
|
-
timeoutMs: retrievalSpawnTimeoutMs,
|
|
415
|
+
onFragmentsAppended: makeAppendHook(openAppendVectorStore),
|
|
416
|
+
onReferenceStored: makeReferenceStoredHook(openAppendVectorStore),
|
|
478
417
|
}),
|
|
479
418
|
dreaming: createDreamingSubagent({
|
|
480
419
|
logger: subagentLogger,
|
|
@@ -493,18 +432,6 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
493
432
|
},
|
|
494
433
|
},
|
|
495
434
|
hooks: {
|
|
496
|
-
// Memory injection lives in core (`createResourceLoader` calls `loadMemory`
|
|
497
|
-
// directly, appended LAST in the system prompt). It does not run from a
|
|
498
|
-
// plugin hook because positioning matters for cache-prefix stability:
|
|
499
|
-
// the daily-stream file grows after every channel turn (memory-logger
|
|
500
|
-
// appends a fragment + watermark) and memory/topics/ changes on every dream.
|
|
501
|
-
// A volatile region in the middle of the system prompt invalidates the
|
|
502
|
-
// entire cacheable suffix below it on every session resurrection
|
|
503
|
-
// (channel sessions evicted by idle GC, container restarts). Pinning
|
|
504
|
-
// memory to the bottom of the system prompt keeps everything above it
|
|
505
|
-
// cacheable across resurrections, at the cost of re-billing only the
|
|
506
|
-
// memory section itself when it grows.
|
|
507
|
-
//
|
|
508
435
|
// Core fires `session.idle` immediately after every prompt completion;
|
|
509
436
|
// the plugin owns the debounce timer so memory-logger only spawns
|
|
510
437
|
// after the user has been quiet for `idleMs`. Re-arming a still-armed
|
|
@@ -547,50 +474,24 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
547
474
|
await fireMemoryLogger(sessionId, 'buffer-trip')
|
|
548
475
|
}
|
|
549
476
|
},
|
|
550
|
-
// memory-retrieval used to run from `session.prompt`, which fires
|
|
551
|
-
// during system-prompt assembly (createResourceLoader) and carries
|
|
552
|
-
// the ASSEMBLING SYSTEM PROMPT as `event.prompt` — not the user's
|
|
553
|
-
// message. The plugin was feeding that string into the subagent as
|
|
554
|
-
// `recentPrompt`, so the LLM keyword-mined TypeClaw's framing prose
|
|
555
|
-
// (`TypeClaw`, `subagent`, `AGENTS.md`, `systemPromptLeak`, etc.)
|
|
556
|
-
// and burned 15+ memory_search calls per session on terms the user
|
|
557
|
-
// never said. `session.turn.start` is the correct trigger: it fires
|
|
558
|
-
// before each `session.prompt(text)` call with the actual text the
|
|
559
|
-
// session is about to receive.
|
|
560
|
-
//
|
|
561
477
|
'session.turn.start': async (event) => {
|
|
562
|
-
if (
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
if (event.retrievalContext === undefined) return
|
|
569
|
-
try {
|
|
570
|
-
let injectedState = injectedMemory.get(event.sessionId)
|
|
571
|
-
if (injectedState === undefined) {
|
|
572
|
-
injectedState = new Map()
|
|
573
|
-
injectedMemory.set(event.sessionId, injectedState)
|
|
574
|
-
}
|
|
575
|
-
event.retrievalContext.results = await renderVectorTurnMemory(
|
|
576
|
-
event,
|
|
577
|
-
ctx.config.injectionBudgetBytes,
|
|
578
|
-
injectedState,
|
|
579
|
-
deps,
|
|
580
|
-
ctx.logger,
|
|
581
|
-
)
|
|
582
|
-
} catch (err) {
|
|
583
|
-
ctx.logger.error(`vector-retrieval failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
478
|
+
if (event.retrievalContext === undefined) return
|
|
479
|
+
try {
|
|
480
|
+
let injectedState = injectedMemory.get(event.sessionId)
|
|
481
|
+
if (injectedState === undefined) {
|
|
482
|
+
injectedState = new Map()
|
|
483
|
+
injectedMemory.set(event.sessionId, injectedState)
|
|
584
484
|
}
|
|
585
|
-
|
|
485
|
+
event.retrievalContext.results = await renderVectorTurnMemory(
|
|
486
|
+
event,
|
|
487
|
+
ctx.config.injectionBudgetBytes,
|
|
488
|
+
injectedState,
|
|
489
|
+
deps,
|
|
490
|
+
ctx.logger,
|
|
491
|
+
)
|
|
492
|
+
} catch (err) {
|
|
493
|
+
ctx.logger.error(`vector-retrieval failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
586
494
|
}
|
|
587
|
-
// Non-vector agents keep memory in the system prompt. The index-mode
|
|
588
|
-
// retrieval subagent must NOT fire for subagent-origin turns (it would
|
|
589
|
-
// recurse: the subagent it spawns triggers another turn.start).
|
|
590
|
-
if (event.origin?.kind === 'subagent') return
|
|
591
|
-
void runMemoryRetrieval(event).catch((err) => {
|
|
592
|
-
ctx.logger.error(`memory-retrieval spawn failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
593
|
-
})
|
|
594
495
|
},
|
|
595
496
|
// The memory-logger spawn is intentionally detached (`void`) instead
|
|
596
497
|
// of awaited. The channel router calls `tearDownLive` synchronously
|
|
@@ -628,9 +529,9 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
628
529
|
// readSize requires an await. fireMemoryLogger itself captures its
|
|
629
530
|
// payload synchronously from `lastIdleEvent` (see fireMemoryLogger
|
|
630
531
|
// comment block), so the `lastIdleEvent.delete` that follows can
|
|
631
|
-
// never race with the chained spawn. The
|
|
632
|
-
//
|
|
633
|
-
//
|
|
532
|
+
// never race with the chained spawn. The bookkeeping deletes are
|
|
533
|
+
// dispatched alongside (not blocking the hook return) to preserve the
|
|
534
|
+
// "session.end returns synchronously"
|
|
634
535
|
// contract that the channel router's tearDownLive path depends on
|
|
635
536
|
// (see the comment block above this hook).
|
|
636
537
|
void (async () => {
|
|
@@ -654,10 +555,6 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
654
555
|
linesAtLastRun.delete(sessionId)
|
|
655
556
|
streamCursorAtLastRun.delete(sessionId)
|
|
656
557
|
})()
|
|
657
|
-
const cacheFilePath = join(ctx.agentDir, 'memory', '.retrieval-cache', `${sessionId}.md`)
|
|
658
|
-
unlink(cacheFilePath).catch((err) => {
|
|
659
|
-
if (!isEnoent(err)) ctx.logger.warn(`[memory] failed to clean retrieval cache: ${err}`)
|
|
660
|
-
})
|
|
661
558
|
},
|
|
662
559
|
},
|
|
663
560
|
doctorChecks: {
|
|
@@ -739,14 +636,8 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
739
636
|
},
|
|
740
637
|
},
|
|
741
638
|
'vector-index': {
|
|
742
|
-
description: 'vector index is consistent with memory
|
|
743
|
-
run: async (dctx) =>
|
|
744
|
-
const config = dctx.config as typeof ctx.config
|
|
745
|
-
if (!vectorEnabled(config)) {
|
|
746
|
-
return { status: 'ok', message: 'vector memory not enabled; skipping index health check' }
|
|
747
|
-
}
|
|
748
|
-
return runVectorIndexDoctor(dctx.agentDir)
|
|
749
|
-
},
|
|
639
|
+
description: 'vector index is available and consistent with memory',
|
|
640
|
+
run: async (dctx) => runVectorIndexDoctor(dctx.agentDir),
|
|
750
641
|
},
|
|
751
642
|
},
|
|
752
643
|
}
|
|
@@ -760,12 +651,6 @@ export function createMemoryPluginForTests(overrides: Partial<MemoryPluginDeps>
|
|
|
760
651
|
return createMemoryPlugin({ ...defaultDeps, ...overrides })
|
|
761
652
|
}
|
|
762
653
|
|
|
763
|
-
function vectorEnabled(config: unknown): boolean {
|
|
764
|
-
if (typeof config !== 'object' || config === null) return false
|
|
765
|
-
const parsed = vectorConfigSchema.safeParse((config as Record<string, unknown>).vector)
|
|
766
|
-
return parsed.success && parsed.data.enabled
|
|
767
|
-
}
|
|
768
|
-
|
|
769
654
|
async function readSize(path: string): Promise<number> {
|
|
770
655
|
try {
|
|
771
656
|
const s = await stat(path)
|