typeclaw 0.50.0 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/dist/types/agent/bash-sandbox-env.d.ts +17 -0
  2. package/dist/types/agent/bash-sandbox-proc.d.ts +8 -0
  3. package/dist/types/agent/bash-sandbox.d.ts +98 -0
  4. package/dist/types/agent/compaction.d.ts +4 -4
  5. package/dist/types/agent/idle-compaction.d.ts +1 -0
  6. package/dist/types/agent/index.d.ts +10 -17
  7. package/dist/types/agent/multimodal/look-at.d.ts +1 -1
  8. package/dist/types/agent/output-allowance.d.ts +4 -0
  9. package/dist/types/agent/plugin-tools.d.ts +8 -75
  10. package/dist/types/agent/prompt-cache-key.d.ts +1 -0
  11. package/dist/types/agent/session-tail.d.ts +12 -0
  12. package/dist/types/agent/subagents.d.ts +5 -0
  13. package/dist/types/agent/system-prompt-composition.d.ts +18 -0
  14. package/dist/types/agent/system-prompt.d.ts +2 -0
  15. package/dist/types/agent/tools/channel-disengage.d.ts +1 -1
  16. package/dist/types/agent/tools/channel-edit.d.ts +1 -1
  17. package/dist/types/agent/tools/channel-fetch-attachment.d.ts +1 -1
  18. package/dist/types/agent/tools/channel-handoff.d.ts +1 -1
  19. package/dist/types/agent/tools/channel-history.d.ts +1 -1
  20. package/dist/types/agent/tools/channel-react.d.ts +1 -1
  21. package/dist/types/agent/tools/channel-read.d.ts +10 -2
  22. package/dist/types/agent/tools/channel-reply.d.ts +1 -1
  23. package/dist/types/agent/tools/channel-send.d.ts +1 -1
  24. package/dist/types/agent/tools/channel-session-search.d.ts +1 -1
  25. package/dist/types/agent/tools/ddg.d.ts +9 -0
  26. package/dist/types/agent/tools/grant-role.d.ts +1 -1
  27. package/dist/types/agent/tools/post-github-review.d.ts +1 -1
  28. package/dist/types/agent/tools/restart.d.ts +1 -1
  29. package/dist/types/agent/tools/skip-response.d.ts +1 -1
  30. package/dist/types/agent/tools/spawn-subagent.d.ts +2 -1
  31. package/dist/types/agent/tools/stream-snapshot.d.ts +2 -2
  32. package/dist/types/agent/tools/subagent-cancel.d.ts +1 -1
  33. package/dist/types/agent/tools/subagent-output.d.ts +1 -1
  34. package/dist/types/agent/tools/todo/index.d.ts +1 -1
  35. package/dist/types/agent/tools/webfetch/adaptive-html.d.ts +3 -3
  36. package/dist/types/agent/tools/webfetch/strategies/readability-process.d.ts +48 -0
  37. package/dist/types/agent/tools/webfetch/tool.d.ts +2 -2
  38. package/dist/types/agent/tools/websearch.d.ts +1 -1
  39. package/dist/types/channels/adapters/github/auth-app.d.ts +25 -0
  40. package/dist/types/channels/adapters/github/auth-pat.d.ts +17 -0
  41. package/dist/types/channels/adapters/github/auth.d.ts +24 -0
  42. package/dist/types/channels/github-review-check.d.ts +1 -1
  43. package/dist/types/channels/github-token-bridge.d.ts +4 -0
  44. package/dist/types/channels/lost-background-context.d.ts +2 -0
  45. package/dist/types/channels/persistence.d.ts +1 -0
  46. package/dist/types/channels/router.d.ts +16 -50
  47. package/dist/types/channels/types.d.ts +1 -0
  48. package/dist/types/coding-agent/tool.d.ts +1 -1
  49. package/dist/types/config/config.d.ts +22 -0
  50. package/dist/types/config/providers.d.ts +50 -1
  51. package/dist/types/container/start.d.ts +1 -0
  52. package/dist/types/eval/cell.d.ts +26 -0
  53. package/dist/types/eval/config.d.ts +19 -0
  54. package/dist/types/eval/dispatch.d.ts +7 -0
  55. package/dist/types/eval/kernel.d.ts +28 -0
  56. package/dist/types/eval/process.d.ts +23 -0
  57. package/dist/types/eval/protocol.d.ts +34 -0
  58. package/dist/types/eval/sandbox.d.ts +12 -0
  59. package/dist/types/eval/tool.d.ts +17 -0
  60. package/dist/types/eval/wire.d.ts +33 -0
  61. package/dist/types/git/resolve-identity.d.ts +2 -0
  62. package/dist/types/init/dockerfile.d.ts +1 -0
  63. package/dist/types/init/gitignore.d.ts +1 -1
  64. package/dist/types/permissions/builtins.d.ts +1 -0
  65. package/dist/types/plugin/types.d.ts +3 -0
  66. package/dist/types/sandbox/canonical-secrets.d.ts +2 -2
  67. package/dist/types/sandbox/coding-agent.d.ts +1 -1
  68. package/dist/types/sandbox/dependency-bins.d.ts +1 -0
  69. package/dist/types/sandbox/dev-source.d.ts +2 -2
  70. package/dist/types/sandbox/env-exposure.d.ts +2 -0
  71. package/dist/types/sandbox/hidden-paths.d.ts +1 -0
  72. package/dist/types/sandbox/index.d.ts +1 -1
  73. package/dist/types/sandbox/policy.d.ts +3 -0
  74. package/dist/types/typevault/cli-path.d.ts +2 -0
  75. package/dist/types/typevault/envelope-store.d.ts +11 -0
  76. package/dist/types/typevault/lifecycle.d.ts +13 -0
  77. package/dist/types/typevault/vault.d.ts +27 -0
  78. package/package.json +8 -7
  79. package/scripts/test.ts +252 -0
  80. package/src/agent/bash-sandbox-env.ts +136 -0
  81. package/src/agent/bash-sandbox-proc.ts +70 -0
  82. package/src/agent/bash-sandbox.ts +239 -0
  83. package/src/agent/compaction.ts +22 -10
  84. package/src/agent/idle-compaction.ts +7 -3
  85. package/src/agent/index.ts +185 -187
  86. package/src/agent/multimodal/look-at.ts +1 -1
  87. package/src/agent/output-allowance.ts +7 -0
  88. package/src/agent/plugin-tools.ts +64 -380
  89. package/src/agent/prompt-cache-key.ts +26 -3
  90. package/src/agent/session-origin.ts +41 -19
  91. package/src/agent/session-tail.ts +28 -0
  92. package/src/agent/subagent-completion-reminder.ts +2 -2
  93. package/src/agent/subagents.ts +21 -2
  94. package/src/agent/system-prompt-composition.ts +159 -0
  95. package/src/agent/system-prompt.ts +43 -5
  96. package/src/agent/todo/scope.ts +1 -1
  97. package/src/agent/tool-error-remediation.ts +18 -3
  98. package/src/agent/tools/channel-disengage.ts +1 -1
  99. package/src/agent/tools/channel-edit.ts +1 -1
  100. package/src/agent/tools/channel-fetch-attachment.ts +1 -1
  101. package/src/agent/tools/channel-handoff.ts +3 -2
  102. package/src/agent/tools/channel-history.ts +2 -1
  103. package/src/agent/tools/channel-react.ts +1 -1
  104. package/src/agent/tools/channel-read.ts +36 -3
  105. package/src/agent/tools/channel-reply.ts +1 -1
  106. package/src/agent/tools/channel-send.ts +4 -3
  107. package/src/agent/tools/channel-session-search.ts +13 -4
  108. package/src/agent/tools/ddg.ts +71 -11
  109. package/src/agent/tools/grant-role.ts +1 -1
  110. package/src/agent/tools/post-github-review.ts +1 -1
  111. package/src/agent/tools/restart.ts +1 -1
  112. package/src/agent/tools/skip-response.ts +1 -1
  113. package/src/agent/tools/spawn-subagent.ts +15 -1
  114. package/src/agent/tools/stream-snapshot.ts +1 -1
  115. package/src/agent/tools/subagent-cancel.ts +1 -1
  116. package/src/agent/tools/subagent-output.ts +1 -1
  117. package/src/agent/tools/todo/index.ts +1 -1
  118. package/src/agent/tools/webfetch/adaptive-html.ts +6 -4
  119. package/src/agent/tools/webfetch/strategies/readability-child.ts +25 -0
  120. package/src/agent/tools/webfetch/strategies/readability-process.ts +185 -0
  121. package/src/agent/tools/webfetch/tool.ts +6 -5
  122. package/src/agent/tools/websearch.ts +1 -1
  123. package/src/agent/tools-without-local-file-operands.ts +4 -0
  124. package/src/browser/dockerfile.ts +1 -1
  125. package/src/browser/fill-secret.ts +364 -0
  126. package/src/browser/registry.ts +6 -6
  127. package/src/bundled-plugins/agent-messenger-cli-auth/command.ts +211 -0
  128. package/src/bundled-plugins/agent-messenger-cli-auth/index.ts +186 -0
  129. package/src/bundled-plugins/backup/index.ts +17 -2
  130. package/src/bundled-plugins/backup/runner.ts +37 -2
  131. package/src/bundled-plugins/browser-agent/browser-agent.ts +51 -14
  132. package/src/bundled-plugins/browser-agent/index.ts +12 -5
  133. package/src/bundled-plugins/explorer/explorer.ts +1 -1
  134. package/src/bundled-plugins/github-cli-auth/gh-review-detect.ts +40 -16
  135. package/src/bundled-plugins/github-cli-auth/graphql-auth-nudge.ts +12 -7
  136. package/src/bundled-plugins/github-cli-auth/index.ts +10 -1
  137. package/src/bundled-plugins/managed-browser/index.ts +130 -2
  138. package/src/bundled-plugins/managed-browser/skills/browserskill/browserskill/SKILL.md +13 -5
  139. package/src/bundled-plugins/managed-browser/skills/omowright/omowright/SKILL.md +56 -5
  140. package/src/bundled-plugins/memory/README.md +23 -2
  141. package/src/bundled-plugins/memory/append-fragment.ts +133 -0
  142. package/src/bundled-plugins/memory/append-tool.ts +80 -149
  143. package/src/bundled-plugins/memory/dreaming.ts +6 -1
  144. package/src/bundled-plugins/memory/index.ts +33 -6
  145. package/src/bundled-plugins/memory/load-memory.ts +2 -4
  146. package/src/bundled-plugins/memory/memory-logger.ts +200 -6
  147. package/src/bundled-plugins/memory/references/store-reference-tool.ts +56 -32
  148. package/src/bundled-plugins/memory/remember-tool.ts +81 -0
  149. package/src/bundled-plugins/memory/search-tool-hybrid.ts +91 -0
  150. package/src/bundled-plugins/memory/search-tool.ts +58 -12
  151. package/src/bundled-plugins/memory/stream-events.ts +2 -0
  152. package/src/bundled-plugins/memory/transcript-cursor.ts +30 -0
  153. package/src/bundled-plugins/memory/transcript-preload.ts +208 -0
  154. package/src/bundled-plugins/memory/transcript-view.ts +36 -0
  155. package/src/bundled-plugins/memory/vector/hybrid.ts +25 -14
  156. package/src/bundled-plugins/memory/watermark.ts +19 -10
  157. package/src/bundled-plugins/reviewer/reviewer.ts +20 -0
  158. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +3 -6
  159. package/src/bundled-plugins/security/policies/outbound-text.ts +20 -0
  160. package/src/bundled-plugins/security/policies/private-surface-read.ts +166 -201
  161. package/src/bundled-plugins/security/policies/system-prompt-leak.ts +2 -7
  162. package/src/bundled-plugins/typevault/grants.ts +119 -0
  163. package/src/bundled-plugins/typevault/index.ts +152 -0
  164. package/src/bundled-plugins/typevault/request.ts +87 -0
  165. package/src/bundled-plugins/typevault/runtime.ts +45 -0
  166. package/src/bundled-plugins/typevault/skills/typevault/SKILL.md +65 -0
  167. package/src/channels/adapter-halt.ts +17 -0
  168. package/src/channels/adapters/discord-bot-reference.ts +5 -4
  169. package/src/channels/adapters/discord-bot.ts +1 -0
  170. package/src/channels/adapters/discord-classify.ts +8 -3
  171. package/src/channels/adapters/discord.ts +29 -0
  172. package/src/channels/adapters/github/history.ts +23 -2
  173. package/src/channels/adapters/github/inbound.ts +60 -1
  174. package/src/channels/adapters/github/index.ts +27 -5
  175. package/src/channels/adapters/github/reconcile-open-prs.ts +7 -1
  176. package/src/channels/adapters/kakaotalk-author-resolver.ts +7 -5
  177. package/src/channels/adapters/kakaotalk-channel-resolver.ts +45 -15
  178. package/src/channels/adapters/kakaotalk-classify.ts +7 -9
  179. package/src/channels/adapters/kakaotalk-list.ts +62 -0
  180. package/src/channels/adapters/kakaotalk-member-cache.ts +67 -0
  181. package/src/channels/adapters/kakaotalk-membership.ts +27 -10
  182. package/src/channels/adapters/kakaotalk-mention.ts +61 -0
  183. package/src/channels/adapters/kakaotalk.ts +159 -125
  184. package/src/channels/adapters/teams-classify.ts +18 -8
  185. package/src/channels/github-review-check.ts +5 -1
  186. package/src/channels/github-review-verdict-coordinator.ts +68 -2
  187. package/src/channels/github-token-bridge.ts +13 -0
  188. package/src/channels/lost-background-context.ts +46 -0
  189. package/src/channels/manager.ts +52 -3
  190. package/src/channels/persistence.ts +5 -0
  191. package/src/channels/router.ts +208 -128
  192. package/src/channels/schema.ts +1 -1
  193. package/src/channels/subagent-completion-bridge.ts +3 -2
  194. package/src/channels/types.ts +3 -0
  195. package/src/cli/hostd.ts +3 -0
  196. package/src/coding-agent/tool.ts +1 -1
  197. package/src/config/config.ts +35 -1
  198. package/src/config/providers.ts +75 -4
  199. package/src/container/start.ts +52 -3
  200. package/src/doctor/channel-checks.ts +33 -0
  201. package/src/doctor/checks.ts +37 -18
  202. package/src/doctor/index.ts +48 -12
  203. package/src/doctor/operational-incidents.ts +16 -1
  204. package/src/doctor/plugin-bridge.ts +14 -1
  205. package/src/doctor/stage.ts +14 -0
  206. package/src/doctor/types.ts +6 -0
  207. package/src/eval/cell.ts +93 -0
  208. package/src/eval/config.ts +32 -0
  209. package/src/eval/dispatch.ts +67 -0
  210. package/src/eval/entry.ts +261 -0
  211. package/src/eval/kernel.ts +154 -0
  212. package/src/eval/process.ts +113 -0
  213. package/src/eval/protocol.ts +26 -0
  214. package/src/eval/redeclare.ts +174 -0
  215. package/src/eval/sandbox.ts +117 -0
  216. package/src/eval/tool.ts +99 -0
  217. package/src/eval/wire.ts +93 -0
  218. package/src/git/resolve-identity.ts +67 -2
  219. package/src/git/system-commit.ts +7 -0
  220. package/src/init/dockerfile.ts +5 -1
  221. package/src/init/gitignore.ts +3 -3
  222. package/src/init/index.ts +1 -0
  223. package/src/init/models-dev.ts +2 -0
  224. package/src/init/validate-api-key.ts +5 -0
  225. package/src/migrations/secrets-v1-to-v2.ts +2 -0
  226. package/src/operations/classify-tool-outcome.ts +1 -1
  227. package/src/permissions/builtins.ts +8 -0
  228. package/src/plugin/core-tool-names.ts +6 -1
  229. package/src/plugin/types.ts +6 -0
  230. package/src/run/bundled-plugins.ts +12 -0
  231. package/src/run/channel-session-factory.ts +3 -0
  232. package/src/run/index.ts +13 -2
  233. package/src/sandbox/build.ts +1 -0
  234. package/src/sandbox/canonical-secrets.ts +6 -0
  235. package/src/sandbox/dependency-bins.ts +13 -1
  236. package/src/sandbox/dev-source.ts +21 -12
  237. package/src/sandbox/env-exposure.ts +20 -0
  238. package/src/sandbox/hidden-paths.ts +1 -1
  239. package/src/sandbox/index.ts +1 -0
  240. package/src/sandbox/policy.ts +5 -0
  241. package/src/secrets/hydrate.ts +4 -1
  242. package/src/secrets/storage.ts +10 -1
  243. package/src/server/command-runner.ts +4 -0
  244. package/src/server/index.ts +54 -12
  245. package/src/skills/typeclaw-config/SKILL.md +36 -2
  246. package/src/skills/typeclaw-memory/SKILL.md +3 -1
  247. package/src/skills/typeclaw-permissions/SKILL.md +8 -8
  248. package/src/skills/typeclaw-troubleshooting/SKILL.md +18 -1
  249. package/src/tunnels/active.ts +14 -0
  250. package/src/typevault/bash-cli.ts +18 -0
  251. package/src/typevault/cli-path.ts +11 -0
  252. package/src/typevault/envelope-helper.ts +17 -0
  253. package/src/typevault/envelope-store.ts +98 -0
  254. package/src/typevault/lifecycle.ts +132 -0
  255. package/src/typevault/prompt-tunnel.ts +33 -0
  256. package/src/typevault/redaction.ts +37 -0
  257. package/src/typevault/vault.ts +283 -0
  258. package/typeclaw.schema.json +92 -10
@@ -1,16 +1,32 @@
1
1
  import { z } from 'zod'
2
2
 
3
+ import { resolveFallbackChain } from '@/agent/model-fallback'
4
+ import { DEFAULT_MAX_OUTPUT_TOKENS, defaultOutputAllowance } from '@/agent/output-allowance'
3
5
  import type { SessionOrigin } from '@/agent/session-origin'
6
+ import { getConfig, resolveModel } from '@/config'
4
7
  import { type Subagent, readTool } from '@/plugin'
5
8
  import { formatLocalDate } from '@/shared'
6
9
 
7
- import { advanceWatermarkTool, createAppendTool, type FragmentsAppendedHook } from './append-tool'
10
+ import {
11
+ advanceWatermarkTool,
12
+ createAdvanceWatermarkTool,
13
+ createAppendTool,
14
+ type FragmentsAppendedHook,
15
+ } from './append-tool'
8
16
  import { findEntryTool } from './find-entry-tool'
9
17
  import { formatMemorySnapshot } from './memory-snapshot'
10
18
  import { streamFilePath, streamsDir } from './paths'
11
19
  import { listReferenceSlugs } from './references/load-references'
12
20
  import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
13
21
  import { readEvents } from './stream-io'
22
+ import { decodeTranscriptCursor } from './transcript-cursor'
23
+ import {
24
+ buildTranscriptPreload,
25
+ PRELOAD_FRAME_BYTES,
26
+ preloadBudgetBytes,
27
+ type PreloadModelLimits,
28
+ type TranscriptPreload,
29
+ } from './transcript-preload'
14
30
  import { readLatestWatermark } from './watermark'
15
31
 
16
32
  export const memoryLoggerPayloadSchema = z.object({
@@ -76,6 +92,10 @@ Tools: \`read\`, \`find_entry\`, \`append\`, \`store_reference\`, and the waterm
76
92
 
77
93
  # Read loop, watermark, and stopping
78
94
 
95
+ An \`entry-part\` may include \`precedingText\`: read-only source overlap immediately before its new \`text\`. Interpret them together to recover instructions crossing a boundary. Capture complete facts supported by the new segment; do not emit a fragment for overlap alone. Only the issued page checkpoint advances progress; overlap never does.
96
+
97
+ The runtime usually preloads a bounded transcript page into your first message. Evaluate that page directly. An \`entry-part\` contains an exact text segment of one large projected source record; its \`id\` is the original evidence anchor. Never treat a partial record as fully consumed. When a page supplies a \`memory-page-checkpoint\`, finish all fragment/reference writes first, then call the watermark-advance tool exactly once with that checkpoint as \`latestEntryId\` and exit. An append does not complete the page. The runtime supplies later pages in fresh sessions; do not use \`read\` or \`find_entry\` to bypass paging. Without a preload, use the read loop below.
98
+
79
99
  Session transcripts are JSONL; each line has an \`id\`. They can be large, and \`read\` truncates output to 50 KB or 2000 lines, returning the line range and next offset. Do not scroll from line 1 through a prefix already covered by the watermark.
80
100
 
81
101
  When a watermark is set, always use \`find_entry\` before \`read\`. It finds the line whose own \`id\` equals the entry id (not \`parentId\`), returns \`line=N, totalLines=T, offset=N+1\`, and lets you resume immediately after the watermark. If it returns "not found" (for example, a compacted parent session), start from \`offset=1\` or, if the transcript is huge and clearly unrelated, write the watermark forward and skip.
@@ -151,6 +171,8 @@ Skip these anti-patterns:
151
171
  - **The agent's own first-person observations.** The agent's persona, model confusion, or self-commentary is not memorable to itself.
152
172
  - **Re-derivable facts.** Anything obvious from the current system prompt, AGENTS.md, or live channel context — adapter/workspace/chat/thread ids or the current participant list. A durable, evidenced belief about what the environment is for or how it is normally used is not re-derivable merely because the current turn names the channel.
153
173
  - **Speculation untethered to a quote.** If no transcript line anchors it, skip.
174
+ - **Unverified agent conclusions and open-investigation status.** The agent's own inference, guess, or "probably X" is not a fact until the user confirms it or the transcript shows decisive evidence (a tool result, document, or record that settles it); never write it as an established finding. Skip in-progress status too: "X is still unknown", "Y was not found", "Z remains unverified". It goes stale the moment the investigation moves on, and a later session reads it as a settled negative and stops looking. Once an investigation reaches a confirmed answer, capture that answer with its evidence.
175
+ - **A user's doubt is not a refutation.** A correction supplies a fact the user knows ("that account is mine", "the invoice was paid in March") and is captured as above. A doubt or objection only questions an inference without supplying the fact ("but I never pay that myself", "that seems off"). Do not turn a doubt into a stored fact that the inference is false, and do not store the agent's retraction as the conclusion. If the doubt carries a durable fact about the user, store only that fact, attributed to the user, without the inference it was aimed at.
154
176
  - **Multi-fragment expansions of one event.** One event produces at most one fragment. A teammate introduction may bundle that person's core durable role/responsibility/preference into one compact fragment, but do not split "new chat", "new participant", "job", "preference", "reaction" into separate memories.
155
177
 
156
178
  # Verbatim references (store_reference tool)
@@ -180,6 +202,8 @@ The \`append\` tool will refuse content containing a recognizable credential pat
180
202
 
181
203
  # Local dedup against today's daily stream
182
204
 
205
+ Facts already saved by a successful \`memory_remember\` tool call in the transcript are already in the stream; do not append them again.
206
+
183
207
  The \`append\` tool refuses byte-equivalent fragments within the same daily stream — content-equality on topic+body modulo whitespace, not marker-equality. If it rejects a fragment already in the same daily stream, rewrite or skip.
184
208
 
185
209
  Do not fight this refusal by changing punctuation or adding filler. If the new transcript only repeats exactly the same fact with no new evidence worth preserving, skip. If it is a true recurrence, rewrite the body to anchor the new occurrence explicitly, so dreaming can see why this line is new evidence rather than a duplicate copy.
@@ -221,7 +245,12 @@ This restricts fragment shape, not whether taught knowledge is captured: store t
221
245
 
222
246
  If you evaluated transcript entries and found nothing worth a fragment, call the watermark-advance tool with \`{source, latestEntryId}\` so the next run does not re-read the same prefix. Do not call \`append\` with fake content just to move the watermark. After your fragment(s) or zero-fragments path advances the watermark to the farthest evaluated entry and the transcript is exhausted, stop with no completion message.`
223
247
 
224
- function buildInitialPrompt(payload: MemoryLoggerPayload, streamFile: string, watermark: string | null): string {
248
+ function buildInitialPrompt(
249
+ payload: MemoryLoggerPayload,
250
+ streamFile: string,
251
+ watermark: string | null,
252
+ preload: TranscriptPreload | null,
253
+ ): string {
225
254
  const lines: string[] = [
226
255
  `Parent session: ${payload.parentSessionId}`,
227
256
  `Transcript file: ${payload.parentTranscriptPath}`,
@@ -249,9 +278,24 @@ function buildInitialPrompt(payload: MemoryLoggerPayload, streamFile: string, wa
249
278
  payload.parentSessionId +
250
279
  '", latestEntryId: "<latestEntryId>" }` instead of a fake fragment.',
251
280
  )
281
+ if (preload !== null) lines.push('', renderPreload(preload))
252
282
  return lines.join('\n')
253
283
  }
254
284
 
285
+ function renderPreload(preload: TranscriptPreload): string {
286
+ if (preload.endLine < preload.startLine) {
287
+ return `Preloaded transcript: nothing past the watermark (totalLines=${preload.totalLines}). Advance nothing; exit.`
288
+ }
289
+ const header = [
290
+ `Preloaded transcript lines ${preload.startLine}-${preload.endLine} of totalLines=${preload.totalLines}${preload.watermarkFound ? ' (starting right after the watermark)' : ''}.`,
291
+ preload.nextOffset === null
292
+ ? 'This reaches the end of the transcript; do not call read on it.'
293
+ : 'More lines remain: the runtime will supply the next page after this page is committed.',
294
+ `After all fragment writes, complete this page with latestEntryId from <memory-page-checkpoint>${preload.checkpoint}</memory-page-checkpoint>.`,
295
+ ]
296
+ return [...header, '<transcript>', preload.text, '</transcript>'].join('\n')
297
+ }
298
+
255
299
  function renderConversationContext(origin: SessionOrigin | undefined): string | null {
256
300
  if (origin === undefined) return null
257
301
  if (origin.kind !== 'channel') return ['Conversation context:', `- Origin: ${origin.kind}`].join('\n')
@@ -296,24 +340,98 @@ export type CreateMemoryLoggerSubagentOptions = {
296
340
  logger?: MemoryLoggerLogger
297
341
  onFragmentsAppended?: FragmentsAppendedHook
298
342
  onReferenceStored?: ReferenceStoredHook
343
+ // Context window and request output allowance of every model the `fast`
344
+ // profile can fall back to.
345
+ fastModelLimits?: () => PreloadModelLimits[]
346
+ }
347
+
348
+ // memory-logger is spawned under a `system` parent origin, so channel model
349
+ // overrides never apply and the global `fast` chain is the one it runs on.
350
+ function defaultFastModelLimits(): PreloadModelLimits[] {
351
+ return resolveFallbackChain(getConfig().models, 'fast').map((ref) => {
352
+ const model = resolveModel(ref)
353
+ return {
354
+ contextWindow: model.contextWindow,
355
+ outputTokens: defaultOutputAllowance(model) ?? DEFAULT_MAX_OUTPUT_TOKENS,
356
+ }
357
+ })
299
358
  }
300
359
 
301
360
  export function createMemoryLoggerSubagent(
302
361
  options: CreateMemoryLoggerSubagentOptions = {},
303
362
  ): Subagent<MemoryLoggerPayload> {
304
363
  const logger = options.logger ?? consoleLogger
364
+ const fastModelLimits = options.fastModelLimits ?? defaultFastModelLimits
305
365
  // The handler runs one logger spawn at a time per agentDir (inFlightKey), so a
306
366
  // single mutable cell safely carries this spawn's origin to the append tool.
307
367
  let currentOrigin: SessionOrigin | undefined
308
368
  let currentTranscriptPath: string | undefined
369
+ let page:
370
+ | {
371
+ source: string
372
+ checkpoint: string | null
373
+ entries: Set<string>
374
+ committed: boolean
375
+ closed: boolean
376
+ pending: number
377
+ }
378
+ | undefined
379
+ const validateCheckpoint = (source: string, checkpoint: string, entry?: string): void => {
380
+ if (
381
+ page !== undefined &&
382
+ (page.closed ||
383
+ page.checkpoint === null ||
384
+ source !== page.source ||
385
+ checkpoint !== page.checkpoint ||
386
+ (entry === undefined && page.pending > 0) ||
387
+ (entry !== undefined && !page.entries.has(entry)))
388
+ ) {
389
+ throw new Error('Memory checkpoint must match the active page; unseen or incomplete evidence cannot be skipped.')
390
+ }
391
+ // Reserve completion before its first I/O await so a parallel tool call
392
+ // cannot start writing evidence behind the durable checkpoint.
393
+ if (page !== undefined && entry === undefined) page.closed = true
394
+ }
395
+ const beginWrite = (): (() => void) | undefined => {
396
+ const active = page
397
+ if (active === undefined) return undefined
398
+ if (active.closed) throw new Error('Memory page is closed; stale writes cannot continue.')
399
+ active.pending++
400
+ return () => {
401
+ active.pending--
402
+ }
403
+ }
309
404
  const appendTool = createAppendTool({
310
405
  onFragmentsAppended: options.onFragmentsAppended,
311
406
  originProvider: () => currentOrigin,
312
407
  referenceSlugResolver: (agentDir) => listReferenceSlugs(agentDir),
313
408
  transcriptPathProvider: () => currentTranscriptPath,
409
+ validateCheckpoint,
410
+ deferWatermark: () => page !== undefined,
411
+ beginWrite,
412
+ })
413
+ const referenceTool = createStoreReferenceTool(options.onReferenceStored)
414
+ const storeReferenceTool = {
415
+ ...referenceTool,
416
+ execute: async (...args: Parameters<typeof referenceTool.execute>) => {
417
+ const finishWrite = beginWrite()
418
+ try {
419
+ return await referenceTool.execute(...args)
420
+ } finally {
421
+ finishWrite?.()
422
+ }
423
+ },
424
+ }
425
+ const pageWatermarkTool = createAdvanceWatermarkTool({
426
+ validateCheckpoint,
427
+ onAdvanced: () => {
428
+ if (page !== undefined) {
429
+ page.committed = true
430
+ page.closed = true
431
+ }
432
+ },
314
433
  })
315
- const storeReferenceTool = createStoreReferenceTool(options.onReferenceStored)
316
- const customTools = [findEntryTool, appendTool, storeReferenceTool, advanceWatermarkTool]
434
+ const customTools = [findEntryTool, appendTool, storeReferenceTool, pageWatermarkTool]
317
435
  return {
318
436
  systemPrompt: MEMORY_LOGGER_SYSTEM_PROMPT,
319
437
  // Logging is "read transcript past the watermark, decide 0-N fragments,
@@ -342,19 +460,93 @@ export function createMemoryLoggerSubagent(
342
460
  exhaustedMessage: memoryLoggerExhaustedMessage,
343
461
  },
344
462
  handler: async (ctx, runSession) => {
463
+ const preloadFor = async (
464
+ path: string,
465
+ mark: string | null,
466
+ basePrompt: string,
467
+ untilLine?: number,
468
+ ): Promise<TranscriptPreload | null> => {
469
+ let models: PreloadModelLimits[]
470
+ try {
471
+ models = fastModelLimits()
472
+ } catch (err) {
473
+ logger.warn(
474
+ `[memory-logger] preload skipped: cannot resolve fast model limits: ${err instanceof Error ? err.message : String(err)}`,
475
+ )
476
+ return null
477
+ }
478
+ const promptBytes =
479
+ Buffer.byteLength(MEMORY_LOGGER_SYSTEM_PROMPT, 'utf8') +
480
+ Buffer.byteLength(basePrompt, 'utf8') +
481
+ PRELOAD_FRAME_BYTES
482
+ let maxBytes = preloadBudgetBytes(models, promptBytes)
483
+ let endLine = untilLine
484
+ while (maxBytes !== null) {
485
+ const preload = await buildTranscriptPreload(path, mark, { maxBytes, untilLine: endLine })
486
+ if (preload === null) return null
487
+ const frameBytes = Buffer.byteLength(renderPreload(preload), 'utf8') - Buffer.byteLength(preload.text, 'utf8')
488
+ const fitted = preloadBudgetBytes(models, promptBytes - PRELOAD_FRAME_BYTES + frameBytes)
489
+ if (fitted === null) return null
490
+ if (Buffer.byteLength(preload.text, 'utf8') <= fitted) return preload
491
+ maxBytes = fitted
492
+ endLine ??= preload.totalLines
493
+ }
494
+ return null
495
+ }
345
496
  currentOrigin = ctx.payload.origin
346
497
  currentTranscriptPath = ctx.payload.parentTranscriptPath
347
498
  const today = formatLocalDate()
348
499
  const memoryDir = streamsDir(ctx.payload.agentDir)
349
500
  const streamFile = streamFilePath(ctx.payload.agentDir, today)
350
- const watermark = await readLatestWatermark(memoryDir, ctx.payload.parentSessionId)
501
+ let watermark = await readLatestWatermark(memoryDir, ctx.payload.parentSessionId)
351
502
  const fragmentsBefore = await countFragments(streamFile)
352
503
  const start = Date.now()
353
504
  logger.info(
354
505
  `[memory-logger] ${ctx.payload.parentSessionId} start stream=${today}.jsonl watermark=${watermark ?? 'none'}`,
355
506
  )
356
507
  try {
357
- await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark) })
508
+ let untilLine: number | undefined
509
+ while (true) {
510
+ const basePrompt = buildInitialPrompt(ctx.payload, streamFile, watermark, null)
511
+ const preload = await preloadFor(ctx.payload.parentTranscriptPath, watermark, basePrompt, untilLine)
512
+ if (preload === null && decodeTranscriptCursor(watermark) !== null) {
513
+ logger.warn(
514
+ `[memory-logger] ${ctx.payload.parentSessionId} partial transcript cursor remains pending: paging is unavailable`,
515
+ )
516
+ break
517
+ }
518
+ untilLine ??= preload?.totalLines
519
+ if (preload !== null) {
520
+ if (preload.startCheckpoint !== null) {
521
+ await advanceWatermarkTool.execute(
522
+ { source: ctx.payload.parentSessionId, latestEntryId: preload.startCheckpoint },
523
+ { agentDir: ctx.agentDir, sessionId: ctx.payload.parentSessionId, signal: undefined, logger },
524
+ )
525
+ }
526
+ page = {
527
+ source: ctx.payload.parentSessionId,
528
+ checkpoint: preload.entryIds.length === 0 ? null : preload.checkpoint,
529
+ entries: new Set(preload.entryIds),
530
+ committed: false,
531
+ closed: false,
532
+ pending: 0,
533
+ }
534
+ } else {
535
+ page = undefined
536
+ }
537
+ try {
538
+ await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark, preload) })
539
+ } finally {
540
+ if (page !== undefined) page.closed = true
541
+ }
542
+ if (!page?.committed || preload?.nextOffset === null) {
543
+ if (page && page.entries.size > 0 && !page.committed) {
544
+ logger.warn(`[memory-logger] ${ctx.payload.parentSessionId} page not committed; evidence remains pending`)
545
+ }
546
+ break
547
+ }
548
+ watermark = page.checkpoint
549
+ }
358
550
  const fragmentsWritten = (await countFragments(streamFile)) - fragmentsBefore
359
551
  logger.info(
360
552
  `[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
@@ -365,6 +557,8 @@ export function createMemoryLoggerSubagent(
365
557
  `[memory-logger] ${ctx.payload.parentSessionId}: run threw: ${message} elapsed_ms=${Date.now() - start}`,
366
558
  )
367
559
  throw err
560
+ } finally {
561
+ if (page !== undefined) page.closed = true
368
562
  }
369
563
  },
370
564
  }
@@ -6,7 +6,7 @@ import { defineTool } from '@/plugin'
6
6
 
7
7
  import { referenceFilePath, referencesDir } from '../paths'
8
8
  import { headingToSlug } from '../slug'
9
- import { renderReference } from './frontmatter'
9
+ import { renderReference, type ReferenceOrigin } from './frontmatter'
10
10
  import { listReferenceSlugs } from './load-references'
11
11
 
12
12
  export type ReferenceStoredHook = (context: { slug: string; body: string }) => Promise<void>
@@ -27,37 +27,61 @@ export function createStoreReferenceTool(onReferenceStored?: ReferenceStoredHook
27
27
  // path; none is a caller-supplied file path. Declared so a title like
28
28
  // "README.md" or a body carrying "/" is not misread as a file operand.
29
29
  fileOperands: { nonFile: ['title', 'body', 'tags'] },
30
- async execute({ title, body, origin, tags }, ctx) {
31
- const existingSlugs = new Set(await listReferenceSlugs(ctx.agentDir))
32
- const slug = headingToSlug(title, existingSlugs)
33
- const created = new Date().toISOString()
34
- const path = referenceFilePath(ctx.agentDir, slug)
35
-
36
- await mkdir(referencesDir(ctx.agentDir), { recursive: true })
37
- await writeFile(
38
- path,
39
- renderReference(
40
- {
41
- title,
42
- origin,
43
- created,
44
- lastAccessed: created,
45
- accessCount: 0,
46
- pinned: false,
47
- demoted: false,
48
- tags: tags ?? [],
49
- },
50
- body,
51
- ),
52
- 'utf8',
53
- )
54
-
55
- if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
56
-
57
- return {
58
- content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
59
- details: { path, slug },
60
- }
30
+ async execute(input, ctx) {
31
+ return storeReference(input, ctx.agentDir, onReferenceStored)
61
32
  },
62
33
  })
63
34
  }
35
+
36
+ export async function storeReference(
37
+ input: {
38
+ readonly title: string
39
+ readonly body: string
40
+ readonly origin: ReferenceOrigin
41
+ readonly tags?: string[] | undefined
42
+ },
43
+ agentDir: string,
44
+ onReferenceStored?: ReferenceStoredHook,
45
+ ) {
46
+ const { title, body, origin, tags } = input
47
+ const existingSlugs = new Set(await listReferenceSlugs(agentDir))
48
+ const created = new Date().toISOString()
49
+ const content = renderReference(
50
+ {
51
+ title,
52
+ origin,
53
+ created,
54
+ lastAccessed: created,
55
+ accessCount: 0,
56
+ pinned: false,
57
+ demoted: false,
58
+ tags: tags ?? [],
59
+ },
60
+ body,
61
+ )
62
+
63
+ await mkdir(referencesDir(agentDir), { recursive: true })
64
+ // Concurrent writers (memory-logger and every main-agent session) can pick
65
+ // the same slug from the same listing, so the file is reserved with an
66
+ // exclusive create and a taken slug moves to the next deduplicated suffix.
67
+ let slug = headingToSlug(title, existingSlugs)
68
+ let path = referenceFilePath(agentDir, slug)
69
+ for (;;) {
70
+ try {
71
+ await writeFile(path, content, { encoding: 'utf8', flag: 'wx' })
72
+ break
73
+ } catch (err) {
74
+ if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST')) throw err
75
+ existingSlugs.add(slug)
76
+ slug = headingToSlug(title, existingSlugs)
77
+ path = referenceFilePath(agentDir, slug)
78
+ }
79
+ }
80
+
81
+ if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
82
+
83
+ return {
84
+ content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
85
+ details: { path, slug },
86
+ }
87
+ }
@@ -0,0 +1,81 @@
1
+ import { z } from 'zod'
2
+
3
+ import { defineTool } from '@/plugin'
4
+ import { formatLocalDate } from '@/shared'
5
+
6
+ import { appendFragment, type FragmentsAppendedHook } from './append-fragment'
7
+ import { provenanceFromOrigin } from './append-tool'
8
+ import { streamFilePath } from './paths'
9
+ import { listReferenceSlugs } from './references/load-references'
10
+ import { storeReference, type ReferenceStoredHook } from './references/store-reference-tool'
11
+
12
+ export function createMemoryRememberTool(
13
+ options: {
14
+ readonly onFragmentsAppended?: FragmentsAppendedHook
15
+ readonly onReferenceStored?: ReferenceStoredHook
16
+ } = {},
17
+ ) {
18
+ return defineTool({
19
+ description:
20
+ 'Remember a durable fact, decision, preference, or correction now. Use when asked to remember something or when durable knowledge is established, not for routine chatter. Supply a short topic and self-contained body in the evidence language; who optionally names the person it is attributable to. Set verbatim to also retain the exact body as a linked reference. The runtime stamps provenance and writes one daily-stream fragment; dreaming consolidates it later.',
21
+ parameters: z.object({
22
+ topic: z.string().min(1),
23
+ body: z.string().min(1),
24
+ who: z.string().min(1).optional(),
25
+ verbatim: z.boolean().optional(),
26
+ }),
27
+ // All arguments are content/control values, never caller-selected paths.
28
+ fileOperands: { nonFile: ['topic', 'body', 'who', 'verbatim'] },
29
+ async execute({ topic, body, who, verbatim }, ctx) {
30
+ const path = streamFilePath(ctx.agentDir, formatLocalDate())
31
+ const { fragment } = await appendFragment(
32
+ {
33
+ topic,
34
+ body,
35
+ who,
36
+ source: ctx.sessionId,
37
+ // An empty anchor is honest for direct calls without a transcript.
38
+ entry: ctx.entryId ?? '',
39
+ kind: 'explicit',
40
+ },
41
+ ctx,
42
+ {
43
+ path,
44
+ where: provenanceFromOrigin(ctx.origin),
45
+ onFragmentsAppended: options.onFragmentsAppended,
46
+ referenceSlugResolver: listReferenceSlugs,
47
+ ...(verbatim
48
+ ? {
49
+ storeReference: async () => {
50
+ // Indexing is best-effort once the reference is persisted, like
51
+ // the fragment hook: a failure must not orphan the stored file
52
+ // or drop the explicit fragment that cites it.
53
+ const { onReferenceStored } = options
54
+ const result = await storeReference(
55
+ { title: topic, body, origin: 'episode' },
56
+ ctx.agentDir,
57
+ onReferenceStored === undefined
58
+ ? undefined
59
+ : async (reference) => {
60
+ try {
61
+ await onReferenceStored(reference)
62
+ } catch (err) {
63
+ ctx.logger.warn(
64
+ `[memory] post-store reference index hook failed: ${err instanceof Error ? err.message : String(err)}`,
65
+ )
66
+ }
67
+ },
68
+ )
69
+ return result.details.slug
70
+ },
71
+ }
72
+ : {}),
73
+ },
74
+ )
75
+ return {
76
+ content: [{ type: 'text' as const, text: `Remembered "${topic}".` }],
77
+ details: { path, fragmentId: fragment.id, references: fragment.references ?? [] },
78
+ }
79
+ },
80
+ })
81
+ }
@@ -0,0 +1,91 @@
1
+ import { createHash } from 'node:crypto'
2
+
3
+ import { loadShard } from './load-shards'
4
+ import { loadReference } from './references/load-references'
5
+ import type { MemorySearchMatch, StreamMatch } from './search-tool'
6
+ import { readAllStreamDays } from './stream-io'
7
+ import type { HybridSearchResult } from './vector/hybrid'
8
+
9
+ // Hybrid results carry excerpts and provenance, while the tool also exposes
10
+ // canonical paths, citation IDs and optional complete bodies.
11
+ export async function hydrateHybridMatches(
12
+ agentDir: string,
13
+ results: HybridSearchResult[],
14
+ full: boolean,
15
+ ): Promise<MemorySearchMatch[]> {
16
+ const streams = new Map<string, StreamMatch>()
17
+ if (results.some((hit) => hit.source === 'stream')) {
18
+ for (const day of await readAllStreamDays(agentDir)) {
19
+ for (const event of day.events) {
20
+ const base = { source: 'stream', streamPath: day.path, date: day.date, excerpt: '' } as const
21
+ switch (event.type) {
22
+ case 'fragment':
23
+ streams.set(`${day.date}#${event.id}`, {
24
+ ...base,
25
+ topic: event.topic,
26
+ eventId: `streams/${day.date}#${event.id}`,
27
+ ...(full ? { fullBody: event.body } : {}),
28
+ })
29
+ break
30
+ case 'legacy_prose':
31
+ streams.set(`${day.date}#legacy-${createHash('sha256').update(event.text).digest('hex').slice(0, 12)}`, {
32
+ ...base,
33
+ topic: '[legacy prose from pre-shard migration]',
34
+ ...(full ? { fullBody: event.text } : {}),
35
+ })
36
+ break
37
+ case 'watermark':
38
+ break
39
+ default:
40
+ event satisfies never
41
+ }
42
+ }
43
+ }
44
+ }
45
+
46
+ const matches = await Promise.all(
47
+ results.map(async (hit): Promise<MemorySearchMatch | null> => {
48
+ switch (hit.source) {
49
+ case 'topic': {
50
+ const shard = await loadShard(agentDir, hit.key)
51
+ if (shard === null) return null
52
+ return {
53
+ source: 'topic',
54
+ shardPath: shard.path,
55
+ slug: hit.key,
56
+ heading: hit.heading,
57
+ excerpt: hit.excerpt,
58
+ ...(hit.provenance === undefined ? {} : { provenance: hit.provenance }),
59
+ ...(full ? { fullBody: shard.body } : {}),
60
+ }
61
+ }
62
+ case 'reference': {
63
+ const reference = await loadReference(agentDir, hit.key)
64
+ if (reference === null) return null
65
+ return {
66
+ source: 'reference',
67
+ slug: hit.key,
68
+ title: hit.heading,
69
+ excerpt: hit.excerpt,
70
+ created: reference.frontmatter.created,
71
+ ...(full ? { fullBody: reference.body } : {}),
72
+ }
73
+ }
74
+ case 'stream': {
75
+ const stream = streams.get(hit.key)
76
+ if (stream === undefined) return null
77
+ return {
78
+ ...stream,
79
+ excerpt: hit.excerpt,
80
+ ...(hit.who === undefined ? {} : { who: hit.who }),
81
+ ...(hit.when === undefined ? {} : { when: hit.when }),
82
+ ...(hit.where === undefined ? {} : { where: hit.where }),
83
+ }
84
+ }
85
+ default:
86
+ return hit.source satisfies never
87
+ }
88
+ }),
89
+ )
90
+ return matches.filter((match) => match !== null)
91
+ }