typeclaw 0.51.0 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/dist/types/agent/bash-sandbox-env.d.ts +2 -0
  2. package/dist/types/agent/bash-sandbox.d.ts +4 -0
  3. package/dist/types/agent/index.d.ts +5 -0
  4. package/dist/types/agent/multimodal/look-at.d.ts +1 -1
  5. package/dist/types/agent/plugin-tools.d.ts +4 -0
  6. package/dist/types/agent/prompt-cache-key.d.ts +1 -0
  7. package/dist/types/agent/session-tail.d.ts +12 -0
  8. package/dist/types/agent/subagents.d.ts +5 -0
  9. package/dist/types/agent/system-prompt-composition.d.ts +1 -0
  10. package/dist/types/agent/system-prompt.d.ts +2 -0
  11. package/dist/types/agent/tools/channel-disengage.d.ts +1 -1
  12. package/dist/types/agent/tools/channel-edit.d.ts +1 -1
  13. package/dist/types/agent/tools/channel-fetch-attachment.d.ts +1 -1
  14. package/dist/types/agent/tools/channel-handoff.d.ts +1 -1
  15. package/dist/types/agent/tools/channel-history.d.ts +1 -1
  16. package/dist/types/agent/tools/channel-react.d.ts +1 -1
  17. package/dist/types/agent/tools/channel-read.d.ts +10 -2
  18. package/dist/types/agent/tools/channel-reply.d.ts +1 -1
  19. package/dist/types/agent/tools/channel-send.d.ts +1 -1
  20. package/dist/types/agent/tools/channel-session-search.d.ts +1 -1
  21. package/dist/types/agent/tools/grant-role.d.ts +1 -1
  22. package/dist/types/agent/tools/post-github-review.d.ts +1 -1
  23. package/dist/types/agent/tools/restart.d.ts +1 -1
  24. package/dist/types/agent/tools/skip-response.d.ts +1 -1
  25. package/dist/types/agent/tools/spawn-subagent.d.ts +2 -1
  26. package/dist/types/agent/tools/stream-snapshot.d.ts +1 -1
  27. package/dist/types/agent/tools/subagent-cancel.d.ts +1 -1
  28. package/dist/types/agent/tools/subagent-output.d.ts +1 -1
  29. package/dist/types/agent/tools/todo/index.d.ts +1 -1
  30. package/dist/types/agent/tools/webfetch/adaptive-html.d.ts +3 -3
  31. package/dist/types/agent/tools/webfetch/strategies/readability-process.d.ts +48 -0
  32. package/dist/types/agent/tools/webfetch/tool.d.ts +2 -2
  33. package/dist/types/agent/tools/websearch.d.ts +1 -1
  34. package/dist/types/channels/adapters/github/auth-app.d.ts +25 -0
  35. package/dist/types/channels/adapters/github/auth-pat.d.ts +17 -0
  36. package/dist/types/channels/adapters/github/auth.d.ts +24 -0
  37. package/dist/types/channels/github-token-bridge.d.ts +4 -0
  38. package/dist/types/channels/lost-background-context.d.ts +2 -0
  39. package/dist/types/channels/persistence.d.ts +1 -0
  40. package/dist/types/channels/router.d.ts +5 -1
  41. package/dist/types/channels/types.d.ts +1 -0
  42. package/dist/types/coding-agent/tool.d.ts +1 -1
  43. package/dist/types/config/config.d.ts +20 -11
  44. package/dist/types/config/providers.d.ts +47 -0
  45. package/dist/types/eval/sandbox.d.ts +2 -0
  46. package/dist/types/git/resolve-identity.d.ts +2 -0
  47. package/dist/types/init/dockerfile.d.ts +1 -0
  48. package/dist/types/init/gitignore.d.ts +1 -1
  49. package/dist/types/permissions/builtins.d.ts +1 -0
  50. package/dist/types/plugin/types.d.ts +3 -0
  51. package/dist/types/sandbox/canonical-secrets.d.ts +2 -2
  52. package/dist/types/sandbox/coding-agent.d.ts +1 -1
  53. package/dist/types/sandbox/dependency-bins.d.ts +1 -0
  54. package/dist/types/sandbox/dev-source.d.ts +2 -2
  55. package/dist/types/sandbox/hidden-paths.d.ts +1 -0
  56. package/dist/types/sandbox/index.d.ts +1 -1
  57. package/dist/types/sandbox/policy.d.ts +3 -0
  58. package/dist/types/typevault/cli-path.d.ts +2 -0
  59. package/dist/types/typevault/envelope-store.d.ts +11 -0
  60. package/dist/types/typevault/lifecycle.d.ts +13 -0
  61. package/dist/types/typevault/vault.d.ts +27 -0
  62. package/package.json +8 -7
  63. package/scripts/test.ts +252 -0
  64. package/src/agent/bash-sandbox-env.ts +33 -1
  65. package/src/agent/bash-sandbox.ts +20 -6
  66. package/src/agent/index.ts +120 -31
  67. package/src/agent/multimodal/look-at.ts +1 -1
  68. package/src/agent/plugin-tools.ts +35 -1
  69. package/src/agent/prompt-cache-key.ts +26 -3
  70. package/src/agent/session-origin.ts +39 -18
  71. package/src/agent/session-tail.ts +28 -0
  72. package/src/agent/subagents.ts +21 -2
  73. package/src/agent/system-prompt-composition.ts +18 -0
  74. package/src/agent/system-prompt.ts +39 -1
  75. package/src/agent/tool-error-remediation.ts +18 -3
  76. package/src/agent/tools/channel-disengage.ts +1 -1
  77. package/src/agent/tools/channel-edit.ts +1 -1
  78. package/src/agent/tools/channel-fetch-attachment.ts +1 -1
  79. package/src/agent/tools/channel-handoff.ts +3 -2
  80. package/src/agent/tools/channel-history.ts +2 -1
  81. package/src/agent/tools/channel-react.ts +1 -1
  82. package/src/agent/tools/channel-read.ts +36 -3
  83. package/src/agent/tools/channel-reply.ts +1 -1
  84. package/src/agent/tools/channel-send.ts +4 -3
  85. package/src/agent/tools/channel-session-search.ts +13 -4
  86. package/src/agent/tools/grant-role.ts +1 -1
  87. package/src/agent/tools/post-github-review.ts +1 -1
  88. package/src/agent/tools/restart.ts +1 -1
  89. package/src/agent/tools/skip-response.ts +1 -1
  90. package/src/agent/tools/spawn-subagent.ts +15 -1
  91. package/src/agent/tools/stream-snapshot.ts +1 -1
  92. package/src/agent/tools/subagent-cancel.ts +1 -1
  93. package/src/agent/tools/subagent-output.ts +1 -1
  94. package/src/agent/tools/todo/index.ts +1 -1
  95. package/src/agent/tools/webfetch/adaptive-html.ts +6 -4
  96. package/src/agent/tools/webfetch/strategies/readability-child.ts +25 -0
  97. package/src/agent/tools/webfetch/strategies/readability-process.ts +185 -0
  98. package/src/agent/tools/webfetch/tool.ts +6 -5
  99. package/src/agent/tools/websearch.ts +1 -1
  100. package/src/browser/dockerfile.ts +1 -1
  101. package/src/browser/fill-secret.ts +92 -35
  102. package/src/browser/registry.ts +6 -6
  103. package/src/bundled-plugins/agent-messenger-cli-auth/command.ts +211 -0
  104. package/src/bundled-plugins/agent-messenger-cli-auth/index.ts +186 -0
  105. package/src/bundled-plugins/backup/index.ts +17 -2
  106. package/src/bundled-plugins/backup/runner.ts +37 -2
  107. package/src/bundled-plugins/browser-agent/browser-agent.ts +51 -14
  108. package/src/bundled-plugins/browser-agent/index.ts +12 -5
  109. package/src/bundled-plugins/managed-browser/index.ts +25 -2
  110. package/src/bundled-plugins/managed-browser/skills/browserskill/browserskill/SKILL.md +13 -5
  111. package/src/bundled-plugins/managed-browser/skills/omowright/omowright/SKILL.md +56 -5
  112. package/src/bundled-plugins/memory/README.md +23 -2
  113. package/src/bundled-plugins/memory/append-fragment.ts +133 -0
  114. package/src/bundled-plugins/memory/append-tool.ts +80 -149
  115. package/src/bundled-plugins/memory/dreaming.ts +2 -0
  116. package/src/bundled-plugins/memory/index.ts +33 -6
  117. package/src/bundled-plugins/memory/memory-logger.ts +133 -11
  118. package/src/bundled-plugins/memory/references/store-reference-tool.ts +56 -32
  119. package/src/bundled-plugins/memory/remember-tool.ts +81 -0
  120. package/src/bundled-plugins/memory/search-tool-hybrid.ts +91 -0
  121. package/src/bundled-plugins/memory/search-tool.ts +58 -12
  122. package/src/bundled-plugins/memory/stream-events.ts +2 -0
  123. package/src/bundled-plugins/memory/transcript-cursor.ts +30 -0
  124. package/src/bundled-plugins/memory/transcript-preload.ts +131 -26
  125. package/src/bundled-plugins/memory/transcript-view.ts +36 -0
  126. package/src/bundled-plugins/memory/watermark.ts +19 -10
  127. package/src/bundled-plugins/reviewer/reviewer.ts +20 -0
  128. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +3 -6
  129. package/src/bundled-plugins/security/policies/outbound-text.ts +20 -0
  130. package/src/bundled-plugins/security/policies/system-prompt-leak.ts +2 -7
  131. package/src/bundled-plugins/typevault/grants.ts +119 -0
  132. package/src/bundled-plugins/typevault/index.ts +152 -0
  133. package/src/bundled-plugins/typevault/request.ts +87 -0
  134. package/src/bundled-plugins/typevault/runtime.ts +45 -0
  135. package/src/bundled-plugins/typevault/skills/typevault/SKILL.md +65 -0
  136. package/src/channels/adapter-halt.ts +17 -0
  137. package/src/channels/adapters/github/history.ts +23 -2
  138. package/src/channels/adapters/github/inbound.ts +60 -1
  139. package/src/channels/adapters/github/index.ts +27 -5
  140. package/src/channels/adapters/github/reconcile-open-prs.ts +7 -1
  141. package/src/channels/adapters/kakaotalk-author-resolver.ts +7 -5
  142. package/src/channels/adapters/kakaotalk-channel-resolver.ts +45 -15
  143. package/src/channels/adapters/kakaotalk-classify.ts +7 -9
  144. package/src/channels/adapters/kakaotalk-list.ts +62 -0
  145. package/src/channels/adapters/kakaotalk-member-cache.ts +67 -0
  146. package/src/channels/adapters/kakaotalk-membership.ts +27 -10
  147. package/src/channels/adapters/kakaotalk-mention.ts +61 -0
  148. package/src/channels/adapters/kakaotalk.ts +159 -125
  149. package/src/channels/adapters/teams-classify.ts +18 -8
  150. package/src/channels/github-token-bridge.ts +13 -0
  151. package/src/channels/lost-background-context.ts +46 -0
  152. package/src/channels/manager.ts +52 -3
  153. package/src/channels/persistence.ts +5 -0
  154. package/src/channels/router.ts +120 -21
  155. package/src/channels/schema.ts +1 -1
  156. package/src/channels/types.ts +3 -0
  157. package/src/coding-agent/tool.ts +1 -1
  158. package/src/config/config.ts +20 -1
  159. package/src/config/providers.ts +65 -0
  160. package/src/container/start.ts +15 -2
  161. package/src/doctor/channel-checks.ts +33 -0
  162. package/src/doctor/checks.ts +12 -2
  163. package/src/eval/sandbox.ts +3 -0
  164. package/src/eval/tool.ts +1 -0
  165. package/src/git/resolve-identity.ts +67 -2
  166. package/src/git/system-commit.ts +7 -0
  167. package/src/init/dockerfile.ts +5 -1
  168. package/src/init/gitignore.ts +3 -3
  169. package/src/init/index.ts +1 -0
  170. package/src/init/models-dev.ts +2 -0
  171. package/src/init/validate-api-key.ts +5 -0
  172. package/src/migrations/secrets-v1-to-v2.ts +2 -0
  173. package/src/permissions/builtins.ts +8 -0
  174. package/src/plugin/types.ts +6 -0
  175. package/src/run/bundled-plugins.ts +12 -0
  176. package/src/run/channel-session-factory.ts +3 -0
  177. package/src/run/index.ts +13 -2
  178. package/src/sandbox/build.ts +1 -0
  179. package/src/sandbox/canonical-secrets.ts +6 -0
  180. package/src/sandbox/dependency-bins.ts +13 -1
  181. package/src/sandbox/dev-source.ts +21 -12
  182. package/src/sandbox/hidden-paths.ts +1 -1
  183. package/src/sandbox/index.ts +1 -0
  184. package/src/sandbox/policy.ts +5 -0
  185. package/src/secrets/hydrate.ts +4 -1
  186. package/src/secrets/storage.ts +10 -1
  187. package/src/server/command-runner.ts +4 -0
  188. package/src/server/index.ts +8 -1
  189. package/src/skills/typeclaw-config/SKILL.md +16 -0
  190. package/src/skills/typeclaw-memory/SKILL.md +3 -1
  191. package/src/skills/typeclaw-permissions/SKILL.md +8 -8
  192. package/src/skills/typeclaw-troubleshooting/SKILL.md +18 -1
  193. package/src/tunnels/active.ts +14 -0
  194. package/src/typevault/bash-cli.ts +18 -0
  195. package/src/typevault/cli-path.ts +11 -0
  196. package/src/typevault/envelope-helper.ts +17 -0
  197. package/src/typevault/envelope-store.ts +98 -0
  198. package/src/typevault/lifecycle.ts +132 -0
  199. package/src/typevault/prompt-tunnel.ts +33 -0
  200. package/src/typevault/redaction.ts +37 -0
  201. package/src/typevault/vault.ts +283 -0
  202. package/typeclaw.schema.json +40 -10
@@ -7,15 +7,15 @@ import type { SessionOrigin } from '@/agent/session-origin'
7
7
  import { defineTool } from '@/plugin'
8
8
  import { formatLocalDate } from '@/shared'
9
9
 
10
- import { fragmentContentHash } from './fragment-parser'
10
+ import { appendFragment, type FragmentsAppendedHook, type ReferenceSlugResolver } from './append-fragment'
11
11
  import { streamFilePath } from './paths'
12
12
  import { sanitizeProvenanceName } from './provenance-sanitize'
13
- import { detectSecrets } from './secret-detector'
14
13
  import { newEventId, timestampFromId } from './stream-events'
15
- import type { FragmentEvent, FragmentProvenance, WatermarkEvent } from './stream-events'
16
- import { appendEvents, readEvents, type FragmentsAppendedContext } from './stream-io'
14
+ import type { FragmentProvenance, WatermarkEvent } from './stream-events'
15
+ import { appendEvents } from './stream-io'
16
+ import { transcriptEntryId } from './transcript-cursor'
17
17
 
18
- export type FragmentsAppendedHook = (fragments: FragmentEvent[], context: FragmentsAppendedContext) => Promise<void>
18
+ export type { FragmentsAppendedHook, ReferenceSlugResolver } from './append-fragment'
19
19
 
20
20
  // Lets the memory-logger thread its spawn-time `payload.origin` into the append
21
21
  // tool so `where` is stamped server-side. A provider closure (not a static
@@ -27,7 +27,6 @@ export type OriginProvider = () => SessionOrigin | undefined
27
27
  // writes the file synchronously before returning the slug, so a same-run store
28
28
  // is already on disk by append time). Lets the append tool strip hallucinated
29
29
  // slugs the model invents without ever calling `store_reference`.
30
- export type ReferenceSlugResolver = (agentDir: string) => Promise<Iterable<string>>
31
30
  export type TranscriptPathProvider = () => string | undefined
32
31
 
33
32
  export type CreateAppendToolOptions = {
@@ -35,6 +34,9 @@ export type CreateAppendToolOptions = {
35
34
  originProvider?: OriginProvider
36
35
  referenceSlugResolver?: ReferenceSlugResolver
37
36
  transcriptPathProvider?: TranscriptPathProvider
37
+ validateCheckpoint?: (source: string, checkpoint: string, entry?: string) => void
38
+ deferWatermark?: () => boolean
39
+ beginWrite?: () => (() => void) | undefined
38
40
  }
39
41
 
40
42
  // Stamped fields are the stable channel coordinates only. Author identity is
@@ -87,81 +89,41 @@ export function createAppendTool(options: CreateAppendToolOptions = {}) {
87
89
  nonFile: ['topic', 'body', 'source', 'entry', 'latestEntryId', 'references', 'who'],
88
90
  },
89
91
  async execute({ topic, body, source, entry, latestEntryId, references, who }, ctx) {
90
- const streamPath = dailyStreamPath(ctx.agentDir)
91
- const where = provenanceFromOrigin(originProvider?.())
92
- const transcriptPath = transcriptPathProvider?.()
93
- if (
94
- transcriptPath !== undefined &&
95
- (await anchorBelongsToOperationalIncident(transcriptPath, entry, latestEntryId))
96
- ) {
97
- throw new Error(
98
- 'Refusing to append: the transcript anchor belongs to a deterministic operational incident sequence. ' +
99
- 'Operational defects are owned by the incident ledger, not semantic memory.',
100
- )
101
- }
102
- // `who` is LLM-supplied and force-committed to memory, so it clears the
103
- // same secret guard as topic/body — a token-shaped display name is refused
104
- // with the same retryable error. (`where` names are origin-derived and the
105
- // agent can't rewrite them, so they are self-redacted in provenanceFromOrigin
106
- // instead of throwing here.)
107
- assertNoSecrets([topic, body, who])
108
-
109
- const hash = fragmentContentHash({ topic, body })
110
- const events = await readEvents(streamPath)
111
- const duplicate = events
112
- .filter((event) => event.type === 'fragment')
113
- .find((event) => fragmentContentHash(event) === hash)
114
- if (duplicate !== undefined) {
115
- throw new Error(
116
- `Refusing to append: fragment "${duplicate.topic}" already exists in ${streamPath} with byte-equivalent content. ` +
117
- `The dreaming subagent will see the existing fragment; do not write it again. If the new occurrence ` +
118
- `is genuinely informative, write a fragment that says so explicitly rather than restating the original.`,
119
- )
120
- }
121
-
122
- const fragmentId = newEventId()
123
- const watermarkId = newEventId()
124
- const fragment: FragmentEvent = {
125
- type: 'fragment',
126
- id: fragmentId,
127
- ts: timestampFromId(fragmentId),
128
- source,
129
- entry,
130
- topic,
131
- body,
132
- }
133
- const validReferences = await resolveValidReferences(references, referenceSlugResolver, ctx)
134
- if (validReferences.length > 0) {
135
- fragment.references = validReferences
136
- }
137
- const safeWho = sanitizeProvenanceName(who)
138
- if (safeWho !== undefined) fragment.who = safeWho
139
- if (where !== undefined) fragment.where = where
140
- const watermark: WatermarkEvent = {
141
- type: 'watermark',
142
- id: watermarkId,
143
- ts: timestampFromId(watermarkId),
144
- source,
145
- entry: latestEntryId,
146
- }
147
-
148
- await mkdir(dirname(streamPath), { recursive: true })
149
- await appendEvents(
150
- streamPath,
151
- [fragment, watermark],
152
- onFragmentsAppended,
153
- onFragmentsAppended
154
- ? (err) => {
155
- ctx.logger?.warn(
156
- `[memory] post-append vector hook failed: ${err instanceof Error ? err.message : String(err)}`,
157
- )
158
- }
159
- : undefined,
160
- )
161
-
162
- return {
163
- content: [{ type: 'text' as const, text: `Appended memory fragment and watermark to ${streamPath}` }],
164
- details: { path: streamPath, fragmentId: fragment.id, watermarkId: watermark.id },
92
+ options.validateCheckpoint?.(source, latestEntryId, entry)
93
+ const finishWrite = options.beginWrite?.()
94
+ try {
95
+ const streamPath = dailyStreamPath(ctx.agentDir)
96
+ const where = provenanceFromOrigin(originProvider?.())
97
+ const transcriptPath = transcriptPathProvider?.()
98
+ if (
99
+ transcriptPath !== undefined &&
100
+ (await anchorBelongsToOperationalIncident(transcriptPath, entry, latestEntryId))
101
+ ) {
102
+ throw new Error(
103
+ 'Refusing to append: the transcript anchor belongs to a deterministic operational incident sequence. ' +
104
+ 'Operational defects are owned by the incident ledger, not semantic memory.',
105
+ )
106
+ }
107
+ const deferred = options.deferWatermark?.() ?? false
108
+ const { fragment, watermark } = await appendFragment({ topic, body, source, entry, references, who }, ctx, {
109
+ path: streamPath,
110
+ where,
111
+ watermarkEntry: deferred ? undefined : latestEntryId,
112
+ onFragmentsAppended,
113
+ referenceSlugResolver,
114
+ })
115
+
116
+ return {
117
+ content: [
118
+ {
119
+ type: 'text' as const,
120
+ text: `Appended memory fragment${deferred ? '; complete the page after all fragments' : ' and watermark'} to ${streamPath}`,
121
+ },
122
+ ],
123
+ details: { path: streamPath, fragmentId: fragment.id, ...(deferred ? {} : { watermarkId: watermark.id }) },
124
+ }
125
+ } finally {
126
+ finishWrite?.()
165
127
  }
166
128
  },
167
129
  })
@@ -191,7 +153,7 @@ export async function anchorBelongsToOperationalIncident(
191
153
  }
192
154
  })
193
155
  const anchorIndex = entries.findIndex((entry) => entryIdOf(entry) === entryId)
194
- const latestIndex = entries.findIndex((entry) => entryIdOf(entry) === latestEntryId)
156
+ const latestIndex = entries.findIndex((entry) => entryIdOf(entry) === transcriptEntryId(latestEntryId))
195
157
  if (anchorIndex < 0 || latestIndex < anchorIndex) return false
196
158
 
197
159
  for (let i = anchorIndex; i <= latestIndex; i++) {
@@ -230,76 +192,45 @@ function containsIncidentMarker(entry: unknown): boolean {
230
192
 
231
193
  export const appendTool = createAppendTool()
232
194
 
233
- export const advanceWatermarkTool = defineTool({
234
- description:
235
- 'Advance the daily-stream watermark without writing a fragment. Use this when you evaluated transcript entries this run but decided none warranted a fragment — still call this once so the next run does not re-read the same prefix. The runtime writes the watermark line and chooses the filename.',
236
- parameters: z.object({
237
- source: z.string().min(1),
238
- latestEntryId: z.string().min(1),
239
- }),
240
- async execute({ source, latestEntryId }, ctx) {
241
- const streamPath = dailyStreamPath(ctx.agentDir)
242
- const watermarkId = newEventId()
243
- const watermark: WatermarkEvent = {
244
- type: 'watermark',
245
- id: watermarkId,
246
- ts: timestampFromId(watermarkId),
247
- source,
248
- entry: latestEntryId,
249
- }
250
-
251
- await mkdir(dirname(streamPath), { recursive: true })
252
- await appendEvents(streamPath, [watermark])
253
-
254
- return {
255
- content: [{ type: 'text' as const, text: `Advanced memory watermark in ${streamPath}` }],
256
- details: { path: streamPath, watermarkId: watermark.id },
257
- }
258
- },
259
- })
260
-
261
- function dailyStreamPath(agentDir: string): string {
262
- return streamFilePath(agentDir, formatLocalDate())
263
- }
195
+ export function createAdvanceWatermarkTool(
196
+ options: {
197
+ validateCheckpoint?: (source: string, checkpoint: string) => void
198
+ onAdvanced?: () => void
199
+ } = {},
200
+ ) {
201
+ return defineTool({
202
+ description:
203
+ 'Advance the daily-stream watermark without writing a fragment. Use this when you evaluated transcript entries this run but decided none warranted a fragment — still call this once so the next run does not re-read the same prefix. The runtime writes the watermark line and chooses the filename.',
204
+ parameters: z.object({
205
+ source: z.string().min(1),
206
+ latestEntryId: z.string().min(1),
207
+ }),
208
+ async execute({ source, latestEntryId }, ctx) {
209
+ options.validateCheckpoint?.(source, latestEntryId)
210
+ const streamPath = dailyStreamPath(ctx.agentDir)
211
+ const watermarkId = newEventId()
212
+ const watermark: WatermarkEvent = {
213
+ type: 'watermark',
214
+ id: watermarkId,
215
+ ts: timestampFromId(watermarkId),
216
+ source,
217
+ entry: latestEntryId,
218
+ }
264
219
 
265
- // The model treats `references[]` as a free-text related-id field, citing empty
266
- // strings, stream paths, and slugs for references it never stored. Every path
267
- // first normalizes: trim, drop blank/whitespace-only entries, and de-duplicate
268
- // — an empty or repeated citation is never meaningful regardless of wiring.
269
- // With a resolver, only slugs backed by a real reference file then survive; the
270
- // rest are dropped and logged so dangling citations never reach the stream.
271
- // Without one (the standalone `appendTool` export, no reference subsystem
272
- // wired) the normalized slugs pass through.
273
- async function resolveValidReferences(
274
- references: string[] | undefined,
275
- resolver: ReferenceSlugResolver | undefined,
276
- ctx: { agentDir: string; logger?: { warn(message: string): void } },
277
- ): Promise<string[]> {
278
- if (references === undefined || references.length === 0) return []
279
- const requested = [...new Set(references.map((slug) => slug.trim()).filter((slug) => slug.length > 0))]
280
- if (resolver === undefined) return requested
220
+ await mkdir(dirname(streamPath), { recursive: true })
221
+ await appendEvents(streamPath, [watermark])
222
+ options.onAdvanced?.()
281
223
 
282
- const known = new Set(await resolver(ctx.agentDir))
283
- const valid = requested.filter((slug) => known.has(slug))
284
- const dropped = requested.filter((slug) => !known.has(slug))
285
- if (dropped.length > 0) {
286
- ctx.logger?.warn(
287
- `[memory] dropped ${dropped.length} unknown reference slug(s) from fragment: ${dropped.join(', ')}. ` +
288
- `references[] may only cite slugs returned by store_reference.`,
289
- )
290
- }
291
- return valid
224
+ return {
225
+ content: [{ type: 'text' as const, text: `Advanced memory watermark in ${streamPath}` }],
226
+ details: { path: streamPath, watermarkId: watermark.id },
227
+ }
228
+ },
229
+ })
292
230
  }
293
231
 
294
- function assertNoSecrets(parts: ReadonlyArray<string | undefined>): void {
295
- const content = parts.filter((part): part is string => part !== undefined).join('\n')
296
- const secrets = detectSecrets(content)
297
- if (secrets.length === 0) return
232
+ export const advanceWatermarkTool = createAdvanceWatermarkTool()
298
233
 
299
- const ruleNames = [...new Set(secrets.map((s) => s.rule))].join(', ')
300
- throw new Error(
301
- `Refusing to append: content contains a recognized credential pattern (${ruleNames}). ` +
302
- `Memory fragments must never quote secret values verbatim. Record the env var name and how it ` +
303
- `was discovered, not the value itself.`,
304
- )
234
+ function dailyStreamPath(agentDir: string): string {
235
+ return streamFilePath(agentDir, formatLocalDate())
305
236
  }
@@ -973,6 +973,8 @@ Dreaming is the offline reflection process that promotes the agent's daily memor
973
973
 
974
974
  # What you do
975
975
 
976
+ Fragments carrying \`kind: "explicit"\` were deliberately saved by the agent: give that intent weight when retaining durable knowledge, while applying the same consolidation, deduplication, contradiction, and citation rules.
977
+
976
978
  Your job is to rebalance topic shards under \`memory/topics/\`. Each shard is one topic, one file. Read existing shards with \`ls memory/topics/\` and \`read memory/topics/<slug>.md\`, then read the **undreamed tail** of every daily stream file the user prompt lists. Each stream line is a JSON object representing a fragment, watermark, or migrated legacy-prose event; focus on fragment events, especially their \`topic\` and \`body\`. Consolidate new fragments into topic shards, rebalance existing shards, and stop.
977
979
 
978
980
  You also distill **muscle memory**: when the streams show a repeated multi-step procedure the user has guided the main agent through enough times that it would save effort to codify, you take action. Muscle memory has three forms, in increasing order of investment — a skill at \`memory/skills/<name>/SKILL.md\` (a codified procedure the next session loads on demand), a **CLI suggestion** recorded as a topic shard (a small command-line tool the main agent may scaffold under \`packages/<name>/\` when the user next asks for that procedure), or a **plugin suggestion** recorded as a topic shard (a typeclaw plugin under \`packages/<name>/\` that hooks into the runtime). You write the skill directly; you only *suggest* CLIs and plugins because they live under \`packages/\`, outside your write sandbox. Long-term memory is passive context: the main agent may use suggestions when a current user request makes them relevant, but a shard alone never authorizes action.
@@ -20,6 +20,7 @@ import {
20
20
  import { createMemoryLoggerSubagent, type MemoryLoggerPayload } from './memory-logger'
21
21
  import { preShardBackupPath, streamFilePath, streamsDir, topicsDir } from './paths'
22
22
  import { bumpReferenceAccess } from './references/load-references'
23
+ import { createMemoryRememberTool } from './remember-tool'
23
24
  import { memoryCommands } from './search-command'
24
25
  import { createMemorySearchTool } from './search-tool'
25
26
  import { type InjectedMemoryState, partitionRetrievedMemoryItems } from './turn-dedup'
@@ -29,6 +30,7 @@ import { hybridSearch, type EmbedFn } from './vector/hybrid'
29
30
  import { makeAppendHook } from './vector/index-on-write'
30
31
  import { makeReferenceStoredHook } from './vector/reference-index-on-write'
31
32
  import { type OpenVectorStore, VectorStore } from './vector/store'
33
+ import { readLatestWatermark } from './watermark'
32
34
 
33
35
  const DEFAULT_IDLE_MS = 60_000
34
36
  const DEFAULT_BUFFER_BYTES = 500_000
@@ -213,6 +215,12 @@ async function renderVectorTurnMemory(
213
215
  logger?: { info: (msg: string) => void },
214
216
  ): Promise<string> {
215
217
  if (isInternalInfraTurn(event.origin)) return ''
218
+ // A runtime wakeup (subagent completion, review-state reminder) has no inbound
219
+ // text. There is nothing to retrieve for: an empty query embeds to a vector that
220
+ // scores every topic alike, so the slug tie-break injected the same first-N
221
+ // headings on every wakeup. The human turn that started the work already
222
+ // carried its own memory, and `memory_search` stays available.
223
+ if (event.userPrompt.trim() === '') return ''
216
224
  const plan = await loadMemoryInjectionPlan(event.agentDir, { injectionBudgetBytes })
217
225
  const isChannel = event.origin?.kind === 'channel'
218
226
  if (plan.mode === 'direct' && isChannel) {
@@ -355,8 +363,8 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
355
363
  const currentSize = await readSize(parentTranscriptPath)
356
364
  const currentLines = await readLineCount(parentTranscriptPath)
357
365
  bytesAtLastRun.set(sessionId, currentSize)
358
- // Real drained watermark — set only here, inside an actual spawn, so
359
- // session.end's byte-equality skip never trusts a buffer-only seed.
366
+ // Record only an actual spawn baseline, not a buffer-only seed.
367
+ // session.end also confirms the durable final-entry checkpoint.
360
368
  bytesAtLastSpawn.set(sessionId, currentSize)
361
369
  // Monotonic: never regress below a baseline the idle gate already
362
370
  // reserved. readLineCount returns 0 on a read error or a
@@ -448,6 +456,10 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
448
456
  },
449
457
  tools: {
450
458
  memory_search: createMemorySearchTool(),
459
+ memory_remember: createMemoryRememberTool({
460
+ onFragmentsAppended: makeAppendHook(openAppendVectorStore),
461
+ onReferenceStored: makeReferenceStoredHook(openAppendVectorStore),
462
+ }),
451
463
  },
452
464
  cronJobs: {
453
465
  [DREAMING_CRON_KEY]: {
@@ -572,10 +584,25 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
572
584
  if (baseline !== undefined && baseline > 0) {
573
585
  const currentSize = await readSize(last.parentTranscriptPath)
574
586
  if (currentSize === baseline) {
575
- ctx.logger.info(
576
- `memory-logger session-end skip ${sessionId} (no new bytes since last spawn at ${baseline})`,
577
- )
578
- skip = true
587
+ try {
588
+ const checkpoint = await readLatestWatermark(streamsDir(ctx.agentDir), sessionId)
589
+ const raw = await readFile(last.parentTranscriptPath, 'utf8')
590
+ const tail: unknown = JSON.parse(raw.trimEnd().split('\n').at(-1) ?? 'null')
591
+ if (
592
+ checkpoint !== null &&
593
+ typeof tail === 'object' &&
594
+ tail !== null &&
595
+ 'id' in tail &&
596
+ tail.id === checkpoint
597
+ ) {
598
+ ctx.logger.info(
599
+ `memory-logger session-end skip ${sessionId} (no new bytes since last spawn at ${baseline})`,
600
+ )
601
+ skip = true
602
+ }
603
+ } catch {
604
+ ctx.logger.warn(`memory-logger ${sessionId} final checkpoint unconfirmed; keeping final sweep`)
605
+ }
579
606
  }
580
607
  }
581
608
  }
@@ -7,13 +7,19 @@ import { getConfig, resolveModel } from '@/config'
7
7
  import { type Subagent, readTool } from '@/plugin'
8
8
  import { formatLocalDate } from '@/shared'
9
9
 
10
- import { advanceWatermarkTool, createAppendTool, type FragmentsAppendedHook } from './append-tool'
10
+ import {
11
+ advanceWatermarkTool,
12
+ createAdvanceWatermarkTool,
13
+ createAppendTool,
14
+ type FragmentsAppendedHook,
15
+ } from './append-tool'
11
16
  import { findEntryTool } from './find-entry-tool'
12
17
  import { formatMemorySnapshot } from './memory-snapshot'
13
18
  import { streamFilePath, streamsDir } from './paths'
14
19
  import { listReferenceSlugs } from './references/load-references'
15
20
  import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
16
21
  import { readEvents } from './stream-io'
22
+ import { decodeTranscriptCursor } from './transcript-cursor'
17
23
  import {
18
24
  buildTranscriptPreload,
19
25
  PRELOAD_FRAME_BYTES,
@@ -86,7 +92,9 @@ Tools: \`read\`, \`find_entry\`, \`append\`, \`store_reference\`, and the waterm
86
92
 
87
93
  # Read loop, watermark, and stopping
88
94
 
89
- The runtime usually preloads the transcript past the watermark into your first message. Evaluate that slice directly: do not call \`find_entry\` or \`read\` for lines it already contains. If it says more lines remain, continue with \`read\` from the offset it gives and follow the loop below from there. A preloaded line ending in a \`truncated\` marker was too large to show whole; judge it from the part shown. Without a preload, use the loop below from the start.
95
+ An \`entry-part\` may include \`precedingText\`: read-only source overlap immediately before its new \`text\`. Interpret them together to recover instructions crossing a boundary. Capture complete facts supported by the new segment; do not emit a fragment for overlap alone. Only the issued page checkpoint advances progress; overlap never does.
96
+
97
+ The runtime usually preloads a bounded transcript page into your first message. Evaluate that page directly. An \`entry-part\` contains an exact text segment of one large projected source record; its \`id\` is the original evidence anchor. Never treat a partial record as fully consumed. When a page supplies a \`memory-page-checkpoint\`, finish all fragment/reference writes first, then call the watermark-advance tool exactly once with that checkpoint as \`latestEntryId\` and exit. An append does not complete the page. The runtime supplies later pages in fresh sessions; do not use \`read\` or \`find_entry\` to bypass paging. Without a preload, use the read loop below.
90
98
 
91
99
  Session transcripts are JSONL; each line has an \`id\`. They can be large, and \`read\` truncates output to 50 KB or 2000 lines, returning the line range and next offset. Do not scroll from line 1 through a prefix already covered by the watermark.
92
100
 
@@ -194,6 +202,8 @@ The \`append\` tool will refuse content containing a recognizable credential pat
194
202
 
195
203
  # Local dedup against today's daily stream
196
204
 
205
+ Facts already saved by a successful \`memory_remember\` tool call in the transcript are already in the stream; do not append them again.
206
+
197
207
  The \`append\` tool refuses byte-equivalent fragments within the same daily stream — content-equality on topic+body modulo whitespace, not marker-equality. If it rejects a fragment already in the same daily stream, rewrite or skip.
198
208
 
199
209
  Do not fight this refusal by changing punctuation or adding filler. If the new transcript only repeats exactly the same fact with no new evidence worth preserving, skip. If it is a true recurrence, rewrite the body to anchor the new occurrence explicitly, so dreaming can see why this line is new evidence rather than a duplicate copy.
@@ -280,7 +290,8 @@ function renderPreload(preload: TranscriptPreload): string {
280
290
  `Preloaded transcript lines ${preload.startLine}-${preload.endLine} of totalLines=${preload.totalLines}${preload.watermarkFound ? ' (starting right after the watermark)' : ''}.`,
281
291
  preload.nextOffset === null
282
292
  ? 'This reaches the end of the transcript; do not call read on it.'
283
- : `More lines remain: after evaluating these, continue with read(path=<transcript>, offset=${preload.nextOffset}).`,
293
+ : 'More lines remain: the runtime will supply the next page after this page is committed.',
294
+ `After all fragment writes, complete this page with latestEntryId from <memory-page-checkpoint>${preload.checkpoint}</memory-page-checkpoint>.`,
284
295
  ]
285
296
  return [...header, '<transcript>', preload.text, '</transcript>'].join('\n')
286
297
  }
@@ -355,14 +366,72 @@ export function createMemoryLoggerSubagent(
355
366
  // single mutable cell safely carries this spawn's origin to the append tool.
356
367
  let currentOrigin: SessionOrigin | undefined
357
368
  let currentTranscriptPath: string | undefined
369
+ let page:
370
+ | {
371
+ source: string
372
+ checkpoint: string | null
373
+ entries: Set<string>
374
+ committed: boolean
375
+ closed: boolean
376
+ pending: number
377
+ }
378
+ | undefined
379
+ const validateCheckpoint = (source: string, checkpoint: string, entry?: string): void => {
380
+ if (
381
+ page !== undefined &&
382
+ (page.closed ||
383
+ page.checkpoint === null ||
384
+ source !== page.source ||
385
+ checkpoint !== page.checkpoint ||
386
+ (entry === undefined && page.pending > 0) ||
387
+ (entry !== undefined && !page.entries.has(entry)))
388
+ ) {
389
+ throw new Error('Memory checkpoint must match the active page; unseen or incomplete evidence cannot be skipped.')
390
+ }
391
+ // Reserve completion before its first I/O await so a parallel tool call
392
+ // cannot start writing evidence behind the durable checkpoint.
393
+ if (page !== undefined && entry === undefined) page.closed = true
394
+ }
395
+ const beginWrite = (): (() => void) | undefined => {
396
+ const active = page
397
+ if (active === undefined) return undefined
398
+ if (active.closed) throw new Error('Memory page is closed; stale writes cannot continue.')
399
+ active.pending++
400
+ return () => {
401
+ active.pending--
402
+ }
403
+ }
358
404
  const appendTool = createAppendTool({
359
405
  onFragmentsAppended: options.onFragmentsAppended,
360
406
  originProvider: () => currentOrigin,
361
407
  referenceSlugResolver: (agentDir) => listReferenceSlugs(agentDir),
362
408
  transcriptPathProvider: () => currentTranscriptPath,
409
+ validateCheckpoint,
410
+ deferWatermark: () => page !== undefined,
411
+ beginWrite,
363
412
  })
364
- const storeReferenceTool = createStoreReferenceTool(options.onReferenceStored)
365
- const customTools = [findEntryTool, appendTool, storeReferenceTool, advanceWatermarkTool]
413
+ const referenceTool = createStoreReferenceTool(options.onReferenceStored)
414
+ const storeReferenceTool = {
415
+ ...referenceTool,
416
+ execute: async (...args: Parameters<typeof referenceTool.execute>) => {
417
+ const finishWrite = beginWrite()
418
+ try {
419
+ return await referenceTool.execute(...args)
420
+ } finally {
421
+ finishWrite?.()
422
+ }
423
+ },
424
+ }
425
+ const pageWatermarkTool = createAdvanceWatermarkTool({
426
+ validateCheckpoint,
427
+ onAdvanced: () => {
428
+ if (page !== undefined) {
429
+ page.committed = true
430
+ page.closed = true
431
+ }
432
+ },
433
+ })
434
+ const customTools = [findEntryTool, appendTool, storeReferenceTool, pageWatermarkTool]
366
435
  return {
367
436
  systemPrompt: MEMORY_LOGGER_SYSTEM_PROMPT,
368
437
  // Logging is "read transcript past the watermark, decide 0-N fragments,
@@ -395,6 +464,7 @@ export function createMemoryLoggerSubagent(
395
464
  path: string,
396
465
  mark: string | null,
397
466
  basePrompt: string,
467
+ untilLine?: number,
398
468
  ): Promise<TranscriptPreload | null> => {
399
469
  let models: PreloadModelLimits[]
400
470
  try {
@@ -409,24 +479,74 @@ export function createMemoryLoggerSubagent(
409
479
  Buffer.byteLength(MEMORY_LOGGER_SYSTEM_PROMPT, 'utf8') +
410
480
  Buffer.byteLength(basePrompt, 'utf8') +
411
481
  PRELOAD_FRAME_BYTES
412
- const maxBytes = preloadBudgetBytes(models, promptBytes)
413
- return maxBytes === null ? null : buildTranscriptPreload(path, mark, { maxBytes })
482
+ let maxBytes = preloadBudgetBytes(models, promptBytes)
483
+ let endLine = untilLine
484
+ while (maxBytes !== null) {
485
+ const preload = await buildTranscriptPreload(path, mark, { maxBytes, untilLine: endLine })
486
+ if (preload === null) return null
487
+ const frameBytes = Buffer.byteLength(renderPreload(preload), 'utf8') - Buffer.byteLength(preload.text, 'utf8')
488
+ const fitted = preloadBudgetBytes(models, promptBytes - PRELOAD_FRAME_BYTES + frameBytes)
489
+ if (fitted === null) return null
490
+ if (Buffer.byteLength(preload.text, 'utf8') <= fitted) return preload
491
+ maxBytes = fitted
492
+ endLine ??= preload.totalLines
493
+ }
494
+ return null
414
495
  }
415
496
  currentOrigin = ctx.payload.origin
416
497
  currentTranscriptPath = ctx.payload.parentTranscriptPath
417
498
  const today = formatLocalDate()
418
499
  const memoryDir = streamsDir(ctx.payload.agentDir)
419
500
  const streamFile = streamFilePath(ctx.payload.agentDir, today)
420
- const watermark = await readLatestWatermark(memoryDir, ctx.payload.parentSessionId)
501
+ let watermark = await readLatestWatermark(memoryDir, ctx.payload.parentSessionId)
421
502
  const fragmentsBefore = await countFragments(streamFile)
422
503
  const start = Date.now()
423
504
  logger.info(
424
505
  `[memory-logger] ${ctx.payload.parentSessionId} start stream=${today}.jsonl watermark=${watermark ?? 'none'}`,
425
506
  )
426
507
  try {
427
- const basePrompt = buildInitialPrompt(ctx.payload, streamFile, watermark, null)
428
- const preload = await preloadFor(ctx.payload.parentTranscriptPath, watermark, basePrompt)
429
- await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark, preload) })
508
+ let untilLine: number | undefined
509
+ while (true) {
510
+ const basePrompt = buildInitialPrompt(ctx.payload, streamFile, watermark, null)
511
+ const preload = await preloadFor(ctx.payload.parentTranscriptPath, watermark, basePrompt, untilLine)
512
+ if (preload === null && decodeTranscriptCursor(watermark) !== null) {
513
+ logger.warn(
514
+ `[memory-logger] ${ctx.payload.parentSessionId} partial transcript cursor remains pending: paging is unavailable`,
515
+ )
516
+ break
517
+ }
518
+ untilLine ??= preload?.totalLines
519
+ if (preload !== null) {
520
+ if (preload.startCheckpoint !== null) {
521
+ await advanceWatermarkTool.execute(
522
+ { source: ctx.payload.parentSessionId, latestEntryId: preload.startCheckpoint },
523
+ { agentDir: ctx.agentDir, sessionId: ctx.payload.parentSessionId, signal: undefined, logger },
524
+ )
525
+ }
526
+ page = {
527
+ source: ctx.payload.parentSessionId,
528
+ checkpoint: preload.entryIds.length === 0 ? null : preload.checkpoint,
529
+ entries: new Set(preload.entryIds),
530
+ committed: false,
531
+ closed: false,
532
+ pending: 0,
533
+ }
534
+ } else {
535
+ page = undefined
536
+ }
537
+ try {
538
+ await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark, preload) })
539
+ } finally {
540
+ if (page !== undefined) page.closed = true
541
+ }
542
+ if (!page?.committed || preload?.nextOffset === null) {
543
+ if (page && page.entries.size > 0 && !page.committed) {
544
+ logger.warn(`[memory-logger] ${ctx.payload.parentSessionId} page not committed; evidence remains pending`)
545
+ }
546
+ break
547
+ }
548
+ watermark = page.checkpoint
549
+ }
430
550
  const fragmentsWritten = (await countFragments(streamFile)) - fragmentsBefore
431
551
  logger.info(
432
552
  `[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
@@ -437,6 +557,8 @@ export function createMemoryLoggerSubagent(
437
557
  `[memory-logger] ${ctx.payload.parentSessionId}: run threw: ${message} elapsed_ms=${Date.now() - start}`,
438
558
  )
439
559
  throw err
560
+ } finally {
561
+ if (page !== undefined) page.closed = true
440
562
  }
441
563
  },
442
564
  }