typeclaw 0.51.0 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/types/agent/bash-sandbox-env.d.ts +2 -0
- package/dist/types/agent/bash-sandbox.d.ts +4 -0
- package/dist/types/agent/index.d.ts +5 -0
- package/dist/types/agent/multimodal/look-at.d.ts +1 -1
- package/dist/types/agent/plugin-tools.d.ts +4 -0
- package/dist/types/agent/prompt-cache-key.d.ts +1 -0
- package/dist/types/agent/session-tail.d.ts +12 -0
- package/dist/types/agent/subagents.d.ts +5 -0
- package/dist/types/agent/system-prompt-composition.d.ts +1 -0
- package/dist/types/agent/system-prompt.d.ts +2 -0
- package/dist/types/agent/tools/channel-disengage.d.ts +1 -1
- package/dist/types/agent/tools/channel-edit.d.ts +1 -1
- package/dist/types/agent/tools/channel-fetch-attachment.d.ts +1 -1
- package/dist/types/agent/tools/channel-handoff.d.ts +1 -1
- package/dist/types/agent/tools/channel-history.d.ts +1 -1
- package/dist/types/agent/tools/channel-react.d.ts +1 -1
- package/dist/types/agent/tools/channel-read.d.ts +10 -2
- package/dist/types/agent/tools/channel-reply.d.ts +1 -1
- package/dist/types/agent/tools/channel-send.d.ts +1 -1
- package/dist/types/agent/tools/channel-session-search.d.ts +1 -1
- package/dist/types/agent/tools/grant-role.d.ts +1 -1
- package/dist/types/agent/tools/post-github-review.d.ts +1 -1
- package/dist/types/agent/tools/restart.d.ts +1 -1
- package/dist/types/agent/tools/skip-response.d.ts +1 -1
- package/dist/types/agent/tools/spawn-subagent.d.ts +2 -1
- package/dist/types/agent/tools/stream-snapshot.d.ts +1 -1
- package/dist/types/agent/tools/subagent-cancel.d.ts +1 -1
- package/dist/types/agent/tools/subagent-output.d.ts +1 -1
- package/dist/types/agent/tools/todo/index.d.ts +1 -1
- package/dist/types/agent/tools/webfetch/adaptive-html.d.ts +3 -3
- package/dist/types/agent/tools/webfetch/strategies/readability-process.d.ts +48 -0
- package/dist/types/agent/tools/webfetch/tool.d.ts +2 -2
- package/dist/types/agent/tools/websearch.d.ts +1 -1
- package/dist/types/channels/adapters/github/auth-app.d.ts +25 -0
- package/dist/types/channels/adapters/github/auth-pat.d.ts +17 -0
- package/dist/types/channels/adapters/github/auth.d.ts +24 -0
- package/dist/types/channels/github-token-bridge.d.ts +4 -0
- package/dist/types/channels/lost-background-context.d.ts +2 -0
- package/dist/types/channels/persistence.d.ts +1 -0
- package/dist/types/channels/router.d.ts +5 -1
- package/dist/types/channels/types.d.ts +1 -0
- package/dist/types/coding-agent/tool.d.ts +1 -1
- package/dist/types/config/config.d.ts +20 -11
- package/dist/types/config/providers.d.ts +47 -0
- package/dist/types/eval/sandbox.d.ts +2 -0
- package/dist/types/git/resolve-identity.d.ts +2 -0
- package/dist/types/init/dockerfile.d.ts +1 -0
- package/dist/types/init/gitignore.d.ts +1 -1
- package/dist/types/permissions/builtins.d.ts +1 -0
- package/dist/types/plugin/types.d.ts +3 -0
- package/dist/types/sandbox/canonical-secrets.d.ts +2 -2
- package/dist/types/sandbox/coding-agent.d.ts +1 -1
- package/dist/types/sandbox/dependency-bins.d.ts +1 -0
- package/dist/types/sandbox/dev-source.d.ts +2 -2
- package/dist/types/sandbox/hidden-paths.d.ts +1 -0
- package/dist/types/sandbox/index.d.ts +1 -1
- package/dist/types/sandbox/policy.d.ts +3 -0
- package/dist/types/typevault/cli-path.d.ts +2 -0
- package/dist/types/typevault/envelope-store.d.ts +11 -0
- package/dist/types/typevault/lifecycle.d.ts +13 -0
- package/dist/types/typevault/vault.d.ts +27 -0
- package/package.json +8 -7
- package/scripts/test.ts +252 -0
- package/src/agent/bash-sandbox-env.ts +33 -1
- package/src/agent/bash-sandbox.ts +20 -6
- package/src/agent/index.ts +120 -31
- package/src/agent/multimodal/look-at.ts +1 -1
- package/src/agent/plugin-tools.ts +35 -1
- package/src/agent/prompt-cache-key.ts +26 -3
- package/src/agent/session-origin.ts +39 -18
- package/src/agent/session-tail.ts +28 -0
- package/src/agent/subagents.ts +21 -2
- package/src/agent/system-prompt-composition.ts +18 -0
- package/src/agent/system-prompt.ts +39 -1
- package/src/agent/tool-error-remediation.ts +18 -3
- package/src/agent/tools/channel-disengage.ts +1 -1
- package/src/agent/tools/channel-edit.ts +1 -1
- package/src/agent/tools/channel-fetch-attachment.ts +1 -1
- package/src/agent/tools/channel-handoff.ts +3 -2
- package/src/agent/tools/channel-history.ts +2 -1
- package/src/agent/tools/channel-react.ts +1 -1
- package/src/agent/tools/channel-read.ts +36 -3
- package/src/agent/tools/channel-reply.ts +1 -1
- package/src/agent/tools/channel-send.ts +4 -3
- package/src/agent/tools/channel-session-search.ts +13 -4
- package/src/agent/tools/grant-role.ts +1 -1
- package/src/agent/tools/post-github-review.ts +1 -1
- package/src/agent/tools/restart.ts +1 -1
- package/src/agent/tools/skip-response.ts +1 -1
- package/src/agent/tools/spawn-subagent.ts +15 -1
- package/src/agent/tools/stream-snapshot.ts +1 -1
- package/src/agent/tools/subagent-cancel.ts +1 -1
- package/src/agent/tools/subagent-output.ts +1 -1
- package/src/agent/tools/todo/index.ts +1 -1
- package/src/agent/tools/webfetch/adaptive-html.ts +6 -4
- package/src/agent/tools/webfetch/strategies/readability-child.ts +25 -0
- package/src/agent/tools/webfetch/strategies/readability-process.ts +185 -0
- package/src/agent/tools/webfetch/tool.ts +6 -5
- package/src/agent/tools/websearch.ts +1 -1
- package/src/browser/dockerfile.ts +1 -1
- package/src/browser/fill-secret.ts +92 -35
- package/src/browser/registry.ts +6 -6
- package/src/bundled-plugins/agent-messenger-cli-auth/command.ts +211 -0
- package/src/bundled-plugins/agent-messenger-cli-auth/index.ts +186 -0
- package/src/bundled-plugins/backup/index.ts +17 -2
- package/src/bundled-plugins/backup/runner.ts +37 -2
- package/src/bundled-plugins/browser-agent/browser-agent.ts +51 -14
- package/src/bundled-plugins/browser-agent/index.ts +12 -5
- package/src/bundled-plugins/managed-browser/index.ts +25 -2
- package/src/bundled-plugins/managed-browser/skills/browserskill/browserskill/SKILL.md +13 -5
- package/src/bundled-plugins/managed-browser/skills/omowright/omowright/SKILL.md +56 -5
- package/src/bundled-plugins/memory/README.md +23 -2
- package/src/bundled-plugins/memory/append-fragment.ts +133 -0
- package/src/bundled-plugins/memory/append-tool.ts +80 -149
- package/src/bundled-plugins/memory/dreaming.ts +2 -0
- package/src/bundled-plugins/memory/index.ts +33 -6
- package/src/bundled-plugins/memory/memory-logger.ts +133 -11
- package/src/bundled-plugins/memory/references/store-reference-tool.ts +56 -32
- package/src/bundled-plugins/memory/remember-tool.ts +81 -0
- package/src/bundled-plugins/memory/search-tool-hybrid.ts +91 -0
- package/src/bundled-plugins/memory/search-tool.ts +58 -12
- package/src/bundled-plugins/memory/stream-events.ts +2 -0
- package/src/bundled-plugins/memory/transcript-cursor.ts +30 -0
- package/src/bundled-plugins/memory/transcript-preload.ts +131 -26
- package/src/bundled-plugins/memory/transcript-view.ts +36 -0
- package/src/bundled-plugins/memory/watermark.ts +19 -10
- package/src/bundled-plugins/reviewer/reviewer.ts +20 -0
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +3 -6
- package/src/bundled-plugins/security/policies/outbound-text.ts +20 -0
- package/src/bundled-plugins/security/policies/system-prompt-leak.ts +2 -7
- package/src/bundled-plugins/typevault/grants.ts +119 -0
- package/src/bundled-plugins/typevault/index.ts +152 -0
- package/src/bundled-plugins/typevault/request.ts +87 -0
- package/src/bundled-plugins/typevault/runtime.ts +45 -0
- package/src/bundled-plugins/typevault/skills/typevault/SKILL.md +65 -0
- package/src/channels/adapter-halt.ts +17 -0
- package/src/channels/adapters/github/history.ts +23 -2
- package/src/channels/adapters/github/inbound.ts +60 -1
- package/src/channels/adapters/github/index.ts +27 -5
- package/src/channels/adapters/github/reconcile-open-prs.ts +7 -1
- package/src/channels/adapters/kakaotalk-author-resolver.ts +7 -5
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +45 -15
- package/src/channels/adapters/kakaotalk-classify.ts +7 -9
- package/src/channels/adapters/kakaotalk-list.ts +62 -0
- package/src/channels/adapters/kakaotalk-member-cache.ts +67 -0
- package/src/channels/adapters/kakaotalk-membership.ts +27 -10
- package/src/channels/adapters/kakaotalk-mention.ts +61 -0
- package/src/channels/adapters/kakaotalk.ts +159 -125
- package/src/channels/adapters/teams-classify.ts +18 -8
- package/src/channels/github-token-bridge.ts +13 -0
- package/src/channels/lost-background-context.ts +46 -0
- package/src/channels/manager.ts +52 -3
- package/src/channels/persistence.ts +5 -0
- package/src/channels/router.ts +120 -21
- package/src/channels/schema.ts +1 -1
- package/src/channels/types.ts +3 -0
- package/src/coding-agent/tool.ts +1 -1
- package/src/config/config.ts +20 -1
- package/src/config/providers.ts +65 -0
- package/src/container/start.ts +15 -2
- package/src/doctor/channel-checks.ts +33 -0
- package/src/doctor/checks.ts +12 -2
- package/src/eval/sandbox.ts +3 -0
- package/src/eval/tool.ts +1 -0
- package/src/git/resolve-identity.ts +67 -2
- package/src/git/system-commit.ts +7 -0
- package/src/init/dockerfile.ts +5 -1
- package/src/init/gitignore.ts +3 -3
- package/src/init/index.ts +1 -0
- package/src/init/models-dev.ts +2 -0
- package/src/init/validate-api-key.ts +5 -0
- package/src/migrations/secrets-v1-to-v2.ts +2 -0
- package/src/permissions/builtins.ts +8 -0
- package/src/plugin/types.ts +6 -0
- package/src/run/bundled-plugins.ts +12 -0
- package/src/run/channel-session-factory.ts +3 -0
- package/src/run/index.ts +13 -2
- package/src/sandbox/build.ts +1 -0
- package/src/sandbox/canonical-secrets.ts +6 -0
- package/src/sandbox/dependency-bins.ts +13 -1
- package/src/sandbox/dev-source.ts +21 -12
- package/src/sandbox/hidden-paths.ts +1 -1
- package/src/sandbox/index.ts +1 -0
- package/src/sandbox/policy.ts +5 -0
- package/src/secrets/hydrate.ts +4 -1
- package/src/secrets/storage.ts +10 -1
- package/src/server/command-runner.ts +4 -0
- package/src/server/index.ts +8 -1
- package/src/skills/typeclaw-config/SKILL.md +16 -0
- package/src/skills/typeclaw-memory/SKILL.md +3 -1
- package/src/skills/typeclaw-permissions/SKILL.md +8 -8
- package/src/skills/typeclaw-troubleshooting/SKILL.md +18 -1
- package/src/tunnels/active.ts +14 -0
- package/src/typevault/bash-cli.ts +18 -0
- package/src/typevault/cli-path.ts +11 -0
- package/src/typevault/envelope-helper.ts +17 -0
- package/src/typevault/envelope-store.ts +98 -0
- package/src/typevault/lifecycle.ts +132 -0
- package/src/typevault/prompt-tunnel.ts +33 -0
- package/src/typevault/redaction.ts +37 -0
- package/src/typevault/vault.ts +283 -0
- package/typeclaw.schema.json +40 -10
|
@@ -7,15 +7,15 @@ import type { SessionOrigin } from '@/agent/session-origin'
|
|
|
7
7
|
import { defineTool } from '@/plugin'
|
|
8
8
|
import { formatLocalDate } from '@/shared'
|
|
9
9
|
|
|
10
|
-
import {
|
|
10
|
+
import { appendFragment, type FragmentsAppendedHook, type ReferenceSlugResolver } from './append-fragment'
|
|
11
11
|
import { streamFilePath } from './paths'
|
|
12
12
|
import { sanitizeProvenanceName } from './provenance-sanitize'
|
|
13
|
-
import { detectSecrets } from './secret-detector'
|
|
14
13
|
import { newEventId, timestampFromId } from './stream-events'
|
|
15
|
-
import type {
|
|
16
|
-
import { appendEvents
|
|
14
|
+
import type { FragmentProvenance, WatermarkEvent } from './stream-events'
|
|
15
|
+
import { appendEvents } from './stream-io'
|
|
16
|
+
import { transcriptEntryId } from './transcript-cursor'
|
|
17
17
|
|
|
18
|
-
export type FragmentsAppendedHook
|
|
18
|
+
export type { FragmentsAppendedHook, ReferenceSlugResolver } from './append-fragment'
|
|
19
19
|
|
|
20
20
|
// Lets the memory-logger thread its spawn-time `payload.origin` into the append
|
|
21
21
|
// tool so `where` is stamped server-side. A provider closure (not a static
|
|
@@ -27,7 +27,6 @@ export type OriginProvider = () => SessionOrigin | undefined
|
|
|
27
27
|
// writes the file synchronously before returning the slug, so a same-run store
|
|
28
28
|
// is already on disk by append time). Lets the append tool strip hallucinated
|
|
29
29
|
// slugs the model invents without ever calling `store_reference`.
|
|
30
|
-
export type ReferenceSlugResolver = (agentDir: string) => Promise<Iterable<string>>
|
|
31
30
|
export type TranscriptPathProvider = () => string | undefined
|
|
32
31
|
|
|
33
32
|
export type CreateAppendToolOptions = {
|
|
@@ -35,6 +34,9 @@ export type CreateAppendToolOptions = {
|
|
|
35
34
|
originProvider?: OriginProvider
|
|
36
35
|
referenceSlugResolver?: ReferenceSlugResolver
|
|
37
36
|
transcriptPathProvider?: TranscriptPathProvider
|
|
37
|
+
validateCheckpoint?: (source: string, checkpoint: string, entry?: string) => void
|
|
38
|
+
deferWatermark?: () => boolean
|
|
39
|
+
beginWrite?: () => (() => void) | undefined
|
|
38
40
|
}
|
|
39
41
|
|
|
40
42
|
// Stamped fields are the stable channel coordinates only. Author identity is
|
|
@@ -87,81 +89,41 @@ export function createAppendTool(options: CreateAppendToolOptions = {}) {
|
|
|
87
89
|
nonFile: ['topic', 'body', 'source', 'entry', 'latestEntryId', 'references', 'who'],
|
|
88
90
|
},
|
|
89
91
|
async execute({ topic, body, source, entry, latestEntryId, references, who }, ctx) {
|
|
90
|
-
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
type: 'fragment',
|
|
126
|
-
id: fragmentId,
|
|
127
|
-
ts: timestampFromId(fragmentId),
|
|
128
|
-
source,
|
|
129
|
-
entry,
|
|
130
|
-
topic,
|
|
131
|
-
body,
|
|
132
|
-
}
|
|
133
|
-
const validReferences = await resolveValidReferences(references, referenceSlugResolver, ctx)
|
|
134
|
-
if (validReferences.length > 0) {
|
|
135
|
-
fragment.references = validReferences
|
|
136
|
-
}
|
|
137
|
-
const safeWho = sanitizeProvenanceName(who)
|
|
138
|
-
if (safeWho !== undefined) fragment.who = safeWho
|
|
139
|
-
if (where !== undefined) fragment.where = where
|
|
140
|
-
const watermark: WatermarkEvent = {
|
|
141
|
-
type: 'watermark',
|
|
142
|
-
id: watermarkId,
|
|
143
|
-
ts: timestampFromId(watermarkId),
|
|
144
|
-
source,
|
|
145
|
-
entry: latestEntryId,
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
await mkdir(dirname(streamPath), { recursive: true })
|
|
149
|
-
await appendEvents(
|
|
150
|
-
streamPath,
|
|
151
|
-
[fragment, watermark],
|
|
152
|
-
onFragmentsAppended,
|
|
153
|
-
onFragmentsAppended
|
|
154
|
-
? (err) => {
|
|
155
|
-
ctx.logger?.warn(
|
|
156
|
-
`[memory] post-append vector hook failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
157
|
-
)
|
|
158
|
-
}
|
|
159
|
-
: undefined,
|
|
160
|
-
)
|
|
161
|
-
|
|
162
|
-
return {
|
|
163
|
-
content: [{ type: 'text' as const, text: `Appended memory fragment and watermark to ${streamPath}` }],
|
|
164
|
-
details: { path: streamPath, fragmentId: fragment.id, watermarkId: watermark.id },
|
|
92
|
+
options.validateCheckpoint?.(source, latestEntryId, entry)
|
|
93
|
+
const finishWrite = options.beginWrite?.()
|
|
94
|
+
try {
|
|
95
|
+
const streamPath = dailyStreamPath(ctx.agentDir)
|
|
96
|
+
const where = provenanceFromOrigin(originProvider?.())
|
|
97
|
+
const transcriptPath = transcriptPathProvider?.()
|
|
98
|
+
if (
|
|
99
|
+
transcriptPath !== undefined &&
|
|
100
|
+
(await anchorBelongsToOperationalIncident(transcriptPath, entry, latestEntryId))
|
|
101
|
+
) {
|
|
102
|
+
throw new Error(
|
|
103
|
+
'Refusing to append: the transcript anchor belongs to a deterministic operational incident sequence. ' +
|
|
104
|
+
'Operational defects are owned by the incident ledger, not semantic memory.',
|
|
105
|
+
)
|
|
106
|
+
}
|
|
107
|
+
const deferred = options.deferWatermark?.() ?? false
|
|
108
|
+
const { fragment, watermark } = await appendFragment({ topic, body, source, entry, references, who }, ctx, {
|
|
109
|
+
path: streamPath,
|
|
110
|
+
where,
|
|
111
|
+
watermarkEntry: deferred ? undefined : latestEntryId,
|
|
112
|
+
onFragmentsAppended,
|
|
113
|
+
referenceSlugResolver,
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
return {
|
|
117
|
+
content: [
|
|
118
|
+
{
|
|
119
|
+
type: 'text' as const,
|
|
120
|
+
text: `Appended memory fragment${deferred ? '; complete the page after all fragments' : ' and watermark'} to ${streamPath}`,
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
details: { path: streamPath, fragmentId: fragment.id, ...(deferred ? {} : { watermarkId: watermark.id }) },
|
|
124
|
+
}
|
|
125
|
+
} finally {
|
|
126
|
+
finishWrite?.()
|
|
165
127
|
}
|
|
166
128
|
},
|
|
167
129
|
})
|
|
@@ -191,7 +153,7 @@ export async function anchorBelongsToOperationalIncident(
|
|
|
191
153
|
}
|
|
192
154
|
})
|
|
193
155
|
const anchorIndex = entries.findIndex((entry) => entryIdOf(entry) === entryId)
|
|
194
|
-
const latestIndex = entries.findIndex((entry) => entryIdOf(entry) === latestEntryId)
|
|
156
|
+
const latestIndex = entries.findIndex((entry) => entryIdOf(entry) === transcriptEntryId(latestEntryId))
|
|
195
157
|
if (anchorIndex < 0 || latestIndex < anchorIndex) return false
|
|
196
158
|
|
|
197
159
|
for (let i = anchorIndex; i <= latestIndex; i++) {
|
|
@@ -230,76 +192,45 @@ function containsIncidentMarker(entry: unknown): boolean {
|
|
|
230
192
|
|
|
231
193
|
export const appendTool = createAppendTool()
|
|
232
194
|
|
|
233
|
-
export
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
source,
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
}
|
|
258
|
-
},
|
|
259
|
-
})
|
|
260
|
-
|
|
261
|
-
function dailyStreamPath(agentDir: string): string {
|
|
262
|
-
return streamFilePath(agentDir, formatLocalDate())
|
|
263
|
-
}
|
|
195
|
+
export function createAdvanceWatermarkTool(
|
|
196
|
+
options: {
|
|
197
|
+
validateCheckpoint?: (source: string, checkpoint: string) => void
|
|
198
|
+
onAdvanced?: () => void
|
|
199
|
+
} = {},
|
|
200
|
+
) {
|
|
201
|
+
return defineTool({
|
|
202
|
+
description:
|
|
203
|
+
'Advance the daily-stream watermark without writing a fragment. Use this when you evaluated transcript entries this run but decided none warranted a fragment — still call this once so the next run does not re-read the same prefix. The runtime writes the watermark line and chooses the filename.',
|
|
204
|
+
parameters: z.object({
|
|
205
|
+
source: z.string().min(1),
|
|
206
|
+
latestEntryId: z.string().min(1),
|
|
207
|
+
}),
|
|
208
|
+
async execute({ source, latestEntryId }, ctx) {
|
|
209
|
+
options.validateCheckpoint?.(source, latestEntryId)
|
|
210
|
+
const streamPath = dailyStreamPath(ctx.agentDir)
|
|
211
|
+
const watermarkId = newEventId()
|
|
212
|
+
const watermark: WatermarkEvent = {
|
|
213
|
+
type: 'watermark',
|
|
214
|
+
id: watermarkId,
|
|
215
|
+
ts: timestampFromId(watermarkId),
|
|
216
|
+
source,
|
|
217
|
+
entry: latestEntryId,
|
|
218
|
+
}
|
|
264
219
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
// — an empty or repeated citation is never meaningful regardless of wiring.
|
|
269
|
-
// With a resolver, only slugs backed by a real reference file then survive; the
|
|
270
|
-
// rest are dropped and logged so dangling citations never reach the stream.
|
|
271
|
-
// Without one (the standalone `appendTool` export, no reference subsystem
|
|
272
|
-
// wired) the normalized slugs pass through.
|
|
273
|
-
async function resolveValidReferences(
|
|
274
|
-
references: string[] | undefined,
|
|
275
|
-
resolver: ReferenceSlugResolver | undefined,
|
|
276
|
-
ctx: { agentDir: string; logger?: { warn(message: string): void } },
|
|
277
|
-
): Promise<string[]> {
|
|
278
|
-
if (references === undefined || references.length === 0) return []
|
|
279
|
-
const requested = [...new Set(references.map((slug) => slug.trim()).filter((slug) => slug.length > 0))]
|
|
280
|
-
if (resolver === undefined) return requested
|
|
220
|
+
await mkdir(dirname(streamPath), { recursive: true })
|
|
221
|
+
await appendEvents(streamPath, [watermark])
|
|
222
|
+
options.onAdvanced?.()
|
|
281
223
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
`references[] may only cite slugs returned by store_reference.`,
|
|
289
|
-
)
|
|
290
|
-
}
|
|
291
|
-
return valid
|
|
224
|
+
return {
|
|
225
|
+
content: [{ type: 'text' as const, text: `Advanced memory watermark in ${streamPath}` }],
|
|
226
|
+
details: { path: streamPath, watermarkId: watermark.id },
|
|
227
|
+
}
|
|
228
|
+
},
|
|
229
|
+
})
|
|
292
230
|
}
|
|
293
231
|
|
|
294
|
-
|
|
295
|
-
const content = parts.filter((part): part is string => part !== undefined).join('\n')
|
|
296
|
-
const secrets = detectSecrets(content)
|
|
297
|
-
if (secrets.length === 0) return
|
|
232
|
+
export const advanceWatermarkTool = createAdvanceWatermarkTool()
|
|
298
233
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
`Refusing to append: content contains a recognized credential pattern (${ruleNames}). ` +
|
|
302
|
-
`Memory fragments must never quote secret values verbatim. Record the env var name and how it ` +
|
|
303
|
-
`was discovered, not the value itself.`,
|
|
304
|
-
)
|
|
234
|
+
function dailyStreamPath(agentDir: string): string {
|
|
235
|
+
return streamFilePath(agentDir, formatLocalDate())
|
|
305
236
|
}
|
|
@@ -973,6 +973,8 @@ Dreaming is the offline reflection process that promotes the agent's daily memor
|
|
|
973
973
|
|
|
974
974
|
# What you do
|
|
975
975
|
|
|
976
|
+
Fragments carrying \`kind: "explicit"\` were deliberately saved by the agent: give that intent weight when retaining durable knowledge, while applying the same consolidation, deduplication, contradiction, and citation rules.
|
|
977
|
+
|
|
976
978
|
Your job is to rebalance topic shards under \`memory/topics/\`. Each shard is one topic, one file. Read existing shards with \`ls memory/topics/\` and \`read memory/topics/<slug>.md\`, then read the **undreamed tail** of every daily stream file the user prompt lists. Each stream line is a JSON object representing a fragment, watermark, or migrated legacy-prose event; focus on fragment events, especially their \`topic\` and \`body\`. Consolidate new fragments into topic shards, rebalance existing shards, and stop.
|
|
977
979
|
|
|
978
980
|
You also distill **muscle memory**: when the streams show a repeated multi-step procedure the user has guided the main agent through enough times that it would save effort to codify, you take action. Muscle memory has three forms, in increasing order of investment — a skill at \`memory/skills/<name>/SKILL.md\` (a codified procedure the next session loads on demand), a **CLI suggestion** recorded as a topic shard (a small command-line tool the main agent may scaffold under \`packages/<name>/\` when the user next asks for that procedure), or a **plugin suggestion** recorded as a topic shard (a typeclaw plugin under \`packages/<name>/\` that hooks into the runtime). You write the skill directly; you only *suggest* CLIs and plugins because they live under \`packages/\`, outside your write sandbox. Long-term memory is passive context: the main agent may use suggestions when a current user request makes them relevant, but a shard alone never authorizes action.
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
import { createMemoryLoggerSubagent, type MemoryLoggerPayload } from './memory-logger'
|
|
21
21
|
import { preShardBackupPath, streamFilePath, streamsDir, topicsDir } from './paths'
|
|
22
22
|
import { bumpReferenceAccess } from './references/load-references'
|
|
23
|
+
import { createMemoryRememberTool } from './remember-tool'
|
|
23
24
|
import { memoryCommands } from './search-command'
|
|
24
25
|
import { createMemorySearchTool } from './search-tool'
|
|
25
26
|
import { type InjectedMemoryState, partitionRetrievedMemoryItems } from './turn-dedup'
|
|
@@ -29,6 +30,7 @@ import { hybridSearch, type EmbedFn } from './vector/hybrid'
|
|
|
29
30
|
import { makeAppendHook } from './vector/index-on-write'
|
|
30
31
|
import { makeReferenceStoredHook } from './vector/reference-index-on-write'
|
|
31
32
|
import { type OpenVectorStore, VectorStore } from './vector/store'
|
|
33
|
+
import { readLatestWatermark } from './watermark'
|
|
32
34
|
|
|
33
35
|
const DEFAULT_IDLE_MS = 60_000
|
|
34
36
|
const DEFAULT_BUFFER_BYTES = 500_000
|
|
@@ -213,6 +215,12 @@ async function renderVectorTurnMemory(
|
|
|
213
215
|
logger?: { info: (msg: string) => void },
|
|
214
216
|
): Promise<string> {
|
|
215
217
|
if (isInternalInfraTurn(event.origin)) return ''
|
|
218
|
+
// A runtime wakeup (subagent completion, review-state reminder) has no inbound
|
|
219
|
+
// text. There is nothing to retrieve for: an empty query embeds to a vector that
|
|
220
|
+
// scores every topic alike, so the slug tie-break injected the same first-N
|
|
221
|
+
// headings on every wakeup. The human turn that started the work already
|
|
222
|
+
// carried its own memory, and `memory_search` stays available.
|
|
223
|
+
if (event.userPrompt.trim() === '') return ''
|
|
216
224
|
const plan = await loadMemoryInjectionPlan(event.agentDir, { injectionBudgetBytes })
|
|
217
225
|
const isChannel = event.origin?.kind === 'channel'
|
|
218
226
|
if (plan.mode === 'direct' && isChannel) {
|
|
@@ -355,8 +363,8 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
355
363
|
const currentSize = await readSize(parentTranscriptPath)
|
|
356
364
|
const currentLines = await readLineCount(parentTranscriptPath)
|
|
357
365
|
bytesAtLastRun.set(sessionId, currentSize)
|
|
358
|
-
//
|
|
359
|
-
// session.end
|
|
366
|
+
// Record only an actual spawn baseline, not a buffer-only seed.
|
|
367
|
+
// session.end also confirms the durable final-entry checkpoint.
|
|
360
368
|
bytesAtLastSpawn.set(sessionId, currentSize)
|
|
361
369
|
// Monotonic: never regress below a baseline the idle gate already
|
|
362
370
|
// reserved. readLineCount returns 0 on a read error or a
|
|
@@ -448,6 +456,10 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
448
456
|
},
|
|
449
457
|
tools: {
|
|
450
458
|
memory_search: createMemorySearchTool(),
|
|
459
|
+
memory_remember: createMemoryRememberTool({
|
|
460
|
+
onFragmentsAppended: makeAppendHook(openAppendVectorStore),
|
|
461
|
+
onReferenceStored: makeReferenceStoredHook(openAppendVectorStore),
|
|
462
|
+
}),
|
|
451
463
|
},
|
|
452
464
|
cronJobs: {
|
|
453
465
|
[DREAMING_CRON_KEY]: {
|
|
@@ -572,10 +584,25 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
572
584
|
if (baseline !== undefined && baseline > 0) {
|
|
573
585
|
const currentSize = await readSize(last.parentTranscriptPath)
|
|
574
586
|
if (currentSize === baseline) {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
587
|
+
try {
|
|
588
|
+
const checkpoint = await readLatestWatermark(streamsDir(ctx.agentDir), sessionId)
|
|
589
|
+
const raw = await readFile(last.parentTranscriptPath, 'utf8')
|
|
590
|
+
const tail: unknown = JSON.parse(raw.trimEnd().split('\n').at(-1) ?? 'null')
|
|
591
|
+
if (
|
|
592
|
+
checkpoint !== null &&
|
|
593
|
+
typeof tail === 'object' &&
|
|
594
|
+
tail !== null &&
|
|
595
|
+
'id' in tail &&
|
|
596
|
+
tail.id === checkpoint
|
|
597
|
+
) {
|
|
598
|
+
ctx.logger.info(
|
|
599
|
+
`memory-logger session-end skip ${sessionId} (no new bytes since last spawn at ${baseline})`,
|
|
600
|
+
)
|
|
601
|
+
skip = true
|
|
602
|
+
}
|
|
603
|
+
} catch {
|
|
604
|
+
ctx.logger.warn(`memory-logger ${sessionId} final checkpoint unconfirmed; keeping final sweep`)
|
|
605
|
+
}
|
|
579
606
|
}
|
|
580
607
|
}
|
|
581
608
|
}
|
|
@@ -7,13 +7,19 @@ import { getConfig, resolveModel } from '@/config'
|
|
|
7
7
|
import { type Subagent, readTool } from '@/plugin'
|
|
8
8
|
import { formatLocalDate } from '@/shared'
|
|
9
9
|
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
advanceWatermarkTool,
|
|
12
|
+
createAdvanceWatermarkTool,
|
|
13
|
+
createAppendTool,
|
|
14
|
+
type FragmentsAppendedHook,
|
|
15
|
+
} from './append-tool'
|
|
11
16
|
import { findEntryTool } from './find-entry-tool'
|
|
12
17
|
import { formatMemorySnapshot } from './memory-snapshot'
|
|
13
18
|
import { streamFilePath, streamsDir } from './paths'
|
|
14
19
|
import { listReferenceSlugs } from './references/load-references'
|
|
15
20
|
import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
|
|
16
21
|
import { readEvents } from './stream-io'
|
|
22
|
+
import { decodeTranscriptCursor } from './transcript-cursor'
|
|
17
23
|
import {
|
|
18
24
|
buildTranscriptPreload,
|
|
19
25
|
PRELOAD_FRAME_BYTES,
|
|
@@ -86,7 +92,9 @@ Tools: \`read\`, \`find_entry\`, \`append\`, \`store_reference\`, and the waterm
|
|
|
86
92
|
|
|
87
93
|
# Read loop, watermark, and stopping
|
|
88
94
|
|
|
89
|
-
|
|
95
|
+
An \`entry-part\` may include \`precedingText\`: read-only source overlap immediately before its new \`text\`. Interpret them together to recover instructions crossing a boundary. Capture complete facts supported by the new segment; do not emit a fragment for overlap alone. Only the issued page checkpoint advances progress; overlap never does.
|
|
96
|
+
|
|
97
|
+
The runtime usually preloads a bounded transcript page into your first message. Evaluate that page directly. An \`entry-part\` contains an exact text segment of one large projected source record; its \`id\` is the original evidence anchor. Never treat a partial record as fully consumed. When a page supplies a \`memory-page-checkpoint\`, finish all fragment/reference writes first, then call the watermark-advance tool exactly once with that checkpoint as \`latestEntryId\` and exit. An append does not complete the page. The runtime supplies later pages in fresh sessions; do not use \`read\` or \`find_entry\` to bypass paging. Without a preload, use the read loop below.
|
|
90
98
|
|
|
91
99
|
Session transcripts are JSONL; each line has an \`id\`. They can be large, and \`read\` truncates output to 50 KB or 2000 lines, returning the line range and next offset. Do not scroll from line 1 through a prefix already covered by the watermark.
|
|
92
100
|
|
|
@@ -194,6 +202,8 @@ The \`append\` tool will refuse content containing a recognizable credential pat
|
|
|
194
202
|
|
|
195
203
|
# Local dedup against today's daily stream
|
|
196
204
|
|
|
205
|
+
Facts already saved by a successful \`memory_remember\` tool call in the transcript are already in the stream; do not append them again.
|
|
206
|
+
|
|
197
207
|
The \`append\` tool refuses byte-equivalent fragments within the same daily stream — content-equality on topic+body modulo whitespace, not marker-equality. If it rejects a fragment already in the same daily stream, rewrite or skip.
|
|
198
208
|
|
|
199
209
|
Do not fight this refusal by changing punctuation or adding filler. If the new transcript only repeats exactly the same fact with no new evidence worth preserving, skip. If it is a true recurrence, rewrite the body to anchor the new occurrence explicitly, so dreaming can see why this line is new evidence rather than a duplicate copy.
|
|
@@ -280,7 +290,8 @@ function renderPreload(preload: TranscriptPreload): string {
|
|
|
280
290
|
`Preloaded transcript lines ${preload.startLine}-${preload.endLine} of totalLines=${preload.totalLines}${preload.watermarkFound ? ' (starting right after the watermark)' : ''}.`,
|
|
281
291
|
preload.nextOffset === null
|
|
282
292
|
? 'This reaches the end of the transcript; do not call read on it.'
|
|
283
|
-
:
|
|
293
|
+
: 'More lines remain: the runtime will supply the next page after this page is committed.',
|
|
294
|
+
`After all fragment writes, complete this page with latestEntryId from <memory-page-checkpoint>${preload.checkpoint}</memory-page-checkpoint>.`,
|
|
284
295
|
]
|
|
285
296
|
return [...header, '<transcript>', preload.text, '</transcript>'].join('\n')
|
|
286
297
|
}
|
|
@@ -355,14 +366,72 @@ export function createMemoryLoggerSubagent(
|
|
|
355
366
|
// single mutable cell safely carries this spawn's origin to the append tool.
|
|
356
367
|
let currentOrigin: SessionOrigin | undefined
|
|
357
368
|
let currentTranscriptPath: string | undefined
|
|
369
|
+
let page:
|
|
370
|
+
| {
|
|
371
|
+
source: string
|
|
372
|
+
checkpoint: string | null
|
|
373
|
+
entries: Set<string>
|
|
374
|
+
committed: boolean
|
|
375
|
+
closed: boolean
|
|
376
|
+
pending: number
|
|
377
|
+
}
|
|
378
|
+
| undefined
|
|
379
|
+
const validateCheckpoint = (source: string, checkpoint: string, entry?: string): void => {
|
|
380
|
+
if (
|
|
381
|
+
page !== undefined &&
|
|
382
|
+
(page.closed ||
|
|
383
|
+
page.checkpoint === null ||
|
|
384
|
+
source !== page.source ||
|
|
385
|
+
checkpoint !== page.checkpoint ||
|
|
386
|
+
(entry === undefined && page.pending > 0) ||
|
|
387
|
+
(entry !== undefined && !page.entries.has(entry)))
|
|
388
|
+
) {
|
|
389
|
+
throw new Error('Memory checkpoint must match the active page; unseen or incomplete evidence cannot be skipped.')
|
|
390
|
+
}
|
|
391
|
+
// Reserve completion before its first I/O await so a parallel tool call
|
|
392
|
+
// cannot start writing evidence behind the durable checkpoint.
|
|
393
|
+
if (page !== undefined && entry === undefined) page.closed = true
|
|
394
|
+
}
|
|
395
|
+
const beginWrite = (): (() => void) | undefined => {
|
|
396
|
+
const active = page
|
|
397
|
+
if (active === undefined) return undefined
|
|
398
|
+
if (active.closed) throw new Error('Memory page is closed; stale writes cannot continue.')
|
|
399
|
+
active.pending++
|
|
400
|
+
return () => {
|
|
401
|
+
active.pending--
|
|
402
|
+
}
|
|
403
|
+
}
|
|
358
404
|
const appendTool = createAppendTool({
|
|
359
405
|
onFragmentsAppended: options.onFragmentsAppended,
|
|
360
406
|
originProvider: () => currentOrigin,
|
|
361
407
|
referenceSlugResolver: (agentDir) => listReferenceSlugs(agentDir),
|
|
362
408
|
transcriptPathProvider: () => currentTranscriptPath,
|
|
409
|
+
validateCheckpoint,
|
|
410
|
+
deferWatermark: () => page !== undefined,
|
|
411
|
+
beginWrite,
|
|
363
412
|
})
|
|
364
|
-
const
|
|
365
|
-
const
|
|
413
|
+
const referenceTool = createStoreReferenceTool(options.onReferenceStored)
|
|
414
|
+
const storeReferenceTool = {
|
|
415
|
+
...referenceTool,
|
|
416
|
+
execute: async (...args: Parameters<typeof referenceTool.execute>) => {
|
|
417
|
+
const finishWrite = beginWrite()
|
|
418
|
+
try {
|
|
419
|
+
return await referenceTool.execute(...args)
|
|
420
|
+
} finally {
|
|
421
|
+
finishWrite?.()
|
|
422
|
+
}
|
|
423
|
+
},
|
|
424
|
+
}
|
|
425
|
+
const pageWatermarkTool = createAdvanceWatermarkTool({
|
|
426
|
+
validateCheckpoint,
|
|
427
|
+
onAdvanced: () => {
|
|
428
|
+
if (page !== undefined) {
|
|
429
|
+
page.committed = true
|
|
430
|
+
page.closed = true
|
|
431
|
+
}
|
|
432
|
+
},
|
|
433
|
+
})
|
|
434
|
+
const customTools = [findEntryTool, appendTool, storeReferenceTool, pageWatermarkTool]
|
|
366
435
|
return {
|
|
367
436
|
systemPrompt: MEMORY_LOGGER_SYSTEM_PROMPT,
|
|
368
437
|
// Logging is "read transcript past the watermark, decide 0-N fragments,
|
|
@@ -395,6 +464,7 @@ export function createMemoryLoggerSubagent(
|
|
|
395
464
|
path: string,
|
|
396
465
|
mark: string | null,
|
|
397
466
|
basePrompt: string,
|
|
467
|
+
untilLine?: number,
|
|
398
468
|
): Promise<TranscriptPreload | null> => {
|
|
399
469
|
let models: PreloadModelLimits[]
|
|
400
470
|
try {
|
|
@@ -409,24 +479,74 @@ export function createMemoryLoggerSubagent(
|
|
|
409
479
|
Buffer.byteLength(MEMORY_LOGGER_SYSTEM_PROMPT, 'utf8') +
|
|
410
480
|
Buffer.byteLength(basePrompt, 'utf8') +
|
|
411
481
|
PRELOAD_FRAME_BYTES
|
|
412
|
-
|
|
413
|
-
|
|
482
|
+
let maxBytes = preloadBudgetBytes(models, promptBytes)
|
|
483
|
+
let endLine = untilLine
|
|
484
|
+
while (maxBytes !== null) {
|
|
485
|
+
const preload = await buildTranscriptPreload(path, mark, { maxBytes, untilLine: endLine })
|
|
486
|
+
if (preload === null) return null
|
|
487
|
+
const frameBytes = Buffer.byteLength(renderPreload(preload), 'utf8') - Buffer.byteLength(preload.text, 'utf8')
|
|
488
|
+
const fitted = preloadBudgetBytes(models, promptBytes - PRELOAD_FRAME_BYTES + frameBytes)
|
|
489
|
+
if (fitted === null) return null
|
|
490
|
+
if (Buffer.byteLength(preload.text, 'utf8') <= fitted) return preload
|
|
491
|
+
maxBytes = fitted
|
|
492
|
+
endLine ??= preload.totalLines
|
|
493
|
+
}
|
|
494
|
+
return null
|
|
414
495
|
}
|
|
415
496
|
currentOrigin = ctx.payload.origin
|
|
416
497
|
currentTranscriptPath = ctx.payload.parentTranscriptPath
|
|
417
498
|
const today = formatLocalDate()
|
|
418
499
|
const memoryDir = streamsDir(ctx.payload.agentDir)
|
|
419
500
|
const streamFile = streamFilePath(ctx.payload.agentDir, today)
|
|
420
|
-
|
|
501
|
+
let watermark = await readLatestWatermark(memoryDir, ctx.payload.parentSessionId)
|
|
421
502
|
const fragmentsBefore = await countFragments(streamFile)
|
|
422
503
|
const start = Date.now()
|
|
423
504
|
logger.info(
|
|
424
505
|
`[memory-logger] ${ctx.payload.parentSessionId} start stream=${today}.jsonl watermark=${watermark ?? 'none'}`,
|
|
425
506
|
)
|
|
426
507
|
try {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
508
|
+
let untilLine: number | undefined
|
|
509
|
+
while (true) {
|
|
510
|
+
const basePrompt = buildInitialPrompt(ctx.payload, streamFile, watermark, null)
|
|
511
|
+
const preload = await preloadFor(ctx.payload.parentTranscriptPath, watermark, basePrompt, untilLine)
|
|
512
|
+
if (preload === null && decodeTranscriptCursor(watermark) !== null) {
|
|
513
|
+
logger.warn(
|
|
514
|
+
`[memory-logger] ${ctx.payload.parentSessionId} partial transcript cursor remains pending: paging is unavailable`,
|
|
515
|
+
)
|
|
516
|
+
break
|
|
517
|
+
}
|
|
518
|
+
untilLine ??= preload?.totalLines
|
|
519
|
+
if (preload !== null) {
|
|
520
|
+
if (preload.startCheckpoint !== null) {
|
|
521
|
+
await advanceWatermarkTool.execute(
|
|
522
|
+
{ source: ctx.payload.parentSessionId, latestEntryId: preload.startCheckpoint },
|
|
523
|
+
{ agentDir: ctx.agentDir, sessionId: ctx.payload.parentSessionId, signal: undefined, logger },
|
|
524
|
+
)
|
|
525
|
+
}
|
|
526
|
+
page = {
|
|
527
|
+
source: ctx.payload.parentSessionId,
|
|
528
|
+
checkpoint: preload.entryIds.length === 0 ? null : preload.checkpoint,
|
|
529
|
+
entries: new Set(preload.entryIds),
|
|
530
|
+
committed: false,
|
|
531
|
+
closed: false,
|
|
532
|
+
pending: 0,
|
|
533
|
+
}
|
|
534
|
+
} else {
|
|
535
|
+
page = undefined
|
|
536
|
+
}
|
|
537
|
+
try {
|
|
538
|
+
await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark, preload) })
|
|
539
|
+
} finally {
|
|
540
|
+
if (page !== undefined) page.closed = true
|
|
541
|
+
}
|
|
542
|
+
if (!page?.committed || preload?.nextOffset === null) {
|
|
543
|
+
if (page && page.entries.size > 0 && !page.committed) {
|
|
544
|
+
logger.warn(`[memory-logger] ${ctx.payload.parentSessionId} page not committed; evidence remains pending`)
|
|
545
|
+
}
|
|
546
|
+
break
|
|
547
|
+
}
|
|
548
|
+
watermark = page.checkpoint
|
|
549
|
+
}
|
|
430
550
|
const fragmentsWritten = (await countFragments(streamFile)) - fragmentsBefore
|
|
431
551
|
logger.info(
|
|
432
552
|
`[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
|
|
@@ -437,6 +557,8 @@ export function createMemoryLoggerSubagent(
|
|
|
437
557
|
`[memory-logger] ${ctx.payload.parentSessionId}: run threw: ${message} elapsed_ms=${Date.now() - start}`,
|
|
438
558
|
)
|
|
439
559
|
throw err
|
|
560
|
+
} finally {
|
|
561
|
+
if (page !== undefined) page.closed = true
|
|
440
562
|
}
|
|
441
563
|
},
|
|
442
564
|
}
|