typeclaw 0.51.0 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/dist/types/agent/bash-sandbox-env.d.ts +2 -0
  2. package/dist/types/agent/bash-sandbox.d.ts +4 -0
  3. package/dist/types/agent/index.d.ts +5 -0
  4. package/dist/types/agent/multimodal/look-at.d.ts +1 -1
  5. package/dist/types/agent/plugin-tools.d.ts +4 -0
  6. package/dist/types/agent/prompt-cache-key.d.ts +1 -0
  7. package/dist/types/agent/session-tail.d.ts +12 -0
  8. package/dist/types/agent/subagents.d.ts +5 -0
  9. package/dist/types/agent/system-prompt-composition.d.ts +1 -0
  10. package/dist/types/agent/system-prompt.d.ts +2 -0
  11. package/dist/types/agent/tools/channel-disengage.d.ts +1 -1
  12. package/dist/types/agent/tools/channel-edit.d.ts +1 -1
  13. package/dist/types/agent/tools/channel-fetch-attachment.d.ts +1 -1
  14. package/dist/types/agent/tools/channel-handoff.d.ts +1 -1
  15. package/dist/types/agent/tools/channel-history.d.ts +1 -1
  16. package/dist/types/agent/tools/channel-react.d.ts +1 -1
  17. package/dist/types/agent/tools/channel-read.d.ts +10 -2
  18. package/dist/types/agent/tools/channel-reply.d.ts +1 -1
  19. package/dist/types/agent/tools/channel-send.d.ts +1 -1
  20. package/dist/types/agent/tools/channel-session-search.d.ts +1 -1
  21. package/dist/types/agent/tools/grant-role.d.ts +1 -1
  22. package/dist/types/agent/tools/post-github-review.d.ts +1 -1
  23. package/dist/types/agent/tools/restart.d.ts +1 -1
  24. package/dist/types/agent/tools/skip-response.d.ts +1 -1
  25. package/dist/types/agent/tools/spawn-subagent.d.ts +2 -1
  26. package/dist/types/agent/tools/stream-snapshot.d.ts +1 -1
  27. package/dist/types/agent/tools/subagent-cancel.d.ts +1 -1
  28. package/dist/types/agent/tools/subagent-output.d.ts +1 -1
  29. package/dist/types/agent/tools/todo/index.d.ts +1 -1
  30. package/dist/types/agent/tools/webfetch/adaptive-html.d.ts +3 -3
  31. package/dist/types/agent/tools/webfetch/strategies/readability-process.d.ts +48 -0
  32. package/dist/types/agent/tools/webfetch/tool.d.ts +2 -2
  33. package/dist/types/agent/tools/websearch.d.ts +1 -1
  34. package/dist/types/channels/adapters/github/auth-app.d.ts +25 -0
  35. package/dist/types/channels/adapters/github/auth-pat.d.ts +17 -0
  36. package/dist/types/channels/adapters/github/auth.d.ts +24 -0
  37. package/dist/types/channels/github-token-bridge.d.ts +4 -0
  38. package/dist/types/channels/lost-background-context.d.ts +2 -0
  39. package/dist/types/channels/persistence.d.ts +1 -0
  40. package/dist/types/channels/router.d.ts +5 -1
  41. package/dist/types/channels/types.d.ts +1 -0
  42. package/dist/types/coding-agent/tool.d.ts +1 -1
  43. package/dist/types/config/config.d.ts +20 -11
  44. package/dist/types/config/providers.d.ts +47 -0
  45. package/dist/types/eval/sandbox.d.ts +2 -0
  46. package/dist/types/git/resolve-identity.d.ts +2 -0
  47. package/dist/types/init/dockerfile.d.ts +1 -0
  48. package/dist/types/init/gitignore.d.ts +1 -1
  49. package/dist/types/permissions/builtins.d.ts +1 -0
  50. package/dist/types/plugin/types.d.ts +3 -0
  51. package/dist/types/sandbox/canonical-secrets.d.ts +2 -2
  52. package/dist/types/sandbox/coding-agent.d.ts +1 -1
  53. package/dist/types/sandbox/dependency-bins.d.ts +1 -0
  54. package/dist/types/sandbox/dev-source.d.ts +2 -2
  55. package/dist/types/sandbox/hidden-paths.d.ts +1 -0
  56. package/dist/types/sandbox/index.d.ts +1 -1
  57. package/dist/types/sandbox/policy.d.ts +3 -0
  58. package/dist/types/typevault/cli-path.d.ts +2 -0
  59. package/dist/types/typevault/envelope-store.d.ts +11 -0
  60. package/dist/types/typevault/lifecycle.d.ts +13 -0
  61. package/dist/types/typevault/vault.d.ts +27 -0
  62. package/package.json +8 -7
  63. package/scripts/test.ts +252 -0
  64. package/src/agent/bash-sandbox-env.ts +33 -1
  65. package/src/agent/bash-sandbox.ts +20 -6
  66. package/src/agent/index.ts +120 -31
  67. package/src/agent/multimodal/look-at.ts +1 -1
  68. package/src/agent/plugin-tools.ts +35 -1
  69. package/src/agent/prompt-cache-key.ts +26 -3
  70. package/src/agent/session-origin.ts +39 -18
  71. package/src/agent/session-tail.ts +28 -0
  72. package/src/agent/subagents.ts +21 -2
  73. package/src/agent/system-prompt-composition.ts +18 -0
  74. package/src/agent/system-prompt.ts +39 -1
  75. package/src/agent/tool-error-remediation.ts +18 -3
  76. package/src/agent/tools/channel-disengage.ts +1 -1
  77. package/src/agent/tools/channel-edit.ts +1 -1
  78. package/src/agent/tools/channel-fetch-attachment.ts +1 -1
  79. package/src/agent/tools/channel-handoff.ts +3 -2
  80. package/src/agent/tools/channel-history.ts +2 -1
  81. package/src/agent/tools/channel-react.ts +1 -1
  82. package/src/agent/tools/channel-read.ts +36 -3
  83. package/src/agent/tools/channel-reply.ts +1 -1
  84. package/src/agent/tools/channel-send.ts +4 -3
  85. package/src/agent/tools/channel-session-search.ts +13 -4
  86. package/src/agent/tools/grant-role.ts +1 -1
  87. package/src/agent/tools/post-github-review.ts +1 -1
  88. package/src/agent/tools/restart.ts +1 -1
  89. package/src/agent/tools/skip-response.ts +1 -1
  90. package/src/agent/tools/spawn-subagent.ts +15 -1
  91. package/src/agent/tools/stream-snapshot.ts +1 -1
  92. package/src/agent/tools/subagent-cancel.ts +1 -1
  93. package/src/agent/tools/subagent-output.ts +1 -1
  94. package/src/agent/tools/todo/index.ts +1 -1
  95. package/src/agent/tools/webfetch/adaptive-html.ts +6 -4
  96. package/src/agent/tools/webfetch/strategies/readability-child.ts +25 -0
  97. package/src/agent/tools/webfetch/strategies/readability-process.ts +185 -0
  98. package/src/agent/tools/webfetch/tool.ts +6 -5
  99. package/src/agent/tools/websearch.ts +1 -1
  100. package/src/browser/dockerfile.ts +1 -1
  101. package/src/browser/fill-secret.ts +92 -35
  102. package/src/browser/registry.ts +6 -6
  103. package/src/bundled-plugins/agent-messenger-cli-auth/command.ts +211 -0
  104. package/src/bundled-plugins/agent-messenger-cli-auth/index.ts +186 -0
  105. package/src/bundled-plugins/backup/index.ts +17 -2
  106. package/src/bundled-plugins/backup/runner.ts +37 -2
  107. package/src/bundled-plugins/browser-agent/browser-agent.ts +51 -14
  108. package/src/bundled-plugins/browser-agent/index.ts +12 -5
  109. package/src/bundled-plugins/managed-browser/index.ts +25 -2
  110. package/src/bundled-plugins/managed-browser/skills/browserskill/browserskill/SKILL.md +13 -5
  111. package/src/bundled-plugins/managed-browser/skills/omowright/omowright/SKILL.md +56 -5
  112. package/src/bundled-plugins/memory/README.md +23 -2
  113. package/src/bundled-plugins/memory/append-fragment.ts +133 -0
  114. package/src/bundled-plugins/memory/append-tool.ts +80 -149
  115. package/src/bundled-plugins/memory/dreaming.ts +2 -0
  116. package/src/bundled-plugins/memory/index.ts +33 -6
  117. package/src/bundled-plugins/memory/memory-logger.ts +133 -11
  118. package/src/bundled-plugins/memory/references/store-reference-tool.ts +56 -32
  119. package/src/bundled-plugins/memory/remember-tool.ts +81 -0
  120. package/src/bundled-plugins/memory/search-tool-hybrid.ts +91 -0
  121. package/src/bundled-plugins/memory/search-tool.ts +58 -12
  122. package/src/bundled-plugins/memory/stream-events.ts +2 -0
  123. package/src/bundled-plugins/memory/transcript-cursor.ts +30 -0
  124. package/src/bundled-plugins/memory/transcript-preload.ts +131 -26
  125. package/src/bundled-plugins/memory/transcript-view.ts +36 -0
  126. package/src/bundled-plugins/memory/watermark.ts +19 -10
  127. package/src/bundled-plugins/reviewer/reviewer.ts +20 -0
  128. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +3 -6
  129. package/src/bundled-plugins/security/policies/outbound-text.ts +20 -0
  130. package/src/bundled-plugins/security/policies/system-prompt-leak.ts +2 -7
  131. package/src/bundled-plugins/typevault/grants.ts +119 -0
  132. package/src/bundled-plugins/typevault/index.ts +152 -0
  133. package/src/bundled-plugins/typevault/request.ts +87 -0
  134. package/src/bundled-plugins/typevault/runtime.ts +45 -0
  135. package/src/bundled-plugins/typevault/skills/typevault/SKILL.md +65 -0
  136. package/src/channels/adapter-halt.ts +17 -0
  137. package/src/channels/adapters/github/history.ts +23 -2
  138. package/src/channels/adapters/github/inbound.ts +60 -1
  139. package/src/channels/adapters/github/index.ts +27 -5
  140. package/src/channels/adapters/github/reconcile-open-prs.ts +7 -1
  141. package/src/channels/adapters/kakaotalk-author-resolver.ts +7 -5
  142. package/src/channels/adapters/kakaotalk-channel-resolver.ts +45 -15
  143. package/src/channels/adapters/kakaotalk-classify.ts +7 -9
  144. package/src/channels/adapters/kakaotalk-list.ts +62 -0
  145. package/src/channels/adapters/kakaotalk-member-cache.ts +67 -0
  146. package/src/channels/adapters/kakaotalk-membership.ts +27 -10
  147. package/src/channels/adapters/kakaotalk-mention.ts +61 -0
  148. package/src/channels/adapters/kakaotalk.ts +159 -125
  149. package/src/channels/adapters/teams-classify.ts +18 -8
  150. package/src/channels/github-token-bridge.ts +13 -0
  151. package/src/channels/lost-background-context.ts +46 -0
  152. package/src/channels/manager.ts +52 -3
  153. package/src/channels/persistence.ts +5 -0
  154. package/src/channels/router.ts +120 -21
  155. package/src/channels/schema.ts +1 -1
  156. package/src/channels/types.ts +3 -0
  157. package/src/coding-agent/tool.ts +1 -1
  158. package/src/config/config.ts +20 -1
  159. package/src/config/providers.ts +65 -0
  160. package/src/container/start.ts +15 -2
  161. package/src/doctor/channel-checks.ts +33 -0
  162. package/src/doctor/checks.ts +12 -2
  163. package/src/eval/sandbox.ts +3 -0
  164. package/src/eval/tool.ts +1 -0
  165. package/src/git/resolve-identity.ts +67 -2
  166. package/src/git/system-commit.ts +7 -0
  167. package/src/init/dockerfile.ts +5 -1
  168. package/src/init/gitignore.ts +3 -3
  169. package/src/init/index.ts +1 -0
  170. package/src/init/models-dev.ts +2 -0
  171. package/src/init/validate-api-key.ts +5 -0
  172. package/src/migrations/secrets-v1-to-v2.ts +2 -0
  173. package/src/permissions/builtins.ts +8 -0
  174. package/src/plugin/types.ts +6 -0
  175. package/src/run/bundled-plugins.ts +12 -0
  176. package/src/run/channel-session-factory.ts +3 -0
  177. package/src/run/index.ts +13 -2
  178. package/src/sandbox/build.ts +1 -0
  179. package/src/sandbox/canonical-secrets.ts +6 -0
  180. package/src/sandbox/dependency-bins.ts +13 -1
  181. package/src/sandbox/dev-source.ts +21 -12
  182. package/src/sandbox/hidden-paths.ts +1 -1
  183. package/src/sandbox/index.ts +1 -0
  184. package/src/sandbox/policy.ts +5 -0
  185. package/src/secrets/hydrate.ts +4 -1
  186. package/src/secrets/storage.ts +10 -1
  187. package/src/server/command-runner.ts +4 -0
  188. package/src/server/index.ts +8 -1
  189. package/src/skills/typeclaw-config/SKILL.md +16 -0
  190. package/src/skills/typeclaw-memory/SKILL.md +3 -1
  191. package/src/skills/typeclaw-permissions/SKILL.md +8 -8
  192. package/src/skills/typeclaw-troubleshooting/SKILL.md +18 -1
  193. package/src/tunnels/active.ts +14 -0
  194. package/src/typevault/bash-cli.ts +18 -0
  195. package/src/typevault/cli-path.ts +11 -0
  196. package/src/typevault/envelope-helper.ts +17 -0
  197. package/src/typevault/envelope-store.ts +98 -0
  198. package/src/typevault/lifecycle.ts +132 -0
  199. package/src/typevault/prompt-tunnel.ts +33 -0
  200. package/src/typevault/redaction.ts +37 -0
  201. package/src/typevault/vault.ts +283 -0
  202. package/typeclaw.schema.json +40 -10
@@ -6,7 +6,7 @@ import { defineTool } from '@/plugin'
6
6
 
7
7
  import { referenceFilePath, referencesDir } from '../paths'
8
8
  import { headingToSlug } from '../slug'
9
- import { renderReference } from './frontmatter'
9
+ import { renderReference, type ReferenceOrigin } from './frontmatter'
10
10
  import { listReferenceSlugs } from './load-references'
11
11
 
12
12
  export type ReferenceStoredHook = (context: { slug: string; body: string }) => Promise<void>
@@ -27,37 +27,61 @@ export function createStoreReferenceTool(onReferenceStored?: ReferenceStoredHook
27
27
  // path; none is a caller-supplied file path. Declared so a title like
28
28
  // "README.md" or a body carrying "/" is not misread as a file operand.
29
29
  fileOperands: { nonFile: ['title', 'body', 'tags'] },
30
- async execute({ title, body, origin, tags }, ctx) {
31
- const existingSlugs = new Set(await listReferenceSlugs(ctx.agentDir))
32
- const slug = headingToSlug(title, existingSlugs)
33
- const created = new Date().toISOString()
34
- const path = referenceFilePath(ctx.agentDir, slug)
35
-
36
- await mkdir(referencesDir(ctx.agentDir), { recursive: true })
37
- await writeFile(
38
- path,
39
- renderReference(
40
- {
41
- title,
42
- origin,
43
- created,
44
- lastAccessed: created,
45
- accessCount: 0,
46
- pinned: false,
47
- demoted: false,
48
- tags: tags ?? [],
49
- },
50
- body,
51
- ),
52
- 'utf8',
53
- )
54
-
55
- if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
56
-
57
- return {
58
- content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
59
- details: { path, slug },
60
- }
30
+ async execute(input, ctx) {
31
+ return storeReference(input, ctx.agentDir, onReferenceStored)
61
32
  },
62
33
  })
63
34
  }
35
+
36
+ export async function storeReference(
37
+ input: {
38
+ readonly title: string
39
+ readonly body: string
40
+ readonly origin: ReferenceOrigin
41
+ readonly tags?: string[] | undefined
42
+ },
43
+ agentDir: string,
44
+ onReferenceStored?: ReferenceStoredHook,
45
+ ) {
46
+ const { title, body, origin, tags } = input
47
+ const existingSlugs = new Set(await listReferenceSlugs(agentDir))
48
+ const created = new Date().toISOString()
49
+ const content = renderReference(
50
+ {
51
+ title,
52
+ origin,
53
+ created,
54
+ lastAccessed: created,
55
+ accessCount: 0,
56
+ pinned: false,
57
+ demoted: false,
58
+ tags: tags ?? [],
59
+ },
60
+ body,
61
+ )
62
+
63
+ await mkdir(referencesDir(agentDir), { recursive: true })
64
+ // Concurrent writers (memory-logger and every main-agent session) can pick
65
+ // the same slug from the same listing, so the file is reserved with an
66
+ // exclusive create and a taken slug moves to the next deduplicated suffix.
67
+ let slug = headingToSlug(title, existingSlugs)
68
+ let path = referenceFilePath(agentDir, slug)
69
+ for (;;) {
70
+ try {
71
+ await writeFile(path, content, { encoding: 'utf8', flag: 'wx' })
72
+ break
73
+ } catch (err) {
74
+ if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST')) throw err
75
+ existingSlugs.add(slug)
76
+ slug = headingToSlug(title, existingSlugs)
77
+ path = referenceFilePath(agentDir, slug)
78
+ }
79
+ }
80
+
81
+ if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
82
+
83
+ return {
84
+ content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
85
+ details: { path, slug },
86
+ }
87
+ }
@@ -0,0 +1,81 @@
1
+ import { z } from 'zod'
2
+
3
+ import { defineTool } from '@/plugin'
4
+ import { formatLocalDate } from '@/shared'
5
+
6
+ import { appendFragment, type FragmentsAppendedHook } from './append-fragment'
7
+ import { provenanceFromOrigin } from './append-tool'
8
+ import { streamFilePath } from './paths'
9
+ import { listReferenceSlugs } from './references/load-references'
10
+ import { storeReference, type ReferenceStoredHook } from './references/store-reference-tool'
11
+
12
+ export function createMemoryRememberTool(
13
+ options: {
14
+ readonly onFragmentsAppended?: FragmentsAppendedHook
15
+ readonly onReferenceStored?: ReferenceStoredHook
16
+ } = {},
17
+ ) {
18
+ return defineTool({
19
+ description:
20
+ 'Remember a durable fact, decision, preference, or correction now. Use when asked to remember something or when durable knowledge is established, not for routine chatter. Supply a short topic and self-contained body in the evidence language; who optionally names the person it is attributable to. Set verbatim to also retain the exact body as a linked reference. The runtime stamps provenance and writes one daily-stream fragment; dreaming consolidates it later.',
21
+ parameters: z.object({
22
+ topic: z.string().min(1),
23
+ body: z.string().min(1),
24
+ who: z.string().min(1).optional(),
25
+ verbatim: z.boolean().optional(),
26
+ }),
27
+ // All arguments are content/control values, never caller-selected paths.
28
+ fileOperands: { nonFile: ['topic', 'body', 'who', 'verbatim'] },
29
+ async execute({ topic, body, who, verbatim }, ctx) {
30
+ const path = streamFilePath(ctx.agentDir, formatLocalDate())
31
+ const { fragment } = await appendFragment(
32
+ {
33
+ topic,
34
+ body,
35
+ who,
36
+ source: ctx.sessionId,
37
+ // An empty anchor is honest for direct calls without a transcript.
38
+ entry: ctx.entryId ?? '',
39
+ kind: 'explicit',
40
+ },
41
+ ctx,
42
+ {
43
+ path,
44
+ where: provenanceFromOrigin(ctx.origin),
45
+ onFragmentsAppended: options.onFragmentsAppended,
46
+ referenceSlugResolver: listReferenceSlugs,
47
+ ...(verbatim
48
+ ? {
49
+ storeReference: async () => {
50
+ // Indexing is best-effort once the reference is persisted, like
51
+ // the fragment hook: a failure must not orphan the stored file
52
+ // or drop the explicit fragment that cites it.
53
+ const { onReferenceStored } = options
54
+ const result = await storeReference(
55
+ { title: topic, body, origin: 'episode' },
56
+ ctx.agentDir,
57
+ onReferenceStored === undefined
58
+ ? undefined
59
+ : async (reference) => {
60
+ try {
61
+ await onReferenceStored(reference)
62
+ } catch (err) {
63
+ ctx.logger.warn(
64
+ `[memory] post-store reference index hook failed: ${err instanceof Error ? err.message : String(err)}`,
65
+ )
66
+ }
67
+ },
68
+ )
69
+ return result.details.slug
70
+ },
71
+ }
72
+ : {}),
73
+ },
74
+ )
75
+ return {
76
+ content: [{ type: 'text' as const, text: `Remembered "${topic}".` }],
77
+ details: { path, fragmentId: fragment.id, references: fragment.references ?? [] },
78
+ }
79
+ },
80
+ })
81
+ }
@@ -0,0 +1,91 @@
1
+ import { createHash } from 'node:crypto'
2
+
3
+ import { loadShard } from './load-shards'
4
+ import { loadReference } from './references/load-references'
5
+ import type { MemorySearchMatch, StreamMatch } from './search-tool'
6
+ import { readAllStreamDays } from './stream-io'
7
+ import type { HybridSearchResult } from './vector/hybrid'
8
+
9
+ // Hybrid results carry excerpts and provenance, while the tool also exposes
10
+ // canonical paths, citation IDs and optional complete bodies.
11
+ export async function hydrateHybridMatches(
12
+ agentDir: string,
13
+ results: HybridSearchResult[],
14
+ full: boolean,
15
+ ): Promise<MemorySearchMatch[]> {
16
+ const streams = new Map<string, StreamMatch>()
17
+ if (results.some((hit) => hit.source === 'stream')) {
18
+ for (const day of await readAllStreamDays(agentDir)) {
19
+ for (const event of day.events) {
20
+ const base = { source: 'stream', streamPath: day.path, date: day.date, excerpt: '' } as const
21
+ switch (event.type) {
22
+ case 'fragment':
23
+ streams.set(`${day.date}#${event.id}`, {
24
+ ...base,
25
+ topic: event.topic,
26
+ eventId: `streams/${day.date}#${event.id}`,
27
+ ...(full ? { fullBody: event.body } : {}),
28
+ })
29
+ break
30
+ case 'legacy_prose':
31
+ streams.set(`${day.date}#legacy-${createHash('sha256').update(event.text).digest('hex').slice(0, 12)}`, {
32
+ ...base,
33
+ topic: '[legacy prose from pre-shard migration]',
34
+ ...(full ? { fullBody: event.text } : {}),
35
+ })
36
+ break
37
+ case 'watermark':
38
+ break
39
+ default:
40
+ event satisfies never
41
+ }
42
+ }
43
+ }
44
+ }
45
+
46
+ const matches = await Promise.all(
47
+ results.map(async (hit): Promise<MemorySearchMatch | null> => {
48
+ switch (hit.source) {
49
+ case 'topic': {
50
+ const shard = await loadShard(agentDir, hit.key)
51
+ if (shard === null) return null
52
+ return {
53
+ source: 'topic',
54
+ shardPath: shard.path,
55
+ slug: hit.key,
56
+ heading: hit.heading,
57
+ excerpt: hit.excerpt,
58
+ ...(hit.provenance === undefined ? {} : { provenance: hit.provenance }),
59
+ ...(full ? { fullBody: shard.body } : {}),
60
+ }
61
+ }
62
+ case 'reference': {
63
+ const reference = await loadReference(agentDir, hit.key)
64
+ if (reference === null) return null
65
+ return {
66
+ source: 'reference',
67
+ slug: hit.key,
68
+ title: hit.heading,
69
+ excerpt: hit.excerpt,
70
+ created: reference.frontmatter.created,
71
+ ...(full ? { fullBody: reference.body } : {}),
72
+ }
73
+ }
74
+ case 'stream': {
75
+ const stream = streams.get(hit.key)
76
+ if (stream === undefined) return null
77
+ return {
78
+ ...stream,
79
+ excerpt: hit.excerpt,
80
+ ...(hit.who === undefined ? {} : { who: hit.who }),
81
+ ...(hit.when === undefined ? {} : { when: hit.when }),
82
+ ...(hit.where === undefined ? {} : { where: hit.where }),
83
+ }
84
+ }
85
+ default:
86
+ return hit.source satisfies never
87
+ }
88
+ }),
89
+ )
90
+ return matches.filter((match) => match !== null)
91
+ }
@@ -1,3 +1,4 @@
1
+ import { existsSync } from 'node:fs'
1
2
  import { writeFile } from 'node:fs/promises'
2
3
 
3
4
  import { z } from 'zod'
@@ -13,9 +14,13 @@ import {
13
14
  } from './provenance-index'
14
15
  import { sanitizeProvenanceName } from './provenance-sanitize'
15
16
  import { renderReference } from './references/frontmatter'
16
- import { loadAllReferences, loadReference, type Reference } from './references/load-references'
17
+ import { bumpReferenceAccess, loadAllReferences, loadReference, type Reference } from './references/load-references'
18
+ import { vectorDbPath } from './search-command'
19
+ import { hydrateHybridMatches } from './search-tool-hybrid'
17
20
  import type { FragmentEvent, FragmentProvenance, LegacyProseEvent, StreamEvent } from './stream-events'
18
21
  import { filterUndreamedEvents, readAllStreamDays, type UndreamedStreamDay } from './stream-io'
22
+ import { hybridSearch, type EmbedFn, type HybridSearchResult } from './vector/hybrid'
23
+ import { VectorStore } from './vector/store'
19
24
 
20
25
  const DEFAULT_MAX_RESULTS = 10
21
26
  const EXCERPT_CONTEXT_LINES = 3
@@ -59,12 +64,13 @@ export type MemorySearchResult = { matches: MemorySearchMatch[]; truncatedAt?: n
59
64
 
60
65
  export type Matcher = (haystack: string) => boolean
61
66
 
62
- export function createMemorySearchTool() {
67
+ export function createMemorySearchTool(embedFn?: EmbedFn) {
63
68
  return defineTool({
64
69
  description:
65
- 'Search long-term memory and recent undreamed fragments, or look up one exact topic/reference slug. Plain search is case-insensitive phrase-first with token-OR fallback; regex mode has no token fallback. Runtime-owned provenance from active topic citations is searchable but never embedded. `workspace`, `chat`, and `thread` scope both topics and recent fragments to matching captured origin IDs or names; legacy `where` remains an alias for `chat`. Scoped topic eligibility, lexical matching, and returned provenance use the same eligible active cited children. Unresolved citations remain explicit in unscoped results and never gain invented metadata.',
70
+ 'Search long-term memory and recent undreamed fragments by meaning and keywords (default mode: hybrid). Use mode: lexical for case-insensitive phrase-first search with token-OR fallback. Use asRegex: true for exact regex semantics without token fallback, or topic for one exact topic/reference slug; both bypass semantic search. If the semantic index is unavailable, the result reports lexical fallback. full: true returns complete bodies. Runtime-owned provenance from active topic citations is searchable but never embedded. `workspace`, `chat`, and `thread` scope both topics and recent fragments to matching captured origin IDs or names; legacy `where` remains an alias for `chat`. Scoped topic eligibility, lexical matching, and returned provenance use the same eligible active cited children. Unresolved citations remain explicit in unscoped results and never gain invented metadata.',
66
71
  parameters: z.object({
67
72
  query: z.string().optional(),
73
+ mode: z.enum(['hybrid', 'lexical']).default('hybrid'),
68
74
  topic: z.string().optional(),
69
75
  asRegex: z.boolean().default(false),
70
76
  full: z.boolean().default(false),
@@ -82,9 +88,12 @@ export function createMemorySearchTool() {
82
88
  // with an agent-root dir, or a `since` date like "2026-01-01", is not
83
89
  // misread as a file operand.
84
90
  fileOperands: {
85
- nonFile: ['query', 'topic', 'since', 'before', 'where', 'workspace', 'chat', 'thread'],
91
+ nonFile: ['query', 'mode', 'topic', 'since', 'before', 'where', 'workspace', 'chat', 'thread'],
86
92
  },
87
- async execute({ query, topic, asRegex, full, maxResults, since, before, where, workspace, chat, thread }, ctx) {
93
+ async execute(
94
+ { query, mode, topic, asRegex, full, maxResults, since, before, where, workspace, chat, thread },
95
+ ctx,
96
+ ) {
88
97
  if ((query === undefined) === (topic === undefined)) {
89
98
  return resultToToolResult({ error: 'provide exactly one of `query` or `topic`' })
90
99
  }
@@ -105,6 +114,37 @@ export function createMemorySearchTool() {
105
114
  if (typeof matcherOrError === 'string') {
106
115
  return resultToToolResult({ error: matcherOrError })
107
116
  }
117
+ const dateFilter = parseReferenceDateFilter(since, before)
118
+ if ('error' in dateFilter) return resultToToolResult(dateFilter)
119
+ let fallbackReason: string | undefined
120
+ if (query !== undefined && mode === 'hybrid' && !asRegex) {
121
+ const dbPath = vectorDbPath(ctx.agentDir)
122
+ if (!existsSync(dbPath)) {
123
+ fallbackReason = 'memory/.vectors/index.db is absent'
124
+ } else {
125
+ try {
126
+ const store = VectorStore.open(dbPath)
127
+ let hits: HybridSearchResult[]
128
+ try {
129
+ hits = await hybridSearch(query, store, ctx.agentDir, maxResults, embedFn, { scope })
130
+ } finally {
131
+ store.close()
132
+ }
133
+ const matches = (await hydrateHybridMatches(ctx.agentDir, hits, full)).filter(
134
+ (match) => match.source !== 'reference' || referenceDateAllowed(match.created, dateFilter),
135
+ )
136
+ await bumpReferenceAccess(
137
+ ctx.agentDir,
138
+ matches.flatMap((match) => (match.source === 'reference' ? [match.slug] : [])),
139
+ { logger: ctx.logger },
140
+ )
141
+ return resultToToolResult({ matches })
142
+ } catch (err) {
143
+ fallbackReason = err instanceof Error ? err.message : String(err)
144
+ }
145
+ }
146
+ ctx.logger.warn(`[memory-search] semantic index unavailable: ${fallbackReason}`)
147
+ }
108
148
  const [shards, allStreamDays, allReferences] = await Promise.all([
109
149
  loadAllShards(ctx.agentDir, { logger: ctx.logger }),
110
150
  readAllStreamDays(ctx.agentDir),
@@ -126,8 +166,6 @@ export function createMemorySearchTool() {
126
166
  }
127
167
  return events.length === 0 ? [] : [{ date: day.date, path: day.path, name: day.name, events }]
128
168
  })
129
- const dateFilter = parseReferenceDateFilter(since, before)
130
- if ('error' in dateFilter) return resultToToolResult(dateFilter)
131
169
 
132
170
  const roomFilteredDays = streamDays
133
171
  const scopedShards = shards.filter((shard) => provenanceIndex.topicEligible(shard.slug, scope))
@@ -154,7 +192,7 @@ export function createMemorySearchTool() {
154
192
  }
155
193
 
156
194
  if (scopedShards.length === 0 && roomFilteredDays.length === 0 && references.length === 0) {
157
- return resultToToolResult({ matches: [], truncatedAt: 0 })
195
+ return resultToToolResult({ matches: [], truncatedAt: 0 }, fallbackReason)
158
196
  }
159
197
 
160
198
  const searchOptions = { full, maxResults, references, topicDocuments, topicProvenance, streamDocuments }
@@ -164,7 +202,7 @@ export function createMemorySearchTool() {
164
202
  if (fallback !== null) result = fallback
165
203
  }
166
204
  if ('matches' in result) await bumpReturnedReferences(allReferences, result.matches)
167
- return resultToToolResult(result)
205
+ return resultToToolResult(result, fallbackReason)
168
206
  },
169
207
  })
170
208
  }
@@ -661,7 +699,11 @@ function parseDateParam(name: string, value: string): Date | string {
661
699
 
662
700
  function referenceCandidateAllowed(reference: Reference, filter: ReferenceDateFilter): boolean {
663
701
  if (reference.frontmatter.demoted) return false
664
- const created = new Date(reference.frontmatter.created)
702
+ return referenceDateAllowed(reference.frontmatter.created, filter)
703
+ }
704
+
705
+ function referenceDateAllowed(createdAt: string, filter: ReferenceDateFilter): boolean {
706
+ const created = new Date(createdAt)
665
707
  if (filter.since !== undefined && created < filter.since) return false
666
708
  if (filter.before !== undefined && created >= filter.before) return false
667
709
  return true
@@ -702,9 +744,13 @@ const EMPTY_RESULT_GUIDANCE =
702
744
  // The empty-set note rides in the LLM-facing `text` ONLY. `details` stays the
703
745
  // pure struct: `keywordLane` reads `searchAll` directly (never this layer) and
704
746
  // the structured tests assert on `details`, so both must see no `note`.
705
- function resultToToolResult(result: MemorySearchResult) {
747
+ function resultToToolResult(result: MemorySearchResult, fallbackReason?: string) {
706
748
  const isEmpty = 'matches' in result && result.matches.length === 0
707
- const text = isEmpty ? JSON.stringify({ ...result, note: EMPTY_RESULT_GUIDANCE }) : JSON.stringify(result)
749
+ const text = JSON.stringify({
750
+ ...result,
751
+ ...(isEmpty ? { note: EMPTY_RESULT_GUIDANCE } : {}),
752
+ ...(fallbackReason === undefined ? {} : { notice: 'Semantic index unavailable; lexical results are shown.' }),
753
+ })
708
754
  return {
709
755
  content: [{ type: 'text' as const, text }],
710
756
  details: result,
@@ -56,6 +56,8 @@ export const fragmentEventSchema = z
56
56
  entry: z.string(),
57
57
  topic: z.string(),
58
58
  body: z.string(),
59
+ // Deliberate main-agent capture; absent on legacy/logger fragments.
60
+ kind: z.literal('explicit').optional(),
59
61
  references: z.array(z.string()).optional(),
60
62
  // WHO the evidence is attributable to (display name/handle of the speaker).
61
63
  // LLM-supplied by the memory-logger, set ONLY when one transcript speaker
@@ -0,0 +1,30 @@
1
+ import { createHash } from 'node:crypto'
2
+
3
+ import { z } from 'zod'
4
+
5
+ const PREFIX = 'memory-page:'
6
+ const cursorSchema = z.object({
7
+ version: z.literal(1),
8
+ file: z.string(),
9
+ entry: z.string().min(1),
10
+ digest: z.string(),
11
+ offset: z.number().int().nonnegative(),
12
+ })
13
+
14
+ export function transcriptDigest(value: string): string {
15
+ return createHash('sha256').update(value).digest('hex')
16
+ }
17
+
18
+ export function encodeTranscriptCursor(path: string, entry: string, text: string, offset: number): string {
19
+ const cursor = { version: 1, file: transcriptDigest(path), entry, digest: transcriptDigest(text), offset }
20
+ return `${PREFIX}${Buffer.from(JSON.stringify(cursor)).toString('base64url')}`
21
+ }
22
+
23
+ export function decodeTranscriptCursor(value: string | null): z.infer<typeof cursorSchema> | null {
24
+ if (!value?.startsWith(PREFIX)) return null
25
+ return cursorSchema.parse(JSON.parse(Buffer.from(value.slice(PREFIX.length), 'base64url').toString('utf8')))
26
+ }
27
+
28
+ export function transcriptEntryId(value: string): string {
29
+ return decodeTranscriptCursor(value)?.entry ?? value
30
+ }
@@ -1,15 +1,13 @@
1
1
  import { readFile } from 'node:fs/promises'
2
2
 
3
- // The runtime hands memory-logger the transcript slice past the watermark in
4
- // its first prompt, so a typical run is one model call instead of a
5
- // find_entry/read/read/read chain. A JSONL line over `maxLineBytes` is cut
6
- // rather than skipped: `read` refuses lines over its 50 KB limit outright,
7
- // which made the subagent silently skip large entries (long cron prompts,
8
- // big tool results) because it has no shell fallback.
3
+ import { decodeTranscriptCursor, encodeTranscriptCursor, transcriptDigest } from './transcript-cursor'
4
+ import { projectTranscriptLine } from './transcript-view'
5
+
6
+ // Pack complete projected records where possible. Large records are paged
7
+ // reversibly; the checkpoint includes the next character offset and digest,
8
+ // so an interrupted or changed entry is replayed rather than skipped.
9
9
  export const PRELOAD_MAX_BYTES = 256 * 1024
10
- export const PRELOAD_MAX_LINE_BYTES = 16 * 1024
11
10
  const MIN_USEFUL_PRELOAD_BYTES = 8 * 1024
12
- const TRUNCATION_MARKER_BYTES = 128
13
11
  // Byte-level BPE tokenizers emit at most one token per UTF-8 byte, so byte
14
12
  // length is a tokenizer-independent upper bound on token count. The overhead
15
13
  // covers tool schemas and the preload header; each model's output allowance is
@@ -41,6 +39,9 @@ export type TranscriptPreload = {
41
39
  totalLines: number
42
40
  truncatedLines: number
43
41
  watermarkFound: boolean
42
+ checkpoint: string | null
43
+ startCheckpoint: string | null
44
+ entryIds: string[]
44
45
  // First unread line when the byte cap stopped the preload early; null when
45
46
  // the preload reaches the end of the transcript.
46
47
  nextOffset: number | null
@@ -49,10 +50,9 @@ export type TranscriptPreload = {
49
50
  export async function buildTranscriptPreload(
50
51
  path: string,
51
52
  watermark: string | null,
52
- options: { maxBytes?: number; maxLineBytes?: number } = {},
53
+ options: { maxBytes?: number; untilLine?: number } = {},
53
54
  ): Promise<TranscriptPreload | null> {
54
55
  const maxBytes = options.maxBytes ?? PRELOAD_MAX_BYTES
55
- const maxLineBytes = Math.min(options.maxLineBytes ?? PRELOAD_MAX_LINE_BYTES, maxBytes - TRUNCATION_MARKER_BYTES)
56
56
  let raw: string
57
57
  try {
58
58
  raw = await readFile(path, 'utf8')
@@ -61,43 +61,148 @@ export async function buildTranscriptPreload(
61
61
  }
62
62
  const lines = raw.length === 0 ? [] : raw.split('\n')
63
63
  if (lines.length > 0 && lines[lines.length - 1] === '') lines.pop()
64
- const totalLines = lines.length
64
+ const totalLines = Math.min(lines.length, options.untilLine ?? lines.length)
65
65
 
66
66
  let startIndex = 0
67
+ let startOffset = 0
67
68
  let watermarkFound = false
69
+ const cursor = decodeTranscriptCursor(watermark)
68
70
  if (watermark !== null) {
69
- const needle = `"id":"${watermark}"`
70
- const found = lines.findIndex((line) => line.includes(needle))
71
+ const entryId = cursor?.entry ?? watermark
72
+ const found = lines.findIndex((line) => recordId(line) === entryId)
71
73
  if (found !== -1) {
72
- startIndex = found + 1
74
+ startIndex = cursor === null ? found + 1 : found
73
75
  watermarkFound = true
76
+ if (cursor !== null) {
77
+ const text = projectTranscriptLine(lines[found] ?? '')
78
+ if (
79
+ cursor.file === transcriptDigest(path) &&
80
+ cursor.digest === transcriptDigest(text) &&
81
+ cursor.offset <= text.length &&
82
+ !isInsideSurrogatePair(text, cursor.offset)
83
+ ) {
84
+ startOffset = cursor.offset
85
+ }
86
+ }
74
87
  }
75
88
  }
76
89
 
77
90
  const out: string[] = []
91
+ const entryIds: string[] = []
78
92
  let bytes = 0
79
- let truncatedLines = 0
93
+ let checkpoint = watermark
94
+ const firstId = recordId(lines[startIndex] ?? '')
95
+ const startCheckpoint =
96
+ firstId === null
97
+ ? null
98
+ : encodeTranscriptCursor(path, firstId, projectTranscriptLine(lines[startIndex] ?? ''), startOffset)
99
+ let partial = false
80
100
  let index = startIndex
81
101
  for (; index < totalLines; index++) {
82
- let line = lines[index] ?? ''
83
- const lineBytes = Buffer.byteLength(line, 'utf8')
84
- if (lineBytes > maxLineBytes) {
85
- line = `${Buffer.from(line, 'utf8').subarray(0, maxLineBytes).toString('utf8')} …[line ${index + 1} truncated: ${Math.round(lineBytes / 1024)}KB total, first ${Math.round(maxLineBytes / 1024)}KB shown]`
86
- truncatedLines++
102
+ const source = lines[index] ?? ''
103
+ const id = recordId(source)
104
+ // A live writer's incomplete tail is never acknowledged.
105
+ if (id === null) break
106
+ const text = projectTranscriptLine(source)
107
+ const offset = index === startIndex ? startOffset : 0
108
+ const separatorBytes = out.length === 0 ? 0 : 1
109
+ if (offset === 0 && bytes + separatorBytes + Buffer.byteLength(text, 'utf8') <= maxBytes) {
110
+ out.push(text)
111
+ entryIds.push(id)
112
+ bytes += separatorBytes + Buffer.byteLength(text, 'utf8')
113
+ checkpoint = id
114
+ continue
115
+ }
116
+ if (out.length > 0) break
117
+
118
+ const context = entryContext(text)
119
+ // Read-only overlap lets a fresh session interpret facts split at the
120
+ // boundary. Reserve most of even a small page for new progress.
121
+ let overlapStart = Math.max(0, offset - Math.min(2048, Math.floor(maxBytes / 16)))
122
+ if (isInsideSurrogatePair(text, overlapStart)) overlapStart++
123
+ const precedingText = offset > 0 ? text.slice(overlapStart, offset) : undefined
124
+ let low = offset
125
+ let high = text.length
126
+ while (low < high) {
127
+ const middle = Math.ceil((low + high) / 2)
128
+ const part = JSON.stringify({
129
+ type: 'entry-part',
130
+ id,
131
+ context,
132
+ offset,
133
+ precedingText,
134
+ text: text.slice(offset, middle),
135
+ })
136
+ if (Buffer.byteLength(part, 'utf8') <= maxBytes) low = middle
137
+ else high = middle - 1
87
138
  }
88
- const cost = Buffer.byteLength(line, 'utf8') + 1
89
- if (out.length > 0 && bytes + cost > maxBytes) break
90
- out.push(line)
91
- bytes += cost
139
+ const end = isInsideSurrogatePair(text, low) ? low - 1 : low
140
+ if (end <= offset) throw new Error('Memory transcript page budget cannot hold the next character.')
141
+ out.push(JSON.stringify({ type: 'entry-part', id, context, offset, precedingText, text: text.slice(offset, end) }))
142
+ entryIds.push(id)
143
+ checkpoint = end === text.length ? id : encodeTranscriptCursor(path, id, text, end)
144
+ partial = end < text.length
145
+ if (end === text.length) index++
146
+ break
92
147
  }
93
148
 
94
149
  return {
95
150
  text: out.join('\n'),
96
151
  startLine: startIndex + 1,
97
- endLine: index,
152
+ endLine: index + (partial ? 1 : 0),
98
153
  totalLines,
99
- truncatedLines,
154
+ truncatedLines: 0,
100
155
  watermarkFound,
156
+ checkpoint,
157
+ startCheckpoint,
158
+ entryIds,
101
159
  nextOffset: index < totalLines ? index + 1 : null,
102
160
  }
103
161
  }
162
+
163
+ function entryContext(text: string): Record<string, unknown> {
164
+ const entry: unknown = JSON.parse(text)
165
+ if (!isRecord(entry)) return {}
166
+ const context: Record<string, unknown> = {}
167
+ for (const key of ['parentId', 'timestamp']) {
168
+ if (key in entry) context[key] = entry[key]
169
+ }
170
+ if (isRecord(entry.message)) {
171
+ for (const key of [
172
+ 'role',
173
+ 'timestamp',
174
+ 'toolCallId',
175
+ 'toolName',
176
+ 'isError',
177
+ 'stopReason',
178
+ 'authorId',
179
+ 'authorName',
180
+ 'who',
181
+ 'where',
182
+ ]) {
183
+ if (key in entry.message) context[`message.${key}`] = entry.message[key]
184
+ }
185
+ }
186
+ return context
187
+ }
188
+
189
+ function isRecord(value: unknown): value is Record<string, unknown> {
190
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
191
+ }
192
+
193
+ function recordId(line: string): string | null {
194
+ try {
195
+ const entry: unknown = JSON.parse(line)
196
+ if (typeof entry !== 'object' || entry === null || !('id' in entry)) return null
197
+ return typeof entry.id === 'string' ? entry.id : null
198
+ } catch (error) {
199
+ if (error instanceof SyntaxError) return null
200
+ throw error
201
+ }
202
+ }
203
+
204
+ function isInsideSurrogatePair(text: string, offset: number): boolean {
205
+ const before = text.charCodeAt(offset - 1)
206
+ const after = text.charCodeAt(offset)
207
+ return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff
208
+ }