typeclaw 0.40.0 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +3 -2
  2. package/auth.schema.json +57 -0
  3. package/package.json +2 -2
  4. package/secrets.schema.json +57 -0
  5. package/src/agent/index.ts +10 -6
  6. package/src/agent/multimodal/look-at.ts +87 -0
  7. package/src/agent/session-origin.ts +1 -0
  8. package/src/agent/tools/channel-history-render.ts +21 -0
  9. package/src/agent/tools/channel-history.ts +2 -17
  10. package/src/agent/tools/channel-read.ts +206 -0
  11. package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +26 -2
  12. package/src/bundled-plugins/github-cli-auth/index.ts +25 -6
  13. package/src/bundled-plugins/memory/index.ts +10 -11
  14. package/src/bundled-plugins/memory/vector/hybrid.ts +73 -6
  15. package/src/bundled-plugins/memory/vector/index-on-write.ts +25 -20
  16. package/src/bundled-plugins/memory/vector/passages.ts +2 -24
  17. package/src/bundled-plugins/memory/vector/reference-index-on-write.ts +28 -23
  18. package/src/bundled-plugins/memory/vector/store.ts +2 -0
  19. package/src/bundled-plugins/memory/vector/truncation.ts +28 -0
  20. package/src/bundled-plugins/planner/planner.ts +4 -2
  21. package/src/bundled-plugins/reviewer/reviewer.ts +9 -1
  22. package/src/bundled-plugins/tool-result-cap/README.md +10 -1
  23. package/src/bundled-plugins/tool-result-cap/cap-result.ts +8 -0
  24. package/src/channels/adapters/discord-bot.ts +112 -0
  25. package/src/channels/adapters/instagram-channel-resolver.ts +113 -0
  26. package/src/channels/adapters/instagram-classify.ts +69 -0
  27. package/src/channels/adapters/instagram-format.ts +8 -0
  28. package/src/channels/adapters/instagram.ts +346 -0
  29. package/src/channels/adapters/kakaotalk-classify.ts +31 -6
  30. package/src/channels/adapters/kakaotalk.ts +1 -1
  31. package/src/channels/adapters/slack-bot.ts +147 -0
  32. package/src/channels/manager.ts +60 -2
  33. package/src/channels/router.ts +64 -0
  34. package/src/channels/schema.ts +5 -0
  35. package/src/channels/types.ts +46 -0
  36. package/src/cli/channel.ts +76 -1
  37. package/src/cli/init.ts +1 -0
  38. package/src/cli/status.ts +37 -14
  39. package/src/config/channels-mutation.ts +2 -1
  40. package/src/doctor/channel-checks.ts +28 -0
  41. package/src/hostd/daemon.ts +59 -19
  42. package/src/hostd/kakao-renewal-manager.ts +35 -10
  43. package/src/hostd/protocol.ts +2 -0
  44. package/src/hostd/tailscale.ts +41 -5
  45. package/src/hostd/webex-renewal-manager.ts +13 -4
  46. package/src/init/index.ts +50 -1
  47. package/src/init/instagram-auth.ts +134 -0
  48. package/src/init/run-owner-claim.ts +1 -0
  49. package/src/permissions/match-rule.ts +1 -1
  50. package/src/permissions/resolve.ts +1 -0
  51. package/src/plugin/index.ts +1 -0
  52. package/src/plugin/manager.ts +14 -0
  53. package/src/plugin/registry.ts +12 -0
  54. package/src/plugin/types.ts +4 -0
  55. package/src/role-claim/match-rule.ts +2 -1
  56. package/src/run/index.ts +46 -20
  57. package/src/secrets/instagram-store.ts +112 -0
  58. package/src/secrets/schema.ts +18 -0
  59. package/src/skills/typeclaw-channel-instagram/SKILL.md +30 -0
  60. package/src/test-helpers/rm-temp-dir.ts +24 -0
  61. package/typeclaw.schema.json +161 -1
@@ -109,6 +109,18 @@ export default definePlugin({
109
109
  }): Promise<HookResult | 'fall-through'> => {
110
110
  const { event, command } = params
111
111
  const review = await noteReviewCommand({ callId: event.callId, command })
112
+ // guard() holds the lease expecting tool.after to release it, but a
113
+ // tool.before block (from ANY guard below) means the command never runs, so
114
+ // tool.after never fires and the lease strands for its full TTL — telling
115
+ // every other session "the in-flight one will post" when it never will (the
116
+ // lost-verdict strand: a review trips the shape guard, the lease strands, the
117
+ // PR ends unreviewed). blockAfterLease() releases on such a block. guard()'s
118
+ // OWN terminal block self-releases and returns directly, bypassing this path.
119
+ let leaseClaimed = false
120
+ const blockAfterLease = async (block: HookResult & { block: true }): Promise<HookResult> => {
121
+ if (leaseClaimed) await verdictGuard.release({ callId: event.callId, succeeded: false })
122
+ return block
123
+ }
112
124
  if (review.detected !== null) {
113
125
  const block = await verdictGuard.guard({
114
126
  callId: event.callId,
@@ -117,8 +129,9 @@ export default definePlugin({
117
129
  verdict: review.detected.verdict,
118
130
  })
119
131
  if (block !== null) return block
132
+ leaseClaimed = true
120
133
  }
121
- if (review.dump !== null) return review.dump
134
+ if (review.dump !== null) return blockAfterLease(review.dump)
122
135
 
123
136
  // Analyze first WITHOUT the fallback: an explicit `-R`/path repo must win,
124
137
  // and we only pay for fallback resolution (a git subprocess) when the
@@ -152,7 +165,7 @@ export default definePlugin({
152
165
  GITHUB_TOKEN: process.env.GITHUB_TOKEN,
153
166
  })
154
167
  if (userEndpointTokens.some((token) => shouldMintAppToken(token, hasAppTokenResolver()))) {
155
- return { block: true, reason: appUserEndpointReason }
168
+ return blockAfterLease({ block: true, reason: appUserEndpointReason })
156
169
  }
157
170
 
158
171
  if (decision.kind === 'pass-through') return 'fall-through'
@@ -200,16 +213,22 @@ export default definePlugin({
200
213
  // bug returns); otherwise block with guidance rather than failing mute.
201
214
  if (!shouldMintAppToken(undefined, hasAppTokenResolver())) {
202
215
  if (decision.kind === 'block') {
203
- return { block: true, reason: decision.reason + buildGhBlockGuidance(decision.code, fallbackRepo) }
216
+ return blockAfterLease({
217
+ block: true,
218
+ reason: decision.reason + buildGhBlockGuidance(decision.code, fallbackRepo),
219
+ })
204
220
  }
205
221
  warnSandboxedPatWithheldOnce()
206
- return { block: true, reason: sandboxedPatWithheldReason }
222
+ return blockAfterLease({ block: true, reason: sandboxedPatWithheldReason })
207
223
  }
208
224
  mintForSandboxedPat = true
209
225
  }
210
226
 
211
227
  if (decision.kind === 'block') {
212
- return { block: true, reason: decision.reason + buildGhBlockGuidance(decision.code, fallbackRepo) }
228
+ return blockAfterLease({
229
+ block: true,
230
+ reason: decision.reason + buildGhBlockGuidance(decision.code, fallbackRepo),
231
+ })
213
232
  }
214
233
 
215
234
  // No App auth (no App-class GH_TOKEN and no live minter): leave whatever
@@ -218,7 +237,7 @@ export default definePlugin({
218
237
  if (!mintForSandboxedPat && !shouldMintAppToken(process.env.GH_TOKEN, hasAppTokenResolver())) return
219
238
 
220
239
  const result = await resolveTokenForRepo(decision.repoSlug)
221
- if (result.kind === 'unavailable') return { block: true, reason: result.reason }
240
+ if (result.kind === 'unavailable') return blockAfterLease({ block: true, reason: result.reason })
222
241
  // Inject via the internal env overlay (delivered to the spawn / bwrap
223
242
  // --setenv by the bash wrapper) so the token never enters the command
224
243
  // string, where it could leak through logs or later hooks. When the repo
@@ -28,7 +28,7 @@ import { embed } from './vector/embedder'
28
28
  import { hybridSearch, type EmbedFn } from './vector/hybrid'
29
29
  import { makeAppendHook } from './vector/index-on-write'
30
30
  import { makeReferenceStoredHook } from './vector/reference-index-on-write'
31
- import { VectorStore } from './vector/store'
31
+ import { type OpenVectorStore, VectorStore } from './vector/store'
32
32
 
33
33
  const DEFAULT_IDLE_MS = 60_000
34
34
  const DEFAULT_BUFFER_BYTES = 500_000
@@ -144,13 +144,13 @@ const VECTOR_TURN_TOP_K = 10
144
144
  export type MemoryPluginDeps = {
145
145
  hybridSearch: typeof hybridSearch
146
146
  queryEmbedFn: EmbedFn
147
- openAppendVectorStore: (agentDir: string) => VectorStore
147
+ openAppendVectorStore: (agentDir: string) => OpenVectorStore
148
148
  }
149
149
 
150
150
  const defaultDeps: MemoryPluginDeps = {
151
151
  hybridSearch,
152
152
  queryEmbedFn: embed,
153
- openAppendVectorStore: (agentDir) => VectorStore.open(join(agentDir, 'memory', '.vectors', 'index.db')),
153
+ openAppendVectorStore: (agentDir) => () => VectorStore.open(join(agentDir, 'memory', '.vectors', 'index.db')),
154
154
  }
155
155
 
156
156
  // TypeClaw-owned infrastructure subagents (memory-logger, dreaming, backup) run
@@ -402,19 +402,18 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
402
402
  error: (m: string) => ctx.logger.error(m),
403
403
  }
404
404
 
405
- // Long-lived, process-lifetime SQLite handle for append-time indexing
406
- // (the on-write fragment/reference hooks below). Intentionally not closed
407
- // by the plugin: it lives as long as the agent runtime and is released on
408
- // process teardown. The per-turn query stores opened in renderVectorTurnMemory
409
- // are the ones that get closed each turn.
410
- const appendVectorStore = deps.openAppendVectorStore(ctx.agentDir)
405
+ // Opener (not an open handle) for append-time indexing. Each on-write hook
406
+ // opens its own short-lived store and closes it when the batch settles, so
407
+ // no bun:sqlite handle outlives a hook call to pin the agent dir (Bun
408
+ // #25964). Boot-only paths that never append never open the DB at all.
409
+ const openAppendVectorStore = deps.openAppendVectorStore(ctx.agentDir)
411
410
 
412
411
  return {
413
412
  subagents: {
414
413
  'memory-logger': createMemoryLoggerSubagent({
415
414
  logger: subagentLogger,
416
- onFragmentsAppended: makeAppendHook(appendVectorStore),
417
- onReferenceStored: makeReferenceStoredHook(appendVectorStore),
415
+ onFragmentsAppended: makeAppendHook(openAppendVectorStore),
416
+ onReferenceStored: makeReferenceStoredHook(openAppendVectorStore),
418
417
  }),
419
418
  dreaming: createDreamingSubagent({
420
419
  logger: subagentLogger,
@@ -19,11 +19,27 @@ import type { Passage } from './passages'
19
19
  import { clearsBaseline, gateRelevance, MARGIN, streamAdmissionBaseline } from './relevance-gate'
20
20
  import { crossScriptMarginScale, dominantScript, type ScriptClass } from './script'
21
21
  import { VectorStore, type ScoredVectorRow, type VectorRow } from './store'
22
+ import { chunkEmbeddableText } from './truncation'
22
23
 
23
24
  export { collectPassages, findMissingPassages, type Passage } from './passages'
24
25
 
25
26
  const RRF_K = 60
26
27
 
28
+ // A long prompt is embedded as multiple ≤512-token chunks (not truncated to one
29
+ // vector), so a memory relevant to ANY part of the prompt — including its tail —
30
+ // can still be retrieved. The cap bounds the per-turn cosine scans: each chunk is
31
+ // a full O(rows) brute-force pass over the store, so an unbounded chunk count
32
+ // would scale per-turn retrieval cost with prompt length. At the cap, the LAST
33
+ // chunks are kept rather than the first: TypeClaw cron/agent prompts front-load
34
+ // fixed framing prose and place the actual payload last, so dropping the head is
35
+ // the lesser loss than re-introducing the tail-truncation this fix removes.
36
+ const MAX_QUERY_CHUNKS = 10
37
+
38
+ function queryEmbeddingChunks(query: string): string[] {
39
+ const chunks = chunkEmbeddableText(query)
40
+ return chunks.length <= MAX_QUERY_CHUNKS ? chunks : chunks.slice(chunks.length - MAX_QUERY_CHUNKS)
41
+ }
42
+
27
43
  export type HybridSearchResult = {
28
44
  source: 'topic' | 'stream' | 'reference'
29
45
  key: string
@@ -46,17 +62,18 @@ export async function hybridSearch(
46
62
  ): Promise<HybridSearchResult[]> {
47
63
  if (topK <= 0) return []
48
64
 
65
+ const queryChunks = queryEmbeddingChunks(query)
49
66
  const [shards, streamDays, references, queryEmbeddings] = await Promise.all([
50
67
  loadAllShards(agentDir),
51
68
  readAllUndreamedStreamDays(agentDir),
52
69
  loadAllReferences(agentDir),
53
- embedFn([query], 'query'),
70
+ embedFn(queryChunks, 'query'),
54
71
  ])
55
72
 
56
73
  const { parentSlugsByFragmentId, supersededFragmentIds } = buildParentLinks(shards)
57
74
  const index = buildContentIndex(shards, streamDays, references, supersededFragmentIds)
58
75
  const vectorRows =
59
- queryEmbeddings[0] === undefined ? [] : gatedVectorLane(queryEmbeddings[0], store, topK, query, index)
76
+ queryEmbeddings.length === 0 ? [] : gatedVectorLane(queryEmbeddings, queryChunks, store, topK, index)
60
77
  const keywordMatches = keywordLane(query, shards, streamDays, references, topK * 2)
61
78
 
62
79
  return fuseLanes(vectorRows, keywordMatches, index, parentSlugsByFragmentId).slice(0, topK)
@@ -85,13 +102,13 @@ export async function hybridSearch(
85
102
  // separate keyword lane. An empty merged lane composes with RRF exactly like a
86
103
  // lane that found nothing, so a genuine keyword hit survives a full no-match.
87
104
  function gatedVectorLane(
88
- queryEmbedding: Float32Array,
105
+ queryEmbeddings: Float32Array[],
106
+ queryChunks: string[],
89
107
  store: VectorStore,
90
108
  topK: number,
91
- query: string,
92
109
  index: Map<string, Omit<HybridSearchResult, 'rrfScore'>>,
93
110
  ): VectorRow[] {
94
- const scored = store.queryScored(queryEmbedding, EMBEDDING_MODEL_ID)
111
+ const { scored, winnerChunkByRowId } = maxScoreAcrossChunks(queryEmbeddings, store)
95
112
  const bandDefiningRows = scored.filter(({ row }) => row.source === 'topic' || row.source === 'reference')
96
113
  const streamRows = scored.filter(({ row }) => row.source === 'stream')
97
114
 
@@ -102,7 +119,14 @@ function gatedVectorLane(
102
119
  // script winner whose contrast sits between the loosened and strict margins
103
120
  // would still be suppressed. Headings (via `index`) carry the language; slugs
104
121
  // are ASCII-kebab even for non-Latin topics.
105
- const topicMargin = MARGIN * crossScriptMarginScale(dominantScript(query), headBandScripts(bandDefiningRows, index))
122
+ //
123
+ // The query-side script comes from the CHUNK that won each head row, not the
124
+ // full prompt: a boilerplate-front prompt (most TypeClaw cron prompts) has a
125
+ // dominant script set by framing prose, not by the payload that actually
126
+ // matched. Using the winning chunk's script keeps the loosening aligned with
127
+ // the text that produced the contrast.
128
+ const queryScript = dominantQueryScript(bandDefiningRows, winnerChunkByRowId, queryChunks)
129
+ const topicMargin = MARGIN * crossScriptMarginScale(queryScript, headBandScripts(bandDefiningRows, index))
106
130
  const bandScores = bandDefiningRows.map(({ score }) => score)
107
131
  const keptBandDefiningRows = bandDefiningRows.slice(0, gateRelevance(bandScores, topK * 2, topicMargin))
108
132
 
@@ -137,6 +161,49 @@ function headBandScripts(
137
161
  return [...scripts]
138
162
  }
139
163
 
164
+ // Scores every store row against ALL query-chunk embeddings and keeps each row's
165
+ // MAX similarity (a row relevant to ANY chunk should rank high; mean would dilute
166
+ // it against unrelated chunks, sum would reward prompt length). Returns the merged
167
+ // `{ row, score }[]` in the same shape `store.queryScored` produced for a single
168
+ // query, plus the chunk index that produced each row's winning score — so the
169
+ // cross-script margin can be judged against the chunk that actually matched.
170
+ function maxScoreAcrossChunks(
171
+ queryEmbeddings: Float32Array[],
172
+ store: VectorStore,
173
+ ): { scored: ScoredVectorRow[]; winnerChunkByRowId: Map<string, number> } {
174
+ const best = new Map<string, { scored: ScoredVectorRow; chunkIndex: number }>()
175
+ queryEmbeddings.forEach((embedding, chunkIndex) => {
176
+ for (const scoredRow of store.queryScored(embedding, EMBEDDING_MODEL_ID)) {
177
+ const existing = best.get(scoredRow.row.id)
178
+ if (existing === undefined || scoredRow.score > existing.scored.score) {
179
+ best.set(scoredRow.row.id, { scored: scoredRow, chunkIndex })
180
+ }
181
+ }
182
+ })
183
+ const scored = [...best.values()].map(({ scored: s }) => s).sort((a, b) => b.score - a.score)
184
+ const winnerChunkByRowId = new Map([...best].map(([id, { chunkIndex }]) => [id, chunkIndex]))
185
+ return { scored, winnerChunkByRowId }
186
+ }
187
+
188
+ // The single query script the cross-script margin is judged with, taken from the
189
+ // chunk that won the TOP-scoring band row. The gate's contrast is top1 minus the
190
+ // non-head median (`gateRelevance`), so the top row alone defines whether the
191
+ // margin loosens — a majority vote across the head would let a crowd of ambient
192
+ // same-script noise rows (each won by a framing chunk) override the cross-script
193
+ // payload chunk that actually won the real top match, keeping the margin strict
194
+ // and suppressing a genuine cross-script hit sitting in the compressed band.
195
+ function dominantQueryScript(
196
+ bandDefiningRows: ScoredVectorRow[],
197
+ winnerChunkByRowId: Map<string, number>,
198
+ queryChunks: string[],
199
+ ): ScriptClass {
200
+ const topRow = bandDefiningRows[0]
201
+ if (topRow === undefined) return 'other'
202
+ const chunkIndex = winnerChunkByRowId.get(topRow.row.id)
203
+ const chunk = chunkIndex === undefined ? undefined : queryChunks[chunkIndex]
204
+ return chunk === undefined ? 'other' : dominantScript(chunk)
205
+ }
206
+
140
207
  // Phrase-first, then token-OR fallback (mirrors `memory_search`). `hybridSearch`'s
141
208
  // query is always the whole user prompt, which never appears verbatim in a shard,
142
209
  // so a phrase-only lane returns nothing every real turn and RRF degenerates to the
@@ -3,32 +3,37 @@ import type { FragmentEvent } from '../stream-events'
3
3
  import type { FragmentsAppendedContext } from '../stream-io'
4
4
  import { embed, EMBEDDING_MODEL_ID } from './embedder'
5
5
  import type { EmbedFn } from './hybrid'
6
- import type { VectorStore } from './store'
6
+ import type { OpenVectorStore } from './store'
7
7
 
8
8
  export function makeAppendHook(
9
- store: VectorStore,
9
+ openStore: OpenVectorStore,
10
10
  embedFn: EmbedFn = embed,
11
11
  ): (fragments: FragmentEvent[], context: FragmentsAppendedContext) => Promise<void> {
12
12
  return async (fragments, context) => {
13
- for (const fragment of fragments) {
14
- const key = `${context.date ?? fragment.ts.slice(0, 10)}#${fragment.id}`
15
- const id = `stream:${key}`
16
- const contentHash = fragmentContentHash(fragment)
17
- const existing = store.getByIds([id])[0]
18
- if (existing?.contentHash === contentHash && existing.model === EMBEDDING_MODEL_ID) continue
13
+ const store = openStore()
14
+ try {
15
+ for (const fragment of fragments) {
16
+ const key = `${context.date ?? fragment.ts.slice(0, 10)}#${fragment.id}`
17
+ const id = `stream:${key}`
18
+ const contentHash = fragmentContentHash(fragment)
19
+ const existing = store.getByIds([id])[0]
20
+ if (existing?.contentHash === contentHash && existing.model === EMBEDDING_MODEL_ID) continue
19
21
 
20
- const text = `${fragment.topic}\n${fragment.body}`
21
- const [embedding] = await embedFn([text], 'passage')
22
- if (embedding === undefined) continue
23
- store.upsert({
24
- id,
25
- source: 'stream',
26
- key,
27
- model: EMBEDDING_MODEL_ID,
28
- dims: embedding.length,
29
- embedding,
30
- contentHash,
31
- })
22
+ const text = `${fragment.topic}\n${fragment.body}`
23
+ const [embedding] = await embedFn([text], 'passage')
24
+ if (embedding === undefined) continue
25
+ store.upsert({
26
+ id,
27
+ source: 'stream',
28
+ key,
29
+ model: EMBEDDING_MODEL_ID,
30
+ dims: embedding.length,
31
+ embedding,
32
+ contentHash,
33
+ })
34
+ }
35
+ } finally {
36
+ store.close()
32
37
  }
33
38
  }
34
39
  }
@@ -8,7 +8,7 @@ import { loadAllReferences } from '../references/load-references'
8
8
  import { readAllUndreamedStreamDays, type UndreamedStreamDay } from '../stream-io'
9
9
  import { EMBEDDING_MODEL_ID } from './embedder'
10
10
  import type { VectorStore } from './store'
11
- import { boundEmbeddableText, fragmentEmbeddableText } from './truncation'
11
+ import { chunkEmbeddableText, fragmentEmbeddableText } from './truncation'
12
12
 
13
13
  export type Passage = {
14
14
  id: string
@@ -52,7 +52,7 @@ export async function referencePassages(agentDir: string): Promise<Passage[]> {
52
52
  // applies at startup.
53
53
  export function referencePassagesForOne(slug: string, body: string, demoted = false): Passage[] {
54
54
  if (demoted) return []
55
- return chunkReferenceBody(body).map((chunk, chunkIdx) => ({
55
+ return chunkEmbeddableText(body).map((chunk, chunkIdx) => ({
56
56
  id: `reference:${slug}#${chunkIdx}`,
57
57
  source: 'reference',
58
58
  key: slug,
@@ -98,28 +98,6 @@ function buildPassages(shards: TopicShard[], streamDays: UndreamedStreamDay[]):
98
98
  ]
99
99
  }
100
100
 
101
- function chunkReferenceBody(body: string): string[] {
102
- if (body.length === 0) return ['']
103
-
104
- const chunks: string[] = []
105
- let remaining = body
106
- while (remaining.length > 0) {
107
- const bounded = boundEmbeddableText(remaining)
108
- if (!bounded.bounded) {
109
- chunks.push(bounded.text)
110
- break
111
- }
112
- if (bounded.text.length === 0) {
113
- chunks.push(remaining[0]!)
114
- remaining = remaining.slice(1)
115
- continue
116
- }
117
- chunks.push(bounded.text)
118
- remaining = remaining.slice(bounded.text.length)
119
- }
120
- return chunks
121
- }
122
-
123
101
  function hashContent(content: string): string {
124
102
  return createHash('sha256').update(content).digest('hex')
125
103
  }
@@ -1,7 +1,7 @@
1
1
  import { embed, EMBEDDING_MODEL_ID } from './embedder'
2
2
  import type { EmbedFn } from './hybrid'
3
3
  import { referencePassagesForOne } from './passages'
4
- import type { VectorStore } from './store'
4
+ import type { OpenVectorStore } from './store'
5
5
 
6
6
  export type ReferenceStoredContext = { slug: string; body: string; demoted?: boolean }
7
7
 
@@ -17,34 +17,39 @@ export type ReferenceStoredContext = { slug: string; body: string; demoted?: boo
17
17
  // exists. We compute the wanted id set, upsert changed chunks, and delete any
18
18
  // existing row for this slug that is not wanted.
19
19
  export function makeReferenceStoredHook(
20
- store: VectorStore,
20
+ openStore: OpenVectorStore,
21
21
  embedFn: EmbedFn = embed,
22
22
  ): (context: ReferenceStoredContext) => Promise<void> {
23
23
  return async ({ slug, body, demoted }) => {
24
- const passages = referencePassagesForOne(slug, body, demoted)
25
- const wantedIds = new Set(passages.map((passage) => passage.id))
24
+ const store = openStore()
25
+ try {
26
+ const passages = referencePassagesForOne(slug, body, demoted)
27
+ const wantedIds = new Set(passages.map((passage) => passage.id))
26
28
 
27
- const prefix = `reference:${slug}#`
28
- const staleIds = store
29
- .getAllMeta()
30
- .flatMap((row) => (row.id.startsWith(prefix) && !wantedIds.has(row.id) ? [row.id] : []))
31
- if (staleIds.length > 0) store.deleteMany(staleIds)
29
+ const prefix = `reference:${slug}#`
30
+ const staleIds = store
31
+ .getAllMeta()
32
+ .flatMap((row) => (row.id.startsWith(prefix) && !wantedIds.has(row.id) ? [row.id] : []))
33
+ if (staleIds.length > 0) store.deleteMany(staleIds)
32
34
 
33
- for (const passage of passages) {
34
- const existing = store.getByIds([passage.id])[0]
35
- if (existing?.contentHash === passage.contentHash && existing.model === EMBEDDING_MODEL_ID) continue
35
+ for (const passage of passages) {
36
+ const existing = store.getByIds([passage.id])[0]
37
+ if (existing?.contentHash === passage.contentHash && existing.model === EMBEDDING_MODEL_ID) continue
36
38
 
37
- const [embedding] = await embedFn([passage.text], 'passage')
38
- if (embedding === undefined) continue
39
- store.upsert({
40
- id: passage.id,
41
- source: 'reference',
42
- key: slug,
43
- model: EMBEDDING_MODEL_ID,
44
- dims: embedding.length,
45
- embedding,
46
- contentHash: passage.contentHash,
47
- })
39
+ const [embedding] = await embedFn([passage.text], 'passage')
40
+ if (embedding === undefined) continue
41
+ store.upsert({
42
+ id: passage.id,
43
+ source: 'reference',
44
+ key: slug,
45
+ model: EMBEDDING_MODEL_ID,
46
+ dims: embedding.length,
47
+ embedding,
48
+ contentHash: passage.contentHash,
49
+ })
50
+ }
51
+ } finally {
52
+ store.close()
48
53
  }
49
54
  }
50
55
  }
@@ -2,6 +2,8 @@ import { Database } from 'bun:sqlite'
2
2
  import { mkdirSync } from 'node:fs'
3
3
  import { dirname } from 'node:path'
4
4
 
5
+ export type OpenVectorStore = () => VectorStore
6
+
5
7
  export type VectorRow = {
6
8
  id: string
7
9
  source: 'topic' | 'stream' | 'reference'
@@ -92,6 +92,34 @@ export function boundEmbeddableText(text: string): BoundedText {
92
92
  return { text: text.slice(0, charBudget), bounded: true, estimatedTokens }
93
93
  }
94
94
 
95
+ // Splits text into ordered ≤budget chunks so NOTHING is dropped — the
96
+ // no-silent-loss alternative to `boundEmbeddableText` for callers that can embed
97
+ // every part (stored references, retrieval queries). An in-budget input yields a
98
+ // single unchanged chunk, so the common short case is `[text]`. Each chunk is the
99
+ // longest budgeted prefix of the remaining text (same forward walk as the bound),
100
+ // guaranteeing forward progress; a degenerate input whose first char alone
101
+ // over-estimates still advances one char at a time rather than looping.
102
+ export function chunkEmbeddableText(text: string): string[] {
103
+ if (text.length === 0) return ['']
104
+ const chunks: string[] = []
105
+ let remaining = text
106
+ while (remaining.length > 0) {
107
+ const bounded = boundEmbeddableText(remaining)
108
+ if (!bounded.bounded) {
109
+ chunks.push(bounded.text)
110
+ break
111
+ }
112
+ if (bounded.text.length === 0) {
113
+ chunks.push(remaining[0]!)
114
+ remaining = remaining.slice(1)
115
+ continue
116
+ }
117
+ chunks.push(bounded.text)
118
+ remaining = remaining.slice(bounded.text.length)
119
+ }
120
+ return chunks
121
+ }
122
+
95
123
  // Returns the longest prefix length (in chars) whose estimateTokens is still
96
124
  // within budget. Recomputes the EXACT same estimate incrementally — CJK chars
97
125
  // at 1 token, plus max(non-CJK char-ratio, word count) — so a bounded prefix can
@@ -38,8 +38,10 @@ export const PLANNER_SKILLS: readonly LoadableSkill[] = [PROJECT_PLAN_SKILL, GEN
38
38
  // request fails loudly. Sized for a thorough `deep`-model plan (research lookups
39
39
  // + a careful decomposition), well above the typical sub-minute plan. This is
40
40
  // liveness for the parent, not hard cancellation. See src/agent/subagents.ts
41
- // `timeoutMs`.
42
- export const PLANNER_SPAWN_TIMEOUT_MS = 600_000
41
+ // `timeoutMs`. 30m matches the reviewer and researcher — every deep-profile
42
+ // subagent shares the same nested-fan-out budget so a real pass is not killed
43
+ // mid-synthesis.
44
+ export const PLANNER_SPAWN_TIMEOUT_MS = 1_800_000
43
45
 
44
46
  // The default write zone for the plan file when the caller does not specify
45
47
  // `outputPath`. MUST be a universally-writable location, because a subagent
@@ -51,7 +51,15 @@ export const REVIEWER_SKILLS: readonly LoadableSkill[] = [
51
51
  // liveness for the parent, not hard cancellation: pi's `session.prompt` takes no
52
52
  // AbortSignal, so the LLM stream may run until the OS reaps it. See
53
53
  // src/agent/subagents.ts `timeoutMs`.
54
- export const REVIEWER_SPAWN_TIMEOUT_MS = 600_000
54
+ //
55
+ // 30m, not the prior 10m: the `deep` profile trades speed for quality, and a
56
+ // real review of a large PR (many changed files + nested scout fan-out + a few
57
+ // web lookups + careful synthesis) routinely exceeds 10m. A 10m ceiling killed
58
+ // both the first and retry reviews of a 27-file PR mid-analysis, leaving the PR
59
+ // with no posted verdict — the exact "stranded review" failure the guards exist
60
+ // to prevent. Matches RESEARCHER_SPAWN_TIMEOUT_MS, the other deep-profile
61
+ // subagent that hit the same nested-fan-out wall.
62
+ export const REVIEWER_SPAWN_TIMEOUT_MS = 1_800_000
55
63
 
56
64
  // The reviewer's read-only contract is enforced in depth: this system prompt
57
65
  // states it, the global bash guards (`secret-exfil-bash`, `git-exfil`) catch
@@ -9,7 +9,7 @@ This plugin is **auto-loaded** by every TypeClaw agent. There is no `plugins[]`
9
9
  `pi-coding-agent`'s built-in tools occasionally return very large payloads that the model only needed once. Two empirically observed cases:
10
10
 
11
11
  1. **`read` on an image file** returns the base64-encoded image inline (e.g. `{type:"image", data:"<3.2MB of base64>"}`). The model uses it on the turn it was asked for, then sees the same 3.2MB of base64 as conversation context on every subsequent prompt — until compaction fires (which is token-driven, not byte-driven, so a single fat blob may sit in context for many turns before compaction is triggered).
12
- 2. **`web_fetch` on a binary URL** (PNG, ZIP, etc.) receives the raw response body, treats it as text, and stores raw binary as a JSON-encoded string. Same effect: 100KB+ of mojibake sits in the transcript permanently.
12
+ 2. **`web_fetch` on a binary URL** (PNG, ZIP, etc.) receives the raw response body, treats it as text, and stores raw binary as a JSON-encoded string. Same effect: 100KB+ of mojibake sits in the transcript permanently. (Historical: this predates `web_fetch`'s own per-strategy `OUTPUT_CAPS`, which now bound its text output to ≤200KB regardless — see the `web_fetch` text exemption below.)
13
13
 
14
14
  The result is a session JSONL file that's tens of megabytes on disk but mostly one or two giant tool results, plus 3-minute first-prompt latencies after container restart because the full transcript gets re-shipped to the LLM as context.
15
15
 
@@ -47,6 +47,15 @@ The cap is per-part, not per-result, so a result with one small text part and on
47
47
 
48
48
  Placeholders carry the literal substring `tool-result-cap:` so future agents (or human operators inspecting a session) can grep for them and recognize that the original payload was intentionally elided rather than truncated by some other layer.
49
49
 
50
+ ## The `web_fetch` text exemption
51
+
52
+ `web_fetch` is exempt from the **text** cap (but not the image cap), hardcoded in `cap-result.ts` via `TEXT_CAP_EXEMPT_TOOLS`. This is not user-configurable because it resolves a collision between two first-party caps:
53
+
54
+ - `web_fetch` has its own per-strategy `OUTPUT_CAPS` in `src/agent/tools/webfetch/types.ts` — `readability` is capped at 200KB, `raw` at 100KB, others between 50KB and 100KB — bounded above by a 5MB transport limit. These are deliberately tuned: `readability` gets a generous budget so a full article reaches the model.
55
+ - This plugin's flat `textMaxBytes` default (32KB) is smaller than every one of those, so without the exemption it would silently clip every `web_fetch` result to ~8K tokens, making the strategy caps meaningless. A `readability` fetch designed for 200KB would never deliver more than 32KB.
56
+
57
+ The exemption is **text-only**: `web_fetch` image parts still go through `imageMaxBytes`, so the binary/base64 transcript-bloat protection (case 2 above) is preserved. The tradeoff is that this plugin no longer provides a 32KB backstop for `web_fetch` text — that responsibility now belongs entirely to `web_fetch`'s own `capOutput()`. User-configured `exemptTools` is a separate, broader knob (skips _all_ capping, image included); the `web_fetch` text exemption is narrower and always on.
58
+
50
59
  ## What's not capped
51
60
 
52
61
  - `details` on tool results (an opaque structured payload provider-specific to each tool — generally small, and mutating it risks breaking tool-specific telemetry).
@@ -14,6 +14,13 @@ export type CapStats = {
14
14
 
15
15
  const ELIDED_MARKER = '[tool-result-cap: '
16
16
 
17
+ // `web_fetch` already bounds its own text via per-strategy OUTPUT_CAPS
18
+ // (src/agent/tools/webfetch/types.ts: 50KB–200KB; readability gets 200KB on
19
+ // purpose), so the flat 32KB textMaxBytes here only collides with it and clips
20
+ // every readability fetch to ~8K tokens. Skip TEXT capping for it; image parts
21
+ // still hit the image cap below, preserving the binary/base64 bloat protection.
22
+ const TEXT_CAP_EXEMPT_TOOLS = new Set(['web_fetch'])
23
+
17
24
  // A capped text part is exactly the placeholder we generated: starts with
18
25
  // `[tool-result-cap: `, ends with `]`, and contains no inner `]`. The shape
19
26
  // check exists for idempotency so a previously-capped entry survives a second
@@ -46,6 +53,7 @@ export function capContentParts(tool: string, content: ContentPart[], options: C
46
53
  continue
47
54
  }
48
55
  if (part.type === 'text') {
56
+ if (TEXT_CAP_EXEMPT_TOOLS.has(tool)) continue
49
57
  const size = part.text.length
50
58
  if (size <= options.textMaxBytes) continue
51
59
  if (isElidedPlaceholderText(part.text)) continue