typeclaw 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/auth.schema.json +57 -0
- package/package.json +2 -2
- package/secrets.schema.json +57 -0
- package/src/agent/index.ts +10 -6
- package/src/agent/multimodal/look-at.ts +87 -0
- package/src/agent/session-origin.ts +1 -0
- package/src/agent/tools/channel-history-render.ts +21 -0
- package/src/agent/tools/channel-history.ts +2 -17
- package/src/agent/tools/channel-read.ts +206 -0
- package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +26 -2
- package/src/bundled-plugins/github-cli-auth/index.ts +25 -6
- package/src/bundled-plugins/memory/index.ts +10 -11
- package/src/bundled-plugins/memory/vector/hybrid.ts +73 -6
- package/src/bundled-plugins/memory/vector/index-on-write.ts +25 -20
- package/src/bundled-plugins/memory/vector/passages.ts +2 -24
- package/src/bundled-plugins/memory/vector/reference-index-on-write.ts +28 -23
- package/src/bundled-plugins/memory/vector/store.ts +2 -0
- package/src/bundled-plugins/memory/vector/truncation.ts +28 -0
- package/src/bundled-plugins/planner/planner.ts +4 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +9 -1
- package/src/bundled-plugins/tool-result-cap/README.md +10 -1
- package/src/bundled-plugins/tool-result-cap/cap-result.ts +8 -0
- package/src/channels/adapters/discord-bot.ts +112 -0
- package/src/channels/adapters/instagram-channel-resolver.ts +113 -0
- package/src/channels/adapters/instagram-classify.ts +69 -0
- package/src/channels/adapters/instagram-format.ts +8 -0
- package/src/channels/adapters/instagram.ts +346 -0
- package/src/channels/adapters/kakaotalk-classify.ts +31 -6
- package/src/channels/adapters/kakaotalk.ts +1 -1
- package/src/channels/adapters/slack-bot.ts +147 -0
- package/src/channels/manager.ts +60 -2
- package/src/channels/router.ts +64 -0
- package/src/channels/schema.ts +5 -0
- package/src/channels/types.ts +46 -0
- package/src/cli/channel.ts +76 -1
- package/src/cli/init.ts +1 -0
- package/src/cli/status.ts +37 -14
- package/src/config/channels-mutation.ts +2 -1
- package/src/doctor/channel-checks.ts +28 -0
- package/src/hostd/daemon.ts +59 -19
- package/src/hostd/kakao-renewal-manager.ts +35 -10
- package/src/hostd/protocol.ts +2 -0
- package/src/hostd/tailscale.ts +41 -5
- package/src/hostd/webex-renewal-manager.ts +13 -4
- package/src/init/index.ts +50 -1
- package/src/init/instagram-auth.ts +134 -0
- package/src/init/run-owner-claim.ts +1 -0
- package/src/permissions/match-rule.ts +1 -1
- package/src/permissions/resolve.ts +1 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/manager.ts +14 -0
- package/src/plugin/registry.ts +12 -0
- package/src/plugin/types.ts +4 -0
- package/src/role-claim/match-rule.ts +2 -1
- package/src/run/index.ts +46 -20
- package/src/secrets/instagram-store.ts +112 -0
- package/src/secrets/schema.ts +18 -0
- package/src/skills/typeclaw-channel-instagram/SKILL.md +30 -0
- package/src/test-helpers/rm-temp-dir.ts +24 -0
- package/typeclaw.schema.json +161 -1
|
@@ -109,6 +109,18 @@ export default definePlugin({
|
|
|
109
109
|
}): Promise<HookResult | 'fall-through'> => {
|
|
110
110
|
const { event, command } = params
|
|
111
111
|
const review = await noteReviewCommand({ callId: event.callId, command })
|
|
112
|
+
// guard() holds the lease expecting tool.after to release it, but a
|
|
113
|
+
// tool.before block (from ANY guard below) means the command never runs, so
|
|
114
|
+
// tool.after never fires and the lease strands for its full TTL — telling
|
|
115
|
+
// every other session "the in-flight one will post" when it never will (the
|
|
116
|
+
// lost-verdict strand: a review trips the shape guard, the lease strands, the
|
|
117
|
+
// PR ends unreviewed). blockAfterLease() releases on such a block. guard()'s
|
|
118
|
+
// OWN terminal block self-releases and returns directly, bypassing this path.
|
|
119
|
+
let leaseClaimed = false
|
|
120
|
+
const blockAfterLease = async (block: HookResult & { block: true }): Promise<HookResult> => {
|
|
121
|
+
if (leaseClaimed) await verdictGuard.release({ callId: event.callId, succeeded: false })
|
|
122
|
+
return block
|
|
123
|
+
}
|
|
112
124
|
if (review.detected !== null) {
|
|
113
125
|
const block = await verdictGuard.guard({
|
|
114
126
|
callId: event.callId,
|
|
@@ -117,8 +129,9 @@ export default definePlugin({
|
|
|
117
129
|
verdict: review.detected.verdict,
|
|
118
130
|
})
|
|
119
131
|
if (block !== null) return block
|
|
132
|
+
leaseClaimed = true
|
|
120
133
|
}
|
|
121
|
-
if (review.dump !== null) return review.dump
|
|
134
|
+
if (review.dump !== null) return blockAfterLease(review.dump)
|
|
122
135
|
|
|
123
136
|
// Analyze first WITHOUT the fallback: an explicit `-R`/path repo must win,
|
|
124
137
|
// and we only pay for fallback resolution (a git subprocess) when the
|
|
@@ -152,7 +165,7 @@ export default definePlugin({
|
|
|
152
165
|
GITHUB_TOKEN: process.env.GITHUB_TOKEN,
|
|
153
166
|
})
|
|
154
167
|
if (userEndpointTokens.some((token) => shouldMintAppToken(token, hasAppTokenResolver()))) {
|
|
155
|
-
return { block: true, reason: appUserEndpointReason }
|
|
168
|
+
return blockAfterLease({ block: true, reason: appUserEndpointReason })
|
|
156
169
|
}
|
|
157
170
|
|
|
158
171
|
if (decision.kind === 'pass-through') return 'fall-through'
|
|
@@ -200,16 +213,22 @@ export default definePlugin({
|
|
|
200
213
|
// bug returns); otherwise block with guidance rather than failing mute.
|
|
201
214
|
if (!shouldMintAppToken(undefined, hasAppTokenResolver())) {
|
|
202
215
|
if (decision.kind === 'block') {
|
|
203
|
-
return {
|
|
216
|
+
return blockAfterLease({
|
|
217
|
+
block: true,
|
|
218
|
+
reason: decision.reason + buildGhBlockGuidance(decision.code, fallbackRepo),
|
|
219
|
+
})
|
|
204
220
|
}
|
|
205
221
|
warnSandboxedPatWithheldOnce()
|
|
206
|
-
return { block: true, reason: sandboxedPatWithheldReason }
|
|
222
|
+
return blockAfterLease({ block: true, reason: sandboxedPatWithheldReason })
|
|
207
223
|
}
|
|
208
224
|
mintForSandboxedPat = true
|
|
209
225
|
}
|
|
210
226
|
|
|
211
227
|
if (decision.kind === 'block') {
|
|
212
|
-
return {
|
|
228
|
+
return blockAfterLease({
|
|
229
|
+
block: true,
|
|
230
|
+
reason: decision.reason + buildGhBlockGuidance(decision.code, fallbackRepo),
|
|
231
|
+
})
|
|
213
232
|
}
|
|
214
233
|
|
|
215
234
|
// No App auth (no App-class GH_TOKEN and no live minter): leave whatever
|
|
@@ -218,7 +237,7 @@ export default definePlugin({
|
|
|
218
237
|
if (!mintForSandboxedPat && !shouldMintAppToken(process.env.GH_TOKEN, hasAppTokenResolver())) return
|
|
219
238
|
|
|
220
239
|
const result = await resolveTokenForRepo(decision.repoSlug)
|
|
221
|
-
if (result.kind === 'unavailable') return { block: true, reason: result.reason }
|
|
240
|
+
if (result.kind === 'unavailable') return blockAfterLease({ block: true, reason: result.reason })
|
|
222
241
|
// Inject via the internal env overlay (delivered to the spawn / bwrap
|
|
223
242
|
// --setenv by the bash wrapper) so the token never enters the command
|
|
224
243
|
// string, where it could leak through logs or later hooks. When the repo
|
|
@@ -28,7 +28,7 @@ import { embed } from './vector/embedder'
|
|
|
28
28
|
import { hybridSearch, type EmbedFn } from './vector/hybrid'
|
|
29
29
|
import { makeAppendHook } from './vector/index-on-write'
|
|
30
30
|
import { makeReferenceStoredHook } from './vector/reference-index-on-write'
|
|
31
|
-
import { VectorStore } from './vector/store'
|
|
31
|
+
import { type OpenVectorStore, VectorStore } from './vector/store'
|
|
32
32
|
|
|
33
33
|
const DEFAULT_IDLE_MS = 60_000
|
|
34
34
|
const DEFAULT_BUFFER_BYTES = 500_000
|
|
@@ -144,13 +144,13 @@ const VECTOR_TURN_TOP_K = 10
|
|
|
144
144
|
export type MemoryPluginDeps = {
|
|
145
145
|
hybridSearch: typeof hybridSearch
|
|
146
146
|
queryEmbedFn: EmbedFn
|
|
147
|
-
openAppendVectorStore: (agentDir: string) =>
|
|
147
|
+
openAppendVectorStore: (agentDir: string) => OpenVectorStore
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
const defaultDeps: MemoryPluginDeps = {
|
|
151
151
|
hybridSearch,
|
|
152
152
|
queryEmbedFn: embed,
|
|
153
|
-
openAppendVectorStore: (agentDir) => VectorStore.open(join(agentDir, 'memory', '.vectors', 'index.db')),
|
|
153
|
+
openAppendVectorStore: (agentDir) => () => VectorStore.open(join(agentDir, 'memory', '.vectors', 'index.db')),
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
// TypeClaw-owned infrastructure subagents (memory-logger, dreaming, backup) run
|
|
@@ -402,19 +402,18 @@ export function createMemoryPlugin(deps: MemoryPluginDeps = defaultDeps) {
|
|
|
402
402
|
error: (m: string) => ctx.logger.error(m),
|
|
403
403
|
}
|
|
404
404
|
|
|
405
|
-
//
|
|
406
|
-
//
|
|
407
|
-
//
|
|
408
|
-
//
|
|
409
|
-
|
|
410
|
-
const appendVectorStore = deps.openAppendVectorStore(ctx.agentDir)
|
|
405
|
+
// Opener (not an open handle) for append-time indexing. Each on-write hook
|
|
406
|
+
// opens its own short-lived store and closes it when the batch settles, so
|
|
407
|
+
// no bun:sqlite handle outlives a hook call to pin the agent dir (Bun
|
|
408
|
+
// #25964). Boot-only paths that never append never open the DB at all.
|
|
409
|
+
const openAppendVectorStore = deps.openAppendVectorStore(ctx.agentDir)
|
|
411
410
|
|
|
412
411
|
return {
|
|
413
412
|
subagents: {
|
|
414
413
|
'memory-logger': createMemoryLoggerSubagent({
|
|
415
414
|
logger: subagentLogger,
|
|
416
|
-
onFragmentsAppended: makeAppendHook(
|
|
417
|
-
onReferenceStored: makeReferenceStoredHook(
|
|
415
|
+
onFragmentsAppended: makeAppendHook(openAppendVectorStore),
|
|
416
|
+
onReferenceStored: makeReferenceStoredHook(openAppendVectorStore),
|
|
418
417
|
}),
|
|
419
418
|
dreaming: createDreamingSubagent({
|
|
420
419
|
logger: subagentLogger,
|
|
@@ -19,11 +19,27 @@ import type { Passage } from './passages'
|
|
|
19
19
|
import { clearsBaseline, gateRelevance, MARGIN, streamAdmissionBaseline } from './relevance-gate'
|
|
20
20
|
import { crossScriptMarginScale, dominantScript, type ScriptClass } from './script'
|
|
21
21
|
import { VectorStore, type ScoredVectorRow, type VectorRow } from './store'
|
|
22
|
+
import { chunkEmbeddableText } from './truncation'
|
|
22
23
|
|
|
23
24
|
export { collectPassages, findMissingPassages, type Passage } from './passages'
|
|
24
25
|
|
|
25
26
|
const RRF_K = 60
|
|
26
27
|
|
|
28
|
+
// A long prompt is embedded as multiple ≤512-token chunks (not truncated to one
|
|
29
|
+
// vector), so a memory relevant to ANY part of the prompt — including its tail —
|
|
30
|
+
// can still be retrieved. The cap bounds the per-turn cosine scans: each chunk is
|
|
31
|
+
// a full O(rows) brute-force pass over the store, so an unbounded chunk count
|
|
32
|
+
// would scale per-turn retrieval cost with prompt length. At the cap, the LAST
|
|
33
|
+
// chunks are kept rather than the first: TypeClaw cron/agent prompts front-load
|
|
34
|
+
// fixed framing prose and place the actual payload last, so dropping the head is
|
|
35
|
+
// the lesser loss than re-introducing the tail-truncation this fix removes.
|
|
36
|
+
const MAX_QUERY_CHUNKS = 10
|
|
37
|
+
|
|
38
|
+
function queryEmbeddingChunks(query: string): string[] {
|
|
39
|
+
const chunks = chunkEmbeddableText(query)
|
|
40
|
+
return chunks.length <= MAX_QUERY_CHUNKS ? chunks : chunks.slice(chunks.length - MAX_QUERY_CHUNKS)
|
|
41
|
+
}
|
|
42
|
+
|
|
27
43
|
export type HybridSearchResult = {
|
|
28
44
|
source: 'topic' | 'stream' | 'reference'
|
|
29
45
|
key: string
|
|
@@ -46,17 +62,18 @@ export async function hybridSearch(
|
|
|
46
62
|
): Promise<HybridSearchResult[]> {
|
|
47
63
|
if (topK <= 0) return []
|
|
48
64
|
|
|
65
|
+
const queryChunks = queryEmbeddingChunks(query)
|
|
49
66
|
const [shards, streamDays, references, queryEmbeddings] = await Promise.all([
|
|
50
67
|
loadAllShards(agentDir),
|
|
51
68
|
readAllUndreamedStreamDays(agentDir),
|
|
52
69
|
loadAllReferences(agentDir),
|
|
53
|
-
embedFn(
|
|
70
|
+
embedFn(queryChunks, 'query'),
|
|
54
71
|
])
|
|
55
72
|
|
|
56
73
|
const { parentSlugsByFragmentId, supersededFragmentIds } = buildParentLinks(shards)
|
|
57
74
|
const index = buildContentIndex(shards, streamDays, references, supersededFragmentIds)
|
|
58
75
|
const vectorRows =
|
|
59
|
-
queryEmbeddings
|
|
76
|
+
queryEmbeddings.length === 0 ? [] : gatedVectorLane(queryEmbeddings, queryChunks, store, topK, index)
|
|
60
77
|
const keywordMatches = keywordLane(query, shards, streamDays, references, topK * 2)
|
|
61
78
|
|
|
62
79
|
return fuseLanes(vectorRows, keywordMatches, index, parentSlugsByFragmentId).slice(0, topK)
|
|
@@ -85,13 +102,13 @@ export async function hybridSearch(
|
|
|
85
102
|
// separate keyword lane. An empty merged lane composes with RRF exactly like a
|
|
86
103
|
// lane that found nothing, so a genuine keyword hit survives a full no-match.
|
|
87
104
|
function gatedVectorLane(
|
|
88
|
-
|
|
105
|
+
queryEmbeddings: Float32Array[],
|
|
106
|
+
queryChunks: string[],
|
|
89
107
|
store: VectorStore,
|
|
90
108
|
topK: number,
|
|
91
|
-
query: string,
|
|
92
109
|
index: Map<string, Omit<HybridSearchResult, 'rrfScore'>>,
|
|
93
110
|
): VectorRow[] {
|
|
94
|
-
const scored =
|
|
111
|
+
const { scored, winnerChunkByRowId } = maxScoreAcrossChunks(queryEmbeddings, store)
|
|
95
112
|
const bandDefiningRows = scored.filter(({ row }) => row.source === 'topic' || row.source === 'reference')
|
|
96
113
|
const streamRows = scored.filter(({ row }) => row.source === 'stream')
|
|
97
114
|
|
|
@@ -102,7 +119,14 @@ function gatedVectorLane(
|
|
|
102
119
|
// script winner whose contrast sits between the loosened and strict margins
|
|
103
120
|
// would still be suppressed. Headings (via `index`) carry the language; slugs
|
|
104
121
|
// are ASCII-kebab even for non-Latin topics.
|
|
105
|
-
|
|
122
|
+
//
|
|
123
|
+
// The query-side script comes from the CHUNK that won each head row, not the
|
|
124
|
+
// full prompt: a boilerplate-front prompt (most TypeClaw cron prompts) has a
|
|
125
|
+
// dominant script set by framing prose, not by the payload that actually
|
|
126
|
+
// matched. Using the winning chunk's script keeps the loosening aligned with
|
|
127
|
+
// the text that produced the contrast.
|
|
128
|
+
const queryScript = dominantQueryScript(bandDefiningRows, winnerChunkByRowId, queryChunks)
|
|
129
|
+
const topicMargin = MARGIN * crossScriptMarginScale(queryScript, headBandScripts(bandDefiningRows, index))
|
|
106
130
|
const bandScores = bandDefiningRows.map(({ score }) => score)
|
|
107
131
|
const keptBandDefiningRows = bandDefiningRows.slice(0, gateRelevance(bandScores, topK * 2, topicMargin))
|
|
108
132
|
|
|
@@ -137,6 +161,49 @@ function headBandScripts(
|
|
|
137
161
|
return [...scripts]
|
|
138
162
|
}
|
|
139
163
|
|
|
164
|
+
// Scores every store row against ALL query-chunk embeddings and keeps each row's
|
|
165
|
+
// MAX similarity (a row relevant to ANY chunk should rank high; mean would dilute
|
|
166
|
+
// it against unrelated chunks, sum would reward prompt length). Returns the merged
|
|
167
|
+
// `{ row, score }[]` in the same shape `store.queryScored` produced for a single
|
|
168
|
+
// query, plus the chunk index that produced each row's winning score — so the
|
|
169
|
+
// cross-script margin can be judged against the chunk that actually matched.
|
|
170
|
+
function maxScoreAcrossChunks(
|
|
171
|
+
queryEmbeddings: Float32Array[],
|
|
172
|
+
store: VectorStore,
|
|
173
|
+
): { scored: ScoredVectorRow[]; winnerChunkByRowId: Map<string, number> } {
|
|
174
|
+
const best = new Map<string, { scored: ScoredVectorRow; chunkIndex: number }>()
|
|
175
|
+
queryEmbeddings.forEach((embedding, chunkIndex) => {
|
|
176
|
+
for (const scoredRow of store.queryScored(embedding, EMBEDDING_MODEL_ID)) {
|
|
177
|
+
const existing = best.get(scoredRow.row.id)
|
|
178
|
+
if (existing === undefined || scoredRow.score > existing.scored.score) {
|
|
179
|
+
best.set(scoredRow.row.id, { scored: scoredRow, chunkIndex })
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
})
|
|
183
|
+
const scored = [...best.values()].map(({ scored: s }) => s).sort((a, b) => b.score - a.score)
|
|
184
|
+
const winnerChunkByRowId = new Map([...best].map(([id, { chunkIndex }]) => [id, chunkIndex]))
|
|
185
|
+
return { scored, winnerChunkByRowId }
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// The single query script the cross-script margin is judged with, taken from the
|
|
189
|
+
// chunk that won the TOP-scoring band row. The gate's contrast is top1 minus the
|
|
190
|
+
// non-head median (`gateRelevance`), so the top row alone defines whether the
|
|
191
|
+
// margin loosens — a majority vote across the head would let a crowd of ambient
|
|
192
|
+
// same-script noise rows (each won by a framing chunk) override the cross-script
|
|
193
|
+
// payload chunk that actually won the real top match, keeping the margin strict
|
|
194
|
+
// and suppressing a genuine cross-script hit sitting in the compressed band.
|
|
195
|
+
function dominantQueryScript(
|
|
196
|
+
bandDefiningRows: ScoredVectorRow[],
|
|
197
|
+
winnerChunkByRowId: Map<string, number>,
|
|
198
|
+
queryChunks: string[],
|
|
199
|
+
): ScriptClass {
|
|
200
|
+
const topRow = bandDefiningRows[0]
|
|
201
|
+
if (topRow === undefined) return 'other'
|
|
202
|
+
const chunkIndex = winnerChunkByRowId.get(topRow.row.id)
|
|
203
|
+
const chunk = chunkIndex === undefined ? undefined : queryChunks[chunkIndex]
|
|
204
|
+
return chunk === undefined ? 'other' : dominantScript(chunk)
|
|
205
|
+
}
|
|
206
|
+
|
|
140
207
|
// Phrase-first, then token-OR fallback (mirrors `memory_search`). `hybridSearch`'s
|
|
141
208
|
// query is always the whole user prompt, which never appears verbatim in a shard,
|
|
142
209
|
// so a phrase-only lane returns nothing every real turn and RRF degenerates to the
|
|
@@ -3,32 +3,37 @@ import type { FragmentEvent } from '../stream-events'
|
|
|
3
3
|
import type { FragmentsAppendedContext } from '../stream-io'
|
|
4
4
|
import { embed, EMBEDDING_MODEL_ID } from './embedder'
|
|
5
5
|
import type { EmbedFn } from './hybrid'
|
|
6
|
-
import type {
|
|
6
|
+
import type { OpenVectorStore } from './store'
|
|
7
7
|
|
|
8
8
|
export function makeAppendHook(
|
|
9
|
-
|
|
9
|
+
openStore: OpenVectorStore,
|
|
10
10
|
embedFn: EmbedFn = embed,
|
|
11
11
|
): (fragments: FragmentEvent[], context: FragmentsAppendedContext) => Promise<void> {
|
|
12
12
|
return async (fragments, context) => {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
const store = openStore()
|
|
14
|
+
try {
|
|
15
|
+
for (const fragment of fragments) {
|
|
16
|
+
const key = `${context.date ?? fragment.ts.slice(0, 10)}#${fragment.id}`
|
|
17
|
+
const id = `stream:${key}`
|
|
18
|
+
const contentHash = fragmentContentHash(fragment)
|
|
19
|
+
const existing = store.getByIds([id])[0]
|
|
20
|
+
if (existing?.contentHash === contentHash && existing.model === EMBEDDING_MODEL_ID) continue
|
|
19
21
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
22
|
+
const text = `${fragment.topic}\n${fragment.body}`
|
|
23
|
+
const [embedding] = await embedFn([text], 'passage')
|
|
24
|
+
if (embedding === undefined) continue
|
|
25
|
+
store.upsert({
|
|
26
|
+
id,
|
|
27
|
+
source: 'stream',
|
|
28
|
+
key,
|
|
29
|
+
model: EMBEDDING_MODEL_ID,
|
|
30
|
+
dims: embedding.length,
|
|
31
|
+
embedding,
|
|
32
|
+
contentHash,
|
|
33
|
+
})
|
|
34
|
+
}
|
|
35
|
+
} finally {
|
|
36
|
+
store.close()
|
|
32
37
|
}
|
|
33
38
|
}
|
|
34
39
|
}
|
|
@@ -8,7 +8,7 @@ import { loadAllReferences } from '../references/load-references'
|
|
|
8
8
|
import { readAllUndreamedStreamDays, type UndreamedStreamDay } from '../stream-io'
|
|
9
9
|
import { EMBEDDING_MODEL_ID } from './embedder'
|
|
10
10
|
import type { VectorStore } from './store'
|
|
11
|
-
import {
|
|
11
|
+
import { chunkEmbeddableText, fragmentEmbeddableText } from './truncation'
|
|
12
12
|
|
|
13
13
|
export type Passage = {
|
|
14
14
|
id: string
|
|
@@ -52,7 +52,7 @@ export async function referencePassages(agentDir: string): Promise<Passage[]> {
|
|
|
52
52
|
// applies at startup.
|
|
53
53
|
export function referencePassagesForOne(slug: string, body: string, demoted = false): Passage[] {
|
|
54
54
|
if (demoted) return []
|
|
55
|
-
return
|
|
55
|
+
return chunkEmbeddableText(body).map((chunk, chunkIdx) => ({
|
|
56
56
|
id: `reference:${slug}#${chunkIdx}`,
|
|
57
57
|
source: 'reference',
|
|
58
58
|
key: slug,
|
|
@@ -98,28 +98,6 @@ function buildPassages(shards: TopicShard[], streamDays: UndreamedStreamDay[]):
|
|
|
98
98
|
]
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
function chunkReferenceBody(body: string): string[] {
|
|
102
|
-
if (body.length === 0) return ['']
|
|
103
|
-
|
|
104
|
-
const chunks: string[] = []
|
|
105
|
-
let remaining = body
|
|
106
|
-
while (remaining.length > 0) {
|
|
107
|
-
const bounded = boundEmbeddableText(remaining)
|
|
108
|
-
if (!bounded.bounded) {
|
|
109
|
-
chunks.push(bounded.text)
|
|
110
|
-
break
|
|
111
|
-
}
|
|
112
|
-
if (bounded.text.length === 0) {
|
|
113
|
-
chunks.push(remaining[0]!)
|
|
114
|
-
remaining = remaining.slice(1)
|
|
115
|
-
continue
|
|
116
|
-
}
|
|
117
|
-
chunks.push(bounded.text)
|
|
118
|
-
remaining = remaining.slice(bounded.text.length)
|
|
119
|
-
}
|
|
120
|
-
return chunks
|
|
121
|
-
}
|
|
122
|
-
|
|
123
101
|
function hashContent(content: string): string {
|
|
124
102
|
return createHash('sha256').update(content).digest('hex')
|
|
125
103
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { embed, EMBEDDING_MODEL_ID } from './embedder'
|
|
2
2
|
import type { EmbedFn } from './hybrid'
|
|
3
3
|
import { referencePassagesForOne } from './passages'
|
|
4
|
-
import type {
|
|
4
|
+
import type { OpenVectorStore } from './store'
|
|
5
5
|
|
|
6
6
|
export type ReferenceStoredContext = { slug: string; body: string; demoted?: boolean }
|
|
7
7
|
|
|
@@ -17,34 +17,39 @@ export type ReferenceStoredContext = { slug: string; body: string; demoted?: boo
|
|
|
17
17
|
// exists. We compute the wanted id set, upsert changed chunks, and delete any
|
|
18
18
|
// existing row for this slug that is not wanted.
|
|
19
19
|
export function makeReferenceStoredHook(
|
|
20
|
-
|
|
20
|
+
openStore: OpenVectorStore,
|
|
21
21
|
embedFn: EmbedFn = embed,
|
|
22
22
|
): (context: ReferenceStoredContext) => Promise<void> {
|
|
23
23
|
return async ({ slug, body, demoted }) => {
|
|
24
|
-
const
|
|
25
|
-
|
|
24
|
+
const store = openStore()
|
|
25
|
+
try {
|
|
26
|
+
const passages = referencePassagesForOne(slug, body, demoted)
|
|
27
|
+
const wantedIds = new Set(passages.map((passage) => passage.id))
|
|
26
28
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
const prefix = `reference:${slug}#`
|
|
30
|
+
const staleIds = store
|
|
31
|
+
.getAllMeta()
|
|
32
|
+
.flatMap((row) => (row.id.startsWith(prefix) && !wantedIds.has(row.id) ? [row.id] : []))
|
|
33
|
+
if (staleIds.length > 0) store.deleteMany(staleIds)
|
|
32
34
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
for (const passage of passages) {
|
|
36
|
+
const existing = store.getByIds([passage.id])[0]
|
|
37
|
+
if (existing?.contentHash === passage.contentHash && existing.model === EMBEDDING_MODEL_ID) continue
|
|
36
38
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
39
|
+
const [embedding] = await embedFn([passage.text], 'passage')
|
|
40
|
+
if (embedding === undefined) continue
|
|
41
|
+
store.upsert({
|
|
42
|
+
id: passage.id,
|
|
43
|
+
source: 'reference',
|
|
44
|
+
key: slug,
|
|
45
|
+
model: EMBEDDING_MODEL_ID,
|
|
46
|
+
dims: embedding.length,
|
|
47
|
+
embedding,
|
|
48
|
+
contentHash: passage.contentHash,
|
|
49
|
+
})
|
|
50
|
+
}
|
|
51
|
+
} finally {
|
|
52
|
+
store.close()
|
|
48
53
|
}
|
|
49
54
|
}
|
|
50
55
|
}
|
|
@@ -92,6 +92,34 @@ export function boundEmbeddableText(text: string): BoundedText {
|
|
|
92
92
|
return { text: text.slice(0, charBudget), bounded: true, estimatedTokens }
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
// Splits text into ordered ≤budget chunks so NOTHING is dropped — the
|
|
96
|
+
// no-silent-loss alternative to `boundEmbeddableText` for callers that can embed
|
|
97
|
+
// every part (stored references, retrieval queries). An in-budget input yields a
|
|
98
|
+
// single unchanged chunk, so the common short case is `[text]`. Each chunk is the
|
|
99
|
+
// longest budgeted prefix of the remaining text (same forward walk as the bound),
|
|
100
|
+
// guaranteeing forward progress; a degenerate input whose first char alone
|
|
101
|
+
// over-estimates still advances one char at a time rather than looping.
|
|
102
|
+
export function chunkEmbeddableText(text: string): string[] {
|
|
103
|
+
if (text.length === 0) return ['']
|
|
104
|
+
const chunks: string[] = []
|
|
105
|
+
let remaining = text
|
|
106
|
+
while (remaining.length > 0) {
|
|
107
|
+
const bounded = boundEmbeddableText(remaining)
|
|
108
|
+
if (!bounded.bounded) {
|
|
109
|
+
chunks.push(bounded.text)
|
|
110
|
+
break
|
|
111
|
+
}
|
|
112
|
+
if (bounded.text.length === 0) {
|
|
113
|
+
chunks.push(remaining[0]!)
|
|
114
|
+
remaining = remaining.slice(1)
|
|
115
|
+
continue
|
|
116
|
+
}
|
|
117
|
+
chunks.push(bounded.text)
|
|
118
|
+
remaining = remaining.slice(bounded.text.length)
|
|
119
|
+
}
|
|
120
|
+
return chunks
|
|
121
|
+
}
|
|
122
|
+
|
|
95
123
|
// Returns the longest prefix length (in chars) whose estimateTokens is still
|
|
96
124
|
// within budget. Recomputes the EXACT same estimate incrementally — CJK chars
|
|
97
125
|
// at 1 token, plus max(non-CJK char-ratio, word count) — so a bounded prefix can
|
|
@@ -38,8 +38,10 @@ export const PLANNER_SKILLS: readonly LoadableSkill[] = [PROJECT_PLAN_SKILL, GEN
|
|
|
38
38
|
// request fails loudly. Sized for a thorough `deep`-model plan (research lookups
|
|
39
39
|
// + a careful decomposition), well above the typical sub-minute plan. This is
|
|
40
40
|
// liveness for the parent, not hard cancellation. See src/agent/subagents.ts
|
|
41
|
-
// `timeoutMs`.
|
|
42
|
-
|
|
41
|
+
// `timeoutMs`. 30m matches the reviewer and researcher — every deep-profile
|
|
42
|
+
// subagent shares the same nested-fan-out budget so a real pass is not killed
|
|
43
|
+
// mid-synthesis.
|
|
44
|
+
export const PLANNER_SPAWN_TIMEOUT_MS = 1_800_000
|
|
43
45
|
|
|
44
46
|
// The default write zone for the plan file when the caller does not specify
|
|
45
47
|
// `outputPath`. MUST be a universally-writable location, because a subagent
|
|
@@ -51,7 +51,15 @@ export const REVIEWER_SKILLS: readonly LoadableSkill[] = [
|
|
|
51
51
|
// liveness for the parent, not hard cancellation: pi's `session.prompt` takes no
|
|
52
52
|
// AbortSignal, so the LLM stream may run until the OS reaps it. See
|
|
53
53
|
// src/agent/subagents.ts `timeoutMs`.
|
|
54
|
-
|
|
54
|
+
//
|
|
55
|
+
// 30m, not the prior 10m: the `deep` profile trades speed for quality, and a
|
|
56
|
+
// real review of a large PR (many changed files + nested scout fan-out + a few
|
|
57
|
+
// web lookups + careful synthesis) routinely exceeds 10m. A 10m ceiling killed
|
|
58
|
+
// both the first and retry reviews of a 27-file PR mid-analysis, leaving the PR
|
|
59
|
+
// with no posted verdict — the exact "stranded review" failure the guards exist
|
|
60
|
+
// to prevent. Matches RESEARCHER_SPAWN_TIMEOUT_MS, the other deep-profile
|
|
61
|
+
// subagent that hit the same nested-fan-out wall.
|
|
62
|
+
export const REVIEWER_SPAWN_TIMEOUT_MS = 1_800_000
|
|
55
63
|
|
|
56
64
|
// The reviewer's read-only contract is enforced in depth: this system prompt
|
|
57
65
|
// states it, the global bash guards (`secret-exfil-bash`, `git-exfil`) catch
|
|
@@ -9,7 +9,7 @@ This plugin is **auto-loaded** by every TypeClaw agent. There is no `plugins[]`
|
|
|
9
9
|
`pi-coding-agent`'s built-in tools occasionally return very large payloads that the model only needed once. Two empirically observed cases:
|
|
10
10
|
|
|
11
11
|
1. **`read` on an image file** returns the base64-encoded image inline (e.g. `{type:"image", data:"<3.2MB of base64>"}`). The model uses it on the turn it was asked for, then sees the same 3.2MB of base64 as conversation context on every subsequent prompt — until compaction fires (which is token-driven, not byte-driven, so a single fat blob may sit in context for many turns before compaction is triggered).
|
|
12
|
-
2. **`web_fetch` on a binary URL** (PNG, ZIP, etc.) receives the raw response body, treats it as text, and stores raw binary as a JSON-encoded string. Same effect: 100KB+ of mojibake sits in the transcript permanently.
|
|
12
|
+
2. **`web_fetch` on a binary URL** (PNG, ZIP, etc.) receives the raw response body, treats it as text, and stores raw binary as a JSON-encoded string. Same effect: 100KB+ of mojibake sits in the transcript permanently. (Historical: this predates `web_fetch`'s own per-strategy `OUTPUT_CAPS`, which now bound its text output to ≤200KB regardless — see the `web_fetch` text exemption below.)
|
|
13
13
|
|
|
14
14
|
The result is a session JSONL file that's tens of megabytes on disk but mostly one or two giant tool results, plus 3-minute first-prompt latencies after container restart because the full transcript gets re-shipped to the LLM as context.
|
|
15
15
|
|
|
@@ -47,6 +47,15 @@ The cap is per-part, not per-result, so a result with one small text part and on
|
|
|
47
47
|
|
|
48
48
|
Placeholders carry the literal substring `tool-result-cap:` so future agents (or human operators inspecting a session) can grep for them and recognize that the original payload was intentionally elided rather than truncated by some other layer.
|
|
49
49
|
|
|
50
|
+
## The `web_fetch` text exemption
|
|
51
|
+
|
|
52
|
+
`web_fetch` is exempt from the **text** cap (but not the image cap), hardcoded in `cap-result.ts` via `TEXT_CAP_EXEMPT_TOOLS`. This is not user-configurable because it resolves a collision between two first-party caps:
|
|
53
|
+
|
|
54
|
+
- `web_fetch` has its own per-strategy `OUTPUT_CAPS` in `src/agent/tools/webfetch/types.ts` — `readability` is capped at 200KB, `raw` at 100KB, others between 50KB and 100KB — bounded above by a 5MB transport limit. These are deliberately tuned: `readability` gets a generous budget so a full article reaches the model.
|
|
55
|
+
- This plugin's flat `textMaxBytes` default (32KB) is smaller than every one of those, so without the exemption it would silently clip every `web_fetch` result to ~8K tokens, making the strategy caps meaningless. A `readability` fetch designed for 200KB would never deliver more than 32KB.
|
|
56
|
+
|
|
57
|
+
The exemption is **text-only**: `web_fetch` image parts still go through `imageMaxBytes`, so the binary/base64 transcript-bloat protection (case 2 above) is preserved. The tradeoff is that this plugin no longer provides a 32KB backstop for `web_fetch` text — that responsibility now belongs entirely to `web_fetch`'s own `capOutput()`. User-configured `exemptTools` is a separate, broader knob (skips _all_ capping, image included); the `web_fetch` text exemption is narrower and always on.
|
|
58
|
+
|
|
50
59
|
## What's not capped
|
|
51
60
|
|
|
52
61
|
- `details` on tool results (an opaque structured payload provider-specific to each tool — generally small, and mutating it risks breaking tool-specific telemetry).
|
|
@@ -14,6 +14,13 @@ export type CapStats = {
|
|
|
14
14
|
|
|
15
15
|
const ELIDED_MARKER = '[tool-result-cap: '
|
|
16
16
|
|
|
17
|
+
// `web_fetch` already bounds its own text via per-strategy OUTPUT_CAPS
|
|
18
|
+
// (src/agent/tools/webfetch/types.ts: 50KB–200KB; readability gets 200KB on
|
|
19
|
+
// purpose), so the flat 32KB textMaxBytes here only collides with it and clips
|
|
20
|
+
// every readability fetch to ~8K tokens. Skip TEXT capping for it; image parts
|
|
21
|
+
// still hit the image cap below, preserving the binary/base64 bloat protection.
|
|
22
|
+
const TEXT_CAP_EXEMPT_TOOLS = new Set(['web_fetch'])
|
|
23
|
+
|
|
17
24
|
// A capped text part is exactly the placeholder we generated: starts with
|
|
18
25
|
// `[tool-result-cap: `, ends with `]`, and contains no inner `]`. The shape
|
|
19
26
|
// check exists for idempotency so a previously-capped entry survives a second
|
|
@@ -46,6 +53,7 @@ export function capContentParts(tool: string, content: ContentPart[], options: C
|
|
|
46
53
|
continue
|
|
47
54
|
}
|
|
48
55
|
if (part.type === 'text') {
|
|
56
|
+
if (TEXT_CAP_EXEMPT_TOOLS.has(tool)) continue
|
|
49
57
|
const size = part.text.length
|
|
50
58
|
if (size <= options.textMaxBytes) continue
|
|
51
59
|
if (isElidedPlaceholderText(part.text)) continue
|