typeclaw 0.46.1 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +2 -2
- package/src/agent/tool-file-safety.ts +18 -9
- package/src/agent/tools/webfetch/strategies/jq.ts +1 -2
- package/src/agent/tools/webfetch/strategies/readability.ts +39 -16
- package/src/agent/tools/webfetch/strategies/selector.ts +4 -2
- package/src/agent/tools/webfetch/strategies/snapshot.ts +3 -2
- package/src/agent/tools/webfetch/tool.ts +3 -3
- package/src/bundled-plugins/backup/index.ts +21 -4
- package/src/bundled-plugins/backup/runner.ts +19 -5
- package/src/bundled-plugins/memory/dreaming.ts +2 -1
- package/src/bundled-plugins/memory/load-shards.ts +31 -3
- package/src/bundled-plugins/memory/memory-logger.ts +2 -1
- package/src/bundled-plugins/memory/memory-snapshot.ts +8 -0
- package/src/bundled-plugins/memory/vector/embedder.ts +28 -9
- package/src/bundled-plugins/memory/vector/hybrid.ts +10 -2
- package/src/bundled-plugins/memory/vector/store.ts +21 -9
- package/src/bundled-plugins/security/policies/git-exfil.ts +45 -27
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +8 -3
- package/src/bundled-plugins/security/policies/private-surface-read.ts +6 -0
- package/src/channels/adapters/telegram-bot-classify.ts +29 -0
- package/src/channels/router.ts +98 -0
- package/src/channels/types.ts +7 -0
- package/src/cli/command-meta.ts +2 -0
- package/src/cli/compose-stats.ts +103 -0
- package/src/cli/compose.ts +13 -0
- package/src/cli/index.ts +1 -0
- package/src/cli/inspect.ts +16 -0
- package/src/cli/stats.ts +134 -0
- package/src/compose/index.ts +1 -0
- package/src/compose/stats.ts +51 -0
- package/src/config/config.ts +14 -0
- package/src/config/index.ts +4 -0
- package/src/container/controller.ts +6 -0
- package/src/container/index.ts +1 -0
- package/src/container/log-archive.ts +448 -0
- package/src/container/shared.ts +49 -19
- package/src/container/start.ts +78 -63
- package/src/container/stats.ts +86 -0
- package/src/container/stop.ts +63 -36
- package/src/init/checkpoint.ts +4 -4
- package/src/init/dockerfile.ts +87 -88
- package/src/init/gitignore.ts +5 -6
- package/src/init/restart-deps-preflight.ts +2 -2
- package/src/init/run-bun-install.ts +16 -2
- package/src/init/validate-api-key.ts +1 -0
- package/src/inspect/item-list.ts +12 -6
- package/src/permissions/permissions.ts +93 -2
- package/src/role-claim/code.ts +9 -5
- package/src/run/index.ts +50 -36
- package/src/run/resource-report.ts +226 -0
- package/src/sandbox/canonical-secrets.ts +7 -0
- package/src/sandbox/index.ts +1 -0
- package/src/sandbox/symlinks.ts +1 -1
- package/src/secrets/export-claude-credentials-file.ts +13 -22
- package/src/secrets/export-codex-auth-file.ts +12 -28
- package/typeclaw.schema.json +14 -0
package/README.md
CHANGED
|
@@ -79,10 +79,11 @@
|
|
|
79
79
|
- 📊 **usage** — reports token and dollar spend by day, model, session, or origin
|
|
80
80
|
- 🔍 **inspect** — replays a session transcript and tails live activity
|
|
81
81
|
- 📜 **logs** — streams container logs with local-time prefixes
|
|
82
|
+
- 📈 **stats** — a live cpu / memory / pid snapshot of the agent container
|
|
82
83
|
|
|
83
84
|
## Compose — manage a fleet from the CLI
|
|
84
85
|
|
|
85
|
-
- 🎼 **Fleet operations** — discover agent folders and start, stop, restart, check status, tail logs, report usage, and run diagnostics across them from the command line
|
|
86
|
+
- 🎼 **Fleet operations** — discover agent folders and start, stop, restart, check status, snapshot resource stats, tail logs, report usage, and run diagnostics across them from the command line
|
|
86
87
|
|
|
87
88
|
Memory loop and subagent architecture are covered in detail in the [Internals docs](https://typeclaw.dev/docs/internals) and [`src/bundled-plugins/memory/README.md`](./src/bundled-plugins/memory/README.md).
|
|
88
89
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "typeclaw",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.47.0",
|
|
4
4
|
"homepage": "https://github.com/typeclaw/typeclaw#readme",
|
|
5
5
|
"bugs": {
|
|
6
6
|
"url": "https://github.com/typeclaw/typeclaw/issues"
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"@mariozechner/pi-tui": "^0.73.1",
|
|
51
51
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
52
52
|
"@mozilla/readability": "^0.6.0",
|
|
53
|
-
"agent-messenger": "^2.32.
|
|
53
|
+
"agent-messenger": "^2.32.3",
|
|
54
54
|
"cheerio": "^1.2.0",
|
|
55
55
|
"citty": "^0.2.2",
|
|
56
56
|
"cron-parser": "^5.5.0",
|
|
@@ -861,15 +861,24 @@ async function streamSnapshot(
|
|
|
861
861
|
callback(null, chunk)
|
|
862
862
|
},
|
|
863
863
|
})
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
864
|
+
// `autoClose: false` leaves the FileHandle to the caller, but the ReadStream
|
|
865
|
+
// opens its OWN fd that `source.close()` never releases and GC never reclaims
|
|
866
|
+
// — one leaked fd per snapshotted file, reaching EMFILE on a long-running
|
|
867
|
+
// agent. Destroy the stream explicitly to free that fd deterministically.
|
|
868
|
+
const readStream = source.createReadStream({ autoClose: false, start: 0 })
|
|
869
|
+
try {
|
|
870
|
+
await pipeline(
|
|
871
|
+
readStream,
|
|
872
|
+
limiter,
|
|
873
|
+
createWriteStream(destination, {
|
|
874
|
+
flags: 'wx',
|
|
875
|
+
mode: 0o400,
|
|
876
|
+
}),
|
|
877
|
+
{ signal },
|
|
878
|
+
)
|
|
879
|
+
} finally {
|
|
880
|
+
readStream.destroy()
|
|
881
|
+
}
|
|
873
882
|
return copied
|
|
874
883
|
}
|
|
875
884
|
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { raw } from 'jq-wasm'
|
|
2
|
-
|
|
3
1
|
export class JqError extends Error {
|
|
4
2
|
constructor(message: string) {
|
|
5
3
|
super(message)
|
|
@@ -17,6 +15,7 @@ export async function applyJq(content: string, query: string): Promise<string> {
|
|
|
17
15
|
}
|
|
18
16
|
|
|
19
17
|
try {
|
|
18
|
+
const { raw } = await import('jq-wasm')
|
|
20
19
|
const result = await raw(parsed as string | object, query)
|
|
21
20
|
if (result.exitCode !== 0 || result.stderr) {
|
|
22
21
|
const detail = result.stderr.trim() || `exit code ${result.exitCode}`
|
|
@@ -1,19 +1,42 @@
|
|
|
1
|
-
import { Readability } from '@mozilla/readability'
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
1
|
+
import type { Readability as ReadabilityClass } from '@mozilla/readability'
|
|
2
|
+
import type TurndownService from 'turndown'
|
|
3
|
+
|
|
4
|
+
// Perf: jsdom (+readability+turndown) costs ~40-60MB RSS at import time. Keep it
|
|
5
|
+
// lazy so idle agents that never call web_fetch don't pay for it. Do NOT hoist to
|
|
6
|
+
// a top-level import.
|
|
7
|
+
type ReadabilityDeps = {
|
|
8
|
+
Readability: typeof ReadabilityClass
|
|
9
|
+
JSDOM: typeof import('jsdom').JSDOM
|
|
10
|
+
turndown: TurndownService
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
let depsPromise: Promise<ReadabilityDeps> | undefined
|
|
14
|
+
|
|
15
|
+
async function loadDeps(): Promise<ReadabilityDeps> {
|
|
16
|
+
depsPromise ??= (async () => {
|
|
17
|
+
const [{ Readability }, { JSDOM }, { default: TurndownCtor }] = await Promise.all([
|
|
18
|
+
import('@mozilla/readability'),
|
|
19
|
+
import('jsdom'),
|
|
20
|
+
import('turndown'),
|
|
21
|
+
])
|
|
22
|
+
const turndown = new TurndownCtor({
|
|
23
|
+
headingStyle: 'atx',
|
|
24
|
+
codeBlockStyle: 'fenced',
|
|
25
|
+
bulletListMarker: '-',
|
|
26
|
+
emDelimiter: '*',
|
|
27
|
+
hr: '---',
|
|
28
|
+
})
|
|
29
|
+
turndown.remove(['script', 'style', 'meta', 'link', 'noscript', 'iframe'])
|
|
30
|
+
return { Readability, JSDOM, turndown }
|
|
31
|
+
})()
|
|
32
|
+
return depsPromise
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
type ReadabilityDocument = ConstructorParameters<typeof ReadabilityClass>[0]
|
|
36
|
+
|
|
37
|
+
export async function applyReadability(html: string, url: string): Promise<string> {
|
|
38
|
+
const { Readability, JSDOM, turndown } = await loadDeps()
|
|
39
|
+
|
|
17
40
|
const dom = new JSDOM(html, { url })
|
|
18
41
|
const document = dom.window.document.cloneNode(true) as unknown as ReadabilityDocument
|
|
19
42
|
const article = new Readability(document).parse()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio'
|
|
1
|
+
import type * as cheerio from 'cheerio'
|
|
2
2
|
|
|
3
3
|
export class SelectorError extends Error {
|
|
4
4
|
constructor(message: string) {
|
|
@@ -7,7 +7,9 @@ export class SelectorError extends Error {
|
|
|
7
7
|
}
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
export function applySelector(html: string, selector: string): string {
|
|
10
|
+
export async function applySelector(html: string, selector: string): Promise<string> {
|
|
11
|
+
const cheerio = await import('cheerio')
|
|
12
|
+
|
|
11
13
|
let $: cheerio.CheerioAPI
|
|
12
14
|
try {
|
|
13
15
|
$ = cheerio.load(html)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio'
|
|
1
|
+
import type * as cheerio from 'cheerio'
|
|
2
2
|
import type { AnyNode, Element } from 'domhandler'
|
|
3
3
|
|
|
4
4
|
const SEMANTIC_TAGS = new Set([
|
|
@@ -65,7 +65,8 @@ const ROLE_FOR_TAG: Record<string, string> = {
|
|
|
65
65
|
label: 'label',
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
export function applySnapshot(html: string): string {
|
|
68
|
+
export async function applySnapshot(html: string): Promise<string> {
|
|
69
|
+
const cheerio = await import('cheerio')
|
|
69
70
|
const $ = cheerio.load(html)
|
|
70
71
|
const lines: string[] = []
|
|
71
72
|
const body = $('body').get(0)
|
|
@@ -226,7 +226,7 @@ async function runStrategy(
|
|
|
226
226
|
case 'raw':
|
|
227
227
|
return applyRaw(body)
|
|
228
228
|
case 'readability':
|
|
229
|
-
return applyReadability(body, url)
|
|
229
|
+
return await applyReadability(body, url)
|
|
230
230
|
case 'jq':
|
|
231
231
|
try {
|
|
232
232
|
return await applyJq(body, params.query ?? '')
|
|
@@ -236,7 +236,7 @@ async function runStrategy(
|
|
|
236
236
|
}
|
|
237
237
|
case 'selector':
|
|
238
238
|
try {
|
|
239
|
-
return applySelector(body, params.selector ?? '')
|
|
239
|
+
return await applySelector(body, params.selector ?? '')
|
|
240
240
|
} catch (error) {
|
|
241
241
|
if (error instanceof SelectorError) throw new Error(error.message)
|
|
242
242
|
throw error
|
|
@@ -255,7 +255,7 @@ async function runStrategy(
|
|
|
255
255
|
throw error
|
|
256
256
|
}
|
|
257
257
|
case 'snapshot':
|
|
258
|
-
return applySnapshot(body)
|
|
258
|
+
return await applySnapshot(body)
|
|
259
259
|
}
|
|
260
260
|
}
|
|
261
261
|
|
|
@@ -299,8 +299,11 @@ async function resolveBackupAuthEnv(
|
|
|
299
299
|
}
|
|
300
300
|
}
|
|
301
301
|
|
|
302
|
-
export async function resolveOriginPushUrl(
|
|
303
|
-
|
|
302
|
+
export async function resolveOriginPushUrl(
|
|
303
|
+
cwd: string,
|
|
304
|
+
bunOverride?: { spawn: typeof Bun.spawn },
|
|
305
|
+
): Promise<string | null> {
|
|
306
|
+
const bun = bunOverride ?? (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
|
|
304
307
|
if (!bun) return null
|
|
305
308
|
const repo = resolveAgentGit(cwd)
|
|
306
309
|
if (!repo) return null
|
|
@@ -311,8 +314,22 @@ export async function resolveOriginPushUrl(cwd: string): Promise<string | null>
|
|
|
311
314
|
stderr: 'ignore',
|
|
312
315
|
env: { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0' },
|
|
313
316
|
})
|
|
314
|
-
|
|
315
|
-
const
|
|
317
|
+
const exited = proc.exited
|
|
318
|
+
const stdoutDrain = new Response(proc.stdout).text()
|
|
319
|
+
let exitCode: number
|
|
320
|
+
let stdout: string
|
|
321
|
+
try {
|
|
322
|
+
;[exitCode, stdout] = await Promise.all([exited, stdoutDrain])
|
|
323
|
+
} catch {
|
|
324
|
+
// A stdout-drain rejection must not leave the git child running: kill it
|
|
325
|
+
// and await exit + drain before returning, so `Promise.all`'s early reject
|
|
326
|
+
// can't orphan the subprocess or its pipe.
|
|
327
|
+
proc.kill()
|
|
328
|
+
await Promise.allSettled([exited, stdoutDrain])
|
|
329
|
+
return null
|
|
330
|
+
}
|
|
331
|
+
if (exitCode !== 0) return null
|
|
332
|
+
const out = stdout.trim()
|
|
316
333
|
return out === '' ? null : out
|
|
317
334
|
} catch {
|
|
318
335
|
return null
|
|
@@ -399,9 +399,9 @@ function sanitizeCommitMessage(raw: string): string {
|
|
|
399
399
|
return rest.length > 0 ? `${subject}\n\n${rest}` : subject
|
|
400
400
|
}
|
|
401
401
|
|
|
402
|
-
export function makeDefaultGitSpawn(): GitSpawn {
|
|
402
|
+
export function makeDefaultGitSpawn(bunOverride?: { spawn: typeof Bun.spawn }): GitSpawn {
|
|
403
403
|
return withIndexLockRetry(async (args, { cwd, timeoutMs, env }) => {
|
|
404
|
-
const bun = (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
|
|
404
|
+
const bun = bunOverride ?? (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
|
|
405
405
|
if (!bun) {
|
|
406
406
|
return { exitCode: 127, stdout: '', stderr: 'Bun runtime not available', timedOut: false }
|
|
407
407
|
}
|
|
@@ -419,9 +419,23 @@ export function makeDefaultGitSpawn(): GitSpawn {
|
|
|
419
419
|
env: { ...process.env, ...NONINTERACTIVE_ENV, ...env },
|
|
420
420
|
signal: controller.signal,
|
|
421
421
|
})
|
|
422
|
-
const
|
|
423
|
-
const
|
|
424
|
-
const
|
|
422
|
+
const exitedDrain = proc.exited
|
|
423
|
+
const stdoutDrain = new Response(proc.stdout).text()
|
|
424
|
+
const stderrDrain = new Response(proc.stderr).text()
|
|
425
|
+
let exitCode: number
|
|
426
|
+
let stdout: string
|
|
427
|
+
let stderr: string
|
|
428
|
+
try {
|
|
429
|
+
;[exitCode, stdout, stderr] = await Promise.all([exitedDrain, stdoutDrain, stderrDrain])
|
|
430
|
+
} catch (drainErr) {
|
|
431
|
+
// A pipe-read rejection must not escape while the git child is still
|
|
432
|
+
// running (the outer `finally` would clear the abort timer and leave a
|
|
433
|
+
// stuck child unbounded). Abort to kill it, then wait for exit and both
|
|
434
|
+
// drains to settle before surfacing the error.
|
|
435
|
+
controller.abort()
|
|
436
|
+
await Promise.allSettled([exitedDrain, stdoutDrain, stderrDrain])
|
|
437
|
+
throw drainErr
|
|
438
|
+
}
|
|
425
439
|
const timedOut = controller.signal.aborted
|
|
426
440
|
return { exitCode, stdout, stderr, timedOut }
|
|
427
441
|
} catch (err) {
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
} from './dreaming-state'
|
|
27
27
|
import { parseShard, renderShard, type ShardFrontmatter } from './frontmatter'
|
|
28
28
|
import { listShardSlugs, loadAllShards, loadShard, type TopicShard } from './load-shards'
|
|
29
|
+
import { formatMemorySnapshot } from './memory-snapshot'
|
|
29
30
|
import { referencesDir, streamFilePath, streamsDir, topicShardPath, topicsDir } from './paths'
|
|
30
31
|
import { renderReference } from './references/frontmatter'
|
|
31
32
|
import { loadAllReferences, type Reference } from './references/load-references'
|
|
@@ -1477,7 +1478,7 @@ export function createDreamingSubagent(options: CreateDreamingSubagentOptions =
|
|
|
1477
1478
|
try {
|
|
1478
1479
|
await commit(ctx.payload.agentDir)
|
|
1479
1480
|
logger.info(
|
|
1480
|
-
`[dreaming] done topics_created=${metrics.topicsCreated} topics_removed=${metrics.topicsRemoved} superseded_new=${metrics.supersededDelta} fragments_dropped=${compaction.fragmentsDropped} over_budget=${overBudget.length} references_demoted=${referencesDemoted} references_evicted=${referencesEvicted} elapsed_ms=${Date.now() - start}`,
|
|
1481
|
+
`[dreaming] done topics_created=${metrics.topicsCreated} topics_removed=${metrics.topicsRemoved} superseded_new=${metrics.supersededDelta} fragments_dropped=${compaction.fragmentsDropped} over_budget=${overBudget.length} references_demoted=${referencesDemoted} references_evicted=${referencesEvicted} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
|
|
1481
1482
|
)
|
|
1482
1483
|
} catch (err) {
|
|
1483
1484
|
const message = err instanceof Error ? err.message : String(err)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readdir, readFile, stat } from 'node:fs/promises'
|
|
1
|
+
import { mkdir, readdir, readFile, stat } from 'node:fs/promises'
|
|
2
2
|
|
|
3
3
|
import { parseShard, type ShardFrontmatter } from './frontmatter'
|
|
4
4
|
import { topicShardPath, topicsDir } from './paths'
|
|
@@ -117,11 +117,23 @@ export async function loadShard(
|
|
|
117
117
|
}
|
|
118
118
|
|
|
119
119
|
export async function listShardSlugs(agentDir: string): Promise<string[]> {
|
|
120
|
+
const dir = topicsDir(agentDir)
|
|
120
121
|
let names: string[]
|
|
121
122
|
try {
|
|
122
|
-
names = await readdir(
|
|
123
|
+
names = await readdir(dir)
|
|
123
124
|
} catch (err) {
|
|
124
|
-
|
|
125
|
+
// ENOENT: the topics dir doesn't exist yet (fresh agent, or a wiped memory
|
|
126
|
+
// tree). Self-heal by recreating it so every downstream reader -- per-turn
|
|
127
|
+
// injection, memory_search, dreaming, the startup index build, and the
|
|
128
|
+
// doctor vector-index check -- sees a real, empty directory instead of a
|
|
129
|
+
// missing one. mkdir is idempotent, so a lost race just no-ops.
|
|
130
|
+
if (isEnoent(err)) return healMissingTopicsDir(dir)
|
|
131
|
+
// ENOTDIR: a bind-mount/tmpfs coherency hiccup can momentarily replace the
|
|
132
|
+
// topics dir (or a parent path component) with a non-directory; readdir
|
|
133
|
+
// then throws ENOTDIR. We can't self-heal this -- mkdir over a regular file
|
|
134
|
+
// would either fail or clobber operator data -- so degrade to empty and let
|
|
135
|
+
// the transient host state resolve itself, exactly as before.
|
|
136
|
+
if (isEnotdir(err)) return []
|
|
125
137
|
throw err
|
|
126
138
|
}
|
|
127
139
|
|
|
@@ -131,6 +143,18 @@ export async function listShardSlugs(agentDir: string): Promise<string[]> {
|
|
|
131
143
|
.sort()
|
|
132
144
|
}
|
|
133
145
|
|
|
146
|
+
async function healMissingTopicsDir(dir: string): Promise<string[]> {
|
|
147
|
+
try {
|
|
148
|
+
await mkdir(dir, { recursive: true })
|
|
149
|
+
} catch (err) {
|
|
150
|
+
// A concurrent healer already created it, or the parent briefly became a
|
|
151
|
+
// non-directory. Either way the empty-list contract still holds; never let
|
|
152
|
+
// the heal attempt turn a self-healing state into a hard error.
|
|
153
|
+
if (!isEnotdir(err)) throw err
|
|
154
|
+
}
|
|
155
|
+
return []
|
|
156
|
+
}
|
|
157
|
+
|
|
134
158
|
// Test-only helper. Clears the in-memory shard cache so tests that exercise
|
|
135
159
|
// the cache invalidation path can simulate a cold start without spinning up a
|
|
136
160
|
// fresh process.
|
|
@@ -180,3 +204,7 @@ function getOrCreateCache(agentDir: string): Map<string, ShardCacheEntry> {
|
|
|
180
204
|
function isEnoent(err: unknown): boolean {
|
|
181
205
|
return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOENT'
|
|
182
206
|
}
|
|
207
|
+
|
|
208
|
+
function isEnotdir(err: unknown): boolean {
|
|
209
|
+
return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOTDIR'
|
|
210
|
+
}
|
|
@@ -6,6 +6,7 @@ import { formatLocalDate } from '@/shared'
|
|
|
6
6
|
|
|
7
7
|
import { advanceWatermarkTool, createAppendTool, type FragmentsAppendedHook } from './append-tool'
|
|
8
8
|
import { findEntryTool } from './find-entry-tool'
|
|
9
|
+
import { formatMemorySnapshot } from './memory-snapshot'
|
|
9
10
|
import { streamFilePath, streamsDir } from './paths'
|
|
10
11
|
import { listReferenceSlugs } from './references/load-references'
|
|
11
12
|
import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
|
|
@@ -352,7 +353,7 @@ export function createMemoryLoggerSubagent(
|
|
|
352
353
|
await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark) })
|
|
353
354
|
const fragmentsWritten = (await countFragments(streamFile)) - fragmentsBefore
|
|
354
355
|
logger.info(
|
|
355
|
-
`[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start}`,
|
|
356
|
+
`[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
|
|
356
357
|
)
|
|
357
358
|
} catch (err) {
|
|
358
359
|
const message = err instanceof Error ? err.message : String(err)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type MemorySample = Pick<NodeJS.MemoryUsage, 'rss' | 'heapUsed' | 'heapTotal' | 'external' | 'arrayBuffers'>
|
|
2
|
+
|
|
3
|
+
const BYTES_PER_MB = 1024 * 1024
|
|
4
|
+
|
|
5
|
+
export function formatMemorySnapshot(sample: MemorySample = process.memoryUsage()): string {
|
|
6
|
+
const mb = (bytes: number): number => Math.round(bytes / BYTES_PER_MB)
|
|
7
|
+
return `rss_mb=${mb(sample.rss)} heapUsed_mb=${mb(sample.heapUsed)} external_mb=${mb(sample.external)} arrayBuffers_mb=${mb(sample.arrayBuffers)}`
|
|
8
|
+
}
|
|
@@ -6,6 +6,7 @@ import { join } from 'node:path'
|
|
|
6
6
|
// values are pulled lazily via `loadTransformers()` below.
|
|
7
7
|
import type { env as TransformersEnvValue, pipeline as TransformersPipeline } from '@huggingface/transformers'
|
|
8
8
|
|
|
9
|
+
import { createKeyedSemaphore } from '@/agent/tools/keyed-semaphore'
|
|
9
10
|
import {
|
|
10
11
|
assertModelCacheCompatible,
|
|
11
12
|
EMBEDDING_DIMS,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
import { getResolvedTransformersVersion } from '@/models/transformers-version'
|
|
17
18
|
|
|
18
19
|
import { homeRoot } from '../../../hostd/paths'
|
|
20
|
+
import { formatMemorySnapshot } from '../memory-snapshot'
|
|
19
21
|
import { resolveEmbedThreadCount } from './embed-threads'
|
|
20
22
|
import { type BoundedText, boundEmbeddableText, MAX_MODEL_TOKENS } from './truncation'
|
|
21
23
|
|
|
@@ -36,6 +38,21 @@ export type EmbedType = 'query' | 'passage'
|
|
|
36
38
|
// loaded once and reused across chunks, so the only cost is sequential passes.
|
|
37
39
|
const EMBED_BATCH_SIZE = 64
|
|
38
40
|
|
|
41
|
+
// Concurrent embeds inside inference at once, process-wide. The intra-op thread
|
|
42
|
+
// cap (embed-threads.ts) bounds ONE embed's footprint, but nothing bounded how
|
|
43
|
+
// MANY embeds run at once: a busy channel turn's hybridSearch, a dreaming pass,
|
|
44
|
+
// and a memory-logger append can all be inside the shared ONNX session
|
|
45
|
+
// simultaneously (concurrent `Run()` is allowed on the Node backend), stacking
|
|
46
|
+
// their activation memory into one RSS spike that the OOM killer takes with a
|
|
47
|
+
// bare SIGKILL. This caps that stack. Chosen 2, not 1: the workload is
|
|
48
|
+
// latency-sensitive per-turn query embeds, so a hard serial would park a live
|
|
49
|
+
// turn behind a whole dreaming batch; 2 keeps one lane free while still bounding
|
|
50
|
+
// the worst-case multiplier. Single fixed key — the limit is global, not
|
|
51
|
+
// per-key.
|
|
52
|
+
const EMBED_CONCURRENCY = 2
|
|
53
|
+
const EMBED_SEMAPHORE_KEY = 'embed'
|
|
54
|
+
const embedSemaphore = createKeyedSemaphore({ concurrency: EMBED_CONCURRENCY })
|
|
55
|
+
|
|
39
56
|
type TransformersEnv = typeof TransformersEnvValue
|
|
40
57
|
type FeatureExtractor = Awaited<ReturnType<typeof TransformersPipeline<'feature-extraction'>>>
|
|
41
58
|
|
|
@@ -154,14 +171,16 @@ export class Embedder {
|
|
|
154
171
|
// upserts) finish in a pass or two and would only spam the logs.
|
|
155
172
|
const reportProgress = prefixed.length >= LARGE_EMBED
|
|
156
173
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
174
|
+
return embedSemaphore.run(EMBED_SEMAPHORE_KEY, async () => {
|
|
175
|
+
const embeddings: Float32Array[] = []
|
|
176
|
+
for (let start = 0; start < prefixed.length; start += EMBED_BATCH_SIZE) {
|
|
177
|
+
const batch = prefixed.slice(start, start + EMBED_BATCH_SIZE)
|
|
178
|
+
const output = await this.extractor(batch, { pooling: 'mean', normalize: true })
|
|
179
|
+
embeddings.push(...toEmbeddings(output.data, batch.length))
|
|
180
|
+
if (reportProgress) logEmbedProgress(embeddings.length, prefixed.length, type)
|
|
181
|
+
}
|
|
182
|
+
return embeddings
|
|
183
|
+
})
|
|
165
184
|
}
|
|
166
185
|
}
|
|
167
186
|
|
|
@@ -213,7 +232,7 @@ function warnIfBounded(results: readonly BoundedText[], type: EmbedType): void {
|
|
|
213
232
|
const LARGE_EMBED = 256
|
|
214
233
|
|
|
215
234
|
function logEmbedBatch(count: number, type: EmbedType): void {
|
|
216
|
-
const line = `[memory] vector embedding: ${count} ${type} input(s) (chunked at ${EMBED_BATCH_SIZE}/pass)`
|
|
235
|
+
const line = `[memory] vector embedding: ${count} ${type} input(s) (chunked at ${EMBED_BATCH_SIZE}/pass) ${formatMemorySnapshot()}`
|
|
217
236
|
if (count >= LARGE_EMBED) {
|
|
218
237
|
console.info(`${line} — large build, this may take a while`)
|
|
219
238
|
} else {
|
|
@@ -24,7 +24,7 @@ import { embed, EMBEDDING_MODEL_ID, type EmbedType } from './embedder'
|
|
|
24
24
|
import type { Passage } from './passages'
|
|
25
25
|
import { clearsBaseline, gateRelevance, MARGIN, streamAdmissionBaseline } from './relevance-gate'
|
|
26
26
|
import { crossScriptMarginScale, dominantScript, type ScriptClass } from './script'
|
|
27
|
-
import { VectorStore, type ScoredVectorRow, type VectorRow } from './store'
|
|
27
|
+
import { scoreRows, VectorStore, type ScoredVectorRow, type VectorRow } from './store'
|
|
28
28
|
import { chunkEmbeddableText } from './truncation'
|
|
29
29
|
|
|
30
30
|
export { collectPassages, findMissingPassages, type Passage } from './passages'
|
|
@@ -241,9 +241,17 @@ function maxScoreAcrossChunks(
|
|
|
241
241
|
queryEmbeddings: Float32Array[],
|
|
242
242
|
store: VectorStore,
|
|
243
243
|
): { scored: ScoredVectorRow[]; winnerChunkByRowId: Map<string, number> } {
|
|
244
|
+
// Decode the compatible corpus ONCE and score every chunk against it, instead
|
|
245
|
+
// of re-reading and re-decoding the whole table per chunk (queryScored did a
|
|
246
|
+
// full SELECT + BLOB decode on each call). Same MAX-across-chunks result, but
|
|
247
|
+
// the O(rows) decode runs once per turn instead of once per chunk.
|
|
248
|
+
const dims = queryEmbeddings[0]?.length
|
|
249
|
+
if (dims === undefined) return { scored: [], winnerChunkByRowId: new Map() }
|
|
250
|
+
const rows = store.loadRows(EMBEDDING_MODEL_ID, dims)
|
|
251
|
+
|
|
244
252
|
const best = new Map<string, { scored: ScoredVectorRow; chunkIndex: number }>()
|
|
245
253
|
queryEmbeddings.forEach((embedding, chunkIndex) => {
|
|
246
|
-
for (const scoredRow of
|
|
254
|
+
for (const scoredRow of scoreRows(rows, embedding)) {
|
|
247
255
|
const existing = best.get(scoredRow.row.id)
|
|
248
256
|
if (existing === undefined || scoredRow.score > existing.scored.score) {
|
|
249
257
|
best.set(scoredRow.row.id, { scored: scoredRow, chunkIndex })
|
|
@@ -95,24 +95,28 @@ export class VectorStore {
|
|
|
95
95
|
// Same cosine scan as `query` but returns every compatible row WITH its score
|
|
96
96
|
// and unsliced, so a caller can reason about the full score distribution (the
|
|
97
97
|
// relevance gate's per-query baseline) before deciding how many to keep.
|
|
98
|
+
queryScored(embedding: Float32Array, modelId: string): ScoredVectorRow[] {
|
|
99
|
+
return scoreRows(this.loadRows(modelId, embedding.length), embedding).sort((a, b) => b.score - a.score)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Decodes every compatible row's embedding BLOB into a Float32Array ONCE. A
|
|
103
|
+
// multi-chunk query scores many embeddings against the SAME corpus, so callers
|
|
104
|
+
// that decode here once and reuse the result across chunks avoid re-reading and
|
|
105
|
+
// re-decoding the whole table per chunk (the per-turn allocation hot path). The
|
|
106
|
+
// returned rows are transient — held only for the current search, never cached
|
|
107
|
+
// across turns (that would trade collectable garbage for permanent resident
|
|
108
|
+
// memory, the opposite of what we want).
|
|
98
109
|
//
|
|
99
110
|
// Filter by embedding identity, not dims alone: a stale row from a different
|
|
100
111
|
// model/dtype variant can share the same dims but lives in an incompatible
|
|
101
112
|
// vector space, so cosine against it is garbage. Excluding it here keeps a
|
|
102
113
|
// partial re-embed (mixed variants mid-rebuild) at reduced recall, never
|
|
103
114
|
// wrong scores.
|
|
104
|
-
|
|
105
|
-
// The query vector's magnitude is identical for every row in this scan, so
|
|
106
|
-
// hoist it out of the per-row cosine instead of recomputing N times (each
|
|
107
|
-
// recompute is 768 multiply-adds + a sqrt). Behavior is unchanged — same
|
|
108
|
-
// cosine values, fewer operations on the brute-force hot path.
|
|
109
|
-
const queryMagnitude = magnitude(embedding)
|
|
115
|
+
loadRows(modelId: string, dims: number): VectorRow[] {
|
|
110
116
|
return this.db
|
|
111
117
|
.query<StoredVectorRow, [string, number]>('SELECT * FROM vectors WHERE model = ? AND dims = ?')
|
|
112
|
-
.all(modelId,
|
|
118
|
+
.all(modelId, dims)
|
|
113
119
|
.map(toVectorRow)
|
|
114
|
-
.map((row) => ({ row, score: cosineSimilarity(embedding, queryMagnitude, row.embedding) }))
|
|
115
|
-
.sort((a, b) => b.score - a.score)
|
|
116
120
|
}
|
|
117
121
|
|
|
118
122
|
deleteOtherModels(modelId: string): void {
|
|
@@ -160,6 +164,14 @@ export class VectorStore {
|
|
|
160
164
|
}
|
|
161
165
|
}
|
|
162
166
|
|
|
167
|
+
// Scores pre-decoded rows against one query embedding. The query vector's
|
|
168
|
+
// magnitude is identical for every row, so hoist it out of the per-row cosine
|
|
169
|
+
// instead of recomputing N times (each recompute is 768 multiply-adds + a sqrt).
|
|
170
|
+
export function scoreRows(rows: VectorRow[], embedding: Float32Array): ScoredVectorRow[] {
|
|
171
|
+
const queryMagnitude = magnitude(embedding)
|
|
172
|
+
return rows.map((row) => ({ row, score: cosineSimilarity(embedding, queryMagnitude, row.embedding) }))
|
|
173
|
+
}
|
|
174
|
+
|
|
163
175
|
function toVectorRow(row: StoredVectorRow): VectorRow {
|
|
164
176
|
return {
|
|
165
177
|
id: row.id,
|
|
@@ -3,33 +3,43 @@ import { ACKNOWLEDGE_GUARDS, type SecurityBlock, isGuardAcknowledged } from '../
|
|
|
3
3
|
import { getRemoteTaint, recordRemoteTaint } from './remote-taint-state'
|
|
4
4
|
|
|
5
5
|
export const GUARD_GIT_EXFIL = 'gitExfil'
|
|
6
|
+
// The coarse, role-gated first-line publication/file-transfer gate. It matches
|
|
7
|
+
// the command STRING only (see checkGitExfilGuard); it has no working
|
|
8
|
+
// directory, no resolved git repository identity, and no view of a checkout's
|
|
9
|
+
// content provenance. It therefore fires regardless of where the command runs
|
|
10
|
+
// — a `/tmp` clone is NOT a trusted boundary, because an earlier turn may have
|
|
11
|
+
// copied identity files or secrets into it, and the effective repository is
|
|
12
|
+
// unknowable from the string alone (`git -C`, `--git-dir`, `GIT_DIR`, `cd`,
|
|
13
|
+
// subshells, aliases). Do NOT re-introduce cwd/path scoping here: it would be
|
|
14
|
+
// a bypass, not a fix (Oracle-reviewed). Member-tier publishing, if ever
|
|
15
|
+
// required, belongs in a separate out-of-band capability, not a path heuristic.
|
|
16
|
+
//
|
|
6
17
|
// Classified `medium` (silent-attack axis). Originally `high`; reclassified
|
|
7
|
-
// because the
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
// by `gitRemoteTainted` even though trusted no longer needs to ack the
|
|
18
|
-
// push itself. Net effect: trusted users can push to remotes the operator
|
|
19
|
-
// configured without per-call acks, but cannot retarget-and-push.
|
|
18
|
+
// because the one narrow retarget-then-push audience-leak shape (re-point
|
|
19
|
+
// origin to an attacker URL, then push) is gated separately by
|
|
20
|
+
// `gitRemoteTainted` (still high). The recorder-vs-checker split is what makes
|
|
21
|
+
// that reclassification safe: the recorder fires for any actor who can run a
|
|
22
|
+
// `git remote set-url` (per-guard bypass OR the medium-tier permission via the
|
|
23
|
+
// OR check), so trusted's first-step set-url still records taint and the
|
|
24
|
+
// second-step push still gets caught by `gitRemoteTainted` even though trusted
|
|
25
|
+
// no longer needs to ack the push itself. Net effect: trusted users can push
|
|
26
|
+
// to remotes the operator configured without per-call acks, but cannot
|
|
27
|
+
// retarget-and-push.
|
|
20
28
|
export const GUARD_GIT_EXFIL_SEVERITY: SecuritySeverity = 'medium'
|
|
21
29
|
export const GUARD_GIT_REMOTE_TAINTED = 'gitRemoteTainted'
|
|
22
|
-
// Classified `high` (audience-leak axis)
|
|
23
|
-
// for git
|
|
24
|
-
//
|
|
25
|
-
// the
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
30
|
+
// Classified `high` (audience-leak axis). This is NOT the complete audience-
|
|
31
|
+
// leak defense for git — it gates ONE specific two-step shape: a push to a
|
|
32
|
+
// remote whose URL was changed mid-session (the breach pattern that motivated
|
|
33
|
+
// the security plugin per PR #134). Its state is keyed by session + remote
|
|
34
|
+
// name, so it does not cover pushes to an initially-malicious remote, direct
|
|
35
|
+
// URL pushes, or destinations established outside its session recorder — those
|
|
36
|
+
// stay under the coarse `gitExfil` gate above. Stays high regardless of how
|
|
37
|
+
// `gitExfil` is classified — the two are independent per-guard strings AND
|
|
38
|
+
// independent tier classifications. The recorder-vs-checker split (see comment
|
|
39
|
+
// on recordGitRemoteTaintIfAny below) is still load-bearing: the recorder
|
|
40
|
+
// fires for anyone who can run the underlying `set-url` command (ack, per-guard
|
|
41
|
+
// `bypassGitExfil`, OR the medium-tier permission — which now includes trusted
|
|
42
|
+
// by default), so the second-step taint check still fires on the eventual push.
|
|
33
43
|
export const GUARD_GIT_REMOTE_TAINTED_SEVERITY: SecuritySeverity = 'high'
|
|
34
44
|
|
|
35
45
|
// Anchors we reuse: a `git` token must be at start-of-line or follow a shell
|
|
@@ -185,9 +195,17 @@ export function checkGitExfilGuard(options: {
|
|
|
185
195
|
return {
|
|
186
196
|
block: true,
|
|
187
197
|
reason: [
|
|
188
|
-
`Guard \`${GUARD_GIT_EXFIL}\` blocked
|
|
189
|
-
//
|
|
190
|
-
|
|
198
|
+
`Guard \`${GUARD_GIT_EXFIL}\` blocked a risky repository-publication or file-transfer command: ${matched.label}.`,
|
|
199
|
+
// Deliberately directory-INDEPENDENT: the guard sees only the command
|
|
200
|
+
// string and cannot prove which repository git will operate on, nor that
|
|
201
|
+
// an external checkout (e.g. a /tmp clone) does not already contain data
|
|
202
|
+
// copied out of the agent folder. A "clean" path is not a security
|
|
203
|
+
// boundary, so this fires wherever the command runs. Pushing a repo,
|
|
204
|
+
// adding/re-pointing a remote, or piping a remote payload to a shell can
|
|
205
|
+
// publish identity files (MEMORY.md, IDENTITY.md, SOUL.md, AGENTS.md) and
|
|
206
|
+
// embedded secrets - including via prompt-injected requests from chat
|
|
207
|
+
// channels.
|
|
208
|
+
'This applies regardless of working directory.',
|
|
191
209
|
`If this is genuinely intentional and the user (not a channel message) explicitly asked for it, retry with \`${ACKNOWLEDGE_GUARDS}.${GUARD_GIT_EXFIL}: true\` in the bash arguments.`,
|
|
192
210
|
].join(' '),
|
|
193
211
|
}
|